Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions backend/collectors/collect_data_files.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ class HiddenDataFileMetadata(HiddenFile):
@dataclass
class DataFileRecord(BaseFile):
format: str
size_gb: str
file_size_in_bytes: str
row_count: int
partition: str
earliest_appearing_snapshot_id: int
Expand Down Expand Up @@ -59,7 +59,7 @@ def _process_data_file_row(self, data_file_row) -> DataFileRecord:
type=self._detect_file_type(data_file_dict["content"]),
file_path=data_file_dict["file_path"],
format=data_file_dict["file_format"],
size_gb=f"{(data_file_dict['file_size_in_bytes'] / 1024**3):.10f}",
file_size_in_bytes=str(data_file_dict["file_size_in_bytes"]),
row_count=data_file_dict["record_count"],
partition=format_partition(data_file_dict["partition"]),
earliest_appearing_snapshot_id=data_file_dict["earliest_snapshot_id"],
Expand Down
11 changes: 9 additions & 2 deletions backend/collectors/collect_snapshots.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,15 @@ def _validate_snapshot_count(snapshots_df: pyspark.sql.DataFrame) -> None:
raise ValueError(f"Too many snapshots to compute. Maximum is {Env.MAX_SNAPSHOTS_TO_COMPUTE}.")

@staticmethod
def _format_summary(summary: dict) -> dict:
return {k: (f"{(int(v) / (1024**3)):.5f} GB" if k.endswith("files-size") else v) for k, v in summary.items()}
def _format_summary(summary: Dict[str, str]) -> Dict[str, str]:
Comment thread
YanivZalach marked this conversation as resolved.
formatted = {}
for key, value in summary.items():
if key.endswith("files-size"):
formatted[f"{key}-bytes"] = str(value)
else:
formatted[key] = value

return formatted

def _parse_snapshot_row(self, snapshot) -> SnapshotRecord:
return SnapshotRecord(
Expand Down
7 changes: 3 additions & 4 deletions backend/iceberg_ports/readable_metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@

RawMetricMap = Optional[list[Dict[str, Any]]]
MetricMap = Optional[Dict[int, Any]]
BYTES_PER_MIB = 1024 * 1024


@dataclass(frozen=True)
Expand Down Expand Up @@ -64,7 +63,7 @@ def convert(self, raw_metrics: RawFileMetrics) -> Dict[str, Dict[str, Any]]:
field.qualified_name: {
"source_id": field.field_id,
"field_type": field.field_type,
"column_size_mib": self._column_size_mib(metrics.column_sizes, field.field_id),
"column_size_in_bytes": self._column_size_in_bytes(metrics.column_sizes, field.field_id),
"value_count": self._metric_value(metrics.value_counts, field.field_id),
"null_value_count": self._metric_value(metrics.null_value_counts, field.field_id),
"nan_value_count": self._metric_value(metrics.nan_value_counts, field.field_id),
Expand Down Expand Up @@ -164,12 +163,12 @@ def _metric_value(metric_map: MetricMap, field_id: int):
return metric_map.get(field_id)

@classmethod
def _column_size_mib(cls, column_sizes: MetricMap, field_id: int):
def _column_size_in_bytes(cls, column_sizes: MetricMap, field_id: int):
column_size_bytes = cls._metric_value(column_sizes, field_id)
if column_size_bytes is None:
return None

return column_size_bytes / BYTES_PER_MIB
return str(column_size_bytes)

def _decode_metric_bound(self, bounds: MetricMap, field: PrimitiveField):
encoded_bound = self._metric_value(bounds, field.field_id)
Expand Down
5 changes: 2 additions & 3 deletions frontend/src/components/DataFileReadableMetricsTable.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,14 @@ import {
formatReadableMetricValue,
type ReadableMetrics,
} from "../utils/readableMetrics";
import { stripByteUnitFromFieldName } from "../shared/lib/formatBytes";

interface DataFileReadableMetricsTableProps {
readableMetrics: ReadableMetrics;
}

const formatMetricLabel = (metricName: string): string =>
metricName === "column_size_mib"
? "column size (MiB)"
: metricName.replaceAll("_", " ");
stripByteUnitFromFieldName(metricName).replaceAll("_", " ");

const DataFileReadableMetricsTable = ({
readableMetrics,
Expand Down
17 changes: 15 additions & 2 deletions frontend/src/components/PanelContent.jsx
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
import { useState } from "react";
import CopyIconButton from "./CopyIconButton";
import {
formatBytesAsMebibytes,
isByteFieldName,
stripByteUnitFromFieldName,
} from "../shared/lib/formatBytes";
import {
UI_BODY_MUTED_ITALIC_CLASS,
UI_FIELD_LABEL_CLASS,
Expand Down Expand Up @@ -112,10 +117,16 @@ export function PanelDetailRow({
relaxedCollapse = false,
collapseLineCount = DEFAULT_COLLAPSE_LINES,
}) {
const isByteField = isByteFieldName(String(label));
const displayLabel = isByteField
? stripByteUnitFromFieldName(String(label))
: label;
const displayValue =
typeof value === "object" && value !== null
? JSON.stringify(value, null, 2)
: value;
: isByteField
? formatBytesAsMebibytes(value)
: value;

const textToCopy =
displayValue != null && displayValue !== "" ? String(displayValue) : "";
Expand All @@ -127,7 +138,9 @@ export function PanelDetailRow({
return (
<div>
<div className="flex items-center justify-between mb-1 gap-2">
<span className={`block ${PANEL_FIELD_LABEL_CLASS}`}>{label}</span>
<span className={`block ${PANEL_FIELD_LABEL_CLASS}`}>
{displayLabel}
</span>
{isCollapsible && (
<button
type="button"
Expand Down
Loading
Loading