Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions diffly/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,15 @@ def main(
)
),
] = [],
percentage_decimals: Annotated[
int,
typer.Option(
Comment thread
Copilot marked this conversation as resolved.
help=(
"The number of decimal places to show for percentages in the summary, "
"e.g. column match rates."
)
),
] = 2,
) -> None:
"""Compare two `parquet` files and print the comparison result."""
if hidden_columns:
Expand Down Expand Up @@ -202,6 +211,7 @@ def main(
hidden_columns=hidden_column,
data_metrics=data_metrics,
change_metrics=change_metrics,
percentage_decimals=percentage_decimals,
)
if output_json:
typer.echo(summary.to_json())
Expand Down
4 changes: 4 additions & 0 deletions diffly/comparison.py
Original file line number Diff line number Diff line change
Expand Up @@ -942,6 +942,7 @@ def summary(
hidden_columns: list[str] | None = None,
data_metrics: Mapping[str, DataMetricFn | DataMetric] | None = None,
change_metrics: Mapping[str, ChangeMetricFn | ChangeMetric] | None = None,
percentage_decimals: int = 2,
) -> Summary:
"""Generate a summary of all aspects of the comparison.

Expand Down Expand Up @@ -995,6 +996,8 @@ def summary(
(default), no metrics are computed; presets are not applied
automatically. Prefer short labels — the summary has a fixed width and
many or long labels degrade rendering.
percentage_decimals: The number of decimal places to show for percentages
in the summary, e.g. column match rates.

Returns:
A summary which can be printed or written to a file.
Expand Down Expand Up @@ -1033,6 +1036,7 @@ def summary(
hidden_columns=hidden_columns,
data_metrics=resolved_data_metrics,
change_metrics=resolved_change_metrics,
percentage_decimals=percentage_decimals,
)

# ----------------------------------- UTILITIES ----------------------------------- #
Expand Down
32 changes: 21 additions & 11 deletions diffly/summary.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,10 @@ def __init__(
hidden_columns: list[str] | None,
data_metrics: Mapping[str, DataMetric] | None,
change_metrics: Mapping[str, ChangeMetric] | None,
percentage_decimals: int,
):
self.slim = slim
self.percentage_decimals = percentage_decimals
self._data = _compute_summary_data(
comparison,
show_perfect_column_matches=show_perfect_column_matches,
Expand Down Expand Up @@ -392,9 +394,13 @@ def _section_row_counts(self) -> RenderableType:
if rows.n_left > 0:
fraction_rows_right = rows.n_right / rows.n_left
if fraction_rows_right > 1:
gain_loss = f"(+{(fraction_rows_right - 1):.2%})"
gain_loss = (
f"(+{(fraction_rows_right - 1):.{self.percentage_decimals}%})"
)
elif fraction_rows_right < 1:
gain_loss = f"(-{(1 - fraction_rows_right):.2%})"
gain_loss = (
f"(-{(1 - fraction_rows_right):.{self.percentage_decimals}%})"
)
else:
gain_loss = "(no change)"

Expand Down Expand Up @@ -424,6 +430,7 @@ def _section_row_matches(self) -> RenderableType:
assert rows.n_joined_unequal is not None
assert rows.n_right_only is not None
n_joined = rows.n_joined_equal + rows.n_joined_unequal
decimals = self.percentage_decimals

columns: list[RenderableType] = []
num_dummy_cols = 5
Expand Down Expand Up @@ -512,31 +519,31 @@ def _section_row_matches(self) -> RenderableType:
grid.add_row(
f"{rows.n_left_only:,}",
f"{self._data._truncated_left_name} only",
f"({_format_fraction_as_percentage(fraction_left_only)})",
f"({_format_fraction_as_percentage(fraction_left_only, decimals)})",
)
grid.add_section()
if rows.n_joined_equal > 0:
fraction_equal = rows.n_joined_equal / n_joined
grid.add_row(
f"{rows.n_joined_equal:,}",
"equal",
f"({_format_fraction_as_percentage(fraction_equal)})",
f"({_format_fraction_as_percentage(fraction_equal, decimals)})",
)
grid.add_section()
if rows.n_joined_unequal > 0:
fraction_unequal = rows.n_joined_unequal / n_joined
grid.add_row(
f"{rows.n_joined_unequal:,}",
"unequal",
f"({_format_fraction_as_percentage(fraction_unequal)})",
f"({_format_fraction_as_percentage(fraction_unequal, decimals)})",
)
grid.add_section()
if rows.n_right_only > 0:
fraction_right_only = rows.n_right_only / rows.n_right
grid.add_row(
f"{rows.n_right_only:,}",
f"{self._data._truncated_right_name} only",
f"({_format_fraction_as_percentage(fraction_right_only)})",
f"({_format_fraction_as_percentage(fraction_right_only, decimals)})",
)
columns.append(grid)

Expand Down Expand Up @@ -594,10 +601,11 @@ def _section_columns(self) -> RenderableType:
if has_top_changes_column:
matches.add_column("Top Changes", justify="right")
max_col_len = max(len(c.name) for c in columns)
decimals = self.percentage_decimals
for col in columns:
row_items: list[RenderableType] = [
Text(col.name, style="cyan"),
f"{_format_fraction_as_percentage(col.match_rate)}",
f"{_format_fraction_as_percentage(col.match_rate, decimals)}",
]
for label in self._data._change_metric_labels:
value = (
Expand Down Expand Up @@ -1228,22 +1236,24 @@ def _format_colname(name: str) -> str:
return f"[cyan]{name}[/cyan]"


def _format_fraction_as_percentage(fraction: float) -> str:
"""Format a fraction as a percentage with two decimal places.
def _format_fraction_as_percentage(fraction: float, decimals: int) -> str:
"""Format a fraction as a percentage with the given number of decimal places.

Only shows 0%/100% if the fraction is exactly 0 or 1, respectively.

Args:
fraction: The fraction to format as a percentage. Must be in the range [0, 1].
decimals: The number of decimal places to show.

Returns:
The formatted percentage, including a percent sign.
"""
assert 0 <= fraction <= 1, "Fraction must be between 0 and 1"
percentage = fraction * 100
if percentage not in [0, 100]:
percentage = min(max(percentage, 0.01), 99.99)
return f"{percentage:.2f}%"
smallest = 10**-decimals
percentage = min(max(percentage, smallest), 100 - smallest)
return f"{percentage:.{decimals}f}%"


def _yellow(raw: Any) -> str:
Expand Down
8 changes: 8 additions & 0 deletions diffly/testing.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@ def assert_collection_equal(
hidden_columns: list[str] | None = None,
data_metrics: Mapping[str, DataMetricFn | DataMetric] | None = None,
change_metrics: Mapping[str, ChangeMetricFn | ChangeMetric] | None = None,
percentage_decimals: int = 2,
) -> None:
"""Assert that two :mod:`dataframely` collections are equal.

Expand Down Expand Up @@ -159,6 +160,8 @@ def assert_collection_equal(
target other column types, construct the metric explicitly with a column
selector. See :mod:`diffly.metrics` for presets. When ``None`` (default), no
change metrics are computed.
percentage_decimals: The number of decimal places to show for percentages in
the summary, e.g. column match rates.

Raises:
CollectionComparisonAssertionError: If the collections are not equal.
Expand Down Expand Up @@ -217,6 +220,7 @@ def assert_collection_equal(
hidden_columns=hidden_columns,
data_metrics=data_metrics,
change_metrics=change_metrics,
percentage_decimals=percentage_decimals,
)
)
}"""
Expand Down Expand Up @@ -251,6 +255,7 @@ def assert_frame_equal(
hidden_columns: list[str] | None = None,
data_metrics: Mapping[str, DataMetricFn | DataMetric] | None = None,
change_metrics: Mapping[str, ChangeMetricFn | ChangeMetric] | None = None,
percentage_decimals: int = 2,
) -> None:
"""Assert that two :mod:`polars` data frames are equal.

Expand Down Expand Up @@ -313,6 +318,8 @@ def assert_frame_equal(
target other column types, construct the metric explicitly with a column
selector. See :mod:`diffly.metrics` for presets. When ``None`` (default), no
change metrics are computed.
percentage_decimals: The number of decimal places to show for percentages in
the summary, e.g. column match rates.

Raises:
FrameComparisonAssertionError: If the data frames are not equal.
Expand Down Expand Up @@ -360,6 +367,7 @@ def assert_frame_equal(
hidden_columns=hidden_columns,
data_metrics=data_metrics,
change_metrics=change_metrics,
percentage_decimals=percentage_decimals,
)
text = textwrap.indent(str(summary), " " * 2)
raise FrameComparisonAssertionError(
Expand Down
21 changes: 21 additions & 0 deletions tests/cli/test_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,27 @@ def test_cli_hidden_columns_alias_warns(tmp_path: Path) -> None:
assert result.exit_code == 0


def test_cli_percentage_decimals(tmp_path: Path) -> None:
left = pl.DataFrame({"id": [1, 2, 3], "x": [1, 2, 3]})
right = pl.DataFrame({"id": [1, 2, 3], "x": [1, 2, 4]})
left.write_parquet(tmp_path / "left.parquet")
right.write_parquet(tmp_path / "right.parquet")

result = runner.invoke(
app,
[
str(tmp_path / "left.parquet"),
str(tmp_path / "right.parquet"),
"--primary-key",
"id",
"--percentage-decimals",
"3",
],
)
assert result.exit_code == 0
assert "66.667%" in result.output


@pytest.mark.parametrize(
"flag, name, expected",
[
Expand Down
43 changes: 34 additions & 9 deletions tests/summary/test_summary.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,18 +65,43 @@ def test_summary_all_columns_join_cols(


@pytest.mark.parametrize(
"fraction, expected",
"fraction, decimals, expected",
[
(0.5, "50.00%"),
(0.123456, "12.35%"),
(0.000001, "0.01%"),
(0.999999, "99.99%"),
(0, "0.00%"),
(1, "100.00%"),
(0.5, 2, "50.00%"),
(0.123456, 2, "12.35%"),
(0.000001, 2, "0.01%"),
(0.999999, 2, "99.99%"),
(0, 2, "0.00%"),
(1, 2, "100.00%"),
(0.123456, 0, "12%"),
(0.000001, 0, "1%"),
(0.999999, 0, "99%"),
(0, 0, "0%"),
(1, 0, "100%"),
(0.123456, 4, "12.3456%"),
(0.0000001, 4, "0.0001%"),
(0.9999999, 4, "99.9999%"),
],
)
def test__format_fraction_as_percentage(fraction: float, expected: str) -> None:
assert _format_fraction_as_percentage(fraction) == expected
def test__format_fraction_as_percentage(
fraction: float, decimals: int, expected: str
) -> None:
assert _format_fraction_as_percentage(fraction, decimals) == expected


def test_summary_percentage_decimals() -> None:
Comment thread
MoritzPotthoffQC marked this conversation as resolved.
Outdated
left = pl.DataFrame({"id": [1, 2, 3], "x": [1, 2, 3]})
right = pl.DataFrame({"id": [1, 2, 3, 4], "x": [1, 2, 4, 5]})
comp = compare_frames(left, right, primary_key="id")

default = str(comp.summary())
assert "66.67%" in default
assert "(+33.33%)" in default

summary = str(comp.summary(percentage_decimals=4))
assert "66.6667%" in summary
assert "(+33.3333%)" in summary
assert "(25.0000%)" in summary


def test_validate_primary_key_hidden_columns() -> None:
Expand Down
Loading