Skip to content

API Reference: Reporting

Report generator

auditml.reporting.report_generator

Unified report generator for a complete AuditML audit.

Orchestrates individual attack reports, DP comparison, and cross-attack comparison into a single output directory with a master summary.

Expected workflow:

  1. Run one or more attacks against a target model.
  2. Optionally run the same attacks against a DP-trained model.
  3. Pass all results to ReportGenerator.
  4. Call generate() to produce the full report.

Output structure::

<output_dir>/
├── summary.txt              # Master text summary
├── audit_summary.json       # Machine-readable summary
├── attacks/
│   ├── mia_threshold/       # Per-attack report dirs
│   ├── mia_shadow/
│   ├── model_inversion/
│   └── attribute_inference/
├── attack_comparison/       # Cross-attack comparison (if 2+ attacks)
│   ├── attack_comparison.json
│   ├── attack_comparison_bar.png
│   └── attack_roc_overlay.png
└── dp_comparison/           # DP vs non-DP (if DP results provided)
    ├── comparison.json
    ├── roc_comparison.png
    └── ...

ReportGenerator

Unified report generator for a full privacy audit.

Parameters:

Name Type Description Default
experiment_name str

Human-readable name for this audit (used in titles and filenames).

 'audit'
attack_results dict[str, tuple[BaseAttack, AttackResult]] | None

Mapping from attack name to (attack_instance, result) tuples. The attack instances are used to call per-attack generate_report().

 None
dp_attack_results dict[str, tuple[BaseAttack, AttackResult]] | None

Same structure but for attacks run against the DP model. If provided, a DP comparison section is added.

 None
epsilon float | None

The privacy budget used for DP training. Required if dp_attack_results is provided.

 None
model_accuracy dict[str, float] | None

Optional dict {"baseline": float, "dp": float} with model test accuracy for the utility vs privacy analysis.

 None
Source code in src/auditml/reporting/report_generator.py 
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
class ReportGenerator:
    """Unified report generator for a full privacy audit.

    Parameters
    ----------
    experiment_name:
        Human-readable name for this audit (used in titles and filenames).
    attack_results:
        Mapping from attack name to ``(attack_instance, result)`` tuples.
        The attack instances are used to call per-attack ``generate_report()``.
    dp_attack_results:
        Same structure but for attacks run against the DP model.
        If provided, a DP comparison section is added.
    epsilon:
        The privacy budget used for DP training. Required if
        ``dp_attack_results`` is provided.
    model_accuracy:
        Optional dict ``{"baseline": float, "dp": float}`` with model
        test accuracy for the utility vs privacy analysis.
    """

    def __init__(
        self,
        experiment_name: str = "audit",
        attack_results: dict[str, tuple[BaseAttack, AttackResult]] | None = None,
        dp_attack_results: dict[str, tuple[BaseAttack, AttackResult]] | None = None,
        epsilon: float | None = None,
        model_accuracy: dict[str, float] | None = None,
    ) -> None:
        self.experiment_name = experiment_name
        self.attack_results = attack_results or {}
        self.dp_attack_results = dp_attack_results or {}
        self.epsilon = epsilon
        self.model_accuracy = model_accuracy or {}

    # ------------------------------------------------------------------
    # Main entry point
    # ------------------------------------------------------------------

    def generate(self, output_dir: str | Path) -> Path:
        """Generate the complete audit report.

        Parameters
        ----------
        output_dir:
            Root directory for the report. Created if it doesn't exist.

        Returns
        -------
        Path
            The output directory.
        """
        out = Path(output_dir)
        out.mkdir(parents=True, exist_ok=True)

        summary_data: dict[str, Any] = {
            "experiment_name": self.experiment_name,
            "timestamp": datetime.now().isoformat(),
            "num_attacks": len(self.attack_results),
            "attack_names": list(self.attack_results.keys()),
            "dp_enabled": len(self.dp_attack_results) > 0,
        }

        # 1. Per-attack reports
        attack_metrics = self._generate_attack_reports(out)
        summary_data["attack_metrics"] = attack_metrics

        # 2. Cross-attack comparison (if 2+ attacks)
        if len(self.attack_results) >= 2:
            self._generate_attack_comparison(out)
            summary_data["attack_comparison"] = True
        else:
            summary_data["attack_comparison"] = False

        # 3. DP comparison (if DP results provided)
        dp_summaries = {}
        if self.dp_attack_results:
            dp_summaries = self._generate_dp_comparisons(out)
            summary_data["dp_comparison"] = dp_summaries
            summary_data["epsilon"] = self.epsilon
        else:
            summary_data["dp_comparison"] = None

        # 4. Model accuracy
        if self.model_accuracy:
            summary_data["model_accuracy"] = self.model_accuracy

        # 5. Write master JSON
        with open(out / "audit_summary.json", "w") as f:
            json.dump(summary_data, f, indent=2, default=str)

        # 6. Write master text summary
        self._write_master_summary(out / "summary.txt", summary_data, dp_summaries)

        # 7. Generate self-contained HTML report
        try:
            from auditml.reporting.html_report import generate_html_report
            dp_metrics = {
                name: attack.evaluate()
                for name, (attack, _) in self.dp_attack_results.items()
            } if self.dp_attack_results else None
            html_path = generate_html_report(
                output_dir=out,
                experiment_name=self.experiment_name,
                attack_results=self.attack_results,
                dp_attack_results=self.dp_attack_results or None,
                attack_metrics=attack_metrics,
                dp_attack_metrics=dp_metrics,
                model_accuracy=self.model_accuracy or None,
                epsilon=self.epsilon,
            )
            summary_data["html_report"] = str(html_path)
            logger.info("HTML report generated at %s", html_path)
        except Exception as exc:
            logger.warning("HTML report generation failed: %s", exc, exc_info=True)

        logger.info("Report generated at %s", out)
        return out

    # ------------------------------------------------------------------
    # Per-attack reports
    # ------------------------------------------------------------------

    def _generate_attack_reports(
        self, output_dir: Path,
    ) -> dict[str, dict[str, float]]:
        """Generate individual report for each attack.

        Returns mapping from attack name to its metrics dict.
        """
        attacks_dir = output_dir / "attacks"
        attacks_dir.mkdir(exist_ok=True)

        metrics_all: dict[str, dict[str, float]] = {}

        for name, (attack, result) in self.attack_results.items():
            attack_dir = attacks_dir / name
            logger.info("Generating report for %s ...", name)

            if hasattr(attack, "generate_report"):
                attack.generate_report(attack_dir)
            else:
                # Fallback: just save metrics
                attack_dir.mkdir(parents=True, exist_ok=True)
                metrics = attack.evaluate()
                with open(attack_dir / "metrics.json", "w") as f:
                    json.dump(metrics, f, indent=2)

            metrics_all[name] = attack.evaluate()

        return metrics_all

    # ------------------------------------------------------------------
    # Cross-attack comparison
    # ------------------------------------------------------------------

    def _generate_attack_comparison(self, output_dir: Path) -> None:
        """Generate cross-attack comparison report."""
        results_only = {
            name: result
            for name, (_, result) in self.attack_results.items()
        }
        comp = AttackComparison(results_only)
        comp.generate_report(output_dir / "attack_comparison")

    # ------------------------------------------------------------------
    # DP comparison
    # ------------------------------------------------------------------

    def _generate_dp_comparisons(
        self, output_dir: Path,
    ) -> dict[str, dict[str, Any]]:
        """Generate DP vs non-DP comparison for each shared attack.

        Returns mapping from attack name to privacy gain summary.
        """
        dp_dir = output_dir / "dp_comparison"
        dp_dir.mkdir(exist_ok=True)

        dp_summaries: dict[str, dict[str, Any]] = {}

        for name in self.attack_results:
            if name not in self.dp_attack_results:
                continue

            _, baseline_result = self.attack_results[name]
            _, dp_result = self.dp_attack_results[name]

            comp = DPComparison(
                baseline_result=baseline_result,
                dp_result=dp_result,
                epsilon=self.epsilon,
                model_accuracy=self.model_accuracy,
            )
            comp.generate_report(dp_dir / name)
            dp_summaries[name] = comp.compute_privacy_gain()

        return dp_summaries

    # ------------------------------------------------------------------
    # Master summary
    # ------------------------------------------------------------------

    def _write_master_summary(
        self,
        path: Path,
        summary_data: dict[str, Any],
        dp_summaries: dict[str, dict[str, Any]],
    ) -> None:
        """Write the top-level human-readable summary."""
        lines = [
            "=" * 70,
            f"AuditML Privacy Audit Report — {self.experiment_name}",
            "=" * 70,
            "",
            f"Generated: {summary_data['timestamp']}",
            f"Attacks run: {summary_data['num_attacks']}",
            f"Attack types: {', '.join(summary_data['attack_names'])}",
            f"DP comparison: {'Yes' if summary_data['dp_enabled'] else 'No'}",
        ]

        if self.model_accuracy:
            lines.append("")
            lines.append("--- Model Utility ---")
            for k, v in self.model_accuracy.items():
                lines.append(f"  {k}: {v:.4f}")

        # Per-attack metrics
        lines.append("")
        lines.append("--- Attack Results ---")
        attack_metrics = summary_data.get("attack_metrics", {})

        if attack_metrics:
            header = f"  {'Attack':<25s}  {'Accuracy':>10s}  {'AUC-ROC':>10s}  {'F1':>10s}"
            lines.append(header)
            lines.append(f"  {'-'*25}  {'-'*10}  {'-'*10}  {'-'*10}")
            for name, m in attack_metrics.items():
                lines.append(
                    f"  {name:<25s}  {m.get('accuracy', 0.0):>10.4f}  "
                    f"{m.get('auc_roc', 0.0):>10.4f}  {m.get('f1', 0.0):>10.4f}"
                )

        # DP comparison
        if dp_summaries:
            lines.append("")
            lines.append("--- DP Privacy Gain ---")
            if self.epsilon is not None:
                lines.append(f"  Epsilon: {self.epsilon:.2f}")
            for name, gain in dp_summaries.items():
                lines.append(
                    f"  {name}: accuracy reduction = "
                    f"{gain.get('attack_accuracy_reduction', 0.0):.4f}, "
                    f"AUC-ROC reduction = "
                    f"{gain.get('auc_roc_reduction', 0.0):.4f}"
                )

        # Report structure
        lines.append("")
        lines.append("--- Report Files ---")
        lines.append("  attacks/<attack_name>/     Per-attack detailed reports")
        if summary_data.get("attack_comparison"):
            lines.append("  attack_comparison/         Cross-attack comparison")
        if summary_data.get("dp_comparison"):
            lines.append("  dp_comparison/<attack>/    DP vs non-DP comparison")
        lines.append("  audit_summary.json         Machine-readable summary")

        lines.append("")
        path.write_text("\n".join(lines))

generate(output_dir: str | Path) -> Path

Generate the complete audit report.

Parameters:

Name Type Description Default
output_dir str | Path

Root directory for the report. Created if it doesn't exist.

 required

Returns:

Type Description
Path

The output directory.
Source code in src/auditml/reporting/report_generator.py 
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
def generate(self, output_dir: str | Path) -> Path:
    """Generate the complete audit report.

    Parameters
    ----------
    output_dir:
        Root directory for the report. Created if it doesn't exist.

    Returns
    -------
    Path
        The output directory.
    """
    out = Path(output_dir)
    out.mkdir(parents=True, exist_ok=True)

    summary_data: dict[str, Any] = {
        "experiment_name": self.experiment_name,
        "timestamp": datetime.now().isoformat(),
        "num_attacks": len(self.attack_results),
        "attack_names": list(self.attack_results.keys()),
        "dp_enabled": len(self.dp_attack_results) > 0,
    }

    # 1. Per-attack reports
    attack_metrics = self._generate_attack_reports(out)
    summary_data["attack_metrics"] = attack_metrics

    # 2. Cross-attack comparison (if 2+ attacks)
    if len(self.attack_results) >= 2:
        self._generate_attack_comparison(out)
        summary_data["attack_comparison"] = True
    else:
        summary_data["attack_comparison"] = False

    # 3. DP comparison (if DP results provided)
    dp_summaries = {}
    if self.dp_attack_results:
        dp_summaries = self._generate_dp_comparisons(out)
        summary_data["dp_comparison"] = dp_summaries
        summary_data["epsilon"] = self.epsilon
    else:
        summary_data["dp_comparison"] = None

    # 4. Model accuracy
    if self.model_accuracy:
        summary_data["model_accuracy"] = self.model_accuracy

    # 5. Write master JSON
    with open(out / "audit_summary.json", "w") as f:
        json.dump(summary_data, f, indent=2, default=str)

    # 6. Write master text summary
    self._write_master_summary(out / "summary.txt", summary_data, dp_summaries)

    # 7. Generate self-contained HTML report
    try:
        from auditml.reporting.html_report import generate_html_report
        dp_metrics = {
            name: attack.evaluate()
            for name, (attack, _) in self.dp_attack_results.items()
        } if self.dp_attack_results else None
        html_path = generate_html_report(
            output_dir=out,
            experiment_name=self.experiment_name,
            attack_results=self.attack_results,
            dp_attack_results=self.dp_attack_results or None,
            attack_metrics=attack_metrics,
            dp_attack_metrics=dp_metrics,
            model_accuracy=self.model_accuracy or None,
            epsilon=self.epsilon,
        )
        summary_data["html_report"] = str(html_path)
        logger.info("HTML report generated at %s", html_path)
    except Exception as exc:
        logger.warning("HTML report generation failed: %s", exc, exc_info=True)

    logger.info("Report generated at %s", out)
    return out

Attack comparison

auditml.reporting.attack_comparison

Cross-attack comparison module.

Compares results from multiple attack types run against the same model, answering questions like:

  • Which attack is the most effective at inferring membership?
  • Which classes are most vulnerable, and does this vary by attack?
  • How do confidence-score distributions differ across attacks?

Expected workflow:

  1. Run two or more attacks (threshold MIA, shadow MIA, model inversion, attribute inference) against the same target model.
  2. Pass all AttackResult objects to AttackComparison.
  3. Call rank_attacks(), generate_report(), etc.

This module is complementary to DPComparison (Task 2.11), which compares the same attack across DP vs non-DP models.

AttackComparison

Compare effectiveness across multiple attack types.

Parameters:

Name Type Description Default
results dict[str, AttackResult]

Mapping from attack name to its AttackResult. At least two entries are required for a meaningful comparison. Example::

{
    "mia_threshold": threshold_result,
    "mia_shadow": shadow_result,
    "model_inversion": inversion_result,
}
required
Source code in src/auditml/reporting/attack_comparison.py 
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
class AttackComparison:
    """Compare effectiveness across multiple attack types.

    Parameters
    ----------
    results:
        Mapping from attack name to its ``AttackResult``. At least two
        entries are required for a meaningful comparison.  Example::

            {
                "mia_threshold": threshold_result,
                "mia_shadow": shadow_result,
                "model_inversion": inversion_result,
            }
    """

    def __init__(self, results: dict[str, AttackResult]) -> None:
        if len(results) < 1:
            raise ValueError("At least one AttackResult is required.")
        self.results = results
        self._metrics: dict[str, dict[str, float]] = {}

    # ------------------------------------------------------------------
    # Metrics
    # ------------------------------------------------------------------

    def compute_all_metrics(self) -> dict[str, dict[str, float]]:
        """Compute standard metrics for every attack.

        Returns
        -------
        dict[str, dict[str, float]]
            Mapping from attack name to its metric dictionary.
        """
        if not self._metrics:
            for name, result in self.results.items():
                self._metrics[name] = BaseAttack._compute_metrics(
                    result.predictions,
                    result.ground_truth,
                    result.confidence_scores,
                )
        return self._metrics

    def rank_attacks(self, metric: str = "auc_roc") -> list[tuple[str, float]]:
        """Rank attacks by a chosen metric (descending).

        Parameters
        ----------
        metric:
            The metric key to rank by (default ``"auc_roc"``).

        Returns
        -------
        list of (attack_name, metric_value), sorted descending.
        """
        all_metrics = self.compute_all_metrics()
        ranked = [
            (name, m.get(metric, 0.0))
            for name, m in all_metrics.items()
        ]
        ranked.sort(key=lambda x: x[1], reverse=True)
        return ranked

    def best_attack(self, metric: str = "auc_roc") -> str:
        """Return the name of the most effective attack.

        Parameters
        ----------
        metric:
            The metric to compare by.

        Returns
        -------
        str
            Name of the top-ranked attack.
        """
        return self.rank_attacks(metric)[0][0]

    def summary_table(self) -> dict[str, dict[str, float]]:
        """Build a table of all attacks and all metrics.

        Returns
        -------
        dict[str, dict[str, float]]
            Outer key = attack name, inner dict = metric values.
        """
        return self.compute_all_metrics()

    def summary_dict(self) -> dict[str, Any]:
        """Return a comprehensive summary for serialisation.

        Returns
        -------
        dict containing ``metrics``, ``ranking``, ``best_attack``, and
        ``attack_names``.
        """
        return {
            "attack_names": list(self.results.keys()),
            "metrics": self.compute_all_metrics(),
            "ranking_by_auc_roc": self.rank_attacks("auc_roc"),
            "ranking_by_accuracy": self.rank_attacks("accuracy"),
            "best_attack_auc_roc": self.best_attack("auc_roc"),
            "best_attack_accuracy": self.best_attack("accuracy"),
        }

    # ------------------------------------------------------------------
    # Report generation
    # ------------------------------------------------------------------

    def generate_report(self, output_dir: str | Path) -> Path:
        """Generate a cross-attack comparison report.

        Creates:

        - ``attack_comparison.json`` — full metrics and rankings
        - ``attack_comparison_bar.png`` — grouped bar chart of metrics
        - ``attack_roc_overlay.png`` — overlaid ROC curves
        - ``summary.txt`` — human-readable summary

        Parameters
        ----------
        output_dir:
            Directory to write report files.

        Returns
        -------
        Path
            The output directory.
        """
        from auditml.reporting.attack_visualization import (
            plot_attack_comparison_bar,
            plot_attack_roc_overlay,
        )

        out = Path(output_dir)
        out.mkdir(parents=True, exist_ok=True)

        # 1. JSON data
        summary = self.summary_dict()
        with open(out / "attack_comparison.json", "w") as f:
            json.dump(summary, f, indent=2, default=str)

        # 2. Bar chart
        plot_attack_comparison_bar(
            metrics=self.compute_all_metrics(),
            save_path=out / "attack_comparison_bar.png",
        )

        # 3. ROC overlay
        plot_attack_roc_overlay(
            results=self.results,
            save_path=out / "attack_roc_overlay.png",
        )

        # 4. Summary text
        self._write_summary(out / "summary.txt")

        return out

    def _write_summary(self, path: Path) -> None:
        """Write a human-readable comparison summary."""
        all_metrics = self.compute_all_metrics()
        ranking = self.rank_attacks("auc_roc")

        lines = [
            "=" * 60,
            "AuditML — Cross-Attack Comparison Report",
            "=" * 60,
            "",
            f"Attacks compared: {len(self.results)}",
            f"Attack names: {', '.join(self.results.keys())}",
            "",
            "--- Ranking by AUC-ROC ---",
        ]
        for i, (name, val) in enumerate(ranking, 1):
            lines.append(f"  {i}. {name:<25s}  AUC-ROC = {val:.4f}")

        lines.append("")
        lines.append("--- Full Metrics Table ---")

        # Header
        metric_keys = ["accuracy", "precision", "recall", "f1", "auc_roc", "auc_pr"]
        header = f"  {'Attack':<25s}"
        for mk in metric_keys:
            header += f"  {mk:>10s}"
        lines.append(header)
        lines.append(f"  {'-'*25}" + f"  {'-'*10}" * len(metric_keys))

        for name in self.results:
            m = all_metrics[name]
            row = f"  {name:<25s}"
            for mk in metric_keys:
                row += f"  {m.get(mk, 0.0):>10.4f}"
            lines.append(row)

        lines.append("")
        lines.append(f"Best attack (AUC-ROC): {self.best_attack('auc_roc')}")
        lines.append(f"Best attack (Accuracy): {self.best_attack('accuracy')}")

        lines.append("")
        path.write_text("\n".join(lines))

compute_all_metrics() -> dict[str, dict[str, float]]

Compute standard metrics for every attack.

Returns:

Type Description
dict[str, dict[str, float]]

Mapping from attack name to its metric dictionary.
Source code in src/auditml/reporting/attack_comparison.py 
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
def compute_all_metrics(self) -> dict[str, dict[str, float]]:
    """Compute standard metrics for every attack.

    Returns
    -------
    dict[str, dict[str, float]]
        Mapping from attack name to its metric dictionary.
    """
    if not self._metrics:
        for name, result in self.results.items():
            self._metrics[name] = BaseAttack._compute_metrics(
                result.predictions,
                result.ground_truth,
                result.confidence_scores,
            )
    return self._metrics

rank_attacks(metric: str = 'auc_roc') -> list[tuple[str, float]]

Rank attacks by a chosen metric (descending).

Parameters:

Name Type Description Default
metric str

The metric key to rank by (default "auc_roc").

 'auc_roc'

Returns:

Type Description
list of (attack_name, metric_value), sorted descending.
Source code in src/auditml/reporting/attack_comparison.py 
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
def rank_attacks(self, metric: str = "auc_roc") -> list[tuple[str, float]]:
    """Rank attacks by a chosen metric (descending).

    Parameters
    ----------
    metric:
        The metric key to rank by (default ``"auc_roc"``).

    Returns
    -------
    list of (attack_name, metric_value), sorted descending.
    """
    all_metrics = self.compute_all_metrics()
    ranked = [
        (name, m.get(metric, 0.0))
        for name, m in all_metrics.items()
    ]
    ranked.sort(key=lambda x: x[1], reverse=True)
    return ranked

best_attack(metric: str = 'auc_roc') -> str

Return the name of the most effective attack.

Parameters:

Name Type Description Default
metric str

The metric to compare by.

 'auc_roc'

Returns:

Type Description
str

Name of the top-ranked attack.
Source code in src/auditml/reporting/attack_comparison.py 
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
def best_attack(self, metric: str = "auc_roc") -> str:
    """Return the name of the most effective attack.

    Parameters
    ----------
    metric:
        The metric to compare by.

    Returns
    -------
    str
        Name of the top-ranked attack.
    """
    return self.rank_attacks(metric)[0][0]

summary_table() -> dict[str, dict[str, float]]

Build a table of all attacks and all metrics.

Returns:

Type Description
dict[str, dict[str, float]]

Outer key = attack name, inner dict = metric values.
Source code in src/auditml/reporting/attack_comparison.py 
112
113
114
115
116
117
118
119
120
def summary_table(self) -> dict[str, dict[str, float]]:
    """Build a table of all attacks and all metrics.

    Returns
    -------
    dict[str, dict[str, float]]
        Outer key = attack name, inner dict = metric values.
    """
    return self.compute_all_metrics()

summary_dict() -> dict[str, Any]

Return a comprehensive summary for serialisation.

Returns:

Type Description
dict containing ``metrics``, ``ranking``, ``best_attack``, and
``attack_names``.
Source code in src/auditml/reporting/attack_comparison.py 
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
def summary_dict(self) -> dict[str, Any]:
    """Return a comprehensive summary for serialisation.

    Returns
    -------
    dict containing ``metrics``, ``ranking``, ``best_attack``, and
    ``attack_names``.
    """
    return {
        "attack_names": list(self.results.keys()),
        "metrics": self.compute_all_metrics(),
        "ranking_by_auc_roc": self.rank_attacks("auc_roc"),
        "ranking_by_accuracy": self.rank_attacks("accuracy"),
        "best_attack_auc_roc": self.best_attack("auc_roc"),
        "best_attack_accuracy": self.best_attack("accuracy"),
    }

generate_report(output_dir: str | Path) -> Path

Generate a cross-attack comparison report.

Creates:

  • attack_comparison.json — full metrics and rankings
  • attack_comparison_bar.png — grouped bar chart of metrics
  • attack_roc_overlay.png — overlaid ROC curves
  • summary.txt — human-readable summary

Parameters:

Name Type Description Default
output_dir str | Path

Directory to write report files.

 required

Returns:

Type Description
Path

The output directory.
Source code in src/auditml/reporting/attack_comparison.py 
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
def generate_report(self, output_dir: str | Path) -> Path:
    """Generate a cross-attack comparison report.

    Creates:

    - ``attack_comparison.json`` — full metrics and rankings
    - ``attack_comparison_bar.png`` — grouped bar chart of metrics
    - ``attack_roc_overlay.png`` — overlaid ROC curves
    - ``summary.txt`` — human-readable summary

    Parameters
    ----------
    output_dir:
        Directory to write report files.

    Returns
    -------
    Path
        The output directory.
    """
    from auditml.reporting.attack_visualization import (
        plot_attack_comparison_bar,
        plot_attack_roc_overlay,
    )

    out = Path(output_dir)
    out.mkdir(parents=True, exist_ok=True)

    # 1. JSON data
    summary = self.summary_dict()
    with open(out / "attack_comparison.json", "w") as f:
        json.dump(summary, f, indent=2, default=str)

    # 2. Bar chart
    plot_attack_comparison_bar(
        metrics=self.compute_all_metrics(),
        save_path=out / "attack_comparison_bar.png",
    )

    # 3. ROC overlay
    plot_attack_roc_overlay(
        results=self.results,
        save_path=out / "attack_roc_overlay.png",
    )

    # 4. Summary text
    self._write_summary(out / "summary.txt")

    return out

DP vs Non-DP comparison

auditml.reporting.comparison

DP vs Non-DP comparison module.

Compares attack results obtained from a standard (non-private) model against a differentially-private (DP) model. The comparison answers the core question: does DP training reduce privacy leakage?

Expected workflow:

  1. Train a standard model and run attacks → baseline_results.
  2. Train a DP model (same architecture, same data) and run the same attacks → dp_results.
  3. Pass both to DPComparison to compute deltas, generate plots, and produce a combined report.

The module is attack-agnostic — it works with any AttackResult from any of the four attack types.

DPComparison

Compare attack effectiveness between standard and DP models.

Parameters:

Name Type Description Default
baseline_result AttackResult

Attack result from the standard (non-DP) model.

 required
dp_result AttackResult

Attack result from the DP-trained model.

 required
baseline_metrics dict[str, float] | None

Pre-computed metrics for the baseline. If None, computed from the result arrays.

 None
dp_metrics dict[str, float] | None

Pre-computed metrics for the DP model. If None, computed from the result arrays.

 None
epsilon float | None

The privacy budget (epsilon) used during DP training.

 None
model_accuracy dict[str, float] | None

Optional dict with {"baseline": float, "dp": float} holding the utility (test accuracy) of each model.

 None
Source code in src/auditml/reporting/comparison.py 
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
class DPComparison:
    """Compare attack effectiveness between standard and DP models.

    Parameters
    ----------
    baseline_result:
        Attack result from the standard (non-DP) model.
    dp_result:
        Attack result from the DP-trained model.
    baseline_metrics:
        Pre-computed metrics for the baseline. If ``None``, computed
        from the result arrays.
    dp_metrics:
        Pre-computed metrics for the DP model. If ``None``, computed
        from the result arrays.
    epsilon:
        The privacy budget (epsilon) used during DP training.
    model_accuracy:
        Optional dict with ``{"baseline": float, "dp": float}``
        holding the utility (test accuracy) of each model.
    """

    def __init__(
        self,
        baseline_result: AttackResult,
        dp_result: AttackResult,
        baseline_metrics: dict[str, float] | None = None,
        dp_metrics: dict[str, float] | None = None,
        epsilon: float | None = None,
        model_accuracy: dict[str, float] | None = None,
    ) -> None:
        self.baseline_result = baseline_result
        self.dp_result = dp_result
        self.epsilon = epsilon
        self.model_accuracy = model_accuracy or {}

        # Compute metrics if not provided
        self.baseline_metrics = baseline_metrics or BaseAttack._compute_metrics(
            baseline_result.predictions,
            baseline_result.ground_truth,
            baseline_result.confidence_scores,
        )
        self.dp_metrics = dp_metrics or BaseAttack._compute_metrics(
            dp_result.predictions,
            dp_result.ground_truth,
            dp_result.confidence_scores,
        )

    # ------------------------------------------------------------------
    # Core comparison
    # ------------------------------------------------------------------

    def compute_deltas(self) -> dict[str, float]:
        """Compute the change in each metric: ``dp_value - baseline_value``.

        Negative deltas mean the DP model is more private (attacks are
        less effective).  The most important metrics:

        - ``accuracy_delta``: negative = DP makes the attack less accurate
        - ``auc_roc_delta``: negative = DP makes the attack less
          discriminative
        - ``tpr_at_1fpr_delta``: negative = DP reduces true positive rate
          at realistic operating points

        Returns
        -------
        dict[str, float]
            Keys are ``"<metric>_delta"`` for each shared metric.
        """
        deltas: dict[str, float] = {}
        for key in self.baseline_metrics:
            if key in self.dp_metrics:
                deltas[f"{key}_delta"] = (
                    self.dp_metrics[key] - self.baseline_metrics[key]
                )
        return deltas

    def compute_privacy_gain(self) -> dict[str, float]:
        """Summarise the privacy improvement from DP training.

        Returns
        -------
        dict with keys:
            - ``attack_accuracy_reduction``: how much attack accuracy dropped
            - ``auc_roc_reduction``: how much AUC-ROC dropped
            - ``baseline_attack_accuracy``: baseline attack accuracy
            - ``dp_attack_accuracy``: DP attack accuracy
            - ``baseline_auc_roc``: baseline AUC-ROC
            - ``dp_auc_roc``: DP AUC-ROC
            - ``epsilon``: privacy budget (if known)
            - ``utility_cost``: drop in model accuracy (if known)
        """
        gain: dict[str, float] = {
            "attack_accuracy_reduction": (
                self.baseline_metrics.get("accuracy", 0.0)
                - self.dp_metrics.get("accuracy", 0.0)
            ),
            "auc_roc_reduction": (
                self.baseline_metrics.get("auc_roc", 0.0)
                - self.dp_metrics.get("auc_roc", 0.0)
            ),
            "baseline_attack_accuracy": self.baseline_metrics.get("accuracy", 0.0),
            "dp_attack_accuracy": self.dp_metrics.get("accuracy", 0.0),
            "baseline_auc_roc": self.baseline_metrics.get("auc_roc", 0.0),
            "dp_auc_roc": self.dp_metrics.get("auc_roc", 0.0),
        }

        if self.epsilon is not None:
            gain["epsilon"] = self.epsilon

        if "baseline" in self.model_accuracy and "dp" in self.model_accuracy:
            gain["utility_cost"] = (
                self.model_accuracy["baseline"] - self.model_accuracy["dp"]
            )
            gain["baseline_model_accuracy"] = self.model_accuracy["baseline"]
            gain["dp_model_accuracy"] = self.model_accuracy["dp"]

        return gain

    def summary_dict(self) -> dict[str, Any]:
        """Return a comprehensive summary combining all comparisons.

        Returns
        -------
        dict
            Contains ``baseline_metrics``, ``dp_metrics``, ``deltas``,
            ``privacy_gain``, and metadata.
        """
        return {
            "baseline_metrics": self.baseline_metrics,
            "dp_metrics": self.dp_metrics,
            "deltas": self.compute_deltas(),
            "privacy_gain": self.compute_privacy_gain(),
            "attack_name": self.baseline_result.attack_name,
            "epsilon": self.epsilon,
            "model_accuracy": self.model_accuracy,
        }

    # ------------------------------------------------------------------
    # Report generation
    # ------------------------------------------------------------------

    def generate_report(self, output_dir: str | Path) -> Path:
        """Generate a comparison report with metrics, plots, and summary.

        Creates:

        - ``comparison.json`` — full comparison data
        - ``comparison_bar_chart.png`` — side-by-side metric comparison
        - ``roc_comparison.png`` — overlaid ROC curves
        - ``score_comparison.png`` — confidence score distributions
        - ``summary.txt`` — human-readable summary

        Parameters
        ----------
        output_dir:
            Directory to write report files.

        Returns
        -------
        Path
            The output directory.
        """
        from auditml.reporting.visualization import (
            plot_metric_comparison,
            plot_roc_comparison,
            plot_score_comparison,
        )

        out = Path(output_dir)
        out.mkdir(parents=True, exist_ok=True)

        # 1. JSON data
        summary = self.summary_dict()
        with open(out / "comparison.json", "w") as f:
            json.dump(summary, f, indent=2, default=str)

        # 2. Bar chart comparing metrics
        plot_metric_comparison(
            baseline_metrics=self.baseline_metrics,
            dp_metrics=self.dp_metrics,
            save_path=out / "comparison_bar_chart.png",
        )

        # 3. ROC curves
        plot_roc_comparison(
            baseline_gt=self.baseline_result.ground_truth,
            baseline_scores=self.baseline_result.confidence_scores,
            dp_gt=self.dp_result.ground_truth,
            dp_scores=self.dp_result.confidence_scores,
            save_path=out / "roc_comparison.png",
        )

        # 4. Score distributions
        plot_score_comparison(
            baseline_scores=self.baseline_result.confidence_scores,
            baseline_gt=self.baseline_result.ground_truth,
            dp_scores=self.dp_result.confidence_scores,
            dp_gt=self.dp_result.ground_truth,
            save_path=out / "score_comparison.png",
        )

        # 5. Summary text
        self._write_summary(out / "summary.txt")

        return out

    def _write_summary(self, path: Path) -> None:
        """Write a human-readable comparison summary."""
        gain = self.compute_privacy_gain()
        deltas = self.compute_deltas()

        lines = [
            "=" * 60,
            "AuditML — DP vs Non-DP Comparison Report",
            "=" * 60,
            "",
            f"Attack: {self.baseline_result.attack_name}",
        ]

        if self.epsilon is not None:
            lines.append(f"DP epsilon: {self.epsilon:.2f}")

        lines.append("")
        lines.append("--- Model Utility ---")
        if self.model_accuracy:
            for k, v in self.model_accuracy.items():
                lines.append(f"  {k}: {v:.4f}")
            if "utility_cost" in gain:
                lines.append(f"  Utility cost (accuracy drop): {gain['utility_cost']:.4f}")
        else:
            lines.append("  (not provided)")

        lines.append("")
        lines.append("--- Attack Metrics ---")
        lines.append(f"  {'Metric':<20s}  {'Baseline':>10s}  {'DP':>10s}  {'Delta':>10s}")
        lines.append(f"  {'-'*20}  {'-'*10}  {'-'*10}  {'-'*10}")
        for key in sorted(self.baseline_metrics.keys()):
            bv = self.baseline_metrics[key]
            dv = self.dp_metrics.get(key, 0.0)
            delta = deltas.get(f"{key}_delta", 0.0)
            lines.append(f"  {key:<20s}  {bv:>10.4f}  {dv:>10.4f}  {delta:>+10.4f}")

        lines.append("")
        lines.append("--- Privacy Gain Summary ---")
        lines.append(f"  Attack accuracy reduction: {gain['attack_accuracy_reduction']:.4f}")
        lines.append(f"  AUC-ROC reduction:         {gain['auc_roc_reduction']:.4f}")

        direction = "improved" if gain["auc_roc_reduction"] > 0 else "worsened"
        lines.append(f"  Privacy {direction} with DP training.")

        lines.append("")
        path.write_text("\n".join(lines))

compute_deltas() -> dict[str, float]

Compute the change in each metric: dp_value - baseline_value.

Negative deltas mean the DP model is more private (attacks are less effective). The most important metrics:

  • accuracy_delta: negative = DP makes the attack less accurate
  • auc_roc_delta: negative = DP makes the attack less discriminative
  • tpr_at_1fpr_delta: negative = DP reduces true positive rate at realistic operating points

Returns:

Type Description
dict[str, float]

Keys are "<metric>_delta" for each shared metric.
Source code in src/auditml/reporting/comparison.py 
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
def compute_deltas(self) -> dict[str, float]:
    """Compute the change in each metric: ``dp_value - baseline_value``.

    Negative deltas mean the DP model is more private (attacks are
    less effective).  The most important metrics:

    - ``accuracy_delta``: negative = DP makes the attack less accurate
    - ``auc_roc_delta``: negative = DP makes the attack less
      discriminative
    - ``tpr_at_1fpr_delta``: negative = DP reduces true positive rate
      at realistic operating points

    Returns
    -------
    dict[str, float]
        Keys are ``"<metric>_delta"`` for each shared metric.
    """
    deltas: dict[str, float] = {}
    for key in self.baseline_metrics:
        if key in self.dp_metrics:
            deltas[f"{key}_delta"] = (
                self.dp_metrics[key] - self.baseline_metrics[key]
            )
    return deltas

compute_privacy_gain() -> dict[str, float]

Summarise the privacy improvement from DP training.

Returns:

Type Description
dict with keys:
  • attack_accuracy_reduction: how much attack accuracy dropped
  • auc_roc_reduction: how much AUC-ROC dropped
  • baseline_attack_accuracy: baseline attack accuracy
  • dp_attack_accuracy: DP attack accuracy
  • baseline_auc_roc: baseline AUC-ROC
  • dp_auc_roc: DP AUC-ROC
  • epsilon: privacy budget (if known)
  • utility_cost: drop in model accuracy (if known)
Source code in src/auditml/reporting/comparison.py 
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
def compute_privacy_gain(self) -> dict[str, float]:
    """Summarise the privacy improvement from DP training.

    Returns
    -------
    dict with keys:
        - ``attack_accuracy_reduction``: how much attack accuracy dropped
        - ``auc_roc_reduction``: how much AUC-ROC dropped
        - ``baseline_attack_accuracy``: baseline attack accuracy
        - ``dp_attack_accuracy``: DP attack accuracy
        - ``baseline_auc_roc``: baseline AUC-ROC
        - ``dp_auc_roc``: DP AUC-ROC
        - ``epsilon``: privacy budget (if known)
        - ``utility_cost``: drop in model accuracy (if known)
    """
    gain: dict[str, float] = {
        "attack_accuracy_reduction": (
            self.baseline_metrics.get("accuracy", 0.0)
            - self.dp_metrics.get("accuracy", 0.0)
        ),
        "auc_roc_reduction": (
            self.baseline_metrics.get("auc_roc", 0.0)
            - self.dp_metrics.get("auc_roc", 0.0)
        ),
        "baseline_attack_accuracy": self.baseline_metrics.get("accuracy", 0.0),
        "dp_attack_accuracy": self.dp_metrics.get("accuracy", 0.0),
        "baseline_auc_roc": self.baseline_metrics.get("auc_roc", 0.0),
        "dp_auc_roc": self.dp_metrics.get("auc_roc", 0.0),
    }

    if self.epsilon is not None:
        gain["epsilon"] = self.epsilon

    if "baseline" in self.model_accuracy and "dp" in self.model_accuracy:
        gain["utility_cost"] = (
            self.model_accuracy["baseline"] - self.model_accuracy["dp"]
        )
        gain["baseline_model_accuracy"] = self.model_accuracy["baseline"]
        gain["dp_model_accuracy"] = self.model_accuracy["dp"]

    return gain

summary_dict() -> dict[str, Any]

Return a comprehensive summary combining all comparisons.

Returns:

Type Description
dict

Contains baseline_metrics, dp_metrics, deltas, privacy_gain, and metadata.
Source code in src/auditml/reporting/comparison.py 
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
def summary_dict(self) -> dict[str, Any]:
    """Return a comprehensive summary combining all comparisons.

    Returns
    -------
    dict
        Contains ``baseline_metrics``, ``dp_metrics``, ``deltas``,
        ``privacy_gain``, and metadata.
    """
    return {
        "baseline_metrics": self.baseline_metrics,
        "dp_metrics": self.dp_metrics,
        "deltas": self.compute_deltas(),
        "privacy_gain": self.compute_privacy_gain(),
        "attack_name": self.baseline_result.attack_name,
        "epsilon": self.epsilon,
        "model_accuracy": self.model_accuracy,
    }

generate_report(output_dir: str | Path) -> Path

Generate a comparison report with metrics, plots, and summary.

Creates:

  • comparison.json — full comparison data
  • comparison_bar_chart.png — side-by-side metric comparison
  • roc_comparison.png — overlaid ROC curves
  • score_comparison.png — confidence score distributions
  • summary.txt — human-readable summary

Parameters:

Name Type Description Default
output_dir str | Path

Directory to write report files.

 required

Returns:

Type Description
Path

The output directory.
Source code in src/auditml/reporting/comparison.py 
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
def generate_report(self, output_dir: str | Path) -> Path:
    """Generate a comparison report with metrics, plots, and summary.

    Creates:

    - ``comparison.json`` — full comparison data
    - ``comparison_bar_chart.png`` — side-by-side metric comparison
    - ``roc_comparison.png`` — overlaid ROC curves
    - ``score_comparison.png`` — confidence score distributions
    - ``summary.txt`` — human-readable summary

    Parameters
    ----------
    output_dir:
        Directory to write report files.

    Returns
    -------
    Path
        The output directory.
    """
    from auditml.reporting.visualization import (
        plot_metric_comparison,
        plot_roc_comparison,
        plot_score_comparison,
    )

    out = Path(output_dir)
    out.mkdir(parents=True, exist_ok=True)

    # 1. JSON data
    summary = self.summary_dict()
    with open(out / "comparison.json", "w") as f:
        json.dump(summary, f, indent=2, default=str)

    # 2. Bar chart comparing metrics
    plot_metric_comparison(
        baseline_metrics=self.baseline_metrics,
        dp_metrics=self.dp_metrics,
        save_path=out / "comparison_bar_chart.png",
    )

    # 3. ROC curves
    plot_roc_comparison(
        baseline_gt=self.baseline_result.ground_truth,
        baseline_scores=self.baseline_result.confidence_scores,
        dp_gt=self.dp_result.ground_truth,
        dp_scores=self.dp_result.confidence_scores,
        save_path=out / "roc_comparison.png",
    )

    # 4. Score distributions
    plot_score_comparison(
        baseline_scores=self.baseline_result.confidence_scores,
        baseline_gt=self.baseline_result.ground_truth,
        dp_scores=self.dp_result.confidence_scores,
        dp_gt=self.dp_result.ground_truth,
        save_path=out / "score_comparison.png",
    )

    # 5. Summary text
    self._write_summary(out / "summary.txt")

    return out

Visualisation

auditml.reporting.visualization

Visualization functions for DP vs Non-DP comparison reports.

Provides side-by-side plots that make it easy to see how DP training affects attack effectiveness.

plot_metric_comparison(baseline_metrics: dict[str, float], dp_metrics: dict[str, float], save_path: str | Path | None = None, title: str = 'Attack Metrics — Baseline vs DP') -> plt.Figure

Side-by-side bar chart comparing attack metrics.

Parameters:

Name Type Description Default
baseline_metrics dict[str, float]

Metrics from the standard (non-DP) model.

 required
dp_metrics dict[str, float]

Metrics from the DP model.

 required
save_path str | Path | None

If given, saves the figure.

 None
title str

Plot title.

 'Attack Metrics — Baseline vs DP'

Returns:

Type Description
Figure
Source code in src/auditml/reporting/visualization.py 
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
def plot_metric_comparison(
    baseline_metrics: dict[str, float],
    dp_metrics: dict[str, float],
    save_path: str | Path | None = None,
    title: str = "Attack Metrics — Baseline vs DP",
) -> plt.Figure:
    """Side-by-side bar chart comparing attack metrics.

    Parameters
    ----------
    baseline_metrics:
        Metrics from the standard (non-DP) model.
    dp_metrics:
        Metrics from the DP model.
    save_path:
        If given, saves the figure.
    title:
        Plot title.

    Returns
    -------
    matplotlib.figure.Figure
    """
    # Select the most informative metrics for the chart
    display_keys = [
        k for k in ("accuracy", "precision", "recall", "f1", "auc_roc", "auc_pr")
        if k in baseline_metrics and k in dp_metrics
    ]

    baseline_vals = [baseline_metrics[k] for k in display_keys]
    dp_vals = [dp_metrics[k] for k in display_keys]

    x = np.arange(len(display_keys))
    width = 0.35

    fig, ax = plt.subplots(figsize=(10, 5))
    bars_base = ax.bar(x - width / 2, baseline_vals, width,
                       label="Baseline (no DP)", color="#2563eb", alpha=0.8)
    bars_dp = ax.bar(x + width / 2, dp_vals, width,
                     label="DP-trained", color="#16a34a", alpha=0.8)

    for bar in bars_base:
        ax.text(bar.get_x() + bar.get_width() / 2, bar.get_height() + 0.01,
                f"{bar.get_height():.3f}", ha="center", va="bottom", fontsize=7)
    for bar in bars_dp:
        ax.text(bar.get_x() + bar.get_width() / 2, bar.get_height() + 0.01,
                f"{bar.get_height():.3f}", ha="center", va="bottom", fontsize=7)

    ax.axhline(0.5, color="grey", linestyle="--", lw=1, alpha=0.7,
               label="Random baseline (0.5)")

    ax.set_xticks(x)
    ax.set_xticklabels([k.replace("_", " ").title() for k in display_keys],
                       fontsize=9)
    ax.set_ylabel("Score")
    ax.set_title(title)
    ax.set_ylim([0, 1.15])
    ax.legend()
    ax.grid(True, axis="y", alpha=0.3)

    fig.tight_layout()
    if save_path is not None:
        fig.savefig(save_path, dpi=150, bbox_inches="tight")
    plt.close(fig)
    return fig

plot_roc_comparison(baseline_gt: np.ndarray, baseline_scores: np.ndarray, dp_gt: np.ndarray, dp_scores: np.ndarray, save_path: str | Path | None = None, title: str = 'ROC Curve — Baseline vs DP') -> plt.Figure

Overlay ROC curves from baseline and DP models.

Parameters:

Name Type Description Default
baseline_gt ndarray

Ground truth for baseline attack.

 required
baseline_scores ndarray

Confidence scores for baseline attack.

 required
dp_gt ndarray

Ground truth for DP attack.

 required
dp_scores ndarray

Confidence scores for DP attack.

 required
save_path str | Path | None

If given, saves the figure.

 None
title str

Plot title.

 'ROC Curve — Baseline vs DP'

Returns:

Type Description
Figure
Source code in src/auditml/reporting/visualization.py 
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
def plot_roc_comparison(
    baseline_gt: np.ndarray,
    baseline_scores: np.ndarray,
    dp_gt: np.ndarray,
    dp_scores: np.ndarray,
    save_path: str | Path | None = None,
    title: str = "ROC Curve — Baseline vs DP",
) -> plt.Figure:
    """Overlay ROC curves from baseline and DP models.

    Parameters
    ----------
    baseline_gt:
        Ground truth for baseline attack.
    baseline_scores:
        Confidence scores for baseline attack.
    dp_gt:
        Ground truth for DP attack.
    dp_scores:
        Confidence scores for DP attack.
    save_path:
        If given, saves the figure.
    title:
        Plot title.

    Returns
    -------
    matplotlib.figure.Figure
    """
    fig, ax = plt.subplots(figsize=(7, 6))

    # Baseline ROC
    fpr_b, tpr_b, _ = roc_curve(baseline_gt, baseline_scores)
    auc_b = auc(fpr_b, tpr_b)
    ax.plot(fpr_b, tpr_b, color="#2563eb", lw=2,
            label=f"Baseline (AUC = {auc_b:.4f})")

    # DP ROC
    fpr_d, tpr_d, _ = roc_curve(dp_gt, dp_scores)
    auc_d = auc(fpr_d, tpr_d)
    ax.plot(fpr_d, tpr_d, color="#16a34a", lw=2,
            label=f"DP (AUC = {auc_d:.4f})")

    # Random baseline
    ax.plot([0, 1], [0, 1], color="grey", lw=1, linestyle="--",
            label="Random (AUC = 0.5)")

    ax.set_xlabel("False Positive Rate")
    ax.set_ylabel("True Positive Rate")
    ax.set_title(title)
    ax.legend(loc="lower right")
    ax.set_xlim([-0.01, 1.01])
    ax.set_ylim([-0.01, 1.01])
    ax.grid(True, alpha=0.3)

    fig.tight_layout()
    if save_path is not None:
        fig.savefig(save_path, dpi=150, bbox_inches="tight")
    plt.close(fig)
    return fig

plot_score_comparison(baseline_scores: np.ndarray, baseline_gt: np.ndarray, dp_scores: np.ndarray, dp_gt: np.ndarray, save_path: str | Path | None = None, title: str = 'Confidence Score Distribution — Baseline vs DP') -> plt.Figure

Four-panel histogram comparing member/non-member score distributions.

Top row: baseline model (members vs non-members). Bottom row: DP model (members vs non-members).

Well-separated distributions indicate a successful attack; overlapping distributions indicate the model is more private.

Parameters:

Name Type Description Default
baseline_scores ndarray

Confidence scores from baseline attack.

 required
baseline_gt ndarray

Ground truth for baseline.

 required
dp_scores ndarray

Confidence scores from DP attack.

 required
dp_gt ndarray

Ground truth for DP.

 required
save_path str | Path | None

If given, saves the figure.

 None
title str

Plot title.

 'Confidence Score Distribution — Baseline vs DP'

Returns:

Type Description
Figure
Source code in src/auditml/reporting/visualization.py 
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
def plot_score_comparison(
    baseline_scores: np.ndarray,
    baseline_gt: np.ndarray,
    dp_scores: np.ndarray,
    dp_gt: np.ndarray,
    save_path: str | Path | None = None,
    title: str = "Confidence Score Distribution — Baseline vs DP",
) -> plt.Figure:
    """Four-panel histogram comparing member/non-member score distributions.

    Top row: baseline model (members vs non-members).
    Bottom row: DP model (members vs non-members).

    Well-separated distributions indicate a successful attack; overlapping
    distributions indicate the model is more private.

    Parameters
    ----------
    baseline_scores:
        Confidence scores from baseline attack.
    baseline_gt:
        Ground truth for baseline.
    dp_scores:
        Confidence scores from DP attack.
    dp_gt:
        Ground truth for DP.
    save_path:
        If given, saves the figure.
    title:
        Plot title.

    Returns
    -------
    matplotlib.figure.Figure
    """
    fig, axes = plt.subplots(1, 2, figsize=(14, 5), sharey=True)

    for ax, scores, gt, label in [
        (axes[0], baseline_scores, baseline_gt, "Baseline"),
        (axes[1], dp_scores, dp_gt, "DP"),
    ]:
        member_mask = gt == 1
        nonmember_mask = gt == 0

        all_vals = np.concatenate([scores[member_mask], scores[nonmember_mask]])
        bins = np.linspace(all_vals.min(), all_vals.max(), 40)

        ax.hist(scores[member_mask], bins=bins, alpha=0.6,
                color="#2563eb", label=f"Members (n={member_mask.sum()})",
                density=True)
        ax.hist(scores[nonmember_mask], bins=bins, alpha=0.6,
                color="#dc2626", label=f"Non-members (n={nonmember_mask.sum()})",
                density=True)

        ax.set_xlabel("Confidence Score")
        ax.set_title(f"{label} Model")
        ax.legend(fontsize=8)
        ax.grid(True, alpha=0.3)

    axes[0].set_ylabel("Density")
    fig.suptitle(title, fontsize=13, fontweight="bold")
    fig.tight_layout()

    if save_path is not None:
        fig.savefig(save_path, dpi=150, bbox_inches="tight")
    plt.close(fig)
    return fig