#!/usr/bin/env python3 # SPDX-License-Identifier: MIT """Cross-notebook acceptance suite for the Computational AMO chapter. The suite reads retained CSV and JSON artifacts, recomputes independent analytic anchors where practical, verifies upstream validation booleans, and emits a machine-readable report plus a compact CSV summary. Hydrogen energies are an analytic benchmark specification in this suite. They do not claim that a radial discretization artifact has been run. The helium, H2+, Rabi, optical Bloch, Jaynes-Cummings, and Doppler groups are backed by retained notebook artifacts. """ from __future__ import annotations import argparse import csv import hashlib import json import math import platform import sys from dataclasses import asdict, dataclass, field from pathlib import Path from typing import Any, Iterable SCHEMA_VERSION = "1.0.0" SUITE_ID = "computational-amo-reproducibility-v1" @dataclass(frozen=True) class Check: """One auditable acceptance statement.""" check_id: str description: str passed: bool actual: Any expected: Any tolerance: str | None = None @dataclass class Benchmark: """One benchmark family and its evidence.""" benchmark_id: str title: str evidence_status: str source_page: str claim_boundary: str artifacts: list[str] = field(default_factory=list) checks: list[Check] = field(default_factory=list) metrics: dict[str, Any] = field(default_factory=dict) @property def passed(self) -> bool: return bool(self.checks) and all(check.passed for check in self.checks) def to_dict(self) -> dict[str, Any]: return { "benchmark_id": self.benchmark_id, "title": self.title, "evidence_status": self.evidence_status, "source_page": self.source_page, "claim_boundary": self.claim_boundary, "artifacts": self.artifacts, "passed": self.passed, "passed_checks": sum(check.passed for check in self.checks), "total_checks": len(self.checks), "checks": [asdict(check) for check in self.checks], "metrics": self.metrics, } def close( check_id: str, description: str, actual: float, expected: float, *, absolute_tolerance: float, relative_tolerance: float = 0.0, ) -> Check: """Construct a finite floating-point comparison.""" finite = math.isfinite(actual) and math.isfinite(expected) passed = finite and math.isclose( actual, expected, abs_tol=absolute_tolerance, rel_tol=relative_tolerance, ) tolerance = ( f"atol={absolute_tolerance:.3g}, " f"rtol={relative_tolerance:.3g}" ) return Check( check_id, description, passed, actual, expected, tolerance, ) def require( check_id: str, description: str, condition: bool, actual: Any, expected: Any, ) -> Check: return Check( check_id, description, bool(condition), actual, expected, ) def read_json(path: Path) -> dict[str, Any]: with path.open(encoding="utf-8") as handle: value = json.load(handle) if not isinstance(value, dict): raise ValueError(f"expected JSON object: {path}") return value def read_csv(path: Path) -> list[dict[str, str]]: with path.open(newline="", encoding="utf-8") as handle: rows = list(csv.DictReader(handle)) if not rows: raise ValueError(f"expected nonempty CSV: {path}") if any(None in row for row in rows): raise ValueError(f"malformed CSV row: {path}") return rows def find_row( rows: Iterable[dict[str, str]], column: str, target: float, *, tolerance: float = 1.0e-12, ) -> dict[str, str]: candidates = list(rows) if not candidates: raise ValueError("cannot search an empty row collection") row = min(candidates, key=lambda item: abs(float(item[column]) - target)) difference = abs(float(row[column]) - target) if difference > tolerance: raise ValueError( f"no row for {column}={target}; nearest difference={difference}" ) return row def find_label( rows: Iterable[dict[str, str]], column: str, label: str, ) -> dict[str, str]: matches = [row for row in rows if row[column] == label] if len(matches) != 1: raise ValueError( f"expected one row with {column}={label!r}, found {len(matches)}" ) return matches[0] def boolean_leaves(value: Any, prefix: str = "") -> dict[str, bool]: """Collect bool leaves without coercing numeric values.""" leaves: dict[str, bool] = {} if isinstance(value, dict): for key, child in value.items(): child_prefix = f"{prefix}.{key}" if prefix else key leaves.update(boolean_leaves(child, child_prefix)) elif isinstance(value, list): for index, child in enumerate(value): leaves.update(boolean_leaves(child, f"{prefix}[{index}]")) elif isinstance(value, bool): leaves[prefix] = value return leaves def metadata_validation_check( benchmark_id: str, metadata: dict[str, Any], ) -> Check: leaves = boolean_leaves(metadata.get("validation", {})) failed = sorted(name for name, passed in leaves.items() if not passed) return require( f"{benchmark_id}.upstream_validation", "every retained upstream validation boolean is true", bool(leaves) and not failed, { "boolean_checks": len(leaves), "failed": failed, }, { "boolean_checks": "one or more", "failed": [], }, ) def recursive_key_values(value: Any, target_key: str) -> list[Any]: results: list[Any] = [] if isinstance(value, dict): for key, child in value.items(): if key == target_key: results.append(child) results.extend(recursive_key_values(child, target_key)) elif isinstance(value, list): for child in value: results.extend(recursive_key_values(child, target_key)) return results def sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def hydrogen_benchmark() -> Benchmark: energies = { n: -1.0 / (2.0 * n * n) for n in (1, 2, 3) } checks = [ close( "hydrogen.energy_1s", "the atomic-unit hydrogen 1s anchor is -1/2 hartree", energies[1], -0.5, absolute_tolerance=0.0, ), close( "hydrogen.energy_n2", "the n=2 Coulomb energy anchor is -1/8 hartree", energies[2], -0.125, absolute_tolerance=0.0, ), close( "hydrogen.energy_n3", "the n=3 Coulomb energy anchor is -1/18 hartree", energies[3], -1.0 / 18.0, absolute_tolerance=0.0, ), require( "hydrogen.coulomb_degeneracy_n2", "the nonrelativistic Coulomb anchor is independent of l at fixed n", energies[2] == -1.0 / (2.0 * 2**2), { "E_2s_Eh": energies[2], "E_2p_Eh": energies[2], }, "equal", ), require( "hydrogen.energy_ordering", "bound energies approach zero from below as n increases", energies[1] < energies[2] < energies[3] < 0.0, [energies[1], energies[2], energies[3]], "E_1 < E_2 < E_3 < 0", ), ] return Benchmark( benchmark_id="hydrogen", title="Hydrogen energy anchors", evidence_status="analytic_specification", source_page=( "/atoms-molecules-light/computational-amo-quantum-chemistry/" "radial-schrodinger-solvers/" ), claim_boundary=( "Defines exact Coulomb anchors for radial solvers; it does not " "certify a retained radial discretization artifact." ), checks=checks, metrics={ "energy_formula_atomic_units": "E_n = -1 / (2 n^2)", "E_1_Eh": energies[1], "E_2_Eh": energies[2], "E_3_Eh": energies[3], }, ) def helium_benchmark(data_dir: Path) -> Benchmark: metadata_name = "variational-helium-metadata.json" summary_name = "variational-helium-summary.csv" metadata = read_json(data_dir / metadata_name) summary = read_csv(data_dir / summary_name) row = find_label(summary, "label", "optimized_analytic") nuclear_charge = 2.0 zeta_expected = nuclear_charge - 5.0 / 16.0 energy_expected = ( zeta_expected**2 - 2.0 * nuclear_charge * zeta_expected + 5.0 * zeta_expected / 8.0 ) exact_row = find_label( summary, "label", "exact_nonrelativistic_reference", ) exact_reference = float(exact_row["total_Eh"]) metadata_zeta = float(metadata["analytic_components"]["zeta"]) metadata_energy = float(metadata["analytic_components"]["total"]) csv_zeta = float(row["zeta"]) csv_energy = float(row["total_Eh"]) checks = [ metadata_validation_check("helium", metadata), close( "helium.metadata_zeta", "metadata exponent agrees with the independent stationary point", metadata_zeta, zeta_expected, absolute_tolerance=1.0e-14, ), close( "helium.metadata_energy", "metadata energy agrees with an independent energy evaluation", metadata_energy, energy_expected, absolute_tolerance=1.0e-14, ), close( "helium.csv_zeta", "summary CSV carries the analytic optimized exponent", csv_zeta, zeta_expected, absolute_tolerance=1.0e-14, ), close( "helium.csv_energy", "summary CSV carries the analytic optimized energy", csv_energy, energy_expected, absolute_tolerance=1.0e-14, ), require( "helium.variational_ordering", "the trial energy remains above the retained exact reference", csv_energy > exact_reference, { "trial_Eh": csv_energy, "reference_Eh": exact_reference, }, "trial_Eh > reference_Eh", ), ] return Benchmark( benchmark_id="helium", title="Variational helium", evidence_status="artifact_verified", source_page=( "/atoms-molecules-light/computational-amo-quantum-chemistry/" "variational-helium-notebook/" ), claim_boundary=( "Certifies the one-parameter effective-charge trial family, not " "the exact helium ground state or a general variational solver." ), artifacts=[metadata_name, summary_name], checks=checks, metrics={ "zeta_optimum": metadata_zeta, "energy_optimum_Eh": metadata_energy, "exact_reference_Eh": exact_reference, "trial_deficit_Eh": metadata_energy - exact_reference, }, ) def h2plus_benchmark(data_dir: Path) -> Benchmark: metadata_name = "molecular-orbital-h2plus-metadata.json" curves_name = "molecular-orbital-h2plus-curves.csv" metadata = read_json(data_dir / metadata_name) rows = read_csv(data_dir / curves_name) row = find_row(rows, "R_bohr", 2.0) separation = 2.0 overlap_expected = math.exp(-separation) * ( 1.0 + separation + separation**2 / 3.0 ) overlap_actual = float(row["overlap"]) gerade_energy = float(row["total_g_Eh"]) metric_norm = float(row["metric_norm_g"]) residual = float(row["residual_g"]) minimum = metadata["lcao_minimum"] checks = [ metadata_validation_check("h2plus", metadata), close( "h2plus.overlap_R2", "the R=2 overlap agrees with the analytic 1s Slater formula", overlap_actual, overlap_expected, absolute_tolerance=2.0e-15, ), close( "h2plus.gerade_energy_R2", "the retained R=2 gerade total-energy anchor is stable", gerade_energy, -0.553771495318482, absolute_tolerance=2.0e-14, ), close( "h2plus.metric_norm", "the R=2 gerade eigenvector is normalized in the overlap metric", metric_norm, 1.0, absolute_tolerance=2.0e-14, ), require( "h2plus.generalized_residual", "the R=2 generalized eigenpair residual is small", residual < 2.0e-14, residual, "< 2e-14", ), close( "h2plus.minimum_location", "the retained minimal-basis equilibrium location is unchanged", float(minimum["R_bohr"]), 2.4928304252534232, absolute_tolerance=2.0e-12, ), close( "h2plus.minimum_energy", "the retained minimal-basis minimum energy is unchanged", float(minimum["total_energy_Eh"]), -0.5648309923708077, absolute_tolerance=2.0e-13, ), ] return Benchmark( benchmark_id="h2plus", title="Minimal-basis H2+", evidence_status="artifact_verified", source_page=( "/atoms-molecules-light/computational-amo-quantum-chemistry/" "molecular-orbital-computation/" ), claim_boundary=( "Certifies the fixed-exponent two-function LCAO model, not a " "basis-set-limit Born-Oppenheimer curve." ), artifacts=[metadata_name, curves_name], checks=checks, metrics={ "overlap_at_R2": overlap_actual, "gerade_total_energy_at_R2_Eh": gerade_energy, "minimum_R_bohr": float(minimum["R_bohr"]), "minimum_energy_Eh": float(minimum["total_energy_Eh"]), }, ) def rabi_benchmark(data_dir: Path) -> Benchmark: metadata_name = "time-dependent-two-level-metadata.json" rabi_name = "time-dependent-two-level-rabi.csv" metadata = read_json(data_dir / metadata_name) rows = read_csv(data_dir / rabi_name) row = find_row(rows, "tau_over_pi", 1.0) resonant_analytic = float(row["P_delta_0p0_analytic"]) resonant_matrix = float(row["P_delta_0p0_matrix"]) matrix_error = float( metadata["rabi"]["validation"][ "max_matrix_minus_analytic_probability" ] ) norm_error = float( metadata["rabi"]["validation"]["max_state_norm_error"] ) checks = [ metadata_validation_check("rabi", metadata), close( "rabi.pi_pulse_analytic", "the resonant analytic pi pulse reaches the excited state", resonant_analytic, 1.0, absolute_tolerance=2.0e-15, ), close( "rabi.pi_pulse_matrix", "matrix propagation reproduces the resonant pi pulse", resonant_matrix, 1.0, absolute_tolerance=2.0e-15, ), require( "rabi.analytic_matrix_agreement", "the full Rabi grid agrees with the closed-form probability", matrix_error < 5.0e-15, matrix_error, "< 5e-15", ), require( "rabi.norm", "closed two-level propagation preserves state norm", norm_error < 1.0e-14, norm_error, "< 1e-14", ), ] return Benchmark( benchmark_id="rabi", title="Coherent Rabi oscillations", evidence_status="artifact_verified", source_page=( "/atoms-molecules-light/computational-amo-quantum-chemistry/" "time-dependent-two-level-systems-notebook/" ), claim_boundary=( "Certifies the declared closed two-state Hamiltonian and " "propagator, not decoherence, leakage, or pulse calibration." ), artifacts=[metadata_name, rabi_name], checks=checks, metrics={ "pi_pulse_probability": resonant_matrix, "max_matrix_formula_error": matrix_error, "max_norm_error": norm_error, }, ) def optical_bloch_benchmark(data_dir: Path) -> Benchmark: metadata_name = "optical-bloch-metadata.json" saturation_name = "optical-bloch-saturation.csv" metadata = read_json(data_dir / metadata_name) rows = read_csv(data_dir / saturation_name) row = find_row(rows, "Omega_over_Gamma", 1.0) formula = float(row["rhoee_formula_gamma_phi_0"]) linear = float(row["rhoee_linear_gamma_phi_0"]) global_formula_error = float( metadata["saturation"]["validation"][ "max_population_formula_error" ] ) linear_error = float( metadata["saturation"]["validation"][ "max_analytic_minus_linear_state" ] ) checks = [ metadata_validation_check("optical_bloch", metadata), close( "optical_bloch.steady_formula", "at Omega/Gamma=1 and zero dephasing, rho_ee is 1/3", formula, 1.0 / 3.0, absolute_tolerance=2.0e-15, ), close( "optical_bloch.steady_linear_solver", "the independent linear steady solver returns 1/3", linear, 1.0 / 3.0, absolute_tolerance=2.0e-15, ), require( "optical_bloch.formula_grid", "the saturation grid agrees with the analytic population", global_formula_error < 2.0e-15, global_formula_error, "< 2e-15", ), require( "optical_bloch.linear_grid", "the analytic and linear steady-state solvers agree", linear_error < 2.0e-15, linear_error, "< 2e-15", ), ] return Benchmark( benchmark_id="optical_bloch", title="Optical Bloch steady state", evidence_status="artifact_verified", source_page=( "/atoms-molecules-light/computational-amo-quantum-chemistry/" "optical-bloch-equation-notebook/" ), claim_boundary=( "Certifies the declared dissipative two-level steady state, not " "multilevel fluorescence or an apparatus detector model." ), artifacts=[metadata_name, saturation_name], checks=checks, metrics={ "rhoee_at_Omega_equal_Gamma": linear, "max_population_formula_error": global_formula_error, "max_analytic_linear_state_error": linear_error, }, ) def cavity_qed_benchmark(data_dir: Path) -> Benchmark: metadata_name = "cavity-qed-metadata.json" spectrum_name = "cavity-qed-dressed-spectrum.csv" metadata = read_json(data_dir / metadata_name) rows = read_csv(data_dir / spectrum_name) row = find_row(rows, "Delta_over_g", 0.0) g = float(metadata["dressed_spectrum"]["g"]) splitting_n0 = float(row["E_upper_n0"]) - float(row["E_lower_n0"]) splitting_n1 = float(row["E_upper_n1"]) - float(row["E_lower_n1"]) expected_n0 = 2.0 * g expected_n1 = 2.0 * g * math.sqrt(2.0) exchange_error = float( metadata["vacuum_rabi"]["validation"][ "max_excited_population_error" ] ) checks = [ metadata_validation_check("cavity_qed", metadata), close( "cavity_qed.resonance_splitting_n0", "the first resonant doublet has splitting 2g", splitting_n0, expected_n0, absolute_tolerance=2.0e-15, ), close( "cavity_qed.resonance_splitting_n1", "the second resonant doublet has splitting 2g sqrt(2)", splitting_n1, expected_n1, absolute_tolerance=2.0e-15, ), require( "cavity_qed.vacuum_rabi", "vacuum Rabi exchange agrees with the analytic population", exchange_error < 5.0e-15, exchange_error, "< 5e-15", ), require( "cavity_qed.sqrt_n_scaling", "the ratio of the two retained splittings is sqrt(2)", math.isclose( splitting_n1 / splitting_n0, math.sqrt(2.0), abs_tol=2.0e-14, rel_tol=0.0, ), splitting_n1 / splitting_n0, math.sqrt(2.0), ), ] return Benchmark( benchmark_id="cavity_qed", title="Jaynes-Cummings splitting", evidence_status="artifact_verified", source_page=( "/atoms-molecules-light/computational-amo-quantum-chemistry/" "cavity-qed-simulation-notebook/" ), claim_boundary=( "Certifies finite-basis Jaynes-Cummings doublets and vacuum " "exchange, not a driven device spectrum or ultrastrong coupling." ), artifacts=[metadata_name, spectrum_name], checks=checks, metrics={ "g": g, "resonance_splitting_n0": splitting_n0, "resonance_splitting_n1": splitting_n1, "max_vacuum_rabi_population_error": exchange_error, }, ) def doppler_benchmark(data_dir: Path) -> Benchmark: metadata_name = "laser-cooling-metadata.json" force_name = "laser-cooling-force-curves.csv" metadata = read_json(data_dir / metadata_name) rows = read_csv(data_dir / force_name) positive = find_row(rows, "k_v_over_Gamma", 0.1) negative = find_row(rows, "k_v_over_Gamma", -0.1) red_positive = float(positive["weak_force_d0p5_s0p1"]) red_negative = float(negative["weak_force_d0p5_s0p1"]) blue_positive = float(positive["weak_force_dm0p5_s0p1"]) friction_optimum = float( metadata["friction"]["validation"][ "numeric_weak_friction_optimum_Delta_over_Gamma" ] ) temperature_optimum = float( metadata["temperature"]["validation"][ "numeric_weak_temperature_optimum_Delta_over_Gamma" ] ) checks = [ metadata_validation_check("doppler", metadata), require( "doppler.red_damping", "red detuning gives negative force at positive velocity", red_positive < 0.0, red_positive, "< 0", ), require( "doppler.blue_antidamping", "blue detuning gives positive force at positive velocity", blue_positive > 0.0, blue_positive, "> 0", ), close( "doppler.odd_force", "the balanced red-detuned force is odd at u=+/-0.1", red_positive + red_negative, 0.0, absolute_tolerance=2.0e-15, ), close( "doppler.friction_optimum", "maximum weak friction occurs at 1/(2 sqrt(3))", friction_optimum, 1.0 / (2.0 * math.sqrt(3.0)), absolute_tolerance=2.0e-8, ), close( "doppler.temperature_optimum", "minimum weak Doppler temperature occurs at Delta/Gamma=1/2", temperature_optimum, 0.5, absolute_tolerance=1.0e-8, ), ] return Benchmark( benchmark_id="doppler", title="Doppler force curve", evidence_status="artifact_verified", source_page=( "/atoms-molecules-light/computational-amo-quantum-chemistry/" "laser-cooling-simulation-notebook/" ), claim_boundary=( "Certifies the balanced weak two-level Doppler benchmark, not " "multilevel sub-Doppler cooling or apparatus capture." ), artifacts=[metadata_name, force_name], checks=checks, metrics={ "force_at_u_0p1_red": red_positive, "force_at_u_0p1_blue": blue_positive, "friction_optimum_Delta_over_Gamma": friction_optimum, "temperature_optimum_Delta_over_Gamma": temperature_optimum, }, ) def infrastructure_benchmark( data_dir: Path, scientific_benchmarks: list[Benchmark], ) -> Benchmark: metadata_names = [ name for benchmark in scientific_benchmarks for name in benchmark.artifacts if name.endswith("-metadata.json") ] metadata_objects = { name: read_json(data_dir / name) for name in metadata_names } declared_outputs: dict[str, list[str]] = { name: list(metadata.get("outputs", [])) for name, metadata in metadata_objects.items() } missing_outputs = sorted( f"{metadata_name}: {output}" for metadata_name, outputs in declared_outputs.items() for output in outputs if not (data_dir / output).is_file() ) licenses = { name: metadata.get("license") for name, metadata in metadata_objects.items() } seed_values = { name: recursive_key_values(metadata, "random_seed") for name, metadata in metadata_objects.items() } non_null_seeds = { name: values for name, values in seed_values.items() if any(value is not None for value in values) } runtime_records: dict[str, Any] = {} for name, metadata in metadata_objects.items(): runtime = metadata.get("runtime") if not isinstance(runtime, dict): runtime = { key: metadata.get(key) for key in ("python", "numpy", "platform") if key in metadata } runtime_records[name] = runtime checks = [ require( "infrastructure.declared_outputs", "every upstream metadata file names outputs that exist", not missing_outputs, missing_outputs, [], ), require( "infrastructure.licenses", "all artifact-producing programs declare the MIT license", bool(licenses) and all( license_name == "MIT" for license_name in licenses.values() ), licenses, "all MIT", ), require( "infrastructure.randomness", "all retained benchmark programs declare deterministic execution", not non_null_seeds, non_null_seeds, {}, ), require( "infrastructure.runtime_records", "each upstream metadata file records a Python runtime", all( isinstance(runtime, dict) and bool(runtime.get("python")) for runtime in runtime_records.values() ), runtime_records, "Python version present for every producer", ), ] return Benchmark( benchmark_id="infrastructure", title="Versioning and dependency policy", evidence_status="artifact_verified", source_page=( "/atoms-molecules-light/computational-amo-quantum-chemistry/" "reproducibility-benchmarks/" ), claim_boundary=( "Certifies retained file presence, licensing, runtime records, " "and declared determinism; it does not prove bitwise portability " "across all hardware and numerical libraries." ), artifacts=sorted(metadata_names), checks=checks, metrics={ "metadata_files": len(metadata_names), "declared_output_files": sum( len(outputs) for outputs in declared_outputs.values() ), "runtime_records": runtime_records, }, ) def artifact_manifest( data_dir: Path, benchmarks: Iterable[Benchmark], ) -> dict[str, dict[str, Any]]: names = sorted( { name for benchmark in benchmarks for name in benchmark.artifacts } ) manifest: dict[str, dict[str, Any]] = {} for name in names: path = data_dir / name if not path.is_file(): raise FileNotFoundError(path) manifest[name] = { "bytes": path.stat().st_size, "sha256": sha256(path), } return manifest def headline_metric(benchmark: Benchmark) -> str: if not benchmark.metrics: return "" key = next(iter(benchmark.metrics)) value = benchmark.metrics[key] if isinstance(value, float): return f"{key}={value:.12g}" return f"{key}={value}" def write_summary(path: Path, benchmarks: list[Benchmark]) -> None: with path.open("w", newline="", encoding="utf-8") as handle: writer = csv.DictWriter( handle, fieldnames=[ "benchmark_id", "title", "evidence_status", "passed", "passed_checks", "total_checks", "headline_metric", "claim_boundary", ], ) writer.writeheader() for benchmark in benchmarks: writer.writerow( { "benchmark_id": benchmark.benchmark_id, "title": benchmark.title, "evidence_status": benchmark.evidence_status, "passed": str(benchmark.passed).lower(), "passed_checks": sum( check.passed for check in benchmark.checks ), "total_checks": len(benchmark.checks), "headline_metric": headline_metric(benchmark), "claim_boundary": benchmark.claim_boundary, } ) def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--data-dir", type=Path, default=Path("public/data/atoms-molecules-light"), help="directory containing retained notebook CSV and JSON artifacts", ) parser.add_argument( "--output-dir", type=Path, default=Path("amo-reproducibility-output"), help="directory for the suite report and summary", ) return parser.parse_args() def main() -> None: args = parse_args() if not args.data_dir.is_dir(): raise FileNotFoundError(args.data_dir) args.output_dir.mkdir(parents=True, exist_ok=True) scientific = [ hydrogen_benchmark(), helium_benchmark(args.data_dir), h2plus_benchmark(args.data_dir), rabi_benchmark(args.data_dir), optical_bloch_benchmark(args.data_dir), cavity_qed_benchmark(args.data_dir), doppler_benchmark(args.data_dir), ] infrastructure = infrastructure_benchmark(args.data_dir, scientific) benchmarks = scientific + [infrastructure] manifest = artifact_manifest(args.data_dir, benchmarks) report_name = "amo-reproducibility-report.json" summary_name = "amo-reproducibility-summary.csv" report_path = args.output_dir / report_name summary_path = args.output_dir / summary_name all_passed = all(benchmark.passed for benchmark in benchmarks) total_checks = sum(len(benchmark.checks) for benchmark in benchmarks) passed_checks = sum( check.passed for benchmark in benchmarks for check in benchmark.checks ) report = { "schema_version": SCHEMA_VERSION, "suite_id": SUITE_ID, "purpose": ( "Cross-notebook acceptance checks for retained Computational " "AMO artifacts and analytic hydrogen anchors." ), "all_benchmarks_passed": all_passed, "counts": { "benchmarks": len(benchmarks), "artifact_backed_benchmarks": sum( benchmark.evidence_status == "artifact_verified" for benchmark in benchmarks ), "analytic_specifications": sum( benchmark.evidence_status == "analytic_specification" for benchmark in benchmarks ), "passed_checks": passed_checks, "total_checks": total_checks, "input_artifacts": len(manifest), }, "benchmarks": [ benchmark.to_dict() for benchmark in benchmarks ], "input_artifacts": manifest, "runtime": { "python": platform.python_version(), "implementation": platform.python_implementation(), "platform": platform.platform(), "random_seed": None, "dependencies": "Python standard library only", }, "policy": { "numeric_contract": ( "compare named observables with benchmark-specific absolute " "tolerances; do not require universal bitwise equality" ), "artifact_contract": ( "record SHA-256 digests and byte counts for every consumed " "CSV and JSON artifact" ), "environment_contract": ( "record producer and suite runtimes; version changes trigger " "review but do not fail solely because a version string differs" ), "randomness_contract": ( "the retained suite and all consumed producers declare no " "random sampling" ), "failure_contract": ( "always emit the report, print failed check identifiers, " "and return a nonzero process status" ), }, "limitations": [ ( "the hydrogen group specifies exact targets but does not " "execute a retained radial discretization" ), ( "artifact checks cannot detect the same conceptual error " "repeated in both a producer and its metadata" ), ( "hash equality identifies bytes, not scientific correctness" ), ( "the suite samples named anchor rows rather than recomputing " "every upstream curve" ), ( "passing model benchmarks does not validate species- or " "apparatus-specific predictions" ), ], "license": "MIT", "outputs": [report_name, summary_name], } report_path.write_text( json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8", ) write_summary(summary_path, benchmarks) for benchmark in benchmarks: status = "PASS" if benchmark.passed else "FAIL" passed = sum(check.passed for check in benchmark.checks) print( f"{status:4s} {benchmark.benchmark_id:18s} " f"{passed}/{len(benchmark.checks)} checks" ) print( f"summary: {passed_checks}/{total_checks} checks passed " f"across {len(benchmarks)} benchmark groups" ) print(f"report: {report_path}") print(f"table: {summary_path}") if not all_passed: failed = [ check.check_id for benchmark in benchmarks for check in benchmark.checks if not check.passed ] print("failed checks:", ", ".join(failed), file=sys.stderr) raise SystemExit(1) if __name__ == "__main__": main()