#!/usr/bin/env python3 """Check published pilot records and regenerate their cost worksheet, offline.""" import argparse import csv import hashlib import json import math import sys from decimal import Decimal from pathlib import Path RECORDS = ( "results.json", "method.json", "fine-tuning-cost.svg", "fine-tuning-time.svg", "pretraining-cost.svg", "pretraining-time.svg", ) def require(condition, message): if not condition: raise ValueError(message) def money(value): amount = Decimal(value) require(amount.is_finite() and amount >= 0, "Invalid recorded charge") return amount def close(actual, expected, label): require(math.isclose(float(actual), float(expected), rel_tol=1e-10, abs_tol=1e-10), label) def reduction(nodus, modal): require(modal > 0, "Reference charge must be positive") return 100 * (1 - nodus / modal) def analyze(directory): entries = [line.split() for line in (directory / "SHA256SUMS.txt").read_text().splitlines()] manifest = {name: digest for digest, name in entries} require(len(entries) == len(RECORDS) and set(manifest) == set(RECORDS), "Original evidence manifest is incomplete") for name in RECORDS: require(hashlib.sha256((directory / name).read_bytes()).hexdigest() == manifest[name], f"Checksum mismatch: {name}") results = json.loads((directory / "results.json").read_text()) method = json.loads((directory / "method.json").read_text()) require(results["schema_version"] == method["schema_version"] == 1, "Unknown record schema") require(results["currency"] == "USD", "Unsupported currency") require(results["all_in_invoice_finalized"] is False, "Accounting scope changed; review required") campaign = results["campaign"] attempts = {attempt["id"]: attempt for attempt in campaign["attempts"]} require(len(attempts) == len(campaign["attempts"]) == 8, "Expected eight distinct paid attempts") require(sum(money(a["recorded_charge_usd"]) for a in attempts.values()) == money(campaign["recorded_usage_usd"]), "Campaign ledger does not reconcile") require({stage["stage"] for stage in results["stages"]} == {"sft", "pretrain"} and len(results["stages"]) == 2, "Expected two experiment stages") rows = [] for stage in results["stages"]: name = stage["stage"] require(stage["accepted_runs_per_platform"] == method["accepted_runs_per_stage_per_platform"] == 1, "Accepted-run count changed") require(stage["seed"] == method["seed"], "Seed metadata differs") native, modal = stage["nodus"], stage["modal"] for platform in ("nodus", "modal"): run = stage[platform] attempt_id = f"{platform}_sft_corrected" if name == "sft" else f"{platform}_pretrain" attempt = attempts[attempt_id] require(attempt["stage"] == name and attempt["execution_platform"].lower() == platform and attempt["role"] == "headline_successful_attempt", "Accepted attempt attribution differs") require(money(attempt["recorded_charge_usd"]) == money(run["recorded_successful_attempt_usd"]), "Accepted attempt charge differs from ledger") for kind in ("baseline", "final"): score = run[kind] if name == "sft": require(0 <= score["correct"] <= score["valid"] <= score["rows"] == 872, "Invalid sentiment evaluation counts") close(score["accuracy"], score["correct"] / score["rows"], "Accuracy does not reconcile") close(score["format_validity"], score["valid"] / score["rows"], "Label validity differs") else: require(score["tokens"] == method["evaluation"]["pretrain"]["target_tokens"], "Evaluation token count differs") close(score["perplexity"], math.exp(score["nll"]), "Perplexity does not reconcile") charges = native["charge_segments"] native_cost, modal_cost = (money(run["recorded_successful_attempt_usd"]) for run in (native, modal)) require(sum(money(charges[key]) for key in ("bootUSD", "runningUSD", "teardownUSD")) == native_cost, "Native charge segments do not reconcile") savings = reduction(native_cost, modal_cost) close(savings, stage["successful_attempt_charge_reduction_percent"], "Charge reduction differs") close(100 * (1 - native["trainer_runtime_seconds"] / modal["trainer_runtime_seconds"]), stage["trainer_time_reduction_percent"], "Trainer runtime reduction differs") require(native["export_files"]["model.safetensors"]["sha256"] == modal["export_files"]["model.safetensors"]["sha256"] == stage["model_weights_sha256"], "Published weight digests differ") metric = "accuracy" if name == "sft" else "perplexity" close(native["final"][metric], modal["final"][metric], "Published final quality differs") row = { "stage": name, "comparison": "successful_attempts", "model": method["model"], "model_revision": method["model_revision"], "gpu": native["gpu"], "seed": stage["seed"], "accepted_runs_per_platform": 1, "observed_at": results["observed_at"], "nodus_recorded_usd": str(native_cost), "modal_recorded_usd": str(modal_cost), "charge_reduction_percent": f"{savings:.6f}", "nodus_trainer_seconds": native["trainer_runtime_seconds"], "modal_trainer_seconds": modal["trainer_runtime_seconds"], "nodus_submit_to_export_seconds": native["submit_to_local_export_seconds"], "modal_submit_to_export_seconds": modal["submit_to_local_export_seconds"], "quality_metric": metric, "nodus_final_quality": native["final"][metric], "modal_final_quality": modal["final"][metric], "all_in_invoice_finalized": False, } rows.append(row) if name == "sft": failed = attempts["nodus_sft_wrapper_failure"] require(failed["role"] == "failed_before_training", "Failure classification differs") combined = native_cost + money(failed["recorded_charge_usd"]) require(combined == money(campaign["nodus_sft_including_failed_attempt_usd"]), "Retry total differs") retry_savings = reduction(combined, modal_cost) close(retry_savings, campaign["nodus_sft_including_failed_attempt_vs_corrected_modal_reduction_percent"], "Retry-inclusive charge reduction differs") retry = dict(row, comparison="nodus_success_and_wrapper_failure_vs_modal_success", nodus_recorded_usd=str(combined), charge_reduction_percent=f"{retry_savings:.6f}") # The ledger does not provide a combined delivery clock for both attempts. for key in retry: if key.endswith("_seconds"): retry[key] = "" rows.append(retry) return rows, campaign def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--csv", type=Path, help="Write the recalculated worksheet to this local file") args = parser.parse_args() try: rows, campaign = analyze(Path(__file__).resolve().parent) if args.csv: with args.csv.open("w", newline="", encoding="utf-8") as output: writer = csv.DictWriter(output, fieldnames=list(rows[0]), lineterminator="\n") writer.writeheader() writer.writerows(rows) print("PASS: six evidence checksums; charge arithmetic, evaluation summaries and reported digest consistency.") for row in rows: print(f'{row["stage"]}: {row["comparison"]}: Nodus ${row["nodus_recorded_usd"]}; ' f'Modal ${row["modal_recorded_usd"]}; {row["charge_reduction_percent"]}% lower recorded charges') print(f'All eight paid attempts: ${campaign["recorded_usage_usd"]} USD. Platform subtotals are not comparable.') print("This verifies published-record consistency, not actual model weights, receipts or training execution.") print("One seed; one accepted run per platform. Residual charges are not finalized. No general savings forecast.") except (OSError, ValueError, KeyError, TypeError, ArithmeticError) as error: print(f"FAIL: {error}", file=sys.stderr) return 1 return 0 if __name__ == "__main__": raise SystemExit(main())