Composable models and A/B tests of the 2024 ideas; add powerlaw.

Models are now a Composite of drift, volatility and (optional) shape
components, so an experiment can swap one part against a fixed control.

btcmodel/experiments.py holds seven experiments built from the ideas in the
old branches (catalogued in docs/2024-ideas.md), each with its hypothesis
and source, and a verdict rule fixed before anything ran. `just ab` runs
them on development data. Results:

- Shrinking the cycle drift, and a power-law trend (plain or reverting),
  beat their controls. The power law beats the random walk by 53-63% at
  3-4 years with unbiased outcomes, so it is promoted to MODELS.
- Every alternative volatility estimate (EWMA blends, other windows,
  reversion to a level or trend) is worse than the trailing 365-day window.
  Cycle-dependent volatility, heavy tails and stretched cycle phase show no
  reliable effect.

btcmodel/__main__.py

@@ -5,6 +5,7 @@ Command line entry point.
     python -m btcmodel backtest            score models on development data
     python -m btcmodel backtest --holdout  score models on outcomes after DEV_CUTOFF
     python -m btcmodel forecast            forecast from the latest price
+    python -m btcmodel ab [NAME ...]       run A/B experiments on development data
 """
 
 import argparse
@@ -13,7 +14,7 @@ from pathlib import Path
 import numpy as np
 import pandas as pd
 
-from . import data, evaluate, plots
+from . import data, evaluate, experiments, plots
 from .models import MODELS
 
 FORECAST_REPORT_HORIZONS = (182, 365, 730, 1095, 1460)
@@ -33,7 +34,11 @@ def main() -> None:
         help=f"score outcomes after {data.DEV_CUTOFF:%Y-%m-%d} (don't use while developing)",
     )
     commands.add_parser("forecast", help="forecast from the latest price")
+    ab = commands.add_parser("ab", help="run A/B experiments on development data")
+    ab.add_argument("names", nargs="*", help="experiments to run (default: all)")
     args = parser.parse_args()
+    if args.command == "ab" and (unknown := set(args.names) - set(experiments.EXPERIMENTS)):
+        parser.error(f"unknown experiments: {', '.join(sorted(unknown))}")
     models = [MODELS[name] for name in args.models]
 
     if args.command == "update":
@@ -43,6 +48,8 @@ def main() -> None:
         run_backtest(models, args.output, args.holdout)
     elif args.command == "forecast":
         run_forecast(models, args.output)
+    elif args.command == "ab":
+        run_ab(args.names or list(experiments.EXPERIMENTS), args.output)
 
 
 def run_backtest(models, output: Path, holdout: bool) -> None:
@@ -70,6 +77,41 @@ def run_backtest(models, output: Path, holdout: bool) -> None:
     print(f"\nwrote {out}/")
 
 
+def run_ab(names: list[str], output: Path) -> None:
+    prices = data.load_prices(until=data.DEV_CUTOFF)
+    verdicts = []
+    for name in names:
+        experiment = experiments.EXPERIMENTS[name]
+        scores, summary = experiments.run(experiment, prices)
+        out = output / "ab" / name
+        out.mkdir(parents=True, exist_ok=True)
+        control = experiment.control.name
+        report = (
+            f"{name}: {experiment.hypothesis}\n(from {experiment.source})\n\n"
+            + evaluate.format_summary(summary, baseline=control)
+        )
+        print(report + "\n")
+        (out / "report.txt").write_text(report + "\n")
+        summary.to_csv(out / "summary.csv", index=False)
+        plots.skill_chart(summary, out / "skill.png", f"{name}: skill vs {control}", control)
+        plots.calibration_chart(summary, out / "calibration.png", f"{name}: interval coverage")
+        for variant in experiment.variants:
+            v = summary[summary.model == variant.name].set_index("horizon")
+            verdicts.append(
+                {
+                    "experiment": name,
+                    "variant": variant.name,
+                    "control": control,
+                    **{evaluate.horizon_label(h): f"{s:+.0%}" for h, s in v["skill"].items()},
+                    "verdict": experiments.verdict(v),
+                }
+            )
+    table = pd.DataFrame(verdicts).to_string(index=False)
+    print(table)
+    (output / "ab").mkdir(parents=True, exist_ok=True)
+    (output / "ab" / "verdicts.txt").write_text(table + "\n")
+
+
 def run_forecast(models, output: Path) -> None:
     prices = data.load_prices()
     horizons = np.arange(1, max(FORECAST_REPORT_HORIZONS) + 1)