""" Command line entry point. python -m btcmodel update fetch new daily prices from Coinbase python -m btcmodel backtest score models on development data python -m btcmodel backtest --holdout score models on outcomes after DEV_CUTOFF python -m btcmodel forecast forecast from the latest price python -m btcmodel ab [NAME ...] run A/B experiments on development data python -m btcmodel snapshot record tracked models' forecasts for the forward test python -m btcmodel forward score recorded forecasts whose targets have passed """ import argparse from pathlib import Path import numpy as np import pandas as pd from . import data, evaluate, experiments, forward, plots from .models import MODELS FORECAST_REPORT_HORIZONS = (182, 365, 730, 1095, 1460) FORECAST_REPORT_LEVELS = (0.05, 0.25, 0.5, 0.75, 0.95) def main() -> None: parser = argparse.ArgumentParser(prog="btcmodel", description="Bitcoin price model") parser.add_argument("-o", "--output", type=Path, default=Path("output")) parser.add_argument("-m", "--models", nargs="+", choices=list(MODELS), default=list(MODELS)) commands = parser.add_subparsers(dest="command", required=True) commands.add_parser("update", help="fetch new daily prices") backtest = commands.add_parser("backtest", help="walk-forward evaluation") backtest.add_argument( "--holdout", action="store_true", help=f"score outcomes after {data.DEV_CUTOFF:%Y-%m-%d} (don't use while developing)", ) commands.add_parser("forecast", help="forecast from the latest price") ab = commands.add_parser("ab", help="run A/B experiments on development data") ab.add_argument("names", nargs="*", help="experiments to run (default: all)") commands.add_parser("snapshot", help="record forecasts for the forward test") commands.add_parser("forward", help="score recorded forecasts") args = parser.parse_args() if args.command == "ab" and (unknown := set(args.names) - set(experiments.EXPERIMENTS)): parser.error(f"unknown experiments: {', '.join(sorted(unknown))}") models = [MODELS[name] for name in args.models] if args.command == "update": added = data.update_coinbase() print(f"added {added} days; latest {data.load_prices().index[-1]:%Y-%m-%d}") elif args.command == "backtest": run_backtest(models, args.output, args.holdout) elif args.command == "forecast": run_forecast(models, args.output) elif args.command == "ab": run_ab(args.names or list(experiments.EXPERIMENTS), args.output) elif args.command == "snapshot": run_snapshot() elif args.command == "forward": run_forward(args.output) def run_backtest(models, output: Path, holdout: bool) -> None: if holdout: name, prices = "holdout", data.load_prices() scores = evaluate.backtest(models, prices, score_after=data.DEV_CUTOFF) else: name, prices = "backtest", data.load_prices(until=data.DEV_CUTOFF) scores = evaluate.backtest(models, prices) out = output / name out.mkdir(parents=True, exist_ok=True) summary = evaluate.summarize(scores) report = ( f"{name}: origins every {evaluate.ORIGIN_STEP_DAYS} days from " f"{evaluate.FIRST_ORIGIN:%Y-%m-%d}, outcomes through {prices.index[-1]:%Y-%m-%d}\n\n" + evaluate.format_summary(summary) ) print(report) (out / "report.txt").write_text(report + "\n") scores.to_csv(out / "scores.csv", index=False) summary.to_csv(out / "summary.csv", index=False) plots.skill_chart(summary, out / "skill.png", f"{name}: skill by horizon") plots.calibration_chart(summary, out / "calibration.png", f"{name}: interval coverage") print(f"\nwrote {out}/") def run_ab(names: list[str], output: Path) -> None: prices = data.load_prices(until=data.DEV_CUTOFF) verdicts = [] for name in names: experiment = experiments.EXPERIMENTS[name] scores, summary = experiments.run(experiment, prices) out = output / "ab" / name out.mkdir(parents=True, exist_ok=True) control = experiment.control.name report = ( f"{name}: {experiment.hypothesis}\n(from {experiment.source})\n\n" + evaluate.format_summary(summary, baseline=control) ) print(report + "\n") (out / "report.txt").write_text(report + "\n") summary.to_csv(out / "summary.csv", index=False) plots.skill_chart(summary, out / "skill.png", f"{name}: skill vs {control}", control) plots.calibration_chart(summary, out / "calibration.png", f"{name}: interval coverage") for variant in experiment.variants: v = summary[summary.model == variant.name].set_index("horizon") verdicts.append( { "experiment": name, "variant": variant.name, "control": control, **{evaluate.horizon_label(h): f"{s:+.0%}" for h, s in v["skill"].items()}, "verdict": experiments.verdict(v), } ) table = pd.DataFrame(verdicts).to_string(index=False) print(table) (output / "ab").mkdir(parents=True, exist_ok=True) (output / "ab" / "verdicts.txt").write_text(table + "\n") def run_snapshot() -> None: prices = data.load_prices() path = forward.snapshot(prices) recorded = [(m, f) for m, f in forward.load_snapshots() if f.origin == prices.index[-1]] medians = pd.DataFrame( {m: [plots.price_formatter(p) for p in np.exp(f.quantile(0.5))] for m, f in recorded}, index=[evaluate.horizon_label(h) for h in evaluate.HORIZONS], ).T print( f"recorded {len(recorded)} models' forecasts from {prices.index[-1]:%Y-%m-%d} in {path}\n" ) print("medians:\n" + medians.to_string()) def run_forward(output: Path) -> None: prices = data.load_prices() scores = forward.score_snapshots(prices) due = forward.next_due(prices) if scores.empty: when = f"; the first is due {due:%Y-%m-%d}" if due is not None else "" print(f"no recorded forecast has reached its target date yet{when}") return out = output / "forward" out.mkdir(parents=True, exist_ok=True) summary = evaluate.summarize(scores, step_days=forward.SNAPSHOT_STEP_DAYS) report = ( f"forward test: {scores.origin.nunique()} snapshots from {scores.origin.min():%Y-%m-%d}, " f"outcomes through {prices.index[-1]:%Y-%m-%d}\n\n" + evaluate.format_summary(summary) ) if due is not None: report += f"\n\nnext outcome due {due:%Y-%m-%d}" print(report) (out / "report.txt").write_text(report + "\n") scores.to_csv(out / "scores.csv", index=False) summary.to_csv(out / "summary.csv", index=False) plots.skill_chart(summary, out / "skill.png", "forward test: skill by horizon") plots.calibration_chart(summary, out / "calibration.png", "forward test: interval coverage") print(f"\nwrote {out}/") def run_forecast(models, output: Path) -> None: prices = data.load_prices() horizons = np.arange(1, max(FORECAST_REPORT_HORIZONS) + 1) forecasts = {m.name: m.forecast(prices, horizons) for m in models} out = output / "forecast" out.mkdir(parents=True, exist_ok=True) rows = [] for name, f in forecasts.items(): for h in FORECAST_REPORT_HORIZONS: row = {"model": name, "date": f.dates[h - 1].date(), "horizon": h} for level in FORECAST_REPORT_LEVELS: row[f"p{level * 100:02.0f}"] = np.exp(f.quantile(level)[h - 1]) rows.append(row) table = pd.DataFrame(rows) table.to_csv(out / "forecast.csv", index=False) shown = table.copy() for column in shown.columns[3:]: shown[column] = shown[column].map(plots.price_formatter) print(f"from {prices.index[-1]:%Y-%m-%d} at {plots.price_formatter(prices.close.iloc[-1])}\n") print(shown.to_string(index=False)) plots.fan_chart(prices, forecasts, out / "fan.png") print(f"\nwrote {out}/") if __name__ == "__main__": main()