Composable models and A/B tests of the 2024 ideas; add powerlaw.

Models are now a Composite of drift, volatility and (optional) shape
components, so an experiment can swap one part against a fixed control.

btcmodel/experiments.py holds seven experiments built from the ideas in the
old branches (catalogued in docs/2024-ideas.md), each with its hypothesis
and source, and a verdict rule fixed before anything ran. `just ab` runs
them on development data. Results:

- Shrinking the cycle drift, and a power-law trend (plain or reverting),
  beat their controls. The power law beats the random walk by 53-63% at
  3-4 years with unbiased outcomes, so it is promoted to MODELS.
- Every alternative volatility estimate (EWMA blends, other windows,
  reversion to a level or trend) is worse than the trailing 365-day window.
  Cycle-dependent volatility, heavy tails and stretched cycle phase show no
  reliable effect.
This commit is contained in:
sam
2026-09-24 03:01:46 -07:00
parent cfc27a38de
commit b0243adf61
18 changed files with 795 additions and 155 deletions
+17 -10
View File
@@ -56,8 +56,12 @@ plt.rcParams.update(
)
def model_color(name: str) -> str:
return SERIES[list(MODELS).index(name) % len(SERIES)]
def model_colors(names) -> dict[str, str]:
"""Registered models keep their MODELS slot; others follow in order of appearance."""
names = list(dict.fromkeys(names))
if all(n in MODELS for n in names):
return {n: SERIES[list(MODELS).index(n)] for n in names}
return {n: SERIES[i % len(SERIES)] for i, n in enumerate(names)}
def price_formatter(x, _=None) -> str:
@@ -74,8 +78,9 @@ def fan_chart(history: pd.DataFrame, forecasts: dict[str, Forecast], path: Path)
)
axes = np.atleast_1d(axes)
shown = history[history.index >= history.index[-1] - pd.Timedelta(days=6 * 365)]
colors = model_colors(forecasts)
for ax, (name, f) in zip(axes, forecasts.items(), strict=True):
color = model_color(name)
color = colors[name]
for c in sorted(COVERAGES, reverse=True):
lo, hi = f.interval(c)
ax.fill_between(f.dates, np.exp(lo), np.exp(hi), color=color, alpha=0.1, lw=0)
@@ -108,15 +113,16 @@ def fan_chart(history: pd.DataFrame, forecasts: dict[str, Forecast], path: Path)
plt.close(fig)
def skill_chart(summary: pd.DataFrame, path: Path, title: str) -> None:
"""CRPS skill vs the random walk, by horizon, with bootstrap intervals."""
def skill_chart(summary: pd.DataFrame, path: Path, title: str, baseline: str = BASELINE) -> None:
"""CRPS skill vs `baseline`, by horizon, with bootstrap intervals."""
horizons = sorted(summary["horizon"].unique())
x = np.arange(len(horizons))
colors = model_colors(summary["model"])
fig, ax = plt.subplots(figsize=(8, 4.5))
ax.axhline(0, color=model_color(BASELINE), lw=2, label=BASELINE)
for name, g in summary[summary.model != BASELINE].groupby("model", sort=False):
ax.axhline(0, color=colors[baseline], lw=2, label=baseline)
for name, g in summary[summary.model != baseline].groupby("model", sort=False):
g = g.set_index("horizon").reindex(horizons)
color = model_color(name)
color = colors[name]
ax.fill_between(x, g.skill_lo, g.skill_hi, color=color, alpha=0.1, lw=0)
ax.plot(x, g.skill, color=color, marker="o", ms=6, mec=SURFACE, mew=2, label=name)
ax.annotate(
@@ -129,7 +135,7 @@ def skill_chart(summary: pd.DataFrame, path: Path, title: str) -> None:
)
ax.set_xticks(x, [horizon_label(h) for h in horizons])
ax.set_xlabel("forecast horizon")
ax.set_ylabel("CRPS skill vs random walk (higher is better)")
ax.set_ylabel(f"CRPS skill vs {baseline} (higher is better)")
ax.yaxis.set_major_formatter(PercentFormatter(1.0, decimals=0))
ax.set_title(title)
ax.legend(loc="lower left")
@@ -142,6 +148,7 @@ def calibration_chart(summary: pd.DataFrame, path: Path, title: str) -> None:
"""How often each nominal interval contained the outcome, by horizon."""
horizons = sorted(summary["horizon"].unique())
x = np.arange(len(horizons))
colors = model_colors(summary["model"])
fig, axes = plt.subplots(1, len(COVERAGES), figsize=(12, 4), sharey=True)
for ax, c in zip(axes, COVERAGES, strict=True):
ax.axhline(c, color=INK_SECONDARY, lw=1)
@@ -153,7 +160,7 @@ def calibration_chart(summary: pd.DataFrame, path: Path, title: str) -> None:
ax.plot(
x,
g[f"cov{c:.0%}"],
color=model_color(name),
color=colors[name],
marker="o",
ms=6,
mec=SURFACE,