Composable models and A/B tests of the 2024 ideas; add powerlaw.
Models are now a Composite of drift, volatility and (optional) shape components, so an experiment can swap one part against a fixed control. btcmodel/experiments.py holds seven experiments built from the ideas in the old branches (catalogued in docs/2024-ideas.md), each with its hypothesis and source, and a verdict rule fixed before anything ran. `just ab` runs them on development data. Results: - Shrinking the cycle drift, and a power-law trend (plain or reverting), beat their controls. The power law beats the random walk by 53-63% at 3-4 years with unbiased outcomes, so it is promoted to MODELS. - Every alternative volatility estimate (EWMA blends, other windows, reversion to a level or trend) is worse than the trailing 365-day window. Cycle-dependent volatility, heavy tails and stretched cycle phase show no reliable effect.
This commit is contained in:
+17
-10
@@ -56,8 +56,12 @@ plt.rcParams.update(
|
||||
)
|
||||
|
||||
|
||||
def model_color(name: str) -> str:
|
||||
return SERIES[list(MODELS).index(name) % len(SERIES)]
|
||||
def model_colors(names) -> dict[str, str]:
|
||||
"""Registered models keep their MODELS slot; others follow in order of appearance."""
|
||||
names = list(dict.fromkeys(names))
|
||||
if all(n in MODELS for n in names):
|
||||
return {n: SERIES[list(MODELS).index(n)] for n in names}
|
||||
return {n: SERIES[i % len(SERIES)] for i, n in enumerate(names)}
|
||||
|
||||
|
||||
def price_formatter(x, _=None) -> str:
|
||||
@@ -74,8 +78,9 @@ def fan_chart(history: pd.DataFrame, forecasts: dict[str, Forecast], path: Path)
|
||||
)
|
||||
axes = np.atleast_1d(axes)
|
||||
shown = history[history.index >= history.index[-1] - pd.Timedelta(days=6 * 365)]
|
||||
colors = model_colors(forecasts)
|
||||
for ax, (name, f) in zip(axes, forecasts.items(), strict=True):
|
||||
color = model_color(name)
|
||||
color = colors[name]
|
||||
for c in sorted(COVERAGES, reverse=True):
|
||||
lo, hi = f.interval(c)
|
||||
ax.fill_between(f.dates, np.exp(lo), np.exp(hi), color=color, alpha=0.1, lw=0)
|
||||
@@ -108,15 +113,16 @@ def fan_chart(history: pd.DataFrame, forecasts: dict[str, Forecast], path: Path)
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def skill_chart(summary: pd.DataFrame, path: Path, title: str) -> None:
|
||||
"""CRPS skill vs the random walk, by horizon, with bootstrap intervals."""
|
||||
def skill_chart(summary: pd.DataFrame, path: Path, title: str, baseline: str = BASELINE) -> None:
|
||||
"""CRPS skill vs `baseline`, by horizon, with bootstrap intervals."""
|
||||
horizons = sorted(summary["horizon"].unique())
|
||||
x = np.arange(len(horizons))
|
||||
colors = model_colors(summary["model"])
|
||||
fig, ax = plt.subplots(figsize=(8, 4.5))
|
||||
ax.axhline(0, color=model_color(BASELINE), lw=2, label=BASELINE)
|
||||
for name, g in summary[summary.model != BASELINE].groupby("model", sort=False):
|
||||
ax.axhline(0, color=colors[baseline], lw=2, label=baseline)
|
||||
for name, g in summary[summary.model != baseline].groupby("model", sort=False):
|
||||
g = g.set_index("horizon").reindex(horizons)
|
||||
color = model_color(name)
|
||||
color = colors[name]
|
||||
ax.fill_between(x, g.skill_lo, g.skill_hi, color=color, alpha=0.1, lw=0)
|
||||
ax.plot(x, g.skill, color=color, marker="o", ms=6, mec=SURFACE, mew=2, label=name)
|
||||
ax.annotate(
|
||||
@@ -129,7 +135,7 @@ def skill_chart(summary: pd.DataFrame, path: Path, title: str) -> None:
|
||||
)
|
||||
ax.set_xticks(x, [horizon_label(h) for h in horizons])
|
||||
ax.set_xlabel("forecast horizon")
|
||||
ax.set_ylabel("CRPS skill vs random walk (higher is better)")
|
||||
ax.set_ylabel(f"CRPS skill vs {baseline} (higher is better)")
|
||||
ax.yaxis.set_major_formatter(PercentFormatter(1.0, decimals=0))
|
||||
ax.set_title(title)
|
||||
ax.legend(loc="lower left")
|
||||
@@ -142,6 +148,7 @@ def calibration_chart(summary: pd.DataFrame, path: Path, title: str) -> None:
|
||||
"""How often each nominal interval contained the outcome, by horizon."""
|
||||
horizons = sorted(summary["horizon"].unique())
|
||||
x = np.arange(len(horizons))
|
||||
colors = model_colors(summary["model"])
|
||||
fig, axes = plt.subplots(1, len(COVERAGES), figsize=(12, 4), sharey=True)
|
||||
for ax, c in zip(axes, COVERAGES, strict=True):
|
||||
ax.axhline(c, color=INK_SECONDARY, lw=1)
|
||||
@@ -153,7 +160,7 @@ def calibration_chart(summary: pd.DataFrame, path: Path, title: str) -> None:
|
||||
ax.plot(
|
||||
x,
|
||||
g[f"cov{c:.0%}"],
|
||||
color=model_color(name),
|
||||
color=colors[name],
|
||||
marker="o",
|
||||
ms=6,
|
||||
mec=SURFACE,
|
||||
|
||||
Reference in New Issue
Block a user