From 249826afc0db814170fcd1670125350a3719553e Mon Sep 17 00:00:00 2001 From: Christophe Vila Date: Sun, 19 Jul 2026 12:57:18 +0200 Subject: [PATCH] Move backfill horizon from a startup env var into the editable profile SMARTRUN_BACKFILL_HORIZON_DAYS was a server-startup-only env var with no UI, defaulting to 3 years -- so editing the unrelated "Rolling window" profile field (for classification, not sync) had no effect on how far back Sync Now reached. Backfill horizon is now Profile.BackfillHorizonDays, read fresh on every Backfill call, with its own field on the Profile page. --- .claude/skills/smartrun-dev/SKILL.md | 2 +- backend/cmd/smartrund/main.go | 3 +- backend/internal/config/config.go | 11 ------- .../migrations/0009_backfill_horizon.sql | 6 ++++ backend/internal/store/profile.go | 18 ++++++---- backend/internal/store/profile_test.go | 7 ++++ backend/internal/sync/service.go | 33 +++++++++++-------- backend/internal/sync/service_test.go | 27 ++++++++++++--- frontend/src/pages/Profile.tsx | 13 ++++++++ frontend/src/types/api.ts | 1 + 10 files changed, 84 insertions(+), 37 deletions(-) create mode 100644 backend/internal/store/migrations/0009_backfill_horizon.sql diff --git a/.claude/skills/smartrun-dev/SKILL.md b/.claude/skills/smartrun-dev/SKILL.md index 0c15859..81edcc0 100644 --- a/.claude/skills/smartrun-dev/SKILL.md +++ b/.claude/skills/smartrun-dev/SKILL.md @@ -70,7 +70,7 @@ Each `workout_kinds.rule_json` is a recursive AND/OR condition tree (`internal/c - `kind_assignments` is **append-only** — always INSERT, never UPDATE. Re-classifying after a rule edit, or a manual override, keeps full history; `current_kind_assignment` (a view) picks the latest row per activity by `id`. - `activities.raw_json`/`details_raw_json` hedge columns store the full original Garmin JSON, so fields not yet modeled in Go can be backfilled later without re-fetching from Garmin. - `garmin_activity_id` is the natural idempotency key for `UpsertActivity` (`ON CONFLICT ... DO UPDATE`), safe to re-run on every sync pass. -- `sync_state` (singleton row) tracks a **backfill watermark** (`earliest_synced_date`, `backfill_complete`) — since Garmin history is immutable once recorded, `Service.Backfill` uses this to resume from where it left off (or no-op entirely if the configured horizon is already fully covered) instead of re-walking years of already-known history against Garmin's API on every call. Widening `BackfillHorizonDays` after a completed backfill correctly triggers resumption further back, not a full re-fetch. +- `sync_state` (singleton row) tracks a **backfill watermark** (`earliest_synced_date`, `backfill_complete`) — since Garmin history is immutable once recorded, `Service.Backfill` uses this to resume from where it left off (or no-op entirely if the configured horizon is already fully covered) instead of re-walking years of already-known history against Garmin's API on every call. `Backfill` reads `Profile.BackfillHorizonDays` fresh on every call (not a fixed `Config` field), so widening it in the Profile page takes effect on the very next sync, no restart needed, and correctly triggers resumption further back rather than a full re-fetch. "Sync now" (`POST /api/sync/run`) calls `Backfill` then `IncrementalSync` then `FillPendingDetails` in one pass — there's no separate "full backfill" trigger anymore. "Reset all" (`POST /api/sync/reset`) is the destructive counterpart: deletes every activity (cascading to laps/samples/kind_assignments) and rewinds the watermark, so the next sync is a genuinely fresh pull — the only way to get already-synced activities re-processed against newer schema fields (e.g. a newly-added metric), since `FillPendingDetails` only ever touches activities whose details were never fetched. - Live sync progress is exposed via `Service.Progress()` (in-memory, mutex-guarded `Done`/`Total` counters set by `FillPendingDetails`, reset to zero when idle) and surfaced through `GET /api/sync/status` (`detail_fill_progress`, plus `activities_pending_details` for the total remaining beyond the current batch). The frontend's `GarminConnection` banner polls this and shows "syncing: N/M activities" live. ## Dev workflow diff --git a/backend/cmd/smartrund/main.go b/backend/cmd/smartrund/main.go index 82cd61c..1861fe0 100644 --- a/backend/cmd/smartrund/main.go +++ b/backend/cmd/smartrund/main.go @@ -45,8 +45,7 @@ func main() { defer garminClient.Close() syncSvc := appsync.NewService(garminClient, db, appsync.Config{ - BackfillHorizonDays: cfg.BackfillHorizonDays, - MinConfidence: cfg.MinConfidence, + MinConfidence: cfg.MinConfidence, }, nil) server := api.NewServer(db, garminClient, syncSvc) diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go index 844c805..d3a1856 100644 --- a/backend/internal/config/config.go +++ b/backend/internal/config/config.go @@ -28,7 +28,6 @@ type Config struct { GarminTokenStore string MinConfidence float64 - BackfillHorizonDays int IncrementalSyncEvery time.Duration } @@ -42,7 +41,6 @@ func Load() (Config, error) { GarminServerPath: os.Getenv("MCP_GARMIN_SERVER"), GarminTokenStore: os.Getenv("GARMIN_TOKENSTORE"), MinConfidence: getEnvFloat("SMARTRUN_MIN_CONFIDENCE", 0.6), - BackfillHorizonDays: getEnvInt("SMARTRUN_BACKFILL_HORIZON_DAYS", 3*365), IncrementalSyncEvery: getEnvDuration("SMARTRUN_INCREMENTAL_SYNC_EVERY", 6*time.Hour), } @@ -71,15 +69,6 @@ func getEnvFloat(key string, def float64) float64 { return def } -func getEnvInt(key string, def int) int { - if v := os.Getenv(key); v != "" { - if n, err := strconv.Atoi(v); err == nil { - return n - } - } - return def -} - func getEnvDuration(key string, def time.Duration) time.Duration { if v := os.Getenv(key); v != "" { if d, err := time.ParseDuration(v); err == nil { diff --git a/backend/internal/store/migrations/0009_backfill_horizon.sql b/backend/internal/store/migrations/0009_backfill_horizon.sql new file mode 100644 index 0000000..86da295 --- /dev/null +++ b/backend/internal/store/migrations/0009_backfill_horizon.sql @@ -0,0 +1,6 @@ +-- Backfill horizon used to be a startup-time env var +-- (SMARTRUN_BACKFILL_HORIZON_DAYS) with no UI at all -- exactly the kind of +-- "tunable analysis-engine parameter" this profile table exists for. +-- Default matches the old env var's default (3 years) so existing +-- deployments keep their current behavior until the user changes it. +ALTER TABLE profile ADD COLUMN backfill_horizon_days INTEGER NOT NULL DEFAULT 1095; diff --git a/backend/internal/store/profile.go b/backend/internal/store/profile.go index 1a4af9f..89d9ffd 100644 --- a/backend/internal/store/profile.go +++ b/backend/internal/store/profile.go @@ -11,8 +11,14 @@ type Profile struct { GarminEmail string GarminPassword string RollingWindowDays int - MaxHeartRate *float64 - RestingHeartRate *float64 + // BackfillHorizonDays bounds how far back "Sync now" reaches when + // walking backward from today; it's read fresh on every sync (not fixed + // at server startup), so changing it here takes effect on the next + // click. Unrelated to RollingWindowDays, which is for the (not yet + // implemented) relative classification window. + BackfillHorizonDays int + MaxHeartRate *float64 + RestingHeartRate *float64 HRZone1MinPct, HRZone1MaxPct float64 HRZone2MinPct, HRZone2MaxPct float64 @@ -30,7 +36,7 @@ type Profile struct { } const profileColumns = ` - garmin_email, garmin_password, rolling_window_days, max_heart_rate, resting_heart_rate, + garmin_email, garmin_password, rolling_window_days, backfill_horizon_days, max_heart_rate, resting_heart_rate, hr_zone1_min_pct, hr_zone1_max_pct, hr_zone2_min_pct, hr_zone2_max_pct, hr_zone3_min_pct, hr_zone3_max_pct, hr_zone4_min_pct, hr_zone4_max_pct, hr_zone5_min_pct, hr_zone5_max_pct, @@ -42,7 +48,7 @@ const profileColumns = ` func (db *DB) GetProfile(ctx context.Context) (Profile, error) { var p Profile err := db.QueryRowContext(ctx, `SELECT `+profileColumns+` FROM profile WHERE id = 1`).Scan( - &p.GarminEmail, &p.GarminPassword, &p.RollingWindowDays, &p.MaxHeartRate, &p.RestingHeartRate, + &p.GarminEmail, &p.GarminPassword, &p.RollingWindowDays, &p.BackfillHorizonDays, &p.MaxHeartRate, &p.RestingHeartRate, &p.HRZone1MinPct, &p.HRZone1MaxPct, &p.HRZone2MinPct, &p.HRZone2MaxPct, &p.HRZone3MinPct, &p.HRZone3MaxPct, &p.HRZone4MinPct, &p.HRZone4MaxPct, &p.HRZone5MinPct, &p.HRZone5MaxPct, @@ -61,14 +67,14 @@ func (db *DB) GetProfile(ctx context.Context) (Profile, error) { func (db *DB) UpdateProfile(ctx context.Context, p Profile) error { _, err := db.ExecContext(ctx, ` UPDATE profile SET - garmin_email=?, garmin_password=?, rolling_window_days=?, max_heart_rate=?, resting_heart_rate=?, + garmin_email=?, garmin_password=?, rolling_window_days=?, backfill_horizon_days=?, max_heart_rate=?, resting_heart_rate=?, hr_zone1_min_pct=?, hr_zone1_max_pct=?, hr_zone2_min_pct=?, hr_zone2_max_pct=?, hr_zone3_min_pct=?, hr_zone3_max_pct=?, hr_zone4_min_pct=?, hr_zone4_max_pct=?, hr_zone5_min_pct=?, hr_zone5_max_pct=?, warmup_minutes=?, cooldown_minutes=?, updated_at=datetime('now') WHERE id = 1`, - p.GarminEmail, p.GarminPassword, p.RollingWindowDays, p.MaxHeartRate, p.RestingHeartRate, + p.GarminEmail, p.GarminPassword, p.RollingWindowDays, p.BackfillHorizonDays, p.MaxHeartRate, p.RestingHeartRate, p.HRZone1MinPct, p.HRZone1MaxPct, p.HRZone2MinPct, p.HRZone2MaxPct, p.HRZone3MinPct, p.HRZone3MaxPct, p.HRZone4MinPct, p.HRZone4MaxPct, p.HRZone5MinPct, p.HRZone5MaxPct, diff --git a/backend/internal/store/profile_test.go b/backend/internal/store/profile_test.go index 5042017..2433e7a 100644 --- a/backend/internal/store/profile_test.go +++ b/backend/internal/store/profile_test.go @@ -22,6 +22,9 @@ func TestProfile_DefaultsThenUpdate(t *testing.T) { if p.WarmupMinutes != 10 || p.CooldownMinutes != 5 { t.Errorf("phase-minute defaults = %+v, want warmup=10, cooldown=5", p) } + if p.BackfillHorizonDays != 1095 { + t.Errorf("BackfillHorizonDays = %d, want 1095 (migration default)", p.BackfillHorizonDays) + } maxHR, restingHR := 190.0, 50.0 p.GarminEmail = "runner@example.com" @@ -30,6 +33,7 @@ func TestProfile_DefaultsThenUpdate(t *testing.T) { p.MaxHeartRate = &maxHR p.RestingHeartRate = &restingHR p.WarmupMinutes = 8 + p.BackfillHorizonDays = 14 if err := db.UpdateProfile(ctx, p); err != nil { t.Fatalf("UpdateProfile: %v", err) @@ -48,4 +52,7 @@ func TestProfile_DefaultsThenUpdate(t *testing.T) { if got.WarmupMinutes != 8 { t.Errorf("WarmupMinutes = %v, want 8", got.WarmupMinutes) } + if got.BackfillHorizonDays != 14 { + t.Errorf("BackfillHorizonDays = %v, want 14", got.BackfillHorizonDays) + } } diff --git a/backend/internal/sync/service.go b/backend/internal/sync/service.go index 9156be4..55750fa 100644 --- a/backend/internal/sync/service.go +++ b/backend/internal/sync/service.go @@ -18,10 +18,10 @@ import ( ) // Config tunes sync behavior. Zero values fall back to sensible defaults in -// NewService. +// NewService. How far back Backfill reaches is not here -- it's +// Profile.BackfillHorizonDays, read fresh on every call so a user-edited +// value takes effect on the next sync without a server restart. type Config struct { - // BackfillHorizonDays bounds how far back a full backfill reaches. - BackfillHorizonDays int // BackfillWindowDays is the page size for each get_activities call // during backfill. BackfillWindowDays int @@ -39,9 +39,6 @@ type Config struct { } func (c Config) withDefaults() Config { - if c.BackfillHorizonDays == 0 { - c.BackfillHorizonDays = 3 * 365 - } if c.BackfillWindowDays == 0 { c.BackfillWindowDays = 90 } @@ -98,19 +95,29 @@ func (s *Service) setProgress(done, total int) { } // Backfill pages backward in Config.BackfillWindowDays windows until -// Config.BackfillHorizonDays is reached or Garmin returns an empty page. -// Safe to re-run: activities are upserted by garmin_activity_id, and thanks -// to the sync_state watermark (Garmin history is immutable once recorded) -// a repeat call only fetches whatever's newer than the last completed -// backfill, or is a fast no-op if the configured horizon is already fully -// covered -- it does not re-walk years of already-known history. +// Profile.BackfillHorizonDays is reached or Garmin returns an empty page. +// The horizon is read fresh from the profile on every call (not fixed at +// server startup), so a user-edited value takes effect on the very next +// sync. Safe to re-run: activities are upserted by garmin_activity_id, and +// thanks to the sync_state watermark (Garmin history is immutable once +// recorded) a repeat call only fetches whatever's newer than the last +// completed backfill, or is a fast no-op if the configured horizon is +// already fully covered -- it does not re-walk years of already-known +// history. Widening the horizon between calls resumes further back instead +// of re-fetching everything. func (s *Service) Backfill(ctx context.Context) error { runID, err := s.db.StartSyncRun(ctx, store.SyncKindBackfill) if err != nil { return err } - horizon := s.now().AddDate(0, 0, -s.cfg.BackfillHorizonDays) + profile, err := s.db.GetProfile(ctx) + if err != nil { + msg := err.Error() + s.db.FinishSyncRun(ctx, runID, 0, &msg) + return fmt.Errorf("load profile: %w", err) + } + horizon := s.now().AddDate(0, 0, -profile.BackfillHorizonDays) state, err := s.db.GetSyncState(ctx) if err != nil { diff --git a/backend/internal/sync/service_test.go b/backend/internal/sync/service_test.go index 4826eba..41bc6d6 100644 --- a/backend/internal/sync/service_test.go +++ b/backend/internal/sync/service_test.go @@ -28,6 +28,21 @@ func fixedNow(t time.Time) func() time.Time { return func() time.Time { return t } } +// setBackfillHorizon sets Profile.BackfillHorizonDays, which Backfill reads +// fresh on every call (it's no longer part of Config). +func setBackfillHorizon(t *testing.T, db *store.DB, days int) { + t.Helper() + ctx := context.Background() + profile, err := db.GetProfile(ctx) + if err != nil { + t.Fatalf("GetProfile: %v", err) + } + profile.BackfillHorizonDays = days + if err := db.UpdateProfile(ctx, profile); err != nil { + t.Fatalf("UpdateProfile: %v", err) + } +} + func TestBackfill_StoresActivitiesAndRecordsSyncRun(t *testing.T) { db := openTestDB(t) ctx := context.Background() @@ -399,8 +414,9 @@ func TestBackfill_SecondRunIsANoOpOnceHorizonFullyCovered(t *testing.T) { m := &mock.Client{Activities: []garmin.Activity{ {ActivityID: 1, ActivityType: garmin.ActivityType{TypeKey: "running"}, StartTimeGMT: "2026-07-05 06:00:00", Distance: 5000, Duration: 1500}, }} - svc := NewService(m, db, Config{BackfillHorizonDays: 10, BackfillWindowDays: 10}, + svc := NewService(m, db, Config{BackfillWindowDays: 10}, fixedNow(time.Date(2026, 7, 11, 0, 0, 0, 0, time.UTC))) + setBackfillHorizon(t, db, 10) if err := svc.Backfill(ctx); err != nil { t.Fatalf("first Backfill: %v", err) @@ -434,8 +450,9 @@ func TestResetAll_AllowsFreshBackfillAfterwards(t *testing.T) { m := &mock.Client{Activities: []garmin.Activity{ {ActivityID: 1, ActivityType: garmin.ActivityType{TypeKey: "running"}, StartTimeGMT: "2026-07-05 06:00:00", Distance: 5000, Duration: 1500}, }} - svc := NewService(m, db, Config{BackfillHorizonDays: 10, BackfillWindowDays: 10}, + svc := NewService(m, db, Config{BackfillWindowDays: 10}, fixedNow(time.Date(2026, 7, 11, 0, 0, 0, 0, time.UTC))) + setBackfillHorizon(t, db, 10) if err := svc.Backfill(ctx); err != nil { t.Fatalf("first Backfill: %v", err) @@ -477,7 +494,8 @@ func TestBackfill_ResumesFromWatermarkWhenHorizonGrows(t *testing.T) { }} now := fixedNow(time.Date(2026, 7, 11, 0, 0, 0, 0, time.UTC)) - svc := NewService(m, db, Config{BackfillHorizonDays: 10, BackfillWindowDays: 10}, now) + svc := NewService(m, db, Config{BackfillWindowDays: 10}, now) + setBackfillHorizon(t, db, 10) if err := svc.Backfill(ctx); err != nil { t.Fatalf("first Backfill: %v", err) } @@ -486,7 +504,8 @@ func TestBackfill_ResumesFromWatermarkWhenHorizonGrows(t *testing.T) { // Simulate the user widening the horizon later -- should resume from the // watermark (not re-fetch the already-covered recent window) but still // make progress toward the new, deeper horizon. - svc2 := NewService(m, db, Config{BackfillHorizonDays: 30, BackfillWindowDays: 10}, now) + setBackfillHorizon(t, db, 30) + svc2 := NewService(m, db, Config{BackfillWindowDays: 10}, now) if err := svc2.Backfill(ctx); err != nil { t.Fatalf("second Backfill: %v", err) } diff --git a/frontend/src/pages/Profile.tsx b/frontend/src/pages/Profile.tsx index ab4fa3d..ab9fa08 100644 --- a/frontend/src/pages/Profile.tsx +++ b/frontend/src/pages/Profile.tsx @@ -118,6 +118,19 @@ export function Profile() { /> +
+ Sync + set("BackfillHorizonDays", v)} + /> +

+ How far back "Sync now" reaches when walking backward from today. Not the same as the classification rolling + window above. +

+
+
Heart rate