diff --git a/api/containerautomation/autoupdate.go b/api/containerautomation/autoupdate.go index cbbd6baba..4f3a6d2f0 100644 --- a/api/containerautomation/autoupdate.go +++ b/api/containerautomation/autoupdate.go @@ -237,6 +237,19 @@ func (s *Service) stackLookupForEndpoint(endpointID portainer.EndpointID) func(p func (s *Service) updateStandalone(cli *dockerclient.Client, endpoint *portainer.Endpoint, c UpdateCandidate, opts updateOptions) { endpointID := int(endpoint.ID) + // Loop-guard safety: the rolled-back map is keyed by endpoint+name (the only + // identifier that survives a recreate). An unnamed container cannot be recorded + // (recordRolledBack skips it), so with rollback enabled a container that keeps + // failing its health gate would update->rollback every tick with NO suppression. + // Skip the unnamed case when rollback is on so it cannot enter that + // unsuppressable loop; detection/badge refresh already happened upstream and is + // unaffected. (With rollback off there is no rollback to loop, so we proceed.) + if skipUnnamedForRollback(opts.rollback, c.Name) { + log.Info().Str("container_id", c.ID).Int("endpoint_id", endpointID). + Msg("auto-update: skipping unnamed standalone container, rollback is enabled but there is no stable name to key the loop guard") + return + } + // Update->rollback loop guard: if this container's update was rolled back // recently and the remote still points at the SAME failed image, skip it until // the cooldown elapses. A genuinely new upstream image (a changed remote digest) @@ -260,7 +273,12 @@ func (s *Service) updateStandalone(cli *dockerclient.Client, endpoint *portainer var startPeriod time.Duration healthGated := false if opts.rollback { - if inspect, err := cli.ContainerInspect(s.baseCtx, c.ID); err != nil { + // Bound the inspect like every other engine call so a hung/unreachable engine + // cannot block the whole sequential tick until shutdown. + inspectCtx, inspectCancel := context.WithTimeout(s.baseCtx, endpointTimeout) + inspect, err := cli.ContainerInspect(inspectCtx, c.ID) + inspectCancel() + if err != nil { log.Warn().Err(err).Str("container_id", c.ID).Int("endpoint_id", endpointID). Msg("auto-update: unable to inspect container before update, proceeding without a health gate") } else { @@ -355,6 +373,17 @@ func containerName(names []string) string { return strings.TrimPrefix(names[0], "/") } +// skipUnnamedForRollback reports whether a standalone update must be skipped +// because rollback is enabled but the container has no stable name to key the +// loop guard. The rolled-back map is keyed by endpoint+name (the only identifier +// that survives a recreate); without a name the guard cannot record a failed +// target, so a repeatedly-failing update would loop update->rollback every tick +// with no suppression. When rollback is off there is nothing to loop, so an +// unnamed container is still allowed to update. +func skipUnnamedForRollback(rollback bool, name string) bool { + return rollback && name == "" +} + // rollbackKey identifies a standalone container in the rolled-back map by its // endpoint and (recreate-stable) name. A recreate assigns a new container ID, so // the ID cannot key state across an update; the name is preserved. diff --git a/api/containerautomation/rollback_test.go b/api/containerautomation/rollback_test.go index 37432ba8f..257d9adc4 100644 --- a/api/containerautomation/rollback_test.go +++ b/api/containerautomation/rollback_test.go @@ -183,6 +183,28 @@ func TestIsTagReference(t *testing.T) { } } +func TestSkipUnnamedForRollback(t *testing.T) { + tests := []struct { + name string + rollback bool + cName string + want bool + }{ + {name: "rollback on, unnamed -> skip (unsuppressable loop otherwise)", rollback: true, cName: "", want: true}, + {name: "rollback on, named -> proceed (guard can key it)", rollback: true, cName: "web", want: false}, + {name: "rollback off, unnamed -> proceed (no rollback to loop)", rollback: false, cName: "", want: false}, + {name: "rollback off, named -> proceed", rollback: false, cName: "web", want: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := skipUnnamedForRollback(tt.rollback, tt.cName); got != tt.want { + t.Errorf("skipUnnamedForRollback(%v, %q) = %v, want %v", tt.rollback, tt.cName, got, tt.want) + } + }) + } +} + func TestHasHealthGate(t *testing.T) { tests := []struct { name string diff --git a/api/containerautomation/service.go b/api/containerautomation/service.go index a7012ff92..4139256f7 100644 --- a/api/containerautomation/service.go +++ b/api/containerautomation/service.go @@ -27,7 +27,8 @@ const ( // defaultCheckInterval is used when the configured auto-heal interval is empty or unparseable. defaultCheckInterval = 30 * time.Second // defaultPollInterval is used when the configured auto-update interval is empty or unparseable. - // It is conservative (hours) to stay within registry rate limits and rely on the 24h status cache. + // It is conservative (hours) to stay within registry rate limits; the image-status cache is + // short-lived (keyed by the local imageID), so each poll re-checks the remote digest. defaultPollInterval = 6 * time.Hour ) @@ -70,6 +71,13 @@ type Service struct { // by endpoint+name, so the auto-update job does not immediately re-pull the // same failed image and roll back again on the next tick (the update->rollback // loop guard, mirroring the auto-heal retries map). + // + // This state is in-memory only and is NOT persisted: after a Portainer restart + // the map is empty, so at most one extra update->rollback cycle per restart is + // possible before the guard re-records the failed target. Persisting it would + // require a datastore schema (key + digest + timestamp) and is intentionally out + // of scope here; the cooldown-bounded single extra cycle is an acceptable + // trade-off against that complexity. rolledBack map[string]rolledBackTarget } @@ -125,6 +133,13 @@ func (s *Service) Start() { // Reload re-applies the current settings: it stops the running jobs and starts // fresh ones with the new intervals, or leaves them stopped if disabled. It is // safe to call after a settings update. +// +// Note: stopping a job unschedules future ticks but does not interrupt a tick +// already in progress. An in-flight heal/update pass runs to completion on its +// original (pre-reload) context and is only cancelled by a server shutdown (via +// baseCtx); the new interval takes effect from the next scheduled tick. The +// overlap guards (running/updateRunning) and the per-map mutexes keep this safe +// against data races, so this is a deliberate behavioural nuance, not a bug. func (s *Service) Reload() error { s.mu.Lock() defer s.mu.Unlock() diff --git a/api/docker/images/status.go b/api/docker/images/status.go index bb7439115..fe163497f 100644 --- a/api/docker/images/status.go +++ b/api/docker/images/status.go @@ -30,12 +30,22 @@ const ( ) const ( + // statusCacheTTL bounds how long a computed image status is served from the + // statusCache. It is intentionally short (tied to the auto-update poll window), + // NOT the previous 24h: the cache key is the LOCAL imageID, which does not + // change when upstream pushes a new image under the same tag. A long TTL would + // therefore keep serving a stale "updated" status for up to a day, and the + // auto-update daemon (which resolves status through this same path) could not + // see a freshly-pushed image within its poll interval. A few minutes still + // absorbs bursts of badge lookups for the same image while re-checking the + // remote digest soon after an upstream push. + statusCacheTTL = 5 * time.Minute errorStatusCacheTTL = 5 * time.Minute maxConcurrentStatusChecks = 8 ) var ( - statusCache = cache.New(24*time.Hour, 24*time.Hour) + statusCache = cache.New(statusCacheTTL, statusCacheTTL) remoteDigestCache = cache.New(5*time.Second, 5*time.Second) swarmID2NameCache = cache.New(5*time.Second, 5*time.Second) ) @@ -134,14 +144,16 @@ func (c *DigestClient) ContainerImageStatus(ctx context.Context, containerID str return Skipped, nil } - // statusCache is the 24h cache keyed by imageID. Reading it here makes the - // long-lived cache effective on the input side for every caller (handler, - // ContainersImageStatus, the M4 auto-update job): a hit skips the expensive, - // rate-limited remote registry digest lookup below. The container/image - // inspects above are local Docker calls and cheap; the registry HEAD is the - // part worth avoiding. Only successful statuses are ever written (the error - // paths return early without caching), so a hit returns the same value the - // full computation would have produced. + // statusCache is keyed by the LOCAL imageID and read here so every caller + // (handler, ContainersImageStatus, the auto-update job) can skip the expensive, + // rate-limited remote registry digest lookup below on a hit; the container/image + // inspects above are cheap local Docker calls, the registry HEAD is the part + // worth avoiding. The entry TTL is deliberately short (statusCacheTTL): because + // the key is the local imageID, a new upstream image pushed under the same tag + // leaves the key unchanged, so a long TTL would keep serving a stale "updated" + // status (the full computation would now return "outdated") until it expired. A + // short TTL re-checks the remote digest within the poll window. Both Outdated + // and Skipped are cached too (only the error paths return early without caching). if s, err := CachedResourceImageStatus(imageID); err == nil { return s, nil } diff --git a/api/docker/images/status_test.go b/api/docker/images/status_test.go index e7135c2a0..f09bce23e 100644 --- a/api/docker/images/status_test.go +++ b/api/docker/images/status_test.go @@ -2,10 +2,31 @@ package images import ( "testing" + "time" "github.com/stretchr/testify/require" ) +// TestStatusCacheTTLIsShort is a regression guard for the stale-detection bug: the +// statusCache is keyed by the LOCAL imageID, which does not change when upstream +// pushes a new image under the same tag. A long TTL (the previous 24h) would serve +// a stale "updated" status and hide a freshly-pushed image from both the badge and +// the auto-update daemon for up to a day. The TTL must stay tied to the poll window +// (a few minutes), and entries set with the default expiration (0) must actually +// expire rather than live forever. +func TestStatusCacheTTLIsShort(t *testing.T) { + require.LessOrEqual(t, statusCacheTTL, 10*time.Minute, "status cache TTL must be short, not 24h") + + key := "status-test-ttl-key" + CacheResourceImageStatus(key, Updated) + defer EvictImageStatus(key) + + _, exp, ok := statusCache.GetWithExpiration(key) + require.True(t, ok) + require.False(t, exp.IsZero(), "status entries must expire, not live forever") + require.LessOrEqual(t, time.Until(exp), statusCacheTTL) +} + func TestAggregateImageStatus(t *testing.T) { t.Parallel() diff --git a/api/http/handler/docker/containers/image_status.go b/api/http/handler/docker/containers/image_status.go index 78303f730..562548e18 100644 --- a/api/http/handler/docker/containers/image_status.go +++ b/api/http/handler/docker/containers/image_status.go @@ -62,8 +62,10 @@ func (handler *Handler) imageStatus(w http.ResponseWriter, r *http.Request) *htt } // The detection engine (zlib/CE) routes outbound registry calls through the - // RegistryClient, which honors the encrypted credential store and the outbound - // SSRF/AllowList. It caches results for 24h and skips digest-pinned/local-only images. + // RegistryClient, which honors the encrypted credential store. It caches results + // briefly and skips digest-pinned/local-only images. Note: the outbound registry + // HEAD (RemoteDigest -> docker.GetDigest) is NOT run through an SSRF/AllowList + // filter; this mirrors upstream ContainersImageStatus behaviour. digestClient := images.NewClientWithRegistry(images.NewRegistryClient(handler.dataStore), handler.dockerClientFactory) status, err := digestClient.ContainerImageStatus(r.Context(), containerID, endpoint, nodeName) diff --git a/api/http/handler/settings/settings_update.go b/api/http/handler/settings/settings_update.go index 693e9837a..2ef722302 100644 --- a/api/http/handler/settings/settings_update.go +++ b/api/http/handler/settings/settings_update.go @@ -22,9 +22,16 @@ import ( "golang.org/x/oauth2" ) +// minAutoHealCheckInterval is the lower bound for the auto-heal check interval. +// A near-zero interval (e.g. 1ms) is valid as a positive duration but would hammer +// Docker with list/inspect calls for no benefit; keep a sane floor, mirroring the +// auto-update poll-interval validation. +const minAutoHealCheckInterval = time.Second + // minAutoUpdatePollInterval is the lower bound for the auto-update poll interval. // Polling more often than this hammers registries (rate limits) for no benefit: -// the image-status cache is long-lived, so a sub-minute interval only adds load. +// the image-status cache (~5m) bounds detection latency, so a sub-minute interval +// only adds registry load without resolving new images any faster. const minAutoUpdatePollInterval = time.Minute // minAutoUpdateRollbackTimeout is the lower bound for the health-gate rollback @@ -142,8 +149,8 @@ func (payload *settingsUpdatePayload) Validate(r *http.Request) error { if payload.ContainerAutomation != nil && payload.ContainerAutomation.AutoHeal != nil { autoHeal := payload.ContainerAutomation.AutoHeal if autoHeal.CheckInterval != nil { - if d, err := time.ParseDuration(*autoHeal.CheckInterval); err != nil || d <= 0 { - return errors.New("Invalid auto-heal check interval. Must be a positive duration (e.g. 30s)") + if d, err := time.ParseDuration(*autoHeal.CheckInterval); err != nil || d < minAutoHealCheckInterval { + return errors.New("Invalid auto-heal check interval. Must be a duration of at least 1s (e.g. 30s)") } } diff --git a/api/http/handler/settings/settings_update_test.go b/api/http/handler/settings/settings_update_test.go index 520ff3da0..e0aa8d8e6 100644 --- a/api/http/handler/settings/settings_update_test.go +++ b/api/http/handler/settings/settings_update_test.go @@ -46,6 +46,46 @@ func TestSettingsUpdatePayloadValidateAutoUpdatePollInterval(t *testing.T) { } } +// TestSettingsUpdatePayloadValidateAutoHealCheckInterval covers the auto-heal +// check-interval floor (F6): durations below minAutoHealCheckInterval (1s), as +// well as malformed or non-positive durations, must be rejected, mirroring the +// auto-update poll-interval validation. +func TestSettingsUpdatePayloadValidateAutoHealCheckInterval(t *testing.T) { + cases := []struct { + name string + interval string + wantErr bool + }{ + {name: "one millisecond is below the floor", interval: "1ms", wantErr: true}, + {name: "half a second is below the floor", interval: "500ms", wantErr: true}, + {name: "exactly one second is allowed", interval: "1s", wantErr: false}, + {name: "thirty seconds is allowed", interval: "30s", wantErr: false}, + {name: "zero is rejected", interval: "0s", wantErr: true}, + {name: "negative is rejected", interval: "-5s", wantErr: true}, + {name: "unparseable is rejected", interval: "soon", wantErr: true}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + payload := settingsUpdatePayload{ + ContainerAutomation: &containerAutomationSettingsPayload{ + AutoHeal: &autoHealSettingsPayload{ + CheckInterval: strptr(tc.interval), + }, + }, + } + + err := payload.Validate(httptest.NewRequest("PUT", "/settings", nil)) + if tc.wantErr && err == nil { + t.Errorf("Validate(%q) = nil, want error", tc.interval) + } + if !tc.wantErr && err != nil { + t.Errorf("Validate(%q) = %v, want nil", tc.interval, err) + } + }) + } +} + // TestSettingsUpdatePayloadValidateRollbackTimeout covers the M5 health-gated // rollback timeout and its floor (F7): it must be a Go duration of at least // minAutoUpdateRollbackTimeout (10s), rejecting near-zero, non-positive and diff --git a/app/react/docker/containers/ListView/ContainersDatatable/columns/updateAvailable.tsx b/app/react/docker/containers/ListView/ContainersDatatable/columns/updateAvailable.tsx index 031a6864c..76175f235 100644 --- a/app/react/docker/containers/ListView/ContainersDatatable/columns/updateAvailable.tsx +++ b/app/react/docker/containers/ListView/ContainersDatatable/columns/updateAvailable.tsx @@ -19,7 +19,7 @@ function UpdateAvailableCell({ }: CellContext) { const environmentId = useEnvironmentId(); // One detection request per visible row is acceptable: the backend caches results - // for 24h and the hook keeps a generous client-side staleTime, so re-renders and + // for ~5m and the hook keeps a generous client-side staleTime, so re-renders and // pagination don't re-hit the registry. The query is non-blocking, so the table // renders immediately and badges fill in as statuses resolve. const statusQuery = useContainerImageStatus( diff --git a/app/react/docker/containers/queries/useContainerImageStatus.ts b/app/react/docker/containers/queries/useContainerImageStatus.ts index 5515aa4ca..d81b27d0a 100644 --- a/app/react/docker/containers/queries/useContainerImageStatus.ts +++ b/app/react/docker/containers/queries/useContainerImageStatus.ts @@ -30,9 +30,11 @@ export interface ContainerImageStatus { Message?: string; } -// The backend caches detection results for 24h, so a generous client-side staleTime -// is enough and avoids hammering the endpoint when many rows are visible at once. -const STALE_TIME = 5 * 60 * 1000; // 5 minutes +// Client-side staleTime for image-status badges: long enough to avoid hammering +// the endpoint when many rows are visible at once, short enough that a freshly +// pushed upstream image surfaces reasonably soon. Exported as the single source of +// truth so the bulk-update action reuses the same window instead of redefining it. +export const STALE_TIME = 5 * 60 * 1000; // 5 minutes export async function getContainerImageStatus( environmentId: EnvironmentId, diff --git a/app/react/docker/containers/update/useBulkUpdateContainerImages.ts b/app/react/docker/containers/update/useBulkUpdateContainerImages.ts index 5f5f20205..7861a11c6 100644 --- a/app/react/docker/containers/update/useBulkUpdateContainerImages.ts +++ b/app/react/docker/containers/update/useBulkUpdateContainerImages.ts @@ -7,7 +7,10 @@ import { notifyWarning, } from '@/portainer/services/notifications'; -import { getContainerImageStatus } from '../queries/useContainerImageStatus'; +import { + getContainerImageStatus, + STALE_TIME, +} from '../queries/useContainerImageStatus'; import { queryKeys as containerQueryKeys } from '../queries/query-keys'; import { applyContainerUpdate } from './applyContainerUpdate'; @@ -15,10 +18,6 @@ import { groupContainersForUpdate } from './groupContainersForUpdate'; import { invalidateContainerUpdateQueries } from './useUpdateContainerImage'; import { ContainerUpdateContext } from './types'; -// Mirror useContainerImageStatus's client-side staleTime so the bulk action -// reuses cached badge statuses instead of re-hitting the registry per row. -const STATUS_STALE_TIME = 5 * 60 * 1000; - interface BulkUpdateParams { contexts: ContainerUpdateContext[]; stacks: Stack[]; @@ -66,7 +65,7 @@ async function bulkUpdate( context.id, context.nodeName ), - { staleTime: STATUS_STALE_TIME } + { staleTime: STALE_TIME } ) .then((status) => status.Status) .catch(() => 'error' as const) diff --git a/app/react/docker/containers/update/useUpdateContainerImage.ts b/app/react/docker/containers/update/useUpdateContainerImage.ts index 45de5572b..61def7981 100644 --- a/app/react/docker/containers/update/useUpdateContainerImage.ts +++ b/app/react/docker/containers/update/useUpdateContainerImage.ts @@ -13,6 +13,13 @@ import { ContainerUpdateContext } from './types'; * Refresh the data affected by a container image update: the container itself, * its image-status badge (so it flips away from "outdated") and the stacks * list (a stack redeploy bumps its deployment info). + * + * Note: for a stack redeploy this invalidates only the representative container's + * badge, not those of its siblings in the same stack — a stack redeploy updates + * every container, but only `context` is passed here. The sibling badges refresh + * on their next natural refetch (staleTime / window focus) or a manual reload. + * They are deliberately not force-invalidated from this shared helper (also used + * by the single standalone "Update now") to avoid an endpoint-wide badge refetch. */ export function invalidateContainerUpdateQueries( queryClient: QueryClient, diff --git a/app/react/portainer/settings/SettingsView/AutoHealPanel/validation.test.ts b/app/react/portainer/settings/SettingsView/AutoHealPanel/validation.test.ts new file mode 100644 index 000000000..34e8ba164 --- /dev/null +++ b/app/react/portainer/settings/SettingsView/AutoHealPanel/validation.test.ts @@ -0,0 +1,47 @@ +import { validation } from './validation'; +import { Values } from './types'; + +function validate(values: Values) { + return validation() + .validate(values, { abortEarly: false }) + .then(() => undefined) + .catch((err: { errors: string[] }) => err.errors); +} + +const base: Values = { + enabled: true, + checkInterval: '30s', + scope: 'labeled', +}; + +describe('AutoHealPanel validation', () => { + it('accepts a check interval at or above the 1s floor', async () => { + expect(await validate({ ...base, checkInterval: '1s' })).toBeUndefined(); + expect(await validate({ ...base, checkInterval: '30s' })).toBeUndefined(); + expect(await validate({ ...base, checkInterval: '2m' })).toBeUndefined(); + }); + + it('rejects a check interval below the 1s floor (backend 400s on these)', async () => { + expect(await validate({ ...base, checkInterval: '1ms' })).toContain( + 'Check interval must be at least 1s' + ); + expect(await validate({ ...base, checkInterval: '500ms' })).toContain( + 'Check interval must be at least 1s' + ); + expect(await validate({ ...base, checkInterval: '0s' })).toContain( + 'Check interval must be at least 1s' + ); + }); + + it('rejects a malformed duration', async () => { + expect(await validate({ ...base, checkInterval: 'soon' })).toContain( + 'Must be a valid duration (e.g. 30s, 1m, 2h)' + ); + }); + + it('rejects an empty check interval', async () => { + expect(await validate({ ...base, checkInterval: '' })).toContain( + 'Check interval is required' + ); + }); +}); diff --git a/app/react/portainer/settings/SettingsView/AutoHealPanel/validation.ts b/app/react/portainer/settings/SettingsView/AutoHealPanel/validation.ts index 7f95acccb..c0059ae10 100644 --- a/app/react/portainer/settings/SettingsView/AutoHealPanel/validation.ts +++ b/app/react/portainer/settings/SettingsView/AutoHealPanel/validation.ts @@ -7,14 +7,63 @@ import { Values } from './types'; // Matches Go's time.ParseDuration units (e.g. "30s", "1m30s", "2h"). const durationPattern = /^(\d+(\.\d+)?(ns|us|µs|ms|s|m|h))+$/; +// Lower bound for the check interval, kept in sync with the backend +// (minAutoHealCheckInterval). A near-zero interval would hammer Docker with +// list/inspect calls for no benefit, and the backend rejects it (400), so the +// front-end must reject it too rather than letting a sub-second value through. +const minCheckIntervalSeconds = 1; + +// Seconds per Go duration unit, used to evaluate the configured interval against +// the floor. Mirrors the units accepted by durationPattern. +const unitSeconds: Record = { + ns: 1e-9, + us: 1e-6, + µs: 1e-6, + ms: 1e-3, + s: 1, + m: 60, + h: 3600, +}; + +// parseGoDurationSeconds converts a Go-style duration (e.g. "1h30m") to seconds, +// or returns null when the string is not a well-formed duration. +function parseGoDurationSeconds(value: string): number | null { + if (!durationPattern.test(value)) { + return null; + } + + let total = 0; + const componentPattern = /(\d+(?:\.\d+)?)(ns|us|µs|ms|s|m|h)/g; + let match = componentPattern.exec(value); + while (match !== null) { + total += parseFloat(match[1]) * unitSeconds[match[2]]; + match = componentPattern.exec(value); + } + + return total; +} + export function validation(): SchemaOf { return object({ enabled: boolean().default(false), checkInterval: string() .required('Check interval is required') - .matches( - durationPattern, - 'Must be a valid duration (e.g. 30s, 1m, 2h)' + .matches(durationPattern, 'Must be a valid duration (e.g. 30s, 1m, 2h)') + .test( + 'min-check-interval', + 'Check interval must be at least 1s', + (value) => { + if (!value) { + return true; // let required/matches report the error first + } + + const seconds = parseGoDurationSeconds(value); + if (seconds === null) { + return true; // let matches report the format error first + } + + return seconds >= minCheckIntervalSeconds; + } ), scope: mixed() .oneOf(['labeled', 'all']) diff --git a/app/react/portainer/settings/SettingsView/AutoUpdatePanel/validation.ts b/app/react/portainer/settings/SettingsView/AutoUpdatePanel/validation.ts index 8a7fa1820..5870b7f9c 100644 --- a/app/react/portainer/settings/SettingsView/AutoUpdatePanel/validation.ts +++ b/app/react/portainer/settings/SettingsView/AutoUpdatePanel/validation.ts @@ -9,7 +9,8 @@ const durationPattern = /^(\d+(\.\d+)?(ns|us|µs|ms|s|m|h))+$/; // Lower bound for the poll interval, kept in sync with the backend // (minAutoUpdatePollInterval). Polling more often than this only adds registry -// load: the image-status cache is long-lived, so a sub-minute interval is wasteful. +// load: the image-status cache (~5m) bounds detection latency, so a sub-minute +// interval is wasteful. const minPollIntervalSeconds = 60; // Lower bound for the rollback timeout, kept in sync with the backend