From 51957d2f9870c36ffac7918a1e73228722808ede Mon Sep 17 00:00:00 2001 From: claude code agent Date: Mon, 29 Jun 2026 08:22:46 +0300 Subject: [PATCH 1/2] feat(automation): native auto-heal daemon (#8, epic #3 M1) Add a native, CE-only auto-heal daemon that restarts Docker containers whose healthcheck reports "unhealthy", replacing the willfarrell/autoheal sidecar. Backend: - New package api/containerautomation (service lifecycle + scheduler job, per-endpoint heal pass, label/scope parsing, in-memory cooldown/retry state). - Settings.ContainerAutomation.AutoHeal {Enabled, CheckInterval, Scope} with fresh-install defaults and a 2.43.0 migration backfilling existing installs. - Settings update handler reloads/stops the job via a small Reloader interface (no import cycle); service bootstrapped from main.go after stack schedules. Frontend: - Global AutoHealPanel in SettingsView (enable / interval / scope) via useUpdateSettingsMutation, plus settings TS types. - Read-only per-container Auto-heal row in the container details view (Docker labels are immutable at runtime; opt-in is set via Create/Edit form labels). Tests: Go unit tests for label/scope resolution and the cooldown/retry decision; vitest for the panel and the per-container row. Co-Authored-By: Claude Opus 4.8 (1M context) --- api/cmd/portainer/main.go | 5 + api/containerautomation/autoheal.go | 165 ++++++++++++++++++ api/containerautomation/autoheal_test.go | 75 ++++++++ api/containerautomation/labels.go | 94 ++++++++++ api/containerautomation/labels_test.go | 80 +++++++++ api/containerautomation/service.go | 149 ++++++++++++++++ api/datastore/init.go | 4 + api/datastore/migrator/migrate_2_43_0.go | 23 +++ api/datastore/migrator/migrator.go | 1 + .../test_data/output_24_to_latest.json | 9 +- api/http/handler/settings/handler.go | 20 ++- api/http/handler/settings/settings_update.go | 42 +++++ api/http/server.go | 3 + api/portainer.go | 9 + .../AutoHealRow.test.tsx | 47 +++++ .../ContainerDetailsSection/AutoHealRow.tsx | 65 +++++++ .../ContainerDetailsSection.tsx | 3 + .../AutoHealPanel/AutoHealPanel.test.tsx | 77 ++++++++ .../AutoHealPanel/AutoHealPanel.tsx | 149 ++++++++++++++++ .../SettingsView/AutoHealPanel/index.ts | 1 + .../SettingsView/AutoHealPanel/types.ts | 7 + .../SettingsView/AutoHealPanel/validation.ts | 23 +++ .../settings/SettingsView/SettingsView.tsx | 3 + app/react/portainer/settings/types.ts | 13 ++ 24 files changed, 1060 insertions(+), 7 deletions(-) create mode 100644 api/containerautomation/autoheal.go create mode 100644 api/containerautomation/autoheal_test.go create mode 100644 api/containerautomation/labels.go create mode 100644 api/containerautomation/labels_test.go create mode 100644 api/containerautomation/service.go create mode 100644 app/react/docker/containers/ItemView/ContainerDetailsSection/AutoHealRow.test.tsx create mode 100644 app/react/docker/containers/ItemView/ContainerDetailsSection/AutoHealRow.tsx create mode 100644 app/react/portainer/settings/SettingsView/AutoHealPanel/AutoHealPanel.test.tsx create mode 100644 app/react/portainer/settings/SettingsView/AutoHealPanel/AutoHealPanel.tsx create mode 100644 app/react/portainer/settings/SettingsView/AutoHealPanel/index.ts create mode 100644 app/react/portainer/settings/SettingsView/AutoHealPanel/types.ts create mode 100644 app/react/portainer/settings/SettingsView/AutoHealPanel/validation.ts diff --git a/api/cmd/portainer/main.go b/api/cmd/portainer/main.go index d5104db6b..b77f6c636 100644 --- a/api/cmd/portainer/main.go +++ b/api/cmd/portainer/main.go @@ -14,6 +14,7 @@ import ( "github.com/portainer/portainer/api/apikey" "github.com/portainer/portainer/api/chisel" "github.com/portainer/portainer/api/cli" + "github.com/portainer/portainer/api/containerautomation" "github.com/portainer/portainer/api/crypto" "github.com/portainer/portainer/api/database" "github.com/portainer/portainer/api/database/boltdb" @@ -577,6 +578,9 @@ func buildServer(flags *portainer.CLIFlags, shutdownCtx context.Context, shutdow log.Fatal().Err(err).Msg("failed to start stack scheduler") } + containerAutomationService := containerautomation.NewService(scheduler, dataStore, dockerClientFactory) + containerAutomationService.Start() + sslDBSettings, err := dataStore.SSLSettings().Settings() if err != nil { log.Fatal().Msg("failed to fetch SSL settings from DB") @@ -650,6 +654,7 @@ func buildServer(flags *portainer.CLIFlags, shutdownCtx context.Context, shutdow DockerClientFactory: dockerClientFactory, KubernetesClientFactory: kubernetesClientFactory, Scheduler: scheduler, + ContainerAutomationService: containerAutomationService, ShutdownTrigger: shutdownTrigger, StackDeployer: stackDeployer, UpgradeService: upgradeService, diff --git a/api/containerautomation/autoheal.go b/api/containerautomation/autoheal.go new file mode 100644 index 000000000..c430161c9 --- /dev/null +++ b/api/containerautomation/autoheal.go @@ -0,0 +1,165 @@ +package containerautomation + +import ( + "context" + "time" + + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/internal/endpointutils" + + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/filters" + "github.com/rs/zerolog/log" +) + +const ( + // retryWindow is the rolling window over which max restarts per container are counted. + retryWindow = 10 * time.Minute + // restartCooldown is the minimum delay between two restarts of the same container, + // giving its healthcheck time to recover before we try again. + restartCooldown = 60 * time.Second + // endpointTimeout bounds Docker API calls for a single endpoint. + endpointTimeout = 30 * time.Second +) + +// retryState tracks restart accounting for a single container across ticks. +type retryState struct { + attempts int + windowStart time.Time + lastRestart time.Time +} + +// retryPolicy holds the cooldown/window parameters applied to a container. +type retryPolicy struct { + maxRetries int + window time.Duration + cooldown time.Duration +} + +// decideRestart is a pure function that decides whether an unhealthy container +// should be restarted now, given its current retry state and policy. It returns +// the decision and the updated state to persist. +// +// Rules, in order: +// - reset the window (and attempts) when the window has elapsed; +// - deny while still within the cooldown since the last restart; +// - deny once the max number of restarts in the current window is reached; +// - otherwise restart, incrementing the attempt counter. +func decideRestart(state retryState, policy retryPolicy, now time.Time) (bool, retryState) { + if state.windowStart.IsZero() || now.Sub(state.windowStart) >= policy.window { + state.windowStart = now + state.attempts = 0 + } + + if !state.lastRestart.IsZero() && now.Sub(state.lastRestart) < policy.cooldown { + return false, state + } + + if state.attempts >= policy.maxRetries { + return false, state + } + + state.attempts++ + state.lastRestart = now + + return true, state +} + +// heal runs a single auto-heal pass over every reachable Docker endpoint. +// It is registered with the scheduler and guarded against overlapping ticks by +// the Service. Errors are logged per endpoint/container so one failure does not +// abort the whole pass; it always returns nil so the scheduler keeps the job. +func (s *Service) heal() error { + if !s.running.CompareAndSwap(false, true) { + log.Debug().Msg("auto-heal: previous run still in progress, skipping tick") + return nil + } + defer s.running.Store(false) + + scope := s.scope() + + endpoints, err := s.dataStore.Endpoint().Endpoints() + if err != nil { + log.Warn().Err(err).Msg("auto-heal: unable to list environments") + return nil + } + + seen := make(map[string]struct{}) + + for i := range endpoints { + endpoint := &endpoints[i] + + // M1 scope: native Docker endpoints only. Kubernetes is not applicable and + // Edge/async endpoints are not reachable synchronously from the scheduler. + if !endpointutils.IsDockerEndpoint(endpoint) || endpointutils.IsEdgeEndpoint(endpoint) { + continue + } + + s.healEndpoint(endpoint, scope, seen) + } + + // Drop retry state for containers that are no longer unhealthy (healed or gone), + // resetting their counters for any future incidents. + s.pruneRetries(seen) + + return nil +} + +// healEndpoint restarts the in-scope unhealthy containers of a single endpoint. +func (s *Service) healEndpoint(endpoint *portainer.Endpoint, scope string, seen map[string]struct{}) { + endpointID := int(endpoint.ID) + + // Swarm note (M1 limitation): we connect to the endpoint's primary node only + // (nodeName ""). Containers scheduled on other Swarm nodes are not healed here; + // per-node iteration is deferred to a later milestone. + clientTimeout := endpointTimeout + cli, err := s.clientFactory.CreateClient(endpoint, "", &clientTimeout) + if err != nil { + log.Warn().Err(err).Int("endpoint_id", endpointID).Msg("auto-heal: unable to create Docker client") + return + } + defer cli.Close() + + ctx, cancel := context.WithTimeout(context.Background(), endpointTimeout) + defer cancel() + + // Filter server-side for unhealthy containers (State.Health.Status == "unhealthy"). + listFilters := filters.NewArgs(filters.Arg("health", "unhealthy")) + containers, err := cli.ContainerList(ctx, container.ListOptions{All: true, Filters: listFilters}) + if err != nil { + log.Warn().Err(err).Int("endpoint_id", endpointID).Msg("auto-heal: unable to list containers") + return + } + + for _, c := range containers { + seen[c.ID] = struct{}{} + + if !InScope(scope, c.Labels) { + continue + } + + policy := retryPolicy{ + maxRetries: MaxRetries(c.Labels), + window: retryWindow, + cooldown: restartCooldown, + } + + ok, newState := decideRestart(s.getRetry(c.ID), policy, time.Now()) + s.setRetry(c.ID, newState) + if !ok { + log.Debug().Str("container_id", c.ID).Int("endpoint_id", endpointID). + Msg("auto-heal: restart skipped (cooldown or max retries reached)") + continue + } + + timeout := StopTimeout(c.Labels) + if err := cli.ContainerRestart(ctx, c.ID, container.StopOptions{Timeout: &timeout}); err != nil { + log.Warn().Err(err).Str("container_id", c.ID).Int("endpoint_id", endpointID). + Msg("auto-heal: failed to restart unhealthy container") + continue + } + + log.Info().Str("container_id", c.ID).Int("endpoint_id", endpointID).Int("attempt", newState.attempts). + Msg("auto-heal: restarted unhealthy container") + } +} diff --git a/api/containerautomation/autoheal_test.go b/api/containerautomation/autoheal_test.go new file mode 100644 index 000000000..34176cd8a --- /dev/null +++ b/api/containerautomation/autoheal_test.go @@ -0,0 +1,75 @@ +package containerautomation + +import ( + "testing" + "time" +) + +func TestDecideRestart(t *testing.T) { + policy := retryPolicy{ + maxRetries: 3, + window: 10 * time.Minute, + cooldown: 60 * time.Second, + } + base := time.Date(2026, 6, 28, 12, 0, 0, 0, time.UTC) + + t.Run("first restart on empty state", func(t *testing.T) { + ok, state := decideRestart(retryState{}, policy, base) + if !ok { + t.Fatal("expected restart on first unhealthy observation") + } + if state.attempts != 1 { + t.Errorf("attempts = %d, want 1", state.attempts) + } + if !state.windowStart.Equal(base) || !state.lastRestart.Equal(base) { + t.Error("windowStart/lastRestart should be set to now") + } + }) + + t.Run("blocked during cooldown", func(t *testing.T) { + _, state := decideRestart(retryState{}, policy, base) + ok, _ := decideRestart(state, policy, base.Add(30*time.Second)) + if ok { + t.Error("expected restart to be blocked within cooldown") + } + }) + + t.Run("allowed after cooldown", func(t *testing.T) { + _, state := decideRestart(retryState{}, policy, base) + ok, state := decideRestart(state, policy, base.Add(61*time.Second)) + if !ok { + t.Error("expected restart allowed after cooldown") + } + if state.attempts != 2 { + t.Errorf("attempts = %d, want 2", state.attempts) + } + }) + + t.Run("max retries enforced within window", func(t *testing.T) { + state := retryState{} + now := base + allowed := 0 + for i := 0; i < 6; i++ { + ok, newState := decideRestart(state, policy, now) + state = newState + if ok { + allowed++ + } + now = now.Add(policy.cooldown + time.Second) + } + if allowed != policy.maxRetries { + t.Errorf("allowed %d restarts, want %d (max per window)", allowed, policy.maxRetries) + } + }) + + t.Run("counter resets after window elapses", func(t *testing.T) { + state := retryState{attempts: 3, windowStart: base, lastRestart: base} + ok, newState := decideRestart(state, policy, base.Add(policy.window+time.Second)) + if !ok { + t.Error("expected restart allowed once the window elapsed") + } + if newState.attempts != 1 { + t.Errorf("attempts = %d, want 1 after window reset", newState.attempts) + } + }) +} diff --git a/api/containerautomation/labels.go b/api/containerautomation/labels.go new file mode 100644 index 000000000..f7d054d49 --- /dev/null +++ b/api/containerautomation/labels.go @@ -0,0 +1,94 @@ +package containerautomation + +import "strconv" + +const ( + // Scope values for the global auto-heal setting. + ScopeLabeled = "labeled" + ScopeAll = "all" + + // Primary labels (with community aliases) controlling per-container auto-heal. + labelEnable = "io.portainer.autoheal.enable" + labelEnableAlias = "autoheal" + labelStopTimeout = "io.portainer.autoheal.stop-timeout" + labelStopTimeoutAlias = "autoheal.stop.timeout" + labelRetries = "io.portainer.autoheal.retries" + + // Defaults used when a label is missing or holds an invalid value. + defaultStopTimeout = 10 + defaultRetries = 3 +) + +// parseEnable resolves the enable label (primary first, alias second). +// It returns the parsed boolean value and whether the label was present at all. +// Invalid values are treated as false but still count as "present". +func parseEnable(labels map[string]string) (enabled bool, present bool) { + raw, ok := labels[labelEnable] + if !ok { + raw, ok = labels[labelEnableAlias] + } + + if !ok { + return false, false + } + + value, err := strconv.ParseBool(raw) + if err != nil { + return false, true + } + + return value, true +} + +// InScope reports whether a container is subject to auto-heal given the global +// scope and the container's labels. +// +// - "all": every container is in scope, unless it explicitly opts out with the +// enable label set to false. +// - "labeled" (default): only containers with the enable label set to true. +func InScope(scope string, labels map[string]string) bool { + enabled, present := parseEnable(labels) + + switch scope { + case ScopeAll: + if present && !enabled { + return false + } + + return true + default: // ScopeLabeled + return present && enabled + } +} + +// StopTimeout returns the per-container stop timeout (in seconds) from labels, +// falling back to the default when missing or invalid. +func StopTimeout(labels map[string]string) int { + return positiveIntLabel(labels, labelStopTimeout, labelStopTimeoutAlias, defaultStopTimeout) +} + +// MaxRetries returns the per-container max restarts per window from labels, +// falling back to the default when missing or invalid. +func MaxRetries(labels map[string]string) int { + return positiveIntLabel(labels, labelRetries, "", defaultRetries) +} + +// positiveIntLabel reads an integer label (primary first, optional alias second) +// and returns it when strictly positive, otherwise the provided default. +func positiveIntLabel(labels map[string]string, key, alias string, fallback int) int { + raw, ok := labels[key] + if !ok && alias != "" { + raw, ok = labels[alias] + } + + if !ok { + return fallback + } + + value, err := strconv.Atoi(raw) + if err != nil || value <= 0 { + return fallback + } + + return value +} diff --git a/api/containerautomation/labels_test.go b/api/containerautomation/labels_test.go new file mode 100644 index 000000000..cb73ff59b --- /dev/null +++ b/api/containerautomation/labels_test.go @@ -0,0 +1,80 @@ +package containerautomation + +import "testing" + +func TestInScope(t *testing.T) { + tests := []struct { + name string + scope string + labels map[string]string + want bool + }{ + {"labeled: no labels", ScopeLabeled, nil, false}, + {"labeled: enable true (primary)", ScopeLabeled, map[string]string{labelEnable: "true"}, true}, + {"labeled: enable true (alias)", ScopeLabeled, map[string]string{labelEnableAlias: "true"}, true}, + {"labeled: enable false", ScopeLabeled, map[string]string{labelEnable: "false"}, false}, + {"labeled: enable bad value", ScopeLabeled, map[string]string{labelEnable: "yepp"}, false}, + {"labeled: primary wins over alias", ScopeLabeled, map[string]string{labelEnable: "true", labelEnableAlias: "false"}, true}, + {"all: no labels", ScopeAll, nil, true}, + {"all: enable true", ScopeAll, map[string]string{labelEnable: "true"}, true}, + {"all: explicit opt-out", ScopeAll, map[string]string{labelEnable: "false"}, false}, + {"all: opt-out via alias", ScopeAll, map[string]string{labelEnableAlias: "0"}, false}, + {"all: bad value is not opt-out", ScopeAll, map[string]string{labelEnable: "nope"}, false}, + {"unknown scope falls back to labeled", "weird", map[string]string{labelEnable: "true"}, true}, + {"unknown scope, no label", "weird", nil, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := InScope(tt.scope, tt.labels); got != tt.want { + t.Errorf("InScope(%q, %v) = %v, want %v", tt.scope, tt.labels, got, tt.want) + } + }) + } +} + +func TestStopTimeout(t *testing.T) { + tests := []struct { + name string + labels map[string]string + want int + }{ + {"default when missing", nil, defaultStopTimeout}, + {"primary value", map[string]string{labelStopTimeout: "25"}, 25}, + {"alias value", map[string]string{labelStopTimeoutAlias: "15"}, 15}, + {"primary wins over alias", map[string]string{labelStopTimeout: "25", labelStopTimeoutAlias: "15"}, 25}, + {"bad value falls back", map[string]string{labelStopTimeout: "abc"}, defaultStopTimeout}, + {"zero falls back", map[string]string{labelStopTimeout: "0"}, defaultStopTimeout}, + {"negative falls back", map[string]string{labelStopTimeout: "-5"}, defaultStopTimeout}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := StopTimeout(tt.labels); got != tt.want { + t.Errorf("StopTimeout(%v) = %d, want %d", tt.labels, got, tt.want) + } + }) + } +} + +func TestMaxRetries(t *testing.T) { + tests := []struct { + name string + labels map[string]string + want int + }{ + {"default when missing", nil, defaultRetries}, + {"explicit value", map[string]string{labelRetries: "5"}, 5}, + {"bad value falls back", map[string]string{labelRetries: "lots"}, defaultRetries}, + {"zero falls back", map[string]string{labelRetries: "0"}, defaultRetries}, + {"no alias for retries", map[string]string{"autoheal.retries": "7"}, defaultRetries}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := MaxRetries(tt.labels); got != tt.want { + t.Errorf("MaxRetries(%v) = %d, want %d", tt.labels, got, tt.want) + } + }) + } +} diff --git a/api/containerautomation/service.go b/api/containerautomation/service.go new file mode 100644 index 000000000..d9a7123e7 --- /dev/null +++ b/api/containerautomation/service.go @@ -0,0 +1,149 @@ +// Package containerautomation provides native container automation that runs as +// a background scheduler job. M1 implements auto-heal: restarting Docker +// containers whose healthcheck reports "unhealthy", replacing the +// willfarrell/autoheal sidecar. +package containerautomation + +import ( + "sync" + "sync/atomic" + "time" + + "github.com/portainer/portainer/api/dataservices" + dockerclient "github.com/portainer/portainer/api/docker/client" + "github.com/portainer/portainer/api/scheduler" + + "github.com/rs/zerolog/log" +) + +// defaultCheckInterval is used when the configured interval is empty or unparseable. +const defaultCheckInterval = 30 * time.Second + +// Service manages the lifecycle of the auto-heal scheduler job and keeps the +// per-container retry state in memory across ticks. +type Service struct { + scheduler *scheduler.Scheduler + dataStore dataservices.DataStore + clientFactory *dockerclient.ClientFactory + + mu sync.Mutex + jobID string + + // running guards against overlapping heal ticks. + running atomic.Bool + + retryMu sync.Mutex + retries map[string]retryState +} + +// NewService creates a new container automation service. Call Start to schedule +// the job according to the persisted settings. +func NewService(scheduler *scheduler.Scheduler, dataStore dataservices.DataStore, clientFactory *dockerclient.ClientFactory) *Service { + return &Service{ + scheduler: scheduler, + dataStore: dataStore, + clientFactory: clientFactory, + retries: make(map[string]retryState), + } +} + +// Start schedules the auto-heal job if it is enabled in the settings. +func (s *Service) Start() { + s.mu.Lock() + defer s.mu.Unlock() + + s.start() +} + +// Reload re-applies the current settings: it stops the running job and starts a +// fresh one with the new interval, or leaves it stopped if auto-heal is now +// disabled. It is safe to call after a settings update. +func (s *Service) Reload() error { + s.mu.Lock() + defer s.mu.Unlock() + + s.stop() + s.start() + + return nil +} + +// start (re)schedules the job from settings. Caller must hold s.mu. +func (s *Service) start() { + settings, err := s.dataStore.Settings().Settings() + if err != nil { + log.Warn().Err(err).Msg("auto-heal: unable to read settings, job not scheduled") + return + } + + autoHeal := settings.ContainerAutomation.AutoHeal + if !autoHeal.Enabled { + return + } + + interval, err := time.ParseDuration(autoHeal.CheckInterval) + if err != nil || interval <= 0 { + log.Warn().Str("interval", autoHeal.CheckInterval).Dur("default", defaultCheckInterval). + Msg("auto-heal: invalid check interval, falling back to default") + interval = defaultCheckInterval + } + + s.jobID = s.scheduler.StartJobEvery(interval, s.heal) + log.Info().Dur("interval", interval).Msg("auto-heal: job scheduled") +} + +// stop cancels the running job, if any. Caller must hold s.mu. +func (s *Service) stop() { + if s.jobID == "" { + return + } + + if err := s.scheduler.StopJob(s.jobID); err != nil { + log.Warn().Err(err).Msg("auto-heal: could not stop the job") + } + + s.jobID = "" +} + +// scope returns the configured auto-heal scope, defaulting to "labeled". +func (s *Service) scope() string { + settings, err := s.dataStore.Settings().Settings() + if err != nil { + return ScopeLabeled + } + + if settings.ContainerAutomation.AutoHeal.Scope == ScopeAll { + return ScopeAll + } + + return ScopeLabeled +} + +// getRetry returns the retry state for a container (zero value if unknown). +func (s *Service) getRetry(containerID string) retryState { + s.retryMu.Lock() + defer s.retryMu.Unlock() + + return s.retries[containerID] +} + +// setRetry stores the retry state for a container. +func (s *Service) setRetry(containerID string, state retryState) { + s.retryMu.Lock() + defer s.retryMu.Unlock() + + s.retries[containerID] = state +} + +// pruneRetries drops retry state for containers not seen unhealthy in the last +// pass, so a recovered (or removed) container starts fresh next time. +func (s *Service) pruneRetries(seen map[string]struct{}) { + s.retryMu.Lock() + defer s.retryMu.Unlock() + + for id := range s.retries { + if _, ok := seen[id]; !ok { + delete(s.retries, id) + } + } +} diff --git a/api/datastore/init.go b/api/datastore/init.go index 191457b16..13a4b4a3d 100644 --- a/api/datastore/init.go +++ b/api/datastore/init.go @@ -62,6 +62,10 @@ func (store *Store) checkOrCreateDefaultSettings() error { EnforceEdgeID: true, } + defaultSettings.ContainerAutomation.AutoHeal.Enabled = false + defaultSettings.ContainerAutomation.AutoHeal.CheckInterval = "30s" + defaultSettings.ContainerAutomation.AutoHeal.Scope = "labeled" + return store.SettingsService.UpdateSettings(defaultSettings) } if err != nil { diff --git a/api/datastore/migrator/migrate_2_43_0.go b/api/datastore/migrator/migrate_2_43_0.go index f45dc722d..b538d2e4d 100644 --- a/api/datastore/migrator/migrate_2_43_0.go +++ b/api/datastore/migrator/migrate_2_43_0.go @@ -248,3 +248,26 @@ func (m *Migrator) migrateCustomTemplateGitConfigToSources_2_43_0() error { return nil } + +// migrateContainerAutomationSettings_2_43_0 backfills the native auto-heal +// defaults into existing installs so the new ContainerAutomation block is +// populated (disabled, 30s interval, "labeled" scope) without changing behavior. +func (m *Migrator) migrateContainerAutomationSettings_2_43_0() error { + log.Info().Msg("backfilling container automation (auto-heal) settings") + + settings, err := m.settingsService.Settings() + if err != nil { + return err + } + + autoHeal := &settings.ContainerAutomation.AutoHeal + if autoHeal.CheckInterval == "" { + autoHeal.CheckInterval = "30s" + } + + if autoHeal.Scope == "" { + autoHeal.Scope = "labeled" + } + + return m.settingsService.UpdateSettings(settings) +} diff --git a/api/datastore/migrator/migrator.go b/api/datastore/migrator/migrator.go index f7fd38fdc..385778809 100644 --- a/api/datastore/migrator/migrator.go +++ b/api/datastore/migrator/migrator.go @@ -275,6 +275,7 @@ func (m *Migrator) initMigrations() { m.addMigrations("2.43.0", m.migrateGitConfigToSources_2_43_0, m.migrateCustomTemplateGitConfigToSources_2_43_0, + m.migrateContainerAutomationSettings_2_43_0, ) // WARNING: do not change migrations that have already been released! diff --git a/api/datastore/test_data/output_24_to_latest.json b/api/datastore/test_data/output_24_to_latest.json index 16d6c3c06..aba1cf55a 100644 --- a/api/datastore/test_data/output_24_to_latest.json +++ b/api/datastore/test_data/output_24_to_latest.json @@ -585,6 +585,13 @@ "AllowStackManagementForRegularUsers": true, "AuthenticationMethod": 1, "BlackListedLabels": [], + "ContainerAutomation": { + "AutoHeal": { + "CheckInterval": "30s", + "Enabled": false, + "Scope": "labeled" + } + }, "Edge": { "CommandInterval": 0, "PingInterval": 0, @@ -921,7 +928,7 @@ } ], "version": { - "VERSION": "{\"SchemaVersion\":\"2.43.0\",\"MigratorCount\":2,\"Edition\":1,\"InstanceID\":\"463d5c47-0ea5-4aca-85b1-405ceefee254\"}" + "VERSION": "{\"SchemaVersion\":\"2.43.0\",\"MigratorCount\":3,\"Edition\":1,\"InstanceID\":\"463d5c47-0ea5-4aca-85b1-405ceefee254\"}" }, "webhooks": null, "workflows": null diff --git a/api/http/handler/settings/handler.go b/api/http/handler/settings/handler.go index 81a9f586c..72d8c4ac7 100644 --- a/api/http/handler/settings/handler.go +++ b/api/http/handler/settings/handler.go @@ -17,15 +17,23 @@ func hideFields(settings *portainer.Settings) { settings.OAuthSettings.KubeSecretKey = nil } +// ContainerAutomationReloader re-applies container automation settings (e.g. the +// auto-heal scheduler job) after a settings change. It is a minimal interface so +// the settings handler does not depend on the concrete service implementation. +type ContainerAutomationReloader interface { + Reload() error +} + // Handler is the HTTP handler used to handle settings operations. type Handler struct { *mux.Router - DataStore dataservices.DataStore - FileService portainer.FileService - JWTService portainer.JWTService - LDAPService portainer.LDAPService - SnapshotService portainer.SnapshotService - SetupTokenRequired bool + DataStore dataservices.DataStore + FileService portainer.FileService + JWTService portainer.JWTService + LDAPService portainer.LDAPService + SnapshotService portainer.SnapshotService + SetupTokenRequired bool + ContainerAutomationService ContainerAutomationReloader } // NewHandler creates a handler to manage settings operations. diff --git a/api/http/handler/settings/settings_update.go b/api/http/handler/settings/settings_update.go index 4a24bf93f..98f8f1ca4 100644 --- a/api/http/handler/settings/settings_update.go +++ b/api/http/handler/settings/settings_update.go @@ -18,6 +18,7 @@ import ( "github.com/portainer/portainer/pkg/validate" "github.com/pkg/errors" + "github.com/rs/zerolog/log" "golang.org/x/oauth2" ) @@ -56,6 +57,18 @@ type settingsUpdatePayload struct { EdgePortainerURL *string `json:"EdgePortainerURL"` // ForceSecureCookies forces the Secure attribute on auth cookies regardless of the detected scheme ForceSecureCookies *bool `example:"false"` + // Native container automation settings (auto-heal) + ContainerAutomation *containerAutomationSettingsPayload +} + +type containerAutomationSettingsPayload struct { + AutoHeal *autoHealSettingsPayload +} + +type autoHealSettingsPayload struct { + Enabled *bool `example:"false"` + CheckInterval *string `example:"30s"` + Scope *string `example:"labeled"` } func (payload *settingsUpdatePayload) Validate(r *http.Request) error { @@ -105,6 +118,19 @@ func (payload *settingsUpdatePayload) Validate(r *http.Request) error { } } + if payload.ContainerAutomation != nil && payload.ContainerAutomation.AutoHeal != nil { + autoHeal := payload.ContainerAutomation.AutoHeal + if autoHeal.CheckInterval != nil { + if d, err := time.ParseDuration(*autoHeal.CheckInterval); err != nil || d <= 0 { + return errors.New("Invalid auto-heal check interval. Must be a positive duration (e.g. 30s)") + } + } + + if autoHeal.Scope != nil && *autoHeal.Scope != "labeled" && *autoHeal.Scope != "all" { + return errors.New("Invalid auto-heal scope. Value must be one of: labeled, all") + } + } + return nil } @@ -138,6 +164,14 @@ func (handler *Handler) settingsUpdate(w http.ResponseWriter, r *http.Request) * return response.TxErrorResponse(err) } + // Re-apply container automation settings so the auto-heal job is rescheduled + // (or stopped) with the new interval/scope after a successful save. + if handler.ContainerAutomationService != nil { + if err := handler.ContainerAutomationService.Reload(); err != nil { + log.Warn().Err(err).Msg("unable to reload container automation settings") + } + } + hideFields(settings) return response.JSON(w, settings) } @@ -236,6 +270,14 @@ func (handler *Handler) updateSettings(tx dataservices.DataStoreTx, payload sett settings.KubectlShellImage = *cmp.Or(payload.KubectlShellImage, &settings.KubectlShellImage) + if payload.ContainerAutomation != nil && payload.ContainerAutomation.AutoHeal != nil { + autoHeal := payload.ContainerAutomation.AutoHeal + current := &settings.ContainerAutomation.AutoHeal + current.Enabled = *cmp.Or(autoHeal.Enabled, ¤t.Enabled) + current.CheckInterval = *cmp.Or(autoHeal.CheckInterval, ¤t.CheckInterval) + current.Scope = *cmp.Or(autoHeal.Scope, ¤t.Scope) + } + if err := tx.Settings().UpdateSettings(settings); err != nil { return nil, httperror.InternalServerError("Unable to persist settings changes inside the database", err) } diff --git a/api/http/server.go b/api/http/server.go index c56d1cd33..d3fa7edc0 100644 --- a/api/http/server.go +++ b/api/http/server.go @@ -11,6 +11,7 @@ import ( portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/adminmonitor" "github.com/portainer/portainer/api/apikey" + "github.com/portainer/portainer/api/containerautomation" "github.com/portainer/portainer/api/crypto" "github.com/portainer/portainer/api/dataservices" "github.com/portainer/portainer/api/docker" @@ -105,6 +106,7 @@ type Server struct { KubernetesDeployer portainer.KubernetesDeployer HelmPackageManager libhelmtypes.HelmPackageManager Scheduler *scheduler.Scheduler + ContainerAutomationService *containerautomation.Service ShutdownTrigger context.CancelFunc StackDeployer deployments.StackDeployer UpgradeService upgrade.Service @@ -238,6 +240,7 @@ func (server *Server) Start(ctx context.Context) error { settingsHandler.LDAPService = server.LDAPService settingsHandler.SnapshotService = server.SnapshotService settingsHandler.SetupTokenRequired = server.SetupToken != "" + settingsHandler.ContainerAutomationService = server.ContainerAutomationService var sslHandler = sslhandler.NewHandler(requestBouncer) sslHandler.SSLService = server.SSLService diff --git a/api/portainer.go b/api/portainer.go index 5b925e9cd..c6d7b2651 100644 --- a/api/portainer.go +++ b/api/portainer.go @@ -1211,6 +1211,15 @@ type ( // ForceSecureCookies forces the Secure attribute on auth cookies regardless of detected scheme. // Enable when Portainer runs behind a TLS-terminating proxy. ForceSecureCookies bool `json:"ForceSecureCookies" example:"false"` + + // ContainerAutomation holds native container automation settings. + ContainerAutomation struct { + AutoHeal struct { + Enabled bool `json:"Enabled"` + CheckInterval string `json:"CheckInterval" example:"30s"` + Scope string `json:"Scope" example:"labeled"` // "labeled" | "all" + } `json:"AutoHeal"` + } `json:"ContainerAutomation"` } // SnapshotJob represents a scheduled job that can create environment(endpoint) snapshots diff --git a/app/react/docker/containers/ItemView/ContainerDetailsSection/AutoHealRow.test.tsx b/app/react/docker/containers/ItemView/ContainerDetailsSection/AutoHealRow.test.tsx new file mode 100644 index 000000000..818ba66a6 --- /dev/null +++ b/app/react/docker/containers/ItemView/ContainerDetailsSection/AutoHealRow.test.tsx @@ -0,0 +1,47 @@ +import { render, screen } from '@testing-library/react'; + +import { AutoHealRow } from './AutoHealRow'; + +function renderRow(labels?: Record) { + return render( + + + + +
+ ); +} + +describe('AutoHealRow', () => { + it('shows enabled when the primary label is true', () => { + renderRow({ 'io.portainer.autoheal.enable': 'true' }); + expect(screen.getByText('Enabled')).toBeInTheDocument(); + }); + + it('shows enabled via the community alias', () => { + renderRow({ autoheal: 'true' }); + expect(screen.getByText('Enabled')).toBeInTheDocument(); + }); + + it('shows opted out when the label is false', () => { + renderRow({ 'io.portainer.autoheal.enable': 'false' }); + expect(screen.getByText('Disabled (opted out)')).toBeInTheDocument(); + }); + + it('shows the global-scope fallback when no label is set', () => { + renderRow({}); + expect( + screen.getByText('Not labeled (follows global scope)') + ).toBeInTheDocument(); + }); + + it('renders stop timeout and retries when present', () => { + renderRow({ + 'io.portainer.autoheal.enable': 'true', + 'io.portainer.autoheal.stop-timeout': '20', + 'io.portainer.autoheal.retries': '5', + }); + expect(screen.getByText('Stop timeout: 20s')).toBeInTheDocument(); + expect(screen.getByText('Max retries: 5')).toBeInTheDocument(); + }); +}); diff --git a/app/react/docker/containers/ItemView/ContainerDetailsSection/AutoHealRow.tsx b/app/react/docker/containers/ItemView/ContainerDetailsSection/AutoHealRow.tsx new file mode 100644 index 000000000..e953534cc --- /dev/null +++ b/app/react/docker/containers/ItemView/ContainerDetailsSection/AutoHealRow.tsx @@ -0,0 +1,65 @@ +import { DetailsTable } from '@@/DetailsTable'; + +interface Props { + labels?: Record; +} + +// Label keys (with community aliases) mirroring the backend +// (api/containerautomation/labels.go). +const ENABLE_LABEL = 'io.portainer.autoheal.enable'; +const ENABLE_LABEL_ALIAS = 'autoheal'; +const STOP_TIMEOUT_LABEL = 'io.portainer.autoheal.stop-timeout'; +const STOP_TIMEOUT_LABEL_ALIAS = 'autoheal.stop.timeout'; +const RETRIES_LABEL = 'io.portainer.autoheal.retries'; + +function parseBool(value?: string) { + if (value === undefined) { + return undefined; + } + return value === 'true' || value === '1'; +} + +/** + * AutoHealRow shows the per-container auto-heal opt-in state, resolved from the + * container's immutable Docker labels. It is read-only: because labels cannot be + * changed on a running container, opt-in is set through the Create/Edit form + * labels and the global behavior is controlled in Settings. + */ +export function AutoHealRow({ labels }: Props) { + const enabled = + parseBool(labels?.[ENABLE_LABEL]) ?? parseBool(labels?.[ENABLE_LABEL_ALIAS]); + + let stateLabel: string; + if (enabled === true) { + stateLabel = 'Enabled'; + } else if (enabled === false) { + stateLabel = 'Disabled (opted out)'; + } else { + stateLabel = 'Not labeled (follows global scope)'; + } + + const stopTimeout = + labels?.[STOP_TIMEOUT_LABEL] ?? labels?.[STOP_TIMEOUT_LABEL_ALIAS]; + const retries = labels?.[RETRIES_LABEL]; + + return ( + +
+
{stateLabel}
+ {stopTimeout && ( +
+ Stop timeout: {stopTimeout}s +
+ )} + {retries && ( +
Max retries: {retries}
+ )} +
+ Set via the {ENABLE_LABEL} label (immutable at runtime; + edit through the container Create/Edit form). Global behavior is + configured in Settings. +
+
+ + ); +} diff --git a/app/react/docker/containers/ItemView/ContainerDetailsSection/ContainerDetailsSection.tsx b/app/react/docker/containers/ItemView/ContainerDetailsSection/ContainerDetailsSection.tsx index 6b13966ac..7ad0824fc 100644 --- a/app/react/docker/containers/ItemView/ContainerDetailsSection/ContainerDetailsSection.tsx +++ b/app/react/docker/containers/ItemView/ContainerDetailsSection/ContainerDetailsSection.tsx @@ -10,6 +10,7 @@ import { Widget } from '@@/Widget'; import { RestartPolicy } from '../../CreateView/RestartPolicyTab/types'; import { RestartPolicySection } from '../RestartPolicySection/RestartPolicySection'; +import { AutoHealRow } from './AutoHealRow'; import { ImageRow } from './ImageRow'; import { PortConfigurationRow } from './PortConfigurationRow'; import { EnvironmentVariablesRow } from './EnvironmentVariablesRow'; @@ -80,6 +81,8 @@ export function ContainerDetailsSection({ /> + + diff --git a/app/react/portainer/settings/SettingsView/AutoHealPanel/AutoHealPanel.test.tsx b/app/react/portainer/settings/SettingsView/AutoHealPanel/AutoHealPanel.test.tsx new file mode 100644 index 000000000..efa402c01 --- /dev/null +++ b/app/react/portainer/settings/SettingsView/AutoHealPanel/AutoHealPanel.test.tsx @@ -0,0 +1,77 @@ +import { render, screen, waitFor } from '@testing-library/react'; +import { HttpResponse, http } from 'msw'; + +import { withTestRouter } from '@/react/test-utils/withRouter'; +import { withTestQueryProvider } from '@/react/test-utils/withTestQuery'; +import { withUserProvider } from '@/react/test-utils/withUserProvider'; +import { server } from '@/setup-tests/server'; + +import { AutoHealPanel } from './AutoHealPanel'; + +function renderComponent() { + const Wrapped = withTestQueryProvider( + withUserProvider(withTestRouter(AutoHealPanel)) + ); + return render(); +} + +describe('AutoHealPanel', () => { + it('renders the auto-heal settings read from the API', async () => { + server.use( + http.get('/api/settings', () => + HttpResponse.json({ + ContainerAutomation: { + AutoHeal: { + Enabled: true, + CheckInterval: '45s', + Scope: 'all', + }, + }, + }) + ) + ); + + renderComponent(); + + expect(await screen.findByText('Container auto-heal')).toBeInTheDocument(); + + await waitFor(() => { + const interval = screen.getByLabelText(/Check interval/i); + expect(interval).toHaveValue('45s'); + }); + + const scope = screen.getByLabelText(/Scope/i); + expect(scope).toHaveValue('all'); + + expect( + screen.getByRole('checkbox', { name: /Enable auto-heal/i }) + ).toBeChecked(); + }); + + it('defaults the interval and scope when missing', async () => { + server.use( + http.get('/api/settings', () => + HttpResponse.json({ + ContainerAutomation: { + AutoHeal: { + Enabled: false, + CheckInterval: '', + Scope: 'labeled', + }, + }, + }) + ) + ); + + renderComponent(); + + await waitFor(() => { + const interval = screen.getByLabelText(/Check interval/i); + expect(interval).toHaveValue('30s'); + }); + + expect( + screen.getByRole('checkbox', { name: /Enable auto-heal/i }) + ).not.toBeChecked(); + }); +}); diff --git a/app/react/portainer/settings/SettingsView/AutoHealPanel/AutoHealPanel.tsx b/app/react/portainer/settings/SettingsView/AutoHealPanel/AutoHealPanel.tsx new file mode 100644 index 000000000..bc8aef6ef --- /dev/null +++ b/app/react/portainer/settings/SettingsView/AutoHealPanel/AutoHealPanel.tsx @@ -0,0 +1,149 @@ +import { HeartPulse } from 'lucide-react'; +import { Field, Form, Formik, useFormikContext } from 'formik'; + +import { notifySuccess } from '@/portainer/services/notifications'; + +import { Widget } from '@@/Widget'; +import { LoadingButton } from '@@/buttons'; +import { FormControl } from '@@/form-components/FormControl'; +import { Input, Select } from '@@/form-components/Input'; +import { SwitchField } from '@@/form-components/SwitchField'; +import { TextTip } from '@@/Tip/TextTip'; + +import { useSettings, useUpdateSettingsMutation } from '../../queries'; +import { AutoHealScope } from '../../types'; + +import { Values } from './types'; +import { validation } from './validation'; + +const scopeOptions: Array<{ value: AutoHealScope; label: string }> = [ + { value: 'labeled', label: 'Labeled containers only' }, + { value: 'all', label: 'All containers' }, +]; + +export function AutoHealPanel() { + const settingsQuery = useSettings((settings) => settings.ContainerAutomation); + const mutation = useUpdateSettingsMutation(); + + if (!settingsQuery.data) { + return null; + } + + const autoHeal = settingsQuery.data.AutoHeal; + const initialValues: Values = { + enabled: autoHeal.Enabled, + checkInterval: autoHeal.CheckInterval || '30s', + scope: autoHeal.Scope || 'labeled', + }; + + return ( + + + +
+ + When enabled, Portainer periodically restarts containers whose Docker + healthcheck reports an unhealthy state, replacing the + willfarrell/autoheal sidecar. Per-container opt-in is controlled with + the io.portainer.autoheal.enable=true label. + +
+ + + + + + + ); + + function handleSubmit(values: Values) { + mutation.mutate( + { + ContainerAutomation: { + AutoHeal: { + Enabled: values.enabled, + CheckInterval: values.checkInterval, + Scope: values.scope, + }, + }, + }, + { + onSuccess() { + notifySuccess('Success', 'Auto-heal settings updated'); + }, + } + ); + } +} + +function InnerForm({ isLoading }: { isLoading: boolean }) { + const { values, setFieldValue, isValid, errors } = useFormikContext(); + + return ( +
+
+
+ setFieldValue('enabled', value)} + labelClass="col-sm-3 col-lg-2" + data-cy="settings-autoHealEnabled" + /> +
+
+ + + + + + + + + +
+
+ + Save auto-heal settings + +
+
+
+ ); +} diff --git a/app/react/portainer/settings/SettingsView/AutoHealPanel/index.ts b/app/react/portainer/settings/SettingsView/AutoHealPanel/index.ts new file mode 100644 index 000000000..1287291af --- /dev/null +++ b/app/react/portainer/settings/SettingsView/AutoHealPanel/index.ts @@ -0,0 +1 @@ +export { AutoHealPanel } from './AutoHealPanel'; diff --git a/app/react/portainer/settings/SettingsView/AutoHealPanel/types.ts b/app/react/portainer/settings/SettingsView/AutoHealPanel/types.ts new file mode 100644 index 000000000..d7c562107 --- /dev/null +++ b/app/react/portainer/settings/SettingsView/AutoHealPanel/types.ts @@ -0,0 +1,7 @@ +import { AutoHealScope } from '../../types'; + +export interface Values { + enabled: boolean; + checkInterval: string; + scope: AutoHealScope; +} diff --git a/app/react/portainer/settings/SettingsView/AutoHealPanel/validation.ts b/app/react/portainer/settings/SettingsView/AutoHealPanel/validation.ts new file mode 100644 index 000000000..7f95acccb --- /dev/null +++ b/app/react/portainer/settings/SettingsView/AutoHealPanel/validation.ts @@ -0,0 +1,23 @@ +import { SchemaOf, boolean, mixed, object, string } from 'yup'; + +import { AutoHealScope } from '../../types'; + +import { Values } from './types'; + +// Matches Go's time.ParseDuration units (e.g. "30s", "1m30s", "2h"). +const durationPattern = /^(\d+(\.\d+)?(ns|us|µs|ms|s|m|h))+$/; + +export function validation(): SchemaOf { + return object({ + enabled: boolean().default(false), + checkInterval: string() + .required('Check interval is required') + .matches( + durationPattern, + 'Must be a valid duration (e.g. 30s, 1m, 2h)' + ), + scope: mixed() + .oneOf(['labeled', 'all']) + .required('Scope is required'), + }); +} diff --git a/app/react/portainer/settings/SettingsView/SettingsView.tsx b/app/react/portainer/settings/SettingsView/SettingsView.tsx index 1f79d8fd1..6dc0c1179 100644 --- a/app/react/portainer/settings/SettingsView/SettingsView.tsx +++ b/app/react/portainer/settings/SettingsView/SettingsView.tsx @@ -10,6 +10,7 @@ import { Settings } from '../types'; import { isBE } from '../../feature-flags/feature-flags.service'; import { ApplicationSettingsPanel } from './ApplicationSettingsPanel'; +import { AutoHealPanel } from './AutoHealPanel'; import { BackupSettingsPanel } from './BackupSettingsView'; import { HelmCertPanel } from './HelmCertPanel'; import { HiddenContainersPanel } from './HiddenContainersPanel/HiddenContainersPanel'; @@ -54,6 +55,8 @@ export function SettingsView() { + + diff --git a/app/react/portainer/settings/types.ts b/app/react/portainer/settings/types.ts index dd882918d..8bbf7e3d7 100644 --- a/app/react/portainer/settings/types.ts +++ b/app/react/portainer/settings/types.ts @@ -127,6 +127,7 @@ export interface Settings { AllowContainerCapabilitiesForRegularUsers: boolean; ForceSecureCookies: boolean; GlobalDeploymentOptions?: GlobalDeploymentOptions; + ContainerAutomation: ContainerAutomationSettings; Edge: { PingInterval: number; SnapshotInterval: number; @@ -136,6 +137,18 @@ export interface Settings { }; } +export type AutoHealScope = 'labeled' | 'all'; + +export interface AutoHealSettings { + Enabled: boolean; + CheckInterval: string; + Scope: AutoHealScope; +} + +export interface ContainerAutomationSettings { + AutoHeal: AutoHealSettings; +} + export interface GlobalDeploymentOptions { /** Hide manual deploy forms in portainer */ hideAddWithForm: boolean; -- 2.52.0 From b233f75ab7142919c73c2a2d9ba6388c9d69ab11 Mon Sep 17 00:00:00 2001 From: claude code agent Date: Mon, 29 Jun 2026 08:39:17 +0300 Subject: [PATCH 2/2] fix(automation): retry-state retention + running-only heal + per-restart ctx (#8 F1-F6) F1: prune retry-state by elapsed window since lastRestart instead of "not seen this tick", so a container flapping through "starting" keeps its cooldown/max-retries accounting (storm guard no longer defeated). Recovered containers quiet for > window are still cleaned up. F2: list running containers only (All:false) so stopped-unhealthy containers are never revived. F3: each ContainerRestart gets its own context (stop-timeout + buffer), separate from the per-endpoint list context, so a slow/hung restart cannot starve the others or exhaust a single shared deadline. F4: start() is idempotent (no-op when a job is already scheduled); Reload still stops first so it always reschedules. F5: frontend parseBool mirrors Go strconv.ParseBool (case-insensitive 1/t/true; present-but-invalid counts as present & false). F6: tests TestPruneRetries and TestRetryStateSurvivesStartingTick lock in the F1 behavior; added AutoHealRow parse cases. Co-Authored-By: Claude Opus 4.8 (1M context) --- api/containerautomation/autoheal.go | 40 +++++++----- api/containerautomation/autoheal_test.go | 62 +++++++++++++++++++ api/containerautomation/service.go | 24 +++++-- .../AutoHealRow.test.tsx | 13 ++++ .../ContainerDetailsSection/AutoHealRow.tsx | 7 ++- 5 files changed, 125 insertions(+), 21 deletions(-) diff --git a/api/containerautomation/autoheal.go b/api/containerautomation/autoheal.go index c430161c9..b011ed6d6 100644 --- a/api/containerautomation/autoheal.go +++ b/api/containerautomation/autoheal.go @@ -18,8 +18,12 @@ const ( // restartCooldown is the minimum delay between two restarts of the same container, // giving its healthcheck time to recover before we try again. restartCooldown = 60 * time.Second - // endpointTimeout bounds Docker API calls for a single endpoint. + // endpointTimeout bounds the container-list call for a single endpoint. endpointTimeout = 30 * time.Second + // restartTimeoutBuffer is added on top of a container's stop-timeout to derive + // the deadline of its own restart context, leaving room for the engine to kill + // and start the container after the graceful stop window elapses. + restartTimeoutBuffer = 15 * time.Second ) // retryState tracks restart accounting for a single container across ticks. @@ -84,8 +88,6 @@ func (s *Service) heal() error { return nil } - seen := make(map[string]struct{}) - for i := range endpoints { endpoint := &endpoints[i] @@ -95,18 +97,20 @@ func (s *Service) heal() error { continue } - s.healEndpoint(endpoint, scope, seen) + s.healEndpoint(endpoint, scope) } - // Drop retry state for containers that are no longer unhealthy (healed or gone), - // resetting their counters for any future incidents. - s.pruneRetries(seen) + // Drop retry state only for containers whose retry window has fully elapsed + // since their last restart. A container that briefly leaves the unhealthy + // filter (e.g. while "starting" after a restart) keeps its accounting, so the + // cooldown / max-retries storm guard survives flapping. + s.pruneRetries(time.Now()) return nil } // healEndpoint restarts the in-scope unhealthy containers of a single endpoint. -func (s *Service) healEndpoint(endpoint *portainer.Endpoint, scope string, seen map[string]struct{}) { +func (s *Service) healEndpoint(endpoint *portainer.Endpoint, scope string) { endpointID := int(endpoint.ID) // Swarm note (M1 limitation): we connect to the endpoint's primary node only @@ -120,20 +124,20 @@ func (s *Service) healEndpoint(endpoint *portainer.Endpoint, scope string, seen } defer cli.Close() - ctx, cancel := context.WithTimeout(context.Background(), endpointTimeout) + listCtx, cancel := context.WithTimeout(context.Background(), endpointTimeout) defer cancel() - // Filter server-side for unhealthy containers (State.Health.Status == "unhealthy"). + // List running unhealthy containers only (All:false). Docker keeps + // Health.Status=="unhealthy" on stopped containers, so listing with All:true + // would let us "restart" (i.e. start) an intentionally-stopped container. listFilters := filters.NewArgs(filters.Arg("health", "unhealthy")) - containers, err := cli.ContainerList(ctx, container.ListOptions{All: true, Filters: listFilters}) + containers, err := cli.ContainerList(listCtx, container.ListOptions{All: false, Filters: listFilters}) if err != nil { log.Warn().Err(err).Int("endpoint_id", endpointID).Msg("auto-heal: unable to list containers") return } for _, c := range containers { - seen[c.ID] = struct{}{} - if !InScope(scope, c.Labels) { continue } @@ -153,7 +157,15 @@ func (s *Service) healEndpoint(endpoint *portainer.Endpoint, scope string, seen } timeout := StopTimeout(c.Labels) - if err := cli.ContainerRestart(ctx, c.ID, container.StopOptions{Timeout: &timeout}); err != nil { + + // Each restart gets its own context, bounded by the container's stop-timeout + // plus a buffer, so one slow restart cannot starve the others and a hung + // engine call is bounded independently of the list deadline. + restartTimeout := time.Duration(timeout)*time.Second + restartTimeoutBuffer + restartCtx, restartCancel := context.WithTimeout(context.Background(), restartTimeout) + err := cli.ContainerRestart(restartCtx, c.ID, container.StopOptions{Timeout: &timeout}) + restartCancel() + if err != nil { log.Warn().Err(err).Str("container_id", c.ID).Int("endpoint_id", endpointID). Msg("auto-heal: failed to restart unhealthy container") continue diff --git a/api/containerautomation/autoheal_test.go b/api/containerautomation/autoheal_test.go index 34176cd8a..83d8cf838 100644 --- a/api/containerautomation/autoheal_test.go +++ b/api/containerautomation/autoheal_test.go @@ -73,3 +73,65 @@ func TestDecideRestart(t *testing.T) { } }) } + +func TestPruneRetries(t *testing.T) { + now := time.Date(2026, 6, 28, 12, 0, 0, 0, time.UTC) + s := &Service{retries: map[string]retryState{ + // within the window -> retained + "fresh": {attempts: 1, windowStart: now.Add(-time.Minute), lastRestart: now.Add(-time.Minute)}, + // exactly at the window boundary -> pruned + "edge": {attempts: 2, windowStart: now.Add(-retryWindow), lastRestart: now.Add(-retryWindow)}, + // long past the window -> pruned + "stale": {attempts: 3, windowStart: now.Add(-2 * retryWindow), lastRestart: now.Add(-2 * retryWindow)}, + }} + + s.pruneRetries(now) + + if _, ok := s.retries["fresh"]; !ok { + t.Error("entry within the retry window should be retained") + } + if _, ok := s.retries["edge"]; ok { + t.Error("entry exactly at the window boundary should be pruned") + } + if _, ok := s.retries["stale"]; ok { + t.Error("entry past the retry window should be pruned") + } +} + +// TestRetryStateSurvivesStartingTick locks in the F1 fix: a container that flaps +// through "starting" right after a restart (and so briefly drops out of the +// health=unhealthy filter) must keep its retry accounting across the tick where +// it is not observed, otherwise the cooldown / max-retries storm guard is +// defeated and the next unhealthy observation triggers an immediate restart. +func TestRetryStateSurvivesStartingTick(t *testing.T) { + policy := retryPolicy{maxRetries: 3, window: retryWindow, cooldown: restartCooldown} + const id = "flapper" + s := &Service{retries: make(map[string]retryState)} + + t0 := time.Date(2026, 6, 28, 12, 0, 0, 0, time.UTC) + + // Tick 1: container is unhealthy -> first restart. + ok, state := decideRestart(s.getRetry(id), policy, t0) + s.setRetry(id, state) + if !ok || state.attempts != 1 { + t.Fatalf("tick 1: ok=%v attempts=%d, want restart with attempts=1", ok, state.attempts) + } + + // Tick 2 (t0+30s): the container is "starting" and not in the unhealthy list. + // Prune must NOT drop its state because the window has not elapsed. + s.pruneRetries(t0.Add(30 * time.Second)) + if _, kept := s.retries[id]; !kept { + t.Fatal("tick 2: retry state was pruned while the container was 'starting'") + } + + // Tick 3 (t0+45s): unhealthy again, still within the cooldown. The surviving + // state must block the restart and the attempt count must not be reset. + ok, state = decideRestart(s.getRetry(id), policy, t0.Add(45*time.Second)) + s.setRetry(id, state) + if ok { + t.Error("tick 3: restart should be blocked by the surviving cooldown") + } + if state.attempts != 1 { + t.Errorf("tick 3: attempts = %d, want 1 (state survived, not reset)", state.attempts) + } +} diff --git a/api/containerautomation/service.go b/api/containerautomation/service.go index d9a7123e7..8dc9853dd 100644 --- a/api/containerautomation/service.go +++ b/api/containerautomation/service.go @@ -68,8 +68,15 @@ func (s *Service) Reload() error { return nil } -// start (re)schedules the job from settings. Caller must hold s.mu. +// start (re)schedules the job from settings. Caller must hold s.mu. It is a +// no-op when a job is already scheduled, so calling Start more than once does +// not leak an orphaned job; Reload first calls stop (clearing jobID) and so +// always reschedules. func (s *Service) start() { + if s.jobID != "" { + return + } + settings, err := s.dataStore.Settings().Settings() if err != nil { log.Warn().Err(err).Msg("auto-heal: unable to read settings, job not scheduled") @@ -135,14 +142,19 @@ func (s *Service) setRetry(containerID string, state retryState) { s.retries[containerID] = state } -// pruneRetries drops retry state for containers not seen unhealthy in the last -// pass, so a recovered (or removed) container starts fresh next time. -func (s *Service) pruneRetries(seen map[string]struct{}) { +// pruneRetries drops retry state for containers whose retry window has fully +// elapsed since their last restart. A container is kept regardless of whether it +// appeared in the current tick: one that briefly leaves the unhealthy filter +// (e.g. while "starting" right after a restart) must not lose its accounting, or +// the cooldown / max-retries storm guard would be defeated. A container that has +// recovered and stayed quiet for longer than the window is cleaned up (fresh +// budget next incident, no unbounded growth). +func (s *Service) pruneRetries(now time.Time) { s.retryMu.Lock() defer s.retryMu.Unlock() - for id := range s.retries { - if _, ok := seen[id]; !ok { + for id, state := range s.retries { + if now.Sub(state.lastRestart) >= retryWindow { delete(s.retries, id) } } diff --git a/app/react/docker/containers/ItemView/ContainerDetailsSection/AutoHealRow.test.tsx b/app/react/docker/containers/ItemView/ContainerDetailsSection/AutoHealRow.test.tsx index 818ba66a6..3bdfb59ad 100644 --- a/app/react/docker/containers/ItemView/ContainerDetailsSection/AutoHealRow.test.tsx +++ b/app/react/docker/containers/ItemView/ContainerDetailsSection/AutoHealRow.test.tsx @@ -23,6 +23,19 @@ describe('AutoHealRow', () => { expect(screen.getByText('Enabled')).toBeInTheDocument(); }); + it.each(['TRUE', 'True', 'T', 't', '1'])( + 'parses the truthy value %s like strconv.ParseBool', + (value) => { + renderRow({ 'io.portainer.autoheal.enable': value }); + expect(screen.getByText('Enabled')).toBeInTheDocument(); + } + ); + + it('treats a present-but-invalid value as opted out', () => { + renderRow({ 'io.portainer.autoheal.enable': 'yepp' }); + expect(screen.getByText('Disabled (opted out)')).toBeInTheDocument(); + }); + it('shows opted out when the label is false', () => { renderRow({ 'io.portainer.autoheal.enable': 'false' }); expect(screen.getByText('Disabled (opted out)')).toBeInTheDocument(); diff --git a/app/react/docker/containers/ItemView/ContainerDetailsSection/AutoHealRow.tsx b/app/react/docker/containers/ItemView/ContainerDetailsSection/AutoHealRow.tsx index e953534cc..063f6e6f9 100644 --- a/app/react/docker/containers/ItemView/ContainerDetailsSection/AutoHealRow.tsx +++ b/app/react/docker/containers/ItemView/ContainerDetailsSection/AutoHealRow.tsx @@ -16,7 +16,12 @@ function parseBool(value?: string) { if (value === undefined) { return undefined; } - return value === 'true' || value === '1'; + // Mirror Go's strconv.ParseBool (used by the backend label parser): accept + // 1/t/true case-insensitively as truthy. Any other present-but-invalid value + // (including 0/f/false and garbage) counts as present & false, matching how + // the backend treats an unparseable enable label. + const normalized = value.toLowerCase(); + return normalized === '1' || normalized === 't' || normalized === 'true'; } /** -- 2.52.0