diff --git a/pkg/libstack/compose/composeplugin.go b/pkg/libstack/compose/composeplugin.go index efd3051b8..cbba48c5e 100644 --- a/pkg/libstack/compose/composeplugin.go +++ b/pkg/libstack/compose/composeplugin.go @@ -75,6 +75,11 @@ func withCli( cli.ConfigFile().AuthConfigs[r.ServerAddress] = r } + if cli.ConfigFile().CredentialsStore == "portainer" { + log.Debug().Msg("completely disabling portainer credential store helper") + cli.ConfigFile().CredentialsStore = "" + } + return cliFn(ctx, cli) } diff --git a/pkg/retry/retry.go b/pkg/retry/retry.go new file mode 100644 index 000000000..1ebba752f --- /dev/null +++ b/pkg/retry/retry.go @@ -0,0 +1,85 @@ +package aws + +import ( + "math/rand/v2" + "time" + + "github.com/rs/zerolog/log" +) + +type Settings struct { + MaxRetries int + RetryTimeFunc func(failures int) time.Duration +} + +var Default = Settings{ + MaxRetries: 3, + RetryTimeFunc: RetryTime, +} + +// sleepFunc is the function used for sleeping between retries +// It can be overridden in tests to avoid actual sleep delays +// TODO: in go 1.25 version replace test with synctime +var sleepFunc = time.Sleep + +// RetryWithWarnings executes a function with retries and backoff, logging warnings on failures +func RetryWithWarnings[T any](operation string, settings Settings, fn func() (T, error)) (T, error) { + timeStart := time.Now() + + var result T + var err error + + for attempt := 1; attempt <= settings.MaxRetries; attempt++ { + result, err = fn() + if err == nil { + if attempt > 1 { + log.Info(). + Str("operation", operation). + Int("attempt", attempt). + Str("duration", time.Since(timeStart).String()). + Msg("operation succeeded after retry") + } + return result, nil + } + + if attempt < settings.MaxRetries { + retryDelay := settings.RetryTimeFunc(attempt) + log.Warn(). + Err(err). + Str("operation", operation). + Int("attempt", attempt). + Int("max_retries", settings.MaxRetries). + Str("retry_delay", retryDelay.String()). // the string version of the duration reads better in logs + Str("duration", time.Since(timeStart).String()). + Msg("operation failed, retrying") + sleepFunc(retryDelay) + } + } + + log.Error(). + Err(err). + Str("operation", operation). + Int("attempts", settings.MaxRetries). + Str("duration", time.Since(timeStart).String()). + Msg("operation failed after all retries") + + return result, err +} + +func RetryTime(failures int) time.Duration { + var baseTime time.Duration + + switch failures { + case 0, 1: + baseTime = 5 * time.Second + case 2: + baseTime = 30 * time.Second + default: + baseTime = 90 * time.Second + } + + // plus random jitter + jitter := time.Duration(rand.Int64N(int64(10 * time.Second))) // 10s as nanoseconds overflows int on 32bit systems + + return baseTime + jitter +} diff --git a/pkg/retry/retry_test.go b/pkg/retry/retry_test.go new file mode 100644 index 000000000..54d452c4d --- /dev/null +++ b/pkg/retry/retry_test.go @@ -0,0 +1,89 @@ +package aws + +import ( + "errors" + "fmt" + "math" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRetryTime(t *testing.T) { + cases := []struct { + failures int + secGte int + secLte int + }{ + {0, 5, 15}, + {1, 5, 15}, + {2, 30, 40}, + {3, 90, 100}, + {4, 90, 100}, + {5, 90, 100}, + } + for _, tt := range cases { + t.Run(fmt.Sprintf("failures=%d", tt.failures), func(t *testing.T) { + previous := time.Duration(math.MaxInt64) + + for range 5 { + sec := RetryTime(tt.failures) + assert.GreaterOrEqual(t, sec, time.Duration(tt.secGte)*time.Second) + assert.LessOrEqual(t, sec, time.Duration(tt.secLte)*time.Second) + assert.NotEqual(t, sec, previous) + previous = sec + } + }) + } +} + +func TestRetryWithWarnings(t *testing.T) { + originalSleepFunc := sleepFunc + defer func() { sleepFunc = originalSleepFunc }() + sleepFunc = func(d time.Duration) {} + + // different number of retries than the default + settings := Settings{ + MaxRetries: 7, + RetryTimeFunc: RetryTime, + } + + cases := []struct { + failures int + expectSuccess bool + }{} + + for i := range settings.MaxRetries + 3 { + cases = append(cases, struct { + failures int + expectSuccess bool + }{i, i < settings.MaxRetries}) + } + + for _, tt := range cases { + t.Run(fmt.Sprintf("failures=%d", tt.failures), func(t *testing.T) { + attempts := 0 + expectedErr := errors.New("temporary error") + result, err := RetryWithWarnings("test-operation", settings, func() (string, error) { + attempts++ + if attempts <= tt.failures { + return "", expectedErr + } + return "success", nil + }) + + if tt.expectSuccess { + require.NoError(t, err) + assert.Equal(t, "success", result) + assert.Equal(t, tt.failures+1, attempts) + } else { + require.Error(t, err) + assert.Equal(t, expectedErr, err) + assert.Equal(t, "", result) + assert.Equal(t, settings.MaxRetries, attempts) + } + }) + } +}