diff --git a/api/dataservices/helpers.go b/api/dataservices/helpers.go index f9dd22380..dfddcfb37 100644 --- a/api/dataservices/helpers.go +++ b/api/dataservices/helpers.go @@ -4,7 +4,9 @@ import ( "errors" "fmt" + portainer "github.com/portainer/portainer/api" perrors "github.com/portainer/portainer/api/dataservices/errors" + gittypes "github.com/portainer/portainer/api/git/types" "github.com/rs/zerolog/log" ) @@ -12,6 +14,86 @@ import ( // ErrStop signals the stop of computation when filtering results var ErrStop = errors.New("stop") +// gitSourceReader is a minimal interface satisfied by both CE and EE DataStoreTx, +// used to avoid duplicating GitSourceForWorkflow across packages. +type gitSourceReader interface { + Workflow() WorkflowService + Source() SourceService +} + +// GitSourceForWorkflow returns the first git-type Source for the given workflow. +// Returns nil, nil when workflowID is 0 or no git source is found. +func GitSourceForWorkflow(tx gitSourceReader, workflowID portainer.WorkflowID) (*portainer.Source, error) { + if workflowID == 0 { + return nil, nil + } + + wf, err := tx.Workflow().Read(workflowID) + if err != nil { + return nil, err + } + + for _, srcID := range wf.SourceIDs { + src, err := tx.Source().Read(srcID) + if err != nil { + return nil, err + } + + if src.Type == portainer.SourceTypeGit { + return src, nil + } + } + + return nil, nil +} + +// GitConfigHashForWorkflow returns the git commit hash for the first git-type Source of the given workflow. +// Returns "" when workflowID is 0, no git source is found, or on error. +func GitConfigHashForWorkflow(tx gitSourceReader, workflowID portainer.WorkflowID) string { + src, err := GitSourceForWorkflow(tx, workflowID) + if err != nil || src == nil || src.GitConfig == nil { + return "" + } + + return src.GitConfig.ConfigHash +} + +// FindOrCreateGitSource returns an existing Source whose URL and authentication match src, +// or creates a new one. ConfigHash is excluded from matching as it is transient runtime state. +func FindOrCreateGitSource(tx gitSourceReader, src *portainer.Source) (*portainer.Source, error) { + existing, err := tx.Source().ReadAll(func(s portainer.Source) bool { + return s.Type == portainer.SourceTypeGit && + s.GitConfig != nil && + s.GitConfig.URL == src.GitConfig.URL && + gitAuthMatches(s.GitConfig.Authentication, src.GitConfig.Authentication) + }) + if err != nil { + return nil, err + } + + if len(existing) > 0 { + return &existing[0], nil + } + + if err := tx.Source().Create(src); err != nil { + return nil, err + } + + return src, nil +} + +func gitAuthMatches(a, b *gittypes.GitAuthentication) bool { + if a == nil && b == nil { + return true + } + + if a == nil || b == nil { + return false + } + + return a.Username == b.Username && a.Password == b.Password && a.GitCredentialID == b.GitCredentialID +} + func IsErrObjectNotFound(e error) bool { return errors.Is(e, perrors.ErrObjectNotFound) } diff --git a/api/dataservices/interface.go b/api/dataservices/interface.go index 281c0bbb0..4819f667f 100644 --- a/api/dataservices/interface.go +++ b/api/dataservices/interface.go @@ -24,6 +24,7 @@ type ( Settings() SettingsService Snapshot() SnapshotService SSLSettings() SSLSettingsService + Source() SourceService Stack() StackService Tag() TagService TeamMembership() TeamMembershipService @@ -32,6 +33,7 @@ type ( User() UserService Version() VersionService Webhook() WebhookService + Workflow() WorkflowService PendingActions() PendingActionsService } @@ -183,6 +185,11 @@ type ( BucketName() string } + // SourceService represents a service for managing GitOps source data + SourceService interface { + BaseCRUD[portainer.Source, portainer.SourceID] + } + // StackService represents a service for managing stack data StackService interface { BaseCRUD[portainer.Stack, portainer.StackID] @@ -245,4 +252,9 @@ type ( WebhookByResourceID(resourceID string) (*portainer.Webhook, error) WebhookByToken(token string) (*portainer.Webhook, error) } + + // WorkflowService represents a service for managing GitOps workflow data + WorkflowService interface { + BaseCRUD[portainer.Workflow, portainer.WorkflowID] + } ) diff --git a/api/dataservices/source/source.go b/api/dataservices/source/source.go new file mode 100644 index 000000000..3d97b4b37 --- /dev/null +++ b/api/dataservices/source/source.go @@ -0,0 +1,50 @@ +package source + +import ( + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/dataservices" +) + +// BucketName represents the name of the bucket where this service stores data. +const BucketName = "sources" + +// Service represents a service for managing GitOps source data. +type Service struct { + dataservices.BaseDataService[portainer.Source, portainer.SourceID] +} + +// NewService creates a new instance of a service. +func NewService(connection portainer.Connection) (*Service, error) { + err := connection.SetServiceName(BucketName) + if err != nil { + return nil, err + } + + return &Service{ + BaseDataService: dataservices.BaseDataService[portainer.Source, portainer.SourceID]{ + Bucket: BucketName, + Connection: connection, + }, + }, nil +} + +func (service *Service) Tx(tx portainer.Transaction) ServiceTx { + return ServiceTx{ + BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.Source, portainer.SourceID]{ + Bucket: BucketName, + Connection: service.Connection, + Tx: tx, + }, + } +} + +// Create creates a new source. +func (service *Service) Create(source *portainer.Source) error { + return service.Connection.CreateObject( + BucketName, + func(id uint64) (int, any) { + source.ID = portainer.SourceID(id) + return int(source.ID), source + }, + ) +} diff --git a/api/dataservices/source/tx.go b/api/dataservices/source/tx.go new file mode 100644 index 000000000..d45cebe74 --- /dev/null +++ b/api/dataservices/source/tx.go @@ -0,0 +1,21 @@ +package source + +import ( + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/dataservices" +) + +type ServiceTx struct { + dataservices.BaseDataServiceTx[portainer.Source, portainer.SourceID] +} + +// Create creates a new source. +func (service ServiceTx) Create(source *portainer.Source) error { + return service.Tx.CreateObject( + BucketName, + func(id uint64) (int, any) { + source.ID = portainer.SourceID(id) + return int(source.ID), source + }, + ) +} diff --git a/api/dataservices/stack/stack.go b/api/dataservices/stack/stack.go index d6a2453c1..78f213867 100644 --- a/api/dataservices/stack/stack.go +++ b/api/dataservices/stack/stack.go @@ -7,6 +7,8 @@ import ( portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/dataservices" dserrors "github.com/portainer/portainer/api/dataservices/errors" + + "github.com/rs/zerolog/log" ) // BucketName represents the name of the bucket where this service stores data. @@ -81,9 +83,21 @@ func (service *Service) GetNextIdentifier() int { // CreateStack creates a new stack. func (service *Service) Create(stack *portainer.Stack) error { + if stack.GitConfig != nil { + log.Warn().Int("stackID", int(stack.ID)).Str("url", stack.GitConfig.URL).Msg("stack persisted with non-nil GitConfig; GitConfig is deprecated, use WorkflowID/Source instead") + } + return service.Connection.CreateObjectWithId(BucketName, int(stack.ID), stack) } +func (service *Service) Update(ID portainer.StackID, stack *portainer.Stack) error { + if stack.GitConfig != nil { + log.Warn().Int("stackID", int(ID)).Str("url", stack.GitConfig.URL).Msg("stack persisted with non-nil GitConfig; GitConfig is deprecated, use WorkflowID/Source instead") + } + + return service.BaseDataService.Update(ID, stack) +} + // StackByWebhookID returns a pointer to a stack object by webhook ID. // It returns nil, errors.ErrObjectNotFound if there's no stack associated with the webhook ID. func (service *Service) StackByWebhookID(id string) (*portainer.Stack, error) { diff --git a/api/dataservices/stack/tx.go b/api/dataservices/stack/tx.go index 478c286ea..50b827e26 100644 --- a/api/dataservices/stack/tx.go +++ b/api/dataservices/stack/tx.go @@ -7,6 +7,8 @@ import ( portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/dataservices" dserrors "github.com/portainer/portainer/api/dataservices/errors" + + "github.com/rs/zerolog/log" ) type ServiceTx struct { @@ -56,9 +58,21 @@ func (service ServiceTx) GetNextIdentifier() int { // CreateStack creates a new stack. func (service ServiceTx) Create(stack *portainer.Stack) error { + if stack.GitConfig != nil { + log.Warn().Int("stackID", int(stack.ID)).Str("url", stack.GitConfig.URL).Msg("stack persisted with non-nil GitConfig; GitConfig is deprecated, use WorkflowID/Source instead") + } + return service.Tx.CreateObjectWithId(BucketName, int(stack.ID), stack) } +func (service ServiceTx) Update(ID portainer.StackID, stack *portainer.Stack) error { + if stack.GitConfig != nil { + log.Warn().Int("stackID", int(ID)).Str("url", stack.GitConfig.URL).Msg("stack persisted with non-nil GitConfig; GitConfig is deprecated, use WorkflowID/Source instead") + } + + return service.BaseDataServiceTx.Update(ID, stack) +} + // StackByWebhookID returns a pointer to a stack object by webhook ID. // It returns nil, errors.ErrObjectNotFound if there's no stack associated with the webhook ID. func (service ServiceTx) StackByWebhookID(id string) (*portainer.Stack, error) { diff --git a/api/dataservices/workflow/service.go b/api/dataservices/workflow/service.go new file mode 100644 index 000000000..8901ecb49 --- /dev/null +++ b/api/dataservices/workflow/service.go @@ -0,0 +1,46 @@ +package workflow + +import ( + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/dataservices" +) + +const BucketName = "workflows" + +type Service struct { + dataservices.BaseDataService[portainer.Workflow, portainer.WorkflowID] +} + +func NewService(connection portainer.Connection) (*Service, error) { + err := connection.SetServiceName(BucketName) + if err != nil { + return nil, err + } + + return &Service{ + BaseDataService: dataservices.BaseDataService[portainer.Workflow, portainer.WorkflowID]{ + Bucket: BucketName, + Connection: connection, + }, + }, nil +} + +func (service *Service) Tx(tx portainer.Transaction) ServiceTx { + return ServiceTx{ + BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.Workflow, portainer.WorkflowID]{ + Bucket: BucketName, + Connection: service.Connection, + Tx: tx, + }, + } +} + +func (service *Service) Create(workflow *portainer.Workflow) error { + return service.Connection.CreateObject( + BucketName, + func(id uint64) (int, any) { + workflow.ID = portainer.WorkflowID(id) + return int(workflow.ID), workflow + }, + ) +} diff --git a/api/dataservices/workflow/tx.go b/api/dataservices/workflow/tx.go new file mode 100644 index 000000000..8fe38fd80 --- /dev/null +++ b/api/dataservices/workflow/tx.go @@ -0,0 +1,20 @@ +package workflow + +import ( + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/dataservices" +) + +type ServiceTx struct { + dataservices.BaseDataServiceTx[portainer.Workflow, portainer.WorkflowID] +} + +func (service ServiceTx) Create(workflow *portainer.Workflow) error { + return service.Tx.CreateObject( + BucketName, + func(id uint64) (int, any) { + workflow.ID = portainer.WorkflowID(id) + return int(workflow.ID), workflow + }, + ) +} diff --git a/api/datastore/migrate_data.go b/api/datastore/migrate_data.go index 2b53bbb9c..74a033334 100644 --- a/api/datastore/migrate_data.go +++ b/api/datastore/migrate_data.go @@ -88,6 +88,8 @@ func (store *Store) newMigratorParameters(version *models.Version, flags *portai EdgeGroupService: store.EdgeGroupService, TunnelServerService: store.TunnelServerService, PendingActionsService: store.PendingActionsService, + SourceService: store.SourceService, + WorkflowService: store.WorkflowService, } } diff --git a/api/datastore/migrator/migrate_2_43_0.go b/api/datastore/migrator/migrate_2_43_0.go new file mode 100644 index 000000000..c9c3abefd --- /dev/null +++ b/api/datastore/migrator/migrate_2_43_0.go @@ -0,0 +1,167 @@ +package migrator + +import ( + "fmt" + + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/dataservices/stack" + gittypes "github.com/portainer/portainer/api/git/types" + + "github.com/rs/zerolog/log" +) + +type legacyRepoConfig struct { + URL string + ReferenceName string + ConfigFilePath string + Authentication *legacyGitAuthentication + ConfigHash string + TLSSkipVerify bool +} + +type legacyGitAuthentication struct { + Username string + Password string + Provider int `json:",omitempty"` + AuthorizationType int `json:",omitempty"` + GitCredentialID int +} + +func (lrc *legacyRepoConfig) toRepoConfig() *gittypes.RepoConfig { + if lrc == nil { + return nil + } + + cfg := &gittypes.RepoConfig{ + URL: lrc.URL, + ReferenceName: lrc.ReferenceName, + ConfigFilePath: lrc.ConfigFilePath, + ConfigHash: lrc.ConfigHash, + TLSSkipVerify: lrc.TLSSkipVerify, + } + + if lrc.Authentication != nil { + cfg.Authentication = &gittypes.GitAuthentication{ + Username: lrc.Authentication.Username, + Password: lrc.Authentication.Password, + Provider: gittypes.GitProvider(lrc.Authentication.Provider), + AuthorizationType: gittypes.GitCredentialAuthType(lrc.Authentication.AuthorizationType), + } + } + + return cfg +} + +type legacyStack struct { + ID int `json:"Id"` + GitConfig *legacyRepoConfig `json:"GitConfig"` + WorkflowID *int +} + +// sourceDedupeKey is the identity used to detect duplicate Sources during migration. +// Two stacks sharing the same URL and credentials must reuse the same Source record. +type sourceDedupeKey struct { + url string + username string + password string +} + +func gitSourceKey(cfg *gittypes.RepoConfig) sourceDedupeKey { + key := sourceDedupeKey{url: cfg.URL} + if cfg.Authentication != nil { + key.username = cfg.Authentication.Username + key.password = cfg.Authentication.Password + } + + return key +} + +func (m *Migrator) migrateGitConfigToSources_2_43_0() error { + log.Info().Msg("migrating git-backed stacks to Source+Workflow records") + + var legacyStacks []legacyStack + + err := m.stackService.Connection.GetAll( + stack.BucketName, + new(legacyStack), + func(obj any) (any, error) { + s, ok := obj.(*legacyStack) + if !ok { + return nil, fmt.Errorf("unexpected type reading stack bucket: %T", obj) + } + + legacyStacks = append(legacyStacks, *s) + + return new(legacyStack), nil + }, + ) + if err != nil { + return err + } + + existingSources, err := m.sourceService.ReadAll() + if err != nil { + return err + } + + sourcesByKey := make(map[sourceDedupeKey]portainer.SourceID, len(existingSources)) + for _, src := range existingSources { + if src.GitConfig != nil { + sourcesByKey[gitSourceKey(src.GitConfig)] = src.ID + } + } + + for _, ls := range legacyStacks { + if ls.GitConfig == nil || (ls.WorkflowID != nil && *ls.WorkflowID != 0) { + continue + } + + cfg := ls.GitConfig.toRepoConfig() + cfg.URL = gittypes.SanitizeURL(cfg.URL) + key := gitSourceKey(cfg) + + var newSrcID portainer.SourceID + + if err := m.stackService.Connection.UpdateTx(func(tx portainer.Transaction) error { + srcID, exists := sourcesByKey[key] + + if !exists { + src := &portainer.Source{ + Name: gittypes.RepoName(cfg.URL), + Type: portainer.SourceTypeGit, + GitConfig: cfg, + } + if err := m.sourceService.Tx(tx).Create(src); err != nil { + return fmt.Errorf("failed to create source for stack %d: %w", ls.ID, err) + } + srcID = src.ID + newSrcID = src.ID + } + + wf := &portainer.Workflow{ + SourceIDs: []portainer.SourceID{srcID}, + } + if err := m.workflowService.Tx(tx).Create(wf); err != nil { + return fmt.Errorf("failed to create workflow for stack %d: %w", ls.ID, err) + } + + liveStack, err := m.stackService.Tx(tx).Read(portainer.StackID(ls.ID)) + if err != nil { + return fmt.Errorf("failed to read stack %d: %w", ls.ID, err) + } + + liveStack.WorkflowID = wf.ID + liveStack.GitConfig = nil + + return m.stackService.Tx(tx).Update(portainer.StackID(ls.ID), liveStack) + }); err != nil { + return fmt.Errorf("failed to migrate stack %d: %w", ls.ID, err) + } + + if newSrcID != 0 { + sourcesByKey[key] = newSrcID + } + } + + return nil +} diff --git a/api/datastore/migrator/migrate_2_43_0_test.go b/api/datastore/migrator/migrate_2_43_0_test.go new file mode 100644 index 000000000..8c8cc1f6e --- /dev/null +++ b/api/datastore/migrator/migrate_2_43_0_test.go @@ -0,0 +1,221 @@ +package migrator + +import ( + "testing" + + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/database/boltdb" + "github.com/portainer/portainer/api/dataservices/source" + "github.com/portainer/portainer/api/dataservices/stack" + "github.com/portainer/portainer/api/dataservices/workflow" + gittypes "github.com/portainer/portainer/api/git/types" + "github.com/portainer/portainer/api/logs" + + "github.com/stretchr/testify/require" +) + +func TestMigrateGitConfigToSources_2_43_0_GitStackMigrated(t *testing.T) { + t.Parallel() + + conn := &boltdb.DbConnection{Path: t.TempDir()} + err := conn.Open() + require.NoError(t, err) + defer logs.CloseAndLogErr(conn) + + stackSvc, err := stack.NewService(conn) + require.NoError(t, err) + sourceSvc, err := source.NewService(conn) + require.NoError(t, err) + workflowSvc, err := workflow.NewService(conn) + require.NoError(t, err) + + m := NewMigrator(&MigratorParameters{ + StackService: stackSvc, + SourceService: sourceSvc, + WorkflowService: workflowSvc, + }) + + gitStack := &portainer.Stack{ + ID: 1, + Name: "git-stack", + GitConfig: &gittypes.RepoConfig{ + URL: "https://github.com/example/repo", + ReferenceName: "refs/heads/main", + ConfigHash: "abc123", + }, + } + err = conn.CreateObjectWithId(stack.BucketName, int(gitStack.ID), gitStack) + require.NoError(t, err) + + err = m.migrateGitConfigToSources_2_43_0() + require.NoError(t, err) + + migrated, err := stackSvc.Read(gitStack.ID) + require.NoError(t, err) + require.NotZero(t, migrated.WorkflowID) + require.Nil(t, migrated.GitConfig) + + wf, err := workflowSvc.Read(migrated.WorkflowID) + require.NoError(t, err) + require.Len(t, wf.SourceIDs, 1) + + src, err := sourceSvc.Read(wf.SourceIDs[0]) + require.NoError(t, err) + require.Equal(t, portainer.SourceTypeGit, src.Type) + require.Equal(t, gitStack.GitConfig.URL, src.GitConfig.URL) + require.Equal(t, gitStack.GitConfig.ReferenceName, src.GitConfig.ReferenceName) +} + +func TestMigrateGitConfigToSources_2_43_0_NonGitStackUntouched(t *testing.T) { + t.Parallel() + + conn := &boltdb.DbConnection{Path: t.TempDir()} + err := conn.Open() + require.NoError(t, err) + defer logs.CloseAndLogErr(conn) + + stackSvc, err := stack.NewService(conn) + require.NoError(t, err) + sourceSvc, err := source.NewService(conn) + require.NoError(t, err) + workflowSvc, err := workflow.NewService(conn) + require.NoError(t, err) + + m := NewMigrator(&MigratorParameters{ + StackService: stackSvc, + SourceService: sourceSvc, + WorkflowService: workflowSvc, + }) + + plainStack := &portainer.Stack{ + ID: 1, + Name: "plain-stack", + } + err = conn.CreateObjectWithId(stack.BucketName, int(plainStack.ID), plainStack) + require.NoError(t, err) + + err = m.migrateGitConfigToSources_2_43_0() + require.NoError(t, err) + + result, err := stackSvc.Read(plainStack.ID) + require.NoError(t, err) + require.Zero(t, result.WorkflowID) + require.Nil(t, result.GitConfig) + + sources, err := sourceSvc.ReadAll() + require.NoError(t, err) + require.Empty(t, sources) + + workflows, err := workflowSvc.ReadAll() + require.NoError(t, err) + require.Empty(t, workflows) +} + +func TestMigrateGitConfigToSources_2_43_0_DuplicateSourcesDeduped(t *testing.T) { + t.Parallel() + + conn := &boltdb.DbConnection{Path: t.TempDir()} + err := conn.Open() + require.NoError(t, err) + defer logs.CloseAndLogErr(conn) + + stackSvc, err := stack.NewService(conn) + require.NoError(t, err) + sourceSvc, err := source.NewService(conn) + require.NoError(t, err) + workflowSvc, err := workflow.NewService(conn) + require.NoError(t, err) + + m := NewMigrator(&MigratorParameters{ + StackService: stackSvc, + SourceService: sourceSvc, + WorkflowService: workflowSvc, + }) + + sharedURL := "https://github.com/example/shared-repo" + + stack1 := &portainer.Stack{ + ID: 1, + Name: "stack-a", + GitConfig: &gittypes.RepoConfig{ + URL: sharedURL, + ReferenceName: "refs/heads/main", + }, + } + stack2 := &portainer.Stack{ + ID: 2, + Name: "stack-b", + GitConfig: &gittypes.RepoConfig{ + URL: sharedURL, + ReferenceName: "refs/heads/develop", + }, + } + err = conn.CreateObjectWithId(stack.BucketName, int(stack1.ID), stack1) + require.NoError(t, err) + err = conn.CreateObjectWithId(stack.BucketName, int(stack2.ID), stack2) + require.NoError(t, err) + + err = m.migrateGitConfigToSources_2_43_0() + require.NoError(t, err) + + sources, err := sourceSvc.ReadAll() + require.NoError(t, err) + require.Len(t, sources, 1, "two stacks with the same URL must share one Source") + + workflows, err := workflowSvc.ReadAll() + require.NoError(t, err) + require.Len(t, workflows, 2, "each stack must get its own Workflow") + + sharedSourceID := sources[0].ID + for _, wf := range workflows { + require.Len(t, wf.SourceIDs, 1) + require.Equal(t, sharedSourceID, wf.SourceIDs[0]) + } +} + +func TestMigrateGitConfigToSources_2_43_0_Idempotent(t *testing.T) { + t.Parallel() + + conn := &boltdb.DbConnection{Path: t.TempDir()} + err := conn.Open() + require.NoError(t, err) + defer logs.CloseAndLogErr(conn) + + stackSvc, err := stack.NewService(conn) + require.NoError(t, err) + sourceSvc, err := source.NewService(conn) + require.NoError(t, err) + workflowSvc, err := workflow.NewService(conn) + require.NoError(t, err) + + m := NewMigrator(&MigratorParameters{ + StackService: stackSvc, + SourceService: sourceSvc, + WorkflowService: workflowSvc, + }) + + gitStack := &portainer.Stack{ + ID: 1, + Name: "git-stack", + GitConfig: &gittypes.RepoConfig{ + URL: "https://github.com/example/repo", + }, + } + err = conn.CreateObjectWithId(stack.BucketName, int(gitStack.ID), gitStack) + require.NoError(t, err) + + err = m.migrateGitConfigToSources_2_43_0() + require.NoError(t, err) + + // Second run must not create duplicate Source/Workflow records + err = m.migrateGitConfigToSources_2_43_0() + require.NoError(t, err) + + sources, err := sourceSvc.ReadAll() + require.NoError(t, err) + require.Len(t, sources, 1) + + workflows, err := workflowSvc.ReadAll() + require.NoError(t, err) + require.Len(t, workflows, 1) +} diff --git a/api/datastore/migrator/migrator.go b/api/datastore/migrator/migrator.go index 683d45e55..3a4bfa91c 100644 --- a/api/datastore/migrator/migrator.go +++ b/api/datastore/migrator/migrator.go @@ -21,12 +21,14 @@ import ( "github.com/portainer/portainer/api/dataservices/schedule" "github.com/portainer/portainer/api/dataservices/settings" "github.com/portainer/portainer/api/dataservices/snapshot" + "github.com/portainer/portainer/api/dataservices/source" "github.com/portainer/portainer/api/dataservices/stack" "github.com/portainer/portainer/api/dataservices/tag" "github.com/portainer/portainer/api/dataservices/teammembership" "github.com/portainer/portainer/api/dataservices/tunnelserver" "github.com/portainer/portainer/api/dataservices/user" "github.com/portainer/portainer/api/dataservices/version" + "github.com/portainer/portainer/api/dataservices/workflow" "github.com/portainer/portainer/api/internal/authorization" "github.com/Masterminds/semver/v3" @@ -64,6 +66,8 @@ type ( edgeGroupService *edgegroup.Service TunnelServerService *tunnelserver.Service pendingActionsService *pendingactions.Service + sourceService *source.Service + workflowService *workflow.Service } // MigratorParameters represents the required parameters to create a new Migrator instance. @@ -94,6 +98,8 @@ type ( EdgeGroupService *edgegroup.Service TunnelServerService *tunnelserver.Service PendingActionsService *pendingactions.Service + SourceService *source.Service + WorkflowService *workflow.Service } ) @@ -126,6 +132,8 @@ func NewMigrator(parameters *MigratorParameters) *Migrator { edgeGroupService: parameters.EdgeGroupService, TunnelServerService: parameters.TunnelServerService, pendingActionsService: parameters.PendingActionsService, + sourceService: parameters.SourceService, + workflowService: parameters.WorkflowService, } migrator.initMigrations() @@ -260,6 +268,8 @@ func (m *Migrator) initMigrations() { m.addMigrations("2.40.0", m.migrateRegistryAccessSASecrets_2_40_0) + m.addMigrations("2.43.0", m.migrateGitConfigToSources_2_43_0) + // WARNING: do not change migrations that have already been released! // Add new migrations above... diff --git a/api/datastore/services.go b/api/datastore/services.go index fb385cb72..163cf92f4 100644 --- a/api/datastore/services.go +++ b/api/datastore/services.go @@ -26,6 +26,7 @@ import ( "github.com/portainer/portainer/api/dataservices/schedule" "github.com/portainer/portainer/api/dataservices/settings" "github.com/portainer/portainer/api/dataservices/snapshot" + "github.com/portainer/portainer/api/dataservices/source" "github.com/portainer/portainer/api/dataservices/ssl" "github.com/portainer/portainer/api/dataservices/stack" "github.com/portainer/portainer/api/dataservices/tag" @@ -35,6 +36,7 @@ import ( "github.com/portainer/portainer/api/dataservices/user" "github.com/portainer/portainer/api/dataservices/version" "github.com/portainer/portainer/api/dataservices/webhook" + "github.com/portainer/portainer/api/dataservices/workflow" "github.com/rs/zerolog/log" "github.com/segmentio/encoding/json" @@ -67,6 +69,7 @@ type Store struct { ScheduleService *schedule.Service SettingsService *settings.Service SnapshotService *snapshot.Service + SourceService *source.Service SSLSettingsService *ssl.Service StackService *stack.Service TagService *tag.Service @@ -76,6 +79,7 @@ type Store struct { UserService *user.Service VersionService *version.Service WebhookService *webhook.Service + WorkflowService *workflow.Service PendingActionsService *pendingactions.Service } @@ -179,6 +183,12 @@ func (store *Store) initServices() error { } store.SnapshotService = snapshotService + sourceService, err := source.NewService(store.connection) + if err != nil { + return err + } + store.SourceService = sourceService + sslSettingsService, err := ssl.NewService(store.connection) if err != nil { return err @@ -239,6 +249,12 @@ func (store *Store) initServices() error { } store.WebhookService = webhookService + workflowService, err := workflow.NewService(store.connection) + if err != nil { + return err + } + store.WorkflowService = workflowService + scheduleService, err := schedule.NewService(store.connection) if err != nil { return err @@ -332,6 +348,11 @@ func (store *Store) Snapshot() dataservices.SnapshotService { return store.SnapshotService } +// Source gives access to the Source data management layer +func (store *Store) Source() dataservices.SourceService { + return store.SourceService +} + // SSLSettings gives access to the SSL Settings data management layer func (store *Store) SSLSettings() dataservices.SSLSettingsService { return store.SSLSettingsService @@ -377,6 +398,11 @@ func (store *Store) Webhook() dataservices.WebhookService { return store.WebhookService } +// Workflow gives access to the Workflow data management layer +func (store *Store) Workflow() dataservices.WorkflowService { + return store.WorkflowService +} + type storeExport struct { CustomTemplate []portainer.CustomTemplate `json:"customtemplates,omitempty"` EdgeGroup []portainer.EdgeGroup `json:"edgegroups,omitempty"` @@ -394,6 +420,7 @@ type storeExport struct { Settings portainer.Settings `json:"settings,omitzero"` Snapshot []portainer.Snapshot `json:"snapshots,omitempty"` SSLSettings portainer.SSLSettings `json:"ssl,omitzero"` + Source []portainer.Source `json:"sources,omitempty"` Stack []portainer.Stack `json:"stacks,omitempty"` Tag []portainer.Tag `json:"tags,omitempty"` TeamMembership []portainer.TeamMembership `json:"team_membership,omitempty"` @@ -402,6 +429,7 @@ type storeExport struct { User []portainer.User `json:"users,omitempty"` Version models.Version `json:"version,omitzero"` Webhook []portainer.Webhook `json:"webhooks,omitempty"` + Workflow []portainer.Workflow `json:"workflows,omitempty"` Metadata map[string]any `json:"metadata,omitempty"` } @@ -536,6 +564,14 @@ func (store *Store) Export(filename string) (err error) { backup.SSLSettings = *settings } + if s, err := store.Source().ReadAll(); err != nil { + if !store.IsErrObjectNotFound(err) { + log.Error().Err(err).Msg("exporting Sources") + } + } else { + backup.Source = s + } + if t, err := store.Stack().ReadAll(); err != nil { if !store.IsErrObjectNotFound(err) { log.Error().Err(err).Msg("exporting Stacks") @@ -592,6 +628,14 @@ func (store *Store) Export(filename string) (err error) { backup.Webhook = webhooks } + if w, err := store.Workflow().ReadAll(); err != nil { + if !store.IsErrObjectNotFound(err) { + log.Error().Err(err).Msg("exporting Workflows") + } + } else { + backup.Workflow = w + } + if version, err := store.Version().Version(); err != nil { if !store.IsErrObjectNotFound(err) { log.Error().Err(err).Msg("exporting Version") @@ -710,6 +754,18 @@ func (store *Store) Import(filename string) (err error) { } } + for _, v := range backup.Source { + if err := store.Source().Update(v.ID, &v); err != nil { + log.Warn().Err(err).Msg("failed to update the source in the database") + } + } + + for _, v := range backup.Workflow { + if err := store.Workflow().Update(v.ID, &v); err != nil { + log.Warn().Err(err).Msg("failed to update the workflow in the database") + } + } + for _, v := range backup.Stack { if err := store.Stack().Update(v.ID, &v); err != nil { log.Warn().Err(err).Msg("failed to update the stack in the database") diff --git a/api/datastore/services_tx.go b/api/datastore/services_tx.go index f8781ca5f..31fcd5370 100644 --- a/api/datastore/services_tx.go +++ b/api/datastore/services_tx.go @@ -74,6 +74,10 @@ func (tx *StoreTx) Snapshot() dataservices.SnapshotService { return tx.store.SnapshotService.Tx(tx.tx) } +func (tx *StoreTx) Source() dataservices.SourceService { + return tx.store.SourceService.Tx(tx.tx) +} + func (tx *StoreTx) SSLSettings() dataservices.SSLSettingsService { return tx.store.SSLSettingsService.Tx(tx.tx) } @@ -102,3 +106,7 @@ func (tx *StoreTx) User() dataservices.UserService { func (tx *StoreTx) Version() dataservices.VersionService { return nil } func (tx *StoreTx) Webhook() dataservices.WebhookService { return nil } + +func (tx *StoreTx) Workflow() dataservices.WorkflowService { + return tx.store.WorkflowService.Tx(tx.tx) +} diff --git a/api/datastore/test_data/output_24_to_latest.json b/api/datastore/test_data/output_24_to_latest.json index e840f28d4..642be4d4a 100644 --- a/api/datastore/test_data/output_24_to_latest.json +++ b/api/datastore/test_data/output_24_to_latest.json @@ -786,6 +786,7 @@ "Kubernetes": null } ], + "sources": null, "ssl": { "certPath": "", "httpEnabled": true, @@ -939,5 +940,6 @@ "version": { "VERSION": "{\"SchemaVersion\":\"2.42.0\",\"MigratorCount\":0,\"Edition\":1,\"InstanceID\":\"463d5c47-0ea5-4aca-85b1-405ceefee254\"}" }, - "webhooks": null + "webhooks": null, + "workflows": null } \ No newline at end of file diff --git a/api/git/types/types.go b/api/git/types/types.go index 166bf10b0..7aca167f1 100644 --- a/api/git/types/types.go +++ b/api/git/types/types.go @@ -53,6 +53,25 @@ func SanitizeURL(rawURL string) string { return u.String() } +// SanitizeRepoConfig returns a copy of gc with the URL sanitized and password cleared, +// safe to return to clients. +func SanitizeRepoConfig(gc *RepoConfig) *RepoConfig { + if gc == nil { + return nil + } + + result := *gc + result.URL = SanitizeURL(result.URL) + + if result.Authentication != nil && result.Authentication.Password != "" { + auth := *result.Authentication + auth.Password = "" + result.Authentication = &auth + } + + return &result +} + type GitAuthentication struct { Username string Password string diff --git a/api/gitops/workflows/fetch.go b/api/gitops/workflows/fetch.go index e91a45d48..1d4bf3501 100644 --- a/api/gitops/workflows/fetch.go +++ b/api/gitops/workflows/fetch.go @@ -9,6 +9,8 @@ import ( "github.com/portainer/portainer/api/http/security" "github.com/portainer/portainer/api/kubernetes/cli" "github.com/portainer/portainer/api/set" + + "github.com/rs/zerolog/log" ) // FetchWorkflows returns all GitOps workflows visible to the given user. @@ -20,12 +22,14 @@ func FetchWorkflows( sc *security.RestrictedRequestContext, endpointIDSet set.Set[portainer.EndpointID], ) ([]Workflow, error) { - var entries []portainer.Stack + var stacks []portainer.Stack var endpointMap map[portainer.EndpointID]portainer.Endpoint + gitConfigs := map[portainer.StackID]*gittypes.RepoConfig{} err := dataStore.ViewTx(func(tx dataservices.DataStoreTx) error { - stacks, err := tx.Stack().ReadAll(func(s portainer.Stack) bool { - return s.GitConfig != nil && (len(endpointIDSet) == 0 || endpointIDSet.Contains(s.EndpointID)) + var err error + stacks, err = tx.Stack().ReadAll(func(s portainer.Stack) bool { + return s.WorkflowID != 0 && (len(endpointIDSet) == 0 || endpointIDSet.Contains(s.EndpointID)) }) if err != nil { return err @@ -41,15 +45,75 @@ func FetchWorkflows( return err } - for i := range stacks { - s := stacks[i] - + // First pass: filter by endpoint/stack-type match and collect workflow IDs. + preFiltered := make([]portainer.Stack, 0, len(stacks)) + workflowIDSet := make(map[portainer.WorkflowID]struct{}, len(stacks)) + for _, s := range stacks { if ep, ok := endpointMap[s.EndpointID]; ok && !EndpointMatchesStackType(ep, s.Type) { continue } - entries = append(entries, s) + preFiltered = append(preFiltered, s) + workflowIDSet[s.WorkflowID] = struct{}{} } + // Batch-load all needed workflows in one scan. + workflows, err := tx.Workflow().ReadAll(func(wf portainer.Workflow) bool { + _, ok := workflowIDSet[wf.ID] + return ok + }) + if err != nil { + return err + } + + workflowMap := make(map[portainer.WorkflowID]portainer.Workflow, len(workflows)) + sourceIDSet := make(map[portainer.SourceID]struct{}) + for _, wf := range workflows { + workflowMap[wf.ID] = wf + for _, sid := range wf.SourceIDs { + sourceIDSet[sid] = struct{}{} + } + } + + // Batch-load all needed sources in one scan. + srcs, err := tx.Source().ReadAll(func(src portainer.Source) bool { + _, ok := sourceIDSet[src.ID] + return ok + }) + if err != nil { + return err + } + + sourceMap := make(map[portainer.SourceID]portainer.Source, len(srcs)) + for _, src := range srcs { + sourceMap[src.ID] = src + } + + // Second pass: build filtered list using in-memory lookups. + var filtered []portainer.Stack + for _, s := range preFiltered { + wf, ok := workflowMap[s.WorkflowID] + if !ok { + log.Warn().Int("stackID", int(s.ID)).Msg("workflow record missing for stack, skipping") + continue + } + + for _, srcID := range wf.SourceIDs { + src, ok := sourceMap[srcID] + if !ok { + log.Warn().Int("stackID", int(s.ID)).Msg("source record missing for stack, skipping") + break + } + + if src.Type == portainer.SourceTypeGit { + gitConfigs[s.ID] = src.GitConfig + break + } + } + + filtered = append(filtered, s) + } + stacks = filtered + return nil }) if err != nil { @@ -61,46 +125,119 @@ func FetchWorkflows( return nil, err } - entries, err = filterK8SStacks(entries, endpointMap, k8sFactory, accessMap) + stacks, err = filterK8SStacks(stacks, endpointMap, k8sFactory, accessMap) if err != nil { return nil, err } - items := make([]Workflow, 0, len(entries)) - for _, s := range entries { - gitEntries := []GitEntries{ - {Name: s.GitConfig.ConfigFilePath, IsFile: true}, - } - for _, additionalPath := range s.AdditionalFiles { - gitEntries = append(gitEntries, GitEntries{Name: additionalPath, IsFile: true}) - } - - source, artifact := computePhases(ctx, gitService, s.GitConfig, gitEntries) - items = append(items, MapStackToWorkflow(s, s.GitConfig, source, artifact)) + items := make([]Workflow, 0, len(stacks)) + for _, s := range stacks { + gitConfig := gitConfigs[s.ID] + source, artifact := ComputeGitPhasesForConfig(ctx, gitService, gitConfig) + items = append(items, MapStackToWorkflow(s, gitConfig, source, artifact)) } return items, nil } -func computePhases(ctx context.Context, gitSvc portainer.GitService, cfg *gittypes.RepoConfig, gitEntries []GitEntries) (source, artifact WorkflowPhaseStatus) { - if gitSvc == nil || cfg == nil { - return WorkflowPhaseStatus{Status: StatusUnknown}, WorkflowPhaseStatus{Status: StatusUnknown} - } - - username, password := gitCredentials(cfg) - return ComputeGitPhases(ctx, cfg.ReferenceName, gitEntries, - func(ctx context.Context) ([]string, error) { - return gitSvc.ListRefs(ctx, cfg.URL, username, password, false, cfg.TLSSkipVerify) - }, - func(ctx context.Context, exts []string, dirOnly bool) ([]string, error) { - return gitSvc.ListFiles(ctx, cfg.URL, cfg.ReferenceName, username, password, dirOnly, false, exts, cfg.TLSSkipVerify) - }, - ) +// SourceStats holds aggregated statistics for a GitOps source. +type SourceStats struct { + WorkflowCount int + EndpointIDs set.Set[portainer.EndpointID] + LastSync int64 } -func gitCredentials(cfg *gittypes.RepoConfig) (username, password string) { - if cfg.Authentication != nil { - return cfg.Authentication.Username, cfg.Authentication.Password +// FetchSourceStats returns all sources and per-source stats for sources accessible to the given user. +// It applies the same access control as FetchWorkflows but skips git phase checks. +func FetchSourceStats( + tx dataservices.DataStoreTx, + k8sFactory *cli.ClientFactory, + sc *security.RestrictedRequestContext, +) ([]portainer.Source, map[portainer.SourceID]SourceStats, error) { + sources, err := tx.Source().ReadAll() + if err != nil { + return nil, nil, err + } + + allStacks, err := tx.Stack().ReadAll(func(s portainer.Stack) bool { return s.WorkflowID != 0 }) + if err != nil { + return nil, nil, err + } + + endpointMap, err := buildEndpointMap(tx, allStacks) + if err != nil { + return nil, nil, err + } + + allStacks, err = filterDockerStacksByAccess(tx, allStacks, sc) + if err != nil { + return nil, nil, err + } + + workflowIDSet := make(map[portainer.WorkflowID]struct{}, len(allStacks)) + preFiltered := make([]portainer.Stack, 0, len(allStacks)) + for _, s := range allStacks { + if ep, ok := endpointMap[s.EndpointID]; ok && !EndpointMatchesStackType(ep, s.Type) { + continue + } + preFiltered = append(preFiltered, s) + workflowIDSet[s.WorkflowID] = struct{}{} + } + + wfs, err := tx.Workflow().ReadAll(func(wf portainer.Workflow) bool { + _, ok := workflowIDSet[wf.ID] + return ok + }) + if err != nil { + return nil, nil, err + } + + wfSources := make(map[portainer.WorkflowID][]portainer.SourceID, len(wfs)) + for _, wf := range wfs { + wfSources[wf.ID] = wf.SourceIDs + } + + stackSourceIDs := make(map[portainer.StackID][]portainer.SourceID) + for _, s := range preFiltered { + if srcIDs := wfSources[s.WorkflowID]; len(srcIDs) > 0 { + stackSourceIDs[s.ID] = srcIDs + } + } + + accessMap, err := buildEndpointAccessMap(k8sFactory, sc, endpointMap) + if err != nil { + return nil, nil, err + } + + stacks, err := filterK8SStacks(preFiltered, endpointMap, k8sFactory, accessMap) + if err != nil { + return nil, nil, err + } + + stats := make(map[portainer.SourceID]SourceStats) + + for _, s := range stacks { + var epIDs []portainer.EndpointID + if s.EndpointID != 0 { + epIDs = []portainer.EndpointID{s.EndpointID} + } + addSourceStats(stats, stackSourceIDs[s.ID], epIDs, StackLastSyncDate(s)) + } + + return sources, stats, nil +} + +func addSourceStats(result map[portainer.SourceID]SourceStats, srcIDs []portainer.SourceID, epIDs []portainer.EndpointID, lastSync int64) { + for _, srcID := range srcIDs { + st := result[srcID] + if st.EndpointIDs == nil { + st.EndpointIDs = make(set.Set[portainer.EndpointID]) + } + st.WorkflowCount++ + for _, epID := range epIDs { + st.EndpointIDs.Add(epID) + } + st.LastSync = max(lastSync, st.LastSync) + result[srcID] = st } - return "", "" } diff --git a/api/gitops/workflows/fetch_test.go b/api/gitops/workflows/fetch_test.go new file mode 100644 index 000000000..62f36b811 --- /dev/null +++ b/api/gitops/workflows/fetch_test.go @@ -0,0 +1,263 @@ +package workflows + +import ( + "strconv" + "testing" + + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/dataservices" + "github.com/portainer/portainer/api/datastore" + gittypes "github.com/portainer/portainer/api/git/types" + "github.com/portainer/portainer/api/http/security" + "github.com/portainer/portainer/api/set" + + "github.com/stretchr/testify/require" +) + +func adminContext() *security.RestrictedRequestContext { + return &security.RestrictedRequestContext{IsAdmin: true, UserID: 1} +} + +func mustCreateGitWorkflow(t *testing.T, tx dataservices.DataStoreTx, stack *portainer.Stack) { + t.Helper() + + cfg := stack.GitConfig + + src := &portainer.Source{Type: portainer.SourceTypeGit, GitConfig: cfg} + require.NoError(t, tx.Source().Create(src)) + + wf := &portainer.Workflow{SourceIDs: []portainer.SourceID{src.ID}} + require.NoError(t, tx.Workflow().Create(wf)) + + stack.WorkflowID = wf.ID + stack.GitConfig = nil + + require.NoError(t, tx.Stack().Create(stack)) +} + +func TestAddSourceStats_NoOp(t *testing.T) { + t.Parallel() + + result := make(map[portainer.SourceID]SourceStats) + addSourceStats(result, nil, nil, 0) + + require.Empty(t, result) +} + +func TestAddSourceStats_AccumulatesWorkflowCount(t *testing.T) { + t.Parallel() + + result := make(map[portainer.SourceID]SourceStats) + addSourceStats(result, []portainer.SourceID{1}, nil, 0) + addSourceStats(result, []portainer.SourceID{1}, nil, 0) + + require.Equal(t, 2, result[1].WorkflowCount) +} + +func TestAddSourceStats_CollectsUniqueEndpointIDs(t *testing.T) { + t.Parallel() + + result := make(map[portainer.SourceID]SourceStats) + addSourceStats(result, []portainer.SourceID{1}, []portainer.EndpointID{10, 20}, 0) + addSourceStats(result, []portainer.SourceID{1}, []portainer.EndpointID{20, 30}, 0) + + require.Len(t, result[1].EndpointIDs, 3) + require.True(t, result[1].EndpointIDs[10]) + require.True(t, result[1].EndpointIDs[20]) + require.True(t, result[1].EndpointIDs[30]) +} + +func TestAddSourceStats_MaxLastSync(t *testing.T) { + t.Parallel() + + result := make(map[portainer.SourceID]SourceStats) + addSourceStats(result, []portainer.SourceID{1}, nil, 100) + addSourceStats(result, []portainer.SourceID{1}, nil, 500) + addSourceStats(result, []portainer.SourceID{1}, nil, 200) + + require.Equal(t, int64(500), result[1].LastSync) +} + +func TestAddSourceStats_MultipleSourceIDs(t *testing.T) { + t.Parallel() + + result := make(map[portainer.SourceID]SourceStats) + addSourceStats(result, []portainer.SourceID{1, 2}, []portainer.EndpointID{10}, 100) + + require.Equal(t, 1, result[1].WorkflowCount) + require.Equal(t, 1, result[2].WorkflowCount) + require.True(t, result[1].EndpointIDs[10]) + require.True(t, result[2].EndpointIDs[10]) +} + +func TestFetchWorkflows_ReturnsOnlyGitopsStacks(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { + mustCreateGitWorkflow(t, tx, &portainer.Stack{ + ID: 1, + Name: "gitops-stack", + GitConfig: &gittypes.RepoConfig{URL: "https://github.com/x/repo"}, + }) + require.NoError(t, tx.Stack().Create(&portainer.Stack{ID: 2, Name: "plain-stack"})) + + return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) + })) + + items, err := FetchWorkflows(t.Context(), store, nil, nil, adminContext(), nil) + require.NoError(t, err) + require.Len(t, items, 1) + require.Equal(t, "gitops-stack", items[0].Name) +} + +func TestFetchWorkflows_FiltersByEndpointID(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { + for i := 1; i <= 3; i++ { + mustCreateGitWorkflow(t, tx, &portainer.Stack{ + ID: portainer.StackID(i), + Name: "stack-" + strconv.Itoa(i), + EndpointID: portainer.EndpointID(i), + GitConfig: &gittypes.RepoConfig{URL: "https://github.com/x/" + strconv.Itoa(i)}, + }) + } + + return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) + })) + + items, err := FetchWorkflows(t.Context(), store, nil, nil, adminContext(), set.ToSet([]portainer.EndpointID{1, 2})) + require.NoError(t, err) + require.Len(t, items, 2) + + names := []string{items[0].Name, items[1].Name} + require.Contains(t, names, "stack-1") + require.Contains(t, names, "stack-2") +} + +func TestFetchWorkflows_EmptyWhenNoGitopsStacks(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { + require.NoError(t, tx.Stack().Create(&portainer.Stack{ID: 1, Name: "plain-1"})) + require.NoError(t, tx.Stack().Create(&portainer.Stack{ID: 2, Name: "plain-2"})) + + return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) + })) + + items, err := FetchWorkflows(t.Context(), store, nil, nil, adminContext(), nil) + require.NoError(t, err) + require.Empty(t, items) +} + +func TestFetchWorkflows_NilEndpointSetReturnsAll(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { + for i := 1; i <= 3; i++ { + mustCreateGitWorkflow(t, tx, &portainer.Stack{ + ID: portainer.StackID(i), + Name: "stack-" + strconv.Itoa(i), + EndpointID: portainer.EndpointID(i), + GitConfig: &gittypes.RepoConfig{URL: "https://github.com/x/" + strconv.Itoa(i)}, + }) + } + + return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) + })) + + items, err := FetchWorkflows(t.Context(), store, nil, nil, adminContext(), nil) + require.NoError(t, err) + require.Len(t, items, 3) +} + +func TestFetchSourceStats_ReturnsAllSources(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { + require.NoError(t, tx.Source().Create(&portainer.Source{Name: "source-1", Type: portainer.SourceTypeGit})) + require.NoError(t, tx.Source().Create(&portainer.Source{Name: "source-2", Type: portainer.SourceTypeGit})) + + return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) + })) + + var sources []portainer.Source + require.NoError(t, store.ViewTx(func(tx dataservices.DataStoreTx) error { + var err error + sources, _, err = FetchSourceStats(tx, nil, adminContext()) + + return err + })) + + require.Len(t, sources, 2) +} + +func TestFetchSourceStats_TracksWorkflowCountAndEndpoints(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + var srcID portainer.SourceID + + require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { + src := &portainer.Source{Name: "shared", Type: portainer.SourceTypeGit} + require.NoError(t, tx.Source().Create(src)) + srcID = src.ID + + for i := 1; i <= 2; i++ { + wf := &portainer.Workflow{SourceIDs: []portainer.SourceID{srcID}} + require.NoError(t, tx.Workflow().Create(wf)) + require.NoError(t, tx.Stack().Create(&portainer.Stack{ + ID: portainer.StackID(i), + Name: "stack-" + strconv.Itoa(i), + EndpointID: portainer.EndpointID(i), + WorkflowID: wf.ID, + })) + } + + return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) + })) + + var stats map[portainer.SourceID]SourceStats + require.NoError(t, store.ViewTx(func(tx dataservices.DataStoreTx) error { + var err error + _, stats, err = FetchSourceStats(tx, nil, adminContext()) + + return err + })) + + st := stats[srcID] + require.Equal(t, 2, st.WorkflowCount) + require.Len(t, st.EndpointIDs, 2) +} + +func TestFetchSourceStats_UnusedSourceHasZeroStats(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + var unusedID portainer.SourceID + + require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { + src := &portainer.Source{Name: "unused", Type: portainer.SourceTypeGit} + require.NoError(t, tx.Source().Create(src)) + unusedID = src.ID + + return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) + })) + + var stats map[portainer.SourceID]SourceStats + require.NoError(t, store.ViewTx(func(tx dataservices.DataStoreTx) error { + var err error + _, stats, err = FetchSourceStats(tx, nil, adminContext()) + + return err + })) + + st := stats[unusedID] + require.Zero(t, st.WorkflowCount) + require.Empty(t, st.EndpointIDs) +} diff --git a/api/gitops/workflows/git_phases.go b/api/gitops/workflows/git_phases.go index 5e2a207dc..4d0e5c276 100644 --- a/api/gitops/workflows/git_phases.go +++ b/api/gitops/workflows/git_phases.go @@ -5,6 +5,9 @@ import ( "fmt" "path" "slices" + + portainer "github.com/portainer/portainer/api" + gittypes "github.com/portainer/portainer/api/git/types" ) // ListRefsFunc lists all git refs for a repository. @@ -19,6 +22,30 @@ type GitEntries struct { IsFile bool } +// ComputeGitPhasesForConfig computes source and artifact phases from a RepoConfig and a GitService. +func ComputeGitPhasesForConfig(ctx context.Context, gitSvc portainer.GitService, cfg *gittypes.RepoConfig) (source, artifact WorkflowPhaseStatus) { + if gitSvc == nil || cfg == nil { + return WorkflowPhaseStatus{Status: StatusUnknown}, WorkflowPhaseStatus{Status: StatusUnknown} + } + + username, password := gitCredentials(cfg) + return ComputeGitPhases(ctx, cfg.ReferenceName, []GitEntries{{Name: cfg.ConfigFilePath, IsFile: true}}, + func(ctx context.Context) ([]string, error) { + return gitSvc.ListRefs(ctx, cfg.URL, username, password, false, cfg.TLSSkipVerify) + }, + func(ctx context.Context, exts []string, dirOnly bool) ([]string, error) { + return gitSvc.ListFiles(ctx, cfg.URL, cfg.ReferenceName, username, password, dirOnly, false, exts, cfg.TLSSkipVerify) + }, + ) +} + +func gitCredentials(cfg *gittypes.RepoConfig) (username, password string) { + if cfg.Authentication != nil { + return cfg.Authentication.Username, cfg.Authentication.Password + } + return "", "" +} + // ComputeGitPhases checks source (ref reachability) and artifact (config file presence). // If source fails, artifact is returned as unknown without making a network call. func ComputeGitPhases(ctx context.Context, referenceName string, configFilePath []GitEntries, listRefs ListRefsFunc, listFiles ListFilesFunc) (source, artifact WorkflowPhaseStatus) { diff --git a/api/gitops/workflows/git_phases_test.go b/api/gitops/workflows/git_phases_test.go index 90a9cb067..956625cec 100644 --- a/api/gitops/workflows/git_phases_test.go +++ b/api/gitops/workflows/git_phases_test.go @@ -34,7 +34,7 @@ func TestComputeGitPhases(t *testing.T) { expectedArtifact Status }{ { - name: "listRefs errors → source error, artifact unknown", + name: "listRefs errors: source error, artifact unknown", referenceName: "refs/heads/main", configFilePath: []GitEntries{{Name: "docker-compose.yml", IsFile: true}}, listRefs: errRefs, @@ -43,7 +43,7 @@ func TestComputeGitPhases(t *testing.T) { expectedArtifact: StatusUnknown, }, { - name: "ref not in list → source error, artifact unknown", + name: "ref not in list: source error, artifact unknown", referenceName: "refs/heads/missing", configFilePath: []GitEntries{{Name: "docker-compose.yml", IsFile: true}}, listRefs: func(_ context.Context) ([]string, error) { @@ -54,7 +54,7 @@ func TestComputeGitPhases(t *testing.T) { expectedArtifact: StatusUnknown, }, { - name: "empty configFilePath → artifact error", + name: "empty configFilePath: artifact error", referenceName: "refs/heads/main", configFilePath: []GitEntries{}, listRefs: okRefs, @@ -63,7 +63,7 @@ func TestComputeGitPhases(t *testing.T) { expectedArtifact: StatusError, }, { - name: "listFiles errors → artifact error", + name: "listFiles errors: artifact error", referenceName: "refs/heads/main", configFilePath: []GitEntries{{Name: "docker-compose.yml", IsFile: true}}, listRefs: okRefs, @@ -72,7 +72,7 @@ func TestComputeGitPhases(t *testing.T) { expectedArtifact: StatusError, }, { - name: "file not in list → artifact error", + name: "file not in list: artifact error", referenceName: "refs/heads/main", configFilePath: []GitEntries{{Name: "docker-compose.yml", IsFile: true}}, listRefs: okRefs, @@ -92,7 +92,7 @@ func TestComputeGitPhases(t *testing.T) { expectedArtifact: StatusHealthy, }, { - name: "empty referenceName → source healthy (default HEAD)", + name: "empty referenceName: source healthy (default HEAD)", referenceName: "", configFilePath: []GitEntries{{Name: "docker-compose.yml", IsFile: true}}, listRefs: okRefs, diff --git a/api/gitops/workflows/mapping.go b/api/gitops/workflows/mapping.go index a9adaecfb..b67a0340d 100644 --- a/api/gitops/workflows/mapping.go +++ b/api/gitops/workflows/mapping.go @@ -27,7 +27,7 @@ func MapStackToWorkflow(s portainer.Stack, gitConfig *gittypes.RepoConfig, sourc Namespace: s.Namespace, }, CreationDate: s.CreationDate, - LastSyncDate: stackLastSyncDate(s), + LastSyncDate: StackLastSyncDate(s), } } @@ -60,7 +60,7 @@ func MapEdgeStackToWorkflow(es portainer.EdgeStack, gitConfig *gittypes.RepoConf } } -func stackLastSyncDate(s portainer.Stack) int64 { +func StackLastSyncDate(s portainer.Stack) int64 { for i := len(s.DeploymentStatus) - 1; i >= 0; i-- { if s.DeploymentStatus[i].Status == portainer.StackStatusActive { return s.DeploymentStatus[i].Time @@ -120,7 +120,6 @@ func resolveEdgeGroupEndpoints(groups []portainer.EdgeGroupID, groupEndpoints ma for _, gid := range groups { for _, epID := range groupEndpoints[gid] { seen.Add(epID) - } } return seen.Keys() diff --git a/api/gitops/workflows/mapping_test.go b/api/gitops/workflows/mapping_test.go index ae11bc490..a1c006e6c 100644 --- a/api/gitops/workflows/mapping_test.go +++ b/api/gitops/workflows/mapping_test.go @@ -4,7 +4,10 @@ import ( "testing" portainer "github.com/portainer/portainer/api" + gittypes "github.com/portainer/portainer/api/git/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestStackLastSyncDate(t *testing.T) { @@ -12,7 +15,7 @@ func TestStackLastSyncDate(t *testing.T) { t.Run("no deployment status", func(t *testing.T) { t.Parallel() - assert.Equal(t, int64(0), stackLastSyncDate(portainer.Stack{})) + assert.Equal(t, int64(0), StackLastSyncDate(portainer.Stack{})) }) t.Run("no active entry", func(t *testing.T) { @@ -20,7 +23,7 @@ func TestStackLastSyncDate(t *testing.T) { s := portainer.Stack{DeploymentStatus: []portainer.StackDeploymentStatus{ {Status: portainer.StackStatusDeploying, Time: 100}, }} - assert.Equal(t, int64(0), stackLastSyncDate(s)) + assert.Equal(t, int64(0), StackLastSyncDate(s)) }) t.Run("last entry is active", func(t *testing.T) { @@ -29,7 +32,7 @@ func TestStackLastSyncDate(t *testing.T) { {Status: portainer.StackStatusDeploying, Time: 50}, {Status: portainer.StackStatusActive, Time: 100}, }} - assert.Equal(t, int64(100), stackLastSyncDate(s)) + assert.Equal(t, int64(100), StackLastSyncDate(s)) }) t.Run("active followed by non-active returns the active time", func(t *testing.T) { @@ -38,7 +41,7 @@ func TestStackLastSyncDate(t *testing.T) { {Status: portainer.StackStatusActive, Time: 100}, {Status: portainer.StackStatusDeploying, Time: 200}, }} - assert.Equal(t, int64(100), stackLastSyncDate(s)) + assert.Equal(t, int64(100), StackLastSyncDate(s)) }) } @@ -147,3 +150,118 @@ func TestEdgeStackTargetStatuses(t *testing.T) { assert.Equal(t, StatusError, result[portainer.EdgeGroupID(20)]) }) } + +func TestMapEdgeStackToWorkflow_DockerPlatform(t *testing.T) { + t.Parallel() + + es := portainer.EdgeStack{ + ID: 1, + Name: "docker-edge", + DeploymentType: portainer.EdgeStackDeploymentCompose, + EdgeGroups: []portainer.EdgeGroupID{1}, + CreationDate: 1587399600, + } + cfg := &gittypes.RepoConfig{URL: "https://github.com/x/repo"} + + w := MapEdgeStackToWorkflow(es, cfg, nil, map[portainer.EdgeGroupID][]portainer.EndpointID{1: {10}}, WorkflowPhaseStatus{Status: StatusHealthy}, WorkflowPhaseStatus{Status: StatusHealthy}) + + require.Equal(t, int(es.ID), w.ID) + require.Equal(t, es.Name, w.Name) + require.Equal(t, TypeEdgeStack, w.Type) + require.Equal(t, DeploymentPlatformDockerStandalone, w.Platform) + require.Equal(t, es.CreationDate, w.CreationDate) + require.Equal(t, cfg, w.GitConfig) + require.Equal(t, []portainer.EdgeGroupID{1}, w.Target.EdgeGroupIDs) +} + +func TestMapEdgeStackToWorkflow_KubernetesPlatform(t *testing.T) { + t.Parallel() + + es := portainer.EdgeStack{ + ID: 2, + Name: "kube-edge", + DeploymentType: portainer.EdgeStackDeploymentKubernetes, + EdgeGroups: []portainer.EdgeGroupID{1}, + } + + w := MapEdgeStackToWorkflow(es, nil, nil, map[portainer.EdgeGroupID][]portainer.EndpointID{}, WorkflowPhaseStatus{Status: StatusUnknown}, WorkflowPhaseStatus{Status: StatusUnknown}) + + require.Equal(t, DeploymentPlatformKubernetes, w.Platform) +} + +func TestMapEdgeStackToWorkflow_GroupStatusesAndResolvedEndpoints(t *testing.T) { + t.Parallel() + + statuses := []portainer.EdgeStackStatusForEnv{ + {EndpointID: 10, Status: []portainer.EdgeStackDeploymentStatus{{Type: portainer.EdgeStackStatusRunning}}}, + {EndpointID: 20, Status: []portainer.EdgeStackDeploymentStatus{{Type: portainer.EdgeStackStatusError, Error: "boom"}}}, + } + groupEndpoints := map[portainer.EdgeGroupID][]portainer.EndpointID{ + 1: {10}, + 2: {20}, + } + es := portainer.EdgeStack{ + ID: 3, + Name: "multi-group", + EdgeGroups: []portainer.EdgeGroupID{1, 2}, + } + + w := MapEdgeStackToWorkflow(es, nil, statuses, groupEndpoints, WorkflowPhaseStatus{Status: StatusUnknown}, WorkflowPhaseStatus{Status: StatusUnknown}) + + require.Equal(t, StatusHealthy, w.Target.GroupStatus[1]) + require.Equal(t, StatusError, w.Target.GroupStatus[2]) + require.Len(t, w.Target.ResolvedEndpointIDs, 2) +} + +func TestPlatformFromStackType(t *testing.T) { + t.Parallel() + + require.Equal(t, DeploymentPlatformKubernetes, platformFromStackType(portainer.KubernetesStack)) + require.Equal(t, DeploymentPlatformDockerSwarm, platformFromStackType(portainer.DockerSwarmStack)) + require.Equal(t, DeploymentPlatformDockerStandalone, platformFromStackType(portainer.DockerComposeStack)) + require.Equal(t, DeploymentPlatformDockerStandalone, platformFromStackType(portainer.StackType(99))) +} + +func TestResolveEdgeGroupEndpoints_Empty(t *testing.T) { + t.Parallel() + + result := resolveEdgeGroupEndpoints(nil, map[portainer.EdgeGroupID][]portainer.EndpointID{}) + require.Empty(t, result) +} + +func TestResolveEdgeGroupEndpoints_DeduplicatesAcrossGroups(t *testing.T) { + t.Parallel() + + groupEndpoints := map[portainer.EdgeGroupID][]portainer.EndpointID{ + 1: {10, 20}, + 2: {20, 30}, + } + + result := resolveEdgeGroupEndpoints([]portainer.EdgeGroupID{1, 2}, groupEndpoints) + + require.Len(t, result, 3) +} + +func TestIsEdgeStackHealthyStatus(t *testing.T) { + t.Parallel() + + healthyTypes := []portainer.EdgeStackStatusType{ + portainer.EdgeStackStatusRunning, + portainer.EdgeStackStatusRolledBack, + portainer.EdgeStackStatusCompleted, + portainer.EdgeStackStatusRemoved, + portainer.EdgeStackStatusRemoteUpdateSuccess, + } + for _, typ := range healthyTypes { + require.True(t, isEdgeStackHealthyStatus(typ)) + } + + unhealthyTypes := []portainer.EdgeStackStatusType{ + portainer.EdgeStackStatusError, + portainer.EdgeStackStatusDeploying, + portainer.EdgeStackStatusPending, + } + for _, typ := range unhealthyTypes { + require.False(t, isEdgeStackHealthyStatus(typ)) + } +} diff --git a/api/gitops/workflows/status_test.go b/api/gitops/workflows/status_test.go index 1a87d65c5..c86917dbd 100644 --- a/api/gitops/workflows/status_test.go +++ b/api/gitops/workflows/status_test.go @@ -111,12 +111,12 @@ func TestDeriveEdgeStackTargetState(t *testing.T) { }{ {"empty", nil, StatusUnknown}, {"all per-env status slices empty", []portainer.EdgeStackStatusForEnv{{EndpointID: 1}}, StatusUnknown}, - {"running → healthy", []portainer.EdgeStackStatusForEnv{ep(1, portainer.EdgeStackStatusRunning)}, StatusHealthy}, - {"deploying → syncing", []portainer.EdgeStackStatusForEnv{ep(1, portainer.EdgeStackStatusDeploying)}, StatusSyncing}, - {"paused deploying → paused", []portainer.EdgeStackStatusForEnv{ep(1, portainer.EdgeStackStatusPausedDeploying)}, StatusPaused}, + {"running: healthy", []portainer.EdgeStackStatusForEnv{ep(1, portainer.EdgeStackStatusRunning)}, StatusHealthy}, + {"deploying: syncing", []portainer.EdgeStackStatusForEnv{ep(1, portainer.EdgeStackStatusDeploying)}, StatusSyncing}, + {"paused deploying: paused", []portainer.EdgeStackStatusForEnv{ep(1, portainer.EdgeStackStatusPausedDeploying)}, StatusPaused}, {"error short-circuits", []portainer.EdgeStackStatusForEnv{ep(1, portainer.EdgeStackStatusError)}, StatusError}, { - "error + running → error (short-circuit, order matters)", + "error + running gives error (short-circuit, order matters)", []portainer.EdgeStackStatusForEnv{ ep(1, portainer.EdgeStackStatusError), ep(2, portainer.EdgeStackStatusRunning), diff --git a/api/http/handler/gitops/sources/create_git.go b/api/http/handler/gitops/sources/create_git.go new file mode 100644 index 000000000..87bf3439a --- /dev/null +++ b/api/http/handler/gitops/sources/create_git.go @@ -0,0 +1,105 @@ +package sources + +import ( + "errors" + "net/http" + "strings" + + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/dataservices" + gittypes "github.com/portainer/portainer/api/git/types" + httperror "github.com/portainer/portainer/pkg/libhttp/error" + "github.com/portainer/portainer/pkg/libhttp/request" + "github.com/portainer/portainer/pkg/libhttp/response" +) + +// GitAuthenticationPayload holds authentication parameters for a git source +type GitAuthenticationPayload struct { + Username string `json:"username"` + Password string `json:"password"` + Provider gittypes.GitProvider `json:"provider"` + AuthorizationType gittypes.GitCredentialAuthType `json:"authorizationType"` +} + +// GitSourceCreatePayload holds the parameters for creating a git-backed source +type GitSourceCreatePayload struct { + Name string `json:"name"` + URL string `json:"url"` + ReferenceName string `json:"referenceName"` + TLSSkipVerify bool `json:"tlsSkipVerify"` + Authentication *GitAuthenticationPayload `json:"authentication"` + ClearAuthentication bool `json:"clearAuthentication"` +} + +// Validate implements the portainer.Validatable interface +func (payload *GitSourceCreatePayload) Validate(_ *http.Request) error { + if strings.TrimSpace(payload.URL) == "" { + return errors.New("url is required") + } + + return nil +} + +// BuildGitSource constructs a portainer.Source from a GitSourceCreatePayload +func BuildGitSource(payload GitSourceCreatePayload) *portainer.Source { + gitConfig := &gittypes.RepoConfig{ + URL: payload.URL, + ReferenceName: payload.ReferenceName, + TLSSkipVerify: payload.TLSSkipVerify, + } + + if payload.Authentication != nil { + gitConfig.Authentication = &gittypes.GitAuthentication{ + Username: payload.Authentication.Username, + Password: payload.Authentication.Password, + Provider: payload.Authentication.Provider, + AuthorizationType: payload.Authentication.AuthorizationType, + } + } + + name := payload.Name + if strings.TrimSpace(name) == "" { + name = gittypes.RepoName(payload.URL) + } + + return &portainer.Source{ + Name: name, + Type: portainer.SourceTypeGit, + GitConfig: gitConfig, + } +} + +// @id GitOpsSourcesCreateGit +// @summary Create a Git source +// @description Creates a new GitOps source backed by a Git repository. +// @description **Access policy**: admin +// @tags gitops +// @security ApiKeyAuth +// @security jwt +// @accept json +// @produce json +// @param body body GitSourceCreatePayload true "Git source details" +// @success 201 {object} portainer.Source +// @failure 400 "Invalid request payload" +// @failure 403 "Access denied" +// @failure 500 "Server error" +// @router /gitops/sources/git [post] +func (h *Handler) gitSourceCreate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { + var payload GitSourceCreatePayload + + if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil { + return httperror.BadRequest("Invalid request payload", err) + } + + src := BuildGitSource(payload) + + if err := h.dataStore.UpdateTx(func(tx dataservices.DataStoreTx) error { + return tx.Source().Create(src) + }); err != nil { + return httperror.InternalServerError("Unable to create source", err) + } + + src.GitConfig = gittypes.SanitizeRepoConfig(src.GitConfig) + + return response.JSONWithStatus(w, src, http.StatusCreated) +} diff --git a/api/http/handler/gitops/sources/create_git_test.go b/api/http/handler/gitops/sources/create_git_test.go new file mode 100644 index 000000000..b8999d962 --- /dev/null +++ b/api/http/handler/gitops/sources/create_git_test.go @@ -0,0 +1,166 @@ +package sources + +import ( + "net/http" + "net/http/httptest" + "testing" + + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/dataservices" + "github.com/portainer/portainer/api/datastore" + + "github.com/segmentio/encoding/json" + "github.com/stretchr/testify/require" +) + +func TestBuildGitSource_DerivesNameFromURL(t *testing.T) { + t.Parallel() + + src := BuildGitSource(GitSourceCreatePayload{ + URL: "https://github.com/org/my-repo.git", + }) + + require.Equal(t, "my-repo", src.Name) + require.Equal(t, portainer.SourceTypeGit, src.Type) + require.Nil(t, src.GitConfig.Authentication) +} + +func TestBuildGitSource_UsesExplicitName(t *testing.T) { + t.Parallel() + + src := BuildGitSource(GitSourceCreatePayload{ + Name: "custom-name", + URL: "https://github.com/org/repo.git", + }) + + require.Equal(t, "custom-name", src.Name) +} + +func TestBuildGitSource_WithAuthentication(t *testing.T) { + t.Parallel() + + src := BuildGitSource(GitSourceCreatePayload{ + URL: "https://github.com/org/repo.git", + Authentication: &GitAuthenticationPayload{ + Username: "alice", + Password: "secret", + }, + }) + + require.NotNil(t, src.GitConfig.Authentication) + require.Equal(t, "alice", src.GitConfig.Authentication.Username) + require.Equal(t, "secret", src.GitConfig.Authentication.Password) +} + +func TestGitSourceCreatePayload_Validate_EmptyURL(t *testing.T) { + t.Parallel() + + err := (&GitSourceCreatePayload{}).Validate(nil) + require.Error(t, err) +} + +func TestGitSourceCreatePayload_Validate_ValidURL(t *testing.T) { + t.Parallel() + + err := (&GitSourceCreatePayload{URL: "https://github.com/org/repo.git"}).Validate(nil) + require.NoError(t, err) +} + +func TestGitSourceCreate_Success(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { + return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) + })) + + h := newTestHandler(t, store) + + body, err := json.Marshal(GitSourceCreatePayload{ + URL: "https://github.com/org/repo.git", + Name: "my-source", + }) + require.NoError(t, err) + + rr := httptest.NewRecorder() + h.ServeHTTP(rr, buildCreateReq(t, 1, body)) + + require.Equal(t, http.StatusCreated, rr.Code) + + var src portainer.Source + err = json.NewDecoder(rr.Body).Decode(&src) + require.NoError(t, err) + require.Equal(t, "my-source", src.Name) + require.Equal(t, portainer.SourceTypeGit, src.Type) + require.NotZero(t, src.ID) + require.NotNil(t, src.GitConfig) + require.Equal(t, "https://github.com/org/repo.git", src.GitConfig.URL) +} + +func TestGitSourceCreate_SanitizesCredentials(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { + return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) + })) + + h := newTestHandler(t, store) + + body, err := json.Marshal(GitSourceCreatePayload{ + URL: "https://github.com/org/repo.git", + Authentication: &GitAuthenticationPayload{ + Username: "alice", + Password: "secret", + }, + }) + require.NoError(t, err) + + rr := httptest.NewRecorder() + h.ServeHTTP(rr, buildCreateReq(t, 1, body)) + + require.Equal(t, http.StatusCreated, rr.Code) + + var src portainer.Source + err = json.NewDecoder(rr.Body).Decode(&src) + require.NoError(t, err) + require.NotNil(t, src.GitConfig) + require.NotNil(t, src.GitConfig.Authentication) + require.Equal(t, "alice", src.GitConfig.Authentication.Username) + require.Empty(t, src.GitConfig.Authentication.Password) +} + +func TestGitSourceCreate_MissingURL(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { + return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) + })) + + h := newTestHandler(t, store) + + body, err := json.Marshal(GitSourceCreatePayload{Name: "no-url"}) + require.NoError(t, err) + + rr := httptest.NewRecorder() + h.ServeHTTP(rr, buildCreateReq(t, 1, body)) + + require.Equal(t, http.StatusBadRequest, rr.Code) +} + +func TestGitSourceCreate_MalformedJSON(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { + return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) + })) + + h := newTestHandler(t, store) + + rr := httptest.NewRecorder() + h.ServeHTTP(rr, buildCreateReq(t, 1, []byte("not-valid-json{"))) + + require.Equal(t, http.StatusBadRequest, rr.Code) +} diff --git a/api/http/handler/gitops/sources/delete.go b/api/http/handler/gitops/sources/delete.go new file mode 100644 index 000000000..1b2e53c64 --- /dev/null +++ b/api/http/handler/gitops/sources/delete.go @@ -0,0 +1,67 @@ +package sources + +import ( + "errors" + "net/http" + "slices" + + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/dataservices" + dserrors "github.com/portainer/portainer/api/dataservices/errors" + httperror "github.com/portainer/portainer/pkg/libhttp/error" + "github.com/portainer/portainer/pkg/libhttp/request" + "github.com/portainer/portainer/pkg/libhttp/response" +) + +var ErrSourceInUse = errors.New("source is used by one or more workflows") + +// @id GitOpsSourcesDelete +// @summary Delete a source +// @description Deletes an existing GitOps source. Returns 409 if the source is referenced by any workflow. +// @description **Access policy**: admin +// @tags gitops +// @security ApiKeyAuth +// @security jwt +// @param id path int true "Source identifier" +// @success 204 "Source deleted" +// @failure 400 "Invalid request" +// @failure 403 "Access denied" +// @failure 404 "Source not found" +// @failure 409 "Source is in use by one or more workflows" +// @failure 500 "Server error" +// @router /gitops/sources/{id} [delete] +func (h *Handler) sourceDelete(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { + sourceID, err := request.RetrieveNumericRouteVariableValue(r, "id") + if err != nil { + return httperror.BadRequest("Invalid source identifier route variable", err) + } + + if err := h.dataStore.UpdateTx(func(tx dataservices.DataStoreTx) error { + if exists, err := tx.Source().Exists(portainer.SourceID(sourceID)); err != nil { + return err + } else if !exists { + return dserrors.ErrObjectNotFound + } + + workflows, err := tx.Workflow().ReadAll() + if err != nil { + return err + } + + for _, wf := range workflows { + if slices.Contains(wf.SourceIDs, portainer.SourceID(sourceID)) { + return ErrSourceInUse + } + } + + return tx.Source().Delete(portainer.SourceID(sourceID)) + }); h.dataStore.IsErrObjectNotFound(err) { + return httperror.NotFound("Unable to find a source with the specified identifier", err) + } else if errors.Is(err, ErrSourceInUse) { + return httperror.Conflict("Source is used by one or more workflows", err) + } else if err != nil { + return httperror.InternalServerError("Unable to delete source", err) + } + + return response.Empty(w) +} diff --git a/api/http/handler/gitops/sources/delete_test.go b/api/http/handler/gitops/sources/delete_test.go new file mode 100644 index 000000000..5b3d012d1 --- /dev/null +++ b/api/http/handler/gitops/sources/delete_test.go @@ -0,0 +1,89 @@ +package sources + +import ( + "net/http" + "net/http/httptest" + "testing" + + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/dataservices" + "github.com/portainer/portainer/api/datastore" + + "github.com/stretchr/testify/require" +) + +func TestSourceDelete_Success(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + var srcID portainer.SourceID + require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { + src := &portainer.Source{Name: "to-delete", Type: portainer.SourceTypeGit} + err := tx.Source().Create(src) + require.NoError(t, err) + srcID = src.ID + + return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) + })) + + h := newTestHandler(t, store) + rr := httptest.NewRecorder() + h.ServeHTTP(rr, buildDeleteReq(t, 1, int(srcID))) + + require.Equal(t, http.StatusNoContent, rr.Code) +} + +func TestSourceDelete_NotFound(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { + return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) + })) + + h := newTestHandler(t, store) + rr := httptest.NewRecorder() + h.ServeHTTP(rr, buildDeleteReq(t, 1, 99)) + + require.Equal(t, http.StatusNotFound, rr.Code) +} + +func TestSourceDelete_InUse(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + var srcID portainer.SourceID + require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { + src := &portainer.Source{Name: "in-use", Type: portainer.SourceTypeGit} + err := tx.Source().Create(src) + require.NoError(t, err) + srcID = src.ID + + wf := &portainer.Workflow{SourceIDs: []portainer.SourceID{src.ID}} + err = tx.Workflow().Create(wf) + require.NoError(t, err) + + return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) + })) + + h := newTestHandler(t, store) + rr := httptest.NewRecorder() + h.ServeHTTP(rr, buildDeleteReq(t, 1, int(srcID))) + + require.Equal(t, http.StatusConflict, rr.Code) +} + +func TestSourceDelete_NonNumericID(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { + return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) + })) + + h := newTestHandler(t, store) + rr := httptest.NewRecorder() + h.ServeHTTP(rr, buildDeleteReqWithRawID(t, 1, "not-a-number")) + + require.Equal(t, http.StatusBadRequest, rr.Code) +} diff --git a/api/http/handler/gitops/sources/get_test.go b/api/http/handler/gitops/sources/get_test.go index 9a653ae7c..e0b48919e 100644 --- a/api/http/handler/gitops/sources/get_test.go +++ b/api/http/handler/gitops/sources/get_test.go @@ -40,7 +40,9 @@ func TestGetSource_ReturnsDetail(t *testing.T) { } require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { - require.NoError(t, tx.Stack().Create(&portainer.Stack{ID: 1, Name: "my-stack", GitConfig: cfg})) + stack := &portainer.Stack{ID: 1, Name: "my-stack"} + createGitWorkflow(t, tx, stack, cfg) + require.NoError(t, tx.Stack().Create(stack)) return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) })) @@ -71,7 +73,9 @@ func TestGetSource_RedactsCredentials(t *testing.T) { } require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { - require.NoError(t, tx.Stack().Create(&portainer.Stack{ID: 1, Name: "secure-stack", GitConfig: cfg})) + stack := &portainer.Stack{ID: 1, Name: "secure-stack"} + createGitWorkflow(t, tx, stack, cfg) + require.NoError(t, tx.Stack().Create(stack)) return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) })) @@ -94,12 +98,13 @@ func TestGetSource_AutoUpdate(t *testing.T) { cfg := gitCfg("https://github.com/org/polled") require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { - require.NoError(t, tx.Stack().Create(&portainer.Stack{ + stack := &portainer.Stack{ ID: 1, Name: "polled-stack", - GitConfig: cfg, AutoUpdate: &portainer.AutoUpdateSettings{Interval: "5m"}, - })) + } + createGitWorkflow(t, tx, stack, cfg) + require.NoError(t, tx.Stack().Create(stack)) return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) })) diff --git a/api/http/handler/gitops/sources/handler.go b/api/http/handler/gitops/sources/handler.go index 52512d927..650ad1e35 100644 --- a/api/http/handler/gitops/sources/handler.go +++ b/api/http/handler/gitops/sources/handler.go @@ -37,10 +37,16 @@ func NewHandler(bouncer security.BouncerService, dataStore dataservices.DataStor k8sFactory: k8sFactory, } + authenticatedRouter := h.PathPrefix("/gitops/sources").Subrouter() + authenticatedRouter.Use(bouncer.AuthenticatedAccess) + authenticatedRouter.Handle("", httperror.LoggerHandler(h.list)).Methods(http.MethodGet) + authenticatedRouter.Handle("/summary", httperror.LoggerHandler(h.summary)).Methods(http.MethodGet) + adminRouter := h.PathPrefix("/gitops/sources").Subrouter() adminRouter.Use(bouncer.AdminAccess) - adminRouter.Handle("", httperror.LoggerHandler(h.list)).Methods(http.MethodGet) - adminRouter.Handle("/summary", httperror.LoggerHandler(h.summary)).Methods(http.MethodGet) + adminRouter.Handle("/git", httperror.LoggerHandler(h.gitSourceCreate)).Methods(http.MethodPost) adminRouter.Handle("/{id}", httperror.LoggerHandler(h.getSource)).Methods(http.MethodGet) + adminRouter.Handle("/{id}", httperror.LoggerHandler(h.gitSourceUpdate)).Methods(http.MethodPut) + adminRouter.Handle("/{id}", httperror.LoggerHandler(h.sourceDelete)).Methods(http.MethodDelete) return h } diff --git a/api/http/handler/gitops/sources/helpers_test.go b/api/http/handler/gitops/sources/helpers_test.go index 44540a4ab..52fef2aa1 100644 --- a/api/http/handler/gitops/sources/helpers_test.go +++ b/api/http/handler/gitops/sources/helpers_test.go @@ -1,6 +1,8 @@ package sources import ( + "bytes" + "fmt" "net/http" "net/http/httptest" "testing" @@ -15,6 +17,26 @@ import ( "github.com/stretchr/testify/require" ) +// createGitWorkflow creates a Source and Workflow for the given config and +// wires them up by setting stack.WorkflowID before creating the stack. +func createGitWorkflow(t *testing.T, tx dataservices.DataStoreTx, stack *portainer.Stack, cfg *gittypes.RepoConfig) { + t.Helper() + + src := &portainer.Source{ + Name: gittypes.RepoName(cfg.URL), + Type: portainer.SourceTypeGit, + GitConfig: cfg, + } + require.NoError(t, tx.Source().Create(src)) + + wf := &portainer.Workflow{ + SourceIDs: []portainer.SourceID{src.ID}, + } + require.NoError(t, tx.Workflow().Create(wf)) + + stack.WorkflowID = wf.ID +} + func newTestHandler(t *testing.T, store dataservices.DataStore) *Handler { t.Helper() return NewHandler(testhelpers.NewTestRequestBouncer(), store, nil, nil) @@ -63,3 +85,66 @@ func gitCfg(url string) *gittypes.RepoConfig { ReferenceName: "refs/heads/main", } } + +func buildCreateReq(t *testing.T, userID portainer.UserID, body []byte) *http.Request { + t.Helper() + req := httptest.NewRequest(http.MethodPost, "/gitops/sources/git", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + req = req.WithContext(security.StoreTokenData(req, &portainer.TokenData{ID: userID})) + req = req.WithContext(security.StoreRestrictedRequestContext(req, &security.RestrictedRequestContext{ + UserID: userID, IsAdmin: true, + })) + return req +} + +func buildUpdateReq(t *testing.T, userID portainer.UserID, id int, body []byte) *http.Request { + t.Helper() + req := httptest.NewRequest(http.MethodPut, fmt.Sprintf("/gitops/sources/%d", id), bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + req = req.WithContext(security.StoreTokenData(req, &portainer.TokenData{ID: userID})) + req = req.WithContext(security.StoreRestrictedRequestContext(req, &security.RestrictedRequestContext{ + UserID: userID, IsAdmin: true, + })) + return req +} + +func buildDeleteReq(t *testing.T, userID portainer.UserID, id int) *http.Request { + t.Helper() + req := httptest.NewRequest(http.MethodDelete, fmt.Sprintf("/gitops/sources/%d", id), nil) + req = req.WithContext(security.StoreTokenData(req, &portainer.TokenData{ID: userID})) + req = req.WithContext(security.StoreRestrictedRequestContext(req, &security.RestrictedRequestContext{ + UserID: userID, IsAdmin: true, + })) + return req +} + +func buildSummaryReq(t *testing.T, userID portainer.UserID) *http.Request { + t.Helper() + req := httptest.NewRequest(http.MethodGet, "/gitops/sources/summary", nil) + req = req.WithContext(security.StoreTokenData(req, &portainer.TokenData{ID: userID})) + req = req.WithContext(security.StoreRestrictedRequestContext(req, &security.RestrictedRequestContext{ + UserID: userID, IsAdmin: true, + })) + return req +} + +func buildUpdateReqWithRawID(t *testing.T, userID portainer.UserID, id string, body []byte) *http.Request { + t.Helper() + req := httptest.NewRequest(http.MethodPut, "/gitops/sources/"+id, bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + req = req.WithContext(security.StoreTokenData(req, &portainer.TokenData{ID: userID})) + req = req.WithContext(security.StoreRestrictedRequestContext(req, &security.RestrictedRequestContext{ + UserID: userID, IsAdmin: true, + })) + return req +} + +func buildDeleteReqWithRawID(t *testing.T, userID portainer.UserID, id string) *http.Request { + t.Helper() + req := httptest.NewRequest(http.MethodDelete, "/gitops/sources/"+id, nil) + req = req.WithContext(security.StoreTokenData(req, &portainer.TokenData{ID: userID})) + req = req.WithContext(security.StoreRestrictedRequestContext(req, &security.RestrictedRequestContext{ + UserID: userID, IsAdmin: true, + })) + return req +} diff --git a/api/http/handler/gitops/sources/list.go b/api/http/handler/gitops/sources/list.go index 0cb95e78a..330475d68 100644 --- a/api/http/handler/gitops/sources/list.go +++ b/api/http/handler/gitops/sources/list.go @@ -7,7 +7,9 @@ import ( "strconv" "strings" - gocache "github.com/patrickmn/go-cache" + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/dataservices" + gittypes "github.com/portainer/portainer/api/git/types" ceWorkflows "github.com/portainer/portainer/api/gitops/workflows" "github.com/portainer/portainer/api/http/security" "github.com/portainer/portainer/api/http/utils/filters" @@ -15,12 +17,14 @@ import ( httperror "github.com/portainer/portainer/pkg/libhttp/error" "github.com/portainer/portainer/pkg/libhttp/request" "github.com/portainer/portainer/pkg/libhttp/response" + + gocache "github.com/patrickmn/go-cache" ) // @id GitOpsSourcesList // @summary List all GitOps sources // @description Returns a deduplicated list of git repositories used across all GitOps workflows. -// @description **Access policy**: admin +// @description **Access policy**: authenticated // @tags gitops // @security ApiKeyAuth // @security jwt @@ -107,16 +111,51 @@ func cacheKey(sc *security.RestrictedRequestContext) string { } func (h *Handler) fetchSources(ctx context.Context, sc *security.RestrictedRequestContext) ([]Source, error) { - workflows, err := ceWorkflows.FetchWorkflows(ctx, h.dataStore, h.gitService, h.k8sFactory, sc, nil) - if err != nil { + var allSrcs []portainer.Source + var stats map[portainer.SourceID]ceWorkflows.SourceStats + + if err := h.dataStore.ViewTx(func(tx dataservices.DataStoreTx) error { + var err error + allSrcs, stats, err = ceWorkflows.FetchSourceStats(tx, h.k8sFactory, sc) + return err + }); err != nil { return nil, err } - byID := workflowsBySourceID(workflows) + result := make([]Source, 0, len(allSrcs)) + for _, src := range allSrcs { + s, accessible := stats[src.ID] + if !accessible && !sc.IsAdmin { + continue + } - sources := make([]Source, 0, len(byID)) - for id, wfs := range byID { - sources = append(sources, buildSource(id, wfs[0].GitConfig.URL, wfs)) + var status ceWorkflows.Status + var sourceErr string + if src.GitConfig != nil { + phase, _ := ceWorkflows.ComputeGitPhasesForConfig(ctx, h.gitService, src.GitConfig) + status = phase.Status + sourceErr = phase.Error + } else { + status = ceWorkflows.StatusUnknown + } + + url := "" + if src.GitConfig != nil { + url = gittypes.SanitizeURL(src.GitConfig.URL) + } + + result = append(result, Source{ + ID: strconv.Itoa(int(src.ID)), + Name: src.Name, + Type: sourceTypeString(src.Type), + URL: url, + Status: status, + Error: sourceErr, + UsedBy: s.WorkflowCount, + Environments: len(s.EndpointIDs), + LastSync: s.LastSync, + }) } - return sources, nil + + return result, nil } diff --git a/api/http/handler/gitops/sources/list_test.go b/api/http/handler/gitops/sources/list_test.go index 88d2b9122..11fdfff16 100644 --- a/api/http/handler/gitops/sources/list_test.go +++ b/api/http/handler/gitops/sources/list_test.go @@ -18,12 +18,18 @@ func TestSourcesList_GroupsByURLAndCredentials(t *testing.T) { _, store := datastore.MustNewTestStore(t, false, true) require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { - require.NoError(t, tx.Stack().Create(&portainer.Stack{ - ID: 1, Name: "stack-a", GitConfig: gitCfg("https://github.com/org/repo"), - })) - require.NoError(t, tx.Stack().Create(&portainer.Stack{ - ID: 2, Name: "stack-b", GitConfig: gitCfg("https://github.com/org/repo"), - })) + cfg := gitCfg("https://github.com/org/repo") + src := &portainer.Source{Name: "repo", Type: portainer.SourceTypeGit, GitConfig: cfg} + require.NoError(t, tx.Source().Create(src)) + + wfA := &portainer.Workflow{SourceIDs: []portainer.SourceID{src.ID}} + require.NoError(t, tx.Workflow().Create(wfA)) + wfB := &portainer.Workflow{SourceIDs: []portainer.SourceID{src.ID}} + require.NoError(t, tx.Workflow().Create(wfB)) + + require.NoError(t, tx.Stack().Create(&portainer.Stack{ID: 1, Name: "stack-a", WorkflowID: wfA.ID})) + require.NoError(t, tx.Stack().Create(&portainer.Stack{ID: 2, Name: "stack-b", WorkflowID: wfB.ID})) + return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) })) @@ -45,8 +51,15 @@ func TestSourcesList_SeparatesCredentials(t *testing.T) { cfg1.Authentication = &gittypes.GitAuthentication{Username: "alice", Password: "pass1"} cfg2 := gitCfg("https://github.com/org/repo") cfg2.Authentication = &gittypes.GitAuthentication{Username: "bob", Password: "pass2"} - require.NoError(t, tx.Stack().Create(&portainer.Stack{ID: 1, Name: "stack-a", GitConfig: cfg1})) - require.NoError(t, tx.Stack().Create(&portainer.Stack{ID: 2, Name: "stack-b", GitConfig: cfg2})) + + stackA := &portainer.Stack{ID: 1, Name: "stack-a"} + createGitWorkflow(t, tx, stackA, cfg1) + require.NoError(t, tx.Stack().Create(stackA)) + + stackB := &portainer.Stack{ID: 2, Name: "stack-b"} + createGitWorkflow(t, tx, stackB, cfg2) + require.NoError(t, tx.Stack().Create(stackB)) + return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) })) @@ -64,9 +77,9 @@ func TestSourcesList_StatusFilter(t *testing.T) { // With nil gitService, source git-phase status is always StatusUnknown. require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { - require.NoError(t, tx.Stack().Create(&portainer.Stack{ - ID: 1, GitConfig: gitCfg("https://github.com/org/app"), - })) + stack := &portainer.Stack{ID: 1} + createGitWorkflow(t, tx, stack, gitCfg("https://github.com/org/app")) + require.NoError(t, tx.Stack().Create(stack)) return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) })) @@ -92,12 +105,14 @@ func TestSourcesList_SearchByURL(t *testing.T) { _, store := datastore.MustNewTestStore(t, false, true) require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { - require.NoError(t, tx.Stack().Create(&portainer.Stack{ - ID: 1, GitConfig: gitCfg("https://github.com/org/app"), - })) - require.NoError(t, tx.Stack().Create(&portainer.Stack{ - ID: 2, GitConfig: gitCfg("https://github.com/org/infra"), - })) + stackA := &portainer.Stack{ID: 1} + createGitWorkflow(t, tx, stackA, gitCfg("https://github.com/org/app")) + require.NoError(t, tx.Stack().Create(stackA)) + + stackB := &portainer.Stack{ID: 2} + createGitWorkflow(t, tx, stackB, gitCfg("https://github.com/org/infra")) + require.NoError(t, tx.Stack().Create(stackB)) + return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) })) diff --git a/api/http/handler/gitops/sources/summary.go b/api/http/handler/gitops/sources/summary.go index ef9f62a27..c9b00db7e 100644 --- a/api/http/handler/gitops/sources/summary.go +++ b/api/http/handler/gitops/sources/summary.go @@ -12,7 +12,7 @@ import ( // @id GitOpsSourcesSummary // @summary Summarize GitOps source status counts // @description Returns a count of sources per status. -// @description **Access policy**: admin +// @description **Access policy**: authenticated // @tags gitops // @security ApiKeyAuth // @security jwt @@ -27,10 +27,6 @@ func (h *Handler) summary(w http.ResponseWriter, r *http.Request) *httperror.Han return httperror.InternalServerError("Unable to retrieve info from request context", err) } - if !securityContext.IsAdmin { - return httperror.Forbidden("Access denied", nil) - } - key := cacheKey(securityContext) sources, err := h.getSources(r.Context(), key, securityContext) diff --git a/api/http/handler/gitops/sources/summary_test.go b/api/http/handler/gitops/sources/summary_test.go new file mode 100644 index 000000000..ba4088a61 --- /dev/null +++ b/api/http/handler/gitops/sources/summary_test.go @@ -0,0 +1,65 @@ +package sources + +import ( + "net/http" + "net/http/httptest" + "testing" + + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/dataservices" + "github.com/portainer/portainer/api/datastore" + ceWorkflows "github.com/portainer/portainer/api/gitops/workflows" + + "github.com/segmentio/encoding/json" + "github.com/stretchr/testify/require" +) + +func TestSourcesSummary_Empty(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { + return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) + })) + + h := newTestHandler(t, store) + rr := httptest.NewRecorder() + h.ServeHTTP(rr, buildSummaryReq(t, 1)) + + require.Equal(t, http.StatusOK, rr.Code) + + var summary ceWorkflows.StatusSummary + err := json.NewDecoder(rr.Body).Decode(&summary) + require.NoError(t, err) + require.Equal(t, ceWorkflows.StatusSummary{}, summary) +} + +func TestSourcesSummary_CountsByStatus(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + // With nil gitService and nil GitConfig, all sources get StatusUnknown. + require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { + for _, name := range []string{"source-a", "source-b", "source-c"} { + err := tx.Source().Create(&portainer.Source{Name: name, Type: portainer.SourceTypeGit}) + require.NoError(t, err) + } + + return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) + })) + + h := newTestHandler(t, store) + rr := httptest.NewRecorder() + h.ServeHTTP(rr, buildSummaryReq(t, 1)) + + require.Equal(t, http.StatusOK, rr.Code) + + var summary ceWorkflows.StatusSummary + err := json.NewDecoder(rr.Body).Decode(&summary) + require.NoError(t, err) + require.Equal(t, 3, summary.Unknown) + require.Zero(t, summary.Healthy) + require.Zero(t, summary.Error) + require.Zero(t, summary.Syncing) + require.Zero(t, summary.Paused) +} diff --git a/api/http/handler/gitops/sources/types.go b/api/http/handler/gitops/sources/types.go index 6e487093d..b39311e97 100644 --- a/api/http/handler/gitops/sources/types.go +++ b/api/http/handler/gitops/sources/types.go @@ -7,6 +7,7 @@ import ( "path" "strings" + portainer "github.com/portainer/portainer/api" gittypes "github.com/portainer/portainer/api/git/types" "github.com/portainer/portainer/api/gitops/workflows" ) @@ -41,6 +42,19 @@ func parseSourceType(s string) (string, error) { } } +func sourceTypeString(t portainer.SourceType) string { + switch t { + case portainer.SourceTypeGit: + return string(SourceTypeGit) + case portainer.SourceTypeHelm: + return string(SourceTypeHelm) + case portainer.SourceTypeRegistry: + return string(SourceTypeOCI) + default: + return string(SourceTypeGit) + } +} + type sourceGroupKey struct { URL string Username string @@ -53,6 +67,7 @@ func gitSourceKey(cfg *gittypes.RepoConfig) sourceGroupKey { key.Username = cfg.Authentication.Username key.Password = cfg.Authentication.Password } + return key } @@ -61,11 +76,8 @@ func sourceID(key sourceGroupKey) string { return hex.EncodeToString(h[:8]) } -// repoName extracts the repository name from a URL. -// e.g. "https://github.com/org/app-config.git" → "app-config" func repoName(rawURL string) string { - base := path.Base(rawURL) - return strings.TrimSuffix(base, ".git") + return strings.TrimSuffix(path.Base(rawURL), ".git") } func worstCaseStatus(statuses []workflows.Status) workflows.Status { @@ -82,5 +94,6 @@ func worstCaseStatus(statuses []workflows.Status) workflows.Status { worst = s } } + return worst } diff --git a/api/http/handler/gitops/sources/update_git.go b/api/http/handler/gitops/sources/update_git.go new file mode 100644 index 000000000..c59659a51 --- /dev/null +++ b/api/http/handler/gitops/sources/update_git.go @@ -0,0 +1,103 @@ +package sources + +import ( + "errors" + "net/http" + "strings" + + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/dataservices" + gittypes "github.com/portainer/portainer/api/git/types" + httperror "github.com/portainer/portainer/pkg/libhttp/error" + "github.com/portainer/portainer/pkg/libhttp/request" + "github.com/portainer/portainer/pkg/libhttp/response" +) + +var ErrNotGitSource = errors.New("source is not a Git source") + +// @id GitOpsSourcesUpdateGit +// @summary Update a Git source +// @description Updates an existing GitOps source backed by a Git repository. +// @description **Access policy**: admin +// @tags gitops +// @security ApiKeyAuth +// @security jwt +// @accept json +// @produce json +// @param id path int true "Source identifier" +// @param body body GitSourceCreatePayload true "Git source details" +// @success 200 {object} portainer.Source +// @failure 400 "Invalid request payload" +// @failure 403 "Access denied" +// @failure 404 "Source not found" +// @failure 500 "Server error" +// @router /gitops/sources/{id} [put] +func (h *Handler) gitSourceUpdate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { + sourceID, err := request.RetrieveNumericRouteVariableValue(r, "id") + if err != nil { + return httperror.BadRequest("Invalid source identifier route variable", err) + } + + var payload GitSourceCreatePayload + + if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil { + return httperror.BadRequest("Invalid request payload", err) + } + + var src *portainer.Source + + if err := h.dataStore.UpdateTx(func(tx dataservices.DataStoreTx) error { + var err error + + if src, err = tx.Source().Read(portainer.SourceID(sourceID)); err != nil { + return err + } + + if src.Type != portainer.SourceTypeGit { + return ErrNotGitSource + } + + name := payload.Name + if strings.TrimSpace(name) == "" { + name = gittypes.RepoName(payload.URL) + } + + src.Name = name + + var existingAuth *gittypes.GitAuthentication + if src.GitConfig != nil { + existingAuth = src.GitConfig.Authentication + } + + src.GitConfig = &gittypes.RepoConfig{ + URL: payload.URL, + ReferenceName: payload.ReferenceName, + TLSSkipVerify: payload.TLSSkipVerify, + } + + if payload.Authentication != nil { + src.GitConfig.Authentication = &gittypes.GitAuthentication{ + Username: payload.Authentication.Username, + Password: payload.Authentication.Password, + Provider: payload.Authentication.Provider, + AuthorizationType: payload.Authentication.AuthorizationType, + } + } else if payload.ClearAuthentication { + src.GitConfig.Authentication = nil + } else { + src.GitConfig.Authentication = existingAuth + } + + return tx.Source().Update(src.ID, src) + }); h.dataStore.IsErrObjectNotFound(err) { + return httperror.NotFound("Unable to find a source with the specified identifier", err) + } else if errors.Is(err, ErrNotGitSource) { + return httperror.BadRequest("Source is not a Git source", err) + } else if err != nil { + return httperror.InternalServerError("Unable to update source", err) + } + + src.GitConfig = gittypes.SanitizeRepoConfig(src.GitConfig) + + return response.JSON(w, src) +} diff --git a/api/http/handler/gitops/sources/update_git_test.go b/api/http/handler/gitops/sources/update_git_test.go new file mode 100644 index 000000000..d5ad1c4c2 --- /dev/null +++ b/api/http/handler/gitops/sources/update_git_test.go @@ -0,0 +1,264 @@ +package sources + +import ( + "net/http" + "net/http/httptest" + "testing" + + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/dataservices" + "github.com/portainer/portainer/api/datastore" + gittypes "github.com/portainer/portainer/api/git/types" + + "github.com/segmentio/encoding/json" + "github.com/stretchr/testify/require" +) + +func TestGitSourceUpdate_Success(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + var srcID portainer.SourceID + require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { + src := &portainer.Source{Name: "old-name", Type: portainer.SourceTypeGit} + err := tx.Source().Create(src) + require.NoError(t, err) + srcID = src.ID + + return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) + })) + + h := newTestHandler(t, store) + + body, err := json.Marshal(GitSourceCreatePayload{ + URL: "https://github.com/org/new.git", + Name: "new-name", + }) + require.NoError(t, err) + + rr := httptest.NewRecorder() + h.ServeHTTP(rr, buildUpdateReq(t, 1, int(srcID), body)) + + require.Equal(t, http.StatusOK, rr.Code) + + var src portainer.Source + err = json.NewDecoder(rr.Body).Decode(&src) + require.NoError(t, err) + require.Equal(t, "new-name", src.Name) + require.NotNil(t, src.GitConfig) + require.Equal(t, "https://github.com/org/new.git", src.GitConfig.URL) +} + +func TestGitSourceUpdate_DerivesNameFromURL(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + var srcID portainer.SourceID + require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { + src := &portainer.Source{Name: "old-name", Type: portainer.SourceTypeGit} + err := tx.Source().Create(src) + require.NoError(t, err) + srcID = src.ID + + return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) + })) + + h := newTestHandler(t, store) + + body, err := json.Marshal(GitSourceCreatePayload{ + URL: "https://github.com/org/my-project.git", + }) + require.NoError(t, err) + + rr := httptest.NewRecorder() + h.ServeHTTP(rr, buildUpdateReq(t, 1, int(srcID), body)) + + require.Equal(t, http.StatusOK, rr.Code) + + var src portainer.Source + err = json.NewDecoder(rr.Body).Decode(&src) + require.NoError(t, err) + require.Equal(t, "my-project", src.Name) +} + +func TestGitSourceUpdate_PreservesAuthWhenNotProvided(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + var srcID portainer.SourceID + require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { + src := &portainer.Source{ + Name: "auth-source", + Type: portainer.SourceTypeGit, + GitConfig: &gittypes.RepoConfig{ + URL: "https://github.com/org/repo.git", + Authentication: &gittypes.GitAuthentication{ + Username: "alice", + Password: "secret", + }, + }, + } + err := tx.Source().Create(src) + require.NoError(t, err) + srcID = src.ID + + return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) + })) + + h := newTestHandler(t, store) + + body, err := json.Marshal(GitSourceCreatePayload{ + URL: "https://github.com/org/repo.git", + Name: "renamed", + }) + require.NoError(t, err) + + rr := httptest.NewRecorder() + h.ServeHTTP(rr, buildUpdateReq(t, 1, int(srcID), body)) + + require.Equal(t, http.StatusOK, rr.Code) + + var stored *portainer.Source + require.NoError(t, store.ViewTx(func(tx dataservices.DataStoreTx) error { + var err error + stored, err = tx.Source().Read(srcID) + return err + })) + require.NotNil(t, stored.GitConfig) + require.NotNil(t, stored.GitConfig.Authentication) + require.Equal(t, "alice", stored.GitConfig.Authentication.Username) + require.Equal(t, "secret", stored.GitConfig.Authentication.Password) +} + +func TestGitSourceUpdate_ClearsAuthWhenRequested(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + var srcID portainer.SourceID + require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { + src := &portainer.Source{ + Name: "auth-source", + Type: portainer.SourceTypeGit, + GitConfig: &gittypes.RepoConfig{ + URL: "https://github.com/org/repo.git", + Authentication: &gittypes.GitAuthentication{ + Username: "alice", + Password: "secret", + }, + }, + } + err := tx.Source().Create(src) + require.NoError(t, err) + srcID = src.ID + + return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) + })) + + h := newTestHandler(t, store) + + body, err := json.Marshal(GitSourceCreatePayload{ + URL: "https://github.com/org/repo.git", + ClearAuthentication: true, + }) + require.NoError(t, err) + + rr := httptest.NewRecorder() + h.ServeHTTP(rr, buildUpdateReq(t, 1, int(srcID), body)) + + require.Equal(t, http.StatusOK, rr.Code) + + var stored *portainer.Source + require.NoError(t, store.ViewTx(func(tx dataservices.DataStoreTx) error { + var err error + stored, err = tx.Source().Read(srcID) + return err + })) + require.NotNil(t, stored.GitConfig) + require.Nil(t, stored.GitConfig.Authentication) +} + +func TestGitSourceUpdate_NotFound(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { + return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) + })) + + h := newTestHandler(t, store) + + body, err := json.Marshal(GitSourceCreatePayload{URL: "https://github.com/org/repo.git"}) + require.NoError(t, err) + + rr := httptest.NewRecorder() + h.ServeHTTP(rr, buildUpdateReq(t, 1, 99, body)) + + require.Equal(t, http.StatusNotFound, rr.Code) +} + +func TestGitSourceUpdate_NotGitSource(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + var srcID portainer.SourceID + require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { + src := &portainer.Source{Name: "helm-source", Type: portainer.SourceTypeHelm} + err := tx.Source().Create(src) + require.NoError(t, err) + srcID = src.ID + + return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) + })) + + h := newTestHandler(t, store) + + body, err := json.Marshal(GitSourceCreatePayload{URL: "https://github.com/org/repo.git"}) + require.NoError(t, err) + + rr := httptest.NewRecorder() + h.ServeHTTP(rr, buildUpdateReq(t, 1, int(srcID), body)) + + require.Equal(t, http.StatusBadRequest, rr.Code) +} + +func TestGitSourceUpdate_MalformedJSON(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + var srcID portainer.SourceID + require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { + src := &portainer.Source{Name: "src", Type: portainer.SourceTypeGit} + err := tx.Source().Create(src) + require.NoError(t, err) + srcID = src.ID + + return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) + })) + + h := newTestHandler(t, store) + + rr := httptest.NewRecorder() + h.ServeHTTP(rr, buildUpdateReq(t, 1, int(srcID), []byte("not-valid-json{"))) + + require.Equal(t, http.StatusBadRequest, rr.Code) +} + +func TestGitSourceUpdate_NonNumericID(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { + return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) + })) + + h := newTestHandler(t, store) + + body, err := json.Marshal(GitSourceCreatePayload{URL: "https://github.com/org/repo.git"}) + require.NoError(t, err) + + req := buildUpdateReqWithRawID(t, 1, "not-a-number", body) + rr := httptest.NewRecorder() + h.ServeHTTP(rr, req) + + require.Equal(t, http.StatusBadRequest, rr.Code) +} diff --git a/api/http/handler/gitops/sources/utils.go b/api/http/handler/gitops/sources/utils.go index cc7ab5cf1..5bf98daf0 100644 --- a/api/http/handler/gitops/sources/utils.go +++ b/api/http/handler/gitops/sources/utils.go @@ -28,9 +28,7 @@ func buildSource(id, url string, wfs []ce.Workflow) Source { if sourceError == "" && wf.Status.Source.Status == ce.StatusError { sourceError = wf.Status.Source.Error } - if wf.LastSyncDate > lastSync { - lastSync = wf.LastSyncDate - } + lastSync = max(lastSync, wf.LastSyncDate) if wf.Target.EndpointID != 0 { endpointIDs.Add(wf.Target.EndpointID) } diff --git a/api/http/handler/gitops/workflows/filter_test.go b/api/http/handler/gitops/workflows/filter_test.go index ccaca02a6..544ae9d70 100644 --- a/api/http/handler/gitops/workflows/filter_test.go +++ b/api/http/handler/gitops/workflows/filter_test.go @@ -28,10 +28,10 @@ func TestWorkflowsList_RBAC_NonAdminNoAccess(t *testing.T) { require.NoError(t, store.Endpoint().Create(&portainer.Endpoint{ID: 1, Name: "test-env"})) // Stack on endpoint 1 WITHOUT resource control — non-admin cannot see it - require.NoError(t, store.StackService.Create(&portainer.Stack{ + createGitStack(t, store, &portainer.Stack{ ID: 1, Name: "no-rc-stack", EndpointID: 1, GitConfig: gitConfig("https://github.com/x/no-rc"), - })) + }) h := NewHandler(store, nil, nil) rr := httptest.NewRecorder() @@ -56,10 +56,10 @@ func TestWorkflowsList_RBAC_NonAdminWithAccess(t *testing.T) { require.NoError(t, store.Endpoint().Create(&portainer.Endpoint{ID: 1, Name: "test-env"})) const stackName = "rc-stack" - require.NoError(t, store.StackService.Create(&portainer.Stack{ + createGitStack(t, store, &portainer.Stack{ ID: 1, Name: stackName, EndpointID: 1, GitConfig: gitConfig("https://github.com/x/rc"), - })) + }) require.NoError(t, store.ResourceControl().Create(&portainer.ResourceControl{ ID: 1, diff --git a/api/http/handler/gitops/workflows/helpers_test.go b/api/http/handler/gitops/workflows/helpers_test.go index 8d49a2b25..f9489a577 100644 --- a/api/http/handler/gitops/workflows/helpers_test.go +++ b/api/http/handler/gitops/workflows/helpers_test.go @@ -6,6 +6,7 @@ import ( "testing" portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/dataservices" gittypes "github.com/portainer/portainer/api/git/types" ce "github.com/portainer/portainer/api/gitops/workflows" "github.com/portainer/portainer/api/http/security" @@ -40,3 +41,20 @@ func decodeWorkflows(t *testing.T, rr *httptest.ResponseRecorder) []ce.Workflow func gitConfig(url string) *gittypes.RepoConfig { return &gittypes.RepoConfig{URL: url, ConfigFilePath: "docker-compose.yml"} } + +// createGitStack creates Source, Workflow, and Stack records with WorkflowID properly wired. +func createGitStack(t *testing.T, tx dataservices.DataStoreTx, stack *portainer.Stack) { + t.Helper() + + if stack.GitConfig != nil { + src := &portainer.Source{GitConfig: stack.GitConfig, Type: portainer.SourceTypeGit} + require.NoError(t, tx.Source().Create(src)) + + wf := &portainer.Workflow{SourceIDs: []portainer.SourceID{src.ID}} + require.NoError(t, tx.Workflow().Create(wf)) + + stack.WorkflowID = wf.ID + } + + require.NoError(t, tx.Stack().Create(stack)) +} diff --git a/api/http/handler/gitops/workflows/list.go b/api/http/handler/gitops/workflows/list.go index 18b09776a..4585e8552 100644 --- a/api/http/handler/gitops/workflows/list.go +++ b/api/http/handler/gitops/workflows/list.go @@ -128,16 +128,13 @@ func (h *Handler) getWorkflows(ctx context.Context, key string, sc *security.Res return slices.Clone(cached.([]svc.Workflow)), nil } - result, err := h.fetchWorkflows(ctx, sc, set.ToSet(endpointIDs)) + result, err := svc.FetchWorkflows(ctx, h.dataStore, h.gitService, h.k8sFactory, sc, set.ToSet(endpointIDs)) if err != nil { return nil, err } h.cache.Set(key, result, gocache.DefaultExpiration) - return slices.Clone(result), nil -} -func (h *Handler) fetchWorkflows(ctx context.Context, sc *security.RestrictedRequestContext, endpointIDSet set.Set[portainer.EndpointID]) ([]svc.Workflow, error) { - return svc.FetchWorkflows(ctx, h.dataStore, h.gitService, h.k8sFactory, sc, endpointIDSet) + return slices.Clone(result), nil } func cacheKey(sc *security.RestrictedRequestContext, endpointIDs []portainer.EndpointID) string { diff --git a/api/http/handler/gitops/workflows/list_test.go b/api/http/handler/gitops/workflows/list_test.go index 9ba5702b4..fd37aba33 100644 --- a/api/http/handler/gitops/workflows/list_test.go +++ b/api/http/handler/gitops/workflows/list_test.go @@ -23,10 +23,10 @@ func TestWorkflowsList_GitConfigFilter(t *testing.T) { _, store := datastore.MustNewTestStore(t, false, true) require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { - require.NoError(t, tx.Stack().Create(&portainer.Stack{ + createGitStack(t, tx, &portainer.Stack{ ID: 1, Name: "gitops-stack", GitConfig: gitConfig("https://github.com/example/repo"), - })) + }) require.NoError(t, tx.Stack().Create(&portainer.Stack{ID: 2, Name: "plain-stack"})) require.NoError(t, tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})) return nil @@ -50,12 +50,12 @@ func TestWorkflowsList_EndpointIDsFilter(t *testing.T) { require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { for i := 1; i <= 3; i++ { - require.NoError(t, tx.Stack().Create(&portainer.Stack{ + createGitStack(t, tx, &portainer.Stack{ ID: portainer.StackID(i), Name: fmt.Sprintf("env%d-stack", i), EndpointID: portainer.EndpointID(i), GitConfig: gitConfig(fmt.Sprintf("https://github.com/x/%d", i)), - })) + }) } require.NoError(t, tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})) return nil @@ -78,11 +78,11 @@ func TestWorkflowsList_Pagination(t *testing.T) { require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { for i := 1; i <= 5; i++ { - require.NoError(t, tx.Stack().Create(&portainer.Stack{ + createGitStack(t, tx, &portainer.Stack{ ID: portainer.StackID(i), Name: fmt.Sprintf("stack-%d", i), GitConfig: gitConfig("https://github.com/x/y"), - })) + }) } require.NoError(t, tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})) @@ -107,7 +107,7 @@ func TestWorkflowsList_Search(t *testing.T) { {ID: 1, Name: "alpha", GitConfig: gitConfig("https://github.com/org/alpha")}, {ID: 2, Name: "beta", GitConfig: gitConfig("https://github.com/org/beta")}, } { - require.NoError(t, tx.Stack().Create(s)) + createGitStack(t, tx, s) } require.NoError(t, tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})) @@ -128,14 +128,14 @@ func TestWorkflowsList_SearchByURL(t *testing.T) { _, store := datastore.MustNewTestStore(t, false, true) require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { - require.NoError(t, tx.Stack().Create(&portainer.Stack{ + createGitStack(t, tx, &portainer.Stack{ ID: 1, Name: "stack-org1", GitConfig: gitConfig("https://github.com/org1/repo"), - })) - require.NoError(t, tx.Stack().Create(&portainer.Stack{ + }) + createGitStack(t, tx, &portainer.Stack{ ID: 2, Name: "stack-org2", GitConfig: gitConfig("https://github.com/org2/repo"), - })) + }) require.NoError(t, tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})) return nil @@ -156,11 +156,11 @@ func TestWorkflowsList_Sort(t *testing.T) { require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { for i, name := range []string{"gamma", "alpha", "beta"} { - require.NoError(t, tx.Stack().Create(&portainer.Stack{ + createGitStack(t, tx, &portainer.Stack{ ID: portainer.StackID(i + 1), Name: name, GitConfig: gitConfig("https://github.com/x/" + name), - })) + }) } require.NoError(t, tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})) return nil @@ -188,10 +188,10 @@ func TestWorkflowsList_Cache(t *testing.T) { _, store := datastore.MustNewTestStore(t, false, true) require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { - require.NoError(t, tx.Stack().Create(&portainer.Stack{ + createGitStack(t, tx, &portainer.Stack{ ID: 1, Name: "initial-stack", GitConfig: gitConfig("https://github.com/x/initial"), - })) + }) require.NoError(t, tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})) return nil @@ -208,10 +208,10 @@ func TestWorkflowsList_Cache(t *testing.T) { require.Len(t, decodeWorkflows(t, rr), 1) // Mutate the store while cache is still warm. - require.NoError(t, store.StackService.Create(&portainer.Stack{ + createGitStack(t, store, &portainer.Stack{ ID: 2, Name: "new-stack", GitConfig: gitConfig("https://github.com/x/new"), - })) + }) // Second request — same cache key, should return stale cached result. rr = httptest.NewRecorder() @@ -235,13 +235,11 @@ func TestWorkflowsList_CacheImmutableAfterSort(t *testing.T) { for i, name := range []string{"alpha", "beta", "gamma"} { require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { - require.NoError(t, tx.Stack().Create( - &portainer.Stack{ - ID: portainer.StackID(i + 1), - Name: name, - GitConfig: gitConfig("https://github.com/x/" + name), - }, - )) + createGitStack(t, tx, &portainer.Stack{ + ID: portainer.StackID(i + 1), + Name: name, + GitConfig: gitConfig("https://github.com/x/" + name), + }) require.NoError(t, tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})) return nil @@ -278,14 +276,14 @@ func TestWorkflowsList_CacheSeparateKeys(t *testing.T) { _, store := datastore.MustNewTestStore(t, false, true) require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { - require.NoError(t, tx.Stack().Create(&portainer.Stack{ + createGitStack(t, tx, &portainer.Stack{ ID: 1, Name: "env1-stack", EndpointID: 1, GitConfig: gitConfig("https://github.com/x/1"), - })) - require.NoError(t, tx.Stack().Create(&portainer.Stack{ + }) + createGitStack(t, tx, &portainer.Stack{ ID: 2, Name: "env2-stack", EndpointID: 2, GitConfig: gitConfig("https://github.com/x/2"), - })) + }) require.NoError(t, tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})) return nil })) @@ -310,15 +308,15 @@ func TestWorkflowsList_StatusFilter(t *testing.T) { _, store := datastore.MustNewTestStore(t, false, true) require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { - require.NoError(t, tx.Stack().Create(&portainer.Stack{ + createGitStack(t, tx, &portainer.Stack{ ID: 1, Name: "healthy-stack", GitConfig: gitConfig("https://github.com/x/1"), - })) - require.NoError(t, tx.Stack().Create(&portainer.Stack{ + }) + createGitStack(t, tx, &portainer.Stack{ ID: 2, Name: "error-stack", GitConfig: gitConfig("https://github.com/x/2"), DeploymentStatus: []portainer.StackDeploymentStatus{{Status: portainer.StackStatusError}}, - })) + }) require.NoError(t, tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})) return nil })) @@ -366,9 +364,9 @@ func TestWorkflowsList_RedactsCredentials(t *testing.T) { cfg.Authentication = &gittypes.GitAuthentication{Username: "user", Password: "s3cr3t"} require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { - require.NoError(t, tx.Stack().Create(&portainer.Stack{ + createGitStack(t, tx, &portainer.Stack{ ID: 1, Name: "secure-stack", GitConfig: cfg, - })) + }) require.NoError(t, tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})) return nil })) diff --git a/api/http/handler/gitops/workflows/status_test.go b/api/http/handler/gitops/workflows/status_test.go index f1323cbe5..b169c41bb 100644 --- a/api/http/handler/gitops/workflows/status_test.go +++ b/api/http/handler/gitops/workflows/status_test.go @@ -22,26 +22,26 @@ func TestWorkflowsList_StackStatusDerivation(t *testing.T) { expectedStatus ce.Status }{ { - name: "no deployment status → healthy", + name: "no deployment status gives healthy", expectedStatus: ce.StatusHealthy, }, { - name: "active → healthy", + name: "active gives healthy", deployStatus: []portainer.StackDeploymentStatus{{Status: portainer.StackStatusActive}}, expectedStatus: ce.StatusHealthy, }, { - name: "error → error", + name: "error gives error", deployStatus: []portainer.StackDeploymentStatus{{Status: portainer.StackStatusError}}, expectedStatus: ce.StatusError, }, { - name: "deploying → syncing", + name: "deploying gives syncing", deployStatus: []portainer.StackDeploymentStatus{{Status: portainer.StackStatusDeploying}}, expectedStatus: ce.StatusSyncing, }, { - name: "inactive → paused", + name: "inactive gives paused", deployStatus: []portainer.StackDeploymentStatus{{Status: portainer.StackStatusInactive}}, expectedStatus: ce.StatusPaused, }, @@ -61,12 +61,12 @@ func TestWorkflowsList_StackStatusDerivation(t *testing.T) { _, store := datastore.MustNewTestStore(t, false, true) require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { - require.NoError(t, tx.Stack().Create(&portainer.Stack{ + createGitStack(t, tx, &portainer.Stack{ ID: 1, Name: "status-stack", DeploymentStatus: tc.deployStatus, GitConfig: gitConfig("https://github.com/x/y"), - })) + }) require.NoError(t, tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})) return nil diff --git a/api/http/handler/gitops/workflows/summary_test.go b/api/http/handler/gitops/workflows/summary_test.go new file mode 100644 index 000000000..24c1b4572 --- /dev/null +++ b/api/http/handler/gitops/workflows/summary_test.go @@ -0,0 +1,92 @@ +package workflows + +import ( + "net/http" + "net/http/httptest" + "testing" + + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/dataservices" + "github.com/portainer/portainer/api/datastore" + ce "github.com/portainer/portainer/api/gitops/workflows" + "github.com/portainer/portainer/api/http/security" + + "github.com/segmentio/encoding/json" + "github.com/stretchr/testify/require" +) + +func buildSummaryReq(t *testing.T, userID portainer.UserID, role portainer.UserRole) *http.Request { + t.Helper() + req := httptest.NewRequest(http.MethodGet, "/gitops/workflows/summary", nil) + ctx := security.StoreTokenData(req, &portainer.TokenData{ID: userID}) + req = req.WithContext(ctx) + ctx = security.StoreRestrictedRequestContext(req, &security.RestrictedRequestContext{ + UserID: userID, + IsAdmin: security.IsAdminRole(role), + }) + return req.WithContext(ctx) +} + +func decodeSummary(t *testing.T, rr *httptest.ResponseRecorder) ce.StatusSummary { + t.Helper() + require.Equal(t, http.StatusOK, rr.Code, "unexpected status: %s", rr.Body.String()) + var s ce.StatusSummary + require.NoError(t, json.NewDecoder(rr.Body).Decode(&s)) + return s +} + +func TestWorkflowsSummary_Empty(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { + return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) + })) + + h := NewHandler(store, nil, nil) + rr := httptest.NewRecorder() + h.ServeHTTP(rr, buildSummaryReq(t, 1, portainer.AdministratorRole)) + + s := decodeSummary(t, rr) + require.Equal(t, ce.StatusSummary{}, s) +} + +func TestWorkflowsSummary_CountsHealthyAndError(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { + // No deployment status, target healthy, effective status = healthy. + createGitStack(t, tx, &portainer.Stack{ + ID: 1, Name: "healthy-stack", + GitConfig: gitConfig("https://github.com/x/1"), + }) + + // Error deployment status, target error, effective status = error. + createGitStack(t, tx, &portainer.Stack{ + ID: 2, Name: "error-stack", + GitConfig: gitConfig("https://github.com/x/2"), + DeploymentStatus: []portainer.StackDeploymentStatus{{Status: portainer.StackStatusError}}, + }) + + // Deploying deployment status, target syncing, effective status = syncing. + createGitStack(t, tx, &portainer.Stack{ + ID: 3, Name: "syncing-stack", + GitConfig: gitConfig("https://github.com/x/3"), + DeploymentStatus: []portainer.StackDeploymentStatus{{Status: portainer.StackStatusDeploying}}, + }) + + return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) + })) + + h := NewHandler(store, nil, nil) + rr := httptest.NewRecorder() + h.ServeHTTP(rr, buildSummaryReq(t, 1, portainer.AdministratorRole)) + + s := decodeSummary(t, rr) + require.Equal(t, 1, s.Healthy) + require.Equal(t, 1, s.Error) + require.Equal(t, 1, s.Syncing) + require.Zero(t, s.Paused) + require.Zero(t, s.Unknown) +} diff --git a/api/http/handler/stacks/response.go b/api/http/handler/stacks/response.go new file mode 100644 index 000000000..129a9f2e3 --- /dev/null +++ b/api/http/handler/stacks/response.go @@ -0,0 +1,47 @@ +package stacks + +import ( + "fmt" + + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/dataservices" + gittypes "github.com/portainer/portainer/api/git/types" +) + +// loadGitConfigFromSource reads the GitConfig and SourceID from the first Git-type Source in the workflow. +func loadGitConfigFromSource(tx dataservices.DataStoreTx, workflowID portainer.WorkflowID) (*gittypes.RepoConfig, portainer.SourceID, error) { + src, err := dataservices.GitSourceForWorkflow(tx, workflowID) + if err != nil || src == nil { + return nil, 0, err + } + + return src.GitConfig, src.ID, nil +} + +// saveSourceGitConfig updates the GitConfig on the Source record identified by sourceID within a transaction. +func saveSourceGitConfig(tx dataservices.DataStoreTx, sourceID portainer.SourceID, gitConfig *gittypes.RepoConfig) error { + src, err := tx.Source().Read(sourceID) + if err != nil { + return fmt.Errorf("failed to read source: %w", err) + } + + src.GitConfig = gitConfig + + return tx.Source().Update(src.ID, src) +} + +// fillStackGitConfig loads GitConfig from Source and sets it on the stack for backwards-compatible responses. +func fillStackGitConfig(tx dataservices.DataStoreTx, stack *portainer.Stack) error { + if stack.WorkflowID == 0 { + return nil + } + + gitConfig, _, err := loadGitConfigFromSource(tx, stack.WorkflowID) + if err != nil { + return err + } + + stack.GitConfig = gittypes.SanitizeRepoConfig(gitConfig) + + return nil +} diff --git a/api/http/handler/stacks/stack_associate.go b/api/http/handler/stacks/stack_associate.go index 309e07635..3c7aba819 100644 --- a/api/http/handler/stacks/stack_associate.go +++ b/api/http/handler/stacks/stack_associate.go @@ -120,9 +120,8 @@ func (handler *Handler) stackAssociate(w http.ResponseWriter, r *http.Request) * stack.ResourceControl = resourceControl - if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" { - // sanitize password in the http response to minimise possible security leaks - stack.GitConfig.Authentication.Password = "" + if err := fillStackGitConfig(handler.DataStore, stack); err != nil { + return httperror.InternalServerError("Unable to load git config for stack", err) } return response.JSON(w, stack) diff --git a/api/http/handler/stacks/stack_create.go b/api/http/handler/stacks/stack_create.go index cb297f9d7..df1864c93 100644 --- a/api/http/handler/stacks/stack_create.go +++ b/api/http/handler/stacks/stack_create.go @@ -133,9 +133,8 @@ func (handler *Handler) decorateStackResponse(w http.ResponseWriter, stack *port stack.ResourceControl = resourceControl - if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" { - // sanitize password in the http response to minimise possible security leaks - stack.GitConfig.Authentication.Password = "" + if err := fillStackGitConfig(handler.DataStore, stack); err != nil { + return httperror.InternalServerError("Unable to load git config for stack", err) } return response.JSON(w, stack) diff --git a/api/http/handler/stacks/stack_delete.go b/api/http/handler/stacks/stack_delete.go index 3c34b8ce0..b9e14fbc5 100644 --- a/api/http/handler/stacks/stack_delete.go +++ b/api/http/handler/stacks/stack_delete.go @@ -8,6 +8,7 @@ import ( "strconv" portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/dataservices" "github.com/portainer/portainer/api/filesystem" httperrors "github.com/portainer/portainer/api/http/errors" "github.com/portainer/portainer/api/http/security" @@ -121,7 +122,14 @@ func (handler *Handler) stackDelete(w http.ResponseWriter, r *http.Request) *htt return httperror.InternalServerError(err.Error(), err) } - if err := handler.DataStore.Stack().Delete(portainer.StackID(id)); err != nil { + if err := handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error { + if stack.WorkflowID != 0 { + if err := tx.Workflow().Delete(stack.WorkflowID); err != nil { + return err + } + } + return tx.Stack().Delete(portainer.StackID(id)) + }); err != nil { return httperror.InternalServerError("Unable to remove the stack from the database", err) } @@ -333,9 +341,16 @@ func (handler *Handler) stackDeleteKubernetesByName(w http.ResponseWriter, r *ht continue } - if err := handler.DataStore.Stack().Delete(stack.ID); err != nil { + if err := handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error { + if stack.WorkflowID != 0 { + if err := tx.Workflow().Delete(stack.WorkflowID); err != nil { + return err + } + } + return tx.Stack().Delete(stack.ID) + }); err != nil { errs = errors.Join(errs, err) - log.Err(err).Msgf("Unable to remove the stack `%d` from the database", stack.ID) + log.Err(err).Int("stack_id", int(stack.ID)).Msg("unable to remove the stack from the database") continue } diff --git a/api/http/handler/stacks/stack_file.go b/api/http/handler/stacks/stack_file.go index 72dc70ee7..c43e7ae2e 100644 --- a/api/http/handler/stacks/stack_file.go +++ b/api/http/handler/stacks/stack_file.go @@ -4,6 +4,7 @@ import ( "net/http" portainer "github.com/portainer/portainer/api" + gittypes "github.com/portainer/portainer/api/git/types" httperrors "github.com/portainer/portainer/api/http/errors" "github.com/portainer/portainer/api/http/security" "github.com/portainer/portainer/api/stacks/stackutils" @@ -93,7 +94,16 @@ func (handler *Handler) stackFile(w http.ResponseWriter, r *http.Request) *httpe } } - if gitStackPendingRedeploy(stack) { + var gitConfig *gittypes.RepoConfig + if stack.WorkflowID != 0 { + var err error + gitConfig, _, err = loadGitConfigFromSource(handler.DataStore, stack.WorkflowID) + if err != nil { + return httperror.InternalServerError("Unable to load git config for stack", err) + } + } + + if gitStackPendingRedeploy(stack, gitConfig) { return httperror.Conflict("Stack git settings have changed. Redeploy the stack to apply the new configuration.", errors.New("git settings updated without redeploy")) } @@ -108,10 +118,11 @@ func (handler *Handler) stackFile(w http.ResponseWriter, r *http.Request) *httpe // gitStackPendingRedeploy returns true when the stack's git settings (URL or config file path) // have been updated via "save settings" but the stack has not yet been redeployed to apply them. // In that state the local clone is stale and the stack file cannot be read from disk. -func gitStackPendingRedeploy(stack *portainer.Stack) bool { - if stack.GitConfig == nil || stack.CurrentDeploymentInfo == nil { +func gitStackPendingRedeploy(stack *portainer.Stack, gitConfig *gittypes.RepoConfig) bool { + if gitConfig == nil || stack.CurrentDeploymentInfo == nil { return false } - return stack.GitConfig.URL != stack.CurrentDeploymentInfo.RepositoryURL || - stack.GitConfig.ConfigFilePath != stack.CurrentDeploymentInfo.ConfigFilePath + + return gitConfig.URL != stack.CurrentDeploymentInfo.RepositoryURL || + gitConfig.ConfigFilePath != stack.CurrentDeploymentInfo.ConfigFilePath } diff --git a/api/http/handler/stacks/stack_file_test.go b/api/http/handler/stacks/stack_file_test.go index 9c8b119fd..f88c15fd7 100644 --- a/api/http/handler/stacks/stack_file_test.go +++ b/api/http/handler/stacks/stack_file_test.go @@ -36,18 +36,27 @@ func TestStackFile_GitPendingRedeploy_Returns409(t *testing.T) { handler.FileService = fileService handler.DataStore = store + src := &portainer.Source{ + Type: portainer.SourceTypeGit, + GitConfig: &gittypes.RepoConfig{ + URL: "https://github.com/portainer/portainer.git", + ConfigFilePath: "docker-compose.yml", + }, + } + require.NoError(t, store.Source().Create(src)) + + wf := &portainer.Workflow{SourceIDs: []portainer.SourceID{src.ID}} + require.NoError(t, store.Workflow().Create(wf)) + stack := &portainer.Stack{ ID: 1, EndpointID: endpoint.ID, Type: portainer.DockerComposeStack, + WorkflowID: wf.ID, CurrentDeploymentInfo: &portainer.StackDeploymentInfo{ RepositoryURL: "https://github.com/portainer/old-repo.git", ConfigFilePath: "docker-compose.yml", }, - GitConfig: &gittypes.RepoConfig{ - URL: "https://github.com/portainer/portainer.git", - ConfigFilePath: "docker-compose.yml", - }, } require.NoError(t, store.Stack().Create(stack)) diff --git a/api/http/handler/stacks/stack_inspect.go b/api/http/handler/stacks/stack_inspect.go index 970c00347..4b33b3d38 100644 --- a/api/http/handler/stacks/stack_inspect.go +++ b/api/http/handler/stacks/stack_inspect.go @@ -91,9 +91,8 @@ func (handler *Handler) stackInspect(w http.ResponseWriter, r *http.Request) *ht } } - if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" { - // sanitize password in the http response to minimise possible security leaks - stack.GitConfig.Authentication.Password = "" + if err := fillStackGitConfig(handler.DataStore, stack); err != nil { + return httperror.InternalServerError("Unable to load git config for stack", err) } return response.JSON(w, stack) diff --git a/api/http/handler/stacks/stack_list.go b/api/http/handler/stacks/stack_list.go index 310d86ef3..592d2e43d 100644 --- a/api/http/handler/stacks/stack_list.go +++ b/api/http/handler/stacks/stack_list.go @@ -79,10 +79,9 @@ func (handler *Handler) stackList(w http.ResponseWriter, r *http.Request) *httpe stacks = authorization.FilterAuthorizedStacks(stacks, user.ID, userTeamIDs) } - for _, stack := range stacks { - if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" { - // sanitize password in the http response to minimise possible security leaks - stack.GitConfig.Authentication.Password = "" + for i := range stacks { + if err := fillStackGitConfig(handler.DataStore, &stacks[i]); err != nil { + return httperror.InternalServerError("Unable to load git config for stack", err) } } diff --git a/api/http/handler/stacks/stack_migrate.go b/api/http/handler/stacks/stack_migrate.go index 276b7e663..8d69e9e12 100644 --- a/api/http/handler/stacks/stack_migrate.go +++ b/api/http/handler/stacks/stack_migrate.go @@ -172,9 +172,8 @@ func (handler *Handler) stackMigrate(w http.ResponseWriter, r *http.Request) *ht } } - if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" { - // sanitize password in the http response to minimise possible security leaks - stack.GitConfig.Authentication.Password = "" + if err := fillStackGitConfig(handler.DataStore, stack); err != nil { + return httperror.InternalServerError("Unable to load git config for stack", err) } return response.JSON(w, stack) diff --git a/api/http/handler/stacks/stack_start.go b/api/http/handler/stacks/stack_start.go index fc94de4d4..1143e3140 100644 --- a/api/http/handler/stacks/stack_start.go +++ b/api/http/handler/stacks/stack_start.go @@ -162,9 +162,8 @@ func (handler *Handler) stackStart(w http.ResponseWriter, r *http.Request) *http return httperror.InternalServerError("Unable to update stack status", err) } - if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" { - // sanitize password in the http response to minimise possible security leaks - stack.GitConfig.Authentication.Password = "" + if err := fillStackGitConfig(handler.DataStore, stack); err != nil { + return httperror.InternalServerError("Unable to load git config for stack", err) } return response.JSON(w, stack) diff --git a/api/http/handler/stacks/stack_stop.go b/api/http/handler/stacks/stack_stop.go index 5df0f6d46..6f15e8d9c 100644 --- a/api/http/handler/stacks/stack_stop.go +++ b/api/http/handler/stacks/stack_stop.go @@ -127,9 +127,8 @@ func (handler *Handler) stackStop(w http.ResponseWriter, r *http.Request) *httpe return httperror.InternalServerError("Unable to update stack status", err) } - if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" { - // sanitize password in the http response to minimise possible security leaks - stack.GitConfig.Authentication.Password = "" + if err := fillStackGitConfig(handler.DataStore, stack); err != nil { + return httperror.InternalServerError("Unable to load git config for stack", err) } return response.JSON(w, stack) diff --git a/api/http/handler/stacks/stack_update.go b/api/http/handler/stacks/stack_update.go index 4156b4729..5f58062cd 100644 --- a/api/http/handler/stacks/stack_update.go +++ b/api/http/handler/stacks/stack_update.go @@ -188,9 +188,8 @@ func (handler *Handler) updateStackInTx(tx dataservices.DataStoreTx, r *http.Req deployGate.startDeploy() - if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" { - // Sanitize password in the http response to minimise possible security leaks - stack.GitConfig.Authentication.Password = "" + if err := fillStackGitConfig(tx, stack); err != nil { + return nil, httperror.InternalServerError("Unable to load git config for stack", err) } return stack, nil @@ -207,7 +206,7 @@ func (handler *Handler) updateAndDeployStack(tx dataservices.DataStoreTx, r *htt return handler.updateComposeStack(tx, r, stack, endpoint, gate) case portainer.KubernetesStack: - return handler.updateKubernetesStack(r, stack, endpoint, gate) + return handler.updateKubernetesStack(tx, r, stack, endpoint, gate) } return httperror.InternalServerError("Unsupported stack", errors.Errorf("unsupported stack type: %v", stack.Type)) @@ -219,7 +218,7 @@ func (handler *Handler) updateComposeStack(tx dataservices.DataStoreTx, r *http. deployments.StopAutoupdate(stack.ID, stack.AutoUpdate.JobID, handler.Scheduler) stack.AutoUpdate = nil } - if stack.GitConfig != nil { + if stack.WorkflowID != 0 { stack.FromAppTemplate = true } @@ -231,9 +230,12 @@ func (handler *Handler) updateComposeStack(tx dataservices.DataStoreTx, r *http. payload.RepullImageAndRedeploy = payload.RepullImageAndRedeploy || payload.PullImage stack.Env = payload.Env - if stack.GitConfig != nil { - // detach from git - stack.GitConfig = nil + if stack.WorkflowID != 0 { + oldWorkflowID := stack.WorkflowID + stack.WorkflowID = 0 + if err := tx.Workflow().Delete(oldWorkflowID); err != nil { + return httperror.InternalServerError("Unable to remove git workflow records from database", err) + } } stackFolder := strconv.Itoa(int(stack.ID)) @@ -299,7 +301,7 @@ func (handler *Handler) updateSwarmStack(tx dataservices.DataStoreTx, r *http.Re deployments.StopAutoupdate(stack.ID, stack.AutoUpdate.JobID, handler.Scheduler) stack.AutoUpdate = nil } - if stack.GitConfig != nil { + if stack.WorkflowID != 0 { stack.FromAppTemplate = true } @@ -310,9 +312,12 @@ func (handler *Handler) updateSwarmStack(tx dataservices.DataStoreTx, r *http.Re payload.RepullImageAndRedeploy = payload.RepullImageAndRedeploy || payload.PullImage stack.Env = payload.Env - if stack.GitConfig != nil { - // detach from git - stack.GitConfig = nil + if stack.WorkflowID != 0 { + oldWorkflowID := stack.WorkflowID + stack.WorkflowID = 0 + if err := tx.Workflow().Delete(oldWorkflowID); err != nil { + return httperror.InternalServerError("Unable to remove git workflow records from database", err) + } } stackFolder := strconv.Itoa(int(stack.ID)) diff --git a/api/http/handler/stacks/stack_update_git.go b/api/http/handler/stacks/stack_update_git.go index 8df53c616..c01e80704 100644 --- a/api/http/handler/stacks/stack_update_git.go +++ b/api/http/handler/stacks/stack_update_git.go @@ -73,11 +73,20 @@ func (handler *Handler) stackUpdateGit(w http.ResponseWriter, r *http.Request) * return httperror.NotFound("Unable to find a stack with the specified identifier inside the database", err) } else if err != nil { return httperror.InternalServerError("Unable to find a stack with the specified identifier inside the database", err) - } else if stack.GitConfig == nil { + } else if stack.WorkflowID == 0 { msg := "No Git config in the found stack" return httperror.InternalServerError(msg, errors.New(msg)) } + gitConfig, sourceID, err := loadGitConfigFromSource(handler.DataStore, stack.WorkflowID) + if err != nil { + return httperror.InternalServerError("Unable to load git config for stack", err) + } + if gitConfig == nil { + msg := "No Git config in the found stack source" + return httperror.InternalServerError(msg, errors.New(msg)) + } + if payload.AutoUpdate != nil && payload.AutoUpdate.Webhook != "" && (stack.AutoUpdate == nil || (stack.AutoUpdate != nil && stack.AutoUpdate.Webhook != payload.AutoUpdate.Webhook)) { @@ -143,24 +152,24 @@ func (handler *Handler) stackUpdateGit(w http.ResponseWriter, r *http.Request) * deployments.StopAutoupdate(stack.ID, stack.AutoUpdate.JobID, handler.Scheduler) } - if stack.CurrentDeploymentInfo == nil && stack.GitConfig != nil { + if stack.CurrentDeploymentInfo == nil { stack.CurrentDeploymentInfo = &portainer.StackDeploymentInfo{ - RepositoryURL: stack.GitConfig.URL, - ReferenceName: stack.GitConfig.ReferenceName, - ConfigFilePath: stack.GitConfig.ConfigFilePath, + RepositoryURL: gitConfig.URL, + ReferenceName: gitConfig.ReferenceName, + ConfigFilePath: gitConfig.ConfigFilePath, AdditionalFiles: stack.AdditionalFiles, - ConfigHash: stack.GitConfig.ConfigHash, + ConfigHash: gitConfig.ConfigHash, } } - //update retrieved stack data based on the payload - stack.GitConfig.ReferenceName = payload.RepositoryReferenceName - stack.GitConfig.TLSSkipVerify = payload.TLSSkipVerify + // Update gitConfig based on payload; the updated config is saved to Source (not stack.GitConfig). + gitConfig.ReferenceName = payload.RepositoryReferenceName + gitConfig.TLSSkipVerify = payload.TLSSkipVerify if payload.RepositoryURL != "" { - stack.GitConfig.URL = payload.RepositoryURL + gitConfig.URL = payload.RepositoryURL } if payload.ConfigFilePath != "" { - stack.GitConfig.ConfigFilePath = payload.ConfigFilePath + gitConfig.ConfigFilePath = payload.ConfigFilePath } if payload.AdditionalFiles != nil { stack.AdditionalFiles = payload.AdditionalFiles @@ -182,27 +191,27 @@ func (handler *Handler) stackUpdateGit(w http.ResponseWriter, r *http.Request) * // When the existing stack is using the custom username/password and the password is not updated, // the stack should keep using the saved username/password - if password == "" && stack.GitConfig != nil && stack.GitConfig.Authentication != nil { - password = stack.GitConfig.Authentication.Password + if password == "" && gitConfig.Authentication != nil { + password = gitConfig.Authentication.Password } - stack.GitConfig.Authentication = &gittypes.GitAuthentication{ + gitConfig.Authentication = &gittypes.GitAuthentication{ Username: payload.RepositoryUsername, Password: password, } if _, err := handler.GitService.LatestCommitID( context.TODO(), - stack.GitConfig.URL, - stack.GitConfig.ReferenceName, - stack.GitConfig.Authentication.Username, - stack.GitConfig.Authentication.Password, - stack.GitConfig.TLSSkipVerify, + gitConfig.URL, + gitConfig.ReferenceName, + gitConfig.Authentication.Username, + gitConfig.Authentication.Password, + gitConfig.TLSSkipVerify, ); err != nil { return httperror.InternalServerError("Unable to fetch git repository", err) } } else { - stack.GitConfig.Authentication = nil + gitConfig.Authentication = nil } if payload.AutoUpdate != nil && payload.AutoUpdate.Interval != "" { @@ -213,17 +222,18 @@ func (handler *Handler) stackUpdateGit(w http.ResponseWriter, r *http.Request) * } } - // Save the updated stack to DB + // Save the updated stack and git config to DB. if err := handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error { - return tx.Stack().Update(stack.ID, stack) + if err := tx.Stack().Update(stack.ID, stack); err != nil { + return err + } + if err := saveSourceGitConfig(tx, sourceID, gitConfig); err != nil { + return err + } + return fillStackGitConfig(tx, stack) }); err != nil { return httperror.InternalServerError("Unable to persist the stack changes inside the database", err) } - if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" { - // sanitize password in the http response to minimise possible security leaks - stack.GitConfig.Authentication.Password = "" - } - return response.JSON(w, stack) } diff --git a/api/http/handler/stacks/stack_update_git_redeploy.go b/api/http/handler/stacks/stack_update_git_redeploy.go index 6f34df75c..afe30b811 100644 --- a/api/http/handler/stacks/stack_update_git_redeploy.go +++ b/api/http/handler/stacks/stack_update_git_redeploy.go @@ -19,6 +19,7 @@ import ( "github.com/portainer/portainer/pkg/libhttp/response" "github.com/pkg/errors" + "github.com/rs/zerolog/log" ) type stackGitRedeployPayload struct { @@ -73,8 +74,16 @@ func (handler *Handler) stackGitRedeploy(w http.ResponseWriter, r *http.Request) return httperror.InternalServerError("Unable to find a stack with the specified identifier inside the database", err) } - if stack.GitConfig == nil { - return httperror.BadRequest("Stack is not created from git", err) + if stack.WorkflowID == 0 { + return httperror.BadRequest("Stack is not created from git", errors.New("stack has no git workflow")) + } + + gitConfig, sourceID, err := loadGitConfigFromSource(handler.DataStore, stack.WorkflowID) + if err != nil { + return httperror.InternalServerError("Unable to load git config for stack", err) + } + if gitConfig == nil { + return httperror.BadRequest("Stack is not created from git", errors.New("stack source has no git config")) } if stack.Status == portainer.StackStatusDeploying { @@ -135,7 +144,7 @@ func (handler *Handler) stackGitRedeploy(w http.ResponseWriter, r *http.Request) } payload.RepullImageAndRedeploy = payload.RepullImageAndRedeploy || payload.PullImage - stack.GitConfig.ReferenceName = cmp.Or(payload.RepositoryReferenceName, stack.GitConfig.ReferenceName) + gitConfig.ReferenceName = cmp.Or(payload.RepositoryReferenceName, gitConfig.ReferenceName) if payload.Env != nil { stack.Env = payload.Env @@ -161,19 +170,19 @@ func (handler *Handler) stackGitRedeploy(w http.ResponseWriter, r *http.Request) // When the existing stack is using the custom username/password and the password is not updated, // the stack should keep using the saved username/password - if repositoryPassword == "" && stack.GitConfig != nil && stack.GitConfig.Authentication != nil { - repositoryPassword = stack.GitConfig.Authentication.Password + if repositoryPassword == "" && gitConfig.Authentication != nil { + repositoryPassword = gitConfig.Authentication.Password } repositoryUsername = payload.RepositoryUsername } cloneOptions := git.CloneOptions{ ProjectPath: stack.ProjectPath, - URL: stack.GitConfig.URL, - ReferenceName: stack.GitConfig.ReferenceName, + URL: gitConfig.URL, + ReferenceName: gitConfig.ReferenceName, Username: repositoryUsername, Password: repositoryPassword, - TLSSkipVerify: stack.GitConfig.TLSSkipVerify, + TLSSkipVerify: gitConfig.TLSSkipVerify, } clean, err := git.CloneWithBackup(context.TODO(), handler.GitService, handler.FileService, cloneOptions) @@ -183,51 +192,82 @@ func (handler *Handler) stackGitRedeploy(w http.ResponseWriter, r *http.Request) defer clean() - deployGate := newDeployGate() - if err := handler.deployStack(r, stack, payload.RepullImageAndRedeploy, endpoint, deployGate); err != nil { - return err - } - - newHash, err := handler.GitService.LatestCommitID(context.TODO(), stack.GitConfig.URL, stack.GitConfig.ReferenceName, repositoryUsername, repositoryPassword, stack.GitConfig.TLSSkipVerify) + newHash, err := handler.GitService.LatestCommitID(context.TODO(), gitConfig.URL, gitConfig.ReferenceName, repositoryUsername, repositoryPassword, gitConfig.TLSSkipVerify) if err != nil { return httperror.InternalServerError("Unable get latest commit id", errors.WithMessagef(err, "failed to fetch latest commit id of the stack %v", stack.ID)) } - stack.GitConfig.ConfigHash = newHash + + oldConfigHash := gitConfig.ConfigHash + gitConfig.ConfigHash = newHash user, err := handler.DataStore.User().Read(securityContext.UserID) if err != nil { return httperror.BadRequest("Cannot find context user", errors.Wrap(err, "failed to fetch the user")) } stack.CurrentDeploymentInfo = &portainer.StackDeploymentInfo{ - RepositoryURL: stack.GitConfig.URL, - ReferenceName: stack.GitConfig.ReferenceName, - ConfigFilePath: stack.GitConfig.ConfigFilePath, + RepositoryURL: gitConfig.URL, + ReferenceName: gitConfig.ReferenceName, + ConfigFilePath: gitConfig.ConfigFilePath, AdditionalFiles: stack.AdditionalFiles, - ConfigHash: stack.GitConfig.ConfigHash, + ConfigHash: newHash, } stack.UpdatedBy = user.Username stack.UpdateDate = time.Now().Unix() stackutils.PrepareStackStatusForDeployment(stack) + postDeploy := func(ctx context.Context, deployErr error) { + if deployErr == nil { + return + } + + if err := handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error { + liveStack, err := tx.Stack().Read(stack.ID) + if err != nil { + return err + } + + if liveStack.CurrentDeploymentInfo != nil { + liveStack.CurrentDeploymentInfo.ConfigHash = oldConfigHash + } + + if err := tx.Stack().Update(liveStack.ID, liveStack); err != nil { + return err + } + + gitConfig.ConfigHash = oldConfigHash + + return saveSourceGitConfig(tx, sourceID, gitConfig) + }); err != nil { + log.Error().Err(err).Int("stack_id", int(stack.ID)).Msg("failed to revert config hash after failed redeploy") + } + } + + deployGate := newDeployGate() + if err := handler.deployStack(r, stack, payload.RepullImageAndRedeploy, endpoint, deployGate, postDeploy); err != nil { + return err + } + if err := handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error { - return tx.Stack().Update(stack.ID, stack) + if err := tx.Stack().Update(stack.ID, stack); err != nil { + return err + } + if err := saveSourceGitConfig(tx, sourceID, gitConfig); err != nil { + return err + } + return fillStackGitConfig(tx, stack) }); err != nil { deployGate.abortDeploy() + return httperror.InternalServerError("Unable to persist the stack changes inside the database", errors.Wrap(err, "failed to update the stack")) } deployGate.startDeploy() - if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" { - // Sanitize password in the http response to minimise possible security leaks - stack.GitConfig.Authentication.Password = "" - } - return response.JSON(w, stack) } -func (handler *Handler) deployStack(r *http.Request, stack *portainer.Stack, pullImage bool, endpoint *portainer.Endpoint, gate *deployGate) *httperror.HandlerError { +func (handler *Handler) deployStack(r *http.Request, stack *portainer.Stack, pullImage bool, endpoint *portainer.Endpoint, gate *deployGate, postDeploy postDeployFunc) *httperror.HandlerError { var deploymentConfiger deployments.StackDeploymentConfiger switch stack.Type { @@ -286,7 +326,7 @@ func (handler *Handler) deployStack(r *http.Request, stack *portainer.Stack, pul return httperror.InternalServerError("Unsupported stack", errors.Errorf("unsupported stack type: %v", stack.Type)) } - go stackDeploy(handler.DataStore, stack.ID, deploymentConfiger, gate, nil) + go stackDeploy(handler.DataStore, stack.ID, deploymentConfiger, gate, postDeploy) return nil } diff --git a/api/http/handler/stacks/stack_update_git_test.go b/api/http/handler/stacks/stack_update_git_test.go index 0d6bfc1b7..185bb18e9 100644 --- a/api/http/handler/stacks/stack_update_git_test.go +++ b/api/http/handler/stacks/stack_update_git_test.go @@ -33,15 +33,35 @@ func TestStackUpdateGitWebhookUniqueness(t *testing.T) { err = store.Endpoint().Create(endpoint) require.NoError(t, err) + src1 := &portainer.Source{ + Type: portainer.SourceTypeGit, + GitConfig: &gittypes.RepoConfig{URL: "https://github.com/portainer/portainer.git"}, + } + err = store.Source().Create(src1) + require.NoError(t, err) + + wf1 := &portainer.Workflow{SourceIDs: []portainer.SourceID{src1.ID}} + err = store.Workflow().Create(wf1) + require.NoError(t, err) + + src2 := &portainer.Source{ + Type: portainer.SourceTypeGit, + GitConfig: &gittypes.RepoConfig{URL: "https://github.com/portainer/portainer.git"}, + } + err = store.Source().Create(src2) + require.NoError(t, err) + + wf2 := &portainer.Workflow{SourceIDs: []portainer.SourceID{src2.ID}} + err = store.Workflow().Create(wf2) + require.NoError(t, err) + stack1 := portainer.Stack{ ID: 456, EndpointID: endpoint.ID, + WorkflowID: wf1.ID, AutoUpdate: &portainer.AutoUpdateSettings{ Webhook: webhook.String(), }, - GitConfig: &gittypes.RepoConfig{ - URL: "https://github.com/portainer/portainer.git", - }, } err = store.Stack().Create(&stack1) @@ -50,6 +70,7 @@ func TestStackUpdateGitWebhookUniqueness(t *testing.T) { stack2 := stack1 stack2.ID++ stack2.AutoUpdate = nil + stack2.WorkflowID = wf2.ID err = store.Stack().Create(&stack2) require.NoError(t, err) diff --git a/api/http/handler/stacks/update_kubernetes_stack.go b/api/http/handler/stacks/update_kubernetes_stack.go index 4b3c309ef..02cda0b46 100644 --- a/api/http/handler/stacks/update_kubernetes_stack.go +++ b/api/http/handler/stacks/update_kubernetes_stack.go @@ -7,6 +7,7 @@ import ( "strconv" portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/dataservices" "github.com/portainer/portainer/api/filesystem" gittypes "github.com/portainer/portainer/api/git/types" "github.com/portainer/portainer/api/git/update" @@ -52,8 +53,16 @@ func (payload *kubernetesGitStackUpdatePayload) Validate(r *http.Request) error return nil } -func (handler *Handler) updateKubernetesStack(r *http.Request, stack *portainer.Stack, endpoint *portainer.Endpoint, gate *deployGate) *httperror.HandlerError { - if stack.GitConfig != nil { +func (handler *Handler) updateKubernetesStack(tx dataservices.DataStoreTx, r *http.Request, stack *portainer.Stack, endpoint *portainer.Endpoint, gate *deployGate) *httperror.HandlerError { + if stack.WorkflowID != 0 { + gitConfig, sourceID, err := loadGitConfigFromSource(tx, stack.WorkflowID) + if err != nil { + return httperror.InternalServerError("Unable to load git config for stack", err) + } + if gitConfig == nil { + return httperror.InternalServerError("Stack has no git config in source", errors.New("source has no git config")) + } + // Stop the autoupdate job if there is any if stack.AutoUpdate != nil { deployments.StopAutoupdate(stack.ID, stack.AutoUpdate.JobID, handler.Scheduler) @@ -65,32 +74,33 @@ func (handler *Handler) updateKubernetesStack(r *http.Request, stack *portainer. return httperror.BadRequest("Invalid request payload", err) } - stack.GitConfig.ReferenceName = payload.RepositoryReferenceName - stack.GitConfig.TLSSkipVerify = payload.TLSSkipVerify - stack.GitConfig.Authentication = nil + gitConfig.ReferenceName = payload.RepositoryReferenceName + gitConfig.TLSSkipVerify = payload.TLSSkipVerify stack.AutoUpdate = payload.AutoUpdate if payload.RepositoryAuthentication { password := payload.RepositoryPassword - if password == "" && stack.GitConfig != nil && stack.GitConfig.Authentication != nil { - password = stack.GitConfig.Authentication.Password + if password == "" && gitConfig.Authentication != nil { + password = gitConfig.Authentication.Password } - stack.GitConfig.Authentication = &gittypes.GitAuthentication{ + gitConfig.Authentication = &gittypes.GitAuthentication{ Username: payload.RepositoryUsername, Password: password, } if _, err := handler.GitService.LatestCommitID( context.TODO(), - stack.GitConfig.URL, - stack.GitConfig.ReferenceName, - stack.GitConfig.Authentication.Username, - stack.GitConfig.Authentication.Password, - stack.GitConfig.TLSSkipVerify, + gitConfig.URL, + gitConfig.ReferenceName, + gitConfig.Authentication.Username, + gitConfig.Authentication.Password, + gitConfig.TLSSkipVerify, ); err != nil { return httperror.InternalServerError("Unable to fetch git repository", err) } + } else { + gitConfig.Authentication = nil } if payload.AutoUpdate != nil && payload.AutoUpdate.Interval != "" { @@ -101,6 +111,10 @@ func (handler *Handler) updateKubernetesStack(r *http.Request, stack *portainer. stack.AutoUpdate.JobID = jobID } + if err := saveSourceGitConfig(tx, sourceID, gitConfig); err != nil { + return httperror.InternalServerError("Unable to update source git config", err) + } + return nil } diff --git a/api/internal/testhelpers/datastore.go b/api/internal/testhelpers/datastore.go index 024ae4697..3a28baf30 100644 --- a/api/internal/testhelpers/datastore.go +++ b/api/internal/testhelpers/datastore.go @@ -24,6 +24,7 @@ type testDatastore struct { helmUserRepository dataservices.HelmUserRepositoryService registry dataservices.RegistryService resourceControl dataservices.ResourceControlService + source dataservices.SourceService apiKeyRepositoryService dataservices.APIKeyRepository role dataservices.RoleService sslSettings dataservices.SSLSettingsService @@ -38,6 +39,7 @@ type testDatastore struct { version dataservices.VersionService webhook dataservices.WebhookService pendingActionsService dataservices.PendingActionsService + workflow dataservices.WorkflowService connection portainer.Connection } @@ -72,7 +74,8 @@ func (d *testDatastore) Registry() dataservices.RegistryService { return d.regis func (d *testDatastore) ResourceControl() dataservices.ResourceControlService { return d.resourceControl } -func (d *testDatastore) Role() dataservices.RoleService { return d.role } +func (d *testDatastore) Source() dataservices.SourceService { return d.source } +func (d *testDatastore) Role() dataservices.RoleService { return d.role } func (d *testDatastore) APIKeyRepository() dataservices.APIKeyRepository { return d.apiKeyRepositoryService } @@ -87,6 +90,7 @@ func (d *testDatastore) TunnelServer() dataservices.TunnelServerService { re func (d *testDatastore) User() dataservices.UserService { return d.user } func (d *testDatastore) Version() dataservices.VersionService { return d.version } func (d *testDatastore) Webhook() dataservices.WebhookService { return d.webhook } +func (d *testDatastore) Workflow() dataservices.WorkflowService { return d.workflow } func (d *testDatastore) PendingActions() dataservices.PendingActionsService { return d.pendingActionsService diff --git a/api/portainer.go b/api/portainer.go index 19704a3e5..f18b1b468 100644 --- a/api/portainer.go +++ b/api/portainer.go @@ -1210,8 +1210,12 @@ type ( AutoUpdate *AutoUpdateSettings `json:"AutoUpdate"` // The stack deployment option Option *StackOption `json:"Option"` - // The git config of this stack - GitConfig *gittypes.RepoConfig + // GitConfig is the git repository configuration for git-backed stacks. + // Deprecated: loaded from Source via WorkflowID; kept for DB backwards-compatibility only. + // Non-migration code must not read or write this field; use Source records instead. + GitConfig *gittypes.RepoConfig `json:"GitConfig"` + // WorkflowID is the ID of the Workflow that owns the Source for this stack. + WorkflowID WorkflowID `json:"WorkflowID,omitempty"` // CurrentDeploymentInfo records the git repository state at the time of the last actual deployment. CurrentDeploymentInfo *StackDeploymentInfo `json:"CurrentDeploymentInfo,omitempty"` // Whether the stack is from a app template @@ -1240,6 +1244,23 @@ type ( // StackType represents the type of the stack (compose v2, stack deploy v3) StackType int + // Source represents a GitOps source that can be referenced by stacks or deployments. + Source struct { + ID SourceID `json:"id" example:"1"` + Name string `json:"name" example:"my-source"` + LastSync int64 `json:"lastSync,omitempty" example:"1587399600"` + Type SourceType `json:"type" example:"1"` + GitConfig *gittypes.RepoConfig `json:"gitConfig,omitempty"` + Registry *Registry `json:"registry,omitempty"` + HelmConfig *HelmConfig `json:"helmConfig,omitempty"` + } + + // SourceID represents a source identifier + SourceID int + + // SourceType represents the type of a source + SourceType int + // Status represents the application status Status struct { // Portainer API version @@ -1519,6 +1540,14 @@ type ( // WebhookType represents the type of resource a webhook is related to WebhookType int + // Workflow represents a GitOps workflow + Workflow struct { + ID WorkflowID `json:"id" example:"1"` + SourceIDs []SourceID `json:"sourceIds"` + } + + WorkflowID int + Snapshot struct { EndpointID EndpointID `json:"EndpointId"` Docker *DockerSnapshot `json:"Docker"` @@ -2158,6 +2187,13 @@ const ( PortainerEE ) +const ( + _ SourceType = iota + SourceTypeGit + SourceTypeRegistry + SourceTypeHelm +) + const ( _ RegistryType = iota // QuayRegistry represents a Quay.io registry diff --git a/api/stacks/deployments/compose_unpacker_cmd_builder.go b/api/stacks/deployments/compose_unpacker_cmd_builder.go index e7b7e6222..f6b889d5a 100644 --- a/api/stacks/deployments/compose_unpacker_cmd_builder.go +++ b/api/stacks/deployments/compose_unpacker_cmd_builder.go @@ -5,6 +5,7 @@ import ( portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/dataservices" + gittypes "github.com/portainer/portainer/api/git/types" "github.com/portainer/portainer/api/internal/registryutils" ) @@ -34,6 +35,7 @@ type unpackerCmdBuilderOptions struct { forceRecreate bool composeDestination string registries []portainer.Registry + gitConfig *gittypes.RepoConfig } type buildCmdFunc func(stack *portainer.Stack, opts unpackerCmdBuilderOptions, registries []string, env []string) []string @@ -65,8 +67,8 @@ func (d *stackDeployer) buildUnpackerCmdForStack(stack *portainer.Stack, operati // deploy [-u username -p password] [--skip-tls-verify] [--force-recreate] [-r] [-k] [--env KEY1=VALUE1 --env KEY2=VALUE2] [...] func buildDeployCmd(stack *portainer.Stack, opts unpackerCmdBuilderOptions, registries []string, env []string) []string { cmd := []string{UnpackerCmdDeploy} - cmd = appendGitAuthIfNeeded(cmd, stack) - cmd = appendSkipTLSVerifyIfNeeded(cmd, stack) + cmd = appendGitAuthIfNeeded(cmd, opts.gitConfig) + cmd = appendSkipTLSVerifyIfNeeded(cmd, opts.gitConfig) cmd = appendForceRecreateIfNeeded(cmd, opts.forceRecreate) if opts.prune { @@ -76,8 +78,8 @@ func buildDeployCmd(stack *portainer.Stack, opts unpackerCmdBuilderOptions, regi cmd = append(cmd, env...) cmd = append(cmd, registries...) cmd = append(cmd, - stack.GitConfig.URL, - stack.GitConfig.ReferenceName, + opts.gitConfig.URL, + opts.gitConfig.ReferenceName, stack.Name, opts.composeDestination, stack.EntryPoint, @@ -89,8 +91,8 @@ func buildDeployCmd(stack *portainer.Stack, opts unpackerCmdBuilderOptions, regi // undeploy [-u username -p password] [-k] [...] func buildUndeployCmd(stack *portainer.Stack, opts unpackerCmdBuilderOptions, registries []string, env []string) []string { cmd := []string{UnpackerCmdUndeploy} - cmd = appendGitAuthIfNeeded(cmd, stack) - cmd = append(cmd, stack.GitConfig.URL, + cmd = appendGitAuthIfNeeded(cmd, opts.gitConfig) + cmd = append(cmd, opts.gitConfig.URL, stack.Name, opts.composeDestination, stack.EntryPoint, @@ -102,13 +104,13 @@ func buildUndeployCmd(stack *portainer.Stack, opts unpackerCmdBuilderOptions, re // deploy [-u username -p password] [--skip-tls-verify] [-k] [--env KEY1=VALUE1 --env KEY2=VALUE2] [...] func buildComposeStartCmd(stack *portainer.Stack, opts unpackerCmdBuilderOptions, registries []string, env []string) []string { cmd := []string{UnpackerCmdDeploy} - cmd = appendGitAuthIfNeeded(cmd, stack) - cmd = appendSkipTLSVerifyIfNeeded(cmd, stack) + cmd = appendGitAuthIfNeeded(cmd, opts.gitConfig) + cmd = appendSkipTLSVerifyIfNeeded(cmd, opts.gitConfig) cmd = append(cmd, "-k") cmd = append(cmd, env...) cmd = append(cmd, registries...) - cmd = append(cmd, stack.GitConfig.URL, - stack.GitConfig.ReferenceName, + cmd = append(cmd, opts.gitConfig.URL, + opts.gitConfig.ReferenceName, stack.Name, opts.composeDestination, stack.EntryPoint, @@ -120,10 +122,10 @@ func buildComposeStartCmd(stack *portainer.Stack, opts unpackerCmdBuilderOptions // undeploy [-u username -p password] [-k] [...] func buildComposeStopCmd(stack *portainer.Stack, opts unpackerCmdBuilderOptions, registries []string, env []string) []string { cmd := []string{UnpackerCmdUndeploy} - cmd = appendGitAuthIfNeeded(cmd, stack) + cmd = appendGitAuthIfNeeded(cmd, opts.gitConfig) cmd = append(cmd, "-k", - stack.GitConfig.URL, + opts.gitConfig.URL, stack.Name, opts.composeDestination, stack.EntryPoint, @@ -135,8 +137,8 @@ func buildComposeStopCmd(stack *portainer.Stack, opts unpackerCmdBuilderOptions, // swarm-deploy [-u username -p password] [--skip-tls-verify] [--force-recreate] [-f] [-r] [-k] [--env KEY1=VALUE1 --env KEY2=VALUE2] [...] func buildSwarmDeployCmd(stack *portainer.Stack, opts unpackerCmdBuilderOptions, registries []string, env []string) []string { cmd := []string{UnpackerCmdSwarmDeploy} - cmd = appendGitAuthIfNeeded(cmd, stack) - cmd = appendSkipTLSVerifyIfNeeded(cmd, stack) + cmd = appendGitAuthIfNeeded(cmd, opts.gitConfig) + cmd = appendSkipTLSVerifyIfNeeded(cmd, opts.gitConfig) cmd = appendForceRecreateIfNeeded(cmd, opts.forceRecreate) if opts.pullImage { cmd = append(cmd, "-f") @@ -148,8 +150,8 @@ func buildSwarmDeployCmd(stack *portainer.Stack, opts unpackerCmdBuilderOptions, cmd = append(cmd, env...) cmd = append(cmd, registries...) - cmd = append(cmd, stack.GitConfig.URL, - stack.GitConfig.ReferenceName, + cmd = append(cmd, opts.gitConfig.URL, + opts.gitConfig.ReferenceName, stack.Name, opts.composeDestination, stack.EntryPoint, @@ -166,11 +168,11 @@ func buildSwarmUndeployCmd(stack *portainer.Stack, opts unpackerCmdBuilderOption // swarm-deploy [-u username -p password] [-f] [-r] [-k] [--skip-tls-verify] [--env KEY1=VALUE1 --env KEY2=VALUE2] [...] func buildSwarmStartCmd(stack *portainer.Stack, opts unpackerCmdBuilderOptions, registries []string, env []string) []string { cmd := []string{UnpackerCmdSwarmDeploy, "-f", "-r", "-k"} - cmd = appendSkipTLSVerifyIfNeeded(cmd, stack) + cmd = appendSkipTLSVerifyIfNeeded(cmd, opts.gitConfig) cmd = append(cmd, getEnv(stack.Env)...) cmd = append(cmd, registries...) - cmd = append(cmd, stack.GitConfig.URL, - stack.GitConfig.ReferenceName, + cmd = append(cmd, opts.gitConfig.URL, + opts.gitConfig.ReferenceName, stack.Name, opts.composeDestination, stack.EntryPoint, @@ -184,16 +186,16 @@ func buildSwarmStopCmd(stack *portainer.Stack, opts unpackerCmdBuilderOptions, r return []string{UnpackerCmdSwarmUndeploy, "-k", stack.Name, opts.composeDestination} } -func appendGitAuthIfNeeded(cmd []string, stack *portainer.Stack) []string { - if stack.GitConfig.Authentication == nil || stack.GitConfig.Authentication.Password == "" { +func appendGitAuthIfNeeded(cmd []string, gc *gittypes.RepoConfig) []string { + if gc == nil || gc.Authentication == nil || gc.Authentication.Password == "" { return cmd } - return append(cmd, "-u", stack.GitConfig.Authentication.Username, "-p", stack.GitConfig.Authentication.Password) + return append(cmd, "-u", gc.Authentication.Username, "-p", gc.Authentication.Password) } -func appendSkipTLSVerifyIfNeeded(cmd []string, stack *portainer.Stack) []string { - if !stack.GitConfig.TLSSkipVerify { +func appendSkipTLSVerifyIfNeeded(cmd []string, gc *gittypes.RepoConfig) []string { + if gc == nil || !gc.TLSSkipVerify { return cmd } @@ -204,6 +206,7 @@ func appendForceRecreateIfNeeded(cmd []string, forceRecreate bool) []string { if forceRecreate { cmd = append(cmd, "--force-recreate") } + return cmd } diff --git a/api/stacks/deployments/deploy.go b/api/stacks/deployments/deploy.go index 7a3f751c2..0e6b1764b 100644 --- a/api/stacks/deployments/deploy.go +++ b/api/stacks/deployments/deploy.go @@ -58,7 +58,7 @@ func RedeployWhenChanged(ctx context.Context, stackID portainer.StackID, deploye func redeployWhenChanged(ctx context.Context, stack *portainer.Stack, deployer StackDeployer, datastore dataservices.DataStore, gitService portainer.GitService, webhook bool) error { log.Debug().Int("stack_id", int(stack.ID)).Msg("redeploying stack") - if stack.GitConfig == nil { + if stack.WorkflowID == 0 { return nil // do nothing if it isn't a git-based stack } @@ -120,21 +120,29 @@ func redeployWhenChangedSecondStage( user *portainer.User, endpoint *portainer.Endpoint, ) error { + gitSrc, err := dataservices.GitSourceForWorkflow(datastore, stack.WorkflowID) + if err != nil { + return errors.WithMessagef(err, "failed to load git config for stack %v", stack.ID) + } + + if gitSrc == nil { + return nil + } + + gitConfig := gitSrc.GitConfig + var gitCommitChangedOrForceUpdate bool - // pendingHash holds the new commit hash to apply after deployment is attempted, - // preventing pre-deployment failures (e.g. registry lookup) from advancing the - // stored hash and causing subsequent polls to skip this commit. - var pendingHash string - if !stack.FromAppTemplate { - updated, newHash, err := update.UpdateGitObject(ctx, gitService, fmt.Sprintf("stack:%d", stack.ID), stack.GitConfig, false, stack.ProjectPath) + updated, newHash, err := update.UpdateGitObject(ctx, gitService, fmt.Sprintf("stack:%d", stack.ID), gitConfig, false, stack.ProjectPath) if err != nil { return err } if updated { - pendingHash = newHash + gitConfig.ConfigHash = newHash + + stack.UpdateDate = time.Now().Unix() gitCommitChangedOrForceUpdate = updated } @@ -154,16 +162,12 @@ func redeployWhenChangedSecondStage( return errors.WithMessagef(err, "failed to set the deploying status for stack %v", stack.ID) } - if pendingHash != "" { - stack.GitConfig.ConfigHash = pendingHash - } - stack.CurrentDeploymentInfo = &portainer.StackDeploymentInfo{ - RepositoryURL: stack.GitConfig.URL, - ReferenceName: stack.GitConfig.ReferenceName, - ConfigFilePath: stack.GitConfig.ConfigFilePath, + RepositoryURL: gitConfig.URL, + ReferenceName: gitConfig.ReferenceName, + ConfigFilePath: gitConfig.ConfigFilePath, AdditionalFiles: stack.AdditionalFiles, - ConfigHash: stack.GitConfig.ConfigHash, + ConfigHash: gitConfig.ConfigHash, } registries, err := getUserRegistries(datastore, user, endpoint.ID) @@ -204,6 +208,7 @@ func redeployWhenChangedSecondStage( default: return errors.Errorf("cannot update stack, type %v is unsupported", stack.Type) } + return nil } @@ -213,7 +218,18 @@ func redeployWhenChangedSecondStage( stack.UpdateDate = time.Now().Unix() stackutils.UpdateStackStatusFromDeploymentResult(stack, deployErr) - return tx.Stack().Update(stack.ID, stack) + if err := tx.Stack().Update(stack.ID, stack); err != nil { + return err + } + + src, err := tx.Source().Read(gitSrc.ID) + if err != nil { + return errors.Wrap(err, "failed to read source") + } + + src.GitConfig = gitConfig + + return tx.Source().Update(src.ID, src) }); err != nil { return errors.WithMessagef(err, "failed to update the stack %v", stack.ID) } diff --git a/api/stacks/deployments/deploy_test.go b/api/stacks/deployments/deploy_test.go index a61a9cf98..2f14f7884 100644 --- a/api/stacks/deployments/deploy_test.go +++ b/api/stacks/deployments/deploy_test.go @@ -234,14 +234,26 @@ func Test_redeployWhenChanged_FailsWhenCannotClone(t *testing.T) { }) require.NoError(t, err, "error creating environment") - err = store.Stack().Create(&portainer.Stack{ - ID: 1, - CreatedBy: "admin", + src := &portainer.Source{ + Type: portainer.SourceTypeGit, GitConfig: &gittypes.RepoConfig{ URL: "url", ReferenceName: "ref", ConfigHash: "oldHash", - }}) + }, + } + err = store.Source().Create(src) + require.NoError(t, err, "failed to create source") + + wf := &portainer.Workflow{SourceIDs: []portainer.SourceID{src.ID}} + err = store.Workflow().Create(wf) + require.NoError(t, err, "failed to create workflow") + + err = store.Stack().Create(&portainer.Stack{ + ID: 1, + CreatedBy: "admin", + WorkflowID: wf.ID, + }) require.NoError(t, err, "failed to create a test stack") err = RedeployWhenChanged(t.Context(), 1, nil, store, testhelpers.NewGitService(cloneErr, "newHash")) diff --git a/api/stacks/deployments/deployer.go b/api/stacks/deployments/deployer.go index 3882163f3..63127bd88 100644 --- a/api/stacks/deployments/deployer.go +++ b/api/stacks/deployments/deployer.go @@ -90,7 +90,7 @@ func (d *stackDeployer) DeployKubernetesStack(ctx context.Context, stack *portai Owner: user.Username, } - if stack.GitConfig == nil { + if stack.WorkflowID == 0 { appLabels.Kind = "content" } else { appLabels.Kind = "git" diff --git a/api/stacks/deployments/deployer_remote.go b/api/stacks/deployments/deployer_remote.go index 17cd9bb88..fd2a167fc 100644 --- a/api/stacks/deployments/deployer_remote.go +++ b/api/stacks/deployments/deployer_remote.go @@ -10,6 +10,7 @@ import ( "time" portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/dataservices" "github.com/portainer/portainer/api/filesystem" "github.com/portainer/portainer/api/logs" "github.com/portainer/portainer/pkg/librand" @@ -167,6 +168,17 @@ func (d *stackDeployer) StopRemoteSwarmStack(ctx context.Context, stack *portain // * wait for deployment to end // * gather deployment logs and bubble them up func (d *stackDeployer) remoteStack(ctx context.Context, stack *portainer.Stack, endpoint *portainer.Endpoint, operation StackRemoteOperation, opts unpackerCmdBuilderOptions) error { + if stack.WorkflowID != 0 && opts.gitConfig == nil { + src, err := dataservices.GitSourceForWorkflow(d.dataStore, stack.WorkflowID) + if err != nil { + return errors.Wrap(err, "failed to load git config for remote stack") + } + + if src != nil { + opts.gitConfig = src.GitConfig + } + } + cli, err := d.createDockerClient(ctx, endpoint) if err != nil { return errors.WithMessage(err, "unable to create docker client") diff --git a/api/stacks/stackbuilders/stack_builder.go b/api/stacks/stackbuilders/stack_builder.go index 5691e7b31..0181c4074 100644 --- a/api/stacks/stackbuilders/stack_builder.go +++ b/api/stacks/stackbuilders/stack_builder.go @@ -74,6 +74,19 @@ func (b *StackBuilder) cleanUp() error { return nil } + if b.stack.WorkflowID != 0 { + if err := b.dataStore.UpdateTx(func(tx dataservices.DataStoreTx) error { + err := tx.Workflow().Delete(b.stack.WorkflowID) + if tx.IsErrObjectNotFound(err) { + return nil + } + + return err + }); err != nil { + log.Error().Err(err).Msg("unable to cleanup orphan workflow records after failed stack creation") + } + } + if err := b.fileService.RemoveDirectory(b.stack.ProjectPath); err != nil { log.Error().Err(err).Msg("unable to cleanup stack creation") } diff --git a/api/stacks/stackbuilders/stack_git_builder.go b/api/stacks/stackbuilders/stack_git_builder.go index 52430a1f3..22940cda8 100644 --- a/api/stacks/stackbuilders/stack_git_builder.go +++ b/api/stacks/stackbuilders/stack_git_builder.go @@ -66,7 +66,36 @@ func (b *GitMethodStackBuilder) prepare(ctx context.Context, payload *StackPaylo // Update the latest commit id repoConfig.ConfigHash = commitHash - b.stack.GitConfig = &repoConfig + + var workflowID portainer.WorkflowID + + if err := b.dataStore.UpdateTx(func(tx dataservices.DataStoreTx) error { + repoConfig.URL = gittypes.SanitizeURL(repoConfig.URL) + + src, err := dataservices.FindOrCreateGitSource(tx, &portainer.Source{ + Name: gittypes.RepoName(repoConfig.URL), + Type: portainer.SourceTypeGit, + GitConfig: &repoConfig, + }) + if err != nil { + return fmt.Errorf("failed to find or create source: %w", err) + } + + wf := &portainer.Workflow{ + SourceIDs: []portainer.SourceID{src.ID}, + } + if err := tx.Workflow().Create(wf); err != nil { + return fmt.Errorf("failed to create workflow: %w", err) + } + + workflowID = wf.ID + + return nil + }); err != nil { + return err + } + + b.stack.WorkflowID = workflowID return nil } diff --git a/api/stacks/stackutils/env.go b/api/stacks/stackutils/env.go index f8aa815be..826bd3834 100644 --- a/api/stacks/stackutils/env.go +++ b/api/stacks/stackutils/env.go @@ -13,7 +13,7 @@ import ( ) // BuildEnvMap builds the environment variable map for stack validation/loading. -// Priority (lowest to highest): OS env → .env file → stack.Env +// Priority (lowest to highest): OS env, .env file, stack.Env func BuildEnvMap(stack *portainer.Stack) map[string]string { env := make(map[string]string, len(os.Environ())) for _, e := range os.Environ() { diff --git a/api/stacks/stackutils/gitops_test.go b/api/stacks/stackutils/gitops_test.go new file mode 100644 index 000000000..b4d8e7aaf --- /dev/null +++ b/api/stacks/stackutils/gitops_test.go @@ -0,0 +1,101 @@ +package stackutils + +import ( + "context" + "errors" + "testing" + + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/git" + gittypes "github.com/portainer/portainer/api/git/types" + + "github.com/stretchr/testify/require" +) + +type mockGitSvc struct { + cloneErr error + commitID string + commitErr error +} + +func (m *mockGitSvc) CloneRepository(_ context.Context, _, _, _, _, _ string, _ bool) error { + return m.cloneErr +} + +func (m *mockGitSvc) LatestCommitID(_ context.Context, _, _, _, _ string, _ bool) (string, error) { + return m.commitID, m.commitErr +} + +func (m *mockGitSvc) ListRefs(_ context.Context, _, _, _ string, _ bool, _ bool) ([]string, error) { + return nil, nil +} + +func (m *mockGitSvc) ListFiles(_ context.Context, _, _, _, _ string, _, _ bool, _ []string, _ bool) ([]string, error) { + return nil, nil +} + +var _ portainer.GitService = (*mockGitSvc)(nil) + +func TestDownloadGitRepository_Success(t *testing.T) { + t.Parallel() + + svc := &mockGitSvc{commitID: "abc123"} + cfg := gittypes.RepoConfig{ + URL: "https://github.com/x/repo", + ReferenceName: "refs/heads/main", + } + + commitID, err := DownloadGitRepository(t.Context(), cfg, svc, func() string { return t.TempDir() }) + require.NoError(t, err) + require.Equal(t, "abc123", commitID) +} + +func TestDownloadGitRepository_NilAuthentication(t *testing.T) { + t.Parallel() + + svc := &mockGitSvc{commitID: "deadbeef"} + cfg := gittypes.RepoConfig{ + URL: "https://github.com/x/repo", + Authentication: nil, + } + + commitID, err := DownloadGitRepository(t.Context(), cfg, svc, func() string { return t.TempDir() }) + require.NoError(t, err) + require.Equal(t, "deadbeef", commitID) +} + +func TestDownloadGitRepository_AuthenticationFailure(t *testing.T) { + t.Parallel() + + svc := &mockGitSvc{cloneErr: gittypes.ErrAuthenticationFailure} + cfg := gittypes.RepoConfig{URL: "https://github.com/x/private"} + + _, err := DownloadGitRepository(t.Context(), cfg, svc, func() string { return t.TempDir() }) + require.Error(t, err) + require.ErrorIs(t, err, git.ErrInvalidGitCredential) +} + +func TestDownloadGitRepository_OtherCloneError(t *testing.T) { + t.Parallel() + + cloneErr := errors.New("network timeout") + svc := &mockGitSvc{cloneErr: cloneErr} + cfg := gittypes.RepoConfig{URL: "https://github.com/x/repo"} + + _, err := DownloadGitRepository(t.Context(), cfg, svc, func() string { return t.TempDir() }) + require.Error(t, err) + require.ErrorIs(t, err, cloneErr) + require.NotErrorIs(t, err, git.ErrInvalidGitCredential) +} + +func TestDownloadGitRepository_LatestCommitIDError(t *testing.T) { + t.Parallel() + + commitErr := errors.New("remote unreachable") + svc := &mockGitSvc{commitErr: commitErr} + cfg := gittypes.RepoConfig{URL: "https://github.com/x/repo"} + + _, err := DownloadGitRepository(t.Context(), cfg, svc, func() string { return t.TempDir() }) + require.Error(t, err) + require.ErrorIs(t, err, commitErr) +}