diff --git a/api/dataservices/helpers.go b/api/dataservices/helpers.go index 7e420ed20..e2733fa6d 100644 --- a/api/dataservices/helpers.go +++ b/api/dataservices/helpers.go @@ -4,9 +4,7 @@ import ( "errors" "fmt" - portainer "github.com/portainer/portainer/api" perrors "github.com/portainer/portainer/api/dataservices/errors" - gittypes "github.com/portainer/portainer/api/git/types" "github.com/rs/zerolog/log" ) @@ -14,86 +12,6 @@ import ( // ErrStop signals the stop of computation when filtering results var ErrStop = errors.New("stop") -// gitSourceReader is a minimal interface satisfied by both CE and EE DataStoreTx, -// used to avoid duplicating GitSourceForWorkflow across packages. -type gitSourceReader interface { - Workflow() WorkflowService - Source() SourceService -} - -// GitSourceForWorkflow returns the first git-type Source for the given workflow. -// Returns nil, nil when workflowID is 0 or no git source is found. -func GitSourceForWorkflow(tx gitSourceReader, workflowID portainer.WorkflowID) (*portainer.Source, error) { - if workflowID == 0 { - return nil, nil - } - - wf, err := tx.Workflow().Read(workflowID) - if err != nil { - return nil, err - } - - for _, srcID := range wf.SourceIDs { - src, err := tx.Source().Read(srcID) - if err != nil { - return nil, err - } - - if src.Type == portainer.SourceTypeGit { - return src, nil - } - } - - return nil, nil -} - -// GitConfigHashForWorkflow returns the git commit hash for the first git-type Source of the given workflow. -// Returns "" when workflowID is 0, no git source is found, or on error. -func GitConfigHashForWorkflow(tx gitSourceReader, workflowID portainer.WorkflowID) string { - src, err := GitSourceForWorkflow(tx, workflowID) - if err != nil || src == nil || src.GitConfig == nil { - return "" - } - - return src.GitConfig.ConfigHash -} - -// FindOrCreateGitSource returns an existing Source whose URL and authentication match src, -// or creates a new one. ConfigHash is excluded from matching as it is transient runtime state. -func FindOrCreateGitSource(tx gitSourceReader, src *portainer.Source) (*portainer.Source, error) { - existing, err := tx.Source().ReadAll(func(s portainer.Source) bool { - return s.Type == portainer.SourceTypeGit && - s.GitConfig != nil && - s.GitConfig.URL == src.GitConfig.URL && - gitAuthMatches(s.GitConfig.Authentication, src.GitConfig.Authentication) - }) - if err != nil { - return nil, err - } - - if len(existing) > 0 { - return &existing[0], nil - } - - if err := tx.Source().Create(src); err != nil { - return nil, err - } - - return src, nil -} - -func gitAuthMatches(a, b *gittypes.GitAuthentication) bool { - if a == nil && b == nil { - return true - } - - if a == nil || b == nil { - return false - } - - return a.Username == b.Username && a.Password == b.Password && a.GitCredentialID == b.GitCredentialID -} - func IsErrObjectNotFound(e error) bool { return errors.Is(e, perrors.ErrObjectNotFound) } diff --git a/api/datastore/migrator/migrate_2_43_0.go b/api/datastore/migrator/migrate_2_43_0.go index c9c3abefd..54a2f38a5 100644 --- a/api/datastore/migrator/migrate_2_43_0.go +++ b/api/datastore/migrator/migrate_2_43_0.go @@ -41,6 +41,12 @@ func (lrc *legacyRepoConfig) toRepoConfig() *gittypes.RepoConfig { } if lrc.Authentication != nil { + if lrc.Authentication.GitCredentialID != 0 { + log.Warn(). + Int("git_credential_id", lrc.Authentication.GitCredentialID). + Msg("stack has a GitCredentialID reference which is not supported in CE; credential reference will be dropped during migration") + } + cfg.Authentication = &gittypes.GitAuthentication{ Username: lrc.Authentication.Username, Password: lrc.Authentication.Password, @@ -138,18 +144,27 @@ func (m *Migrator) migrateGitConfigToSources_2_43_0() error { newSrcID = src.ID } - wf := &portainer.Workflow{ - SourceIDs: []portainer.SourceID{srcID}, - } - if err := m.workflowService.Tx(tx).Create(wf); err != nil { - return fmt.Errorf("failed to create workflow for stack %d: %w", ls.ID, err) - } - liveStack, err := m.stackService.Tx(tx).Read(portainer.StackID(ls.ID)) if err != nil { return fmt.Errorf("failed to read stack %d: %w", ls.ID, err) } + wf := &portainer.Workflow{ + Name: liveStack.Name, + Artifacts: []portainer.ArtifactSources{{ + Artifact: portainer.Artifact{ + ReferenceName: cfg.ReferenceName, + ConfigFilePath: cfg.ConfigFilePath, + ConfigHash: cfg.ConfigHash, + StackID: portainer.StackID(ls.ID), + }, + SourceIDs: []portainer.SourceID{srcID}, + }}, + } + if err := m.workflowService.Tx(tx).Create(wf); err != nil { + return fmt.Errorf("failed to create workflow for stack %d: %w", ls.ID, err) + } + liveStack.WorkflowID = wf.ID liveStack.GitConfig = nil diff --git a/api/datastore/migrator/migrate_2_43_0_test.go b/api/datastore/migrator/migrate_2_43_0_test.go index 8c8cc1f6e..9b6c72224 100644 --- a/api/datastore/migrator/migrate_2_43_0_test.go +++ b/api/datastore/migrator/migrate_2_43_0_test.go @@ -57,9 +57,10 @@ func TestMigrateGitConfigToSources_2_43_0_GitStackMigrated(t *testing.T) { wf, err := workflowSvc.Read(migrated.WorkflowID) require.NoError(t, err) - require.Len(t, wf.SourceIDs, 1) + require.Len(t, wf.Artifacts, 1) + require.Len(t, wf.Artifacts[0].SourceIDs, 1) - src, err := sourceSvc.Read(wf.SourceIDs[0]) + src, err := sourceSvc.Read(wf.Artifacts[0].SourceIDs[0]) require.NoError(t, err) require.Equal(t, portainer.SourceTypeGit, src.Type) require.Equal(t, gitStack.GitConfig.URL, src.GitConfig.URL) @@ -168,8 +169,9 @@ func TestMigrateGitConfigToSources_2_43_0_DuplicateSourcesDeduped(t *testing.T) sharedSourceID := sources[0].ID for _, wf := range workflows { - require.Len(t, wf.SourceIDs, 1) - require.Equal(t, sharedSourceID, wf.SourceIDs[0]) + require.Len(t, wf.Artifacts, 1) + require.Len(t, wf.Artifacts[0].SourceIDs, 1) + require.Equal(t, sharedSourceID, wf.Artifacts[0].SourceIDs[0]) } } diff --git a/api/git/types/types.go b/api/git/types/types.go index 7aca167f1..ee0d4afdb 100644 --- a/api/git/types/types.go +++ b/api/git/types/types.go @@ -1,6 +1,7 @@ package gittypes import ( + "cmp" "errors" "net/url" "path" @@ -40,6 +41,24 @@ func RepoName(rawURL string) string { return strings.TrimSuffix(path.Base(rawURL), ".git") } +// NormalizeURL returns a canonical form of rawURL for deduplication purposes: +// scheme and host are lowercased, embedded credentials are removed, trailing +// slashes and the .git suffix are stripped from the path. If the scheme is +// absent it defaults to https. +func NormalizeURL(rawURL string) (string, error) { + u, err := url.Parse(rawURL) + if err != nil { + return "", err + } + + u.Scheme = strings.ToLower(cmp.Or(u.Scheme, "https")) + u.Host = strings.ToLower(u.Host) + u.User = nil + u.Path = strings.TrimSuffix(strings.TrimRight(u.Path, "/"), ".git") + + return u.String(), nil +} + // SanitizeURL strips any userinfo (username/password) embedded in rawURL, // returning a URL safe to store or return to clients. func SanitizeURL(rawURL string) string { diff --git a/api/git/types/types_test.go b/api/git/types/types_test.go new file mode 100644 index 000000000..5e113787d --- /dev/null +++ b/api/git/types/types_test.go @@ -0,0 +1,26 @@ +package gittypes + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNormalizeURL(t *testing.T) { + t.Parallel() + + f := func(input, expected string) { + t.Helper() + got, err := NormalizeURL(input) + require.NoError(t, err) + require.Equal(t, expected, got) + } + + f("https://github.com/org/repo.git", "https://github.com/org/repo") + f("https://github.com/org/repo/", "https://github.com/org/repo") + f("https://github.com/org/repo.git/", "https://github.com/org/repo") + f("HTTPS://github.com/org/repo", "https://github.com/org/repo") + f("https://GitHub.COM/org/repo", "https://github.com/org/repo") + f("https://user:pass@github.com/org/repo.git", "https://github.com/org/repo") + f("https://github.com/org/repo", "https://github.com/org/repo") +} diff --git a/api/gitops/workflows/fetch.go b/api/gitops/workflows/fetch.go index 1d4bf3501..257d2a288 100644 --- a/api/gitops/workflows/fetch.go +++ b/api/gitops/workflows/fetch.go @@ -48,12 +48,12 @@ func FetchWorkflows( // First pass: filter by endpoint/stack-type match and collect workflow IDs. preFiltered := make([]portainer.Stack, 0, len(stacks)) workflowIDSet := make(map[portainer.WorkflowID]struct{}, len(stacks)) - for _, s := range stacks { - if ep, ok := endpointMap[s.EndpointID]; ok && !EndpointMatchesStackType(ep, s.Type) { + for _, stack := range stacks { + if ep, ok := endpointMap[stack.EndpointID]; ok && !EndpointMatchesStackType(ep, stack.Type) { continue } - preFiltered = append(preFiltered, s) - workflowIDSet[s.WorkflowID] = struct{}{} + preFiltered = append(preFiltered, stack) + workflowIDSet[stack.WorkflowID] = struct{}{} } // Batch-load all needed workflows in one scan. @@ -66,18 +66,16 @@ func FetchWorkflows( } workflowMap := make(map[portainer.WorkflowID]portainer.Workflow, len(workflows)) - sourceIDSet := make(map[portainer.SourceID]struct{}) + var allArtifacts []portainer.ArtifactSources for _, wf := range workflows { workflowMap[wf.ID] = wf - for _, sid := range wf.SourceIDs { - sourceIDSet[sid] = struct{}{} - } + allArtifacts = append(allArtifacts, wf.Artifacts...) } + sourceSet := ArtifactsToSourceSet(allArtifacts...) // Batch-load all needed sources in one scan. srcs, err := tx.Source().ReadAll(func(src portainer.Source) bool { - _, ok := sourceIDSet[src.ID] - return ok + return sourceSet.Contains(src.ID) }) if err != nil { return err @@ -90,27 +88,34 @@ func FetchWorkflows( // Second pass: build filtered list using in-memory lookups. var filtered []portainer.Stack - for _, s := range preFiltered { - wf, ok := workflowMap[s.WorkflowID] + for _, stack := range preFiltered { + wf, ok := workflowMap[stack.WorkflowID] if !ok { - log.Warn().Int("stackID", int(s.ID)).Msg("workflow record missing for stack, skipping") + log.Warn().Int("stackID", int(stack.ID)).Msg("workflow record missing for stack, skipping") continue } - for _, srcID := range wf.SourceIDs { - src, ok := sourceMap[srcID] - if !ok { - log.Warn().Int("stackID", int(s.ID)).Msg("source record missing for stack, skipping") - break + outer: + for _, as := range wf.Artifacts { + if as.Artifact.StackID != stack.ID { + continue } - if src.Type == portainer.SourceTypeGit { - gitConfigs[s.ID] = src.GitConfig - break + for _, srcID := range as.SourceIDs { + src, ok := sourceMap[srcID] + if !ok { + log.Warn().Int("stackID", int(stack.ID)).Msg("source record missing for stack, skipping") + break outer + } + + if src.Type == portainer.SourceTypeGit { + gitConfigs[stack.ID] = MergeSourceAndArtifact(&src, &as.Artifact) + break outer + } } } - filtered = append(filtered, s) + filtered = append(filtered, stack) } stacks = filtered @@ -131,10 +136,10 @@ func FetchWorkflows( } items := make([]Workflow, 0, len(stacks)) - for _, s := range stacks { - gitConfig := gitConfigs[s.ID] + for _, stack := range stacks { + gitConfig := gitConfigs[stack.ID] source, artifact := ComputeGitPhasesForConfig(ctx, gitService, gitConfig) - items = append(items, MapStackToWorkflow(s, gitConfig, source, artifact)) + items = append(items, MapStackToWorkflow(stack, gitConfig, source, artifact)) } return items, nil @@ -176,12 +181,12 @@ func FetchSourceStats( workflowIDSet := make(map[portainer.WorkflowID]struct{}, len(allStacks)) preFiltered := make([]portainer.Stack, 0, len(allStacks)) - for _, s := range allStacks { - if ep, ok := endpointMap[s.EndpointID]; ok && !EndpointMatchesStackType(ep, s.Type) { + for _, stack := range allStacks { + if ep, ok := endpointMap[stack.EndpointID]; ok && !EndpointMatchesStackType(ep, stack.Type) { continue } - preFiltered = append(preFiltered, s) - workflowIDSet[s.WorkflowID] = struct{}{} + preFiltered = append(preFiltered, stack) + workflowIDSet[stack.WorkflowID] = struct{}{} } wfs, err := tx.Workflow().ReadAll(func(wf portainer.Workflow) bool { @@ -194,13 +199,15 @@ func FetchSourceStats( wfSources := make(map[portainer.WorkflowID][]portainer.SourceID, len(wfs)) for _, wf := range wfs { - wfSources[wf.ID] = wf.SourceIDs + for _, as := range wf.Artifacts { + wfSources[wf.ID] = append(wfSources[wf.ID], as.SourceIDs...) + } } stackSourceIDs := make(map[portainer.StackID][]portainer.SourceID) - for _, s := range preFiltered { - if srcIDs := wfSources[s.WorkflowID]; len(srcIDs) > 0 { - stackSourceIDs[s.ID] = srcIDs + for _, stack := range preFiltered { + if srcIDs := wfSources[stack.WorkflowID]; len(srcIDs) > 0 { + stackSourceIDs[stack.ID] = srcIDs } } @@ -216,12 +223,12 @@ func FetchSourceStats( stats := make(map[portainer.SourceID]SourceStats) - for _, s := range stacks { + for _, stack := range stacks { var epIDs []portainer.EndpointID - if s.EndpointID != 0 { - epIDs = []portainer.EndpointID{s.EndpointID} + if stack.EndpointID != 0 { + epIDs = []portainer.EndpointID{stack.EndpointID} } - addSourceStats(stats, stackSourceIDs[s.ID], epIDs, StackLastSyncDate(s)) + addSourceStats(stats, stackSourceIDs[stack.ID], epIDs, StackLastSyncDate(stack)) } return sources, stats, nil diff --git a/api/gitops/workflows/fetch_test.go b/api/gitops/workflows/fetch_test.go index 62f36b811..e40de7a9c 100644 --- a/api/gitops/workflows/fetch_test.go +++ b/api/gitops/workflows/fetch_test.go @@ -26,7 +26,10 @@ func mustCreateGitWorkflow(t *testing.T, tx dataservices.DataStoreTx, stack *por src := &portainer.Source{Type: portainer.SourceTypeGit, GitConfig: cfg} require.NoError(t, tx.Source().Create(src)) - wf := &portainer.Workflow{SourceIDs: []portainer.SourceID{src.ID}} + wf := &portainer.Workflow{Artifacts: []portainer.ArtifactSources{{ + Artifact: portainer.Artifact{StackID: stack.ID}, + SourceIDs: []portainer.SourceID{src.ID}, + }}} require.NoError(t, tx.Workflow().Create(wf)) stack.WorkflowID = wf.ID @@ -209,7 +212,7 @@ func TestFetchSourceStats_TracksWorkflowCountAndEndpoints(t *testing.T) { srcID = src.ID for i := 1; i <= 2; i++ { - wf := &portainer.Workflow{SourceIDs: []portainer.SourceID{srcID}} + wf := &portainer.Workflow{Artifacts: []portainer.ArtifactSources{{SourceIDs: []portainer.SourceID{srcID}}}} require.NoError(t, tx.Workflow().Create(wf)) require.NoError(t, tx.Stack().Create(&portainer.Stack{ ID: portainer.StackID(i), diff --git a/api/gitops/workflows/mapping.go b/api/gitops/workflows/mapping.go index b67a0340d..fec1c70ec 100644 --- a/api/gitops/workflows/mapping.go +++ b/api/gitops/workflows/mapping.go @@ -115,6 +115,18 @@ func isEdgeStackHealthyStatus(t portainer.EdgeStackStatusType) bool { return false } +// ArtifactsToSourceSet returns the set of all SourceIDs referenced by the given artifact-source mappings +func ArtifactsToSourceSet(artifacts ...portainer.ArtifactSources) set.Set[portainer.SourceID] { + s := make(set.Set[portainer.SourceID]) + for _, a := range artifacts { + for _, sid := range a.SourceIDs { + s.Add(sid) + } + } + + return s +} + func resolveEdgeGroupEndpoints(groups []portainer.EdgeGroupID, groupEndpoints map[portainer.EdgeGroupID][]portainer.EndpointID) []portainer.EndpointID { seen := set.Set[portainer.EndpointID]{} for _, gid := range groups { diff --git a/api/gitops/workflows/source_artifact.go b/api/gitops/workflows/source_artifact.go new file mode 100644 index 000000000..81c11aec1 --- /dev/null +++ b/api/gitops/workflows/source_artifact.go @@ -0,0 +1,253 @@ +package workflows + +import ( + "fmt" + + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/dataservices" + gittypes "github.com/portainer/portainer/api/git/types" +) + +// gitSourceStore is the minimal intersection of CE and EE DataStoreTx that these functions need. +type gitSourceStore interface { + Workflow() dataservices.WorkflowService + Source() dataservices.SourceService +} + +// GitSourceAndArtifactForStack returns the git Source and the Artifact matching stackID +// from the workflow identified by workflowID. +// Source carries the shared fields (URL, auth, TLS); Artifact carries the stack-specific fields (ref, path, hash). +// Returns nil, nil, nil when workflowID is 0 or no matching entry is found. +func GitSourceAndArtifactForStack(tx gitSourceStore, workflowID portainer.WorkflowID, stackID portainer.StackID) (*portainer.Source, *portainer.Artifact, error) { + if workflowID == 0 { + return nil, nil, nil + } + + wf, err := tx.Workflow().Read(workflowID) + if err != nil { + return nil, nil, err + } + + for i, as := range wf.Artifacts { + if as.Artifact.StackID != stackID { + continue + } + + for _, srcID := range as.SourceIDs { + src, err := tx.Source().Read(srcID) + if err != nil { + return nil, nil, err + } + + if src.Type == portainer.SourceTypeGit { + return src, &wf.Artifacts[i].Artifact, nil + } + } + } + + return nil, nil, nil +} + +// GitSourceAndArtifactForEdgeStack returns the git Source and the Artifact matching edgeStackID. +// Returns nil, nil, nil when workflowID is 0 or no matching entry is found. +func GitSourceAndArtifactForEdgeStack(tx gitSourceStore, workflowID portainer.WorkflowID, edgeStackID portainer.EdgeStackID) (*portainer.Source, *portainer.Artifact, error) { + if workflowID == 0 { + return nil, nil, nil + } + + wf, err := tx.Workflow().Read(workflowID) + if err != nil { + return nil, nil, err + } + + for i, as := range wf.Artifacts { + if as.Artifact.EdgeStackID != edgeStackID { + continue + } + + for _, srcID := range as.SourceIDs { + src, err := tx.Source().Read(srcID) + if err != nil { + return nil, nil, err + } + + if src.Type == portainer.SourceTypeGit { + return src, &wf.Artifacts[i].Artifact, nil + } + } + } + + return nil, nil, nil +} + +// MergeSourceAndArtifact builds a RepoConfig by combining shared fields from src (URL, auth, TLS) +// with stack-specific fields from artifact (ref, path, hash). +func MergeSourceAndArtifact(src *portainer.Source, artifact *portainer.Artifact) *gittypes.RepoConfig { + if src == nil || src.GitConfig == nil { + return nil + } + + cfg := &gittypes.RepoConfig{ + URL: src.GitConfig.URL, + Authentication: src.GitConfig.Authentication, + TLSSkipVerify: src.GitConfig.TLSSkipVerify, + } + + if artifact != nil { + cfg.ReferenceName = artifact.ReferenceName + cfg.ConfigFilePath = artifact.ConfigFilePath + cfg.ConfigHash = artifact.ConfigHash + } + + return cfg +} + +// UpdateArtifactForStack finds the Artifact matching stackID in the workflow and applies fn to it, +// then persists the updated Workflow. A no-op if no matching Artifact is found. +func UpdateArtifactForStack(tx gitSourceStore, workflowID portainer.WorkflowID, stackID portainer.StackID, fn func(*portainer.Artifact)) error { + wf, err := tx.Workflow().Read(workflowID) + if err != nil { + return err + } + + for i, as := range wf.Artifacts { + if as.Artifact.StackID == stackID { + fn(&wf.Artifacts[i].Artifact) + + return tx.Workflow().Update(workflowID, wf) + } + } + + return nil +} + +// UpdateArtifactForEdgeStack finds the Artifact matching edgeStackID in the workflow and applies fn to it, +// then persists the updated Workflow. A no-op if no matching Artifact is found. +func UpdateArtifactForEdgeStack(tx gitSourceStore, workflowID portainer.WorkflowID, edgeStackID portainer.EdgeStackID, fn func(*portainer.Artifact)) error { + wf, err := tx.Workflow().Read(workflowID) + if err != nil { + return err + } + + for i, as := range wf.Artifacts { + if as.Artifact.EdgeStackID == edgeStackID { + fn(&wf.Artifacts[i].Artifact) + + return tx.Workflow().Update(workflowID, wf) + } + } + + return nil +} + +// FindOrCreateGitSource returns an existing Source whose URL and authentication match cfg, +// or creates a new one. Only URL, authentication, and TLSSkipVerify are stored on the Source; +// per-stack fields (ReferenceName, ConfigFilePath, ConfigHash) belong in the Artifact. +func FindOrCreateGitSource(tx gitSourceStore, src *portainer.Source) (*portainer.Source, error) { + src.GitConfig.URL = gittypes.SanitizeURL(src.GitConfig.URL) + + existing, err := tx.Source().ReadAll(func(s portainer.Source) bool { + return s.Type == portainer.SourceTypeGit && + s.GitConfig != nil && + s.GitConfig.URL == src.GitConfig.URL && + gitAuthMatches(s.GitConfig.Authentication, src.GitConfig.Authentication) + }) + if err != nil { + return nil, err + } + + if len(existing) > 0 { + return &existing[0], nil + } + + toCreate := &portainer.Source{ + Name: src.Name, + Type: portainer.SourceTypeGit, + GitConfig: &gittypes.RepoConfig{ + URL: src.GitConfig.URL, + Authentication: src.GitConfig.Authentication, + TLSSkipVerify: src.GitConfig.TLSSkipVerify, + }, + } + + if err := tx.Source().Create(toCreate); err != nil { + return nil, err + } + + return toCreate, nil +} + +// SaveWorkflowGitConfig persists URL/auth/TLS on the Source and ref/path/hash on the Artifact +// matched by matchArtifact. When the URL changes, an existing or new Source is located via +// FindOrCreateGitSource and the Workflow's SourceID is updated atomically alongside the Artifact fields. +func SaveWorkflowGitConfig(tx gitSourceStore, workflowID portainer.WorkflowID, matchArtifact func(portainer.Artifact) bool, oldSourceID portainer.SourceID, cfg *gittypes.RepoConfig) error { + src, err := tx.Source().Read(oldSourceID) + if err != nil { + return fmt.Errorf("failed to read source: %w", err) + } + + if src.GitConfig == nil { + return fmt.Errorf("source %d has no git configuration", oldSourceID) + } + + newSourceID := oldSourceID + + if cfg.URL != src.GitConfig.URL { + newSrc, err := FindOrCreateGitSource(tx, &portainer.Source{ + Name: gittypes.RepoName(cfg.URL), + Type: portainer.SourceTypeGit, + GitConfig: cfg, + }) + if err != nil { + return fmt.Errorf("failed to find or create source: %w", err) + } + + newSourceID = newSrc.ID + } else { + src.GitConfig.Authentication = cfg.Authentication + src.GitConfig.TLSSkipVerify = cfg.TLSSkipVerify + + if err := tx.Source().Update(src.ID, src); err != nil { + return fmt.Errorf("failed to update source: %w", err) + } + } + + wf, err := tx.Workflow().Read(workflowID) + if err != nil { + return fmt.Errorf("failed to read workflow: %w", err) + } + + for i, as := range wf.Artifacts { + if !matchArtifact(as.Artifact) { + continue + } + + wf.Artifacts[i].Artifact.ReferenceName = cfg.ReferenceName + wf.Artifacts[i].Artifact.ConfigFilePath = cfg.ConfigFilePath + wf.Artifacts[i].Artifact.ConfigHash = cfg.ConfigHash + + if newSourceID != oldSourceID { + for j, sID := range as.SourceIDs { + if sID == oldSourceID { + wf.Artifacts[i].SourceIDs[j] = newSourceID + } + } + } + + break + } + + return tx.Workflow().Update(workflowID, wf) +} + +func gitAuthMatches(a, b *gittypes.GitAuthentication) bool { + if a == nil && b == nil { + return true + } + + if a == nil || b == nil { + return false + } + + return a.Username == b.Username && a.Password == b.Password && a.GitCredentialID == b.GitCredentialID +} diff --git a/api/gitops/workflows/source_artifact_test.go b/api/gitops/workflows/source_artifact_test.go new file mode 100644 index 000000000..c7173bee9 --- /dev/null +++ b/api/gitops/workflows/source_artifact_test.go @@ -0,0 +1,752 @@ +package workflows + +import ( + "testing" + + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/dataservices" + "github.com/portainer/portainer/api/datastore" + gittypes "github.com/portainer/portainer/api/git/types" + + "github.com/stretchr/testify/require" +) + +func TestMergeSourceAndArtifact_NilSourceReturnsNil(t *testing.T) { + t.Parallel() + + require.Nil(t, MergeSourceAndArtifact(nil, nil)) +} + +func TestMergeSourceAndArtifact_NilGitConfigReturnsNil(t *testing.T) { + t.Parallel() + + src := &portainer.Source{Type: portainer.SourceTypeGit} + require.Nil(t, MergeSourceAndArtifact(src, nil)) +} + +func TestMergeSourceAndArtifact_NilArtifactLeavesPerStackFieldsEmpty(t *testing.T) { + t.Parallel() + + src := &portainer.Source{ + GitConfig: &gittypes.RepoConfig{ + URL: "https://github.com/example/repo", + TLSSkipVerify: true, + Authentication: &gittypes.GitAuthentication{ + Username: "user", + Password: "pass", + }, + }, + } + + cfg := MergeSourceAndArtifact(src, nil) + require.NotNil(t, cfg) + require.Equal(t, "https://github.com/example/repo", cfg.URL) + require.True(t, cfg.TLSSkipVerify) + require.Equal(t, "user", cfg.Authentication.Username) + require.Empty(t, cfg.ReferenceName) + require.Empty(t, cfg.ConfigFilePath) + require.Empty(t, cfg.ConfigHash) +} + +func TestMergeSourceAndArtifact_MergesAllFieldsFromArtifact(t *testing.T) { + t.Parallel() + + src := &portainer.Source{ + GitConfig: &gittypes.RepoConfig{ + URL: "https://github.com/example/repo", + TLSSkipVerify: true, + }, + } + artifact := &portainer.Artifact{ + ReferenceName: "refs/heads/main", + ConfigFilePath: "docker-compose.yml", + ConfigHash: "abc123", + } + + cfg := MergeSourceAndArtifact(src, artifact) + require.NotNil(t, cfg) + require.Equal(t, "https://github.com/example/repo", cfg.URL) + require.True(t, cfg.TLSSkipVerify) + require.Equal(t, "refs/heads/main", cfg.ReferenceName) + require.Equal(t, "docker-compose.yml", cfg.ConfigFilePath) + require.Equal(t, "abc123", cfg.ConfigHash) +} + +func TestGitSourceAndArtifactForStack_ZeroWorkflowIDReturnsNil(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + var src *portainer.Source + var artifact *portainer.Artifact + err := store.ViewTx(func(tx dataservices.DataStoreTx) error { + var txErr error + src, artifact, txErr = GitSourceAndArtifactForStack(tx, 0, 1) + return txErr + }) + require.NoError(t, err) + require.Nil(t, src) + require.Nil(t, artifact) +} + +func TestGitSourceAndArtifactForStack_ReturnsMatchingSourceAndArtifact(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + var workflowID portainer.WorkflowID + err := store.UpdateTx(func(tx dataservices.DataStoreTx) error { + gitSrc := &portainer.Source{ + Type: portainer.SourceTypeGit, + GitConfig: &gittypes.RepoConfig{URL: "https://github.com/example/repo"}, + } + err := tx.Source().Create(gitSrc) + require.NoError(t, err) + + wf := &portainer.Workflow{ + Artifacts: []portainer.ArtifactSources{{ + Artifact: portainer.Artifact{ + StackID: 42, + ReferenceName: "refs/heads/main", + ConfigFilePath: "docker-compose.yml", + ConfigHash: "abc123", + }, + SourceIDs: []portainer.SourceID{gitSrc.ID}, + }}, + } + err = tx.Workflow().Create(wf) + require.NoError(t, err) + workflowID = wf.ID + + return nil + }) + require.NoError(t, err) + + var src *portainer.Source + var artifact *portainer.Artifact + err = store.ViewTx(func(tx dataservices.DataStoreTx) error { + var txErr error + src, artifact, txErr = GitSourceAndArtifactForStack(tx, workflowID, 42) + return txErr + }) + require.NoError(t, err) + require.NotNil(t, src) + require.Equal(t, portainer.SourceTypeGit, src.Type) + require.NotNil(t, artifact) + require.Equal(t, portainer.StackID(42), artifact.StackID) + require.Equal(t, "refs/heads/main", artifact.ReferenceName) + require.Equal(t, "docker-compose.yml", artifact.ConfigFilePath) + require.Equal(t, "abc123", artifact.ConfigHash) +} + +func TestGitSourceAndArtifactForStack_NoMatchingArtifactReturnsNil(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + var workflowID portainer.WorkflowID + err := store.UpdateTx(func(tx dataservices.DataStoreTx) error { + src := &portainer.Source{ + Type: portainer.SourceTypeGit, + GitConfig: &gittypes.RepoConfig{URL: "https://github.com/example/repo"}, + } + err := tx.Source().Create(src) + require.NoError(t, err) + + wf := &portainer.Workflow{ + Artifacts: []portainer.ArtifactSources{{ + Artifact: portainer.Artifact{StackID: 1}, + SourceIDs: []portainer.SourceID{src.ID}, + }}, + } + err = tx.Workflow().Create(wf) + require.NoError(t, err) + workflowID = wf.ID + + return nil + }) + require.NoError(t, err) + + var src *portainer.Source + var artifact *portainer.Artifact + err = store.ViewTx(func(tx dataservices.DataStoreTx) error { + var txErr error + src, artifact, txErr = GitSourceAndArtifactForStack(tx, workflowID, 99) + return txErr + }) + require.NoError(t, err) + require.Nil(t, src) + require.Nil(t, artifact) +} + +func TestGitSourceAndArtifactForStack_NonGitSourceSkipped(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + var workflowID portainer.WorkflowID + err := store.UpdateTx(func(tx dataservices.DataStoreTx) error { + nonGitSrc := &portainer.Source{Type: portainer.SourceType(99)} + err := tx.Source().Create(nonGitSrc) + require.NoError(t, err) + + wf := &portainer.Workflow{ + Artifacts: []portainer.ArtifactSources{{ + Artifact: portainer.Artifact{StackID: 1}, + SourceIDs: []portainer.SourceID{nonGitSrc.ID}, + }}, + } + err = tx.Workflow().Create(wf) + require.NoError(t, err) + workflowID = wf.ID + + return nil + }) + require.NoError(t, err) + + var src *portainer.Source + var artifact *portainer.Artifact + err = store.ViewTx(func(tx dataservices.DataStoreTx) error { + var txErr error + src, artifact, txErr = GitSourceAndArtifactForStack(tx, workflowID, 1) + return txErr + }) + require.NoError(t, err) + require.Nil(t, src) + require.Nil(t, artifact) +} + +func TestGitSourceAndArtifactForEdgeStack_ZeroWorkflowIDReturnsNil(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + var src *portainer.Source + var artifact *portainer.Artifact + err := store.ViewTx(func(tx dataservices.DataStoreTx) error { + var txErr error + src, artifact, txErr = GitSourceAndArtifactForEdgeStack(tx, 0, 1) + return txErr + }) + require.NoError(t, err) + require.Nil(t, src) + require.Nil(t, artifact) +} + +func TestGitSourceAndArtifactForEdgeStack_ReturnsMatchingSourceAndArtifact(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + var workflowID portainer.WorkflowID + err := store.UpdateTx(func(tx dataservices.DataStoreTx) error { + gitSrc := &portainer.Source{ + Type: portainer.SourceTypeGit, + GitConfig: &gittypes.RepoConfig{URL: "https://github.com/example/edge-repo"}, + } + err := tx.Source().Create(gitSrc) + require.NoError(t, err) + + wf := &portainer.Workflow{ + Artifacts: []portainer.ArtifactSources{{ + Artifact: portainer.Artifact{ + EdgeStackID: 5, + ReferenceName: "refs/heads/edge", + ConfigFilePath: "edge.yml", + }, + SourceIDs: []portainer.SourceID{gitSrc.ID}, + }}, + } + err = tx.Workflow().Create(wf) + require.NoError(t, err) + workflowID = wf.ID + + return nil + }) + require.NoError(t, err) + + var src *portainer.Source + var artifact *portainer.Artifact + err = store.ViewTx(func(tx dataservices.DataStoreTx) error { + var txErr error + src, artifact, txErr = GitSourceAndArtifactForEdgeStack(tx, workflowID, 5) + return txErr + }) + require.NoError(t, err) + require.NotNil(t, src) + require.Equal(t, portainer.SourceTypeGit, src.Type) + require.NotNil(t, artifact) + require.Equal(t, portainer.EdgeStackID(5), artifact.EdgeStackID) + require.Equal(t, "refs/heads/edge", artifact.ReferenceName) +} + +func TestUpdateArtifactForStack_NoMatchingArtifactIsNoOp(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + var workflowID portainer.WorkflowID + err := store.UpdateTx(func(tx dataservices.DataStoreTx) error { + wf := &portainer.Workflow{ + Artifacts: []portainer.ArtifactSources{{ + Artifact: portainer.Artifact{StackID: 1, ConfigHash: "original"}, + }}, + } + err := tx.Workflow().Create(wf) + require.NoError(t, err) + workflowID = wf.ID + + return nil + }) + require.NoError(t, err) + + err = store.UpdateTx(func(tx dataservices.DataStoreTx) error { + return UpdateArtifactForStack(tx, workflowID, 99, func(a *portainer.Artifact) { + a.ConfigHash = "changed" + }) + }) + require.NoError(t, err) + + wf, err := store.Workflow().Read(workflowID) + require.NoError(t, err) + require.Equal(t, "original", wf.Artifacts[0].Artifact.ConfigHash) +} + +func TestUpdateArtifactForStack_AppliesFnAndPersists(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + var workflowID portainer.WorkflowID + err := store.UpdateTx(func(tx dataservices.DataStoreTx) error { + wf := &portainer.Workflow{ + Artifacts: []portainer.ArtifactSources{{ + Artifact: portainer.Artifact{StackID: 1, ConfigHash: "old-hash"}, + }}, + } + err := tx.Workflow().Create(wf) + require.NoError(t, err) + workflowID = wf.ID + + return nil + }) + require.NoError(t, err) + + err = store.UpdateTx(func(tx dataservices.DataStoreTx) error { + return UpdateArtifactForStack(tx, workflowID, 1, func(a *portainer.Artifact) { + a.ConfigHash = "new-hash" + }) + }) + require.NoError(t, err) + + wf, err := store.Workflow().Read(workflowID) + require.NoError(t, err) + require.Equal(t, "new-hash", wf.Artifacts[0].Artifact.ConfigHash) +} + +func TestUpdateArtifactForEdgeStack_AppliesFnAndPersists(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + var workflowID portainer.WorkflowID + err := store.UpdateTx(func(tx dataservices.DataStoreTx) error { + wf := &portainer.Workflow{ + Artifacts: []portainer.ArtifactSources{{ + Artifact: portainer.Artifact{EdgeStackID: 7, ConfigHash: "old-hash"}, + }}, + } + err := tx.Workflow().Create(wf) + require.NoError(t, err) + workflowID = wf.ID + + return nil + }) + require.NoError(t, err) + + err = store.UpdateTx(func(tx dataservices.DataStoreTx) error { + return UpdateArtifactForEdgeStack(tx, workflowID, 7, func(a *portainer.Artifact) { + a.ConfigHash = "new-hash" + }) + }) + require.NoError(t, err) + + wf, err := store.Workflow().Read(workflowID) + require.NoError(t, err) + require.Equal(t, "new-hash", wf.Artifacts[0].Artifact.ConfigHash) +} + +func TestFindOrCreateGitSource_CreatesNewSource(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + var src *portainer.Source + err := store.UpdateTx(func(tx dataservices.DataStoreTx) error { + var txErr error + src, txErr = FindOrCreateGitSource(tx, &portainer.Source{ + Name: "my-repo", + Type: portainer.SourceTypeGit, + GitConfig: &gittypes.RepoConfig{ + URL: "https://github.com/example/repo", + }, + }) + return txErr + }) + require.NoError(t, err) + require.NotNil(t, src) + require.NotZero(t, src.ID) + require.Equal(t, "https://github.com/example/repo", src.GitConfig.URL) +} + +func TestFindOrCreateGitSource_ReusesExistingSourceForSameURLAndAuth(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + makeSource := func(tx dataservices.DataStoreTx) (*portainer.Source, error) { + return FindOrCreateGitSource(tx, &portainer.Source{ + Type: portainer.SourceTypeGit, + GitConfig: &gittypes.RepoConfig{ + URL: "https://github.com/example/repo", + }, + }) + } + + var firstID portainer.SourceID + err := store.UpdateTx(func(tx dataservices.DataStoreTx) error { + s, err := makeSource(tx) + if err != nil { + return err + } + firstID = s.ID + + return nil + }) + require.NoError(t, err) + + var secondID portainer.SourceID + err = store.UpdateTx(func(tx dataservices.DataStoreTx) error { + s, err := makeSource(tx) + if err != nil { + return err + } + secondID = s.ID + + return nil + }) + require.NoError(t, err) + require.Equal(t, firstID, secondID) + + sources, err := store.Source().ReadAll() + require.NoError(t, err) + require.Len(t, sources, 1) +} + +func TestFindOrCreateGitSource_DifferentAuthCreatesNewSource(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + err := store.UpdateTx(func(tx dataservices.DataStoreTx) error { + _, txErr := FindOrCreateGitSource(tx, &portainer.Source{ + Type: portainer.SourceTypeGit, + GitConfig: &gittypes.RepoConfig{ + URL: "https://github.com/example/repo", + Authentication: &gittypes.GitAuthentication{Username: "alice", Password: "pass1"}, + }, + }) + return txErr + }) + require.NoError(t, err) + + err = store.UpdateTx(func(tx dataservices.DataStoreTx) error { + _, txErr := FindOrCreateGitSource(tx, &portainer.Source{ + Type: portainer.SourceTypeGit, + GitConfig: &gittypes.RepoConfig{ + URL: "https://github.com/example/repo", + Authentication: &gittypes.GitAuthentication{Username: "bob", Password: "pass2"}, + }, + }) + return txErr + }) + require.NoError(t, err) + + sources, err := store.Source().ReadAll() + require.NoError(t, err) + require.Len(t, sources, 2) +} + +func TestSaveWorkflowGitConfig_UpdatesArtifactAndSourceWhenURLUnchanged(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + var workflowID portainer.WorkflowID + var sourceID portainer.SourceID + + err := store.UpdateTx(func(tx dataservices.DataStoreTx) error { + src := &portainer.Source{ + Type: portainer.SourceTypeGit, + GitConfig: &gittypes.RepoConfig{ + URL: "https://github.com/example/repo", + TLSSkipVerify: false, + Authentication: &gittypes.GitAuthentication{ + Username: "old-user", + Password: "old-pass", + }, + }, + } + err := tx.Source().Create(src) + require.NoError(t, err) + sourceID = src.ID + + wf := &portainer.Workflow{ + Artifacts: []portainer.ArtifactSources{{ + Artifact: portainer.Artifact{ + StackID: 1, + ReferenceName: "refs/heads/main", + ConfigFilePath: "docker-compose.yml", + ConfigHash: "old-hash", + }, + SourceIDs: []portainer.SourceID{sourceID}, + }}, + } + err = tx.Workflow().Create(wf) + require.NoError(t, err) + workflowID = wf.ID + + return nil + }) + require.NoError(t, err) + + newCfg := &gittypes.RepoConfig{ + URL: "https://github.com/example/repo", + TLSSkipVerify: true, + Authentication: &gittypes.GitAuthentication{ + Username: "new-user", + Password: "new-pass", + }, + ReferenceName: "refs/heads/dev", + ConfigFilePath: "compose.yml", + ConfigHash: "new-hash", + } + + err = store.UpdateTx(func(tx dataservices.DataStoreTx) error { + return SaveWorkflowGitConfig(tx, workflowID, func(a portainer.Artifact) bool { + return a.StackID == 1 + }, sourceID, newCfg) + }) + require.NoError(t, err) + + wf, err := store.Workflow().Read(workflowID) + require.NoError(t, err) + require.Equal(t, "refs/heads/dev", wf.Artifacts[0].Artifact.ReferenceName) + require.Equal(t, "compose.yml", wf.Artifacts[0].Artifact.ConfigFilePath) + require.Equal(t, "new-hash", wf.Artifacts[0].Artifact.ConfigHash) + require.Equal(t, sourceID, wf.Artifacts[0].SourceIDs[0]) + + src, err := store.Source().Read(sourceID) + require.NoError(t, err) + require.Equal(t, "new-user", src.GitConfig.Authentication.Username) + require.Equal(t, "new-pass", src.GitConfig.Authentication.Password) + require.True(t, src.GitConfig.TLSSkipVerify) +} + +func TestSaveWorkflowGitConfig_CreatesNewSourceOnURLChange(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + var workflowID portainer.WorkflowID + var oldSourceID portainer.SourceID + + err := store.UpdateTx(func(tx dataservices.DataStoreTx) error { + src := &portainer.Source{ + Type: portainer.SourceTypeGit, + GitConfig: &gittypes.RepoConfig{URL: "https://github.com/example/old-repo"}, + } + err := tx.Source().Create(src) + require.NoError(t, err) + oldSourceID = src.ID + + wf := &portainer.Workflow{ + Artifacts: []portainer.ArtifactSources{{ + Artifact: portainer.Artifact{StackID: 1}, + SourceIDs: []portainer.SourceID{oldSourceID}, + }}, + } + err = tx.Workflow().Create(wf) + require.NoError(t, err) + workflowID = wf.ID + + return nil + }) + require.NoError(t, err) + + newCfg := &gittypes.RepoConfig{URL: "https://github.com/example/new-repo"} + + err = store.UpdateTx(func(tx dataservices.DataStoreTx) error { + return SaveWorkflowGitConfig(tx, workflowID, func(a portainer.Artifact) bool { + return a.StackID == 1 + }, oldSourceID, newCfg) + }) + require.NoError(t, err) + + wf, err := store.Workflow().Read(workflowID) + require.NoError(t, err) + newSourceID := wf.Artifacts[0].SourceIDs[0] + require.NotEqual(t, oldSourceID, newSourceID) + + newSrc, err := store.Source().Read(newSourceID) + require.NoError(t, err) + require.Equal(t, "https://github.com/example/new-repo", newSrc.GitConfig.URL) +} + +func TestSaveWorkflowGitConfig_ReusesExistingSourceOnURLChange(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + var workflowID portainer.WorkflowID + var oldSourceID, existingSourceID portainer.SourceID + + err := store.UpdateTx(func(tx dataservices.DataStoreTx) error { + old := &portainer.Source{ + Type: portainer.SourceTypeGit, + GitConfig: &gittypes.RepoConfig{URL: "https://github.com/example/old-repo"}, + } + err := tx.Source().Create(old) + require.NoError(t, err) + oldSourceID = old.ID + + existing := &portainer.Source{ + Type: portainer.SourceTypeGit, + GitConfig: &gittypes.RepoConfig{URL: "https://github.com/example/shared-repo"}, + } + err = tx.Source().Create(existing) + require.NoError(t, err) + existingSourceID = existing.ID + + wf := &portainer.Workflow{ + Artifacts: []portainer.ArtifactSources{{ + Artifact: portainer.Artifact{StackID: 1}, + SourceIDs: []portainer.SourceID{oldSourceID}, + }}, + } + err = tx.Workflow().Create(wf) + require.NoError(t, err) + workflowID = wf.ID + + return nil + }) + require.NoError(t, err) + + newCfg := &gittypes.RepoConfig{URL: "https://github.com/example/shared-repo"} + + err = store.UpdateTx(func(tx dataservices.DataStoreTx) error { + return SaveWorkflowGitConfig(tx, workflowID, func(a portainer.Artifact) bool { + return a.StackID == 1 + }, oldSourceID, newCfg) + }) + require.NoError(t, err) + + wf, err := store.Workflow().Read(workflowID) + require.NoError(t, err) + require.Equal(t, existingSourceID, wf.Artifacts[0].SourceIDs[0]) + + sources, err := store.Source().ReadAll() + require.NoError(t, err) + require.Len(t, sources, 2) +} + +func TestSaveWorkflowGitConfig_NilGitConfigReturnsError(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + var workflowID portainer.WorkflowID + var sourceID portainer.SourceID + + err := store.UpdateTx(func(tx dataservices.DataStoreTx) error { + src := &portainer.Source{Type: portainer.SourceTypeGit} + err := tx.Source().Create(src) + require.NoError(t, err) + sourceID = src.ID + + wf := &portainer.Workflow{ + Artifacts: []portainer.ArtifactSources{{ + Artifact: portainer.Artifact{StackID: 1}, + SourceIDs: []portainer.SourceID{sourceID}, + }}, + } + err = tx.Workflow().Create(wf) + require.NoError(t, err) + workflowID = wf.ID + + return nil + }) + require.NoError(t, err) + + err = store.UpdateTx(func(tx dataservices.DataStoreTx) error { + return SaveWorkflowGitConfig(tx, workflowID, func(a portainer.Artifact) bool { + return a.StackID == 1 + }, sourceID, &gittypes.RepoConfig{URL: "https://github.com/example/repo"}) + }) + require.Error(t, err) +} + +func TestSaveWorkflowGitConfig_OnlyMatchingArtifactUpdated(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + var workflowID portainer.WorkflowID + var sourceID portainer.SourceID + + err := store.UpdateTx(func(tx dataservices.DataStoreTx) error { + src := &portainer.Source{ + Type: portainer.SourceTypeGit, + GitConfig: &gittypes.RepoConfig{URL: "https://github.com/example/repo"}, + } + err := tx.Source().Create(src) + require.NoError(t, err) + sourceID = src.ID + + wf := &portainer.Workflow{ + Artifacts: []portainer.ArtifactSources{ + { + Artifact: portainer.Artifact{StackID: 1, ConfigHash: "hash-1"}, + SourceIDs: []portainer.SourceID{sourceID}, + }, + { + Artifact: portainer.Artifact{StackID: 2, ConfigHash: "hash-2"}, + SourceIDs: []portainer.SourceID{sourceID}, + }, + }, + } + err = tx.Workflow().Create(wf) + require.NoError(t, err) + workflowID = wf.ID + + return nil + }) + require.NoError(t, err) + + err = store.UpdateTx(func(tx dataservices.DataStoreTx) error { + return SaveWorkflowGitConfig(tx, workflowID, func(a portainer.Artifact) bool { + return a.StackID == 1 + }, sourceID, &gittypes.RepoConfig{ + URL: "https://github.com/example/repo", + ConfigHash: "updated-hash", + }) + }) + require.NoError(t, err) + + wf, err := store.Workflow().Read(workflowID) + require.NoError(t, err) + require.Equal(t, "updated-hash", wf.Artifacts[0].Artifact.ConfigHash) + require.Equal(t, "hash-2", wf.Artifacts[1].Artifact.ConfigHash) +} + +func TestFindOrCreateGitSource_StripsEmbeddedCredentialsFromURL(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + var src *portainer.Source + err := store.UpdateTx(func(tx dataservices.DataStoreTx) error { + var txErr error + src, txErr = FindOrCreateGitSource(tx, &portainer.Source{ + Type: portainer.SourceTypeGit, + GitConfig: &gittypes.RepoConfig{ + URL: "https://user:secret@github.com/example/repo", + }, + }) + return txErr + }) + require.NoError(t, err) + require.Equal(t, "https://github.com/example/repo", src.GitConfig.URL) +} diff --git a/api/http/handler/gitops/sources/create_git.go b/api/http/handler/gitops/sources/create_git.go index 87bf3439a..23ae0309e 100644 --- a/api/http/handler/gitops/sources/create_git.go +++ b/api/http/handler/gitops/sources/create_git.go @@ -25,7 +25,6 @@ type GitAuthenticationPayload struct { type GitSourceCreatePayload struct { Name string `json:"name"` URL string `json:"url"` - ReferenceName string `json:"referenceName"` TLSSkipVerify bool `json:"tlsSkipVerify"` Authentication *GitAuthenticationPayload `json:"authentication"` ClearAuthentication bool `json:"clearAuthentication"` @@ -44,7 +43,6 @@ func (payload *GitSourceCreatePayload) Validate(_ *http.Request) error { func BuildGitSource(payload GitSourceCreatePayload) *portainer.Source { gitConfig := &gittypes.RepoConfig{ URL: payload.URL, - ReferenceName: payload.ReferenceName, TLSSkipVerify: payload.TLSSkipVerify, } diff --git a/api/http/handler/gitops/sources/delete.go b/api/http/handler/gitops/sources/delete.go index 1b2e53c64..381a15adf 100644 --- a/api/http/handler/gitops/sources/delete.go +++ b/api/http/handler/gitops/sources/delete.go @@ -49,7 +49,9 @@ func (h *Handler) sourceDelete(w http.ResponseWriter, r *http.Request) *httperro } for _, wf := range workflows { - if slices.Contains(wf.SourceIDs, portainer.SourceID(sourceID)) { + if slices.ContainsFunc(wf.Artifacts, func(as portainer.ArtifactSources) bool { + return slices.Contains(as.SourceIDs, portainer.SourceID(sourceID)) + }) { return ErrSourceInUse } } diff --git a/api/http/handler/gitops/sources/delete_test.go b/api/http/handler/gitops/sources/delete_test.go index 5b3d012d1..f148e962e 100644 --- a/api/http/handler/gitops/sources/delete_test.go +++ b/api/http/handler/gitops/sources/delete_test.go @@ -59,7 +59,7 @@ func TestSourceDelete_InUse(t *testing.T) { require.NoError(t, err) srcID = src.ID - wf := &portainer.Workflow{SourceIDs: []portainer.SourceID{src.ID}} + wf := &portainer.Workflow{Artifacts: []portainer.ArtifactSources{{SourceIDs: []portainer.SourceID{src.ID}}}} err = tx.Workflow().Create(wf) require.NoError(t, err) diff --git a/api/http/handler/gitops/sources/get.go b/api/http/handler/gitops/sources/get.go index a83e1b108..fea6d0d0a 100644 --- a/api/http/handler/gitops/sources/get.go +++ b/api/http/handler/gitops/sources/get.go @@ -2,8 +2,10 @@ package sources import ( "net/http" + "strconv" portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/dataservices" gittypes "github.com/portainer/portainer/api/git/types" ce "github.com/portainer/portainer/api/gitops/workflows" "github.com/portainer/portainer/api/http/security" @@ -40,16 +42,17 @@ type SourceDetail struct { // @security ApiKeyAuth // @security jwt // @produce json -// @param id path string true "Source ID" +// @param id path int true "Source identifier" // @success 200 {object} SourceDetail +// @failure 400 "Invalid request" // @failure 403 "Access denied" // @failure 404 "Source not found" // @failure 500 "Server error" // @router /gitops/sources/{id} [get] func (h *Handler) getSource(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { - id, err := request.RetrieveRouteVariableValue(r, "id") + srcID, err := request.RetrieveNumericRouteVariableValue(r, "id") if err != nil { - return httperror.BadRequest("Invalid source ID", err) + return httperror.BadRequest("Invalid source identifier route variable", err) } securityContext, err := security.RetrieveRestrictedRequestContext(r) @@ -57,6 +60,18 @@ func (h *Handler) getSource(w http.ResponseWriter, r *http.Request) *httperror.H return httperror.InternalServerError("Unable to retrieve info from request context", err) } + var src *portainer.Source + + if err := h.dataStore.ViewTx(func(tx dataservices.DataStoreTx) error { + var err error + src, err = tx.Source().Read(portainer.SourceID(srcID)) + return err + }); h.dataStore.IsErrObjectNotFound(err) { + return httperror.NotFound("Unable to find a source with the specified identifier", err) + } else if err != nil { + return httperror.InternalServerError("Unable to retrieve source", err) + } + workflows, err := ce.FetchWorkflows(r.Context(), h.dataStore, h.gitService, h.k8sFactory, securityContext, nil) if err != nil { return httperror.InternalServerError("Unable to retrieve workflows", err) @@ -64,16 +79,26 @@ func (h *Handler) getSource(w http.ResponseWriter, r *http.Request) *httperror.H byID := workflowsBySourceID(workflows) - wfs, ok := byID[id] - if !ok || len(wfs) == 0 || wfs[0].GitConfig == nil { - return httperror.NotFound("Source not found", nil) + var wfs []ce.Workflow + if src.GitConfig != nil { + wfs = byID[sourceID(gitSourceKey(src.GitConfig))] + } + + var autoUpdate *portainer.AutoUpdateSettings + if len(wfs) > 0 { + autoUpdate = wfs[0].AutoUpdate + } + + id := strconv.Itoa(int(src.ID)) + url := "" + if src.GitConfig != nil { + url = src.GitConfig.URL } - url := wfs[0].GitConfig.URL detail := SourceDetail{ Source: buildSource(id, url, wfs), - Connection: buildConnectionInfo(wfs[0].GitConfig), - AutoUpdate: buildAutoUpdateInfo(wfs[0].AutoUpdate), + Connection: buildConnectionInfo(src.GitConfig), + AutoUpdate: buildAutoUpdateInfo(autoUpdate), Workflows: redactWorkflowCredentials(wfs), } return response.JSON(w, detail) diff --git a/api/http/handler/gitops/sources/get_test.go b/api/http/handler/gitops/sources/get_test.go index e0b48919e..c82bce7a2 100644 --- a/api/http/handler/gitops/sources/get_test.go +++ b/api/http/handler/gitops/sources/get_test.go @@ -3,6 +3,7 @@ package sources import ( "net/http" "net/http/httptest" + "strconv" "testing" portainer "github.com/portainer/portainer/api" @@ -24,7 +25,7 @@ func TestGetSource_NotFound(t *testing.T) { h := newTestHandler(t, store) rr := httptest.NewRecorder() - h.ServeHTTP(rr, buildGetReq(t, 1, "nonexistent-id")) + h.ServeHTTP(rr, buildGetReq(t, 1, "999")) assert.Equal(t, http.StatusNotFound, rr.Code) } @@ -39,20 +40,21 @@ func TestGetSource_ReturnsDetail(t *testing.T) { TLSSkipVerify: true, } + var srcID portainer.SourceID + require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { stack := &portainer.Stack{ID: 1, Name: "my-stack"} - createGitWorkflow(t, tx, stack, cfg) + srcID = createGitWorkflow(t, tx, stack, cfg) require.NoError(t, tx.Stack().Create(stack)) return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) })) - id := sourceID(gitSourceKey(cfg)) h := newTestHandler(t, store) rr := httptest.NewRecorder() - h.ServeHTTP(rr, buildGetReq(t, 1, id)) + h.ServeHTTP(rr, buildGetReq(t, 1, strconv.Itoa(int(srcID)))) detail := decodeSourceDetail(t, rr) - assert.Equal(t, id, detail.ID) + assert.Equal(t, strconv.Itoa(int(srcID)), detail.ID) assert.Equal(t, "repo", detail.Name) assert.Equal(t, 1, detail.UsedBy) require.NotNil(t, detail.Connection) @@ -72,17 +74,18 @@ func TestGetSource_RedactsCredentials(t *testing.T) { Authentication: &gittypes.GitAuthentication{Username: "user", Password: "s3cr3t"}, } + var srcID portainer.SourceID + require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { stack := &portainer.Stack{ID: 1, Name: "secure-stack"} - createGitWorkflow(t, tx, stack, cfg) + srcID = createGitWorkflow(t, tx, stack, cfg) require.NoError(t, tx.Stack().Create(stack)) return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) })) - id := sourceID(gitSourceKey(cfg)) h := newTestHandler(t, store) rr := httptest.NewRecorder() - h.ServeHTTP(rr, buildGetReq(t, 1, id)) + h.ServeHTTP(rr, buildGetReq(t, 1, strconv.Itoa(int(srcID)))) detail := decodeSourceDetail(t, rr) require.Len(t, detail.Workflows, 1) @@ -97,21 +100,23 @@ func TestGetSource_AutoUpdate(t *testing.T) { _, store := datastore.MustNewTestStore(t, false, true) cfg := gitCfg("https://github.com/org/polled") + + var srcID portainer.SourceID + require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { stack := &portainer.Stack{ ID: 1, Name: "polled-stack", AutoUpdate: &portainer.AutoUpdateSettings{Interval: "5m"}, } - createGitWorkflow(t, tx, stack, cfg) + srcID = createGitWorkflow(t, tx, stack, cfg) require.NoError(t, tx.Stack().Create(stack)) return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) })) - id := sourceID(gitSourceKey(cfg)) h := newTestHandler(t, store) rr := httptest.NewRecorder() - h.ServeHTTP(rr, buildGetReq(t, 1, id)) + h.ServeHTTP(rr, buildGetReq(t, 1, strconv.Itoa(int(srcID)))) detail := decodeSourceDetail(t, rr) require.NotNil(t, detail.AutoUpdate) diff --git a/api/http/handler/gitops/sources/helpers_test.go b/api/http/handler/gitops/sources/helpers_test.go index 52fef2aa1..6acb04975 100644 --- a/api/http/handler/gitops/sources/helpers_test.go +++ b/api/http/handler/gitops/sources/helpers_test.go @@ -19,7 +19,7 @@ import ( // createGitWorkflow creates a Source and Workflow for the given config and // wires them up by setting stack.WorkflowID before creating the stack. -func createGitWorkflow(t *testing.T, tx dataservices.DataStoreTx, stack *portainer.Stack, cfg *gittypes.RepoConfig) { +func createGitWorkflow(t *testing.T, tx dataservices.DataStoreTx, stack *portainer.Stack, cfg *gittypes.RepoConfig) portainer.SourceID { t.Helper() src := &portainer.Source{ @@ -30,11 +30,20 @@ func createGitWorkflow(t *testing.T, tx dataservices.DataStoreTx, stack *portain require.NoError(t, tx.Source().Create(src)) wf := &portainer.Workflow{ - SourceIDs: []portainer.SourceID{src.ID}, + Artifacts: []portainer.ArtifactSources{{ + Artifact: portainer.Artifact{ + StackID: stack.ID, + ReferenceName: cfg.ReferenceName, + ConfigFilePath: cfg.ConfigFilePath, + }, + SourceIDs: []portainer.SourceID{src.ID}, + }}, } require.NoError(t, tx.Workflow().Create(wf)) stack.WorkflowID = wf.ID + + return src.ID } func newTestHandler(t *testing.T, store dataservices.DataStore) *Handler { diff --git a/api/http/handler/gitops/sources/list_test.go b/api/http/handler/gitops/sources/list_test.go index 11fdfff16..f81097520 100644 --- a/api/http/handler/gitops/sources/list_test.go +++ b/api/http/handler/gitops/sources/list_test.go @@ -22,9 +22,9 @@ func TestSourcesList_GroupsByURLAndCredentials(t *testing.T) { src := &portainer.Source{Name: "repo", Type: portainer.SourceTypeGit, GitConfig: cfg} require.NoError(t, tx.Source().Create(src)) - wfA := &portainer.Workflow{SourceIDs: []portainer.SourceID{src.ID}} + wfA := &portainer.Workflow{Artifacts: []portainer.ArtifactSources{{SourceIDs: []portainer.SourceID{src.ID}}}} require.NoError(t, tx.Workflow().Create(wfA)) - wfB := &portainer.Workflow{SourceIDs: []portainer.SourceID{src.ID}} + wfB := &portainer.Workflow{Artifacts: []portainer.ArtifactSources{{SourceIDs: []portainer.SourceID{src.ID}}}} require.NoError(t, tx.Workflow().Create(wfB)) require.NoError(t, tx.Stack().Create(&portainer.Stack{ID: 1, Name: "stack-a", WorkflowID: wfA.ID})) diff --git a/api/http/handler/gitops/sources/update_git.go b/api/http/handler/gitops/sources/update_git.go index c59659a51..4a7d2bc47 100644 --- a/api/http/handler/gitops/sources/update_git.go +++ b/api/http/handler/gitops/sources/update_git.go @@ -3,7 +3,6 @@ package sources import ( "errors" "net/http" - "strings" portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/dataservices" @@ -13,7 +12,10 @@ import ( "github.com/portainer/portainer/pkg/libhttp/response" ) -var ErrNotGitSource = errors.New("source is not a Git source") +var ( + ErrNotGitSource = errors.New("source is not a Git source") + ErrDuplicateSourceURL = errors.New("a source with this URL already exists") +) // @id GitOpsSourcesUpdateGit // @summary Update a Git source @@ -30,6 +32,7 @@ var ErrNotGitSource = errors.New("source is not a Git source") // @failure 400 "Invalid request payload" // @failure 403 "Access denied" // @failure 404 "Source not found" +// @failure 409 "A source with this URL already exists" // @failure 500 "Server error" // @router /gitops/sources/{id} [put] func (h *Handler) gitSourceUpdate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { @@ -57,34 +60,38 @@ func (h *Handler) gitSourceUpdate(w http.ResponseWriter, r *http.Request) *httpe return ErrNotGitSource } - name := payload.Name - if strings.TrimSpace(name) == "" { - name = gittypes.RepoName(payload.URL) + normalizedURL, err := gittypes.NormalizeURL(payload.URL) + if err != nil { + return err } - src.Name = name + existing, err := tx.Source().ReadAll(func(s portainer.Source) bool { + if s.ID == src.ID || s.Type != portainer.SourceTypeGit || s.GitConfig == nil { + return false + } + + normalized, err := gittypes.NormalizeURL(s.GitConfig.URL) + + return err == nil && normalized == normalizedURL + }) + if err != nil { + return err + } + + if len(existing) > 0 { + return ErrDuplicateSourceURL + } var existingAuth *gittypes.GitAuthentication if src.GitConfig != nil { existingAuth = src.GitConfig.Authentication } - src.GitConfig = &gittypes.RepoConfig{ - URL: payload.URL, - ReferenceName: payload.ReferenceName, - TLSSkipVerify: payload.TLSSkipVerify, - } + updated := BuildGitSource(payload) + src.Name = updated.Name + src.GitConfig = updated.GitConfig - if payload.Authentication != nil { - src.GitConfig.Authentication = &gittypes.GitAuthentication{ - Username: payload.Authentication.Username, - Password: payload.Authentication.Password, - Provider: payload.Authentication.Provider, - AuthorizationType: payload.Authentication.AuthorizationType, - } - } else if payload.ClearAuthentication { - src.GitConfig.Authentication = nil - } else { + if payload.Authentication == nil && !payload.ClearAuthentication { src.GitConfig.Authentication = existingAuth } @@ -93,6 +100,8 @@ func (h *Handler) gitSourceUpdate(w http.ResponseWriter, r *http.Request) *httpe return httperror.NotFound("Unable to find a source with the specified identifier", err) } else if errors.Is(err, ErrNotGitSource) { return httperror.BadRequest("Source is not a Git source", err) + } else if errors.Is(err, ErrDuplicateSourceURL) { + return httperror.Conflict("A source with this URL already exists", err) } else if err != nil { return httperror.InternalServerError("Unable to update source", err) } diff --git a/api/http/handler/gitops/sources/update_git_test.go b/api/http/handler/gitops/sources/update_git_test.go index d5ad1c4c2..857aa6a0e 100644 --- a/api/http/handler/gitops/sources/update_git_test.go +++ b/api/http/handler/gitops/sources/update_git_test.go @@ -196,6 +196,43 @@ func TestGitSourceUpdate_NotFound(t *testing.T) { require.Equal(t, http.StatusNotFound, rr.Code) } +func TestGitSourceUpdate_ConflictOnDuplicateURL(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + var srcID portainer.SourceID + require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { + existing := &portainer.Source{ + Name: "existing", + Type: portainer.SourceTypeGit, + GitConfig: &gittypes.RepoConfig{ + URL: "https://github.com/org/existing.git", + }, + } + err := tx.Source().Create(existing) + require.NoError(t, err) + + src := &portainer.Source{Name: "other", Type: portainer.SourceTypeGit} + err = tx.Source().Create(src) + require.NoError(t, err) + srcID = src.ID + + return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) + })) + + h := newTestHandler(t, store) + + body, err := json.Marshal(GitSourceCreatePayload{ + URL: "https://github.com/org/existing.git", + }) + require.NoError(t, err) + + rr := httptest.NewRecorder() + h.ServeHTTP(rr, buildUpdateReq(t, 1, int(srcID), body)) + + require.Equal(t, http.StatusConflict, rr.Code) +} + func TestGitSourceUpdate_NotGitSource(t *testing.T) { t.Parallel() _, store := datastore.MustNewTestStore(t, false, true) diff --git a/api/http/handler/gitops/workflows/helpers_test.go b/api/http/handler/gitops/workflows/helpers_test.go index f9489a577..289ed877d 100644 --- a/api/http/handler/gitops/workflows/helpers_test.go +++ b/api/http/handler/gitops/workflows/helpers_test.go @@ -50,7 +50,15 @@ func createGitStack(t *testing.T, tx dataservices.DataStoreTx, stack *portainer. src := &portainer.Source{GitConfig: stack.GitConfig, Type: portainer.SourceTypeGit} require.NoError(t, tx.Source().Create(src)) - wf := &portainer.Workflow{SourceIDs: []portainer.SourceID{src.ID}} + wf := &portainer.Workflow{Artifacts: []portainer.ArtifactSources{{ + Artifact: portainer.Artifact{ + StackID: stack.ID, + ReferenceName: stack.GitConfig.ReferenceName, + ConfigFilePath: stack.GitConfig.ConfigFilePath, + ConfigHash: stack.GitConfig.ConfigHash, + }, + SourceIDs: []portainer.SourceID{src.ID}, + }}} require.NoError(t, tx.Workflow().Create(wf)) stack.WorkflowID = wf.ID diff --git a/api/http/handler/stacks/response.go b/api/http/handler/stacks/response.go index 129a9f2e3..5f0316031 100644 --- a/api/http/handler/stacks/response.go +++ b/api/http/handler/stacks/response.go @@ -1,42 +1,36 @@ package stacks import ( - "fmt" - portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/dataservices" gittypes "github.com/portainer/portainer/api/git/types" + "github.com/portainer/portainer/api/gitops/workflows" ) -// loadGitConfigFromSource reads the GitConfig and SourceID from the first Git-type Source in the workflow. -func loadGitConfigFromSource(tx dataservices.DataStoreTx, workflowID portainer.WorkflowID) (*gittypes.RepoConfig, portainer.SourceID, error) { - src, err := dataservices.GitSourceForWorkflow(tx, workflowID) +// loadGitConfigForStack reads the merged GitConfig (Source URL/auth/TLS + Artifact ref/path/hash) +// and the SourceID for the given stack. +func loadGitConfigForStack(tx dataservices.DataStoreTx, workflowID portainer.WorkflowID, stackID portainer.StackID) (*gittypes.RepoConfig, portainer.SourceID, error) { + src, artifact, err := workflows.GitSourceAndArtifactForStack(tx, workflowID, stackID) if err != nil || src == nil { return nil, 0, err } - return src.GitConfig, src.ID, nil + return workflows.MergeSourceAndArtifact(src, artifact), src.ID, nil } -// saveSourceGitConfig updates the GitConfig on the Source record identified by sourceID within a transaction. -func saveSourceGitConfig(tx dataservices.DataStoreTx, sourceID portainer.SourceID, gitConfig *gittypes.RepoConfig) error { - src, err := tx.Source().Read(sourceID) - if err != nil { - return fmt.Errorf("failed to read source: %w", err) - } - - src.GitConfig = gitConfig - - return tx.Source().Update(src.ID, src) +func saveStackGitConfig(tx dataservices.DataStoreTx, workflowID portainer.WorkflowID, stackID portainer.StackID, oldSourceID portainer.SourceID, cfg *gittypes.RepoConfig) error { + return workflows.SaveWorkflowGitConfig(tx, workflowID, func(a portainer.Artifact) bool { + return a.StackID == stackID + }, oldSourceID, cfg) } -// fillStackGitConfig loads GitConfig from Source and sets it on the stack for backwards-compatible responses. +// fillStackGitConfig populates stack.GitConfig from the merged Source+Artifact for backwards-compatible responses. func fillStackGitConfig(tx dataservices.DataStoreTx, stack *portainer.Stack) error { if stack.WorkflowID == 0 { return nil } - gitConfig, _, err := loadGitConfigFromSource(tx, stack.WorkflowID) + gitConfig, _, err := loadGitConfigForStack(tx, stack.WorkflowID, stack.ID) if err != nil { return err } diff --git a/api/http/handler/stacks/stack_file.go b/api/http/handler/stacks/stack_file.go index c43e7ae2e..19519b993 100644 --- a/api/http/handler/stacks/stack_file.go +++ b/api/http/handler/stacks/stack_file.go @@ -97,7 +97,7 @@ func (handler *Handler) stackFile(w http.ResponseWriter, r *http.Request) *httpe var gitConfig *gittypes.RepoConfig if stack.WorkflowID != 0 { var err error - gitConfig, _, err = loadGitConfigFromSource(handler.DataStore, stack.WorkflowID) + gitConfig, _, err = loadGitConfigForStack(handler.DataStore, stack.WorkflowID, stack.ID) if err != nil { return httperror.InternalServerError("Unable to load git config for stack", err) } diff --git a/api/http/handler/stacks/stack_file_test.go b/api/http/handler/stacks/stack_file_test.go index f88c15fd7..5a931017d 100644 --- a/api/http/handler/stacks/stack_file_test.go +++ b/api/http/handler/stacks/stack_file_test.go @@ -36,6 +36,8 @@ func TestStackFile_GitPendingRedeploy_Returns409(t *testing.T) { handler.FileService = fileService handler.DataStore = store + const stackID = portainer.StackID(1) + src := &portainer.Source{ Type: portainer.SourceTypeGit, GitConfig: &gittypes.RepoConfig{ @@ -45,11 +47,14 @@ func TestStackFile_GitPendingRedeploy_Returns409(t *testing.T) { } require.NoError(t, store.Source().Create(src)) - wf := &portainer.Workflow{SourceIDs: []portainer.SourceID{src.ID}} + wf := &portainer.Workflow{Artifacts: []portainer.ArtifactSources{{ + Artifact: portainer.Artifact{StackID: stackID}, + SourceIDs: []portainer.SourceID{src.ID}, + }}} require.NoError(t, store.Workflow().Create(wf)) stack := &portainer.Stack{ - ID: 1, + ID: stackID, EndpointID: endpoint.ID, Type: portainer.DockerComposeStack, WorkflowID: wf.ID, diff --git a/api/http/handler/stacks/stack_update_git.go b/api/http/handler/stacks/stack_update_git.go index c01e80704..db762cc20 100644 --- a/api/http/handler/stacks/stack_update_git.go +++ b/api/http/handler/stacks/stack_update_git.go @@ -78,7 +78,7 @@ func (handler *Handler) stackUpdateGit(w http.ResponseWriter, r *http.Request) * return httperror.InternalServerError(msg, errors.New(msg)) } - gitConfig, sourceID, err := loadGitConfigFromSource(handler.DataStore, stack.WorkflowID) + gitConfig, sourceID, err := loadGitConfigForStack(handler.DataStore, stack.WorkflowID, stack.ID) if err != nil { return httperror.InternalServerError("Unable to load git config for stack", err) } @@ -227,7 +227,7 @@ func (handler *Handler) stackUpdateGit(w http.ResponseWriter, r *http.Request) * if err := tx.Stack().Update(stack.ID, stack); err != nil { return err } - if err := saveSourceGitConfig(tx, sourceID, gitConfig); err != nil { + if err := saveStackGitConfig(tx, stack.WorkflowID, stack.ID, sourceID, gitConfig); err != nil { return err } return fillStackGitConfig(tx, stack) diff --git a/api/http/handler/stacks/stack_update_git_redeploy.go b/api/http/handler/stacks/stack_update_git_redeploy.go index afe30b811..ff9be2597 100644 --- a/api/http/handler/stacks/stack_update_git_redeploy.go +++ b/api/http/handler/stacks/stack_update_git_redeploy.go @@ -9,6 +9,7 @@ import ( portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/dataservices" "github.com/portainer/portainer/api/git" + "github.com/portainer/portainer/api/gitops/workflows" httperrors "github.com/portainer/portainer/api/http/errors" "github.com/portainer/portainer/api/http/security" k "github.com/portainer/portainer/api/kubernetes" @@ -78,7 +79,7 @@ func (handler *Handler) stackGitRedeploy(w http.ResponseWriter, r *http.Request) return httperror.BadRequest("Stack is not created from git", errors.New("stack has no git workflow")) } - gitConfig, sourceID, err := loadGitConfigFromSource(handler.DataStore, stack.WorkflowID) + gitConfig, sourceID, err := loadGitConfigForStack(handler.DataStore, stack.WorkflowID, stack.ID) if err != nil { return httperror.InternalServerError("Unable to load git config for stack", err) } @@ -235,9 +236,9 @@ func (handler *Handler) stackGitRedeploy(w http.ResponseWriter, r *http.Request) return err } - gitConfig.ConfigHash = oldConfigHash - - return saveSourceGitConfig(tx, sourceID, gitConfig) + return workflows.UpdateArtifactForStack(tx, stack.WorkflowID, stack.ID, func(a *portainer.Artifact) { + a.ConfigHash = oldConfigHash + }) }); err != nil { log.Error().Err(err).Int("stack_id", int(stack.ID)).Msg("failed to revert config hash after failed redeploy") } @@ -252,9 +253,10 @@ func (handler *Handler) stackGitRedeploy(w http.ResponseWriter, r *http.Request) if err := tx.Stack().Update(stack.ID, stack); err != nil { return err } - if err := saveSourceGitConfig(tx, sourceID, gitConfig); err != nil { + if err := saveStackGitConfig(tx, stack.WorkflowID, stack.ID, sourceID, gitConfig); err != nil { return err } + return fillStackGitConfig(tx, stack) }); err != nil { deployGate.abortDeploy() diff --git a/api/http/handler/stacks/stack_update_git_test.go b/api/http/handler/stacks/stack_update_git_test.go index 185bb18e9..786acfa8d 100644 --- a/api/http/handler/stacks/stack_update_git_test.go +++ b/api/http/handler/stacks/stack_update_git_test.go @@ -33,6 +33,9 @@ func TestStackUpdateGitWebhookUniqueness(t *testing.T) { err = store.Endpoint().Create(endpoint) require.NoError(t, err) + const stack1ID = portainer.StackID(456) + const stack2ID = portainer.StackID(457) + src1 := &portainer.Source{ Type: portainer.SourceTypeGit, GitConfig: &gittypes.RepoConfig{URL: "https://github.com/portainer/portainer.git"}, @@ -40,7 +43,10 @@ func TestStackUpdateGitWebhookUniqueness(t *testing.T) { err = store.Source().Create(src1) require.NoError(t, err) - wf1 := &portainer.Workflow{SourceIDs: []portainer.SourceID{src1.ID}} + wf1 := &portainer.Workflow{Artifacts: []portainer.ArtifactSources{{ + Artifact: portainer.Artifact{StackID: stack1ID}, + SourceIDs: []portainer.SourceID{src1.ID}, + }}} err = store.Workflow().Create(wf1) require.NoError(t, err) @@ -51,12 +57,15 @@ func TestStackUpdateGitWebhookUniqueness(t *testing.T) { err = store.Source().Create(src2) require.NoError(t, err) - wf2 := &portainer.Workflow{SourceIDs: []portainer.SourceID{src2.ID}} + wf2 := &portainer.Workflow{Artifacts: []portainer.ArtifactSources{{ + Artifact: portainer.Artifact{StackID: stack2ID}, + SourceIDs: []portainer.SourceID{src2.ID}, + }}} err = store.Workflow().Create(wf2) require.NoError(t, err) stack1 := portainer.Stack{ - ID: 456, + ID: stack1ID, EndpointID: endpoint.ID, WorkflowID: wf1.ID, AutoUpdate: &portainer.AutoUpdateSettings{ @@ -68,7 +77,7 @@ func TestStackUpdateGitWebhookUniqueness(t *testing.T) { require.NoError(t, err) stack2 := stack1 - stack2.ID++ + stack2.ID = stack2ID stack2.AutoUpdate = nil stack2.WorkflowID = wf2.ID diff --git a/api/http/handler/stacks/update_kubernetes_stack.go b/api/http/handler/stacks/update_kubernetes_stack.go index 02cda0b46..7b144c058 100644 --- a/api/http/handler/stacks/update_kubernetes_stack.go +++ b/api/http/handler/stacks/update_kubernetes_stack.go @@ -55,7 +55,7 @@ func (payload *kubernetesGitStackUpdatePayload) Validate(r *http.Request) error func (handler *Handler) updateKubernetesStack(tx dataservices.DataStoreTx, r *http.Request, stack *portainer.Stack, endpoint *portainer.Endpoint, gate *deployGate) *httperror.HandlerError { if stack.WorkflowID != 0 { - gitConfig, sourceID, err := loadGitConfigFromSource(tx, stack.WorkflowID) + gitConfig, sourceID, err := loadGitConfigForStack(tx, stack.WorkflowID, stack.ID) if err != nil { return httperror.InternalServerError("Unable to load git config for stack", err) } @@ -111,7 +111,7 @@ func (handler *Handler) updateKubernetesStack(tx dataservices.DataStoreTx, r *ht stack.AutoUpdate.JobID = jobID } - if err := saveSourceGitConfig(tx, sourceID, gitConfig); err != nil { + if err := saveStackGitConfig(tx, stack.WorkflowID, stack.ID, sourceID, gitConfig); err != nil { return httperror.InternalServerError("Unable to update source git config", err) } diff --git a/api/portainer.go b/api/portainer.go index acfdbaf66..a449e9aa5 100644 --- a/api/portainer.go +++ b/api/portainer.go @@ -1544,10 +1544,26 @@ type ( // WebhookType represents the type of resource a webhook is related to WebhookType int + // Artifact represents a GitOps artifact produced by a source + Artifact struct { + StackID StackID `json:"stackId,omitempty"` + EdgeStackID EdgeStackID `json:"edgeStackId,omitempty"` + ReferenceName string `json:"referenceName,omitempty" example:"refs/heads/main"` + ConfigFilePath string `json:"configFilePath,omitempty" example:"portainer.yaml"` + ConfigHash string `json:"configHash,omitempty" example:"abc123"` + } + + // ArtifactSources is one entry in a Workflow's ordered artifact-to-sources mapping + ArtifactSources struct { + Artifact Artifact `json:"artifact"` + SourceIDs []SourceID `json:"sourceIds,omitempty"` + } + // Workflow represents a GitOps workflow Workflow struct { - ID WorkflowID `json:"id" example:"1"` - SourceIDs []SourceID `json:"sourceIds"` + ID WorkflowID `json:"id" example:"1"` + Name string `json:"name,omitempty" example:"my-workflow"` + Artifacts []ArtifactSources `json:"artifacts,omitempty"` } WorkflowID int diff --git a/api/stacks/deployments/deploy.go b/api/stacks/deployments/deploy.go index 0e6b1764b..9e54fcc74 100644 --- a/api/stacks/deployments/deploy.go +++ b/api/stacks/deployments/deploy.go @@ -12,6 +12,7 @@ import ( "github.com/portainer/portainer/api/crypto" "github.com/portainer/portainer/api/dataservices" "github.com/portainer/portainer/api/git/update" + "github.com/portainer/portainer/api/gitops/workflows" "github.com/portainer/portainer/api/http/security" "github.com/portainer/portainer/api/scheduler" "github.com/portainer/portainer/api/stacks/stackutils" @@ -120,7 +121,7 @@ func redeployWhenChangedSecondStage( user *portainer.User, endpoint *portainer.Endpoint, ) error { - gitSrc, err := dataservices.GitSourceForWorkflow(datastore, stack.WorkflowID) + gitSrc, artifact, err := workflows.GitSourceAndArtifactForStack(datastore, stack.WorkflowID, stack.ID) if err != nil { return errors.WithMessagef(err, "failed to load git config for stack %v", stack.ID) } @@ -129,7 +130,7 @@ func redeployWhenChangedSecondStage( return nil } - gitConfig := gitSrc.GitConfig + gitConfig := workflows.MergeSourceAndArtifact(gitSrc, artifact) var gitCommitChangedOrForceUpdate bool @@ -222,14 +223,11 @@ func redeployWhenChangedSecondStage( return err } - src, err := tx.Source().Read(gitSrc.ID) - if err != nil { - return errors.Wrap(err, "failed to read source") - } + newHash := gitConfig.ConfigHash - src.GitConfig = gitConfig - - return tx.Source().Update(src.ID, src) + return workflows.UpdateArtifactForStack(tx, stack.WorkflowID, stack.ID, func(a *portainer.Artifact) { + a.ConfigHash = newHash + }) }); err != nil { return errors.WithMessagef(err, "failed to update the stack %v", stack.ID) } diff --git a/api/stacks/deployments/deploy_test.go b/api/stacks/deployments/deploy_test.go index 2f14f7884..023742ce5 100644 --- a/api/stacks/deployments/deploy_test.go +++ b/api/stacks/deployments/deploy_test.go @@ -12,6 +12,7 @@ import ( portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/crypto" + "github.com/portainer/portainer/api/dataservices" "github.com/portainer/portainer/api/datastore" gittypes "github.com/portainer/portainer/api/git/types" "github.com/portainer/portainer/api/internal/testhelpers" @@ -194,20 +195,30 @@ func Test_redeployWhenChanged_DoesNothingWhenNoGitChanges(t *testing.T) { err := store.User().Create(admin) require.NoError(t, err, "error creating an admin") - err = store.Endpoint().Create(&portainer.Endpoint{ - ID: 0, - }) + err = store.Endpoint().Create(&portainer.Endpoint{ID: 0}) require.NoError(t, err, "error creating environment") + src := &portainer.Source{ + Type: portainer.SourceTypeGit, + GitConfig: &gittypes.RepoConfig{ + URL: "url", + ReferenceName: "ref", + ConfigHash: "oldHash", + }, + } + err = store.Source().Create(src) + require.NoError(t, err, "failed to create source") + + wf := &portainer.Workflow{Artifacts: []portainer.ArtifactSources{{SourceIDs: []portainer.SourceID{src.ID}}}} + err = store.Workflow().Create(wf) + require.NoError(t, err, "failed to create workflow") + err = store.Stack().Create(&portainer.Stack{ ID: 1, CreatedBy: "admin", ProjectPath: tmpDir, - GitConfig: &gittypes.RepoConfig{ - URL: "url", - ReferenceName: "ref", - ConfigHash: "oldHash", - }}) + WorkflowID: wf.ID, + }) require.NoError(t, err, "failed to create a test stack") err = RedeployWhenChanged(t.Context(), 1, nil, store, testhelpers.NewGitService(nil, "oldHash")) @@ -245,7 +256,10 @@ func Test_redeployWhenChanged_FailsWhenCannotClone(t *testing.T) { err = store.Source().Create(src) require.NoError(t, err, "failed to create source") - wf := &portainer.Workflow{SourceIDs: []portainer.SourceID{src.ID}} + wf := &portainer.Workflow{Artifacts: []portainer.ArtifactSources{{ + Artifact: portainer.Artifact{StackID: 1}, + SourceIDs: []portainer.SourceID{src.ID}, + }}} err = store.Workflow().Create(wf) require.NoError(t, err, "failed to create workflow") @@ -261,10 +275,10 @@ func Test_redeployWhenChanged_FailsWhenCannotClone(t *testing.T) { require.ErrorIs(t, err, cloneErr, "should failed to clone but didn't, check test setup") } -func Test_redeployWhenChanged(t *testing.T) { - t.Parallel() - _, store := datastore.MustNewTestStore(t, false, true) +func setupRedeployStore(t *testing.T, stackType portainer.StackType) (dataservices.DataStore, portainer.StackID) { + t.Helper() + _, store := datastore.MustNewTestStore(t, false, true) tmpDir := t.TempDir() err := store.Endpoint().Create(&portainer.Endpoint{ID: 1}) @@ -274,47 +288,61 @@ func Test_redeployWhenChanged(t *testing.T) { err = store.User().Create(&portainer.User{Username: username, Role: portainer.AdministratorRole}) require.NoError(t, err, "error creating a user") - stack := portainer.Stack{ - ID: 1, - EndpointID: 1, - ProjectPath: tmpDir, - UpdatedBy: username, + src := &portainer.Source{ + Type: portainer.SourceTypeGit, GitConfig: &gittypes.RepoConfig{ URL: "url", ReferenceName: "ref", ConfigHash: "oldHash", }, } + err = store.Source().Create(src) + require.NoError(t, err, "failed to create source") - err = store.Stack().Create(&stack) + wf := &portainer.Workflow{Artifacts: []portainer.ArtifactSources{{SourceIDs: []portainer.SourceID{src.ID}}}} + err = store.Workflow().Create(wf) + require.NoError(t, err, "failed to create workflow") + + const stackID portainer.StackID = 1 + + err = store.Stack().Create(&portainer.Stack{ + ID: stackID, + EndpointID: 1, + ProjectPath: tmpDir, + UpdatedBy: username, + WorkflowID: wf.ID, + Type: stackType, + }) require.NoError(t, err, "failed to create a test stack") - t.Run("can deploy docker compose stack", func(t *testing.T) { - stack.Type = portainer.DockerComposeStack - err = store.Stack().Update(stack.ID, &stack) - require.NoError(t, err) + return store, stackID +} - err = RedeployWhenChanged(t.Context(), 1, noopDeployer{}, store, testhelpers.NewGitService(nil, "newHash")) - require.NoError(t, err) - }) +func Test_redeployWhenChanged_DockerComposeStack(t *testing.T) { + t.Parallel() - t.Run("can deploy docker swarm stack", func(t *testing.T) { - stack.Type = portainer.DockerSwarmStack - err = store.Stack().Update(stack.ID, &stack) - require.NoError(t, err) + store, stackID := setupRedeployStore(t, portainer.DockerComposeStack) - err = RedeployWhenChanged(t.Context(), 1, noopDeployer{}, store, testhelpers.NewGitService(nil, "newHash")) - require.NoError(t, err) - }) + err := RedeployWhenChanged(t.Context(), stackID, noopDeployer{}, store, testhelpers.NewGitService(nil, "newHash")) + require.NoError(t, err) +} - t.Run("can deploy kube app", func(t *testing.T) { - stack.Type = portainer.KubernetesStack - err = store.Stack().Update(stack.ID, &stack) - require.NoError(t, err) +func Test_redeployWhenChanged_DockerSwarmStack(t *testing.T) { + t.Parallel() - err = RedeployWhenChanged(t.Context(), 1, noopDeployer{}, store, testhelpers.NewGitService(nil, "newHash")) - require.NoError(t, err) - }) + store, stackID := setupRedeployStore(t, portainer.DockerSwarmStack) + + err := RedeployWhenChanged(t.Context(), stackID, noopDeployer{}, store, testhelpers.NewGitService(nil, "newHash")) + require.NoError(t, err) +} + +func Test_redeployWhenChanged_KubernetesStack(t *testing.T) { + t.Parallel() + + store, stackID := setupRedeployStore(t, portainer.KubernetesStack) + + err := RedeployWhenChanged(t.Context(), stackID, noopDeployer{}, store, testhelpers.NewGitService(nil, "newHash")) + require.NoError(t, err) } func Test_getUserRegistries(t *testing.T) { diff --git a/api/stacks/deployments/deployer_remote.go b/api/stacks/deployments/deployer_remote.go index fd2a167fc..1360b7675 100644 --- a/api/stacks/deployments/deployer_remote.go +++ b/api/stacks/deployments/deployer_remote.go @@ -10,8 +10,8 @@ import ( "time" portainer "github.com/portainer/portainer/api" - "github.com/portainer/portainer/api/dataservices" "github.com/portainer/portainer/api/filesystem" + "github.com/portainer/portainer/api/gitops/workflows" "github.com/portainer/portainer/api/logs" "github.com/portainer/portainer/pkg/librand" @@ -169,13 +169,13 @@ func (d *stackDeployer) StopRemoteSwarmStack(ctx context.Context, stack *portain // * gather deployment logs and bubble them up func (d *stackDeployer) remoteStack(ctx context.Context, stack *portainer.Stack, endpoint *portainer.Endpoint, operation StackRemoteOperation, opts unpackerCmdBuilderOptions) error { if stack.WorkflowID != 0 && opts.gitConfig == nil { - src, err := dataservices.GitSourceForWorkflow(d.dataStore, stack.WorkflowID) + src, artifact, err := workflows.GitSourceAndArtifactForStack(d.dataStore, stack.WorkflowID, stack.ID) if err != nil { return errors.Wrap(err, "failed to load git config for remote stack") } if src != nil { - opts.gitConfig = src.GitConfig + opts.gitConfig = workflows.MergeSourceAndArtifact(src, artifact) } } diff --git a/api/stacks/stackbuilders/stack_git_builder.go b/api/stacks/stackbuilders/stack_git_builder.go index 22940cda8..4e242f8f1 100644 --- a/api/stacks/stackbuilders/stack_git_builder.go +++ b/api/stacks/stackbuilders/stack_git_builder.go @@ -9,6 +9,7 @@ import ( "github.com/portainer/portainer/api/dataservices" "github.com/portainer/portainer/api/filesystem" gittypes "github.com/portainer/portainer/api/git/types" + "github.com/portainer/portainer/api/gitops/workflows" "github.com/portainer/portainer/api/scheduler" "github.com/portainer/portainer/api/stacks/deployments" "github.com/portainer/portainer/api/stacks/stackutils" @@ -72,7 +73,7 @@ func (b *GitMethodStackBuilder) prepare(ctx context.Context, payload *StackPaylo if err := b.dataStore.UpdateTx(func(tx dataservices.DataStoreTx) error { repoConfig.URL = gittypes.SanitizeURL(repoConfig.URL) - src, err := dataservices.FindOrCreateGitSource(tx, &portainer.Source{ + src, err := workflows.FindOrCreateGitSource(tx, &portainer.Source{ Name: gittypes.RepoName(repoConfig.URL), Type: portainer.SourceTypeGit, GitConfig: &repoConfig, @@ -82,7 +83,16 @@ func (b *GitMethodStackBuilder) prepare(ctx context.Context, payload *StackPaylo } wf := &portainer.Workflow{ - SourceIDs: []portainer.SourceID{src.ID}, + Name: b.stack.Name, + Artifacts: []portainer.ArtifactSources{{ + Artifact: portainer.Artifact{ + ReferenceName: repoConfig.ReferenceName, + ConfigFilePath: repoConfig.ConfigFilePath, + ConfigHash: repoConfig.ConfigHash, + StackID: b.stack.ID, + }, + SourceIDs: []portainer.SourceID{src.ID}, + }}, } if err := tx.Workflow().Create(wf); err != nil { return fmt.Errorf("failed to create workflow: %w", err)