diff --git a/api/dataservices/stack/stack.go b/api/dataservices/stack/stack.go index 78f213867..9c54c2878 100644 --- a/api/dataservices/stack/stack.go +++ b/api/dataservices/stack/stack.go @@ -130,7 +130,7 @@ func (service *Service) RefreshableStacks() ([]portainer.Stack, error) { BucketName, &portainer.Stack{}, dataservices.FilterFn(&stacks, func(e portainer.Stack) bool { - return e.AutoUpdate != nil && e.AutoUpdate.Interval != "" + return e.WorkflowID != 0 && e.AutoUpdate != nil && e.AutoUpdate.Interval != "" }), ) } diff --git a/api/dataservices/stack/tests/stack_test.go b/api/dataservices/stack/tests/stack_test.go index f58c04ed9..1cafe95e3 100644 --- a/api/dataservices/stack/tests/stack_test.go +++ b/api/dataservices/stack/tests/stack_test.go @@ -93,14 +93,15 @@ func Test_RefreshableStacks(t *testing.T) { staticStack := portainer.Stack{ID: 1} stackWithWebhook := portainer.Stack{ID: 2, AutoUpdate: &portainer.AutoUpdateSettings{Webhook: "webhook"}} - refreshableStack := portainer.Stack{ID: 3, AutoUpdate: &portainer.AutoUpdateSettings{Interval: "1m"}} + intervalNoWorkflow := portainer.Stack{ID: 3, AutoUpdate: &portainer.AutoUpdateSettings{Interval: "1m"}} + refreshableStack := portainer.Stack{ID: 4, WorkflowID: 1, AutoUpdate: &portainer.AutoUpdateSettings{Interval: "1m"}} - for _, stack := range []*portainer.Stack{&staticStack, &stackWithWebhook, &refreshableStack} { + for _, stack := range []*portainer.Stack{&staticStack, &stackWithWebhook, &intervalNoWorkflow, &refreshableStack} { err := store.Stack().Create(stack) require.NoError(t, err) } stacks, err := store.Stack().RefreshableStacks() require.NoError(t, err) - assert.ElementsMatch(t, []portainer.Stack{refreshableStack}, stacks) + require.ElementsMatch(t, []portainer.Stack{refreshableStack}, stacks) } diff --git a/api/dataservices/stack/tx.go b/api/dataservices/stack/tx.go index 50b827e26..c05216e25 100644 --- a/api/dataservices/stack/tx.go +++ b/api/dataservices/stack/tx.go @@ -106,7 +106,7 @@ func (service ServiceTx) RefreshableStacks() ([]portainer.Stack, error) { BucketName, &portainer.Stack{}, dataservices.FilterFn(&stacks, func(e portainer.Stack) bool { - return e.AutoUpdate != nil && e.AutoUpdate.Interval != "" + return e.WorkflowID != 0 && e.AutoUpdate != nil && e.AutoUpdate.Interval != "" }), ) } diff --git a/api/gitops/workflows/fetch.go b/api/gitops/workflows/fetch.go index 257d2a288..f657498ba 100644 --- a/api/gitops/workflows/fetch.go +++ b/api/gitops/workflows/fetch.go @@ -16,115 +16,105 @@ import ( // FetchWorkflows returns all GitOps workflows visible to the given user. func FetchWorkflows( ctx context.Context, - dataStore dataservices.DataStore, + tx dataservices.DataStoreTx, gitService portainer.GitService, k8sFactory *cli.ClientFactory, sc *security.RestrictedRequestContext, endpointIDSet set.Set[portainer.EndpointID], ) ([]Workflow, error) { - var stacks []portainer.Stack - var endpointMap map[portainer.EndpointID]portainer.Endpoint gitConfigs := map[portainer.StackID]*gittypes.RepoConfig{} - err := dataStore.ViewTx(func(tx dataservices.DataStoreTx) error { - var err error - stacks, err = tx.Stack().ReadAll(func(s portainer.Stack) bool { - return s.WorkflowID != 0 && (len(endpointIDSet) == 0 || endpointIDSet.Contains(s.EndpointID)) - }) - if err != nil { - return err - } - - endpointMap, err = buildEndpointMap(tx, stacks) - if err != nil { - return err - } - - stacks, err = filterDockerStacksByAccess(tx, stacks, sc) - if err != nil { - return err - } - - // First pass: filter by endpoint/stack-type match and collect workflow IDs. - preFiltered := make([]portainer.Stack, 0, len(stacks)) - workflowIDSet := make(map[portainer.WorkflowID]struct{}, len(stacks)) - for _, stack := range stacks { - if ep, ok := endpointMap[stack.EndpointID]; ok && !EndpointMatchesStackType(ep, stack.Type) { - continue - } - preFiltered = append(preFiltered, stack) - workflowIDSet[stack.WorkflowID] = struct{}{} - } - - // Batch-load all needed workflows in one scan. - workflows, err := tx.Workflow().ReadAll(func(wf portainer.Workflow) bool { - _, ok := workflowIDSet[wf.ID] - return ok - }) - if err != nil { - return err - } - - workflowMap := make(map[portainer.WorkflowID]portainer.Workflow, len(workflows)) - var allArtifacts []portainer.ArtifactSources - for _, wf := range workflows { - workflowMap[wf.ID] = wf - allArtifacts = append(allArtifacts, wf.Artifacts...) - } - sourceSet := ArtifactsToSourceSet(allArtifacts...) - - // Batch-load all needed sources in one scan. - srcs, err := tx.Source().ReadAll(func(src portainer.Source) bool { - return sourceSet.Contains(src.ID) - }) - if err != nil { - return err - } - - sourceMap := make(map[portainer.SourceID]portainer.Source, len(srcs)) - for _, src := range srcs { - sourceMap[src.ID] = src - } - - // Second pass: build filtered list using in-memory lookups. - var filtered []portainer.Stack - for _, stack := range preFiltered { - wf, ok := workflowMap[stack.WorkflowID] - if !ok { - log.Warn().Int("stackID", int(stack.ID)).Msg("workflow record missing for stack, skipping") - continue - } - - outer: - for _, as := range wf.Artifacts { - if as.Artifact.StackID != stack.ID { - continue - } - - for _, srcID := range as.SourceIDs { - src, ok := sourceMap[srcID] - if !ok { - log.Warn().Int("stackID", int(stack.ID)).Msg("source record missing for stack, skipping") - break outer - } - - if src.Type == portainer.SourceTypeGit { - gitConfigs[stack.ID] = MergeSourceAndArtifact(&src, &as.Artifact) - break outer - } - } - } - - filtered = append(filtered, stack) - } - stacks = filtered - - return nil + stacks, err := tx.Stack().ReadAll(func(s portainer.Stack) bool { + return s.WorkflowID != 0 && (len(endpointIDSet) == 0 || endpointIDSet.Contains(s.EndpointID)) }) if err != nil { return nil, err } + endpointMap, err := buildEndpointMap(tx, stacks) + if err != nil { + return nil, err + } + + stacks, err = filterDockerStacksByAccess(tx, stacks, sc) + if err != nil { + return nil, err + } + + // First pass: filter by endpoint/stack-type match and collect workflow IDs. + preFiltered := make([]portainer.Stack, 0, len(stacks)) + workflowIDSet := make(map[portainer.WorkflowID]struct{}, len(stacks)) + for _, stack := range stacks { + if ep, ok := endpointMap[stack.EndpointID]; ok && !EndpointMatchesStackType(ep, stack.Type) { + continue + } + preFiltered = append(preFiltered, stack) + workflowIDSet[stack.WorkflowID] = struct{}{} + } + + // Batch-load all needed workflows in one scan. + wfs, err := tx.Workflow().ReadAll(func(wf portainer.Workflow) bool { + _, ok := workflowIDSet[wf.ID] + return ok + }) + if err != nil { + return nil, err + } + + workflowMap := make(map[portainer.WorkflowID]portainer.Workflow, len(wfs)) + var allArtifacts []portainer.ArtifactSources + for _, wf := range wfs { + workflowMap[wf.ID] = wf + allArtifacts = append(allArtifacts, wf.Artifacts...) + } + sourceSet := ArtifactsToSourceSet(allArtifacts...) + + // Batch-load all needed sources in one scan. + srcs, err := tx.Source().ReadAll(func(src portainer.Source) bool { + return sourceSet.Contains(src.ID) + }) + if err != nil { + return nil, err + } + + sourceMap := make(map[portainer.SourceID]portainer.Source, len(srcs)) + for _, src := range srcs { + sourceMap[src.ID] = src + } + + // Second pass: build filtered list using in-memory lookups. + var filtered []portainer.Stack + for _, stack := range preFiltered { + wf, ok := workflowMap[stack.WorkflowID] + if !ok { + log.Warn().Int("stackID", int(stack.ID)).Msg("workflow record missing for stack, skipping") + continue + } + + outer: + for _, as := range wf.Artifacts { + if as.Artifact.StackID != stack.ID { + continue + } + + for _, srcID := range as.SourceIDs { + src, ok := sourceMap[srcID] + if !ok { + log.Warn().Int("stackID", int(stack.ID)).Msg("source record missing for stack, skipping") + break outer + } + + if src.Type == portainer.SourceTypeGit { + gitConfigs[stack.ID] = MergeSourceAndArtifact(&src, &as.Artifact) + break outer + } + } + } + + filtered = append(filtered, stack) + } + stacks = filtered + accessMap, err := buildEndpointAccessMap(k8sFactory, sc, endpointMap) if err != nil { return nil, err diff --git a/api/gitops/workflows/fetch_test.go b/api/gitops/workflows/fetch_test.go index e40de7a9c..fdfbedd26 100644 --- a/api/gitops/workflows/fetch_test.go +++ b/api/gitops/workflows/fetch_test.go @@ -108,8 +108,12 @@ func TestFetchWorkflows_ReturnsOnlyGitopsStacks(t *testing.T) { return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) })) - items, err := FetchWorkflows(t.Context(), store, nil, nil, adminContext(), nil) - require.NoError(t, err) + var items []Workflow + require.NoError(t, store.ViewTx(func(tx dataservices.DataStoreTx) error { + var err error + items, err = FetchWorkflows(t.Context(), tx, nil, nil, adminContext(), nil) + return err + })) require.Len(t, items, 1) require.Equal(t, "gitops-stack", items[0].Name) } @@ -131,8 +135,12 @@ func TestFetchWorkflows_FiltersByEndpointID(t *testing.T) { return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) })) - items, err := FetchWorkflows(t.Context(), store, nil, nil, adminContext(), set.ToSet([]portainer.EndpointID{1, 2})) - require.NoError(t, err) + var items []Workflow + require.NoError(t, store.ViewTx(func(tx dataservices.DataStoreTx) error { + var err error + items, err = FetchWorkflows(t.Context(), tx, nil, nil, adminContext(), set.ToSet([]portainer.EndpointID{1, 2})) + return err + })) require.Len(t, items, 2) names := []string{items[0].Name, items[1].Name} @@ -151,8 +159,12 @@ func TestFetchWorkflows_EmptyWhenNoGitopsStacks(t *testing.T) { return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) })) - items, err := FetchWorkflows(t.Context(), store, nil, nil, adminContext(), nil) - require.NoError(t, err) + var items []Workflow + require.NoError(t, store.ViewTx(func(tx dataservices.DataStoreTx) error { + var err error + items, err = FetchWorkflows(t.Context(), tx, nil, nil, adminContext(), nil) + return err + })) require.Empty(t, items) } @@ -173,8 +185,12 @@ func TestFetchWorkflows_NilEndpointSetReturnsAll(t *testing.T) { return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) })) - items, err := FetchWorkflows(t.Context(), store, nil, nil, adminContext(), nil) - require.NoError(t, err) + var items []Workflow + require.NoError(t, store.ViewTx(func(tx dataservices.DataStoreTx) error { + var err error + items, err = FetchWorkflows(t.Context(), tx, nil, nil, adminContext(), nil) + return err + })) require.Len(t, items, 3) } diff --git a/api/http/handler/gitops/sources/create_git.go b/api/http/handler/gitops/sources/create_git.go index 23ae0309e..ce680d895 100644 --- a/api/http/handler/gitops/sources/create_git.go +++ b/api/http/handler/gitops/sources/create_git.go @@ -23,11 +23,10 @@ type GitAuthenticationPayload struct { // GitSourceCreatePayload holds the parameters for creating a git-backed source type GitSourceCreatePayload struct { - Name string `json:"name"` - URL string `json:"url"` - TLSSkipVerify bool `json:"tlsSkipVerify"` - Authentication *GitAuthenticationPayload `json:"authentication"` - ClearAuthentication bool `json:"clearAuthentication"` + Name string `json:"name"` + URL string `json:"url"` + TLSSkipVerify bool `json:"tlsSkipVerify"` + Authentication *GitAuthenticationPayload `json:"authentication"` } // Validate implements the portainer.Validatable interface diff --git a/api/http/handler/gitops/sources/get.go b/api/http/handler/gitops/sources/get.go index fea6d0d0a..449563afc 100644 --- a/api/http/handler/gitops/sources/get.go +++ b/api/http/handler/gitops/sources/get.go @@ -61,10 +61,15 @@ func (h *Handler) getSource(w http.ResponseWriter, r *http.Request) *httperror.H } var src *portainer.Source + var workflows []ce.Workflow if err := h.dataStore.ViewTx(func(tx dataservices.DataStoreTx) error { var err error src, err = tx.Source().Read(portainer.SourceID(srcID)) + if err != nil { + return err + } + workflows, err = ce.FetchWorkflows(r.Context(), tx, h.gitService, h.k8sFactory, securityContext, nil) return err }); h.dataStore.IsErrObjectNotFound(err) { return httperror.NotFound("Unable to find a source with the specified identifier", err) @@ -72,11 +77,6 @@ func (h *Handler) getSource(w http.ResponseWriter, r *http.Request) *httperror.H return httperror.InternalServerError("Unable to retrieve source", err) } - workflows, err := ce.FetchWorkflows(r.Context(), h.dataStore, h.gitService, h.k8sFactory, securityContext, nil) - if err != nil { - return httperror.InternalServerError("Unable to retrieve workflows", err) - } - byID := workflowsBySourceID(workflows) var wfs []ce.Workflow diff --git a/api/http/handler/gitops/sources/update_git.go b/api/http/handler/gitops/sources/update_git.go index 4a7d2bc47..1a5fd6530 100644 --- a/api/http/handler/gitops/sources/update_git.go +++ b/api/http/handler/gitops/sources/update_git.go @@ -91,8 +91,10 @@ func (h *Handler) gitSourceUpdate(w http.ResponseWriter, r *http.Request) *httpe src.Name = updated.Name src.GitConfig = updated.GitConfig - if payload.Authentication == nil && !payload.ClearAuthentication { + if payload.Authentication == nil { src.GitConfig.Authentication = existingAuth + } else if *payload.Authentication == (GitAuthenticationPayload{}) { + src.GitConfig.Authentication = nil } return tx.Source().Update(src.ID, src) diff --git a/api/http/handler/gitops/sources/update_git_test.go b/api/http/handler/gitops/sources/update_git_test.go index 857aa6a0e..dd344f8fa 100644 --- a/api/http/handler/gitops/sources/update_git_test.go +++ b/api/http/handler/gitops/sources/update_git_test.go @@ -157,8 +157,8 @@ func TestGitSourceUpdate_ClearsAuthWhenRequested(t *testing.T) { h := newTestHandler(t, store) body, err := json.Marshal(GitSourceCreatePayload{ - URL: "https://github.com/org/repo.git", - ClearAuthentication: true, + URL: "https://github.com/org/repo.git", + Authentication: &GitAuthenticationPayload{}, }) require.NoError(t, err) @@ -177,6 +177,58 @@ func TestGitSourceUpdate_ClearsAuthWhenRequested(t *testing.T) { require.Nil(t, stored.GitConfig.Authentication) } +func TestGitSourceUpdate_ReplacesAuthWhenProvided(t *testing.T) { + t.Parallel() + _, store := datastore.MustNewTestStore(t, false, true) + + var srcID portainer.SourceID + require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error { + src := &portainer.Source{ + Name: "auth-source", + Type: portainer.SourceTypeGit, + GitConfig: &gittypes.RepoConfig{ + URL: "https://github.com/org/repo.git", + Authentication: &gittypes.GitAuthentication{ + Username: "alice", + Password: "secret", + }, + }, + } + err := tx.Source().Create(src) + require.NoError(t, err) + srcID = src.ID + + return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}) + })) + + h := newTestHandler(t, store) + + body, err := json.Marshal(GitSourceCreatePayload{ + URL: "https://github.com/org/repo.git", + Authentication: &GitAuthenticationPayload{ + Username: "bob", + Password: "new-secret", + }, + }) + require.NoError(t, err) + + rr := httptest.NewRecorder() + h.ServeHTTP(rr, buildUpdateReq(t, 1, int(srcID), body)) + + require.Equal(t, http.StatusOK, rr.Code) + + var stored *portainer.Source + require.NoError(t, store.ViewTx(func(tx dataservices.DataStoreTx) error { + var err error + stored, err = tx.Source().Read(srcID) + return err + })) + require.NotNil(t, stored.GitConfig) + require.NotNil(t, stored.GitConfig.Authentication) + require.Equal(t, "bob", stored.GitConfig.Authentication.Username) + require.Equal(t, "new-secret", stored.GitConfig.Authentication.Password) +} + func TestGitSourceUpdate_NotFound(t *testing.T) { t.Parallel() _, store := datastore.MustNewTestStore(t, false, true) diff --git a/api/http/handler/gitops/workflows/list.go b/api/http/handler/gitops/workflows/list.go index 4585e8552..e3c7551c2 100644 --- a/api/http/handler/gitops/workflows/list.go +++ b/api/http/handler/gitops/workflows/list.go @@ -10,6 +10,7 @@ import ( gocache "github.com/patrickmn/go-cache" portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/dataservices" svc "github.com/portainer/portainer/api/gitops/workflows" "github.com/portainer/portainer/api/http/security" "github.com/portainer/portainer/api/http/utils/filters" @@ -128,7 +129,12 @@ func (h *Handler) getWorkflows(ctx context.Context, key string, sc *security.Res return slices.Clone(cached.([]svc.Workflow)), nil } - result, err := svc.FetchWorkflows(ctx, h.dataStore, h.gitService, h.k8sFactory, sc, set.ToSet(endpointIDs)) + var result []svc.Workflow + err := h.dataStore.ViewTx(func(tx dataservices.DataStoreTx) error { + var err error + result, err = svc.FetchWorkflows(ctx, tx, h.gitService, h.k8sFactory, sc, set.ToSet(endpointIDs)) + return err + }) if err != nil { return nil, err }