Update community branch (#13208)

Co-authored-by: Hannah Cooper <hannah.cooper@portainer.io>
Co-authored-by: Chaim Lev-Ari <chiptus@users.noreply.github.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Ali <83188384+testA113@users.noreply.github.com>
Co-authored-by: Steven Kang <skan070@gmail.com>
Co-authored-by: Josiah Clumont <josiah.clumont@portainer.io>
Co-authored-by: nickl-portainer <nicholas.loomans@portainer.io>
Co-authored-by: andres-portainer <91705312+andres-portainer@users.noreply.github.com>
Co-authored-by: Oscar Zhou <100548325+oscarzhou-portainer@users.noreply.github.com>
Co-authored-by: ferreiraborgesaxel-design <ferreiraborgesaxel-design@users.noreply.github.com>

api/http/handler/gitops/sources/create_git.go

@@ -60,9 +60,9 @@ func BuildGitSource(payload GitSourceCreatePayload) *portainer.Source {
 	}
 
 	return &portainer.Source{
-		Name:      name,
-		Type:      portainer.SourceTypeGit,
-		GitConfig: gitConfig,
+		Name: name,
+		Type: portainer.SourceTypeGit,
+		Git:  gitConfig,
 	}
 }
 
@@ -96,7 +96,7 @@ func (h *Handler) gitSourceCreate(w http.ResponseWriter, r *http.Request) *httpe
 		return httperror.InternalServerError("Unable to create source", err)
 	}
 
-	src.GitConfig = gittypes.SanitizeRepoConfig(src.GitConfig)
+	src.Git = gittypes.SanitizeRepoConfig(src.Git)
 
 	return response.JSONWithStatus(w, src, http.StatusCreated)
 }

api/http/handler/gitops/sources/create_git_test.go

@@ -22,7 +22,7 @@ func TestBuildGitSource_DerivesNameFromURL(t *testing.T) {
 
 	require.Equal(t, "my-repo", src.Name)
 	require.Equal(t, portainer.SourceTypeGit, src.Type)
-	require.Nil(t, src.GitConfig.Authentication)
+	require.Nil(t, src.Git.Authentication)
 }
 
 func TestBuildGitSource_UsesExplicitName(t *testing.T) {
@@ -47,9 +47,9 @@ func TestBuildGitSource_WithAuthentication(t *testing.T) {
 		},
 	})
 
-	require.NotNil(t, src.GitConfig.Authentication)
-	require.Equal(t, "alice", src.GitConfig.Authentication.Username)
-	require.Equal(t, "secret", src.GitConfig.Authentication.Password)
+	require.NotNil(t, src.Git.Authentication)
+	require.Equal(t, "alice", src.Git.Authentication.Username)
+	require.Equal(t, "secret", src.Git.Authentication.Password)
 }
 
 func TestGitSourceCreatePayload_Validate_EmptyURL(t *testing.T) {
@@ -93,8 +93,8 @@ func TestGitSourceCreate_Success(t *testing.T) {
 	require.Equal(t, "my-source", src.Name)
 	require.Equal(t, portainer.SourceTypeGit, src.Type)
 	require.NotZero(t, src.ID)
-	require.NotNil(t, src.GitConfig)
-	require.Equal(t, "https://github.com/org/repo.git", src.GitConfig.URL)
+	require.NotNil(t, src.Git)
+	require.Equal(t, "https://github.com/org/repo.git", src.Git.URL)
 }
 
 func TestGitSourceCreate_SanitizesCredentials(t *testing.T) {
@@ -124,10 +124,10 @@ func TestGitSourceCreate_SanitizesCredentials(t *testing.T) {
 	var src portainer.Source
 	err = json.NewDecoder(rr.Body).Decode(&src)
 	require.NoError(t, err)
-	require.NotNil(t, src.GitConfig)
-	require.NotNil(t, src.GitConfig.Authentication)
-	require.Equal(t, "alice", src.GitConfig.Authentication.Username)
-	require.Empty(t, src.GitConfig.Authentication.Password)
+	require.NotNil(t, src.Git)
+	require.NotNil(t, src.Git.Authentication)
+	require.Equal(t, "alice", src.Git.Authentication.Username)
+	require.Empty(t, src.Git.Authentication.Password)
 }
 
 func TestGitSourceCreate_MissingURL(t *testing.T) {

api/http/handler/gitops/sources/delete.go

@@ -13,11 +13,11 @@ import (
 	"github.com/portainer/portainer/pkg/libhttp/response"
 )
 
-var ErrSourceInUse = errors.New("source is used by one or more workflows")
+var ErrSourceInUse = errors.New("source is used by one or more workflows or custom templates")
 
 // @id GitOpsSourcesDelete
 // @summary Delete a source
-// @description Deletes an existing GitOps source. Returns 409 if the source is referenced by any workflow.
+// @description Deletes an existing GitOps source. Returns 409 if the source is referenced by any workflow or custom template.
 // @description **Access policy**: admin
 // @tags gitops
 // @security ApiKeyAuth
@@ -27,7 +27,7 @@ var ErrSourceInUse = errors.New("source is used by one or more workflows")
 // @failure 400 "Invalid request"
 // @failure 403 "Access denied"
 // @failure 404 "Source not found"
-// @failure 409 "Source is in use by one or more workflows"
+// @failure 409 "Source is in use by one or more workflows or custom templates"
 // @failure 500 "Server error"
 // @router /gitops/sources/{id} [delete]
 func (h *Handler) sourceDelete(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
@@ -49,18 +49,33 @@ func (h *Handler) sourceDelete(w http.ResponseWriter, r *http.Request) *httperro
 		}
 
 		for _, wf := range workflows {
-			if slices.ContainsFunc(wf.Artifacts, func(as portainer.ArtifactSources) bool {
-				return slices.Contains(as.SourceIDs, portainer.SourceID(sourceID))
+			if slices.ContainsFunc(wf.Artifacts, func(as portainer.Artifact) bool {
+				return slices.ContainsFunc(as.Files, func(f portainer.ArtifactFile) bool {
+					return f.SourceID == portainer.SourceID(sourceID)
+				})
 			}) {
 				return ErrSourceInUse
 			}
 		}
 
+		templates, err := tx.CustomTemplate().ReadAll(func(t portainer.CustomTemplate) bool {
+			return t.Artifact != nil && slices.ContainsFunc(t.Artifact.Files, func(f portainer.ArtifactFile) bool {
+				return f.SourceID == portainer.SourceID(sourceID)
+			})
+		})
+		if err != nil {
+			return err
+		}
+
+		if len(templates) > 0 {
+			return ErrSourceInUse
+		}
+
 		return tx.Source().Delete(portainer.SourceID(sourceID))
 	}); h.dataStore.IsErrObjectNotFound(err) {
 		return httperror.NotFound("Unable to find a source with the specified identifier", err)
 	} else if errors.Is(err, ErrSourceInUse) {
-		return httperror.Conflict("Source is used by one or more workflows", err)
+		return httperror.Conflict("Source is used by one or more workflows or custom templates", err)
 	} else if err != nil {
 		return httperror.InternalServerError("Unable to delete source", err)
 	}

api/http/handler/gitops/sources/delete_test.go

@@ -14,6 +14,7 @@ import (
 
 func TestSourceDelete_Success(t *testing.T) {
 	t.Parallel()
+
 	_, store := datastore.MustNewTestStore(t, false, true)
 
 	var srcID portainer.SourceID
@@ -35,6 +36,7 @@ func TestSourceDelete_Success(t *testing.T) {
 
 func TestSourceDelete_NotFound(t *testing.T) {
 	t.Parallel()
+
 	_, store := datastore.MustNewTestStore(t, false, true)
 
 	require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
@@ -50,6 +52,7 @@ func TestSourceDelete_NotFound(t *testing.T) {
 
 func TestSourceDelete_InUse(t *testing.T) {
 	t.Parallel()
+
 	_, store := datastore.MustNewTestStore(t, false, true)
 
 	var srcID portainer.SourceID
@@ -59,7 +62,7 @@ func TestSourceDelete_InUse(t *testing.T) {
 		require.NoError(t, err)
 		srcID = src.ID
 
-		wf := &portainer.Workflow{Artifacts: []portainer.ArtifactSources{{SourceIDs: []portainer.SourceID{src.ID}}}}
+		wf := &portainer.Workflow{Artifacts: []portainer.Artifact{{Files: []portainer.ArtifactFile{{SourceID: src.ID}}}}}
 		err = tx.Workflow().Create(wf)
 		require.NoError(t, err)
 
@@ -75,6 +78,7 @@ func TestSourceDelete_InUse(t *testing.T) {
 
 func TestSourceDelete_NonNumericID(t *testing.T) {
 	t.Parallel()
+
 	_, store := datastore.MustNewTestStore(t, false, true)
 
 	require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
@@ -87,3 +91,34 @@ func TestSourceDelete_NonNumericID(t *testing.T) {
 
 	require.Equal(t, http.StatusBadRequest, rr.Code)
 }
+
+func TestSourceDelete_InUseByCustomTemplate(t *testing.T) {
+	t.Parallel()
+
+	_, store := datastore.MustNewTestStore(t, false, true)
+
+	var srcID portainer.SourceID
+	require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
+		src := &portainer.Source{Name: "in-use-by-template", Type: portainer.SourceTypeGit}
+		err := tx.Source().Create(src)
+		require.NoError(t, err)
+		srcID = src.ID
+
+		ct := &portainer.CustomTemplate{
+			ID: 1,
+			Artifact: &portainer.Artifact{
+				Files: []portainer.ArtifactFile{{SourceID: src.ID}},
+			},
+		}
+		err = tx.CustomTemplate().Create(ct)
+		require.NoError(t, err)
+
+		return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
+	}))
+
+	h := newTestHandler(t, store)
+	rr := httptest.NewRecorder()
+	h.ServeHTTP(rr, buildDeleteReq(t, 1, int(srcID)))
+
+	require.Equal(t, http.StatusConflict, rr.Code)
+}

api/http/handler/gitops/sources/fetch.go

@@ -13,8 +13,10 @@ import (
 // FetchSourceWorkflows returns the workflows and stats for a single source.
 func FetchSourceWorkflows(tx dataservices.DataStoreTx, src *portainer.Source) ([]ce.Workflow, ce.SourceStats, error) {
 	wfs, err := tx.Workflow().ReadAll(func(wf portainer.Workflow) bool {
-		return slices.ContainsFunc(wf.Artifacts, func(artifact portainer.ArtifactSources) bool {
-			return slices.Contains(artifact.SourceIDs, src.ID)
+		return slices.ContainsFunc(wf.Artifacts, func(artifact portainer.Artifact) bool {
+			return slices.ContainsFunc(artifact.Files, func(f portainer.ArtifactFile) bool {
+				return f.SourceID == src.ID
+			})
 		})
 	})
 	if err != nil {
@@ -40,7 +42,7 @@ func FetchSourceWorkflows(tx dataservices.DataStoreTx, src *portainer.Source) ([
 	stats := ce.SourceStats{EndpointIDs: set.Set[portainer.EndpointID]{}}
 
 	for _, stacks := range stacks {
-		items = append(items, ce.MapStackToWorkflow(stacks, src.GitConfig, unknown, unknown))
+		items = append(items, ce.MapStackToWorkflow(stacks, src.Git, unknown, unknown))
 		stats.WorkflowCount++
 		if stacks.EndpointID != 0 {
 			stats.EndpointIDs.Add(stacks.EndpointID)

api/http/handler/gitops/sources/get.go

@@ -80,7 +80,7 @@ func (h *Handler) getSource(w http.ResponseWriter, r *http.Request) *httperror.H
 		return httperror.InternalServerError("Unable to retrieve source", err)
 	}
 
-	detail := BuildSourceDetail(h.buildSource(r.Context(), source, stats), source.GitConfig, sourceWfs)
+	detail := BuildSourceDetail(h.buildSource(r.Context(), source, stats), source.Git, sourceWfs)
 	return response.JSON(w, detail)
 }
 

api/http/handler/gitops/sources/helpers_test.go

@@ -23,20 +23,20 @@ func createGitWorkflow(t *testing.T, tx dataservices.DataStoreTx, stack *portain
 	t.Helper()
 
 	src := &portainer.Source{
-		Name:      gittypes.RepoName(cfg.URL),
-		Type:      portainer.SourceTypeGit,
-		GitConfig: cfg,
+		Name: gittypes.RepoName(cfg.URL),
+		Type: portainer.SourceTypeGit,
+		Git:  cfg,
 	}
 	require.NoError(t, tx.Source().Create(src))
 
 	wf := &portainer.Workflow{
-		Artifacts: []portainer.ArtifactSources{{
-			Artifact: portainer.Artifact{
-				StackID:        stack.ID,
-				ReferenceName:  cfg.ReferenceName,
-				ConfigFilePath: cfg.ConfigFilePath,
-			},
-			SourceIDs: []portainer.SourceID{src.ID},
+		Artifacts: []portainer.Artifact{{
+			StackID: stack.ID,
+			Files: []portainer.ArtifactFile{{
+				SourceID: src.ID,
+				Path:     cfg.ConfigFilePath,
+				Ref:      cfg.ReferenceName,
+			}},
 		}},
 	}
 	require.NoError(t, tx.Workflow().Create(wf))

api/http/handler/gitops/sources/list_test.go

@@ -19,12 +19,13 @@ func TestSourcesList_GroupsByURLAndCredentials(t *testing.T) {
 
 	require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
 		cfg := gitCfg("https://github.com/org/repo")
-		src := &portainer.Source{Name: "repo", Type: portainer.SourceTypeGit, GitConfig: cfg}
+		src := &portainer.Source{Name: "repo", Type: portainer.SourceTypeGit, Git: cfg}
 		require.NoError(t, tx.Source().Create(src))
 
-		wfA := &portainer.Workflow{Artifacts: []portainer.ArtifactSources{{SourceIDs: []portainer.SourceID{src.ID}}}}
+		wfA := &portainer.Workflow{Artifacts: []portainer.Artifact{{Files: []portainer.ArtifactFile{{SourceID: src.ID}}}}}
 		require.NoError(t, tx.Workflow().Create(wfA))
-		wfB := &portainer.Workflow{Artifacts: []portainer.ArtifactSources{{SourceIDs: []portainer.SourceID{src.ID}}}}
+
+		wfB := &portainer.Workflow{Artifacts: []portainer.Artifact{{Files: []portainer.ArtifactFile{{SourceID: src.ID}}}}}
 		require.NoError(t, tx.Workflow().Create(wfB))
 
 		require.NoError(t, tx.Stack().Create(&portainer.Stack{ID: 1, Name: "stack-a", WorkflowID: wfA.ID}))

api/http/handler/gitops/sources/source_connection.go

@@ -58,11 +58,11 @@ func (h *Handler) sourceTestConnection(w http.ResponseWriter, r *http.Request) *
 		return httperror.InternalServerError("Unable to apply source changes", err)
 	}
 
-	if src.GitConfig == nil {
+	if src.Git == nil {
 		return httperror.InternalServerError("Source has no git configuration", nil)
 	}
 
-	result := testSourceConnection(r.Context(), h.gitService, src.GitConfig)
+	result := testSourceConnection(r.Context(), h.gitService, src.Git)
 
 	return response.JSON(w, result)
 }

api/http/handler/gitops/sources/update_git.go

@@ -104,7 +104,7 @@ func (h *Handler) gitSourceUpdate(w http.ResponseWriter, r *http.Request) *httpe
 		return httperror.InternalServerError("Unable to update source", err)
 	}
 
-	src.GitConfig = gittypes.SanitizeRepoConfig(src.GitConfig)
+	src.Git = gittypes.SanitizeRepoConfig(src.Git)
 
 	return response.JSON(w, src)
 }
@@ -119,7 +119,7 @@ func ApplyGitSourceChanges(src *portainer.Source, payload GitSourceUpdatePayload
 		src.Name = *payload.Name
 	}
 
-	gitConfig := src.GitConfig
+	gitConfig := src.Git
 	if gitConfig == nil {
 		gitConfig = &gittypes.RepoConfig{}
 	}
@@ -167,7 +167,7 @@ func ApplyGitSourceChanges(src *portainer.Source, payload GitSourceUpdatePayload
 	}
 
 	gitConfig.Authentication = auth
-	src.GitConfig = gitConfig
+	src.Git = gitConfig
 
 	return nil
 }

api/http/handler/gitops/sources/update_git_test.go

@@ -45,8 +45,8 @@ func TestGitSourceUpdate_Success(t *testing.T) {
 	err = json.NewDecoder(rr.Body).Decode(&src)
 	require.NoError(t, err)
 	require.Equal(t, "new-name", src.Name)
-	require.NotNil(t, src.GitConfig)
-	require.Equal(t, "https://github.com/org/new.git", src.GitConfig.URL)
+	require.NotNil(t, src.Git)
+	require.Equal(t, "https://github.com/org/new.git", src.Git.URL)
 }
 
 func TestGitSourceUpdate_PreservesAuthWhenNotProvided(t *testing.T) {
@@ -58,7 +58,7 @@ func TestGitSourceUpdate_PreservesAuthWhenNotProvided(t *testing.T) {
 		src := &portainer.Source{
 			Name: "auth-source",
 			Type: portainer.SourceTypeGit,
-			GitConfig: &gittypes.RepoConfig{
+			Git: &gittypes.RepoConfig{
 				URL: "https://github.com/org/repo.git",
 				Authentication: &gittypes.GitAuthentication{
 					Username: "alice",
@@ -92,10 +92,10 @@ func TestGitSourceUpdate_PreservesAuthWhenNotProvided(t *testing.T) {
 		stored, err = tx.Source().Read(srcID)
 		return err
 	}))
-	require.NotNil(t, stored.GitConfig)
-	require.NotNil(t, stored.GitConfig.Authentication)
-	require.Equal(t, "alice", stored.GitConfig.Authentication.Username)
-	require.Equal(t, "secret", stored.GitConfig.Authentication.Password)
+	require.NotNil(t, stored.Git)
+	require.NotNil(t, stored.Git.Authentication)
+	require.Equal(t, "alice", stored.Git.Authentication.Username)
+	require.Equal(t, "secret", stored.Git.Authentication.Password)
 }
 
 func TestGitSourceUpdate_ClearsAuthWhenRequested(t *testing.T) {
@@ -107,7 +107,7 @@ func TestGitSourceUpdate_ClearsAuthWhenRequested(t *testing.T) {
 		src := &portainer.Source{
 			Name: "auth-source",
 			Type: portainer.SourceTypeGit,
-			GitConfig: &gittypes.RepoConfig{
+			Git: &gittypes.RepoConfig{
 				URL: "https://github.com/org/repo.git",
 				Authentication: &gittypes.GitAuthentication{
 					Username: "alice",
@@ -141,8 +141,8 @@ func TestGitSourceUpdate_ClearsAuthWhenRequested(t *testing.T) {
 		stored, err = tx.Source().Read(srcID)
 		return err
 	}))
-	require.NotNil(t, stored.GitConfig)
-	require.Nil(t, stored.GitConfig.Authentication)
+	require.NotNil(t, stored.Git)
+	require.Nil(t, stored.Git.Authentication)
 }
 
 func TestGitSourceUpdate_ReplacesAuthWhenProvided(t *testing.T) {
@@ -154,7 +154,7 @@ func TestGitSourceUpdate_ReplacesAuthWhenProvided(t *testing.T) {
 		src := &portainer.Source{
 			Name: "auth-source",
 			Type: portainer.SourceTypeGit,
-			GitConfig: &gittypes.RepoConfig{
+			Git: &gittypes.RepoConfig{
 				URL: "https://github.com/org/repo.git",
 				Authentication: &gittypes.GitAuthentication{
 					Username: "alice",
@@ -191,10 +191,10 @@ func TestGitSourceUpdate_ReplacesAuthWhenProvided(t *testing.T) {
 		stored, err = tx.Source().Read(srcID)
 		return err
 	}))
-	require.NotNil(t, stored.GitConfig)
-	require.NotNil(t, stored.GitConfig.Authentication)
-	require.Equal(t, "bob", stored.GitConfig.Authentication.Username)
-	require.Equal(t, "new-secret", stored.GitConfig.Authentication.Password)
+	require.NotNil(t, stored.Git)
+	require.NotNil(t, stored.Git.Authentication)
+	require.Equal(t, "bob", stored.Git.Authentication.Username)
+	require.Equal(t, "new-secret", stored.Git.Authentication.Password)
 }
 
 func TestGitSourceUpdate_NotFound(t *testing.T) {
@@ -225,7 +225,7 @@ func TestGitSourceUpdate_ConflictOnDuplicateURL(t *testing.T) {
 		existing := &portainer.Source{
 			Name: "existing",
 			Type: portainer.SourceTypeGit,
-			GitConfig: &gittypes.RepoConfig{
+			Git: &gittypes.RepoConfig{
 				URL: "https://github.com/org/existing.git",
 			},
 		}

api/http/handler/gitops/sources/utils.go

@@ -11,8 +11,8 @@ import (
 func (h *Handler) buildSource(ctx context.Context, src *portainer.Source, stats ce.SourceStats) Source {
 	var status ce.Status
 	var sourceErr string
-	if src.GitConfig != nil {
-		phase, _ := ce.ComputeGitPhasesForConfig(ctx, h.gitService, src.GitConfig)
+	if src.Git != nil {
+		phase, _ := ce.ComputeGitPhasesForConfig(ctx, h.gitService, src.Git)
 		status = phase.Status
 		sourceErr = phase.Error
 	} else {
@@ -21,10 +21,10 @@ func (h *Handler) buildSource(ctx context.Context, src *portainer.Source, stats
 
 	url := ""
 	var provider gittypes.GitProvider
-	if src.GitConfig != nil {
-		url = gittypes.SanitizeURL(src.GitConfig.URL)
-		if src.GitConfig.Authentication != nil {
-			provider = src.GitConfig.Authentication.Provider
+	if src.Git != nil {
+		url = gittypes.SanitizeURL(src.Git.URL)
+		if src.Git.Authentication != nil {
+			provider = src.Git.Authentication.Provider
 		}
 	}