feat(template/gitops): support to add/update k8s custom template with git repository method

api/http/handler/customtemplates/customtemplate_create.go

@@ -3,7 +3,6 @@ package customtemplates
 import (
 	"encoding/json"
 	"errors"
-	"fmt"
 	"net/http"
 	"os"
 	"regexp"
@@ -18,6 +17,7 @@ import (
 	gittypes "github.com/portainer/portainer/api/git/types"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/internal/authorization"
+	"github.com/portainer/portainer/api/stacks/stackutils"
 	"github.com/rs/zerolog/log"
 )
 
@@ -218,6 +218,8 @@ type customTemplateFromGitRepositoryPayload struct {
 	ComposeFilePathInRepository string `example:"docker-compose.yml" default:"docker-compose.yml"`
 	// Definitions of variables in the stack file
 	Variables []portainer.CustomTemplateVariableDefinition
+	// IsComposeFormat indicates if the Kubernetes template is created from a Docker Compose file
+	IsComposeFormat bool `example:"false"`
 }
 
 func (payload *customTemplateFromGitRepositoryPayload) Validate(r *http.Request) error {
@@ -237,14 +239,10 @@ func (payload *customTemplateFromGitRepositoryPayload) Validate(r *http.Request)
 		payload.ComposeFilePathInRepository = filesystem.ComposeFileDefaultName
 	}
 
-	if payload.Type == portainer.KubernetesStack {
-		return errors.New("Creating a Kubernetes custom template from git is not supported")
-	}
-
 	if payload.Platform != portainer.CustomTemplatePlatformLinux && payload.Platform != portainer.CustomTemplatePlatformWindows {
 		return errors.New("Invalid custom template platform")
 	}
-	if payload.Type != portainer.DockerSwarmStack && payload.Type != portainer.DockerComposeStack {
+	if payload.Type != portainer.DockerSwarmStack && payload.Type != portainer.DockerComposeStack && payload.Type != portainer.KubernetesStack {
 		return errors.New("Invalid custom template type")
 	}
 	if !isValidNote(payload.Note) {
@@ -268,35 +266,44 @@ func (handler *Handler) createCustomTemplateFromGitRepository(r *http.Request) (
 
 	customTemplateID := handler.DataStore.CustomTemplate().GetNextIdentifier()
 	customTemplate := &portainer.CustomTemplate{
-		ID:          portainer.CustomTemplateID(customTemplateID),
-		Title:       payload.Title,
-		EntryPoint:  payload.ComposeFilePathInRepository,
-		Description: payload.Description,
-		Note:        payload.Note,
-		Platform:    payload.Platform,
-		Type:        payload.Type,
-		Logo:        payload.Logo,
-		Variables:   payload.Variables,
+		ID:              portainer.CustomTemplateID(customTemplateID),
+		Title:           payload.Title,
+		Description:     payload.Description,
+		Note:            payload.Note,
+		Platform:        payload.Platform,
+		Type:            payload.Type,
+		Logo:            payload.Logo,
+		Variables:       payload.Variables,
+		IsComposeFormat: payload.IsComposeFormat,
 	}
 
-	projectPath := handler.FileService.GetCustomTemplateProjectPath(strconv.Itoa(customTemplateID))
+	getProjectPath := func() string {
+		return handler.FileService.GetCustomTemplateProjectPath(strconv.Itoa(customTemplateID))
+	}
+	projectPath := getProjectPath()
 	customTemplate.ProjectPath = projectPath
 
-	repositoryUsername := payload.RepositoryUsername
-	repositoryPassword := payload.RepositoryPassword
-	if !payload.RepositoryAuthentication {
-		repositoryUsername = ""
-		repositoryPassword = ""
+	gitConfig := &gittypes.RepoConfig{
+		URL:            payload.RepositoryURL,
+		ReferenceName:  payload.RepositoryReferenceName,
+		ConfigFilePath: payload.ComposeFilePathInRepository,
 	}
 
-	err = handler.GitService.CloneRepository(projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, repositoryUsername, repositoryPassword)
-	if err != nil {
-		if err == gittypes.ErrAuthenticationFailure {
-			return nil, fmt.Errorf("invalid git credential")
+	if payload.RepositoryAuthentication {
+		gitConfig.Authentication = &gittypes.GitAuthentication{
+			Username: payload.RepositoryUsername,
+			Password: payload.RepositoryPassword,
 		}
+	}
+
+	commitHash, err := stackutils.DownloadGitRepository(*gitConfig, handler.GitService, getProjectPath)
+	if err != nil {
 		return nil, err
 	}
 
+	gitConfig.ConfigHash = commitHash
+	customTemplate.GitConfig = gitConfig
+
 	isValidProject := true
 	defer func() {
 		if !isValidProject {
@@ -306,7 +313,7 @@ func (handler *Handler) createCustomTemplateFromGitRepository(r *http.Request) (
 		}
 	}()
 
-	entryPath := filesystem.JoinPaths(projectPath, customTemplate.EntryPoint)
+	entryPath := filesystem.JoinPaths(projectPath, gitConfig.ConfigFilePath)
 
 	exists, err := handler.FileService.FileExists(entryPath)
 	if err != nil || !exists {

api/http/handler/customtemplates/customtemplate_file.go

@@ -40,7 +40,11 @@ func (handler *Handler) customTemplateFile(w http.ResponseWriter, r *http.Reques
 		return httperror.InternalServerError("Unable to find a custom template with the specified identifier inside the database", err)
 	}
 
-	fileContent, err := handler.FileService.GetFileContent(customTemplate.ProjectPath, customTemplate.EntryPoint)
+	entryPath := customTemplate.EntryPoint
+	if customTemplate.GitConfig != nil {
+		entryPath = customTemplate.GitConfig.ConfigFilePath
+	}
+	fileContent, err := handler.FileService.GetFileContent(customTemplate.ProjectPath, entryPath)
 	if err != nil {
 		return httperror.InternalServerError("Unable to retrieve custom template file from disk", err)
 	}

api/http/handler/customtemplates/customtemplate_git_fetch.go

@@ -0,0 +1,101 @@
+package customtemplates
+
+import (
+	"fmt"
+	"net/http"
+	"os"
+
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/stacks/stackutils"
+)
+
+// @id CustomTemplateGitFetch
+// @summary Fetch the latest config file content based on custom template's git repository configuration
+// @description Retrieve details about a template created from git repository method.
+// @description **Access policy**: authenticated
+// @tags custom_templates
+// @security ApiKeyAuth
+// @security jwt
+// @produce json
+// @param id path int true "Template identifier"
+// @success 200 {object} portaineree.CustomTemplate "Success"
+// @failure 400 "Invalid request"
+// @failure 500 "Server error"
+// @router /custom_templates/{id}/git_fetch [put]
+func (handler *Handler) customTemplateGitFetch(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	customTemplateID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	if err != nil {
+		return httperror.BadRequest("Invalid Custom template identifier route variable", err)
+	}
+
+	customTemplate, err := handler.DataStore.CustomTemplate().CustomTemplate(portainer.CustomTemplateID(customTemplateID))
+	if handler.DataStore.IsErrObjectNotFound(err) {
+		return httperror.NotFound("Unable to find a custom template with the specified identifier inside the database", err)
+	} else if err != nil {
+		return httperror.InternalServerError("Unable to find a custom template with the specified identifier inside the database", err)
+	}
+
+	if customTemplate.GitConfig != nil {
+		// back up the current custom template folder
+		backupPath, err := backupCustomTemplate(customTemplate.ProjectPath)
+		if err != nil {
+			return httperror.InternalServerError("Failed to backup the custom template folder", err)
+		}
+
+		// remove backup custom template folder
+		defer cleanUpBackupCustomTemplate(backupPath)
+
+		commitHash, err := stackutils.DownloadGitRepository(*customTemplate.GitConfig, handler.GitService, func() string {
+			return customTemplate.ProjectPath
+		})
+		if err != nil {
+			err = rollbackCustomTemplate(backupPath, customTemplate.ProjectPath)
+			if err != nil {
+				return httperror.InternalServerError("Failed to rollback the custom template folder", err)
+			}
+			return httperror.InternalServerError(err.Error(), err)
+		}
+
+		if customTemplate.GitConfig.ConfigHash != commitHash {
+			customTemplate.GitConfig.ConfigHash = commitHash
+		}
+
+		err = handler.DataStore.CustomTemplate().UpdateCustomTemplate(customTemplate.ID, customTemplate)
+		if err != nil {
+			return httperror.InternalServerError("Unable to persist custom template changes inside the database", err)
+		}
+	}
+
+	return response.JSON(w, customTemplate)
+}
+
+func backupCustomTemplate(projectPath string) (string, error) {
+	stat, err := os.Stat(projectPath)
+	if err != nil {
+		return "", err
+	}
+
+	backupPath := fmt.Sprintf("%s-backup", projectPath)
+	err = os.Rename(projectPath, backupPath)
+	if err != nil {
+		return "", err
+	}
+
+	err = os.Mkdir(projectPath, stat.Mode())
+	if err != nil {
+		return backupPath, err
+	}
+	return backupPath, nil
+}
+
+func rollbackCustomTemplate(backupPath, projectPath string) error {
+	os.RemoveAll(projectPath)
+	return os.Rename(backupPath, projectPath)
+}
+
+func cleanUpBackupCustomTemplate(backupPath string) error {
+	return os.RemoveAll(backupPath)
+}

api/http/handler/customtemplates/customtemplate_update.go

@@ -3,15 +3,17 @@ package customtemplates
 import (
 	"errors"
 	"net/http"
-	"strconv"
 
 	"github.com/asaskevich/govalidator"
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/filesystem"
+	gittypes "github.com/portainer/portainer/api/git/types"
 	httperrors "github.com/portainer/portainer/api/http/errors"
 	"github.com/portainer/portainer/api/http/security"
+	"github.com/portainer/portainer/api/stacks/stackutils"
 )
 
 type customTemplateUpdatePayload struct {
@@ -29,18 +31,37 @@ type customTemplateUpdatePayload struct {
 	Platform portainer.CustomTemplatePlatform `example:"1" enums:"1,2"`
 	// Type of created stack (1 - swarm, 2 - compose, 3 - kubernetes)
 	Type portainer.StackType `example:"1" enums:"1,2,3" validate:"required"`
+	// URL of a Git repository hosting the Stack file
+	RepositoryURL string `example:"https://github.com/openfaas/faas" validate:"required"`
+	// Reference name of a Git repository hosting the Stack file
+	RepositoryReferenceName string `example:"refs/heads/master"`
+	// Use basic authentication to clone the Git repository
+	RepositoryAuthentication bool `example:"true"`
+	// Username used in basic authentication. Required when RepositoryAuthentication is true
+	// and RepositoryGitCredentialID is 0
+	RepositoryUsername string `example:"myGitUsername"`
+	// Password used in basic authentication. Required when RepositoryAuthentication is true
+	// and RepositoryGitCredentialID is 0
+	RepositoryPassword string `example:"myGitPassword"`
+	// GitCredentialID used to identify the bound git credential. Required when RepositoryAuthentication
+	// is true and RepositoryUsername/RepositoryPassword are not provided
+	RepositoryGitCredentialID int `example:"0"`
+	// Path to the Stack file inside the Git repository
+	ComposeFilePathInRepository string `example:"docker-compose.yml" default:"docker-compose.yml"`
 	// Content of stack file
 	FileContent string `validate:"required"`
 	// Definitions of variables in the stack file
 	Variables []portainer.CustomTemplateVariableDefinition
+	// IsComposeFormat indicates if the Kubernetes template is created from a Docker Compose file
+	IsComposeFormat bool `example:"false"`
 }
 
 func (payload *customTemplateUpdatePayload) Validate(r *http.Request) error {
 	if govalidator.IsNull(payload.Title) {
 		return errors.New("Invalid custom template title")
 	}
-	if govalidator.IsNull(payload.FileContent) {
-		return errors.New("Invalid file content")
+	if govalidator.IsNull(payload.FileContent) && govalidator.IsNull(payload.RepositoryURL) {
+		return errors.New("Either file content or git repository url need to be provided")
 	}
 	if payload.Type != portainer.KubernetesStack && payload.Platform != portainer.CustomTemplatePlatformLinux && payload.Platform != portainer.CustomTemplatePlatformWindows {
 		return errors.New("Invalid custom template platform")
@@ -55,6 +76,16 @@ func (payload *customTemplateUpdatePayload) Validate(r *http.Request) error {
 		return errors.New("Invalid note. <img> tag is not supported")
 	}
 
+	if govalidator.IsNull(payload.RepositoryURL) || !govalidator.IsURL(payload.RepositoryURL) {
+		return errors.New("Invalid repository URL. Must correspond to a valid URL format")
+	}
+	if payload.RepositoryAuthentication && (govalidator.IsNull(payload.RepositoryUsername) || govalidator.IsNull(payload.RepositoryPassword)) {
+		return errors.New("Invalid repository credentials. Username and password must be specified when authentication is enabled")
+	}
+	if govalidator.IsNull(payload.ComposeFilePathInRepository) {
+		payload.ComposeFilePathInRepository = filesystem.ComposeFileDefaultName
+	}
+
 	err := validateVariablesDefinitions(payload.Variables)
 	if err != nil {
 		return err
@@ -120,12 +151,6 @@ func (handler *Handler) customTemplateUpdate(w http.ResponseWriter, r *http.Requ
 		return httperror.Forbidden("Access denied to resource", httperrors.ErrResourceAccessDenied)
 	}
 
-	templateFolder := strconv.Itoa(customTemplateID)
-	_, err = handler.FileService.StoreCustomTemplateFileFromBytes(templateFolder, customTemplate.EntryPoint, []byte(payload.FileContent))
-	if err != nil {
-		return httperror.InternalServerError("Unable to persist updated custom template file on disk", err)
-	}
-
 	customTemplate.Title = payload.Title
 	customTemplate.Logo = payload.Logo
 	customTemplate.Description = payload.Description
@@ -133,6 +158,32 @@ func (handler *Handler) customTemplateUpdate(w http.ResponseWriter, r *http.Requ
 	customTemplate.Platform = payload.Platform
 	customTemplate.Type = payload.Type
 	customTemplate.Variables = payload.Variables
+	customTemplate.IsComposeFormat = payload.IsComposeFormat
+
+	if payload.RepositoryURL != "" {
+		gitConfig := &gittypes.RepoConfig{
+			URL:            payload.RepositoryURL,
+			ReferenceName:  payload.RepositoryReferenceName,
+			ConfigFilePath: payload.ComposeFilePathInRepository,
+		}
+
+		if payload.RepositoryAuthentication {
+			gitConfig.Authentication = &gittypes.GitAuthentication{
+				Username: payload.RepositoryUsername,
+				Password: payload.RepositoryPassword,
+			}
+		}
+
+		commitHash, err := stackutils.DownloadGitRepository(*gitConfig, handler.GitService, func() string {
+			return customTemplate.ProjectPath
+		})
+		if err != nil {
+			return httperror.InternalServerError(err.Error(), err)
+		}
+
+		gitConfig.ConfigHash = commitHash
+		customTemplate.GitConfig = gitConfig
+	}
 
 	err = handler.DataStore.CustomTemplate().UpdateCustomTemplate(customTemplate.ID, customTemplate)
 	if err != nil {

api/http/handler/customtemplates/handler.go

@@ -36,6 +36,8 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler {
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.customTemplateUpdate))).Methods(http.MethodPut)
 	h.Handle("/custom_templates/{id}",
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.customTemplateDelete))).Methods(http.MethodDelete)
+	h.Handle("/custom_templates/{id}/git_fetch",
+		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.customTemplateGitFetch))).Methods(http.MethodPut)
 	return h
 }
 

api/http/handler/stacks/create_kubernetes_stack.go

@@ -22,14 +22,17 @@ type kubernetesStringDeploymentPayload struct {
 	ComposeFormat    bool
 	Namespace        string
 	StackFileContent string
+	// Whether the stack is from a app template
+	FromAppTemplate bool `example:"false"`
 }
 
-func createStackPayloadFromK8sFileContentPayload(name, namespace, fileContent string, composeFormat bool) stackbuilders.StackPayload {
+func createStackPayloadFromK8sFileContentPayload(name, namespace, fileContent string, composeFormat, fromAppTemplate bool) stackbuilders.StackPayload {
 	return stackbuilders.StackPayload{
 		StackName:        name,
 		Namespace:        namespace,
 		StackFileContent: fileContent,
 		ComposeFormat:    composeFormat,
+		FromAppTemplate:  fromAppTemplate,
 	}
 }
 
@@ -142,7 +145,7 @@ func (handler *Handler) createKubernetesStackFromFileContent(w http.ResponseWrit
 		return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A stack with the name '%s' already exists", payload.StackName), Err: stackutils.ErrStackAlreadyExists}
 	}
 
-	stackPayload := createStackPayloadFromK8sFileContentPayload(payload.StackName, payload.Namespace, payload.StackFileContent, payload.ComposeFormat)
+	stackPayload := createStackPayloadFromK8sFileContentPayload(payload.StackName, payload.Namespace, payload.StackFileContent, payload.ComposeFormat, payload.FromAppTemplate)
 
 	k8sStackBuilder := stackbuilders.CreateK8sStackFileContentBuilder(handler.DataStore,
 		handler.FileService,

api/portainer.go

@@ -165,6 +165,9 @@ type (
 		Type            StackType        `json:"Type" example:"1"`
 		ResourceControl *ResourceControl `json:"ResourceControl"`
 		Variables       []CustomTemplateVariableDefinition
+		GitConfig       *gittypes.RepoConfig `json:"GitConfig"`
+		// IsComposeFormat indicates if the Kubernetes template is created from a Docker Compose file
+		IsComposeFormat bool `example:"false"`
 	}
 
 	// CustomTemplateID represents a custom template identifier

api/stacks/stackbuilders/k8s_file_content_builder.go

@@ -53,6 +53,7 @@ func (b *K8sStackFileContentBuilder) SetUniqueInfo(payload *StackPayload) FileCo
 	b.stack.Namespace = payload.Namespace
 	b.stack.CreatedBy = b.User.Username
 	b.stack.IsComposeFormat = payload.ComposeFormat
+	b.stack.FromAppTemplate = payload.FromAppTemplate
 	return b
 }
 

api/stacks/stackbuilders/stack_git_builder.go

@@ -1,6 +1,7 @@
 package stackbuilders
 
 import (
+	"fmt"
 	"strconv"
 	"time"
 
@@ -76,7 +77,12 @@ func (b *GitMethodStackBuilder) SetGitRepository(payload *StackPayload) GitMetho
 	// Set the project path on the disk
 	b.stack.ProjectPath = b.fileService.GetStackProjectPath(stackFolder)
 
-	commitHash, err := stackutils.DownloadGitRepository(b.stack.ID, repoConfig, b.gitService, b.fileService)
+	getProjectPath := func() string {
+		stackFolder := fmt.Sprintf("%d", b.stack.ID)
+		return b.fileService.GetStackProjectPath(stackFolder)
+	}
+
+	commitHash, err := stackutils.DownloadGitRepository(repoConfig, b.gitService, getProjectPath)
 	if err != nil {
 		b.err = httperror.InternalServerError(err.Error(), err)
 		return b

api/stacks/stackutils/gitops.go

@@ -16,7 +16,7 @@ var (
 
 // DownloadGitRepository downloads the target git repository on the disk
 // The first return value represents the commit hash of the downloaded git repository
-func DownloadGitRepository(stackID portainer.StackID, config gittypes.RepoConfig, gitService portainer.GitService, fileService portainer.FileService) (string, error) {
+func DownloadGitRepository(config gittypes.RepoConfig, gitService portainer.GitService, getProjectPath func() string) (string, error) {
 	username := ""
 	password := ""
 	if config.Authentication != nil {
@@ -24,9 +24,7 @@ func DownloadGitRepository(stackID portainer.StackID, config gittypes.RepoConfig
 		password = config.Authentication.Password
 	}
 
-	stackFolder := fmt.Sprintf("%d", stackID)
-	projectPath := fileService.GetStackProjectPath(stackFolder)
-
+	projectPath := getProjectPath()
 	err := gitService.CloneRepository(projectPath, config.URL, config.ReferenceName, username, password)
 	if err != nil {
 		if err == gittypes.ErrAuthenticationFailure {