From aedae9b4caebdb396f4ef908aed405fce513fa2b Mon Sep 17 00:00:00 2001 From: oscarzhou Date: Tue, 18 Oct 2022 15:28:32 +1300 Subject: [PATCH] feat(template/gitops): support to add/update k8s custom template with git repository method --- .../customtemplates/customtemplate_create.go | 59 +++++----- .../customtemplates/customtemplate_file.go | 6 +- .../customtemplate_git_fetch.go | 101 ++++++++++++++++++ .../customtemplates/customtemplate_update.go | 69 ++++++++++-- api/http/handler/customtemplates/handler.go | 2 + .../handler/stacks/create_kubernetes_stack.go | 7 +- api/portainer.go | 3 + .../stackbuilders/k8s_file_content_builder.go | 1 + api/stacks/stackbuilders/stack_git_builder.go | 8 +- api/stacks/stackutils/gitops.go | 6 +- 10 files changed, 219 insertions(+), 43 deletions(-) create mode 100644 api/http/handler/customtemplates/customtemplate_git_fetch.go diff --git a/api/http/handler/customtemplates/customtemplate_create.go b/api/http/handler/customtemplates/customtemplate_create.go index c026a540e..f5d69375c 100644 --- a/api/http/handler/customtemplates/customtemplate_create.go +++ b/api/http/handler/customtemplates/customtemplate_create.go @@ -3,7 +3,6 @@ package customtemplates import ( "encoding/json" "errors" - "fmt" "net/http" "os" "regexp" @@ -18,6 +17,7 @@ import ( gittypes "github.com/portainer/portainer/api/git/types" "github.com/portainer/portainer/api/http/security" "github.com/portainer/portainer/api/internal/authorization" + "github.com/portainer/portainer/api/stacks/stackutils" "github.com/rs/zerolog/log" ) @@ -218,6 +218,8 @@ type customTemplateFromGitRepositoryPayload struct { ComposeFilePathInRepository string `example:"docker-compose.yml" default:"docker-compose.yml"` // Definitions of variables in the stack file Variables []portainer.CustomTemplateVariableDefinition + // IsComposeFormat indicates if the Kubernetes template is created from a Docker Compose file + IsComposeFormat bool `example:"false"` } func (payload *customTemplateFromGitRepositoryPayload) Validate(r *http.Request) error { @@ -237,14 +239,10 @@ func (payload *customTemplateFromGitRepositoryPayload) Validate(r *http.Request) payload.ComposeFilePathInRepository = filesystem.ComposeFileDefaultName } - if payload.Type == portainer.KubernetesStack { - return errors.New("Creating a Kubernetes custom template from git is not supported") - } - if payload.Platform != portainer.CustomTemplatePlatformLinux && payload.Platform != portainer.CustomTemplatePlatformWindows { return errors.New("Invalid custom template platform") } - if payload.Type != portainer.DockerSwarmStack && payload.Type != portainer.DockerComposeStack { + if payload.Type != portainer.DockerSwarmStack && payload.Type != portainer.DockerComposeStack && payload.Type != portainer.KubernetesStack { return errors.New("Invalid custom template type") } if !isValidNote(payload.Note) { @@ -268,35 +266,44 @@ func (handler *Handler) createCustomTemplateFromGitRepository(r *http.Request) ( customTemplateID := handler.DataStore.CustomTemplate().GetNextIdentifier() customTemplate := &portainer.CustomTemplate{ - ID: portainer.CustomTemplateID(customTemplateID), - Title: payload.Title, - EntryPoint: payload.ComposeFilePathInRepository, - Description: payload.Description, - Note: payload.Note, - Platform: payload.Platform, - Type: payload.Type, - Logo: payload.Logo, - Variables: payload.Variables, + ID: portainer.CustomTemplateID(customTemplateID), + Title: payload.Title, + Description: payload.Description, + Note: payload.Note, + Platform: payload.Platform, + Type: payload.Type, + Logo: payload.Logo, + Variables: payload.Variables, + IsComposeFormat: payload.IsComposeFormat, } - projectPath := handler.FileService.GetCustomTemplateProjectPath(strconv.Itoa(customTemplateID)) + getProjectPath := func() string { + return handler.FileService.GetCustomTemplateProjectPath(strconv.Itoa(customTemplateID)) + } + projectPath := getProjectPath() customTemplate.ProjectPath = projectPath - repositoryUsername := payload.RepositoryUsername - repositoryPassword := payload.RepositoryPassword - if !payload.RepositoryAuthentication { - repositoryUsername = "" - repositoryPassword = "" + gitConfig := &gittypes.RepoConfig{ + URL: payload.RepositoryURL, + ReferenceName: payload.RepositoryReferenceName, + ConfigFilePath: payload.ComposeFilePathInRepository, } - err = handler.GitService.CloneRepository(projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, repositoryUsername, repositoryPassword) - if err != nil { - if err == gittypes.ErrAuthenticationFailure { - return nil, fmt.Errorf("invalid git credential") + if payload.RepositoryAuthentication { + gitConfig.Authentication = &gittypes.GitAuthentication{ + Username: payload.RepositoryUsername, + Password: payload.RepositoryPassword, } + } + + commitHash, err := stackutils.DownloadGitRepository(*gitConfig, handler.GitService, getProjectPath) + if err != nil { return nil, err } + gitConfig.ConfigHash = commitHash + customTemplate.GitConfig = gitConfig + isValidProject := true defer func() { if !isValidProject { @@ -306,7 +313,7 @@ func (handler *Handler) createCustomTemplateFromGitRepository(r *http.Request) ( } }() - entryPath := filesystem.JoinPaths(projectPath, customTemplate.EntryPoint) + entryPath := filesystem.JoinPaths(projectPath, gitConfig.ConfigFilePath) exists, err := handler.FileService.FileExists(entryPath) if err != nil || !exists { diff --git a/api/http/handler/customtemplates/customtemplate_file.go b/api/http/handler/customtemplates/customtemplate_file.go index 7666bae25..3749d6c66 100644 --- a/api/http/handler/customtemplates/customtemplate_file.go +++ b/api/http/handler/customtemplates/customtemplate_file.go @@ -40,7 +40,11 @@ func (handler *Handler) customTemplateFile(w http.ResponseWriter, r *http.Reques return httperror.InternalServerError("Unable to find a custom template with the specified identifier inside the database", err) } - fileContent, err := handler.FileService.GetFileContent(customTemplate.ProjectPath, customTemplate.EntryPoint) + entryPath := customTemplate.EntryPoint + if customTemplate.GitConfig != nil { + entryPath = customTemplate.GitConfig.ConfigFilePath + } + fileContent, err := handler.FileService.GetFileContent(customTemplate.ProjectPath, entryPath) if err != nil { return httperror.InternalServerError("Unable to retrieve custom template file from disk", err) } diff --git a/api/http/handler/customtemplates/customtemplate_git_fetch.go b/api/http/handler/customtemplates/customtemplate_git_fetch.go new file mode 100644 index 000000000..236314c6f --- /dev/null +++ b/api/http/handler/customtemplates/customtemplate_git_fetch.go @@ -0,0 +1,101 @@ +package customtemplates + +import ( + "fmt" + "net/http" + "os" + + httperror "github.com/portainer/libhttp/error" + "github.com/portainer/libhttp/request" + "github.com/portainer/libhttp/response" + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/stacks/stackutils" +) + +// @id CustomTemplateGitFetch +// @summary Fetch the latest config file content based on custom template's git repository configuration +// @description Retrieve details about a template created from git repository method. +// @description **Access policy**: authenticated +// @tags custom_templates +// @security ApiKeyAuth +// @security jwt +// @produce json +// @param id path int true "Template identifier" +// @success 200 {object} portaineree.CustomTemplate "Success" +// @failure 400 "Invalid request" +// @failure 500 "Server error" +// @router /custom_templates/{id}/git_fetch [put] +func (handler *Handler) customTemplateGitFetch(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { + customTemplateID, err := request.RetrieveNumericRouteVariableValue(r, "id") + if err != nil { + return httperror.BadRequest("Invalid Custom template identifier route variable", err) + } + + customTemplate, err := handler.DataStore.CustomTemplate().CustomTemplate(portainer.CustomTemplateID(customTemplateID)) + if handler.DataStore.IsErrObjectNotFound(err) { + return httperror.NotFound("Unable to find a custom template with the specified identifier inside the database", err) + } else if err != nil { + return httperror.InternalServerError("Unable to find a custom template with the specified identifier inside the database", err) + } + + if customTemplate.GitConfig != nil { + // back up the current custom template folder + backupPath, err := backupCustomTemplate(customTemplate.ProjectPath) + if err != nil { + return httperror.InternalServerError("Failed to backup the custom template folder", err) + } + + // remove backup custom template folder + defer cleanUpBackupCustomTemplate(backupPath) + + commitHash, err := stackutils.DownloadGitRepository(*customTemplate.GitConfig, handler.GitService, func() string { + return customTemplate.ProjectPath + }) + if err != nil { + err = rollbackCustomTemplate(backupPath, customTemplate.ProjectPath) + if err != nil { + return httperror.InternalServerError("Failed to rollback the custom template folder", err) + } + return httperror.InternalServerError(err.Error(), err) + } + + if customTemplate.GitConfig.ConfigHash != commitHash { + customTemplate.GitConfig.ConfigHash = commitHash + } + + err = handler.DataStore.CustomTemplate().UpdateCustomTemplate(customTemplate.ID, customTemplate) + if err != nil { + return httperror.InternalServerError("Unable to persist custom template changes inside the database", err) + } + } + + return response.JSON(w, customTemplate) +} + +func backupCustomTemplate(projectPath string) (string, error) { + stat, err := os.Stat(projectPath) + if err != nil { + return "", err + } + + backupPath := fmt.Sprintf("%s-backup", projectPath) + err = os.Rename(projectPath, backupPath) + if err != nil { + return "", err + } + + err = os.Mkdir(projectPath, stat.Mode()) + if err != nil { + return backupPath, err + } + return backupPath, nil +} + +func rollbackCustomTemplate(backupPath, projectPath string) error { + os.RemoveAll(projectPath) + return os.Rename(backupPath, projectPath) +} + +func cleanUpBackupCustomTemplate(backupPath string) error { + return os.RemoveAll(backupPath) +} diff --git a/api/http/handler/customtemplates/customtemplate_update.go b/api/http/handler/customtemplates/customtemplate_update.go index 7b0a5e750..313145b4d 100644 --- a/api/http/handler/customtemplates/customtemplate_update.go +++ b/api/http/handler/customtemplates/customtemplate_update.go @@ -3,15 +3,17 @@ package customtemplates import ( "errors" "net/http" - "strconv" "github.com/asaskevich/govalidator" httperror "github.com/portainer/libhttp/error" "github.com/portainer/libhttp/request" "github.com/portainer/libhttp/response" portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/filesystem" + gittypes "github.com/portainer/portainer/api/git/types" httperrors "github.com/portainer/portainer/api/http/errors" "github.com/portainer/portainer/api/http/security" + "github.com/portainer/portainer/api/stacks/stackutils" ) type customTemplateUpdatePayload struct { @@ -29,18 +31,37 @@ type customTemplateUpdatePayload struct { Platform portainer.CustomTemplatePlatform `example:"1" enums:"1,2"` // Type of created stack (1 - swarm, 2 - compose, 3 - kubernetes) Type portainer.StackType `example:"1" enums:"1,2,3" validate:"required"` + // URL of a Git repository hosting the Stack file + RepositoryURL string `example:"https://github.com/openfaas/faas" validate:"required"` + // Reference name of a Git repository hosting the Stack file + RepositoryReferenceName string `example:"refs/heads/master"` + // Use basic authentication to clone the Git repository + RepositoryAuthentication bool `example:"true"` + // Username used in basic authentication. Required when RepositoryAuthentication is true + // and RepositoryGitCredentialID is 0 + RepositoryUsername string `example:"myGitUsername"` + // Password used in basic authentication. Required when RepositoryAuthentication is true + // and RepositoryGitCredentialID is 0 + RepositoryPassword string `example:"myGitPassword"` + // GitCredentialID used to identify the bound git credential. Required when RepositoryAuthentication + // is true and RepositoryUsername/RepositoryPassword are not provided + RepositoryGitCredentialID int `example:"0"` + // Path to the Stack file inside the Git repository + ComposeFilePathInRepository string `example:"docker-compose.yml" default:"docker-compose.yml"` // Content of stack file FileContent string `validate:"required"` // Definitions of variables in the stack file Variables []portainer.CustomTemplateVariableDefinition + // IsComposeFormat indicates if the Kubernetes template is created from a Docker Compose file + IsComposeFormat bool `example:"false"` } func (payload *customTemplateUpdatePayload) Validate(r *http.Request) error { if govalidator.IsNull(payload.Title) { return errors.New("Invalid custom template title") } - if govalidator.IsNull(payload.FileContent) { - return errors.New("Invalid file content") + if govalidator.IsNull(payload.FileContent) && govalidator.IsNull(payload.RepositoryURL) { + return errors.New("Either file content or git repository url need to be provided") } if payload.Type != portainer.KubernetesStack && payload.Platform != portainer.CustomTemplatePlatformLinux && payload.Platform != portainer.CustomTemplatePlatformWindows { return errors.New("Invalid custom template platform") @@ -55,6 +76,16 @@ func (payload *customTemplateUpdatePayload) Validate(r *http.Request) error { return errors.New("Invalid note. tag is not supported") } + if govalidator.IsNull(payload.RepositoryURL) || !govalidator.IsURL(payload.RepositoryURL) { + return errors.New("Invalid repository URL. Must correspond to a valid URL format") + } + if payload.RepositoryAuthentication && (govalidator.IsNull(payload.RepositoryUsername) || govalidator.IsNull(payload.RepositoryPassword)) { + return errors.New("Invalid repository credentials. Username and password must be specified when authentication is enabled") + } + if govalidator.IsNull(payload.ComposeFilePathInRepository) { + payload.ComposeFilePathInRepository = filesystem.ComposeFileDefaultName + } + err := validateVariablesDefinitions(payload.Variables) if err != nil { return err @@ -120,12 +151,6 @@ func (handler *Handler) customTemplateUpdate(w http.ResponseWriter, r *http.Requ return httperror.Forbidden("Access denied to resource", httperrors.ErrResourceAccessDenied) } - templateFolder := strconv.Itoa(customTemplateID) - _, err = handler.FileService.StoreCustomTemplateFileFromBytes(templateFolder, customTemplate.EntryPoint, []byte(payload.FileContent)) - if err != nil { - return httperror.InternalServerError("Unable to persist updated custom template file on disk", err) - } - customTemplate.Title = payload.Title customTemplate.Logo = payload.Logo customTemplate.Description = payload.Description @@ -133,6 +158,32 @@ func (handler *Handler) customTemplateUpdate(w http.ResponseWriter, r *http.Requ customTemplate.Platform = payload.Platform customTemplate.Type = payload.Type customTemplate.Variables = payload.Variables + customTemplate.IsComposeFormat = payload.IsComposeFormat + + if payload.RepositoryURL != "" { + gitConfig := &gittypes.RepoConfig{ + URL: payload.RepositoryURL, + ReferenceName: payload.RepositoryReferenceName, + ConfigFilePath: payload.ComposeFilePathInRepository, + } + + if payload.RepositoryAuthentication { + gitConfig.Authentication = &gittypes.GitAuthentication{ + Username: payload.RepositoryUsername, + Password: payload.RepositoryPassword, + } + } + + commitHash, err := stackutils.DownloadGitRepository(*gitConfig, handler.GitService, func() string { + return customTemplate.ProjectPath + }) + if err != nil { + return httperror.InternalServerError(err.Error(), err) + } + + gitConfig.ConfigHash = commitHash + customTemplate.GitConfig = gitConfig + } err = handler.DataStore.CustomTemplate().UpdateCustomTemplate(customTemplate.ID, customTemplate) if err != nil { diff --git a/api/http/handler/customtemplates/handler.go b/api/http/handler/customtemplates/handler.go index 7bc9b4b3d..4fa682778 100644 --- a/api/http/handler/customtemplates/handler.go +++ b/api/http/handler/customtemplates/handler.go @@ -36,6 +36,8 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler { bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.customTemplateUpdate))).Methods(http.MethodPut) h.Handle("/custom_templates/{id}", bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.customTemplateDelete))).Methods(http.MethodDelete) + h.Handle("/custom_templates/{id}/git_fetch", + bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.customTemplateGitFetch))).Methods(http.MethodPut) return h } diff --git a/api/http/handler/stacks/create_kubernetes_stack.go b/api/http/handler/stacks/create_kubernetes_stack.go index 09cc8f04a..ae5bc90c3 100644 --- a/api/http/handler/stacks/create_kubernetes_stack.go +++ b/api/http/handler/stacks/create_kubernetes_stack.go @@ -22,14 +22,17 @@ type kubernetesStringDeploymentPayload struct { ComposeFormat bool Namespace string StackFileContent string + // Whether the stack is from a app template + FromAppTemplate bool `example:"false"` } -func createStackPayloadFromK8sFileContentPayload(name, namespace, fileContent string, composeFormat bool) stackbuilders.StackPayload { +func createStackPayloadFromK8sFileContentPayload(name, namespace, fileContent string, composeFormat, fromAppTemplate bool) stackbuilders.StackPayload { return stackbuilders.StackPayload{ StackName: name, Namespace: namespace, StackFileContent: fileContent, ComposeFormat: composeFormat, + FromAppTemplate: fromAppTemplate, } } @@ -142,7 +145,7 @@ func (handler *Handler) createKubernetesStackFromFileContent(w http.ResponseWrit return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A stack with the name '%s' already exists", payload.StackName), Err: stackutils.ErrStackAlreadyExists} } - stackPayload := createStackPayloadFromK8sFileContentPayload(payload.StackName, payload.Namespace, payload.StackFileContent, payload.ComposeFormat) + stackPayload := createStackPayloadFromK8sFileContentPayload(payload.StackName, payload.Namespace, payload.StackFileContent, payload.ComposeFormat, payload.FromAppTemplate) k8sStackBuilder := stackbuilders.CreateK8sStackFileContentBuilder(handler.DataStore, handler.FileService, diff --git a/api/portainer.go b/api/portainer.go index 11ae98781..d7b33075a 100644 --- a/api/portainer.go +++ b/api/portainer.go @@ -165,6 +165,9 @@ type ( Type StackType `json:"Type" example:"1"` ResourceControl *ResourceControl `json:"ResourceControl"` Variables []CustomTemplateVariableDefinition + GitConfig *gittypes.RepoConfig `json:"GitConfig"` + // IsComposeFormat indicates if the Kubernetes template is created from a Docker Compose file + IsComposeFormat bool `example:"false"` } // CustomTemplateID represents a custom template identifier diff --git a/api/stacks/stackbuilders/k8s_file_content_builder.go b/api/stacks/stackbuilders/k8s_file_content_builder.go index ee0335465..783c2110b 100644 --- a/api/stacks/stackbuilders/k8s_file_content_builder.go +++ b/api/stacks/stackbuilders/k8s_file_content_builder.go @@ -53,6 +53,7 @@ func (b *K8sStackFileContentBuilder) SetUniqueInfo(payload *StackPayload) FileCo b.stack.Namespace = payload.Namespace b.stack.CreatedBy = b.User.Username b.stack.IsComposeFormat = payload.ComposeFormat + b.stack.FromAppTemplate = payload.FromAppTemplate return b } diff --git a/api/stacks/stackbuilders/stack_git_builder.go b/api/stacks/stackbuilders/stack_git_builder.go index 212a73e62..b96da7178 100644 --- a/api/stacks/stackbuilders/stack_git_builder.go +++ b/api/stacks/stackbuilders/stack_git_builder.go @@ -1,6 +1,7 @@ package stackbuilders import ( + "fmt" "strconv" "time" @@ -76,7 +77,12 @@ func (b *GitMethodStackBuilder) SetGitRepository(payload *StackPayload) GitMetho // Set the project path on the disk b.stack.ProjectPath = b.fileService.GetStackProjectPath(stackFolder) - commitHash, err := stackutils.DownloadGitRepository(b.stack.ID, repoConfig, b.gitService, b.fileService) + getProjectPath := func() string { + stackFolder := fmt.Sprintf("%d", b.stack.ID) + return b.fileService.GetStackProjectPath(stackFolder) + } + + commitHash, err := stackutils.DownloadGitRepository(repoConfig, b.gitService, getProjectPath) if err != nil { b.err = httperror.InternalServerError(err.Error(), err) return b diff --git a/api/stacks/stackutils/gitops.go b/api/stacks/stackutils/gitops.go index 325e13637..e7fee5bfb 100644 --- a/api/stacks/stackutils/gitops.go +++ b/api/stacks/stackutils/gitops.go @@ -16,7 +16,7 @@ var ( // DownloadGitRepository downloads the target git repository on the disk // The first return value represents the commit hash of the downloaded git repository -func DownloadGitRepository(stackID portainer.StackID, config gittypes.RepoConfig, gitService portainer.GitService, fileService portainer.FileService) (string, error) { +func DownloadGitRepository(config gittypes.RepoConfig, gitService portainer.GitService, getProjectPath func() string) (string, error) { username := "" password := "" if config.Authentication != nil { @@ -24,9 +24,7 @@ func DownloadGitRepository(stackID portainer.StackID, config gittypes.RepoConfig password = config.Authentication.Password } - stackFolder := fmt.Sprintf("%d", stackID) - projectPath := fileService.GetStackProjectPath(stackFolder) - + projectPath := getProjectPath() err := gitService.CloneRepository(projectPath, config.URL, config.ReferenceName, username, password) if err != nil { if err == gittypes.ErrAuthenticationFailure {