diff --git a/api/http/handler/stacks/create_compose_stack.go b/api/http/handler/stacks/create_compose_stack.go index 2c78ee0f9..a12538aff 100644 --- a/api/http/handler/stacks/create_compose_stack.go +++ b/api/http/handler/stacks/create_compose_stack.go @@ -112,8 +112,9 @@ type composeStackFromGitRepositoryPayload struct { // Password used in basic authentication. Required when RepositoryAuthentication is true. RepositoryPassword string `example:"myGitPassword"` // Path to the Stack file inside the Git repository - ComposeFilePathInRepository string `example:"docker-compose.yml" default:"docker-compose.yml"` - + ComposeFile string `example:"docker-compose.yml" default:"docker-compose.yml"` + AdditionalFiles []string + AutoUpdate *portainer.StackAutoUpdate // A list of environment variables used during stack deployment Env []portainer.Pair } @@ -126,8 +127,11 @@ func (payload *composeStackFromGitRepositoryPayload) Validate(r *http.Request) e if govalidator.IsNull(payload.RepositoryURL) || !govalidator.IsURL(payload.RepositoryURL) { return errors.New("Invalid repository URL. Must correspond to a valid URL format") } - if payload.RepositoryAuthentication && (govalidator.IsNull(payload.RepositoryUsername) || govalidator.IsNull(payload.RepositoryPassword)) { - return errors.New("Invalid repository credentials. Username and password must be specified when authentication is enabled") + if govalidator.IsNull(payload.RepositoryReferenceName) { + return errors.New("Invalid RepositoryReferenceName") + } + if payload.RepositoryAuthentication && govalidator.IsNull(payload.RepositoryPassword) { + return errors.New("Invalid repository credentials. Password must be specified when authentication is enabled") } return nil @@ -141,8 +145,8 @@ func (handler *Handler) createComposeStackFromGitRepository(w http.ResponseWrite } payload.Name = handler.ComposeStackManager.NormalizeStackName(payload.Name) - if payload.ComposeFilePathInRepository == "" { - payload.ComposeFilePathInRepository = filesystem.ComposeFileDefaultName + if payload.ComposeFile == "" { + payload.ComposeFile = filesystem.ComposeFileDefaultName } isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, false) @@ -156,16 +160,29 @@ func (handler *Handler) createComposeStackFromGitRepository(w http.ResponseWrite stackID := handler.DataStore.Stack().GetNextIdentifier() stack := &portainer.Stack{ - ID: portainer.StackID(stackID), - Name: payload.Name, - Type: portainer.DockerComposeStack, - EndpointID: endpoint.ID, - EntryPoint: payload.ComposeFilePathInRepository, - Env: payload.Env, + ID: portainer.StackID(stackID), + Name: payload.Name, + Type: portainer.DockerComposeStack, + EndpointID: endpoint.ID, + EntryPoint: payload.ComposeFile, + AdditionalFiles: payload.AdditionalFiles, + AutoUpdate: payload.AutoUpdate, + Env: payload.Env, + GitConfig: &portainer.GitConfig{ + URL: payload.RepositoryURL, + ReferenceName: payload.RepositoryReferenceName, + }, Status: portainer.StackStatusActive, CreationDate: time.Now().Unix(), } + if payload.RepositoryAuthentication { + stack.GitConfig.Authentication = &portainer.GitAuthentication{ + Username: payload.RepositoryUsername, + Password: payload.RepositoryPassword, + } + } + projectPath := handler.FileService.GetStackProjectPath(strconv.Itoa(int(stack.ID))) stack.ProjectPath = projectPath diff --git a/api/http/handler/stacks/create_swarm_stack.go b/api/http/handler/stacks/create_swarm_stack.go index e34df227e..6838a3bb2 100644 --- a/api/http/handler/stacks/create_swarm_stack.go +++ b/api/http/handler/stacks/create_swarm_stack.go @@ -119,7 +119,9 @@ type swarmStackFromGitRepositoryPayload struct { // Password used in basic authentication. Required when RepositoryAuthentication is true. RepositoryPassword string `example:"myGitPassword"` // Path to the Stack file inside the Git repository - ComposeFilePathInRepository string `example:"docker-compose.yml" default:"docker-compose.yml"` + ComposeFile string `example:"docker-compose.yml" default:"docker-compose.yml"` + AdditionalFiles []string + AutoUpdate *portainer.StackAutoUpdate } func (payload *swarmStackFromGitRepositoryPayload) Validate(r *http.Request) error { @@ -132,11 +134,14 @@ func (payload *swarmStackFromGitRepositoryPayload) Validate(r *http.Request) err if govalidator.IsNull(payload.RepositoryURL) || !govalidator.IsURL(payload.RepositoryURL) { return errors.New("Invalid repository URL. Must correspond to a valid URL format") } - if payload.RepositoryAuthentication && (govalidator.IsNull(payload.RepositoryUsername) || govalidator.IsNull(payload.RepositoryPassword)) { - return errors.New("Invalid repository credentials. Username and password must be specified when authentication is enabled") + if govalidator.IsNull(payload.RepositoryReferenceName) { + return errors.New("Invalid RepositoryReferenceName") } - if govalidator.IsNull(payload.ComposeFilePathInRepository) { - payload.ComposeFilePathInRepository = filesystem.ComposeFileDefaultName + if payload.RepositoryAuthentication && govalidator.IsNull(payload.RepositoryPassword) { + return errors.New("Invalid repository credentials. Password must be specified when authentication is enabled") + } + if govalidator.IsNull(payload.ComposeFile) { + payload.ComposeFile = filesystem.ComposeFileDefaultName } return nil } @@ -159,17 +164,30 @@ func (handler *Handler) createSwarmStackFromGitRepository(w http.ResponseWriter, stackID := handler.DataStore.Stack().GetNextIdentifier() stack := &portainer.Stack{ - ID: portainer.StackID(stackID), - Name: payload.Name, - Type: portainer.DockerSwarmStack, - SwarmID: payload.SwarmID, - EndpointID: endpoint.ID, - EntryPoint: payload.ComposeFilePathInRepository, + ID: portainer.StackID(stackID), + Name: payload.Name, + Type: portainer.DockerSwarmStack, + SwarmID: payload.SwarmID, + EndpointID: endpoint.ID, + EntryPoint: payload.ComposeFile, + AdditionalFiles: payload.AdditionalFiles, + AutoUpdate: payload.AutoUpdate, + GitConfig: &portainer.GitConfig{ + URL: payload.RepositoryURL, + ReferenceName: payload.RepositoryReferenceName, + }, Env: payload.Env, Status: portainer.StackStatusActive, CreationDate: time.Now().Unix(), } + if payload.RepositoryAuthentication { + stack.GitConfig.Authentication = &portainer.GitAuthentication{ + Username: payload.RepositoryUsername, + Password: payload.RepositoryPassword, + } + } + projectPath := handler.FileService.GetStackProjectPath(strconv.Itoa(int(stack.ID))) stack.ProjectPath = projectPath diff --git a/api/http/handler/stacks/handler.go b/api/http/handler/stacks/handler.go index f61952515..83c5eb770 100644 --- a/api/http/handler/stacks/handler.go +++ b/api/http/handler/stacks/handler.go @@ -54,6 +54,8 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler { bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackDelete))).Methods(http.MethodDelete) h.Handle("/stacks/{id}", bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackUpdate))).Methods(http.MethodPut) + h.Handle("/stacks/{id}/git", + bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackGitUpdate))).Methods(http.MethodPost) h.Handle("/stacks/{id}/file", bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackFile))).Methods(http.MethodGet) h.Handle("/stacks/{id}/migrate", diff --git a/api/http/handler/stacks/helper.go b/api/http/handler/stacks/helper.go new file mode 100644 index 000000000..f0b4b929b --- /dev/null +++ b/api/http/handler/stacks/helper.go @@ -0,0 +1,27 @@ +package stacks + +import ( + "errors" + "time" + + "github.com/asaskevich/govalidator" + portainer "github.com/portainer/portainer/api" +) + +func validateStackAutoUpdate(autoUpdate *portainer.StackAutoUpdate) error { + if autoUpdate == nil { + return nil + } + if autoUpdate.Interval == "" && autoUpdate.Webhook == "" { + return errors.New("Both Interval and Webhook fields are empty") + } + if autoUpdate.Webhook != "" && !govalidator.IsUUID(autoUpdate.Webhook) { + return errors.New("Invalid Webhook format") + } + if autoUpdate.Interval != "" { + if _, err := time.ParseDuration(autoUpdate.Interval); err != nil { + return errors.New("Invalid Interval format") + } + } + return nil +} diff --git a/api/http/handler/stacks/helper_test.go b/api/http/handler/stacks/helper_test.go new file mode 100644 index 000000000..29cb4c8f0 --- /dev/null +++ b/api/http/handler/stacks/helper_test.go @@ -0,0 +1,31 @@ +package stacks + +import ( + "testing" + + portainer "github.com/portainer/portainer/api" + "github.com/stretchr/testify/assert" +) + +func Test_ValidateStackAutoUpdate_Valid(t *testing.T) { + mock := &portainer.StackAutoUpdate{ + Webhook: "8dce8c2f-9ca1-482b-ad20-271e86536ada", + Interval: "5h30m40s10ms", + } + if err := validateStackAutoUpdate(mock); err != nil { + t.Errorf("wrong validation, mock data: %v should be valid, but got error: %v", mock, err) + } +} + +func Test_ValidateStackAutoUpdate_InValid(t *testing.T) { + mock := &portainer.StackAutoUpdate{} + err := validateStackAutoUpdate(mock) + assert.Error(t, err) + mock.Webhook = "fake-web-hook" + err = validateStackAutoUpdate(mock) + assert.Error(t, err) + mock.Webhook = "" + mock.Interval = "1dd2hh3mm" + err = validateStackAutoUpdate(mock) + assert.Error(t, err) +} diff --git a/api/http/handler/stacks/stack_update_git.go b/api/http/handler/stacks/stack_update_git.go new file mode 100644 index 000000000..87f4120aa --- /dev/null +++ b/api/http/handler/stacks/stack_update_git.go @@ -0,0 +1,206 @@ +package stacks + +import ( + "errors" + "net/http" + "time" + + "github.com/asaskevich/govalidator" + httperror "github.com/portainer/libhttp/error" + "github.com/portainer/libhttp/request" + "github.com/portainer/libhttp/response" + portainer "github.com/portainer/portainer/api" + bolterrors "github.com/portainer/portainer/api/bolt/errors" + httperrors "github.com/portainer/portainer/api/http/errors" + "github.com/portainer/portainer/api/http/security" + "github.com/portainer/portainer/api/internal/stackutils" +) + +type stackGitUpdatePayload struct { + AutoUpdate *portainer.StackAutoUpdate + Env []portainer.Pair + RepositoryReferenceName string +} + +func (payload *stackGitUpdatePayload) Validate(r *http.Request) error { + if govalidator.IsNull(payload.RepositoryReferenceName) { + return errors.New("Invalid RepositoryReferenceName") + } + if err := validateStackAutoUpdate(payload.AutoUpdate); err != nil { + return err + } + return nil +} + +// @id Stacks +// @summary Update and redeploy an existing stack (with Git config) +// @description Update and redeploy an existing stack (with Git config) +// @description **Access policy**: authenticated +// @tags stacks +// @security jwt +// @produce json +// @param id path int true "Stack identifier" +// @param endpointId query int false "Stacks created before version 1.18.0 might not have an associated endpoint identifier. Use this optional parameter to set the endpoint identifier used by the stack." +// @param body body stackGitUpdatePayload true "Stack Git config" +// @success 200 {object} portainer.Stack "Success" +// @failure 400 "Invalid request" +// @failure 403 "Permission denied" +// @failure 404 "Not found" +// @failure 500 "Server error" +// @router /stacks/{id}/git +func (handler *Handler) stackGitUpdate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { + stackID, err := request.RetrieveNumericRouteVariableValue(r, "id") + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid stack identifier route variable", Err: err} + } + + var payload stackGitUpdatePayload + err = request.DecodeAndValidateJSONPayload(r, &payload) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid request payload", Err: err} + } + + stack, err := handler.DataStore.Stack().Stack(portainer.StackID(stackID)) + if err == bolterrors.ErrObjectNotFound { + return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find a stack with the specified identifier inside the database", Err: err} + } else if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find a stack with the specified identifier inside the database", Err: err} + } else if stack.GitConfig == nil { + msg := "No Git config in the found stack" + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: msg, Err: errors.New(msg)} + } + + // TODO: this is a work-around for stacks created with Portainer version >= 1.17.1 + // The EndpointID property is not available for these stacks, this API endpoint + // can use the optional EndpointID query parameter to associate a valid endpoint identifier to the stack. + endpointID, err := request.RetrieveNumericQueryParameter(r, "endpointId", true) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid query parameter: endpointId", Err: err} + } + if endpointID != int(stack.EndpointID) { + stack.EndpointID = portainer.EndpointID(endpointID) + } + + endpoint, err := handler.DataStore.Endpoint().Endpoint(stack.EndpointID) + if err == bolterrors.ErrObjectNotFound { + return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find the endpoint associated to the stack inside the database", Err: err} + } else if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find the endpoint associated to the stack inside the database", Err: err} + } + + err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Permission denied to access endpoint", Err: err} + } + + resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve a resource control associated to the stack", Err: err} + } + + securityContext, err := security.RetrieveRestrictedRequestContext(r) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve info from request context", Err: err} + } + + access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to verify user authorizations to validate stack access", Err: err} + } + if !access { + return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Access denied to resource", Err: httperrors.ErrResourceAccessDenied} + } + + //update retrieved stack data based on payload + stack.GitConfig.ReferenceName = payload.RepositoryReferenceName + stack.AutoUpdate = payload.AutoUpdate + stack.Env = payload.Env + + //compose git clone params based on stack info + gitCloneParams := &cloneRepositoryParameters{ + url: stack.GitConfig.URL, + referenceName: stack.GitConfig.ReferenceName, + path: stack.ProjectPath, + } + if stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" { + gitCloneParams.authentication = true + gitCloneParams.username = stack.GitConfig.Authentication.Username + gitCloneParams.password = stack.GitConfig.Authentication.Password + } + + //distribute requests based on stack types + //use composed gitCloneParams + var updateError *httperror.HandlerError + switch stack.Type { + case portainer.DockerComposeStack: + updateError = handler.updateComposeStackFromGit(r, stack, endpoint, gitCloneParams) + case portainer.DockerSwarmStack: + updateError = handler.updateSwarmStackFromGit(r, stack, endpoint, gitCloneParams) + default: + msg := "Unsupported stack type" + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: msg, Err: errors.New(msg)} + } + if updateError != nil { + return updateError + } + + //save the updated stack to DB + err = handler.DataStore.Stack().UpdateStack(stack.ID, stack) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist the stack changes inside the database", Err: err} + } + + return response.JSON(w, stack) +} + +func (handler *Handler) updateComposeStackFromGit(r *http.Request, stack *portainer.Stack, endpoint *portainer.Endpoint, gitCloneParams *cloneRepositoryParameters) *httperror.HandlerError { + doCleanUp := true + defer handler.cleanUp(stack, &doCleanUp) + + err := handler.cloneGitRepository(gitCloneParams) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to clone git repository", Err: err} + } + + config, configErr := handler.createComposeDeployConfig(r, stack, endpoint) + if configErr != nil { + return configErr + } + + err = handler.deployComposeStack(config) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: err.Error(), Err: err} + } + + stack.UpdateDate = time.Now().Unix() + stack.UpdatedBy = config.user.Username + + doCleanUp = false + return nil +} + +func (handler *Handler) updateSwarmStackFromGit(r *http.Request, stack *portainer.Stack, endpoint *portainer.Endpoint, gitCloneParams *cloneRepositoryParameters) *httperror.HandlerError { + doCleanUp := true + defer handler.cleanUp(stack, &doCleanUp) + + err := handler.cloneGitRepository(gitCloneParams) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to clone git repository", Err: err} + } + + config, configErr := handler.createSwarmDeployConfig(r, stack, endpoint, false) + if configErr != nil { + return configErr + } + + err = handler.deploySwarmStack(config) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: err.Error(), Err: err} + } + + stack.UpdateDate = time.Now().Unix() + stack.UpdatedBy = config.user.Username + + doCleanUp = false + return nil +} diff --git a/api/portainer.go b/api/portainer.go index 282a8b5f4..14adaf754 100644 --- a/api/portainer.go +++ b/api/portainer.go @@ -681,6 +681,12 @@ type ( SwarmID string `json:"SwarmId" example:"jpofkc0i9uo9wtx1zesuk649w"` // Path to the Stack file EntryPoint string `json:"EntryPoint" example:"docker-compose.yml"` + // Additional Stack files + AdditionalFiles []string `json:"AdditionalFiles" example:""` + // Stack auto update settings + AutoUpdate *StackAutoUpdate `json:"AutoUpdate" example:""` + // Git settings + GitConfig *GitConfig `json:"GitConfig" example:""` // A list of environment variables used during stack deployment Env []Pair `json:"Env" example:""` // @@ -699,6 +705,26 @@ type ( UpdatedBy string `example:"bob"` } + // GitConfig represents the Git settings for a stack + GitConfig struct { + URL string + ReferenceName string + Authentication *GitAuthentication + ConfigHash []byte + } + + // GitAuthentication represents the credentials for accessing privete Git repos + GitAuthentication struct { + Username string + Password string + } + + // StackAutoUpdate represents the git auto sync config for stack deployment + StackAutoUpdate struct { + Interval string + Webhook string + } + // StackID represents a stack identifier (it must be composed of Name + "_" + SwarmID to create a unique identifier) StackID int