diff --git a/api/filesystem/filesystem.go b/api/filesystem/filesystem.go index 62dac019d..85c519844 100644 --- a/api/filesystem/filesystem.go +++ b/api/filesystem/filesystem.go @@ -279,13 +279,7 @@ func (service *Service) WriteJSONToFile(path string, content interface{}) error // FileExists checks for the existence of the specified file. func (service *Service) FileExists(filePath string) (bool, error) { - if _, err := os.Stat(filePath); err != nil { - if os.IsNotExist(err) { - return false, nil - } - return false, err - } - return true, nil + return FileExists(filePath) } // KeyPairFilesExist checks for the existence of the key files. @@ -510,3 +504,14 @@ func (service *Service) GetTemporaryPath() (string, error) { func (service *Service) GetDatastorePath() string { return service.dataStorePath } + +// FileExists checks for the existence of the specified file. +func FileExists(filePath string) (bool, error) { + if _, err := os.Stat(filePath); err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + return true, nil +} diff --git a/api/filesystem/filesystem_test.go b/api/filesystem/filesystem_test.go new file mode 100644 index 000000000..9688dacc8 --- /dev/null +++ b/api/filesystem/filesystem_test.go @@ -0,0 +1,68 @@ +package filesystem + +import ( + "fmt" + "math/rand" + "os" + "path" + "testing" + + "github.com/stretchr/testify/assert" +) + +func createService(t *testing.T) *Service { + dataStorePath := path.Join(os.TempDir(), t.Name()) + + service, err := NewService(dataStorePath, "") + assert.NoError(t, err, "NewService should not fail") + + t.Cleanup(func() { + os.RemoveAll(dataStorePath) + }) + + return service +} + +func Test_fileSystemService_FileExists_whenFileExistsShouldReturnTrue(t *testing.T) { + service := createService(t) + testHelperFileExists_fileExists(t, service.FileExists) +} + +func Test_fileSystemService_FileExists_whenFileNotExistsShouldReturnFalse(t *testing.T) { + service := createService(t) + testHelperFileExists_fileNotExists(t, service.FileExists) +} + +func Test_FileExists_whenFileExistsShouldReturnTrue(t *testing.T) { + testHelperFileExists_fileExists(t, FileExists) +} + +func Test_FileExists_whenFileNotExistsShouldReturnFalse(t *testing.T) { + testHelperFileExists_fileNotExists(t, FileExists) +} + +func testHelperFileExists_fileExists(t *testing.T, checker func(path string) (bool, error)) { + file, err := os.CreateTemp("", t.Name()) + assert.NoError(t, err, "CreateTemp should not fail") + + t.Cleanup(func() { + os.RemoveAll(file.Name()) + }) + + exists, err := checker(file.Name()) + assert.NoError(t, err, "FileExists should not fail") + + assert.True(t, exists) +} + +func testHelperFileExists_fileNotExists(t *testing.T, checker func(path string) (bool, error)) { + filePath := path.Join(os.TempDir(), fmt.Sprintf("%s%d", t.Name(), rand.Int())) + + err := os.RemoveAll(filePath) + assert.NoError(t, err, "RemoveAll should not fail") + + exists, err := checker(filePath) + assert.NoError(t, err, "FileExists should not fail") + + assert.False(t, exists) +} diff --git a/api/git/azure_integration_test.go b/api/git/azure_integration_test.go index 85ee8f707..9f3a60c2a 100644 --- a/api/git/azure_integration_test.go +++ b/api/git/azure_integration_test.go @@ -2,12 +2,13 @@ package git import ( "fmt" - "github.com/docker/docker/pkg/ioutils" - _ "github.com/joho/godotenv/autoload" - "github.com/stretchr/testify/assert" "os" "path/filepath" "testing" + + "github.com/docker/docker/pkg/ioutils" + _ "github.com/joho/godotenv/autoload" + "github.com/stretchr/testify/assert" ) func TestService_ClonePublicRepository_Azure(t *testing.T) { @@ -54,7 +55,7 @@ func TestService_ClonePublicRepository_Azure(t *testing.T) { assert.NoError(t, err) defer os.RemoveAll(dst) repositoryUrl := fmt.Sprintf(tt.args.repositoryURLFormat, tt.args.password) - err = service.ClonePublicRepository(repositoryUrl, tt.args.referenceName, dst) + err = service.CloneRepository(repositoryUrl, tt.args.referenceName, dst, "", "") assert.NoError(t, err) assert.FileExists(t, filepath.Join(dst, "README.md")) }) @@ -72,7 +73,7 @@ func TestService_ClonePrivateRepository_Azure(t *testing.T) { defer os.RemoveAll(dst) repositoryUrl := "https://portainer.visualstudio.com/Playground/_git/dev_integration" - err = service.ClonePrivateRepositoryWithBasicAuth(repositoryUrl, "refs/heads/main", dst, "", pat) + err = service.CloneRepository(repositoryUrl, "refs/heads/main", dst, "", pat) assert.NoError(t, err) assert.FileExists(t, filepath.Join(dst, "README.md")) } diff --git a/api/git/git.go b/api/git/git.go index 51eeac898..9415cc2d0 100644 --- a/api/git/git.go +++ b/api/git/git.go @@ -3,12 +3,13 @@ package git import ( "context" "crypto/tls" - "github.com/pkg/errors" "net/http" "os" "path/filepath" "time" + "github.com/pkg/errors" + "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/transport/client" @@ -27,7 +28,7 @@ type downloader interface { download(ctx context.Context, dst string, opt cloneOptions) error } -type gitClient struct{ +type gitClient struct { preserveGitDirectory bool } @@ -86,26 +87,18 @@ func NewService() *Service { } } -// ClonePublicRepository clones a public git repository using the specified URL in the specified +// CloneRepository clones a git repository using the specified URL in the specified // destination folder. -func (service *Service) ClonePublicRepository(repositoryURL, referenceName, destination string) error { - return service.cloneRepository(destination, cloneOptions{ - repositoryUrl: repositoryURL, - referenceName: referenceName, - depth: 1, - }) -} - -// ClonePrivateRepositoryWithBasicAuth clones a private git repository using the specified URL in the specified -// destination folder. It will use the specified Username and Password for basic HTTP authentication. -func (service *Service) ClonePrivateRepositoryWithBasicAuth(repositoryURL, referenceName, destination, username, password string) error { - return service.cloneRepository(destination, cloneOptions{ +func (service *Service) CloneRepository(repositoryURL, referenceName, destination, username, password string) error { + options := cloneOptions{ repositoryUrl: repositoryURL, username: username, password: password, referenceName: referenceName, depth: 1, - }) + } + + return service.cloneRepository(destination, options) } func (service *Service) cloneRepository(destination string, options cloneOptions) error { diff --git a/api/git/git_integration_test.go b/api/git/git_integration_test.go index 0249a629a..5c9103690 100644 --- a/api/git/git_integration_test.go +++ b/api/git/git_integration_test.go @@ -1,11 +1,12 @@ package git import ( - "github.com/docker/docker/pkg/ioutils" - "github.com/stretchr/testify/assert" "os" "path/filepath" "testing" + + "github.com/docker/docker/pkg/ioutils" + "github.com/stretchr/testify/assert" ) func TestService_ClonePrivateRepository_GitHub(t *testing.T) { @@ -20,7 +21,7 @@ func TestService_ClonePrivateRepository_GitHub(t *testing.T) { defer os.RemoveAll(dst) repositoryUrl := "https://github.com/portainer/private-test-repository.git" - err = service.ClonePrivateRepositoryWithBasicAuth(repositoryUrl, "refs/heads/main", dst, username, pat) + err = service.CloneRepository(repositoryUrl, "refs/heads/main", dst, username, pat) assert.NoError(t, err) assert.FileExists(t, filepath.Join(dst, "README.md")) } diff --git a/api/git/git_test.go b/api/git/git_test.go index 8276a0925..8ffba3364 100644 --- a/api/git/git_test.go +++ b/api/git/git_test.go @@ -2,16 +2,17 @@ package git import ( "context" - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing/object" - "github.com/pkg/errors" - "github.com/portainer/portainer/api/archive" - "github.com/stretchr/testify/assert" "io/ioutil" "log" "os" "path/filepath" "testing" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/pkg/errors" + "github.com/portainer/portainer/api/archive" + "github.com/stretchr/testify/assert" ) var bareRepoDir string @@ -59,7 +60,7 @@ func Test_ClonePublicRepository_Shallow(t *testing.T) { } defer os.RemoveAll(dir) t.Logf("Cloning into %s", dir) - err = service.ClonePublicRepository(repositoryURL, referenceName, dir) + err = service.CloneRepository(repositoryURL, referenceName, dir, "", "") assert.NoError(t, err) assert.Equal(t, 1, getCommitHistoryLength(t, err, dir), "cloned repo has incorrect depth") } @@ -74,9 +75,11 @@ func Test_ClonePublicRepository_NoGitDirectory(t *testing.T) { if err != nil { t.Fatalf("failed to create a temp dir") } + defer os.RemoveAll(dir) + t.Logf("Cloning into %s", dir) - err = service.ClonePublicRepository(repositoryURL, referenceName, dir) + err = service.CloneRepository(repositoryURL, referenceName, dir, "", "") assert.NoError(t, err) assert.NoDirExists(t, filepath.Join(dir, ".git")) } diff --git a/api/git/types/types.go b/api/git/types/types.go new file mode 100644 index 000000000..2a91f61b6 --- /dev/null +++ b/api/git/types/types.go @@ -0,0 +1,7 @@ +package gittypes + +type RepoConfig struct { + URL string + ReferenceName string + ConfigFilePath string +} diff --git a/api/http/handler/customtemplates/customtemplate_create.go b/api/http/handler/customtemplates/customtemplate_create.go index b8768b74c..9433affa2 100644 --- a/api/http/handler/customtemplates/customtemplate_create.go +++ b/api/http/handler/customtemplates/customtemplate_create.go @@ -236,16 +236,7 @@ func (handler *Handler) createCustomTemplateFromGitRepository(r *http.Request) ( projectPath := handler.FileService.GetCustomTemplateProjectPath(strconv.Itoa(customTemplateID)) customTemplate.ProjectPath = projectPath - gitCloneParams := &cloneRepositoryParameters{ - url: payload.RepositoryURL, - referenceName: payload.RepositoryReferenceName, - path: projectPath, - authentication: payload.RepositoryAuthentication, - username: payload.RepositoryUsername, - password: payload.RepositoryPassword, - } - - err = handler.cloneGitRepository(gitCloneParams) + err = handler.GitService.CloneRepository(projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, payload.RepositoryUsername, payload.RepositoryPassword) if err != nil { return nil, err } diff --git a/api/http/handler/customtemplates/git.go b/api/http/handler/customtemplates/git.go deleted file mode 100644 index b4f3e3211..000000000 --- a/api/http/handler/customtemplates/git.go +++ /dev/null @@ -1,17 +0,0 @@ -package customtemplates - -type cloneRepositoryParameters struct { - url string - referenceName string - path string - authentication bool - username string - password string -} - -func (handler *Handler) cloneGitRepository(parameters *cloneRepositoryParameters) error { - if parameters.authentication { - return handler.GitService.ClonePrivateRepositoryWithBasicAuth(parameters.url, parameters.referenceName, parameters.path, parameters.username, parameters.password) - } - return handler.GitService.ClonePublicRepository(parameters.url, parameters.referenceName, parameters.path) -} diff --git a/api/http/handler/edgestacks/edgestack_create.go b/api/http/handler/edgestacks/edgestack_create.go index 0e5a74d7c..b0b904faa 100644 --- a/api/http/handler/edgestacks/edgestack_create.go +++ b/api/http/handler/edgestacks/edgestack_create.go @@ -212,16 +212,7 @@ func (handler *Handler) createSwarmStackFromGitRepository(r *http.Request) (*por projectPath := handler.FileService.GetEdgeStackProjectPath(strconv.Itoa(int(stack.ID))) stack.ProjectPath = projectPath - gitCloneParams := &cloneRepositoryParameters{ - url: payload.RepositoryURL, - referenceName: payload.RepositoryReferenceName, - path: projectPath, - authentication: payload.RepositoryAuthentication, - username: payload.RepositoryUsername, - password: payload.RepositoryPassword, - } - - err = handler.cloneGitRepository(gitCloneParams) + err = handler.GitService.CloneRepository(projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, payload.RepositoryUsername, payload.RepositoryPassword) if err != nil { return nil, err } diff --git a/api/http/handler/edgestacks/git.go b/api/http/handler/edgestacks/git.go deleted file mode 100644 index 855fa72bc..000000000 --- a/api/http/handler/edgestacks/git.go +++ /dev/null @@ -1,17 +0,0 @@ -package edgestacks - -type cloneRepositoryParameters struct { - url string - referenceName string - path string - authentication bool - username string - password string -} - -func (handler *Handler) cloneGitRepository(parameters *cloneRepositoryParameters) error { - if parameters.authentication { - return handler.GitService.ClonePrivateRepositoryWithBasicAuth(parameters.url, parameters.referenceName, parameters.path, parameters.username, parameters.password) - } - return handler.GitService.ClonePublicRepository(parameters.url, parameters.referenceName, parameters.path) -} diff --git a/api/http/handler/stacks/create_compose_stack.go b/api/http/handler/stacks/create_compose_stack.go index 2c78ee0f9..4f8e38c50 100644 --- a/api/http/handler/stacks/create_compose_stack.go +++ b/api/http/handler/stacks/create_compose_stack.go @@ -169,19 +169,10 @@ func (handler *Handler) createComposeStackFromGitRepository(w http.ResponseWrite projectPath := handler.FileService.GetStackProjectPath(strconv.Itoa(int(stack.ID))) stack.ProjectPath = projectPath - gitCloneParams := &cloneRepositoryParameters{ - url: payload.RepositoryURL, - referenceName: payload.RepositoryReferenceName, - path: projectPath, - authentication: payload.RepositoryAuthentication, - username: payload.RepositoryUsername, - password: payload.RepositoryPassword, - } - doCleanUp := true defer handler.cleanUp(stack, &doCleanUp) - err = handler.cloneGitRepository(gitCloneParams) + err = handler.cloneAndSaveConfig(stack, projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, payload.ComposeFilePathInRepository, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword) if err != nil { return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to clone git repository", Err: err} } diff --git a/api/http/handler/stacks/create_swarm_stack.go b/api/http/handler/stacks/create_swarm_stack.go index e34df227e..747f1b456 100644 --- a/api/http/handler/stacks/create_swarm_stack.go +++ b/api/http/handler/stacks/create_swarm_stack.go @@ -173,21 +173,12 @@ func (handler *Handler) createSwarmStackFromGitRepository(w http.ResponseWriter, projectPath := handler.FileService.GetStackProjectPath(strconv.Itoa(int(stack.ID))) stack.ProjectPath = projectPath - gitCloneParams := &cloneRepositoryParameters{ - url: payload.RepositoryURL, - referenceName: payload.RepositoryReferenceName, - path: projectPath, - authentication: payload.RepositoryAuthentication, - username: payload.RepositoryUsername, - password: payload.RepositoryPassword, - } - doCleanUp := true defer handler.cleanUp(stack, &doCleanUp) - err = handler.cloneGitRepository(gitCloneParams) + err = handler.cloneAndSaveConfig(stack, projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, payload.ComposeFilePathInRepository, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to clone git repository", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to clone git repository", Err: err} } config, configErr := handler.createSwarmDeployConfig(r, stack, endpoint, false) diff --git a/api/http/handler/stacks/git.go b/api/http/handler/stacks/git.go deleted file mode 100644 index b3e2448f7..000000000 --- a/api/http/handler/stacks/git.go +++ /dev/null @@ -1,17 +0,0 @@ -package stacks - -type cloneRepositoryParameters struct { - url string - referenceName string - path string - authentication bool - username string - password string -} - -func (handler *Handler) cloneGitRepository(parameters *cloneRepositoryParameters) error { - if parameters.authentication { - return handler.GitService.ClonePrivateRepositoryWithBasicAuth(parameters.url, parameters.referenceName, parameters.path, parameters.username, parameters.password) - } - return handler.GitService.ClonePublicRepository(parameters.url, parameters.referenceName, parameters.path) -} diff --git a/api/http/handler/stacks/stack_create.go b/api/http/handler/stacks/stack_create.go index 7da0fd386..eae955cf5 100644 --- a/api/http/handler/stacks/stack_create.go +++ b/api/http/handler/stacks/stack_create.go @@ -2,6 +2,7 @@ package stacks import ( "errors" + "fmt" "log" "net/http" @@ -12,6 +13,7 @@ import ( "github.com/portainer/libhttp/response" portainer "github.com/portainer/portainer/api" bolterrors "github.com/portainer/portainer/api/bolt/errors" + gittypes "github.com/portainer/portainer/api/git/types" httperrors "github.com/portainer/portainer/api/http/errors" "github.com/portainer/portainer/api/http/security" "github.com/portainer/portainer/api/internal/authorization" @@ -226,3 +228,18 @@ func (handler *Handler) decorateStackResponse(w http.ResponseWriter, stack *port stack.ResourceControl = resourceControl return response.JSON(w, stack) } + +func (handler *Handler) cloneAndSaveConfig(stack *portainer.Stack, projectPath, repositoryURL, refName, configFilePath string, auth bool, username, password string) error { + + err := handler.GitService.CloneRepository(projectPath, repositoryURL, refName, username, password) + if err != nil { + return fmt.Errorf("unable to clone git repository: %w", err) + } + + stack.GitConfig = &gittypes.RepoConfig{ + URL: repositoryURL, + ReferenceName: refName, + ConfigFilePath: configFilePath, + } + return nil +} diff --git a/api/http/handler/stacks/stack_create_test.go b/api/http/handler/stacks/stack_create_test.go new file mode 100644 index 000000000..414948378 --- /dev/null +++ b/api/http/handler/stacks/stack_create_test.go @@ -0,0 +1,29 @@ +package stacks + +import ( + "testing" + + portainer "github.com/portainer/portainer/api" + gittypes "github.com/portainer/portainer/api/git/types" + "github.com/portainer/portainer/api/http/security" + "github.com/portainer/portainer/api/internal/testhelpers" + "github.com/stretchr/testify/assert" +) + +func Test_stackHandler_cloneAndSaveConfig_shouldCallGitCloneAndSaveConfigOnStack(t *testing.T) { + handler := NewHandler(&security.RequestBouncer{}) + handler.GitService = testhelpers.NewGitService() + + url := "url" + refName := "ref" + configPath := "path" + stack := &portainer.Stack{} + err := handler.cloneAndSaveConfig(stack, "", url, refName, configPath, false, "", "") + assert.NoError(t, err, "clone and save should not fail") + + assert.Equal(t, gittypes.RepoConfig{ + URL: url, + ReferenceName: refName, + ConfigFilePath: configPath, + }, *stack.GitConfig) +} diff --git a/api/http/handler/templates/git.go b/api/http/handler/templates/git.go deleted file mode 100644 index cc94668cd..000000000 --- a/api/http/handler/templates/git.go +++ /dev/null @@ -1,17 +0,0 @@ -package templates - -type cloneRepositoryParameters struct { - url string - referenceName string - path string - authentication bool - username string - password string -} - -func (handler *Handler) cloneGitRepository(parameters *cloneRepositoryParameters) error { - if parameters.authentication { - return handler.GitService.ClonePrivateRepositoryWithBasicAuth(parameters.url, parameters.referenceName, parameters.path, parameters.username, parameters.password) - } - return handler.GitService.ClonePublicRepository(parameters.url, parameters.referenceName, parameters.path) -} diff --git a/api/http/handler/templates/template_file.go b/api/http/handler/templates/template_file.go index 359471ce6..12f6b5b70 100644 --- a/api/http/handler/templates/template_file.go +++ b/api/http/handler/templates/template_file.go @@ -63,12 +63,7 @@ func (handler *Handler) templateFile(w http.ResponseWriter, r *http.Request) *ht defer handler.cleanUp(projectPath) - gitCloneParams := &cloneRepositoryParameters{ - url: payload.RepositoryURL, - path: projectPath, - } - - err = handler.cloneGitRepository(gitCloneParams) + err = handler.GitService.CloneRepository(projectPath, payload.RepositoryURL, "", "", "") if err != nil { return &httperror.HandlerError{http.StatusInternalServerError, "Unable to clone git repository", err} } diff --git a/api/internal/testhelpers/git_service.go b/api/internal/testhelpers/git_service.go new file mode 100644 index 000000000..a51c431c6 --- /dev/null +++ b/api/internal/testhelpers/git_service.go @@ -0,0 +1,12 @@ +package testhelpers + +type gitService struct{} + +// NewGitService creates new mock for portainer.GitService. +func NewGitService() *gitService { + return &gitService{} +} + +func (service *gitService) CloneRepository(destination string, repositoryURL, referenceName string, auth bool, username, password string) error { + return nil +} diff --git a/api/portainer.go b/api/portainer.go index 2292aab1e..3ce533ac2 100644 --- a/api/portainer.go +++ b/api/portainer.go @@ -3,6 +3,8 @@ package portainer import ( "io" "time" + + gittypes "github.com/portainer/portainer/api/git/types" ) type ( @@ -697,6 +699,8 @@ type ( UpdateDate int64 `example:"1587399600"` // The username which last updated this stack UpdatedBy string `example:"bob"` + // The git config of this stack + GitConfig *gittypes.RepoConfig } // StackID represents a stack identifier (it must be composed of Name + "_" + SwarmID to create a unique identifier) @@ -1138,8 +1142,7 @@ type ( // GitService represents a service for managing Git GitService interface { - ClonePublicRepository(repositoryURL, referenceName string, destination string) error - ClonePrivateRepositoryWithBasicAuth(repositoryURL, referenceName string, destination, username, password string) error + CloneRepository(destination string, repositoryURL, referenceName, username, password string) error } // JWTService represents a service for managing JWT tokens