add newline at end of file

.github/ISSUE_TEMPLATE/Bug_report.md

@@ -30,7 +30,7 @@ A clear and concise description of what you expected to happen.
 
 **Portainer Logs**
 Provide the logs of your Portainer container or Service.
-You can see how [here](https://documentation.portainer.io/r/portainer-logs)
+You can see how [here](https://documentation.portainer.io/archive/1.23.2/faq/#how-do-i-get-the-logs-from-portainer)
 
 **Steps to reproduce the issue:**
 

api/bolt/backup.go

@@ -1,142 +0,0 @@
-package bolt
-
-import (
-	"fmt"
-	"os"
-	"path"
-	"time"
-
-	plog "github.com/portainer/portainer/api/bolt/log"
-)
-
-var backupDefaults = struct {
-	backupDir        string
-	commonDir        string
-	databaseFileName string
-}{
-	"backups",
-	"common",
-	databaseFileName,
-}
-
-var backupLog = plog.NewScopedLog("bolt, backup")
-
-//
-// Backup Helpers
-//
-
-// createBackupFolders create initial folders for backups
-func (store *Store) createBackupFolders() {
-	// create common dir
-	commonDir := store.commonBackupDir()
-	if exists, _ := store.fileService.FileExists(commonDir); !exists {
-		if err := os.MkdirAll(commonDir, 0700); err != nil {
-			backupLog.Error("Error while creating common backup folder", err)
-		}
-	}
-}
-
-func (store *Store) databasePath() string {
-	return path.Join(store.path, databaseFileName)
-}
-
-func (store *Store) commonBackupDir() string {
-	return path.Join(store.path, backupDefaults.backupDir, backupDefaults.commonDir)
-}
-
-func (store *Store) copyDBFile(from string, to string) error {
-	backupLog.Info(fmt.Sprintf("Copying db file from %s to %s", from, to))
-	err := store.fileService.Copy(from, to, true)
-	if err != nil {
-		backupLog.Error("Failed", err)
-	}
-	return err
-}
-
-// BackupOptions provide a helper to inject backup options
-type BackupOptions struct {
-	Version        int
-	BackupDir      string
-	BackupFileName string
-	BackupPath     string
-}
-
-func (store *Store) setupOptions(options *BackupOptions) *BackupOptions {
-	if options == nil {
-		options = &BackupOptions{}
-	}
-	if options.Version == 0 {
-		options.Version, _ = store.version()
-	}
-	if options.BackupDir == "" {
-		options.BackupDir = store.commonBackupDir()
-	}
-	if options.BackupFileName == "" {
-		options.BackupFileName = fmt.Sprintf("%s.%s.%s", backupDefaults.databaseFileName, fmt.Sprintf("%03d", options.Version), time.Now().Format("20060102150405"))
-	}
-	if options.BackupPath == "" {
-		options.BackupPath = path.Join(options.BackupDir, options.BackupFileName)
-	}
-	return options
-}
-
-// BackupWithOptions backup current database with options
-func (store *Store) BackupWithOptions(options *BackupOptions) (string, error) {
-	backupLog.Info("creating db backup")
-	store.createBackupFolders()
-
-	options = store.setupOptions(options)
-
-	return options.BackupPath, store.copyDBFile(store.databasePath(), options.BackupPath)
-}
-
-// RestoreWithOptions previously saved backup for the current Edition  with options
-// Restore strategies:
-// - default: restore latest from current edition
-// - restore a specific
-func (store *Store) RestoreWithOptions(options *BackupOptions) error {
-	options = store.setupOptions(options)
-
-	// Check if backup file exist before restoring
-	_, err := os.Stat(options.BackupPath)
-	if os.IsNotExist(err) {
-		backupLog.Error(fmt.Sprintf("Backup file to restore does not exist %s", options.BackupPath), err)
-		return err
-	}
-
-	err = store.Close()
-	if err != nil {
-		backupLog.Error("Error while closing store before restore", err)
-		return err
-	}
-
-	backupLog.Info("Restoring db backup")
-	err = store.copyDBFile(options.BackupPath, store.databasePath())
-	if err != nil {
-		return err
-	}
-
-	return store.Open()
-}
-
-// RemoveWithOptions removes backup database based on supplied options
-func (store *Store) RemoveWithOptions(options *BackupOptions) error {
-	backupLog.Info("Removing db backup")
-
-	options = store.setupOptions(options)
-	_, err := os.Stat(options.BackupPath)
-
-	if os.IsNotExist(err) {
-		backupLog.Error(fmt.Sprintf("Backup file to remove does not exist %s", options.BackupPath), err)
-		return err
-	}
-
-	backupLog.Info(fmt.Sprintf("Removing db file at %s", options.BackupPath))
-	err = os.Remove(options.BackupPath)
-	if err != nil {
-		backupLog.Error("Failed", err)
-		return err
-	}
-
-	return nil
-}

api/bolt/backup_test.go

@@ -1,116 +0,0 @@
-package bolt
-
-import (
-	"fmt"
-	"os"
-	"path"
-	"path/filepath"
-	"testing"
-
-	portainer "github.com/portainer/portainer/api"
-)
-
-// isFileExist is helper function to check for file existence
-func isFileExist(path string) bool {
-	matches, err := filepath.Glob(path)
-	if err != nil {
-		return false
-	}
-	return len(matches) > 0
-}
-
-func TestCreateBackupFolders(t *testing.T) {
-	store, teardown := MustNewTestStore(false)
-	defer teardown()
-
-	backupPath := path.Join(store.path, backupDefaults.backupDir)
-
-	if isFileExist(backupPath) {
-		t.Error("Expect backups folder to not exist")
-	}
-
-	store.createBackupFolders()
-	if !isFileExist(backupPath) {
-		t.Error("Expect backups folder to exist")
-	}
-}
-
-func TestStoreCreation(t *testing.T) {
-	store, teardown := MustNewTestStore(true)
-	defer teardown()
-
-	if store == nil {
-		t.Error("Expect to create a store")
-	}
-
-	if store.edition() != portainer.PortainerCE {
-		t.Error("Expect to get CE Edition")
-	}
-}
-
-func TestBackup(t *testing.T) {
-	store, teardown := MustNewTestStore(true)
-	defer teardown()
-
-	t.Run("Backup should create default db backup", func(t *testing.T) {
-		store.VersionService.StoreDBVersion(portainer.DBVersion)
-		store.BackupWithOptions(nil)
-
-		backupFileName := path.Join(store.path, "backups", "common", fmt.Sprintf("portainer.db.%03d.*", portainer.DBVersion))
-		if !isFileExist(backupFileName) {
-			t.Errorf("Expect backup file to be created %s", backupFileName)
-		}
-	})
-
-	t.Run("BackupWithOption should create a name specific backup at common path", func(t *testing.T) {
-		store.BackupWithOptions(&BackupOptions{
-			BackupFileName: beforePortainerVersionUpgradeBackup,
-			BackupDir:      store.commonBackupDir(),
-		})
-		backupFileName := path.Join(store.path, "backups", "common", beforePortainerVersionUpgradeBackup)
-		if !isFileExist(backupFileName) {
-			t.Errorf("Expect backup file to be created %s", backupFileName)
-		}
-	})
-}
-
-func TestRemoveWithOptions(t *testing.T) {
-	store, teardown := MustNewTestStore(true)
-	defer teardown()
-
-	t.Run("successfully removes file if existent", func(t *testing.T) {
-		store.createBackupFolders()
-		options := &BackupOptions{
-			BackupDir:      store.commonBackupDir(),
-			BackupFileName: "test.txt",
-		}
-
-		filePath := path.Join(options.BackupDir, options.BackupFileName)
-		f, err := os.Create(filePath)
-		if err != nil {
-			t.Fatalf("file should be created; err=%s", err)
-		}
-		f.Close()
-
-		err = store.RemoveWithOptions(options)
-		if err != nil {
-			t.Errorf("RemoveWithOptions should successfully remove file; err=%w", err)
-		}
-
-		if isFileExist(f.Name()) {
-			t.Errorf("RemoveWithOptions should successfully remove file; file=%s", f.Name())
-		}
-	})
-
-	t.Run("fails to removes file if non-existent", func(t *testing.T) {
-		options := &BackupOptions{
-			BackupDir:      store.commonBackupDir(),
-			BackupFileName: "test.txt",
-		}
-
-		err := store.RemoveWithOptions(options)
-		if err == nil {
-			t.Error("RemoveWithOptions should fail for non-existent file")
-		}
-	})
-}

api/bolt/bolttest/datastore.go

@@ -1,4 +1,4 @@
-package bolt
+package bolttest
 
 import (
 	"io/ioutil"
@@ -6,12 +6,13 @@ import (
 	"os"
 
 	"github.com/pkg/errors"
+	"github.com/portainer/portainer/api/bolt"
 	"github.com/portainer/portainer/api/filesystem"
 )
 
 var errTempDir = errors.New("can't create a temp dir")
 
-func MustNewTestStore(init bool) (*Store, func()) {
+func MustNewTestStore(init bool) (*bolt.Store, func()) {
 	store, teardown, err := NewTestStore(init)
 	if err != nil {
 		if !errors.Is(err, errTempDir) {
@@ -23,7 +24,7 @@ func MustNewTestStore(init bool) (*Store, func()) {
 	return store, teardown
 }
 
-func NewTestStore(init bool) (*Store, func(), error) {
+func NewTestStore(init bool) (*bolt.Store, func(), error) {
 	// Creates unique temp directory in a concurrency friendly manner.
 	dataStorePath, err := ioutil.TempDir("", "boltdb")
 	if err != nil {
@@ -35,7 +36,11 @@ func NewTestStore(init bool) (*Store, func(), error) {
 		return nil, nil, err
 	}
 
-	store := NewStore(dataStorePath, fileService)
+	store, err := bolt.NewStore(dataStorePath, fileService)
+	if err != nil {
+		return nil, nil, err
+	}
+
 	err = store.Open()
 	if err != nil {
 		return nil, nil, err
@@ -55,7 +60,7 @@ func NewTestStore(init bool) (*Store, func(), error) {
 	return store, teardown, nil
 }
 
-func teardown(store *Store, dataStorePath string) {
+func teardown(store *bolt.Store, dataStorePath string) {
 	err := store.Close()
 	if err != nil {
 		log.Fatalln(err)

api/bolt/datastore.go

@@ -2,6 +2,7 @@ package bolt
 
 import (
 	"io"
+	"log"
 	"path"
 	"time"
 
@@ -20,6 +21,7 @@ import (
 	"github.com/portainer/portainer/api/bolt/errors"
 	"github.com/portainer/portainer/api/bolt/extension"
 	"github.com/portainer/portainer/api/bolt/internal"
+	"github.com/portainer/portainer/api/bolt/migrator"
 	"github.com/portainer/portainer/api/bolt/registry"
 	"github.com/portainer/portainer/api/bolt/resourcecontrol"
 	"github.com/portainer/portainer/api/bolt/role"
@@ -34,6 +36,7 @@ import (
 	"github.com/portainer/portainer/api/bolt/user"
 	"github.com/portainer/portainer/api/bolt/version"
 	"github.com/portainer/portainer/api/bolt/webhook"
+	"github.com/portainer/portainer/api/internal/authorization"
 )
 
 const (
@@ -73,14 +76,6 @@ type Store struct {
 	WebhookService            *webhook.Service
 }
 
-func (store *Store) version() (int, error) {
-	version, err := store.VersionService.DBVersion()
-	if err == errors.ErrObjectNotFound {
-		version = 0
-	}
-	return version, err
-}
-
 func (store *Store) edition() portainer.SoftwareEdition {
 	edition, err := store.VersionService.Edition()
 	if err == errors.ErrObjectNotFound {
@@ -90,13 +85,25 @@ func (store *Store) edition() portainer.SoftwareEdition {
 }
 
 // NewStore initializes a new Store and the associated services
-func NewStore(storePath string, fileService portainer.FileService) *Store {
-	return &Store{
+func NewStore(storePath string, fileService portainer.FileService) (*Store, error) {
+	store := &Store{
 		path:        storePath,
 		fileService: fileService,
 		isNew:       true,
 		connection:  &internal.DbConnection{},
 	}
+
+	databasePath := path.Join(storePath, databaseFileName)
+	databaseFileExists, err := fileService.FileExists(databasePath)
+	if err != nil {
+		return nil, err
+	}
+
+	if databaseFileExists {
+		store.isNew = false
+	}
+
+	return store, nil
 }
 
 // Open opens and initializes the BoltDB database.
@@ -108,17 +115,7 @@ func (store *Store) Open() error {
 	}
 	store.connection.DB = db
 
-	err = store.initServices()
-	if err != nil {
-		return err
-	}
-
-	// if we have DBVersion in the database then ensure we flag this as NOT a new store
-	if _, err := store.VersionService.DBVersion(); err == nil {
-		store.isNew = false
-	}
-
-	return nil
+	return store.initServices()
 }
 
 // Close closes the BoltDB database.
@@ -136,6 +133,64 @@ func (store *Store) IsNew() bool {
 	return store.isNew
 }
 
+// CheckCurrentEdition checks if current edition is community edition
+func (store *Store) CheckCurrentEdition() error {
+	if store.edition() != portainer.PortainerCE {
+		return errors.ErrWrongDBEdition
+	}
+	return nil
+}
+
+// MigrateData automatically migrate the data based on the DBVersion.
+// This process is only triggered on an existing database, not if the database was just created.
+// if force is true, then migrate regardless.
+func (store *Store) MigrateData(force bool) error {
+	if store.isNew && !force {
+		return store.VersionService.StoreDBVersion(portainer.DBVersion)
+	}
+
+	version, err := store.VersionService.DBVersion()
+	if err == errors.ErrObjectNotFound {
+		version = 0
+	} else if err != nil {
+		return err
+	}
+
+	if version < portainer.DBVersion {
+		migratorParams := &migrator.Parameters{
+			DB:                      store.connection.DB,
+			DatabaseVersion:         version,
+			EndpointGroupService:    store.EndpointGroupService,
+			EndpointService:         store.EndpointService,
+			EndpointRelationService: store.EndpointRelationService,
+			ExtensionService:        store.ExtensionService,
+			RegistryService:         store.RegistryService,
+			ResourceControlService:  store.ResourceControlService,
+			RoleService:             store.RoleService,
+			ScheduleService:         store.ScheduleService,
+			SettingsService:         store.SettingsService,
+			StackService:            store.StackService,
+			TagService:              store.TagService,
+			TeamMembershipService:   store.TeamMembershipService,
+			UserService:             store.UserService,
+			VersionService:          store.VersionService,
+			FileService:             store.fileService,
+			DockerhubService:        store.DockerHubService,
+			AuthorizationService:    authorization.NewService(store),
+		}
+		migrator := migrator.NewMigrator(migratorParams)
+
+		log.Printf("Migrating database from version %v to %v.\n", version, portainer.DBVersion)
+		err = migrator.Migrate()
+		if err != nil {
+			log.Printf("An error occurred during database migration: %s\n", err)
+			return err
+		}
+	}
+
+	return nil
+}
+
 // BackupTo backs up db to a provided writer.
 // It does hot backup and doesn't block other database reads and writes
 func (store *Store) BackupTo(w io.Writer) error {
@@ -144,11 +199,3 @@ func (store *Store) BackupTo(w io.Writer) error {
 		return err
 	})
 }
-
-// CheckCurrentEdition checks if current edition is community edition
-func (store *Store) CheckCurrentEdition() error {
-	if store.edition() != portainer.PortainerCE {
-		return errors.ErrWrongDBEdition
-	}
-	return nil
-}

api/bolt/init.go

@@ -47,7 +47,6 @@ func (store *Store) Init() error {
 			HelmRepositoryURL:        portainer.DefaultHelmRepositoryURL,
 			UserSessionTimeout:       portainer.DefaultUserSessionTimeout,
 			KubeconfigExpiry:         portainer.DefaultKubeconfigExpiry,
-			KubectlShellImage:        portainer.DefaultKubectlShellImage,
 		}
 
 		err = store.SettingsService.UpdateSettings(defaultSettings)

api/bolt/migrate_data.go

@@ -1,146 +0,0 @@
-package bolt
-
-import (
-	"fmt"
-
-	"github.com/portainer/portainer/api/cli"
-
-	werrors "github.com/pkg/errors"
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/bolt/errors"
-	plog "github.com/portainer/portainer/api/bolt/log"
-	"github.com/portainer/portainer/api/bolt/migrator"
-	"github.com/portainer/portainer/api/internal/authorization"
-)
-
-const beforePortainerVersionUpgradeBackup = "portainer.db.bak"
-
-var migrateLog = plog.NewScopedLog("bolt, migrate")
-
-// FailSafeMigrate backup and restore DB if migration fail
-func (store *Store) FailSafeMigrate(migrator *migrator.Migrator) error {
-	defer func() {
-		if err := recover(); err != nil {
-			migrateLog.Info(fmt.Sprintf("Error during migration, recovering [%v]", err))
-			store.Rollback(true)
-		}
-	}()
-	return migrator.Migrate()
-}
-
-// MigrateData automatically migrate the data based on the DBVersion.
-// This process is only triggered on an existing database, not if the database was just created.
-// if force is true, then migrate regardless.
-func (store *Store) MigrateData(force bool) error {
-	if store.isNew && !force {
-		return store.VersionService.StoreDBVersion(portainer.DBVersion)
-	}
-
-	migrator, err := store.newMigrator()
-	if err != nil {
-		return err
-	}
-
-	// backup db file before upgrading DB to support rollback
-	isUpdating, err := store.VersionService.IsUpdating()
-	if err != nil && err != errors.ErrObjectNotFound {
-		return err
-	}
-
-	if !isUpdating && migrator.Version() != portainer.DBVersion {
-		err = store.backupVersion(migrator)
-		if err != nil {
-			return werrors.Wrapf(err, "failed to backup database")
-		}
-	}
-
-	if migrator.Version() < portainer.DBVersion {
-		migrateLog.Info(fmt.Sprintf("Migrating database from version %v to %v.\n", migrator.Version(), portainer.DBVersion))
-		err = store.FailSafeMigrate(migrator)
-		if err != nil {
-			migrateLog.Error("An error occurred during database migration", err)
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (store *Store) newMigrator() (*migrator.Migrator, error) {
-	version, err := store.version()
-	if err != nil {
-		return nil, err
-	}
-
-	migratorParams := &migrator.Parameters{
-		DB:                      store.connection.DB,
-		DatabaseVersion:         version,
-		EndpointGroupService:    store.EndpointGroupService,
-		EndpointService:         store.EndpointService,
-		EndpointRelationService: store.EndpointRelationService,
-		ExtensionService:        store.ExtensionService,
-		RegistryService:         store.RegistryService,
-		ResourceControlService:  store.ResourceControlService,
-		RoleService:             store.RoleService,
-		ScheduleService:         store.ScheduleService,
-		SettingsService:         store.SettingsService,
-		StackService:            store.StackService,
-		TagService:              store.TagService,
-		TeamMembershipService:   store.TeamMembershipService,
-		UserService:             store.UserService,
-		VersionService:          store.VersionService,
-		FileService:             store.fileService,
-		DockerhubService:        store.DockerHubService,
-		AuthorizationService:    authorization.NewService(store),
-	}
-	return migrator.NewMigrator(migratorParams), nil
-}
-
-// getBackupRestoreOptions returns options to store db at common backup dir location; used by:
-// - db backup prior to version upgrade
-// - db rollback
-func getBackupRestoreOptions(store *Store) *BackupOptions {
-	return &BackupOptions{
-		BackupDir:      store.commonBackupDir(),
-		BackupFileName: beforePortainerVersionUpgradeBackup,
-	}
-}
-
-// backupVersion will backup the database or panic if any errors occur
-func (store *Store) backupVersion(migrator *migrator.Migrator) error {
-	migrateLog.Info("Backing up database prior to version upgrade...")
-
-	options := getBackupRestoreOptions(store)
-
-	_, err := store.BackupWithOptions(options)
-	if err != nil {
-		migrateLog.Error("An error occurred during database backup", err)
-		removalErr := store.RemoveWithOptions(options)
-		if removalErr != nil {
-			migrateLog.Error("An error occurred during store removal prior to backup", err)
-		}
-		return err
-	}
-
-	return nil
-}
-
-// Rollback to a pre-upgrade backup copy/snapshot of portainer.db
-func (store *Store) Rollback(force bool) error {
-
-	if !force {
-		confirmed, err := cli.Confirm("Are you sure you want to rollback your database to the previous backup?")
-		if err != nil || !confirmed {
-			return err
-		}
-	}
-
-	options := getBackupRestoreOptions(store)
-
-	err := store.RestoreWithOptions(options)
-	if err != nil {
-		return err
-	}
-
-	return store.Close()
-}

api/bolt/migrate_data_test.go

@@ -1,172 +0,0 @@
-package bolt
-
-import (
-	"fmt"
-	"log"
-	"strings"
-	"testing"
-
-	portainer "github.com/portainer/portainer/api"
-)
-
-// testVersion is a helper which tests current store version against wanted version
-func testVersion(store *Store, versionWant int, t *testing.T) {
-	if v, _ := store.version(); v != versionWant {
-		t.Errorf("Expect store version to be %d but was %d", versionWant, v)
-	}
-}
-
-func TestMigrateData(t *testing.T) {
-	t.Run("MigrateData for New Store & Re-Open Check", func(t *testing.T) {
-		store, teardown := MustNewTestStore(false)
-		defer teardown()
-
-		if !store.IsNew() {
-			t.Error("Expect a new DB")
-		}
-
-		store.MigrateData(false)
-
-		testVersion(store, portainer.DBVersion, t)
-		store.Close()
-
-		store.Open()
-		if store.IsNew() {
-			t.Error("Expect store to NOT be new DB")
-		}
-	})
-
-	tests := []struct {
-		version         int
-		expectedVersion int
-	}{
-		{version: 2, expectedVersion: portainer.DBVersion},
-		{version: 21, expectedVersion: portainer.DBVersion},
-	}
-	for _, tc := range tests {
-		store, teardown := MustNewTestStore(true)
-		defer teardown()
-
-		// Setup data
-		store.VersionService.StoreDBVersion(tc.version)
-
-		// Required roles by migrations 22.2
-		store.RoleService.CreateRole(&portainer.Role{ID: 1})
-		store.RoleService.CreateRole(&portainer.Role{ID: 2})
-		store.RoleService.CreateRole(&portainer.Role{ID: 3})
-		store.RoleService.CreateRole(&portainer.Role{ID: 4})
-
-		t.Run(fmt.Sprintf("MigrateData for version %d", tc.version), func(t *testing.T) {
-			store.MigrateData(true)
-			testVersion(store, tc.expectedVersion, t)
-		})
-
-		t.Run(fmt.Sprintf("Restoring DB after migrateData for version %d", tc.version), func(t *testing.T) {
-			store.Rollback(true)
-			store.Open()
-			testVersion(store, tc.version, t)
-		})
-	}
-
-	t.Run("Error in MigrateData should restore backup before MigrateData", func(t *testing.T) {
-		store, teardown := MustNewTestStore(false)
-		defer teardown()
-
-		version := 2
-		store.VersionService.StoreDBVersion(version)
-
-		store.MigrateData(true)
-
-		testVersion(store, version, t)
-	})
-
-	t.Run("MigrateData should create backup file upon update", func(t *testing.T) {
-		store, teardown := MustNewTestStore(false)
-		defer teardown()
-		store.VersionService.StoreDBVersion(0)
-
-		store.MigrateData(true)
-
-		options := store.setupOptions(getBackupRestoreOptions(store))
-
-		if !isFileExist(options.BackupPath) {
-			t.Errorf("Backup file should exist; file=%s", options.BackupPath)
-		}
-	})
-
-	t.Run("MigrateData should fail to create backup if database file is set to updating", func(t *testing.T) {
-		store, teardown := MustNewTestStore(false)
-		defer teardown()
-
-		store.VersionService.StoreIsUpdating(true)
-
-		store.MigrateData(true)
-
-		options := store.setupOptions(getBackupRestoreOptions(store))
-
-		if isFileExist(options.BackupPath) {
-			t.Errorf("Backup file should not exist for dirty database; file=%s", options.BackupPath)
-		}
-	})
-
-	t.Run("MigrateData should not create backup on startup if portainer version matches db", func(t *testing.T) {
-		store, teardown := MustNewTestStore(false)
-		defer teardown()
-
-		store.MigrateData(true)
-
-		options := store.setupOptions(getBackupRestoreOptions(store))
-
-		if isFileExist(options.BackupPath) {
-			t.Errorf("Backup file should not exist for dirty database; file=%s", options.BackupPath)
-		}
-	})
-
-}
-
-func Test_getBackupRestoreOptions(t *testing.T) {
-	store, teardown := MustNewTestStore(false)
-	defer teardown()
-
-	options := getBackupRestoreOptions(store)
-
-	wantDir := store.commonBackupDir()
-	if !strings.HasSuffix(options.BackupDir, wantDir) {
-		log.Fatalf("incorrect backup dir; got=%s, want=%s", options.BackupDir, wantDir)
-	}
-
-	wantFilename := "portainer.db.bak"
-	if options.BackupFileName != wantFilename {
-		log.Fatalf("incorrect backup file; got=%s, want=%s", options.BackupFileName, wantFilename)
-	}
-}
-
-func TestRollback(t *testing.T) {
-	t.Run("Rollback should restore upgrade after backup", func(t *testing.T) {
-		version := 21
-		store, teardown := MustNewTestStore(false)
-		defer teardown()
-		store.VersionService.StoreDBVersion(version)
-
-		_, err := store.BackupWithOptions(getBackupRestoreOptions(store))
-		if err != nil {
-			log.Fatal(err)
-		}
-
-		// Change the current edition
-		err = store.VersionService.StoreDBVersion(version + 10)
-		if err != nil {
-			log.Fatal(err)
-		}
-
-		err = store.Rollback(true)
-		if err != nil {
-			t.Logf("Rollback failed: %s", err)
-			t.Fail()
-			return
-		}
-
-		store.Open()
-		testVersion(store, version, t)
-	})
-}

api/bolt/migrator/migrate_ce.go

@@ -1,327 +0,0 @@
-package migrator
-
-import (
-	"fmt"
-
-	werrors "github.com/pkg/errors"
-	portainer "github.com/portainer/portainer/api"
-)
-
-func migrationError(err error, context string) error {
-	return werrors.Wrap(err, "failed in "+context)
-}
-
-// Migrate checks the database version and migrate the existing data to the most recent data model.
-func (m *Migrator) Migrate() error {
-	// set DB to updating status
-	err := m.versionService.StoreIsUpdating(true)
-	if err != nil {
-		return migrationError(err, "StoreIsUpdating")
-	}
-
-	// Portainer < 1.12
-	if m.currentDBVersion < 1 {
-		err := m.updateAdminUserToDBVersion1()
-		if err != nil {
-			return migrationError(err, "updateAdminUserToDBVersion1")
-		}
-	}
-
-	// Portainer 1.12.x
-	if m.currentDBVersion < 2 {
-		err := m.updateResourceControlsToDBVersion2()
-		if err != nil {
-			return migrationError(err, "updateResourceControlsToDBVersion2")
-		}
-		err = m.updateEndpointsToDBVersion2()
-		if err != nil {
-			return migrationError(err, "updateEndpointsToDBVersion2")
-		}
-	}
-
-	// Portainer 1.13.x
-	if m.currentDBVersion < 3 {
-		err := m.updateSettingsToDBVersion3()
-		if err != nil {
-			return migrationError(err, "updateSettingsToDBVersion3")
-		}
-	}
-
-	// Portainer 1.14.0
-	if m.currentDBVersion < 4 {
-		err := m.updateEndpointsToDBVersion4()
-		if err != nil {
-			return migrationError(err, "updateEndpointsToDBVersion4")
-		}
-	}
-
-	// https://github.com/portainer/portainer/issues/1235
-	if m.currentDBVersion < 5 {
-		err := m.updateSettingsToVersion5()
-		if err != nil {
-			return migrationError(err, "updateSettingsToVersion5")
-		}
-	}
-
-	// https://github.com/portainer/portainer/issues/1236
-	if m.currentDBVersion < 6 {
-		err := m.updateSettingsToVersion6()
-		if err != nil {
-			return migrationError(err, "updateSettingsToVersion6")
-		}
-	}
-
-	// https://github.com/portainer/portainer/issues/1449
-	if m.currentDBVersion < 7 {
-		err := m.updateSettingsToVersion7()
-		if err != nil {
-			return migrationError(err, "updateSettingsToVersion7")
-		}
-	}
-
-	if m.currentDBVersion < 8 {
-		err := m.updateEndpointsToVersion8()
-		if err != nil {
-			return migrationError(err, "updateEndpointsToVersion8")
-		}
-	}
-
-	// https: //github.com/portainer/portainer/issues/1396
-	if m.currentDBVersion < 9 {
-		err := m.updateEndpointsToVersion9()
-		if err != nil {
-			return migrationError(err, "updateEndpointsToVersion9")
-		}
-	}
-
-	// https://github.com/portainer/portainer/issues/461
-	if m.currentDBVersion < 10 {
-		err := m.updateEndpointsToVersion10()
-		if err != nil {
-			return migrationError(err, "updateEndpointsToVersion10")
-		}
-	}
-
-	// https://github.com/portainer/portainer/issues/1906
-	if m.currentDBVersion < 11 {
-		err := m.updateEndpointsToVersion11()
-		if err != nil {
-			return migrationError(err, "updateEndpointsToVersion11")
-		}
-	}
-
-	// Portainer 1.18.0
-	if m.currentDBVersion < 12 {
-		err := m.updateEndpointsToVersion12()
-		if err != nil {
-			return migrationError(err, "updateEndpointsToVersion12")
-		}
-
-		err = m.updateEndpointGroupsToVersion12()
-		if err != nil {
-			return migrationError(err, "updateEndpointGroupsToVersion12")
-		}
-
-		err = m.updateStacksToVersion12()
-		if err != nil {
-			return migrationError(err, "updateStacksToVersion12")
-		}
-	}
-
-	// Portainer 1.19.0
-	if m.currentDBVersion < 13 {
-		err := m.updateSettingsToVersion13()
-		if err != nil {
-			return migrationError(err, "updateSettingsToVersion13")
-		}
-	}
-
-	// Portainer 1.19.2
-	if m.currentDBVersion < 14 {
-		err := m.updateResourceControlsToDBVersion14()
-		if err != nil {
-			return migrationError(err, "updateResourceControlsToDBVersion14")
-		}
-	}
-
-	// Portainer 1.20.0
-	if m.currentDBVersion < 15 {
-		err := m.updateSettingsToDBVersion15()
-		if err != nil {
-			return migrationError(err, "updateSettingsToDBVersion15")
-		}
-
-		err = m.updateTemplatesToVersion15()
-		if err != nil {
-			return migrationError(err, "updateTemplatesToVersion15")
-		}
-	}
-
-	if m.currentDBVersion < 16 {
-		err := m.updateSettingsToDBVersion16()
-		if err != nil {
-			return migrationError(err, "updateSettingsToDBVersion16")
-		}
-	}
-
-	// Portainer 1.20.1
-	if m.currentDBVersion < 17 {
-		err := m.updateExtensionsToDBVersion17()
-		if err != nil {
-			return migrationError(err, "updateExtensionsToDBVersion17")
-		}
-	}
-
-	// Portainer 1.21.0
-	if m.currentDBVersion < 18 {
-		err := m.updateUsersToDBVersion18()
-		if err != nil {
-			return migrationError(err, "updateUsersToDBVersion18")
-		}
-
-		err = m.updateEndpointsToDBVersion18()
-		if err != nil {
-			return migrationError(err, "updateEndpointsToDBVersion18")
-		}
-
-		err = m.updateEndpointGroupsToDBVersion18()
-		if err != nil {
-			return migrationError(err, "updateEndpointGroupsToDBVersion18")
-		}
-
-		err = m.updateRegistriesToDBVersion18()
-		if err != nil {
-			return migrationError(err, "updateRegistriesToDBVersion18")
-		}
-	}
-
-	// Portainer 1.22.0
-	if m.currentDBVersion < 19 {
-		err := m.updateSettingsToDBVersion19()
-		if err != nil {
-			return migrationError(err, "updateSettingsToDBVersion19")
-		}
-	}
-
-	// Portainer 1.22.1
-	if m.currentDBVersion < 20 {
-		err := m.updateUsersToDBVersion20()
-		if err != nil {
-			return migrationError(err, "updateUsersToDBVersion20")
-		}
-
-		err = m.updateSettingsToDBVersion20()
-		if err != nil {
-			return migrationError(err, "updateSettingsToDBVersion20")
-		}
-
-		err = m.updateSchedulesToDBVersion20()
-		if err != nil {
-			return migrationError(err, "updateSchedulesToDBVersion20")
-		}
-	}
-
-	// Portainer 1.23.0
-	// DBVersion 21 is missing as it was shipped as via hotfix 1.22.2
-	if m.currentDBVersion < 22 {
-		err := m.updateResourceControlsToDBVersion22()
-		if err != nil {
-			return migrationError(err, "updateResourceControlsToDBVersion22")
-		}
-
-		err = m.updateUsersAndRolesToDBVersion22()
-		if err != nil {
-			return migrationError(err, "updateUsersAndRolesToDBVersion22")
-		}
-	}
-
-	// Portainer 1.24.0
-	if m.currentDBVersion < 23 {
-		err := m.updateTagsToDBVersion23()
-		if err != nil {
-			return migrationError(err, "updateTagsToDBVersion23")
-		}
-
-		err = m.updateEndpointsAndEndpointGroupsToDBVersion23()
-		if err != nil {
-			return migrationError(err, "updateEndpointsAndEndpointGroupsToDBVersion23")
-		}
-	}
-
-	// Portainer 1.24.1
-	if m.currentDBVersion < 24 {
-		err := m.updateSettingsToDB24()
-		if err != nil {
-			return migrationError(err, "updateSettingsToDB24")
-		}
-	}
-
-	// Portainer 2.0.0
-	if m.currentDBVersion < 25 {
-		err := m.updateSettingsToDB25()
-		if err != nil {
-			return migrationError(err, "updateSettingsToDB25")
-		}
-
-		err = m.updateStacksToDB24()
-		if err != nil {
-			return migrationError(err, "updateStacksToDB24")
-		}
-	}
-
-	// Portainer 2.1.0
-	if m.currentDBVersion < 26 {
-		err := m.updateEndpointSettingsToDB25()
-		if err != nil {
-			return migrationError(err, "updateEndpointSettingsToDB25")
-		}
-	}
-
-	// Portainer 2.2.0
-	if m.currentDBVersion < 27 {
-		err := m.updateStackResourceControlToDB27()
-		if err != nil {
-			return migrationError(err, "updateStackResourceControlToDB27")
-		}
-	}
-
-	// Portainer 2.6.0
-	if m.currentDBVersion < 30 {
-		err := m.migrateDBVersionToDB30()
-		if err != nil {
-			return migrationError(err, "migrateDBVersionToDB30")
-		}
-	}
-
-	// Portainer 2.9.0
-	if m.currentDBVersion < 32 {
-		err := m.migrateDBVersionToDB32()
-		if err != nil {
-			return migrationError(err, "migrateDBVersionToDB32")
-		}
-	}
-
-	// Portainer 2.9.1
-	if m.currentDBVersion < 33 {
-		err := m.migrateDBVersionToDB33()
-		if err != nil {
-			return migrationError(err, "migrateDBVersionToDB33")
-		}
-	}
-
-	// Portainer 2.10
-	if m.currentDBVersion < 34 {
-		if err := m.migrateDBVersionToDB34(); err != nil {
-			return migrationError(err, "migrateDBVersionToDB34")
-		}
-	}
-
-	err = m.versionService.StoreDBVersion(portainer.DBVersion)
-	if err != nil {
-		return migrationError(err, "StoreDBVersion")
-	}
-	migrateLog.Info(fmt.Sprintf("Updated DB version to %d", portainer.DBVersion))
-
-	// reset DB updating status
-	return m.versionService.StoreIsUpdating(false)
-}

api/bolt/migrator/migrate_dbversion31.go

@@ -176,8 +176,7 @@ func (m *Migrator) updateVolumeResourceControlToDB32() error {
 
 		endpointDockerID, err := snapshotutils.FetchDockerID(snapshot)
 		if err != nil {
-			log.Printf("[WARN] [bolt,migrator,v31] [message: failed fetching environment docker id] [err: %s]", err)
-			continue
+			return fmt.Errorf("failed fetching environment docker id: %w", err)
 		}
 
 		if volumesData, done := snapshot.SnapshotRaw.Volumes.(map[string]interface{}); done {
@@ -214,11 +213,7 @@ func findResourcesToUpdateForDB32(dockerID string, volumesData map[string]interf
 	volumes := volumesData["Volumes"].([]interface{})
 	for _, volumeMeta := range volumes {
 		volume := volumeMeta.(map[string]interface{})
-		volumeName, nameExist := volume["Name"].(string)
-		if !nameExist {
-			continue
-		}
-
+		volumeName := volume["Name"].(string)
 		oldResourceID := fmt.Sprintf("%s%s", volumeName, volume["CreatedAt"].(string))
 		resourceControl, ok := volumeResourceControls[oldResourceID]
 

api/bolt/migrator/migrate_dbversion32.go

@@ -1,21 +0,0 @@
-package migrator
-
-import portainer "github.com/portainer/portainer/api"
-
-func (m *Migrator) migrateDBVersionToDB33() error {
-	if err := m.migrateSettingsToDB33(); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (m *Migrator) migrateSettingsToDB33() error {
-	settings, err := m.settingsService.Settings()
-	if err != nil {
-		return err
-	}
-
-	settings.KubectlShellImage = portainer.DefaultKubectlShellImage
-	return m.settingsService.UpdateSettings(settings)
-}

api/bolt/migrator/migrate_dbversion33.go

@@ -4,7 +4,7 @@ import (
 	portainer "github.com/portainer/portainer/api"
 )
 
-func (m *Migrator) migrateDBVersionToDB34() error {
+func (m *Migrator) migrateDBVersionTo33() error {
 	err := migrateStackEntryPoint(m.stackService)
 	if err != nil {
 		return err

api/bolt/migrator/migrate_dbversion33_test.go

@@ -14,7 +14,7 @@ import (
 )
 
 func TestMigrateStackEntryPoint(t *testing.T) {
-	dbConn, err := bolt.Open(path.Join(t.TempDir(), "portainer-ee-mig-34.db"), 0600, &bolt.Options{Timeout: 1 * time.Second})
+	dbConn, err := bolt.Open(path.Join(t.TempDir(), "portainer-ee-mig-33.db"), 0600, &bolt.Options{Timeout: 1 * time.Second})
 	assert.NoError(t, err, "failed to init testing DB connection")
 	defer dbConn.Close()
 

api/bolt/migrator/migrator.go

@@ -27,9 +27,8 @@ var migrateLog = plog.NewScopedLog("bolt, migrate")
 type (
 	// Migrator defines a service to migrate data after a Portainer version update.
 	Migrator struct {
-		db               *bolt.DB
-		currentDBVersion int
-
+		currentDBVersion        int
+		db                      *bolt.DB
 		endpointGroupService    *endpointgroup.Service
 		endpointService         *endpoint.Service
 		endpointRelationService *endpointrelation.Service
@@ -98,7 +97,295 @@ func NewMigrator(parameters *Parameters) *Migrator {
 	}
 }
 
-// Version exposes version of database
-func (migrator *Migrator) Version() int {
-	return migrator.currentDBVersion
+// Migrate checks the database version and migrate the existing data to the most recent data model.
+func (m *Migrator) Migrate() error {
+	// Portainer < 1.12
+	if m.currentDBVersion < 1 {
+		err := m.updateAdminUserToDBVersion1()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.12.x
+	if m.currentDBVersion < 2 {
+		err := m.updateResourceControlsToDBVersion2()
+		if err != nil {
+			return err
+		}
+		err = m.updateEndpointsToDBVersion2()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.13.x
+	if m.currentDBVersion < 3 {
+		err := m.updateSettingsToDBVersion3()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.14.0
+	if m.currentDBVersion < 4 {
+		err := m.updateEndpointsToDBVersion4()
+		if err != nil {
+			return err
+		}
+	}
+
+	// https://github.com/portainer/portainer/issues/1235
+	if m.currentDBVersion < 5 {
+		err := m.updateSettingsToVersion5()
+		if err != nil {
+			return err
+		}
+	}
+
+	// https://github.com/portainer/portainer/issues/1236
+	if m.currentDBVersion < 6 {
+		err := m.updateSettingsToVersion6()
+		if err != nil {
+			return err
+		}
+	}
+
+	// https://github.com/portainer/portainer/issues/1449
+	if m.currentDBVersion < 7 {
+		err := m.updateSettingsToVersion7()
+		if err != nil {
+			return err
+		}
+	}
+
+	if m.currentDBVersion < 8 {
+		err := m.updateEndpointsToVersion8()
+		if err != nil {
+			return err
+		}
+	}
+
+	// https: //github.com/portainer/portainer/issues/1396
+	if m.currentDBVersion < 9 {
+		err := m.updateEndpointsToVersion9()
+		if err != nil {
+			return err
+		}
+	}
+
+	// https://github.com/portainer/portainer/issues/461
+	if m.currentDBVersion < 10 {
+		err := m.updateEndpointsToVersion10()
+		if err != nil {
+			return err
+		}
+	}
+
+	// https://github.com/portainer/portainer/issues/1906
+	if m.currentDBVersion < 11 {
+		err := m.updateEndpointsToVersion11()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.18.0
+	if m.currentDBVersion < 12 {
+		err := m.updateEndpointsToVersion12()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateEndpointGroupsToVersion12()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateStacksToVersion12()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.19.0
+	if m.currentDBVersion < 13 {
+		err := m.updateSettingsToVersion13()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.19.2
+	if m.currentDBVersion < 14 {
+		err := m.updateResourceControlsToDBVersion14()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.20.0
+	if m.currentDBVersion < 15 {
+		err := m.updateSettingsToDBVersion15()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateTemplatesToVersion15()
+		if err != nil {
+			return err
+		}
+	}
+
+	if m.currentDBVersion < 16 {
+		err := m.updateSettingsToDBVersion16()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.20.1
+	if m.currentDBVersion < 17 {
+		err := m.updateExtensionsToDBVersion17()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.21.0
+	if m.currentDBVersion < 18 {
+		err := m.updateUsersToDBVersion18()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateEndpointsToDBVersion18()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateEndpointGroupsToDBVersion18()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateRegistriesToDBVersion18()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.22.0
+	if m.currentDBVersion < 19 {
+		err := m.updateSettingsToDBVersion19()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.22.1
+	if m.currentDBVersion < 20 {
+		err := m.updateUsersToDBVersion20()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateSettingsToDBVersion20()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateSchedulesToDBVersion20()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.23.0
+	// DBVersion 21 is missing as it was shipped as via hotfix 1.22.2
+	if m.currentDBVersion < 22 {
+		err := m.updateResourceControlsToDBVersion22()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateUsersAndRolesToDBVersion22()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.24.0
+	if m.currentDBVersion < 23 {
+		err := m.updateTagsToDBVersion23()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateEndpointsAndEndpointGroupsToDBVersion23()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.24.1
+	if m.currentDBVersion < 24 {
+		err := m.updateSettingsToDB24()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 2.0.0
+	if m.currentDBVersion < 25 {
+		err := m.updateSettingsToDB25()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateStacksToDB24()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 2.1.0
+	if m.currentDBVersion < 26 {
+		err := m.updateEndpointSettingsToDB25()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 2.2.0
+	if m.currentDBVersion < 27 {
+		err := m.updateStackResourceControlToDB27()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 2.6.0
+	if m.currentDBVersion < 30 {
+		err := m.migrateDBVersionToDB30()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 2.9.0
+	if m.currentDBVersion < 32 {
+		err := m.migrateDBVersionToDB32()
+		if err != nil {
+			return err
+		}
+	}
+
+	if m.currentDBVersion < 33 {
+		if err := m.migrateDBVersionTo33(); err != nil {
+			return err
+		}
+	}
+
+	return m.versionService.StoreDBVersion(portainer.DBVersion)
 }

api/bolt/stack/tests/stack_test.go

@@ -4,12 +4,18 @@ import (
 	"testing"
 	"time"
 
-	"github.com/gofrs/uuid"
-	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt"
+
 	bolterrors "github.com/portainer/portainer/api/bolt/errors"
-	"github.com/portainer/portainer/api/filesystem"
+
+	"github.com/portainer/portainer/api/bolt/bolttest"
+
+	"github.com/gofrs/uuid"
+
 	"github.com/stretchr/testify/assert"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/filesystem"
 )
 
 func newGuidString(t *testing.T) string {
@@ -29,7 +35,7 @@ func TestService_StackByWebhookID(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping test in short mode. Normally takes ~1s to run.")
 	}
-	store, teardown := bolt.MustNewTestStore(true)
+	store, teardown := bolttest.MustNewTestStore(true)
 	defer teardown()
 
 	b := stackBuilder{t: t, store: store}
@@ -87,7 +93,7 @@ func Test_RefreshableStacks(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping test in short mode. Normally takes ~1s to run.")
 	}
-	store, teardown := bolt.MustNewTestStore(true)
+	store, teardown := bolttest.MustNewTestStore(true)
 	defer teardown()
 
 	staticStack := portainer.Stack{ID: 1}

api/bolt/version/version.go

@@ -15,7 +15,6 @@ const (
 	versionKey  = "DB_VERSION"
 	instanceKey = "INSTANCE_ID"
 	editionKey  = "EDITION"
-	updatingKey = "DB_UPDATING"
 )
 
 // Service represents a service to manage stored versions.
@@ -84,21 +83,6 @@ func (service *Service) StoreDBVersion(version int) error {
 	})
 }
 
-// IsUpdating retrieves the database updating status.
-func (service *Service) IsUpdating() (bool, error) {
-	isUpdating, err := service.getKey(updatingKey)
-	if err != nil {
-		return false, err
-	}
-
-	return strconv.ParseBool(string(isUpdating))
-}
-
-// StoreIsUpdating store the database updating status.
-func (service *Service) StoreIsUpdating(isUpdating bool) error {
-	return service.setKey(updatingKey, strconv.FormatBool(isUpdating))
-}
-
 // InstanceID retrieves the stored instance ID.
 func (service *Service) InstanceID() (string, error) {
 	var data []byte

api/chisel/tunnel.go

@@ -56,32 +56,6 @@ func (service *Service) GetTunnelDetails(endpointID portainer.EndpointID) *porta
 	}
 }
 
-// GetActiveTunnel retrieves an active tunnel which allows communicating with edge agent
-func (service *Service) GetActiveTunnel(endpoint *portainer.Endpoint) (*portainer.TunnelDetails, error) {
-	tunnel := service.GetTunnelDetails(endpoint.ID)
-	if tunnel.Status == portainer.EdgeAgentIdle || tunnel.Status == portainer.EdgeAgentManagementRequired {
-		err := service.SetTunnelStatusToRequired(endpoint.ID)
-		if err != nil {
-			return nil, fmt.Errorf("failed opening tunnel to endpoint: %w", err)
-		}
-
-		if endpoint.EdgeCheckinInterval == 0 {
-			settings, err := service.dataStore.Settings().Settings()
-			if err != nil {
-				return nil, fmt.Errorf("failed fetching settings from db: %w", err)
-			}
-
-			endpoint.EdgeCheckinInterval = settings.EdgeAgentCheckinInterval
-		}
-
-		waitForAgentToConnect := time.Duration(endpoint.EdgeCheckinInterval) * time.Second
-		time.Sleep(waitForAgentToConnect * 2)
-	}
-	tunnel = service.GetTunnelDetails(endpoint.ID)
-
-	return tunnel, nil
-}
-
 // SetTunnelStatusToActive update the status of the tunnel associated to the specified environment(endpoint).
 // It sets the status to ACTIVE.
 func (service *Service) SetTunnelStatusToActive(endpointID portainer.EndpointID) {

api/cli/cli.go

@@ -47,7 +47,6 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 		SSL:                       kingpin.Flag("ssl", "Secure Portainer instance using SSL (deprecated)").Default(defaultSSL).Bool(),
 		SSLCert:                   kingpin.Flag("sslcert", "Path to the SSL certificate used to secure the Portainer instance").String(),
 		SSLKey:                    kingpin.Flag("sslkey", "Path to the SSL key used to secure the Portainer instance").String(),
-		Rollback:                  kingpin.Flag("rollback", "Rollback the database store to the previous version").Bool(),
 		SnapshotInterval:          kingpin.Flag("snapshot-interval", "Duration between each environment snapshot job").Default(defaultSnapshotInterval).String(),
 		AdminPassword:             kingpin.Flag("admin-password", "Hashed admin password").String(),
 		AdminPasswordFile:         kingpin.Flag("admin-password-file", "Path to the file containing the password for the admin user").String(),

api/cli/confirm.go

@@ -1,24 +0,0 @@
-package cli
-
-import (
-	"bufio"
-	"log"
-	"os"
-	"strings"
-)
-
-// Confirm starts a rollback db cli application
-func Confirm(message string) (bool, error) {
-	log.Printf("%s [y/N]", message)
-
-	reader := bufio.NewReader(os.Stdin)
-	answer, err := reader.ReadString('\n')
-	if err != nil {
-		return false, err
-	}
-	answer = strings.Replace(answer, "\n", "", -1)
-	answer = strings.ToLower(answer)
-
-	return answer == "y" || answer == "yes", nil
-
-}

api/cmd/portainer/main.go

@@ -56,22 +56,15 @@ func initFileService(dataStorePath string) portainer.FileService {
 	return fileService
 }
 
-func initDataStore(dataStorePath string, rollback bool, fileService portainer.FileService, shutdownCtx context.Context) portainer.DataStore {
-	store := bolt.NewStore(dataStorePath, fileService)
-	err := store.Open()
+func initDataStore(dataStorePath string, fileService portainer.FileService, shutdownCtx context.Context) portainer.DataStore {
+	store, err := bolt.NewStore(dataStorePath, fileService)
 	if err != nil {
-		log.Fatalf("failed opening store: %v", err)
+		log.Fatalf("failed creating data store: %v", err)
 	}
 
-	if rollback {
-		err := store.Rollback(false)
-		if err != nil {
-			log.Fatalf("failed rolling back: %s", err)
-		}
-
-		log.Println("Exiting rollback")
-		os.Exit(0)
-		return nil
+	err = store.Open()
+	if err != nil {
+		log.Fatalf("failed opening store: %v", err)
 	}
 
 	err = store.Init()
@@ -106,8 +99,8 @@ func initSwarmStackManager(assetsPath string, configPath string, signatureServic
 	return exec.NewSwarmStackManager(assetsPath, configPath, signatureService, fileService, reverseTunnelService)
 }
 
-func initKubernetesDeployer(kubernetesTokenCacheManager *kubeproxy.TokenCacheManager, kubernetesClientFactory *kubecli.ClientFactory, dataStore portainer.DataStore, reverseTunnelService portainer.ReverseTunnelService, signatureService portainer.DigitalSignatureService, proxyManager *proxy.Manager, assetsPath string) portainer.KubernetesDeployer {
-	return exec.NewKubernetesDeployer(kubernetesTokenCacheManager, kubernetesClientFactory, dataStore, reverseTunnelService, signatureService, proxyManager, assetsPath)
+func initKubernetesDeployer(kubernetesTokenCacheManager *kubeproxy.TokenCacheManager, kubernetesClientFactory *kubecli.ClientFactory, dataStore portainer.DataStore, reverseTunnelService portainer.ReverseTunnelService, signatureService portainer.DigitalSignatureService, assetsPath string) portainer.KubernetesDeployer {
+	return exec.NewKubernetesDeployer(kubernetesTokenCacheManager, kubernetesClientFactory, dataStore, reverseTunnelService, signatureService, assetsPath)
 }
 
 func initHelmPackageManager(assetsPath string) (libhelm.HelmPackageManager, error) {
@@ -406,7 +399,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 
 	fileService := initFileService(*flags.Data)
 
-	dataStore := initDataStore(*flags.Data, *flags.Rollback, fileService, shutdownCtx)
+	dataStore := initDataStore(*flags.Data, fileService, shutdownCtx)
 
 	if err := dataStore.CheckCurrentEdition(); err != nil {
 		log.Fatal(err)
@@ -476,7 +469,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		log.Fatalf("failed initializing swarm stack manager: %s", err)
 	}
 
-	kubernetesDeployer := initKubernetesDeployer(kubernetesTokenCacheManager, kubernetesClientFactory, dataStore, reverseTunnelService, digitalSignatureService, proxyManager, *flags.Assets)
+	kubernetesDeployer := initKubernetesDeployer(kubernetesTokenCacheManager, kubernetesClientFactory, dataStore, reverseTunnelService, digitalSignatureService, *flags.Assets)
 
 	helmPackageManager, err := initHelmPackageManager(*flags.Assets)
 	if err != nil {
@@ -549,7 +542,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 	}
 
 	scheduler := scheduler.NewScheduler(shutdownCtx)
-	stackDeployer := stacks.NewStackDeployer(swarmStackManager, composeStackManager, kubernetesDeployer)
+	stackDeployer := stacks.NewStackDeployer(swarmStackManager, composeStackManager)
 	stacks.StartStackSchedules(scheduler, stackDeployer, dataStore, gitService)
 
 	return &http.Server{

api/exec/common.go

@@ -1,5 +0,0 @@
-package exec
-
-import "regexp"
-
-var stackNameNormalizeRegex = regexp.MustCompile("[^-_a-z0-9]+")

api/exec/compose_stack.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"regexp"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -46,7 +47,7 @@ func (manager *ComposeStackManager) ComposeSyntaxMaxVersion() string {
 func (manager *ComposeStackManager) Up(ctx context.Context, stack *portainer.Stack, endpoint *portainer.Endpoint) error {
 	url, proxy, err := manager.fetchEndpointProxy(endpoint)
 	if err != nil {
-		return errors.Wrap(err, "failed to fetch environment proxy")
+		return errors.Wrap(err, "failed to featch environment proxy")
 	}
 
 	if proxy != nil {
@@ -79,8 +80,9 @@ func (manager *ComposeStackManager) Down(ctx context.Context, stack *portainer.S
 }
 
 // NormalizeStackName returns a new stack name with unsupported characters replaced
-func (manager *ComposeStackManager) NormalizeStackName(name string) string {
-	return stackNameNormalizeRegex.ReplaceAllString(strings.ToLower(name), "")
+func (w *ComposeStackManager) NormalizeStackName(name string) string {
+	r := regexp.MustCompile("[^a-z0-9]+")
+	return r.ReplaceAllString(strings.ToLower(name), "")
 }
 
 func (manager *ComposeStackManager) fetchEndpointProxy(endpoint *portainer.Endpoint) (string, *factory.ProxyServer, error) {
@@ -88,7 +90,7 @@ func (manager *ComposeStackManager) fetchEndpointProxy(endpoint *portainer.Endpo
 		return "", nil, nil
 	}
 
-	proxy, err := manager.proxyManager.CreateAgentProxyServer(endpoint)
+	proxy, err := manager.proxyManager.CreateComposeProxyServer(endpoint)
 	if err != nil {
 		return "", nil, err
 	}

api/exec/exectest/kubernetes_mocks.go

@@ -1,23 +0,0 @@
-package exectest
-
-import (
-	portainer "github.com/portainer/portainer/api"
-)
-
-type kubernetesMockDeployer struct{}
-
-func NewKubernetesDeployer() portainer.KubernetesDeployer {
-	return &kubernetesMockDeployer{}
-}
-
-func (deployer *kubernetesMockDeployer) Deploy(userID portainer.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) {
-	return "", nil
-}
-
-func (deployer *kubernetesMockDeployer) Remove(userID portainer.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) {
-	return "", nil
-}
-
-func (deployer *kubernetesMockDeployer) ConvertCompose(data []byte) ([]byte, error) {
-	return nil, nil
-}

api/exec/kubernetes_deploy.go

@@ -2,19 +2,24 @@ package exec
 
 import (
 	"bytes"
+	"encoding/json"
+	"errors"
 	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
 	"os/exec"
 	"path"
 	"runtime"
 	"strings"
+	"time"
 
-	"github.com/pkg/errors"
-	"github.com/portainer/portainer/api/http/proxy"
-	"github.com/portainer/portainer/api/http/proxy/factory"
 	"github.com/portainer/portainer/api/http/proxy/factory/kubernetes"
+	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/kubernetes/cli"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/crypto"
 )
 
 // KubernetesDeployer represents a service to deploy resources inside a Kubernetes environment(endpoint).
@@ -25,11 +30,10 @@ type KubernetesDeployer struct {
 	signatureService            portainer.DigitalSignatureService
 	kubernetesClientFactory     *cli.ClientFactory
 	kubernetesTokenCacheManager *kubernetes.TokenCacheManager
-	proxyManager                *proxy.Manager
 }
 
 // NewKubernetesDeployer initializes a new KubernetesDeployer service.
-func NewKubernetesDeployer(kubernetesTokenCacheManager *kubernetes.TokenCacheManager, kubernetesClientFactory *cli.ClientFactory, datastore portainer.DataStore, reverseTunnelService portainer.ReverseTunnelService, signatureService portainer.DigitalSignatureService, proxyManager *proxy.Manager, binaryPath string) *KubernetesDeployer {
+func NewKubernetesDeployer(kubernetesTokenCacheManager *kubernetes.TokenCacheManager, kubernetesClientFactory *cli.ClientFactory, datastore portainer.DataStore, reverseTunnelService portainer.ReverseTunnelService, signatureService portainer.DigitalSignatureService, binaryPath string) *KubernetesDeployer {
 	return &KubernetesDeployer{
 		binaryPath:                  binaryPath,
 		dataStore:                   datastore,
@@ -37,33 +41,32 @@ func NewKubernetesDeployer(kubernetesTokenCacheManager *kubernetes.TokenCacheMan
 		signatureService:            signatureService,
 		kubernetesClientFactory:     kubernetesClientFactory,
 		kubernetesTokenCacheManager: kubernetesTokenCacheManager,
-		proxyManager:                proxyManager,
 	}
 }
 
-func (deployer *KubernetesDeployer) getToken(userID portainer.UserID, endpoint *portainer.Endpoint, setLocalAdminToken bool) (string, error) {
-	kubeCLI, err := deployer.kubernetesClientFactory.GetKubeClient(endpoint)
+func (deployer *KubernetesDeployer) getToken(request *http.Request, endpoint *portainer.Endpoint, setLocalAdminToken bool) (string, error) {
+	tokenData, err := security.RetrieveTokenData(request)
+	if err != nil {
+		return "", err
+	}
+
+	kubecli, err := deployer.kubernetesClientFactory.GetKubeClient(endpoint)
 	if err != nil {
 		return "", err
 	}
 
 	tokenCache := deployer.kubernetesTokenCacheManager.GetOrCreateTokenCache(int(endpoint.ID))
 
-	tokenManager, err := kubernetes.NewTokenManager(kubeCLI, deployer.dataStore, tokenCache, setLocalAdminToken)
+	tokenManager, err := kubernetes.NewTokenManager(kubecli, deployer.dataStore, tokenCache, setLocalAdminToken)
 	if err != nil {
 		return "", err
 	}
 
-	user, err := deployer.dataStore.User().User(userID)
-	if err != nil {
-		return "", errors.Wrap(err, "failed to fetch the user")
-	}
-
-	if user.Role == portainer.AdministratorRole {
+	if tokenData.Role == portainer.AdministratorRole {
 		return tokenManager.GetAdminServiceAccountToken(), nil
 	}
 
-	token, err := tokenManager.GetUserServiceAccountToken(int(user.ID), endpoint.ID)
+	token, err := tokenManager.GetUserServiceAccountToken(int(tokenData.ID), endpoint.ID)
 	if err != nil {
 		return "", err
 	}
@@ -74,62 +77,156 @@ func (deployer *KubernetesDeployer) getToken(userID portainer.UserID, endpoint *
 	return token, nil
 }
 
-// Deploy upserts Kubernetes resources defined in manifest(s)
-func (deployer *KubernetesDeployer) Deploy(userID portainer.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) {
-	return deployer.command("apply", userID, endpoint, manifestFiles, namespace)
-}
-
-// Remove deletes Kubernetes resources defined in manifest(s)
-func (deployer *KubernetesDeployer) Remove(userID portainer.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) {
-	return deployer.command("delete", userID, endpoint, manifestFiles, namespace)
-}
-
-func (deployer *KubernetesDeployer) command(operation string, userID portainer.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) {
-	token, err := deployer.getToken(userID, endpoint, endpoint.Type == portainer.KubernetesLocalEnvironment)
-	if err != nil {
-		return "", errors.Wrap(err, "failed generating a user token")
-	}
-
-	command := path.Join(deployer.binaryPath, "kubectl")
-	if runtime.GOOS == "windows" {
-		command = path.Join(deployer.binaryPath, "kubectl.exe")
-	}
-
-	args := []string{"--token", token}
-	if namespace != "" {
-		args = append(args, "--namespace", namespace)
-	}
-
-	if endpoint.Type == portainer.AgentOnKubernetesEnvironment || endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment {
-		url, proxy, err := deployer.getAgentURL(endpoint)
+// Deploy will deploy a Kubernetes manifest inside a specific namespace in a Kubernetes environment(endpoint).
+// Otherwise it will use kubectl to deploy the manifest.
+func (deployer *KubernetesDeployer) Deploy(request *http.Request, endpoint *portainer.Endpoint, stackConfig string, namespace string) (string, error) {
+	if endpoint.Type == portainer.KubernetesLocalEnvironment {
+		token, err := deployer.getToken(request, endpoint, true)
 		if err != nil {
-			return "", errors.WithMessage(err, "failed generating endpoint URL")
+			return "", err
 		}
 
-		defer proxy.Close()
-		args = append(args, "--server", url)
+		command := path.Join(deployer.binaryPath, "kubectl")
+		if runtime.GOOS == "windows" {
+			command = path.Join(deployer.binaryPath, "kubectl.exe")
+		}
+
+		args := make([]string, 0)
+		args = append(args, "--server", endpoint.URL)
 		args = append(args, "--insecure-skip-tls-verify")
+		args = append(args, "--token", token)
+		args = append(args, "--namespace", namespace)
+		args = append(args, "apply", "-f", "-")
+
+		var stderr bytes.Buffer
+		cmd := exec.Command(command, args...)
+		cmd.Stderr = &stderr
+		cmd.Stdin = strings.NewReader(stackConfig)
+
+		output, err := cmd.Output()
+		if err != nil {
+			return "", errors.New(stderr.String())
+		}
+
+		return string(output), nil
 	}
 
-	if operation == "delete" {
-		args = append(args, "--ignore-not-found=true")
+	// agent
+
+	endpointURL := endpoint.URL
+	if endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment {
+		tunnel := deployer.reverseTunnelService.GetTunnelDetails(endpoint.ID)
+		if tunnel.Status == portainer.EdgeAgentIdle {
+
+			err := deployer.reverseTunnelService.SetTunnelStatusToRequired(endpoint.ID)
+			if err != nil {
+				return "", err
+			}
+
+			settings, err := deployer.dataStore.Settings().Settings()
+			if err != nil {
+				return "", err
+			}
+
+			waitForAgentToConnect := time.Duration(settings.EdgeAgentCheckinInterval) * time.Second
+			time.Sleep(waitForAgentToConnect * 2)
+		}
+
+		endpointURL = fmt.Sprintf("http://127.0.0.1:%d", tunnel.Port)
 	}
 
-	args = append(args, operation)
-	for _, path := range manifestFiles {
-		args = append(args, "-f", strings.TrimSpace(path))
+	transport := &http.Transport{}
+
+	if endpoint.TLSConfig.TLS {
+		tlsConfig, err := crypto.CreateTLSConfigurationFromDisk(endpoint.TLSConfig.TLSCACertPath, endpoint.TLSConfig.TLSCertPath, endpoint.TLSConfig.TLSKeyPath, endpoint.TLSConfig.TLSSkipVerify)
+		if err != nil {
+			return "", err
+		}
+		transport.TLSClientConfig = tlsConfig
 	}
 
-	var stderr bytes.Buffer
-	cmd := exec.Command(command, args...)
-	cmd.Stderr = &stderr
+	httpCli := &http.Client{
+		Transport: transport,
+	}
 
-	output, err := cmd.Output()
+	if !strings.HasPrefix(endpointURL, "http") {
+		endpointURL = fmt.Sprintf("https://%s", endpointURL)
+	}
+
+	url, err := url.Parse(fmt.Sprintf("%s/v2/kubernetes/stack", endpointURL))
 	if err != nil {
-		return "", errors.Wrapf(err, "failed to execute kubectl command: %q", stderr.String())
+		return "", err
 	}
 
-	return string(output), nil
+	reqPayload, err := json.Marshal(
+		struct {
+			StackConfig string
+			Namespace   string
+		}{
+			StackConfig: stackConfig,
+			Namespace:   namespace,
+		})
+	if err != nil {
+		return "", err
+	}
+
+	req, err := http.NewRequest(http.MethodPost, url.String(), bytes.NewReader(reqPayload))
+	if err != nil {
+		return "", err
+	}
+
+	signature, err := deployer.signatureService.CreateSignature(portainer.PortainerAgentSignatureMessage)
+	if err != nil {
+		return "", err
+	}
+
+	token, err := deployer.getToken(request, endpoint, false)
+	if err != nil {
+		return "", err
+	}
+
+	req.Header.Set(portainer.PortainerAgentPublicKeyHeader, deployer.signatureService.EncodedPublicKey())
+	req.Header.Set(portainer.PortainerAgentSignatureHeader, signature)
+	req.Header.Set(portainer.PortainerAgentKubernetesSATokenHeader, token)
+
+	resp, err := httpCli.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		var errorResponseData struct {
+			Message string
+			Details string
+		}
+		err = json.NewDecoder(resp.Body).Decode(&errorResponseData)
+		if err != nil {
+			output, parseStringErr := ioutil.ReadAll(resp.Body)
+			if parseStringErr != nil {
+				return "", parseStringErr
+			}
+
+			return "", fmt.Errorf("Failed parsing, body: %s, error: %w", output, err)
+
+		}
+
+		return "", fmt.Errorf("Deployment to agent failed: %s", errorResponseData.Details)
+	}
+
+	var responseData struct{ Output string }
+	err = json.NewDecoder(resp.Body).Decode(&responseData)
+	if err != nil {
+		parsedOutput, parseStringErr := ioutil.ReadAll(resp.Body)
+		if parseStringErr != nil {
+			return "", parseStringErr
+		}
+
+		return "", fmt.Errorf("Failed decoding, body: %s, err: %w", parsedOutput, err)
+	}
+
+	return responseData.Output, nil
+
 }
 
 // ConvertCompose leverages the kompose binary to deploy a compose compliant manifest.
@@ -154,12 +251,3 @@ func (deployer *KubernetesDeployer) ConvertCompose(data []byte) ([]byte, error)
 
 	return output, nil
 }
-
-func (deployer *KubernetesDeployer) getAgentURL(endpoint *portainer.Endpoint) (string, *factory.ProxyServer, error) {
-	proxy, err := deployer.proxyManager.CreateAgentProxyServer(endpoint)
-	if err != nil {
-		return "", nil, err
-	}
-
-	return fmt.Sprintf("http://127.0.0.1:%d/kubernetes", proxy.Port), proxy, nil
-}

api/exec/swarm_stack.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"regexp"
 	"runtime"
 	"strings"
 
@@ -189,7 +190,8 @@ func (manager *SwarmStackManager) retrieveConfigurationFromDisk(path string) (ma
 }
 
 func (manager *SwarmStackManager) NormalizeStackName(name string) string {
-	return stackNameNormalizeRegex.ReplaceAllString(strings.ToLower(name), "")
+	r := regexp.MustCompile("[^a-z0-9]+")
+	return r.ReplaceAllString(strings.ToLower(name), "")
 }
 
 func configureFilePaths(args []string, filePaths []string) []string {

api/filesystem/write.go

@@ -1,23 +0,0 @@
-package filesystem
-
-import (
-	"os"
-	"path/filepath"
-
-	"github.com/pkg/errors"
-)
-
-func WriteToFile(dst string, content []byte) error {
-	if err := os.MkdirAll(filepath.Dir(dst), 0744); err != nil {
-		return errors.Wrapf(err, "failed to create filestructure for the path %q", dst)
-	}
-
-	file, err := os.Create(dst)
-	if err != nil {
-		return errors.Wrapf(err, "failed to open a file %q", dst)
-	}
-	defer file.Close()
-
-	_, err = file.Write(content)
-	return errors.Wrapf(err, "failed to write a file %q", dst)
-}

api/filesystem/write_test.go

@@ -1,48 +0,0 @@
-package filesystem
-
-import (
-	"io/ioutil"
-	"path"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func Test_WriteFile_CanStoreContentInANewFile(t *testing.T) {
-	tmpDir := t.TempDir()
-	tmpFilePath := path.Join(tmpDir, "dummy")
-
-	content := []byte("content")
-	err := WriteToFile(tmpFilePath, content)
-	assert.NoError(t, err)
-
-	fileContent, _ := ioutil.ReadFile(tmpFilePath)
-	assert.Equal(t, content, fileContent)
-}
-
-func Test_WriteFile_CanOverwriteExistingFile(t *testing.T) {
-	tmpDir := t.TempDir()
-	tmpFilePath := path.Join(tmpDir, "dummy")
-
-	err := WriteToFile(tmpFilePath, []byte("content"))
-	assert.NoError(t, err)
-
-	content := []byte("new content")
-	err = WriteToFile(tmpFilePath, content)
-	assert.NoError(t, err)
-
-	fileContent, _ := ioutil.ReadFile(tmpFilePath)
-	assert.Equal(t, content, fileContent)
-}
-
-func Test_WriteFile_CanWriteANestedPath(t *testing.T) {
-	tmpDir := t.TempDir()
-	tmpFilePath := path.Join(tmpDir, "dir", "sub-dir", "dummy")
-
-	content := []byte("content")
-	err := WriteToFile(tmpFilePath, content)
-	assert.NoError(t, err)
-
-	fileContent, _ := ioutil.ReadFile(tmpFilePath)
-	assert.Equal(t, content, fileContent)
-}

api/go.mod

@@ -3,48 +3,57 @@ module github.com/portainer/portainer/api
 go 1.16
 
 require (
-	github.com/Microsoft/go-winio v0.4.17
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
+	github.com/Microsoft/go-winio v0.4.16
+	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
 	github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535
 	github.com/boltdb/bolt v1.3.1
-	github.com/containerd/containerd v1.5.7 // indirect
+	github.com/containerd/containerd v1.3.1 // indirect
 	github.com/coreos/go-semver v0.3.0
 	github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
-	github.com/docker/cli v20.10.9+incompatible
-	github.com/docker/docker v20.10.9+incompatible
+	github.com/docker/cli v0.0.0-20191126203649-54d085b857e9
+	github.com/docker/distribution v2.7.1+incompatible // indirect
+	github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0
 	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-units v0.4.0 // indirect
 	github.com/g07cha/defender v0.0.0-20180505193036-5665c627c814
 	github.com/go-git/go-git/v5 v5.3.0
 	github.com/go-ldap/ldap/v3 v3.1.8
-	github.com/gofrs/uuid v4.0.0+incompatible
-	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
+	github.com/gofrs/uuid v3.2.0+incompatible
 	github.com/gorilla/mux v1.7.3
 	github.com/gorilla/securecookie v1.1.1
-	github.com/gorilla/websocket v1.4.2
+	github.com/gorilla/websocket v1.4.1
 	github.com/joho/godotenv v1.3.0
 	github.com/jpillora/chisel v0.0.0-20190724232113-f3a8df20e389
-	github.com/json-iterator/go v1.1.11
+	github.com/json-iterator/go v1.1.10
 	github.com/koding/websocketproxy v0.0.0-20181220232114-7ed82d81a28c
-	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
+	github.com/mattn/go-shellwords v1.0.6 // indirect
+	github.com/mitchellh/mapstructure v1.1.2 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.0.1 // indirect
 	github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6
 	github.com/pkg/errors v0.9.1
 	github.com/portainer/docker-compose-wrapper v0.0.0-20210909083948-8be0d98451a1
 	github.com/portainer/libcrypto v0.0.0-20210422035235-c652195c5c3a
-	github.com/portainer/libhelm v0.0.0-20210929000907-825e93d62108
+	github.com/portainer/libhelm v0.0.0-20210913052337-365741c1c320
 	github.com/portainer/libhttp v0.0.0-20190806161843-ba068f58be33
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/sirupsen/logrus v1.8.1
 	github.com/stretchr/testify v1.7.0
-	github.com/swaggo/swag v1.7.3
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
 	golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2
-	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
-	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
+	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
-	k8s.io/api v0.22.2
-	k8s.io/apimachinery v0.22.2
-	k8s.io/client-go v0.22.2
+	gotest.tools v2.2.0+incompatible // indirect
+	k8s.io/api v0.17.2
+	k8s.io/apimachinery v0.17.2
+	k8s.io/client-go v0.17.2
 )
+
+replace github.com/docker/docker => github.com/docker/engine v1.4.2-0.20200204220554-5f6d6f3f2203
+
+replace golang.org/x/sys => golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456

api/http/handler/auth/authenticate.go

@@ -39,7 +39,6 @@ func (payload *authenticatePayload) Validate(r *http.Request) error {
 
 // @id AuthenticateUser
 // @summary Authenticate
-// @description **Access policy**: public
 // @description Use this environment(endpoint) to authenticate against Portainer using a username and password.
 // @tags auth
 // @accept json

api/http/handler/auth/authenticate_oauth.go

@@ -44,7 +44,6 @@ func (handler *Handler) authenticateOAuth(code string, settings *portainer.OAuth
 
 // @id ValidateOAuth
 // @summary Authenticate with OAuth
-// @description **Access policy**: public
 // @tags auth
 // @accept json
 // @produce json

api/http/handler/auth/logout.go

@@ -10,7 +10,6 @@ import (
 
 // @id Logout
 // @summary Logout
-// @description **Access policy**: authenticated
 // @security jwt
 // @tags auth
 // @success 204 "Success"

api/http/handler/backup/backup.go

@@ -27,9 +27,8 @@ func (p *backupPayload) Validate(r *http.Request) error {
 // @description **Access policy**: admin
 // @tags backup
 // @security jwt
-// @accept json
 // @produce octet-stream
-// @param body body backupPayload false "An object contains the password to encrypt the backup with"
+// @param Password body string false "Password to encrypt the backup with"
 // @success 200 "Success"
 // @failure 400 "Invalid request"
 // @failure 500 "Server error"

api/http/handler/backup/restore.go

@@ -22,9 +22,10 @@ type restorePayload struct {
 // @description Triggers a system restore using provided backup file
 // @description **Access policy**: public
 // @tags backup
-// @accept json
-// @param restorePayload body restorePayload true "Restore request payload"
-// @success 200 "Success"
+// @param FileContent body []byte true "Content of the backup"
+// @param FileName body string true "File name"
+// @param Password body string false "Password to decrypt the backup with"
+// @success 200  "Success"
 // @failure 400 "Invalid request"
 // @failure 500 "Server error"
 // @router /restore [post]

api/http/handler/customtemplates/customtemplate_create.go

@@ -3,7 +3,6 @@ package customtemplates
 import (
 	"errors"
 	"net/http"
-	"regexp"
 	"strconv"
 
 	"github.com/asaskevich/govalidator"
@@ -130,20 +129,9 @@ func (payload *customTemplateFromFileContentPayload) Validate(r *http.Request) e
 	if payload.Type != portainer.KubernetesStack && payload.Type != portainer.DockerSwarmStack && payload.Type != portainer.DockerComposeStack {
 		return errors.New("Invalid custom template type")
 	}
-	if !isValidNote(payload.Note) {
-		return errors.New("Invalid note. <img> tag is not supported")
-	}
 	return nil
 }
 
-func isValidNote(note string) bool {
-	if govalidator.IsNull(note) {
-		return true
-	}
-	match, _ := regexp.MatchString("<img", note)
-	return !match
-}
-
 func (handler *Handler) createCustomTemplateFromFileContent(r *http.Request) (*portainer.CustomTemplate, error) {
 	var payload customTemplateFromFileContentPayload
 	err := request.DecodeAndValidateJSONPayload(r, &payload)
@@ -230,9 +218,6 @@ func (payload *customTemplateFromGitRepositoryPayload) Validate(r *http.Request)
 	if payload.Type != portainer.DockerSwarmStack && payload.Type != portainer.DockerComposeStack {
 		return errors.New("Invalid custom template type")
 	}
-	if !isValidNote(payload.Note) {
-		return errors.New("Invalid note. <img> tag is not supported")
-	}
 	return nil
 }
 
@@ -300,9 +285,6 @@ func (payload *customTemplateFromFileUploadPayload) Validate(r *http.Request) er
 	payload.Logo = logo
 
 	note, _ := request.RetrieveMultiPartFormValue(r, "Note", true)
-	if !isValidNote(note) {
-		return errors.New("Invalid note. <img> tag is not supported")
-	}
 	payload.Note = note
 
 	typeNumeral, _ := request.RetrieveNumericMultiPartFormValue(r, "Type", true)

api/http/handler/customtemplates/customtemplate_delete.go

@@ -16,7 +16,7 @@ import (
 // @id CustomTemplateDelete
 // @summary Remove a template
 // @description Remove a template.
-// @description **Access policy**: authenticated
+// @description **Access policy**: authorized
 // @tags custom_templates
 // @security jwt
 // @param id path int true "Template identifier"

api/http/handler/customtemplates/customtemplate_file.go

@@ -18,7 +18,7 @@ type fileResponse struct {
 // @id CustomTemplateFile
 // @summary Get Template stack file content.
 // @description Retrieve the content of the Stack file for the specified custom template
-// @description **Access policy**: authenticated
+// @description **Access policy**: authorized
 // @tags custom_templates
 // @security jwt
 // @produce json

api/http/handler/customtemplates/customtemplate_inspect.go

@@ -19,6 +19,7 @@ import (
 // @description **Access policy**: authenticated
 // @tags custom_templates
 // @security jwt
+// @accept json
 // @produce json
 // @param id path int true "Template identifier"
 // @success 200 {object} portainer.CustomTemplate "Success"

api/http/handler/customtemplates/customtemplate_update.go

@@ -51,9 +51,6 @@ func (payload *customTemplateUpdatePayload) Validate(r *http.Request) error {
 	if govalidator.IsNull(payload.Description) {
 		return errors.New("Invalid custom template description")
 	}
-	if !isValidNote(payload.Note) {
-		return errors.New("Invalid note. <img> tag is not supported")
-	}
 	return nil
 }
 

api/http/handler/edgegroups/edgegroup_create.go

@@ -34,7 +34,7 @@ func (payload *edgeGroupCreatePayload) Validate(r *http.Request) error {
 
 // @id EdgeGroupCreate
 // @summary Create an EdgeGroup
-// @description **Access policy**: administrator
+// @description
 // @tags edge_groups
 // @security jwt
 // @accept json

api/http/handler/edgegroups/edgegroup_delete.go

@@ -13,9 +13,11 @@ import (
 
 // @id EdgeGroupDelete
 // @summary Deletes an EdgeGroup
-// @description **Access policy**: administrator
+// @description
 // @tags edge_groups
 // @security jwt
+// @accept json
+// @produce json
 // @param id path int true "EdgeGroup Id"
 // @success 204
 // @failure 503 "Edge compute features are disabled"

api/http/handler/edgegroups/edgegroup_inspect.go

@@ -12,9 +12,10 @@ import (
 
 // @id EdgeGroupInspect
 // @summary Inspects an EdgeGroup
-// @description **Access policy**: administrator
+// @description
 // @tags edge_groups
 // @security jwt
+// @accept json
 // @produce json
 // @param id path int true "EdgeGroup Id"
 // @success 200 {object} portainer.EdgeGroup

api/http/handler/edgegroups/edgegroup_list.go

@@ -17,9 +17,10 @@ type decoratedEdgeGroup struct {
 
 // @id EdgeGroupList
 // @summary list EdgeGroups
-// @description **Access policy**: administrator
+// @description
 // @tags edge_groups
 // @security jwt
+// @accept json
 // @produce json
 // @success 200 {array} portainer.EdgeGroup{HasEdgeStack=bool} "EdgeGroups"
 // @failure 500

api/http/handler/edgegroups/edgegroup_update.go

@@ -36,7 +36,7 @@ func (payload *edgeGroupUpdatePayload) Validate(r *http.Request) error {
 
 // @id EgeGroupUpdate
 // @summary Updates an EdgeGroup
-// @description **Access policy**: administrator
+// @description
 // @tags edge_groups
 // @security jwt
 // @accept json

api/http/handler/edgejobs/edgejob_create.go

@@ -16,9 +16,10 @@ import (
 
 // @id EdgeJobCreate
 // @summary Create an EdgeJob
-// @description **Access policy**: administrator
+// @description
 // @tags edge_jobs
 // @security jwt
+// @accept json
 // @produce json
 // @param method query string true "Creation Method" Enums(file, string)
 // @param body_string body edgeJobCreateFromFileContentPayload true "EdgeGroup data when method is string"

api/http/handler/edgejobs/edgejob_delete.go

@@ -13,9 +13,11 @@ import (
 
 // @id EdgeJobDelete
 // @summary Delete an EdgeJob
-// @description **Access policy**: administrator
+// @description
 // @tags edge_jobs
 // @security jwt
+// @accept json
+// @produce json
 // @param id path string true "EdgeJob Id"
 // @success 204
 // @failure 500

api/http/handler/edgejobs/edgejob_file.go

@@ -16,9 +16,10 @@ type edgeJobFileResponse struct {
 
 // @id EdgeJobFile
 // @summary Fetch a file of an EdgeJob
-// @description **Access policy**: administrator
+// @description
 // @tags edge_jobs
 // @security jwt
+// @accept json
 // @produce json
 // @param id path string true "EdgeJob Id"
 // @success 200 {object} edgeJobFileResponse

api/http/handler/edgejobs/edgejob_inspect.go

@@ -17,9 +17,10 @@ type edgeJobInspectResponse struct {
 
 // @id EdgeJobInspect
 // @summary Inspect an EdgeJob
-// @description **Access policy**: administrator
+// @description
 // @tags edge_jobs
 // @security jwt
+// @accept json
 // @produce json
 // @param id path string true "EdgeJob Id"
 // @success 200 {object} portainer.EdgeJob

api/http/handler/edgejobs/edgejob_list.go

@@ -9,9 +9,10 @@ import (
 
 // @id EdgeJobList
 // @summary Fetch EdgeJobs list
-// @description **Access policy**: administrator
+// @description
 // @tags edge_jobs
 // @security jwt
+// @accept json
 // @produce json
 // @success 200 {array} portainer.EdgeJob
 // @failure 500

api/http/handler/edgejobs/edgejob_tasklogs_clear.go

@@ -13,9 +13,10 @@ import (
 
 // @id EdgeJobTasksClear
 // @summary Clear the log for a specifc task on an EdgeJob
-// @description **Access policy**: administrator
+// @description
 // @tags edge_jobs
 // @security jwt
+// @accept json
 // @produce json
 // @param id path string true "EdgeJob Id"
 // @param taskID path string true "Task Id"

api/http/handler/edgejobs/edgejob_tasklogs_collect.go

@@ -12,9 +12,10 @@ import (
 
 // @id EdgeJobTasksCollect
 // @summary Collect the log for a specifc task on an EdgeJob
-// @description **Access policy**: administrator
+// @description
 // @tags edge_jobs
 // @security jwt
+// @accept json
 // @produce json
 // @param id path string true "EdgeJob Id"
 // @param taskID path string true "Task Id"

api/http/handler/edgejobs/edgejob_tasklogs_inspect.go

@@ -15,9 +15,10 @@ type fileResponse struct {
 
 // @id EdgeJobTaskLogsInspect
 // @summary Fetch the log for a specifc task on an EdgeJob
-// @description **Access policy**: administrator
+// @description
 // @tags edge_jobs
 // @security jwt
+// @accept json
 // @produce json
 // @param id path string true "EdgeJob Id"
 // @param taskID path string true "Task Id"

api/http/handler/edgejobs/edgejob_tasks_list.go

@@ -19,9 +19,10 @@ type taskContainer struct {
 
 // @id EdgeJobTasksList
 // @summary Fetch the list of tasks on an EdgeJob
-// @description **Access policy**: administrator
+// @description
 // @tags edge_jobs
 // @security jwt
+// @accept json
 // @produce json
 // @param id path string true "EdgeJob Id"
 // @success 200 {array} taskContainer

api/http/handler/edgejobs/edgejob_update.go

@@ -30,7 +30,7 @@ func (payload *edgeJobUpdatePayload) Validate(r *http.Request) error {
 
 // @id EdgeJobUpdate
 // @summary Update an EdgeJob
-// @description **Access policy**: administrator
+// @description
 // @tags edge_jobs
 // @security jwt
 // @accept json

api/http/handler/edgestacks/edgestack_create.go

@@ -19,9 +19,10 @@ import (
 
 // @id EdgeStackCreate
 // @summary Create an EdgeStack
-// @description **Access policy**: administrator
+// @description
 // @tags edge_stacks
 // @security jwt
+// @accept json
 // @produce json
 // @param method query string true "Creation Method" Enums(file,string,repository)
 // @param body_string body swarmStackFromFileContentPayload true "Required when using method=string"

api/http/handler/edgestacks/edgestack_delete.go

@@ -13,9 +13,11 @@ import (
 
 // @id EdgeStackDelete
 // @summary Delete an EdgeStack
-// @description **Access policy**: administrator
+// @description
 // @tags edge_stacks
 // @security jwt
+// @accept json
+// @produce json
 // @param id path string true "EdgeStack Id"
 // @success 204
 // @failure 500

api/http/handler/edgestacks/edgestack_file.go

@@ -17,9 +17,10 @@ type stackFileResponse struct {
 
 // @id EdgeStackFile
 // @summary Fetches the stack file for an EdgeStack
-// @description **Access policy**: administrator
+// @description
 // @tags edge_stacks
 // @security jwt
+// @accept json
 // @produce json
 // @param id path string true "EdgeStack Id"
 // @success 200 {object} stackFileResponse

api/http/handler/edgestacks/edgestack_inspect.go

@@ -12,9 +12,10 @@ import (
 
 // @id EdgeStackInspect
 // @summary Inspect an EdgeStack
-// @description **Access policy**: administrator
+// @description
 // @tags edge_stacks
 // @security jwt
+// @accept json
 // @produce json
 // @param id path string true "EdgeStack Id"
 // @success 200 {object} portainer.EdgeStack

api/http/handler/edgestacks/edgestack_list.go

@@ -9,9 +9,10 @@ import (
 
 // @id EdgeStackList
 // @summary Fetches the list of EdgeStacks
-// @description **Access policy**: administrator
+// @description
 // @tags edge_stacks
 // @security jwt
+// @accept json
 // @produce json
 // @success 200 {array} portainer.EdgeStack
 // @failure 500

api/http/handler/edgestacks/edgestack_update.go

@@ -33,7 +33,7 @@ func (payload *updateEdgeStackPayload) Validate(r *http.Request) error {
 
 // @id EdgeStackUpdate
 // @summary Update an EdgeStack
-// @description **Access policy**: administrator
+// @description
 // @tags edge_stacks
 // @security jwt
 // @accept json

api/http/handler/edgetemplates/edgetemplate_list.go

@@ -17,7 +17,7 @@ type templateFileFormat struct {
 
 // @id EdgeTemplateList
 // @summary Fetches the list of Edge Templates
-// @description **Access policy**: administrator
+// @description
 // @tags edge_templates
 // @security jwt
 // @accept json

api/http/handler/endpointedge/endpoint_edgejob_logs.go

@@ -21,7 +21,7 @@ func (payload *logsPayload) Validate(r *http.Request) error {
 
 // endpointEdgeJobsLogs
 // @summary Inspect an EdgeJob Log
-// @description **Access policy**: public
+// @description
 // @tags edge, endpoints
 // @accept json
 // @produce json

api/http/handler/endpointedge/endpoint_edgestack_inspect.go

@@ -19,7 +19,7 @@ type configResponse struct {
 }
 
 // @summary Inspect an Edge Stack for an Environment(Endpoint)
-// @description **Access policy**: public
+// @description
 // @tags edge, endpoints, edge_stacks
 // @accept json
 // @produce json

api/http/handler/endpointgroups/endpointgroup_delete.go

@@ -17,6 +17,8 @@ import (
 // @description **Access policy**: administrator
 // @tags endpoint_groups
 // @security jwt
+// @accept json
+// @produce json
 // @param id path int true "EndpointGroup identifier"
 // @success 204 "Success"
 // @failure 400 "Invalid request"

api/http/handler/endpoints/endpoint_association_delete.go

@@ -27,7 +27,7 @@ import (
 // @failure 400 "Invalid request"
 // @failure 404 "Environment(Endpoint) not found"
 // @failure 500 "Server error"
-// @router /endpoints/{id}/association [put]
+// @router /api/endpoints/{id}/association [put]
 func (handler *Handler) endpointAssociationDelete(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
 	if err != nil {

api/http/handler/endpoints/endpoint_dockerhub_status.go

@@ -18,27 +18,11 @@ import (
 )
 
 type dockerhubStatusResponse struct {
-	// Remaiming images to pull
 	Remaining int `json:"remaining"`
-	// Daily limit
-	Limit int `json:"limit"`
+	Limit     int `json:"limit"`
 }
 
-// @id endpointDockerhubStatus
-// @summary fetch docker pull limits
-// @description get docker pull limits for a docker hub registry in the environment
-// @description **Access policy**:
-// @tags endpoints
-// @security jwt
-// @produce json
-// @param id path int true "endpoint ID"
-// @param registryId path int true "registry ID"
-// @success 200 {object} dockerhubStatusResponse "Success"
-// @failure 400 "Invalid request"
-// @failure 403 "Permission denied"
-// @failure 404 "registry or endpoint not found"
-// @failure 500 "Server error"
-// @router /endpoints/{id}/dockerhub/{registryId} [get]
+// GET request on /api/endpoints/{id}/dockerhub/{registryId}
 func (handler *Handler) endpointDockerhubStatus(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
 	if err != nil {

api/http/handler/endpoints/endpoint_extension_add.go

@@ -29,12 +29,6 @@ func (payload *endpointExtensionAddPayload) Validate(r *http.Request) error {
 	return nil
 }
 
-// @id endpointExtensionAdd
-// @tags endpoints
-// @deprecated
-// @param id path int true "Environment(Endpoint) identifier"
-// @success 204 "Success"
-// @router /endpoints/{id}/extensions [post]
 func (handler *Handler) endpointExtensionAdd(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
 	if err != nil {

api/http/handler/endpoints/endpoint_extension_remove.go

@@ -12,13 +12,6 @@ import (
 	"github.com/portainer/portainer/api/bolt/errors"
 )
 
-// @id endpointExtensionRemove
-// @tags endpoints
-// @deprecated
-// @param id path int true "Environment(Endpoint) identifier"
-// @param extensionType path string true "Extension Type"
-// @success 204 "Success"
-// @router /endpoints/{id}/extensions/{extensionType} [delete]
 func (handler *Handler) endpointExtensionRemove(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
 	if err != nil {

api/http/handler/endpoints/endpoint_registries_inspect.go

@@ -12,20 +12,7 @@ import (
 	"github.com/portainer/portainer/api/http/security"
 )
 
-// @id endpointRegistryInspect
-// @summary get registry for environment
-// @description **Access policy**: authenticated
-// @tags endpoints
-// @security jwt
-// @produce json
-// @param id path int true "identifier"
-// @param registryId path int true "Registry identifier"
-// @success 200 {object} portainer.Registry "Success"
-// @failure 400 "Invalid request"
-// @failure 403 "Permission denied"
-// @failure 404 "Registry not found"
-// @failure 500 "Server error"
-// @router /endpoints/{id}/registries/{registryId} [get]
+// GET request on /endpoints/{id}/registries/{registryId}
 func (handler *Handler) endpointRegistryInspect(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
 	if err != nil {

api/http/handler/endpoints/endpoint_registries_list.go

@@ -13,18 +13,7 @@ import (
 	"github.com/portainer/portainer/api/internal/endpointutils"
 )
 
-// @id endpointRegistriesList
-// @summary List Registries on environment
-// @description List all registries based on the current user authorizations in current environment.
-// @description **Access policy**: authenticated
-// @tags endpoints
-// @param namespace query string false "required if kubernetes environment, will show registries by namespace"
-// @security jwt
-// @produce json
-// @param id path int true "Environment(Endpoint) identifier"
-// @success 200 {array} portainer.Registry "Success"
-// @failure 500 "Server error"
-// @router /endpoints/{id}/registries [get]
+// GET request on /endpoints/{id}/registries?namespace
 func (handler *Handler) endpointRegistriesList(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	securityContext, err := security.RetrieveRestrictedRequestContext(r)
 	if err != nil {

api/http/handler/endpoints/endpoint_registry_access.go

@@ -21,22 +21,7 @@ func (payload *registryAccessPayload) Validate(r *http.Request) error {
 	return nil
 }
 
-// @id endpointRegistryAccess
-// @summary update registry access for environment
-// @description **Access policy**: authenticated
-// @tags endpoints
-// @security jwt
-// @accept json
-// @produce json
-// @param id path int true "Environment(Endpoint) identifier"
-// @param registryId path int true "Registry identifier"
-// @param body body registryAccessPayload true "details"
-// @success 204 "Success"
-// @failure 400 "Invalid request"
-// @failure 403 "Permission denied"
-// @failure 404 "Endpoint not found"
-// @failure 500 "Server error"
-// @router /endpoints/{id}/registries/{registryId} [put]
+// PUT request on /endpoints/{id}/registries/{registryId}
 func (handler *Handler) endpointRegistryAccess(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
 	if err != nil {

api/http/handler/endpoints/endpoint_settings_update.go

@@ -36,9 +36,9 @@ func (payload *endpointSettingsUpdatePayload) Validate(r *http.Request) error {
 }
 
 // @id EndpointSettingsUpdate
-// @summary Update settings for an environment(endpoint)
-// @description Update settings for an environment(endpoint).
-// @description **Access policy**: authenticated
+// @summary Update settings for an environments(endpoints)
+// @description Update settings for an environments(endpoints).
+// @description **Access policy**: administrator
 // @security jwt
 // @tags endpoints
 // @accept json
@@ -49,7 +49,7 @@ func (payload *endpointSettingsUpdatePayload) Validate(r *http.Request) error {
 // @failure 400 "Invalid request"
 // @failure 404 "Environment(Endpoint) not found"
 // @failure 500 "Server error"
-// @router /endpoints/{id}/settings [put]
+// @router /api/endpoints/{id}/settings [put]
 func (handler *Handler) endpointSettingsUpdate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
 	if err != nil {

api/http/handler/endpoints/endpoint_snapshot.go

@@ -12,9 +12,9 @@ import (
 )
 
 // @id EndpointSnapshot
-// @summary Snapshots an environment(endpoint)
-// @description Snapshots an environment(endpoint)
-// @description **Access policy**: administrator
+// @summary Snapshots an environments(endpoints)
+// @description Snapshots an environments(endpoints)
+// @description **Access policy**: restricted
 // @tags endpoints
 // @security jwt
 // @param id path int true "Environment(Endpoint) identifier"

api/http/handler/endpoints/endpoint_update.go

@@ -56,7 +56,7 @@ func (payload *endpointUpdatePayload) Validate(r *http.Request) error {
 // @id EndpointUpdate
 // @summary Update an environment(endpoint)
 // @description Update an environment(endpoint).
-// @description **Access policy**: authenticated
+// @description **Access policy**: administrator
 // @security jwt
 // @tags endpoints
 // @accept json

api/http/handler/handler.go

@@ -18,7 +18,6 @@ import (
 	"github.com/portainer/portainer/api/http/handler/file"
 	"github.com/portainer/portainer/api/http/handler/helm"
 	"github.com/portainer/portainer/api/http/handler/kubernetes"
-	"github.com/portainer/portainer/api/http/handler/ldap"
 	"github.com/portainer/portainer/api/http/handler/motd"
 	"github.com/portainer/portainer/api/http/handler/registries"
 	"github.com/portainer/portainer/api/http/handler/resourcecontrols"
@@ -54,7 +53,6 @@ type Handler struct {
 	HelmTemplatesHandler   *helm.Handler
 	KubernetesHandler      *kubernetes.Handler
 	FileHandler            *file.Handler
-	LDAPHandler            *ldap.Handler
 	MOTDHandler            *motd.Handler
 	RegistryHandler        *registries.Handler
 	ResourceControlHandler *resourcecontrols.Handler
@@ -74,7 +72,7 @@ type Handler struct {
 }
 
 // @title PortainerCE API
-// @version 2.9.1
+// @version 2.9.0
 // @description.markdown api-description.md
 // @termsOfService
 
@@ -191,8 +189,6 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		default:
 			http.StripPrefix("/api", h.EndpointHandler).ServeHTTP(w, r)
 		}
-	case strings.HasPrefix(r.URL.Path, "/api/ldap"):
-		http.StripPrefix("/api", h.LDAPHandler).ServeHTTP(w, r)
 	case strings.HasPrefix(r.URL.Path, "/api/motd"):
 		http.StripPrefix("/api", h.MOTDHandler).ServeHTTP(w, r)
 	case strings.HasPrefix(r.URL.Path, "/api/registries"):

api/http/handler/helm/handler.go

@@ -27,17 +27,15 @@ type Handler struct {
 	requestBouncer     requestBouncer
 	dataStore          portainer.DataStore
 	kubeConfigService  kubernetes.KubeConfigService
-	kubernetesDeployer portainer.KubernetesDeployer
 	helmPackageManager libhelm.HelmPackageManager
 }
 
-// NewHandler creates a handler to manage endpoint group operations.
-func NewHandler(bouncer requestBouncer, dataStore portainer.DataStore, kubernetesDeployer portainer.KubernetesDeployer, helmPackageManager libhelm.HelmPackageManager, kubeConfigService kubernetes.KubeConfigService) *Handler {
+// NewHandler creates a handler to manage environment(endpoint) group operations.
+func NewHandler(bouncer requestBouncer, dataStore portainer.DataStore, helmPackageManager libhelm.HelmPackageManager, kubeConfigService kubernetes.KubeConfigService) *Handler {
 	h := &Handler{
 		Router:             mux.NewRouter(),
 		requestBouncer:     bouncer,
 		dataStore:          dataStore,
-		kubernetesDeployer: kubernetesDeployer,
 		helmPackageManager: helmPackageManager,
 		kubeConfigService:  kubeConfigService,
 	}

api/http/handler/helm/helm_delete.go

@@ -12,9 +12,11 @@ import (
 // @id HelmDelete
 // @summary Delete Helm Release
 // @description
-// @description **Access policy**: authenticated
+// @description **Access policy**: authorized
 // @tags helm
 // @security jwt
+// @accept json
+// @produce json
 // @param id path int true "Environment(Endpoint) identifier"
 // @param release path string true "The name of the release/application to uninstall"
 // @param namespace query string true "An optional namespace"

api/http/handler/helm/helm_delete_test.go

@@ -9,12 +9,11 @@ import (
 	"github.com/portainer/libhelm/binary/test"
 	"github.com/portainer/libhelm/options"
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/exec/exectest"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/kubernetes"
 	"github.com/stretchr/testify/assert"
 
-	"github.com/portainer/portainer/api/bolt"
+	bolt "github.com/portainer/portainer/api/bolt/bolttest"
 	helper "github.com/portainer/portainer/api/internal/testhelpers"
 )
 
@@ -30,10 +29,9 @@ func Test_helmDelete(t *testing.T) {
 	err = store.User().CreateUser(&portainer.User{Username: "admin", Role: portainer.AdministratorRole})
 	is.NoError(err, "Error creating a user")
 
-	kubernetesDeployer := exectest.NewKubernetesDeployer()
 	helmPackageManager := test.NewMockHelmBinaryPackageManager("")
 	kubeConfigService := kubernetes.NewKubeConfigCAService("", "")
-	h := NewHandler(helper.NewTestRequestBouncer(), store, kubernetesDeployer, helmPackageManager, kubeConfigService)
+	h := NewHandler(helper.NewTestRequestBouncer(), store, helmPackageManager, kubeConfigService)
 
 	is.NotNil(h, "Handler should not fail")
 

api/http/handler/helm/helm_install.go

@@ -1,23 +1,18 @@
 package helm
 
 import (
+	"errors"
 	"fmt"
-	"io/ioutil"
 	"net/http"
 	"os"
 	"strings"
 
-	"github.com/pkg/errors"
 	"github.com/portainer/libhelm/options"
 	"github.com/portainer/libhelm/release"
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
-	"github.com/portainer/portainer/api/http/middlewares"
-	"github.com/portainer/portainer/api/http/security"
-	"github.com/portainer/portainer/api/kubernetes"
 	"github.com/portainer/portainer/api/kubernetes/validation"
-	"golang.org/x/sync/errgroup"
 )
 
 type installChartPayload struct {
@@ -36,7 +31,7 @@ var errChartNameInvalid = errors.New("invalid chart name. " +
 // @id HelmInstall
 // @summary Install Helm Chart
 // @description
-// @description **Access policy**: authenticated
+// @description **Access policy**: authorized
 // @tags helm
 // @security jwt
 // @accept json
@@ -136,98 +131,5 @@ func (handler *Handler) installChart(r *http.Request, p installChartPayload) (*r
 	if err != nil {
 		return nil, err
 	}
-
-	manifest, err := handler.applyPortainerLabelsToHelmAppManifest(r, installOpts, release.Manifest)
-	if err != nil {
-		return nil, err
-	}
-
-	err = handler.updateHelmAppManifest(r, manifest, installOpts.Namespace)
-	if err != nil {
-		return nil, err
-	}
-
 	return release, nil
 }
-
-// applyPortainerLabelsToHelmAppManifest will patch all the resources deployed in the helm release manifest
-// with portainer specific labels. This is to mark the resources as managed by portainer - hence the helm apps
-// wont appear external in the portainer UI.
-func (handler *Handler) applyPortainerLabelsToHelmAppManifest(r *http.Request, installOpts options.InstallOptions, manifest string) ([]byte, error) {
-	// Patch helm release by adding with portainer labels to all deployed resources
-	tokenData, err := security.RetrieveTokenData(r)
-	if err != nil {
-		return nil, errors.Wrap(err, "unable to retrieve user details from authentication token")
-	}
-	user, err := handler.dataStore.User().User(tokenData.ID)
-	if err != nil {
-		return nil, errors.Wrap(err, "unable to load user information from the database")
-	}
-
-	appLabels := kubernetes.GetHelmAppLabels(installOpts.Name, user.Username)
-	labeledManifest, err := kubernetes.AddAppLabels([]byte(manifest), appLabels)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to label helm release manifest")
-	}
-
-	return labeledManifest, nil
-}
-
-// updateHelmAppManifest will update the resources of helm release manifest with portainer labels using kubectl.
-// The resources of the manifest will be updated in parallel and individuallly since resources of a chart
-// can be deployed to different namespaces.
-// NOTE: These updates will need to be re-applied when upgrading the helm release
-func (handler *Handler) updateHelmAppManifest(r *http.Request, manifest []byte, namespace string) error {
-	endpoint, err := middlewares.FetchEndpoint(r)
-	if err != nil {
-		return errors.Wrap(err, "unable to find an endpoint on request context")
-	}
-
-	tokenData, err := security.RetrieveTokenData(r)
-	if err != nil {
-		return errors.Wrap(err, "unable to retrieve user details from authentication token")
-	}
-
-	// extract list of yaml resources from helm manifest
-	yamlResources, err := kubernetes.ExtractDocuments(manifest, nil)
-	if err != nil {
-		return errors.Wrap(err, "unable to extract documents from helm release manifest")
-	}
-
-	// deploy individual resources in parallel
-	g := new(errgroup.Group)
-	for _, resource := range yamlResources {
-		resource := resource // https://golang.org/doc/faq#closures_and_goroutines
-		g.Go(func() error {
-			tmpfile, err := ioutil.TempFile("", "helm-manifest-*")
-			if err != nil {
-				return errors.Wrap(err, "failed to create a tmp helm manifest file")
-			}
-			defer func() {
-				tmpfile.Close()
-				os.Remove(tmpfile.Name())
-			}()
-
-			if _, err := tmpfile.Write(resource); err != nil {
-				return errors.Wrap(err, "failed to write a tmp helm manifest file")
-			}
-
-			// get resource namespace, fallback to provided namespace if not explicit on resource
-			resourceNamespace, err := kubernetes.GetNamespace(resource)
-			if err != nil {
-				return err
-			}
-			if resourceNamespace == "" {
-				resourceNamespace = namespace
-			}
-
-			_, err = handler.kubernetesDeployer.Deploy(tokenData.ID, endpoint, []string{tmpfile.Name()}, resourceNamespace)
-			return err
-		})
-	}
-	if err := g.Wait(); err != nil {
-		return errors.Wrap(err, "unable to patch helm release using kubectl")
-	}
-
-	return nil
-}

api/http/handler/helm/helm_install_test.go

@@ -12,8 +12,7 @@ import (
 	"github.com/portainer/libhelm/options"
 	"github.com/portainer/libhelm/release"
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/bolt"
-	"github.com/portainer/portainer/api/exec/exectest"
+	bolt "github.com/portainer/portainer/api/bolt/bolttest"
 	"github.com/portainer/portainer/api/http/security"
 	helper "github.com/portainer/portainer/api/internal/testhelpers"
 	"github.com/portainer/portainer/api/kubernetes"
@@ -32,10 +31,9 @@ func Test_helmInstall(t *testing.T) {
 	err = store.User().CreateUser(&portainer.User{Username: "admin", Role: portainer.AdministratorRole})
 	is.NoError(err, "error creating a user")
 
-	kubernetesDeployer := exectest.NewKubernetesDeployer()
 	helmPackageManager := test.NewMockHelmBinaryPackageManager("")
 	kubeConfigService := kubernetes.NewKubeConfigCAService("", "")
-	h := NewHandler(helper.NewTestRequestBouncer(), store, kubernetesDeployer, helmPackageManager, kubeConfigService)
+	h := NewHandler(helper.NewTestRequestBouncer(), store, helmPackageManager, kubeConfigService)
 
 	is.NotNil(h, "Handler should not fail")
 

api/http/handler/helm/helm_list.go

@@ -12,7 +12,7 @@ import (
 // @id HelmList
 // @summary List Helm Releases
 // @description
-// @description **Access policy**: authenticated
+// @description **Access policy**: authorized
 // @tags helm
 // @security jwt
 // @accept json

api/http/handler/helm/helm_list_test.go

@@ -11,11 +11,10 @@ import (
 	"github.com/portainer/libhelm/options"
 	"github.com/portainer/libhelm/release"
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/exec/exectest"
 	"github.com/portainer/portainer/api/kubernetes"
 	"github.com/stretchr/testify/assert"
 
-	"github.com/portainer/portainer/api/bolt"
+	bolt "github.com/portainer/portainer/api/bolt/bolttest"
 	helper "github.com/portainer/portainer/api/internal/testhelpers"
 )
 
@@ -29,10 +28,9 @@ func Test_helmList(t *testing.T) {
 	err = store.User().CreateUser(&portainer.User{Username: "admin", Role: portainer.AdministratorRole})
 	assert.NoError(t, err, "error creating a user")
 
-	kubernetesDeployer := exectest.NewKubernetesDeployer()
 	helmPackageManager := test.NewMockHelmBinaryPackageManager("")
 	kubeConfigService := kubernetes.NewKubeConfigCAService("", "")
-	h := NewHandler(helper.NewTestRequestBouncer(), store, kubernetesDeployer, helmPackageManager, kubeConfigService)
+	h := NewHandler(helper.NewTestRequestBouncer(), store, helmPackageManager, kubeConfigService)
 
 	// Install a single chart.  We expect to get these values back
 	options := options.InstallOptions{Name: "nginx-1", Chart: "nginx", Namespace: "default"}

api/http/handler/helm/helm_repo_search.go

@@ -13,7 +13,7 @@ import (
 // @id HelmRepoSearch
 // @summary Search Helm Charts
 // @description
-// @description **Access policy**: authenticated
+// @description **Access policy**: authorized
 // @tags helm
 // @param repo query string true "Helm repository URL"
 // @security jwt

api/http/handler/helm/helm_show.go

@@ -15,8 +15,8 @@ import (
 // @id HelmShow
 // @summary Show Helm Chart Information
 // @description
-// @description **Access policy**: authenticated
-// @tags helm
+// @description **Access policy**: authorized
+// @tags helm_chart
 // @param repo query string true "Helm repository URL"
 // @param chart query string true "Chart name"
 // @param command path string true "chart/values/readme"

api/http/handler/kubernetes/kubernetes_config.go

@@ -17,7 +17,7 @@ import (
 // @id GetKubernetesConfig
 // @summary Generates kubeconfig file enabling client communication with k8s api server
 // @description Generates kubeconfig file enabling client communication with k8s api server
-// @description **Access policy**: authenticated
+// @description **Access policy**: authorized
 // @tags kubernetes
 // @security jwt
 // @accept json

api/http/handler/kubernetes/kubernetes_nodes_limits.go

@@ -13,7 +13,7 @@ import (
 // @id getKubernetesNodesLimits
 // @summary Get CPU and memory limits of all nodes within k8s cluster
 // @description Get CPU and memory limits of all nodes within k8s cluster
-// @description **Access policy**: authenticated
+// @description **Access policy**: authorized
 // @tags kubernetes
 // @security jwt
 // @accept json

api/http/handler/ldap/handler.go

@@ -1,53 +0,0 @@
-package ldap
-
-import (
-	"net/http"
-
-	"github.com/gorilla/mux"
-	httperror "github.com/portainer/libhttp/error"
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/filesystem"
-	"github.com/portainer/portainer/api/http/security"
-)
-
-// Handler is the HTTP handler used to handle LDAP search Operations
-type Handler struct {
-	*mux.Router
-	DataStore   portainer.DataStore
-	FileService portainer.FileService
-	LDAPService portainer.LDAPService
-}
-
-// NewHandler returns a new Handler
-func NewHandler(bouncer *security.RequestBouncer) *Handler {
-	h := &Handler{
-		Router: mux.NewRouter(),
-	}
-
-	h.Handle("/ldap/check",
-		bouncer.AdminAccess(httperror.LoggerHandler(h.ldapCheck))).Methods(http.MethodPost)
-
-	return h
-}
-
-func (handler *Handler) prefillSettings(ldapSettings *portainer.LDAPSettings) error {
-	if !ldapSettings.AnonymousMode && ldapSettings.Password == "" {
-		settings, err := handler.DataStore.Settings().Settings()
-		if err != nil {
-			return err
-		}
-
-		ldapSettings.Password = settings.LDAPSettings.Password
-	}
-
-	if (ldapSettings.TLSConfig.TLS || ldapSettings.StartTLS) && !ldapSettings.TLSConfig.TLSSkipVerify {
-		caCertPath, err := handler.FileService.GetPathForTLSFile(filesystem.LDAPStorePath, portainer.TLSFileCA)
-		if err != nil {
-			return err
-		}
-
-		ldapSettings.TLSConfig.TLSCACertPath = caCertPath
-	}
-
-	return nil
-}

api/http/handler/motd/motd.go

@@ -26,9 +26,7 @@ type motdData struct {
 	Style         string            `json:"style"`
 }
 
-// @id MOTD
 // @summary fetches the message of the day
-// @description **Access policy**: restricted
 // @tags motd
 // @security jwt
 // @produce json

api/http/handler/registries/registry_configure.go

@@ -81,7 +81,7 @@ func (payload *registryConfigurePayload) Validate(r *http.Request) error {
 // @id RegistryConfigure
 // @summary Configures a registry
 // @description Configures a registry.
-// @description **Access policy**: restricted
+// @description **Access policy**: admin
 // @tags registries
 // @security jwt
 // @accept json

api/http/handler/registries/registry_create.go

@@ -62,7 +62,7 @@ func (payload *registryCreatePayload) Validate(_ *http.Request) error {
 // @id RegistryCreate
 // @summary Create a new registry
 // @description Create a new registry.
-// @description **Access policy**: restricted
+// @description **Access policy**: administrator
 // @tags registries
 // @security jwt
 // @accept json

api/http/handler/registries/registry_delete.go

@@ -15,7 +15,7 @@ import (
 // @id RegistryDelete
 // @summary Remove a registry
 // @description Remove a registry
-// @description **Access policy**: restricted
+// @description **Access policy**: administrator
 // @tags registries
 // @security jwt
 // @param id path int true "Registry identifier"

api/http/handler/registries/registry_inspect.go

@@ -14,7 +14,7 @@ import (
 // @id RegistryInspect
 // @summary Inspect a registry
 // @description Retrieve details about a registry.
-// @description **Access policy**: restricted
+// @description **Access policy**: administrator
 // @tags registries
 // @security jwt
 // @produce json

api/http/handler/registries/registry_update.go

@@ -37,7 +37,7 @@ func (payload *registryUpdatePayload) Validate(r *http.Request) error {
 // @id RegistryUpdate
 // @summary Update a registry
 // @description Update a registry
-// @description **Access policy**: restricted
+// @description **Access policy**: administrator
 // @tags registries
 // @security jwt
 // @accept json

api/http/handler/resourcecontrols/resourcecontrol_update.go

@@ -38,7 +38,7 @@ func (payload *resourceControlUpdatePayload) Validate(r *http.Request) error {
 // @id ResourceControlUpdate
 // @summary Update a resource control
 // @description Update a resource control
-// @description **Access policy**: authenticated
+// @description **Access policy**: restricted
 // @tags resource_controls
 // @security jwt
 // @accept json

api/http/handler/settings/handler.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/gorilla/mux"
 	httperror "github.com/portainer/libhttp/error"
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/http/security"
 )
 
@@ -35,6 +35,8 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler {
 		bouncer.AdminAccess(httperror.LoggerHandler(h.settingsUpdate))).Methods(http.MethodPut)
 	h.Handle("/settings/public",
 		bouncer.PublicAccess(httperror.LoggerHandler(h.settingsPublic))).Methods(http.MethodGet)
+	h.Handle("/settings/authentication/checkLDAP",
+		bouncer.AdminAccess(httperror.LoggerHandler(h.settingsLDAPCheck))).Methods(http.MethodPut)
 
 	return h
 }

api/http/handler/settings/settings_ldap_check.go

@@ -1,4 +1,4 @@
-package ldap
+package settings
 
 import (
 	"net/http"
@@ -7,43 +7,42 @@ import (
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/filesystem"
 )
 
-type checkPayload struct {
+type settingsLDAPCheckPayload struct {
 	LDAPSettings portainer.LDAPSettings
 }
 
-func (payload *checkPayload) Validate(r *http.Request) error {
+func (payload *settingsLDAPCheckPayload) Validate(r *http.Request) error {
 	return nil
 }
 
-// @id LDAPCheck
+// @id SettingsLDAPCheck
 // @summary Test LDAP connectivity
 // @description Test LDAP connectivity using LDAP details
 // @description **Access policy**: administrator
-// @tags ldap
+// @tags settings
 // @security jwt
 // @accept json
-// @param body body checkPayload true "details"
+// @param body body settingsLDAPCheckPayload true "details"
 // @success 204 "Success"
 // @failure 400 "Invalid request"
 // @failure 500 "Server error"
-// @router /ldap/check [post]
-func (handler *Handler) ldapCheck(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	var payload checkPayload
+// @router /settings/ldap/check [put]
+func (handler *Handler) settingsLDAPCheck(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	var payload settingsLDAPCheckPayload
 	err := request.DecodeAndValidateJSONPayload(r, &payload)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
 	}
 
-	settings := &payload.LDAPSettings
-
-	err = handler.prefillSettings(settings)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to fetch default settings", err}
+	if (payload.LDAPSettings.TLSConfig.TLS || payload.LDAPSettings.StartTLS) && !payload.LDAPSettings.TLSConfig.TLSSkipVerify {
+		caCertPath, _ := handler.FileService.GetPathForTLSFile(filesystem.LDAPStorePath, portainer.TLSFileCA)
+		payload.LDAPSettings.TLSConfig.TLSCACertPath = caCertPath
 	}
 
-	err = handler.LDAPService.TestConnectivity(settings)
+	err = handler.LDAPService.TestConnectivity(&payload.LDAPSettings)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to connect to LDAP server", err}
 	}

api/http/handler/settings/settings_update.go

@@ -40,8 +40,6 @@ type settingsUpdatePayload struct {
 	EnableTelemetry *bool `example:"false"`
 	// Helm repository URL
 	HelmRepositoryURL *string `example:"https://charts.bitnami.com/bitnami"`
-	// Kubectl Shell Image
-	KubectlShellImage *string `example:"portainer/kubectl-shell:latest"`
 }
 
 func (payload *settingsUpdatePayload) Validate(r *http.Request) error {
@@ -180,10 +178,6 @@ func (handler *Handler) settingsUpdate(w http.ResponseWriter, r *http.Request) *
 		return tlsError
 	}
 
-	if payload.KubectlShellImage != nil {
-		settings.KubectlShellImage = *payload.KubectlShellImage
-	}
-
 	err = handler.DataStore.Settings().UpdateSettings(settings)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist settings changes inside the database", err}

api/http/handler/stacks/autoupdate.go

@@ -17,8 +17,10 @@ func startAutoupdate(stackID portainer.StackID, interval string, scheduler *sche
 		return "", &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Unable to parse stack's auto update interval", Err: err}
 	}
 
-	jobID = scheduler.StartJobEvery(d, func() error {
-		return stacks.RedeployWhenChanged(stackID, stackDeployer, datastore, gitService)
+	jobID = scheduler.StartJobEvery(d, func() {
+		if err := stacks.RedeployWhenChanged(stackID, stackDeployer, datastore, gitService); err != nil {
+			log.Printf("[ERROR] [http,stacks] [message: failed redeploying] [err: %s]\n", err)
+		}
 	})
 
 	return jobID, nil

api/http/handler/stacks/create_compose_stack.go

@@ -46,12 +46,13 @@ func (handler *Handler) createComposeStackFromFileContent(w http.ResponseWriter,
 
 	payload.Name = handler.ComposeStackManager.NormalizeStackName(payload.Name)
 
-	isUnique, err := handler.checkUniqueStackNameInDocker(endpoint, payload.Name, 0, false)
+	isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, false)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to check for name collision", err}
 	}
 	if !isUnique {
-		return stackExistsError(payload.Name)
+		errorMessage := fmt.Sprintf("A stack with the name '%s' is already running", payload.Name)
+		return &httperror.HandlerError{http.StatusConflict, errorMessage, errors.New(errorMessage)}
 	}
 
 	stackID := handler.DataStore.Stack().GetNextIdentifier()
@@ -151,12 +152,12 @@ func (handler *Handler) createComposeStackFromGitRepository(w http.ResponseWrite
 		payload.ComposeFile = filesystem.ComposeFileDefaultName
 	}
 
-	isUnique, err := handler.checkUniqueStackNameInDocker(endpoint, payload.Name, 0, false)
+	isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, false)
 	if err != nil {
 		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err}
 	}
 	if !isUnique {
-		return stackExistsError(payload.Name)
+		return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A stack with the name '%s' already exists", payload.Name), Err: errStackAlreadyExists}
 	}
 
 	//make sure the webhook ID is unique
@@ -207,11 +208,11 @@ func (handler *Handler) createComposeStackFromGitRepository(w http.ResponseWrite
 		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to clone git repository", Err: err}
 	}
 
-	commitID, err := handler.latestCommitID(payload.RepositoryURL, payload.RepositoryReferenceName, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword)
+	commitId, err := handler.latestCommitID(payload.RepositoryURL, payload.RepositoryReferenceName, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword)
 	if err != nil {
 		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to fetch git repository id", Err: err}
 	}
-	stack.GitConfig.ConfigHash = commitID
+	stack.GitConfig.ConfigHash = commitId
 
 	config, configErr := handler.createComposeDeployConfig(r, stack, endpoint)
 	if configErr != nil {
@@ -280,12 +281,13 @@ func (handler *Handler) createComposeStackFromFileUpload(w http.ResponseWriter,
 
 	payload.Name = handler.ComposeStackManager.NormalizeStackName(payload.Name)
 
-	isUnique, err := handler.checkUniqueStackNameInDocker(endpoint, payload.Name, 0, false)
+	isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, false)
 	if err != nil {
 		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err}
 	}
 	if !isUnique {
-		return stackExistsError(payload.Name)
+		errorMessage := fmt.Sprintf("A stack with the name '%s' already exists", payload.Name)
+		return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: errorMessage, Err: errors.New(errorMessage)}
 	}
 
 	stackID := handler.DataStore.Stack().GetNextIdentifier()

api/http/handler/stacks/create_kubernetes_stack.go

@@ -2,8 +2,9 @@ package stacks
 
 import (
 	"fmt"
+	"io/ioutil"
 	"net/http"
-	"os"
+	"path/filepath"
 	"strconv"
 	"time"
 
@@ -18,19 +19,16 @@ import (
 	"github.com/portainer/portainer/api/filesystem"
 	gittypes "github.com/portainer/portainer/api/git/types"
 	"github.com/portainer/portainer/api/http/client"
-	"github.com/portainer/portainer/api/internal/stackutils"
 	k "github.com/portainer/portainer/api/kubernetes"
 )
 
 type kubernetesStringDeploymentPayload struct {
-	StackName        string
 	ComposeFormat    bool
 	Namespace        string
 	StackFileContent string
 }
 
 type kubernetesGitDeploymentPayload struct {
-	StackName                string
 	ComposeFormat            bool
 	Namespace                string
 	RepositoryURL            string
@@ -38,13 +36,10 @@ type kubernetesGitDeploymentPayload struct {
 	RepositoryAuthentication bool
 	RepositoryUsername       string
 	RepositoryPassword       string
-	ManifestFile             string
-	AdditionalFiles          []string
-	AutoUpdate               *portainer.StackAutoUpdate
+	FilePathInRepository     string
 }
 
 type kubernetesManifestURLDeploymentPayload struct {
-	StackName     string
 	Namespace     string
 	ComposeFormat bool
 	ManifestURL   string
@@ -57,9 +52,6 @@ func (payload *kubernetesStringDeploymentPayload) Validate(r *http.Request) erro
 	if govalidator.IsNull(payload.Namespace) {
 		return errors.New("Invalid namespace")
 	}
-	if govalidator.IsNull(payload.StackName) {
-		return errors.New("Invalid stack name")
-	}
 	return nil
 }
 
@@ -73,18 +65,12 @@ func (payload *kubernetesGitDeploymentPayload) Validate(r *http.Request) error {
 	if payload.RepositoryAuthentication && govalidator.IsNull(payload.RepositoryPassword) {
 		return errors.New("Invalid repository credentials. Password must be specified when authentication is enabled")
 	}
-	if govalidator.IsNull(payload.ManifestFile) {
-		return errors.New("Invalid manifest file in repository")
+	if govalidator.IsNull(payload.FilePathInRepository) {
+		return errors.New("Invalid file path in repository")
 	}
 	if govalidator.IsNull(payload.RepositoryReferenceName) {
 		payload.RepositoryReferenceName = defaultGitReferenceName
 	}
-	if err := validateStackAutoUpdate(payload.AutoUpdate); err != nil {
-		return err
-	}
-	if govalidator.IsNull(payload.StackName) {
-		return errors.New("Invalid stack name")
-	}
 	return nil
 }
 
@@ -92,9 +78,6 @@ func (payload *kubernetesManifestURLDeploymentPayload) Validate(r *http.Request)
 	if govalidator.IsNull(payload.ManifestURL) || !govalidator.IsURL(payload.ManifestURL) {
 		return errors.New("Invalid manifest URL")
 	}
-	if govalidator.IsNull(payload.StackName) {
-		return errors.New("Invalid stack name")
-	}
 	return nil
 }
 
@@ -112,13 +95,6 @@ func (handler *Handler) createKubernetesStackFromFileContent(w http.ResponseWrit
 	if err != nil {
 		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to load user information from the database", Err: err}
 	}
-	isUnique, err := handler.checkUniqueStackName(endpoint, payload.StackName, 0)
-	if err != nil {
-		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err}
-	}
-	if !isUnique {
-		return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A stack with the name '%s' already exists", payload.StackName), Err: errStackAlreadyExists}
-	}
 
 	stackID := handler.DataStore.Stack().GetNextIdentifier()
 	stack := &portainer.Stack{
@@ -126,7 +102,6 @@ func (handler *Handler) createKubernetesStackFromFileContent(w http.ResponseWrit
 		Type:            portainer.KubernetesStack,
 		EndpointID:      endpoint.ID,
 		EntryPoint:      filesystem.ManifestFileDefaultName,
-		Name:            payload.StackName,
 		Namespace:       payload.Namespace,
 		Status:          portainer.StackStatusActive,
 		CreationDate:    time.Now().Unix(),
@@ -149,11 +124,11 @@ func (handler *Handler) createKubernetesStackFromFileContent(w http.ResponseWrit
 	doCleanUp := true
 	defer handler.cleanUp(stack, &doCleanUp)
 
-	output, err := handler.deployKubernetesStack(user.ID, endpoint, stack, k.KubeAppLabels{
-		StackID:   stackID,
-		StackName: stack.Name,
-		Owner:     stack.CreatedBy,
-		Kind:      "content",
+	output, err := handler.deployKubernetesStack(r, endpoint, payload.StackFileContent, payload.ComposeFormat, payload.Namespace, k.KubeAppLabels{
+		StackID: stackID,
+		Name:    stack.Name,
+		Owner:   stack.CreatedBy,
+		Kind:    "content",
 	})
 
 	if err != nil {
@@ -165,11 +140,12 @@ func (handler *Handler) createKubernetesStackFromFileContent(w http.ResponseWrit
 		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist the Kubernetes stack inside the database", Err: err}
 	}
 
+	doCleanUp = false
+
 	resp := &createKubernetesStackResponse{
 		Output: output,
 	}
 
-	doCleanUp = false
 	return response.JSON(w, resp)
 }
 
@@ -183,44 +159,23 @@ func (handler *Handler) createKubernetesStackFromGitRepository(w http.ResponseWr
 	if err != nil {
 		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to load user information from the database", Err: err}
 	}
-	isUnique, err := handler.checkUniqueStackName(endpoint, payload.StackName, 0)
-	if err != nil {
-		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err}
-	}
-	if !isUnique {
-		return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A stack with the name '%s' already exists", payload.StackName), Err: errStackAlreadyExists}
-	}
-
-	//make sure the webhook ID is unique
-	if payload.AutoUpdate != nil && payload.AutoUpdate.Webhook != "" {
-		isUnique, err := handler.checkUniqueWebhookID(payload.AutoUpdate.Webhook)
-		if err != nil {
-			return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for webhook ID collision", Err: err}
-		}
-		if !isUnique {
-			return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("Webhook ID: %s already exists", payload.AutoUpdate.Webhook), Err: errWebhookIDAlreadyExists}
-		}
-	}
 
 	stackID := handler.DataStore.Stack().GetNextIdentifier()
 	stack := &portainer.Stack{
 		ID:         portainer.StackID(stackID),
 		Type:       portainer.KubernetesStack,
 		EndpointID: endpoint.ID,
-		EntryPoint: payload.ManifestFile,
+		EntryPoint: payload.FilePathInRepository,
 		GitConfig: &gittypes.RepoConfig{
 			URL:            payload.RepositoryURL,
 			ReferenceName:  payload.RepositoryReferenceName,
-			ConfigFilePath: payload.ManifestFile,
+			ConfigFilePath: payload.FilePathInRepository,
 		},
 		Namespace:       payload.Namespace,
-		Name:            payload.StackName,
 		Status:          portainer.StackStatusActive,
 		CreationDate:    time.Now().Unix(),
 		CreatedBy:       user.Username,
 		IsComposeFormat: payload.ComposeFormat,
-		AutoUpdate:      payload.AutoUpdate,
-		AdditionalFiles: payload.AdditionalFiles,
 	}
 
 	if payload.RepositoryAuthentication {
@@ -242,48 +197,33 @@ func (handler *Handler) createKubernetesStackFromGitRepository(w http.ResponseWr
 	}
 	stack.GitConfig.ConfigHash = commitID
 
-	repositoryUsername := payload.RepositoryUsername
-	repositoryPassword := payload.RepositoryPassword
-	if !payload.RepositoryAuthentication {
-		repositoryUsername = ""
-		repositoryPassword = ""
-	}
-
-	err = handler.GitService.CloneRepository(projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, repositoryUsername, repositoryPassword)
+	stackFileContent, err := handler.cloneManifestContentFromGitRepo(&payload, stack.ProjectPath)
 	if err != nil {
-		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Failed to clone git repository", Err: err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Failed to process manifest from Git repository", Err: err}
 	}
 
-	output, err := handler.deployKubernetesStack(user.ID, endpoint, stack, k.KubeAppLabels{
-		StackID:   stackID,
-		StackName: stack.Name,
-		Owner:     stack.CreatedBy,
-		Kind:      "git",
+	output, err := handler.deployKubernetesStack(r, endpoint, stackFileContent, payload.ComposeFormat, payload.Namespace, k.KubeAppLabels{
+		StackID: stackID,
+		Name:    stack.Name,
+		Owner:   stack.CreatedBy,
+		Kind:    "git",
 	})
 
 	if err != nil {
 		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to deploy Kubernetes stack", Err: err}
 	}
 
-	if payload.AutoUpdate != nil && payload.AutoUpdate.Interval != "" {
-		jobID, e := startAutoupdate(stack.ID, stack.AutoUpdate.Interval, handler.Scheduler, handler.StackDeployer, handler.DataStore, handler.GitService)
-		if e != nil {
-			return e
-		}
-
-		stack.AutoUpdate.JobID = jobID
-	}
-
 	err = handler.DataStore.Stack().CreateStack(stack)
 	if err != nil {
 		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist the stack inside the database", Err: err}
 	}
 
+	doCleanUp = false
+
 	resp := &createKubernetesStackResponse{
 		Output: output,
 	}
 
-	doCleanUp = false
 	return response.JSON(w, resp)
 }
 
@@ -297,13 +237,6 @@ func (handler *Handler) createKubernetesStackFromManifestURL(w http.ResponseWrit
 	if err != nil {
 		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to load user information from the database", Err: err}
 	}
-	isUnique, err := handler.checkUniqueStackName(endpoint, payload.StackName, 0)
-	if err != nil {
-		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err}
-	}
-	if !isUnique {
-		return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A stack with the name '%s' already exists", payload.StackName), Err: errStackAlreadyExists}
-	}
 
 	stackID := handler.DataStore.Stack().GetNextIdentifier()
 	stack := &portainer.Stack{
@@ -312,7 +245,6 @@ func (handler *Handler) createKubernetesStackFromManifestURL(w http.ResponseWrit
 		EndpointID:      endpoint.ID,
 		EntryPoint:      filesystem.ManifestFileDefaultName,
 		Namespace:       payload.Namespace,
-		Name:            payload.StackName,
 		Status:          portainer.StackStatusActive,
 		CreationDate:    time.Now().Unix(),
 		CreatedBy:       user.Username,
@@ -335,11 +267,11 @@ func (handler *Handler) createKubernetesStackFromManifestURL(w http.ResponseWrit
 	doCleanUp := true
 	defer handler.cleanUp(stack, &doCleanUp)
 
-	output, err := handler.deployKubernetesStack(user.ID, endpoint, stack, k.KubeAppLabels{
-		StackID:   stackID,
-		StackName: stack.Name,
-		Owner:     stack.CreatedBy,
-		Kind:      "url",
+	output, err := handler.deployKubernetesStack(r, endpoint, string(manifestContent), payload.ComposeFormat, payload.Namespace, k.KubeAppLabels{
+		StackID: stackID,
+		Name:    stack.Name,
+		Owner:   stack.CreatedBy,
+		Kind:    "url",
 	})
 	if err != nil {
 		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to deploy Kubernetes stack", Err: err}
@@ -359,14 +291,42 @@ func (handler *Handler) createKubernetesStackFromManifestURL(w http.ResponseWrit
 	return response.JSON(w, resp)
 }
 
-func (handler *Handler) deployKubernetesStack(userID portainer.UserID, endpoint *portainer.Endpoint, stack *portainer.Stack, appLabels k.KubeAppLabels) (string, error) {
+func (handler *Handler) deployKubernetesStack(request *http.Request, endpoint *portainer.Endpoint, stackConfig string, composeFormat bool, namespace string, appLabels k.KubeAppLabels) (string, error) {
 	handler.stackCreationMutex.Lock()
 	defer handler.stackCreationMutex.Unlock()
 
-	manifestFilePaths, tempDir, err := stackutils.CreateTempK8SDeploymentFiles(stack, handler.KubernetesDeployer, appLabels)
-	if err != nil {
-		return "", errors.Wrap(err, "failed to create temp kub deployment files")
+	manifest := []byte(stackConfig)
+	if composeFormat {
+		convertedConfig, err := handler.KubernetesDeployer.ConvertCompose(manifest)
+		if err != nil {
+			return "", errors.Wrap(err, "failed to convert docker compose file to a kube manifest")
+		}
+		manifest = convertedConfig
 	}
-	defer os.RemoveAll(tempDir)
-	return handler.KubernetesDeployer.Deploy(userID, endpoint, manifestFilePaths, stack.Namespace)
+
+	manifest, err := k.AddAppLabels(manifest, appLabels)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to add application labels")
+	}
+
+	return handler.KubernetesDeployer.Deploy(request, endpoint, string(manifest), namespace)
+}
+
+func (handler *Handler) cloneManifestContentFromGitRepo(gitInfo *kubernetesGitDeploymentPayload, projectPath string) (string, error) {
+	repositoryUsername := gitInfo.RepositoryUsername
+	repositoryPassword := gitInfo.RepositoryPassword
+	if !gitInfo.RepositoryAuthentication {
+		repositoryUsername = ""
+		repositoryPassword = ""
+	}
+
+	err := handler.GitService.CloneRepository(projectPath, gitInfo.RepositoryURL, gitInfo.RepositoryReferenceName, repositoryUsername, repositoryPassword)
+	if err != nil {
+		return "", err
+	}
+	content, err := ioutil.ReadFile(filepath.Join(projectPath, gitInfo.FilePathInRepository))
+	if err != nil {
+		return "", err
+	}
+	return string(content), nil
 }