fix(dockerhub-migration): prevent duplicate migrated dockerhub entries EE-2042 (#6084)

* add missing changes to make updateDockerhubToDB32 idempotent

* fix(migration) make dockerhub registry migration idempotent EE-2042

* add tests for bad migrations

.github/ISSUE_TEMPLATE/Bug_report.md

@@ -30,7 +30,7 @@ A clear and concise description of what you expected to happen.
 
 **Portainer Logs**
 Provide the logs of your Portainer container or Service.
-You can see how [here](https://documentation.portainer.io/archive/1.23.2/faq/#how-do-i-get-the-logs-from-portainer)
+You can see how [here](https://documentation.portainer.io/r/portainer-logs)
 
 **Steps to reproduce the issue:**
 

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Portainer Business
+    url: https://www.portainer.io/portainerbusiness 
+    about: Would you and your co-workers benefit from our enterprise edition which provides functionality to deploy Portainer at scale? 

README.md

@@ -44,7 +44,7 @@ Portainer CE is an open source project and is supported by the community. You ca
 Learn more about Portainers community support channels [here.](https://www.portainer.io/help_about)
 
 - Issues: https://github.com/portainer/portainer/issues
-- Slack (chat): https://portainer.io/slack/
+- Slack (chat): [https://portainer.slack.com/](https://join.slack.com/t/portainer/shared_invite/zt-txh3ljab-52QHTyjCqbe5RibC2lcjKA)
 
 You can join the Portainer Community by visiting community.portainer.io. This will give you advance notice of events, content and other related Portainer content.
 
@@ -59,7 +59,7 @@ You can join the Portainer Community by visiting community.portainer.io. This wi
 
 ## WORK FOR US
 
-If you are a developer, and our code in this repo makes sense to you, we would love to hear from you. We are always on the hunt for awesome devs, either freelance or employed. Drop us a line to info@portainer.io with your details and we will be in touch. 
+If you are a developer, and our code in this repo makes sense to you, we would love to hear from you. We are always on the hunt for awesome devs, either freelance or employed. Drop us a line to info@portainer.io with your details and we will be in touch.
 
 ## Privacy
 

api/bolt/backup.go

@@ -0,0 +1,142 @@
+package bolt
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"time"
+
+	plog "github.com/portainer/portainer/api/bolt/log"
+)
+
+var backupDefaults = struct {
+	backupDir        string
+	commonDir        string
+	databaseFileName string
+}{
+	"backups",
+	"common",
+	databaseFileName,
+}
+
+var backupLog = plog.NewScopedLog("bolt, backup")
+
+//
+// Backup Helpers
+//
+
+// createBackupFolders create initial folders for backups
+func (store *Store) createBackupFolders() {
+	// create common dir
+	commonDir := store.commonBackupDir()
+	if exists, _ := store.fileService.FileExists(commonDir); !exists {
+		if err := os.MkdirAll(commonDir, 0700); err != nil {
+			backupLog.Error("Error while creating common backup folder", err)
+		}
+	}
+}
+
+func (store *Store) databasePath() string {
+	return path.Join(store.path, databaseFileName)
+}
+
+func (store *Store) commonBackupDir() string {
+	return path.Join(store.path, backupDefaults.backupDir, backupDefaults.commonDir)
+}
+
+func (store *Store) copyDBFile(from string, to string) error {
+	backupLog.Info(fmt.Sprintf("Copying db file from %s to %s", from, to))
+	err := store.fileService.Copy(from, to, true)
+	if err != nil {
+		backupLog.Error("Failed", err)
+	}
+	return err
+}
+
+// BackupOptions provide a helper to inject backup options
+type BackupOptions struct {
+	Version        int
+	BackupDir      string
+	BackupFileName string
+	BackupPath     string
+}
+
+func (store *Store) setupOptions(options *BackupOptions) *BackupOptions {
+	if options == nil {
+		options = &BackupOptions{}
+	}
+	if options.Version == 0 {
+		options.Version, _ = store.version()
+	}
+	if options.BackupDir == "" {
+		options.BackupDir = store.commonBackupDir()
+	}
+	if options.BackupFileName == "" {
+		options.BackupFileName = fmt.Sprintf("%s.%s.%s", backupDefaults.databaseFileName, fmt.Sprintf("%03d", options.Version), time.Now().Format("20060102150405"))
+	}
+	if options.BackupPath == "" {
+		options.BackupPath = path.Join(options.BackupDir, options.BackupFileName)
+	}
+	return options
+}
+
+// BackupWithOptions backup current database with options
+func (store *Store) BackupWithOptions(options *BackupOptions) (string, error) {
+	backupLog.Info("creating db backup")
+	store.createBackupFolders()
+
+	options = store.setupOptions(options)
+
+	return options.BackupPath, store.copyDBFile(store.databasePath(), options.BackupPath)
+}
+
+// RestoreWithOptions previously saved backup for the current Edition  with options
+// Restore strategies:
+// - default: restore latest from current edition
+// - restore a specific
+func (store *Store) RestoreWithOptions(options *BackupOptions) error {
+	options = store.setupOptions(options)
+
+	// Check if backup file exist before restoring
+	_, err := os.Stat(options.BackupPath)
+	if os.IsNotExist(err) {
+		backupLog.Error(fmt.Sprintf("Backup file to restore does not exist %s", options.BackupPath), err)
+		return err
+	}
+
+	err = store.Close()
+	if err != nil {
+		backupLog.Error("Error while closing store before restore", err)
+		return err
+	}
+
+	backupLog.Info("Restoring db backup")
+	err = store.copyDBFile(options.BackupPath, store.databasePath())
+	if err != nil {
+		return err
+	}
+
+	return store.Open()
+}
+
+// RemoveWithOptions removes backup database based on supplied options
+func (store *Store) RemoveWithOptions(options *BackupOptions) error {
+	backupLog.Info("Removing db backup")
+
+	options = store.setupOptions(options)
+	_, err := os.Stat(options.BackupPath)
+
+	if os.IsNotExist(err) {
+		backupLog.Error(fmt.Sprintf("Backup file to remove does not exist %s", options.BackupPath), err)
+		return err
+	}
+
+	backupLog.Info(fmt.Sprintf("Removing db file at %s", options.BackupPath))
+	err = os.Remove(options.BackupPath)
+	if err != nil {
+		backupLog.Error("Failed", err)
+		return err
+	}
+
+	return nil
+}

api/bolt/backup_test.go

@@ -0,0 +1,116 @@
+package bolt
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+)
+
+// isFileExist is helper function to check for file existence
+func isFileExist(path string) bool {
+	matches, err := filepath.Glob(path)
+	if err != nil {
+		return false
+	}
+	return len(matches) > 0
+}
+
+func TestCreateBackupFolders(t *testing.T) {
+	store, teardown := MustNewTestStore(false)
+	defer teardown()
+
+	backupPath := path.Join(store.path, backupDefaults.backupDir)
+
+	if isFileExist(backupPath) {
+		t.Error("Expect backups folder to not exist")
+	}
+
+	store.createBackupFolders()
+	if !isFileExist(backupPath) {
+		t.Error("Expect backups folder to exist")
+	}
+}
+
+func TestStoreCreation(t *testing.T) {
+	store, teardown := MustNewTestStore(true)
+	defer teardown()
+
+	if store == nil {
+		t.Error("Expect to create a store")
+	}
+
+	if store.edition() != portainer.PortainerCE {
+		t.Error("Expect to get CE Edition")
+	}
+}
+
+func TestBackup(t *testing.T) {
+	store, teardown := MustNewTestStore(true)
+	defer teardown()
+
+	t.Run("Backup should create default db backup", func(t *testing.T) {
+		store.VersionService.StoreDBVersion(portainer.DBVersion)
+		store.BackupWithOptions(nil)
+
+		backupFileName := path.Join(store.path, "backups", "common", fmt.Sprintf("portainer.db.%03d.*", portainer.DBVersion))
+		if !isFileExist(backupFileName) {
+			t.Errorf("Expect backup file to be created %s", backupFileName)
+		}
+	})
+
+	t.Run("BackupWithOption should create a name specific backup at common path", func(t *testing.T) {
+		store.BackupWithOptions(&BackupOptions{
+			BackupFileName: beforePortainerVersionUpgradeBackup,
+			BackupDir:      store.commonBackupDir(),
+		})
+		backupFileName := path.Join(store.path, "backups", "common", beforePortainerVersionUpgradeBackup)
+		if !isFileExist(backupFileName) {
+			t.Errorf("Expect backup file to be created %s", backupFileName)
+		}
+	})
+}
+
+func TestRemoveWithOptions(t *testing.T) {
+	store, teardown := MustNewTestStore(true)
+	defer teardown()
+
+	t.Run("successfully removes file if existent", func(t *testing.T) {
+		store.createBackupFolders()
+		options := &BackupOptions{
+			BackupDir:      store.commonBackupDir(),
+			BackupFileName: "test.txt",
+		}
+
+		filePath := path.Join(options.BackupDir, options.BackupFileName)
+		f, err := os.Create(filePath)
+		if err != nil {
+			t.Fatalf("file should be created; err=%s", err)
+		}
+		f.Close()
+
+		err = store.RemoveWithOptions(options)
+		if err != nil {
+			t.Errorf("RemoveWithOptions should successfully remove file; err=%w", err)
+		}
+
+		if isFileExist(f.Name()) {
+			t.Errorf("RemoveWithOptions should successfully remove file; file=%s", f.Name())
+		}
+	})
+
+	t.Run("fails to removes file if non-existent", func(t *testing.T) {
+		options := &BackupOptions{
+			BackupDir:      store.commonBackupDir(),
+			BackupFileName: "test.txt",
+		}
+
+		err := store.RemoveWithOptions(options)
+		if err == nil {
+			t.Error("RemoveWithOptions should fail for non-existent file")
+		}
+	})
+}

api/bolt/datastore.go

@@ -2,7 +2,6 @@ package bolt
 
 import (
 	"io"
-	"log"
 	"path"
 	"time"
 
@@ -21,7 +20,6 @@ import (
 	"github.com/portainer/portainer/api/bolt/errors"
 	"github.com/portainer/portainer/api/bolt/extension"
 	"github.com/portainer/portainer/api/bolt/internal"
-	"github.com/portainer/portainer/api/bolt/migrator"
 	"github.com/portainer/portainer/api/bolt/registry"
 	"github.com/portainer/portainer/api/bolt/resourcecontrol"
 	"github.com/portainer/portainer/api/bolt/role"
@@ -36,7 +34,6 @@ import (
 	"github.com/portainer/portainer/api/bolt/user"
 	"github.com/portainer/portainer/api/bolt/version"
 	"github.com/portainer/portainer/api/bolt/webhook"
-	"github.com/portainer/portainer/api/internal/authorization"
 )
 
 const (
@@ -76,6 +73,14 @@ type Store struct {
 	WebhookService            *webhook.Service
 }
 
+func (store *Store) version() (int, error) {
+	version, err := store.VersionService.DBVersion()
+	if err == errors.ErrObjectNotFound {
+		version = 0
+	}
+	return version, err
+}
+
 func (store *Store) edition() portainer.SoftwareEdition {
 	edition, err := store.VersionService.Edition()
 	if err == errors.ErrObjectNotFound {
@@ -85,25 +90,13 @@ func (store *Store) edition() portainer.SoftwareEdition {
 }
 
 // NewStore initializes a new Store and the associated services
-func NewStore(storePath string, fileService portainer.FileService) (*Store, error) {
-	store := &Store{
+func NewStore(storePath string, fileService portainer.FileService) *Store {
+	return &Store{
 		path:        storePath,
 		fileService: fileService,
 		isNew:       true,
 		connection:  &internal.DbConnection{},
 	}
-
-	databasePath := path.Join(storePath, databaseFileName)
-	databaseFileExists, err := fileService.FileExists(databasePath)
-	if err != nil {
-		return nil, err
-	}
-
-	if databaseFileExists {
-		store.isNew = false
-	}
-
-	return store, nil
 }
 
 // Open opens and initializes the BoltDB database.
@@ -115,7 +108,17 @@ func (store *Store) Open() error {
 	}
 	store.connection.DB = db
 
-	return store.initServices()
+	err = store.initServices()
+	if err != nil {
+		return err
+	}
+
+	// if we have DBVersion in the database then ensure we flag this as NOT a new store
+	if _, err := store.VersionService.DBVersion(); err == nil {
+		store.isNew = false
+	}
+
+	return nil
 }
 
 // Close closes the BoltDB database.
@@ -133,64 +136,6 @@ func (store *Store) IsNew() bool {
 	return store.isNew
 }
 
-// CheckCurrentEdition checks if current edition is community edition
-func (store *Store) CheckCurrentEdition() error {
-	if store.edition() != portainer.PortainerCE {
-		return errors.ErrWrongDBEdition
-	}
-	return nil
-}
-
-// MigrateData automatically migrate the data based on the DBVersion.
-// This process is only triggered on an existing database, not if the database was just created.
-// if force is true, then migrate regardless.
-func (store *Store) MigrateData(force bool) error {
-	if store.isNew && !force {
-		return store.VersionService.StoreDBVersion(portainer.DBVersion)
-	}
-
-	version, err := store.VersionService.DBVersion()
-	if err == errors.ErrObjectNotFound {
-		version = 0
-	} else if err != nil {
-		return err
-	}
-
-	if version < portainer.DBVersion {
-		migratorParams := &migrator.Parameters{
-			DB:                      store.connection.DB,
-			DatabaseVersion:         version,
-			EndpointGroupService:    store.EndpointGroupService,
-			EndpointService:         store.EndpointService,
-			EndpointRelationService: store.EndpointRelationService,
-			ExtensionService:        store.ExtensionService,
-			RegistryService:         store.RegistryService,
-			ResourceControlService:  store.ResourceControlService,
-			RoleService:             store.RoleService,
-			ScheduleService:         store.ScheduleService,
-			SettingsService:         store.SettingsService,
-			StackService:            store.StackService,
-			TagService:              store.TagService,
-			TeamMembershipService:   store.TeamMembershipService,
-			UserService:             store.UserService,
-			VersionService:          store.VersionService,
-			FileService:             store.fileService,
-			DockerhubService:        store.DockerHubService,
-			AuthorizationService:    authorization.NewService(store),
-		}
-		migrator := migrator.NewMigrator(migratorParams)
-
-		log.Printf("Migrating database from version %v to %v.\n", version, portainer.DBVersion)
-		err = migrator.Migrate()
-		if err != nil {
-			log.Printf("An error occurred during database migration: %s\n", err)
-			return err
-		}
-	}
-
-	return nil
-}
-
 // BackupTo backs up db to a provided writer.
 // It does hot backup and doesn't block other database reads and writes
 func (store *Store) BackupTo(w io.Writer) error {
@@ -199,3 +144,11 @@ func (store *Store) BackupTo(w io.Writer) error {
 		return err
 	})
 }
+
+// CheckCurrentEdition checks if current edition is community edition
+func (store *Store) CheckCurrentEdition() error {
+	if store.edition() != portainer.PortainerCE {
+		return errors.ErrWrongDBEdition
+	}
+	return nil
+}

api/bolt/init.go

@@ -47,6 +47,7 @@ func (store *Store) Init() error {
 			HelmRepositoryURL:        portainer.DefaultHelmRepositoryURL,
 			UserSessionTimeout:       portainer.DefaultUserSessionTimeout,
 			KubeconfigExpiry:         portainer.DefaultKubeconfigExpiry,
+			KubectlShellImage:        portainer.DefaultKubectlShellImage,
 		}
 
 		err = store.SettingsService.UpdateSettings(defaultSettings)

api/bolt/migrate_data.go

@@ -0,0 +1,149 @@
+package bolt
+
+import (
+	"fmt"
+
+	"github.com/portainer/portainer/api/cli"
+
+	werrors "github.com/pkg/errors"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/errors"
+	plog "github.com/portainer/portainer/api/bolt/log"
+	"github.com/portainer/portainer/api/bolt/migrator"
+	"github.com/portainer/portainer/api/internal/authorization"
+)
+
+const beforePortainerVersionUpgradeBackup = "portainer.db.bak"
+
+var migrateLog = plog.NewScopedLog("bolt, migrate")
+
+// FailSafeMigrate backup and restore DB if migration fail
+func (store *Store) FailSafeMigrate(migrator *migrator.Migrator) (err error) {
+	defer func() {
+		if e := recover(); e != nil {
+			store.Rollback(true)
+			err = fmt.Errorf("%v", e)
+		}
+	}()
+
+	// !Important: we must use a named return value in the function definition and not a local
+	// !variable referenced from the closure or else the return value will be incorrectly set
+	return migrator.Migrate()
+}
+
+// MigrateData automatically migrate the data based on the DBVersion.
+// This process is only triggered on an existing database, not if the database was just created.
+// if force is true, then migrate regardless.
+func (store *Store) MigrateData(force bool) error {
+	if store.isNew && !force {
+		return store.VersionService.StoreDBVersion(portainer.DBVersion)
+	}
+
+	migrator, err := store.newMigrator()
+	if err != nil {
+		return err
+	}
+
+	// backup db file before upgrading DB to support rollback
+	isUpdating, err := store.VersionService.IsUpdating()
+	if err != nil && err != errors.ErrObjectNotFound {
+		return err
+	}
+
+	if !isUpdating && migrator.Version() != portainer.DBVersion {
+		err = store.backupVersion(migrator)
+		if err != nil {
+			return werrors.Wrapf(err, "failed to backup database")
+		}
+	}
+
+	if migrator.Version() < portainer.DBVersion {
+		migrateLog.Info(fmt.Sprintf("Migrating database from version %v to %v.\n", migrator.Version(), portainer.DBVersion))
+		err = store.FailSafeMigrate(migrator)
+		if err != nil {
+			migrateLog.Error("An error occurred during database migration", err)
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (store *Store) newMigrator() (*migrator.Migrator, error) {
+	version, err := store.version()
+	if err != nil {
+		return nil, err
+	}
+
+	migratorParams := &migrator.Parameters{
+		DB:                      store.connection.DB,
+		DatabaseVersion:         version,
+		EndpointGroupService:    store.EndpointGroupService,
+		EndpointService:         store.EndpointService,
+		EndpointRelationService: store.EndpointRelationService,
+		ExtensionService:        store.ExtensionService,
+		RegistryService:         store.RegistryService,
+		ResourceControlService:  store.ResourceControlService,
+		RoleService:             store.RoleService,
+		ScheduleService:         store.ScheduleService,
+		SettingsService:         store.SettingsService,
+		StackService:            store.StackService,
+		TagService:              store.TagService,
+		TeamMembershipService:   store.TeamMembershipService,
+		UserService:             store.UserService,
+		VersionService:          store.VersionService,
+		FileService:             store.fileService,
+		DockerhubService:        store.DockerHubService,
+		AuthorizationService:    authorization.NewService(store),
+	}
+	return migrator.NewMigrator(migratorParams), nil
+}
+
+// getBackupRestoreOptions returns options to store db at common backup dir location; used by:
+// - db backup prior to version upgrade
+// - db rollback
+func getBackupRestoreOptions(store *Store) *BackupOptions {
+	return &BackupOptions{
+		BackupDir:      store.commonBackupDir(),
+		BackupFileName: beforePortainerVersionUpgradeBackup,
+	}
+}
+
+// backupVersion will backup the database or panic if any errors occur
+func (store *Store) backupVersion(migrator *migrator.Migrator) error {
+	migrateLog.Info("Backing up database prior to version upgrade...")
+
+	options := getBackupRestoreOptions(store)
+
+	_, err := store.BackupWithOptions(options)
+	if err != nil {
+		migrateLog.Error("An error occurred during database backup", err)
+		removalErr := store.RemoveWithOptions(options)
+		if removalErr != nil {
+			migrateLog.Error("An error occurred during store removal prior to backup", err)
+		}
+		return err
+	}
+
+	return nil
+}
+
+// Rollback to a pre-upgrade backup copy/snapshot of portainer.db
+func (store *Store) Rollback(force bool) error {
+
+	if !force {
+		confirmed, err := cli.Confirm("Are you sure you want to rollback your database to the previous backup?")
+		if err != nil || !confirmed {
+			return err
+		}
+	}
+
+	options := getBackupRestoreOptions(store)
+
+	err := store.RestoreWithOptions(options)
+	if err != nil {
+		return err
+	}
+
+	return store.Close()
+}

api/bolt/migrate_data_test.go

@@ -0,0 +1,172 @@
+package bolt
+
+import (
+	"fmt"
+	"log"
+	"strings"
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+)
+
+// testVersion is a helper which tests current store version against wanted version
+func testVersion(store *Store, versionWant int, t *testing.T) {
+	if v, _ := store.version(); v != versionWant {
+		t.Errorf("Expect store version to be %d but was %d", versionWant, v)
+	}
+}
+
+func TestMigrateData(t *testing.T) {
+	t.Run("MigrateData for New Store & Re-Open Check", func(t *testing.T) {
+		store, teardown := MustNewTestStore(false)
+		defer teardown()
+
+		if !store.IsNew() {
+			t.Error("Expect a new DB")
+		}
+
+		store.MigrateData(false)
+
+		testVersion(store, portainer.DBVersion, t)
+		store.Close()
+
+		store.Open()
+		if store.IsNew() {
+			t.Error("Expect store to NOT be new DB")
+		}
+	})
+
+	tests := []struct {
+		version         int
+		expectedVersion int
+	}{
+		{version: 2, expectedVersion: portainer.DBVersion},
+		{version: 21, expectedVersion: portainer.DBVersion},
+	}
+	for _, tc := range tests {
+		store, teardown := MustNewTestStore(true)
+		defer teardown()
+
+		// Setup data
+		store.VersionService.StoreDBVersion(tc.version)
+
+		// Required roles by migrations 22.2
+		store.RoleService.CreateRole(&portainer.Role{ID: 1})
+		store.RoleService.CreateRole(&portainer.Role{ID: 2})
+		store.RoleService.CreateRole(&portainer.Role{ID: 3})
+		store.RoleService.CreateRole(&portainer.Role{ID: 4})
+
+		t.Run(fmt.Sprintf("MigrateData for version %d", tc.version), func(t *testing.T) {
+			store.MigrateData(true)
+			testVersion(store, tc.expectedVersion, t)
+		})
+
+		t.Run(fmt.Sprintf("Restoring DB after migrateData for version %d", tc.version), func(t *testing.T) {
+			store.Rollback(true)
+			store.Open()
+			testVersion(store, tc.version, t)
+		})
+	}
+
+	t.Run("Error in MigrateData should restore backup before MigrateData", func(t *testing.T) {
+		store, teardown := MustNewTestStore(false)
+		defer teardown()
+
+		version := 2
+		store.VersionService.StoreDBVersion(version)
+
+		store.MigrateData(true)
+
+		testVersion(store, version, t)
+	})
+
+	t.Run("MigrateData should create backup file upon update", func(t *testing.T) {
+		store, teardown := MustNewTestStore(false)
+		defer teardown()
+		store.VersionService.StoreDBVersion(0)
+
+		store.MigrateData(true)
+
+		options := store.setupOptions(getBackupRestoreOptions(store))
+
+		if !isFileExist(options.BackupPath) {
+			t.Errorf("Backup file should exist; file=%s", options.BackupPath)
+		}
+	})
+
+	t.Run("MigrateData should fail to create backup if database file is set to updating", func(t *testing.T) {
+		store, teardown := MustNewTestStore(false)
+		defer teardown()
+
+		store.VersionService.StoreIsUpdating(true)
+
+		store.MigrateData(true)
+
+		options := store.setupOptions(getBackupRestoreOptions(store))
+
+		if isFileExist(options.BackupPath) {
+			t.Errorf("Backup file should not exist for dirty database; file=%s", options.BackupPath)
+		}
+	})
+
+	t.Run("MigrateData should not create backup on startup if portainer version matches db", func(t *testing.T) {
+		store, teardown := MustNewTestStore(false)
+		defer teardown()
+
+		store.MigrateData(true)
+
+		options := store.setupOptions(getBackupRestoreOptions(store))
+
+		if isFileExist(options.BackupPath) {
+			t.Errorf("Backup file should not exist for dirty database; file=%s", options.BackupPath)
+		}
+	})
+
+}
+
+func Test_getBackupRestoreOptions(t *testing.T) {
+	store, teardown := MustNewTestStore(false)
+	defer teardown()
+
+	options := getBackupRestoreOptions(store)
+
+	wantDir := store.commonBackupDir()
+	if !strings.HasSuffix(options.BackupDir, wantDir) {
+		log.Fatalf("incorrect backup dir; got=%s, want=%s", options.BackupDir, wantDir)
+	}
+
+	wantFilename := "portainer.db.bak"
+	if options.BackupFileName != wantFilename {
+		log.Fatalf("incorrect backup file; got=%s, want=%s", options.BackupFileName, wantFilename)
+	}
+}
+
+func TestRollback(t *testing.T) {
+	t.Run("Rollback should restore upgrade after backup", func(t *testing.T) {
+		version := 21
+		store, teardown := MustNewTestStore(false)
+		defer teardown()
+		store.VersionService.StoreDBVersion(version)
+
+		_, err := store.BackupWithOptions(getBackupRestoreOptions(store))
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		// Change the current edition
+		err = store.VersionService.StoreDBVersion(version + 10)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		err = store.Rollback(true)
+		if err != nil {
+			t.Logf("Rollback failed: %s", err)
+			t.Fail()
+			return
+		}
+
+		store.Open()
+		testVersion(store, version, t)
+	})
+}

api/bolt/migrator/migrate_ce.go

@@ -0,0 +1,334 @@
+package migrator
+
+import (
+	"fmt"
+
+	werrors "github.com/pkg/errors"
+	portainer "github.com/portainer/portainer/api"
+)
+
+func migrationError(err error, context string) error {
+	return werrors.Wrap(err, "failed in "+context)
+}
+
+// Migrate checks the database version and migrate the existing data to the most recent data model.
+func (m *Migrator) Migrate() error {
+	// set DB to updating status
+	err := m.versionService.StoreIsUpdating(true)
+	if err != nil {
+		return migrationError(err, "StoreIsUpdating")
+	}
+
+	// Portainer < 1.12
+	if m.currentDBVersion < 1 {
+		err := m.updateAdminUserToDBVersion1()
+		if err != nil {
+			return migrationError(err, "updateAdminUserToDBVersion1")
+		}
+	}
+
+	// Portainer 1.12.x
+	if m.currentDBVersion < 2 {
+		err := m.updateResourceControlsToDBVersion2()
+		if err != nil {
+			return migrationError(err, "updateResourceControlsToDBVersion2")
+		}
+		err = m.updateEndpointsToDBVersion2()
+		if err != nil {
+			return migrationError(err, "updateEndpointsToDBVersion2")
+		}
+	}
+
+	// Portainer 1.13.x
+	if m.currentDBVersion < 3 {
+		err := m.updateSettingsToDBVersion3()
+		if err != nil {
+			return migrationError(err, "updateSettingsToDBVersion3")
+		}
+	}
+
+	// Portainer 1.14.0
+	if m.currentDBVersion < 4 {
+		err := m.updateEndpointsToDBVersion4()
+		if err != nil {
+			return migrationError(err, "updateEndpointsToDBVersion4")
+		}
+	}
+
+	// https://github.com/portainer/portainer/issues/1235
+	if m.currentDBVersion < 5 {
+		err := m.updateSettingsToVersion5()
+		if err != nil {
+			return migrationError(err, "updateSettingsToVersion5")
+		}
+	}
+
+	// https://github.com/portainer/portainer/issues/1236
+	if m.currentDBVersion < 6 {
+		err := m.updateSettingsToVersion6()
+		if err != nil {
+			return migrationError(err, "updateSettingsToVersion6")
+		}
+	}
+
+	// https://github.com/portainer/portainer/issues/1449
+	if m.currentDBVersion < 7 {
+		err := m.updateSettingsToVersion7()
+		if err != nil {
+			return migrationError(err, "updateSettingsToVersion7")
+		}
+	}
+
+	if m.currentDBVersion < 8 {
+		err := m.updateEndpointsToVersion8()
+		if err != nil {
+			return migrationError(err, "updateEndpointsToVersion8")
+		}
+	}
+
+	// https: //github.com/portainer/portainer/issues/1396
+	if m.currentDBVersion < 9 {
+		err := m.updateEndpointsToVersion9()
+		if err != nil {
+			return migrationError(err, "updateEndpointsToVersion9")
+		}
+	}
+
+	// https://github.com/portainer/portainer/issues/461
+	if m.currentDBVersion < 10 {
+		err := m.updateEndpointsToVersion10()
+		if err != nil {
+			return migrationError(err, "updateEndpointsToVersion10")
+		}
+	}
+
+	// https://github.com/portainer/portainer/issues/1906
+	if m.currentDBVersion < 11 {
+		err := m.updateEndpointsToVersion11()
+		if err != nil {
+			return migrationError(err, "updateEndpointsToVersion11")
+		}
+	}
+
+	// Portainer 1.18.0
+	if m.currentDBVersion < 12 {
+		err := m.updateEndpointsToVersion12()
+		if err != nil {
+			return migrationError(err, "updateEndpointsToVersion12")
+		}
+
+		err = m.updateEndpointGroupsToVersion12()
+		if err != nil {
+			return migrationError(err, "updateEndpointGroupsToVersion12")
+		}
+
+		err = m.updateStacksToVersion12()
+		if err != nil {
+			return migrationError(err, "updateStacksToVersion12")
+		}
+	}
+
+	// Portainer 1.19.0
+	if m.currentDBVersion < 13 {
+		err := m.updateSettingsToVersion13()
+		if err != nil {
+			return migrationError(err, "updateSettingsToVersion13")
+		}
+	}
+
+	// Portainer 1.19.2
+	if m.currentDBVersion < 14 {
+		err := m.updateResourceControlsToDBVersion14()
+		if err != nil {
+			return migrationError(err, "updateResourceControlsToDBVersion14")
+		}
+	}
+
+	// Portainer 1.20.0
+	if m.currentDBVersion < 15 {
+		err := m.updateSettingsToDBVersion15()
+		if err != nil {
+			return migrationError(err, "updateSettingsToDBVersion15")
+		}
+
+		err = m.updateTemplatesToVersion15()
+		if err != nil {
+			return migrationError(err, "updateTemplatesToVersion15")
+		}
+	}
+
+	if m.currentDBVersion < 16 {
+		err := m.updateSettingsToDBVersion16()
+		if err != nil {
+			return migrationError(err, "updateSettingsToDBVersion16")
+		}
+	}
+
+	// Portainer 1.20.1
+	if m.currentDBVersion < 17 {
+		err := m.updateExtensionsToDBVersion17()
+		if err != nil {
+			return migrationError(err, "updateExtensionsToDBVersion17")
+		}
+	}
+
+	// Portainer 1.21.0
+	if m.currentDBVersion < 18 {
+		err := m.updateUsersToDBVersion18()
+		if err != nil {
+			return migrationError(err, "updateUsersToDBVersion18")
+		}
+
+		err = m.updateEndpointsToDBVersion18()
+		if err != nil {
+			return migrationError(err, "updateEndpointsToDBVersion18")
+		}
+
+		err = m.updateEndpointGroupsToDBVersion18()
+		if err != nil {
+			return migrationError(err, "updateEndpointGroupsToDBVersion18")
+		}
+
+		err = m.updateRegistriesToDBVersion18()
+		if err != nil {
+			return migrationError(err, "updateRegistriesToDBVersion18")
+		}
+	}
+
+	// Portainer 1.22.0
+	if m.currentDBVersion < 19 {
+		err := m.updateSettingsToDBVersion19()
+		if err != nil {
+			return migrationError(err, "updateSettingsToDBVersion19")
+		}
+	}
+
+	// Portainer 1.22.1
+	if m.currentDBVersion < 20 {
+		err := m.updateUsersToDBVersion20()
+		if err != nil {
+			return migrationError(err, "updateUsersToDBVersion20")
+		}
+
+		err = m.updateSettingsToDBVersion20()
+		if err != nil {
+			return migrationError(err, "updateSettingsToDBVersion20")
+		}
+
+		err = m.updateSchedulesToDBVersion20()
+		if err != nil {
+			return migrationError(err, "updateSchedulesToDBVersion20")
+		}
+	}
+
+	// Portainer 1.23.0
+	// DBVersion 21 is missing as it was shipped as via hotfix 1.22.2
+	if m.currentDBVersion < 22 {
+		err := m.updateResourceControlsToDBVersion22()
+		if err != nil {
+			return migrationError(err, "updateResourceControlsToDBVersion22")
+		}
+
+		err = m.updateUsersAndRolesToDBVersion22()
+		if err != nil {
+			return migrationError(err, "updateUsersAndRolesToDBVersion22")
+		}
+	}
+
+	// Portainer 1.24.0
+	if m.currentDBVersion < 23 {
+		err := m.updateTagsToDBVersion23()
+		if err != nil {
+			return migrationError(err, "updateTagsToDBVersion23")
+		}
+
+		err = m.updateEndpointsAndEndpointGroupsToDBVersion23()
+		if err != nil {
+			return migrationError(err, "updateEndpointsAndEndpointGroupsToDBVersion23")
+		}
+	}
+
+	// Portainer 1.24.1
+	if m.currentDBVersion < 24 {
+		err := m.updateSettingsToDB24()
+		if err != nil {
+			return migrationError(err, "updateSettingsToDB24")
+		}
+	}
+
+	// Portainer 2.0.0
+	if m.currentDBVersion < 25 {
+		err := m.updateSettingsToDB25()
+		if err != nil {
+			return migrationError(err, "updateSettingsToDB25")
+		}
+
+		err = m.updateStacksToDB24()
+		if err != nil {
+			return migrationError(err, "updateStacksToDB24")
+		}
+	}
+
+	// Portainer 2.1.0
+	if m.currentDBVersion < 26 {
+		err := m.updateEndpointSettingsToDB25()
+		if err != nil {
+			return migrationError(err, "updateEndpointSettingsToDB25")
+		}
+	}
+
+	// Portainer 2.2.0
+	if m.currentDBVersion < 27 {
+		err := m.updateStackResourceControlToDB27()
+		if err != nil {
+			return migrationError(err, "updateStackResourceControlToDB27")
+		}
+	}
+
+	// Portainer 2.6.0
+	if m.currentDBVersion < 30 {
+		err := m.migrateDBVersionToDB30()
+		if err != nil {
+			return migrationError(err, "migrateDBVersionToDB30")
+		}
+	}
+
+	// Portainer 2.9.0
+	if m.currentDBVersion < 32 {
+		err := m.migrateDBVersionToDB32()
+		if err != nil {
+			return migrationError(err, "migrateDBVersionToDB32")
+		}
+	}
+
+	// Portainer 2.9.1, 2.9.2
+	if m.currentDBVersion < 33 {
+		err := m.migrateDBVersionToDB33()
+		if err != nil {
+			return migrationError(err, "migrateDBVersionToDB33")
+		}
+	}
+
+	// Portainer 2.10
+	if m.currentDBVersion < 34 {
+		if err := m.migrateDBVersionToDB34(); err != nil {
+			return migrationError(err, "migrateDBVersionToDB34")
+		}
+	}
+
+	// Portainer 2.9.3 (yep out of order, but 2.10 is EE only)
+	if m.currentDBVersion < 35 {
+		if err := m.migrateDBVersionToDB35(); err != nil {
+			return migrationError(err, "migrateDBVersionToDB35")
+		}
+	}
+
+	err = m.versionService.StoreDBVersion(portainer.DBVersion)
+	if err != nil {
+		return migrationError(err, "StoreDBVersion")
+	}
+	migrateLog.Info(fmt.Sprintf("Updated DB version to %d", portainer.DBVersion))
+
+	// reset DB updating status
+	return m.versionService.StoreIsUpdating(false)
+}

api/bolt/migrator/migrate_dbversion31.go

@@ -2,6 +2,7 @@ package migrator
 
 import (
 	"fmt"
+	"log"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/errors"
@@ -99,6 +100,32 @@ func (m *Migrator) updateDockerhubToDB32() error {
 		RegistryAccesses: portainer.RegistryAccesses{},
 	}
 
+	// The following code will make this function idempotent.
+	// i.e. if run again, it will not change the data.  It will ensure that
+	// we only have one migrated registry entry. Duplicates will be removed
+	// if they exist and which has been happening due to earlier migration bugs
+	migrated := false
+	registries, _ := m.registryService.Registries()
+	for _, r := range registries {
+		if r.Type == registry.Type &&
+			r.Name == registry.Name &&
+			r.URL == registry.URL &&
+			r.Authentication == registry.Authentication {
+
+			if !migrated {
+				// keep this one entry
+				migrated = true
+			} else {
+				// delete subsequent duplicates
+				m.registryService.DeleteRegistry(portainer.RegistryID(r.ID))
+			}
+		}
+	}
+
+	if migrated {
+		return nil
+	}
+
 	endpoints, err := m.endpointService.Endpoints()
 	if err != nil {
 		return err
@@ -167,6 +194,7 @@ func (m *Migrator) updateVolumeResourceControlToDB32() error {
 
 		totalSnapshots := len(endpoint.Snapshots)
 		if totalSnapshots == 0 {
+			log.Println("[DEBUG] [volume migration] [message: no snapshot found]")
 			continue
 		}
 
@@ -174,11 +202,13 @@ func (m *Migrator) updateVolumeResourceControlToDB32() error {
 
 		endpointDockerID, err := snapshotutils.FetchDockerID(snapshot)
 		if err != nil {
-			return fmt.Errorf("failed fetching environment docker id: %w", err)
+			log.Printf("[WARN] [bolt,migrator,v31] [message: failed fetching environment docker id] [err: %s]", err)
+			continue
 		}
 
 		if volumesData, done := snapshot.SnapshotRaw.Volumes.(map[string]interface{}); done {
 			if volumesData["Volumes"] == nil {
+				log.Println("[DEBUG] [volume migration] [message: no volume data found]")
 				continue
 			}
 
@@ -199,7 +229,7 @@ func (m *Migrator) updateVolumeResourceControlToDB32() error {
 			if err != nil {
 				return fmt.Errorf("failed deleting resource control %d: %w", resourceControl.ID, err)
 			}
-
+			log.Printf("[DEBUG] [volume migration] [message: legacy resource control(%s) has been deleted]", resourceControl.ResourceID)
 		}
 	}
 
@@ -210,8 +240,16 @@ func findResourcesToUpdateForDB32(dockerID string, volumesData map[string]interf
 	volumes := volumesData["Volumes"].([]interface{})
 	for _, volumeMeta := range volumes {
 		volume := volumeMeta.(map[string]interface{})
-		volumeName := volume["Name"].(string)
-		oldResourceID := fmt.Sprintf("%s%s", volumeName, volume["CreatedAt"].(string))
+		volumeName, nameExist := volume["Name"].(string)
+		if !nameExist {
+			continue
+		}
+		createTime, createTimeExist := volume["CreatedAt"].(string)
+		if !createTimeExist {
+			continue
+		}
+
+		oldResourceID := fmt.Sprintf("%s%s", volumeName, createTime)
 		resourceControl, ok := volumeResourceControls[oldResourceID]
 
 		if ok {

api/bolt/migrator/migrate_dbversion32.go

@@ -0,0 +1,21 @@
+package migrator
+
+import portainer "github.com/portainer/portainer/api"
+
+func (m *Migrator) migrateDBVersionToDB33() error {
+	if err := m.migrateSettingsToDB33(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (m *Migrator) migrateSettingsToDB33() error {
+	settings, err := m.settingsService.Settings()
+	if err != nil {
+		return err
+	}
+
+	settings.KubectlShellImage = portainer.DefaultKubectlShellImage
+	return m.settingsService.UpdateSettings(settings)
+}

api/bolt/migrator/migrate_dbversion33.go

@@ -4,7 +4,7 @@ import (
 	portainer "github.com/portainer/portainer/api"
 )
 
-func (m *Migrator) migrateDBVersionTo33() error {
+func (m *Migrator) migrateDBVersionToDB34() error {
 	err := migrateStackEntryPoint(m.stackService)
 	if err != nil {
 		return err

api/bolt/migrator/migrate_dbversion33_test.go

@@ -14,7 +14,7 @@ import (
 )
 
 func TestMigrateStackEntryPoint(t *testing.T) {
-	dbConn, err := bolt.Open(path.Join(t.TempDir(), "portainer-ee-mig-33.db"), 0600, &bolt.Options{Timeout: 1 * time.Second})
+	dbConn, err := bolt.Open(path.Join(t.TempDir(), "portainer-ee-mig-34.db"), 0600, &bolt.Options{Timeout: 1 * time.Second})
 	assert.NoError(t, err, "failed to init testing DB connection")
 	defer dbConn.Close()
 

api/bolt/migrator/migrate_dbversion34.go

@@ -0,0 +1,11 @@
+package migrator
+
+func (m *Migrator) migrateDBVersionToDB35() error {
+	// These should have been migrated already, but due to an earlier bug and a bunch of duplicates,
+	// calling it again will now fix the issue as the function has been repaired.
+	err := m.updateDockerhubToDB32()
+	if err != nil {
+		return err
+	}
+	return nil
+}

api/bolt/migrator/migrate_dbversion34_test.go

@@ -0,0 +1,108 @@
+package migrator
+
+import (
+	"os"
+	"path"
+	"testing"
+	"time"
+
+	"github.com/boltdb/bolt"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/dockerhub"
+	"github.com/portainer/portainer/api/bolt/endpoint"
+	"github.com/portainer/portainer/api/bolt/internal"
+	"github.com/portainer/portainer/api/bolt/registry"
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	db35TestFile = "portainer-mig-35.db"
+	username     = "portainer"
+	password     = "password"
+)
+
+func setupDB35Test(t *testing.T) *Migrator {
+	is := assert.New(t)
+	dbConn, err := bolt.Open(path.Join(t.TempDir(), db35TestFile), 0600, &bolt.Options{Timeout: 1 * time.Second})
+	is.NoError(err, "failed to init testing DB connection")
+
+	// Create an old style dockerhub authenticated account
+	dockerhubService, err := dockerhub.NewService(&internal.DbConnection{DB: dbConn})
+	is.NoError(err, "failed to init testing registry service")
+	err = dockerhubService.UpdateDockerHub(&portainer.DockerHub{true, username, password})
+	is.NoError(err, "failed to create dockerhub account")
+
+	registryService, err := registry.NewService(&internal.DbConnection{DB: dbConn})
+	is.NoError(err, "failed to init testing registry service")
+
+	endpointService, err := endpoint.NewService(&internal.DbConnection{DB: dbConn})
+	is.NoError(err, "failed to init endpoint service")
+
+	m := &Migrator{
+		db:               dbConn,
+		dockerhubService: dockerhubService,
+		registryService:  registryService,
+		endpointService:  endpointService,
+	}
+
+	return m
+}
+
+// TestUpdateDockerhubToDB32 tests a normal upgrade
+func TestUpdateDockerhubToDB32(t *testing.T) {
+	is := assert.New(t)
+	m := setupDB35Test(t)
+	defer m.db.Close()
+	defer os.Remove(db35TestFile)
+
+	if err := m.updateDockerhubToDB32(); err != nil {
+		t.Errorf("failed to update settings: %v", err)
+	}
+
+	// Verify we have a single registry were created
+	registries, err := m.registryService.Registries()
+	is.NoError(err, "failed to read registries from the RegistryService")
+	is.Equal(len(registries), 1, "only one migrated registry expected")
+}
+
+// TestUpdateDockerhubToDB32_with_duplicate_migrations tests an upgrade where in earlier versions a broken migration
+// created a large number of duplicate "dockerhub migrated" registry entries.
+func TestUpdateDockerhubToDB32_with_duplicate_migrations(t *testing.T) {
+	is := assert.New(t)
+	m := setupDB35Test(t)
+	defer m.db.Close()
+	defer os.Remove(db35TestFile)
+
+	// Create lots of duplicate entries...
+	registry := &portainer.Registry{
+		Type:             portainer.DockerHubRegistry,
+		Name:             "Dockerhub (authenticated - migrated)",
+		URL:              "docker.io",
+		Authentication:   true,
+		Username:         "portainer",
+		Password:         "password",
+		RegistryAccesses: portainer.RegistryAccesses{},
+	}
+
+	for i := 1; i < 150; i++ {
+		err := m.registryService.CreateRegistry(registry)
+		assert.NoError(t, err, "create registry failed")
+	}
+
+	// Verify they were created
+	registries, err := m.registryService.Registries()
+	is.NoError(err, "failed to read registries from the RegistryService")
+	is.Condition(func() bool {
+		return len(registries) > 1
+	}, "expected multiple duplicate registry entries")
+
+	// Now run the migrator
+	if err := m.updateDockerhubToDB32(); err != nil {
+		t.Errorf("failed to update settings: %v", err)
+	}
+
+	// Verify we have a single registry were created
+	registries, err = m.registryService.Registries()
+	is.NoError(err, "failed to read registries from the RegistryService")
+	is.Equal(len(registries), 1, "only one migrated registry expected")
+}

api/bolt/migrator/migrator.go

@@ -27,8 +27,9 @@ var migrateLog = plog.NewScopedLog("bolt, migrate")
 type (
 	// Migrator defines a service to migrate data after a Portainer version update.
 	Migrator struct {
-		currentDBVersion        int
-		db                      *bolt.DB
+		db               *bolt.DB
+		currentDBVersion int
+
 		endpointGroupService    *endpointgroup.Service
 		endpointService         *endpoint.Service
 		endpointRelationService *endpointrelation.Service
@@ -97,295 +98,7 @@ func NewMigrator(parameters *Parameters) *Migrator {
 	}
 }
 
-// Migrate checks the database version and migrate the existing data to the most recent data model.
-func (m *Migrator) Migrate() error {
-	// Portainer < 1.12
-	if m.currentDBVersion < 1 {
-		err := m.updateAdminUserToDBVersion1()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.12.x
-	if m.currentDBVersion < 2 {
-		err := m.updateResourceControlsToDBVersion2()
-		if err != nil {
-			return err
-		}
-		err = m.updateEndpointsToDBVersion2()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.13.x
-	if m.currentDBVersion < 3 {
-		err := m.updateSettingsToDBVersion3()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.14.0
-	if m.currentDBVersion < 4 {
-		err := m.updateEndpointsToDBVersion4()
-		if err != nil {
-			return err
-		}
-	}
-
-	// https://github.com/portainer/portainer/issues/1235
-	if m.currentDBVersion < 5 {
-		err := m.updateSettingsToVersion5()
-		if err != nil {
-			return err
-		}
-	}
-
-	// https://github.com/portainer/portainer/issues/1236
-	if m.currentDBVersion < 6 {
-		err := m.updateSettingsToVersion6()
-		if err != nil {
-			return err
-		}
-	}
-
-	// https://github.com/portainer/portainer/issues/1449
-	if m.currentDBVersion < 7 {
-		err := m.updateSettingsToVersion7()
-		if err != nil {
-			return err
-		}
-	}
-
-	if m.currentDBVersion < 8 {
-		err := m.updateEndpointsToVersion8()
-		if err != nil {
-			return err
-		}
-	}
-
-	// https: //github.com/portainer/portainer/issues/1396
-	if m.currentDBVersion < 9 {
-		err := m.updateEndpointsToVersion9()
-		if err != nil {
-			return err
-		}
-	}
-
-	// https://github.com/portainer/portainer/issues/461
-	if m.currentDBVersion < 10 {
-		err := m.updateEndpointsToVersion10()
-		if err != nil {
-			return err
-		}
-	}
-
-	// https://github.com/portainer/portainer/issues/1906
-	if m.currentDBVersion < 11 {
-		err := m.updateEndpointsToVersion11()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.18.0
-	if m.currentDBVersion < 12 {
-		err := m.updateEndpointsToVersion12()
-		if err != nil {
-			return err
-		}
-
-		err = m.updateEndpointGroupsToVersion12()
-		if err != nil {
-			return err
-		}
-
-		err = m.updateStacksToVersion12()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.19.0
-	if m.currentDBVersion < 13 {
-		err := m.updateSettingsToVersion13()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.19.2
-	if m.currentDBVersion < 14 {
-		err := m.updateResourceControlsToDBVersion14()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.20.0
-	if m.currentDBVersion < 15 {
-		err := m.updateSettingsToDBVersion15()
-		if err != nil {
-			return err
-		}
-
-		err = m.updateTemplatesToVersion15()
-		if err != nil {
-			return err
-		}
-	}
-
-	if m.currentDBVersion < 16 {
-		err := m.updateSettingsToDBVersion16()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.20.1
-	if m.currentDBVersion < 17 {
-		err := m.updateExtensionsToDBVersion17()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.21.0
-	if m.currentDBVersion < 18 {
-		err := m.updateUsersToDBVersion18()
-		if err != nil {
-			return err
-		}
-
-		err = m.updateEndpointsToDBVersion18()
-		if err != nil {
-			return err
-		}
-
-		err = m.updateEndpointGroupsToDBVersion18()
-		if err != nil {
-			return err
-		}
-
-		err = m.updateRegistriesToDBVersion18()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.22.0
-	if m.currentDBVersion < 19 {
-		err := m.updateSettingsToDBVersion19()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.22.1
-	if m.currentDBVersion < 20 {
-		err := m.updateUsersToDBVersion20()
-		if err != nil {
-			return err
-		}
-
-		err = m.updateSettingsToDBVersion20()
-		if err != nil {
-			return err
-		}
-
-		err = m.updateSchedulesToDBVersion20()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.23.0
-	// DBVersion 21 is missing as it was shipped as via hotfix 1.22.2
-	if m.currentDBVersion < 22 {
-		err := m.updateResourceControlsToDBVersion22()
-		if err != nil {
-			return err
-		}
-
-		err = m.updateUsersAndRolesToDBVersion22()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.24.0
-	if m.currentDBVersion < 23 {
-		err := m.updateTagsToDBVersion23()
-		if err != nil {
-			return err
-		}
-
-		err = m.updateEndpointsAndEndpointGroupsToDBVersion23()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.24.1
-	if m.currentDBVersion < 24 {
-		err := m.updateSettingsToDB24()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 2.0.0
-	if m.currentDBVersion < 25 {
-		err := m.updateSettingsToDB25()
-		if err != nil {
-			return err
-		}
-
-		err = m.updateStacksToDB24()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 2.1.0
-	if m.currentDBVersion < 26 {
-		err := m.updateEndpointSettingsToDB25()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 2.2.0
-	if m.currentDBVersion < 27 {
-		err := m.updateStackResourceControlToDB27()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 2.6.0
-	if m.currentDBVersion < 30 {
-		err := m.migrateDBVersionToDB30()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 2.9.0
-	if m.currentDBVersion < 32 {
-		err := m.migrateDBVersionToDB32()
-		if err != nil {
-			return err
-		}
-	}
-
-	if m.currentDBVersion < 33 {
-		if err := m.migrateDBVersionTo33(); err != nil {
-			return err
-		}
-	}
-
-	return m.versionService.StoreDBVersion(portainer.DBVersion)
+// Version exposes version of database
+func (migrator *Migrator) Version() int {
+	return migrator.currentDBVersion
 }

api/bolt/stack/stack.go

@@ -192,8 +192,8 @@ func (service *Service) RefreshableStacks() ([]portainer.Stack, error) {
 		bucket := tx.Bucket([]byte(BucketName))
 		cursor := bucket.Cursor()
 
-		var stack portainer.Stack
 		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			stack := portainer.Stack{}
 			err := internal.UnmarshalObject(v, &stack)
 			if err != nil {
 				return err

api/bolt/stack/tests/stack_test.go

@@ -4,18 +4,12 @@ import (
 	"testing"
 	"time"
 
-	"github.com/portainer/portainer/api/bolt"
-
-	bolterrors "github.com/portainer/portainer/api/bolt/errors"
-
-	"github.com/portainer/portainer/api/bolt/bolttest"
-
 	"github.com/gofrs/uuid"
-
-	"github.com/stretchr/testify/assert"
-
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt"
+	bolterrors "github.com/portainer/portainer/api/bolt/errors"
 	"github.com/portainer/portainer/api/filesystem"
+	"github.com/stretchr/testify/assert"
 )
 
 func newGuidString(t *testing.T) string {
@@ -35,7 +29,7 @@ func TestService_StackByWebhookID(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping test in short mode. Normally takes ~1s to run.")
 	}
-	store, teardown := bolttest.MustNewTestStore(true)
+	store, teardown := bolt.MustNewTestStore(true)
 	defer teardown()
 
 	b := stackBuilder{t: t, store: store}
@@ -93,7 +87,7 @@ func Test_RefreshableStacks(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping test in short mode. Normally takes ~1s to run.")
 	}
-	store, teardown := bolttest.MustNewTestStore(true)
+	store, teardown := bolt.MustNewTestStore(true)
 	defer teardown()
 
 	staticStack := portainer.Stack{ID: 1}

api/bolt/teststore.go

@@ -1,4 +1,4 @@
-package bolttest
+package bolt
 
 import (
 	"io/ioutil"
@@ -6,13 +6,12 @@ import (
 	"os"
 
 	"github.com/pkg/errors"
-	"github.com/portainer/portainer/api/bolt"
 	"github.com/portainer/portainer/api/filesystem"
 )
 
 var errTempDir = errors.New("can't create a temp dir")
 
-func MustNewTestStore(init bool) (*bolt.Store, func()) {
+func MustNewTestStore(init bool) (*Store, func()) {
 	store, teardown, err := NewTestStore(init)
 	if err != nil {
 		if !errors.Is(err, errTempDir) {
@@ -24,7 +23,7 @@ func MustNewTestStore(init bool) (*bolt.Store, func()) {
 	return store, teardown
 }
 
-func NewTestStore(init bool) (*bolt.Store, func(), error) {
+func NewTestStore(init bool) (*Store, func(), error) {
 	// Creates unique temp directory in a concurrency friendly manner.
 	dataStorePath, err := ioutil.TempDir("", "boltdb")
 	if err != nil {
@@ -36,11 +35,7 @@ func NewTestStore(init bool) (*bolt.Store, func(), error) {
 		return nil, nil, err
 	}
 
-	store, err := bolt.NewStore(dataStorePath, fileService)
-	if err != nil {
-		return nil, nil, err
-	}
-
+	store := NewStore(dataStorePath, fileService)
 	err = store.Open()
 	if err != nil {
 		return nil, nil, err
@@ -60,7 +55,7 @@ func NewTestStore(init bool) (*bolt.Store, func(), error) {
 	return store, teardown, nil
 }
 
-func teardown(store *bolt.Store, dataStorePath string) {
+func teardown(store *Store, dataStorePath string) {
 	err := store.Close()
 	if err != nil {
 		log.Fatalln(err)

api/bolt/version/version.go

@@ -15,6 +15,7 @@ const (
 	versionKey  = "DB_VERSION"
 	instanceKey = "INSTANCE_ID"
 	editionKey  = "EDITION"
+	updatingKey = "DB_UPDATING"
 )
 
 // Service represents a service to manage stored versions.
@@ -83,6 +84,21 @@ func (service *Service) StoreDBVersion(version int) error {
 	})
 }
 
+// IsUpdating retrieves the database updating status.
+func (service *Service) IsUpdating() (bool, error) {
+	isUpdating, err := service.getKey(updatingKey)
+	if err != nil {
+		return false, err
+	}
+
+	return strconv.ParseBool(string(isUpdating))
+}
+
+// StoreIsUpdating store the database updating status.
+func (service *Service) StoreIsUpdating(isUpdating bool) error {
+	return service.setKey(updatingKey, strconv.FormatBool(isUpdating))
+}
+
 // InstanceID retrieves the stored instance ID.
 func (service *Service) InstanceID() (string, error) {
 	var data []byte

api/chisel/service.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"log"
+	"net/http"
 	"strconv"
 	"time"
 
@@ -42,6 +43,55 @@ func NewService(dataStore portainer.DataStore, shutdownCtx context.Context) *Ser
 	}
 }
 
+// pingAgent ping the given agent so that the agent can keep the tunnel alive
+func (service *Service) pingAgent(endpointID portainer.EndpointID) error{
+	tunnel := service.GetTunnelDetails(endpointID)
+	requestURL := fmt.Sprintf("http://127.0.0.1:%d/ping", tunnel.Port)
+	req, err := http.NewRequest(http.MethodHead, requestURL, nil)
+	if err != nil {
+		return err
+	}
+
+	httpClient := &http.Client{
+		Timeout: 3 * time.Second,
+	}
+	_, err = httpClient.Do(req)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// KeepTunnelAlive keeps the tunnel of the given environment for maxAlive duration, or until ctx is done
+func (service *Service) KeepTunnelAlive(endpointID portainer.EndpointID, ctx context.Context, maxAlive time.Duration) {
+	go func() {
+		log.Printf("[DEBUG] [chisel,KeepTunnelAlive] [endpoint_id: %d] [message: start for %.0f minutes]\n", endpointID, maxAlive.Minutes())
+		maxAliveTicker := time.NewTicker(maxAlive)
+		defer maxAliveTicker.Stop()
+		pingTicker := time.NewTicker(tunnelCleanupInterval)
+		defer pingTicker.Stop()
+
+		for {
+			select {
+			case <-pingTicker.C:
+				service.SetTunnelStatusToActive(endpointID)
+				err := service.pingAgent(endpointID)
+				if err != nil {
+					log.Printf("[DEBUG] [chisel,KeepTunnelAlive] [endpoint_id: %d] [warning: ping agent err=%s]\n", endpointID, err)
+				}
+			case <-maxAliveTicker.C:
+				log.Printf("[DEBUG] [chisel,KeepTunnelAlive] [endpoint_id: %d] [message: stop as %.0f minutes timeout]\n", endpointID, maxAlive.Minutes())
+				return
+			case <-ctx.Done():
+				err := ctx.Err()
+				log.Printf("[DEBUG] [chisel,KeepTunnelAlive] [endpoint_id: %d] [message: stop as err=%s]\n", endpointID, err)
+				return
+			}
+		}
+	}()
+}
+
 // StartTunnelServer starts a tunnel server on the specified addr and port.
 // It uses a seed to generate a new private/public key pair. If the seed cannot
 // be found inside the database, it will generate a new one randomly and persist it.

api/chisel/tunnel.go

@@ -56,6 +56,32 @@ func (service *Service) GetTunnelDetails(endpointID portainer.EndpointID) *porta
 	}
 }
 
+// GetActiveTunnel retrieves an active tunnel which allows communicating with edge agent
+func (service *Service) GetActiveTunnel(endpoint *portainer.Endpoint) (*portainer.TunnelDetails, error) {
+	tunnel := service.GetTunnelDetails(endpoint.ID)
+	if tunnel.Status == portainer.EdgeAgentIdle || tunnel.Status == portainer.EdgeAgentManagementRequired {
+		err := service.SetTunnelStatusToRequired(endpoint.ID)
+		if err != nil {
+			return nil, fmt.Errorf("failed opening tunnel to endpoint: %w", err)
+		}
+
+		if endpoint.EdgeCheckinInterval == 0 {
+			settings, err := service.dataStore.Settings().Settings()
+			if err != nil {
+				return nil, fmt.Errorf("failed fetching settings from db: %w", err)
+			}
+
+			endpoint.EdgeCheckinInterval = settings.EdgeAgentCheckinInterval
+		}
+
+		waitForAgentToConnect := time.Duration(endpoint.EdgeCheckinInterval) * time.Second
+		time.Sleep(waitForAgentToConnect * 2)
+	}
+	tunnel = service.GetTunnelDetails(endpoint.ID)
+
+	return tunnel, nil
+}
+
 // SetTunnelStatusToActive update the status of the tunnel associated to the specified environment(endpoint).
 // It sets the status to ACTIVE.
 func (service *Service) SetTunnelStatusToActive(endpointID portainer.EndpointID) {

api/cli/cli.go

@@ -47,6 +47,7 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 		SSL:                       kingpin.Flag("ssl", "Secure Portainer instance using SSL (deprecated)").Default(defaultSSL).Bool(),
 		SSLCert:                   kingpin.Flag("sslcert", "Path to the SSL certificate used to secure the Portainer instance").String(),
 		SSLKey:                    kingpin.Flag("sslkey", "Path to the SSL key used to secure the Portainer instance").String(),
+		Rollback:                  kingpin.Flag("rollback", "Rollback the database store to the previous version").Bool(),
 		SnapshotInterval:          kingpin.Flag("snapshot-interval", "Duration between each environment snapshot job").Default(defaultSnapshotInterval).String(),
 		AdminPassword:             kingpin.Flag("admin-password", "Hashed admin password").String(),
 		AdminPasswordFile:         kingpin.Flag("admin-password-file", "Path to the file containing the password for the admin user").String(),

api/cli/confirm.go

@@ -0,0 +1,24 @@
+package cli
+
+import (
+	"bufio"
+	"log"
+	"os"
+	"strings"
+)
+
+// Confirm starts a rollback db cli application
+func Confirm(message string) (bool, error) {
+	log.Printf("%s [y/N]", message)
+
+	reader := bufio.NewReader(os.Stdin)
+	answer, err := reader.ReadString('\n')
+	if err != nil {
+		return false, err
+	}
+	answer = strings.Replace(answer, "\n", "", -1)
+	answer = strings.ToLower(answer)
+
+	return answer == "y" || answer == "yes", nil
+
+}

api/cmd/portainer/main.go

@@ -56,17 +56,24 @@ func initFileService(dataStorePath string) portainer.FileService {
 	return fileService
 }
 
-func initDataStore(dataStorePath string, fileService portainer.FileService, shutdownCtx context.Context) portainer.DataStore {
-	store, err := bolt.NewStore(dataStorePath, fileService)
-	if err != nil {
-		log.Fatalf("failed creating data store: %v", err)
-	}
-
-	err = store.Open()
+func initDataStore(dataStorePath string, rollback bool, fileService portainer.FileService, shutdownCtx context.Context) portainer.DataStore {
+	store := bolt.NewStore(dataStorePath, fileService)
+	err := store.Open()
 	if err != nil {
 		log.Fatalf("failed opening store: %v", err)
 	}
 
+	if rollback {
+		err := store.Rollback(false)
+		if err != nil {
+			log.Fatalf("failed rolling back: %s", err)
+		}
+
+		log.Println("Exiting rollback")
+		os.Exit(0)
+		return nil
+	}
+
 	err = store.Init()
 	if err != nil {
 		log.Fatalf("failed initializing data store: %v", err)
@@ -99,8 +106,8 @@ func initSwarmStackManager(assetsPath string, configPath string, signatureServic
 	return exec.NewSwarmStackManager(assetsPath, configPath, signatureService, fileService, reverseTunnelService)
 }
 
-func initKubernetesDeployer(kubernetesTokenCacheManager *kubeproxy.TokenCacheManager, kubernetesClientFactory *kubecli.ClientFactory, dataStore portainer.DataStore, reverseTunnelService portainer.ReverseTunnelService, signatureService portainer.DigitalSignatureService, assetsPath string) portainer.KubernetesDeployer {
-	return exec.NewKubernetesDeployer(kubernetesTokenCacheManager, kubernetesClientFactory, dataStore, reverseTunnelService, signatureService, assetsPath)
+func initKubernetesDeployer(kubernetesTokenCacheManager *kubeproxy.TokenCacheManager, kubernetesClientFactory *kubecli.ClientFactory, dataStore portainer.DataStore, reverseTunnelService portainer.ReverseTunnelService, signatureService portainer.DigitalSignatureService, proxyManager *proxy.Manager, assetsPath string) portainer.KubernetesDeployer {
+	return exec.NewKubernetesDeployer(kubernetesTokenCacheManager, kubernetesClientFactory, dataStore, reverseTunnelService, signatureService, proxyManager, assetsPath)
 }
 
 func initHelmPackageManager(assetsPath string) (libhelm.HelmPackageManager, error) {
@@ -399,7 +406,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 
 	fileService := initFileService(*flags.Data)
 
-	dataStore := initDataStore(*flags.Data, fileService, shutdownCtx)
+	dataStore := initDataStore(*flags.Data, *flags.Rollback, fileService, shutdownCtx)
 
 	if err := dataStore.CheckCurrentEdition(); err != nil {
 		log.Fatal(err)
@@ -469,7 +476,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		log.Fatalf("failed initializing swarm stack manager: %s", err)
 	}
 
-	kubernetesDeployer := initKubernetesDeployer(kubernetesTokenCacheManager, kubernetesClientFactory, dataStore, reverseTunnelService, digitalSignatureService, *flags.Assets)
+	kubernetesDeployer := initKubernetesDeployer(kubernetesTokenCacheManager, kubernetesClientFactory, dataStore, reverseTunnelService, digitalSignatureService, proxyManager, *flags.Assets)
 
 	helmPackageManager, err := initHelmPackageManager(*flags.Assets)
 	if err != nil {
@@ -542,7 +549,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 	}
 
 	scheduler := scheduler.NewScheduler(shutdownCtx)
-	stackDeployer := stacks.NewStackDeployer(swarmStackManager, composeStackManager)
+	stackDeployer := stacks.NewStackDeployer(swarmStackManager, composeStackManager, kubernetesDeployer)
 	stacks.StartStackSchedules(scheduler, stackDeployer, dataStore, gitService)
 
 	return &http.Server{

api/exec/common.go

@@ -0,0 +1,5 @@
+package exec
+
+import "regexp"
+
+var stackNameNormalizeRegex = regexp.MustCompile("[^-_a-z0-9]+")

api/exec/compose_stack.go

@@ -6,7 +6,6 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"regexp"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -47,7 +46,7 @@ func (manager *ComposeStackManager) ComposeSyntaxMaxVersion() string {
 func (manager *ComposeStackManager) Up(ctx context.Context, stack *portainer.Stack, endpoint *portainer.Endpoint) error {
 	url, proxy, err := manager.fetchEndpointProxy(endpoint)
 	if err != nil {
-		return errors.Wrap(err, "failed to featch environment proxy")
+		return errors.Wrap(err, "failed to fetch environment proxy")
 	}
 
 	if proxy != nil {
@@ -80,9 +79,8 @@ func (manager *ComposeStackManager) Down(ctx context.Context, stack *portainer.S
 }
 
 // NormalizeStackName returns a new stack name with unsupported characters replaced
-func (w *ComposeStackManager) NormalizeStackName(name string) string {
-	r := regexp.MustCompile("[^a-z0-9]+")
-	return r.ReplaceAllString(strings.ToLower(name), "")
+func (manager *ComposeStackManager) NormalizeStackName(name string) string {
+	return stackNameNormalizeRegex.ReplaceAllString(strings.ToLower(name), "")
 }
 
 func (manager *ComposeStackManager) fetchEndpointProxy(endpoint *portainer.Endpoint) (string, *factory.ProxyServer, error) {
@@ -90,7 +88,7 @@ func (manager *ComposeStackManager) fetchEndpointProxy(endpoint *portainer.Endpo
 		return "", nil, nil
 	}
 
-	proxy, err := manager.proxyManager.CreateComposeProxyServer(endpoint)
+	proxy, err := manager.proxyManager.CreateAgentProxyServer(endpoint)
 	if err != nil {
 		return "", nil, err
 	}

api/exec/exectest/kubernetes_mocks.go

@@ -0,0 +1,23 @@
+package exectest
+
+import (
+	portainer "github.com/portainer/portainer/api"
+)
+
+type kubernetesMockDeployer struct{}
+
+func NewKubernetesDeployer() portainer.KubernetesDeployer {
+	return &kubernetesMockDeployer{}
+}
+
+func (deployer *kubernetesMockDeployer) Deploy(userID portainer.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) {
+	return "", nil
+}
+
+func (deployer *kubernetesMockDeployer) Remove(userID portainer.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) {
+	return "", nil
+}
+
+func (deployer *kubernetesMockDeployer) ConvertCompose(data []byte) ([]byte, error) {
+	return nil, nil
+}

api/exec/kubernetes_deploy.go

@@ -2,24 +2,19 @@ package exec
 
 import (
 	"bytes"
-	"encoding/json"
-	"errors"
 	"fmt"
-	"io/ioutil"
-	"net/http"
-	"net/url"
 	"os/exec"
 	"path"
 	"runtime"
 	"strings"
-	"time"
 
+	"github.com/pkg/errors"
+	"github.com/portainer/portainer/api/http/proxy"
+	"github.com/portainer/portainer/api/http/proxy/factory"
 	"github.com/portainer/portainer/api/http/proxy/factory/kubernetes"
-	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/kubernetes/cli"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/crypto"
 )
 
 // KubernetesDeployer represents a service to deploy resources inside a Kubernetes environment(endpoint).
@@ -30,10 +25,11 @@ type KubernetesDeployer struct {
 	signatureService            portainer.DigitalSignatureService
 	kubernetesClientFactory     *cli.ClientFactory
 	kubernetesTokenCacheManager *kubernetes.TokenCacheManager
+	proxyManager                *proxy.Manager
 }
 
 // NewKubernetesDeployer initializes a new KubernetesDeployer service.
-func NewKubernetesDeployer(kubernetesTokenCacheManager *kubernetes.TokenCacheManager, kubernetesClientFactory *cli.ClientFactory, datastore portainer.DataStore, reverseTunnelService portainer.ReverseTunnelService, signatureService portainer.DigitalSignatureService, binaryPath string) *KubernetesDeployer {
+func NewKubernetesDeployer(kubernetesTokenCacheManager *kubernetes.TokenCacheManager, kubernetesClientFactory *cli.ClientFactory, datastore portainer.DataStore, reverseTunnelService portainer.ReverseTunnelService, signatureService portainer.DigitalSignatureService, proxyManager *proxy.Manager, binaryPath string) *KubernetesDeployer {
 	return &KubernetesDeployer{
 		binaryPath:                  binaryPath,
 		dataStore:                   datastore,
@@ -41,32 +37,33 @@ func NewKubernetesDeployer(kubernetesTokenCacheManager *kubernetes.TokenCacheMan
 		signatureService:            signatureService,
 		kubernetesClientFactory:     kubernetesClientFactory,
 		kubernetesTokenCacheManager: kubernetesTokenCacheManager,
+		proxyManager:                proxyManager,
 	}
 }
 
-func (deployer *KubernetesDeployer) getToken(request *http.Request, endpoint *portainer.Endpoint, setLocalAdminToken bool) (string, error) {
-	tokenData, err := security.RetrieveTokenData(request)
-	if err != nil {
-		return "", err
-	}
-
-	kubecli, err := deployer.kubernetesClientFactory.GetKubeClient(endpoint)
+func (deployer *KubernetesDeployer) getToken(userID portainer.UserID, endpoint *portainer.Endpoint, setLocalAdminToken bool) (string, error) {
+	kubeCLI, err := deployer.kubernetesClientFactory.GetKubeClient(endpoint)
 	if err != nil {
 		return "", err
 	}
 
 	tokenCache := deployer.kubernetesTokenCacheManager.GetOrCreateTokenCache(int(endpoint.ID))
 
-	tokenManager, err := kubernetes.NewTokenManager(kubecli, deployer.dataStore, tokenCache, setLocalAdminToken)
+	tokenManager, err := kubernetes.NewTokenManager(kubeCLI, deployer.dataStore, tokenCache, setLocalAdminToken)
 	if err != nil {
 		return "", err
 	}
 
-	if tokenData.Role == portainer.AdministratorRole {
+	user, err := deployer.dataStore.User().User(userID)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to fetch the user")
+	}
+
+	if user.Role == portainer.AdministratorRole {
 		return tokenManager.GetAdminServiceAccountToken(), nil
 	}
 
-	token, err := tokenManager.GetUserServiceAccountToken(int(tokenData.ID), endpoint.ID)
+	token, err := tokenManager.GetUserServiceAccountToken(int(user.ID), endpoint.ID)
 	if err != nil {
 		return "", err
 	}
@@ -77,156 +74,62 @@ func (deployer *KubernetesDeployer) getToken(request *http.Request, endpoint *po
 	return token, nil
 }
 
-// Deploy will deploy a Kubernetes manifest inside a specific namespace in a Kubernetes environment(endpoint).
-// Otherwise it will use kubectl to deploy the manifest.
-func (deployer *KubernetesDeployer) Deploy(request *http.Request, endpoint *portainer.Endpoint, stackConfig string, namespace string) (string, error) {
-	if endpoint.Type == portainer.KubernetesLocalEnvironment {
-		token, err := deployer.getToken(request, endpoint, true)
-		if err != nil {
-			return "", err
-		}
+// Deploy upserts Kubernetes resources defined in manifest(s)
+func (deployer *KubernetesDeployer) Deploy(userID portainer.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) {
+	return deployer.command("apply", userID, endpoint, manifestFiles, namespace)
+}
 
-		command := path.Join(deployer.binaryPath, "kubectl")
-		if runtime.GOOS == "windows" {
-			command = path.Join(deployer.binaryPath, "kubectl.exe")
-		}
+// Remove deletes Kubernetes resources defined in manifest(s)
+func (deployer *KubernetesDeployer) Remove(userID portainer.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) {
+	return deployer.command("delete", userID, endpoint, manifestFiles, namespace)
+}
 
-		args := make([]string, 0)
-		args = append(args, "--server", endpoint.URL)
-		args = append(args, "--insecure-skip-tls-verify")
-		args = append(args, "--token", token)
+func (deployer *KubernetesDeployer) command(operation string, userID portainer.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) {
+	token, err := deployer.getToken(userID, endpoint, endpoint.Type == portainer.KubernetesLocalEnvironment)
+	if err != nil {
+		return "", errors.Wrap(err, "failed generating a user token")
+	}
+
+	command := path.Join(deployer.binaryPath, "kubectl")
+	if runtime.GOOS == "windows" {
+		command = path.Join(deployer.binaryPath, "kubectl.exe")
+	}
+
+	args := []string{"--token", token}
+	if namespace != "" {
 		args = append(args, "--namespace", namespace)
-		args = append(args, "apply", "-f", "-")
+	}
 
-		var stderr bytes.Buffer
-		cmd := exec.Command(command, args...)
-		cmd.Stderr = &stderr
-		cmd.Stdin = strings.NewReader(stackConfig)
-
-		output, err := cmd.Output()
+	if endpoint.Type == portainer.AgentOnKubernetesEnvironment || endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment {
+		url, proxy, err := deployer.getAgentURL(endpoint)
 		if err != nil {
-			return "", errors.New(stderr.String())
+			return "", errors.WithMessage(err, "failed generating endpoint URL")
 		}
 
-		return string(output), nil
+		defer proxy.Close()
+		args = append(args, "--server", url)
+		args = append(args, "--insecure-skip-tls-verify")
 	}
 
-	// agent
-
-	endpointURL := endpoint.URL
-	if endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment {
-		tunnel := deployer.reverseTunnelService.GetTunnelDetails(endpoint.ID)
-		if tunnel.Status == portainer.EdgeAgentIdle {
-
-			err := deployer.reverseTunnelService.SetTunnelStatusToRequired(endpoint.ID)
-			if err != nil {
-				return "", err
-			}
-
-			settings, err := deployer.dataStore.Settings().Settings()
-			if err != nil {
-				return "", err
-			}
-
-			waitForAgentToConnect := time.Duration(settings.EdgeAgentCheckinInterval) * time.Second
-			time.Sleep(waitForAgentToConnect * 2)
-		}
-
-		endpointURL = fmt.Sprintf("http://127.0.0.1:%d", tunnel.Port)
+	if operation == "delete" {
+		args = append(args, "--ignore-not-found=true")
 	}
 
-	transport := &http.Transport{}
-
-	if endpoint.TLSConfig.TLS {
-		tlsConfig, err := crypto.CreateTLSConfigurationFromDisk(endpoint.TLSConfig.TLSCACertPath, endpoint.TLSConfig.TLSCertPath, endpoint.TLSConfig.TLSKeyPath, endpoint.TLSConfig.TLSSkipVerify)
-		if err != nil {
-			return "", err
-		}
-		transport.TLSClientConfig = tlsConfig
+	args = append(args, operation)
+	for _, path := range manifestFiles {
+		args = append(args, "-f", strings.TrimSpace(path))
 	}
 
-	httpCli := &http.Client{
-		Transport: transport,
-	}
+	var stderr bytes.Buffer
+	cmd := exec.Command(command, args...)
+	cmd.Stderr = &stderr
 
-	if !strings.HasPrefix(endpointURL, "http") {
-		endpointURL = fmt.Sprintf("https://%s", endpointURL)
-	}
-
-	url, err := url.Parse(fmt.Sprintf("%s/v2/kubernetes/stack", endpointURL))
+	output, err := cmd.Output()
 	if err != nil {
-		return "", err
+		return "", errors.Wrapf(err, "failed to execute kubectl command: %q", stderr.String())
 	}
 
-	reqPayload, err := json.Marshal(
-		struct {
-			StackConfig string
-			Namespace   string
-		}{
-			StackConfig: stackConfig,
-			Namespace:   namespace,
-		})
-	if err != nil {
-		return "", err
-	}
-
-	req, err := http.NewRequest(http.MethodPost, url.String(), bytes.NewReader(reqPayload))
-	if err != nil {
-		return "", err
-	}
-
-	signature, err := deployer.signatureService.CreateSignature(portainer.PortainerAgentSignatureMessage)
-	if err != nil {
-		return "", err
-	}
-
-	token, err := deployer.getToken(request, endpoint, false)
-	if err != nil {
-		return "", err
-	}
-
-	req.Header.Set(portainer.PortainerAgentPublicKeyHeader, deployer.signatureService.EncodedPublicKey())
-	req.Header.Set(portainer.PortainerAgentSignatureHeader, signature)
-	req.Header.Set(portainer.PortainerAgentKubernetesSATokenHeader, token)
-
-	resp, err := httpCli.Do(req)
-	if err != nil {
-		return "", err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		var errorResponseData struct {
-			Message string
-			Details string
-		}
-		err = json.NewDecoder(resp.Body).Decode(&errorResponseData)
-		if err != nil {
-			output, parseStringErr := ioutil.ReadAll(resp.Body)
-			if parseStringErr != nil {
-				return "", parseStringErr
-			}
-
-			return "", fmt.Errorf("Failed parsing, body: %s, error: %w", output, err)
-
-		}
-
-		return "", fmt.Errorf("Deployment to agent failed: %s", errorResponseData.Details)
-	}
-
-	var responseData struct{ Output string }
-	err = json.NewDecoder(resp.Body).Decode(&responseData)
-	if err != nil {
-		parsedOutput, parseStringErr := ioutil.ReadAll(resp.Body)
-		if parseStringErr != nil {
-			return "", parseStringErr
-		}
-
-		return "", fmt.Errorf("Failed decoding, body: %s, err: %w", parsedOutput, err)
-	}
-
-	return responseData.Output, nil
-
+	return string(output), nil
 }
 
 // ConvertCompose leverages the kompose binary to deploy a compose compliant manifest.
@@ -251,3 +154,12 @@ func (deployer *KubernetesDeployer) ConvertCompose(data []byte) ([]byte, error)
 
 	return output, nil
 }
+
+func (deployer *KubernetesDeployer) getAgentURL(endpoint *portainer.Endpoint) (string, *factory.ProxyServer, error) {
+	proxy, err := deployer.proxyManager.CreateAgentProxyServer(endpoint)
+	if err != nil {
+		return "", nil, err
+	}
+
+	return fmt.Sprintf("http://127.0.0.1:%d/kubernetes", proxy.Port), proxy, nil
+}

api/exec/swarm_stack.go

@@ -8,7 +8,6 @@ import (
 	"os"
 	"os/exec"
 	"path"
-	"regexp"
 	"runtime"
 	"strings"
 
@@ -190,8 +189,7 @@ func (manager *SwarmStackManager) retrieveConfigurationFromDisk(path string) (ma
 }
 
 func (manager *SwarmStackManager) NormalizeStackName(name string) string {
-	r := regexp.MustCompile("[^a-z0-9]+")
-	return r.ReplaceAllString(strings.ToLower(name), "")
+	return stackNameNormalizeRegex.ReplaceAllString(strings.ToLower(name), "")
 }
 
 func configureFilePaths(args []string, filePaths []string) []string {

api/filesystem/write.go

@@ -0,0 +1,23 @@
+package filesystem
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+)
+
+func WriteToFile(dst string, content []byte) error {
+	if err := os.MkdirAll(filepath.Dir(dst), 0744); err != nil {
+		return errors.Wrapf(err, "failed to create filestructure for the path %q", dst)
+	}
+
+	file, err := os.Create(dst)
+	if err != nil {
+		return errors.Wrapf(err, "failed to open a file %q", dst)
+	}
+	defer file.Close()
+
+	_, err = file.Write(content)
+	return errors.Wrapf(err, "failed to write a file %q", dst)
+}

api/filesystem/write_test.go

@@ -0,0 +1,48 @@
+package filesystem
+
+import (
+	"io/ioutil"
+	"path"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_WriteFile_CanStoreContentInANewFile(t *testing.T) {
+	tmpDir := t.TempDir()
+	tmpFilePath := path.Join(tmpDir, "dummy")
+
+	content := []byte("content")
+	err := WriteToFile(tmpFilePath, content)
+	assert.NoError(t, err)
+
+	fileContent, _ := ioutil.ReadFile(tmpFilePath)
+	assert.Equal(t, content, fileContent)
+}
+
+func Test_WriteFile_CanOverwriteExistingFile(t *testing.T) {
+	tmpDir := t.TempDir()
+	tmpFilePath := path.Join(tmpDir, "dummy")
+
+	err := WriteToFile(tmpFilePath, []byte("content"))
+	assert.NoError(t, err)
+
+	content := []byte("new content")
+	err = WriteToFile(tmpFilePath, content)
+	assert.NoError(t, err)
+
+	fileContent, _ := ioutil.ReadFile(tmpFilePath)
+	assert.Equal(t, content, fileContent)
+}
+
+func Test_WriteFile_CanWriteANestedPath(t *testing.T) {
+	tmpDir := t.TempDir()
+	tmpFilePath := path.Join(tmpDir, "dir", "sub-dir", "dummy")
+
+	content := []byte("content")
+	err := WriteToFile(tmpFilePath, content)
+	assert.NoError(t, err)
+
+	fileContent, _ := ioutil.ReadFile(tmpFilePath)
+	assert.Equal(t, content, fileContent)
+}

api/go.mod

@@ -38,7 +38,7 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/portainer/docker-compose-wrapper v0.0.0-20210909083948-8be0d98451a1
 	github.com/portainer/libcrypto v0.0.0-20210422035235-c652195c5c3a
-	github.com/portainer/libhelm v0.0.0-20210906035629-b5635edd5d97
+	github.com/portainer/libhelm v0.0.0-20210929000907-825e93d62108
 	github.com/portainer/libhttp v0.0.0-20190806161843-ba068f58be33
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/sirupsen/logrus v1.8.1
@@ -46,6 +46,7 @@ require (
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
 	golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 	gotest.tools v2.2.0+incompatible // indirect

api/go.sum

@@ -206,14 +206,12 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/portainer/docker-compose-wrapper v0.0.0-20210909011155-9ff375eac059 h1:98v0k3x3ZXa09NaHP/HmSA83rcN8cuE/zTKo6xvNmoM=
-github.com/portainer/docker-compose-wrapper v0.0.0-20210909011155-9ff375eac059/go.mod h1:WxDlJWZxCnicdLCPnLNEv7/gRhjeIVuCGmsv+iOPH3c=
 github.com/portainer/docker-compose-wrapper v0.0.0-20210909083948-8be0d98451a1 h1:0ZGSu3Atz7RHMDsoITHV676igRfsb51mlgELGo37ELU=
 github.com/portainer/docker-compose-wrapper v0.0.0-20210909083948-8be0d98451a1/go.mod h1:WxDlJWZxCnicdLCPnLNEv7/gRhjeIVuCGmsv+iOPH3c=
 github.com/portainer/libcrypto v0.0.0-20210422035235-c652195c5c3a h1:qY8TbocN75n5PDl16o0uVr5MevtM5IhdwSelXEd4nFM=
 github.com/portainer/libcrypto v0.0.0-20210422035235-c652195c5c3a/go.mod h1:n54EEIq+MM0NNtqLeCby8ljL+l275VpolXO0ibHegLE=
-github.com/portainer/libhelm v0.0.0-20210906035629-b5635edd5d97 h1:ZcRVgWHTac8V7WU9TUBr73H3e5ajVFYTPjPl9TWULDA=
-github.com/portainer/libhelm v0.0.0-20210906035629-b5635edd5d97/go.mod h1:YvYAk7krKTzB+rFwDr0jQ3sQu2BtiXK1AR0sZH7nhJA=
+github.com/portainer/libhelm v0.0.0-20210929000907-825e93d62108 h1:5e8KAnDa2G3cEHK7aV/ue8lOaoQwBZUzoALslwWkR04=
+github.com/portainer/libhelm v0.0.0-20210929000907-825e93d62108/go.mod h1:YvYAk7krKTzB+rFwDr0jQ3sQu2BtiXK1AR0sZH7nhJA=
 github.com/portainer/libhttp v0.0.0-20190806161843-ba068f58be33 h1:H8HR2dHdBf8HANSkUyVw4o8+4tegGcd+zyKZ3e599II=
 github.com/portainer/libhttp v0.0.0-20190806161843-ba068f58be33/go.mod h1:Y2TfgviWI4rT2qaOTHr+hq6MdKIE5YjgQAu7qwptTV0=
 github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
@@ -278,6 +276,8 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=

api/http/handler/customtemplates/customtemplate_create.go

@@ -3,6 +3,7 @@ package customtemplates
 import (
 	"errors"
 	"net/http"
+	"regexp"
 	"strconv"
 
 	"github.com/asaskevich/govalidator"
@@ -129,9 +130,20 @@ func (payload *customTemplateFromFileContentPayload) Validate(r *http.Request) e
 	if payload.Type != portainer.KubernetesStack && payload.Type != portainer.DockerSwarmStack && payload.Type != portainer.DockerComposeStack {
 		return errors.New("Invalid custom template type")
 	}
+	if !isValidNote(payload.Note) {
+		return errors.New("Invalid note. <img> tag is not supported")
+	}
 	return nil
 }
 
+func isValidNote(note string) bool {
+	if govalidator.IsNull(note) {
+		return true
+	}
+	match, _ := regexp.MatchString("<img", note)
+	return !match
+}
+
 func (handler *Handler) createCustomTemplateFromFileContent(r *http.Request) (*portainer.CustomTemplate, error) {
 	var payload customTemplateFromFileContentPayload
 	err := request.DecodeAndValidateJSONPayload(r, &payload)
@@ -218,6 +230,9 @@ func (payload *customTemplateFromGitRepositoryPayload) Validate(r *http.Request)
 	if payload.Type != portainer.DockerSwarmStack && payload.Type != portainer.DockerComposeStack {
 		return errors.New("Invalid custom template type")
 	}
+	if !isValidNote(payload.Note) {
+		return errors.New("Invalid note. <img> tag is not supported")
+	}
 	return nil
 }
 
@@ -285,6 +300,9 @@ func (payload *customTemplateFromFileUploadPayload) Validate(r *http.Request) er
 	payload.Logo = logo
 
 	note, _ := request.RetrieveMultiPartFormValue(r, "Note", true)
+	if !isValidNote(note) {
+		return errors.New("Invalid note. <img> tag is not supported")
+	}
 	payload.Note = note
 
 	typeNumeral, _ := request.RetrieveNumericMultiPartFormValue(r, "Type", true)

api/http/handler/customtemplates/customtemplate_update.go

@@ -51,6 +51,9 @@ func (payload *customTemplateUpdatePayload) Validate(r *http.Request) error {
 	if govalidator.IsNull(payload.Description) {
 		return errors.New("Invalid custom template description")
 	}
+	if !isValidNote(payload.Note) {
+		return errors.New("Invalid note. <img> tag is not supported")
+	}
 	return nil
 }
 

api/http/handler/handler.go

@@ -18,6 +18,7 @@ import (
 	"github.com/portainer/portainer/api/http/handler/file"
 	"github.com/portainer/portainer/api/http/handler/helm"
 	"github.com/portainer/portainer/api/http/handler/kubernetes"
+	"github.com/portainer/portainer/api/http/handler/ldap"
 	"github.com/portainer/portainer/api/http/handler/motd"
 	"github.com/portainer/portainer/api/http/handler/registries"
 	"github.com/portainer/portainer/api/http/handler/resourcecontrols"
@@ -53,6 +54,7 @@ type Handler struct {
 	HelmTemplatesHandler   *helm.Handler
 	KubernetesHandler      *kubernetes.Handler
 	FileHandler            *file.Handler
+	LDAPHandler            *ldap.Handler
 	MOTDHandler            *motd.Handler
 	RegistryHandler        *registries.Handler
 	ResourceControlHandler *resourcecontrols.Handler
@@ -72,7 +74,7 @@ type Handler struct {
 }
 
 // @title PortainerCE API
-// @version 2.9.0
+// @version 2.9.3
 // @description.markdown api-description.md
 // @termsOfService
 
@@ -189,6 +191,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		default:
 			http.StripPrefix("/api", h.EndpointHandler).ServeHTTP(w, r)
 		}
+	case strings.HasPrefix(r.URL.Path, "/api/ldap"):
+		http.StripPrefix("/api", h.LDAPHandler).ServeHTTP(w, r)
 	case strings.HasPrefix(r.URL.Path, "/api/motd"):
 		http.StripPrefix("/api", h.MOTDHandler).ServeHTTP(w, r)
 	case strings.HasPrefix(r.URL.Path, "/api/registries"):

api/http/handler/helm/handler.go

@@ -27,15 +27,17 @@ type Handler struct {
 	requestBouncer     requestBouncer
 	dataStore          portainer.DataStore
 	kubeConfigService  kubernetes.KubeConfigService
+	kubernetesDeployer portainer.KubernetesDeployer
 	helmPackageManager libhelm.HelmPackageManager
 }
 
-// NewHandler creates a handler to manage environment(endpoint) group operations.
-func NewHandler(bouncer requestBouncer, dataStore portainer.DataStore, helmPackageManager libhelm.HelmPackageManager, kubeConfigService kubernetes.KubeConfigService) *Handler {
+// NewHandler creates a handler to manage endpoint group operations.
+func NewHandler(bouncer requestBouncer, dataStore portainer.DataStore, kubernetesDeployer portainer.KubernetesDeployer, helmPackageManager libhelm.HelmPackageManager, kubeConfigService kubernetes.KubeConfigService) *Handler {
 	h := &Handler{
 		Router:             mux.NewRouter(),
 		requestBouncer:     bouncer,
 		dataStore:          dataStore,
+		kubernetesDeployer: kubernetesDeployer,
 		helmPackageManager: helmPackageManager,
 		kubeConfigService:  kubeConfigService,
 	}

api/http/handler/helm/helm_delete_test.go

@@ -9,11 +9,12 @@ import (
 	"github.com/portainer/libhelm/binary/test"
 	"github.com/portainer/libhelm/options"
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/exec/exectest"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/kubernetes"
 	"github.com/stretchr/testify/assert"
 
-	bolt "github.com/portainer/portainer/api/bolt/bolttest"
+	"github.com/portainer/portainer/api/bolt"
 	helper "github.com/portainer/portainer/api/internal/testhelpers"
 )
 
@@ -29,9 +30,10 @@ func Test_helmDelete(t *testing.T) {
 	err = store.User().CreateUser(&portainer.User{Username: "admin", Role: portainer.AdministratorRole})
 	is.NoError(err, "Error creating a user")
 
+	kubernetesDeployer := exectest.NewKubernetesDeployer()
 	helmPackageManager := test.NewMockHelmBinaryPackageManager("")
 	kubeConfigService := kubernetes.NewKubeConfigCAService("", "")
-	h := NewHandler(helper.NewTestRequestBouncer(), store, helmPackageManager, kubeConfigService)
+	h := NewHandler(helper.NewTestRequestBouncer(), store, kubernetesDeployer, helmPackageManager, kubeConfigService)
 
 	is.NotNil(h, "Handler should not fail")
 

api/http/handler/helm/helm_install.go

@@ -1,18 +1,23 @@
 package helm
 
 import (
-	"errors"
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"os"
 	"strings"
 
+	"github.com/pkg/errors"
 	"github.com/portainer/libhelm/options"
 	"github.com/portainer/libhelm/release"
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
+	"github.com/portainer/portainer/api/http/middlewares"
+	"github.com/portainer/portainer/api/http/security"
+	"github.com/portainer/portainer/api/kubernetes"
 	"github.com/portainer/portainer/api/kubernetes/validation"
+	"golang.org/x/sync/errgroup"
 )
 
 type installChartPayload struct {
@@ -131,5 +136,98 @@ func (handler *Handler) installChart(r *http.Request, p installChartPayload) (*r
 	if err != nil {
 		return nil, err
 	}
+
+	manifest, err := handler.applyPortainerLabelsToHelmAppManifest(r, installOpts, release.Manifest)
+	if err != nil {
+		return nil, err
+	}
+
+	err = handler.updateHelmAppManifest(r, manifest, installOpts.Namespace)
+	if err != nil {
+		return nil, err
+	}
+
 	return release, nil
 }
+
+// applyPortainerLabelsToHelmAppManifest will patch all the resources deployed in the helm release manifest
+// with portainer specific labels. This is to mark the resources as managed by portainer - hence the helm apps
+// wont appear external in the portainer UI.
+func (handler *Handler) applyPortainerLabelsToHelmAppManifest(r *http.Request, installOpts options.InstallOptions, manifest string) ([]byte, error) {
+	// Patch helm release by adding with portainer labels to all deployed resources
+	tokenData, err := security.RetrieveTokenData(r)
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to retrieve user details from authentication token")
+	}
+	user, err := handler.dataStore.User().User(tokenData.ID)
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to load user information from the database")
+	}
+
+	appLabels := kubernetes.GetHelmAppLabels(installOpts.Name, user.Username)
+	labeledManifest, err := kubernetes.AddAppLabels([]byte(manifest), appLabels)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to label helm release manifest")
+	}
+
+	return labeledManifest, nil
+}
+
+// updateHelmAppManifest will update the resources of helm release manifest with portainer labels using kubectl.
+// The resources of the manifest will be updated in parallel and individuallly since resources of a chart
+// can be deployed to different namespaces.
+// NOTE: These updates will need to be re-applied when upgrading the helm release
+func (handler *Handler) updateHelmAppManifest(r *http.Request, manifest []byte, namespace string) error {
+	endpoint, err := middlewares.FetchEndpoint(r)
+	if err != nil {
+		return errors.Wrap(err, "unable to find an endpoint on request context")
+	}
+
+	tokenData, err := security.RetrieveTokenData(r)
+	if err != nil {
+		return errors.Wrap(err, "unable to retrieve user details from authentication token")
+	}
+
+	// extract list of yaml resources from helm manifest
+	yamlResources, err := kubernetes.ExtractDocuments(manifest, nil)
+	if err != nil {
+		return errors.Wrap(err, "unable to extract documents from helm release manifest")
+	}
+
+	// deploy individual resources in parallel
+	g := new(errgroup.Group)
+	for _, resource := range yamlResources {
+		resource := resource // https://golang.org/doc/faq#closures_and_goroutines
+		g.Go(func() error {
+			tmpfile, err := ioutil.TempFile("", "helm-manifest-*")
+			if err != nil {
+				return errors.Wrap(err, "failed to create a tmp helm manifest file")
+			}
+			defer func() {
+				tmpfile.Close()
+				os.Remove(tmpfile.Name())
+			}()
+
+			if _, err := tmpfile.Write(resource); err != nil {
+				return errors.Wrap(err, "failed to write a tmp helm manifest file")
+			}
+
+			// get resource namespace, fallback to provided namespace if not explicit on resource
+			resourceNamespace, err := kubernetes.GetNamespace(resource)
+			if err != nil {
+				return err
+			}
+			if resourceNamespace == "" {
+				resourceNamespace = namespace
+			}
+
+			_, err = handler.kubernetesDeployer.Deploy(tokenData.ID, endpoint, []string{tmpfile.Name()}, resourceNamespace)
+			return err
+		})
+	}
+	if err := g.Wait(); err != nil {
+		return errors.Wrap(err, "unable to patch helm release using kubectl")
+	}
+
+	return nil
+}

api/http/handler/helm/helm_install_test.go

@@ -12,7 +12,8 @@ import (
 	"github.com/portainer/libhelm/options"
 	"github.com/portainer/libhelm/release"
 	portainer "github.com/portainer/portainer/api"
-	bolt "github.com/portainer/portainer/api/bolt/bolttest"
+	"github.com/portainer/portainer/api/bolt"
+	"github.com/portainer/portainer/api/exec/exectest"
 	"github.com/portainer/portainer/api/http/security"
 	helper "github.com/portainer/portainer/api/internal/testhelpers"
 	"github.com/portainer/portainer/api/kubernetes"
@@ -31,9 +32,10 @@ func Test_helmInstall(t *testing.T) {
 	err = store.User().CreateUser(&portainer.User{Username: "admin", Role: portainer.AdministratorRole})
 	is.NoError(err, "error creating a user")
 
+	kubernetesDeployer := exectest.NewKubernetesDeployer()
 	helmPackageManager := test.NewMockHelmBinaryPackageManager("")
 	kubeConfigService := kubernetes.NewKubeConfigCAService("", "")
-	h := NewHandler(helper.NewTestRequestBouncer(), store, helmPackageManager, kubeConfigService)
+	h := NewHandler(helper.NewTestRequestBouncer(), store, kubernetesDeployer, helmPackageManager, kubeConfigService)
 
 	is.NotNil(h, "Handler should not fail")
 

api/http/handler/helm/helm_list_test.go

@@ -11,10 +11,11 @@ import (
 	"github.com/portainer/libhelm/options"
 	"github.com/portainer/libhelm/release"
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/exec/exectest"
 	"github.com/portainer/portainer/api/kubernetes"
 	"github.com/stretchr/testify/assert"
 
-	bolt "github.com/portainer/portainer/api/bolt/bolttest"
+	"github.com/portainer/portainer/api/bolt"
 	helper "github.com/portainer/portainer/api/internal/testhelpers"
 )
 
@@ -28,9 +29,10 @@ func Test_helmList(t *testing.T) {
 	err = store.User().CreateUser(&portainer.User{Username: "admin", Role: portainer.AdministratorRole})
 	assert.NoError(t, err, "error creating a user")
 
+	kubernetesDeployer := exectest.NewKubernetesDeployer()
 	helmPackageManager := test.NewMockHelmBinaryPackageManager("")
 	kubeConfigService := kubernetes.NewKubeConfigCAService("", "")
-	h := NewHandler(helper.NewTestRequestBouncer(), store, helmPackageManager, kubeConfigService)
+	h := NewHandler(helper.NewTestRequestBouncer(), store, kubernetesDeployer, helmPackageManager, kubeConfigService)
 
 	// Install a single chart.  We expect to get these values back
 	options := options.InstallOptions{Name: "nginx-1", Chart: "nginx", Namespace: "default"}

api/http/handler/helm/helm_repo_search.go

@@ -6,7 +6,6 @@ import (
 	"net/url"
 
 	"github.com/pkg/errors"
-	"github.com/portainer/libhelm"
 	"github.com/portainer/libhelm/options"
 	httperror "github.com/portainer/libhttp/error"
 )
@@ -40,7 +39,7 @@ func (handler *Handler) helmRepoSearch(w http.ResponseWriter, r *http.Request) *
 		Repo: repo,
 	}
 
-	result, err := libhelm.SearchRepo(searchOpts)
+	result, err := handler.helmPackageManager.SearchRepo(searchOpts)
 	if err != nil {
 		return &httperror.HandlerError{
 			StatusCode: http.StatusInternalServerError,

api/http/handler/helm/helm_repo_search_test.go

@@ -9,13 +9,12 @@ import (
 	"testing"
 
 	"github.com/portainer/libhelm/binary/test"
-	"github.com/stretchr/testify/assert"
 
 	helper "github.com/portainer/portainer/api/internal/testhelpers"
+	"github.com/stretchr/testify/assert"
 )
 
 func Test_helmRepoSearch(t *testing.T) {
-	helper.IntegrationTest(t)
 	is := assert.New(t)
 
 	helmPackageManager := test.NewMockHelmBinaryPackageManager("")

api/http/handler/ldap/handler.go

@@ -0,0 +1,53 @@
+package ldap
+
+import (
+	"net/http"
+
+	"github.com/gorilla/mux"
+	httperror "github.com/portainer/libhttp/error"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/filesystem"
+	"github.com/portainer/portainer/api/http/security"
+)
+
+// Handler is the HTTP handler used to handle LDAP search Operations
+type Handler struct {
+	*mux.Router
+	DataStore   portainer.DataStore
+	FileService portainer.FileService
+	LDAPService portainer.LDAPService
+}
+
+// NewHandler returns a new Handler
+func NewHandler(bouncer *security.RequestBouncer) *Handler {
+	h := &Handler{
+		Router: mux.NewRouter(),
+	}
+
+	h.Handle("/ldap/check",
+		bouncer.AdminAccess(httperror.LoggerHandler(h.ldapCheck))).Methods(http.MethodPost)
+
+	return h
+}
+
+func (handler *Handler) prefillSettings(ldapSettings *portainer.LDAPSettings) error {
+	if !ldapSettings.AnonymousMode && ldapSettings.Password == "" {
+		settings, err := handler.DataStore.Settings().Settings()
+		if err != nil {
+			return err
+		}
+
+		ldapSettings.Password = settings.LDAPSettings.Password
+	}
+
+	if (ldapSettings.TLSConfig.TLS || ldapSettings.StartTLS) && !ldapSettings.TLSConfig.TLSSkipVerify {
+		caCertPath, err := handler.FileService.GetPathForTLSFile(filesystem.LDAPStorePath, portainer.TLSFileCA)
+		if err != nil {
+			return err
+		}
+
+		ldapSettings.TLSConfig.TLSCACertPath = caCertPath
+	}
+
+	return nil
+}

api/http/handler/ldap/ldap_check.go

@@ -1,4 +1,4 @@
-package settings
+package ldap
 
 import (
 	"net/http"
@@ -7,42 +7,43 @@ import (
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/filesystem"
 )
 
-type settingsLDAPCheckPayload struct {
+type checkPayload struct {
 	LDAPSettings portainer.LDAPSettings
 }
 
-func (payload *settingsLDAPCheckPayload) Validate(r *http.Request) error {
+func (payload *checkPayload) Validate(r *http.Request) error {
 	return nil
 }
 
-// @id SettingsLDAPCheck
+// @id LDAPCheck
 // @summary Test LDAP connectivity
 // @description Test LDAP connectivity using LDAP details
 // @description **Access policy**: administrator
-// @tags settings
+// @tags ldap
 // @security jwt
 // @accept json
-// @param body body settingsLDAPCheckPayload true "details"
+// @param body body checkPayload true "details"
 // @success 204 "Success"
 // @failure 400 "Invalid request"
 // @failure 500 "Server error"
-// @router /settings/ldap/check [put]
-func (handler *Handler) settingsLDAPCheck(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	var payload settingsLDAPCheckPayload
+// @router /ldap/check [post]
+func (handler *Handler) ldapCheck(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	var payload checkPayload
 	err := request.DecodeAndValidateJSONPayload(r, &payload)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
 	}
 
-	if (payload.LDAPSettings.TLSConfig.TLS || payload.LDAPSettings.StartTLS) && !payload.LDAPSettings.TLSConfig.TLSSkipVerify {
-		caCertPath, _ := handler.FileService.GetPathForTLSFile(filesystem.LDAPStorePath, portainer.TLSFileCA)
-		payload.LDAPSettings.TLSConfig.TLSCACertPath = caCertPath
+	settings := &payload.LDAPSettings
+
+	err = handler.prefillSettings(settings)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to fetch default settings", err}
 	}
 
-	err = handler.LDAPService.TestConnectivity(&payload.LDAPSettings)
+	err = handler.LDAPService.TestConnectivity(settings)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to connect to LDAP server", err}
 	}

api/http/handler/settings/handler.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/gorilla/mux"
 	httperror "github.com/portainer/libhttp/error"
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/http/security"
 )
 
@@ -35,8 +35,6 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler {
 		bouncer.AdminAccess(httperror.LoggerHandler(h.settingsUpdate))).Methods(http.MethodPut)
 	h.Handle("/settings/public",
 		bouncer.PublicAccess(httperror.LoggerHandler(h.settingsPublic))).Methods(http.MethodGet)
-	h.Handle("/settings/authentication/checkLDAP",
-		bouncer.AdminAccess(httperror.LoggerHandler(h.settingsLDAPCheck))).Methods(http.MethodPut)
 
 	return h
 }

api/http/handler/settings/settings_update.go

@@ -40,6 +40,8 @@ type settingsUpdatePayload struct {
 	EnableTelemetry *bool `example:"false"`
 	// Helm repository URL
 	HelmRepositoryURL *string `example:"https://charts.bitnami.com/bitnami"`
+	// Kubectl Shell Image
+	KubectlShellImage *string `example:"portainer/kubectl-shell:latest"`
 }
 
 func (payload *settingsUpdatePayload) Validate(r *http.Request) error {
@@ -52,11 +54,8 @@ func (payload *settingsUpdatePayload) Validate(r *http.Request) error {
 	if payload.TemplatesURL != nil && *payload.TemplatesURL != "" && !govalidator.IsURL(*payload.TemplatesURL) {
 		return errors.New("Invalid external templates URL. Must correspond to a valid URL format")
 	}
-	if payload.HelmRepositoryURL != nil && *payload.HelmRepositoryURL != "" {
-		err := libhelm.ValidateHelmRepositoryURL(*payload.HelmRepositoryURL)
-		if err != nil {
-			return errors.Wrap(err, "Invalid Helm repository URL. Must correspond to a valid URL format")
-		}
+	if payload.HelmRepositoryURL != nil && *payload.HelmRepositoryURL != "" && !govalidator.IsURL(*payload.HelmRepositoryURL) {
+		return errors.New("Invalid Helm repository URL. Must correspond to a valid URL format")
 	}
 	if payload.UserSessionTimeout != nil {
 		_, err := time.ParseDuration(*payload.UserSessionTimeout)
@@ -112,7 +111,16 @@ func (handler *Handler) settingsUpdate(w http.ResponseWriter, r *http.Request) *
 	}
 
 	if payload.HelmRepositoryURL != nil {
-		settings.HelmRepositoryURL = strings.TrimSuffix(strings.ToLower(*payload.HelmRepositoryURL), "/")
+		newHelmRepo := strings.TrimSuffix(strings.ToLower(*payload.HelmRepositoryURL), "/")
+
+		if newHelmRepo != settings.HelmRepositoryURL && newHelmRepo != portainer.DefaultHelmRepositoryURL {
+			err := libhelm.ValidateHelmRepositoryURL(*payload.HelmRepositoryURL)
+			if err != nil {
+				return &httperror.HandlerError{http.StatusBadRequest, "Invalid Helm repository URL. Must correspond to a valid URL format", err}
+			}
+		}
+
+		settings.HelmRepositoryURL = newHelmRepo
 	}
 
 	if payload.BlackListedLabels != nil {
@@ -178,6 +186,10 @@ func (handler *Handler) settingsUpdate(w http.ResponseWriter, r *http.Request) *
 		return tlsError
 	}
 
+	if payload.KubectlShellImage != nil {
+		settings.KubectlShellImage = *payload.KubectlShellImage
+	}
+
 	err = handler.DataStore.Settings().UpdateSettings(settings)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist settings changes inside the database", err}

api/http/handler/stacks/autoupdate.go

@@ -17,10 +17,8 @@ func startAutoupdate(stackID portainer.StackID, interval string, scheduler *sche
 		return "", &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Unable to parse stack's auto update interval", Err: err}
 	}
 
-	jobID = scheduler.StartJobEvery(d, func() {
-		if err := stacks.RedeployWhenChanged(stackID, stackDeployer, datastore, gitService); err != nil {
-			log.Printf("[ERROR] [http,stacks] [message: failed redeploying] [err: %s]\n", err)
-		}
+	jobID = scheduler.StartJobEvery(d, func() error {
+		return stacks.RedeployWhenChanged(stackID, stackDeployer, datastore, gitService)
 	})
 
 	return jobID, nil

api/http/handler/stacks/create_compose_stack.go

@@ -46,13 +46,12 @@ func (handler *Handler) createComposeStackFromFileContent(w http.ResponseWriter,
 
 	payload.Name = handler.ComposeStackManager.NormalizeStackName(payload.Name)
 
-	isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, false)
+	isUnique, err := handler.checkUniqueStackNameInDocker(endpoint, payload.Name, 0, false)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to check for name collision", err}
 	}
 	if !isUnique {
-		errorMessage := fmt.Sprintf("A stack with the name '%s' is already running", payload.Name)
-		return &httperror.HandlerError{http.StatusConflict, errorMessage, errors.New(errorMessage)}
+		return stackExistsError(payload.Name)
 	}
 
 	stackID := handler.DataStore.Stack().GetNextIdentifier()
@@ -152,12 +151,12 @@ func (handler *Handler) createComposeStackFromGitRepository(w http.ResponseWrite
 		payload.ComposeFile = filesystem.ComposeFileDefaultName
 	}
 
-	isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, false)
+	isUnique, err := handler.checkUniqueStackNameInDocker(endpoint, payload.Name, 0, false)
 	if err != nil {
 		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err}
 	}
 	if !isUnique {
-		return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A stack with the name '%s' already exists", payload.Name), Err: errStackAlreadyExists}
+		return stackExistsError(payload.Name)
 	}
 
 	//make sure the webhook ID is unique
@@ -208,11 +207,11 @@ func (handler *Handler) createComposeStackFromGitRepository(w http.ResponseWrite
 		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to clone git repository", Err: err}
 	}
 
-	commitId, err := handler.latestCommitID(payload.RepositoryURL, payload.RepositoryReferenceName, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword)
+	commitID, err := handler.latestCommitID(payload.RepositoryURL, payload.RepositoryReferenceName, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword)
 	if err != nil {
 		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to fetch git repository id", Err: err}
 	}
-	stack.GitConfig.ConfigHash = commitId
+	stack.GitConfig.ConfigHash = commitID
 
 	config, configErr := handler.createComposeDeployConfig(r, stack, endpoint)
 	if configErr != nil {
@@ -281,13 +280,12 @@ func (handler *Handler) createComposeStackFromFileUpload(w http.ResponseWriter,
 
 	payload.Name = handler.ComposeStackManager.NormalizeStackName(payload.Name)
 
-	isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, false)
+	isUnique, err := handler.checkUniqueStackNameInDocker(endpoint, payload.Name, 0, false)
 	if err != nil {
 		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err}
 	}
 	if !isUnique {
-		errorMessage := fmt.Sprintf("A stack with the name '%s' already exists", payload.Name)
-		return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: errorMessage, Err: errors.New(errorMessage)}
+		return stackExistsError(payload.Name)
 	}
 
 	stackID := handler.DataStore.Stack().GetNextIdentifier()

api/http/handler/stacks/create_kubernetes_stack.go

@@ -2,9 +2,8 @@ package stacks
 
 import (
 	"fmt"
-	"io/ioutil"
 	"net/http"
-	"path/filepath"
+	"os"
 	"strconv"
 	"time"
 
@@ -19,16 +18,19 @@ import (
 	"github.com/portainer/portainer/api/filesystem"
 	gittypes "github.com/portainer/portainer/api/git/types"
 	"github.com/portainer/portainer/api/http/client"
+	"github.com/portainer/portainer/api/internal/stackutils"
 	k "github.com/portainer/portainer/api/kubernetes"
 )
 
 type kubernetesStringDeploymentPayload struct {
+	StackName        string
 	ComposeFormat    bool
 	Namespace        string
 	StackFileContent string
 }
 
 type kubernetesGitDeploymentPayload struct {
+	StackName                string
 	ComposeFormat            bool
 	Namespace                string
 	RepositoryURL            string
@@ -36,10 +38,13 @@ type kubernetesGitDeploymentPayload struct {
 	RepositoryAuthentication bool
 	RepositoryUsername       string
 	RepositoryPassword       string
-	FilePathInRepository     string
+	ManifestFile             string
+	AdditionalFiles          []string
+	AutoUpdate               *portainer.StackAutoUpdate
 }
 
 type kubernetesManifestURLDeploymentPayload struct {
+	StackName     string
 	Namespace     string
 	ComposeFormat bool
 	ManifestURL   string
@@ -52,6 +57,9 @@ func (payload *kubernetesStringDeploymentPayload) Validate(r *http.Request) erro
 	if govalidator.IsNull(payload.Namespace) {
 		return errors.New("Invalid namespace")
 	}
+	if govalidator.IsNull(payload.StackName) {
+		return errors.New("Invalid stack name")
+	}
 	return nil
 }
 
@@ -65,12 +73,18 @@ func (payload *kubernetesGitDeploymentPayload) Validate(r *http.Request) error {
 	if payload.RepositoryAuthentication && govalidator.IsNull(payload.RepositoryPassword) {
 		return errors.New("Invalid repository credentials. Password must be specified when authentication is enabled")
 	}
-	if govalidator.IsNull(payload.FilePathInRepository) {
-		return errors.New("Invalid file path in repository")
+	if govalidator.IsNull(payload.ManifestFile) {
+		return errors.New("Invalid manifest file in repository")
 	}
 	if govalidator.IsNull(payload.RepositoryReferenceName) {
 		payload.RepositoryReferenceName = defaultGitReferenceName
 	}
+	if err := validateStackAutoUpdate(payload.AutoUpdate); err != nil {
+		return err
+	}
+	if govalidator.IsNull(payload.StackName) {
+		return errors.New("Invalid stack name")
+	}
 	return nil
 }
 
@@ -78,6 +92,9 @@ func (payload *kubernetesManifestURLDeploymentPayload) Validate(r *http.Request)
 	if govalidator.IsNull(payload.ManifestURL) || !govalidator.IsURL(payload.ManifestURL) {
 		return errors.New("Invalid manifest URL")
 	}
+	if govalidator.IsNull(payload.StackName) {
+		return errors.New("Invalid stack name")
+	}
 	return nil
 }
 
@@ -95,6 +112,13 @@ func (handler *Handler) createKubernetesStackFromFileContent(w http.ResponseWrit
 	if err != nil {
 		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to load user information from the database", Err: err}
 	}
+	isUnique, err := handler.checkUniqueStackName(endpoint, payload.StackName, 0)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err}
+	}
+	if !isUnique {
+		return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A stack with the name '%s' already exists", payload.StackName), Err: errStackAlreadyExists}
+	}
 
 	stackID := handler.DataStore.Stack().GetNextIdentifier()
 	stack := &portainer.Stack{
@@ -102,6 +126,7 @@ func (handler *Handler) createKubernetesStackFromFileContent(w http.ResponseWrit
 		Type:            portainer.KubernetesStack,
 		EndpointID:      endpoint.ID,
 		EntryPoint:      filesystem.ManifestFileDefaultName,
+		Name:            payload.StackName,
 		Namespace:       payload.Namespace,
 		Status:          portainer.StackStatusActive,
 		CreationDate:    time.Now().Unix(),
@@ -124,11 +149,11 @@ func (handler *Handler) createKubernetesStackFromFileContent(w http.ResponseWrit
 	doCleanUp := true
 	defer handler.cleanUp(stack, &doCleanUp)
 
-	output, err := handler.deployKubernetesStack(r, endpoint, payload.StackFileContent, payload.ComposeFormat, payload.Namespace, k.KubeAppLabels{
-		StackID: stackID,
-		Name:    stack.Name,
-		Owner:   stack.CreatedBy,
-		Kind:    "content",
+	output, err := handler.deployKubernetesStack(user.ID, endpoint, stack, k.KubeAppLabels{
+		StackID:   stackID,
+		StackName: stack.Name,
+		Owner:     stack.CreatedBy,
+		Kind:      "content",
 	})
 
 	if err != nil {
@@ -140,12 +165,11 @@ func (handler *Handler) createKubernetesStackFromFileContent(w http.ResponseWrit
 		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist the Kubernetes stack inside the database", Err: err}
 	}
 
-	doCleanUp = false
-
 	resp := &createKubernetesStackResponse{
 		Output: output,
 	}
 
+	doCleanUp = false
 	return response.JSON(w, resp)
 }
 
@@ -159,23 +183,44 @@ func (handler *Handler) createKubernetesStackFromGitRepository(w http.ResponseWr
 	if err != nil {
 		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to load user information from the database", Err: err}
 	}
+	isUnique, err := handler.checkUniqueStackName(endpoint, payload.StackName, 0)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err}
+	}
+	if !isUnique {
+		return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A stack with the name '%s' already exists", payload.StackName), Err: errStackAlreadyExists}
+	}
+
+	//make sure the webhook ID is unique
+	if payload.AutoUpdate != nil && payload.AutoUpdate.Webhook != "" {
+		isUnique, err := handler.checkUniqueWebhookID(payload.AutoUpdate.Webhook)
+		if err != nil {
+			return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for webhook ID collision", Err: err}
+		}
+		if !isUnique {
+			return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("Webhook ID: %s already exists", payload.AutoUpdate.Webhook), Err: errWebhookIDAlreadyExists}
+		}
+	}
 
 	stackID := handler.DataStore.Stack().GetNextIdentifier()
 	stack := &portainer.Stack{
 		ID:         portainer.StackID(stackID),
 		Type:       portainer.KubernetesStack,
 		EndpointID: endpoint.ID,
-		EntryPoint: payload.FilePathInRepository,
+		EntryPoint: payload.ManifestFile,
 		GitConfig: &gittypes.RepoConfig{
 			URL:            payload.RepositoryURL,
 			ReferenceName:  payload.RepositoryReferenceName,
-			ConfigFilePath: payload.FilePathInRepository,
+			ConfigFilePath: payload.ManifestFile,
 		},
 		Namespace:       payload.Namespace,
+		Name:            payload.StackName,
 		Status:          portainer.StackStatusActive,
 		CreationDate:    time.Now().Unix(),
 		CreatedBy:       user.Username,
 		IsComposeFormat: payload.ComposeFormat,
+		AutoUpdate:      payload.AutoUpdate,
+		AdditionalFiles: payload.AdditionalFiles,
 	}
 
 	if payload.RepositoryAuthentication {
@@ -197,33 +242,48 @@ func (handler *Handler) createKubernetesStackFromGitRepository(w http.ResponseWr
 	}
 	stack.GitConfig.ConfigHash = commitID
 
-	stackFileContent, err := handler.cloneManifestContentFromGitRepo(&payload, stack.ProjectPath)
-	if err != nil {
-		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Failed to process manifest from Git repository", Err: err}
+	repositoryUsername := payload.RepositoryUsername
+	repositoryPassword := payload.RepositoryPassword
+	if !payload.RepositoryAuthentication {
+		repositoryUsername = ""
+		repositoryPassword = ""
 	}
 
-	output, err := handler.deployKubernetesStack(r, endpoint, stackFileContent, payload.ComposeFormat, payload.Namespace, k.KubeAppLabels{
-		StackID: stackID,
-		Name:    stack.Name,
-		Owner:   stack.CreatedBy,
-		Kind:    "git",
+	err = handler.GitService.CloneRepository(projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, repositoryUsername, repositoryPassword)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Failed to clone git repository", Err: err}
+	}
+
+	output, err := handler.deployKubernetesStack(user.ID, endpoint, stack, k.KubeAppLabels{
+		StackID:   stackID,
+		StackName: stack.Name,
+		Owner:     stack.CreatedBy,
+		Kind:      "git",
 	})
 
 	if err != nil {
 		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to deploy Kubernetes stack", Err: err}
 	}
 
+	if payload.AutoUpdate != nil && payload.AutoUpdate.Interval != "" {
+		jobID, e := startAutoupdate(stack.ID, stack.AutoUpdate.Interval, handler.Scheduler, handler.StackDeployer, handler.DataStore, handler.GitService)
+		if e != nil {
+			return e
+		}
+
+		stack.AutoUpdate.JobID = jobID
+	}
+
 	err = handler.DataStore.Stack().CreateStack(stack)
 	if err != nil {
 		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist the stack inside the database", Err: err}
 	}
 
-	doCleanUp = false
-
 	resp := &createKubernetesStackResponse{
 		Output: output,
 	}
 
+	doCleanUp = false
 	return response.JSON(w, resp)
 }
 
@@ -237,6 +297,13 @@ func (handler *Handler) createKubernetesStackFromManifestURL(w http.ResponseWrit
 	if err != nil {
 		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to load user information from the database", Err: err}
 	}
+	isUnique, err := handler.checkUniqueStackName(endpoint, payload.StackName, 0)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err}
+	}
+	if !isUnique {
+		return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A stack with the name '%s' already exists", payload.StackName), Err: errStackAlreadyExists}
+	}
 
 	stackID := handler.DataStore.Stack().GetNextIdentifier()
 	stack := &portainer.Stack{
@@ -245,6 +312,7 @@ func (handler *Handler) createKubernetesStackFromManifestURL(w http.ResponseWrit
 		EndpointID:      endpoint.ID,
 		EntryPoint:      filesystem.ManifestFileDefaultName,
 		Namespace:       payload.Namespace,
+		Name:            payload.StackName,
 		Status:          portainer.StackStatusActive,
 		CreationDate:    time.Now().Unix(),
 		CreatedBy:       user.Username,
@@ -267,11 +335,11 @@ func (handler *Handler) createKubernetesStackFromManifestURL(w http.ResponseWrit
 	doCleanUp := true
 	defer handler.cleanUp(stack, &doCleanUp)
 
-	output, err := handler.deployKubernetesStack(r, endpoint, string(manifestContent), payload.ComposeFormat, payload.Namespace, k.KubeAppLabels{
-		StackID: stackID,
-		Name:    stack.Name,
-		Owner:   stack.CreatedBy,
-		Kind:    "url",
+	output, err := handler.deployKubernetesStack(user.ID, endpoint, stack, k.KubeAppLabels{
+		StackID:   stackID,
+		StackName: stack.Name,
+		Owner:     stack.CreatedBy,
+		Kind:      "url",
 	})
 	if err != nil {
 		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to deploy Kubernetes stack", Err: err}
@@ -291,42 +359,14 @@ func (handler *Handler) createKubernetesStackFromManifestURL(w http.ResponseWrit
 	return response.JSON(w, resp)
 }
 
-func (handler *Handler) deployKubernetesStack(request *http.Request, endpoint *portainer.Endpoint, stackConfig string, composeFormat bool, namespace string, appLabels k.KubeAppLabels) (string, error) {
+func (handler *Handler) deployKubernetesStack(userID portainer.UserID, endpoint *portainer.Endpoint, stack *portainer.Stack, appLabels k.KubeAppLabels) (string, error) {
 	handler.stackCreationMutex.Lock()
 	defer handler.stackCreationMutex.Unlock()
 
-	manifest := []byte(stackConfig)
-	if composeFormat {
-		convertedConfig, err := handler.KubernetesDeployer.ConvertCompose(manifest)
-		if err != nil {
-			return "", errors.Wrap(err, "failed to convert docker compose file to a kube manifest")
-		}
-		manifest = convertedConfig
-	}
-
-	manifest, err := k.AddAppLabels(manifest, appLabels)
+	manifestFilePaths, tempDir, err := stackutils.CreateTempK8SDeploymentFiles(stack, handler.KubernetesDeployer, appLabels)
 	if err != nil {
-		return "", errors.Wrap(err, "failed to add application labels")
+		return "", errors.Wrap(err, "failed to create temp kub deployment files")
 	}
-
-	return handler.KubernetesDeployer.Deploy(request, endpoint, string(manifest), namespace)
-}
-
-func (handler *Handler) cloneManifestContentFromGitRepo(gitInfo *kubernetesGitDeploymentPayload, projectPath string) (string, error) {
-	repositoryUsername := gitInfo.RepositoryUsername
-	repositoryPassword := gitInfo.RepositoryPassword
-	if !gitInfo.RepositoryAuthentication {
-		repositoryUsername = ""
-		repositoryPassword = ""
-	}
-
-	err := handler.GitService.CloneRepository(projectPath, gitInfo.RepositoryURL, gitInfo.RepositoryReferenceName, repositoryUsername, repositoryPassword)
-	if err != nil {
-		return "", err
-	}
-	content, err := ioutil.ReadFile(filepath.Join(projectPath, gitInfo.FilePathInRepository))
-	if err != nil {
-		return "", err
-	}
-	return string(content), nil
+	defer os.RemoveAll(tempDir)
+	return handler.KubernetesDeployer.Deploy(userID, endpoint, manifestFilePaths, stack.Namespace)
 }

api/http/handler/stacks/create_kubernetes_stack_test.go

@@ -1,68 +0,0 @@
-package stacks
-
-import (
-	"io/ioutil"
-	"os"
-	"path"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-type git struct {
-	content string
-}
-
-func (g *git) CloneRepository(destination string, repositoryURL, referenceName, username, password string) error {
-	return g.ClonePublicRepository(repositoryURL, referenceName, destination)
-}
-func (g *git) ClonePublicRepository(repositoryURL string, referenceName string, destination string) error {
-	return ioutil.WriteFile(path.Join(destination, "deployment.yml"), []byte(g.content), 0755)
-}
-func (g *git) ClonePrivateRepositoryWithBasicAuth(repositoryURL, referenceName string, destination, username, password string) error {
-	return g.ClonePublicRepository(repositoryURL, referenceName, destination)
-}
-
-func (g *git) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) {
-	return "", nil
-}
-
-func TestCloneAndConvertGitRepoFile(t *testing.T) {
-	dir, err := os.MkdirTemp("", "kube-create-stack")
-	assert.NoError(t, err, "failed to create a tmp dir")
-	defer os.RemoveAll(dir)
-
-	content := `apiVersion: apps/v1
-	kind: Deployment
-	metadata:
-		name: nginx-deployment
-		labels:
-			app: nginx
-	spec:
-		replicas: 3
-		selector:
-			matchLabels:
-				app: nginx
-		template:
-			metadata:
-				labels:
-					app: nginx
-			spec:
-				containers:
-				- name: nginx
-					image: nginx:1.14.2
-					ports:
-					- containerPort: 80`
-
-	h := &Handler{
-		GitService: &git{
-			content: content,
-		},
-	}
-	gitInfo := &kubernetesGitDeploymentPayload{
-		FilePathInRepository: "deployment.yml",
-	}
-	fileContent, err := h.cloneManifestContentFromGitRepo(gitInfo, dir)
-	assert.NoError(t, err, "failed to clone or convert the file from Git repo")
-	assert.Equal(t, content, fileContent)
-}

api/http/handler/stacks/create_swarm_stack.go

@@ -51,13 +51,13 @@ func (handler *Handler) createSwarmStackFromFileContent(w http.ResponseWriter, r
 
 	payload.Name = handler.SwarmStackManager.NormalizeStackName(payload.Name)
 
-	isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, true)
+	isUnique, err := handler.checkUniqueStackNameInDocker(endpoint, payload.Name, 0, true)
+
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to check for name collision", err}
 	}
 	if !isUnique {
-		errorMessage := fmt.Sprintf("A stack with the name '%s' is already running", payload.Name)
-		return &httperror.HandlerError{http.StatusConflict, errorMessage, errors.New(errorMessage)}
+		return stackExistsError(payload.Name)
 	}
 
 	stackID := handler.DataStore.Stack().GetNextIdentifier()
@@ -161,12 +161,12 @@ func (handler *Handler) createSwarmStackFromGitRepository(w http.ResponseWriter,
 
 	payload.Name = handler.SwarmStackManager.NormalizeStackName(payload.Name)
 
-	isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, true)
+	isUnique, err := handler.checkUniqueStackNameInDocker(endpoint, payload.Name, 0, true)
 	if err != nil {
 		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err}
 	}
 	if !isUnique {
-		return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A stack with the name '%s' already exists", payload.Name), Err: errStackAlreadyExists}
+		return stackExistsError(payload.Name)
 	}
 
 	//make sure the webhook ID is unique
@@ -218,11 +218,11 @@ func (handler *Handler) createSwarmStackFromGitRepository(w http.ResponseWriter,
 		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to clone git repository", Err: err}
 	}
 
-	commitId, err := handler.latestCommitID(payload.RepositoryURL, payload.RepositoryReferenceName, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword)
+	commitID, err := handler.latestCommitID(payload.RepositoryURL, payload.RepositoryReferenceName, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword)
 	if err != nil {
 		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to fetch git repository id", Err: err}
 	}
-	stack.GitConfig.ConfigHash = commitId
+	stack.GitConfig.ConfigHash = commitID
 
 	config, configErr := handler.createSwarmDeployConfig(r, stack, endpoint, false)
 	if configErr != nil {
@@ -298,13 +298,13 @@ func (handler *Handler) createSwarmStackFromFileUpload(w http.ResponseWriter, r
 
 	payload.Name = handler.SwarmStackManager.NormalizeStackName(payload.Name)
 
-	isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, true)
+	isUnique, err := handler.checkUniqueStackNameInDocker(endpoint, payload.Name, 0, true)
+
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to check for name collision", err}
 	}
 	if !isUnique {
-		errorMessage := fmt.Sprintf("A stack with the name '%s' is already running", payload.Name)
-		return &httperror.HandlerError{http.StatusConflict, errorMessage, errors.New(errorMessage)}
+		return stackExistsError(payload.Name)
 	}
 
 	stackID := handler.DataStore.Stack().GetNextIdentifier()

api/http/handler/stacks/handler.go

@@ -45,6 +45,12 @@ type Handler struct {
 	StackDeployer       stacks.StackDeployer
 }
 
+func stackExistsError(name string) (*httperror.HandlerError){
+	msg := fmt.Sprintf("A stack with the normalized name '%s' already exists", name)
+	err := errors.New(msg)
+	return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: msg, Err: err}
+}
+
 // NewHandler creates a handler to manage stack operations.
 func NewHandler(bouncer *security.RequestBouncer) *Handler {
 	h := &Handler{
@@ -127,7 +133,7 @@ func (handler *Handler) userCanCreateStack(securityContext *security.RestrictedR
 	return handler.userIsAdminOrEndpointAdmin(user, endpointID)
 }
 
-func (handler *Handler) checkUniqueName(endpoint *portainer.Endpoint, name string, stackID portainer.StackID, swarmMode bool) (bool, error) {
+func (handler *Handler) checkUniqueStackName(endpoint *portainer.Endpoint, name string, stackID portainer.StackID) (bool, error) {
 	stacks, err := handler.DataStore.Stack().Stacks()
 	if err != nil {
 		return false, err
@@ -139,6 +145,15 @@ func (handler *Handler) checkUniqueName(endpoint *portainer.Endpoint, name strin
 		}
 	}
 
+	return true, nil
+}
+
+func (handler *Handler) checkUniqueStackNameInDocker(endpoint *portainer.Endpoint, name string, stackID portainer.StackID, swarmMode bool) (bool, error) {
+	isUniqueStackName, err := handler.checkUniqueStackName(endpoint, name, stackID)
+	if err != nil {
+		return false, err
+	}
+
 	dockerClient, err := handler.DockerClientFactory.CreateClient(endpoint, "")
 	if err != nil {
 		return false, err
@@ -171,7 +186,7 @@ func (handler *Handler) checkUniqueName(endpoint *portainer.Endpoint, name strin
 		}
 	}
 
-	return true, nil
+	return isUniqueStackName, nil
 }
 
 func (handler *Handler) checkUniqueWebhookID(webhookID string) (bool, error) {

api/http/handler/stacks/stack_delete.go

@@ -2,15 +2,20 @@ package stacks
 
 import (
 	"context"
-	"errors"
+	"fmt"
+	"io/ioutil"
 	"net/http"
+	"os"
+	"path"
 	"strconv"
 
+	"github.com/pkg/errors"
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
 	bolterrors "github.com/portainer/portainer/api/bolt/errors"
+	"github.com/portainer/portainer/api/filesystem"
 	httperrors "github.com/portainer/portainer/api/http/errors"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/internal/stackutils"
@@ -34,12 +39,12 @@ import (
 func (handler *Handler) stackDelete(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	stackID, err := request.RetrieveRouteVariableValue(r, "id")
 	if err != nil {
-		return &httperror.HandlerError{http.StatusBadRequest, "Invalid stack identifier route variable", err}
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid stack identifier route variable", Err: err}
 	}
 
 	securityContext, err := security.RetrieveRestrictedRequestContext(r)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve info from request context", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve info from request context", Err: err}
 	}
 
 	externalStack, _ := request.RetrieveBooleanQueryParameter(r, "external", true)
@@ -49,52 +54,52 @@ func (handler *Handler) stackDelete(w http.ResponseWriter, r *http.Request) *htt
 
 	id, err := strconv.Atoi(stackID)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusBadRequest, "Invalid stack identifier route variable", err}
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid stack identifier route variable", Err: err}
 	}
 
 	stack, err := handler.DataStore.Stack().Stack(portainer.StackID(id))
 	if err == bolterrors.ErrObjectNotFound {
-		return &httperror.HandlerError{http.StatusNotFound, "Unable to find a stack with the specified identifier inside the database", err}
+		return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find a stack with the specified identifier inside the database", Err: err}
 	} else if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a stack with the specified identifier inside the database", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find a stack with the specified identifier inside the database", Err: err}
 	}
 
 	endpointID, err := request.RetrieveNumericQueryParameter(r, "endpointId", true)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusBadRequest, "Invalid query parameter: endpointId", err}
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid query parameter: endpointId", Err: err}
 	}
 
 	isOrphaned := portainer.EndpointID(endpointID) != stack.EndpointID
 
 	if isOrphaned && !securityContext.IsAdmin {
-		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to remove orphaned stack", errors.New("Permission denied to remove orphaned stack")}
+		return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Permission denied to remove orphaned stack", Err: errors.New("Permission denied to remove orphaned stack")}
 	}
 
 	endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
 	if err == bolterrors.ErrObjectNotFound {
-		return &httperror.HandlerError{http.StatusNotFound, "Unable to find the environment associated to the stack inside the database", err}
+		return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find the endpoint associated to the stack inside the database", Err: err}
 	} else if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find the environment associated to the stack inside the database", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find the endpoint associated to the stack inside the database", Err: err}
 	}
 
 	resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a resource control associated to the stack", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve a resource control associated to the stack", Err: err}
 	}
 
 	if !isOrphaned {
 		err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint)
 		if err != nil {
-			return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access environment", err}
+			return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Permission denied to access endpoint", Err: err}
 		}
 
 		if stack.Type == portainer.DockerSwarmStack || stack.Type == portainer.DockerComposeStack {
 			access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl)
 			if err != nil {
-				return &httperror.HandlerError{http.StatusInternalServerError, "Unable to verify user authorizations to validate stack access", err}
+				return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to verify user authorizations to validate stack access", Err: err}
 			}
 			if !access {
-				return &httperror.HandlerError{http.StatusForbidden, "Access denied to resource", httperrors.ErrResourceAccessDenied}
+				return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Access denied to resource", Err: httperrors.ErrResourceAccessDenied}
 			}
 		}
 	}
@@ -104,26 +109,26 @@ func (handler *Handler) stackDelete(w http.ResponseWriter, r *http.Request) *htt
 		stopAutoupdate(stack.ID, stack.AutoUpdate.JobID, *handler.Scheduler)
 	}
 
-	err = handler.deleteStack(stack, endpoint)
+	err = handler.deleteStack(securityContext.UserID, stack, endpoint)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, err.Error(), err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: err.Error(), Err: err}
 	}
 
 	err = handler.DataStore.Stack().DeleteStack(portainer.StackID(id))
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to remove the stack from the database", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to remove the stack from the database", Err: err}
 	}
 
 	if resourceControl != nil {
 		err = handler.DataStore.ResourceControl().DeleteResourceControl(resourceControl.ID)
 		if err != nil {
-			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to remove the associated resource control from the database", err}
+			return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to remove the associated resource control from the database", Err: err}
 		}
 	}
 
 	err = handler.FileService.RemoveDirectory(stack.ProjectPath)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to remove stack files from disk", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to remove stack files from disk", Err: err}
 	}
 
 	return response.Empty(w)
@@ -132,31 +137,31 @@ func (handler *Handler) stackDelete(w http.ResponseWriter, r *http.Request) *htt
 func (handler *Handler) deleteExternalStack(r *http.Request, w http.ResponseWriter, stackName string, securityContext *security.RestrictedRequestContext) *httperror.HandlerError {
 	endpointID, err := request.RetrieveNumericQueryParameter(r, "endpointId", false)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusBadRequest, "Invalid query parameter: endpointId", err}
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid query parameter: endpointId", Err: err}
 	}
 
 	if !securityContext.IsAdmin {
-		return &httperror.HandlerError{http.StatusUnauthorized, "Permission denied to delete the stack", httperrors.ErrUnauthorized}
+		return &httperror.HandlerError{StatusCode: http.StatusUnauthorized, Message: "Permission denied to delete the stack", Err: httperrors.ErrUnauthorized}
 	}
 
 	stack, err := handler.DataStore.Stack().StackByName(stackName)
 	if err != nil && err != bolterrors.ErrObjectNotFound {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to check for stack existence inside the database", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for stack existence inside the database", Err: err}
 	}
 	if stack != nil {
-		return &httperror.HandlerError{http.StatusBadRequest, "A stack with this name exists inside the database. Cannot use external delete method", errors.New("A tag already exists with this name")}
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "A stack with this name exists inside the database. Cannot use external delete method", Err: errors.New("A tag already exists with this name")}
 	}
 
 	endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
 	if err == bolterrors.ErrObjectNotFound {
-		return &httperror.HandlerError{http.StatusNotFound, "Unable to find the environment associated to the stack inside the database", err}
+		return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find the endpoint associated to the stack inside the database", Err: err}
 	} else if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find the environment associated to the stack inside the database", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find the endpoint associated to the stack inside the database", Err: err}
 	}
 
 	err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access environment", err}
+		return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Permission denied to access endpoint", Err: err}
 	}
 
 	stack = &portainer.Stack{
@@ -164,18 +169,57 @@ func (handler *Handler) deleteExternalStack(r *http.Request, w http.ResponseWrit
 		Type: portainer.DockerSwarmStack,
 	}
 
-	err = handler.deleteStack(stack, endpoint)
+	err = handler.deleteStack(securityContext.UserID, stack, endpoint)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to delete stack", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to delete stack", Err: err}
 	}
 
 	return response.Empty(w)
 }
 
-func (handler *Handler) deleteStack(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
+func (handler *Handler) deleteStack(userID portainer.UserID, stack *portainer.Stack, endpoint *portainer.Endpoint) error {
 	if stack.Type == portainer.DockerSwarmStack {
 		return handler.SwarmStackManager.Remove(stack, endpoint)
 	}
+	if stack.Type == portainer.DockerComposeStack {
+		return handler.ComposeStackManager.Down(context.TODO(), stack, endpoint)
+	}
+	if stack.Type == portainer.KubernetesStack {
+		var manifestFiles []string
 
-	return handler.ComposeStackManager.Down(context.TODO(), stack, endpoint)
+		//if it is a compose format kub stack, create a temp dir and convert the manifest files into it
+		//then process the remove operation
+		if stack.IsComposeFormat {
+			fileNames := append([]string{stack.EntryPoint}, stack.AdditionalFiles...)
+			tmpDir, err := ioutil.TempDir("", "kub_delete")
+			if err != nil {
+				return errors.Wrap(err, "failed to create temp directory for deleting kub stack")
+			}
+			defer os.RemoveAll(tmpDir)
+
+			for _, fileName := range fileNames {
+				manifestFilePath := path.Join(tmpDir, fileName)
+				manifestContent, err := ioutil.ReadFile(path.Join(stack.ProjectPath, fileName))
+				if err != nil {
+					return errors.Wrap(err, "failed to read manifest file")
+				}
+
+				manifestContent, err = handler.KubernetesDeployer.ConvertCompose(manifestContent)
+				if err != nil {
+					return errors.Wrap(err, "failed to convert docker compose file to a kube manifest")
+				}
+
+				err = filesystem.WriteToFile(manifestFilePath, []byte(manifestContent))
+				if err != nil {
+					return errors.Wrap(err, "failed to create temp manifest file")
+				}
+				manifestFiles = append(manifestFiles, manifestFilePath)
+			}
+		} else {
+			manifestFiles = stackutils.GetStackFilePaths(stack)
+		}
+		out, err := handler.KubernetesDeployer.Remove(userID, endpoint, manifestFiles, stack.Namespace)
+		return errors.WithMessagef(err, "failed to remove kubernetes resources: %q", out)
+	}
+	return fmt.Errorf("unsupported stack type: %v", stack.Type)
 }

api/http/handler/stacks/stack_migrate.go

@@ -50,52 +50,54 @@ func (payload *stackMigratePayload) Validate(r *http.Request) error {
 func (handler *Handler) stackMigrate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	stackID, err := request.RetrieveNumericRouteVariableValue(r, "id")
 	if err != nil {
-		return &httperror.HandlerError{http.StatusBadRequest, "Invalid stack identifier route variable", err}
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid stack identifier route variable", Err: err}
 	}
 
 	var payload stackMigratePayload
 	err = request.DecodeAndValidateJSONPayload(r, &payload)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid request payload", Err: err}
 	}
 
 	stack, err := handler.DataStore.Stack().Stack(portainer.StackID(stackID))
 	if err == bolterrors.ErrObjectNotFound {
-		return &httperror.HandlerError{http.StatusNotFound, "Unable to find a stack with the specified identifier inside the database", err}
+		return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find a stack with the specified identifier inside the database", Err: err}
 	} else if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a stack with the specified identifier inside the database", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find a stack with the specified identifier inside the database", Err: err}
+	}
+
+	if stack.Type == portainer.KubernetesStack {
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Migrating a kubernetes stack is not supported", Err: err}
 	}
 
 	endpoint, err := handler.DataStore.Endpoint().Endpoint(stack.EndpointID)
 	if err == bolterrors.ErrObjectNotFound {
-		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an environment with the specified identifier inside the database", err}
+		return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find an endpoint with the specified identifier inside the database", Err: err}
 	} else if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an environment with the specified identifier inside the database", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find an endpoint with the specified identifier inside the database", Err: err}
 	}
 
 	err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access environment", err}
+		return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Permission denied to access endpoint", Err: err}
 	}
 
-	if stack.Type == portainer.DockerSwarmStack || stack.Type == portainer.DockerComposeStack {
-		resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl)
-		if err != nil {
-			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a resource control associated to the stack", err}
-		}
+	securityContext, err := security.RetrieveRestrictedRequestContext(r)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve info from request context", Err: err}
+	}
 
-		securityContext, err := security.RetrieveRestrictedRequestContext(r)
-		if err != nil {
-			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve info from request context", err}
-		}
+	resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve a resource control associated to the stack", Err: err}
+	}
 
-		access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl)
-		if err != nil {
-			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to verify user authorizations to validate stack access", err}
-		}
-		if !access {
-			return &httperror.HandlerError{http.StatusForbidden, "Access denied to resource", httperrors.ErrResourceAccessDenied}
-		}
+	access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to verify user authorizations to validate stack access", Err: err}
+	}
+	if !access {
+		return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Access denied to resource", Err: httperrors.ErrResourceAccessDenied}
 	}
 
 	// TODO: this is a work-around for stacks created with Portainer version >= 1.17.1
@@ -103,7 +105,7 @@ func (handler *Handler) stackMigrate(w http.ResponseWriter, r *http.Request) *ht
 	// can use the optional EndpointID query parameter to associate a valid environment(endpoint) identifier to the stack.
 	endpointID, err := request.RetrieveNumericQueryParameter(r, "endpointId", true)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusBadRequest, "Invalid query parameter: endpointId", err}
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid query parameter: endpointId", Err: err}
 	}
 	if endpointID != int(stack.EndpointID) {
 		stack.EndpointID = portainer.EndpointID(endpointID)
@@ -111,9 +113,9 @@ func (handler *Handler) stackMigrate(w http.ResponseWriter, r *http.Request) *ht
 
 	targetEndpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(payload.EndpointID))
 	if err == bolterrors.ErrObjectNotFound {
-		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an environment with the specified identifier inside the database", err}
+		return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find an endpoint with the specified identifier inside the database", Err: err}
 	} else if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an environment with the specified identifier inside the database", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find an endpoint with the specified identifier inside the database", Err: err}
 	}
 
 	stack.EndpointID = portainer.EndpointID(payload.EndpointID)
@@ -126,14 +128,14 @@ func (handler *Handler) stackMigrate(w http.ResponseWriter, r *http.Request) *ht
 		stack.Name = payload.Name
 	}
 
-	isUnique, err := handler.checkUniqueName(targetEndpoint, stack.Name, stack.ID, stack.SwarmID != "")
+	isUnique, err := handler.checkUniqueStackNameInDocker(targetEndpoint, stack.Name, stack.ID, stack.SwarmID != "")
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to check for name collision", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err}
 	}
 
 	if !isUnique {
-		errorMessage := fmt.Sprintf("A stack with the name '%s' is already running on environment '%s'", stack.Name, targetEndpoint.Name)
-		return &httperror.HandlerError{http.StatusConflict, errorMessage, errors.New(errorMessage)}
+		errorMessage := fmt.Sprintf("A stack with the name '%s' is already running on endpoint '%s'", stack.Name, targetEndpoint.Name)
+		return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: errorMessage, Err: errors.New(errorMessage)}
 	}
 
 	migrationError := handler.migrateStack(r, stack, targetEndpoint)
@@ -142,14 +144,14 @@ func (handler *Handler) stackMigrate(w http.ResponseWriter, r *http.Request) *ht
 	}
 
 	stack.Name = oldName
-	err = handler.deleteStack(stack, endpoint)
+	err = handler.deleteStack(securityContext.UserID, stack, endpoint)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, err.Error(), err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: err.Error(), Err: err}
 	}
 
 	err = handler.DataStore.Stack().UpdateStack(stack.ID, stack)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the stack changes inside the database", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist the stack changes inside the database", Err: err}
 	}
 
 	if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" {
@@ -175,7 +177,7 @@ func (handler *Handler) migrateComposeStack(r *http.Request, stack *portainer.St
 
 	err := handler.deployComposeStack(config)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, err.Error(), err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: err.Error(), Err: err}
 	}
 
 	return nil
@@ -189,7 +191,7 @@ func (handler *Handler) migrateSwarmStack(r *http.Request, stack *portainer.Stac
 
 	err := handler.deploySwarmStack(config)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, err.Error(), err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: err.Error(), Err: err}
 	}
 
 	return nil

api/http/handler/stacks/stack_start.go

@@ -33,59 +33,61 @@ import (
 func (handler *Handler) stackStart(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	stackID, err := request.RetrieveNumericRouteVariableValue(r, "id")
 	if err != nil {
-		return &httperror.HandlerError{http.StatusBadRequest, "Invalid stack identifier route variable", err}
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid stack identifier route variable", Err: err}
 	}
 
 	securityContext, err := security.RetrieveRestrictedRequestContext(r)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve info from request context", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve info from request context", Err: err}
 	}
 
 	stack, err := handler.DataStore.Stack().Stack(portainer.StackID(stackID))
 	if err == bolterrors.ErrObjectNotFound {
-		return &httperror.HandlerError{http.StatusNotFound, "Unable to find a stack with the specified identifier inside the database", err}
+		return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find a stack with the specified identifier inside the database", Err: err}
 	} else if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a stack with the specified identifier inside the database", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find a stack with the specified identifier inside the database", Err: err}
+	}
+
+	if stack.Type == portainer.KubernetesStack {
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Starting a kubernetes stack is not supported", Err: err}
 	}
 
 	endpoint, err := handler.DataStore.Endpoint().Endpoint(stack.EndpointID)
 	if err == bolterrors.ErrObjectNotFound {
-		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an environment with the specified identifier inside the database", err}
+		return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find an endpoint with the specified identifier inside the database", Err: err}
 	} else if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an environment with the specified identifier inside the database", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find an endpoint with the specified identifier inside the database", Err: err}
 	}
 
 	err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access environment", err}
+		return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Permission denied to access endpoint", Err: err}
 	}
 
-	isUnique, err := handler.checkUniqueName(endpoint, stack.Name, stack.ID, stack.SwarmID != "")
+	isUnique, err := handler.checkUniqueStackNameInDocker(endpoint, stack.Name, stack.ID, stack.SwarmID != "")
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to check for name collision", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err}
 	}
 	if !isUnique {
 		errorMessage := fmt.Sprintf("A stack with the name '%s' is already running", stack.Name)
-		return &httperror.HandlerError{http.StatusConflict, errorMessage, errors.New(errorMessage)}
+		return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: errorMessage, Err: errors.New(errorMessage)}
 	}
 
-	if stack.Type == portainer.DockerSwarmStack || stack.Type == portainer.DockerComposeStack {
-		resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl)
-		if err != nil {
-			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a resource control associated to the stack", err}
-		}
+	resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve a resource control associated to the stack", Err: err}
+	}
 
-		access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl)
-		if err != nil {
-			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to verify user authorizations to validate stack access", err}
-		}
-		if !access {
-			return &httperror.HandlerError{http.StatusForbidden, "Access denied to resource", httperrors.ErrResourceAccessDenied}
-		}
+	access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to verify user authorizations to validate stack access", Err: err}
+	}
+	if !access {
+		return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Access denied to resource", Err: httperrors.ErrResourceAccessDenied}
 	}
 
 	if stack.Status == portainer.StackStatusActive {
-		return &httperror.HandlerError{http.StatusBadRequest, "Stack is already active", errors.New("Stack is already active")}
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Stack is already active", Err: errors.New("Stack is already active")}
 	}
 
 	if stack.AutoUpdate != nil && stack.AutoUpdate.Interval != "" {
@@ -101,13 +103,13 @@ func (handler *Handler) stackStart(w http.ResponseWriter, r *http.Request) *http
 
 	err = handler.startStack(stack, endpoint)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to start stack", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to start stack", Err: err}
 	}
 
 	stack.Status = portainer.StackStatusActive
 	err = handler.DataStore.Stack().UpdateStack(stack.ID, stack)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to update stack status", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to update stack status", Err: err}
 	}
 
 	if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" {

api/http/handler/stacks/stack_stop.go

@@ -46,6 +46,10 @@ func (handler *Handler) stackStop(w http.ResponseWriter, r *http.Request) *httpe
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a stack with the specified identifier inside the database", err}
 	}
 
+	if stack.Type == portainer.KubernetesStack {
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Stopping a kubernetes stack is not supported", Err: err}
+	}
+
 	endpoint, err := handler.DataStore.Endpoint().Endpoint(stack.EndpointID)
 	if err == bolterrors.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an environment with the specified identifier inside the database", err}
@@ -58,19 +62,17 @@ func (handler *Handler) stackStop(w http.ResponseWriter, r *http.Request) *httpe
 		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access environment", err}
 	}
 
-	if stack.Type == portainer.DockerSwarmStack || stack.Type == portainer.DockerComposeStack {
-		resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl)
-		if err != nil {
-			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a resource control associated to the stack", err}
-		}
+	resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a resource control associated to the stack", err}
+	}
 
-		access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl)
-		if err != nil {
-			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to verify user authorizations to validate stack access", err}
-		}
-		if !access {
-			return &httperror.HandlerError{http.StatusForbidden, "Access denied to resource", httperrors.ErrResourceAccessDenied}
-		}
+	access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to verify user authorizations to validate stack access", err}
+	}
+	if !access {
+		return &httperror.HandlerError{http.StatusForbidden, "Access denied to resource", httperrors.ErrResourceAccessDenied}
 	}
 
 	if stack.Status == portainer.StackStatusInactive {

api/http/handler/stacks/stack_update_git.go

@@ -1,10 +1,11 @@
 package stacks
 
 import (
-	"errors"
 	"net/http"
+	"time"
 
 	"github.com/asaskevich/govalidator"
+	"github.com/pkg/errors"
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
@@ -98,17 +99,22 @@ func (handler *Handler) stackUpdateGit(w http.ResponseWriter, r *http.Request) *
 		return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Permission denied to access environment", Err: err}
 	}
 
+	securityContext, err := security.RetrieveRestrictedRequestContext(r)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve info from request context", Err: err}
+	}
+
+	user, err := handler.DataStore.User().User(securityContext.UserID)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Cannot find context user", Err: errors.Wrap(err, "failed to fetch the user")}
+	}
+
 	if stack.Type == portainer.DockerSwarmStack || stack.Type == portainer.DockerComposeStack {
 		resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl)
 		if err != nil {
 			return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve a resource control associated to the stack", Err: err}
 		}
 
-		securityContext, err := security.RetrieveRestrictedRequestContext(r)
-		if err != nil {
-			return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve info from request context", Err: err}
-		}
-
 		access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl)
 		if err != nil {
 			return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to verify user authorizations to validate stack access", Err: err}
@@ -127,6 +133,8 @@ func (handler *Handler) stackUpdateGit(w http.ResponseWriter, r *http.Request) *
 	stack.GitConfig.ReferenceName = payload.RepositoryReferenceName
 	stack.AutoUpdate = payload.AutoUpdate
 	stack.Env = payload.Env
+	stack.UpdatedBy = user.Username
+	stack.UpdateDate = time.Now().Unix()
 
 	stack.GitConfig.Authentication = nil
 	if payload.RepositoryAuthentication {

api/http/handler/stacks/stack_update_git_redeploy.go

@@ -2,10 +2,8 @@ package stacks
 
 import (
 	"fmt"
-	"io/ioutil"
 	"log"
 	"net/http"
-	"path/filepath"
 	"time"
 
 	"github.com/asaskevich/govalidator"
@@ -216,15 +214,15 @@ func (handler *Handler) deployStack(r *http.Request, stack *portainer.Stack, end
 		if stack.Namespace == "" {
 			return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Invalid namespace", Err: errors.New("Namespace must not be empty when redeploying kubernetes stacks")}
 		}
-		content, err := ioutil.ReadFile(filepath.Join(stack.ProjectPath, stack.GitConfig.ConfigFilePath))
+		tokenData, err := security.RetrieveTokenData(r)
 		if err != nil {
-			return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to read deployment.yml manifest file", Err: errors.Wrap(err, "failed to read manifest file")}
+			return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Failed to retrieve user token data", Err: err}
 		}
-		_, err = handler.deployKubernetesStack(r, endpoint, string(content), stack.IsComposeFormat, stack.Namespace, k.KubeAppLabels{
-			StackID: int(stack.ID),
-			Name:    stack.Name,
-			Owner:   stack.CreatedBy,
-			Kind:    "git",
+		_, err = handler.deployKubernetesStack(tokenData.ID, endpoint, stack, k.KubeAppLabels{
+			StackID:   int(stack.ID),
+			StackName: stack.Name,
+			Owner:     tokenData.Username,
+			Kind:      "git",
 		})
 		if err != nil {
 			return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to redeploy Kubernetes stack", Err: errors.WithMessage(err, "failed to deploy kube application")}

api/http/handler/stacks/update_kubernetes_stack.go

@@ -2,7 +2,10 @@ package stacks
 
 import (
 	"fmt"
+	"io/ioutil"
 	"net/http"
+	"os"
+	"path"
 	"strconv"
 
 	"github.com/asaskevich/govalidator"
@@ -10,7 +13,9 @@ import (
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/filesystem"
 	gittypes "github.com/portainer/portainer/api/git/types"
+	"github.com/portainer/portainer/api/http/security"
 	k "github.com/portainer/portainer/api/kubernetes"
 )
 
@@ -23,6 +28,7 @@ type kubernetesGitStackUpdatePayload struct {
 	RepositoryAuthentication bool
 	RepositoryUsername       string
 	RepositoryPassword       string
+	AutoUpdate               *portainer.StackAutoUpdate
 }
 
 func (payload *kubernetesFileStackUpdatePayload) Validate(r *http.Request) error {
@@ -36,12 +42,20 @@ func (payload *kubernetesGitStackUpdatePayload) Validate(r *http.Request) error
 	if govalidator.IsNull(payload.RepositoryReferenceName) {
 		payload.RepositoryReferenceName = defaultGitReferenceName
 	}
+	if err := validateStackAutoUpdate(payload.AutoUpdate); err != nil {
+		return err
+	}
 	return nil
 }
 
 func (handler *Handler) updateKubernetesStack(r *http.Request, stack *portainer.Stack, endpoint *portainer.Endpoint) *httperror.HandlerError {
 
 	if stack.GitConfig != nil {
+		//stop the autoupdate job if there is any
+		if stack.AutoUpdate != nil {
+			stopAutoupdate(stack.ID, stack.AutoUpdate.JobID, *handler.Scheduler)
+		}
+
 		var payload kubernetesGitStackUpdatePayload
 
 		if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil {
@@ -49,6 +63,8 @@ func (handler *Handler) updateKubernetesStack(r *http.Request, stack *portainer.
 		}
 
 		stack.GitConfig.ReferenceName = payload.RepositoryReferenceName
+		stack.AutoUpdate = payload.AutoUpdate
+
 		if payload.RepositoryAuthentication {
 			password := payload.RepositoryPassword
 			if password == "" && stack.GitConfig != nil && stack.GitConfig.Authentication != nil {
@@ -61,6 +77,15 @@ func (handler *Handler) updateKubernetesStack(r *http.Request, stack *portainer.
 		} else {
 			stack.GitConfig.Authentication = nil
 		}
+
+		if payload.AutoUpdate != nil && payload.AutoUpdate.Interval != "" {
+			jobID, e := startAutoupdate(stack.ID, stack.AutoUpdate.Interval, handler.Scheduler, handler.StackDeployer, handler.DataStore, handler.GitService)
+			if e != nil {
+				return e
+			}
+			stack.AutoUpdate.JobID = jobID
+		}
+
 		return nil
 	}
 
@@ -71,11 +96,27 @@ func (handler *Handler) updateKubernetesStack(r *http.Request, stack *portainer.
 		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid request payload", Err: err}
 	}
 
-	_, err = handler.deployKubernetesStack(r, endpoint, payload.StackFileContent, stack.IsComposeFormat, stack.Namespace, k.KubeAppLabels{
-		StackID: int(stack.ID),
-		Name:    stack.Name,
-		Owner:   stack.CreatedBy,
-		Kind:    "content",
+	tokenData, err := security.RetrieveTokenData(r)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Failed to retrieve user token data", Err: err}
+	}
+
+	tempFileDir, _ := ioutil.TempDir("", "kub_file_content")
+	defer os.RemoveAll(tempFileDir)
+
+	if err := filesystem.WriteToFile(path.Join(tempFileDir, stack.EntryPoint), []byte(payload.StackFileContent)); err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Failed to persist deployment file in a temp directory", Err: err}
+	}
+
+	//use temp dir as the stack project path for deployment
+	//so if the deployment failed, the original file won't be over-written
+	stack.ProjectPath = tempFileDir
+
+	_, err = handler.deployKubernetesStack(tokenData.ID, endpoint, stack, k.KubeAppLabels{
+		StackID:   int(stack.ID),
+		StackName: stack.Name,
+		Owner:     stack.CreatedBy,
+		Kind:      "content",
 	})
 
 	if err != nil {
@@ -83,7 +124,7 @@ func (handler *Handler) updateKubernetesStack(r *http.Request, stack *portainer.
 	}
 
 	stackFolder := strconv.Itoa(int(stack.ID))
-	_, err = handler.FileService.StoreStackFileFromBytes(stackFolder, stack.EntryPoint, []byte(payload.StackFileContent))
+	projectPath, err := handler.FileService.StoreStackFileFromBytes(stackFolder, stack.EntryPoint, []byte(payload.StackFileContent))
 	if err != nil {
 		fileType := "Manifest"
 		if stack.IsComposeFormat {
@@ -92,6 +133,7 @@ func (handler *Handler) updateKubernetesStack(r *http.Request, stack *portainer.
 		errMsg := fmt.Sprintf("Unable to persist Kubernetes %s file on disk", fileType)
 		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: errMsg, Err: err}
 	}
+	stack.ProjectPath = projectPath
 
 	return nil
 }

api/http/handler/stacks/webhook_invoke.go

@@ -1,10 +1,10 @@
 package stacks
 
 import (
-	"log"
 	"net/http"
 
 	"github.com/gofrs/uuid"
+	"github.com/sirupsen/logrus"
 
 	"github.com/portainer/libhttp/response"
 
@@ -31,7 +31,10 @@ func (handler *Handler) webhookInvoke(w http.ResponseWriter, r *http.Request) *h
 	}
 
 	if err = stacks.RedeployWhenChanged(stack.ID, handler.StackDeployer, handler.DataStore, handler.GitService); err != nil {
-		log.Printf("[ERROR] %s\n", err)
+		if _, ok := err.(*stacks.StackAuthorMissingErr); ok {
+			return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: "Autoupdate for the stack isn't available", Err: err}
+		}
+		logrus.WithError(err).Error("failed to update the stack")
 		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Failed to update the stack", Err: err}
 	}
 

api/http/handler/stacks/webhook_invoke_test.go

@@ -6,15 +6,13 @@ import (
 	"testing"
 
 	"github.com/gofrs/uuid"
-	"github.com/stretchr/testify/assert"
-
 	portainer "github.com/portainer/portainer/api"
-
-	"github.com/portainer/portainer/api/bolt/bolttest"
+	"github.com/portainer/portainer/api/bolt"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestHandler_webhookInvoke(t *testing.T) {
-	store, teardown := bolttest.MustNewTestStore(true)
+	store, teardown := bolt.MustNewTestStore(true)
 	defer teardown()
 
 	webhookID := newGuidString(t)

api/http/handler/users/user_inspect.go

@@ -15,6 +15,7 @@ import (
 // @id UserInspect
 // @summary Inspect a user
 // @description Retrieve details about a user.
+// @description User passwords are filtered out, and should never be accessible.
 // @description **Access policy**: administrator
 // @tags users
 // @security jwt

api/http/handler/users/user_list.go

@@ -12,6 +12,7 @@ import (
 // @summary List users
 // @description List Portainer users.
 // @description Non-administrator users will only be able to list other non-administrator user accounts.
+// @description User passwords are filtered out, and should never be accessible.
 // @description **Access policy**: restricted
 // @tags users
 // @security jwt

api/http/handler/websocket/proxy.go

@@ -35,6 +35,9 @@ func (handler *Handler) proxyEdgeAgentWebsocketRequest(w http.ResponseWriter, r
 	}
 
 	handler.ReverseTunnelService.SetTunnelStatusToActive(params.endpoint.ID)
+
+	handler.ReverseTunnelService.KeepTunnelAlive(params.endpoint.ID, r.Context(), portainer.WebSocketKeepAlive)
+
 	proxy.ServeHTTP(w, r)
 
 	return nil

api/http/handler/websocket/shell_pod.go

@@ -10,13 +10,21 @@ import (
 	"github.com/portainer/portainer/api/http/security"
 )
 
-// websocketShellPodExec handles GET requests on /websocket/pod?token=<token>&endpointId=<endpointID>
-// The request will be upgraded to the websocket protocol.
-// Authentication and access is controlled via the mandatory token query parameter.
-// The request will proxy input from the client to the pod via long-lived websocket connection.
-// The following query parameters are mandatory:
-// * token: JWT token used for authentication against this environment(endpoint)
-// * endpointId: environment(endpoint) ID of the environment(endpoint) where the resource is located
+// @summary Execute a websocket on kubectl shell pod
+// @description The request will be upgraded to the websocket protocol. The request will proxy input from the client to the pod via long-lived websocket connection.
+// @description **Access policy**: authenticated
+// @security jwt
+// @tags websocket
+// @accept json
+// @produce json
+// @param endpointId query int true "environment(endpoint) ID of the environment(endpoint) where the resource is located"
+// @param token query string true "JWT token used for authentication against this environment(endpoint)"
+// @success 200
+// @failure 400
+// @failure 403
+// @failure 404
+// @failure 500
+// @router /websocket/kubernetes-shell [get]
 func (handler *Handler) websocketShellPodExec(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	endpointID, err := request.RetrieveNumericQueryParameter(r, "endpointId", false)
 	if err != nil {
@@ -45,7 +53,12 @@ func (handler *Handler) websocketShellPodExec(w http.ResponseWriter, r *http.Req
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find serviceaccount associated with user", err}
 	}
 
-	shellPod, err := cli.CreateUserShellPod(r.Context(), serviceAccount.Name)
+	settings, err := handler.DataStore.Settings().Settings()
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable read settings", err}
+	}
+
+	shellPod, err := cli.CreateUserShellPod(r.Context(), serviceAccount.Name, settings.KubectlShellImage)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to create user shell", err}
 	}

api/http/proxy/factory/agent.go

@@ -6,29 +6,37 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"strings"
 
+	"github.com/pkg/errors"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/crypto"
-	"github.com/portainer/portainer/api/http/proxy/factory/dockercompose"
+	"github.com/portainer/portainer/api/http/proxy/factory/agent"
+	"github.com/portainer/portainer/api/internal/endpointutils"
 )
 
-// ProxyServer provide an extedned proxy with a local server to forward requests
+// ProxyServer provide an extended proxy with a local server to forward requests
 type ProxyServer struct {
 	server *http.Server
 	Port   int
 }
 
-func (factory *ProxyFactory) NewDockerComposeAgentProxy(endpoint *portainer.Endpoint) (*ProxyServer, error) {
+// NewAgentProxy creates a new instance of ProxyServer that wrap http requests with agent headers
+func (factory *ProxyFactory) NewAgentProxy(endpoint *portainer.Endpoint) (*ProxyServer, error) {
+	urlString := endpoint.URL
 
-	if endpoint.Type == portainer.EdgeAgentOnDockerEnvironment {
-		return &ProxyServer{
-			Port: factory.reverseTunnelService.GetTunnelDetails(endpoint.ID).Port,
-		}, nil
+	if endpointutils.IsEdgeEndpoint((endpoint)) {
+		tunnel, err := factory.reverseTunnelService.GetActiveTunnel(endpoint)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed starting tunnel")
+		}
+
+		urlString = fmt.Sprintf("http://127.0.0.1:%d", tunnel.Port)
 	}
 
-	endpointURL, err := url.Parse(endpoint.URL)
+	endpointURL, err := parseURL(urlString)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrapf(err, "failed parsing url %s", endpoint.URL)
 	}
 
 	endpointURL.Scheme = "http"
@@ -37,7 +45,7 @@ func (factory *ProxyFactory) NewDockerComposeAgentProxy(endpoint *portainer.Endp
 	if endpoint.TLSConfig.TLS || endpoint.TLSConfig.TLSSkipVerify {
 		config, err := crypto.CreateTLSConfigurationFromDisk(endpoint.TLSConfig.TLSCACertPath, endpoint.TLSConfig.TLSCertPath, endpoint.TLSConfig.TLSKeyPath, endpoint.TLSConfig.TLSSkipVerify)
 		if err != nil {
-			return nil, err
+			return nil, errors.WithMessage(err, "failed generating tls configuration")
 		}
 
 		httpTransport.TLSClientConfig = config
@@ -46,7 +54,7 @@ func (factory *ProxyFactory) NewDockerComposeAgentProxy(endpoint *portainer.Endp
 
 	proxy := newSingleHostReverseProxyWithHostHeader(endpointURL)
 
-	proxy.Transport = dockercompose.NewAgentTransport(factory.signatureService, httpTransport)
+	proxy.Transport = agent.NewTransport(factory.signatureService, httpTransport)
 
 	proxyServer := &ProxyServer{
 		server: &http.Server{
@@ -57,7 +65,7 @@ func (factory *ProxyFactory) NewDockerComposeAgentProxy(endpoint *portainer.Endp
 
 	err = proxyServer.start()
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "failed starting proxy server")
 	}
 
 	return proxyServer, nil
@@ -91,3 +99,15 @@ func (proxy *ProxyServer) Close() {
 		proxy.server.Close()
 	}
 }
+
+// parseURL parses the endpointURL using url.Parse.
+//
+// to prevent an error when url has port but no protocol prefix
+// we add `//` prefix if needed
+func parseURL(endpointURL string) (*url.URL, error) {
+	if !strings.HasPrefix(endpointURL, "http") && !strings.HasPrefix(endpointURL, "tcp") && !strings.HasPrefix(endpointURL, "//") {
+		endpointURL = fmt.Sprintf("//%s", endpointURL)
+	}
+
+	return url.Parse(endpointURL)
+}

api/http/proxy/factory/agent/transport.go

@@ -1,4 +1,4 @@
-package dockercompose
+package agent
 
 import (
 	"net/http"
@@ -7,17 +7,17 @@ import (
 )
 
 type (
-	// AgentTransport is an http.Transport wrapper that adds custom http headers to communicate to an Agent
-	AgentTransport struct {
+	// Transport is an http.Transport wrapper that adds custom http headers to communicate to an Agent
+	Transport struct {
 		httpTransport      *http.Transport
 		signatureService   portainer.DigitalSignatureService
 		endpointIdentifier portainer.EndpointID
 	}
 )
 
-// NewAgentTransport returns a new transport that can be used to send signed requests to a Portainer agent
-func NewAgentTransport(signatureService portainer.DigitalSignatureService, httpTransport *http.Transport) *AgentTransport {
-	transport := &AgentTransport{
+// NewTransport returns a new transport that can be used to send signed requests to a Portainer agent
+func NewTransport(signatureService portainer.DigitalSignatureService, httpTransport *http.Transport) *Transport {
+	transport := &Transport{
 		httpTransport:    httpTransport,
 		signatureService: signatureService,
 	}
@@ -26,8 +26,7 @@ func NewAgentTransport(signatureService portainer.DigitalSignatureService, httpT
 }
 
 // RoundTrip is the implementation of the the http.RoundTripper interface
-func (transport *AgentTransport) RoundTrip(request *http.Request) (*http.Response, error) {
-
+func (transport *Transport) RoundTrip(request *http.Request) (*http.Response, error) {
 	signature, err := transport.signatureService.CreateSignature(portainer.PortainerAgentSignatureMessage)
 	if err != nil {
 		return nil, err

api/http/proxy/factory/kubernetes/namespaces.go

@@ -46,5 +46,19 @@ func (transport *baseTransport) proxyNamespaceDeleteOperation(request *http.Requ
 			}
 		}
 	}
+
+	stacks, err := transport.dataStore.Stack().Stacks()
+	if err != nil {
+		return nil, err
+	}
+
+	for _, s := range stacks {
+		if s.Namespace == namespace && s.EndpointID == transport.endpoint.ID {
+			if err := transport.dataStore.Stack().DeleteStack(s.ID); err != nil {
+				return nil, err
+			}
+		}
+	}
+
 	return transport.executeKubernetesRequest(request)
 }

api/http/proxy/factory/kubernetes/secrets.go

@@ -1,170 +0,0 @@
-package kubernetes
-
-import (
-	"net/http"
-	"path"
-
-	"github.com/portainer/portainer/api/http/proxy/factory/utils"
-	"github.com/portainer/portainer/api/http/security"
-	"github.com/portainer/portainer/api/kubernetes/privateregistries"
-	v1 "k8s.io/api/core/v1"
-)
-
-func (transport *baseTransport) proxySecretRequest(request *http.Request, namespace, requestPath string) (*http.Response, error) {
-	switch request.Method {
-	case "POST":
-		return transport.proxySecretCreationOperation(request)
-	case "GET":
-		if path.Base(requestPath) == "secrets" {
-			return transport.proxySecretListOperation(request)
-		}
-		return transport.proxySecretInspectOperation(request)
-	case "PUT":
-		return transport.proxySecretUpdateOperation(request)
-	case "DELETE":
-		return transport.proxySecretDeleteOperation(request, namespace)
-	default:
-		return transport.executeKubernetesRequest(request)
-	}
-}
-
-func (transport *baseTransport) proxySecretCreationOperation(request *http.Request) (*http.Response, error) {
-	body, err := utils.GetRequestAsMap(request)
-	if err != nil {
-		return nil, err
-	}
-
-	if isSecretRepresentPrivateRegistry(body) {
-		return utils.WriteAccessDeniedResponse()
-	}
-
-	err = utils.RewriteRequest(request, body)
-	if err != nil {
-		return nil, err
-	}
-
-	return transport.executeKubernetesRequest(request)
-}
-
-func (transport *baseTransport) proxySecretListOperation(request *http.Request) (*http.Response, error) {
-	response, err := transport.executeKubernetesRequest(request)
-	if err != nil {
-		return nil, err
-	}
-
-	isAdmin, err := security.IsAdmin(request)
-	if err != nil {
-		return nil, err
-	}
-
-	if isAdmin {
-		return response, nil
-	}
-
-	body, err := utils.GetResponseAsJSONObject(response)
-	if err != nil {
-		return nil, err
-	}
-
-	items := utils.GetArrayObject(body, "items")
-
-	if items == nil {
-		utils.RewriteResponse(response, body, response.StatusCode)
-		return response, nil
-	}
-
-	filteredItems := []interface{}{}
-	for _, item := range items {
-		itemObj := item.(map[string]interface{})
-		if !isSecretRepresentPrivateRegistry(itemObj) {
-			filteredItems = append(filteredItems, item)
-		}
-	}
-
-	body["items"] = filteredItems
-
-	utils.RewriteResponse(response, body, response.StatusCode)
-	return response, nil
-}
-
-func (transport *baseTransport) proxySecretInspectOperation(request *http.Request) (*http.Response, error) {
-	response, err := transport.executeKubernetesRequest(request)
-	if err != nil {
-		return nil, err
-	}
-
-	isAdmin, err := security.IsAdmin(request)
-	if err != nil {
-		return nil, err
-	}
-
-	if isAdmin {
-		return response, nil
-	}
-
-	body, err := utils.GetResponseAsJSONObject(response)
-	if err != nil {
-		return nil, err
-	}
-
-	if isSecretRepresentPrivateRegistry(body) {
-		return utils.WriteAccessDeniedResponse()
-	}
-
-	err = utils.RewriteResponse(response, body, response.StatusCode)
-	if err != nil {
-		return nil, err
-	}
-
-	return response, nil
-}
-
-func (transport *baseTransport) proxySecretUpdateOperation(request *http.Request) (*http.Response, error) {
-	body, err := utils.GetRequestAsMap(request)
-	if err != nil {
-		return nil, err
-	}
-
-	if isSecretRepresentPrivateRegistry(body) {
-		return utils.WriteAccessDeniedResponse()
-	}
-
-	err = utils.RewriteRequest(request, body)
-	if err != nil {
-		return nil, err
-	}
-
-	return transport.executeKubernetesRequest(request)
-}
-
-func (transport *baseTransport) proxySecretDeleteOperation(request *http.Request, namespace string) (*http.Response, error) {
-	kcl, err := transport.k8sClientFactory.GetKubeClient(transport.endpoint)
-	if err != nil {
-		return nil, err
-	}
-
-	secretName := path.Base(request.RequestURI)
-
-	isRegistry, err := kcl.IsRegistrySecret(namespace, secretName)
-	if err != nil {
-		return nil, err
-	}
-
-	if isRegistry {
-		return utils.WriteAccessDeniedResponse()
-	}
-
-	return transport.executeKubernetesRequest(request)
-}
-
-func isSecretRepresentPrivateRegistry(secret map[string]interface{}) bool {
-    if secret["type"] == nil || secret["type"].(string) != string(v1.SecretTypeDockerConfigJson) {
-		return false
-	}
-
-	metadata := utils.GetJSONObject(secret, "metadata")
-	annotations := utils.GetJSONObject(metadata, "annotations")
-	_, ok := annotations[privateregistries.RegistryIDLabel]
-
-	return ok
-}

api/http/proxy/factory/kubernetes/transport.go

@@ -66,8 +66,6 @@ func (transport *baseTransport) proxyNamespacedRequest(request *http.Request, fu
 	}
 
 	switch {
-	case strings.HasPrefix(requestPath, "secrets"):
-		return transport.proxySecretRequest(request, namespace, requestPath)
 	case requestPath == "" && request.Method == "DELETE":
 		return transport.proxyNamespaceDeleteOperation(request, namespace)
 	default:
@@ -79,6 +77,18 @@ func (transport *baseTransport) executeKubernetesRequest(request *http.Request)
 
 	resp, err := transport.httpTransport.RoundTrip(request)
 
+	// This fix was made to resolve a k8s e2e test, more detailed investigation should be done later.
+	if err == nil && resp.StatusCode == http.StatusMovedPermanently {
+		oldLocation := resp.Header.Get("Location")
+		if oldLocation != "" {
+			stripedPrefix := strings.TrimSuffix(request.RequestURI, request.URL.Path)
+			// local proxy strips "/kubernetes" but agent proxy and edge agent proxy do not
+			stripedPrefix = strings.TrimSuffix(stripedPrefix, "/kubernetes")
+			newLocation := stripedPrefix + "/kubernetes" + oldLocation
+			resp.Header.Set("Location", newLocation)
+		}
+	}
+
 	return resp, err
 }
 

api/http/proxy/manager.go

@@ -48,10 +48,10 @@ func (manager *Manager) CreateAndRegisterEndpointProxy(endpoint *portainer.Endpo
 	return proxy, nil
 }
 
-// CreateComposeProxyServer creates a new HTTP reverse proxy based on environment(endpoint) properties and and adds it to the registered proxies.
+// CreateAgentProxyServer creates a new HTTP reverse proxy based on environment(endpoint) properties and and adds it to the registered proxies.
 // It can also be used to create a new HTTP reverse proxy and replace an already registered proxy.
-func (manager *Manager) CreateComposeProxyServer(endpoint *portainer.Endpoint) (*factory.ProxyServer, error) {
-	return manager.proxyFactory.NewDockerComposeAgentProxy(endpoint)
+func (manager *Manager) CreateAgentProxyServer(endpoint *portainer.Endpoint) (*factory.ProxyServer, error) {
+	return manager.proxyFactory.NewAgentProxy(endpoint)
 }
 
 // GetEndpointProxy returns the proxy associated to a key

api/http/server.go

@@ -29,6 +29,7 @@ import (
 	"github.com/portainer/portainer/api/http/handler/file"
 	"github.com/portainer/portainer/api/http/handler/helm"
 	kubehandler "github.com/portainer/portainer/api/http/handler/kubernetes"
+	"github.com/portainer/portainer/api/http/handler/ldap"
 	"github.com/portainer/portainer/api/http/handler/motd"
 	"github.com/portainer/portainer/api/http/handler/registries"
 	"github.com/portainer/portainer/api/http/handler/resourcecontrols"
@@ -171,10 +172,15 @@ func (server *Server) Start() error {
 
 	var fileHandler = file.NewHandler(filepath.Join(server.AssetsPath, "public"))
 
-	var endpointHelmHandler = helm.NewHandler(requestBouncer, server.DataStore, server.HelmPackageManager, server.KubeConfigService)
+	var endpointHelmHandler = helm.NewHandler(requestBouncer, server.DataStore, server.KubernetesDeployer, server.HelmPackageManager, server.KubeConfigService)
 
 	var helmTemplatesHandler = helm.NewTemplateHandler(requestBouncer, server.HelmPackageManager)
 
+	var ldapHandler = ldap.NewHandler(requestBouncer)
+	ldapHandler.DataStore = server.DataStore
+	ldapHandler.FileService = server.FileService
+	ldapHandler.LDAPService = server.LDAPService
+
 	var motdHandler = motd.NewHandler(requestBouncer)
 
 	var registryHandler = registries.NewHandler(requestBouncer)
@@ -255,6 +261,7 @@ func (server *Server) Start() error {
 		EndpointEdgeHandler:    endpointEdgeHandler,
 		EndpointProxyHandler:   endpointProxyHandler,
 		FileHandler:            fileHandler,
+		LDAPHandler:            ldapHandler,
 		HelmTemplatesHandler:   helmTemplatesHandler,
 		KubernetesHandler:      kubernetesHandler,
 		MOTDHandler:            motdHandler,

api/internal/endpointutils/endpointutils.go

@@ -11,7 +11,7 @@ func IsLocalEndpoint(endpoint *portainer.Endpoint) bool {
 	return strings.HasPrefix(endpoint.URL, "unix://") || strings.HasPrefix(endpoint.URL, "npipe://") || endpoint.Type == 5
 }
 
-// IsKubernetesEndpoint returns true if this is a kubernetes environment(endpoint)
+// IsKubernetesEndpoint returns true if this is a kubernetes endpoint
 func IsKubernetesEndpoint(endpoint *portainer.Endpoint) bool {
 	return endpoint.Type == portainer.KubernetesLocalEnvironment ||
 		endpoint.Type == portainer.AgentOnKubernetesEnvironment ||
@@ -24,3 +24,8 @@ func IsDockerEndpoint(endpoint *portainer.Endpoint) bool {
 		endpoint.Type == portainer.AgentOnDockerEnvironment ||
 		endpoint.Type == portainer.EdgeAgentOnDockerEnvironment
 }
+
+// IsEdgeEndpoint returns true if this is an Edge endpoint
+func IsEdgeEndpoint(endpoint *portainer.Endpoint) bool {
+	return endpoint.Type == portainer.EdgeAgentOnDockerEnvironment || endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment
+}

api/internal/stackutils/stackutils.go

@@ -2,9 +2,13 @@ package stackutils
 
 import (
 	"fmt"
+	"io/ioutil"
 	"path"
 
+	"github.com/pkg/errors"
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/filesystem"
+	k "github.com/portainer/portainer/api/kubernetes"
 )
 
 // ResourceControlID returns the stack resource control id
@@ -20,3 +24,39 @@ func GetStackFilePaths(stack *portainer.Stack) []string {
 	}
 	return filePaths
 }
+
+// CreateTempK8SDeploymentFiles reads manifest files from original stack project path
+// then add app labels into the file contents and create temp files for deployment
+// return temp file paths and temp dir
+func CreateTempK8SDeploymentFiles(stack *portainer.Stack, kubeDeployer portainer.KubernetesDeployer, appLabels k.KubeAppLabels) ([]string, string, error) {
+	fileNames := append([]string{stack.EntryPoint}, stack.AdditionalFiles...)
+	var manifestFilePaths []string
+	tmpDir, err := ioutil.TempDir("", "kub_deployment")
+	if err != nil {
+		return nil, "", errors.Wrap(err, "failed to create temp kub deployment directory")
+	}
+
+	for _, fileName := range fileNames {
+		manifestFilePath := path.Join(tmpDir, fileName)
+		manifestContent, err := ioutil.ReadFile(path.Join(stack.ProjectPath, fileName))
+		if err != nil {
+			return nil, "", errors.Wrap(err, "failed to read manifest file")
+		}
+		if stack.IsComposeFormat {
+			manifestContent, err = kubeDeployer.ConvertCompose(manifestContent)
+			if err != nil {
+				return nil, "", errors.Wrap(err, "failed to convert docker compose file to a kube manifest")
+			}
+		}
+		manifestContent, err = k.AddAppLabels(manifestContent, appLabels.ToMap())
+		if err != nil {
+			return nil, "", errors.Wrap(err, "failed to add application labels")
+		}
+		err = filesystem.WriteToFile(manifestFilePath, []byte(manifestContent))
+		if err != nil {
+			return nil, "", errors.Wrap(err, "failed to create temp manifest file")
+		}
+		manifestFilePaths = append(manifestFilePaths, manifestFilePath)
+	}
+	return manifestFilePaths, tmpDir, nil
+}

api/internal/testhelpers/datastore.go

@@ -38,7 +38,7 @@ func (d *datastore) Close() error                                        { retur
 func (d *datastore) CheckCurrentEdition() error                          { return nil }
 func (d *datastore) IsNew() bool                                         { return false }
 func (d *datastore) MigrateData(force bool) error                        { return nil }
-func (d *datastore) RollbackToCE() error                                 { return nil }
+func (d *datastore) Rollback(force bool) error                           { return nil }
 func (d *datastore) CustomTemplate() portainer.CustomTemplateService     { return d.customTemplate }
 func (d *datastore) EdgeGroup() portainer.EdgeGroupService               { return d.edgeGroup }
 func (d *datastore) EdgeJob() portainer.EdgeJobService                   { return d.edgeJob }

api/kubernetes/cli/client.go

@@ -1,14 +1,13 @@
 package cli
 
 import (
-	"errors"
 	"fmt"
 	"net/http"
 	"strconv"
 	"sync"
-	"time"
 
 	cmap "github.com/orcaman/concurrent-map"
+	"github.com/pkg/errors"
 
 	portainer "github.com/portainer/portainer/api"
 	"k8s.io/client-go/kubernetes"
@@ -116,36 +115,18 @@ func (rt *agentHeaderRoundTripper) RoundTrip(req *http.Request) (*http.Response,
 func (factory *ClientFactory) buildAgentClient(endpoint *portainer.Endpoint) (*kubernetes.Clientset, error) {
 	endpointURL := fmt.Sprintf("https://%s/kubernetes", endpoint.URL)
 
-	return factory.createRemoteClient(endpointURL);
+	return factory.createRemoteClient(endpointURL)
 }
 
 func (factory *ClientFactory) buildEdgeClient(endpoint *portainer.Endpoint) (*kubernetes.Clientset, error) {
-	tunnel := factory.reverseTunnelService.GetTunnelDetails(endpoint.ID)
-
-	if tunnel.Status == portainer.EdgeAgentIdle {
-		err := factory.reverseTunnelService.SetTunnelStatusToRequired(endpoint.ID)
-		if err != nil {
-			return nil, fmt.Errorf("failed opening tunnel to environment: %w", err)
-		}
-
-		if endpoint.EdgeCheckinInterval == 0 {
-			settings, err := factory.dataStore.Settings().Settings()
-			if err != nil {
-				return nil, fmt.Errorf("failed fetching settings from db: %w", err)
-			}
-
-			endpoint.EdgeCheckinInterval = settings.EdgeAgentCheckinInterval
-		}
-
-		waitForAgentToConnect := time.Duration(endpoint.EdgeCheckinInterval) * time.Second
-		time.Sleep(waitForAgentToConnect * 2)
-
-		tunnel = factory.reverseTunnelService.GetTunnelDetails(endpoint.ID)
+	tunnel, err := factory.reverseTunnelService.GetActiveTunnel(endpoint)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed activating tunnel")
 	}
 
 	endpointURL := fmt.Sprintf("http://127.0.0.1:%d/kubernetes", tunnel.Port)
 
-	return factory.createRemoteClient(endpointURL);
+	return factory.createRemoteClient(endpointURL)
 }
 
 func (factory *ClientFactory) createRemoteClient(endpointURL string) (*kubernetes.Clientset, error) {

api/kubernetes/cli/pod.go

@@ -12,18 +12,14 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-const shellPodImage = "portainer/kubectl-shell"
-
 // CreateUserShellPod will create a kubectl based shell for the specified user by mounting their respective service account.
 // The lifecycle of the pod is managed in this function; this entails management of the following pod operations:
 // - The shell pod will be scoped to specified service accounts access permissions
 // - The shell pod will be automatically removed if it's not ready after specified period of time
 // - The shell pod will be automatically removed after a specified max life (prevent zombie pods)
 // - The shell pod will be automatically removed if request is cancelled (or client closes websocket connection)
-func (kcl *KubeClient) CreateUserShellPod(ctx context.Context, serviceAccountName string) (*portainer.KubernetesShellPod, error) {
-	// Schedule the pod for automatic removal
-	maxPodKeepAlive := 1 * time.Hour
-	maxPodKeepAliveSecondsStr := fmt.Sprintf("%d", int(maxPodKeepAlive.Seconds()))
+func (kcl *KubeClient) CreateUserShellPod(ctx context.Context, serviceAccountName, shellPodImage string) (*portainer.KubernetesShellPod, error) {
+	maxPodKeepAliveSecondsStr := fmt.Sprintf("%d", int(portainer.WebSocketKeepAlive.Seconds()))
 
 	podPrefix := userShellPodPrefix(serviceAccountName)
 
@@ -81,7 +77,7 @@ func (kcl *KubeClient) CreateUserShellPod(ctx context.Context, serviceAccountNam
 	// Handle pod lifecycle/cleanup - terminate pod after maxPodKeepAlive or upon request (long-lived) cancellation
 	go func() {
 		select {
-		case <-time.After(maxPodKeepAlive):
+		case <-time.After(portainer.WebSocketKeepAlive):
 			log.Println("[DEBUG] [internal,kubernetes/pod] [message: pod removal schedule duration exceeded]")
 			kcl.cli.CoreV1().Pods(portainerNamespace).Delete(shellPod.Name, nil)
 		case <-ctx.Done():

api/kubernetes/cli/role.go

@@ -9,7 +9,7 @@ import (
 func getPortainerUserDefaultPolicies() []rbacv1.PolicyRule {
 	return []rbacv1.PolicyRule{
 		{
-			Verbs:     []string{"list"},
+			Verbs:     []string{"list", "get"},
 			Resources: []string{"namespaces", "nodes"},
 			APIGroups: []string{""},
 		},
@@ -18,6 +18,11 @@ func getPortainerUserDefaultPolicies() []rbacv1.PolicyRule {
 			Resources: []string{"storageclasses"},
 			APIGroups: []string{"storage.k8s.io"},
 		},
+		{
+			Verbs:     []string{"list", "get"},
+			Resources: []string{"namespaces", "pods", "nodes"},
+			APIGroups: []string{"metrics.k8s.io"},
+		},
 	}
 }
 

api/kubernetes/yaml.go

@@ -11,21 +11,66 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
+const (
+	labelPortainerAppStack   = "io.portainer.kubernetes.application.stack"
+	labelPortainerAppStackID = "io.portainer.kubernetes.application.stackid"
+	labelPortainerAppName    = "io.portainer.kubernetes.application.name"
+	labelPortainerAppOwner   = "io.portainer.kubernetes.application.owner"
+	labelPortainerAppKind    = "io.portainer.kubernetes.application.kind"
+)
+
+// KubeAppLabels are labels applied to all resources deployed in a kubernetes stack
 type KubeAppLabels struct {
-	StackID int
-	Name    string
-	Owner   string
-	Kind    string
+	StackID   int
+	StackName string
+	Owner     string
+	Kind      string
+}
+
+// ToMap converts KubeAppLabels to a map[string]string
+func (kal *KubeAppLabels) ToMap() map[string]string {
+	return map[string]string{
+		labelPortainerAppStackID: strconv.Itoa(kal.StackID),
+		labelPortainerAppStack:   kal.StackName,
+		labelPortainerAppName:    kal.StackName,
+		labelPortainerAppOwner:   kal.Owner,
+		labelPortainerAppKind:    kal.Kind,
+	}
+}
+
+// GetHelmAppLabels returns the labels to be applied to portainer deployed helm applications
+func GetHelmAppLabels(name, owner string) map[string]string {
+	return map[string]string{
+		labelPortainerAppName:  name,
+		labelPortainerAppOwner: owner,
+	}
 }
 
 // AddAppLabels adds required labels to "Resource"->metadata->labels.
 // It'll add those labels to all Resource (nodes with a kind property exluding a list) it can find in provided yaml.
 // Items in the yaml file could either be organised as a list or broken into multi documents.
-func AddAppLabels(manifestYaml []byte, appLabels KubeAppLabels) ([]byte, error) {
+func AddAppLabels(manifestYaml []byte, appLabels map[string]string) ([]byte, error) {
 	if bytes.Equal(manifestYaml, []byte("")) {
 		return manifestYaml, nil
 	}
 
+	postProcessYaml := func(yamlDoc interface{}) error {
+		addResourceLabels(yamlDoc, appLabels)
+		return nil
+	}
+
+	docs, err := ExtractDocuments(manifestYaml, postProcessYaml)
+	if err != nil {
+		return nil, err
+	}
+
+	return bytes.Join(docs, []byte("---\n")), nil
+}
+
+// ExtractDocuments extracts all the documents from a yaml file
+// Optionally post-process each document with a function, which can modify the document in place.
+// Pass in nil for postProcessYaml to skip post-processing.
+func ExtractDocuments(manifestYaml []byte, postProcessYaml func(interface{}) error) ([][]byte, error) {
 	docs := make([][]byte, 0)
 	yamlDecoder := yaml.NewDecoder(bytes.NewReader(manifestYaml))
 
@@ -43,7 +88,12 @@ func AddAppLabels(manifestYaml []byte, appLabels KubeAppLabels) ([]byte, error)
 			break
 		}
 
-		addResourceLabels(m, appLabels)
+		// optionally post-process yaml
+		if postProcessYaml != nil {
+			if err := postProcessYaml(m); err != nil {
+				return nil, errors.Wrap(err, "failed to post process yaml document")
+			}
+		}
 
 		var out bytes.Buffer
 		yamlEncoder := yaml.NewEncoder(&out)
@@ -55,10 +105,29 @@ func AddAppLabels(manifestYaml []byte, appLabels KubeAppLabels) ([]byte, error)
 		docs = append(docs, out.Bytes())
 	}
 
-	return bytes.Join(docs, []byte("---\n")), nil
+	return docs, nil
 }
 
-func addResourceLabels(yamlDoc interface{}, appLabels KubeAppLabels) {
+// GetNamespace returns the namespace of a kubernetes resource from its metadata
+// It returns an empty string if namespace is not found in the resource
+func GetNamespace(manifestYaml []byte) (string, error) {
+	yamlDecoder := yaml.NewDecoder(bytes.NewReader(manifestYaml))
+	m := make(map[string]interface{})
+	err := yamlDecoder.Decode(&m)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to unmarshal yaml manifest when obtaining namespace")
+	}
+
+	if _, ok := m["metadata"]; ok {
+		if namespace, ok := m["metadata"].(map[string]interface{})["namespace"]; ok {
+			return namespace.(string), nil
+		}
+	}
+
+	return "", nil
+}
+
+func addResourceLabels(yamlDoc interface{}, appLabels map[string]string) {
 	m, ok := yamlDoc.(map[string]interface{})
 	if !ok {
 		return
@@ -82,7 +151,7 @@ func addResourceLabels(yamlDoc interface{}, appLabels KubeAppLabels) {
 	}
 }
 
-func addLabels(obj map[string]interface{}, appLabels KubeAppLabels) {
+func addLabels(obj map[string]interface{}, appLabels map[string]string) {
 	metadata := make(map[string]interface{})
 	if m, ok := obj["metadata"]; ok {
 		metadata = m.(map[string]interface{})
@@ -95,18 +164,11 @@ func addLabels(obj map[string]interface{}, appLabels KubeAppLabels) {
 		}
 	}
 
-	name := appLabels.Name
-	if appLabels.Name == "" {
-		if n, ok := metadata["name"]; ok {
-			name = n.(string)
-		}
+	// merge app labels with existing labels
+	for k, v := range appLabels {
+		labels[k] = v
 	}
 
-	labels["io.portainer.kubernetes.application.stackid"] = strconv.Itoa(appLabels.StackID)
-	labels["io.portainer.kubernetes.application.name"] = name
-	labels["io.portainer.kubernetes.application.owner"] = appLabels.Owner
-	labels["io.portainer.kubernetes.application.kind"] = appLabels.Kind
-
 	metadata["labels"] = labels
 	obj["metadata"] = metadata
 }

api/kubernetes/yaml_test.go

@@ -39,6 +39,7 @@ metadata:
     io.portainer.kubernetes.application.kind: git
     io.portainer.kubernetes.application.name: best-name
     io.portainer.kubernetes.application.owner: best-owner
+    io.portainer.kubernetes.application.stack: best-name
     io.portainer.kubernetes.application.stackid: "123"
   name: busybox
 spec:
@@ -86,6 +87,7 @@ metadata:
     io.portainer.kubernetes.application.kind: git
     io.portainer.kubernetes.application.name: best-name
     io.portainer.kubernetes.application.owner: best-owner
+    io.portainer.kubernetes.application.stack: best-name
     io.portainer.kubernetes.application.stackid: "123"
   name: busybox
 spec:
@@ -174,6 +176,7 @@ items:
         io.portainer.kubernetes.application.kind: git
         io.portainer.kubernetes.application.name: best-name
         io.portainer.kubernetes.application.owner: best-owner
+        io.portainer.kubernetes.application.stack: best-name
         io.portainer.kubernetes.application.stackid: "123"
       name: web
     spec:
@@ -194,6 +197,7 @@ items:
         io.portainer.kubernetes.application.kind: git
         io.portainer.kubernetes.application.name: best-name
         io.portainer.kubernetes.application.owner: best-owner
+        io.portainer.kubernetes.application.stack: best-name
         io.portainer.kubernetes.application.stackid: "123"
       name: redis
     spec:
@@ -216,6 +220,7 @@ items:
         io.portainer.kubernetes.application.kind: git
         io.portainer.kubernetes.application.name: best-name
         io.portainer.kubernetes.application.owner: best-owner
+        io.portainer.kubernetes.application.stack: best-name
         io.portainer.kubernetes.application.stackid: "123"
       name: web
     spec:
@@ -297,6 +302,7 @@ metadata:
     io.portainer.kubernetes.application.kind: git
     io.portainer.kubernetes.application.name: best-name
     io.portainer.kubernetes.application.owner: best-owner
+    io.portainer.kubernetes.application.stack: best-name
     io.portainer.kubernetes.application.stackid: "123"
   name: busybox
 spec:
@@ -322,6 +328,7 @@ metadata:
     io.portainer.kubernetes.application.kind: git
     io.portainer.kubernetes.application.name: best-name
     io.portainer.kubernetes.application.owner: best-owner
+    io.portainer.kubernetes.application.stack: best-name
     io.portainer.kubernetes.application.stackid: "123"
   name: web
 spec:
@@ -340,6 +347,7 @@ metadata:
     io.portainer.kubernetes.application.kind: git
     io.portainer.kubernetes.application.name: best-name
     io.portainer.kubernetes.application.owner: best-owner
+    io.portainer.kubernetes.application.stack: best-name
     io.portainer.kubernetes.application.stackid: "123"
   name: busybox
 spec:
@@ -388,6 +396,7 @@ metadata:
     io.portainer.kubernetes.application.kind: git
     io.portainer.kubernetes.application.name: best-name
     io.portainer.kubernetes.application.owner: best-owner
+    io.portainer.kubernetes.application.stack: best-name
     io.portainer.kubernetes.application.stackid: "123"
   name: web
 spec:
@@ -402,10 +411,174 @@ spec:
 	}
 
 	labels := KubeAppLabels{
-		StackID: 123,
-		Name:    "best-name",
-		Owner:   "best-owner",
-		Kind:    "git",
+		StackID:   123,
+		StackName: "best-name",
+		Owner:     "best-owner",
+		Kind:      "git",
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result, err := AddAppLabels([]byte(tt.input), labels.ToMap())
+			assert.NoError(t, err)
+			assert.Equal(t, tt.wantOutput, string(result))
+		})
+	}
+}
+
+func Test_AddAppLabels_HelmApp(t *testing.T) {
+	labels := GetHelmAppLabels("best-name", "best-owner")
+
+	tests := []struct {
+		name       string
+		input      string
+		wantOutput string
+	}{
+		{
+			name: "bitnami nginx configmap",
+			input: `apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nginx-test-server-block
+  labels:
+    app.kubernetes.io/name: nginx
+    helm.sh/chart: nginx-9.5.4
+    app.kubernetes.io/instance: nginx-test
+    app.kubernetes.io/managed-by: Helm
+data:
+  server-blocks-paths.conf: |-
+    include  "/opt/bitnami/nginx/conf/server_blocks/ldap/*.conf";
+    include  "/opt/bitnami/nginx/conf/server_blocks/common/*.conf";
+`,
+			wantOutput: `apiVersion: v1
+data:
+  server-blocks-paths.conf: |-
+    include  "/opt/bitnami/nginx/conf/server_blocks/ldap/*.conf";
+    include  "/opt/bitnami/nginx/conf/server_blocks/common/*.conf";
+kind: ConfigMap
+metadata:
+  labels:
+    app.kubernetes.io/instance: nginx-test
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: nginx
+    helm.sh/chart: nginx-9.5.4
+    io.portainer.kubernetes.application.name: best-name
+    io.portainer.kubernetes.application.owner: best-owner
+  name: nginx-test-server-block
+`,
+		},
+		{
+			name: "bitnami nginx service",
+			input: `apiVersion: v1
+kind: Service
+metadata:
+  name: nginx-test
+  labels:
+    app.kubernetes.io/name: nginx
+    helm.sh/chart: nginx-9.5.4
+    app.kubernetes.io/instance: nginx-test
+    app.kubernetes.io/managed-by: Helm
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: "Cluster"
+  ports:
+    - name: http
+      port: 80
+      targetPort: http
+  selector:
+    app.kubernetes.io/name: nginx
+    app.kubernetes.io/instance: nginx-test
+`,
+			wantOutput: `apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/instance: nginx-test
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: nginx
+    helm.sh/chart: nginx-9.5.4
+    io.portainer.kubernetes.application.name: best-name
+    io.portainer.kubernetes.application.owner: best-owner
+  name: nginx-test
+spec:
+  externalTrafficPolicy: Cluster
+  ports:
+    - name: http
+      port: 80
+      targetPort: http
+  selector:
+    app.kubernetes.io/instance: nginx-test
+    app.kubernetes.io/name: nginx
+  type: LoadBalancer
+`,
+		},
+		{
+			name: "bitnami nginx deployment",
+			input: `apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-test
+  labels:
+    app.kubernetes.io/name: nginx
+    helm.sh/chart: nginx-9.5.4
+    app.kubernetes.io/instance: nginx-test
+    app.kubernetes.io/managed-by: Helm
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: nginx
+      app.kubernetes.io/instance: nginx-test
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: nginx
+        helm.sh/chart: nginx-9.5.4
+        app.kubernetes.io/instance: nginx-test
+        app.kubernetes.io/managed-by: Helm
+    spec:
+      automountServiceAccountToken: false
+      shareProcessNamespace: false
+      serviceAccountName: default
+      containers:
+        - name: nginx
+          image: docker.io/bitnami/nginx:1.21.3-debian-10-r0
+          imagePullPolicy: "IfNotPresent"
+`,
+			wantOutput: `apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/instance: nginx-test
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: nginx
+    helm.sh/chart: nginx-9.5.4
+    io.portainer.kubernetes.application.name: best-name
+    io.portainer.kubernetes.application.owner: best-owner
+  name: nginx-test
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: nginx-test
+      app.kubernetes.io/name: nginx
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/instance: nginx-test
+        app.kubernetes.io/managed-by: Helm
+        app.kubernetes.io/name: nginx
+        helm.sh/chart: nginx-9.5.4
+    spec:
+      automountServiceAccountToken: false
+      containers:
+        - image: docker.io/bitnami/nginx:1.21.3-debian-10-r0
+          imagePullPolicy: IfNotPresent
+          name: nginx
+      serviceAccountName: default
+      shareProcessNamespace: false
+`,
+		},
 	}
 
 	for _, tt := range tests {
@@ -417,77 +590,124 @@ spec:
 	}
 }
 
-func Test_AddAppLabels_PickingName_WhenLabelNameIsEmpty(t *testing.T) {
+func Test_DocumentSeperator(t *testing.T) {
 	labels := KubeAppLabels{
-		StackID: 123,
-		Owner:   "best-owner",
-		Kind:    "git",
+		StackID:   123,
+		StackName: "best-name",
+		Owner:     "best-owner",
+		Kind:      "git",
 	}
 
 	input := `apiVersion: v1
 kind: Service
 metadata:
-  name: web
-spec:
-  ports:
-    - name: "5000"
-      port: 5000
-      targetPort: 5000
+  labels:
+    io.kompose.service: database
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    io.kompose.service: backend
 `
-
 	expected := `apiVersion: v1
 kind: Service
 metadata:
   labels:
+    io.kompose.service: database
     io.portainer.kubernetes.application.kind: git
-    io.portainer.kubernetes.application.name: web
+    io.portainer.kubernetes.application.name: best-name
     io.portainer.kubernetes.application.owner: best-owner
+    io.portainer.kubernetes.application.stack: best-name
+    io.portainer.kubernetes.application.stackid: "123"
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    io.kompose.service: backend
+    io.portainer.kubernetes.application.kind: git
+    io.portainer.kubernetes.application.name: best-name
+    io.portainer.kubernetes.application.owner: best-owner
+    io.portainer.kubernetes.application.stack: best-name
     io.portainer.kubernetes.application.stackid: "123"
-  name: web
-spec:
-  ports:
-    - name: "5000"
-      port: 5000
-      targetPort: 5000
 `
-
-	result, err := AddAppLabels([]byte(input), labels)
+	result, err := AddAppLabels([]byte(input), labels.ToMap())
 	assert.NoError(t, err)
 	assert.Equal(t, expected, string(result))
 }
 
-func Test_AddAppLabels_PickingName_WhenLabelAndMetadataNameAreEmpty(t *testing.T) {
-	labels := KubeAppLabels{
-		StackID: 123,
-		Owner:   "best-owner",
-		Kind:    "git",
+func Test_GetNamespace(t *testing.T) {
+	tests := []struct {
+		name  string
+		input string
+		want  string
+	}{
+		{
+			name: "valid namespace",
+			input: `apiVersion: v1
+kind: Namespace
+metadata:
+  namespace: test-namespace
+`,
+			want: "test-namespace",
+		},
+		{
+			name: "invalid namespace",
+			input: `apiVersion: v1
+kind: Namespace
+`,
+			want: "",
+		},
 	}
 
-	input := `apiVersion: v1
-kind: Service
-spec:
-  ports:
-    - name: "5000"
-      port: 5000
-      targetPort: 5000
-`
-
-	expected := `apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    io.portainer.kubernetes.application.kind: git
-    io.portainer.kubernetes.application.name: ""
-    io.portainer.kubernetes.application.owner: best-owner
-    io.portainer.kubernetes.application.stackid: "123"
-spec:
-  ports:
-    - name: "5000"
-      port: 5000
-      targetPort: 5000
-`
-
-	result, err := AddAppLabels([]byte(input), labels)
-	assert.NoError(t, err)
-	assert.Equal(t, expected, string(result))
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result, err := GetNamespace([]byte(tt.input))
+			assert.NoError(t, err)
+			assert.Equal(t, tt.want, result)
+		})
+	}
+}
+
+func Test_ExtractDocuments(t *testing.T) {
+	tests := []struct {
+		name  string
+		input string
+		want  []string
+	}{
+		{
+			name: "multiple documents",
+			input: `apiVersion: v1
+kind: Namespace
+---
+apiVersion: v1
+kind: Service
+`,
+			want: []string{`apiVersion: v1
+kind: Namespace
+`, `apiVersion: v1
+kind: Service
+`},
+		},
+		{
+			name: "single document",
+			input: `apiVersion: v1
+kind: Namespace
+`,
+			want: []string{`apiVersion: v1
+kind: Namespace
+`},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			results, err := ExtractDocuments([]byte(tt.input), nil)
+			assert.NoError(t, err)
+			for i := range results {
+				assert.Equal(t, tt.want[i], string(results[i]))
+			}
+		})
+	}
 }

api/ldap/ldap.go

@@ -1,11 +1,11 @@
 package ldap
 
 import (
-	"errors"
 	"fmt"
 	"strings"
 
 	ldap "github.com/go-ldap/ldap/v3"
+	"github.com/pkg/errors"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/crypto"
 	httperrors "github.com/portainer/portainer/api/http/errors"
@@ -20,55 +20,28 @@ var (
 // Service represents a service used to authenticate users against a LDAP/AD.
 type Service struct{}
 
-func searchUser(username string, conn *ldap.Conn, settings []portainer.LDAPSearchSettings) (string, error) {
-	var userDN string
-	found := false
-	usernameEscaped := ldap.EscapeFilter(username)
-
-	for _, searchSettings := range settings {
-		searchRequest := ldap.NewSearchRequest(
-			searchSettings.BaseDN,
-			ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false,
-			fmt.Sprintf("(&%s(%s=%s))", searchSettings.Filter, searchSettings.UserNameAttribute, usernameEscaped),
-			[]string{"dn"},
-			nil,
-		)
-
-		// Deliberately skip errors on the search request so that we can jump to other search settings
-		// if any issue arise with the current one.
-		sr, err := conn.Search(searchRequest)
-		if err != nil {
-			continue
-		}
-
-		if len(sr.Entries) == 1 {
-			found = true
-			userDN = sr.Entries[0].DN
-			break
-		}
+func createConnection(settings *portainer.LDAPSettings) (*ldap.Conn, error) {
+	conn, err := createConnectionForURL(settings.URL, settings)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed creating LDAP connection")
 	}
 
-	if !found {
-		return "", errUserNotFound
-	}
-
-	return userDN, nil
+	return conn, nil
 }
 
-func createConnection(settings *portainer.LDAPSettings) (*ldap.Conn, error) {
-
+func createConnectionForURL(url string, settings *portainer.LDAPSettings) (*ldap.Conn, error) {
 	if settings.TLSConfig.TLS || settings.StartTLS {
 		config, err := crypto.CreateTLSConfigurationFromDisk(settings.TLSConfig.TLSCACertPath, settings.TLSConfig.TLSCertPath, settings.TLSConfig.TLSKeyPath, settings.TLSConfig.TLSSkipVerify)
 		if err != nil {
 			return nil, err
 		}
-		config.ServerName = strings.Split(settings.URL, ":")[0]
+		config.ServerName = strings.Split(url, ":")[0]
 
 		if settings.TLSConfig.TLS {
-			return ldap.DialTLS("tcp", settings.URL, config)
+			return ldap.DialTLS("tcp", url, config)
 		}
 
-		conn, err := ldap.Dial("tcp", settings.URL)
+		conn, err := ldap.Dial("tcp", url)
 		if err != nil {
 			return nil, err
 		}
@@ -81,7 +54,7 @@ func createConnection(settings *portainer.LDAPSettings) (*ldap.Conn, error) {
 		return conn, nil
 	}
 
-	return ldap.Dial("tcp", settings.URL)
+	return ldap.Dial("tcp", url)
 }
 
 // AuthenticateUser is used to authenticate a user against a LDAP/AD.
@@ -133,13 +106,157 @@ func (*Service) GetUserGroups(username string, settings *portainer.LDAPSettings)
 		return nil, err
 	}
 
-	userGroups := getGroups(userDN, connection, settings.GroupSearchSettings)
+	userGroups := getGroupsByUser(userDN, connection, settings.GroupSearchSettings)
 
 	return userGroups, nil
 }
 
+// SearchUsers searches for users with the specified settings
+func (*Service) SearchUsers(settings *portainer.LDAPSettings) ([]string, error) {
+	connection, err := createConnection(settings)
+	if err != nil {
+		return nil, err
+	}
+	defer connection.Close()
+
+	if !settings.AnonymousMode {
+		err = connection.Bind(settings.ReaderDN, settings.Password)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	users := map[string]bool{}
+
+	for _, searchSettings := range settings.SearchSettings {
+		searchRequest := ldap.NewSearchRequest(
+			searchSettings.BaseDN,
+			ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false,
+			searchSettings.Filter,
+			[]string{"dn", searchSettings.UserNameAttribute},
+			nil,
+		)
+
+		sr, err := connection.Search(searchRequest)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, user := range sr.Entries {
+			username := user.GetAttributeValue(searchSettings.UserNameAttribute)
+			if username != "" {
+				users[username] = true
+			}
+		}
+	}
+
+	usersList := []string{}
+	for user := range users {
+		usersList = append(usersList, user)
+	}
+
+	return usersList, nil
+}
+
+// SearchGroups searches for groups with the specified settings
+func (*Service) SearchGroups(settings *portainer.LDAPSettings) ([]portainer.LDAPUser, error) {
+	type groupSet map[string]bool
+
+	connection, err := createConnection(settings)
+	if err != nil {
+		return nil, err
+	}
+	defer connection.Close()
+
+	if !settings.AnonymousMode {
+		err = connection.Bind(settings.ReaderDN, settings.Password)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	userGroups := map[string]groupSet{}
+
+	for _, searchSettings := range settings.GroupSearchSettings {
+		searchRequest := ldap.NewSearchRequest(
+			searchSettings.GroupBaseDN,
+			ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false,
+			searchSettings.GroupFilter,
+			[]string{"cn", searchSettings.GroupAttribute},
+			nil,
+		)
+
+		sr, err := connection.Search(searchRequest)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, entry := range sr.Entries {
+			members := entry.GetAttributeValues(searchSettings.GroupAttribute)
+			for _, username := range members {
+				_, ok := userGroups[username]
+				if !ok {
+					userGroups[username] = groupSet{}
+				}
+				userGroups[username][entry.GetAttributeValue("cn")] = true
+			}
+		}
+	}
+
+	users := []portainer.LDAPUser{}
+
+	for username, groups := range userGroups {
+		groupList := []string{}
+		for group := range groups {
+			groupList = append(groupList, group)
+		}
+		user := portainer.LDAPUser{
+			Name:   username,
+			Groups: groupList,
+		}
+		users = append(users, user)
+	}
+
+	return users, nil
+}
+
+func searchUser(username string, conn *ldap.Conn, settings []portainer.LDAPSearchSettings) (string, error) {
+	var userDN string
+	found := false
+	usernameEscaped := ldap.EscapeFilter(username)
+
+	for _, searchSettings := range settings {
+		searchRequest := ldap.NewSearchRequest(
+			searchSettings.BaseDN,
+			ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false,
+			fmt.Sprintf("(&%s(%s=%s))", searchSettings.Filter, searchSettings.UserNameAttribute, usernameEscaped),
+			[]string{"dn"},
+			nil,
+		)
+
+		// Deliberately skip errors on the search request so that we can jump to other search settings
+		// if any issue arise with the current one.
+		sr, err := conn.Search(searchRequest)
+		if err != nil {
+			continue
+		}
+
+		if len(sr.Entries) == 1 {
+			found = true
+			userDN = sr.Entries[0].DN
+			break
+		}
+	}
+
+	if !found {
+		return "", errUserNotFound
+	}
+
+	return userDN, nil
+}
+
 // Get a list of group names for specified user from LDAP/AD
-func getGroups(userDN string, conn *ldap.Conn, settings []portainer.LDAPGroupSearchSettings) []string {
+func getGroupsByUser(userDN string, conn *ldap.Conn, settings []portainer.LDAPGroupSearchSettings) []string {
 	groups := make([]string, 0)
 	userDNEscaped := ldap.EscapeFilter(userDN)
 
@@ -179,9 +296,18 @@ func (*Service) TestConnectivity(settings *portainer.LDAPSettings) error {
 	}
 	defer connection.Close()
 
-	err = connection.Bind(settings.ReaderDN, settings.Password)
-	if err != nil {
-		return err
+	if !settings.AnonymousMode {
+		err = connection.Bind(settings.ReaderDN, settings.Password)
+		if err != nil {
+			return err
+		}
+
+	} else {
+		err = connection.UnauthenticatedBind("")
+		if err != nil {
+			return err
+		}
 	}
+
 	return nil
 }

api/portainer.go

@@ -3,7 +3,6 @@ package portainer
 import (
 	"context"
 	"io"
-	"net/http"
 	"time"
 
 	gittypes "github.com/portainer/portainer/api/git/types"
@@ -66,6 +65,7 @@ type (
 		SSL                       *bool
 		SSLCert                   *string
 		SSLKey                    *string
+		Rollback                  *bool
 		SnapshotInterval          *string
 	}
 
@@ -148,7 +148,7 @@ type (
 		Name         string       `json:"Name"`
 		Dynamic      bool         `json:"Dynamic"`
 		TagIDs       []TagID      `json:"TagIds"`
-		Endpoints    []EndpointID `json:"Environments"`
+		Endpoints    []EndpointID `json:"Endpoints"`
 		PartialMatch bool         `json:"PartialMatch"`
 	}
 
@@ -161,7 +161,7 @@ type (
 		ID             EdgeJobID                          `json:"Id" example:"1"`
 		Created        int64                              `json:"Created"`
 		CronExpression string                             `json:"CronExpression"`
-		Endpoints      map[EndpointID]EdgeJobEndpointMeta `json:"Environments"`
+		Endpoints      map[EndpointID]EdgeJobEndpointMeta `json:"Endpoints"`
 		Name           string                             `json:"Name"`
 		ScriptPath     string                             `json:"ScriptPath"`
 		Recurring      bool                               `json:"Recurring"`
@@ -188,7 +188,7 @@ type (
 		CronExpression string       `json:"CronExpression"`
 		Script         string       `json:"Script"`
 		Version        int          `json:"Version"`
-		Endpoints      []EndpointID `json:"Environments"`
+		Endpoints      []EndpointID `json:"Endpoints"`
 	}
 
 	//EdgeStack represents an edge stack
@@ -513,6 +513,12 @@ type (
 		AutoCreateUsers bool `json:"AutoCreateUsers" example:"true"`
 	}
 
+	// LDAPUser represents a LDAP user
+	LDAPUser struct {
+		Name   string
+		Groups []string
+	}
+
 	// LicenseInformation represents information about an extension license
 	LicenseInformation struct {
 		LicenseKey string `json:"LicenseKey,omitempty"`
@@ -713,6 +719,8 @@ type (
 		EnableTelemetry bool `json:"EnableTelemetry" example:"false"`
 		// Helm repository URL, defaults to "https://charts.bitnami.com/bitnami"
 		HelmRepositoryURL string `json:"HelmRepositoryURL" example:"https://charts.bitnami.com/bitnami"`
+		// KubectlImage, defaults to portainer/kubectl-shell
+		KubectlShellImage string `json:"KubectlShellImage" example:"portainer/kubectl-shell"`
 
 		// Deprecated fields
 		DisplayDonationHeader       bool
@@ -1022,7 +1030,7 @@ type (
 		// User Identifier
 		ID       UserID `json:"Id" example:"1"`
 		Username string `json:"Username" example:"bob"`
-		Password string `json:"Password,omitempty" example:"passwd"`
+		Password string `json:"Password,omitempty" swaggerignore:"true"`
 		// User Theme
 		UserTheme string `example:"dark"`
 		// User role (1 for administrator account and 2 for regular account)
@@ -1103,6 +1111,7 @@ type (
 		Close() error
 		IsNew() bool
 		MigrateData(force bool) error
+		Rollback(force bool) error
 		CheckCurrentEdition() error
 		BackupTo(w io.Writer) error
 
@@ -1204,6 +1213,7 @@ type (
 	FileService interface {
 		GetDockerConfigPath() string
 		GetFileContent(filePath string) ([]byte, error)
+		Copy(fromFilePath string, toFilePath string, deleteIfExists bool) error
 		Rename(oldPath, newPath string) error
 		RemoveDirectory(directoryPath string) error
 		StoreTLSFileFromBytes(folder string, fileType TLSFileType, data []byte) (string, error)
@@ -1261,7 +1271,7 @@ type (
 		SetupUserServiceAccount(userID int, teamIDs []int, restrictDefaultNamespace bool) error
 		GetServiceAccount(tokendata *TokenData) (*v1.ServiceAccount, error)
 		GetServiceAccountBearerToken(userID int) (string, error)
-		CreateUserShellPod(ctx context.Context, serviceAccountName string) (*KubernetesShellPod, error)
+		CreateUserShellPod(ctx context.Context, serviceAccountName, shellPodImage string) (*KubernetesShellPod, error)
 		StartExecProcess(token string, useAdminToken bool, namespace, podName, containerName string, command []string, stdin io.Reader, stdout io.Writer, errChan chan error)
 		NamespaceAccessPoliciesDeleteNamespace(namespace string) error
 		GetNodesLimits() (K8sNodesLimits, error)
@@ -1276,7 +1286,8 @@ type (
 
 	// KubernetesDeployer represents a service to deploy a manifest inside a Kubernetes environment(endpoint)
 	KubernetesDeployer interface {
-		Deploy(request *http.Request, endpoint *Endpoint, data string, namespace string) (string, error)
+		Deploy(userID UserID, endpoint *Endpoint, manifestFiles []string, namespace string) (string, error)
+		Remove(userID UserID, endpoint *Endpoint, manifestFiles []string, namespace string) (string, error)
 		ConvertCompose(data []byte) ([]byte, error)
 	}
 
@@ -1290,6 +1301,8 @@ type (
 		AuthenticateUser(username, password string, settings *LDAPSettings) error
 		TestConnectivity(settings *LDAPSettings) error
 		GetUserGroups(username string, settings *LDAPSettings) ([]string, error)
+		SearchGroups(settings *LDAPSettings) ([]LDAPUser, error)
+		SearchUsers(settings *LDAPSettings) ([]string, error)
 	}
 
 	// OAuthService represents a service used to authenticate users using OAuth
@@ -1324,7 +1337,9 @@ type (
 		SetTunnelStatusToActive(endpointID EndpointID)
 		SetTunnelStatusToRequired(endpointID EndpointID) error
 		SetTunnelStatusToIdle(endpointID EndpointID)
+		KeepTunnelAlive(endpointID EndpointID, ctx context.Context, maxKeepAlive time.Duration)
 		GetTunnelDetails(endpointID EndpointID) *TunnelDetails
+		GetActiveTunnel(endpoint *Endpoint) (*TunnelDetails, error)
 		AddEdgeJob(endpointID EndpointID, edgeJob *EdgeJob)
 		RemoveEdgeJob(edgeJobID EdgeJobID)
 	}
@@ -1455,9 +1470,9 @@ type (
 
 const (
 	// APIVersion is the version number of the Portainer API
-	APIVersion = "2.9.0"
+	APIVersion = "2.9.3"
 	// DBVersion is the version number of the Portainer database
-	DBVersion = 32
+	DBVersion = 35
 	// ComposeSyntaxMaxVersion is a maximum supported version of the docker compose syntax
 	ComposeSyntaxMaxVersion = "3.9"
 	// AssetsServerURL represents the URL of the Portainer asset server
@@ -1493,6 +1508,10 @@ const (
 	DefaultUserSessionTimeout = "8h"
 	// DefaultUserSessionTimeout represents the default timeout after which the user session is cleared
 	DefaultKubeconfigExpiry = "0"
+	// DefaultKubectlShellImage represents the default image and tag for the kubectl shell
+	DefaultKubectlShellImage = "portainer/kubectl-shell"
+	// WebSocketKeepAlive web socket keep alive for edge environments
+	WebSocketKeepAlive = 1 * time.Hour
 )
 
 const (

api/scheduler/scheduler.go

@@ -8,11 +8,12 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/robfig/cron/v3"
+	"github.com/sirupsen/logrus"
 )
 
 type Scheduler struct {
-	crontab     *cron.Cron
-	shutdownCtx context.Context
+	crontab    *cron.Cron
+	activeJobs map[cron.EntryID]context.CancelFunc
 }
 
 func NewScheduler(ctx context.Context) *Scheduler {
@@ -20,7 +21,8 @@ func NewScheduler(ctx context.Context) *Scheduler {
 	crontab.Start()
 
 	s := &Scheduler{
-		crontab: crontab,
+		crontab:    crontab,
+		activeJobs: make(map[cron.EntryID]context.CancelFunc),
 	}
 
 	if ctx != nil {
@@ -43,8 +45,10 @@ func (s *Scheduler) Shutdown() error {
 	ctx := s.crontab.Stop()
 	<-ctx.Done()
 
-	for _, j := range s.crontab.Entries() {
-		s.crontab.Remove(j.ID)
+	for _, job := range s.crontab.Entries() {
+		if cancel, ok := s.activeJobs[job.ID]; ok {
+			cancel()
+		}
 	}
 
 	err := ctx.Err()
@@ -60,14 +64,36 @@ func (s *Scheduler) StopJob(jobID string) error {
 	if err != nil {
 		return errors.Wrapf(err, "failed convert jobID %q to int", jobID)
 	}
-	s.crontab.Remove(cron.EntryID(id))
+	entryID := cron.EntryID(id)
+	if cancel, ok := s.activeJobs[entryID]; ok {
+		cancel()
+	}
 
 	return nil
 }
 
 // StartJobEvery schedules a new periodic job with a given duration.
-// Returns job id that could be used to stop the given job
-func (s *Scheduler) StartJobEvery(duration time.Duration, job func()) string {
-	entryId := s.crontab.Schedule(cron.Every(duration), cron.FuncJob(job))
-	return strconv.Itoa(int(entryId))
+// Returns job id that could be used to stop the given job.
+// When job run returns an error, that job won't be run again.
+func (s *Scheduler) StartJobEvery(duration time.Duration, job func() error) string {
+	ctx, cancel := context.WithCancel(context.Background())
+
+	j := cron.FuncJob(func() {
+		if err := job(); err != nil {
+			logrus.Debug("job returned an error")
+			cancel()
+		}
+	})
+
+	entryID := s.crontab.Schedule(cron.Every(duration), j)
+
+	s.activeJobs[entryID] = cancel
+
+	go func(entryID cron.EntryID) {
+		<-ctx.Done()
+		logrus.Debug("job cancelled, stopping")
+		s.crontab.Remove(entryID)
+	}(entryID)
+
+	return strconv.Itoa(int(entryID))
 }

api/scheduler/scheduler_test.go

@@ -9,49 +9,92 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func Test_CanStartAndTerminate(t *testing.T) {
-	s := NewScheduler(context.Background())
-	s.StartJobEvery(1*time.Minute, func() { fmt.Println("boop") })
+var jobInterval = time.Second
 
-	err := s.Shutdown()
-	assert.NoError(t, err, "Shutdown should return no errors")
-	assert.Empty(t, s.crontab.Entries(), "all jobs should have been removed")
-}
-
-func Test_CanTerminateByCancellingContext(t *testing.T) {
-	ctx, cancel := context.WithCancel(context.Background())
-	s := NewScheduler(ctx)
-	s.StartJobEvery(1*time.Minute, func() { fmt.Println("boop") })
-
-	cancel()
-
-	for i := 0; i < 100; i++ {
-		if len(s.crontab.Entries()) == 0 {
-			return
-		}
-		time.Sleep(10 * time.Millisecond)
-	}
-	t.Fatal("all jobs are expected to be cleaned by now; it might be a timing issue, otherwise implementation defect")
-}
-
-func Test_StartAndStopJob(t *testing.T) {
+func Test_ScheduledJobRuns(t *testing.T) {
 	s := NewScheduler(context.Background())
 	defer s.Shutdown()
 
-	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), 2*jobInterval)
 
-	var jobOne string
 	var workDone bool
-	jobOne = s.StartJobEvery(time.Second, func() {
-		assert.Equal(t, 1, len(s.crontab.Entries()), "scheduler should have one active job")
+	s.StartJobEvery(jobInterval, func() error {
 		workDone = true
 
-		s.StopJob(jobOne)
 		cancel()
+		return nil
 	})
 
 	<-ctx.Done()
 	assert.True(t, workDone, "value should been set in the job")
-	assert.Equal(t, 0, len(s.crontab.Entries()), "scheduler should have no active jobs")
-
+}
+
+func Test_JobCanBeStopped(t *testing.T) {
+	s := NewScheduler(context.Background())
+	defer s.Shutdown()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 2*jobInterval)
+
+	var workDone bool
+	jobID := s.StartJobEvery(jobInterval, func() error {
+		workDone = true
+
+		cancel()
+		return nil
+	})
+	s.StopJob(jobID)
+
+	<-ctx.Done()
+	assert.False(t, workDone, "job shouldn't had a chance to run")
+}
+
+func Test_JobShouldStop_UponError(t *testing.T) {
+	s := NewScheduler(context.Background())
+	defer s.Shutdown()
+
+	var acc int
+	s.StartJobEvery(jobInterval, func() error {
+		acc++
+		return fmt.Errorf("failed")
+	})
+
+	<-time.After(3 * jobInterval)
+	assert.Equal(t, 1, acc, "job stop after the first run because it returns an error")
+}
+
+func Test_CanTerminateAllJobs_ByShuttingDownScheduler(t *testing.T) {
+	s := NewScheduler(context.Background())
+
+	ctx, cancel := context.WithTimeout(context.Background(), 2*jobInterval)
+
+	var workDone bool
+	s.StartJobEvery(jobInterval, func() error {
+		workDone = true
+
+		cancel()
+		return nil
+	})
+
+	s.Shutdown()
+
+	<-ctx.Done()
+	assert.False(t, workDone, "job shouldn't had a chance to run")
+}
+
+func Test_CanTerminateAllJobs_ByCancellingParentContext(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 2*jobInterval)
+	s := NewScheduler(ctx)
+
+	var workDone bool
+	s.StartJobEvery(jobInterval, func() error {
+		workDone = true
+
+		cancel()
+		return nil
+	})
+
+	cancel()
+
+	<-ctx.Done()
+	assert.False(t, workDone, "job shouldn't had a chance to run")
 }

api/stacks/deploy.go

@@ -1,15 +1,29 @@
 package stacks
 
 import (
+	"fmt"
 	"strings"
 	"time"
 
 	"github.com/pkg/errors"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/http/security"
+	log "github.com/sirupsen/logrus"
 )
 
+type StackAuthorMissingErr struct {
+	stackID    int
+	authorName string
+}
+
+func (e *StackAuthorMissingErr) Error() string {
+	return fmt.Sprintf("stack's %v author %s is missing", e.stackID, e.authorName)
+}
+
 func RedeployWhenChanged(stackID portainer.StackID, deployer StackDeployer, datastore portainer.DataStore, gitService portainer.GitService) error {
+	logger := log.WithFields(log.Fields{"stackID": stackID})
+	logger.Debug("redeploying stack")
+
 	stack, err := datastore.Stack().Stack(stackID)
 	if err != nil {
 		return errors.WithMessagef(err, "failed to get the stack %v", stackID)
@@ -19,6 +33,17 @@ func RedeployWhenChanged(stackID portainer.StackID, deployer StackDeployer, data
 		return nil // do nothing if it isn't a git-based stack
 	}
 
+	author := stack.UpdatedBy
+	if author == "" {
+		author = stack.CreatedBy
+	}
+
+	user, err := datastore.User().UserByUsername(author)
+	if err != nil {
+		logger.WithFields(log.Fields{"author": author, "stack": stack.Name, "endpointID": stack.EndpointID}).Warn("cannot autoupdate a stack, stack author user is missing")
+		return &StackAuthorMissingErr{int(stack.ID), author}
+	}
+
 	username, password := "", ""
 	if stack.GitConfig.Authentication != nil {
 		username, password = stack.GitConfig.Authentication.Username, stack.GitConfig.Authentication.Password
@@ -54,12 +79,7 @@ func RedeployWhenChanged(stackID portainer.StackID, deployer StackDeployer, data
 		return errors.WithMessagef(err, "failed to find the environment %v associated to the stack %v", stack.EndpointID, stack.ID)
 	}
 
-	author := stack.UpdatedBy
-	if author == "" {
-		author = stack.CreatedBy
-	}
-
-	registries, err := getUserRegistries(datastore, author, endpoint.ID)
+	registries, err := getUserRegistries(datastore, user, endpoint.ID)
 	if err != nil {
 		return err
 	}
@@ -75,6 +95,12 @@ func RedeployWhenChanged(stackID portainer.StackID, deployer StackDeployer, data
 		if err != nil {
 			return errors.WithMessagef(err, "failed to deploy a docker compose stack %v", stackID)
 		}
+	case portainer.KubernetesStack:
+		logger.Debugf("deploying a kube app")
+		err := deployer.DeployKubernetesStack(stack, endpoint, user)
+		if err != nil {
+			return errors.WithMessagef(err, "failed to deploy a kubternetes app stack %v", stackID)
+		}
 	default:
 		return errors.Errorf("cannot update stack, type %v is unsupported", stack.Type)
 	}
@@ -88,24 +114,19 @@ func RedeployWhenChanged(stackID portainer.StackID, deployer StackDeployer, data
 	return nil
 }
 
-func getUserRegistries(datastore portainer.DataStore, authorUsername string, endpointID portainer.EndpointID) ([]portainer.Registry, error) {
+func getUserRegistries(datastore portainer.DataStore, user *portainer.User, endpointID portainer.EndpointID) ([]portainer.Registry, error) {
 	registries, err := datastore.Registry().Registries()
 	if err != nil {
 		return nil, errors.WithMessage(err, "unable to retrieve registries from the database")
 	}
 
-	user, err := datastore.User().UserByUsername(authorUsername)
-	if err != nil {
-		return nil, errors.WithMessagef(err, "failed to fetch a stack's author [%s]", authorUsername)
-	}
-
 	if user.Role == portainer.AdministratorRole {
 		return registries, nil
 	}
 
 	userMemberships, err := datastore.TeamMembership().TeamMembershipsByUserID(user.ID)
 	if err != nil {
-		return nil, errors.WithMessagef(err, "failed to fetch memberships of the stack author [%s]", authorUsername)
+		return nil, errors.WithMessagef(err, "failed to fetch memberships of the stack author [%s]", user.Username)
 	}
 
 	filteredRegistries := make([]portainer.Registry, 0, len(registries))

api/stacks/deploy_test.go

@@ -7,7 +7,7 @@ import (
 	"testing"
 
 	portainer "github.com/portainer/portainer/api"
-	bolt "github.com/portainer/portainer/api/bolt/bolttest"
+	"github.com/portainer/portainer/api/bolt"
 	gittypes "github.com/portainer/portainer/api/git/types"
 	"github.com/stretchr/testify/assert"
 )
@@ -35,6 +35,10 @@ func (s *noopDeployer) DeployComposeStack(stack *portainer.Stack, endpoint *port
 	return nil
 }
 
+func (s *noopDeployer) DeployKubernetesStack(stack *portainer.Stack, endpoint *portainer.Endpoint, user *portainer.User) error {
+	return nil
+}
+
 func Test_redeployWhenChanged_FailsWhenCannotFindStack(t *testing.T) {
 	store, teardown := bolt.MustNewTestStore(true)
 	defer teardown()
@@ -48,7 +52,11 @@ func Test_redeployWhenChanged_DoesNothingWhenNotAGitBasedStack(t *testing.T) {
 	store, teardown := bolt.MustNewTestStore(true)
 	defer teardown()
 
-	err := store.Stack().CreateStack(&portainer.Stack{ID: 1})
+	admin := &portainer.User{ID: 1, Username: "admin"}
+	err := store.User().CreateUser(admin)
+	assert.NoError(t, err, "error creating an admin")
+
+	err = store.Stack().CreateStack(&portainer.Stack{ID: 1, CreatedBy: "admin"})
 	assert.NoError(t, err, "failed to create a test stack")
 
 	err = RedeployWhenChanged(1, nil, store, &gitService{nil, ""})
@@ -61,8 +69,13 @@ func Test_redeployWhenChanged_DoesNothingWhenNoGitChanges(t *testing.T) {
 
 	tmpDir, _ := ioutil.TempDir("", "stack")
 
-	err := store.Stack().CreateStack(&portainer.Stack{
+	admin := &portainer.User{ID: 1, Username: "admin"}
+	err := store.User().CreateUser(admin)
+	assert.NoError(t, err, "error creating an admin")
+
+	err = store.Stack().CreateStack(&portainer.Stack{
 		ID:          1,
+		CreatedBy:   "admin",
 		ProjectPath: tmpDir,
 		GitConfig: &gittypes.RepoConfig{
 			URL:           "url",
@@ -80,8 +93,13 @@ func Test_redeployWhenChanged_FailsWhenCannotClone(t *testing.T) {
 	store, teardown := bolt.MustNewTestStore(true)
 	defer teardown()
 
-	err := store.Stack().CreateStack(&portainer.Stack{
-		ID: 1,
+	admin := &portainer.User{ID: 1, Username: "admin"}
+	err := store.User().CreateUser(admin)
+	assert.NoError(t, err, "error creating an admin")
+
+	err = store.Stack().CreateStack(&portainer.Stack{
+		ID:        1,
+		CreatedBy: "admin",
 		GitConfig: &gittypes.RepoConfig{
 			URL:           "url",
 			ReferenceName: "ref",
@@ -136,12 +154,12 @@ func Test_redeployWhenChanged(t *testing.T) {
 		assert.NoError(t, err)
 	})
 
-	t.Run("can NOT deploy kube stack", func(t *testing.T) {
+	t.Run("can deploy kube app", func(t *testing.T) {
 		stack.Type = portainer.KubernetesStack
 		store.Stack().UpdateStack(stack.ID, &stack)
 
 		err = RedeployWhenChanged(1, &noopDeployer{}, store, &gitService{nil, "newHash"})
-		assert.EqualError(t, err, "cannot update stack, type 3 is unsupported")
+		assert.NoError(t, err)
 	})
 }
 
@@ -151,12 +169,12 @@ func Test_getUserRegistries(t *testing.T) {
 
 	endpointID := 123
 
-	admin := portainer.User{ID: 1, Username: "admin", Role: portainer.AdministratorRole}
-	err := store.User().CreateUser(&admin)
+	admin := &portainer.User{ID: 1, Username: "admin", Role: portainer.AdministratorRole}
+	err := store.User().CreateUser(admin)
 	assert.NoError(t, err, "error creating an admin")
 
-	user := portainer.User{ID: 2, Username: "user", Role: portainer.StandardUserRole}
-	err = store.User().CreateUser(&user)
+	user := &portainer.User{ID: 2, Username: "user", Role: portainer.StandardUserRole}
+	err = store.User().CreateUser(user)
 	assert.NoError(t, err, "error creating a user")
 
 	team := portainer.Team{ID: 1, Name: "team"}
@@ -208,13 +226,13 @@ func Test_getUserRegistries(t *testing.T) {
 	assert.NoError(t, err, "couldn't create a registry")
 
 	t.Run("admin should has access to all registries", func(t *testing.T) {
-		registries, err := getUserRegistries(store, admin.Username, portainer.EndpointID(endpointID))
+		registries, err := getUserRegistries(store, admin, portainer.EndpointID(endpointID))
 		assert.NoError(t, err)
 		assert.ElementsMatch(t, []portainer.Registry{registryReachableByUser, registryReachableByTeam, registryRestricted}, registries)
 	})
 
 	t.Run("regular user has access to registries allowed to him and/or his team", func(t *testing.T) {
-		registries, err := getUserRegistries(store, user.Username, portainer.EndpointID(endpointID))
+		registries, err := getUserRegistries(store, user, portainer.EndpointID(endpointID))
 		assert.NoError(t, err)
 		assert.ElementsMatch(t, []portainer.Registry{registryReachableByUser, registryReachableByTeam}, registries)
 	})

api/stacks/deployer.go

@@ -2,27 +2,36 @@ package stacks
 
 import (
 	"context"
+	"os"
 	"sync"
 
+	"github.com/pkg/errors"
+
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/internal/stackutils"
+	k "github.com/portainer/portainer/api/kubernetes"
 )
 
 type StackDeployer interface {
 	DeploySwarmStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry, prune bool) error
 	DeployComposeStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry) error
+	DeployKubernetesStack(stack *portainer.Stack, endpoint *portainer.Endpoint, user *portainer.User) error
 }
 
 type stackDeployer struct {
 	lock                *sync.Mutex
 	swarmStackManager   portainer.SwarmStackManager
 	composeStackManager portainer.ComposeStackManager
+	kubernetesDeployer  portainer.KubernetesDeployer
 }
 
-func NewStackDeployer(swarmStackManager portainer.SwarmStackManager, composeStackManager portainer.ComposeStackManager) *stackDeployer {
+// NewStackDeployer inits a stackDeployer struct with a SwarmStackManager, a ComposeStackManager and a KubernetesDeployer
+func NewStackDeployer(swarmStackManager portainer.SwarmStackManager, composeStackManager portainer.ComposeStackManager, kubernetesDeployer portainer.KubernetesDeployer) *stackDeployer {
 	return &stackDeployer{
 		lock:                &sync.Mutex{},
 		swarmStackManager:   swarmStackManager,
 		composeStackManager: composeStackManager,
+		kubernetesDeployer:  kubernetesDeployer,
 	}
 }
 
@@ -45,3 +54,33 @@ func (d *stackDeployer) DeployComposeStack(stack *portainer.Stack, endpoint *por
 
 	return d.composeStackManager.Up(context.TODO(), stack, endpoint)
 }
+
+func (d *stackDeployer) DeployKubernetesStack(stack *portainer.Stack, endpoint *portainer.Endpoint, user *portainer.User) error {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+
+	appLabels := k.KubeAppLabels{
+		StackID:   int(stack.ID),
+		StackName: stack.Name,
+		Owner:     user.Username,
+	}
+
+	if stack.GitConfig == nil {
+		appLabels.Kind = "content"
+	} else {
+		appLabels.Kind = "git"
+	}
+
+	manifestFilePaths, tempDir, err := stackutils.CreateTempK8SDeploymentFiles(stack, d.kubernetesDeployer, appLabels)
+	if err != nil {
+		return errors.Wrap(err, "failed to create temp kub deployment files")
+	}
+	defer os.RemoveAll(tempDir)
+
+	_, err = d.kubernetesDeployer.Deploy(user.ID, endpoint, manifestFilePaths, stack.Namespace)
+	if err != nil {
+		return errors.Wrap(err, "failed to deploy kubernetes application")
+	}
+
+	return nil
+}

api/stacks/scheduled.go

@@ -1,7 +1,6 @@
 package stacks
 
 import (
-	"log"
 	"time"
 
 	"github.com/pkg/errors"
@@ -19,10 +18,9 @@ func StartStackSchedules(scheduler *scheduler.Scheduler, stackdeployer StackDepl
 		if err != nil {
 			return errors.Wrap(err, "Unable to parse auto update interval")
 		}
-		jobID := scheduler.StartJobEvery(d, func() {
-			if err := RedeployWhenChanged(stack.ID, stackdeployer, datastore, gitService); err != nil {
-				log.Printf("[ERROR] %s\n", err)
-			}
+		stackID := stack.ID // to be captured by the scheduled function
+		jobID := scheduler.StartJobEvery(d, func() error {
+			return RedeployWhenChanged(stackID, stackdeployer, datastore, gitService)
 		})
 
 		stack.AutoUpdate.JobID = jobID

app/assets/css/app.css

@@ -816,10 +816,6 @@ json-tree .branch-preview {
 }
 /* !spinkit override */
 
-.w-full {
-  width: 100%;
-}
-
 /* uib-typeahead override */
 #scrollable-dropdown-menu .dropdown-menu {
   max-height: 300px;
@@ -827,17 +823,33 @@ json-tree .branch-preview {
 }
 /* !uib-typeahead override */
 
-.my-8 {
-  margin-top: 2rem;
-  margin-bottom: 2rem;
-}
-
 .kubectl-shell {
   display: block;
   text-align: center;
   padding-bottom: 5px;
 }
 
+.w-full {
+  width: 100%;
+}
+
+.flex {
+  display: flex;
+}
+
+.block {
+  display: block;
+}
+
+.items-center {
+  align-items: center;
+}
+
+.my-8 {
+  margin-top: 2rem;
+  margin-bottom: 2rem;
+}
+
 .text-wrap {
   word-break: break-all;
   white-space: normal;

app/assets/css/theme.css

@@ -88,8 +88,14 @@ html {
 
   --green-1: #164;
   --green-2: #1ec863;
+  --green-3: #23ae89;
+
+  --orange-1: #e86925;
+
+  --BE-only: var(--orange-1);
 }
 
+/* Default Theme */
 :root {
   --bg-card-color: var(--grey-10);
   --bg-main-color: var(--white-color);
@@ -150,6 +156,8 @@ html {
   --bg-btn-focus: var(--grey-59);
   --bg-boxselector-disabled-color: var(--white-color);
   --bg-small-select-color: var(--white-color);
+  --bg-app-datatable-thead: var(--grey-23);
+  --bg-app-datatable-tbody: var(--grey-24);
 
   --text-main-color: var(--grey-7);
   --text-body-color: var(--grey-6);
@@ -318,6 +326,8 @@ html {
   --bg-btn-focus: var(--grey-3);
   --bg-boxselector-disabled-color: var(--grey-54);
   --bg-small-select-color: var(--grey-2);
+  --bg-app-datatable-thead: var(--grey-1);
+  --bg-app-datatable-tbody: var(--grey-1);
 
   --text-main-color: var(--white-color);
   --text-body-color: var(--white-color);
@@ -485,6 +495,8 @@ html {
   --bg-boxselector-color: var(--black-color);
   --bg-boxselector-disabled-color: var(--black-color);
   --bg-small-select-color: var(--black-color);
+  --bg-app-datatable-thead: var(--black-color);
+  --bg-app-datatable-tbody: var(--black-color);
 
   --text-main-color: var(--white-color);
   --text-body-color: var(--white-color);

app/assets/css/vendor-override.css

@@ -287,7 +287,7 @@ json-tree .branch-preview {
   margin-top: 15px;
 }
 
-.summary {
+.bold {
   color: var(--text-summary-color);
   font-weight: 700;
 }
@@ -396,4 +396,19 @@ input:-webkit-autofill {
 .btn-primary:hover {
   color: var(--white-color) !important;
 }
+
+.btn-danger:hover {
+  color: var(--white-color);
+}
 /* Overide Vendor CSS */
+
+.btn.disabled,
+.btn[disabled],
+fieldset[disabled] .btn {
+  pointer-events: none;
+  touch-action: none;
+}
+
+.multiSelect.inlineBlock button {
+  margin: 0;
+}