feat(app/k8s): update ingress scheme from v1beta1 to v1 (#5466)

Co-authored-by: Simon Meng <simon.meng@portainer.io>

.github/workflows/label-conflcts.yaml

@@ -0,0 +1,15 @@
+on:
+  push:
+    branches:
+      - develop
+      - 'release/**'
+jobs:
+  triage:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: mschilde/auto-label-merge-conflicts@master
+        with:
+          CONFLICT_LABEL_NAME: 'has conflicts'
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          MAX_RETRIES: 5
+          WAIT_MS: 5000

README.md

@@ -1,5 +1,5 @@
 <p align="center">
-  <img title="portainer" src='https://github.com/portainer/portainer/blob/develop/app/assets/images/logo_alt.png?raw=true' />
+  <img title="portainer" src='https://github.com/portainer/portainer/blob/develop/app/assets/images/portainer-github-banner.png?raw=true' />
 </p>
 
 **Portainer CE** is a lightweight ‘universal’ management GUI that can be used to **easily** manage Docker, Swarm, Kubernetes and ACI environments. It is designed to be as **simple** to deploy as it is to use.
@@ -8,6 +8,8 @@ Portainer consists of a single container that can run on any cluster. It can be
 
 **Portainer** allows you to manage all your orchestrator resources (containers, images, volumes, networks and more) through a super-simple graphical interface.
 
+A fully supported version of Portainer is available for business use. Visit http://www.portainer.io to learn more
+
 ## Demo
 
 You can try out the public demo instance: http://demo.portainer.io/ (login with the username **admin** and the password **tryportainer**).
@@ -55,6 +57,10 @@ You can join the Portainer Community by visiting community.portainer.io. This wi
 
 - Here at Portainer, we believe in [responsible disclosure](https://en.wikipedia.org/wiki/Responsible_disclosure) of security issues. If you have found a security issue, please report it to <security@portainer.io>.
 
+## WORK FOR US
+
+If you are a developer, and our code in this repo makes sense to you, we would love to hear from you. We are always on the hunt for awesome devs, either freelance or employed. Drop us a line to info@portainer.io with your details and we will be in touch. 
+
 ## Privacy
 
 **To make sure we focus our development effort in the right places we need to know which features get used most often. To give us this information we use [Matomo Analytics](https://matomo.org/), which is hosted in Germany and is fully GDPR compliant.**

api/backup/backup.go

@@ -10,6 +10,7 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/archive"
 	"github.com/portainer/portainer/api/crypto"
+	"github.com/portainer/portainer/api/filesystem"
 	"github.com/portainer/portainer/api/http/offlinegate"
 )
 
@@ -32,7 +33,7 @@ func CreateBackupArchive(password string, gate *offlinegate.OfflineGate, datasto
 	}
 
 	for _, filename := range filesToBackup {
-		err := copyPath(filepath.Join(filestorePath, filename), backupDirPath)
+		err := filesystem.CopyPath(filepath.Join(filestorePath, filename), backupDirPath)
 		if err != nil {
 			return "", errors.Wrap(err, "Failed to create backup file")
 		}

api/backup/copy_test.go

@@ -1,105 +0,0 @@
-package backup
-
-import (
-	"io/ioutil"
-	"os"
-	"path"
-	"path/filepath"
-	"testing"
-
-	"github.com/docker/docker/pkg/ioutils"
-	"github.com/stretchr/testify/assert"
-)
-
-func listFiles(dir string) []string {
-	items := make([]string, 0)
-	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
-		if path == dir {
-			return nil
-		}
-		items = append(items, path)
-		return nil
-	})
-
-	return items
-}
-
-func contains(t *testing.T, list []string, path string) {
-	assert.Contains(t, list, path)
-	copyContent, _ := ioutil.ReadFile(path)
-	assert.Equal(t, "content\n", string(copyContent))
-}
-
-func Test_copyFile_returnsError_whenSourceDoesNotExist(t *testing.T) {
-	tmpdir, _ := ioutils.TempDir("", "backup")
-	defer os.RemoveAll(tmpdir)
-
-	err := copyFile("does-not-exist", tmpdir)
-	assert.NotNil(t, err)
-}
-
-func Test_copyFile_shouldMakeAbackup(t *testing.T) {
-	tmpdir, _ := ioutils.TempDir("", "backup")
-	defer os.RemoveAll(tmpdir)
-
-	content := []byte("content")
-	ioutil.WriteFile(path.Join(tmpdir, "origin"), content, 0600)
-
-	err := copyFile(path.Join(tmpdir, "origin"), path.Join(tmpdir, "copy"))
-	assert.Nil(t, err)
-
-	copyContent, _ := ioutil.ReadFile(path.Join(tmpdir, "copy"))
-	assert.Equal(t, content, copyContent)
-}
-
-func Test_copyDir_shouldCopyAllFilesAndDirectories(t *testing.T) {
-	destination, _ := ioutils.TempDir("", "destination")
-	defer os.RemoveAll(destination)
-	err := copyDir("./test_assets/copy_test", destination)
-	assert.Nil(t, err)
-
-	createdFiles := listFiles(destination)
-
-	contains(t, createdFiles, filepath.Join(destination, "copy_test", "outer"))
-	contains(t, createdFiles, filepath.Join(destination, "copy_test", "dir", ".dotfile"))
-	contains(t, createdFiles, filepath.Join(destination, "copy_test", "dir", "inner"))
-}
-
-func Test_backupPath_shouldSkipWhenNotExist(t *testing.T) {
-	tmpdir, _ := ioutils.TempDir("", "backup")
-	defer os.RemoveAll(tmpdir)
-
-	err := copyPath("does-not-exists", tmpdir)
-	assert.Nil(t, err)
-
-	assert.Empty(t, listFiles(tmpdir))
-}
-
-func Test_backupPath_shouldCopyFile(t *testing.T) {
-	tmpdir, _ := ioutils.TempDir("", "backup")
-	defer os.RemoveAll(tmpdir)
-
-	content := []byte("content")
-	ioutil.WriteFile(path.Join(tmpdir, "file"), content, 0600)
-
-	os.MkdirAll(path.Join(tmpdir, "backup"), 0700)
-	err := copyPath(path.Join(tmpdir, "file"), path.Join(tmpdir, "backup"))
-	assert.Nil(t, err)
-
-	copyContent, err := ioutil.ReadFile(path.Join(tmpdir, "backup", "file"))
-	assert.Nil(t, err)
-	assert.Equal(t, content, copyContent)
-}
-
-func Test_backupPath_shouldCopyDir(t *testing.T) {
-	destination, _ := ioutils.TempDir("", "destination")
-	defer os.RemoveAll(destination)
-	err := copyPath("./test_assets/copy_test", destination)
-	assert.Nil(t, err)
-
-	createdFiles := listFiles(destination)
-
-	contains(t, createdFiles, filepath.Join(destination, "copy_test", "outer"))
-	contains(t, createdFiles, filepath.Join(destination, "copy_test", "dir", ".dotfile"))
-	contains(t, createdFiles, filepath.Join(destination, "copy_test", "dir", "inner"))
-}

api/backup/restore.go

@@ -11,6 +11,7 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/archive"
 	"github.com/portainer/portainer/api/crypto"
+	"github.com/portainer/portainer/api/filesystem"
 	"github.com/portainer/portainer/api/http/offlinegate"
 )
 
@@ -59,7 +60,7 @@ func extractArchive(r io.Reader, destinationDirPath string) error {
 
 func restoreFiles(srcDir string, destinationDir string) error {
 	for _, filename := range filesToRestore {
-		err := copyPath(filepath.Join(srcDir, filename), destinationDir)
+		err := filesystem.CopyPath(filepath.Join(srcDir, filename), destinationDir)
 		if err != nil {
 			return err
 		}

api/bolt/datastore.go

@@ -25,6 +25,7 @@ import (
 	"github.com/portainer/portainer/api/bolt/role"
 	"github.com/portainer/portainer/api/bolt/schedule"
 	"github.com/portainer/portainer/api/bolt/settings"
+	"github.com/portainer/portainer/api/bolt/ssl"
 	"github.com/portainer/portainer/api/bolt/stack"
 	"github.com/portainer/portainer/api/bolt/tag"
 	"github.com/portainer/portainer/api/bolt/team"
@@ -61,6 +62,7 @@ type Store struct {
 	RoleService             *role.Service
 	ScheduleService         *schedule.Service
 	SettingsService         *settings.Service
+	SSLSettingsService      *ssl.Service
 	StackService            *stack.Service
 	TagService              *tag.Service
 	TeamMembershipService   *teammembership.Service
@@ -114,6 +116,7 @@ func (store *Store) Open() error {
 }
 
 // Close closes the BoltDB database.
+// Safe to being called multiple times.
 func (store *Store) Close() error {
 	if store.connection.DB != nil {
 		return store.connection.Close()

api/bolt/init.go

@@ -55,6 +55,22 @@ func (store *Store) Init() error {
 		return err
 	}
 
+	_, err = store.SSLSettings().Settings()
+	if err != nil {
+		if err != errors.ErrObjectNotFound {
+			return err
+		}
+
+		defaultSSLSettings := &portainer.SSLSettings{
+			HTTPEnabled: true,
+		}
+
+		err = store.SSLSettings().UpdateSettings(defaultSSLSettings)
+		if err != nil {
+			return err
+		}
+	}
+
 	groups, err := store.EndpointGroupService.EndpointGroups()
 	if err != nil {
 		return err

api/bolt/migrator/migrate_dbversion31.go

@@ -5,7 +5,7 @@ import (
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/errors"
-	endpointutils "github.com/portainer/portainer/api/internal/endpoint"
+	"github.com/portainer/portainer/api/internal/endpointutils"
 	snapshotutils "github.com/portainer/portainer/api/internal/snapshot"
 )
 

api/bolt/migrator/migrate_dbversion33.go

@@ -0,0 +1,32 @@
+package migrator
+
+import (
+	portainer "github.com/portainer/portainer/api"
+)
+
+func (m *Migrator) migrateDBVersionTo33() error {
+	err := migrateStackEntryPoint(m.stackService)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func migrateStackEntryPoint(stackService portainer.StackService) error {
+	stacks, err := stackService.Stacks()
+	if err != nil {
+		return err
+	}
+	for i := range stacks {
+		stack := &stacks[i]
+		if stack.GitConfig == nil {
+			continue
+		}
+		stack.GitConfig.ConfigFilePath = stack.EntryPoint
+		if err := stackService.UpdateStack(stack.ID, stack); err != nil {
+			return err
+		}
+	}
+	return nil
+}

api/bolt/migrator/migrate_dbversion33_test.go

@@ -0,0 +1,51 @@
+package migrator
+
+import (
+	"path"
+	"testing"
+	"time"
+
+	"github.com/boltdb/bolt"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
+	"github.com/portainer/portainer/api/bolt/stack"
+	gittypes "github.com/portainer/portainer/api/git/types"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMigrateStackEntryPoint(t *testing.T) {
+	dbConn, err := bolt.Open(path.Join(t.TempDir(), "portainer-ee-mig-33.db"), 0600, &bolt.Options{Timeout: 1 * time.Second})
+	assert.NoError(t, err, "failed to init testing DB connection")
+	defer dbConn.Close()
+
+	stackService, err := stack.NewService(&internal.DbConnection{DB: dbConn})
+	assert.NoError(t, err, "failed to init testing Stack service")
+
+	stacks := []*portainer.Stack{
+		{
+			ID:         1,
+			EntryPoint: "dir/sub/compose.yml",
+		},
+		{
+			ID:         2,
+			EntryPoint: "dir/sub/compose.yml",
+			GitConfig:  &gittypes.RepoConfig{},
+		},
+	}
+
+	for _, s := range stacks {
+		err := stackService.CreateStack(s)
+		assert.NoError(t, err, "failed to create stack")
+	}
+
+	err = migrateStackEntryPoint(stackService)
+	assert.NoError(t, err, "failed to migrate entry point to Git ConfigFilePath")
+
+	s, err := stackService.Stack(1)
+	assert.NoError(t, err)
+	assert.Nil(t, s.GitConfig, "first stack should not have git config")
+
+	s, err = stackService.Stack(2)
+	assert.NoError(t, err)
+	assert.Equal(t, "dir/sub/compose.yml", s.GitConfig.ConfigFilePath, "second stack should have config file path migrated")
+}

api/bolt/migrator/migrator.go

@@ -381,5 +381,11 @@ func (m *Migrator) Migrate() error {
 		}
 	}
 
+	if m.currentDBVersion < 33 {
+		if err := m.migrateDBVersionTo33(); err != nil {
+			return err
+		}
+	}
+
 	return m.versionService.StoreDBVersion(portainer.DBVersion)
 }

api/bolt/services.go

@@ -16,6 +16,7 @@ import (
 	"github.com/portainer/portainer/api/bolt/role"
 	"github.com/portainer/portainer/api/bolt/schedule"
 	"github.com/portainer/portainer/api/bolt/settings"
+	"github.com/portainer/portainer/api/bolt/ssl"
 	"github.com/portainer/portainer/api/bolt/stack"
 	"github.com/portainer/portainer/api/bolt/tag"
 	"github.com/portainer/portainer/api/bolt/team"
@@ -105,6 +106,12 @@ func (store *Store) initServices() error {
 	}
 	store.SettingsService = settingsService
 
+	sslSettingsService, err := ssl.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.SSLSettingsService = sslSettingsService
+
 	stackService, err := stack.NewService(store.connection)
 	if err != nil {
 		return err
@@ -217,6 +224,11 @@ func (store *Store) Settings() portainer.SettingsService {
 	return store.SettingsService
 }
 
+// SSLSettings gives access to the SSL Settings data management layer
+func (store *Store) SSLSettings() portainer.SSLSettingsService {
+	return store.SSLSettingsService
+}
+
 // Stack gives access to the Stack data management layer
 func (store *Store) Stack() portainer.StackService {
 	return store.StackService

api/bolt/ssl/ssl.go

@@ -0,0 +1,46 @@
+package ssl
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "ssl"
+	key        = "SSL"
+)
+
+// Service represents a service for managing ssl data.
+type Service struct {
+	connection *internal.DbConnection
+}
+
+// NewService creates a new instance of a service.
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		connection: connection,
+	}, nil
+}
+
+// Settings retrieve the ssl settings object.
+func (service *Service) Settings() (*portainer.SSLSettings, error) {
+	var settings portainer.SSLSettings
+
+	err := internal.GetObject(service.connection, BucketName, []byte(key), &settings)
+	if err != nil {
+		return nil, err
+	}
+
+	return &settings, nil
+}
+
+// UpdateSettings persists a SSLSettings object.
+func (service *Service) UpdateSettings(settings *portainer.SSLSettings) error {
+	return internal.UpdateObject(service.connection, BucketName, []byte(key), settings)
+}

api/bolt/stack/stack.go

@@ -1,11 +1,14 @@
 package stack
 
 import (
+	"strings"
+
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/errors"
 	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
+	pkgerrors "github.com/pkg/errors"
 )
 
 const (
@@ -133,3 +136,76 @@ func (service *Service) DeleteStack(ID portainer.StackID) error {
 	identifier := internal.Itob(int(ID))
 	return internal.DeleteObject(service.connection, BucketName, identifier)
 }
+
+// StackByWebhookID returns a pointer to a stack object by webhook ID.
+// It returns nil, errors.ErrObjectNotFound if there's no stack associated with the webhook ID.
+func (service *Service) StackByWebhookID(id string) (*portainer.Stack, error) {
+	if id == "" {
+		return nil, pkgerrors.New("webhook ID can't be empty string")
+	}
+	var stack portainer.Stack
+	found := false
+
+	err := service.connection.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+		cursor := bucket.Cursor()
+
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var t struct {
+				AutoUpdate *struct {
+					WebhookID string `json:"Webhook"`
+				} `json:"AutoUpdate"`
+			}
+
+			err := internal.UnmarshalObject(v, &t)
+			if err != nil {
+				return err
+			}
+
+			if t.AutoUpdate != nil && strings.EqualFold(t.AutoUpdate.WebhookID, id) {
+				found = true
+				err := internal.UnmarshalObject(v, &stack)
+				if err != nil {
+					return err
+				}
+				break
+			}
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+	if !found {
+		return nil, errors.ErrObjectNotFound
+	}
+
+	return &stack, nil
+}
+
+// RefreshableStacks returns stacks that are configured for a periodic update
+func (service *Service) RefreshableStacks() ([]portainer.Stack, error) {
+	stacks := make([]portainer.Stack, 0)
+	err := service.connection.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+		cursor := bucket.Cursor()
+
+		var stack portainer.Stack
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			err := internal.UnmarshalObject(v, &stack)
+			if err != nil {
+				return err
+			}
+
+			if stack.AutoUpdate != nil && stack.AutoUpdate.Interval != "" {
+				stacks = append(stacks, stack)
+			}
+		}
+
+		return nil
+	})
+
+	return stacks, err
+}

api/bolt/stack/tests/stack_test.go

@@ -0,0 +1,111 @@
+package tests
+
+import (
+	"testing"
+	"time"
+
+	"github.com/portainer/portainer/api/bolt"
+
+	bolterrors "github.com/portainer/portainer/api/bolt/errors"
+
+	"github.com/portainer/portainer/api/bolt/bolttest"
+
+	"github.com/gofrs/uuid"
+
+	"github.com/stretchr/testify/assert"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/filesystem"
+)
+
+func newGuidString(t *testing.T) string {
+	uuid, err := uuid.NewV4()
+	assert.NoError(t, err)
+
+	return uuid.String()
+}
+
+type stackBuilder struct {
+	t     *testing.T
+	count int
+	store *bolt.Store
+}
+
+func TestService_StackByWebhookID(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping test in short mode. Normally takes ~1s to run.")
+	}
+	store, teardown := bolttest.MustNewTestStore(true)
+	defer teardown()
+
+	b := stackBuilder{t: t, store: store}
+	b.createNewStack(newGuidString(t))
+	for i := 0; i < 10; i++ {
+		b.createNewStack("")
+	}
+	webhookID := newGuidString(t)
+	stack := b.createNewStack(webhookID)
+
+	// can find a stack by webhook ID
+	got, err := store.StackService.StackByWebhookID(webhookID)
+	assert.NoError(t, err)
+	assert.Equal(t, stack, *got)
+
+	// returns nil and object not found error if there's no stack associated with the webhook
+	got, err = store.StackService.StackByWebhookID(newGuidString(t))
+	assert.Nil(t, got)
+	assert.ErrorIs(t, err, bolterrors.ErrObjectNotFound)
+}
+
+func (b *stackBuilder) createNewStack(webhookID string) portainer.Stack {
+	b.count++
+	stack := portainer.Stack{
+		ID:           portainer.StackID(b.count),
+		Name:         "Name",
+		Type:         portainer.DockerComposeStack,
+		EndpointID:   2,
+		EntryPoint:   filesystem.ComposeFileDefaultName,
+		Env:          []portainer.Pair{{"Name1", "Value1"}},
+		Status:       portainer.StackStatusActive,
+		CreationDate: time.Now().Unix(),
+		ProjectPath:  "/tmp/project",
+		CreatedBy:    "test",
+	}
+
+	if webhookID == "" {
+		if b.count%2 == 0 {
+			stack.AutoUpdate = &portainer.StackAutoUpdate{
+				Interval: "",
+				Webhook:  "",
+			}
+		} // else keep AutoUpdate nil
+	} else {
+		stack.AutoUpdate = &portainer.StackAutoUpdate{Webhook: webhookID}
+	}
+
+	err := b.store.StackService.CreateStack(&stack)
+	assert.NoError(b.t, err)
+
+	return stack
+}
+
+func Test_RefreshableStacks(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping test in short mode. Normally takes ~1s to run.")
+	}
+	store, teardown := bolttest.MustNewTestStore(true)
+	defer teardown()
+
+	staticStack := portainer.Stack{ID: 1}
+	stackWithWebhook := portainer.Stack{ID: 2, AutoUpdate: &portainer.StackAutoUpdate{Webhook: "webhook"}}
+	refreshableStack := portainer.Stack{ID: 3, AutoUpdate: &portainer.StackAutoUpdate{Interval: "1m"}}
+
+	for _, stack := range []*portainer.Stack{&staticStack, &stackWithWebhook, &refreshableStack} {
+		err := store.Stack().CreateStack(stack)
+		assert.NoError(t, err)
+	}
+
+	stacks, err := store.Stack().RefreshableStacks()
+	assert.NoError(t, err)
+	assert.ElementsMatch(t, []portainer.Stack{refreshableStack}, stacks)
+}

api/cli/cli.go

@@ -30,6 +30,7 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 
 	flags := &portainer.CLIFlags{
 		Addr:                      kingpin.Flag("bind", "Address and port to serve Portainer").Default(defaultBindAddress).Short('p').String(),
+		AddrHTTPS:                 kingpin.Flag("bind-https", "Address and port to serve Portainer via https").Default(defaultHTTPSBindAddress).String(),
 		TunnelAddr:                kingpin.Flag("tunnel-addr", "Address to serve the tunnel server").Default(defaultTunnelServerAddress).String(),
 		TunnelPort:                kingpin.Flag("tunnel-port", "Port to serve the tunnel server").Default(defaultTunnelServerPort).String(),
 		Assets:                    kingpin.Flag("assets", "Path to the assets").Default(defaultAssetsDirectory).Short('a').String(),
@@ -42,9 +43,10 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 		TLSCacert:                 kingpin.Flag("tlscacert", "Path to the CA").Default(defaultTLSCACertPath).String(),
 		TLSCert:                   kingpin.Flag("tlscert", "Path to the TLS certificate file").Default(defaultTLSCertPath).String(),
 		TLSKey:                    kingpin.Flag("tlskey", "Path to the TLS key").Default(defaultTLSKeyPath).String(),
-		SSL:                       kingpin.Flag("ssl", "Secure Portainer instance using SSL").Default(defaultSSL).Bool(),
-		SSLCert:                   kingpin.Flag("sslcert", "Path to the SSL certificate used to secure the Portainer instance").Default(defaultSSLCertPath).String(),
-		SSLKey:                    kingpin.Flag("sslkey", "Path to the SSL key used to secure the Portainer instance").Default(defaultSSLKeyPath).String(),
+		HTTPDisabled:              kingpin.Flag("http-disabled", "Serve portainer only on https").Default(defaultHTTPDisabled).Bool(),
+		SSL:                       kingpin.Flag("ssl", "Secure Portainer instance using SSL (deprecated)").Default(defaultSSL).Bool(),
+		SSLCert:                   kingpin.Flag("sslcert", "Path to the SSL certificate used to secure the Portainer instance").String(),
+		SSLKey:                    kingpin.Flag("sslkey", "Path to the SSL key used to secure the Portainer instance").String(),
 		SnapshotInterval:          kingpin.Flag("snapshot-interval", "Duration between each endpoint snapshot job").Default(defaultSnapshotInterval).String(),
 		AdminPassword:             kingpin.Flag("admin-password", "Hashed admin password").String(),
 		AdminPasswordFile:         kingpin.Flag("admin-password-file", "Path to the file containing the password for the admin user").String(),
@@ -92,6 +94,10 @@ func displayDeprecationWarnings(flags *portainer.CLIFlags) {
 	if *flags.NoAnalytics {
 		log.Println("Warning: The --no-analytics flag has been kept to allow migration of instances running a previous version of Portainer with this flag enabled, to version 2.0 where enabling this flag will have no effect.")
 	}
+
+	if *flags.SSL {
+		log.Println("Warning: SSL is enabled by default and there is no need for the --ssl flag. It has been kept to allow migration of instances running a previous version of Portainer with this flag enabled")
+	}
 }
 
 func validateEndpointURL(endpointURL string) error {

api/cli/defaults.go

@@ -4,6 +4,7 @@ package cli
 
 const (
 	defaultBindAddress         = ":9000"
+	defaultHTTPSBindAddress    = ":9443"
 	defaultTunnelServerAddress = "0.0.0.0"
 	defaultTunnelServerPort    = "8000"
 	defaultDataDirectory       = "/data"
@@ -13,6 +14,7 @@ const (
 	defaultTLSCACertPath       = "/certs/ca.pem"
 	defaultTLSCertPath         = "/certs/cert.pem"
 	defaultTLSKeyPath          = "/certs/key.pem"
+	defaultHTTPDisabled        = "false"
 	defaultSSL                 = "false"
 	defaultSSLCertPath         = "/certs/portainer.crt"
 	defaultSSLKeyPath          = "/certs/portainer.key"

api/cli/defaults_windows.go

@@ -2,6 +2,7 @@ package cli
 
 const (
 	defaultBindAddress         = ":9000"
+	defaultHTTPSBindAddress    = ":9443"
 	defaultTunnelServerAddress = "0.0.0.0"
 	defaultTunnelServerPort    = "8000"
 	defaultDataDirectory       = "C:\\data"
@@ -11,6 +12,7 @@ const (
 	defaultTLSCACertPath       = "C:\\certs\\ca.pem"
 	defaultTLSCertPath         = "C:\\certs\\cert.pem"
 	defaultTLSKeyPath          = "C:\\certs\\key.pem"
+	defaultHTTPDisabled        = "false"
 	defaultSSL                 = "false"
 	defaultSSLCertPath         = "C:\\certs\\portainer.crt"
 	defaultSSLKeyPath          = "C:\\certs\\portainer.key"

api/cmd/portainer/main.go

@@ -6,7 +6,6 @@ import (
 	"os"
 	"strings"
 
-	wrapper "github.com/portainer/docker-compose-wrapper"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt"
 	"github.com/portainer/portainer/api/chisel"
@@ -24,12 +23,15 @@ import (
 	"github.com/portainer/portainer/api/internal/authorization"
 	"github.com/portainer/portainer/api/internal/edge"
 	"github.com/portainer/portainer/api/internal/snapshot"
+	"github.com/portainer/portainer/api/internal/ssl"
 	"github.com/portainer/portainer/api/jwt"
 	"github.com/portainer/portainer/api/kubernetes"
 	kubecli "github.com/portainer/portainer/api/kubernetes/cli"
 	"github.com/portainer/portainer/api/ldap"
 	"github.com/portainer/portainer/api/libcompose"
 	"github.com/portainer/portainer/api/oauth"
+	"github.com/portainer/portainer/api/scheduler"
+	"github.com/portainer/portainer/api/stacks"
 )
 
 func initCLI() *portainer.CLIFlags {
@@ -54,7 +56,7 @@ func initFileService(dataStorePath string) portainer.FileService {
 	return fileService
 }
 
-func initDataStore(dataStorePath string, fileService portainer.FileService) portainer.DataStore {
+func initDataStore(dataStorePath string, fileService portainer.FileService, shutdownCtx context.Context) portainer.DataStore {
 	store, err := bolt.NewStore(dataStorePath, fileService)
 	if err != nil {
 		log.Fatalf("failed creating data store: %v", err)
@@ -74,18 +76,21 @@ func initDataStore(dataStorePath string, fileService portainer.FileService) port
 	if err != nil {
 		log.Fatalf("failed migration: %v", err)
 	}
+
+	go shutdownDatastore(shutdownCtx, store)
 	return store
 }
 
+func shutdownDatastore(shutdownCtx context.Context, datastore portainer.DataStore) {
+	<-shutdownCtx.Done()
+	datastore.Close()
+}
+
 func initComposeStackManager(assetsPath string, dataStorePath string, reverseTunnelService portainer.ReverseTunnelService, proxyManager *proxy.Manager) portainer.ComposeStackManager {
 	composeWrapper, err := exec.NewComposeStackManager(assetsPath, dataStorePath, proxyManager)
 	if err != nil {
-		if err == wrapper.ErrBinaryNotFound {
-			log.Printf("[INFO] [message: docker-compose binary not found, falling back to libcompose]")
-			return libcompose.NewComposeStackManager(dataStorePath, reverseTunnelService)
-		}
-
-		log.Fatalf("failed initalizing compose stack manager; err=%s", err)
+		log.Printf("[INFO] [main,compose] [message: falling-back to libcompose] [error: %s]", err)
+		return libcompose.NewComposeStackManager(dataStorePath, reverseTunnelService)
 	}
 
 	return composeWrapper
@@ -136,6 +141,23 @@ func initGitService() portainer.GitService {
 	return git.NewService()
 }
 
+func initSSLService(addr, dataPath, certPath, keyPath string, fileService portainer.FileService, dataStore portainer.DataStore, shutdownTrigger context.CancelFunc) (*ssl.Service, error) {
+	slices := strings.Split(addr, ":")
+	host := slices[0]
+	if host == "" {
+		host = "0.0.0.0"
+	}
+
+	sslService := ssl.NewService(fileService, dataStore, shutdownTrigger)
+
+	err := sslService.Init(host, certPath, keyPath)
+	if err != nil {
+		return nil, err
+	}
+
+	return sslService, nil
+}
+
 func initDockerClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService) *docker.ClientFactory {
 	return docker.NewClientFactory(signatureService, reverseTunnelService)
 }
@@ -156,9 +178,10 @@ func initSnapshotService(snapshotInterval string, dataStore portainer.DataStore,
 	return snapshotService, nil
 }
 
-func initStatus(flags *portainer.CLIFlags) *portainer.Status {
+func initStatus(instanceID string) *portainer.Status {
 	return &portainer.Status{
-		Version: portainer.APIVersion,
+		Version:    portainer.APIVersion,
+		InstanceID: instanceID,
 	}
 }
 
@@ -182,7 +205,26 @@ func updateSettingsFromFlags(dataStore portainer.DataStore, flags *portainer.CLI
 		settings.BlackListedLabels = *flags.Labels
 	}
 
-	return dataStore.Settings().UpdateSettings(settings)
+	err = dataStore.Settings().UpdateSettings(settings)
+	if err != nil {
+		return err
+	}
+
+	httpEnabled := !*flags.HTTPDisabled
+
+	sslSettings, err := dataStore.SSLSettings().Settings()
+	if err != nil {
+		return err
+	}
+
+	sslSettings.HTTPEnabled = httpEnabled
+
+	err = dataStore.SSLSettings().UpdateSettings(sslSettings)
+	if err != nil {
+		return err
+	}
+
+	return nil
 }
 
 func loadAndParseKeyPair(fileService portainer.FileService, signatureService portainer.DigitalSignatureService) error {
@@ -354,7 +396,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 
 	fileService := initFileService(*flags.Data)
 
-	dataStore := initDataStore(*flags.Data, fileService)
+	dataStore := initDataStore(*flags.Data, fileService, shutdownCtx)
 
 	if err := dataStore.CheckCurrentEdition(); err != nil {
 		log.Fatal(err)
@@ -375,6 +417,11 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 
 	digitalSignatureService := initDigitalSignatureService()
 
+	sslService, err := initSSLService(*flags.AddrHTTPS, *flags.Data, *flags.SSLCert, *flags.SSLKey, fileService, dataStore, shutdownTrigger)
+	if err != nil {
+		log.Fatal(err)
+	}
+
 	err = initKeyPair(fileService, digitalSignatureService)
 	if err != nil {
 		log.Fatalf("failed initializing key pai: %v", err)
@@ -422,7 +469,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		log.Fatalf("failed loading edge jobs from database: %v", err)
 	}
 
-	applicationStatus := initStatus(flags)
+	applicationStatus := initStatus(instanceID)
 
 	err = initEndpoint(flags, dataStore, snapshotService)
 	if err != nil {
@@ -467,14 +514,25 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 
 	err = reverseTunnelService.StartTunnelServer(*flags.TunnelAddr, *flags.TunnelPort, snapshotService)
 	if err != nil {
-		log.Fatalf("failed starting license service: %s", err)
+		log.Fatalf("failed starting tunnel server: %s", err)
 	}
 
+	sslSettings, err := dataStore.SSLSettings().Settings()
+	if err != nil {
+		log.Fatalf("failed to fetch ssl settings from DB")
+	}
+
+	scheduler := scheduler.NewScheduler(shutdownCtx)
+	stackDeployer := stacks.NewStackDeployer(swarmStackManager, composeStackManager)
+	stacks.StartStackSchedules(scheduler, stackDeployer, dataStore, gitService)
+
 	return &http.Server{
 		AuthorizationService:        authorizationService,
 		ReverseTunnelService:        reverseTunnelService,
 		Status:                      applicationStatus,
 		BindAddress:                 *flags.Addr,
+		BindAddressHTTPS:            *flags.AddrHTTPS,
+		HTTPEnabled:                 sslSettings.HTTPEnabled,
 		AssetsPath:                  *flags.Assets,
 		DataStore:                   dataStore,
 		SwarmStackManager:           swarmStackManager,
@@ -490,13 +548,13 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		KubernetesTokenCacheManager: kubernetesTokenCacheManager,
 		SignatureService:            digitalSignatureService,
 		SnapshotService:             snapshotService,
-		SSL:                         *flags.SSL,
-		SSLCert:                     *flags.SSLCert,
-		SSLKey:                      *flags.SSLKey,
+		SSLService:                  sslService,
 		DockerClientFactory:         dockerClientFactory,
 		KubernetesClientFactory:     kubernetesClientFactory,
+		Scheduler:                   scheduler,
 		ShutdownCtx:                 shutdownCtx,
 		ShutdownTrigger:             shutdownTrigger,
+		StackDeployer:               stackDeployer,
 	}
 }
 
@@ -507,8 +565,8 @@ func main() {
 
 	for {
 		server := buildServer(flags)
-		log.Printf("Starting Portainer %s on %s\n", portainer.APIVersion, *flags.Addr)
+		log.Printf("[INFO] [cmd,main] Starting Portainer version %s\n", portainer.APIVersion)
 		err := server.Start()
-		log.Printf("Http server exited: %s\n", err)
+		log.Printf("[INFO] [cmd,main] Http server exited: %s\n", err)
 	}
 }

api/docker/client.go

@@ -42,7 +42,7 @@ func (factory *ClientFactory) CreateClient(endpoint *portainer.Endpoint, nodeNam
 	} else if endpoint.Type == portainer.AgentOnDockerEnvironment {
 		return createAgentClient(endpoint, factory.signatureService, nodeName)
 	} else if endpoint.Type == portainer.EdgeAgentOnDockerEnvironment {
-		return createEdgeClient(endpoint, factory.reverseTunnelService, nodeName)
+		return createEdgeClient(endpoint, factory.signatureService, factory.reverseTunnelService, nodeName)
 	}
 
 	if strings.HasPrefix(endpoint.URL, "unix://") || strings.HasPrefix(endpoint.URL, "npipe://") {
@@ -71,13 +71,22 @@ func createTCPClient(endpoint *portainer.Endpoint) (*client.Client, error) {
 	)
 }
 
-func createEdgeClient(endpoint *portainer.Endpoint, reverseTunnelService portainer.ReverseTunnelService, nodeName string) (*client.Client, error) {
+func createEdgeClient(endpoint *portainer.Endpoint, signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService, nodeName string) (*client.Client, error) {
 	httpCli, err := httpClient(endpoint)
 	if err != nil {
 		return nil, err
 	}
 
-	headers := map[string]string{}
+	signature, err := signatureService.CreateSignature(portainer.PortainerAgentSignatureMessage)
+	if err != nil {
+		return nil, err
+	}
+
+	headers := map[string]string{
+		portainer.PortainerAgentPublicKeyHeader: signatureService.EncodedPublicKey(),
+		portainer.PortainerAgentSignatureHeader: signature,
+	}
+
 	if nodeName != "" {
 		headers[portainer.PortainerAgentTargetHeader] = nodeName
 	}

api/exec/compose_stack.go

@@ -7,8 +7,8 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/pkg/errors"
 	wrapper "github.com/portainer/docker-compose-wrapper"
-
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/http/proxy"
 	"github.com/portainer/portainer/api/http/proxy/factory"
@@ -35,12 +35,6 @@ func NewComposeStackManager(binaryPath string, configPath string, proxyManager *
 	}, nil
 }
 
-// NormalizeStackName returns a new stack name with unsupported characters replaced
-func (w *ComposeStackManager) NormalizeStackName(name string) string {
-	r := regexp.MustCompile("[^a-z0-9]+")
-	return r.ReplaceAllString(strings.ToLower(name), "")
-}
-
 // ComposeSyntaxMaxVersion returns the maximum supported version of the docker compose syntax
 func (w *ComposeStackManager) ComposeSyntaxMaxVersion() string {
 	return portainer.ComposeSyntaxMaxVersion
@@ -50,7 +44,7 @@ func (w *ComposeStackManager) ComposeSyntaxMaxVersion() string {
 func (w *ComposeStackManager) Up(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
 	url, proxy, err := w.fetchEndpointProxy(endpoint)
 	if err != nil {
-		return err
+		return errors.Wrap(err, "failed to featch endpoint proxy")
 	}
 
 	if proxy != nil {
@@ -59,13 +53,12 @@ func (w *ComposeStackManager) Up(stack *portainer.Stack, endpoint *portainer.End
 
 	envFilePath, err := createEnvFile(stack)
 	if err != nil {
-		return err
+		return errors.Wrap(err, "failed to create env file")
 	}
 
-	filePath := stackFilePath(stack)
-
-	_, err = w.wrapper.Up([]string{filePath}, url, stack.Name, envFilePath, w.configPath)
-	return err
+	filePaths := append([]string{stack.EntryPoint}, stack.AdditionalFiles...)
+	_, err = w.wrapper.Up(filePaths, stack.ProjectPath, url, stack.Name, envFilePath, w.configPath)
+	return errors.Wrap(err, "failed to deploy a stack")
 }
 
 // Down stops and removes containers, networks, images, and volumes. Wraps `docker-compose down --remove-orphans` command
@@ -78,14 +71,16 @@ func (w *ComposeStackManager) Down(stack *portainer.Stack, endpoint *portainer.E
 		defer proxy.Close()
 	}
 
-	filePath := stackFilePath(stack)
+	filePaths := append([]string{stack.EntryPoint}, stack.AdditionalFiles...)
 
-	_, err = w.wrapper.Down([]string{filePath}, url, stack.Name)
+	_, err = w.wrapper.Down(filePaths, stack.ProjectPath, url, stack.Name)
 	return err
 }
 
-func stackFilePath(stack *portainer.Stack) string {
-	return path.Join(stack.ProjectPath, stack.EntryPoint)
+// NormalizeStackName returns a new stack name with unsupported characters replaced
+func (w *ComposeStackManager) NormalizeStackName(name string) string {
+	r := regexp.MustCompile("[^a-z0-9]+")
+	return r.ReplaceAllString(strings.ToLower(name), "")
 }
 
 func (w *ComposeStackManager) fetchEndpointProxy(endpoint *portainer.Endpoint) (string, *factory.ProxyServer, error) {
@@ -118,5 +113,5 @@ func createEnvFile(stack *portainer.Stack) (string, error) {
 	}
 	envfile.Close()
 
-	return envFilePath, nil
+	return "stack.env", nil
 }

api/exec/compose_stack_integration_test.go

@@ -1,5 +1,3 @@
-// +build integration
-
 package exec
 
 import (

api/exec/compose_stack_test.go

@@ -10,47 +10,6 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func Test_stackFilePath(t *testing.T) {
-	tests := []struct {
-		name     string
-		stack    *portainer.Stack
-		expected string
-	}{
-		// {
-		// 	name:     "should return empty result if stack is missing",
-		// 	stack:    nil,
-		// 	expected: "",
-		// },
-		// {
-		// 	name:     "should return empty result if stack don't have entrypoint",
-		// 	stack:    &portainer.Stack{},
-		// 	expected: "",
-		// },
-		{
-			name: "should allow file name and dir",
-			stack: &portainer.Stack{
-				ProjectPath: "dir",
-				EntryPoint:  "file",
-			},
-			expected: path.Join("dir", "file"),
-		},
-		{
-			name: "should allow file name only",
-			stack: &portainer.Stack{
-				EntryPoint: "file",
-			},
-			expected: "file",
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			result := stackFilePath(tt.stack)
-			assert.Equal(t, tt.expected, result)
-		})
-	}
-}
-
 func Test_createEnvFile(t *testing.T) {
 	dir := t.TempDir()
 
@@ -60,11 +19,6 @@ func Test_createEnvFile(t *testing.T) {
 		expected     string
 		expectedFile bool
 	}{
-		// {
-		// 	name:     "should not add env file option if stack is missing",
-		// 	stack:    nil,
-		// 	expected: "",
-		// },
 		{
 			name: "should not add env file option if stack doesn't have env variables",
 			stack: &portainer.Stack{
@@ -98,7 +52,7 @@ func Test_createEnvFile(t *testing.T) {
 			result, _ := createEnvFile(tt.stack)
 
 			if tt.expected != "" {
-				assert.Equal(t, path.Join(tt.stack.ProjectPath, "stack.env"), result)
+				assert.Equal(t, "stack.env", result)
 
 				f, _ := os.Open(path.Join(dir, "stack.env"))
 				content, _ := ioutil.ReadAll(f)

api/exec/swarm_stack.go

@@ -13,6 +13,7 @@ import (
 	"strings"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/internal/stackutils"
 )
 
 // SwarmStackManager represents a service for managing stacks.
@@ -63,22 +64,23 @@ func (manager *SwarmStackManager) Logout(endpoint *portainer.Endpoint) error {
 
 // Deploy executes the docker stack deploy command.
 func (manager *SwarmStackManager) Deploy(stack *portainer.Stack, prune bool, endpoint *portainer.Endpoint) error {
-	stackFilePath := path.Join(stack.ProjectPath, stack.EntryPoint)
+	filePaths := stackutils.GetStackFilePaths(stack)
 	command, args := manager.prepareDockerCommandAndArgs(manager.binaryPath, manager.dataPath, endpoint)
 
 	if prune {
-		args = append(args, "stack", "deploy", "--prune", "--with-registry-auth", "--compose-file", stackFilePath, stack.Name)
+		args = append(args, "stack", "deploy", "--prune", "--with-registry-auth")
 	} else {
-		args = append(args, "stack", "deploy", "--with-registry-auth", "--compose-file", stackFilePath, stack.Name)
+		args = append(args, "stack", "deploy", "--with-registry-auth")
 	}
 
+	args = configureFilePaths(args, filePaths)
+	args = append(args, stack.Name)
+
 	env := make([]string, 0)
 	for _, envvar := range stack.Env {
 		env = append(env, envvar.Name+"="+envvar.Value)
 	}
-
-	stackFolder := path.Dir(stackFilePath)
-	return runCommandAndCaptureStdErr(command, args, env, stackFolder)
+	return runCommandAndCaptureStdErr(command, args, env, stack.ProjectPath)
 }
 
 // Remove executes the docker stack rm command.
@@ -191,3 +193,10 @@ func (manager *SwarmStackManager) NormalizeStackName(name string) string {
 	r := regexp.MustCompile("[^a-z0-9]+")
 	return r.ReplaceAllString(strings.ToLower(name), "")
 }
+
+func configureFilePaths(args []string, filePaths []string) []string {
+	for _, path := range filePaths {
+		args = append(args, "--compose-file", path)
+	}
+	return args
+}

api/exec/swarm_stack_test.go

@@ -0,0 +1,15 @@
+package exec
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestConfigFilePaths(t *testing.T) {
+	args := []string{"stack", "deploy", "--with-registry-auth"}
+	filePaths := []string{"dir/file", "dir/file-two", "dir/file-three"}
+	expected := []string{"stack", "deploy", "--with-registry-auth", "--compose-file", "dir/file", "--compose-file", "dir/file-two", "--compose-file", "dir/file-three"}
+	output := configureFilePaths(args, filePaths)
+	assert.ElementsMatch(t, expected, output, "wrong output file paths")
+}

api/filesystem/copy.go

@@ -1,4 +1,4 @@
-package backup
+package filesystem
 
 import (
 	"errors"
@@ -8,7 +8,8 @@ import (
 	"strings"
 )
 
-func copyPath(path string, toDir string) error {
+// CopyPath copies file or directory defined by the path to the toDir path
+func CopyPath(path string, toDir string) error {
 	info, err := os.Stat(path)
 	if err != nil && errors.Is(err, os.ErrNotExist) {
 		// skip copy if file does not exist
@@ -20,17 +21,30 @@ func copyPath(path string, toDir string) error {
 		return copyFile(path, destination)
 	}
 
-	return copyDir(path, toDir)
+	return CopyDir(path, toDir, true)
 }
 
-func copyDir(fromDir, toDir string) error {
+// CopyDir copies contents of fromDir to toDir.
+// When keepParent is true, contents will be copied with their immediate parent dir,
+// i.e. given /from/dirA and /to/dirB with keepParent == true, result will be /to/dirB/dirA/<children>
+func CopyDir(fromDir, toDir string, keepParent bool) error {
 	cleanedSourcePath := filepath.Clean(fromDir)
 	parentDirectory := filepath.Dir(cleanedSourcePath)
 	err := filepath.Walk(cleanedSourcePath, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
-		destination := filepath.Join(toDir, strings.TrimPrefix(path, parentDirectory))
+		var destination string
+		if keepParent {
+			destination = filepath.Join(toDir, strings.TrimPrefix(path, parentDirectory))
+		} else {
+			destination = filepath.Join(toDir, strings.TrimPrefix(path, cleanedSourcePath))
+		}
+
+		if destination == "" {
+			return nil
+		}
+
 		if info.IsDir() {
 			return nil // skip directory creations
 		}

api/filesystem/copy_test.go

@@ -0,0 +1,92 @@
+package filesystem
+
+import (
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_copyFile_returnsError_whenSourceDoesNotExist(t *testing.T) {
+	tmpdir, _ := ioutil.TempDir("", "backup")
+	defer os.RemoveAll(tmpdir)
+
+	err := copyFile("does-not-exist", tmpdir)
+	assert.Error(t, err)
+}
+
+func Test_copyFile_shouldMakeAbackup(t *testing.T) {
+	tmpdir, _ := ioutil.TempDir("", "backup")
+	defer os.RemoveAll(tmpdir)
+
+	content := []byte("content")
+	ioutil.WriteFile(path.Join(tmpdir, "origin"), content, 0600)
+
+	err := copyFile(path.Join(tmpdir, "origin"), path.Join(tmpdir, "copy"))
+	assert.NoError(t, err)
+
+	copyContent, _ := ioutil.ReadFile(path.Join(tmpdir, "copy"))
+	assert.Equal(t, content, copyContent)
+}
+
+func Test_CopyDir_shouldCopyAllFilesAndDirectories(t *testing.T) {
+	destination, _ := ioutil.TempDir("", "destination")
+	defer os.RemoveAll(destination)
+	err := CopyDir("./testdata/copy_test", destination, true)
+	assert.NoError(t, err)
+
+	assert.FileExists(t, filepath.Join(destination, "copy_test", "outer"))
+	assert.FileExists(t, filepath.Join(destination, "copy_test", "dir", ".dotfile"))
+	assert.FileExists(t, filepath.Join(destination, "copy_test", "dir", "inner"))
+}
+
+func Test_CopyDir_shouldCopyOnlyDirContents(t *testing.T) {
+	destination, _ := ioutil.TempDir("", "destination")
+	defer os.RemoveAll(destination)
+	err := CopyDir("./testdata/copy_test", destination, false)
+	assert.NoError(t, err)
+
+	assert.FileExists(t, filepath.Join(destination, "outer"))
+	assert.FileExists(t, filepath.Join(destination, "dir", ".dotfile"))
+	assert.FileExists(t, filepath.Join(destination, "dir", "inner"))
+}
+
+func Test_CopyPath_shouldSkipWhenNotExist(t *testing.T) {
+	tmpdir, _ := ioutil.TempDir("", "backup")
+	defer os.RemoveAll(tmpdir)
+
+	err := CopyPath("does-not-exists", tmpdir)
+	assert.NoError(t, err)
+
+	assert.NoFileExists(t, tmpdir)
+}
+
+func Test_CopyPath_shouldCopyFile(t *testing.T) {
+	tmpdir, _ := ioutil.TempDir("", "backup")
+	defer os.RemoveAll(tmpdir)
+
+	content := []byte("content")
+	ioutil.WriteFile(path.Join(tmpdir, "file"), content, 0600)
+
+	os.MkdirAll(path.Join(tmpdir, "backup"), 0700)
+	err := CopyPath(path.Join(tmpdir, "file"), path.Join(tmpdir, "backup"))
+	assert.NoError(t, err)
+
+	copyContent, err := ioutil.ReadFile(path.Join(tmpdir, "backup", "file"))
+	assert.NoError(t, err)
+	assert.Equal(t, content, copyContent)
+}
+
+func Test_CopyPath_shouldCopyDir(t *testing.T) {
+	destination, _ := ioutil.TempDir("", "destination")
+	defer os.RemoveAll(destination)
+	err := CopyPath("./testdata/copy_test", destination)
+	assert.NoError(t, err)
+
+	assert.FileExists(t, filepath.Join(destination, "copy_test", "outer"))
+	assert.FileExists(t, filepath.Join(destination, "copy_test", "dir", ".dotfile"))
+	assert.FileExists(t, filepath.Join(destination, "copy_test", "dir", "inner"))
+}

api/filesystem/filesystem.go

@@ -50,6 +50,12 @@ const (
 	CustomTemplateStorePath = "custom_templates"
 	// TempPath represent the subfolder where temporary files are saved
 	TempPath = "tmp"
+	// SSLCertPath represents the default ssl certificates path
+	SSLCertPath = "certs"
+	// DefaultSSLCertFilename represents the default ssl certificate file name
+	DefaultSSLCertFilename = "cert.pem"
+	// DefaultSSLKeyFilename represents the default ssl key file name
+	DefaultSSLKeyFilename = "key.pem"
 )
 
 // ErrUndefinedTLSFileType represents an error returned on undefined TLS file type
@@ -74,6 +80,11 @@ func NewService(dataStorePath, fileStorePath string) (*Service, error) {
 		return nil, err
 	}
 
+	err = service.createDirectoryInStore(SSLCertPath)
+	if err != nil {
+		return nil, err
+	}
+
 	err = service.createDirectoryInStore(TLSStorePath)
 	if err != nil {
 		return nil, err
@@ -108,6 +119,66 @@ func (service *Service) GetStackProjectPath(stackIdentifier string) string {
 	return path.Join(service.fileStorePath, ComposeStorePath, stackIdentifier)
 }
 
+// Copy copies the file on fromFilePath to toFilePath
+// if toFilePath exists func will fail unless deleteIfExists is true
+func (service *Service) Copy(fromFilePath string, toFilePath string, deleteIfExists bool) error {
+	exists, err := service.FileExists(fromFilePath)
+	if err != nil {
+		return err
+	}
+
+	if !exists {
+		return errors.New("File doesn't exist")
+	}
+
+	finput, err := os.Open(fromFilePath)
+	if err != nil {
+		return err
+	}
+
+	defer finput.Close()
+
+	exists, err = service.FileExists(toFilePath)
+	if err != nil {
+		return err
+	}
+
+	if exists {
+		if !deleteIfExists {
+			return errors.New("Destination file exists")
+		}
+
+		err := os.Remove(toFilePath)
+		if err != nil {
+			return err
+		}
+	}
+
+	foutput, err := os.Create(toFilePath)
+	if err != nil {
+		return err
+	}
+
+	defer foutput.Close()
+
+	buf := make([]byte, 1024)
+	for {
+		n, err := finput.Read(buf)
+		if err != nil && err != io.EOF {
+			return err
+		}
+		if n == 0 {
+			break
+		}
+
+		if _, err := foutput.Write(buf[:n]); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 // StoreStackFileFromBytes creates a subfolder in the ComposeStorePath and stores a new file from bytes.
 // It returns the path to the folder where the file is stored.
 func (service *Service) StoreStackFileFromBytes(stackIdentifier, fileName string, data []byte) (string, error) {
@@ -507,6 +578,58 @@ func (service *Service) GetDatastorePath() string {
 	return service.dataStorePath
 }
 
+func (service *Service) wrapFileStore(filepath string) string {
+	return path.Join(service.fileStorePath, filepath)
+}
+
+func defaultCertPathUnderFileStore() (string, string) {
+	certPath := path.Join(SSLCertPath, DefaultSSLCertFilename)
+	keyPath := path.Join(SSLCertPath, DefaultSSLKeyFilename)
+	return certPath, keyPath
+}
+
+// GetDefaultSSLCertsPath returns the ssl certs path
+func (service *Service) GetDefaultSSLCertsPath() (string, string) {
+	certPath, keyPath := defaultCertPathUnderFileStore()
+	return service.wrapFileStore(certPath), service.wrapFileStore(keyPath)
+}
+
+// StoreSSLCertPair stores a ssl certificate pair
+func (service *Service) StoreSSLCertPair(cert, key []byte) (string, string, error) {
+	certPath, keyPath := defaultCertPathUnderFileStore()
+
+	r := bytes.NewReader(cert)
+	err := service.createFileInStore(certPath, r)
+	if err != nil {
+		return "", "", err
+	}
+
+	r = bytes.NewReader(key)
+	err = service.createFileInStore(keyPath, r)
+	if err != nil {
+		return "", "", err
+	}
+
+	return service.wrapFileStore(certPath), service.wrapFileStore(keyPath), nil
+}
+
+// CopySSLCertPair copies a ssl certificate pair
+func (service *Service) CopySSLCertPair(certPath, keyPath string) (string, string, error) {
+	defCertPath, defKeyPath := service.GetDefaultSSLCertsPath()
+
+	err := service.Copy(certPath, defCertPath, false)
+	if err != nil {
+		return "", "", err
+	}
+
+	err = service.Copy(keyPath, defKeyPath, false)
+	if err != nil {
+		return "", "", err
+	}
+
+	return defCertPath, defKeyPath, nil
+}
+
 // FileExists checks for the existence of the specified file.
 func FileExists(filePath string) (bool, error) {
 	if _, err := os.Stat(filePath); err != nil {

api/git/azure.go

@@ -2,15 +2,17 @@ package git
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
-	"github.com/pkg/errors"
-	"github.com/portainer/portainer/api/archive"
 	"io"
 	"io/ioutil"
 	"net/http"
 	"net/url"
 	"os"
 	"strings"
+
+	"github.com/pkg/errors"
+	"github.com/portainer/portainer/api/archive"
 )
 
 const (
@@ -37,7 +39,7 @@ type azureDownloader struct {
 
 func NewAzureDownloader(client *http.Client) *azureDownloader {
 	return &azureDownloader{
-		client: client,
+		client:  client,
 		baseUrl: "https://dev.azure.com",
 	}
 }
@@ -100,6 +102,57 @@ func (a *azureDownloader) downloadZipFromAzureDevOps(ctx context.Context, option
 	return zipFile.Name(), nil
 }
 
+func (a *azureDownloader) latestCommitID(ctx context.Context, options fetchOptions) (string, error) {
+	config, err := parseUrl(options.repositoryUrl)
+	if err != nil {
+		return "", errors.WithMessage(err, "failed to parse url")
+	}
+
+	refsUrl, err := a.buildRefsUrl(config, options.referenceName)
+	if err != nil {
+		return "", errors.WithMessage(err, "failed to build azure refs url")
+	}
+
+	req, err := http.NewRequestWithContext(ctx, "GET", refsUrl, nil)
+	if options.username != "" || options.password != "" {
+		req.SetBasicAuth(options.username, options.password)
+	} else if config.username != "" || config.password != "" {
+		req.SetBasicAuth(config.username, config.password)
+	}
+
+	if err != nil {
+		return "", errors.WithMessage(err, "failed to create a new HTTP request")
+	}
+
+	resp, err := a.client.Do(req)
+	if err != nil {
+		return "", errors.WithMessage(err, "failed to make an HTTP request")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("failed to get repository refs with a status \"%v\"", resp.Status)
+	}
+
+	var refs struct {
+		Value []struct {
+			Name     string `json:"name"`
+			ObjectId string `json:"objectId"`
+		}
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&refs); err != nil {
+		return "", errors.Wrap(err, "could not parse Azure Refs response")
+	}
+
+	for _, ref := range refs.Value {
+		if strings.EqualFold(ref.Name, options.referenceName) {
+			return ref.ObjectId, nil
+		}
+	}
+
+	return "", errors.Errorf("could not find ref %q in the repository", options.referenceName)
+}
+
 func parseUrl(rawUrl string) (*azureOptions, error) {
 	if strings.HasPrefix(rawUrl, "https://") || strings.HasPrefix(rawUrl, "http://") {
 		return parseHttpUrl(rawUrl)
@@ -193,6 +246,27 @@ func (a *azureDownloader) buildDownloadUrl(config *azureOptions, referenceName s
 	return u.String(), nil
 }
 
+func (a *azureDownloader) buildRefsUrl(config *azureOptions, referenceName string) (string, error) {
+	rawUrl := fmt.Sprintf("%s/%s/%s/_apis/git/repositories/%s/refs",
+		a.baseUrl,
+		url.PathEscape(config.organisation),
+		url.PathEscape(config.project),
+		url.PathEscape(config.repository))
+	u, err := url.Parse(rawUrl)
+
+	if err != nil {
+		return "", errors.Wrapf(err, "failed to parse refs url path %s", rawUrl)
+	}
+
+	// filterContains=main&api-version=6.0
+	q := u.Query()
+	q.Set("filterContains", formatReferenceName(referenceName))
+	q.Set("api-version", "6.0")
+	u.RawQuery = q.Encode()
+
+	return u.String(), nil
+}
+
 const (
 	branchPrefix = "refs/heads/"
 	tagPrefix    = "refs/tags/"

api/git/azure_integration_test.go

@@ -78,6 +78,18 @@ func TestService_ClonePrivateRepository_Azure(t *testing.T) {
 	assert.FileExists(t, filepath.Join(dst, "README.md"))
 }
 
+func TestService_LatestCommitID_Azure(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	pat := getRequiredValue(t, "AZURE_DEVOPS_PAT")
+	service := NewService()
+
+	repositoryUrl := "https://portainer.visualstudio.com/Playground/_git/dev_integration"
+	id, err := service.LatestCommitID(repositoryUrl, "refs/heads/main", "", pat)
+	assert.NoError(t, err)
+	assert.NotEmpty(t, id, "cannot guarantee commit id, but it should be not empty")
+}
+
 func getRequiredValue(t *testing.T, name string) string {
 	value, ok := os.LookupEnv(name)
 	if !ok {

api/git/azure_test.go

@@ -2,11 +2,12 @@ package git
 
 import (
 	"context"
-	"github.com/stretchr/testify/assert"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
 func Test_buildDownloadUrl(t *testing.T) {
@@ -27,6 +28,23 @@ func Test_buildDownloadUrl(t *testing.T) {
 	}
 }
 
+func Test_buildRefsUrl(t *testing.T) {
+	a := NewAzureDownloader(nil)
+	u, err := a.buildRefsUrl(&azureOptions{
+		organisation: "organisation",
+		project:      "project",
+		repository:   "repository",
+	}, "refs/heads/main")
+
+	expectedUrl, _ := url.Parse("https://dev.azure.com/organisation/project/_apis/git/repositories/repository/refs?filterContains=main&api-version=6.0")
+	actualUrl, _ := url.Parse(u)
+	assert.NoError(t, err)
+	assert.Equal(t, expectedUrl.Host, actualUrl.Host)
+	assert.Equal(t, expectedUrl.Scheme, actualUrl.Scheme)
+	assert.Equal(t, expectedUrl.Path, actualUrl.Path)
+	assert.Equal(t, expectedUrl.Query(), actualUrl.Query())
+}
+
 func Test_parseAzureUrl(t *testing.T) {
 	type args struct {
 		url string
@@ -248,3 +266,110 @@ func Test_azureDownloader_downloadZipFromAzureDevOps(t *testing.T) {
 		})
 	}
 }
+
+func Test_azureDownloader_latestCommitID(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		response := `{
+			"value": [
+				{
+					"name": "refs/heads/feature/calcApp",
+					"objectId": "ffe9cba521f00d7f60e322845072238635edb451",
+					"creator": {
+						"displayName": "Normal Paulk",
+						"url": "https://vssps.dev.azure.com/fabrikam/_apis/Identities/ac5aaba6-a66a-4e1d-b508-b060ec624fa9",
+						"_links": {
+							"avatar": {
+								"href": "https://dev.azure.com/fabrikam/_apis/GraphProfile/MemberAvatars/aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5"
+							}
+						},
+						"id": "ac5aaba6-a66a-4e1d-b508-b060ec624fa9",
+						"uniqueName": "dev@mailserver.com",
+						"imageUrl": "https://dev.azure.com/fabrikam/_api/_common/identityImage?id=ac5aaba6-a66a-4e1d-b508-b060ec624fa9",
+						"descriptor": "aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5"
+					},
+					"url": "https://dev.azure.com/fabrikam/7484f783-66a3-4f27-b7cd-6b08b0b077ed/_apis/git/repositories/d3d1760b-311c-4175-a726-20dfc6a7f885/refs?filter=heads%2Ffeature%2FcalcApp"
+				},
+				{
+					"name": "refs/heads/feature/replacer",
+					"objectId": "917131a709996c5cfe188c3b57e9a6ad90e8b85c",
+					"creator": {
+						"displayName": "Normal Paulk",
+						"url": "https://vssps.dev.azure.com/fabrikam/_apis/Identities/ac5aaba6-a66a-4e1d-b508-b060ec624fa9",
+						"_links": {
+							"avatar": {
+								"href": "https://dev.azure.com/fabrikam/_apis/GraphProfile/MemberAvatars/aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5"
+							}
+						},
+						"id": "ac5aaba6-a66a-4e1d-b508-b060ec624fa9",
+						"uniqueName": "dev@mailserver.com",
+						"imageUrl": "https://dev.azure.com/fabrikam/_api/_common/identityImage?id=ac5aaba6-a66a-4e1d-b508-b060ec624fa9",
+						"descriptor": "aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5"
+					},
+					"url": "https://dev.azure.com/fabrikam/7484f783-66a3-4f27-b7cd-6b08b0b077ed/_apis/git/repositories/d3d1760b-311c-4175-a726-20dfc6a7f885/refs?filter=heads%2Ffeature%2Freplacer"
+				},
+				{
+					"name": "refs/heads/master",
+					"objectId": "ffe9cba521f00d7f60e322845072238635edb451",
+					"creator": {
+						"displayName": "Normal Paulk",
+						"url": "https://vssps.dev.azure.com/fabrikam/_apis/Identities/ac5aaba6-a66a-4e1d-b508-b060ec624fa9",
+						"_links": {
+							"avatar": {
+								"href": "https://dev.azure.com/fabrikam/_apis/GraphProfile/MemberAvatars/aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5"
+							}
+						},
+						"id": "ac5aaba6-a66a-4e1d-b508-b060ec624fa9",
+						"uniqueName": "dev@mailserver.com",
+						"imageUrl": "https://dev.azure.com/fabrikam/_api/_common/identityImage?id=ac5aaba6-a66a-4e1d-b508-b060ec624fa9",
+						"descriptor": "aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5"
+					},
+					"url": "https://dev.azure.com/fabrikam/7484f783-66a3-4f27-b7cd-6b08b0b077ed/_apis/git/repositories/d3d1760b-311c-4175-a726-20dfc6a7f885/refs?filter=heads%2Fmaster"
+				}
+			],
+			"count": 3
+		}`
+		w.Header().Set("Content-Type", "application/json")
+		w.Write([]byte(response))
+	}))
+	defer server.Close()
+
+	a := &azureDownloader{
+		client:  server.Client(),
+		baseUrl: server.URL,
+	}
+
+	tests := []struct {
+		name    string
+		args    fetchOptions
+		want    string
+		wantErr bool
+	}{
+		{
+			name: "should be able to parse response",
+			args: fetchOptions{
+				referenceName: "refs/heads/master",
+				repositoryUrl: "https://dev.azure.com/Organisation/Project/_git/Repository"},
+			want:    "ffe9cba521f00d7f60e322845072238635edb451",
+			wantErr: false,
+		},
+		{
+			name: "should be able to parse response",
+			args: fetchOptions{
+				referenceName: "refs/heads/unknown",
+				repositoryUrl: "https://dev.azure.com/Organisation/Project/_git/Repository"},
+			want:    "",
+			wantErr: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			id, err := a.latestCommitID(context.Background(), tt.args)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("azureDownloader.latestCommitID() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			assert.Equal(t, tt.want, id)
+		})
+	}
+}

api/git/git.go

@@ -6,16 +6,26 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/pkg/errors"
 
 	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/config"
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/go-git/go-git/v5/plumbing/transport/client"
 	githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
+	"github.com/go-git/go-git/v5/storage/memory"
 )
 
+type fetchOptions struct {
+	repositoryUrl string
+	username      string
+	password      string
+	referenceName string
+}
+
 type cloneOptions struct {
 	repositoryUrl string
 	username      string
@@ -26,6 +36,7 @@ type cloneOptions struct {
 
 type downloader interface {
 	download(ctx context.Context, dst string, opt cloneOptions) error
+	latestCommitID(ctx context.Context, opt fetchOptions) (string, error)
 }
 
 type gitClient struct {
@@ -36,13 +47,7 @@ func (c gitClient) download(ctx context.Context, dst string, opt cloneOptions) e
 	gitOptions := git.CloneOptions{
 		URL:   opt.repositoryUrl,
 		Depth: opt.depth,
-	}
-
-	if opt.password != "" || opt.username != "" {
-		gitOptions.Auth = &githttp.BasicAuth{
-			Username: opt.username,
-			Password: opt.password,
-		}
+		Auth:  getAuth(opt.username, opt.password),
 	}
 
 	if opt.referenceName != "" {
@@ -62,6 +67,44 @@ func (c gitClient) download(ctx context.Context, dst string, opt cloneOptions) e
 	return nil
 }
 
+func (c gitClient) latestCommitID(ctx context.Context, opt fetchOptions) (string, error) {
+	remote := git.NewRemote(memory.NewStorage(), &config.RemoteConfig{
+		Name: "origin",
+		URLs: []string{opt.repositoryUrl},
+	})
+
+	listOptions := &git.ListOptions{
+		Auth: getAuth(opt.username, opt.password),
+	}
+
+	refs, err := remote.List(listOptions)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to list repository refs")
+	}
+
+	for _, ref := range refs {
+		if strings.EqualFold(ref.Name().String(), opt.referenceName) {
+			return ref.Hash().String(), nil
+		}
+	}
+
+	return "", errors.Errorf("could not find ref %q in the repository", opt.referenceName)
+}
+
+func getAuth(username, password string) *githttp.BasicAuth {
+	if password != "" {
+		if username == "" {
+			username = "token"
+		}
+
+		return &githttp.BasicAuth{
+			Username: username,
+			Password: password,
+		}
+	}
+	return nil
+}
+
 // Service represents a service for managing Git.
 type Service struct {
 	httpsCli *http.Client
@@ -74,6 +117,7 @@ func NewService() *Service {
 	httpsCli := &http.Client{
 		Transport: &http.Transport{
 			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+			Proxy:           http.ProxyFromEnvironment,
 		},
 		Timeout: 300 * time.Second,
 	}
@@ -108,3 +152,19 @@ func (service *Service) cloneRepository(destination string, options cloneOptions
 
 	return service.git.download(context.TODO(), destination, options)
 }
+
+// LatestCommitID returns SHA1 of the latest commit of the specified reference
+func (service *Service) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) {
+	options := fetchOptions{
+		repositoryUrl: repositoryURL,
+		username:      username,
+		password:      password,
+		referenceName: referenceName,
+	}
+
+	if isAzureUrl(options.repositoryUrl) {
+		return service.azure.latestCommitID(context.TODO(), options)
+	}
+
+	return service.git.latestCommitID(context.TODO(), options)
+}

api/git/git_integration_test.go

@@ -12,7 +12,7 @@ import (
 func TestService_ClonePrivateRepository_GitHub(t *testing.T) {
 	ensureIntegrationTest(t)
 
-	pat := getRequiredValue(t, "GITHUB_PAT")
+	accessToken := getRequiredValue(t, "GITHUB_PAT")
 	username := getRequiredValue(t, "GITHUB_USERNAME")
 	service := NewService()
 
@@ -21,7 +21,20 @@ func TestService_ClonePrivateRepository_GitHub(t *testing.T) {
 	defer os.RemoveAll(dst)
 
 	repositoryUrl := "https://github.com/portainer/private-test-repository.git"
-	err = service.CloneRepository(dst, repositoryUrl, "refs/heads/main", username, pat)
+	err = service.CloneRepository(dst, repositoryUrl, "refs/heads/main", username, accessToken)
 	assert.NoError(t, err)
 	assert.FileExists(t, filepath.Join(dst, "README.md"))
 }
+
+func TestService_LatestCommitID_GitHub(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	accessToken := getRequiredValue(t, "GITHUB_PAT")
+	username := getRequiredValue(t, "GITHUB_USERNAME")
+	service := NewService()
+
+	repositoryUrl := "https://github.com/portainer/private-test-repository.git"
+	id, err := service.LatestCommitID(repositoryUrl, "refs/heads/main", username, accessToken)
+	assert.NoError(t, err)
+	assert.NotEmpty(t, id, "cannot guarantee commit id, but it should be not empty")
+}

api/git/git_test.go

@@ -105,7 +105,19 @@ func Test_cloneRepository(t *testing.T) {
 	})
 
 	assert.NoError(t, err)
-	assert.Equal(t, 3, getCommitHistoryLength(t, err, dir), "cloned repo has incorrect depth")
+	assert.Equal(t, 4, getCommitHistoryLength(t, err, dir), "cloned repo has incorrect depth")
+}
+
+func Test_latestCommitID(t *testing.T) {
+	service := Service{git: gitClient{preserveGitDirectory: true}} // no need for http client since the test access the repo via file system.
+
+	repositoryURL := bareRepoDir
+	referenceName := "refs/heads/main"
+
+	id, err := service.LatestCommitID(repositoryURL, referenceName, "", "")
+
+	assert.NoError(t, err)
+	assert.Equal(t, "68dcaa7bd452494043c64252ab90db0f98ecf8d2", id)
 }
 
 func getCommitHistoryLength(t *testing.T, err error, dir string) int {
@@ -137,6 +149,10 @@ func (t *testDownloader) download(_ context.Context, _ string, _ cloneOptions) e
 	return nil
 }
 
+func (t *testDownloader) latestCommitID(_ context.Context, _ fetchOptions) (string, error) {
+	return "", nil
+}
+
 func Test_cloneRepository_azure(t *testing.T) {
 	tests := []struct {
 		name   string

api/git/types/types.go

@@ -1,10 +1,20 @@
 package gittypes
 
+// RepoConfig represents a configuration for a repo
 type RepoConfig struct {
 	// The repo url
-	URL string `example:"https://github.com/portainer/portainer-ee.git"`
+	URL string `example:"https://github.com/portainer/portainer.git"`
 	// The reference name
 	ReferenceName string `example:"refs/heads/branch_name"`
 	// Path to where the config file is in this url/refName
 	ConfigFilePath string `example:"docker-compose.yml"`
+	// Git credentials
+	Authentication *GitAuthentication
+	// Repository hash
+	ConfigHash string `example:"bc4c183d756879ea4d173315338110b31004b8e0"`
+}
+
+type GitAuthentication struct {
+	Username string
+	Password string
 }

api/go.mod

@@ -27,10 +27,11 @@ require (
 	github.com/mitchellh/mapstructure v1.1.2 // indirect
 	github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6
 	github.com/pkg/errors v0.9.1
-	github.com/portainer/docker-compose-wrapper v0.0.0-20210527221011-0a1418224b92
+	github.com/portainer/docker-compose-wrapper v0.0.0-20210810234209-d01bc85eb481
 	github.com/portainer/libcompose v0.5.3
-	github.com/portainer/libcrypto v0.0.0-20190723020515-23ebe86ab2c2
+	github.com/portainer/libcrypto v0.0.0-20210422035235-c652195c5c3a
 	github.com/portainer/libhttp v0.0.0-20190806161843-ba068f58be33
+	github.com/robfig/cron/v3 v3.0.1
 	github.com/sirupsen/logrus v1.8.1
 	github.com/stretchr/testify v1.7.0
 	golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2

api/go.sum

@@ -241,12 +241,12 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/portainer/docker-compose-wrapper v0.0.0-20210527221011-0a1418224b92 h1:Hh7SHCf3SJblVywU0TTn5lpTKsH5W23LAKH5sqWggig=
-github.com/portainer/docker-compose-wrapper v0.0.0-20210527221011-0a1418224b92/go.mod h1:PF2O2O4UNYWdtPcp6n/mIKpKk+f1jhFTezS8txbf+XM=
+github.com/portainer/docker-compose-wrapper v0.0.0-20210810234209-d01bc85eb481 h1:5c8N9Gh21Ja/9EIpfyHFmQvTCKgOjnRhosmo0ZshkFk=
+github.com/portainer/docker-compose-wrapper v0.0.0-20210810234209-d01bc85eb481/go.mod h1:WxDlJWZxCnicdLCPnLNEv7/gRhjeIVuCGmsv+iOPH3c=
 github.com/portainer/libcompose v0.5.3 h1:tE4WcPuGvo+NKeDkDWpwNavNLZ5GHIJ4RvuZXsI9uI8=
 github.com/portainer/libcompose v0.5.3/go.mod h1:7SKd/ho69rRKHDFSDUwkbMcol2TMKU5OslDsajr8Ro8=
-github.com/portainer/libcrypto v0.0.0-20190723020515-23ebe86ab2c2 h1:0PfgGLys9yHr4rtnirg0W0Cjvv6/DzxBIZk5sV59208=
-github.com/portainer/libcrypto v0.0.0-20190723020515-23ebe86ab2c2/go.mod h1:/wIeGwJOMYc1JplE/OvYMO5korce39HddIfI8VKGyAM=
+github.com/portainer/libcrypto v0.0.0-20210422035235-c652195c5c3a h1:qY8TbocN75n5PDl16o0uVr5MevtM5IhdwSelXEd4nFM=
+github.com/portainer/libcrypto v0.0.0-20210422035235-c652195c5c3a/go.mod h1:n54EEIq+MM0NNtqLeCby8ljL+l275VpolXO0ibHegLE=
 github.com/portainer/libhttp v0.0.0-20190806161843-ba068f58be33 h1:H8HR2dHdBf8HANSkUyVw4o8+4tegGcd+zyKZ3e599II=
 github.com/portainer/libhttp v0.0.0-20190806161843-ba068f58be33/go.mod h1:Y2TfgviWI4rT2qaOTHr+hq6MdKIE5YjgQAu7qwptTV0=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
@@ -263,6 +263,8 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE=
 github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
+github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
+github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
 github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=

api/http/handler/auth/authenticate.go

@@ -5,7 +5,6 @@ import (
 	"log"
 	"net/http"
 	"strings"
-	"time"
 
 	"github.com/asaskevich/govalidator"
 	httperror "github.com/portainer/libhttp/error"
@@ -134,14 +133,6 @@ func (handler *Handler) writeToken(w http.ResponseWriter, user *portainer.User)
 	return handler.persistAndWriteToken(w, composeTokenData(user))
 }
 
-func (handler *Handler) writeTokenForOAuth(w http.ResponseWriter, user *portainer.User, expiryTime *time.Time) *httperror.HandlerError {
-	token, err := handler.JWTService.GenerateTokenForOAuth(composeTokenData(user), expiryTime)
-	if err != nil {
-		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to generate JWT token", Err: err}
-	}
-	return response.JSON(w, &authenticateResponse{JWT: token})
-}
-
 func (handler *Handler) persistAndWriteToken(w http.ResponseWriter, tokenData *portainer.TokenData) *httperror.HandlerError {
 	token, err := handler.JWTService.GenerateToken(tokenData)
 	if err != nil {

api/http/handler/auth/authenticate_oauth.go

@@ -4,7 +4,6 @@ import (
 	"errors"
 	"log"
 	"net/http"
-	"time"
 
 	"github.com/asaskevich/govalidator"
 	httperror "github.com/portainer/libhttp/error"
@@ -26,21 +25,21 @@ func (payload *oauthPayload) Validate(r *http.Request) error {
 	return nil
 }
 
-func (handler *Handler) authenticateOAuth(code string, settings *portainer.OAuthSettings) (string, *time.Time, error) {
+func (handler *Handler) authenticateOAuth(code string, settings *portainer.OAuthSettings) (string, error) {
 	if code == "" {
-		return "", nil, errors.New("Invalid OAuth authorization code")
+		return "", errors.New("Invalid OAuth authorization code")
 	}
 
 	if settings == nil {
-		return "", nil, errors.New("Invalid OAuth configuration")
+		return "", errors.New("Invalid OAuth configuration")
 	}
 
-	username, expiryTime, err := handler.OAuthService.Authenticate(code, settings)
+	username, err := handler.OAuthService.Authenticate(code, settings)
 	if err != nil {
-		return "", nil, err
+		return "", err
 	}
 
-	return username, expiryTime, nil
+	return username, nil
 }
 
 // @id ValidateOAuth
@@ -70,7 +69,7 @@ func (handler *Handler) validateOAuth(w http.ResponseWriter, r *http.Request) *h
 		return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "OAuth authentication is not enabled", Err: errors.New("OAuth authentication is not enabled")}
 	}
 
-	username, expiryTime, err := handler.authenticateOAuth(payload.Code, &settings.OAuthSettings)
+	username, err := handler.authenticateOAuth(payload.Code, &settings.OAuthSettings)
 	if err != nil {
 		log.Printf("[DEBUG] - OAuth authentication error: %s", err)
 		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to authenticate through OAuth", Err: httperrors.ErrUnauthorized}
@@ -111,5 +110,5 @@ func (handler *Handler) validateOAuth(w http.ResponseWriter, r *http.Request) *h
 
 	}
 
-	return handler.writeTokenForOAuth(w, user, expiryTime)
+	return handler.writeToken(w, user)
 }

api/http/handler/endpointproxy/handler.go

@@ -29,6 +29,10 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler {
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.proxyRequestsToDockerAPI)))
 	h.PathPrefix("/{id}/kubernetes").Handler(
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.proxyRequestsToKubernetesAPI)))
+	h.PathPrefix("/{id}/agent/docker").Handler(
+		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.proxyRequestsToDockerAPI)))
+	h.PathPrefix("/{id}/agent/kubernetes").Handler(
+		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.proxyRequestsToKubernetesAPI)))
 	h.PathPrefix("/{id}/storidge").Handler(
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.proxyRequestsToStoridgeAPI)))
 	return h

api/http/handler/endpointproxy/proxy_docker.go

@@ -3,6 +3,7 @@ package endpointproxy
 import (
 	"errors"
 	"strconv"
+	"strings"
 	"time"
 
 	httperror "github.com/portainer/libhttp/error"
@@ -65,6 +66,12 @@ func (handler *Handler) proxyRequestsToDockerAPI(w http.ResponseWriter, r *http.
 	}
 
 	id := strconv.Itoa(endpointID)
-	http.StripPrefix("/"+id+"/docker", proxy).ServeHTTP(w, r)
+
+	prefix := "/" + id + "/agent/docker";
+	if !strings.HasPrefix(r.URL.Path, prefix) {
+		prefix = "/" + id + "/docker";
+	}
+
+	http.StripPrefix(prefix, proxy).ServeHTTP(w, r)
 	return nil
 }

api/http/handler/endpointproxy/proxy_kubernetes.go

@@ -65,17 +65,18 @@ func (handler *Handler) proxyRequestsToKubernetesAPI(w http.ResponseWriter, r *h
 		}
 	}
 
+	//  For KubernetesLocalEnvironment
 	requestPrefix := fmt.Sprintf("/%d/kubernetes", endpointID)
+
 	if endpoint.Type == portainer.AgentOnKubernetesEnvironment || endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment {
-		if isKubernetesRequest(strings.TrimPrefix(r.URL.String(), requestPrefix)) {
-			requestPrefix = fmt.Sprintf("/%d", endpointID)
+		requestPrefix = fmt.Sprintf("/%d", endpointID)
+
+		agentPrefix := fmt.Sprintf("/%d/agent/kubernetes", endpointID)
+		if strings.HasPrefix(r.URL.Path, agentPrefix) {
+			requestPrefix = agentPrefix
 		}
 	}
 
 	http.StripPrefix(requestPrefix, proxy).ServeHTTP(w, r)
 	return nil
 }
-
-func isKubernetesRequest(requestURL string) bool {
-	return strings.HasPrefix(requestURL, "/api") || strings.HasPrefix(requestURL, "/healthz")
-}

api/http/handler/endpoints/endpoint_association_delete.go

@@ -1,14 +1,17 @@
 package endpoints
 
 import (
+	"encoding/base64"
 	"errors"
-	"net/http"
-
+	"fmt"
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
 	bolterrors "github.com/portainer/portainer/api/bolt/errors"
+	"net/http"
+	"regexp"
+	"strings"
 )
 
 // @id EndpointAssociationDelete
@@ -45,6 +48,11 @@ func (handler *Handler) endpointAssociationDelete(w http.ResponseWriter, r *http
 	endpoint.Snapshots = []portainer.DockerSnapshot{}
 	endpoint.Kubernetes.Snapshots = []portainer.KubernetesSnapshot{}
 
+	endpoint.EdgeKey, err = handler.updateEdgeKey(endpoint.EdgeKey)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Invalid EdgeKey", err}
+	}
+
 	err = handler.DataStore.Endpoint().UpdateEndpoint(portainer.EndpointID(endpointID), endpoint)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Failed persisting endpoint in database", err}
@@ -54,3 +62,27 @@ func (handler *Handler) endpointAssociationDelete(w http.ResponseWriter, r *http
 
 	return response.JSON(w, endpoint)
 }
+
+func (handler *Handler) updateEdgeKey(edgeKey string) (string, error) {
+	oldEdgeKeyByte, err := base64.RawStdEncoding.DecodeString(edgeKey)
+	if err != nil {
+		return "", err
+	}
+
+	oldEdgeKeyStr := string(oldEdgeKeyByte)
+
+	httpPort := getPort(handler.BindAddress)
+	httpsPort := getPort(handler.BindAddressHTTPS)
+
+	// replace "http://" with "https://" and replace ":9000" with ":9443", in the case of default values
+	// oldEdgeKeyStr example: http://10.116.1.178:9000|10.116.1.178:8000|46:99:4a:8d:a6:de:6a:bd:d8:e2:1c:99:81:60:54:55|52
+	r := regexp.MustCompile(fmt.Sprintf("^(http://)([^|]+)(:%s)(|.*)", httpPort))
+	newEdgeKeyStr := r.ReplaceAllString(oldEdgeKeyStr, fmt.Sprintf("https://$2:%s$4", httpsPort))
+
+	return base64.RawStdEncoding.EncodeToString([]byte(newEdgeKeyStr)), nil
+}
+
+func getPort(url string) string {
+	items := strings.Split(url, ":")
+	return items[len(items) - 1]
+}

api/http/handler/endpoints/endpoint_registries_list.go

@@ -10,7 +10,7 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	bolterrors "github.com/portainer/portainer/api/bolt/errors"
 	"github.com/portainer/portainer/api/http/security"
-	endpointutils "github.com/portainer/portainer/api/internal/endpoint"
+	"github.com/portainer/portainer/api/internal/endpointutils"
 )
 
 // GET request on /endpoints/{id}/registries?namespace

api/http/handler/endpoints/handler.go

@@ -32,6 +32,8 @@ type Handler struct {
 	K8sClientFactory     *cli.ClientFactory
 	ComposeStackManager  portainer.ComposeStackManager
 	AuthorizationService *authorization.Service
+	BindAddress          string
+	BindAddressHTTPS     string
 }
 
 // NewHandler creates a handler to manage endpoint operations.

api/http/handler/handler.go

@@ -16,11 +16,13 @@ import (
 	"github.com/portainer/portainer/api/http/handler/endpointproxy"
 	"github.com/portainer/portainer/api/http/handler/endpoints"
 	"github.com/portainer/portainer/api/http/handler/file"
+	"github.com/portainer/portainer/api/http/handler/kubernetes"
 	"github.com/portainer/portainer/api/http/handler/motd"
 	"github.com/portainer/portainer/api/http/handler/registries"
 	"github.com/portainer/portainer/api/http/handler/resourcecontrols"
 	"github.com/portainer/portainer/api/http/handler/roles"
 	"github.com/portainer/portainer/api/http/handler/settings"
+	"github.com/portainer/portainer/api/http/handler/ssl"
 	"github.com/portainer/portainer/api/http/handler/stacks"
 	"github.com/portainer/portainer/api/http/handler/status"
 	"github.com/portainer/portainer/api/http/handler/tags"
@@ -46,12 +48,14 @@ type Handler struct {
 	EndpointGroupHandler   *endpointgroups.Handler
 	EndpointHandler        *endpoints.Handler
 	EndpointProxyHandler   *endpointproxy.Handler
+	KubernetesHandler      *kubernetes.Handler
 	FileHandler            *file.Handler
 	MOTDHandler            *motd.Handler
 	RegistryHandler        *registries.Handler
 	ResourceControlHandler *resourcecontrols.Handler
 	RoleHandler            *roles.Handler
 	SettingsHandler        *settings.Handler
+	SSLHandler             *ssl.Handler
 	StackHandler           *stacks.Handler
 	StatusHandler          *status.Handler
 	TagHandler             *tags.Handler
@@ -65,7 +69,7 @@ type Handler struct {
 }
 
 // @title PortainerCE API
-// @version 2.1.1
+// @version 2.6.3
 // @description.markdown api-description.md
 // @termsOfService
 
@@ -100,6 +104,8 @@ type Handler struct {
 // @tag.description Manage Docker environments
 // @tag.name endpoint_groups
 // @tag.description Manage endpoint groups
+// @tag.name kubernetes
+// @tag.description Manage Kubernetes cluster
 // @tag.name motd
 // @tag.description Fetch the message of the day
 // @tag.name registries
@@ -126,6 +132,8 @@ type Handler struct {
 // @tag.description Manage App Templates
 // @tag.name stacks
 // @tag.description Manage stacks
+// @tag.name ssl
+// @tag.description Manage ssl settings
 // @tag.name upload
 // @tag.description Upload files
 // @tag.name webhooks
@@ -156,6 +164,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		http.StripPrefix("/api", h.EdgeTemplatesHandler).ServeHTTP(w, r)
 	case strings.HasPrefix(r.URL.Path, "/api/endpoint_groups"):
 		http.StripPrefix("/api", h.EndpointGroupHandler).ServeHTTP(w, r)
+	case strings.HasPrefix(r.URL.Path, "/api/kubernetes"):
+		http.StripPrefix("/api", h.KubernetesHandler).ServeHTTP(w, r)
 	case strings.HasPrefix(r.URL.Path, "/api/endpoints"):
 		switch {
 		case strings.Contains(r.URL.Path, "/docker/"):
@@ -166,6 +176,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			http.StripPrefix("/api/endpoints", h.EndpointProxyHandler).ServeHTTP(w, r)
 		case strings.Contains(r.URL.Path, "/azure/"):
 			http.StripPrefix("/api/endpoints", h.EndpointProxyHandler).ServeHTTP(w, r)
+		case strings.Contains(r.URL.Path, "/agent/"):
+			http.StripPrefix("/api/endpoints", h.EndpointProxyHandler).ServeHTTP(w, r)
 		case strings.Contains(r.URL.Path, "/edge/"):
 			http.StripPrefix("/api/endpoints", h.EndpointEdgeHandler).ServeHTTP(w, r)
 		default:
@@ -193,6 +205,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		http.StripPrefix("/api", h.UploadHandler).ServeHTTP(w, r)
 	case strings.HasPrefix(r.URL.Path, "/api/users"):
 		http.StripPrefix("/api", h.UserHandler).ServeHTTP(w, r)
+	case strings.HasPrefix(r.URL.Path, "/api/ssl"):
+		http.StripPrefix("/api", h.SSLHandler).ServeHTTP(w, r)
 	case strings.HasPrefix(r.URL.Path, "/api/teams"):
 		http.StripPrefix("/api", h.TeamHandler).ServeHTTP(w, r)
 	case strings.HasPrefix(r.URL.Path, "/api/team_memberships"):

api/http/handler/kubernetes/handler.go

@@ -0,0 +1,68 @@
+package kubernetes
+
+import (
+	"errors"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	httperror "github.com/portainer/libhttp/error"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/http/middlewares"
+	"github.com/portainer/portainer/api/http/security"
+	"github.com/portainer/portainer/api/internal/authorization"
+	"github.com/portainer/portainer/api/internal/endpointutils"
+	"github.com/portainer/portainer/api/kubernetes/cli"
+)
+
+// Handler is the HTTP handler which will natively deal with to external endpoints.
+type Handler struct {
+	*mux.Router
+	dataStore               portainer.DataStore
+	kubernetesClientFactory *cli.ClientFactory
+	authorizationService    *authorization.Service
+}
+
+// NewHandler creates a handler to process pre-proxied requests to external APIs.
+func NewHandler(bouncer *security.RequestBouncer, authorizationService *authorization.Service, dataStore portainer.DataStore, kubernetesClientFactory *cli.ClientFactory) *Handler {
+	h := &Handler{
+		Router:                  mux.NewRouter(),
+		dataStore:               dataStore,
+		kubernetesClientFactory: kubernetesClientFactory,
+		authorizationService:    authorizationService,
+	}
+
+	kubeRouter := h.PathPrefix("/kubernetes/{id}").Subrouter()
+
+	kubeRouter.Use(bouncer.AuthenticatedAccess)
+	kubeRouter.Use(middlewares.WithEndpoint(dataStore.Endpoint(), "id"))
+	kubeRouter.Use(kubeOnlyMiddleware)
+
+	kubeRouter.PathPrefix("/config").Handler(
+		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.getKubernetesConfig))).Methods(http.MethodGet)
+
+	// namespaces
+	// in the future this piece of code might be in another package (or a few different packages - namespaces/namespace?)
+	// to keep it simple, we've decided to leave it like this.
+	namespaceRouter := kubeRouter.PathPrefix("/namespaces/{namespace}").Subrouter()
+	namespaceRouter.Handle("/system", bouncer.RestrictedAccess(httperror.LoggerHandler(h.namespacesToggleSystem))).Methods(http.MethodPut)
+
+	return h
+}
+
+func kubeOnlyMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(rw http.ResponseWriter, request *http.Request) {
+		endpoint, err := middlewares.FetchEndpoint(request)
+		if err != nil {
+			httperror.WriteError(rw, http.StatusInternalServerError, "Unable to find an endpoint on request context", err)
+			return
+		}
+
+		if !endpointutils.IsKubernetesEndpoint(endpoint) {
+			errMessage := "Endpoint is not a kubernetes endpoint"
+			httperror.WriteError(rw, http.StatusBadRequest, errMessage, errors.New(errMessage))
+			return
+		}
+
+		next.ServeHTTP(rw, request)
+	})
+}

api/http/handler/kubernetes/kubernetes_config.go

@@ -0,0 +1,126 @@
+package kubernetes
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	portainer "github.com/portainer/portainer/api"
+	bolterrors "github.com/portainer/portainer/api/bolt/errors"
+	httperrors "github.com/portainer/portainer/api/http/errors"
+	"github.com/portainer/portainer/api/http/security"
+	kcli "github.com/portainer/portainer/api/kubernetes/cli"
+
+	"net/http"
+)
+
+// @id GetKubernetesConfig
+// @summary Generates kubeconfig file enabling client communication with k8s api server
+// @description Generates kubeconfig file enabling client communication with k8s api server
+// @description **Access policy**: authorized
+// @tags kubernetes
+// @security jwt
+// @accept json
+// @produce json
+// @param id path int true "Endpoint identifier"
+// @success 200 "Success"
+// @failure 400 "Invalid request"
+// @failure 401 "Unauthorized"
+// @failure 403 "Permission denied"
+// @failure 404 "Endpoint or ServiceAccount not found"
+// @failure 500 "Server error"
+// @router /kubernetes/{id}/config [get]
+func (handler *Handler) getKubernetesConfig(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint identifier route variable", err}
+	}
+
+	endpoint, err := handler.dataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
+	if err == bolterrors.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
+	} else if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint with the specified identifier inside the database", err}
+	}
+
+	bearerToken, err := extractBearerToken(r)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusUnauthorized, "Unauthorized", err}
+	}
+
+	tokenData, err := security.RetrieveTokenData(r)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access endpoint", err}
+	}
+
+	cli, err := handler.kubernetesClientFactory.GetKubeClient(endpoint)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to create Kubernetes client", err}
+	}
+
+	apiServerURL := getProxyUrl(r, endpointID)
+
+	config, err := cli.GetKubeConfig(r.Context(), apiServerURL, bearerToken, tokenData)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusNotFound, "Unable to generate Kubeconfig", err}
+	}
+
+	filenameBase := fmt.Sprintf("%s-%s", tokenData.Username, endpoint.Name)
+	contentAcceptHeader := r.Header.Get("Accept")
+	if contentAcceptHeader == "text/yaml" {
+		yaml, err := kcli.GenerateYAML(config)
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Failed to generate Kubeconfig", err}
+		}
+
+		w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; %s.yaml", filenameBase))
+		return YAML(w, yaml)
+	}
+
+	w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; %s.json", filenameBase))
+	return response.JSON(w, config)
+}
+
+// extractBearerToken extracts user's portainer bearer token from request auth header
+func extractBearerToken(r *http.Request) (string, error) {
+	token := ""
+	tokens := r.Header["Authorization"]
+	if len(tokens) >= 1 {
+		token = tokens[0]
+		token = strings.TrimPrefix(token, "Bearer ")
+	}
+	if token == "" {
+		return "", httperrors.ErrUnauthorized
+	}
+	return token, nil
+}
+
+// getProxyUrl generates portainer proxy url which acts as proxy to k8s api server
+func getProxyUrl(r *http.Request, endpointID int) string {
+	return fmt.Sprintf("https://%s/api/endpoints/%d/kubernetes", r.Host, endpointID)
+}
+
+// YAML writes yaml response as string to writer. Returns a pointer to a HandlerError if encoding fails.
+// This could be moved to a more useful place; but that place is most likely not in this project.
+// It should actually go in https://github.com/portainer/libhttp - since that is from where we use response.JSON.
+// We use `data interface{}` as parameter - since im trying to keep it as close to (or the same as) response.JSON method signature:
+// https://github.com/portainer/libhttp/blob/d20481a3da823c619887c440a22fdf4fa8f318f2/response/response.go#L13
+func YAML(rw http.ResponseWriter, data interface{}) *httperror.HandlerError {
+	rw.Header().Set("Content-Type", "text/yaml")
+
+	strData, ok := data.(string)
+	if !ok {
+		return &httperror.HandlerError{
+			StatusCode: http.StatusInternalServerError,
+			Message:    "Unable to write YAML response",
+			Err:        errors.New("failed to convert input to string"),
+		}
+	}
+
+	fmt.Fprint(rw, strData)
+
+	return nil
+}

api/http/handler/kubernetes/namespaces_toggle_system.go

@@ -0,0 +1,65 @@
+package kubernetes
+
+import (
+	"net/http"
+
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	"github.com/portainer/portainer/api/http/middlewares"
+)
+
+type namespacesToggleSystemPayload struct {
+	// Toggle the system state of this namespace to true or false
+	System bool `example:"true"`
+}
+
+func (payload *namespacesToggleSystemPayload) Validate(r *http.Request) error {
+	return nil
+}
+
+// @id KubernetesNamespacesToggleSystem
+// @summary Toggle the system state for a namespace
+// @description  Toggle the system state for a namespace
+// @description **Access policy**: administrator or endpoint admin
+// @security jwt
+// @tags kubernetes
+// @accept json
+// @param id path int true "Endpoint identifier"
+// @param namespace path string true "Namespace name"
+// @param body body namespacesToggleSystemPayload true "Update details"
+// @success 200 "Success"
+// @failure 400 "Invalid request"
+// @failure 404 "Endpoint not found"
+// @failure 500 "Server error"
+// @router /kubernetes/{id}/namespaces/{namespace}/system [put]
+func (handler *Handler) namespacesToggleSystem(rw http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	endpoint, err := middlewares.FetchEndpoint(r)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint on request context", err}
+	}
+
+	namespaceName, err := request.RetrieveRouteVariableValue(r, "namespace")
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid namespace identifier route variable", err}
+	}
+
+	var payload namespacesToggleSystemPayload
+	err = request.DecodeAndValidateJSONPayload(r, &payload)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
+	}
+
+	kubeClient, err := handler.kubernetesClientFactory.GetKubeClient(endpoint)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to create kubernetes client", err}
+	}
+
+	err = kubeClient.ToggleSystemState(namespaceName, payload.System)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to toggle system status", err}
+	}
+
+	return response.Empty(rw)
+
+}

api/http/handler/ssl/handler.go

@@ -0,0 +1,29 @@
+package ssl
+
+import (
+	"net/http"
+
+	"github.com/gorilla/mux"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/portainer/api/http/security"
+	"github.com/portainer/portainer/api/internal/ssl"
+)
+
+// Handler is the HTTP handler used to handle MOTD operations.
+type Handler struct {
+	*mux.Router
+	SSLService *ssl.Service
+}
+
+// NewHandler returns a new Handler
+func NewHandler(bouncer *security.RequestBouncer) *Handler {
+	h := &Handler{
+		Router: mux.NewRouter(),
+	}
+	h.Handle("/ssl",
+		bouncer.AdminAccess(httperror.LoggerHandler(h.sslInspect))).Methods(http.MethodGet)
+	h.Handle("/ssl",
+		bouncer.AdminAccess(httperror.LoggerHandler(h.sslUpdate))).Methods(http.MethodPut)
+
+	return h
+}

api/http/handler/ssl/ssl_inspect.go

@@ -0,0 +1,29 @@
+package ssl
+
+import (
+	"net/http"
+
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/response"
+)
+
+// @id SSLInspect
+// @summary Inspect the ssl settings
+// @description Retrieve the ssl settings.
+// @description **Access policy**: administrator
+// @tags ssl
+// @security jwt
+// @produce json
+// @success 200 {object} portainer.SSLSettings "Success"
+// @failure 400 "Invalid request"
+// @failure 403 "Permission denied to access settings"
+// @failure 500 "Server error"
+// @router /ssl [get]
+func (handler *Handler) sslInspect(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	settings, err := handler.SSLService.GetSSLSettings()
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Failed to fetch certificate info", err}
+	}
+
+	return response.JSON(w, settings)
+}

api/http/handler/ssl/ssl_update.go

@@ -0,0 +1,62 @@
+package ssl
+
+import (
+	"errors"
+	"net/http"
+
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+)
+
+type sslUpdatePayload struct {
+	Cert        *string
+	Key         *string
+	HTTPEnabled *bool
+}
+
+func (payload *sslUpdatePayload) Validate(r *http.Request) error {
+	if (payload.Cert == nil || payload.Key == nil) && payload.Cert != payload.Key {
+		return errors.New("both certificate and key files should be provided")
+	}
+
+	return nil
+}
+
+// @id SSLUpdate
+// @summary Update the ssl settings
+// @description Update the ssl settings.
+// @description **Access policy**: administrator
+// @tags ssl
+// @security jwt
+// @accept json
+// @produce json
+// @param body body sslUpdatePayload true "SSL Settings"
+// @success 204 "Success"
+// @failure 400 "Invalid request"
+// @failure 403 "Permission denied to access settings"
+// @failure 500 "Server error"
+// @router /ssl [put]
+func (handler *Handler) sslUpdate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	var payload sslUpdatePayload
+	err := request.DecodeAndValidateJSONPayload(r, &payload)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
+	}
+
+	if payload.Cert != nil {
+		err = handler.SSLService.SetCertificates([]byte(*payload.Cert), []byte(*payload.Key))
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Failed to save certificate", err}
+		}
+	}
+
+	if payload.HTTPEnabled != nil {
+		err = handler.SSLService.SetHTTPEnabled(*payload.HTTPEnabled)
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Failed to force https", err}
+		}
+	}
+
+	return response.Empty(w)
+}

api/http/handler/stacks/autoupdate.go

@@ -0,0 +1,38 @@
+package stacks
+
+import (
+	"log"
+	"net/http"
+	"time"
+
+	httperror "github.com/portainer/libhttp/error"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/scheduler"
+	"github.com/portainer/portainer/api/stacks"
+)
+
+func startAutoupdate(stackID portainer.StackID, interval string, scheduler *scheduler.Scheduler, stackDeployer stacks.StackDeployer, datastore portainer.DataStore, gitService portainer.GitService) (jobID string, e *httperror.HandlerError) {
+	d, err := time.ParseDuration(interval)
+	if err != nil {
+		return "", &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Unable to parse stack's auto update interval", Err: err}
+	}
+
+	jobID = scheduler.StartJobEvery(d, func() {
+		if err := stacks.RedeployWhenChanged(stackID, stackDeployer, datastore, gitService); err != nil {
+			log.Printf("[ERROR] [http,stacks] [message: failed redeploying] [err: %s]\n", err)
+		}
+	})
+
+	return jobID, nil
+}
+
+func stopAutoupdate(stackID portainer.StackID, jobID string, scheduler scheduler.Scheduler) {
+	if jobID == "" {
+		return
+	}
+
+	if err := scheduler.StopJob(jobID); err != nil {
+		log.Printf("[WARN] could not stop the job for the stack %v", stackID)
+	}
+
+}

api/http/handler/stacks/create_compose_stack.go

@@ -1,7 +1,6 @@
 package stacks
 
 import (
-	"errors"
 	"fmt"
 	"net/http"
 	"path"
@@ -9,10 +8,12 @@ import (
 	"time"
 
 	"github.com/asaskevich/govalidator"
+	"github.com/pkg/errors"
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/filesystem"
+	gittypes "github.com/portainer/portainer/api/git/types"
 	"github.com/portainer/portainer/api/http/security"
 )
 
@@ -100,7 +101,6 @@ func (handler *Handler) createComposeStackFromFileContent(w http.ResponseWriter,
 type composeStackFromGitRepositoryPayload struct {
 	// Name of the stack
 	Name string `example:"myStack" validate:"required"`
-
 	// URL of a Git repository hosting the Stack file
 	RepositoryURL string `example:"https://github.com/openfaas/faas" validate:"required"`
 	// Reference name of a Git repository hosting the Stack file
@@ -112,8 +112,11 @@ type composeStackFromGitRepositoryPayload struct {
 	// Password used in basic authentication. Required when RepositoryAuthentication is true.
 	RepositoryPassword string `example:"myGitPassword"`
 	// Path to the Stack file inside the Git repository
-	ComposeFilePathInRepository string `example:"docker-compose.yml" default:"docker-compose.yml"`
-
+	ComposeFile string `example:"docker-compose.yml" default:"docker-compose.yml"`
+	// Applicable when deploying with multiple stack files
+	AdditionalFiles []string `example:"[nz.compose.yml, uat.compose.yml]"`
+	// Optional auto update configuration
+	AutoUpdate *portainer.StackAutoUpdate
 	// A list of environment variables used during stack deployment
 	Env []portainer.Pair
 }
@@ -122,14 +125,18 @@ func (payload *composeStackFromGitRepositoryPayload) Validate(r *http.Request) e
 	if govalidator.IsNull(payload.Name) {
 		return errors.New("Invalid stack name")
 	}
-
 	if govalidator.IsNull(payload.RepositoryURL) || !govalidator.IsURL(payload.RepositoryURL) {
 		return errors.New("Invalid repository URL. Must correspond to a valid URL format")
 	}
-	if payload.RepositoryAuthentication && (govalidator.IsNull(payload.RepositoryUsername) || govalidator.IsNull(payload.RepositoryPassword)) {
-		return errors.New("Invalid repository credentials. Username and password must be specified when authentication is enabled")
+	if govalidator.IsNull(payload.RepositoryReferenceName) {
+		payload.RepositoryReferenceName = defaultGitReferenceName
+	}
+	if payload.RepositoryAuthentication && govalidator.IsNull(payload.RepositoryPassword) {
+		return errors.New("Invalid repository credentials. Password must be specified when authentication is enabled")
+	}
+	if err := validateStackAutoUpdate(payload.AutoUpdate); err != nil {
+		return err
 	}
-
 	return nil
 }
 
@@ -141,42 +148,72 @@ func (handler *Handler) createComposeStackFromGitRepository(w http.ResponseWrite
 	}
 
 	payload.Name = handler.ComposeStackManager.NormalizeStackName(payload.Name)
-	if payload.ComposeFilePathInRepository == "" {
-		payload.ComposeFilePathInRepository = filesystem.ComposeFileDefaultName
+	if payload.ComposeFile == "" {
+		payload.ComposeFile = filesystem.ComposeFileDefaultName
 	}
 
 	isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, false)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to check for name collision", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err}
 	}
 	if !isUnique {
-		errorMessage := fmt.Sprintf("A stack with the name '%s' already exists", payload.Name)
-		return &httperror.HandlerError{http.StatusConflict, errorMessage, errors.New(errorMessage)}
+		return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A stack with the name '%s' already exists", payload.Name), Err: errStackAlreadyExists}
+	}
+
+	//make sure the webhook ID is unique
+	if payload.AutoUpdate != nil && payload.AutoUpdate.Webhook != "" {
+		isUnique, err := handler.checkUniqueWebhookID(payload.AutoUpdate.Webhook)
+		if err != nil {
+			return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for webhook ID collision", Err: err}
+		}
+		if !isUnique {
+			return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("Webhook ID: %s already exists", payload.AutoUpdate.Webhook), Err: errWebhookIDAlreadyExists}
+		}
 	}
 
 	stackID := handler.DataStore.Stack().GetNextIdentifier()
 	stack := &portainer.Stack{
-		ID:           portainer.StackID(stackID),
-		Name:         payload.Name,
-		Type:         portainer.DockerComposeStack,
-		EndpointID:   endpoint.ID,
-		EntryPoint:   payload.ComposeFilePathInRepository,
-		Env:          payload.Env,
+		ID:              portainer.StackID(stackID),
+		Name:            payload.Name,
+		Type:            portainer.DockerComposeStack,
+		EndpointID:      endpoint.ID,
+		EntryPoint:      payload.ComposeFile,
+		AdditionalFiles: payload.AdditionalFiles,
+		AutoUpdate:      payload.AutoUpdate,
+		Env:             payload.Env,
+		GitConfig: &gittypes.RepoConfig{
+			URL:            payload.RepositoryURL,
+			ReferenceName:  payload.RepositoryReferenceName,
+			ConfigFilePath: payload.ComposeFile,
+		},
 		Status:       portainer.StackStatusActive,
 		CreationDate: time.Now().Unix(),
 	}
 
+	if payload.RepositoryAuthentication {
+		stack.GitConfig.Authentication = &gittypes.GitAuthentication{
+			Username: payload.RepositoryUsername,
+			Password: payload.RepositoryPassword,
+		}
+	}
+
 	projectPath := handler.FileService.GetStackProjectPath(strconv.Itoa(int(stack.ID)))
 	stack.ProjectPath = projectPath
 
 	doCleanUp := true
 	defer handler.cleanUp(stack, &doCleanUp)
 
-	err = handler.cloneAndSaveConfig(stack, projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, payload.ComposeFilePathInRepository, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword)
+	err = handler.clone(projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword)
 	if err != nil {
 		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to clone git repository", Err: err}
 	}
 
+	commitId, err := handler.latestCommitID(payload.RepositoryURL, payload.RepositoryReferenceName, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to fetch git repository id", Err: err}
+	}
+	stack.GitConfig.ConfigHash = commitId
+
 	config, configErr := handler.createComposeDeployConfig(r, stack, endpoint)
 	if configErr != nil {
 		return configErr
@@ -187,6 +224,15 @@ func (handler *Handler) createComposeStackFromGitRepository(w http.ResponseWrite
 		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: err.Error(), Err: err}
 	}
 
+	if payload.AutoUpdate != nil && payload.AutoUpdate.Interval != "" {
+		jobID, e := startAutoupdate(stack.ID, stack.AutoUpdate.Interval, handler.Scheduler, handler.StackDeployer, handler.DataStore, handler.GitService)
+		if e != nil {
+			return e
+		}
+
+		stack.AutoUpdate.JobID = jobID
+	}
+
 	stack.CreatedBy = config.user.Username
 
 	err = handler.DataStore.Stack().CreateStack(stack)
@@ -331,7 +377,7 @@ func (handler *Handler) createComposeDeployConfig(r *http.Request, stack *portai
 func (handler *Handler) deployComposeStack(config *composeStackDeploymentConfig) error {
 	isAdminOrEndpointAdmin, err := handler.userIsAdminOrEndpointAdmin(config.user, config.endpoint.ID)
 	if err != nil {
-		return err
+		return errors.Wrap(err, "failed to check user priviliges deploying a stack")
 	}
 
 	securitySettings := &config.endpoint.SecuritySettings
@@ -344,15 +390,17 @@ func (handler *Handler) deployComposeStack(config *composeStackDeploymentConfig)
 		!securitySettings.AllowContainerCapabilitiesForRegularUsers) &&
 		!isAdminOrEndpointAdmin {
 
-		composeFilePath := path.Join(config.stack.ProjectPath, config.stack.EntryPoint)
-		stackContent, err := handler.FileService.GetFileContent(composeFilePath)
-		if err != nil {
-			return err
-		}
+		for _, file := range append([]string{config.stack.EntryPoint}, config.stack.AdditionalFiles...) {
+			path := path.Join(config.stack.ProjectPath, file)
+			stackContent, err := handler.FileService.GetFileContent(path)
+			if err != nil {
+				return errors.Wrapf(err, "failed to get stack file content `%q`", path)
+			}
 
-		err = handler.isValidStackFile(stackContent, securitySettings)
-		if err != nil {
-			return err
+			err = handler.isValidStackFile(stackContent, securitySettings)
+			if err != nil {
+				return errors.Wrap(err, "compose file is invalid")
+			}
 		}
 	}
 
@@ -363,7 +411,7 @@ func (handler *Handler) deployComposeStack(config *composeStackDeploymentConfig)
 
 	err = handler.ComposeStackManager.Up(config.stack, config.endpoint)
 	if err != nil {
-		return err
+		return errors.Wrap(err, "failed to start up the stack")
 	}
 
 	return handler.SwarmStackManager.Logout(config.endpoint)

api/http/handler/stacks/create_kubernetes_stack_test.go

@@ -23,6 +23,10 @@ func (g *git) ClonePrivateRepositoryWithBasicAuth(repositoryURL, referenceName s
 	return g.ClonePublicRepository(repositoryURL, referenceName, destination)
 }
 
+func (g *git) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) {
+	return "", nil
+}
+
 func TestCloneAndConvertGitRepoFile(t *testing.T) {
 	dir, err := os.MkdirTemp("", "kube-create-stack")
 	assert.NoError(t, err, "failed to create a tmp dir")

api/http/handler/stacks/create_swarm_stack.go

@@ -13,6 +13,7 @@ import (
 	"github.com/portainer/libhttp/request"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/filesystem"
+	gittypes "github.com/portainer/portainer/api/git/types"
 	"github.com/portainer/portainer/api/http/security"
 )
 
@@ -121,7 +122,11 @@ type swarmStackFromGitRepositoryPayload struct {
 	// Password used in basic authentication. Required when RepositoryAuthentication is true.
 	RepositoryPassword string `example:"myGitPassword"`
 	// Path to the Stack file inside the Git repository
-	ComposeFilePathInRepository string `example:"docker-compose.yml" default:"docker-compose.yml"`
+	ComposeFile string `example:"docker-compose.yml" default:"docker-compose.yml"`
+	// Applicable when deploying with multiple stack files
+	AdditionalFiles []string `example:"[nz.compose.yml, uat.compose.yml]"`
+	// Optional auto update configuration
+	AutoUpdate *portainer.StackAutoUpdate
 }
 
 func (payload *swarmStackFromGitRepositoryPayload) Validate(r *http.Request) error {
@@ -134,11 +139,14 @@ func (payload *swarmStackFromGitRepositoryPayload) Validate(r *http.Request) err
 	if govalidator.IsNull(payload.RepositoryURL) || !govalidator.IsURL(payload.RepositoryURL) {
 		return errors.New("Invalid repository URL. Must correspond to a valid URL format")
 	}
-	if payload.RepositoryAuthentication && (govalidator.IsNull(payload.RepositoryUsername) || govalidator.IsNull(payload.RepositoryPassword)) {
-		return errors.New("Invalid repository credentials. Username and password must be specified when authentication is enabled")
+	if govalidator.IsNull(payload.RepositoryReferenceName) {
+		payload.RepositoryReferenceName = defaultGitReferenceName
 	}
-	if govalidator.IsNull(payload.ComposeFilePathInRepository) {
-		payload.ComposeFilePathInRepository = filesystem.ComposeFileDefaultName
+	if payload.RepositoryAuthentication && govalidator.IsNull(payload.RepositoryPassword) {
+		return errors.New("Invalid repository credentials. Password must be specified when authentication is enabled")
+	}
+	if err := validateStackAutoUpdate(payload.AutoUpdate); err != nil {
+		return err
 	}
 	return nil
 }
@@ -147,44 +155,74 @@ func (handler *Handler) createSwarmStackFromGitRepository(w http.ResponseWriter,
 	var payload swarmStackFromGitRepositoryPayload
 	err := request.DecodeAndValidateJSONPayload(r, &payload)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid request payload", Err: err}
 	}
 
 	payload.Name = handler.SwarmStackManager.NormalizeStackName(payload.Name)
 
 	isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, true)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to check for name collision", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err}
 	}
 	if !isUnique {
-		errorMessage := fmt.Sprintf("A stack with the name '%s' is already running", payload.Name)
-		return &httperror.HandlerError{http.StatusConflict, errorMessage, errors.New(errorMessage)}
+		return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A stack with the name '%s' already exists", payload.Name), Err: errStackAlreadyExists}
+	}
+
+	//make sure the webhook ID is unique
+	if payload.AutoUpdate != nil && payload.AutoUpdate.Webhook != "" {
+		isUnique, err := handler.checkUniqueWebhookID(payload.AutoUpdate.Webhook)
+		if err != nil {
+			return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for webhook ID collision", Err: err}
+		}
+		if !isUnique {
+			return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("Webhook ID: %s already exists", payload.AutoUpdate.Webhook), Err: errWebhookIDAlreadyExists}
+		}
 	}
 
 	stackID := handler.DataStore.Stack().GetNextIdentifier()
 	stack := &portainer.Stack{
-		ID:           portainer.StackID(stackID),
-		Name:         payload.Name,
-		Type:         portainer.DockerSwarmStack,
-		SwarmID:      payload.SwarmID,
-		EndpointID:   endpoint.ID,
-		EntryPoint:   payload.ComposeFilePathInRepository,
+		ID:              portainer.StackID(stackID),
+		Name:            payload.Name,
+		Type:            portainer.DockerSwarmStack,
+		SwarmID:         payload.SwarmID,
+		EndpointID:      endpoint.ID,
+		EntryPoint:      payload.ComposeFile,
+		AdditionalFiles: payload.AdditionalFiles,
+		AutoUpdate:      payload.AutoUpdate,
+		GitConfig: &gittypes.RepoConfig{
+			URL:            payload.RepositoryURL,
+			ReferenceName:  payload.RepositoryReferenceName,
+			ConfigFilePath: payload.ComposeFile,
+		},
 		Env:          payload.Env,
 		Status:       portainer.StackStatusActive,
 		CreationDate: time.Now().Unix(),
 	}
 
+	if payload.RepositoryAuthentication {
+		stack.GitConfig.Authentication = &gittypes.GitAuthentication{
+			Username: payload.RepositoryUsername,
+			Password: payload.RepositoryPassword,
+		}
+	}
+
 	projectPath := handler.FileService.GetStackProjectPath(strconv.Itoa(int(stack.ID)))
 	stack.ProjectPath = projectPath
 
 	doCleanUp := true
 	defer handler.cleanUp(stack, &doCleanUp)
 
-	err = handler.cloneAndSaveConfig(stack, projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, payload.ComposeFilePathInRepository, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword)
+	err = handler.clone(projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword)
 	if err != nil {
 		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to clone git repository", Err: err}
 	}
 
+	commitId, err := handler.latestCommitID(payload.RepositoryURL, payload.RepositoryReferenceName, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to fetch git repository id", Err: err}
+	}
+	stack.GitConfig.ConfigHash = commitId
+
 	config, configErr := handler.createSwarmDeployConfig(r, stack, endpoint, false)
 	if configErr != nil {
 		return configErr
@@ -192,14 +230,23 @@ func (handler *Handler) createSwarmStackFromGitRepository(w http.ResponseWriter,
 
 	err = handler.deploySwarmStack(config)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, err.Error(), err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: err.Error(), Err: err}
+	}
+
+	if payload.AutoUpdate != nil && payload.AutoUpdate.Interval != "" {
+		jobID, e := startAutoupdate(stack.ID, stack.AutoUpdate.Interval, handler.Scheduler, handler.StackDeployer, handler.DataStore, handler.GitService)
+		if e != nil {
+			return e
+		}
+
+		stack.AutoUpdate.JobID = jobID
 	}
 
 	stack.CreatedBy = config.user.Username
 
 	err = handler.DataStore.Stack().CreateStack(stack)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the stack inside the database", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist the stack inside the database", Err: err}
 	}
 
 	doCleanUp = false
@@ -350,16 +397,17 @@ func (handler *Handler) deploySwarmStack(config *swarmStackDeploymentConfig) err
 	settings := &config.endpoint.SecuritySettings
 
 	if !settings.AllowBindMountsForRegularUsers && !isAdminOrEndpointAdmin {
-		composeFilePath := path.Join(config.stack.ProjectPath, config.stack.EntryPoint)
+		for _, file := range append([]string{config.stack.EntryPoint}, config.stack.AdditionalFiles...) {
+			path := path.Join(config.stack.ProjectPath, file)
+			stackContent, err := handler.FileService.GetFileContent(path)
+			if err != nil {
+				return err
+			}
 
-		stackContent, err := handler.FileService.GetFileContent(composeFilePath)
-		if err != nil {
-			return err
-		}
-
-		err = handler.isValidStackFile(stackContent, settings)
-		if err != nil {
-			return err
+			err = handler.isValidStackFile(stackContent, settings)
+			if err != nil {
+				return err
+			}
 		}
 	}
 

api/http/handler/stacks/handler.go

@@ -2,23 +2,30 @@ package stacks
 
 import (
 	"context"
-	"errors"
+	"fmt"
 	"net/http"
 	"strings"
 	"sync"
 
 	"github.com/docker/docker/api/types"
 	"github.com/gorilla/mux"
+	"github.com/pkg/errors"
 	httperror "github.com/portainer/libhttp/error"
 	portainer "github.com/portainer/portainer/api"
+	bolterrors "github.com/portainer/portainer/api/bolt/errors"
 	"github.com/portainer/portainer/api/docker"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/internal/authorization"
+	"github.com/portainer/portainer/api/scheduler"
+	"github.com/portainer/portainer/api/stacks"
 )
 
+const defaultGitReferenceName = "refs/heads/master"
+
 var (
-	errStackAlreadyExists = errors.New("A stack already exists with this name")
-	errStackNotExternal   = errors.New("Not an external stack")
+	errStackAlreadyExists     = errors.New("A stack already exists with this name")
+	errWebhookIDAlreadyExists = errors.New("A webhook ID already exists")
+	errStackNotExternal       = errors.New("Not an external stack")
 )
 
 // Handler is the HTTP handler used to handle stack operations.
@@ -34,6 +41,8 @@ type Handler struct {
 	SwarmStackManager   portainer.SwarmStackManager
 	ComposeStackManager portainer.ComposeStackManager
 	KubernetesDeployer  portainer.KubernetesDeployer
+	Scheduler           *scheduler.Scheduler
+	StackDeployer       stacks.StackDeployer
 }
 
 // NewHandler creates a handler to manage stack operations.
@@ -57,7 +66,9 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler {
 	h.Handle("/stacks/{id}",
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackUpdate))).Methods(http.MethodPut)
 	h.Handle("/stacks/{id}/git",
-		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackUpdateGit))).Methods(http.MethodPut)
+		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackUpdateGit))).Methods(http.MethodPost)
+	h.Handle("/stacks/{id}/git/redeploy",
+		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackGitRedeploy))).Methods(http.MethodPut)
 	h.Handle("/stacks/{id}/file",
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackFile))).Methods(http.MethodGet)
 	h.Handle("/stacks/{id}/migrate",
@@ -66,6 +77,9 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler {
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackStart))).Methods(http.MethodPost)
 	h.Handle("/stacks/{id}/stop",
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackStop))).Methods(http.MethodPost)
+	h.Handle("/stacks/webhooks/{webhookID}",
+		httperror.LoggerHandler(h.webhookInvoke)).Methods(http.MethodPost)
+
 	return h
 }
 
@@ -159,3 +173,34 @@ func (handler *Handler) checkUniqueName(endpoint *portainer.Endpoint, name strin
 
 	return true, nil
 }
+
+func (handler *Handler) checkUniqueWebhookID(webhookID string) (bool, error) {
+	_, err := handler.DataStore.Stack().StackByWebhookID(webhookID)
+	if err == bolterrors.ErrObjectNotFound {
+		return true, nil
+	}
+	return false, err
+}
+
+func (handler *Handler) clone(projectPath, repositoryURL, refName string, auth bool, username, password string) error {
+	if !auth {
+		username = ""
+		password = ""
+	}
+
+	err := handler.GitService.CloneRepository(projectPath, repositoryURL, refName, username, password)
+	if err != nil {
+		return fmt.Errorf("unable to clone git repository: %w", err)
+	}
+
+	return nil
+}
+
+func (handler *Handler) latestCommitID(repositoryURL, refName string, auth bool, username, password string) (string, error) {
+	if !auth {
+		username = ""
+		password = ""
+	}
+
+	return handler.GitService.LatestCommitID(repositoryURL, refName, username, password)
+}

api/http/handler/stacks/helper.go

@@ -0,0 +1,24 @@
+package stacks
+
+import (
+	"time"
+
+	"github.com/asaskevich/govalidator"
+	"github.com/pkg/errors"
+	portainer "github.com/portainer/portainer/api"
+)
+
+func validateStackAutoUpdate(autoUpdate *portainer.StackAutoUpdate) error {
+	if autoUpdate == nil {
+		return nil
+	}
+	if autoUpdate.Webhook != "" && !govalidator.IsUUID(autoUpdate.Webhook) {
+		return errors.New("invalid Webhook format")
+	}
+	if autoUpdate.Interval != "" {
+		if _, err := time.ParseDuration(autoUpdate.Interval); err != nil {
+			return errors.New("invalid Interval format")
+		}
+	}
+	return nil
+}

api/http/handler/stacks/helper_test.go

@@ -0,0 +1,42 @@
+package stacks
+
+import (
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_ValidateStackAutoUpdate(t *testing.T) {
+	tests := []struct {
+		name    string
+		value   *portainer.StackAutoUpdate
+		wantErr bool
+	}{
+		{
+			name:    "webhook is not a valid UUID",
+			value:   &portainer.StackAutoUpdate{Webhook: "fake-webhook"},
+			wantErr: true,
+		},
+		{
+			name:    "incorrect interval value",
+			value:   &portainer.StackAutoUpdate{Interval: "1dd2hh3mm"},
+			wantErr: true,
+		},
+		{
+			name: "valid auto update",
+			value: &portainer.StackAutoUpdate{
+				Webhook:  "8dce8c2f-9ca1-482b-ad20-271e86536ada",
+				Interval: "5h30m40s10ms",
+			},
+			wantErr: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := validateStackAutoUpdate(tt.value)
+			assert.Equalf(t, tt.wantErr, err != nil, "received %+v", err)
+		})
+	}
+}

api/http/handler/stacks/stack_associate.go

@@ -2,6 +2,9 @@ package stacks
 
 import (
 	"fmt"
+	"net/http"
+	"time"
+
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
@@ -9,8 +12,6 @@ import (
 	bolterrors "github.com/portainer/portainer/api/bolt/errors"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/internal/stackutils"
-	"net/http"
-	"time"
 )
 
 // PUT request on /api/stacks/:id/associate?endpointId=<endpointId>&swarmId=<swarmId>&orphanedRunning=<orphanedRunning>
@@ -87,5 +88,10 @@ func (handler *Handler) stackAssociate(w http.ResponseWriter, r *http.Request) *
 
 	stack.ResourceControl = resourceControl
 
+	if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" {
+		// sanitize password in the http response to minimise possible security leaks
+		stack.GitConfig.Authentication.Password = ""
+	}
+
 	return response.JSON(w, stack)
 }

api/http/handler/stacks/stack_create.go

@@ -1,19 +1,17 @@
 package stacks
 
 import (
-	"errors"
-	"fmt"
 	"log"
 	"net/http"
 
 	"github.com/docker/cli/cli/compose/loader"
 	"github.com/docker/cli/cli/compose/types"
+	"github.com/pkg/errors"
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
 	bolterrors "github.com/portainer/portainer/api/bolt/errors"
-	gittypes "github.com/portainer/portainer/api/git/types"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/internal/authorization"
 	"github.com/portainer/portainer/api/internal/endpointutils"
@@ -129,7 +127,7 @@ func (handler *Handler) createComposeStack(w http.ResponseWriter, r *http.Reques
 		return handler.createComposeStackFromFileUpload(w, r, endpoint, userID)
 	}
 
-	return &httperror.HandlerError{http.StatusBadRequest, "Invalid value for query parameter: method. Value must be one of: string, repository or file", errors.New(request.ErrInvalidQueryParameter)}
+	return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid value for query parameter: method. Value must be one of: string, repository or file", Err: errors.New(request.ErrInvalidQueryParameter)}
 }
 
 func (handler *Handler) createSwarmStack(w http.ResponseWriter, r *http.Request, method string, endpoint *portainer.Endpoint, userID portainer.UserID) *httperror.HandlerError {
@@ -142,7 +140,7 @@ func (handler *Handler) createSwarmStack(w http.ResponseWriter, r *http.Request,
 		return handler.createSwarmStackFromFileUpload(w, r, endpoint, userID)
 	}
 
-	return &httperror.HandlerError{http.StatusBadRequest, "Invalid value for query parameter: method. Value must be one of: string, repository or file", errors.New(request.ErrInvalidQueryParameter)}
+	return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid value for query parameter: method. Value must be one of: string, repository or file", Err: errors.New(request.ErrInvalidQueryParameter)}
 }
 
 func (handler *Handler) createKubernetesStack(w http.ResponseWriter, r *http.Request, method string, endpoint *portainer.Endpoint) *httperror.HandlerError {
@@ -232,24 +230,11 @@ func (handler *Handler) decorateStackResponse(w http.ResponseWriter, stack *port
 	}
 
 	stack.ResourceControl = resourceControl
+
+	if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" {
+		// sanitize password in the http response to minimise possible security leaks
+		stack.GitConfig.Authentication.Password = ""
+	}
+
 	return response.JSON(w, stack)
 }
-
-func (handler *Handler) cloneAndSaveConfig(stack *portainer.Stack, projectPath, repositoryURL, refName, configFilePath string, auth bool, username, password string) error {
-	if !auth {
-		username = ""
-		password = ""
-	}
-
-	err := handler.GitService.CloneRepository(projectPath, repositoryURL, refName, username, password)
-	if err != nil {
-		return fmt.Errorf("unable to clone git repository: %w", err)
-	}
-
-	stack.GitConfig = &gittypes.RepoConfig{
-		URL:            repositoryURL,
-		ReferenceName:  refName,
-		ConfigFilePath: configFilePath,
-	}
-	return nil
-}

api/http/handler/stacks/stack_create_test.go

@@ -1,29 +0,0 @@
-package stacks
-
-import (
-	"testing"
-
-	portainer "github.com/portainer/portainer/api"
-	gittypes "github.com/portainer/portainer/api/git/types"
-	"github.com/portainer/portainer/api/http/security"
-	"github.com/portainer/portainer/api/internal/testhelpers"
-	"github.com/stretchr/testify/assert"
-)
-
-func Test_stackHandler_cloneAndSaveConfig_shouldCallGitCloneAndSaveConfigOnStack(t *testing.T) {
-	handler := NewHandler(&security.RequestBouncer{})
-	handler.GitService = testhelpers.NewGitService()
-
-	url := "url"
-	refName := "ref"
-	configPath := "path"
-	stack := &portainer.Stack{}
-	err := handler.cloneAndSaveConfig(stack, "", url, refName, configPath, false, "", "")
-	assert.NoError(t, err, "clone and save should not fail")
-
-	assert.Equal(t, gittypes.RepoConfig{
-		URL:            url,
-		ReferenceName:  refName,
-		ConfigFilePath: configPath,
-	}, *stack.GitConfig)
-}

api/http/handler/stacks/stack_delete.go

@@ -96,6 +96,11 @@ func (handler *Handler) stackDelete(w http.ResponseWriter, r *http.Request) *htt
 		}
 	}
 
+	// stop scheduler updates of the stack before removal
+	if stack.AutoUpdate != nil {
+		stopAutoupdate(stack.ID, stack.AutoUpdate.JobID, *handler.Scheduler)
+	}
+
 	err = handler.deleteStack(stack, endpoint)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, err.Error(), err}

api/http/handler/stacks/stack_inspect.go

@@ -78,5 +78,10 @@ func (handler *Handler) stackInspect(w http.ResponseWriter, r *http.Request) *ht
 		}
 	}
 
+	if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" {
+		// sanitize password in the http response to minimise possible security leaks
+		stack.GitConfig.Authentication.Password = ""
+	}
+
 	return response.JSON(w, stack)
 }

api/http/handler/stacks/stack_list.go

@@ -1,9 +1,10 @@
 package stacks
 
 import (
-	httperrors "github.com/portainer/portainer/api/http/errors"
 	"net/http"
 
+	httperrors "github.com/portainer/portainer/api/http/errors"
+
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
@@ -80,6 +81,13 @@ func (handler *Handler) stackList(w http.ResponseWriter, r *http.Request) *httpe
 		stacks = authorization.FilterAuthorizedStacks(stacks, user, userTeamIDs)
 	}
 
+	for _, stack := range stacks {
+		if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" {
+			// sanitize password in the http response to minimise possible security leaks
+			stack.GitConfig.Authentication.Password = ""
+		}
+	}
+
 	return response.JSON(w, stacks)
 }
 

api/http/handler/stacks/stack_migrate.go

@@ -150,6 +150,11 @@ func (handler *Handler) stackMigrate(w http.ResponseWriter, r *http.Request) *ht
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the stack changes inside the database", err}
 	}
 
+	if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" {
+		// sanitize password in the http response to minimise possible security leaks
+		stack.GitConfig.Authentication.Password = ""
+	}
+
 	return response.JSON(w, stack)
 }
 

api/http/handler/stacks/stack_start.go

@@ -85,6 +85,17 @@ func (handler *Handler) stackStart(w http.ResponseWriter, r *http.Request) *http
 		return &httperror.HandlerError{http.StatusBadRequest, "Stack is already active", errors.New("Stack is already active")}
 	}
 
+	if stack.AutoUpdate != nil && stack.AutoUpdate.Interval != "" {
+		stopAutoupdate(stack.ID, stack.AutoUpdate.JobID, *handler.Scheduler)
+
+		jobID, e := startAutoupdate(stack.ID, stack.AutoUpdate.Interval, handler.Scheduler, handler.StackDeployer, handler.DataStore, handler.GitService)
+		if e != nil {
+			return e
+		}
+
+		stack.AutoUpdate.JobID = jobID
+	}
+
 	err = handler.startStack(stack, endpoint)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to start stack", err}
@@ -96,6 +107,11 @@ func (handler *Handler) stackStart(w http.ResponseWriter, r *http.Request) *http
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to update stack status", err}
 	}
 
+	if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" {
+		// sanitize password in the http response to minimise possible security leaks
+		stack.GitConfig.Authentication.Password = ""
+	}
+
 	return response.JSON(w, stack)
 }
 

api/http/handler/stacks/stack_stop.go

@@ -74,6 +74,12 @@ func (handler *Handler) stackStop(w http.ResponseWriter, r *http.Request) *httpe
 		return &httperror.HandlerError{http.StatusBadRequest, "Stack is already inactive", errors.New("Stack is already inactive")}
 	}
 
+	// stop scheduler updates of the stack before stopping
+	if stack.AutoUpdate != nil && stack.AutoUpdate.JobID != "" {
+		stopAutoupdate(stack.ID, stack.AutoUpdate.JobID, *handler.Scheduler)
+		stack.AutoUpdate.JobID = ""
+	}
+
 	err = handler.stopStack(stack, endpoint)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to stop stack", err}
@@ -85,6 +91,11 @@ func (handler *Handler) stackStop(w http.ResponseWriter, r *http.Request) *httpe
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to update stack status", err}
 	}
 
+	if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" {
+		// sanitize password in the http response to minimise possible security leaks
+		stack.GitConfig.Authentication.Password = ""
+	}
+
 	return response.JSON(w, stack)
 }
 

api/http/handler/stacks/stack_update.go

@@ -128,6 +128,11 @@ func (handler *Handler) stackUpdate(w http.ResponseWriter, r *http.Request) *htt
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the stack changes inside the database", err}
 	}
 
+	if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" {
+		// sanitize password in the http response to minimise possible security leaks
+		stack.GitConfig.Authentication.Password = ""
+	}
+
 	return response.JSON(w, stack)
 }
 

api/http/handler/stacks/stack_update_git.go

@@ -2,10 +2,7 @@ package stacks
 
 import (
 	"errors"
-	"fmt"
-	"log"
 	"net/http"
-	"time"
 
 	"github.com/asaskevich/govalidator"
 	httperror "github.com/portainer/libhttp/error"
@@ -13,22 +10,28 @@ import (
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
 	bolterrors "github.com/portainer/portainer/api/bolt/errors"
-	"github.com/portainer/portainer/api/filesystem"
+	gittypes "github.com/portainer/portainer/api/git/types"
 	httperrors "github.com/portainer/portainer/api/http/errors"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/internal/stackutils"
 )
 
-type updateStackGitPayload struct {
+type stackGitUpdatePayload struct {
+	AutoUpdate               *portainer.StackAutoUpdate
+	Env                      []portainer.Pair
 	RepositoryReferenceName  string
 	RepositoryAuthentication bool
 	RepositoryUsername       string
 	RepositoryPassword       string
 }
 
-func (payload *updateStackGitPayload) Validate(r *http.Request) error {
-	if payload.RepositoryAuthentication && (govalidator.IsNull(payload.RepositoryUsername) || govalidator.IsNull(payload.RepositoryPassword)) {
-		return errors.New("Invalid repository credentials. Username and password must be specified when authentication is enabled")
+func (payload *stackGitUpdatePayload) Validate(r *http.Request) error {
+	if govalidator.IsNull(payload.RepositoryReferenceName) {
+		payload.RepositoryReferenceName = defaultGitReferenceName
+	}
+
+	if err := validateStackAutoUpdate(payload.AutoUpdate); err != nil {
+		return err
 	}
 	return nil
 }
@@ -53,18 +56,23 @@ func (payload *updateStackGitPayload) Validate(r *http.Request) error {
 func (handler *Handler) stackUpdateGit(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	stackID, err := request.RetrieveNumericRouteVariableValue(r, "id")
 	if err != nil {
-		return &httperror.HandlerError{http.StatusBadRequest, "Invalid stack identifier route variable", err}
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid stack identifier route variable", Err: err}
+	}
+
+	var payload stackGitUpdatePayload
+	err = request.DecodeAndValidateJSONPayload(r, &payload)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid request payload", Err: err}
 	}
 
 	stack, err := handler.DataStore.Stack().Stack(portainer.StackID(stackID))
 	if err == bolterrors.ErrObjectNotFound {
-		return &httperror.HandlerError{http.StatusNotFound, "Unable to find a stack with the specified identifier inside the database", err}
+		return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find a stack with the specified identifier inside the database", Err: err}
 	} else if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a stack with the specified identifier inside the database", err}
-	}
-
-	if stack.GitConfig == nil {
-		return &httperror.HandlerError{http.StatusBadRequest, "Stack is not created from git", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find a stack with the specified identifier inside the database", Err: err}
+	} else if stack.GitConfig == nil {
+		msg := "No Git config in the found stack"
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: msg, Err: errors.New(msg)}
 	}
 
 	// TODO: this is a work-around for stacks created with Portainer version >= 1.17.1
@@ -72,7 +80,7 @@ func (handler *Handler) stackUpdateGit(w http.ResponseWriter, r *http.Request) *
 	// can use the optional EndpointID query parameter to associate a valid endpoint identifier to the stack.
 	endpointID, err := request.RetrieveNumericQueryParameter(r, "endpointId", true)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusBadRequest, "Invalid query parameter: endpointId", err}
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid query parameter: endpointId", Err: err}
 	}
 	if endpointID != int(stack.EndpointID) {
 		stack.EndpointID = portainer.EndpointID(endpointID)
@@ -80,117 +88,75 @@ func (handler *Handler) stackUpdateGit(w http.ResponseWriter, r *http.Request) *
 
 	endpoint, err := handler.DataStore.Endpoint().Endpoint(stack.EndpointID)
 	if err == bolterrors.ErrObjectNotFound {
-		return &httperror.HandlerError{http.StatusNotFound, "Unable to find the endpoint associated to the stack inside the database", err}
+		return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find the endpoint associated to the stack inside the database", Err: err}
 	} else if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find the endpoint associated to the stack inside the database", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find the endpoint associated to the stack inside the database", Err: err}
 	}
 
 	err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access endpoint", err}
+		return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Permission denied to access endpoint", Err: err}
 	}
 
 	resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a resource control associated to the stack", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve a resource control associated to the stack", Err: err}
 	}
 
 	securityContext, err := security.RetrieveRestrictedRequestContext(r)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve info from request context", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve info from request context", Err: err}
 	}
 
 	access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to verify user authorizations to validate stack access", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to verify user authorizations to validate stack access", Err: err}
 	}
 	if !access {
-		return &httperror.HandlerError{http.StatusForbidden, "Access denied to resource", httperrors.ErrResourceAccessDenied}
+		return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Access denied to resource", Err: httperrors.ErrResourceAccessDenied}
 	}
 
-	var payload updateStackGitPayload
-	err = request.DecodeAndValidateJSONPayload(r, &payload)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
+	//stop the autoupdate job if there is any
+	if stack.AutoUpdate != nil {
+		stopAutoupdate(stack.ID, stack.AutoUpdate.JobID, *handler.Scheduler)
 	}
 
+	//update retrieved stack data based on the payload
 	stack.GitConfig.ReferenceName = payload.RepositoryReferenceName
+	stack.AutoUpdate = payload.AutoUpdate
+	stack.Env = payload.Env
 
-	backupProjectPath := fmt.Sprintf("%s-old", stack.ProjectPath)
-	err = filesystem.MoveDirectory(stack.ProjectPath, backupProjectPath)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to move git repository directory", err}
+	stack.GitConfig.Authentication = nil
+	if payload.RepositoryAuthentication {
+		password := payload.RepositoryPassword
+		if password == "" && stack.GitConfig != nil && stack.GitConfig.Authentication != nil {
+			password = stack.GitConfig.Authentication.Password
+		}
+		stack.GitConfig.Authentication = &gittypes.GitAuthentication{
+			Username: payload.RepositoryUsername,
+			Password: password,
+		}
 	}
 
-	repositoryUsername := payload.RepositoryUsername
-	repositoryPassword := payload.RepositoryPassword
-	if !payload.RepositoryAuthentication {
-		repositoryUsername = ""
-		repositoryPassword = ""
-	}
-
-	err = handler.GitService.CloneRepository(stack.ProjectPath, stack.GitConfig.URL, payload.RepositoryReferenceName, repositoryUsername, repositoryPassword)
-	if err != nil {
-		restoreError := filesystem.MoveDirectory(backupProjectPath, stack.ProjectPath)
-		if restoreError != nil {
-			log.Printf("[WARN] [http,stacks,git] [error: %s] [message: failed restoring backup folder]", restoreError)
+	if payload.AutoUpdate != nil && payload.AutoUpdate.Interval != "" {
+		jobID, e := startAutoupdate(stack.ID, stack.AutoUpdate.Interval, handler.Scheduler, handler.StackDeployer, handler.DataStore, handler.GitService)
+		if e != nil {
+			return e
 		}
 
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to clone git repository", err}
-	}
-
-	defer func() {
-		err = handler.FileService.RemoveDirectory(backupProjectPath)
-		if err != nil {
-			log.Printf("[WARN] [http,stacks,git] [error: %s] [message: unable to remove git repository directory]", err)
-		}
-	}()
-
-	httpErr := handler.deployStack(r, stack, endpoint)
-	if httpErr != nil {
-		return httpErr
+		stack.AutoUpdate.JobID = jobID
 	}
 
+	//save the updated stack to DB
 	err = handler.DataStore.Stack().UpdateStack(stack.ID, stack)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the stack changes inside the database", err}
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist the stack changes inside the database", Err: err}
+	}
+
+	if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" {
+		// sanitize password in the http response to minimise possible security leaks
+		stack.GitConfig.Authentication.Password = ""
 	}
 
 	return response.JSON(w, stack)
 }
-
-func (handler *Handler) deployStack(r *http.Request, stack *portainer.Stack, endpoint *portainer.Endpoint) *httperror.HandlerError {
-	if stack.Type == portainer.DockerSwarmStack {
-		config, httpErr := handler.createSwarmDeployConfig(r, stack, endpoint, false)
-		if httpErr != nil {
-			return httpErr
-		}
-
-		err := handler.deploySwarmStack(config)
-		if err != nil {
-			return &httperror.HandlerError{http.StatusInternalServerError, err.Error(), err}
-		}
-
-		stack.UpdateDate = time.Now().Unix()
-		stack.UpdatedBy = config.user.Username
-		stack.Status = portainer.StackStatusActive
-
-		return nil
-	}
-
-	config, httpErr := handler.createComposeDeployConfig(r, stack, endpoint)
-	if httpErr != nil {
-		return httpErr
-	}
-
-	err := handler.deployComposeStack(config)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, err.Error(), err}
-	}
-
-	stack.UpdateDate = time.Now().Unix()
-	stack.UpdatedBy = config.user.Username
-	stack.Status = portainer.StackStatusActive
-
-	return nil
-}

api/http/handler/stacks/stack_update_git_redeploy.go

@@ -0,0 +1,190 @@
+package stacks
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	"time"
+
+	"github.com/asaskevich/govalidator"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	portainer "github.com/portainer/portainer/api"
+	bolterrors "github.com/portainer/portainer/api/bolt/errors"
+	"github.com/portainer/portainer/api/filesystem"
+	httperrors "github.com/portainer/portainer/api/http/errors"
+	"github.com/portainer/portainer/api/http/security"
+	"github.com/portainer/portainer/api/internal/stackutils"
+)
+
+type stackGitRedployPayload struct {
+	RepositoryReferenceName  string
+	RepositoryAuthentication bool
+	RepositoryUsername       string
+	RepositoryPassword       string
+	Env                      []portainer.Pair
+}
+
+func (payload *stackGitRedployPayload) Validate(r *http.Request) error {
+	if govalidator.IsNull(payload.RepositoryReferenceName) {
+		payload.RepositoryReferenceName = defaultGitReferenceName
+	}
+
+	return nil
+}
+
+// PUT request on /api/stacks/:id/git?endpointId=<endpointId>
+func (handler *Handler) stackGitRedeploy(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	stackID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid stack identifier route variable", Err: err}
+	}
+
+	stack, err := handler.DataStore.Stack().Stack(portainer.StackID(stackID))
+	if err == bolterrors.ErrObjectNotFound {
+		return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find a stack with the specified identifier inside the database", Err: err}
+	} else if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find a stack with the specified identifier inside the database", Err: err}
+	}
+
+	if stack.GitConfig == nil {
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Stack is not created from git", Err: err}
+	}
+
+	// TODO: this is a work-around for stacks created with Portainer version >= 1.17.1
+	// The EndpointID property is not available for these stacks, this API endpoint
+	// can use the optional EndpointID query parameter to associate a valid endpoint identifier to the stack.
+	endpointID, err := request.RetrieveNumericQueryParameter(r, "endpointId", true)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid query parameter: endpointId", Err: err}
+	}
+	if endpointID != int(stack.EndpointID) {
+		stack.EndpointID = portainer.EndpointID(endpointID)
+	}
+
+	endpoint, err := handler.DataStore.Endpoint().Endpoint(stack.EndpointID)
+	if err == bolterrors.ErrObjectNotFound {
+		return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find the endpoint associated to the stack inside the database", Err: err}
+	} else if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find the endpoint associated to the stack inside the database", Err: err}
+	}
+
+	err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Permission denied to access endpoint", Err: err}
+	}
+
+	resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve a resource control associated to the stack", Err: err}
+	}
+
+	securityContext, err := security.RetrieveRestrictedRequestContext(r)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve info from request context", Err: err}
+	}
+
+	access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to verify user authorizations to validate stack access", Err: err}
+	}
+	if !access {
+		return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Access denied to resource", Err: httperrors.ErrResourceAccessDenied}
+	}
+
+	var payload stackGitRedployPayload
+	err = request.DecodeAndValidateJSONPayload(r, &payload)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid request payload", Err: err}
+	}
+
+	stack.GitConfig.ReferenceName = payload.RepositoryReferenceName
+	stack.Env = payload.Env
+
+	backupProjectPath := fmt.Sprintf("%s-old", stack.ProjectPath)
+	err = filesystem.MoveDirectory(stack.ProjectPath, backupProjectPath)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to move git repository directory", Err: err}
+	}
+
+	repositoryUsername := ""
+	repositoryPassword := ""
+	if payload.RepositoryAuthentication {
+		repositoryPassword = payload.RepositoryPassword
+		if repositoryPassword == "" && stack.GitConfig != nil && stack.GitConfig.Authentication != nil {
+			repositoryPassword = stack.GitConfig.Authentication.Password
+		}
+		repositoryUsername = payload.RepositoryUsername
+	}
+
+	err = handler.GitService.CloneRepository(stack.ProjectPath, stack.GitConfig.URL, payload.RepositoryReferenceName, repositoryUsername, repositoryPassword)
+	if err != nil {
+		restoreError := filesystem.MoveDirectory(backupProjectPath, stack.ProjectPath)
+		if restoreError != nil {
+			log.Printf("[WARN] [http,stacks,git] [error: %s] [message: failed restoring backup folder]", restoreError)
+		}
+
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to clone git repository", Err: err}
+	}
+
+	defer func() {
+		err = handler.FileService.RemoveDirectory(backupProjectPath)
+		if err != nil {
+			log.Printf("[WARN] [http,stacks,git] [error: %s] [message: unable to remove git repository directory]", err)
+		}
+	}()
+
+	httpErr := handler.deployStack(r, stack, endpoint)
+	if httpErr != nil {
+		return httpErr
+	}
+
+	err = handler.DataStore.Stack().UpdateStack(stack.ID, stack)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist the stack changes inside the database", Err: err}
+	}
+
+	if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" {
+		// sanitize password in the http response to minimise possible security leaks
+		stack.GitConfig.Authentication.Password = ""
+	}
+
+	return response.JSON(w, stack)
+}
+
+func (handler *Handler) deployStack(r *http.Request, stack *portainer.Stack, endpoint *portainer.Endpoint) *httperror.HandlerError {
+	if stack.Type == portainer.DockerSwarmStack {
+		config, httpErr := handler.createSwarmDeployConfig(r, stack, endpoint, false)
+		if httpErr != nil {
+			return httpErr
+		}
+
+		err := handler.deploySwarmStack(config)
+		if err != nil {
+			return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: err.Error(), Err: err}
+		}
+
+		stack.UpdateDate = time.Now().Unix()
+		stack.UpdatedBy = config.user.Username
+		stack.Status = portainer.StackStatusActive
+
+		return nil
+	}
+
+	config, httpErr := handler.createComposeDeployConfig(r, stack, endpoint)
+	if httpErr != nil {
+		return httpErr
+	}
+
+	err := handler.deployComposeStack(config)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: err.Error(), Err: err}
+	}
+
+	stack.UpdateDate = time.Now().Unix()
+	stack.UpdatedBy = config.user.Username
+	stack.Status = portainer.StackStatusActive
+
+	return nil
+}

api/http/handler/stacks/webhook_invoke.go

@@ -0,0 +1,54 @@
+package stacks
+
+import (
+	"log"
+	"net/http"
+
+	"github.com/gofrs/uuid"
+
+	"github.com/portainer/libhttp/response"
+
+	bolterrors "github.com/portainer/portainer/api/bolt/errors"
+	"github.com/portainer/portainer/api/stacks"
+
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+)
+
+func (handler *Handler) webhookInvoke(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	webhookID, err := retrieveUUIDRouteVariableValue(r, "webhookID")
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid webhook identifier route variable", Err: err}
+	}
+
+	stack, err := handler.DataStore.Stack().StackByWebhookID(webhookID.String())
+	if err != nil {
+		statusCode := http.StatusInternalServerError
+		if err == bolterrors.ErrObjectNotFound {
+			statusCode = http.StatusNotFound
+		}
+		return &httperror.HandlerError{StatusCode: statusCode, Message: "Unable to find the stack by webhook ID", Err: err}
+	}
+
+	if err = stacks.RedeployWhenChanged(stack.ID, handler.StackDeployer, handler.DataStore, handler.GitService); err != nil {
+		log.Printf("[ERROR] %s\n", err)
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Failed to update the stack", Err: err}
+	}
+
+	return response.Empty(w)
+}
+
+func retrieveUUIDRouteVariableValue(r *http.Request, name string) (uuid.UUID, error) {
+	webhookID, err := request.RetrieveRouteVariableValue(r, name)
+	if err != nil {
+		return uuid.Nil, err
+	}
+
+	uid, err := uuid.FromString(webhookID)
+
+	if err != nil {
+		return uuid.Nil, err
+	}
+
+	return uid, nil
+}

api/http/handler/stacks/webhook_invoke_test.go

@@ -0,0 +1,59 @@
+package stacks
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/gofrs/uuid"
+	"github.com/stretchr/testify/assert"
+
+	portainer "github.com/portainer/portainer/api"
+
+	"github.com/portainer/portainer/api/bolt/bolttest"
+)
+
+func TestHandler_webhookInvoke(t *testing.T) {
+	store, teardown := bolttest.MustNewTestStore(true)
+	defer teardown()
+
+	webhookID := newGuidString(t)
+	store.StackService.CreateStack(&portainer.Stack{
+		AutoUpdate: &portainer.StackAutoUpdate{
+			Webhook: webhookID,
+		},
+	})
+
+	h := NewHandler(nil)
+	h.DataStore = store
+
+	t.Run("invalid uuid results in http.StatusBadRequest", func(t *testing.T) {
+		w := httptest.NewRecorder()
+		req := newRequest("notuuid")
+		h.Router.ServeHTTP(w, req)
+		assert.Equal(t, http.StatusBadRequest, w.Code)
+	})
+	t.Run("registered webhook ID in http.StatusNoContent", func(t *testing.T) {
+		w := httptest.NewRecorder()
+		req := newRequest(webhookID)
+		h.Router.ServeHTTP(w, req)
+		assert.Equal(t, http.StatusNoContent, w.Code)
+	})
+	t.Run("unregistered webhook ID in http.StatusNotFound", func(t *testing.T) {
+		w := httptest.NewRecorder()
+		req := newRequest(newGuidString(t))
+		h.Router.ServeHTTP(w, req)
+		assert.Equal(t, http.StatusNotFound, w.Code)
+	})
+}
+
+func newGuidString(t *testing.T) string {
+	uuid, err := uuid.NewV4()
+	assert.NoError(t, err)
+
+	return uuid.String()
+}
+
+func newRequest(webhookID string) *http.Request {
+	return httptest.NewRequest(http.MethodPost, "/stacks/webhooks/"+webhookID, nil)
+}

api/http/handler/websocket/handler.go

@@ -36,5 +36,7 @@ func NewHandler(kubernetesTokenCacheManager *kubernetes.TokenCacheManager, bounc
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.websocketAttach)))
 	h.PathPrefix("/websocket/pod").Handler(
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.websocketPodExec)))
+	h.PathPrefix("/websocket/kubernetes-shell").Handler(
+		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.websocketShellPodExec)))
 	return h
 }

api/http/handler/websocket/pod.go

@@ -2,12 +2,13 @@ package websocket
 
 import (
 	"fmt"
-	"github.com/portainer/portainer/api/http/security"
 	"io"
 	"log"
 	"net/http"
 	"strings"
 
+	"github.com/portainer/portainer/api/http/security"
+
 	"github.com/gorilla/websocket"
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
@@ -73,14 +74,14 @@ func (handler *Handler) websocketPodExec(w http.ResponseWriter, r *http.Request)
 		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access endpoint", err}
 	}
 
-	token, useAdminToken, err := handler.getToken(r, endpoint, false)
+	serviceAccountToken, isAdminToken, err := handler.getToken(r, endpoint, false)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to get user service account token", err}
 	}
 
 	params := &webSocketRequestParams{
 		endpoint: endpoint,
-		token:    token,
+		token:    serviceAccountToken,
 	}
 
 	r.Header.Del("Origin")
@@ -99,6 +100,28 @@ func (handler *Handler) websocketPodExec(w http.ResponseWriter, r *http.Request)
 		return nil
 	}
 
+	cli, err := handler.KubernetesClientFactory.GetKubeClient(endpoint)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to create Kubernetes client", err}
+	}
+
+	handlerErr := handler.hijackPodExecStartOperation(w, r, cli, serviceAccountToken, isAdminToken, endpoint, namespace, podName, containerName, command)
+	if handlerErr != nil {
+		return handlerErr
+	}
+
+	return nil
+}
+
+func (handler *Handler) hijackPodExecStartOperation(
+	w http.ResponseWriter,
+	r *http.Request,
+	cli portainer.KubeClient,
+	serviceAccountToken string,
+	isAdminToken bool,
+	endpoint *portainer.Endpoint,
+	namespace, podName, containerName, command string,
+) *httperror.HandlerError {
 	commandArray := strings.Split(command, " ")
 
 	websocketConn, err := handler.connectionUpgrader.Upgrade(w, r, nil)
@@ -116,12 +139,7 @@ func (handler *Handler) websocketPodExec(w http.ResponseWriter, r *http.Request)
 	go streamFromWebsocketToWriter(websocketConn, stdinWriter, errorChan)
 	go streamFromReaderToWebsocket(websocketConn, stdoutReader, errorChan)
 
-	cli, err := handler.KubernetesClientFactory.GetKubeClient(endpoint)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to create Kubernetes client", err}
-	}
-
-	err = cli.StartExecProcess(token, useAdminToken, namespace, podName, containerName, commandArray, stdinReader, stdoutWriter)
+	err = cli.StartExecProcess(serviceAccountToken, isAdminToken, namespace, podName, containerName, commandArray, stdinReader, stdoutWriter)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to start exec process inside container", err}
 	}

api/http/handler/websocket/proxy.go

@@ -22,7 +22,14 @@ func (handler *Handler) proxyEdgeAgentWebsocketRequest(w http.ResponseWriter, r
 	endpointURL.Scheme = "ws"
 	proxy := websocketproxy.NewProxy(endpointURL)
 
+	signature, err := handler.SignatureService.CreateSignature(portainer.PortainerAgentSignatureMessage)
+	if err != nil {
+		return err
+	}
+
 	proxy.Director = func(incoming *http.Request, out http.Header) {
+		out.Set(portainer.PortainerAgentPublicKeyHeader, handler.SignatureService.EncodedPublicKey())
+		out.Set(portainer.PortainerAgentSignatureHeader, signature)
 		out.Set(portainer.PortainerAgentTargetHeader, params.nodeName)
 		out.Set(portainer.PortainerAgentKubernetesSATokenHeader, params.token)
 	}

api/http/handler/websocket/shell_pod.go

@@ -0,0 +1,106 @@
+package websocket
+
+import (
+	"net/http"
+
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	portainer "github.com/portainer/portainer/api"
+	bolterrors "github.com/portainer/portainer/api/bolt/errors"
+	"github.com/portainer/portainer/api/http/security"
+)
+
+// websocketShellPodExec handles GET requests on /websocket/pod?token=<token>&endpointId=<endpointID>
+// The request will be upgraded to the websocket protocol.
+// Authentication and access is controlled via the mandatory token query parameter.
+// The request will proxy input from the client to the pod via long-lived websocket connection.
+// The following query parameters are mandatory:
+// * token: JWT token used for authentication against this endpoint
+// * endpointId: endpoint ID of the endpoint where the resource is located
+func (handler *Handler) websocketShellPodExec(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	endpointID, err := request.RetrieveNumericQueryParameter(r, "endpointId", false)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid query parameter: endpointId", err}
+	}
+
+	endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
+	if err == bolterrors.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusNotFound, "Unable to find the endpoint associated to the stack inside the database", err}
+	} else if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find the endpoint associated to the stack inside the database", err}
+	}
+
+	tokenData, err := security.RetrieveTokenData(r)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access endpoint", err}
+	}
+
+	cli, err := handler.KubernetesClientFactory.GetKubeClient(endpoint)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to create Kubernetes client", err}
+	}
+
+	serviceAccount, err := cli.GetServiceAccount(tokenData)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find serviceaccount associated with user", err}
+	}
+
+	shellPod, err := cli.CreateUserShellPod(r.Context(), serviceAccount.Name)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to create user shell", err}
+	}
+
+	// Modifying request params mid-flight before forewarding to K8s API server (websocket)
+	q := r.URL.Query()
+
+	q.Add("namespace", shellPod.Namespace)
+	q.Add("podName", shellPod.PodName)
+	q.Add("containerName", shellPod.ContainerName)
+	q.Add("command", shellPod.ShellExecCommand)
+
+	r.URL.RawQuery = q.Encode()
+
+	// Modify url path mid-flight before forewarding to k8s API server (websocket)
+	r.URL.Path = "/websocket/pod"
+
+	/*
+		Note: The following websocket proxying logic is duplicated from `api/http/handler/websocket/pod.go`
+	*/
+	params := &webSocketRequestParams{
+		endpoint: endpoint,
+	}
+
+	r.Header.Del("Origin")
+
+	if endpoint.Type == portainer.AgentOnKubernetesEnvironment {
+		err := handler.proxyAgentWebsocketRequest(w, r, params)
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to proxy websocket request to agent", err}
+		}
+		return nil
+	} else if endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment {
+		err := handler.proxyEdgeAgentWebsocketRequest(w, r, params)
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to proxy websocket request to Edge agent", err}
+		}
+		return nil
+	}
+
+	handlerErr := handler.hijackPodExecStartOperation(
+		w,
+		r,
+		cli,
+		"",
+		true,
+		endpoint,
+		shellPod.Namespace,
+		shellPod.PodName,
+		shellPod.ContainerName,
+		shellPod.ShellExecCommand,
+	)
+	if handlerErr != nil {
+		return handlerErr
+	}
+
+	return nil
+}

api/http/middlewares/endpoint.go

@@ -0,0 +1,58 @@
+package middlewares
+
+import (
+	"context"
+	"errors"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	httperror "github.com/portainer/libhttp/error"
+	requesthelpers "github.com/portainer/libhttp/request"
+	portainer "github.com/portainer/portainer/api"
+	bolterrors "github.com/portainer/portainer/api/bolt/errors"
+)
+
+const (
+	contextEndpoint = "endpoint"
+)
+
+func WithEndpoint(endpointService portainer.EndpointService, endpointIDParam string) mux.MiddlewareFunc {
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(rw http.ResponseWriter, request *http.Request) {
+			if endpointIDParam == "" {
+				endpointIDParam = "id"
+			}
+
+			endpointID, err := requesthelpers.RetrieveNumericRouteVariableValue(request, endpointIDParam)
+			if err != nil {
+				httperror.WriteError(rw, http.StatusBadRequest, "Invalid endpoint identifier route variable", err)
+				return
+			}
+
+			endpoint, err := endpointService.Endpoint(portainer.EndpointID(endpointID))
+			if err != nil {
+				statusCode := http.StatusInternalServerError
+
+				if err == bolterrors.ErrObjectNotFound {
+					statusCode = http.StatusNotFound
+				}
+				httperror.WriteError(rw, statusCode, "Unable to find an endpoint with the specified identifier inside the database", err)
+				return
+			}
+
+			ctx := context.WithValue(request.Context(), contextEndpoint, endpoint)
+
+			next.ServeHTTP(rw, request.WithContext(ctx))
+
+		})
+	}
+}
+
+func FetchEndpoint(request *http.Request) (*portainer.Endpoint, error) {
+	contextData := request.Context().Value(contextEndpoint)
+	if contextData == nil {
+		return nil, errors.New("Unable to find endpoint data in request context")
+	}
+
+	return contextData.(*portainer.Endpoint), nil
+}

api/http/proxy/factory/docker/transport.go

@@ -92,7 +92,7 @@ func (transport *Transport) ProxyDockerRequest(request *http.Request) (*http.Res
 	requestPath := apiVersionRe.ReplaceAllString(request.URL.Path, "")
 	request.URL.Path = requestPath
 
-	if transport.endpoint.Type == portainer.AgentOnDockerEnvironment {
+	if transport.endpoint.Type == portainer.AgentOnDockerEnvironment || transport.endpoint.Type == portainer.EdgeAgentOnDockerEnvironment {
 		signature, err := transport.signatureService.CreateSignature(portainer.PortainerAgentSignatureMessage)
 		if err != nil {
 			return nil, err

api/http/proxy/factory/kubernetes.go

@@ -72,7 +72,7 @@ func (factory *ProxyFactory) newKubernetesEdgeHTTPProxy(endpoint *portainer.Endp
 
 	endpointURL.Scheme = "http"
 	proxy := newSingleHostReverseProxyWithHostHeader(endpointURL)
-	proxy.Transport = kubernetes.NewEdgeTransport(factory.reverseTunnelService, endpoint, tokenManager, factory.kubernetesClientFactory, factory.dataStore)
+	proxy.Transport = kubernetes.NewEdgeTransport(factory.dataStore, factory.signatureService, factory.reverseTunnelService, endpoint, tokenManager, factory.kubernetesClientFactory)
 
 	return proxy, nil
 }

api/http/proxy/factory/kubernetes/edge_transport.go

@@ -10,11 +10,12 @@ import (
 
 type edgeTransport struct {
 	*baseTransport
+	signatureService     portainer.DigitalSignatureService
 	reverseTunnelService portainer.ReverseTunnelService
 }
 
 // NewAgentTransport returns a new transport that can be used to send signed requests to a Portainer Edge agent
-func NewEdgeTransport(reverseTunnelService portainer.ReverseTunnelService, endpoint *portainer.Endpoint, tokenManager *tokenManager, k8sClientFactory *cli.ClientFactory, dataStore portainer.DataStore) *edgeTransport {
+func NewEdgeTransport(dataStore portainer.DataStore, signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService, endpoint *portainer.Endpoint, tokenManager *tokenManager, k8sClientFactory *cli.ClientFactory) *edgeTransport {
 	transport := &edgeTransport{
 		baseTransport: newBaseTransport(
 			&http.Transport{},
@@ -24,6 +25,7 @@ func NewEdgeTransport(reverseTunnelService portainer.ReverseTunnelService, endpo
 			dataStore,
 		),
 		reverseTunnelService: reverseTunnelService,
+		signatureService:     signatureService,
 	}
 
 	return transport
@@ -45,6 +47,14 @@ func (transport *edgeTransport) RoundTrip(request *http.Request) (*http.Response
 		}
 	}
 
+	signature, err := transport.signatureService.CreateSignature(portainer.PortainerAgentSignatureMessage)
+	if err != nil {
+		return nil, err
+	}
+
+	request.Header.Set(portainer.PortainerAgentPublicKeyHeader, transport.signatureService.EncodedPublicKey())
+	request.Header.Set(portainer.PortainerAgentSignatureHeader, signature)
+
 	response, err := transport.baseTransport.RoundTrip(request)
 
 	if err == nil {

api/http/proxy/factory/utils/json.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"mime"
 
 	"gopkg.in/yaml.v3"
 )
@@ -69,7 +70,13 @@ func getBody(body io.ReadCloser, contentType string, isGzip bool) (interface{},
 }
 
 func marshal(contentType string, data interface{}) ([]byte, error) {
-	switch contentType {
+	// Note: contentType can look like: "application/json" or "application/json; charset=utf-8"
+	mediaType, _, err := mime.ParseMediaType(contentType)
+	if err != nil {
+		return nil, err
+	}
+
+	switch mediaType {
 	case "application/yaml":
 		return yaml.Marshal(data)
 	case "application/json", "":
@@ -80,7 +87,13 @@ func marshal(contentType string, data interface{}) ([]byte, error) {
 }
 
 func unmarshal(contentType string, body []byte, returnBody interface{}) error {
-	switch contentType {
+	// Note: contentType can look look like: "application/json" or "application/json; charset=utf-8"
+	mediaType, _, err := mime.ParseMediaType(contentType)
+	if err != nil {
+		return err
+	}
+
+	switch mediaType {
 	case "application/yaml":
 		return yaml.Unmarshal(body, returnBody)
 	case "application/json", "":

api/http/server.go

@@ -2,6 +2,7 @@ package http
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
 	"log"
 	"net/http"
@@ -25,11 +26,13 @@ import (
 	"github.com/portainer/portainer/api/http/handler/endpointproxy"
 	"github.com/portainer/portainer/api/http/handler/endpoints"
 	"github.com/portainer/portainer/api/http/handler/file"
+	kubehandler "github.com/portainer/portainer/api/http/handler/kubernetes"
 	"github.com/portainer/portainer/api/http/handler/motd"
 	"github.com/portainer/portainer/api/http/handler/registries"
 	"github.com/portainer/portainer/api/http/handler/resourcecontrols"
 	"github.com/portainer/portainer/api/http/handler/roles"
 	"github.com/portainer/portainer/api/http/handler/settings"
+	sslhandler "github.com/portainer/portainer/api/http/handler/ssl"
 	"github.com/portainer/portainer/api/http/handler/stacks"
 	"github.com/portainer/portainer/api/http/handler/status"
 	"github.com/portainer/portainer/api/http/handler/tags"
@@ -45,13 +48,18 @@ import (
 	"github.com/portainer/portainer/api/http/proxy/factory/kubernetes"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/internal/authorization"
+	"github.com/portainer/portainer/api/internal/ssl"
 	"github.com/portainer/portainer/api/kubernetes/cli"
+	"github.com/portainer/portainer/api/scheduler"
+	stackdeployer "github.com/portainer/portainer/api/stacks"
 )
 
 // Server implements the portainer.Server interface
 type Server struct {
-	AuthorizationService 		*authorization.Service
+	AuthorizationService        *authorization.Service
 	BindAddress                 string
+	BindAddressHTTPS            string
+	HTTPEnabled                 bool
 	AssetsPath                  string
 	Status                      *portainer.Status
 	ReverseTunnelService        portainer.ReverseTunnelService
@@ -69,14 +77,14 @@ type Server struct {
 	ProxyManager                *proxy.Manager
 	KubernetesTokenCacheManager *kubernetes.TokenCacheManager
 	Handler                     *handler.Handler
-	SSL                         bool
-	SSLCert                     string
-	SSLKey                      string
+	SSLService                  *ssl.Service
 	DockerClientFactory         *docker.ClientFactory
 	KubernetesClientFactory     *cli.ClientFactory
 	KubernetesDeployer          portainer.KubernetesDeployer
+	Scheduler                   *scheduler.Scheduler
 	ShutdownCtx                 context.Context
 	ShutdownTrigger             context.CancelFunc
+	StackDeployer               stackdeployer.StackDeployer
 }
 
 // Start starts the HTTP server
@@ -135,6 +143,8 @@ func (server *Server) Start() error {
 	endpointHandler.ReverseTunnelService = server.ReverseTunnelService
 	endpointHandler.ComposeStackManager = server.ComposeStackManager
 	endpointHandler.AuthorizationService = server.AuthorizationService
+	endpointHandler.BindAddress = server.BindAddress
+	endpointHandler.BindAddressHTTPS = server.BindAddressHTTPS
 
 	var endpointEdgeHandler = endpointedge.NewHandler(requestBouncer)
 	endpointEdgeHandler.DataStore = server.DataStore
@@ -150,6 +160,8 @@ func (server *Server) Start() error {
 	endpointProxyHandler.ProxyManager = server.ProxyManager
 	endpointProxyHandler.ReverseTunnelService = server.ReverseTunnelService
 
+	var kubernetesHandler = kubehandler.NewHandler(requestBouncer, server.AuthorizationService, server.DataStore, server.KubernetesClientFactory)
+
 	var fileHandler = file.NewHandler(filepath.Join(server.AssetsPath, "public"))
 
 	var motdHandler = motd.NewHandler(requestBouncer)
@@ -170,14 +182,19 @@ func (server *Server) Start() error {
 	settingsHandler.LDAPService = server.LDAPService
 	settingsHandler.SnapshotService = server.SnapshotService
 
+	var sslHandler = sslhandler.NewHandler(requestBouncer)
+	sslHandler.SSLService = server.SSLService
+
 	var stackHandler = stacks.NewHandler(requestBouncer)
 	stackHandler.DataStore = server.DataStore
 	stackHandler.DockerClientFactory = server.DockerClientFactory
 	stackHandler.FileService = server.FileService
-	stackHandler.SwarmStackManager = server.SwarmStackManager
-	stackHandler.ComposeStackManager = server.ComposeStackManager
 	stackHandler.KubernetesDeployer = server.KubernetesDeployer
 	stackHandler.GitService = server.GitService
+	stackHandler.Scheduler = server.Scheduler
+	stackHandler.SwarmStackManager = server.SwarmStackManager
+	stackHandler.ComposeStackManager = server.ComposeStackManager
+	stackHandler.StackDeployer = server.StackDeployer
 
 	var tagHandler = tags.NewHandler(requestBouncer)
 	tagHandler.DataStore = server.DataStore
@@ -225,11 +242,13 @@ func (server *Server) Start() error {
 		EndpointHandler:        endpointHandler,
 		EndpointEdgeHandler:    endpointEdgeHandler,
 		EndpointProxyHandler:   endpointProxyHandler,
+		KubernetesHandler:      kubernetesHandler,
 		FileHandler:            fileHandler,
 		MOTDHandler:            motdHandler,
 		RegistryHandler:        registryHandler,
 		ResourceControlHandler: resourceControlHandler,
 		SettingsHandler:        settingsHandler,
+		SSLHandler:             sslHandler,
 		StatusHandler:          statusHandler,
 		StackHandler:           stackHandler,
 		TagHandler:             tagHandler,
@@ -242,31 +261,48 @@ func (server *Server) Start() error {
 		WebhookHandler:         webhookHandler,
 	}
 
-	httpServer := &http.Server{
-		Addr:    server.BindAddress,
-		Handler: server.Handler,
-	}
-	httpServer.Handler = offlineGate.WaitingMiddleware(time.Minute, httpServer.Handler)
+	handler := offlineGate.WaitingMiddleware(time.Minute, server.Handler)
 
-	if server.SSL {
-		httpServer.TLSConfig = crypto.CreateServerTLSConfiguration()
-		return httpServer.ListenAndServeTLS(server.SSLCert, server.SSLKey)
+	if server.HTTPEnabled {
+		go func() {
+			log.Printf("[INFO] [http,server] [message: starting HTTP server on port %s]", server.BindAddress)
+			httpServer := &http.Server{
+				Addr:    server.BindAddress,
+				Handler: handler,
+			}
+
+			go shutdown(server.ShutdownCtx, httpServer)
+			err := httpServer.ListenAndServe()
+			if err != nil && err != http.ErrServerClosed {
+				log.Printf("[ERROR] [message: http server failed] [error: %s]", err)
+			}
+		}()
 	}
 
-	go server.shutdown(httpServer)
+	log.Printf("[INFO] [http,server] [message: starting HTTPS server on port %s]", server.BindAddressHTTPS)
+	httpsServer := &http.Server{
+		Addr:    server.BindAddressHTTPS,
+		Handler: handler,
+	}
 
-	return httpServer.ListenAndServe()
+	httpsServer.TLSConfig = crypto.CreateServerTLSConfiguration()
+	httpsServer.TLSConfig.GetCertificate = func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
+		return server.SSLService.GetRawCertificate(), nil
+	}
+
+	go shutdown(server.ShutdownCtx, httpsServer)
+	return httpsServer.ListenAndServeTLS("", "")
 }
 
-func (server *Server) shutdown(httpServer *http.Server) {
-	<-server.ShutdownCtx.Done()
+func shutdown(shutdownCtx context.Context, httpServer *http.Server) {
+	<-shutdownCtx.Done()
 
-	log.Println("[DEBUG] Shutting down http server")
+	log.Println("[DEBUG] [http,server] [message: shutting down http server]")
 	shutdownTimeout, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
 
 	err := httpServer.Shutdown(shutdownTimeout)
 	if err != nil {
-		fmt.Printf("Failed shutdown http server: %s \n", err)
+		fmt.Printf("[ERROR] [http,server] [message: failed shutdown http server] [error: %s]", err)
 	}
 }

api/internal/endpoint/endpoint.go

@@ -1,17 +0,0 @@
-package endpoint
-
-import portainer "github.com/portainer/portainer/api"
-
-// IsKubernetesEndpoint returns true if this is a kubernetes endpoint
-func IsKubernetesEndpoint(endpoint *portainer.Endpoint) bool {
-	return endpoint.Type == portainer.KubernetesLocalEnvironment ||
-		endpoint.Type == portainer.AgentOnKubernetesEnvironment ||
-		endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment
-}
-
-// IsDockerEndpoint returns true if this is a docker endpoint
-func IsDockerEndpoint(endpoint *portainer.Endpoint) bool {
-	return endpoint.Type == portainer.DockerEnvironment ||
-		endpoint.Type == portainer.AgentOnDockerEnvironment ||
-		endpoint.Type == portainer.EdgeAgentOnDockerEnvironment
-}

api/internal/endpointutils/endpointutils.go

@@ -6,6 +6,7 @@ import (
 	portainer "github.com/portainer/portainer/api"
 )
 
+// IsLocalEndpoint returns true if this is a local endpoint
 func IsLocalEndpoint(endpoint *portainer.Endpoint) bool {
 	return strings.HasPrefix(endpoint.URL, "unix://") || strings.HasPrefix(endpoint.URL, "npipe://") || endpoint.Type == 5
 }

api/internal/ssl/ssl.go

@@ -0,0 +1,170 @@
+package ssl
+
+import (
+	"context"
+	"crypto/tls"
+	"log"
+	"os"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/portainer/libcrypto"
+	portainer "github.com/portainer/portainer/api"
+)
+
+// Service represents a service to manage SSL certificates
+type Service struct {
+	fileService     portainer.FileService
+	dataStore       portainer.DataStore
+	rawCert         *tls.Certificate
+	shutdownTrigger context.CancelFunc
+}
+
+// NewService returns a pointer to a new Service
+func NewService(fileService portainer.FileService, dataStore portainer.DataStore, shutdownTrigger context.CancelFunc) *Service {
+	return &Service{
+		fileService:     fileService,
+		dataStore:       dataStore,
+		shutdownTrigger: shutdownTrigger,
+	}
+}
+
+// Init initializes the service
+func (service *Service) Init(host, certPath, keyPath string) error {
+	settings, err := service.GetSSLSettings()
+	if err != nil {
+		return errors.Wrap(err, "failed fetching ssl settings")
+	}
+
+	// certificates already exist
+	if settings.CertPath != "" && settings.KeyPath != "" {
+		err := service.cacheCertificate(settings.CertPath, settings.KeyPath)
+		if err != nil && !os.IsNotExist(err) {
+			return err
+		}
+
+		// continue if certs don't exist
+		if err == nil {
+			return nil
+		}
+	}
+
+	pathSupplied := certPath != "" && keyPath != ""
+	if pathSupplied {
+		newCertPath, newKeyPath, err := service.fileService.CopySSLCertPair(certPath, keyPath)
+		if err != nil {
+			return errors.Wrap(err, "failed copying supplied certs")
+		}
+
+		return service.cacheInfo(newCertPath, newKeyPath, false)
+	}
+
+	// path not supplied and certificates doesn't exist - generate self signed
+	certPath, keyPath = service.fileService.GetDefaultSSLCertsPath()
+
+	err = service.generateSelfSignedCertificates(host, certPath, keyPath)
+	if err != nil {
+		return errors.Wrap(err, "failed generating self signed certs")
+	}
+
+	return service.cacheInfo(certPath, keyPath, true)
+
+}
+
+// GetRawCertificate gets the raw certificate
+func (service *Service) GetRawCertificate() *tls.Certificate {
+	return service.rawCert
+}
+
+// GetSSLSettings gets the certificate info
+func (service *Service) GetSSLSettings() (*portainer.SSLSettings, error) {
+	return service.dataStore.SSLSettings().Settings()
+}
+
+// SetCertificates sets the certificates
+func (service *Service) SetCertificates(certData, keyData []byte) error {
+	if len(certData) == 0 || len(keyData) == 0 {
+		return errors.New("missing certificate files")
+	}
+
+	_, err := tls.X509KeyPair(certData, keyData)
+	if err != nil {
+		return err
+	}
+
+	certPath, keyPath, err := service.fileService.StoreSSLCertPair(certData, keyData)
+	if err != nil {
+		return err
+	}
+
+	service.cacheInfo(certPath, keyPath, false)
+
+	service.shutdownTrigger()
+
+	return nil
+}
+
+func (service *Service) SetHTTPEnabled(httpEnabled bool) error {
+	settings, err := service.dataStore.SSLSettings().Settings()
+	if err != nil {
+		return err
+	}
+
+	if settings.HTTPEnabled == httpEnabled {
+		return nil
+	}
+
+	settings.HTTPEnabled = httpEnabled
+
+	err = service.dataStore.SSLSettings().UpdateSettings(settings)
+	if err != nil {
+		return err
+	}
+
+	service.shutdownTrigger()
+
+	return nil
+}
+
+func (service *Service) cacheCertificate(certPath, keyPath string) error {
+	rawCert, err := tls.LoadX509KeyPair(certPath, keyPath)
+	if err != nil {
+		return err
+	}
+
+	service.rawCert = &rawCert
+
+	return nil
+}
+
+func (service *Service) cacheInfo(certPath, keyPath string, selfSigned bool) error {
+	err := service.cacheCertificate(certPath, keyPath)
+	if err != nil {
+		return err
+	}
+
+	settings, err := service.dataStore.SSLSettings().Settings()
+	if err != nil {
+		return err
+	}
+
+	settings.CertPath = certPath
+	settings.KeyPath = keyPath
+	settings.SelfSigned = selfSigned
+
+	err = service.dataStore.SSLSettings().UpdateSettings(settings)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (service *Service) generateSelfSignedCertificates(ip, certPath, keyPath string) error {
+	if ip == "" {
+		return errors.New("host can't be empty")
+	}
+
+	log.Printf("[INFO] [internal,ssl] [message: no cert files found, generating self signed ssl certificates]")
+	return libcrypto.GenerateCertsForHost("localhost", ip, certPath, keyPath, time.Now().AddDate(5, 0, 0))
+}

api/internal/stackutils/stackutils.go

@@ -2,6 +2,7 @@ package stackutils
 
 import (
 	"fmt"
+	"path"
 
 	portainer "github.com/portainer/portainer/api"
 )
@@ -10,3 +11,12 @@ import (
 func ResourceControlID(endpointID portainer.EndpointID, name string) string {
 	return fmt.Sprintf("%d_%s", endpointID, name)
 }
+
+// GetStackFilePaths returns a list of file paths based on stack project path
+func GetStackFilePaths(stack *portainer.Stack) []string {
+	var filePaths []string
+	for _, file := range append([]string{stack.EntryPoint}, stack.AdditionalFiles...) {
+		filePaths = append(filePaths, path.Join(stack.ProjectPath, file))
+	}
+	return filePaths
+}

api/internal/stackutils/stackutils_test.go

@@ -0,0 +1,26 @@
+package stackutils
+
+import (
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_GetStackFilePaths(t *testing.T) {
+	stack := &portainer.Stack{
+		ProjectPath: "/tmp/stack/1",
+		EntryPoint:  "file-one.yml",
+	}
+
+	t.Run("stack doesn't have additional files", func(t *testing.T) {
+		expected := []string{"/tmp/stack/1/file-one.yml"}
+		assert.ElementsMatch(t, expected, GetStackFilePaths(stack))
+	})
+
+	t.Run("stack has additional files", func(t *testing.T) {
+		stack.AdditionalFiles = []string{"file-two.yml", "file-three.yml"}
+		expected := []string{"/tmp/stack/1/file-one.yml", "/tmp/stack/1/file-two.yml", "/tmp/stack/1/file-three.yml"}
+		assert.ElementsMatch(t, expected, GetStackFilePaths(stack))
+	})
+}

api/internal/testhelpers/datastore.go

@@ -17,6 +17,7 @@ type datastore struct {
 	registry         portainer.RegistryService
 	resourceControl  portainer.ResourceControlService
 	role             portainer.RoleService
+	sslSettings      portainer.SSLSettingsService
 	settings         portainer.SettingsService
 	stack            portainer.StackService
 	tag              portainer.TagService
@@ -47,6 +48,7 @@ func (d *datastore) Registry() portainer.RegistryService                 { retur
 func (d *datastore) ResourceControl() portainer.ResourceControlService   { return d.resourceControl }
 func (d *datastore) Role() portainer.RoleService                         { return d.role }
 func (d *datastore) Settings() portainer.SettingsService                 { return d.settings }
+func (d *datastore) SSLSettings() portainer.SSLSettingsService           { return d.sslSettings }
 func (d *datastore) Stack() portainer.StackService                       { return d.stack }
 func (d *datastore) Tag() portainer.TagService                           { return d.tag }
 func (d *datastore) TeamMembership() portainer.TeamMembershipService     { return d.teamMembership }

api/kubernetes/cli/client.go

@@ -115,26 +115,8 @@ func (rt *agentHeaderRoundTripper) RoundTrip(req *http.Request) (*http.Response,
 
 func (factory *ClientFactory) buildAgentClient(endpoint *portainer.Endpoint) (*kubernetes.Clientset, error) {
 	endpointURL := fmt.Sprintf("https://%s/kubernetes", endpoint.URL)
-	signature, err := factory.signatureService.CreateSignature(portainer.PortainerAgentSignatureMessage)
-	if err != nil {
-		return nil, err
-	}
 
-	config, err := clientcmd.BuildConfigFromFlags(endpointURL, "")
-	if err != nil {
-		return nil, err
-	}
-	config.Insecure = true
-
-	config.Wrap(func(rt http.RoundTripper) http.RoundTripper {
-		return &agentHeaderRoundTripper{
-			signatureHeader: signature,
-			publicKeyHeader: factory.signatureService.EncodedPublicKey(),
-			roundTripper:    rt,
-		}
-	})
-
-	return kubernetes.NewForConfig(config)
+	return factory.createRemoteClient(endpointURL);
 }
 
 func (factory *ClientFactory) buildEdgeClient(endpoint *portainer.Endpoint) (*kubernetes.Clientset, error) {
@@ -163,12 +145,29 @@ func (factory *ClientFactory) buildEdgeClient(endpoint *portainer.Endpoint) (*ku
 
 	endpointURL := fmt.Sprintf("http://127.0.0.1:%d/kubernetes", tunnel.Port)
 
+	return factory.createRemoteClient(endpointURL);
+}
+
+func (factory *ClientFactory) createRemoteClient(endpointURL string) (*kubernetes.Clientset, error) {
+	signature, err := factory.signatureService.CreateSignature(portainer.PortainerAgentSignatureMessage)
+	if err != nil {
+		return nil, err
+	}
+
 	config, err := clientcmd.BuildConfigFromFlags(endpointURL, "")
 	if err != nil {
 		return nil, err
 	}
 	config.Insecure = true
 
+	config.Wrap(func(rt http.RoundTripper) http.RoundTripper {
+		return &agentHeaderRoundTripper{
+			signatureHeader: signature,
+			publicKeyHeader: factory.signatureService.EncodedPublicKey(),
+			roundTripper:    rt,
+		}
+	})
+
 	return kubernetes.NewForConfig(config)
 }
 

api/kubernetes/cli/kubeconfig.go

@@ -0,0 +1,66 @@
+package cli
+
+import (
+	"context"
+	"fmt"
+
+	portainer "github.com/portainer/portainer/api"
+	clientV1 "k8s.io/client-go/tools/clientcmd/api/v1"
+)
+
+// GetKubeConfig returns kubeconfig for the current user based on:
+// - portainer server url
+// - portainer user bearer token
+// - portainer token data - which maps to k8s service account
+func (kcl *KubeClient) GetKubeConfig(ctx context.Context, apiServerURL string, bearerToken string, tokenData *portainer.TokenData) (*clientV1.Config, error) {
+	serviceAccount, err := kcl.GetServiceAccount(tokenData)
+	if err != nil {
+		errText := fmt.Sprintf("unable to find serviceaccount associated with user; username=%s", tokenData.Username)
+		return nil, fmt.Errorf("%s; err=%w", errText, err)
+	}
+
+	kubeconfig := generateKubeconfig(apiServerURL, bearerToken, serviceAccount.Name)
+
+	return kubeconfig, nil
+}
+
+// generateKubeconfig will generate and return kubeconfig resource - usable by `kubectl` cli
+// which will allow the client to connect directly to k8s server endpoint via portainer (proxy)
+func generateKubeconfig(apiServerURL, bearerToken, serviceAccountName string) *clientV1.Config {
+	const (
+		KubeConfigPortainerContext = "portainer-ctx"
+		KubeConfigPortainerCluster = "portainer-cluster"
+	)
+
+	return &clientV1.Config{
+		APIVersion:     "v1",
+		Kind:           "Config",
+		CurrentContext: KubeConfigPortainerContext,
+		Contexts: []clientV1.NamedContext{
+			{
+				Name: KubeConfigPortainerContext,
+				Context: clientV1.Context{
+					AuthInfo: serviceAccountName,
+					Cluster:  KubeConfigPortainerCluster,
+				},
+			},
+		},
+		Clusters: []clientV1.NamedCluster{
+			{
+				Name: KubeConfigPortainerCluster,
+				Cluster: clientV1.Cluster{
+					Server:                apiServerURL,
+					InsecureSkipTLSVerify: true,
+				},
+			},
+		},
+		AuthInfos: []clientV1.NamedAuthInfo{
+			{
+				Name: serviceAccountName,
+				AuthInfo: clientV1.AuthInfo{
+					Token: bearerToken,
+				},
+			},
+		},
+	}
+}

api/kubernetes/cli/kubeconfig_test.go

@@ -0,0 +1,150 @@
+package cli
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+	v1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	kfake "k8s.io/client-go/kubernetes/fake"
+)
+
+func Test_GetKubeConfig(t *testing.T) {
+
+	t.Run("returns error if SA non-existent", func(t *testing.T) {
+		k := &KubeClient{
+			cli:        kfake.NewSimpleClientset(),
+			instanceID: "test",
+		}
+
+		tokenData := &portainer.TokenData{
+			ID:       1,
+			Role:     portainer.AdministratorRole,
+			Username: portainerClusterAdminServiceAccountName,
+		}
+
+		_, err := k.GetKubeConfig(context.Background(), "localhost", "abc", tokenData)
+
+		if err == nil {
+			t.Error("GetKubeConfig should fail as service account does not exist")
+		}
+		if k8sErr := errors.Unwrap(err); !k8serrors.IsNotFound(k8sErr) {
+			t.Error("GetKubeConfig should fail with service account not found k8s error")
+		}
+	})
+
+	t.Run("successfully obtains kubeconfig for cluster admin", func(t *testing.T) {
+		k := &KubeClient{
+			cli:        kfake.NewSimpleClientset(),
+			instanceID: "test",
+		}
+
+		tokenData := &portainer.TokenData{
+			Role:     portainer.AdministratorRole,
+			Username: portainerClusterAdminServiceAccountName,
+		}
+		serviceAccount := &v1.ServiceAccount{
+			ObjectMeta: metav1.ObjectMeta{Name: tokenData.Username},
+		}
+
+		k.cli.CoreV1().ServiceAccounts(portainerNamespace).Create(serviceAccount)
+		defer k.cli.CoreV1().ServiceAccounts(portainerNamespace).Delete(serviceAccount.Name, nil)
+
+		_, err := k.GetKubeConfig(context.Background(), "localhost", "abc", tokenData)
+
+		if err != nil {
+			t.Errorf("GetKubeConfig should succeed; err=%s", err)
+		}
+	})
+
+	t.Run("successfully obtains kubeconfig for standard user", func(t *testing.T) {
+		k := &KubeClient{
+			cli:        kfake.NewSimpleClientset(),
+			instanceID: "test",
+		}
+
+		tokenData := &portainer.TokenData{
+			ID:   1,
+			Role: portainer.StandardUserRole,
+		}
+		nonAdminUserName := userServiceAccountName(int(tokenData.ID), k.instanceID)
+		serviceAccount := &v1.ServiceAccount{
+			ObjectMeta: metav1.ObjectMeta{Name: nonAdminUserName},
+		}
+
+		k.cli.CoreV1().ServiceAccounts(portainerNamespace).Create(serviceAccount)
+		defer k.cli.CoreV1().ServiceAccounts(portainerNamespace).Delete(serviceAccount.Name, nil)
+
+		_, err := k.GetKubeConfig(context.Background(), "localhost", "abc", tokenData)
+
+		if err != nil {
+			t.Errorf("GetKubeConfig should succeed; err=%s", err)
+		}
+	})
+}
+
+func Test_generateKubeconfig(t *testing.T) {
+	apiServerURL, bearerToken, serviceAccountName := "localhost", "test-token", "test-user"
+
+	t.Run("generates Config resource kind", func(t *testing.T) {
+		config := generateKubeconfig(apiServerURL, bearerToken, serviceAccountName)
+		want := "Config"
+		if config.Kind != want {
+			t.Errorf("generateKubeconfig resource kind should be %s", want)
+		}
+	})
+
+	t.Run("generates v1 version", func(t *testing.T) {
+		config := generateKubeconfig(apiServerURL, bearerToken, serviceAccountName)
+		want := "v1"
+		if config.APIVersion != want {
+			t.Errorf("generateKubeconfig api version should be %s", want)
+		}
+	})
+
+	t.Run("generates single entry context cluster and authinfo", func(t *testing.T) {
+		config := generateKubeconfig(apiServerURL, bearerToken, serviceAccountName)
+		if len(config.Contexts) != 1 {
+			t.Error("generateKubeconfig should generate single context configuration")
+		}
+		if len(config.Clusters) != 1 {
+			t.Error("generateKubeconfig should generate single cluster configuration")
+		}
+		if len(config.AuthInfos) != 1 {
+			t.Error("generateKubeconfig should generate single user configuration")
+		}
+	})
+
+	t.Run("sets default context appropriately", func(t *testing.T) {
+		config := generateKubeconfig(apiServerURL, bearerToken, serviceAccountName)
+		want := "portainer-ctx"
+		if config.CurrentContext != want {
+			t.Errorf("generateKubeconfig set cluster to be %s", want)
+		}
+	})
+
+	t.Run("generates cluster with InsecureSkipTLSVerify to be set to true", func(t *testing.T) {
+		config := generateKubeconfig(apiServerURL, bearerToken, serviceAccountName)
+		if config.Clusters[0].Cluster.InsecureSkipTLSVerify != true {
+			t.Error("generateKubeconfig default cluster InsecureSkipTLSVerify should be true")
+		}
+	})
+
+	t.Run("should contain passed in value", func(t *testing.T) {
+		config := generateKubeconfig(apiServerURL, bearerToken, serviceAccountName)
+		if config.Clusters[0].Cluster.Server != apiServerURL {
+			t.Errorf("generateKubeconfig default cluster server url should be %s", apiServerURL)
+		}
+
+		if config.AuthInfos[0].Name != serviceAccountName {
+			t.Errorf("generateKubeconfig default authinfo name should be %s", serviceAccountName)
+		}
+
+		if config.AuthInfos[0].AuthInfo.Token != bearerToken {
+			t.Errorf("generateKubeconfig default authinfo user token should be %s", bearerToken)
+		}
+	})
+}

api/kubernetes/cli/namespace.go

@@ -0,0 +1,73 @@
+package cli
+
+import (
+	"strconv"
+
+	"github.com/pkg/errors"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	systemNamespaceLabel = "io.portainer.kubernetes.namespace.system"
+)
+
+func defaultSystemNamespaces() map[string]struct{} {
+	return map[string]struct{}{
+		"kube-system":     {},
+		"kube-public":     {},
+		"kube-node-lease": {},
+		"portainer":       {},
+	}
+}
+
+func isSystemNamespace(namespace v1.Namespace) bool {
+	systemLabelValue, hasSystemLabel := namespace.Labels[systemNamespaceLabel]
+	if hasSystemLabel {
+		return systemLabelValue == "true"
+	}
+
+	systemNamespaces := defaultSystemNamespaces()
+
+	_, isSystem := systemNamespaces[namespace.Name]
+
+	return isSystem
+}
+
+// ToggleSystemState will set a namespace as a system namespace, or remove this state
+// if isSystem is true it will set `systemNamespaceLabel` to "true" and false otherwise
+// this will skip if namespace is "default" or if the required state is already set
+func (kcl *KubeClient) ToggleSystemState(namespaceName string, isSystem bool) error {
+	if namespaceName == "default" {
+		return nil
+	}
+
+	nsService := kcl.cli.CoreV1().Namespaces()
+
+	namespace, err := nsService.Get(namespaceName, metav1.GetOptions{})
+	if err != nil {
+		return errors.Wrap(err, "failed fetching namespace object")
+	}
+
+	if isSystemNamespace(*namespace) == isSystem {
+		return nil
+	}
+
+	if namespace.Labels == nil {
+		namespace.Labels = map[string]string{}
+	}
+
+	namespace.Labels[systemNamespaceLabel] = strconv.FormatBool(isSystem)
+
+	_, err = nsService.Update(namespace)
+	if err != nil {
+		return errors.Wrap(err, "failed updating namespace object")
+	}
+
+	if isSystem {
+		return kcl.NamespaceAccessPoliciesDeleteNamespace(namespaceName)
+	}
+
+	return nil
+
+}

api/kubernetes/cli/namespace_test.go

@@ -0,0 +1,185 @@
+package cli
+
+import (
+	"strconv"
+	"sync"
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/stretchr/testify/assert"
+
+	core "k8s.io/api/core/v1"
+	ktypes "k8s.io/api/core/v1"
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	kfake "k8s.io/client-go/kubernetes/fake"
+)
+
+func Test_ToggleSystemState(t *testing.T) {
+	t.Run("should skip is default (exit without error)", func(t *testing.T) {
+		nsName := "default"
+		kcl := &KubeClient{
+			cli:        kfake.NewSimpleClientset(&core.Namespace{ObjectMeta: meta.ObjectMeta{Name: nsName}}),
+			instanceID: "instance",
+			lock:       &sync.Mutex{},
+		}
+
+		err := kcl.ToggleSystemState(nsName, true)
+		assert.NoError(t, err)
+
+		ns, err := kcl.cli.CoreV1().Namespaces().Get(nsName, meta.GetOptions{})
+		assert.NoError(t, err)
+
+		_, exists := ns.Labels[systemNamespaceLabel]
+		assert.False(t, exists, "system label should not exists")
+	})
+
+	t.Run("should fail if namespace doesn't exist", func(t *testing.T) {
+		nsName := "not-exist"
+		kcl := &KubeClient{
+			cli:        kfake.NewSimpleClientset(),
+			instanceID: "instance",
+			lock:       &sync.Mutex{},
+		}
+
+		err := kcl.ToggleSystemState(nsName, true)
+		assert.Error(t, err)
+
+	})
+
+	t.Run("if called with the same state, should skip (exit without error)", func(t *testing.T) {
+		nsName := "namespace"
+		tests := []struct {
+			isSystem bool
+		}{
+			{isSystem: true},
+			{isSystem: false},
+		}
+
+		for _, test := range tests {
+			t.Run(strconv.FormatBool(test.isSystem), func(t *testing.T) {
+				kcl := &KubeClient{
+					cli: kfake.NewSimpleClientset(&core.Namespace{ObjectMeta: meta.ObjectMeta{Name: nsName, Labels: map[string]string{
+						systemNamespaceLabel: strconv.FormatBool(test.isSystem),
+					}}}),
+					instanceID: "instance",
+					lock:       &sync.Mutex{},
+				}
+
+				err := kcl.ToggleSystemState(nsName, test.isSystem)
+				assert.NoError(t, err)
+
+				ns, err := kcl.cli.CoreV1().Namespaces().Get(nsName, meta.GetOptions{})
+				assert.NoError(t, err)
+
+				assert.Equal(t, test.isSystem, isSystemNamespace(*ns))
+			})
+		}
+	})
+
+	t.Run("for regular namespace if isSystem is true and doesn't have a label, should set the label to true", func(t *testing.T) {
+		nsName := "namespace"
+
+		kcl := &KubeClient{
+			cli:        kfake.NewSimpleClientset(&core.Namespace{ObjectMeta: meta.ObjectMeta{Name: nsName}}),
+			instanceID: "instance",
+			lock:       &sync.Mutex{},
+		}
+
+		err := kcl.ToggleSystemState(nsName, true)
+		assert.NoError(t, err)
+
+		ns, err := kcl.cli.CoreV1().Namespaces().Get(nsName, meta.GetOptions{})
+		assert.NoError(t, err)
+
+		labelValue, exists := ns.Labels[systemNamespaceLabel]
+		assert.True(t, exists, "system label should exists")
+
+		assert.Equal(t, "true", labelValue)
+	})
+
+	t.Run("for default system namespace if isSystem is false and doesn't have a label, should set the label to false", func(t *testing.T) {
+		nsName := "portainer"
+
+		kcl := &KubeClient{
+			cli:        kfake.NewSimpleClientset(&core.Namespace{ObjectMeta: meta.ObjectMeta{Name: nsName}}),
+			instanceID: "instance",
+			lock:       &sync.Mutex{},
+		}
+
+		err := kcl.ToggleSystemState(nsName, false)
+		assert.NoError(t, err)
+
+		ns, err := kcl.cli.CoreV1().Namespaces().Get(nsName, meta.GetOptions{})
+		assert.NoError(t, err)
+
+		labelValue, exists := ns.Labels[systemNamespaceLabel]
+		assert.True(t, exists, "system label should exists")
+
+		assert.Equal(t, "false", labelValue)
+	})
+
+	t.Run("for system namespace (with label), if called with false, should set the label", func(t *testing.T) {
+		nsName := "namespace"
+
+		kcl := &KubeClient{
+			cli: kfake.NewSimpleClientset(&core.Namespace{ObjectMeta: meta.ObjectMeta{Name: nsName, Labels: map[string]string{
+				systemNamespaceLabel: "true",
+			}}}),
+			instanceID: "instance",
+			lock:       &sync.Mutex{},
+		}
+
+		err := kcl.ToggleSystemState(nsName, false)
+		assert.NoError(t, err)
+
+		ns, err := kcl.cli.CoreV1().Namespaces().Get(nsName, meta.GetOptions{})
+		assert.NoError(t, err)
+
+		labelValue, exists := ns.Labels[systemNamespaceLabel]
+		assert.True(t, exists, "system label should exists")
+		assert.Equal(t, "false", labelValue)
+	})
+
+	t.Run("for non system namespace (with label), if called with true, should set the label, and remove accesses", func(t *testing.T) {
+		nsName := "ns1"
+
+		namespace := &core.Namespace{ObjectMeta: meta.ObjectMeta{Name: nsName, Labels: map[string]string{
+			systemNamespaceLabel: "false",
+		}}}
+
+		config := &ktypes.ConfigMap{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      portainerConfigMapName,
+				Namespace: portainerNamespace,
+			},
+			Data: map[string]string{
+				"NamespaceAccessPolicies": `{"ns1":{"UserAccessPolicies":{"2":{"RoleId":0}}}, "ns2":{"UserAccessPolicies":{"2":{"RoleId":0}}}}`,
+			},
+		}
+
+		kcl := &KubeClient{
+			cli:        kfake.NewSimpleClientset(namespace, config),
+			instanceID: "instance",
+			lock:       &sync.Mutex{},
+		}
+
+		err := kcl.ToggleSystemState(nsName, true)
+		assert.NoError(t, err)
+
+		ns, err := kcl.cli.CoreV1().Namespaces().Get(nsName, meta.GetOptions{})
+		assert.NoError(t, err)
+
+		labelValue, exists := ns.Labels[systemNamespaceLabel]
+		assert.True(t, exists, "system label should exists")
+		assert.Equal(t, "true", labelValue)
+
+		expectedPolicies := map[string]portainer.K8sNamespaceAccessPolicy{
+			"ns2": {UserAccessPolicies: portainer.UserAccessPolicies{2: {RoleID: 0}}},
+		}
+		actualPolicies, err := kcl.GetNamespaceAccessPolicies()
+		assert.NoError(t, err, "failed to fetch policies")
+		assert.Equal(t, expectedPolicies, actualPolicies)
+
+	})
+}

api/kubernetes/cli/naming.go

@@ -1,16 +1,20 @@
 package cli
 
-import "fmt"
+import (
+	"fmt"
+)
 
 const (
-	defaultNamespace                    = "default"
-	portainerNamespace                  = "portainer"
-	portainerUserCRName                 = "portainer-cr-user"
-	portainerUserCRBName                = "portainer-crb-user"
-	portainerUserServiceAccountPrefix   = "portainer-sa-user"
-	portainerRBPrefix                   = "portainer-rb"
-	portainerConfigMapName              = "portainer-config"
-	portainerConfigMapAccessPoliciesKey = "NamespaceAccessPolicies"
+	defaultNamespace                        = "default"
+	portainerNamespace                      = "portainer"
+	portainerUserCRName                     = "portainer-cr-user"
+	portainerUserCRBName                    = "portainer-crb-user"
+	portainerClusterAdminServiceAccountName = "portainer-sa-clusteradmin"
+	portainerUserServiceAccountPrefix       = "portainer-sa-user"
+	portainerRBPrefix                       = "portainer-rb"
+	portainerConfigMapName                  = "portainer-config"
+	portainerConfigMapAccessPoliciesKey     = "NamespaceAccessPolicies"
+	portainerShellPodPrefix                 = "portainer-pod-kubectl-shell"
 )
 
 func userServiceAccountName(userID int, instanceID string) string {
@@ -24,3 +28,7 @@ func userServiceAccountTokenSecretName(serviceAccountName string, instanceID str
 func namespaceClusterRoleBindingName(namespace string, instanceID string) string {
 	return fmt.Sprintf("%s-%s-%s", portainerRBPrefix, instanceID, namespace)
 }
+
+func userShellPodPrefix(serviceAccountName string) string {
+	return fmt.Sprintf("%s-%s-", portainerShellPodPrefix, serviceAccountName)
+}