chore: bump version to 2.31.0 (#789)

Co-authored-by: James Player <james.player@portainer.io>
Co-authored-by: Cara Ryan <cara.ryan@portainer.io>
Co-authored-by: stevensbkang <skan070@gmail.com>

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -91,10 +91,21 @@ body:
   - type: dropdown
     attributes:
       label: Portainer version
-      description: We only provide support for current versions of Portainer as per the lifecycle policy linked above. If you are on an older version of Portainer we recommend [upgrading first](https://docs.portainer.io/start/upgrade) in case your bug has already been fixed.
+      description: We only provide support for current versions of Portainer as per the lifecycle policy linked above. If you are on an older version of Portainer we recommend [updating first](https://docs.portainer.io/start/upgrade) in case your bug has already been fixed.
       multiple: false
       options:
+        - '2.30.1'
+        - '2.30.0'
+        - '2.29.2'
+        - '2.29.1'
+        - '2.29.0'
+        - '2.28.1'
         - '2.28.0'
+        - '2.27.6'
+        - '2.27.5'
+        - '2.27.4'
+        - '2.27.3'
+        - '2.27.2'
         - '2.27.1'
         - '2.27.0'
         - '2.26.1'
@@ -111,16 +122,6 @@ body:
         - '2.21.2'
         - '2.21.1'
         - '2.21.0'
-        - '2.20.3'
-        - '2.20.2'
-        - '2.20.1'
-        - '2.20.0'
-        - '2.19.5'
-        - '2.19.4'
-        - '2.19.3'
-        - '2.19.2'
-        - '2.19.1'
-        - '2.19.0'
     validations:
       required: true
 

.golangci.yaml

@@ -12,6 +12,7 @@ linters:
     - copyloopvar
     - intrange
     - perfsprint
+    - ineffassign
 
 linters-settings:
   depguard:

api/archive/zip.go

@@ -2,7 +2,6 @@ package archive
 
 import (
 	"archive/zip"
-	"bytes"
 	"fmt"
 	"io"
 	"os"
@@ -12,50 +11,6 @@ import (
 	"github.com/pkg/errors"
 )
 
-// UnzipArchive will unzip an archive from bytes into the dest destination folder on disk
-func UnzipArchive(archiveData []byte, dest string) error {
-	zipReader, err := zip.NewReader(bytes.NewReader(archiveData), int64(len(archiveData)))
-	if err != nil {
-		return err
-	}
-
-	for _, zipFile := range zipReader.File {
-		err := extractFileFromArchive(zipFile, dest)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func extractFileFromArchive(file *zip.File, dest string) error {
-	f, err := file.Open()
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-
-	data, err := io.ReadAll(f)
-	if err != nil {
-		return err
-	}
-
-	fpath := filepath.Join(dest, file.Name)
-
-	outFile, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode())
-	if err != nil {
-		return err
-	}
-
-	_, err = io.Copy(outFile, bytes.NewReader(data))
-	if err != nil {
-		return err
-	}
-
-	return outFile.Close()
-}
-
 // UnzipFile will decompress a zip archive, moving all files and folders
 // within the zip file (parameter 1) to an output directory (parameter 2).
 func UnzipFile(src string, dest string) error {
@@ -76,11 +31,11 @@ func UnzipFile(src string, dest string) error {
 		if f.FileInfo().IsDir() {
 			// Make Folder
 			os.MkdirAll(p, os.ModePerm)
+
 			continue
 		}
 
-		err = unzipFile(f, p)
-		if err != nil {
+		if err := unzipFile(f, p); err != nil {
 			return err
 		}
 	}
@@ -93,20 +48,20 @@ func unzipFile(f *zip.File, p string) error {
 	if err := os.MkdirAll(filepath.Dir(p), os.ModePerm); err != nil {
 		return errors.Wrapf(err, "unzipFile: can't make a path %s", p)
 	}
+
 	outFile, err := os.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
 	if err != nil {
 		return errors.Wrapf(err, "unzipFile: can't create file %s", p)
 	}
 	defer outFile.Close()
+
 	rc, err := f.Open()
 	if err != nil {
 		return errors.Wrapf(err, "unzipFile: can't open zip file %s in the archive", f.Name)
 	}
 	defer rc.Close()
 
-	_, err = io.Copy(outFile, rc)
-
-	if err != nil {
+	if _, err = io.Copy(outFile, rc); err != nil {
 		return errors.Wrapf(err, "unzipFile: can't copy an archived file content")
 	}
 

api/cli/cli.go

@@ -60,6 +60,7 @@ func CLIFlags() *portainer.CLIFlags {
 		LogLevel:                  kingpin.Flag("log-level", "Set the minimum logging level to show").Default("INFO").Enum("DEBUG", "INFO", "WARN", "ERROR"),
 		LogMode:                   kingpin.Flag("log-mode", "Set the logging output mode").Default("PRETTY").Enum("NOCOLOR", "PRETTY", "JSON"),
 		KubectlShellImage:         kingpin.Flag("kubectl-shell-image", "Kubectl shell image").Envar(portainer.KubectlShellImageEnvVar).Default(portainer.DefaultKubectlShellImage).String(),
+		PullLimitCheckDisabled:    kingpin.Flag("pull-limit-check-disabled", "Pull limit check").Envar(portainer.PullLimitCheckDisabledEnvVar).Default(defaultPullLimitCheckDisabled).Bool(),
 	}
 }
 

api/cli/defaults.go

@@ -4,20 +4,21 @@
 package cli
 
 const (
-	defaultBindAddress         = ":9000"
-	defaultHTTPSBindAddress    = ":9443"
-	defaultTunnelServerAddress = "0.0.0.0"
-	defaultTunnelServerPort    = "8000"
-	defaultDataDirectory       = "/data"
-	defaultAssetsDirectory     = "./"
-	defaultTLS                 = "false"
-	defaultTLSSkipVerify       = "false"
-	defaultTLSCACertPath       = "/certs/ca.pem"
-	defaultTLSCertPath         = "/certs/cert.pem"
-	defaultTLSKeyPath          = "/certs/key.pem"
-	defaultHTTPDisabled        = "false"
-	defaultHTTPEnabled         = "false"
-	defaultSSL                 = "false"
-	defaultBaseURL             = "/"
-	defaultSecretKeyName       = "portainer"
+	defaultBindAddress            = ":9000"
+	defaultHTTPSBindAddress       = ":9443"
+	defaultTunnelServerAddress    = "0.0.0.0"
+	defaultTunnelServerPort       = "8000"
+	defaultDataDirectory          = "/data"
+	defaultAssetsDirectory        = "./"
+	defaultTLS                    = "false"
+	defaultTLSSkipVerify          = "false"
+	defaultTLSCACertPath          = "/certs/ca.pem"
+	defaultTLSCertPath            = "/certs/cert.pem"
+	defaultTLSKeyPath             = "/certs/key.pem"
+	defaultHTTPDisabled           = "false"
+	defaultHTTPEnabled            = "false"
+	defaultSSL                    = "false"
+	defaultBaseURL                = "/"
+	defaultSecretKeyName          = "portainer"
+	defaultPullLimitCheckDisabled = "false"
 )

api/cli/defaults_windows.go

@@ -1,21 +1,22 @@
 package cli
 
 const (
-	defaultBindAddress         = ":9000"
-	defaultHTTPSBindAddress    = ":9443"
-	defaultTunnelServerAddress = "0.0.0.0"
-	defaultTunnelServerPort    = "8000"
-	defaultDataDirectory       = "C:\\data"
-	defaultAssetsDirectory     = "./"
-	defaultTLS                 = "false"
-	defaultTLSSkipVerify       = "false"
-	defaultTLSCACertPath       = "C:\\certs\\ca.pem"
-	defaultTLSCertPath         = "C:\\certs\\cert.pem"
-	defaultTLSKeyPath          = "C:\\certs\\key.pem"
-	defaultHTTPDisabled        = "false"
-	defaultHTTPEnabled         = "false"
-	defaultSSL                 = "false"
-	defaultSnapshotInterval    = "5m"
-	defaultBaseURL             = "/"
-	defaultSecretKeyName       = "portainer"
+	defaultBindAddress            = ":9000"
+	defaultHTTPSBindAddress       = ":9443"
+	defaultTunnelServerAddress    = "0.0.0.0"
+	defaultTunnelServerPort       = "8000"
+	defaultDataDirectory          = "C:\\data"
+	defaultAssetsDirectory        = "./"
+	defaultTLS                    = "false"
+	defaultTLSSkipVerify          = "false"
+	defaultTLSCACertPath          = "C:\\certs\\ca.pem"
+	defaultTLSCertPath            = "C:\\certs\\cert.pem"
+	defaultTLSKeyPath             = "C:\\certs\\key.pem"
+	defaultHTTPDisabled           = "false"
+	defaultHTTPEnabled            = "false"
+	defaultSSL                    = "false"
+	defaultSnapshotInterval       = "5m"
+	defaultBaseURL                = "/"
+	defaultSecretKeyName          = "portainer"
+	defaultPullLimitCheckDisabled = "false"
 )

api/cli/pairlistbool.go

@@ -1,45 +0,0 @@
-package cli
-
-import (
-	"strings"
-
-	portainer "github.com/portainer/portainer/api"
-
-	"gopkg.in/alecthomas/kingpin.v2"
-)
-
-type pairListBool []portainer.Pair
-
-// Set implementation for a list of portainer.Pair
-func (l *pairListBool) Set(value string) error {
-	p := new(portainer.Pair)
-
-	// default to true.  example setting=true is equivalent to setting
-	parts := strings.SplitN(value, "=", 2)
-	if len(parts) != 2 {
-		p.Name = parts[0]
-		p.Value = "true"
-	} else {
-		p.Name = parts[0]
-		p.Value = parts[1]
-	}
-
-	*l = append(*l, *p)
-	return nil
-}
-
-// String implementation for a list of pair
-func (l *pairListBool) String() string {
-	return ""
-}
-
-// IsCumulative implementation for a list of pair
-func (l *pairListBool) IsCumulative() bool {
-	return true
-}
-
-func BoolPairs(s kingpin.Settings) (target *[]portainer.Pair) {
-	target = new([]portainer.Pair)
-	s.SetValue((*pairListBool)(target))
-	return
-}

api/cmd/portainer/main.go

@@ -39,6 +39,7 @@ import (
 	"github.com/portainer/portainer/api/kubernetes"
 	kubecli "github.com/portainer/portainer/api/kubernetes/cli"
 	"github.com/portainer/portainer/api/ldap"
+	"github.com/portainer/portainer/api/logs"
 	"github.com/portainer/portainer/api/oauth"
 	"github.com/portainer/portainer/api/pendingactions"
 	"github.com/portainer/portainer/api/pendingactions/actions"
@@ -166,8 +167,8 @@ func checkDBSchemaServerVersionMatch(dbStore dataservices.DataStore, serverVersi
 	return v.SchemaVersion == serverVersion && v.Edition == serverEdition
 }
 
-func initKubernetesDeployer(kubernetesTokenCacheManager *kubeproxy.TokenCacheManager, kubernetesClientFactory *kubecli.ClientFactory, dataStore dataservices.DataStore, reverseTunnelService portainer.ReverseTunnelService, signatureService portainer.DigitalSignatureService, proxyManager *proxy.Manager, assetsPath string) portainer.KubernetesDeployer {
-	return exec.NewKubernetesDeployer(kubernetesTokenCacheManager, kubernetesClientFactory, dataStore, reverseTunnelService, signatureService, proxyManager, assetsPath)
+func initKubernetesDeployer(kubernetesTokenCacheManager *kubeproxy.TokenCacheManager, kubernetesClientFactory *kubecli.ClientFactory, dataStore dataservices.DataStore, reverseTunnelService portainer.ReverseTunnelService, signatureService portainer.DigitalSignatureService, proxyManager *proxy.Manager) portainer.KubernetesDeployer {
+	return exec.NewKubernetesDeployer(kubernetesTokenCacheManager, kubernetesClientFactory, dataStore, reverseTunnelService, signatureService, proxyManager)
 }
 
 func initHelmPackageManager() (libhelmtypes.HelmPackageManager, error) {
@@ -422,7 +423,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		log.Fatal().Err(err).Msg("failed initializing swarm stack manager")
 	}
 
-	kubernetesDeployer := initKubernetesDeployer(kubernetesTokenCacheManager, kubernetesClientFactory, dataStore, reverseTunnelService, signatureService, proxyManager, *flags.Assets)
+	kubernetesDeployer := initKubernetesDeployer(kubernetesTokenCacheManager, kubernetesClientFactory, dataStore, reverseTunnelService, signatureService, proxyManager)
 
 	pendingActionsService := pendingactions.NewService(dataStore, kubernetesClientFactory)
 	pendingActionsService.RegisterHandler(actions.CleanNAPWithOverridePolicies, handlers.NewHandlerCleanNAPWithOverridePolicies(authorizationService, dataStore))
@@ -576,17 +577,18 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		AdminCreationDone:           adminCreationDone,
 		PendingActionsService:       pendingActionsService,
 		PlatformService:             platformService,
+		PullLimitCheckDisabled:      *flags.PullLimitCheckDisabled,
 	}
 }
 
 func main() {
-	configureLogger()
-	setLoggingMode("PRETTY")
+	logs.ConfigureLogger()
+	logs.SetLoggingMode("PRETTY")
 
 	flags := initCLI()
 
-	setLoggingLevel(*flags.LogLevel)
-	setLoggingMode(*flags.LogMode)
+	logs.SetLoggingLevel(*flags.LogLevel)
+	logs.SetLoggingMode(*flags.LogMode)
 
 	for {
 		server := buildServer(flags)

api/connection.go

@@ -6,8 +6,10 @@ import (
 
 type ReadTransaction interface {
 	GetObject(bucketName string, key []byte, object any) error
+	GetRawBytes(bucketName string, key []byte) ([]byte, error)
 	GetAll(bucketName string, obj any, append func(o any) (any, error)) error
 	GetAllWithKeyPrefix(bucketName string, keyPrefix []byte, obj any, append func(o any) (any, error)) error
+	KeyExists(bucketName string, key []byte) (bool, error)
 }
 
 type Transaction interface {

api/database/boltdb/db.go

@@ -244,6 +244,32 @@ func (connection *DbConnection) GetObject(bucketName string, key []byte, object
 	})
 }
 
+func (connection *DbConnection) GetRawBytes(bucketName string, key []byte) ([]byte, error) {
+	var value []byte
+
+	err := connection.ViewTx(func(tx portainer.Transaction) error {
+		var err error
+		value, err = tx.GetRawBytes(bucketName, key)
+
+		return err
+	})
+
+	return value, err
+}
+
+func (connection *DbConnection) KeyExists(bucketName string, key []byte) (bool, error) {
+	var exists bool
+
+	err := connection.ViewTx(func(tx portainer.Transaction) error {
+		var err error
+		exists, err = tx.KeyExists(bucketName, key)
+
+		return err
+	})
+
+	return exists, err
+}
+
 func (connection *DbConnection) getEncryptionKey() []byte {
 	if !connection.isEncrypted {
 		return nil

api/database/boltdb/tx.go

@@ -6,6 +6,7 @@ import (
 
 	dserrors "github.com/portainer/portainer/api/dataservices/errors"
 
+	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
 	bolt "go.etcd.io/bbolt"
 )
@@ -31,6 +32,33 @@ func (tx *DbTransaction) GetObject(bucketName string, key []byte, object any) er
 	return tx.conn.UnmarshalObject(value, object)
 }
 
+func (tx *DbTransaction) GetRawBytes(bucketName string, key []byte) ([]byte, error) {
+	bucket := tx.tx.Bucket([]byte(bucketName))
+
+	value := bucket.Get(key)
+	if value == nil {
+		return nil, fmt.Errorf("%w (bucket=%s, key=%s)", dserrors.ErrObjectNotFound, bucketName, keyToString(key))
+	}
+
+	if tx.conn.getEncryptionKey() != nil {
+		var err error
+
+		if value, err = decrypt(value, tx.conn.getEncryptionKey()); err != nil {
+			return value, errors.Wrap(err, "Failed decrypting object")
+		}
+	}
+
+	return value, nil
+}
+
+func (tx *DbTransaction) KeyExists(bucketName string, key []byte) (bool, error) {
+	bucket := tx.tx.Bucket([]byte(bucketName))
+
+	value := bucket.Get(key)
+
+	return value != nil, nil
+}
+
 func (tx *DbTransaction) UpdateObject(bucketName string, key []byte, object any) error {
 	data, err := tx.conn.MarshalObject(object)
 	if err != nil {

api/dataservices/base.go

@@ -9,6 +9,7 @@ import (
 type BaseCRUD[T any, I constraints.Integer] interface {
 	Create(element *T) error
 	Read(ID I) (*T, error)
+	Exists(ID I) (bool, error)
 	ReadAll() ([]T, error)
 	Update(ID I, element *T) error
 	Delete(ID I) error
@@ -42,6 +43,19 @@ func (service BaseDataService[T, I]) Read(ID I) (*T, error) {
 	})
 }
 
+func (service BaseDataService[T, I]) Exists(ID I) (bool, error) {
+	var exists bool
+
+	err := service.Connection.ViewTx(func(tx portainer.Transaction) error {
+		var err error
+		exists, err = service.Tx(tx).Exists(ID)
+
+		return err
+	})
+
+	return exists, err
+}
+
 func (service BaseDataService[T, I]) ReadAll() ([]T, error) {
 	var collection = make([]T, 0)
 

api/dataservices/base_tx.go

@@ -28,6 +28,12 @@ func (service BaseDataServiceTx[T, I]) Read(ID I) (*T, error) {
 	return &element, nil
 }
 
+func (service BaseDataServiceTx[T, I]) Exists(ID I) (bool, error) {
+	identifier := service.Connection.ConvertToKey(int(ID))
+
+	return service.Tx.KeyExists(service.Bucket, identifier)
+}
+
 func (service BaseDataServiceTx[T, I]) ReadAll() ([]T, error) {
 	var collection = make([]T, 0)
 

api/dataservices/edgestackstatus/edgestackstatus.go

@@ -0,0 +1,89 @@
+package edgestackstatus
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+)
+
+var _ dataservices.EdgeStackStatusService = &Service{}
+
+const BucketName = "edge_stack_status"
+
+type Service struct {
+	conn portainer.Connection
+}
+
+func (service *Service) BucketName() string {
+	return BucketName
+}
+
+func NewService(connection portainer.Connection) (*Service, error) {
+	if err := connection.SetServiceName(BucketName); err != nil {
+		return nil, err
+	}
+
+	return &Service{conn: connection}, nil
+}
+
+func (s *Service) Tx(tx portainer.Transaction) ServiceTx {
+	return ServiceTx{
+		service: s,
+		tx:      tx,
+	}
+}
+
+func (s *Service) Create(edgeStackID portainer.EdgeStackID, endpointID portainer.EndpointID, status *portainer.EdgeStackStatusForEnv) error {
+	return s.conn.UpdateTx(func(tx portainer.Transaction) error {
+		return s.Tx(tx).Create(edgeStackID, endpointID, status)
+	})
+}
+
+func (s *Service) Read(edgeStackID portainer.EdgeStackID, endpointID portainer.EndpointID) (*portainer.EdgeStackStatusForEnv, error) {
+	var element *portainer.EdgeStackStatusForEnv
+
+	return element, s.conn.ViewTx(func(tx portainer.Transaction) error {
+		var err error
+		element, err = s.Tx(tx).Read(edgeStackID, endpointID)
+
+		return err
+	})
+}
+
+func (s *Service) ReadAll(edgeStackID portainer.EdgeStackID) ([]portainer.EdgeStackStatusForEnv, error) {
+	var collection = make([]portainer.EdgeStackStatusForEnv, 0)
+
+	return collection, s.conn.ViewTx(func(tx portainer.Transaction) error {
+		var err error
+		collection, err = s.Tx(tx).ReadAll(edgeStackID)
+
+		return err
+	})
+}
+
+func (s *Service) Update(edgeStackID portainer.EdgeStackID, endpointID portainer.EndpointID, status *portainer.EdgeStackStatusForEnv) error {
+	return s.conn.UpdateTx(func(tx portainer.Transaction) error {
+		return s.Tx(tx).Update(edgeStackID, endpointID, status)
+	})
+}
+
+func (s *Service) Delete(edgeStackID portainer.EdgeStackID, endpointID portainer.EndpointID) error {
+	return s.conn.UpdateTx(func(tx portainer.Transaction) error {
+		return s.Tx(tx).Delete(edgeStackID, endpointID)
+	})
+}
+
+func (s *Service) DeleteAll(edgeStackID portainer.EdgeStackID) error {
+	return s.conn.UpdateTx(func(tx portainer.Transaction) error {
+		return s.Tx(tx).DeleteAll(edgeStackID)
+	})
+}
+
+func (s *Service) Clear(edgeStackID portainer.EdgeStackID, relatedEnvironmentsIDs []portainer.EndpointID) error {
+	return s.conn.UpdateTx(func(tx portainer.Transaction) error {
+		return s.Tx(tx).Clear(edgeStackID, relatedEnvironmentsIDs)
+	})
+}
+
+func (s *Service) key(edgeStackID portainer.EdgeStackID, endpointID portainer.EndpointID) []byte {
+	return append(s.conn.ConvertToKey(int(edgeStackID)), s.conn.ConvertToKey(int(endpointID))...)
+}

api/dataservices/edgestackstatus/tx.go

@@ -0,0 +1,95 @@
+package edgestackstatus
+
+import (
+	"fmt"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+)
+
+var _ dataservices.EdgeStackStatusService = &Service{}
+
+type ServiceTx struct {
+	service *Service
+	tx      portainer.Transaction
+}
+
+func (service ServiceTx) Create(edgeStackID portainer.EdgeStackID, endpointID portainer.EndpointID, status *portainer.EdgeStackStatusForEnv) error {
+	identifier := service.service.key(edgeStackID, endpointID)
+	return service.tx.CreateObjectWithStringId(BucketName, identifier, status)
+}
+
+func (s ServiceTx) Read(edgeStackID portainer.EdgeStackID, endpointID portainer.EndpointID) (*portainer.EdgeStackStatusForEnv, error) {
+	var status portainer.EdgeStackStatusForEnv
+	identifier := s.service.key(edgeStackID, endpointID)
+
+	if err := s.tx.GetObject(BucketName, identifier, &status); err != nil {
+		return nil, err
+	}
+
+	return &status, nil
+}
+
+func (s ServiceTx) ReadAll(edgeStackID portainer.EdgeStackID) ([]portainer.EdgeStackStatusForEnv, error) {
+	keyPrefix := s.service.conn.ConvertToKey(int(edgeStackID))
+
+	statuses := make([]portainer.EdgeStackStatusForEnv, 0)
+
+	if err := s.tx.GetAllWithKeyPrefix(BucketName, keyPrefix, &portainer.EdgeStackStatusForEnv{}, dataservices.AppendFn(&statuses)); err != nil {
+		return nil, fmt.Errorf("unable to retrieve EdgeStackStatus for EdgeStack %d: %w", edgeStackID, err)
+	}
+
+	return statuses, nil
+}
+
+func (s ServiceTx) Update(edgeStackID portainer.EdgeStackID, endpointID portainer.EndpointID, status *portainer.EdgeStackStatusForEnv) error {
+	identifier := s.service.key(edgeStackID, endpointID)
+	return s.tx.UpdateObject(BucketName, identifier, status)
+}
+
+func (s ServiceTx) Delete(edgeStackID portainer.EdgeStackID, endpointID portainer.EndpointID) error {
+	identifier := s.service.key(edgeStackID, endpointID)
+	return s.tx.DeleteObject(BucketName, identifier)
+}
+
+func (s ServiceTx) DeleteAll(edgeStackID portainer.EdgeStackID) error {
+	keyPrefix := s.service.conn.ConvertToKey(int(edgeStackID))
+
+	statuses := make([]portainer.EdgeStackStatusForEnv, 0)
+
+	if err := s.tx.GetAllWithKeyPrefix(BucketName, keyPrefix, &portainer.EdgeStackStatusForEnv{}, dataservices.AppendFn(&statuses)); err != nil {
+		return fmt.Errorf("unable to retrieve EdgeStackStatus for EdgeStack %d: %w", edgeStackID, err)
+	}
+
+	for _, status := range statuses {
+		if err := s.tx.DeleteObject(BucketName, s.service.key(edgeStackID, status.EndpointID)); err != nil {
+			return fmt.Errorf("unable to delete EdgeStackStatus for EdgeStack %d and Endpoint %d: %w", edgeStackID, status.EndpointID, err)
+		}
+	}
+
+	return nil
+}
+
+func (s ServiceTx) Clear(edgeStackID portainer.EdgeStackID, relatedEnvironmentsIDs []portainer.EndpointID) error {
+	for _, envID := range relatedEnvironmentsIDs {
+		existingStatus, err := s.Read(edgeStackID, envID)
+		if err != nil && !dataservices.IsErrObjectNotFound(err) {
+			return fmt.Errorf("unable to retrieve status for environment %d: %w", envID, err)
+		}
+
+		var deploymentInfo portainer.StackDeploymentInfo
+		if existingStatus != nil {
+			deploymentInfo = existingStatus.DeploymentInfo
+		}
+
+		if err := s.Update(edgeStackID, envID, &portainer.EdgeStackStatusForEnv{
+			EndpointID:     envID,
+			Status:         []portainer.EdgeStackDeploymentStatus{},
+			DeploymentInfo: deploymentInfo,
+		}); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

api/dataservices/endpointrelation/tx.go

@@ -93,6 +93,10 @@ func (service ServiceTx) AddEndpointRelationsForEdgeStack(endpointIDs []portaine
 		}
 	}
 
+	service.service.mu.Lock()
+	service.service.endpointRelationsCache = nil
+	service.service.mu.Unlock()
+
 	if err := service.service.updateStackFnTx(service.tx, edgeStackID, func(edgeStack *portainer.EdgeStack) {
 		edgeStack.NumDeployments += len(endpointIDs)
 	}); err != nil {
@@ -119,6 +123,10 @@ func (service ServiceTx) RemoveEndpointRelationsForEdgeStack(endpointIDs []porta
 		}
 	}
 
+	service.service.mu.Lock()
+	service.service.endpointRelationsCache = nil
+	service.service.mu.Unlock()
+
 	if err := service.service.updateStackFnTx(service.tx, edgeStackID, func(edgeStack *portainer.EdgeStack) {
 		edgeStack.NumDeployments -= len(endpointIDs)
 	}); err != nil {

api/dataservices/interface.go

@@ -12,6 +12,7 @@ type (
 		EdgeGroup() EdgeGroupService
 		EdgeJob() EdgeJobService
 		EdgeStack() EdgeStackService
+		EdgeStackStatus() EdgeStackStatusService
 		Endpoint() EndpointService
 		EndpointGroup() EndpointGroupService
 		EndpointRelation() EndpointRelationService
@@ -39,8 +40,8 @@ type (
 		Open() (newStore bool, err error)
 		Init() error
 		Close() error
-		UpdateTx(func(DataStoreTx) error) error
-		ViewTx(func(DataStoreTx) error) error
+		UpdateTx(func(tx DataStoreTx) error) error
+		ViewTx(func(tx DataStoreTx) error) error
 		MigrateData() error
 		Rollback(force bool) error
 		CheckCurrentEdition() error
@@ -89,6 +90,16 @@ type (
 		BucketName() string
 	}
 
+	EdgeStackStatusService interface {
+		Create(edgeStackID portainer.EdgeStackID, endpointID portainer.EndpointID, status *portainer.EdgeStackStatusForEnv) error
+		Read(edgeStackID portainer.EdgeStackID, endpointID portainer.EndpointID) (*portainer.EdgeStackStatusForEnv, error)
+		ReadAll(edgeStackID portainer.EdgeStackID) ([]portainer.EdgeStackStatusForEnv, error)
+		Update(edgeStackID portainer.EdgeStackID, endpointID portainer.EndpointID, status *portainer.EdgeStackStatusForEnv) error
+		Delete(edgeStackID portainer.EdgeStackID, endpointID portainer.EndpointID) error
+		DeleteAll(edgeStackID portainer.EdgeStackID) error
+		Clear(edgeStackID portainer.EdgeStackID, relatedEnvironmentsIDs []portainer.EndpointID) error
+	}
+
 	// EndpointService represents a service for managing environment(endpoint) data
 	EndpointService interface {
 		Endpoint(ID portainer.EndpointID) (*portainer.Endpoint, error)
@@ -159,6 +170,7 @@ type (
 
 	SnapshotService interface {
 		BaseCRUD[portainer.Snapshot, portainer.EndpointID]
+		ReadWithoutSnapshotRaw(ID portainer.EndpointID) (*portainer.Snapshot, error)
 	}
 
 	// SSLSettingsService represents a service for managing application settings

api/dataservices/snapshot/snapshot.go

@@ -38,3 +38,33 @@ func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
 func (service *Service) Create(snapshot *portainer.Snapshot) error {
 	return service.Connection.CreateObjectWithId(BucketName, int(snapshot.EndpointID), snapshot)
 }
+
+func (service *Service) ReadWithoutSnapshotRaw(ID portainer.EndpointID) (*portainer.Snapshot, error) {
+	var snapshot *portainer.Snapshot
+
+	err := service.Connection.ViewTx(func(tx portainer.Transaction) error {
+		var err error
+		snapshot, err = service.Tx(tx).ReadWithoutSnapshotRaw(ID)
+
+		return err
+	})
+
+	return snapshot, err
+}
+
+func (service *Service) ReadRawMessage(ID portainer.EndpointID) (*portainer.SnapshotRawMessage, error) {
+	var snapshot *portainer.SnapshotRawMessage
+
+	err := service.Connection.ViewTx(func(tx portainer.Transaction) error {
+		var err error
+		snapshot, err = service.Tx(tx).ReadRawMessage(ID)
+
+		return err
+	})
+
+	return snapshot, err
+}
+
+func (service *Service) CreateRawMessage(snapshot *portainer.SnapshotRawMessage) error {
+	return service.Connection.CreateObjectWithId(BucketName, int(snapshot.EndpointID), snapshot)
+}

api/dataservices/snapshot/tx.go

@@ -12,3 +12,42 @@ type ServiceTx struct {
 func (service ServiceTx) Create(snapshot *portainer.Snapshot) error {
 	return service.Tx.CreateObjectWithId(BucketName, int(snapshot.EndpointID), snapshot)
 }
+
+func (service ServiceTx) ReadWithoutSnapshotRaw(ID portainer.EndpointID) (*portainer.Snapshot, error) {
+	var snapshot struct {
+		Docker *struct {
+			X struct{} `json:"DockerSnapshotRaw"`
+			*portainer.DockerSnapshot
+		} `json:"Docker"`
+
+		portainer.Snapshot
+	}
+
+	identifier := service.Connection.ConvertToKey(int(ID))
+
+	if err := service.Tx.GetObject(service.Bucket, identifier, &snapshot); err != nil {
+		return nil, err
+	}
+
+	if snapshot.Docker != nil {
+		snapshot.Snapshot.Docker = snapshot.Docker.DockerSnapshot
+	}
+
+	return &snapshot.Snapshot, nil
+}
+
+func (service ServiceTx) ReadRawMessage(ID portainer.EndpointID) (*portainer.SnapshotRawMessage, error) {
+	var snapshot = portainer.SnapshotRawMessage{}
+
+	identifier := service.Connection.ConvertToKey(int(ID))
+
+	if err := service.Tx.GetObject(service.Bucket, identifier, &snapshot); err != nil {
+		return nil, err
+	}
+
+	return &snapshot, nil
+}
+
+func (service ServiceTx) CreateRawMessage(snapshot *portainer.SnapshotRawMessage) error {
+	return service.Tx.CreateObjectWithId(BucketName, int(snapshot.EndpointID), snapshot)
+}

api/datastore/migrate_data.go

@@ -40,13 +40,11 @@ func (store *Store) MigrateData() error {
 	}
 
 	// before we alter anything in the DB, create a backup
-	_, err = store.Backup("")
-	if err != nil {
+	if _, err := store.Backup(""); err != nil {
 		return errors.Wrap(err, "while backing up database")
 	}
 
-	err = store.FailSafeMigrate(migrator, version)
-	if err != nil {
+	if err := store.FailSafeMigrate(migrator, version); err != nil {
 		err = errors.Wrap(err, "failed to migrate database")
 
 		log.Warn().Err(err).Msg("migration failed, restoring database to previous version")
@@ -85,6 +83,7 @@ func (store *Store) newMigratorParameters(version *models.Version, flags *portai
 		DockerhubService:        store.DockerHubService,
 		AuthorizationService:    authorization.NewService(store),
 		EdgeStackService:        store.EdgeStackService,
+		EdgeStackStatusService:  store.EdgeStackStatusService,
 		EdgeJobService:          store.EdgeJobService,
 		TunnelServerService:     store.TunnelServerService,
 		PendingActionsService:   store.PendingActionsService,
@@ -140,8 +139,7 @@ func (store *Store) connectionRollback(force bool) error {
 		}
 	}
 
-	err := store.Restore()
-	if err != nil {
+	if err := store.Restore(); err != nil {
 		return err
 	}
 

api/datastore/migrator/migrate_2_31_0.go

@@ -0,0 +1,31 @@
+package migrator
+
+import portainer "github.com/portainer/portainer/api"
+
+func (m *Migrator) migrateEdgeStacksStatuses_2_31_0() error {
+	edgeStacks, err := m.edgeStackService.EdgeStacks()
+	if err != nil {
+		return err
+	}
+
+	for _, edgeStack := range edgeStacks {
+		for envID, status := range edgeStack.Status {
+			if err := m.edgeStackStatusService.Create(edgeStack.ID, envID, &portainer.EdgeStackStatusForEnv{
+				EndpointID:       envID,
+				Status:           status.Status,
+				DeploymentInfo:   status.DeploymentInfo,
+				ReadyRePullImage: status.ReadyRePullImage,
+			}); err != nil {
+				return err
+			}
+		}
+
+		edgeStack.Status = nil
+
+		if err := m.edgeStackService.UpdateEdgeStack(edgeStack.ID, &edgeStack); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

api/datastore/migrator/migrate_dbversion20.go

@@ -18,8 +18,7 @@ func (m *Migrator) updateResourceControlsToDBVersion22() error {
 	for _, resourceControl := range legacyResourceControls {
 		resourceControl.AdministratorsOnly = false
 
-		err := m.resourceControlService.Update(resourceControl.ID, &resourceControl)
-		if err != nil {
+		if err := m.resourceControlService.Update(resourceControl.ID, &resourceControl); err != nil {
 			return err
 		}
 	}
@@ -42,8 +41,8 @@ func (m *Migrator) updateUsersAndRolesToDBVersion22() error {
 
 	for _, user := range legacyUsers {
 		user.PortainerAuthorizations = authorization.DefaultPortainerAuthorizations()
-		err = m.userService.Update(user.ID, &user)
-		if err != nil {
+
+		if err := m.userService.Update(user.ID, &user); err != nil {
 			return err
 		}
 	}
@@ -52,38 +51,47 @@ func (m *Migrator) updateUsersAndRolesToDBVersion22() error {
 	if err != nil {
 		return err
 	}
+
 	endpointAdministratorRole.Priority = 1
 	endpointAdministratorRole.Authorizations = authorization.DefaultEndpointAuthorizationsForEndpointAdministratorRole()
 
-	err = m.roleService.Update(endpointAdministratorRole.ID, endpointAdministratorRole)
+	if err := m.roleService.Update(endpointAdministratorRole.ID, endpointAdministratorRole); err != nil {
+		return err
+	}
 
 	helpDeskRole, err := m.roleService.Read(portainer.RoleID(2))
 	if err != nil {
 		return err
 	}
+
 	helpDeskRole.Priority = 2
 	helpDeskRole.Authorizations = authorization.DefaultEndpointAuthorizationsForHelpDeskRole(settings.AllowVolumeBrowserForRegularUsers)
 
-	err = m.roleService.Update(helpDeskRole.ID, helpDeskRole)
+	if err := m.roleService.Update(helpDeskRole.ID, helpDeskRole); err != nil {
+		return err
+	}
 
 	standardUserRole, err := m.roleService.Read(portainer.RoleID(3))
 	if err != nil {
 		return err
 	}
+
 	standardUserRole.Priority = 3
 	standardUserRole.Authorizations = authorization.DefaultEndpointAuthorizationsForStandardUserRole(settings.AllowVolumeBrowserForRegularUsers)
 
-	err = m.roleService.Update(standardUserRole.ID, standardUserRole)
+	if err := m.roleService.Update(standardUserRole.ID, standardUserRole); err != nil {
+		return err
+	}
 
 	readOnlyUserRole, err := m.roleService.Read(portainer.RoleID(4))
 	if err != nil {
 		return err
 	}
+
 	readOnlyUserRole.Priority = 4
 	readOnlyUserRole.Authorizations = authorization.DefaultEndpointAuthorizationsForReadOnlyUserRole(settings.AllowVolumeBrowserForRegularUsers)
 
-	err = m.roleService.Update(readOnlyUserRole.ID, readOnlyUserRole)
-	if err != nil {
+	if err := m.roleService.Update(readOnlyUserRole.ID, readOnlyUserRole); err != nil {
 		return err
 	}
 

api/datastore/migrator/migrator.go

@@ -3,12 +3,12 @@ package migrator
 import (
 	"errors"
 
-	"github.com/Masterminds/semver"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/database/models"
 	"github.com/portainer/portainer/api/dataservices/dockerhub"
 	"github.com/portainer/portainer/api/dataservices/edgejob"
 	"github.com/portainer/portainer/api/dataservices/edgestack"
+	"github.com/portainer/portainer/api/dataservices/edgestackstatus"
 	"github.com/portainer/portainer/api/dataservices/endpoint"
 	"github.com/portainer/portainer/api/dataservices/endpointgroup"
 	"github.com/portainer/portainer/api/dataservices/endpointrelation"
@@ -27,6 +27,8 @@ import (
 	"github.com/portainer/portainer/api/dataservices/user"
 	"github.com/portainer/portainer/api/dataservices/version"
 	"github.com/portainer/portainer/api/internal/authorization"
+
+	"github.com/Masterminds/semver"
 	"github.com/rs/zerolog/log"
 )
 
@@ -56,6 +58,7 @@ type (
 		authorizationService    *authorization.Service
 		dockerhubService        *dockerhub.Service
 		edgeStackService        *edgestack.Service
+		edgeStackStatusService  *edgestackstatus.Service
 		edgeJobService          *edgejob.Service
 		TunnelServerService     *tunnelserver.Service
 		pendingActionsService   *pendingactions.Service
@@ -84,6 +87,7 @@ type (
 		AuthorizationService    *authorization.Service
 		DockerhubService        *dockerhub.Service
 		EdgeStackService        *edgestack.Service
+		EdgeStackStatusService  *edgestackstatus.Service
 		EdgeJobService          *edgejob.Service
 		TunnelServerService     *tunnelserver.Service
 		PendingActionsService   *pendingactions.Service
@@ -114,6 +118,7 @@ func NewMigrator(parameters *MigratorParameters) *Migrator {
 		authorizationService:    parameters.AuthorizationService,
 		dockerhubService:        parameters.DockerhubService,
 		edgeStackService:        parameters.EdgeStackService,
+		edgeStackStatusService:  parameters.EdgeStackStatusService,
 		edgeJobService:          parameters.EdgeJobService,
 		TunnelServerService:     parameters.TunnelServerService,
 		pendingActionsService:   parameters.PendingActionsService,
@@ -242,6 +247,8 @@ func (m *Migrator) initMigrations() {
 		m.migratePendingActionsDataForDB130,
 	)
 
+	m.addMigrations("2.31.0", m.migrateEdgeStacksStatuses_2_31_0)
+
 	// Add new migrations above...
 	// One function per migration, each versions migration funcs in the same file.
 }

api/datastore/services.go

@@ -13,6 +13,7 @@ import (
 	"github.com/portainer/portainer/api/dataservices/edgegroup"
 	"github.com/portainer/portainer/api/dataservices/edgejob"
 	"github.com/portainer/portainer/api/dataservices/edgestack"
+	"github.com/portainer/portainer/api/dataservices/edgestackstatus"
 	"github.com/portainer/portainer/api/dataservices/endpoint"
 	"github.com/portainer/portainer/api/dataservices/endpointgroup"
 	"github.com/portainer/portainer/api/dataservices/endpointrelation"
@@ -39,6 +40,8 @@ import (
 	"github.com/segmentio/encoding/json"
 )
 
+var _ dataservices.DataStore = &Store{}
+
 // Store defines the implementation of portainer.DataStore using
 // BoltDB as the storage system.
 type Store struct {
@@ -51,6 +54,7 @@ type Store struct {
 	EdgeGroupService          *edgegroup.Service
 	EdgeJobService            *edgejob.Service
 	EdgeStackService          *edgestack.Service
+	EdgeStackStatusService    *edgestackstatus.Service
 	EndpointGroupService      *endpointgroup.Service
 	EndpointService           *endpoint.Service
 	EndpointRelationService   *endpointrelation.Service
@@ -109,6 +113,12 @@ func (store *Store) initServices() error {
 	store.EdgeStackService = edgeStackService
 	endpointRelationService.RegisterUpdateStackFunction(edgeStackService.UpdateEdgeStackFunc, edgeStackService.UpdateEdgeStackFuncTx)
 
+	edgeStackStatusService, err := edgestackstatus.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.EdgeStackStatusService = edgeStackStatusService
+
 	edgeGroupService, err := edgegroup.NewService(store.connection)
 	if err != nil {
 		return err
@@ -269,6 +279,10 @@ func (store *Store) EdgeStack() dataservices.EdgeStackService {
 	return store.EdgeStackService
 }
 
+func (store *Store) EdgeStackStatus() dataservices.EdgeStackStatusService {
+	return store.EdgeStackStatusService
+}
+
 // Environment(Endpoint) gives access to the Environment(Endpoint) data management layer
 func (store *Store) Endpoint() dataservices.EndpointService {
 	return store.EndpointService

api/datastore/services_tx.go

@@ -32,6 +32,10 @@ func (tx *StoreTx) EdgeStack() dataservices.EdgeStackService {
 	return tx.store.EdgeStackService.Tx(tx.tx)
 }
 
+func (tx *StoreTx) EdgeStackStatus() dataservices.EdgeStackStatusService {
+	return tx.store.EdgeStackStatusService.Tx(tx.tx)
+}
+
 func (tx *StoreTx) Endpoint() dataservices.EndpointService {
 	return tx.store.EndpointService.Tx(tx.tx)
 }

api/datastore/test_data/output_24_to_latest.json

@@ -8,6 +8,7 @@
     }
   ],
   "edge_stack": null,
+  "edge_stack_status": null,
   "edgegroups": null,
   "edgejobs": null,
   "endpoint_groups": [
@@ -610,7 +611,7 @@
       "RequiredPasswordLength": 12
     },
     "KubeconfigExpiry": "0",
-    "KubectlShellImage": "portainer/kubectl-shell:2.28.0",
+    "KubectlShellImage": "portainer/kubectl-shell:2.31.0",
     "LDAPSettings": {
       "AnonymousMode": true,
       "AutoCreateUsers": true,
@@ -678,14 +679,11 @@
           "Images": null,
           "Info": {
             "Architecture": "",
-            "BridgeNfIp6tables": false,
-            "BridgeNfIptables": false,
             "CDISpecDirs": null,
             "CPUSet": false,
             "CPUShares": false,
             "CgroupDriver": "",
             "ContainerdCommit": {
-              "Expected": "",
               "ID": ""
             },
             "Containers": 0,
@@ -709,7 +707,6 @@
             "IndexServerAddress": "",
             "InitBinary": "",
             "InitCommit": {
-              "Expected": "",
               "ID": ""
             },
             "Isolation": "",
@@ -738,7 +735,6 @@
             },
             "RegistryConfig": null,
             "RuncCommit": {
-              "Expected": "",
               "ID": ""
             },
             "Runtimes": null,
@@ -943,7 +939,7 @@
     }
   ],
   "version": {
-    "VERSION": "{\"SchemaVersion\":\"2.28.0\",\"MigratorCount\":0,\"Edition\":1,\"InstanceID\":\"463d5c47-0ea5-4aca-85b1-405ceefee254\"}"
+    "VERSION": "{\"SchemaVersion\":\"2.31.0\",\"MigratorCount\":1,\"Edition\":1,\"InstanceID\":\"463d5c47-0ea5-4aca-85b1-405ceefee254\"}"
   },
   "webhooks": null
 }

api/datastore/validate/validate.go

@@ -1,15 +0,0 @@
-package validate
-
-import (
-	"github.com/go-playground/validator/v10"
-	portainer "github.com/portainer/portainer/api"
-)
-
-var validate *validator.Validate
-
-func ValidateLDAPSettings(ldp *portainer.LDAPSettings) error {
-	validate = validator.New()
-	registerValidationMethods(validate)
-
-	return validate.Struct(ldp)
-}

api/datastore/validate/validate_test.go

@@ -1,61 +0,0 @@
-package validate
-
-import (
-	"testing"
-
-	portainer "github.com/portainer/portainer/api"
-)
-
-func TestValidateLDAPSettings(t *testing.T) {
-
-	tests := []struct {
-		name    string
-		ldap    portainer.LDAPSettings
-		wantErr bool
-	}{
-		{
-			name:    "Empty LDAP Settings",
-			ldap:    portainer.LDAPSettings{},
-			wantErr: true,
-		},
-		{
-			name: "With URL",
-			ldap: portainer.LDAPSettings{
-				AnonymousMode: true,
-				URL:           "192.168.0.1:323",
-			},
-			wantErr: false,
-		},
-		{
-			name: "Validate URL and URLs",
-			ldap: portainer.LDAPSettings{
-				AnonymousMode: true,
-				URL:           "192.168.0.1:323",
-			},
-			wantErr: false,
-		},
-		{
-			name: "validate client ldap",
-			ldap: portainer.LDAPSettings{
-				AnonymousMode: false,
-				ReaderDN:      "CN=LDAP API Service Account",
-				Password:      "Qu**dfUUU**",
-				URL:           "aukdc15.pgc.co:389",
-				TLSConfig: portainer.TLSConfiguration{
-					TLS:           false,
-					TLSSkipVerify: false,
-				},
-			},
-			wantErr: false,
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			err := ValidateLDAPSettings(&tt.ldap)
-			if (err == nil) == tt.wantErr {
-				t.Errorf("No error expected but got %s", err)
-			}
-		})
-	}
-}

api/datastore/validate/validationMethods.go

@@ -1,17 +0,0 @@
-package validate
-
-import (
-	"github.com/go-playground/validator/v10"
-)
-
-func registerValidationMethods(v *validator.Validate) {
-	v.RegisterValidation("validate_bool", ValidateBool)
-}
-
-/**
- * Validation methods below are being used for custom validation
- */
-func ValidateBool(fl validator.FieldLevel) bool {
-	_, ok := fl.Field().Interface().(bool)
-	return ok
-}

api/docker/client/client.go

@@ -73,19 +73,6 @@ func createLocalClient(endpoint *portainer.Endpoint) (*client.Client, error) {
 	)
 }
 
-func CreateClientFromEnv() (*client.Client, error) {
-	return client.NewClientWithOpts(
-		client.FromEnv,
-		client.WithAPIVersionNegotiation(),
-	)
-}
-
-func CreateSimpleClient() (*client.Client, error) {
-	return client.NewClientWithOpts(
-		client.WithAPIVersionNegotiation(),
-	)
-}
-
 func createTCPClient(endpoint *portainer.Endpoint, timeout *time.Duration) (*client.Client, error) {
 	httpCli, err := httpClient(endpoint, timeout)
 	if err != nil {

api/docker/images/digest.go

@@ -38,10 +38,10 @@ func NewClientWithRegistry(registryClient *RegistryClient, clientFactory *docker
 func (c *DigestClient) RemoteDigest(image Image) (digest.Digest, error) {
 	ctx, cancel := c.timeoutContext()
 	defer cancel()
+
 	// Docker references with both a tag and digest are currently not supported
 	if image.Tag != "" && image.Digest != "" {
-		err := image.trimDigest()
-		if err != nil {
+		if err := image.TrimDigest(); err != nil {
 			return "", err
 		}
 	}
@@ -69,7 +69,7 @@ func (c *DigestClient) RemoteDigest(image Image) (digest.Digest, error) {
 	// Retrieve remote digest through HEAD request
 	rmDigest, err := docker.GetDigest(ctx, sysCtx, rmRef)
 	if err != nil {
-		// fallback to public registry for hub
+		// Fallback to public registry for hub
 		if image.HubLink != "" {
 			rmDigest, err = docker.GetDigest(ctx, c.sysCtx, rmRef)
 			if err == nil {
@@ -131,8 +131,7 @@ func ParseRepoDigests(repoDigests []string) []digest.Digest {
 func ParseRepoTags(repoTags []string) []*Image {
 	images := make([]*Image, 0)
 	for _, repoTag := range repoTags {
-		image := ParseRepoTag(repoTag)
-		if image != nil {
+		if image := ParseRepoTag(repoTag); image != nil {
 			images = append(images, image)
 		}
 	}
@@ -147,7 +146,7 @@ func ParseRepoDigest(repoDigest string) digest.Digest {
 
 	d, err := digest.Parse(strings.Split(repoDigest, "@")[1])
 	if err != nil {
-		log.Warn().Msgf("Skip invalid repo digest item: %s [error: %v]", repoDigest, err)
+		log.Warn().Err(err).Str("digest", repoDigest).Msg("skip invalid repo item")
 
 		return ""
 	}

api/docker/images/image.go

@@ -26,7 +26,7 @@ type Image struct {
 	Digest  digest.Digest
 	HubLink string
 	named   reference.Named
-	opts    ParseImageOptions
+	Opts    ParseImageOptions `json:"-"`
 }
 
 // ParseImageOptions holds image options for parsing.
@@ -43,9 +43,10 @@ func (i *Image) Name() string {
 // FullName return the real full name may include Tag or Digest of the image, Tag first.
 func (i *Image) FullName() string {
 	if i.Tag == "" {
-		return fmt.Sprintf("%s@%s", i.Name(), i.Digest)
+		return i.Name() + "@" + i.Digest.String()
 	}
-	return fmt.Sprintf("%s:%s", i.Name(), i.Tag)
+
+	return i.Name() + ":" + i.Tag
 }
 
 // String returns the string representation of an image, including Tag and Digest if existed.
@@ -66,22 +67,25 @@ func (i *Image) Reference() string {
 func (i *Image) WithDigest(digest digest.Digest) (err error) {
 	i.Digest = digest
 	i.named, err = reference.WithDigest(i.named, digest)
+
 	return err
 }
 
 func (i *Image) WithTag(tag string) (err error) {
 	i.Tag = tag
 	i.named, err = reference.WithTag(i.named, tag)
+
 	return err
 }
 
-func (i *Image) trimDigest() error {
+func (i *Image) TrimDigest() error {
 	i.Digest = ""
 	named, err := ParseImage(ParseImageOptions{Name: i.FullName()})
 	if err != nil {
 		return err
 	}
 	i.named = &named
+
 	return nil
 }
 
@@ -92,11 +96,12 @@ func ParseImage(parseOpts ParseImageOptions) (Image, error) {
 	if err != nil {
 		return Image{}, errors.Wrapf(err, "parsing image %s failed", parseOpts.Name)
 	}
+
 	// Add the latest lag if they did not provide one.
 	named = reference.TagNameOnly(named)
 
 	i := Image{
-		opts:   parseOpts,
+		Opts:   parseOpts,
 		named:  named,
 		Domain: reference.Domain(named),
 		Path:   reference.Path(named),
@@ -122,15 +127,16 @@ func ParseImage(parseOpts ParseImageOptions) (Image, error) {
 }
 
 func (i *Image) hubLink() (string, error) {
-	if i.opts.HubTpl != "" {
+	if i.Opts.HubTpl != "" {
 		var out bytes.Buffer
 		tmpl, err := template.New("tmpl").
 			Option("missingkey=error").
-			Parse(i.opts.HubTpl)
+			Parse(i.Opts.HubTpl)
 		if err != nil {
 			return "", err
 		}
 		err = tmpl.Execute(&out, i)
+
 		return out.String(), err
 	}
 
@@ -142,6 +148,7 @@ func (i *Image) hubLink() (string, error) {
 			prefix = "_"
 			path = strings.Replace(i.Path, "library/", "", 1)
 		}
+
 		return "https://hub.docker.com/" + prefix + "/" + path, nil
 	case "docker.bintray.io", "jfrog-docker-reg2.bintray.io":
 		return "https://bintray.com/jfrog/reg2/" + strings.ReplaceAll(i.Path, "/", "%3A"), nil

api/docker/images/image_test.go

@@ -16,7 +16,7 @@ func TestImageParser(t *testing.T) {
 		})
 		is.NoError(err, "")
 		is.Equal("docker.io/portainer/portainer-ee:latest", image.FullName())
-		is.Equal("portainer/portainer-ee", image.opts.Name)
+		is.Equal("portainer/portainer-ee", image.Opts.Name)
 		is.Equal("latest", image.Tag)
 		is.Equal("portainer/portainer-ee", image.Path)
 		is.Equal("docker.io", image.Domain)
@@ -32,7 +32,7 @@ func TestImageParser(t *testing.T) {
 		})
 		is.NoError(err, "")
 		is.Equal("gcr.io/k8s-minikube/kicbase@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2", image.FullName())
-		is.Equal("gcr.io/k8s-minikube/kicbase@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2", image.opts.Name)
+		is.Equal("gcr.io/k8s-minikube/kicbase@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2", image.Opts.Name)
 		is.Equal("", image.Tag)
 		is.Equal("k8s-minikube/kicbase", image.Path)
 		is.Equal("gcr.io", image.Domain)
@@ -49,7 +49,7 @@ func TestImageParser(t *testing.T) {
 		})
 		is.NoError(err, "")
 		is.Equal("gcr.io/k8s-minikube/kicbase:v0.0.30", image.FullName())
-		is.Equal("gcr.io/k8s-minikube/kicbase:v0.0.30@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2", image.opts.Name)
+		is.Equal("gcr.io/k8s-minikube/kicbase:v0.0.30@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2", image.Opts.Name)
 		is.Equal("v0.0.30", image.Tag)
 		is.Equal("k8s-minikube/kicbase", image.Path)
 		is.Equal("gcr.io", image.Domain)
@@ -71,7 +71,7 @@ func TestUpdateParsedImage(t *testing.T) {
 		is.NoError(err, "")
 		_ = image.WithTag("v0.0.31")
 		is.Equal("gcr.io/k8s-minikube/kicbase:v0.0.31", image.FullName())
-		is.Equal("gcr.io/k8s-minikube/kicbase:v0.0.30@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2", image.opts.Name)
+		is.Equal("gcr.io/k8s-minikube/kicbase:v0.0.30@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2", image.Opts.Name)
 		is.Equal("v0.0.31", image.Tag)
 		is.Equal("k8s-minikube/kicbase", image.Path)
 		is.Equal("gcr.io", image.Domain)
@@ -89,7 +89,7 @@ func TestUpdateParsedImage(t *testing.T) {
 		is.NoError(err, "")
 		_ = image.WithDigest("sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b3")
 		is.Equal("gcr.io/k8s-minikube/kicbase:v0.0.30", image.FullName())
-		is.Equal("gcr.io/k8s-minikube/kicbase:v0.0.30@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2", image.opts.Name)
+		is.Equal("gcr.io/k8s-minikube/kicbase:v0.0.30@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2", image.Opts.Name)
 		is.Equal("v0.0.30", image.Tag)
 		is.Equal("k8s-minikube/kicbase", image.Path)
 		is.Equal("gcr.io", image.Domain)
@@ -105,9 +105,9 @@ func TestUpdateParsedImage(t *testing.T) {
 			Name: "gcr.io/k8s-minikube/kicbase:v0.0.30@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2",
 		})
 		is.NoError(err, "")
-		_ = image.trimDigest()
+		_ = image.TrimDigest()
 		is.Equal("gcr.io/k8s-minikube/kicbase:v0.0.30", image.FullName())
-		is.Equal("gcr.io/k8s-minikube/kicbase:v0.0.30@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2", image.opts.Name)
+		is.Equal("gcr.io/k8s-minikube/kicbase:v0.0.30@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2", image.Opts.Name)
 		is.Equal("v0.0.30", image.Tag)
 		is.Equal("k8s-minikube/kicbase", image.Path)
 		is.Equal("gcr.io", image.Domain)

api/docker/images/registry.go

@@ -29,7 +29,7 @@ func (c *RegistryClient) RegistryAuth(image Image) (string, string, error) {
 		return "", "", err
 	}
 
-	registry, err := findBestMatchRegistry(image.opts.Name, registries)
+	registry, err := findBestMatchRegistry(image.Opts.Name, registries)
 	if err != nil {
 		return "", "", err
 	}
@@ -59,7 +59,7 @@ func (c *RegistryClient) EncodedRegistryAuth(image Image) (string, error) {
 		return "", err
 	}
 
-	registry, err := findBestMatchRegistry(image.opts.Name, registries)
+	registry, err := findBestMatchRegistry(image.Opts.Name, registries)
 	if err != nil {
 		return "", err
 	}

api/exec/exectest/kubernetes_mocks.go

@@ -4,10 +4,12 @@ import (
 	portainer "github.com/portainer/portainer/api"
 )
 
-type kubernetesMockDeployer struct{}
+type kubernetesMockDeployer struct {
+	portainer.KubernetesDeployer
+}
 
 // NewKubernetesDeployer creates a mock kubernetes deployer
-func NewKubernetesDeployer() portainer.KubernetesDeployer {
+func NewKubernetesDeployer() *kubernetesMockDeployer {
 	return &kubernetesMockDeployer{}
 }
 
@@ -18,3 +20,7 @@ func (deployer *kubernetesMockDeployer) Deploy(userID portainer.UserID, endpoint
 func (deployer *kubernetesMockDeployer) Remove(userID portainer.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) {
 	return "", nil
 }
+
+func (deployer *kubernetesMockDeployer) Restart(userID portainer.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) {
+	return "", nil
+}

api/exec/kubernetes_deploy.go

@@ -1,13 +1,8 @@
 package exec
 
 import (
-	"bytes"
+	"context"
 	"fmt"
-	"os"
-	"os/exec"
-	"path"
-	"runtime"
-	"strings"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
@@ -15,13 +10,17 @@ import (
 	"github.com/portainer/portainer/api/http/proxy/factory"
 	"github.com/portainer/portainer/api/http/proxy/factory/kubernetes"
 	"github.com/portainer/portainer/api/kubernetes/cli"
+	"github.com/portainer/portainer/pkg/libkubectl"
 
 	"github.com/pkg/errors"
 )
 
+const (
+	defaultServerURL = "https://kubernetes.default.svc"
+)
+
 // KubernetesDeployer represents a service to deploy resources inside a Kubernetes environment(endpoint).
 type KubernetesDeployer struct {
-	binaryPath                  string
 	dataStore                   dataservices.DataStore
 	reverseTunnelService        portainer.ReverseTunnelService
 	signatureService            portainer.DigitalSignatureService
@@ -31,9 +30,8 @@ type KubernetesDeployer struct {
 }
 
 // NewKubernetesDeployer initializes a new KubernetesDeployer service.
-func NewKubernetesDeployer(kubernetesTokenCacheManager *kubernetes.TokenCacheManager, kubernetesClientFactory *cli.ClientFactory, datastore dataservices.DataStore, reverseTunnelService portainer.ReverseTunnelService, signatureService portainer.DigitalSignatureService, proxyManager *proxy.Manager, binaryPath string) *KubernetesDeployer {
+func NewKubernetesDeployer(kubernetesTokenCacheManager *kubernetes.TokenCacheManager, kubernetesClientFactory *cli.ClientFactory, datastore dataservices.DataStore, reverseTunnelService portainer.ReverseTunnelService, signatureService portainer.DigitalSignatureService, proxyManager *proxy.Manager) *KubernetesDeployer {
 	return &KubernetesDeployer{
-		binaryPath:                  binaryPath,
 		dataStore:                   datastore,
 		reverseTunnelService:        reverseTunnelService,
 		signatureService:            signatureService,
@@ -78,63 +76,56 @@ func (deployer *KubernetesDeployer) getToken(userID portainer.UserID, endpoint *
 }
 
 // Deploy upserts Kubernetes resources defined in manifest(s)
-func (deployer *KubernetesDeployer) Deploy(userID portainer.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) {
-	return deployer.command("apply", userID, endpoint, manifestFiles, namespace)
+func (deployer *KubernetesDeployer) Deploy(userID portainer.UserID, endpoint *portainer.Endpoint, resources []string, namespace string) (string, error) {
+	return deployer.command("apply", userID, endpoint, resources, namespace)
 }
 
 // Remove deletes Kubernetes resources defined in manifest(s)
-func (deployer *KubernetesDeployer) Remove(userID portainer.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) {
-	return deployer.command("delete", userID, endpoint, manifestFiles, namespace)
+func (deployer *KubernetesDeployer) Remove(userID portainer.UserID, endpoint *portainer.Endpoint, resources []string, namespace string) (string, error) {
+	return deployer.command("delete", userID, endpoint, resources, namespace)
 }
 
-func (deployer *KubernetesDeployer) command(operation string, userID portainer.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) {
+func (deployer *KubernetesDeployer) command(operation string, userID portainer.UserID, endpoint *portainer.Endpoint, resources []string, namespace string) (string, error) {
 	token, err := deployer.getToken(userID, endpoint, endpoint.Type == portainer.KubernetesLocalEnvironment)
 	if err != nil {
 		return "", errors.Wrap(err, "failed generating a user token")
 	}
 
-	command := path.Join(deployer.binaryPath, "kubectl")
-	if runtime.GOOS == "windows" {
-		command = path.Join(deployer.binaryPath, "kubectl.exe")
-	}
-
-	args := []string{"--token", token}
-	if namespace != "" {
-		args = append(args, "--namespace", namespace)
-	}
-
+	serverURL := defaultServerURL
 	if endpoint.Type == portainer.AgentOnKubernetesEnvironment || endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment {
 		url, proxy, err := deployer.getAgentURL(endpoint)
 		if err != nil {
 			return "", errors.WithMessage(err, "failed generating endpoint URL")
 		}
-
 		defer proxy.Close()
-		args = append(args, "--server", url)
-		args = append(args, "--insecure-skip-tls-verify")
+
+		serverURL = url
 	}
 
-	if operation == "delete" {
-		args = append(args, "--ignore-not-found=true")
-	}
-
-	args = append(args, operation)
-	for _, path := range manifestFiles {
-		args = append(args, "-f", strings.TrimSpace(path))
-	}
-
-	var stderr bytes.Buffer
-	cmd := exec.Command(command, args...)
-	cmd.Env = os.Environ()
-	cmd.Env = append(cmd.Env, "POD_NAMESPACE=default")
-	cmd.Stderr = &stderr
-
-	output, err := cmd.Output()
+	client, err := libkubectl.NewClient(&libkubectl.ClientAccess{
+		Token:     token,
+		ServerUrl: serverURL,
+	}, namespace, "", true)
 	if err != nil {
-		return "", errors.Wrapf(err, "failed to execute kubectl command: %q", stderr.String())
+		return "", errors.Wrap(err, "failed to create kubectl client")
 	}
 
-	return string(output), nil
+	operations := map[string]func(context.Context, []string) (string, error){
+		"apply":  client.Apply,
+		"delete": client.Delete,
+	}
+
+	operationFunc, ok := operations[operation]
+	if !ok {
+		return "", errors.Errorf("unsupported operation: %s", operation)
+	}
+
+	output, err := operationFunc(context.Background(), resources)
+	if err != nil {
+		return "", errors.Wrapf(err, "failed to execute kubectl %s command", operation)
+	}
+
+	return output, nil
 }
 
 func (deployer *KubernetesDeployer) getAgentURL(endpoint *portainer.Endpoint) (string, *factory.ProxyServer, error) {

api/exec/kubernetes_deploy_test.go

@@ -0,0 +1,173 @@
+package exec
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+type mockKubectlClient struct {
+	applyFunc          func(ctx context.Context, files []string) error
+	deleteFunc         func(ctx context.Context, files []string) error
+	rolloutRestartFunc func(ctx context.Context, resources []string) error
+}
+
+func (m *mockKubectlClient) Apply(ctx context.Context, files []string) error {
+	if m.applyFunc != nil {
+		return m.applyFunc(ctx, files)
+	}
+	return nil
+}
+
+func (m *mockKubectlClient) Delete(ctx context.Context, files []string) error {
+	if m.deleteFunc != nil {
+		return m.deleteFunc(ctx, files)
+	}
+	return nil
+}
+
+func (m *mockKubectlClient) RolloutRestart(ctx context.Context, resources []string) error {
+	if m.rolloutRestartFunc != nil {
+		return m.rolloutRestartFunc(ctx, resources)
+	}
+	return nil
+}
+
+func testExecuteKubectlOperation(client *mockKubectlClient, operation string, manifestFiles []string) error {
+	operations := map[string]func(context.Context, []string) error{
+		"apply":           client.Apply,
+		"delete":          client.Delete,
+		"rollout-restart": client.RolloutRestart,
+	}
+
+	operationFunc, ok := operations[operation]
+	if !ok {
+		return fmt.Errorf("unsupported operation: %s", operation)
+	}
+
+	if err := operationFunc(context.Background(), manifestFiles); err != nil {
+		return fmt.Errorf("failed to execute kubectl %s command: %w", operation, err)
+	}
+
+	return nil
+}
+
+func TestExecuteKubectlOperation_Apply_Success(t *testing.T) {
+	called := false
+	mockClient := &mockKubectlClient{
+		applyFunc: func(ctx context.Context, files []string) error {
+			called = true
+			assert.Equal(t, []string{"manifest1.yaml", "manifest2.yaml"}, files)
+			return nil
+		},
+	}
+
+	manifests := []string{"manifest1.yaml", "manifest2.yaml"}
+	err := testExecuteKubectlOperation(mockClient, "apply", manifests)
+
+	assert.NoError(t, err)
+	assert.True(t, called)
+}
+
+func TestExecuteKubectlOperation_Apply_Error(t *testing.T) {
+	expectedErr := errors.New("kubectl apply failed")
+	called := false
+	mockClient := &mockKubectlClient{
+		applyFunc: func(ctx context.Context, files []string) error {
+			called = true
+			assert.Equal(t, []string{"error.yaml"}, files)
+			return expectedErr
+		},
+	}
+
+	manifests := []string{"error.yaml"}
+	err := testExecuteKubectlOperation(mockClient, "apply", manifests)
+
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), expectedErr.Error())
+	assert.True(t, called)
+}
+
+func TestExecuteKubectlOperation_Delete_Success(t *testing.T) {
+	called := false
+	mockClient := &mockKubectlClient{
+		deleteFunc: func(ctx context.Context, files []string) error {
+			called = true
+			assert.Equal(t, []string{"manifest1.yaml"}, files)
+			return nil
+		},
+	}
+
+	manifests := []string{"manifest1.yaml"}
+	err := testExecuteKubectlOperation(mockClient, "delete", manifests)
+
+	assert.NoError(t, err)
+	assert.True(t, called)
+}
+
+func TestExecuteKubectlOperation_Delete_Error(t *testing.T) {
+	expectedErr := errors.New("kubectl delete failed")
+	called := false
+	mockClient := &mockKubectlClient{
+		deleteFunc: func(ctx context.Context, files []string) error {
+			called = true
+			assert.Equal(t, []string{"error.yaml"}, files)
+			return expectedErr
+		},
+	}
+
+	manifests := []string{"error.yaml"}
+	err := testExecuteKubectlOperation(mockClient, "delete", manifests)
+
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), expectedErr.Error())
+	assert.True(t, called)
+}
+
+func TestExecuteKubectlOperation_RolloutRestart_Success(t *testing.T) {
+	called := false
+	mockClient := &mockKubectlClient{
+		rolloutRestartFunc: func(ctx context.Context, resources []string) error {
+			called = true
+			assert.Equal(t, []string{"deployment/nginx"}, resources)
+			return nil
+		},
+	}
+
+	resources := []string{"deployment/nginx"}
+	err := testExecuteKubectlOperation(mockClient, "rollout-restart", resources)
+
+	assert.NoError(t, err)
+	assert.True(t, called)
+}
+
+func TestExecuteKubectlOperation_RolloutRestart_Error(t *testing.T) {
+	expectedErr := errors.New("kubectl rollout restart failed")
+	called := false
+	mockClient := &mockKubectlClient{
+		rolloutRestartFunc: func(ctx context.Context, resources []string) error {
+			called = true
+			assert.Equal(t, []string{"deployment/error"}, resources)
+			return expectedErr
+		},
+	}
+
+	resources := []string{"deployment/error"}
+	err := testExecuteKubectlOperation(mockClient, "rollout-restart", resources)
+
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), expectedErr.Error())
+	assert.True(t, called)
+}
+
+func TestExecuteKubectlOperation_UnsupportedOperation(t *testing.T) {
+	mockClient := &mockKubectlClient{}
+
+	err := testExecuteKubectlOperation(mockClient, "unsupported", []string{})
+
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "unsupported operation")
+}

api/filesystem/copy.go

@@ -68,7 +68,7 @@ func copyFile(src, dst string) error {
 	defer from.Close()
 
 	// has to include 'execute' bit, otherwise fails. MkdirAll follows `mkdir -m` restrictions
-	if err := os.MkdirAll(filepath.Dir(dst), 0744); err != nil {
+	if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
 		return err
 	}
 	to, err := os.Create(dst)

api/git/update/validate.go

@@ -3,9 +3,9 @@ package update
 import (
 	"time"
 
-	"github.com/asaskevich/govalidator"
 	portainer "github.com/portainer/portainer/api"
 	httperrors "github.com/portainer/portainer/api/http/errors"
+	"github.com/portainer/portainer/pkg/validate"
 )
 
 func ValidateAutoUpdateSettings(autoUpdate *portainer.AutoUpdateSettings) error {
@@ -17,7 +17,7 @@ func ValidateAutoUpdateSettings(autoUpdate *portainer.AutoUpdateSettings) error
 		return httperrors.NewInvalidPayloadError("Webhook or Interval must be provided")
 	}
 
-	if autoUpdate.Webhook != "" && !govalidator.IsUUID(autoUpdate.Webhook) {
+	if autoUpdate.Webhook != "" && !validate.IsUUID(autoUpdate.Webhook) {
 		return httperrors.NewInvalidPayloadError("invalid Webhook format")
 	}
 

api/git/validate.go

@@ -1,19 +1,17 @@
 package git
 
 import (
-	"github.com/asaskevich/govalidator"
-
 	gittypes "github.com/portainer/portainer/api/git/types"
 	httperrors "github.com/portainer/portainer/api/http/errors"
+	"github.com/portainer/portainer/pkg/validate"
 )
 
 func ValidateRepoConfig(repoConfig *gittypes.RepoConfig) error {
-	if len(repoConfig.URL) == 0 || !govalidator.IsURL(repoConfig.URL) {
+	if len(repoConfig.URL) == 0 || !validate.IsURL(repoConfig.URL) {
 		return httperrors.NewInvalidPayloadError("Invalid repository URL. Must correspond to a valid URL format")
 	}
 
 	return ValidateRepoAuthentication(repoConfig.Authentication)
-
 }
 
 func ValidateRepoAuthentication(auth *gittypes.GitAuthentication) error {

api/http/handler/customtemplates/customtemplate_create.go

@@ -16,8 +16,8 @@ import (
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
 	"github.com/portainer/portainer/pkg/libhttp/response"
+	"github.com/portainer/portainer/pkg/validate"
 
-	"github.com/asaskevich/govalidator"
 	"github.com/rs/zerolog/log"
 	"github.com/segmentio/encoding/json"
 )
@@ -228,7 +228,7 @@ func (payload *customTemplateFromGitRepositoryPayload) Validate(r *http.Request)
 	if len(payload.Description) == 0 {
 		return errors.New("Invalid custom template description")
 	}
-	if len(payload.RepositoryURL) == 0 || !govalidator.IsURL(payload.RepositoryURL) {
+	if len(payload.RepositoryURL) == 0 || !validate.IsURL(payload.RepositoryURL) {
 		return errors.New("Invalid repository URL. Must correspond to a valid URL format")
 	}
 	if payload.RepositoryAuthentication && (len(payload.RepositoryUsername) == 0 || len(payload.RepositoryPassword) == 0) {

api/http/handler/customtemplates/customtemplate_update.go

@@ -15,8 +15,7 @@ import (
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
 	"github.com/portainer/portainer/pkg/libhttp/response"
-
-	"github.com/asaskevich/govalidator"
+	"github.com/portainer/portainer/pkg/validate"
 )
 
 type customTemplateUpdatePayload struct {
@@ -170,7 +169,7 @@ func (handler *Handler) customTemplateUpdate(w http.ResponseWriter, r *http.Requ
 	customTemplate.EdgeTemplate = payload.EdgeTemplate
 
 	if payload.RepositoryURL != "" {
-		if !govalidator.IsURL(payload.RepositoryURL) {
+		if !validate.IsURL(payload.RepositoryURL) {
 			return httperror.BadRequest("Invalid repository URL. Must correspond to a valid URL format", err)
 		}
 

api/http/handler/docker/dashboard.go

@@ -6,6 +6,7 @@ import (
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/image"
+	"github.com/docker/docker/api/types/network"
 	"github.com/docker/docker/api/types/swarm"
 	"github.com/docker/docker/api/types/volume"
 	portainer "github.com/portainer/portainer/api"
@@ -116,12 +117,12 @@ func (h *Handler) dashboard(w http.ResponseWriter, r *http.Request) *httperror.H
 			return err
 		}
 
-		networks, err := cli.NetworkList(r.Context(), types.NetworkListOptions{})
+		networks, err := cli.NetworkList(r.Context(), network.ListOptions{})
 		if err != nil {
 			return httperror.InternalServerError("Unable to retrieve Docker networks", err)
 		}
 
-		networks, err = utils.FilterByResourceControl(tx, networks, portainer.NetworkResourceControl, context, func(c types.NetworkResource) string {
+		networks, err = utils.FilterByResourceControl(tx, networks, portainer.NetworkResourceControl, context, func(c network.Summary) string {
 			return c.Name
 		})
 		if err != nil {

api/http/handler/edgegroups/edgegroup_update.go

@@ -24,10 +24,6 @@ type edgeGroupUpdatePayload struct {
 }
 
 func (payload *edgeGroupUpdatePayload) Validate(r *http.Request) error {
-	if len(payload.Name) == 0 {
-		return errors.New("invalid Edge group name")
-	}
-
 	if payload.Dynamic && len(payload.TagIDs) == 0 {
 		return errors.New("tagIDs is mandatory for a dynamic Edge group")
 	}
@@ -35,7 +31,7 @@ func (payload *edgeGroupUpdatePayload) Validate(r *http.Request) error {
 	return nil
 }
 
-// @id EgeGroupUpdate
+// @id EdgeGroupUpdate
 // @summary Updates an EdgeGroup
 // @description **Access policy**: administrator
 // @tags edge_groups

api/http/handler/edgejobs/edgejob_create.go

@@ -15,8 +15,7 @@ import (
 	"github.com/portainer/portainer/api/internal/endpointutils"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
-
-	"github.com/asaskevich/govalidator"
+	"github.com/portainer/portainer/pkg/validate"
 )
 
 type edgeJobBasePayload struct {
@@ -53,7 +52,7 @@ func (payload *edgeJobCreateFromFileContentPayload) Validate(r *http.Request) er
 		return errors.New("invalid Edge job name")
 	}
 
-	if !govalidator.Matches(payload.Name, `^[a-zA-Z0-9][a-zA-Z0-9_.-]*$`) {
+	if !validate.Matches(payload.Name, `^[a-zA-Z0-9][a-zA-Z0-9_.-]*$`) {
 		return errors.New("invalid Edge job name format. Allowed characters are: [a-zA-Z0-9_.-]")
 	}
 
@@ -136,7 +135,7 @@ func (payload *edgeJobCreateFromFilePayload) Validate(r *http.Request) error {
 		return errors.New("invalid Edge job name")
 	}
 
-	if !govalidator.Matches(name, `^[a-zA-Z0-9][a-zA-Z0-9_.-]+$`) {
+	if !validate.Matches(name, `^[a-zA-Z0-9][a-zA-Z0-9_.-]+$`) {
 		return errors.New("invalid Edge job name format. Allowed characters are: [a-zA-Z0-9_.-]")
 	}
 	payload.Name = name

api/http/handler/edgejobs/edgejob_update.go

@@ -14,8 +14,7 @@ import (
 	"github.com/portainer/portainer/api/internal/endpointutils"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
-
-	"github.com/asaskevich/govalidator"
+	"github.com/portainer/portainer/pkg/validate"
 )
 
 type edgeJobUpdatePayload struct {
@@ -28,7 +27,7 @@ type edgeJobUpdatePayload struct {
 }
 
 func (payload *edgeJobUpdatePayload) Validate(r *http.Request) error {
-	if payload.Name != nil && !govalidator.Matches(*payload.Name, `^[a-zA-Z0-9][a-zA-Z0-9_.-]+$`) {
+	if payload.Name != nil && !validate.Matches(*payload.Name, `^[a-zA-Z0-9][a-zA-Z0-9_.-]+$`) {
 		return errors.New("invalid Edge job name format. Allowed characters are: [a-zA-Z0-9_.-]")
 	}
 

api/http/handler/edgestacks/edgestack_create_file.go

@@ -101,8 +101,7 @@ func (payload *edgeStackFromFileUploadPayload) Validate(r *http.Request) error {
 // @router /edge_stacks/create/file [post]
 func (handler *Handler) createEdgeStackFromFileUpload(r *http.Request, tx dataservices.DataStoreTx, dryrun bool) (*portainer.EdgeStack, error) {
 	payload := &edgeStackFromFileUploadPayload{}
-	err := payload.Validate(r)
-	if err != nil {
+	if err := payload.Validate(r); err != nil {
 		return nil, err
 	}
 

api/http/handler/edgestacks/edgestack_create_git.go

@@ -11,8 +11,8 @@ import (
 	httperrors "github.com/portainer/portainer/api/http/errors"
 	"github.com/portainer/portainer/pkg/edge"
 	"github.com/portainer/portainer/pkg/libhttp/request"
+	"github.com/portainer/portainer/pkg/validate"
 
-	"github.com/asaskevich/govalidator"
 	"github.com/pkg/errors"
 )
 
@@ -59,7 +59,7 @@ func (payload *edgeStackFromGitRepositoryPayload) Validate(r *http.Request) erro
 		return httperrors.NewInvalidPayloadError("Invalid stack name. Stack name must only consist of lowercase alpha characters, numbers, hyphens, or underscores as well as start with a lowercase character or number")
 	}
 
-	if len(payload.RepositoryURL) == 0 || !govalidator.IsURL(payload.RepositoryURL) {
+	if len(payload.RepositoryURL) == 0 || !validate.IsURL(payload.RepositoryURL) {
 		return httperrors.NewInvalidPayloadError("Invalid repository URL. Must correspond to a valid URL format")
 	}
 
@@ -103,8 +103,7 @@ func (payload *edgeStackFromGitRepositoryPayload) Validate(r *http.Request) erro
 // @router /edge_stacks/create/repository [post]
 func (handler *Handler) createEdgeStackFromGitRepository(r *http.Request, tx dataservices.DataStoreTx, dryrun bool, userID portainer.UserID) (*portainer.EdgeStack, error) {
 	var payload edgeStackFromGitRepositoryPayload
-	err := request.DecodeAndValidateJSONPayload(r, &payload)
-	if err != nil {
+	if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil {
 		return nil, err
 	}
 
@@ -137,11 +136,9 @@ func (handler *Handler) createEdgeStackFromGitRepository(r *http.Request, tx dat
 }
 
 func (handler *Handler) storeManifestFromGitRepository(tx dataservices.DataStoreTx, stackFolder string, relatedEndpointIds []portainer.EndpointID, deploymentType portainer.EdgeStackDeploymentType, currentUserID portainer.UserID, repositoryConfig gittypes.RepoConfig) (composePath, manifestPath, projectPath string, err error) {
-	hasWrongType, err := hasWrongEnvironmentType(tx.Endpoint(), relatedEndpointIds, deploymentType)
-	if err != nil {
+	if hasWrongType, err := hasWrongEnvironmentType(tx.Endpoint(), relatedEndpointIds, deploymentType); err != nil {
 		return "", "", "", fmt.Errorf("unable to check for existence of non fitting environments: %w", err)
-	}
-	if hasWrongType {
+	} else if hasWrongType {
 		return "", "", "", errors.New("edge stack with config do not match the environment type")
 	}
 
@@ -153,8 +150,7 @@ func (handler *Handler) storeManifestFromGitRepository(tx dataservices.DataStore
 		repositoryPassword = repositoryConfig.Authentication.Password
 	}
 
-	err = handler.GitService.CloneRepository(projectPath, repositoryConfig.URL, repositoryConfig.ReferenceName, repositoryUsername, repositoryPassword, repositoryConfig.TLSSkipVerify)
-	if err != nil {
+	if err := handler.GitService.CloneRepository(projectPath, repositoryConfig.URL, repositoryConfig.ReferenceName, repositoryUsername, repositoryPassword, repositoryConfig.TLSSkipVerify); err != nil {
 		return "", "", "", err
 	}
 

api/http/handler/edgestacks/edgestack_create_string.go

@@ -76,8 +76,7 @@ func (payload *edgeStackFromStringPayload) Validate(r *http.Request) error {
 // @router /edge_stacks/create/string [post]
 func (handler *Handler) createEdgeStackFromFileContent(r *http.Request, tx dataservices.DataStoreTx, dryrun bool) (*portainer.EdgeStack, error) {
 	var payload edgeStackFromStringPayload
-	err := request.DecodeAndValidateJSONPayload(r, &payload)
-	if err != nil {
+	if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil {
 		return nil, err
 	}
 
@@ -96,11 +95,9 @@ func (handler *Handler) createEdgeStackFromFileContent(r *http.Request, tx datas
 }
 
 func (handler *Handler) storeFileContent(tx dataservices.DataStoreTx, stackFolder string, deploymentType portainer.EdgeStackDeploymentType, relatedEndpointIds []portainer.EndpointID, fileContent []byte) (composePath, manifestPath, projectPath string, err error) {
-	hasWrongType, err := hasWrongEnvironmentType(tx.Endpoint(), relatedEndpointIds, deploymentType)
-	if err != nil {
+	if hasWrongType, err := hasWrongEnvironmentType(tx.Endpoint(), relatedEndpointIds, deploymentType); err != nil {
 		return "", "", "", fmt.Errorf("unable to check for existence of non fitting environments: %w", err)
-	}
-	if hasWrongType {
+	} else if hasWrongType {
 		return "", "", "", errors.New("edge stack with config do not match the environment type")
 	}
 
@@ -124,7 +121,6 @@ func (handler *Handler) storeFileContent(tx dataservices.DataStoreTx, stackFolde
 		}
 
 		return "", manifestPath, projectPath, nil
-
 	}
 
 	errMessage := fmt.Sprintf("invalid deployment type: %d", deploymentType)

api/http/handler/edgestacks/edgestack_create_test.go

@@ -8,6 +8,7 @@ import (
 	"testing"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/stretchr/testify/require"
 
 	"github.com/segmentio/encoding/json"
 )
@@ -28,9 +29,7 @@ func TestCreateAndInspect(t *testing.T) {
 	}
 
 	err := handler.DataStore.EdgeGroup().Create(&edgeGroup)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 
 	endpointRelation := portainer.EndpointRelation{
 		EndpointID: endpoint.ID,
@@ -38,9 +37,7 @@ func TestCreateAndInspect(t *testing.T) {
 	}
 
 	err = handler.DataStore.EndpointRelation().Create(&endpointRelation)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 
 	payload := edgeStackFromStringPayload{
 		Name:             "test-stack",
@@ -50,16 +47,14 @@ func TestCreateAndInspect(t *testing.T) {
 	}
 
 	jsonPayload, err := json.Marshal(payload)
-	if err != nil {
-		t.Fatal("JSON marshal error:", err)
-	}
+	require.NoError(t, err)
+
 	r := bytes.NewBuffer(jsonPayload)
 
 	// Create EdgeStack
 	req, err := http.NewRequest(http.MethodPost, "/edge_stacks/create/string", r)
-	if err != nil {
-		t.Fatal("request error:", err)
-	}
+	require.NoError(t, err)
+
 	req.Header.Add("x-api-key", rawAPIKey)
 	rec := httptest.NewRecorder()
 	handler.ServeHTTP(rec, req)
@@ -70,15 +65,11 @@ func TestCreateAndInspect(t *testing.T) {
 
 	data := portainer.EdgeStack{}
 	err = json.NewDecoder(rec.Body).Decode(&data)
-	if err != nil {
-		t.Fatal("error decoding response:", err)
-	}
+	require.NoError(t, err)
 
 	// Inspect
 	req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("/edge_stacks/%d", data.ID), nil)
-	if err != nil {
-		t.Fatal("request error:", err)
-	}
+	require.NoError(t, err)
 
 	req.Header.Add("x-api-key", rawAPIKey)
 	rec = httptest.NewRecorder()
@@ -90,9 +81,7 @@ func TestCreateAndInspect(t *testing.T) {
 
 	data = portainer.EdgeStack{}
 	err = json.NewDecoder(rec.Body).Decode(&data)
-	if err != nil {
-		t.Fatal("error decoding response:", err)
-	}
+	require.NoError(t, err)
 
 	if payload.Name != data.Name {
 		t.Fatalf("expected EdgeStack Name %s, found %s", payload.Name, data.Name)

api/http/handler/edgestacks/edgestack_delete.go

@@ -30,10 +30,9 @@ func (handler *Handler) edgeStackDelete(w http.ResponseWriter, r *http.Request)
 		return httperror.BadRequest("Invalid edge stack identifier route variable", err)
 	}
 
-	err = handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
+	if err := handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
 		return handler.deleteEdgeStack(tx, portainer.EdgeStackID(edgeStackID))
-	})
-	if err != nil {
+	}); err != nil {
 		var httpErr *httperror.HandlerError
 		if errors.As(err, &httpErr) {
 			return httpErr

api/http/handler/edgestacks/edgestack_delete_test.go

@@ -8,9 +8,10 @@ import (
 	"testing"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/stretchr/testify/assert"
 
 	"github.com/segmentio/encoding/json"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 // Delete
@@ -23,9 +24,7 @@ func TestDeleteAndInspect(t *testing.T) {
 
 	// Inspect
 	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("/edge_stacks/%d", edgeStack.ID), nil)
-	if err != nil {
-		t.Fatal("request error:", err)
-	}
+	require.NoError(t, err)
 
 	req.Header.Add("x-api-key", rawAPIKey)
 	rec := httptest.NewRecorder()
@@ -37,9 +36,7 @@ func TestDeleteAndInspect(t *testing.T) {
 
 	data := portainer.EdgeStack{}
 	err = json.NewDecoder(rec.Body).Decode(&data)
-	if err != nil {
-		t.Fatal("error decoding response:", err)
-	}
+	require.NoError(t, err)
 
 	if data.ID != edgeStack.ID {
 		t.Fatalf("expected EdgeStackID %d, found %d", int(edgeStack.ID), data.ID)
@@ -47,9 +44,7 @@ func TestDeleteAndInspect(t *testing.T) {
 
 	// Delete
 	req, err = http.NewRequest(http.MethodDelete, fmt.Sprintf("/edge_stacks/%d", edgeStack.ID), nil)
-	if err != nil {
-		t.Fatal("request error:", err)
-	}
+	require.NoError(t, err)
 
 	req.Header.Add("x-api-key", rawAPIKey)
 	rec = httptest.NewRecorder()
@@ -61,9 +56,7 @@ func TestDeleteAndInspect(t *testing.T) {
 
 	// Inspect
 	req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("/edge_stacks/%d", edgeStack.ID), nil)
-	if err != nil {
-		t.Fatal("request error:", err)
-	}
+	require.NoError(t, err)
 
 	req.Header.Add("x-api-key", rawAPIKey)
 	rec = httptest.NewRecorder()
@@ -117,15 +110,12 @@ func TestDeleteEdgeStack_RemoveProjectFolder(t *testing.T) {
 	}
 
 	var buf bytes.Buffer
-	if err := json.NewEncoder(&buf).Encode(payload); err != nil {
-		t.Fatal("error encoding payload:", err)
-	}
+	err := json.NewEncoder(&buf).Encode(payload)
+	require.NoError(t, err)
 
 	// Create
 	req, err := http.NewRequest(http.MethodPost, "/edge_stacks/create/string", &buf)
-	if err != nil {
-		t.Fatal("request error:", err)
-	}
+	require.NoError(t, err)
 
 	req.Header.Add("x-api-key", rawAPIKey)
 	rec := httptest.NewRecorder()
@@ -138,9 +128,8 @@ func TestDeleteEdgeStack_RemoveProjectFolder(t *testing.T) {
 	assert.DirExists(t, handler.FileService.GetEdgeStackProjectPath("1"))
 
 	// Delete
-	if req, err = http.NewRequest(http.MethodDelete, "/edge_stacks/1", nil); err != nil {
-		t.Fatal("request error:", err)
-	}
+	req, err = http.NewRequest(http.MethodDelete, "/edge_stacks/1", nil)
+	require.NoError(t, err)
 
 	req.Header.Add("x-api-key", rawAPIKey)
 	rec = httptest.NewRecorder()

api/http/handler/edgestacks/edgestack_inspect.go

@@ -4,6 +4,7 @@ import (
 	"net/http"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
 	"github.com/portainer/portainer/pkg/libhttp/response"
@@ -33,5 +34,35 @@ func (handler *Handler) edgeStackInspect(w http.ResponseWriter, r *http.Request)
 		return handlerDBErr(err, "Unable to find an edge stack with the specified identifier inside the database")
 	}
 
+	if err := fillEdgeStackStatus(handler.DataStore, edgeStack); err != nil {
+		return handlerDBErr(err, "Unable to retrieve edge stack status from the database")
+	}
+
 	return response.JSON(w, edgeStack)
 }
+
+func fillEdgeStackStatus(tx dataservices.DataStoreTx, edgeStack *portainer.EdgeStack) error {
+	status, err := tx.EdgeStackStatus().ReadAll(edgeStack.ID)
+	if err != nil {
+		return err
+	}
+
+	edgeStack.Status = make(map[portainer.EndpointID]portainer.EdgeStackStatus, len(status))
+
+	emptyStatus := make([]portainer.EdgeStackDeploymentStatus, 0)
+
+	for _, s := range status {
+		if s.Status == nil {
+			s.Status = emptyStatus
+		}
+
+		edgeStack.Status[s.EndpointID] = portainer.EdgeStackStatus{
+			Status:           s.Status,
+			EndpointID:       s.EndpointID,
+			DeploymentInfo:   s.DeploymentInfo,
+			ReadyRePullImage: s.ReadyRePullImage,
+		}
+	}
+
+	return nil
+}

api/http/handler/edgestacks/edgestack_list.go

@@ -25,5 +25,11 @@ func (handler *Handler) edgeStackList(w http.ResponseWriter, r *http.Request) *h
 		return httperror.InternalServerError("Unable to retrieve edge stacks from the database", err)
 	}
 
+	for i := range edgeStacks {
+		if err := fillEdgeStackStatus(handler.DataStore, &edgeStacks[i]); err != nil {
+			return handlerDBErr(err, "Unable to retrieve edge stack status from the database")
+		}
+	}
+
 	return response.JSON(w, edgeStacks)
 }

api/http/handler/edgestacks/edgestack_status_update.go

@@ -9,11 +9,10 @@ import (
 	"time"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
 	"github.com/portainer/portainer/pkg/libhttp/response"
-
-	"github.com/rs/zerolog/log"
 )
 
 type updateStatusPayload struct {
@@ -78,12 +77,25 @@ func (handler *Handler) edgeStackStatusUpdate(w http.ResponseWriter, r *http.Req
 		return httperror.Forbidden("Permission denied to access environment", fmt.Errorf("unauthorized edge endpoint operation: %w. Environment name: %s", err, endpoint.Name))
 	}
 
-	updateFn := func(stack *portainer.EdgeStack) (*portainer.EdgeStack, error) {
-		return handler.updateEdgeStackStatus(stack, stack.ID, payload)
-	}
+	var stack *portainer.EdgeStack
 
-	stack, err := handler.stackCoordinator.UpdateStatus(r, portainer.EdgeStackID(stackID), updateFn)
-	if err != nil {
+	if err := handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
+		var err error
+		stack, err = tx.EdgeStack().EdgeStack(portainer.EdgeStackID(stackID))
+		if err != nil {
+			if dataservices.IsErrObjectNotFound(err) {
+				return nil
+			}
+
+			return httperror.InternalServerError("Unable to retrieve Edge stack from the database", err)
+		}
+
+		if err := handler.updateEdgeStackStatus(tx, stack, stack.ID, payload); err != nil {
+			return httperror.InternalServerError("Unable to update Edge stack status", err)
+		}
+
+		return nil
+	}); err != nil {
 		var httpErr *httperror.HandlerError
 		if errors.As(err, &httpErr) {
 			return httpErr
@@ -96,43 +108,34 @@ func (handler *Handler) edgeStackStatusUpdate(w http.ResponseWriter, r *http.Req
 		return nil
 	}
 
+	if err := fillEdgeStackStatus(handler.DataStore, stack); err != nil {
+		return handlerDBErr(err, "Unable to retrieve edge stack status from the database")
+	}
+
 	return response.JSON(w, stack)
 }
 
-func (handler *Handler) updateEdgeStackStatus(stack *portainer.EdgeStack, stackID portainer.EdgeStackID, payload updateStatusPayload) (*portainer.EdgeStack, error) {
+func (handler *Handler) updateEdgeStackStatus(tx dataservices.DataStoreTx, stack *portainer.EdgeStack, stackID portainer.EdgeStackID, payload updateStatusPayload) error {
 	if payload.Version > 0 && payload.Version < stack.Version {
-		return stack, nil
+		return nil
 	}
 
 	status := *payload.Status
 
-	log.Debug().
-		Int("stackID", int(stackID)).
-		Int("status", int(status)).
-		Msg("Updating stack status")
-
 	deploymentStatus := portainer.EdgeStackDeploymentStatus{
 		Type:  status,
 		Error: payload.Error,
 		Time:  payload.Time,
 	}
 
-	updateEnvStatus(payload.EndpointID, stack, deploymentStatus)
-
-	return stack, nil
-}
-
-func updateEnvStatus(environmentId portainer.EndpointID, stack *portainer.EdgeStack, deploymentStatus portainer.EdgeStackDeploymentStatus) {
 	if deploymentStatus.Type == portainer.EdgeStackStatusRemoved {
-		delete(stack.Status, environmentId)
-
-		return
+		return tx.EdgeStackStatus().Delete(stackID, payload.EndpointID)
 	}
 
-	environmentStatus, ok := stack.Status[environmentId]
-	if !ok {
-		environmentStatus = portainer.EdgeStackStatus{
-			EndpointID: environmentId,
+	environmentStatus, err := tx.EdgeStackStatus().Read(stackID, payload.EndpointID)
+	if err != nil {
+		environmentStatus = &portainer.EdgeStackStatusForEnv{
+			EndpointID: payload.EndpointID,
 			Status:     []portainer.EdgeStackDeploymentStatus{},
 		}
 	}
@@ -143,5 +146,5 @@ func updateEnvStatus(environmentId portainer.EndpointID, stack *portainer.EdgeSt
 		environmentStatus.Status = append(environmentStatus.Status, deploymentStatus)
 	}
 
-	stack.Status[environmentId] = environmentStatus
+	return tx.EdgeStackStatus().Update(stackID, payload.EndpointID, environmentStatus)
 }

api/http/handler/edgestacks/edgestack_status_update_coordinator.go

@@ -1,155 +0,0 @@
-package edgestacks
-
-import (
-	"errors"
-	"fmt"
-	"net/http"
-
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
-
-	"github.com/rs/zerolog/log"
-)
-
-type statusRequest struct {
-	respCh   chan statusResponse
-	stackID  portainer.EdgeStackID
-	updateFn statusUpdateFn
-}
-
-type statusResponse struct {
-	Stack *portainer.EdgeStack
-	Error error
-}
-
-type statusUpdateFn func(*portainer.EdgeStack) (*portainer.EdgeStack, error)
-
-type EdgeStackStatusUpdateCoordinator struct {
-	updateCh  chan statusRequest
-	dataStore dataservices.DataStore
-}
-
-var errAnotherStackUpdateInProgress = errors.New("another stack update is in progress")
-
-func NewEdgeStackStatusUpdateCoordinator(dataStore dataservices.DataStore) *EdgeStackStatusUpdateCoordinator {
-	return &EdgeStackStatusUpdateCoordinator{
-		updateCh:  make(chan statusRequest),
-		dataStore: dataStore,
-	}
-}
-
-func (c *EdgeStackStatusUpdateCoordinator) Start() {
-	for {
-		c.loop()
-	}
-}
-
-func (c *EdgeStackStatusUpdateCoordinator) loop() {
-	u := <-c.updateCh
-
-	respChs := []chan statusResponse{u.respCh}
-
-	var stack *portainer.EdgeStack
-
-	err := c.dataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
-		// 1. Load the edge stack
-		var err error
-
-		stack, err = loadEdgeStack(tx, u.stackID)
-		if err != nil {
-			return err
-		}
-
-		// Return early when the agent tries to update the status on a deleted stack
-		if stack == nil {
-			return nil
-		}
-
-		// 2. Mutate the edge stack opportunistically until there are no more pending updates
-		for {
-			stack, err = u.updateFn(stack)
-			if err != nil {
-				return err
-			}
-
-			if m, ok := c.getNextUpdate(stack.ID); ok {
-				u = m
-			} else {
-				break
-			}
-
-			respChs = append(respChs, u.respCh)
-		}
-
-		// 3. Save the changes back to the database
-		if err := tx.EdgeStack().UpdateEdgeStack(stack.ID, stack); err != nil {
-			return handlerDBErr(fmt.Errorf("unable to update Edge stack: %w.", err), "Unable to persist the stack changes inside the database")
-		}
-
-		return nil
-	})
-
-	// 4. Send back the responses
-	for _, ch := range respChs {
-		ch <- statusResponse{Stack: stack, Error: err}
-	}
-}
-
-func loadEdgeStack(tx dataservices.DataStoreTx, stackID portainer.EdgeStackID) (*portainer.EdgeStack, error) {
-	stack, err := tx.EdgeStack().EdgeStack(stackID)
-	if err != nil {
-		if dataservices.IsErrObjectNotFound(err) {
-			// Skip the error when the agent tries to update the status on a deleted stack
-			log.Debug().
-				Err(err).
-				Int("stackID", int(stackID)).
-				Msg("Unable to find a stack inside the database, skipping error")
-
-			return nil, nil
-		}
-
-		return nil, fmt.Errorf("unable to retrieve Edge stack from the database: %w.", err)
-	}
-
-	return stack, nil
-}
-
-func (c *EdgeStackStatusUpdateCoordinator) getNextUpdate(stackID portainer.EdgeStackID) (statusRequest, bool) {
-	for {
-		select {
-		case u := <-c.updateCh:
-			// Discard the update and let the agent retry
-			if u.stackID != stackID {
-				u.respCh <- statusResponse{Error: errAnotherStackUpdateInProgress}
-
-				continue
-			}
-
-			return u, true
-
-		default:
-			return statusRequest{}, false
-		}
-	}
-}
-
-func (c *EdgeStackStatusUpdateCoordinator) UpdateStatus(r *http.Request, stackID portainer.EdgeStackID, updateFn statusUpdateFn) (*portainer.EdgeStack, error) {
-	respCh := make(chan statusResponse)
-	defer close(respCh)
-
-	msg := statusRequest{
-		respCh:   respCh,
-		stackID:  stackID,
-		updateFn: updateFn,
-	}
-
-	select {
-	case c.updateCh <- msg:
-		r := <-respCh
-
-		return r.Stack, r.Error
-
-	case <-r.Context().Done():
-		return nil, r.Context().Err()
-	}
-}

api/http/handler/edgestacks/edgestack_status_update_test.go

@@ -10,6 +10,7 @@ import (
 	portainer "github.com/portainer/portainer/api"
 
 	"github.com/segmentio/encoding/json"
+	"github.com/stretchr/testify/require"
 )
 
 // Update Status
@@ -28,15 +29,11 @@ func TestUpdateStatusAndInspect(t *testing.T) {
 	}
 
 	jsonPayload, err := json.Marshal(payload)
-	if err != nil {
-		t.Fatal("request error:", err)
-	}
+	require.NoError(t, err)
 
 	r := bytes.NewBuffer(jsonPayload)
 	req, err := http.NewRequest(http.MethodPut, fmt.Sprintf("/edge_stacks/%d/status", edgeStack.ID), r)
-	if err != nil {
-		t.Fatal("request error:", err)
-	}
+	require.NoError(t, err)
 
 	req.Header.Set(portainer.PortainerAgentEdgeIDHeader, endpoint.EdgeID)
 	rec := httptest.NewRecorder()
@@ -48,9 +45,7 @@ func TestUpdateStatusAndInspect(t *testing.T) {
 
 	// Get updated edge stack
 	req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("/edge_stacks/%d", edgeStack.ID), nil)
-	if err != nil {
-		t.Fatal("request error:", err)
-	}
+	require.NoError(t, err)
 
 	req.Header.Add("x-api-key", rawAPIKey)
 	rec = httptest.NewRecorder()
@@ -62,14 +57,10 @@ func TestUpdateStatusAndInspect(t *testing.T) {
 
 	updatedStack := portainer.EdgeStack{}
 	err = json.NewDecoder(rec.Body).Decode(&updatedStack)
-	if err != nil {
-		t.Fatal("error decoding response:", err)
-	}
+	require.NoError(t, err)
 
 	endpointStatus, ok := updatedStack.Status[payload.EndpointID]
-	if !ok {
-		t.Fatal("Missing status")
-	}
+	require.True(t, ok)
 
 	lastStatus := endpointStatus.Status[len(endpointStatus.Status)-1]
 
@@ -84,8 +75,8 @@ func TestUpdateStatusAndInspect(t *testing.T) {
 	if endpointStatus.EndpointID != payload.EndpointID {
 		t.Fatalf("expected EndpointID %d, found %d", payload.EndpointID, endpointStatus.EndpointID)
 	}
-
 }
+
 func TestUpdateStatusWithInvalidPayload(t *testing.T) {
 	handler, _ := setupHandler(t)
 
@@ -136,15 +127,11 @@ func TestUpdateStatusWithInvalidPayload(t *testing.T) {
 	for _, tc := range cases {
 		t.Run(tc.Name, func(t *testing.T) {
 			jsonPayload, err := json.Marshal(tc.Payload)
-			if err != nil {
-				t.Fatal("request error:", err)
-			}
+			require.NoError(t, err)
 
 			r := bytes.NewBuffer(jsonPayload)
 			req, err := http.NewRequest(http.MethodPut, fmt.Sprintf("/edge_stacks/%d/status", edgeStack.ID), r)
-			if err != nil {
-				t.Fatal("request error:", err)
-			}
+			require.NoError(t, err)
 
 			req.Header.Set(portainer.PortainerAgentEdgeIDHeader, endpoint.EdgeID)
 			rec := httptest.NewRecorder()

api/http/handler/edgestacks/edgestack_test.go

@@ -17,6 +17,7 @@ import (
 	"github.com/portainer/portainer/api/jwt"
 
 	"github.com/pkg/errors"
+	"github.com/stretchr/testify/require"
 )
 
 // Helpers
@@ -51,27 +52,21 @@ func setupHandler(t *testing.T) (*Handler, string) {
 		t.Fatal(err)
 	}
 
-	coord := NewEdgeStackStatusUpdateCoordinator(store)
-	go coord.Start()
-
 	handler := NewHandler(
 		security.NewRequestBouncer(store, jwtService, apiKeyService),
 		store,
 		edgestacks.NewService(store),
-		coord,
 	)
 
 	handler.FileService = fs
 
 	settings, err := handler.DataStore.Settings().Settings()
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
+
 	settings.EnableEdgeComputeFeatures = true
 
-	if err := handler.DataStore.Settings().UpdateSettings(settings); err != nil {
-		t.Fatal(err)
-	}
+	err = handler.DataStore.Settings().UpdateSettings(settings)
+	require.NoError(t, err)
 
 	handler.GitService = testhelpers.NewGitService(errors.New("Clone error"), "git-service-id")
 
@@ -90,9 +85,8 @@ func createEndpointWithId(t *testing.T, store dataservices.DataStore, endpointID
 		LastCheckInDate: time.Now().Unix(),
 	}
 
-	if err := store.Endpoint().Create(&endpoint); err != nil {
-		t.Fatal(err)
-	}
+	err := store.Endpoint().Create(&endpoint)
+	require.NoError(t, err)
 
 	return endpoint
 }
@@ -113,15 +107,13 @@ func createEdgeStack(t *testing.T, store dataservices.DataStore, endpointID port
 		PartialMatch: false,
 	}
 
-	if err := store.EdgeGroup().Create(&edgeGroup); err != nil {
-		t.Fatal(err)
-	}
+	err := store.EdgeGroup().Create(&edgeGroup)
+	require.NoError(t, err)
 
 	edgeStackID := portainer.EdgeStackID(14)
 	edgeStack := portainer.EdgeStack{
 		ID:             edgeStackID,
 		Name:           "test-edge-stack-" + strconv.Itoa(int(edgeStackID)),
-		Status:         map[portainer.EndpointID]portainer.EdgeStackStatus{},
 		CreationDate:   time.Now().Unix(),
 		EdgeGroups:     []portainer.EdgeGroupID{edgeGroup.ID},
 		ProjectPath:    "/project/path",
@@ -138,13 +130,11 @@ func createEdgeStack(t *testing.T, store dataservices.DataStore, endpointID port
 		},
 	}
 
-	if err := store.EdgeStack().Create(edgeStack.ID, &edgeStack); err != nil {
-		t.Fatal(err)
-	}
+	err = store.EdgeStack().Create(edgeStack.ID, &edgeStack)
+	require.NoError(t, err)
 
-	if err := store.EndpointRelation().Create(&endpointRelation); err != nil {
-		t.Fatal(err)
-	}
+	err = store.EndpointRelation().Create(&endpointRelation)
+	require.NoError(t, err)
 
 	return edgeStack
 }
@@ -155,8 +145,8 @@ func createEdgeGroup(t *testing.T, store dataservices.DataStore) portainer.EdgeG
 		Name: "EdgeGroup 1",
 	}
 
-	if err := store.EdgeGroup().Create(&edgeGroup); err != nil {
-		t.Fatal(err)
-	}
+	err := store.EdgeGroup().Create(&edgeGroup)
+	require.NoError(t, err)
+
 	return edgeGroup
 }

api/http/handler/edgestacks/edgestack_update.go

@@ -74,6 +74,10 @@ func (handler *Handler) edgeStackUpdate(w http.ResponseWriter, r *http.Request)
 		return httperror.InternalServerError("Unexpected error", err)
 	}
 
+	if err := fillEdgeStackStatus(handler.DataStore, stack); err != nil {
+		return handlerDBErr(err, "Unable to retrieve edge stack status from the database")
+	}
+
 	return response.JSON(w, stack)
 }
 
@@ -120,7 +124,7 @@ func (handler *Handler) updateEdgeStack(tx dataservices.DataStoreTx, stackID por
 	stack.EdgeGroups = groupsIds
 
 	if payload.UpdateVersion {
-		if err := handler.updateStackVersion(stack, payload.DeploymentType, []byte(payload.StackFileContent), "", relatedEndpointIds); err != nil {
+		if err := handler.updateStackVersion(tx, stack, payload.DeploymentType, []byte(payload.StackFileContent), "", relatedEndpointIds); err != nil {
 			return nil, httperror.InternalServerError("Unable to update stack version", err)
 		}
 	}
@@ -145,11 +149,15 @@ func (handler *Handler) handleChangeEdgeGroups(tx dataservices.DataStoreTx, edge
 	relatedEnvironmentsToRemove := oldRelatedEnvironmentsSet.Difference(newRelatedEnvironmentsSet)
 
 	if len(relatedEnvironmentsToRemove) > 0 {
-		tx.EndpointRelation().RemoveEndpointRelationsForEdgeStack(relatedEnvironmentsToRemove.Keys(), edgeStackID)
+		if err := tx.EndpointRelation().RemoveEndpointRelationsForEdgeStack(relatedEnvironmentsToRemove.Keys(), edgeStackID); err != nil {
+			return nil, nil, errors.WithMessage(err, "Unable to remove edge stack relations from the database")
+		}
 	}
 
 	if len(relatedEnvironmentsToAdd) > 0 {
-		tx.EndpointRelation().AddEndpointRelationsForEdgeStack(relatedEnvironmentsToAdd.Keys(), edgeStackID)
+		if err := tx.EndpointRelation().AddEndpointRelationsForEdgeStack(relatedEnvironmentsToAdd.Keys(), edgeStackID); err != nil {
+			return nil, nil, errors.WithMessage(err, "Unable to add edge stack relations to the database")
+		}
 	}
 
 	return newRelatedEnvironmentIDs, relatedEnvironmentsToAdd, nil

api/http/handler/edgestacks/edgestack_update_test.go

@@ -25,9 +25,8 @@ func TestUpdateAndInspect(t *testing.T) {
 	endpointID := portainer.EndpointID(6)
 	newEndpoint := createEndpointWithId(t, handler.DataStore, endpointID)
 
-	if err := handler.DataStore.Endpoint().Create(&newEndpoint); err != nil {
-		t.Fatal(err)
-	}
+	err := handler.DataStore.Endpoint().Create(&newEndpoint)
+	require.NoError(t, err)
 
 	endpointRelation := portainer.EndpointRelation{
 		EndpointID: endpointID,
@@ -36,9 +35,8 @@ func TestUpdateAndInspect(t *testing.T) {
 		},
 	}
 
-	if err := handler.DataStore.EndpointRelation().Create(&endpointRelation); err != nil {
-		t.Fatal(err)
-	}
+	err = handler.DataStore.EndpointRelation().Create(&endpointRelation)
+	require.NoError(t, err)
 
 	newEdgeGroup := portainer.EdgeGroup{
 		ID:           2,
@@ -49,9 +47,8 @@ func TestUpdateAndInspect(t *testing.T) {
 		PartialMatch: false,
 	}
 
-	if err := handler.DataStore.EdgeGroup().Create(&newEdgeGroup); err != nil {
-		t.Fatal(err)
-	}
+	err = handler.DataStore.EdgeGroup().Create(&newEdgeGroup)
+	require.NoError(t, err)
 
 	payload := updateEdgeStackPayload{
 		StackFileContent: "update-test",
@@ -61,15 +58,11 @@ func TestUpdateAndInspect(t *testing.T) {
 	}
 
 	jsonPayload, err := json.Marshal(payload)
-	if err != nil {
-		t.Fatal("request error:", err)
-	}
+	require.NoError(t, err)
 
 	r := bytes.NewBuffer(jsonPayload)
 	req, err := http.NewRequest(http.MethodPut, fmt.Sprintf("/edge_stacks/%d", edgeStack.ID), r)
-	if err != nil {
-		t.Fatal("request error:", err)
-	}
+	require.NoError(t, err)
 
 	req.Header.Add("x-api-key", rawAPIKey)
 	rec := httptest.NewRecorder()
@@ -81,9 +74,7 @@ func TestUpdateAndInspect(t *testing.T) {
 
 	// Get updated edge stack
 	req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("/edge_stacks/%d", edgeStack.ID), nil)
-	if err != nil {
-		t.Fatal("request error:", err)
-	}
+	require.NoError(t, err)
 
 	req.Header.Add("x-api-key", rawAPIKey)
 	rec = httptest.NewRecorder()
@@ -94,9 +85,8 @@ func TestUpdateAndInspect(t *testing.T) {
 	}
 
 	updatedStack := portainer.EdgeStack{}
-	if err := json.NewDecoder(rec.Body).Decode(&updatedStack); err != nil {
-		t.Fatal("error decoding response:", err)
-	}
+	err = json.NewDecoder(rec.Body).Decode(&updatedStack)
+	require.NoError(t, err)
 
 	if payload.UpdateVersion && updatedStack.Version != edgeStack.Version+1 {
 		t.Fatalf("expected EdgeStack version %d, found %d", edgeStack.Version+1, updatedStack.Version+1)
@@ -226,15 +216,11 @@ func TestUpdateWithInvalidPayload(t *testing.T) {
 	for _, tc := range cases {
 		t.Run(tc.Name, func(t *testing.T) {
 			jsonPayload, err := json.Marshal(tc.Payload)
-			if err != nil {
-				t.Fatal("request error:", err)
-			}
+			require.NoError(t, err)
 
 			r := bytes.NewBuffer(jsonPayload)
 			req, err := http.NewRequest(http.MethodPut, fmt.Sprintf("/edge_stacks/%d", edgeStack.ID), r)
-			if err != nil {
-				t.Fatal("request error:", err)
-			}
+			require.NoError(t, err)
 
 			req.Header.Add("x-api-key", rawAPIKey)
 			rec := httptest.NewRecorder()

api/http/handler/edgestacks/handler.go

@@ -22,17 +22,15 @@ type Handler struct {
 	GitService         portainer.GitService
 	edgeStacksService  *edgestackservice.Service
 	KubernetesDeployer portainer.KubernetesDeployer
-	stackCoordinator   *EdgeStackStatusUpdateCoordinator
 }
 
 // NewHandler creates a handler to manage environment(endpoint) group operations.
-func NewHandler(bouncer security.BouncerService, dataStore dataservices.DataStore, edgeStacksService *edgestackservice.Service, stackCoordinator *EdgeStackStatusUpdateCoordinator) *Handler {
+func NewHandler(bouncer security.BouncerService, dataStore dataservices.DataStore, edgeStacksService *edgestackservice.Service) *Handler {
 	h := &Handler{
 		Router:            mux.NewRouter(),
 		requestBouncer:    bouncer,
 		DataStore:         dataStore,
 		edgeStacksService: edgeStacksService,
-		stackCoordinator:  stackCoordinator,
 	}
 
 	h.Handle("/edge_stacks/create/{method}",

api/http/handler/edgestacks/utils_update_stack_version.go

@@ -5,15 +5,18 @@ import (
 	"strconv"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/filesystem"
-	edgestackutils "github.com/portainer/portainer/api/internal/edge/edgestacks"
 
 	"github.com/rs/zerolog/log"
 )
 
-func (handler *Handler) updateStackVersion(stack *portainer.EdgeStack, deploymentType portainer.EdgeStackDeploymentType, config []byte, oldGitHash string, relatedEnvironmentsIDs []portainer.EndpointID) error {
-	stack.Version = stack.Version + 1
-	stack.Status = edgestackutils.NewStatus(stack.Status, relatedEnvironmentsIDs)
+func (handler *Handler) updateStackVersion(tx dataservices.DataStoreTx, stack *portainer.EdgeStack, deploymentType portainer.EdgeStackDeploymentType, config []byte, oldGitHash string, relatedEnvironmentsIDs []portainer.EndpointID) error {
+	stack.Version++
+
+	if err := tx.EdgeStackStatus().Clear(stack.ID, relatedEnvironmentsIDs); err != nil {
+		return err
+	}
 
 	return handler.storeStackFile(stack, deploymentType, config)
 }

api/http/handler/endpointedge/endpointedge_status_inspect_test.go

@@ -287,11 +287,8 @@ func TestEdgeStackStatus(t *testing.T) {
 
 	edgeStackID := portainer.EdgeStackID(17)
 	edgeStack := portainer.EdgeStack{
-		ID:   edgeStackID,
-		Name: "test-edge-stack-17",
-		Status: map[portainer.EndpointID]portainer.EdgeStackStatus{
-			endpointID: {},
-		},
+		ID:             edgeStackID,
+		Name:           "test-edge-stack-17",
 		CreationDate:   time.Now().Unix(),
 		EdgeGroups:     []portainer.EdgeGroupID{1, 2},
 		ProjectPath:    "/project/path",

api/http/handler/endpoints/endpoint_delete.go

@@ -214,14 +214,9 @@ func (handler *Handler) deleteEndpoint(tx dataservices.DataStoreTx, endpointID p
 		log.Warn().Err(err).Msg("Unable to retrieve edge stacks from the database")
 	}
 
-	for idx := range edgeStacks {
-		edgeStack := &edgeStacks[idx]
-		if _, ok := edgeStack.Status[endpoint.ID]; ok {
-			delete(edgeStack.Status, endpoint.ID)
-
-			if err := tx.EdgeStack().UpdateEdgeStack(edgeStack.ID, edgeStack); err != nil {
-				log.Warn().Err(err).Msg("Unable to update edge stack")
-			}
+	for _, edgeStack := range edgeStacks {
+		if err := tx.EdgeStackStatus().Delete(edgeStack.ID, endpoint.ID); err != nil {
+			log.Warn().Err(err).Msg("Unable to delete edge stack status")
 		}
 	}
 

api/http/handler/endpoints/endpoint_dockerhub_status.go

@@ -80,6 +80,13 @@ func (handler *Handler) endpointDockerhubStatus(w http.ResponseWriter, r *http.R
 		}
 	}
 
+	if handler.PullLimitCheckDisabled {
+		return response.JSON(w, &dockerhubStatusResponse{
+			Limit:     10,
+			Remaining: 10,
+		})
+	}
+
 	httpClient := client.NewHTTPClient()
 	token, err := getDockerHubToken(httpClient, registry)
 	if err != nil {

api/http/handler/endpoints/endpoint_inspect.go

@@ -19,6 +19,8 @@ import (
 // @security jwt
 // @produce json
 // @param id path int true "Environment(Endpoint) identifier"
+// @param excludeSnapshot query bool false "if true, the snapshot data won't be retrieved"
+// @param excludeSnapshotRaw query bool false "if true, the SnapshotRaw field won't be retrieved"
 // @success 200 {object} portainer.Endpoint "Success"
 // @failure 400 "Invalid request"
 // @failure 404 "Environment(Endpoint) not found"
@@ -37,8 +39,7 @@ func (handler *Handler) endpointInspect(w http.ResponseWriter, r *http.Request)
 		return httperror.InternalServerError("Unable to find an environment with the specified identifier inside the database", err)
 	}
 
-	err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint)
-	if err != nil {
+	if err := handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint); err != nil {
 		return httperror.Forbidden("Permission denied to access environment", err)
 	}
 
@@ -51,9 +52,11 @@ func (handler *Handler) endpointInspect(w http.ResponseWriter, r *http.Request)
 	endpointutils.UpdateEdgeEndpointHeartbeat(endpoint, settings)
 	endpoint.ComposeSyntaxMaxVersion = handler.ComposeStackManager.ComposeSyntaxMaxVersion()
 
-	if !excludeSnapshot(r) {
-		err = handler.SnapshotService.FillSnapshotData(endpoint)
-		if err != nil {
+	excludeSnapshot, _ := request.RetrieveBooleanQueryParameter(r, "excludeSnapshot", true)
+	excludeRaw, _ := request.RetrieveBooleanQueryParameter(r, "excludeSnapshotRaw", true)
+
+	if !excludeSnapshot {
+		if err := handler.SnapshotService.FillSnapshotData(endpoint, !excludeRaw); err != nil {
 			return httperror.InternalServerError("Unable to add snapshot data", err)
 		}
 	}
@@ -83,9 +86,3 @@ func (handler *Handler) endpointInspect(w http.ResponseWriter, r *http.Request)
 
 	return response.JSON(w, endpoint)
 }
-
-func excludeSnapshot(r *http.Request) bool {
-	excludeSnapshot, _ := request.RetrieveBooleanQueryParameter(r, "excludeSnapshot", true)
-
-	return excludeSnapshot
-}

api/http/handler/endpoints/endpoint_list.go

@@ -38,15 +38,19 @@ const (
 // @param tagIds query []int false "search environments(endpoints) with these tags (depends on tagsPartialMatch)"
 // @param tagsPartialMatch query bool false "If true, will return environment(endpoint) which has one of tagIds, if false (or missing) will return only environments(endpoints) that has all the tags"
 // @param endpointIds query []int false "will return only these environments(endpoints)"
+// @param excludeIds query []int false "will exclude these environments(endpoints)"
 // @param provisioned query bool false "If true, will return environment(endpoint) that were provisioned"
 // @param agentVersions query []string false "will return only environments with on of these agent versions"
 // @param edgeAsync query bool false "if exists true show only edge async agents, false show only standard edge agents. if missing, will show both types (relevant only for edge agents)"
 // @param edgeDeviceUntrusted query bool false "if true, show only untrusted edge agents, if false show only trusted edge agents (relevant only for edge agents)"
 // @param edgeCheckInPassedSeconds query number false "if bigger then zero, show only edge agents that checked-in in the last provided seconds (relevant only for edge agents)"
 // @param excludeSnapshots query bool false "if true, the snapshot data won't be retrieved"
+// @param excludeSnapshotRaw query bool false "if true, the SnapshotRaw field won't be retrieved"
 // @param name query string false "will return only environments(endpoints) with this name"
 // @param edgeStackId query portainer.EdgeStackID false "will return the environements of the specified edge stack"
 // @param edgeStackStatus query string false "only applied when edgeStackId exists. Filter the returned environments based on their deployment status in the stack (not the environment status!)" Enum("Pending", "Ok", "Error", "Acknowledged", "Remove", "RemoteUpdateSuccess", "ImagesPulled")
+// @param edgeGroupIds query []int false "List environments(endpoints) of these edge groups"
+// @param excludeEdgeGroupIds query []int false "Exclude environments(endpoints) of these edge groups"
 // @success 200 {array} portainer.Endpoint "Endpoints"
 // @failure 500 "Server error"
 // @router /endpoints [get]
@@ -59,6 +63,7 @@ func (handler *Handler) endpointList(w http.ResponseWriter, r *http.Request) *ht
 	limit, _ := request.RetrieveNumericQueryParameter(r, "limit", true)
 	sortField, _ := request.RetrieveQueryParameter(r, "sort", true)
 	sortOrder, _ := request.RetrieveQueryParameter(r, "order", true)
+	excludeRaw, _ := request.RetrieveBooleanQueryParameter(r, "excludeSnapshotRaw", true)
 
 	endpointGroups, err := handler.DataStore.EndpointGroup().ReadAll()
 	if err != nil {
@@ -105,14 +110,16 @@ func (handler *Handler) endpointList(w http.ResponseWriter, r *http.Request) *ht
 
 	for idx := range paginatedEndpoints {
 		hideFields(&paginatedEndpoints[idx])
+
 		paginatedEndpoints[idx].ComposeSyntaxMaxVersion = handler.ComposeStackManager.ComposeSyntaxMaxVersion()
 		if paginatedEndpoints[idx].EdgeCheckinInterval == 0 {
 			paginatedEndpoints[idx].EdgeCheckinInterval = settings.EdgeAgentCheckinInterval
 		}
+
 		endpointutils.UpdateEdgeEndpointHeartbeat(&paginatedEndpoints[idx], settings)
+
 		if !query.excludeSnapshots {
-			err = handler.SnapshotService.FillSnapshotData(&paginatedEndpoints[idx])
-			if err != nil {
+			if err := handler.SnapshotService.FillSnapshotData(&paginatedEndpoints[idx], !excludeRaw); err != nil {
 				return httperror.InternalServerError("Unable to add snapshot data", err)
 			}
 		}
@@ -120,6 +127,7 @@ func (handler *Handler) endpointList(w http.ResponseWriter, r *http.Request) *ht
 
 	w.Header().Set("X-Total-Count", strconv.Itoa(filteredEndpointCount))
 	w.Header().Set("X-Total-Available", strconv.Itoa(totalAvailableEndpoints))
+
 	return response.JSON(w, paginatedEndpoints)
 }
 
@@ -130,18 +138,8 @@ func paginateEndpoints(endpoints []portainer.Endpoint, start, limit int) []porta
 
 	endpointCount := len(endpoints)
 
-	if start < 0 {
-		start = 0
-	}
-
-	if start > endpointCount {
-		start = endpointCount
-	}
-
-	end := start + limit
-	if end > endpointCount {
-		end = endpointCount
-	}
+	start = min(max(start, 0), endpointCount)
+	end := min(start+limit, endpointCount)
 
 	return endpoints[start:end]
 }
@@ -151,8 +149,10 @@ func getEndpointGroup(groupID portainer.EndpointGroupID, groups []portainer.Endp
 	for _, group := range groups {
 		if group.ID == groupID {
 			endpointGroup = group
+
 			break
 		}
 	}
+
 	return endpointGroup
 }

api/http/handler/endpoints/endpoint_update.go

@@ -272,7 +272,7 @@ func (handler *Handler) endpointUpdate(w http.ResponseWriter, r *http.Request) *
 		}
 	}
 
-	if err := handler.SnapshotService.FillSnapshotData(endpoint); err != nil {
+	if err := handler.SnapshotService.FillSnapshotData(endpoint, true); err != nil {
 		return httperror.InternalServerError("Unable to add snapshot data", err)
 	}
 

api/http/handler/endpoints/filter.go

@@ -37,6 +37,8 @@ type EnvironmentsQuery struct {
 	edgeStackId              portainer.EdgeStackID
 	edgeStackStatus          *portainer.EdgeStackStatusType
 	excludeIds               []portainer.EndpointID
+	edgeGroupIds             []portainer.EdgeGroupID
+	excludeEdgeGroupIds      []portainer.EdgeGroupID
 }
 
 func parseQuery(r *http.Request) (EnvironmentsQuery, error) {
@@ -77,6 +79,16 @@ func parseQuery(r *http.Request) (EnvironmentsQuery, error) {
 		return EnvironmentsQuery{}, err
 	}
 
+	edgeGroupIDs, err := getNumberArrayQueryParameter[portainer.EdgeGroupID](r, "edgeGroupIds")
+	if err != nil {
+		return EnvironmentsQuery{}, err
+	}
+
+	excludeEdgeGroupIds, err := getNumberArrayQueryParameter[portainer.EdgeGroupID](r, "excludeEdgeGroupIds")
+	if err != nil {
+		return EnvironmentsQuery{}, err
+	}
+
 	agentVersions := getArrayQueryParameter(r, "agentVersions")
 
 	name, _ := request.RetrieveQueryParameter(r, "name", true)
@@ -117,6 +129,8 @@ func parseQuery(r *http.Request) (EnvironmentsQuery, error) {
 		edgeCheckInPassedSeconds: edgeCheckInPassedSeconds,
 		edgeStackId:              portainer.EdgeStackID(edgeStackId),
 		edgeStackStatus:          edgeStackStatus,
+		edgeGroupIds:             edgeGroupIDs,
+		excludeEdgeGroupIds:      excludeEdgeGroupIds,
 	}, nil
 }
 
@@ -143,6 +157,14 @@ func (handler *Handler) filterEndpointsByQuery(
 		filteredEndpoints = filterEndpointsByGroupIDs(filteredEndpoints, query.groupIds)
 	}
 
+	if len(query.edgeGroupIds) > 0 {
+		filteredEndpoints, edgeGroups = filterEndpointsByEdgeGroupIDs(filteredEndpoints, edgeGroups, query.edgeGroupIds)
+	}
+
+	if len(query.excludeEdgeGroupIds) > 0 {
+		filteredEndpoints, edgeGroups = filterEndpointsByExcludeEdgeGroupIDs(filteredEndpoints, edgeGroups, query.excludeEdgeGroupIds)
+	}
+
 	if query.name != "" {
 		filteredEndpoints = filterEndpointsByName(filteredEndpoints, query.name)
 	}
@@ -225,19 +247,17 @@ func (handler *Handler) filterEndpointsByQuery(
 	return filteredEndpoints, totalAvailableEndpoints, nil
 }
 
-func endpointStatusInStackMatchesFilter(edgeStackStatus map[portainer.EndpointID]portainer.EdgeStackStatus, envId portainer.EndpointID, statusFilter portainer.EdgeStackStatusType) bool {
-	status, ok := edgeStackStatus[envId]
-
+func endpointStatusInStackMatchesFilter(stackStatus *portainer.EdgeStackStatusForEnv, envId portainer.EndpointID, statusFilter portainer.EdgeStackStatusType) bool {
 	// consider that if the env has no status in the stack it is in Pending state
 	if statusFilter == portainer.EdgeStackStatusPending {
-		return !ok || len(status.Status) == 0
+		return stackStatus == nil || len(stackStatus.Status) == 0
 	}
 
-	if !ok {
+	if stackStatus == nil {
 		return false
 	}
 
-	return slices.ContainsFunc(status.Status, func(s portainer.EdgeStackDeploymentStatus) bool {
+	return slices.ContainsFunc(stackStatus.Status, func(s portainer.EdgeStackDeploymentStatus) bool {
 		return s.Type == statusFilter
 	})
 }
@@ -269,7 +289,12 @@ func filterEndpointsByEdgeStack(endpoints []portainer.Endpoint, edgeStackId port
 	if statusFilter != nil {
 		n := 0
 		for _, envId := range envIds {
-			if endpointStatusInStackMatchesFilter(stack.Status, envId, *statusFilter) {
+			edgeStackStatus, err := datastore.EdgeStackStatus().Read(edgeStackId, envId)
+			if err != nil {
+				return nil, errors.WithMessagef(err, "Unable to retrieve edge stack status for environment %d", envId)
+			}
+
+			if endpointStatusInStackMatchesFilter(edgeStackStatus, envId, *statusFilter) {
 				envIds[n] = envId
 				n++
 			}
@@ -295,6 +320,70 @@ func filterEndpointsByGroupIDs(endpoints []portainer.Endpoint, endpointGroupIDs
 	return endpoints[:n]
 }
 
+func filterEndpointsByEdgeGroupIDs(endpoints []portainer.Endpoint, edgeGroups []portainer.EdgeGroup, edgeGroupIDs []portainer.EdgeGroupID) ([]portainer.Endpoint, []portainer.EdgeGroup) {
+	edgeGroupIDFilterSet := make(map[portainer.EdgeGroupID]struct{}, len(edgeGroupIDs))
+	for _, id := range edgeGroupIDs {
+		edgeGroupIDFilterSet[id] = struct{}{}
+	}
+
+	n := 0
+	for _, edgeGroup := range edgeGroups {
+		if _, exists := edgeGroupIDFilterSet[edgeGroup.ID]; exists {
+			edgeGroups[n] = edgeGroup
+			n++
+		}
+	}
+	edgeGroups = edgeGroups[:n]
+
+	endpointIDSet := make(map[portainer.EndpointID]struct{})
+	for _, edgeGroup := range edgeGroups {
+		for _, endpointID := range edgeGroup.Endpoints {
+			endpointIDSet[endpointID] = struct{}{}
+		}
+	}
+
+	n = 0
+	for _, endpoint := range endpoints {
+		if _, exists := endpointIDSet[endpoint.ID]; exists {
+			endpoints[n] = endpoint
+			n++
+		}
+	}
+
+	return endpoints[:n], edgeGroups
+}
+
+func filterEndpointsByExcludeEdgeGroupIDs(endpoints []portainer.Endpoint, edgeGroups []portainer.EdgeGroup, excludeEdgeGroupIds []portainer.EdgeGroupID) ([]portainer.Endpoint, []portainer.EdgeGroup) {
+	excludeEdgeGroupIDSet := make(map[portainer.EdgeGroupID]struct{}, len(excludeEdgeGroupIds))
+	for _, id := range excludeEdgeGroupIds {
+		excludeEdgeGroupIDSet[id] = struct{}{}
+	}
+
+	n := 0
+	excludeEndpointIDSet := make(map[portainer.EndpointID]struct{})
+	for _, edgeGroup := range edgeGroups {
+		if _, ok := excludeEdgeGroupIDSet[edgeGroup.ID]; ok {
+			for _, endpointID := range edgeGroup.Endpoints {
+				excludeEndpointIDSet[endpointID] = struct{}{}
+			}
+		} else {
+			edgeGroups[n] = edgeGroup
+			n++
+		}
+	}
+	edgeGroups = edgeGroups[:n]
+
+	n = 0
+	for _, endpoint := range endpoints {
+		if _, ok := excludeEndpointIDSet[endpoint.ID]; !ok {
+			endpoints[n] = endpoint
+			n++
+		}
+	}
+
+	return endpoints[:n], edgeGroups
+}
+
 func filterEndpointsBySearchCriteria(
 	endpoints []portainer.Endpoint,
 	endpointGroups []portainer.EndpointGroup,

api/http/handler/endpoints/handler.go

@@ -26,19 +26,20 @@ func hideFields(endpoint *portainer.Endpoint) {
 // Handler is the HTTP handler used to handle environment(endpoint) operations.
 type Handler struct {
 	*mux.Router
-	requestBouncer        security.BouncerService
-	DataStore             dataservices.DataStore
-	FileService           portainer.FileService
-	ProxyManager          *proxy.Manager
-	ReverseTunnelService  portainer.ReverseTunnelService
-	SnapshotService       portainer.SnapshotService
-	K8sClientFactory      *cli.ClientFactory
-	ComposeStackManager   portainer.ComposeStackManager
-	AuthorizationService  *authorization.Service
-	DockerClientFactory   *dockerclient.ClientFactory
-	BindAddress           string
-	BindAddressHTTPS      string
-	PendingActionsService *pendingactions.PendingActionsService
+	requestBouncer         security.BouncerService
+	DataStore              dataservices.DataStore
+	FileService            portainer.FileService
+	ProxyManager           *proxy.Manager
+	ReverseTunnelService   portainer.ReverseTunnelService
+	SnapshotService        portainer.SnapshotService
+	K8sClientFactory       *cli.ClientFactory
+	ComposeStackManager    portainer.ComposeStackManager
+	AuthorizationService   *authorization.Service
+	DockerClientFactory    *dockerclient.ClientFactory
+	BindAddress            string
+	BindAddressHTTPS       string
+	PendingActionsService  *pendingactions.PendingActionsService
+	PullLimitCheckDisabled bool
 }
 
 // NewHandler creates a handler to manage environment(endpoint) operations.

api/http/handler/gitops/git_repo_file_preview.go

@@ -9,8 +9,7 @@ import (
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
 	"github.com/portainer/portainer/pkg/libhttp/response"
-
-	"github.com/asaskevich/govalidator"
+	"github.com/portainer/portainer/pkg/validate"
 )
 
 type fileResponse struct {
@@ -29,7 +28,7 @@ type repositoryFilePreviewPayload struct {
 }
 
 func (payload *repositoryFilePreviewPayload) Validate(r *http.Request) error {
-	if len(payload.Repository) == 0 || !govalidator.IsURL(payload.Repository) {
+	if len(payload.Repository) == 0 || !validate.IsURL(payload.Repository) {
 		return errors.New("invalid repository URL. Must correspond to a valid URL format")
 	}
 

api/http/handler/handler.go

@@ -81,7 +81,7 @@ type Handler struct {
 }
 
 // @title PortainerCE API
-// @version 2.28.0
+// @version 2.31.0
 // @description.markdown api-description.md
 // @termsOfService
 

api/http/handler/helm/handler.go

@@ -54,6 +54,18 @@ func NewHandler(bouncer security.BouncerService, dataStore dataservices.DataStor
 	h.Handle("/{id}/kubernetes/helm",
 		httperror.LoggerHandler(h.helmInstall)).Methods(http.MethodPost)
 
+	// `helm get all [RELEASE_NAME]`
+	h.Handle("/{id}/kubernetes/helm/{release}",
+		httperror.LoggerHandler(h.helmGet)).Methods(http.MethodGet)
+
+	// `helm history [RELEASE_NAME]`
+	h.Handle("/{id}/kubernetes/helm/{release}/history",
+		httperror.LoggerHandler(h.helmGetHistory)).Methods(http.MethodGet)
+
+	// `helm rollback [RELEASE_NAME] [REVISION]`
+	h.Handle("/{id}/kubernetes/helm/{release}/rollback",
+		httperror.LoggerHandler(h.helmRollback)).Methods(http.MethodPost)
+
 	return h
 }
 

api/http/handler/helm/helm_delete_test.go

@@ -42,7 +42,7 @@ func Test_helmDelete(t *testing.T) {
 
 	// Install a single chart directly, to be deleted by the handler
 	options := options.InstallOptions{Name: "nginx-1", Chart: "nginx", Namespace: "default"}
-	h.helmPackageManager.Install(options)
+	h.helmPackageManager.Upgrade(options)
 
 	t.Run("helmDelete succeeds with admin user", func(t *testing.T) {
 		req := httptest.NewRequest(http.MethodDelete, "/1/kubernetes/helm/"+options.Name, nil)

api/http/handler/helm/helm_get.go

@@ -0,0 +1,67 @@
+package helm
+
+import (
+	"net/http"
+
+	"github.com/portainer/portainer/pkg/libhelm/options"
+	_ "github.com/portainer/portainer/pkg/libhelm/release"
+	httperror "github.com/portainer/portainer/pkg/libhttp/error"
+	"github.com/portainer/portainer/pkg/libhttp/request"
+	"github.com/portainer/portainer/pkg/libhttp/response"
+)
+
+// @id HelmGet
+// @summary Get a helm release
+// @description Get details of a helm release by release name
+// @description **Access policy**: authenticated
+// @tags helm
+// @security ApiKeyAuth || jwt
+// @produce json
+// @param id path int true "Environment(Endpoint) identifier"
+// @param name path string true "Helm release name"
+// @param namespace query string false "specify an optional namespace"
+// @param showResources query boolean false "show resources of the release"
+// @param revision query int false "specify an optional revision"
+// @success 200 {object} release.Release "Success"
+// @failure 400 "Invalid request payload, such as missing required fields or fields not meeting validation criteria."
+// @failure 401 "Unauthorized access - the user is not authenticated or does not have the necessary permissions. Ensure that you have provided a valid API key or JWT token, and that you have the required permissions."
+// @failure 403 "Permission denied - the user is authenticated but does not have the necessary permissions to access the requested resource or perform the specified operation. Check your user roles and permissions."
+// @failure 404 "Unable to find an environment with the specified identifier."
+// @failure 500 "Server error occurred while attempting to retrieve the release."
+// @router /endpoints/{id}/kubernetes/helm/{name} [get]
+func (handler *Handler) helmGet(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	release, err := request.RetrieveRouteVariableValue(r, "release")
+	if err != nil {
+		return httperror.BadRequest("No release specified", err)
+	}
+
+	clusterAccess, httperr := handler.getHelmClusterAccess(r)
+	if httperr != nil {
+		return httperr
+	}
+
+	// build the get options
+	getOpts := options.GetOptions{
+		KubernetesClusterAccess: clusterAccess,
+		Name:                    release,
+	}
+	namespace, _ := request.RetrieveQueryParameter(r, "namespace", true)
+	// optional namespace.  The library defaults to "default"
+	if namespace != "" {
+		getOpts.Namespace = namespace
+	}
+	showResources, _ := request.RetrieveBooleanQueryParameter(r, "showResources", true)
+	getOpts.ShowResources = showResources
+	revision, _ := request.RetrieveNumericQueryParameter(r, "revision", true)
+	// optional revision.  The library defaults to the latest revision if not specified
+	if revision > 0 {
+		getOpts.Revision = revision
+	}
+
+	releases, err := handler.helmPackageManager.Get(getOpts)
+	if err != nil {
+		return httperror.InternalServerError("Helm returned an error", err)
+	}
+
+	return response.JSON(w, releases)
+}

api/http/handler/helm/helm_get_test.go

@@ -0,0 +1,66 @@
+package helm
+
+import (
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/datastore"
+	"github.com/portainer/portainer/api/exec/exectest"
+	"github.com/portainer/portainer/api/http/security"
+	"github.com/portainer/portainer/api/internal/testhelpers"
+	helper "github.com/portainer/portainer/api/internal/testhelpers"
+	"github.com/portainer/portainer/api/jwt"
+	"github.com/portainer/portainer/api/kubernetes"
+	"github.com/portainer/portainer/pkg/libhelm/options"
+	"github.com/portainer/portainer/pkg/libhelm/release"
+	"github.com/portainer/portainer/pkg/libhelm/test"
+
+	"github.com/segmentio/encoding/json"
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_helmGet(t *testing.T) {
+	is := assert.New(t)
+
+	_, store := datastore.MustNewTestStore(t, true, true)
+
+	err := store.Endpoint().Create(&portainer.Endpoint{ID: 1})
+	is.NoError(err, "Error creating environment")
+
+	err = store.User().Create(&portainer.User{Username: "admin", Role: portainer.AdministratorRole})
+	is.NoError(err, "Error creating a user")
+
+	jwtService, err := jwt.NewService("1h", store)
+	is.NoError(err, "Error initiating jwt service")
+
+	kubernetesDeployer := exectest.NewKubernetesDeployer()
+	helmPackageManager := test.NewMockHelmPackageManager()
+	kubeClusterAccessService := kubernetes.NewKubeClusterAccessService("", "", "")
+	h := NewHandler(helper.NewTestRequestBouncer(), store, jwtService, kubernetesDeployer, helmPackageManager, kubeClusterAccessService)
+
+	is.NotNil(h, "Handler should not fail")
+
+	// Install a single chart, to be retrieved by the handler
+	options := options.InstallOptions{Name: "nginx-1", Chart: "nginx", Namespace: "default"}
+	h.helmPackageManager.Upgrade(options)
+
+	t.Run("helmGet sucessfuly retrieves helm release", func(t *testing.T) {
+		req := httptest.NewRequest(http.MethodGet, "/1/kubernetes/helm/"+options.Name+"?namespace="+options.Namespace, nil)
+		ctx := security.StoreTokenData(req, &portainer.TokenData{ID: 1, Username: "admin", Role: 1})
+		req = req.WithContext(ctx)
+		testhelpers.AddTestSecurityCookie(req, "Bearer dummytoken")
+
+		rr := httptest.NewRecorder()
+		h.ServeHTTP(rr, req)
+
+		data := release.Release{}
+		body, err := io.ReadAll(rr.Body)
+		is.NoError(err, "ReadAll should not return error")
+		json.Unmarshal(body, &data)
+		is.Equal(http.StatusOK, rr.Code, "Status should be 200")
+		is.Equal("nginx-1", data.Name)
+	})
+}

api/http/handler/helm/helm_history.go

@@ -0,0 +1,58 @@
+package helm
+
+import (
+	"net/http"
+
+	"github.com/portainer/portainer/pkg/libhelm/options"
+	_ "github.com/portainer/portainer/pkg/libhelm/release"
+	httperror "github.com/portainer/portainer/pkg/libhttp/error"
+	"github.com/portainer/portainer/pkg/libhttp/request"
+	"github.com/portainer/portainer/pkg/libhttp/response"
+)
+
+// @id HelmGetHistory
+// @summary Get a historical list of releases
+// @description Get a historical list of releases by release name
+// @description **Access policy**: authenticated
+// @tags helm
+// @security ApiKeyAuth || jwt
+// @produce json
+// @param id path int true "Environment(Endpoint) identifier"
+// @param name path string true "Helm release name"
+// @param namespace query string false "specify an optional namespace"
+// @success 200 {array} release.Release "Success"
+// @failure 400 "Invalid request payload, such as missing required fields or fields not meeting validation criteria."
+// @failure 401 "Unauthorized access - the user is not authenticated or does not have the necessary permissions. Ensure that you have provided a valid API key or JWT token, and that you have the required permissions."
+// @failure 403 "Permission denied - the user is authenticated but does not have the necessary permissions to access the requested resource or perform the specified operation. Check your user roles and permissions."
+// @failure 404 "Unable to find an environment with the specified identifier."
+// @failure 500 "Server error occurred while attempting to retrieve the historical list of releases."
+// @router /endpoints/{id}/kubernetes/helm/{release}/history [get]
+func (handler *Handler) helmGetHistory(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	release, err := request.RetrieveRouteVariableValue(r, "release")
+	if err != nil {
+		return httperror.BadRequest("No release specified", err)
+	}
+
+	clusterAccess, httperr := handler.getHelmClusterAccess(r)
+	if httperr != nil {
+		return httperr
+	}
+
+	historyOptions := options.HistoryOptions{
+		KubernetesClusterAccess: clusterAccess,
+		Name:                    release,
+	}
+
+	// optional namespace.  The library defaults to "default"
+	namespace, _ := request.RetrieveQueryParameter(r, "namespace", true)
+	if namespace != "" {
+		historyOptions.Namespace = namespace
+	}
+
+	releases, err := handler.helmPackageManager.GetHistory(historyOptions)
+	if err != nil {
+		return httperror.InternalServerError("Helm returned an error", err)
+	}
+
+	return response.JSON(w, releases)
+}

api/http/handler/helm/helm_history_test.go

@@ -0,0 +1,67 @@
+package helm
+
+import (
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/datastore"
+	"github.com/portainer/portainer/api/exec/exectest"
+	"github.com/portainer/portainer/api/http/security"
+	"github.com/portainer/portainer/api/internal/testhelpers"
+	helper "github.com/portainer/portainer/api/internal/testhelpers"
+	"github.com/portainer/portainer/api/jwt"
+	"github.com/portainer/portainer/api/kubernetes"
+	"github.com/portainer/portainer/pkg/libhelm/options"
+	"github.com/portainer/portainer/pkg/libhelm/release"
+	"github.com/portainer/portainer/pkg/libhelm/test"
+
+	"github.com/segmentio/encoding/json"
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_helmGetHistory(t *testing.T) {
+	is := assert.New(t)
+
+	_, store := datastore.MustNewTestStore(t, true, true)
+
+	err := store.Endpoint().Create(&portainer.Endpoint{ID: 1})
+	is.NoError(err, "Error creating environment")
+
+	err = store.User().Create(&portainer.User{Username: "admin", Role: portainer.AdministratorRole})
+	is.NoError(err, "Error creating a user")
+
+	jwtService, err := jwt.NewService("1h", store)
+	is.NoError(err, "Error initiating jwt service")
+
+	kubernetesDeployer := exectest.NewKubernetesDeployer()
+	helmPackageManager := test.NewMockHelmPackageManager()
+	kubeClusterAccessService := kubernetes.NewKubeClusterAccessService("", "", "")
+	h := NewHandler(helper.NewTestRequestBouncer(), store, jwtService, kubernetesDeployer, helmPackageManager, kubeClusterAccessService)
+
+	is.NotNil(h, "Handler should not fail")
+
+	// Install a single chart, to be retrieved by the handler
+	options := options.InstallOptions{Name: "nginx-1", Chart: "nginx", Namespace: "default"}
+	h.helmPackageManager.Upgrade(options)
+
+	t.Run("helmGetHistory sucessfuly retrieves helm release history", func(t *testing.T) {
+		req := httptest.NewRequest(http.MethodGet, "/1/kubernetes/helm/"+options.Name+"/history?namespace="+options.Namespace, nil)
+		ctx := security.StoreTokenData(req, &portainer.TokenData{ID: 1, Username: "admin", Role: 1})
+		req = req.WithContext(ctx)
+		testhelpers.AddTestSecurityCookie(req, "Bearer dummytoken")
+
+		rr := httptest.NewRecorder()
+		h.ServeHTTP(rr, req)
+
+		data := []release.Release{}
+		body, err := io.ReadAll(rr.Body)
+		is.NoError(err, "ReadAll should not return error")
+		json.Unmarshal(body, &data)
+		is.Equal(http.StatusOK, rr.Code, "Status should be 200")
+		is.Equal(1, len(data))
+		is.Equal("nginx-1", data[0].Name)
+	})
+}

api/http/handler/helm/helm_install.go

@@ -26,6 +26,8 @@ type installChartPayload struct {
 	Chart     string `json:"chart"`
 	Repo      string `json:"repo"`
 	Values    string `json:"values"`
+	Version   string `json:"version"`
+	Atomic    bool   `json:"atomic"`
 }
 
 var errChartNameInvalid = errors.New("invalid chart name. " +
@@ -101,8 +103,10 @@ func (handler *Handler) installChart(r *http.Request, p installChartPayload) (*r
 	installOpts := options.InstallOptions{
 		Name:                    p.Name,
 		Chart:                   p.Chart,
+		Version:                 p.Version,
 		Namespace:               p.Namespace,
 		Repo:                    p.Repo,
+		Atomic:                  p.Atomic,
 		KubernetesClusterAccess: clusterAccess,
 	}
 
@@ -125,7 +129,7 @@ func (handler *Handler) installChart(r *http.Request, p installChartPayload) (*r
 		installOpts.ValuesFile = file.Name()
 	}
 
-	release, err := handler.helmPackageManager.Install(installOpts)
+	release, err := handler.helmPackageManager.Upgrade(installOpts)
 	if err != nil {
 		return nil, err
 	}
@@ -192,7 +196,7 @@ func (handler *Handler) updateHelmAppManifest(r *http.Request, manifest []byte,
 	g := new(errgroup.Group)
 	for _, resource := range yamlResources {
 		g.Go(func() error {
-			tmpfile, err := os.CreateTemp("", "helm-manifest-*")
+			tmpfile, err := os.CreateTemp("", "helm-manifest-*.yaml")
 			if err != nil {
 				return errors.Wrap(err, "failed to create a tmp helm manifest file")
 			}

api/http/handler/helm/helm_list_test.go

@@ -43,7 +43,7 @@ func Test_helmList(t *testing.T) {
 
 	// Install a single chart.  We expect to get these values back
 	options := options.InstallOptions{Name: "nginx-1", Chart: "nginx", Namespace: "default"}
-	h.helmPackageManager.Install(options)
+	h.helmPackageManager.Upgrade(options)
 
 	t.Run("helmList", func(t *testing.T) {
 		req := httptest.NewRequest(http.MethodGet, "/1/kubernetes/helm", nil)

api/http/handler/helm/helm_repo_search.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/portainer/portainer/pkg/libhelm/options"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
+	"github.com/portainer/portainer/pkg/libhttp/request"
 
 	"github.com/pkg/errors"
 )
@@ -17,6 +18,8 @@ import (
 // @description **Access policy**: authenticated
 // @tags helm
 // @param repo query string true "Helm repository URL"
+// @param chart query string false "Helm chart name"
+// @param useCache query string false "If true will use cache to search"
 // @security ApiKeyAuth
 // @security jwt
 // @produce json
@@ -32,13 +35,19 @@ func (handler *Handler) helmRepoSearch(w http.ResponseWriter, r *http.Request) *
 		return httperror.BadRequest("Bad request", errors.New("missing `repo` query parameter"))
 	}
 
+	chart, _ := request.RetrieveQueryParameter(r, "chart", false)
+	// If true will useCache to search, will always add to cache after
+	useCache, _ := request.RetrieveBooleanQueryParameter(r, "useCache", false)
+
 	_, err := url.ParseRequestURI(repo)
 	if err != nil {
 		return httperror.BadRequest("Bad request", errors.Wrap(err, fmt.Sprintf("provided URL %q is not valid", repo)))
 	}
 
 	searchOpts := options.SearchRepoOptions{
-		Repo: repo,
+		Repo:     repo,
+		Chart:    chart,
+		UseCache: useCache,
 	}
 
 	result, err := handler.helmPackageManager.SearchRepo(searchOpts)

api/http/handler/helm/helm_rollback.go

@@ -0,0 +1,105 @@
+package helm
+
+import (
+	"net/http"
+	"time"
+
+	"github.com/portainer/portainer/pkg/libhelm/options"
+	_ "github.com/portainer/portainer/pkg/libhelm/release"
+	httperror "github.com/portainer/portainer/pkg/libhttp/error"
+	"github.com/portainer/portainer/pkg/libhttp/request"
+	"github.com/portainer/portainer/pkg/libhttp/response"
+)
+
+// @id HelmRollback
+// @summary Rollback a helm release
+// @description Rollback a helm release to a previous revision
+// @description **Access policy**: authenticated
+// @tags helm
+// @security ApiKeyAuth || jwt
+// @produce json
+// @param id path int true "Environment(Endpoint) identifier"
+// @param release path string true "Helm release name"
+// @param namespace query string false "specify an optional namespace"
+// @param revision query int false "specify the revision to rollback to (defaults to previous revision if not specified)"
+// @param wait query boolean false "wait for resources to be ready (default: false)"
+// @param waitForJobs query boolean false "wait for jobs to complete before marking the release as successful (default: false)"
+// @param recreate query boolean false "performs pods restart for the resource if applicable (default: true)"
+// @param force query boolean false "force resource update through delete/recreate if needed (default: false)"
+// @param timeout query int false "time to wait for any individual Kubernetes operation in seconds (default: 300)"
+// @success 200 {object} release.Release "Success"
+// @failure 400 "Invalid request payload, such as missing required fields or fields not meeting validation criteria."
+// @failure 401 "Unauthorized access - the user is not authenticated or does not have the necessary permissions. Ensure that you have provided a valid API key or JWT token, and that you have the required permissions."
+// @failure 403 "Permission denied - the user is authenticated but does not have the necessary permissions to access the requested resource or perform the specified operation. Check your user roles and permissions."
+// @failure 404 "Unable to find an environment with the specified identifier or release name."
+// @failure 500 "Server error occurred while attempting to rollback the release."
+// @router /endpoints/{id}/kubernetes/helm/{release}/rollback [post]
+func (handler *Handler) helmRollback(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	release, err := request.RetrieveRouteVariableValue(r, "release")
+	if err != nil {
+		return httperror.BadRequest("No release specified", err)
+	}
+
+	clusterAccess, httperr := handler.getHelmClusterAccess(r)
+	if httperr != nil {
+		return httperr
+	}
+
+	// build the rollback options
+	rollbackOpts := options.RollbackOptions{
+		KubernetesClusterAccess: clusterAccess,
+		Name:                    release,
+		// Set default values
+		Recreate: true,            // Default to recreate pods (restart)
+		Timeout:  5 * time.Minute, // Default timeout of 5 minutes
+	}
+
+	namespace, _ := request.RetrieveQueryParameter(r, "namespace", true)
+	// optional namespace. The library defaults to "default"
+	if namespace != "" {
+		rollbackOpts.Namespace = namespace
+	}
+
+	revision, _ := request.RetrieveNumericQueryParameter(r, "revision", true)
+	// optional revision. If not specified, it will rollback to the previous revision
+	if revision > 0 {
+		rollbackOpts.Version = revision
+	}
+
+	// Default for wait is false, only set to true if explicitly requested
+	wait, err := request.RetrieveBooleanQueryParameter(r, "wait", true)
+	if err == nil {
+		rollbackOpts.Wait = wait
+	}
+
+	// Default for waitForJobs is false, only set to true if explicitly requested
+	waitForJobs, err := request.RetrieveBooleanQueryParameter(r, "waitForJobs", true)
+	if err == nil {
+		rollbackOpts.WaitForJobs = waitForJobs
+	}
+
+	// Default for recreate is true (set above), override if specified
+	recreate, err := request.RetrieveBooleanQueryParameter(r, "recreate", true)
+	if err == nil {
+		rollbackOpts.Recreate = recreate
+	}
+
+	// Default for force is false, only set to true if explicitly requested
+	force, err := request.RetrieveBooleanQueryParameter(r, "force", true)
+	if err == nil {
+		rollbackOpts.Force = force
+	}
+
+	timeout, _ := request.RetrieveNumericQueryParameter(r, "timeout", true)
+	// Override default timeout if specified
+	if timeout > 0 {
+		rollbackOpts.Timeout = time.Duration(timeout) * time.Second
+	}
+
+	releaseInfo, err := handler.helmPackageManager.Rollback(rollbackOpts)
+	if err != nil {
+		return httperror.InternalServerError("Failed to rollback helm release", err)
+	}
+
+	return response.JSON(w, releaseInfo)
+}

api/http/handler/helm/helm_show.go

@@ -20,6 +20,7 @@ import (
 // @tags helm
 // @param repo query string true "Helm repository URL"
 // @param chart query string true "Chart name"
+// @param version query string true "Chart version"
 // @param command path string true "chart/values/readme"
 // @security ApiKeyAuth
 // @security jwt
@@ -45,6 +46,11 @@ func (handler *Handler) helmShow(w http.ResponseWriter, r *http.Request) *httper
 		return httperror.BadRequest("Bad request", errors.New("missing `chart` query parameter"))
 	}
 
+	version, err := request.RetrieveQueryParameter(r, "version", true)
+	if err != nil {
+		return httperror.BadRequest("Bad request", errors.Wrap(err, fmt.Sprintf("provided version %q is not valid", version)))
+	}
+
 	cmd, err := request.RetrieveRouteVariableValue(r, "command")
 	if err != nil {
 		cmd = "all"
@@ -55,6 +61,7 @@ func (handler *Handler) helmShow(w http.ResponseWriter, r *http.Request) *httper
 		OutputFormat: options.ShowOutputFormat(cmd),
 		Chart:        chart,
 		Repo:         repo,
+		Version:      version,
 	}
 	result, err := handler.helmPackageManager.Show(showOptions)
 	if err != nil {

api/http/handler/kubernetes/client.go

@@ -30,8 +30,8 @@ func (handler *Handler) prepareKubeClient(r *http.Request) (*cli.KubeClient, *ht
 		log.Error().Err(err).Str("context", "prepareKubeClient").Msg("Unable to get a privileged Kubernetes client for the user.")
 		return nil, httperror.InternalServerError("Unable to get a privileged Kubernetes client for the user.", err)
 	}
-	pcli.IsKubeAdmin = cli.IsKubeAdmin
-	pcli.NonAdminNamespaces = cli.NonAdminNamespaces
+	pcli.SetIsKubeAdmin(cli.GetIsKubeAdmin())
+	pcli.SetClientNonAdminNamespaces(cli.GetClientNonAdminNamespaces())
 
 	return pcli, nil
 }

api/http/handler/kubernetes/cluster_role_bindings.go

@@ -32,7 +32,7 @@ func (handler *Handler) getAllKubernetesClusterRoleBindings(w http.ResponseWrite
 		return httperror.Forbidden("User is not authorized to fetch cluster role bindings from the Kubernetes cluster.", httpErr)
 	}
 
-	if !cli.IsKubeAdmin {
+	if !cli.GetIsKubeAdmin() {
 		log.Error().Str("context", "getAllKubernetesClusterRoleBindings").Msg("user is not authorized to fetch cluster role bindings from the Kubernetes cluster.")
 		return httperror.Forbidden("User is not authorized to fetch cluster role bindings from the Kubernetes cluster.", nil)
 	}

api/http/handler/kubernetes/cluster_roles.go

@@ -32,7 +32,7 @@ func (handler *Handler) getAllKubernetesClusterRoles(w http.ResponseWriter, r *h
 		return httperror.Forbidden("User is not authorized to fetch cluster roles from the Kubernetes cluster.", httpErr)
 	}
 
-	if !cli.IsKubeAdmin {
+	if !cli.GetIsKubeAdmin() {
 		log.Error().Str("context", "getAllKubernetesClusterRoles").Msg("user is not authorized to fetch cluster roles from the Kubernetes cluster.")
 		return httperror.Forbidden("User is not authorized to fetch cluster roles from the Kubernetes cluster.", nil)
 	}

api/http/handler/kubernetes/config.go

@@ -167,6 +167,16 @@ func (handler *Handler) buildConfig(r *http.Request, tokenData *portainer.TokenD
 func (handler *Handler) buildCluster(r *http.Request, endpoint portainer.Endpoint, isInternal bool) clientV1.NamedCluster {
 	kubeConfigInternal := handler.kubeClusterAccessService.GetClusterDetails(r.Host, endpoint.ID, isInternal)
 
+	if isInternal {
+		return clientV1.NamedCluster{
+			Name: buildClusterName(endpoint.Name),
+			Cluster: clientV1.Cluster{
+				Server:                kubeConfigInternal.ClusterServerURL,
+				InsecureSkipTLSVerify: true,
+			},
+		}
+	}
+
 	selfSignedCert := false
 	serverUrl, err := url.Parse(kubeConfigInternal.ClusterServerURL)
 	if err != nil {

api/http/handler/kubernetes/configmaps.go

@@ -146,13 +146,11 @@ func (handler *Handler) getAllKubernetesConfigMaps(r *http.Request) ([]models.K8
 	}
 
 	if isUsed {
-		configMapsWithApplications, err := cli.CombineConfigMapsWithApplications(configMaps)
+		err = cli.SetConfigMapsIsUsed(&configMaps)
 		if err != nil {
 			log.Error().Err(err).Str("context", "getAllKubernetesConfigMaps").Msg("Unable to combine configMaps with associated applications")
 			return nil, httperror.InternalServerError("Unable to combine configMaps with associated applications", err)
 		}
-
-		return configMapsWithApplications, nil
 	}
 
 	return configMaps, nil

api/http/handler/kubernetes/deprecated_routes.go

@@ -36,11 +36,14 @@ func deprecatedNamespaceParser(w http.ResponseWriter, r *http.Request) (string,
 
 	// Restore the original body for further use
 	bodyBytes, err := io.ReadAll(r.Body)
+	if err != nil {
+		return "", httperror.InternalServerError("Unable to read request body", err)
+	}
+
 	r.Body = io.NopCloser(bytes.NewBuffer(bodyBytes))
 
 	payload := models.K8sNamespaceDetails{}
-	err = request.DecodeAndValidateJSONPayload(r, &payload)
-	if err != nil {
+	if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil {
 		return "", httperror.BadRequest("Invalid request. Unable to parse namespace payload", err)
 	}
 	namespaceName := payload.Name

api/http/handler/kubernetes/describe.go

@@ -0,0 +1,73 @@
+package kubernetes
+
+import (
+	"net/http"
+
+	httperror "github.com/portainer/portainer/pkg/libhttp/error"
+	"github.com/portainer/portainer/pkg/libhttp/request"
+	"github.com/portainer/portainer/pkg/libhttp/response"
+	"github.com/portainer/portainer/pkg/libkubectl"
+	"github.com/rs/zerolog/log"
+)
+
+type describeResourceResponse struct {
+	Describe string `json:"describe"`
+}
+
+// @id DescribeResource
+// @summary Get a description of a kubernetes resource
+// @description Get a description of a kubernetes resource.
+// @description **Access policy**: Authenticated user.
+// @tags kubernetes
+// @security ApiKeyAuth || jwt
+// @produce json
+// @param id path int true "Environment identifier"
+// @param name query string true "Resource name"
+// @param kind query string true "Resource kind"
+// @param namespace query string false "Namespace"
+// @success 200 {object} describeResourceResponse "Success"
+// @failure 400 "Invalid request payload, such as missing required fields or fields not meeting validation criteria."
+// @failure 401 "Unauthorized access - the user is not authenticated or does not have the necessary permissions. Ensure that you have provided a valid API key or JWT token, and that you have the required permissions."
+// @failure 403 "Permission denied - the user is authenticated but does not have the necessary permissions to access the requested resource or perform the specified operation. Check your user roles and permissions."
+// @failure 404 "Unable to find an environment with the specified identifier."
+// @failure 500 "Server error occurred while attempting to retrieve resource description"
+// @router /kubernetes/{id}/describe [get]
+func (handler *Handler) describeResource(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	name, err := request.RetrieveQueryParameter(r, "name", false)
+	if err != nil {
+		log.Error().Err(err).Str("context", "describeResource").Msg("Invalid query parameter name")
+		return httperror.BadRequest("an error occurred during the describeResource operation, invalid query parameter name. Error: ", err)
+	}
+
+	kind, err := request.RetrieveQueryParameter(r, "kind", false)
+	if err != nil {
+		log.Error().Err(err).Str("context", "describeResource").Msg("Invalid query parameter kind")
+		return httperror.BadRequest("an error occurred during the describeResource operation, invalid query parameter kind. Error: ", err)
+	}
+
+	namespace, err := request.RetrieveQueryParameter(r, "namespace", true)
+	if err != nil {
+		log.Error().Err(err).Str("context", "describeResource").Msg("Invalid query parameter namespace")
+		return httperror.BadRequest("an error occurred during the describeResource operation, invalid query parameter namespace. Error: ", err)
+	}
+
+	// fetches the token and the correct server URL for the endpoint, similar to getHelmClusterAccess
+	libKubectlAccess, err := handler.getLibKubectlAccess(r)
+	if err != nil {
+		return httperror.InternalServerError("an error occurred during the describeResource operation, failed to get libKubectlAccess. Error: ", err)
+	}
+
+	client, err := libkubectl.NewClient(libKubectlAccess, namespace, "", true)
+	if err != nil {
+		log.Error().Err(err).Str("context", "describeResource").Msg("Failed to create kubernetes client")
+		return httperror.InternalServerError("an error occurred during the describeResource operation, failed to create kubernetes client. Error: ", err)
+	}
+
+	out, err := client.Describe(namespace, name, kind)
+	if err != nil {
+		log.Error().Err(err).Str("context", "describeResource").Msg("Failed to describe kubernetes resource")
+		return httperror.InternalServerError("an error occurred during the describeResource operation, failed to describe kubernetes resource. Error: ", err)
+	}
+
+	return response.JSON(w, describeResourceResponse{Describe: out})
+}

api/http/handler/kubernetes/event.go

@@ -0,0 +1,102 @@
+package kubernetes
+
+import (
+	"net/http"
+
+	httperror "github.com/portainer/portainer/pkg/libhttp/error"
+	"github.com/portainer/portainer/pkg/libhttp/request"
+	"github.com/portainer/portainer/pkg/libhttp/response"
+	"github.com/rs/zerolog/log"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+)
+
+// @id getKubernetesEventsForNamespace
+// @summary Gets kubernetes events for namespace
+// @description Get events by optional query param resourceId for a given namespace.
+// @description **Access policy**: Authenticated user.
+// @tags kubernetes
+// @security ApiKeyAuth || jwt
+// @produce json
+// @param id path int true "Environment identifier"
+// @param namespace path string true "The namespace name the events are associated to"
+// @param resourceId query string false "The resource id of the involved kubernetes object" example:"e5b021b6-4bce-4c06-bd3b-6cca906797aa"
+// @success 200 {object} models.Event[] "Success"
+// @failure 400 "Invalid request payload, such as missing required fields or fields not meeting validation criteria."
+// @failure 401 "Unauthorized access - the user is not authenticated or does not have the necessary permissions. Ensure that you have provided a valid API key or JWT token, and that you have the required permissions."
+// @failure 403 "Permission denied - the user is authenticated but does not have the necessary permissions to access the requested resource or perform the specified operation. Check your user roles and permissions."
+// @failure 500 "Server error occurred while attempting to retrieve the events within the specified namespace."
+// @router /kubernetes/{id}/namespaces/{namespace}/events [get]
+func (handler *Handler) getKubernetesEventsForNamespace(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	namespace, err := request.RetrieveRouteVariableValue(r, "namespace")
+	if err != nil {
+		log.Error().Err(err).Str("context", "getKubernetesEvents").Str("namespace", namespace).Msg("Unable to retrieve namespace identifier route variable")
+		return httperror.BadRequest("Unable to retrieve namespace identifier route variable", err)
+	}
+
+	resourceId, err := request.RetrieveQueryParameter(r, "resourceId", true)
+	if err != nil {
+		log.Error().Err(err).Str("context", "getKubernetesEvents").Msg("Unable to retrieve resourceId query parameter")
+		return httperror.BadRequest("Unable to retrieve resourceId query parameter", err)
+	}
+
+	cli, httpErr := handler.getProxyKubeClient(r)
+	if httpErr != nil {
+		log.Error().Err(httpErr).Str("context", "getKubernetesEvents").Str("resourceId", resourceId).Msg("Unable to get a Kubernetes client for the user")
+		return httperror.InternalServerError("Unable to get a Kubernetes client for the user", httpErr)
+	}
+
+	events, err := cli.GetEvents(namespace, resourceId)
+	if err != nil {
+		if k8serrors.IsUnauthorized(err) || k8serrors.IsForbidden(err) {
+			log.Error().Err(err).Str("context", "getKubernetesEvents").Msg("Unauthorized access to the Kubernetes API")
+			return httperror.Forbidden("Unauthorized access to the Kubernetes API", err)
+		}
+
+		log.Error().Err(err).Str("context", "getKubernetesEvents").Msg("Unable to retrieve events")
+		return httperror.InternalServerError("Unable to retrieve events", err)
+	}
+
+	return response.JSON(w, events)
+}
+
+// @id getAllKubernetesEvents
+// @summary Gets kubernetes events
+// @description Get events by query param resourceId
+// @description **Access policy**: Authenticated user.
+// @tags kubernetes
+// @security ApiKeyAuth || jwt
+// @produce json
+// @param id path int true "Environment identifier"
+// @param resourceId query string false "The resource id of the involved kubernetes object"  example:"e5b021b6-4bce-4c06-bd3b-6cca906797aa"
+// @success 200 {object} models.Event[] "Success"
+// @failure 400 "Invalid request payload, such as missing required fields or fields not meeting validation criteria."
+// @failure 401 "Unauthorized access - the user is not authenticated or does not have the necessary permissions. Ensure that you have provided a valid API key or JWT token, and that you have the required permissions."
+// @failure 403 "Permission denied - the user is authenticated but does not have the necessary permissions to access the requested resource or perform the specified operation. Check your user roles and permissions."
+// @failure 500 "Server error occurred while attempting to retrieve the events."
+// @router /kubernetes/{id}/events [get]
+func (handler *Handler) getAllKubernetesEvents(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	resourceId, err := request.RetrieveQueryParameter(r, "resourceId", true)
+	if err != nil {
+		log.Error().Err(err).Str("context", "getKubernetesEvents").Msg("Unable to retrieve resourceId query parameter")
+		return httperror.BadRequest("Unable to retrieve resourceId query parameter", err)
+	}
+
+	cli, httpErr := handler.getProxyKubeClient(r)
+	if httpErr != nil {
+		log.Error().Err(httpErr).Str("context", "getKubernetesEvents").Str("resourceId", resourceId).Msg("Unable to get a Kubernetes client for the user")
+		return httperror.InternalServerError("Unable to get a Kubernetes client for the user", httpErr)
+	}
+
+	events, err := cli.GetEvents("", resourceId)
+	if err != nil {
+		if k8serrors.IsUnauthorized(err) || k8serrors.IsForbidden(err) {
+			log.Error().Err(err).Str("context", "getKubernetesEvents").Msg("Unauthorized access to the Kubernetes API")
+			return httperror.Forbidden("Unauthorized access to the Kubernetes API", err)
+		}
+
+		log.Error().Err(err).Str("context", "getKubernetesEvents").Msg("Unable to retrieve events")
+		return httperror.InternalServerError("Unable to retrieve events", err)
+	}
+
+	return response.JSON(w, events)
+}

api/http/handler/kubernetes/event_test.go

@@ -0,0 +1,60 @@
+package kubernetes
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/datastore"
+	"github.com/portainer/portainer/api/http/security"
+	"github.com/portainer/portainer/api/internal/authorization"
+	"github.com/portainer/portainer/api/internal/testhelpers"
+	"github.com/portainer/portainer/api/jwt"
+	"github.com/portainer/portainer/api/kubernetes"
+	kubeClient "github.com/portainer/portainer/api/kubernetes/cli"
+	"github.com/stretchr/testify/assert"
+)
+
+// Currently this test just tests the HTTP Handler is setup correctly, in the future we should move the ClientFactory to a mock in order
+// test the logic in event.go
+func TestGetKubernetesEvents(t *testing.T) {
+	is := assert.New(t)
+
+	_, store := datastore.MustNewTestStore(t, true, true)
+
+	err := store.Endpoint().Create(&portainer.Endpoint{
+		ID:   1,
+		Type: portainer.AgentOnKubernetesEnvironment,
+	},
+	)
+	is.NoError(err, "error creating environment")
+
+	err = store.User().Create(&portainer.User{Username: "admin", Role: portainer.AdministratorRole})
+	is.NoError(err, "error creating a user")
+
+	jwtService, err := jwt.NewService("1h", store)
+	is.NoError(err, "Error initiating jwt service")
+
+	tk, _, _ := jwtService.GenerateToken(&portainer.TokenData{ID: 1, Username: "admin", Role: portainer.AdministratorRole})
+
+	kubeClusterAccessService := kubernetes.NewKubeClusterAccessService("", "", "")
+
+	cli := testhelpers.NewKubernetesClient()
+	factory, _ := kubeClient.NewClientFactory(nil, nil, store, "", "", "")
+
+	authorizationService := authorization.NewService(store)
+	handler := NewHandler(testhelpers.NewTestRequestBouncer(), authorizationService, store, jwtService, kubeClusterAccessService,
+		factory, cli)
+	is.NotNil(handler, "Handler should not fail")
+
+	req := httptest.NewRequest(http.MethodGet, "/kubernetes/1/events?resourceId=8", nil)
+	ctx := security.StoreTokenData(req, &portainer.TokenData{ID: 1, Username: "admin", Role: 1})
+	req = req.WithContext(ctx)
+	testhelpers.AddTestSecurityCookie(req, tk)
+
+	rr := httptest.NewRecorder()
+	handler.ServeHTTP(rr, req)
+
+	is.Equal(http.StatusOK, rr.Code, "Status should be 200")
+}

api/http/handler/kubernetes/handler.go

@@ -15,6 +15,7 @@ import (
 	"github.com/portainer/portainer/api/kubernetes/cli"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
+	"github.com/portainer/portainer/pkg/libkubectl"
 	"github.com/rs/zerolog/log"
 
 	"github.com/gorilla/mux"
@@ -57,6 +58,7 @@ func NewHandler(bouncer security.BouncerService, authorizationService *authoriza
 	endpointRouter.Handle("/configmaps/count", httperror.LoggerHandler(h.getAllKubernetesConfigMapsCount)).Methods(http.MethodGet)
 	endpointRouter.Handle("/cron_jobs", httperror.LoggerHandler(h.getAllKubernetesCronJobs)).Methods(http.MethodGet)
 	endpointRouter.Handle("/cron_jobs/delete", httperror.LoggerHandler(h.deleteKubernetesCronJobs)).Methods(http.MethodPost)
+	endpointRouter.Handle("/events", httperror.LoggerHandler(h.getAllKubernetesEvents)).Methods(http.MethodGet)
 	endpointRouter.Handle("/jobs", httperror.LoggerHandler(h.getAllKubernetesJobs)).Methods(http.MethodGet)
 	endpointRouter.Handle("/jobs/delete", httperror.LoggerHandler(h.deleteKubernetesJobs)).Methods(http.MethodPost)
 	endpointRouter.Handle("/cluster_roles", httperror.LoggerHandler(h.getAllKubernetesClusterRoles)).Methods(http.MethodGet)
@@ -102,12 +104,14 @@ func NewHandler(bouncer security.BouncerService, authorizationService *authoriza
 	endpointRouter.Handle("/cluster_roles/delete", httperror.LoggerHandler(h.deleteClusterRoles)).Methods(http.MethodPost)
 	endpointRouter.Handle("/cluster_role_bindings", httperror.LoggerHandler(h.getAllKubernetesClusterRoleBindings)).Methods(http.MethodGet)
 	endpointRouter.Handle("/cluster_role_bindings/delete", httperror.LoggerHandler(h.deleteClusterRoleBindings)).Methods(http.MethodPost)
+	endpointRouter.Handle("/describe", httperror.LoggerHandler(h.describeResource)).Methods(http.MethodGet)
 
 	// namespaces
 	// in the future this piece of code might be in another package (or a few different packages - namespaces/namespace?)
 	// to keep it simple, we've decided to leave it like this.
 	namespaceRouter := endpointRouter.PathPrefix("/namespaces/{namespace}").Subrouter()
 	namespaceRouter.Handle("/configmaps/{configmap}", httperror.LoggerHandler(h.getKubernetesConfigMap)).Methods(http.MethodGet)
+	namespaceRouter.Handle("/events", httperror.LoggerHandler(h.getKubernetesEventsForNamespace)).Methods(http.MethodGet)
 	namespaceRouter.Handle("/system", bouncer.RestrictedAccess(httperror.LoggerHandler(h.namespacesToggleSystem))).Methods(http.MethodPut)
 	namespaceRouter.Handle("/ingresscontrollers", httperror.LoggerHandler(h.getKubernetesIngressControllersByNamespace)).Methods(http.MethodGet)
 	namespaceRouter.Handle("/ingresscontrollers", httperror.LoggerHandler(h.updateKubernetesIngressControllersByNamespace)).Methods(http.MethodPut)
@@ -131,7 +135,7 @@ func NewHandler(bouncer security.BouncerService, authorizationService *authoriza
 // getProxyKubeClient gets a kubeclient for the user.  It's generally what you want as it retrieves the kubeclient
 // from the Authorization token of the currently logged in user.  The kubeclient that is not from the proxy is actually using
 // admin permissions.  If you're unsure which one to use, use this.
-func (h *Handler) getProxyKubeClient(r *http.Request) (*cli.KubeClient, *httperror.HandlerError) {
+func (h *Handler) getProxyKubeClient(r *http.Request) (portainer.KubeClient, *httperror.HandlerError) {
 	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
 	if err != nil {
 		return nil, httperror.BadRequest(fmt.Sprintf("an error occurred during the getProxyKubeClient operation, the environment identifier route variable is invalid for /api/kubernetes/%d. Error: ", endpointID), err)
@@ -251,7 +255,7 @@ func (handler *Handler) kubeClientMiddleware(next http.Handler) http.Handler {
 			return
 		}
 		serverURL.Scheme = "https"
-		serverURL.Host = "localhost" + handler.KubernetesClientFactory.AddrHTTPS
+		serverURL.Host = "localhost" + handler.KubernetesClientFactory.GetAddrHTTPS()
 		config.Clusters[0].Cluster.Server = serverURL.String()
 
 		yaml, err := cli.GenerateYAML(config)
@@ -269,3 +273,36 @@ func (handler *Handler) kubeClientMiddleware(next http.Handler) http.Handler {
 		next.ServeHTTP(w, r)
 	})
 }
+
+func (handler *Handler) getLibKubectlAccess(r *http.Request) (*libkubectl.ClientAccess, error) {
+	tokenData, err := security.RetrieveTokenData(r)
+	if err != nil {
+		return nil, httperror.InternalServerError("Unable to retrieve user authentication token", err)
+	}
+
+	bearerToken, _, err := handler.JwtService.GenerateToken(tokenData)
+	if err != nil {
+		return nil, httperror.Unauthorized("Unauthorized", err)
+	}
+
+	endpoint, err := middlewares.FetchEndpoint(r)
+	if err != nil {
+		return nil, httperror.InternalServerError("Unable to find the Kubernetes endpoint associated to the request.", err)
+	}
+
+	sslSettings, err := handler.DataStore.SSLSettings().Settings()
+	if err != nil {
+		return nil, httperror.InternalServerError("Unable to retrieve settings from the database", err)
+	}
+
+	hostURL := "localhost"
+	if !sslSettings.SelfSigned {
+		hostURL = r.Host
+	}
+
+	kubeConfigInternal := handler.kubeClusterAccessService.GetClusterDetails(hostURL, endpoint.ID, true)
+	return &libkubectl.ClientAccess{
+		Token:     bearerToken,
+		ServerUrl: kubeConfigInternal.ClusterServerURL,
+	}, nil
+}

api/http/handler/kubernetes/secrets.go

@@ -130,13 +130,11 @@ func (handler *Handler) getAllKubernetesSecrets(r *http.Request) ([]models.K8sSe
 	}
 
 	if isUsed {
-		secretsWithApplications, err := cli.CombineSecretsWithApplications(secrets)
+		err = cli.SetSecretsIsUsed(&secrets)
 		if err != nil {
 			log.Error().Err(err).Str("context", "GetAllKubernetesSecrets").Msg("Unable to combine secrets with associated applications")
 			return nil, httperror.InternalServerError("unable to combine secrets with associated applications. Error: ", err)
 		}
-
-		return secretsWithApplications, nil
 	}
 
 	return secrets, nil

api/http/handler/motd/motd.go

@@ -7,7 +7,9 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/http/client"
 	"github.com/portainer/portainer/pkg/libcrypto"
+	libclient "github.com/portainer/portainer/pkg/libhttp/client"
 	"github.com/portainer/portainer/pkg/libhttp/response"
+	"github.com/rs/zerolog/log"
 
 	"github.com/segmentio/encoding/json"
 )
@@ -37,6 +39,12 @@ type motdData struct {
 // @success 200 {object} motdResponse
 // @router /motd [get]
 func (handler *Handler) motd(w http.ResponseWriter, r *http.Request) {
+	if err := libclient.ExternalRequestDisabled(portainer.MessageOfTheDayURL); err != nil {
+		log.Debug().Err(err).Msg("External request disabled: MOTD")
+		response.JSON(w, &motdResponse{Message: ""})
+		return
+	}
+
 	motd, err := client.Get(portainer.MessageOfTheDayURL, 0)
 	if err != nil {
 		response.JSON(w, &motdResponse{Message: ""})

api/http/handler/settings/settings_update.go

@@ -14,8 +14,8 @@ import (
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
 	"github.com/portainer/portainer/pkg/libhttp/response"
+	"github.com/portainer/portainer/pkg/validate"
 
-	"github.com/asaskevich/govalidator"
 	"github.com/pkg/errors"
 	"golang.org/x/oauth2"
 )
@@ -62,15 +62,15 @@ func (payload *settingsUpdatePayload) Validate(r *http.Request) error {
 		return errors.New("Invalid authentication method value. Value must be one of: 1 (internal), 2 (LDAP/AD) or 3 (OAuth)")
 	}
 
-	if payload.LogoURL != nil && *payload.LogoURL != "" && !govalidator.IsURL(*payload.LogoURL) {
+	if payload.LogoURL != nil && *payload.LogoURL != "" && !validate.IsURL(*payload.LogoURL) {
 		return errors.New("Invalid logo URL. Must correspond to a valid URL format")
 	}
 
-	if payload.TemplatesURL != nil && *payload.TemplatesURL != "" && !govalidator.IsURL(*payload.TemplatesURL) {
+	if payload.TemplatesURL != nil && *payload.TemplatesURL != "" && !validate.IsURL(*payload.TemplatesURL) {
 		return errors.New("Invalid external templates URL. Must correspond to a valid URL format")
 	}
 
-	if payload.HelmRepositoryURL != nil && *payload.HelmRepositoryURL != "" && !govalidator.IsURL(*payload.HelmRepositoryURL) {
+	if payload.HelmRepositoryURL != nil && *payload.HelmRepositoryURL != "" && !validate.IsURL(*payload.HelmRepositoryURL) {
 		return errors.New("Invalid Helm repository URL. Must correspond to a valid URL format")
 	}
 

api/http/handler/stacks/create_compose_stack.go

@@ -14,8 +14,8 @@ import (
 	"github.com/portainer/portainer/api/stacks/stackutils"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
+	"github.com/portainer/portainer/pkg/validate"
 
-	"github.com/asaskevich/govalidator"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
 )
@@ -205,7 +205,7 @@ func (payload *composeStackFromGitRepositoryPayload) Validate(r *http.Request) e
 	if len(payload.Name) == 0 {
 		return errors.New("Invalid stack name")
 	}
-	if len(payload.RepositoryURL) == 0 || !govalidator.IsURL(payload.RepositoryURL) {
+	if len(payload.RepositoryURL) == 0 || !validate.IsURL(payload.RepositoryURL) {
 		return errors.New("Invalid repository URL. Must correspond to a valid URL format")
 	}
 	if payload.RepositoryAuthentication && len(payload.RepositoryPassword) == 0 {

api/http/handler/stacks/create_kubernetes_stack.go

@@ -15,8 +15,8 @@ import (
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
 	"github.com/portainer/portainer/pkg/libhttp/response"
+	"github.com/portainer/portainer/pkg/validate"
 
-	"github.com/asaskevich/govalidator"
 	"github.com/pkg/errors"
 )
 
@@ -96,7 +96,7 @@ func (payload *kubernetesStringDeploymentPayload) Validate(r *http.Request) erro
 }
 
 func (payload *kubernetesGitDeploymentPayload) Validate(r *http.Request) error {
-	if len(payload.RepositoryURL) == 0 || !govalidator.IsURL(payload.RepositoryURL) {
+	if len(payload.RepositoryURL) == 0 || !validate.IsURL(payload.RepositoryURL) {
 		return errors.New("Invalid repository URL. Must correspond to a valid URL format")
 	}
 
@@ -112,7 +112,7 @@ func (payload *kubernetesGitDeploymentPayload) Validate(r *http.Request) error {
 }
 
 func (payload *kubernetesManifestURLDeploymentPayload) Validate(r *http.Request) error {
-	if len(payload.ManifestURL) == 0 || !govalidator.IsURL(payload.ManifestURL) {
+	if len(payload.ManifestURL) == 0 || !validate.IsURL(payload.ManifestURL) {
 		return errors.New("Invalid manifest URL")
 	}
 

api/http/handler/stacks/create_swarm_stack.go

@@ -11,8 +11,8 @@ import (
 	"github.com/portainer/portainer/api/stacks/stackutils"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
+	valid "github.com/portainer/portainer/pkg/validate"
 
-	"github.com/asaskevich/govalidator"
 	"github.com/pkg/errors"
 )
 
@@ -142,7 +142,7 @@ func (payload *swarmStackFromGitRepositoryPayload) Validate(r *http.Request) err
 	if len(payload.SwarmID) == 0 {
 		return errors.New("Invalid Swarm ID")
 	}
-	if len(payload.RepositoryURL) == 0 || !govalidator.IsURL(payload.RepositoryURL) {
+	if len(payload.RepositoryURL) == 0 || !valid.IsURL(payload.RepositoryURL) {
 		return errors.New("Invalid repository URL. Must correspond to a valid URL format")
 	}
 	if payload.RepositoryAuthentication && len(payload.RepositoryPassword) == 0 {

api/http/handler/system/nodes_count.go

@@ -33,7 +33,7 @@ func (handler *Handler) systemNodesCount(w http.ResponseWriter, r *http.Request)
 	var nodes int
 
 	for _, endpoint := range endpoints {
-		if err := snapshot.FillSnapshotData(handler.dataStore, &endpoint); err != nil {
+		if err := snapshot.FillSnapshotData(handler.dataStore, &endpoint, false); err != nil {
 			return httperror.InternalServerError("Unable to add snapshot data", err)
 		}