[WIP] Optimize buildEdgeStacks()

* chore(edge): add unit tests to edgestatus inspect endpoint EE-3088

.dockerignore

@@ -1,3 +1,5 @@
 *
 !dist
 !build
+!metadata.json
+!docker-extension/build

.github/workflows/lint.yml

@@ -18,17 +18,12 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - name: Check out Git repository
-        uses: actions/checkout@v2
-
-      - name: Set up Node.js
-        uses: actions/setup-node@v1
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
         with:
-          node-version: 12
-
-      # ESLint and Prettier must be in `package.json`
-      - name: Install Node.js dependencies
-        run: yarn --frozen-lockfile
+          node-version: '14'
+          cache: 'yarn'
+      - run: yarn --frozen-lockfile
 
       - name: Run linters
         uses: wearerequired/lint-action@v1
@@ -39,3 +34,5 @@ jobs:
           prettier_dir: app/
           gofmt: true
           gofmt_dir: api/
+      - name: Typecheck
+        uses: icrawl/action-tsc@v1

.github/workflows/nightly-security-scan.yml

@@ -0,0 +1,230 @@
+name: Nightly Code Security Scan
+
+on: 
+  schedule:
+    - cron: '0 8 * * *'
+  workflow_dispatch:
+    
+jobs:
+  client-dependencies:
+    name: Client dependency check
+    runs-on: ubuntu-latest
+    if: >- # only run for develop branch
+      github.ref == 'refs/heads/develop' 
+    outputs:
+      js: ${{ steps.set-matrix.outputs.js_result }}
+    steps:
+      - uses: actions/checkout@master
+
+      - name: Run Snyk to check for vulnerabilities
+        uses: snyk/actions/node@master
+        continue-on-error: true # To make sure that artifact upload gets called
+        env:
+          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+        with:
+          json: true
+
+      - name: Upload js security scan result as artifact 
+        uses: actions/upload-artifact@v3
+        with:
+          name: js-security-scan-develop-result
+          path: snyk.json
+
+      - name: Export scan result to html file 
+        run: | 
+          $(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 summary -report-type=snyk -path="/data/snyk.json" -output-type=table -export -export-filename="/data/js-result")
+
+      - name: Upload js result html file
+        uses: actions/upload-artifact@v3
+        with:
+          name: html-js-result-${{github.run_id}}
+          path: js-result.html
+
+      - name: Analyse the js result
+        id: set-matrix
+        run: | 
+          result=$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 summary -report-type=snyk -path="/data/snyk.json" -output-type=matrix)
+          echo "::set-output name=js_result::${result}"
+
+  server-dependencies:
+    name: Server dependency check
+    runs-on: ubuntu-latest
+    if: >- # only run for develop branch
+      github.ref == 'refs/heads/develop' 
+    outputs:
+      go: ${{ steps.set-matrix.outputs.go_result }}
+    steps:
+      - uses: actions/checkout@master
+
+      - name: Download go modules
+        run: cd ./api && go get -t -v -d ./...
+
+      - name: Run Snyk to check for vulnerabilities
+        uses: snyk/actions/golang@master
+        continue-on-error: true # To make sure that artifact upload gets called
+        env:
+          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+        with:
+          args: --file=./api/go.mod
+          json: true
+
+      - name: Upload go security scan result as artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: go-security-scan-develop-result
+          path: snyk.json
+
+      - name: Export scan result to html file 
+        run: | 
+          $(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 summary -report-type=snyk -path="/data/snyk.json" -output-type=table -export -export-filename="/data/go-result")
+
+      - name: Upload go result html file
+        uses: actions/upload-artifact@v3
+        with:
+          name: html-go-result-${{github.run_id}}
+          path: go-result.html
+
+      - name: Analyse the go result
+        id: set-matrix
+        run: | 
+          result=$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 summary -report-type=snyk -path="/data/snyk.json" -output-type=matrix)
+          echo "::set-output name=go_result::${result}"
+
+  image-vulnerability:
+    name: Build docker image and Image vulnerability check
+    runs-on: ubuntu-latest
+    if: >-
+      github.ref == 'refs/heads/develop'
+    outputs:
+      image: ${{ steps.set-matrix.outputs.image_result }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@master
+
+      - name: Use golang 1.18
+        uses: actions/setup-go@v3
+        with:
+          go-version: '1.18'
+
+      - name: Use Node.js 12.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 12.x
+
+      - name: Install packages and build
+        run: yarn install && yarn build
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: build/linux/Dockerfile
+          tags: trivy-portainer:${{ github.sha }}
+          outputs: type=docker,dest=/tmp/trivy-portainer-image.tar
+
+      - name: Load docker image
+        run: |
+          docker load --input /tmp/trivy-portainer-image.tar
+
+      - name: Run Trivy vulnerability scanner
+        uses: docker://docker.io/aquasec/trivy:latest
+        continue-on-error: true 
+        with:
+          args: image --ignore-unfixed=true --vuln-type="os,library" --exit-code=1 --format="json" --output="image-trivy.json" --no-progress trivy-portainer:${{ github.sha }}  
+
+      - name: Upload image security scan result as artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: image-security-scan-develop-result
+          path: image-trivy.json
+
+      - name: Export scan result to html file 
+        run: | 
+          $(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 summary -report-type=trivy -path="/data/image-trivy.json" -output-type=table -export -export-filename="/data/image-result")
+
+      - name: Upload go result html file
+        uses: actions/upload-artifact@v3
+        with:
+          name: html-image-result-${{github.run_id}}
+          path: image-result.html
+
+      - name: Analyse the trivy result
+        id: set-matrix
+        run: | 
+          result=$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 summary -report-type=trivy -path="/data/image-trivy.json" -output-type=matrix)
+          echo "::set-output name=image_result::${result}"
+
+  result-analysis:
+    name: Analyse scan result
+    needs: [client-dependencies, server-dependencies, image-vulnerability]
+    runs-on: ubuntu-latest
+    if: >-
+      github.ref == 'refs/heads/develop'
+    strategy:
+      matrix: 
+        js: ${{fromJson(needs.client-dependencies.outputs.js)}}
+        go: ${{fromJson(needs.server-dependencies.outputs.go)}}
+        image: ${{fromJson(needs.image-vulnerability.outputs.image)}}
+    steps:
+      - name: Display the results of js, go and image
+        run: |
+          echo ${{ matrix.js.status }}
+          echo ${{ matrix.go.status }}
+          echo ${{ matrix.image.status }}
+          echo ${{ matrix.js.summary }}
+          echo ${{ matrix.go.summary }}
+          echo ${{ matrix.image.summary }}
+
+      - name: Send Slack message
+        if: >- 
+          matrix.js.status == 'failure' ||
+          matrix.go.status == 'failure' || 
+          matrix.image.status == 'failure'
+        uses: slackapi/slack-github-action@v1.18.0
+        with:
+          payload: |
+            {
+              "blocks": [
+                {
+                  "type": "section",
+                  "text": {
+                    "type": "mrkdwn",
+                    "text": "Code Scanning Result (*${{ github.repository }}*)\n*<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|GitHub Actions Workflow URL>*"
+                  }
+                }
+              ],
+              "attachments": [
+                {
+                  "color": "#FF0000",
+                  "blocks": [
+                    {
+                      "type": "section",
+                      "text": {
+                        "type": "mrkdwn",
+                        "text": "*JS dependency check*: *${{ matrix.js.status }}*\n${{ matrix.js.summary }}"
+                      }
+                    },
+                    {
+                      "type": "section",
+                      "text": {
+                        "type": "mrkdwn",
+                        "text": "*Go dependency check*: *${{ matrix.go.status }}*\n${{ matrix.go.summary }}"
+                      }
+                    },
+                    {
+                      "type": "section",
+                      "text": {
+                        "type": "mrkdwn",
+                        "text": "*Image vulnerability check*: *${{ matrix.image.status }}*\n${{ matrix.image.summary }}\n"
+                      }
+                    }
+                  ]
+                }
+              ]
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SECURITY_SLACK_WEBHOOK_URL }}
+          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK

.github/workflows/pr-security.yml

@@ -0,0 +1,233 @@
+name: PR Code Security Scan
+
+on:
+  pull_request_review:
+    types:
+      - submitted
+      - edited
+    paths:
+      - 'package.json'
+      - 'api/go.mod'
+      - 'gruntfile.js'
+      - 'build/linux/Dockerfile'
+      - 'build/linux/alpine.Dockerfile'
+      - 'build/windows/Dockerfile'
+    
+jobs:
+  client-dependencies:
+    name: Client dependency check
+    runs-on: ubuntu-latest
+    if: >-
+      github.event.pull_request &&
+      github.event.review.body == '/scan'
+    outputs:
+      jsdiff: ${{ steps.set-diff-matrix.outputs.js_diff_result }}
+    steps:
+      - uses: actions/checkout@master
+
+      - name: Run Snyk to check for vulnerabilities
+        uses: snyk/actions/node@master
+        continue-on-error: true # To make sure that artifact upload gets called
+        env:
+          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+        with:
+          json: true
+
+      - name: Upload js security scan result as artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: js-security-scan-feat-result
+          path: snyk.json
+
+      - name: Download artifacts from develop branch
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          mv ./snyk.json ./js-snyk-feature.json
+          (gh run download -n js-security-scan-develop-result -R ${{ github.repository }} 2>&1 >/dev/null) || :
+          if [[ -e ./snyk.json ]]; then
+            mv ./snyk.json ./js-snyk-develop.json
+          else
+            echo "null" > ./js-snyk-develop.json
+          fi
+
+      - name: Export scan result to html file 
+        run: | 
+          $(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 diff -report-type=snyk -path="/data/js-snyk-feature.json" -compare-to="/data/js-snyk-develop.json" -output-type=table -export -export-filename="/data/js-result")
+
+      - name: Upload js result html file
+        uses: actions/upload-artifact@v3
+        with:
+          name: html-js-result-compare-to-develop-${{github.run_id}}
+          path: js-result.html
+
+      - name: Analyse the js diff result
+        id: set-diff-matrix
+        run: | 
+          result=$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 diff -report-type=snyk -path="/data/js-snyk-feature.json" -compare-to="./data/js-snyk-develop.json" -output-type=matrix)
+          echo "::set-output name=js_diff_result::${result}"
+
+  server-dependencies:
+    name: Server dependency check
+    runs-on: ubuntu-latest
+    if: >-
+      github.event.pull_request &&
+      github.event.review.body == '/scan'
+    outputs:
+      godiff: ${{ steps.set-diff-matrix.outputs.go_diff_result }}
+    steps:
+      - uses: actions/checkout@master
+
+      - name: Download go modules
+        run: cd ./api && go get -t -v -d ./...
+
+      - name: Run Snyk to check for vulnerabilities
+        uses: snyk/actions/golang@master
+        continue-on-error: true # To make sure that artifact upload gets called
+        env:
+          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+        with:
+          args: --file=./api/go.mod
+          json: true
+
+      - name: Upload go security scan result as artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: go-security-scan-feature-result
+          path: snyk.json
+
+      - name: Download artifacts from develop branch
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          mv ./snyk.json ./go-snyk-feature.json
+          (gh run download -n go-security-scan-develop-result -R ${{ github.repository }} 2>&1 >/dev/null) || :
+          if [[ -e ./snyk.json ]]; then
+            mv ./snyk.json ./go-snyk-develop.json
+          else
+            echo "null" > ./go-snyk-develop.json
+          fi
+
+      - name: Export scan result to html file 
+        run: | 
+          $(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 diff -report-type=snyk -path="/data/go-snyk-feature.json" -compare-to="/data/go-snyk-develop.json" -output-type=table -export -export-filename="/data/go-result")
+
+      - name: Upload go result html file
+        uses: actions/upload-artifact@v3
+        with:
+          name: html-go-result-compare-to-develop-${{github.run_id}}
+          path: go-result.html
+
+      - name: Analyse the go diff result
+        id: set-diff-matrix
+        run: | 
+          result=$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 diff -report-type=snyk -path="/data/go-snyk-feature.json" -compare-to="/data/go-snyk-develop.json" -output-type=matrix)
+          echo "::set-output name=go_diff_result::${result}"
+
+  image-vulnerability:
+    name: Build docker image and Image vulnerability check
+    runs-on: ubuntu-latest
+    if: >-
+      github.event.pull_request &&
+      github.event.review.body == '/scan'
+    outputs:
+      imagediff: ${{ steps.set-diff-matrix.outputs.image_diff_result }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@master
+
+      - name: Use golang 1.18
+        uses: actions/setup-go@v3
+        with:
+          go-version: '1.18'
+
+      - name: Use Node.js 12.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 12.x
+
+      - name: Install packages and build
+        run: yarn install && yarn build
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: build/linux/Dockerfile
+          tags: trivy-portainer:${{ github.sha }}
+          outputs: type=docker,dest=/tmp/trivy-portainer-image.tar
+
+      - name: Load docker image
+        run: |
+          docker load --input /tmp/trivy-portainer-image.tar
+
+      - name: Run Trivy vulnerability scanner
+        uses: docker://docker.io/aquasec/trivy:latest
+        continue-on-error: true 
+        with:
+          args: image --ignore-unfixed=true --vuln-type="os,library" --exit-code=1 --format="json" --output="image-trivy.json" --no-progress trivy-portainer:${{ github.sha }}  
+
+      - name: Upload image security scan result as artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: image-security-scan-feature-result
+          path: image-trivy.json
+
+      - name: Download artifacts from develop branch
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          mv ./image-trivy.json ./image-trivy-feature.json
+          (gh run download -n image-security-scan-develop-result -R ${{ github.repository }} 2>&1 >/dev/null) || :
+          if [[ -e ./image-trivy.json ]]; then
+            mv ./image-trivy.json ./image-trivy-develop.json
+          else
+            echo "null" > ./image-trivy-develop.json
+          fi
+
+      - name: Export scan result to html file 
+        run: | 
+          $(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 diff -report-type=trivy -path="/data/image-trivy-feature.json" -compare-to="/data/image-trivy-develop.json" -output-type=table -export -export-filename="/data/image-result")
+
+      - name: Upload image result html file
+        uses: actions/upload-artifact@v3
+        with:
+          name: html-image-result-compare-to-develop-${{github.run_id}}
+          path: image-result.html
+
+      - name: Analyse the image diff result
+        id: set-diff-matrix
+        run: | 
+          result=$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 diff -report-type=trivy -path="/data/image-trivy-feature.json" -compare-to="./data/image-trivy-develop.json" -output-type=matrix)
+          echo "::set-output name=image_diff_result::${result}"
+
+  result-analysis:
+    name: Analyse scan result compared to develop
+    needs: [client-dependencies, server-dependencies, image-vulnerability]
+    runs-on: ubuntu-latest
+    if: >-
+      github.event.pull_request &&
+      github.event.review.body == '/scan'
+    strategy:
+      matrix: 
+        jsdiff: ${{fromJson(needs.client-dependencies.outputs.jsdiff)}}
+        godiff: ${{fromJson(needs.server-dependencies.outputs.godiff)}}
+        imagediff: ${{fromJson(needs.image-vulnerability.outputs.imagediff)}}
+    steps:
+
+      - name: Check job status of diff result
+        if: >-
+          matrix.jsdiff.status == 'failure' ||
+          matrix.godiff.status == 'failure' ||
+          matrix.imagediff.status == 'failure' 
+        run: |
+          echo ${{ matrix.jsdiff.status }}
+          echo ${{ matrix.godiff.status }}
+          echo ${{ matrix.imagediff.status }}
+          echo ${{ matrix.jsdiff.summary }}
+          echo ${{ matrix.godiff.summary }}
+          echo ${{ matrix.imagediff.summary }}
+          exit 1

.github/workflows/test-client.yaml

@@ -1,11 +0,0 @@
-name: Test Frontend
-on: push
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Install modules
-        run: yarn --frozen-lockfile
-      - name: Run tests
-        run: yarn test:client

.github/workflows/test.yaml

@@ -0,0 +1,29 @@
+name: Test
+on: push
+jobs:
+  test-client:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+          cache: 'yarn'
+      - run: yarn --frozen-lockfile
+
+      - name: Run tests
+        run: yarn test:client
+  # test-server:
+  #   runs-on: ubuntu-latest
+  #   env:
+  #     GOPRIVATE: "github.com/portainer"
+  #   steps:
+  #     - uses: actions/checkout@v3
+  #     - uses: actions/setup-go@v3
+  #       with:
+  #         go-version: '1.18'
+  #     - name: Run tests
+  #       run: |
+  #         cd api
+  #         go test ./...

.prettierignore

@@ -1 +1,2 @@
-dist
+dist
+api/datastore/test_data

.storybook/main.js

@@ -16,6 +16,9 @@ module.exports = {
             exportLocalsConvention: 'camelCaseOnly',
           },
         },
+        postcssLoaderOptions: {
+          implementation: require('postcss'),
+        },
       },
     },
   ],

api/adminmonitor/admin_monitor.go

@@ -3,32 +3,48 @@ package adminmonitor
 import (
 	"context"
 	"log"
+	"net/http"
+	"strings"
+	"sync"
 	"time"
 
+	httperror "github.com/portainer/libhttp/error"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
 )
 
 var logFatalf = log.Fatalf
 
+const RedirectReasonAdminInitTimeout string = "AdminInitTimeout"
+
 type Monitor struct {
-	timeout          time.Duration
-	datastore        dataservices.DataStore
-	shutdownCtx      context.Context
-	cancellationFunc context.CancelFunc
+	timeout           time.Duration
+	datastore         dataservices.DataStore
+	shutdownCtx       context.Context
+	cancellationFunc  context.CancelFunc
+	mu                sync.Mutex
+	adminInitDisabled bool
 }
 
-// New creates a monitor that when started will wait for the timeout duration and then shutdown the application unless it has been initialized.
+// New creates a monitor that when started will wait for the timeout duration and then sends the timeout signal to disable the application
 func New(timeout time.Duration, datastore dataservices.DataStore, shutdownCtx context.Context) *Monitor {
 	return &Monitor{
-		timeout:     timeout,
-		datastore:   datastore,
-		shutdownCtx: shutdownCtx,
+		timeout:           timeout,
+		datastore:         datastore,
+		shutdownCtx:       shutdownCtx,
+		adminInitDisabled: false,
 	}
 }
 
 // Starts starts the monitor. Active monitor could be stopped or shuttted down by cancelling the shutdown context.
 func (m *Monitor) Start() {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	if m.cancellationFunc != nil {
+		return
+	}
+
 	cancellationCtx, cancellationFunc := context.WithCancel(context.Background())
 	m.cancellationFunc = cancellationFunc
 
@@ -41,7 +57,11 @@ func (m *Monitor) Start() {
 				logFatalf("%s", err)
 			}
 			if !initialized {
-				logFatalf("[FATAL] [internal,init] No administrator account was created in %f mins. Shutting down the Portainer instance for security reasons", m.timeout.Minutes())
+				log.Println("[INFO] [internal,init] The Portainer instance timed out for security purposes. To re-enable your Portainer instance, you will need to restart Portainer")
+				m.mu.Lock()
+				defer m.mu.Unlock()
+				m.adminInitDisabled = true
+				return
 			}
 		case <-cancellationCtx.Done():
 			log.Println("[DEBUG] [internal,init] [message: canceling initialization monitor]")
@@ -53,6 +73,9 @@ func (m *Monitor) Start() {
 
 // Stop stops monitor. Safe to call even if monitor wasn't started.
 func (m *Monitor) Stop() {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
 	if m.cancellationFunc == nil {
 		return
 	}
@@ -68,3 +91,25 @@ func (m *Monitor) WasInitialized() (bool, error) {
 	}
 	return len(users) > 0, nil
 }
+
+func (m *Monitor) WasInstanceDisabled() bool {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return m.adminInitDisabled
+}
+
+// WithRedirect checks whether administrator initialisation timeout. If so, it will return the error with redirect reason.
+// Otherwise, it will pass through the request to next
+func (m *Monitor) WithRedirect(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if m.WasInstanceDisabled() {
+			if strings.HasPrefix(r.RequestURI, "/api") && r.RequestURI != "/api/status" && r.RequestURI != "/api/settings/public" {
+				w.Header().Set("redirect-reason", RedirectReasonAdminInitTimeout)
+				httperror.WriteError(w, http.StatusSeeOther, "Administrator initialization timeout", nil)
+				return
+			}
+		}
+
+		next.ServeHTTP(w, r)
+	})
+}

api/adminmonitor/admin_monitor_test.go

@@ -21,6 +21,18 @@ func Test_stopCouldBeCalledMultipleTimes(t *testing.T) {
 	monitor.Stop()
 }
 
+func Test_startOrStopCouldBeCalledMultipleTimesConcurrently(t *testing.T) {
+	monitor := New(1*time.Minute, nil, context.Background())
+
+	go monitor.Start()
+	monitor.Start()
+
+	go monitor.Stop()
+	monitor.Stop()
+
+	time.Sleep(2 * time.Second)
+}
+
 func Test_canStopStartedMonitor(t *testing.T) {
 	monitor := New(1*time.Minute, nil, context.Background())
 	monitor.Start()
@@ -30,21 +42,13 @@ func Test_canStopStartedMonitor(t *testing.T) {
 	assert.Nil(t, monitor.cancellationFunc, "cancellation function should absent in stopped monitor")
 }
 
-func Test_start_shouldFatalAfterTimeout_ifNotInitialized(t *testing.T) {
+func Test_start_shouldDisableInstanceAfterTimeout_ifNotInitialized(t *testing.T) {
 	timeout := 10 * time.Millisecond
 
 	datastore := i.NewDatastore(i.WithUsers([]portainer.User{}))
-
-	var fataled bool
-	origLogFatalf := logFatalf
-	logFatalf = func(s string, v ...interface{}) { fataled = true }
-	defer func() {
-		logFatalf = origLogFatalf
-	}()
-
 	monitor := New(timeout, datastore, context.Background())
 	monitor.Start()
-	<-time.After(2 * timeout)
 
-	assert.True(t, fataled, "monitor should been timeout and fatal")
+	<-time.After(20 * timeout)
+	assert.True(t, monitor.WasInstanceDisabled(), "monitor should have been timeout and instance is disabled")
 }

api/api-description.md

@@ -50,4 +50,15 @@ Instead, it acts as a reverse-proxy to the Docker HTTP API. This means that you
 
 To do so, you can use the `/endpoints/{id}/docker` Portainer API environment(endpoint) (which is not documented below due to Swagger limitations). This environment(endpoint) has a restricted access policy so you still need to be authenticated to be able to query this environment(endpoint). Any query on this environment(endpoint) will be proxied to the Docker API of the associated environment(endpoint) (requests and responses objects are the same as documented in the Docker API).
 
+# Private Registry
+
+Using private registry, you will need to pass a based64 encoded JSON string ‘{"registryId":\<registryID value\>}’ inside the Request Header. The parameter name is "X-Registry-Auth".
+\<registryID value\> - The registry ID where the repository was created.
+
+Example:
+
+```
+eyJyZWdpc3RyeUlkIjoxfQ==
+```
+
 **NOTE**: You can find more information on how to query the Docker API in the [Docker official documentation](https://docs.docker.com/engine/api/v1.30/) as well as in [this Portainer example](https://documentation.portainer.io/api/api-examples/).

api/apikey/apikey.go

@@ -2,7 +2,6 @@ package apikey
 
 import (
 	"crypto/rand"
-	"github.com/portainer/portainer/api/database"
 	"io"
 
 	portainer "github.com/portainer/portainer/api"
@@ -13,11 +12,11 @@ type APIKeyService interface {
 	HashRaw(rawKey string) []byte
 	GenerateApiKey(user portainer.User, description string) (string, *portainer.APIKey, error)
 	GetAPIKey(apiKeyID portainer.APIKeyID) (*portainer.APIKey, error)
-	GetAPIKeys(userID database.UserID) ([]portainer.APIKey, error)
+	GetAPIKeys(userID portainer.UserID) ([]portainer.APIKey, error)
 	GetDigestUserAndKey(digest []byte) (portainer.User, portainer.APIKey, error)
 	UpdateAPIKey(apiKey *portainer.APIKey) error
 	DeleteAPIKey(apiKeyID portainer.APIKeyID) error
-	InvalidateUserKeyCache(userId database.UserID) bool
+	InvalidateUserKeyCache(userId portainer.UserID) bool
 }
 
 // generateRandomKey generates a random key of specified length

api/apikey/cache.go

@@ -3,7 +3,6 @@ package apikey
 import (
 	lru "github.com/hashicorp/golang-lru"
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/database"
 )
 
 const defaultAPIKeyCacheSize = 1024
@@ -58,7 +57,7 @@ func (c *apiKeyCache) Delete(digest []byte) {
 }
 
 // InvalidateUserKeyCache loops through all the api-keys associated to a user and removes them from the cache
-func (c *apiKeyCache) InvalidateUserKeyCache(userId database.UserID) bool {
+func (c *apiKeyCache) InvalidateUserKeyCache(userId portainer.UserID) bool {
 	present := false
 	for _, k := range c.cache.Keys() {
 		user, _, _ := c.Get([]byte(k.(string)))

api/apikey/service.go

@@ -4,7 +4,6 @@ import (
 	"crypto/sha256"
 	"encoding/base64"
 	"fmt"
-	"github.com/portainer/portainer/api/database"
 	"time"
 
 	"github.com/pkg/errors"
@@ -71,7 +70,7 @@ func (a *apiKeyService) GetAPIKey(apiKeyID portainer.APIKeyID) (*portainer.APIKe
 }
 
 // GetAPIKeys returns all the API keys associated to a user.
-func (a *apiKeyService) GetAPIKeys(userID database.UserID) ([]portainer.APIKey, error) {
+func (a *apiKeyService) GetAPIKeys(userID portainer.UserID) ([]portainer.APIKey, error) {
 	return a.apiKeyRepository.GetAPIKeysByUserID(userID)
 }
 
@@ -123,6 +122,6 @@ func (a *apiKeyService) DeleteAPIKey(apiKeyID portainer.APIKeyID) error {
 	return a.apiKeyRepository.DeleteAPIKey(apiKeyID)
 }
 
-func (a *apiKeyService) InvalidateUserKeyCache(userId database.UserID) bool {
+func (a *apiKeyService) InvalidateUserKeyCache(userId portainer.UserID) bool {
 	return a.cache.InvalidateUserKeyCache(userId)
 }

api/apikey/service_test.go

@@ -20,7 +20,7 @@ func Test_SatisfiesAPIKeyServiceInterface(t *testing.T) {
 func Test_GenerateApiKey(t *testing.T) {
 	is := assert.New(t)
 
-	_, store, teardown := datastore.MustNewTestStore(true)
+	_, store, teardown := datastore.MustNewTestStore(true, true)
 	defer teardown()
 
 	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
@@ -74,7 +74,7 @@ func Test_GenerateApiKey(t *testing.T) {
 func Test_GetAPIKey(t *testing.T) {
 	is := assert.New(t)
 
-	_, store, teardown := datastore.MustNewTestStore(true)
+	_, store, teardown := datastore.MustNewTestStore(true, true)
 	defer teardown()
 
 	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
@@ -94,7 +94,7 @@ func Test_GetAPIKey(t *testing.T) {
 func Test_GetAPIKeys(t *testing.T) {
 	is := assert.New(t)
 
-	_, store, teardown := datastore.MustNewTestStore(true)
+	_, store, teardown := datastore.MustNewTestStore(true, true)
 	defer teardown()
 
 	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
@@ -115,7 +115,7 @@ func Test_GetAPIKeys(t *testing.T) {
 func Test_GetDigestUserAndKey(t *testing.T) {
 	is := assert.New(t)
 
-	_, store, teardown := datastore.MustNewTestStore(true)
+	_, store, teardown := datastore.MustNewTestStore(true, true)
 	defer teardown()
 
 	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
@@ -151,7 +151,7 @@ func Test_GetDigestUserAndKey(t *testing.T) {
 func Test_UpdateAPIKey(t *testing.T) {
 	is := assert.New(t)
 
-	_, store, teardown := datastore.MustNewTestStore(true)
+	_, store, teardown := datastore.MustNewTestStore(true, true)
 	defer teardown()
 
 	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
@@ -199,7 +199,7 @@ func Test_UpdateAPIKey(t *testing.T) {
 func Test_DeleteAPIKey(t *testing.T) {
 	is := assert.New(t)
 
-	_, store, teardown := datastore.MustNewTestStore(true)
+	_, store, teardown := datastore.MustNewTestStore(true, true)
 	defer teardown()
 
 	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
@@ -240,7 +240,7 @@ func Test_DeleteAPIKey(t *testing.T) {
 func Test_InvalidateUserKeyCache(t *testing.T) {
 	is := assert.New(t)
 
-	_, store, teardown := datastore.MustNewTestStore(true)
+	_, store, teardown := datastore.MustNewTestStore(true, true)
 	defer teardown()
 
 	service := NewAPIKeyService(store.APIKeyRepository(), store.User())

api/chisel/schedules.go

@@ -1,16 +1,13 @@
 package chisel
 
 import (
-	"github.com/portainer/portainer/api/database"
-	"github.com/portainer/portainer/api/dataservices/edgejob"
-	"strconv"
-
 	portainer "github.com/portainer/portainer/api"
 )
 
 // AddEdgeJob register an EdgeJob inside the tunnel details associated to an environment(endpoint).
-func (service *Service) AddEdgeJob(endpointID database.EndpointID, edgeJob *edgejob.EdgeJob) {
-	tunnel := service.GetTunnelDetails(endpointID)
+func (service *Service) AddEdgeJob(endpointID portainer.EndpointID, edgeJob *portainer.EdgeJob) {
+	service.mu.Lock()
+	tunnel := service.getTunnelDetails(endpointID)
 
 	existingJobIndex := -1
 	for idx, existingJob := range tunnel.Jobs {
@@ -26,24 +23,25 @@ func (service *Service) AddEdgeJob(endpointID database.EndpointID, edgeJob *edge
 		tunnel.Jobs[existingJobIndex] = *edgeJob
 	}
 
-	key := strconv.Itoa(int(endpointID))
-	service.tunnelDetailsMap.Set(key, tunnel)
+	service.mu.Unlock()
 }
 
 // RemoveEdgeJob will remove the specified Edge job from each tunnel it was registered with.
-func (service *Service) RemoveEdgeJob(edgeJobID edgejob.EdgeJobID) {
-	for item := range service.tunnelDetailsMap.IterBuffered() {
-		tunnelDetails := item.Val.(*portainer.TunnelDetails)
+func (service *Service) RemoveEdgeJob(edgeJobID portainer.EdgeJobID) {
+	service.mu.Lock()
 
-		updatedJobs := make([]edgejob.EdgeJob, 0)
-		for _, edgeJob := range tunnelDetails.Jobs {
-			if edgeJob.ID == edgeJobID {
-				continue
+	for _, tunnel := range service.tunnelDetailsMap {
+		// Filter in-place
+		n := 0
+		for _, edgeJob := range tunnel.Jobs {
+			if edgeJob.ID != edgeJobID {
+				tunnel.Jobs[n] = edgeJob
+				n++
 			}
-			updatedJobs = append(updatedJobs, edgeJob)
 		}
 
-		tunnelDetails.Jobs = updatedJobs
-		service.tunnelDetailsMap.Set(item.Key, tunnelDetails)
+		tunnel.Jobs = tunnel.Jobs[:n]
 	}
+
+	service.mu.Unlock()
 }

api/chisel/service.go

@@ -3,18 +3,16 @@ package chisel
 import (
 	"context"
 	"fmt"
-	"github.com/portainer/portainer/api/database"
-	"github.com/portainer/portainer/api/http/proxy"
 	"log"
 	"net/http"
-	"strconv"
+	"sync"
 	"time"
 
 	"github.com/dchest/uniuri"
 	chserver "github.com/jpillora/chisel/server"
-	cmap "github.com/orcaman/concurrent-map"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
+	"github.com/portainer/portainer/api/http/proxy"
 )
 
 const (
@@ -29,25 +27,26 @@ const (
 type Service struct {
 	serverFingerprint string
 	serverPort        string
-	tunnelDetailsMap  cmap.ConcurrentMap
+	tunnelDetailsMap  map[portainer.EndpointID]*portainer.TunnelDetails
 	dataStore         dataservices.DataStore
 	snapshotService   portainer.SnapshotService
 	chiselServer      *chserver.Server
 	shutdownCtx       context.Context
 	ProxyManager      *proxy.Manager
+	mu                sync.Mutex
 }
 
 // NewService returns a pointer to a new instance of Service
 func NewService(dataStore dataservices.DataStore, shutdownCtx context.Context) *Service {
 	return &Service{
-		tunnelDetailsMap: cmap.New(),
+		tunnelDetailsMap: make(map[portainer.EndpointID]*portainer.TunnelDetails),
 		dataStore:        dataStore,
 		shutdownCtx:      shutdownCtx,
 	}
 }
 
 // pingAgent ping the given agent so that the agent can keep the tunnel alive
-func (service *Service) pingAgent(endpointID database.EndpointID) error {
+func (service *Service) pingAgent(endpointID portainer.EndpointID) error {
 	tunnel := service.GetTunnelDetails(endpointID)
 	requestURL := fmt.Sprintf("http://127.0.0.1:%d/ping", tunnel.Port)
 	req, err := http.NewRequest(http.MethodHead, requestURL, nil)
@@ -59,15 +58,11 @@ func (service *Service) pingAgent(endpointID database.EndpointID) error {
 		Timeout: 3 * time.Second,
 	}
 	_, err = httpClient.Do(req)
-	if err != nil {
-		return err
-	}
-
-	return nil
+	return err
 }
 
 // KeepTunnelAlive keeps the tunnel of the given environment for maxAlive duration, or until ctx is done
-func (service *Service) KeepTunnelAlive(endpointID database.EndpointID, ctx context.Context, maxAlive time.Duration) {
+func (service *Service) KeepTunnelAlive(endpointID portainer.EndpointID, ctx context.Context, maxAlive time.Duration) {
 	go func() {
 		log.Printf("[DEBUG] [chisel,KeepTunnelAlive] [endpoint_id: %d] [message: start for %.0f minutes]\n", endpointID, maxAlive.Minutes())
 		maxAliveTicker := time.NewTicker(maxAlive)
@@ -186,49 +181,44 @@ func (service *Service) startTunnelVerificationLoop() {
 }
 
 func (service *Service) checkTunnels() {
-	for item := range service.tunnelDetailsMap.IterBuffered() {
-		tunnel := item.Val.(*portainer.TunnelDetails)
+	tunnels := make(map[portainer.EndpointID]portainer.TunnelDetails)
 
+	service.mu.Lock()
+	for key, tunnel := range service.tunnelDetailsMap {
+		tunnels[key] = *tunnel
+	}
+	service.mu.Unlock()
+
+	for endpointID, tunnel := range tunnels {
 		if tunnel.LastActivity.IsZero() || tunnel.Status == portainer.EdgeAgentIdle {
 			continue
 		}
 
 		elapsed := time.Since(tunnel.LastActivity)
-		log.Printf("[DEBUG] [chisel,monitoring] [endpoint_id: %s] [status: %s] [status_time_seconds: %f] [message: environment tunnel monitoring]", item.Key, tunnel.Status, elapsed.Seconds())
+		log.Printf("[DEBUG] [chisel,monitoring] [endpoint_id: %d] [status: %s] [status_time_seconds: %f] [message: environment tunnel monitoring]", endpointID, tunnel.Status, elapsed.Seconds())
 
 		if tunnel.Status == portainer.EdgeAgentManagementRequired && elapsed.Seconds() < requiredTimeout.Seconds() {
 			continue
 		} else if tunnel.Status == portainer.EdgeAgentManagementRequired && elapsed.Seconds() > requiredTimeout.Seconds() {
-			log.Printf("[DEBUG] [chisel,monitoring] [endpoint_id: %s] [status: %s] [status_time_seconds: %f] [timeout_seconds: %f] [message: REQUIRED state timeout exceeded]", item.Key, tunnel.Status, elapsed.Seconds(), requiredTimeout.Seconds())
+			log.Printf("[DEBUG] [chisel,monitoring] [endpoint_id: %d] [status: %s] [status_time_seconds: %f] [timeout_seconds: %f] [message: REQUIRED state timeout exceeded]", endpointID, tunnel.Status, elapsed.Seconds(), requiredTimeout.Seconds())
 		}
 
 		if tunnel.Status == portainer.EdgeAgentActive && elapsed.Seconds() < activeTimeout.Seconds() {
 			continue
 		} else if tunnel.Status == portainer.EdgeAgentActive && elapsed.Seconds() > activeTimeout.Seconds() {
-			log.Printf("[DEBUG] [chisel,monitoring] [endpoint_id: %s] [status: %s] [status_time_seconds: %f] [timeout_seconds: %f] [message: ACTIVE state timeout exceeded]", item.Key, tunnel.Status, elapsed.Seconds(), activeTimeout.Seconds())
+			log.Printf("[DEBUG] [chisel,monitoring] [endpoint_id: %d] [status: %s] [status_time_seconds: %f] [timeout_seconds: %f] [message: ACTIVE state timeout exceeded]", endpointID, tunnel.Status, elapsed.Seconds(), activeTimeout.Seconds())
 
-			endpointID, err := strconv.Atoi(item.Key)
+			err := service.snapshotEnvironment(endpointID, tunnel.Port)
 			if err != nil {
-				log.Printf("[ERROR] [chisel,snapshot,conversion] Invalid environment identifier (id: %s): %s", item.Key, err)
-			}
-
-			err = service.snapshotEnvironment(database.EndpointID(endpointID), tunnel.Port)
-			if err != nil {
-				log.Printf("[ERROR] [snapshot] Unable to snapshot Edge environment (id: %s): %s", item.Key, err)
+				log.Printf("[ERROR] [snapshot] Unable to snapshot Edge environment (id: %d): %s", endpointID, err)
 			}
 		}
 
-		endpointID, err := strconv.Atoi(item.Key)
-		if err != nil {
-			log.Printf("[ERROR] [chisel,conversion] Invalid environment identifier (id: %s): %s", item.Key, err)
-			continue
-		}
-
-		service.SetTunnelStatusToIdle(database.EndpointID(endpointID))
+		service.SetTunnelStatusToIdle(portainer.EndpointID(endpointID))
 	}
 }
 
-func (service *Service) snapshotEnvironment(endpointID database.EndpointID, tunnelPort int) error {
+func (service *Service) snapshotEnvironment(endpointID portainer.EndpointID, tunnelPort int) error {
 	endpoint, err := service.dataStore.Endpoint().Endpoint(endpointID)
 	if err != nil {
 		return err

api/chisel/tunnel.go

@@ -3,16 +3,12 @@ package chisel
 import (
 	"encoding/base64"
 	"fmt"
-	"github.com/portainer/portainer/api/database"
-	"github.com/portainer/portainer/api/dataservices/edgejob"
 	"math/rand"
-	"strconv"
 	"strings"
 	"time"
 
-	"github.com/portainer/libcrypto"
-
 	"github.com/dchest/uniuri"
+	"github.com/portainer/libcrypto"
 	portainer "github.com/portainer/portainer/api"
 )
 
@@ -21,13 +17,13 @@ const (
 	maxAvailablePort = 65535
 )
 
+// NOTE: it needs to be called with the lock acquired
 // getUnusedPort is used to generate an unused random port in the dynamic port range.
 // Dynamic ports (also called private ports) are 49152 to 65535.
 func (service *Service) getUnusedPort() int {
 	port := randomInt(minAvailablePort, maxAvailablePort)
 
-	for item := range service.tunnelDetailsMap.IterBuffered() {
-		tunnel := item.Val.(*portainer.TunnelDetails)
+	for _, tunnel := range service.tunnelDetailsMap {
 		if tunnel.Port == port {
 			return service.getUnusedPort()
 		}
@@ -40,26 +36,32 @@ func randomInt(min, max int) int {
 	return min + rand.Intn(max-min)
 }
 
+// NOTE: it needs to be called with the lock acquired
+func (service *Service) getTunnelDetails(endpointID portainer.EndpointID) *portainer.TunnelDetails {
+
+	if tunnel, ok := service.tunnelDetailsMap[endpointID]; ok {
+		return tunnel
+	}
+
+	tunnel := &portainer.TunnelDetails{
+		Status: portainer.EdgeAgentIdle,
+	}
+
+	service.tunnelDetailsMap[endpointID] = tunnel
+
+	return tunnel
+}
+
 // GetTunnelDetails returns information about the tunnel associated to an environment(endpoint).
-func (service *Service) GetTunnelDetails(endpointID database.EndpointID) *portainer.TunnelDetails {
-	key := strconv.Itoa(int(endpointID))
+func (service *Service) GetTunnelDetails(endpointID portainer.EndpointID) portainer.TunnelDetails {
+	service.mu.Lock()
+	defer service.mu.Unlock()
 
-	if item, ok := service.tunnelDetailsMap.Get(key); ok {
-		tunnelDetails := item.(*portainer.TunnelDetails)
-		return tunnelDetails
-	}
-
-	jobs := make([]edgejob.EdgeJob, 0)
-	return &portainer.TunnelDetails{
-		Status:      portainer.EdgeAgentIdle,
-		Port:        0,
-		Jobs:        jobs,
-		Credentials: "",
-	}
+	return *service.getTunnelDetails(endpointID)
 }
 
 // GetActiveTunnel retrieves an active tunnel which allows communicating with edge agent
-func (service *Service) GetActiveTunnel(endpoint *portainer.Endpoint) (*portainer.TunnelDetails, error) {
+func (service *Service) GetActiveTunnel(endpoint *portainer.Endpoint) (portainer.TunnelDetails, error) {
 	tunnel := service.GetTunnelDetails(endpoint.ID)
 
 	if tunnel.Status == portainer.EdgeAgentActive {
@@ -70,13 +72,13 @@ func (service *Service) GetActiveTunnel(endpoint *portainer.Endpoint) (*portaine
 	if tunnel.Status == portainer.EdgeAgentIdle || tunnel.Status == portainer.EdgeAgentManagementRequired {
 		err := service.SetTunnelStatusToRequired(endpoint.ID)
 		if err != nil {
-			return nil, fmt.Errorf("failed opening tunnel to endpoint: %w", err)
+			return portainer.TunnelDetails{}, fmt.Errorf("failed opening tunnel to endpoint: %w", err)
 		}
 
 		if endpoint.EdgeCheckinInterval == 0 {
 			settings, err := service.dataStore.Settings().Settings()
 			if err != nil {
-				return nil, fmt.Errorf("failed fetching settings from db: %w", err)
+				return portainer.TunnelDetails{}, fmt.Errorf("failed fetching settings from db: %w", err)
 			}
 
 			endpoint.EdgeCheckinInterval = settings.EdgeAgentCheckinInterval
@@ -85,29 +87,27 @@ func (service *Service) GetActiveTunnel(endpoint *portainer.Endpoint) (*portaine
 		time.Sleep(2 * time.Duration(endpoint.EdgeCheckinInterval) * time.Second)
 	}
 
-	tunnel = service.GetTunnelDetails(endpoint.ID)
-
-	return tunnel, nil
+	return service.GetTunnelDetails(endpoint.ID), nil
 }
 
 // SetTunnelStatusToActive update the status of the tunnel associated to the specified environment(endpoint).
 // It sets the status to ACTIVE.
-func (service *Service) SetTunnelStatusToActive(endpointID database.EndpointID) {
-	tunnel := service.GetTunnelDetails(endpointID)
+func (service *Service) SetTunnelStatusToActive(endpointID portainer.EndpointID) {
+	service.mu.Lock()
+	tunnel := service.getTunnelDetails(endpointID)
 	tunnel.Status = portainer.EdgeAgentActive
 	tunnel.Credentials = ""
 	tunnel.LastActivity = time.Now()
-
-	key := strconv.Itoa(int(endpointID))
-	service.tunnelDetailsMap.Set(key, tunnel)
+	service.mu.Unlock()
 }
 
 // SetTunnelStatusToIdle update the status of the tunnel associated to the specified environment(endpoint).
 // It sets the status to IDLE.
 // It removes any existing credentials associated to the tunnel.
-func (service *Service) SetTunnelStatusToIdle(endpointID database.EndpointID) {
-	tunnel := service.GetTunnelDetails(endpointID)
+func (service *Service) SetTunnelStatusToIdle(endpointID portainer.EndpointID) {
+	service.mu.Lock()
 
+	tunnel := service.getTunnelDetails(endpointID)
 	tunnel.Status = portainer.EdgeAgentIdle
 	tunnel.Port = 0
 	tunnel.LastActivity = time.Now()
@@ -118,10 +118,9 @@ func (service *Service) SetTunnelStatusToIdle(endpointID database.EndpointID) {
 		service.chiselServer.DeleteUser(strings.Split(credentials, ":")[0])
 	}
 
-	key := strconv.Itoa(int(endpointID))
-	service.tunnelDetailsMap.Set(key, tunnel)
-
 	service.ProxyManager.DeleteEndpointProxy(endpointID)
+
+	service.mu.Unlock()
 }
 
 // SetTunnelStatusToRequired update the status of the tunnel associated to the specified environment(endpoint).
@@ -129,8 +128,11 @@ func (service *Service) SetTunnelStatusToIdle(endpointID database.EndpointID) {
 // If no port is currently associated to the tunnel, it will associate a random unused port to the tunnel
 // and generate temporary credentials that can be used to establish a reverse tunnel on that port.
 // Credentials are encrypted using the Edge ID associated to the environment(endpoint).
-func (service *Service) SetTunnelStatusToRequired(endpointID database.EndpointID) error {
-	tunnel := service.GetTunnelDetails(endpointID)
+func (service *Service) SetTunnelStatusToRequired(endpointID portainer.EndpointID) error {
+	tunnel := service.getTunnelDetails(endpointID)
+
+	service.mu.Lock()
+	defer service.mu.Unlock()
 
 	if tunnel.Port == 0 {
 		endpoint, err := service.dataStore.Endpoint().Endpoint(endpointID)
@@ -154,9 +156,6 @@ func (service *Service) SetTunnelStatusToRequired(endpointID database.EndpointID
 			return err
 		}
 		tunnel.Credentials = credentials
-
-		key := strconv.Itoa(int(endpointID))
-		service.tunnelDetailsMap.Set(key, tunnel)
 	}
 
 	return nil

api/cli/cli.go

@@ -51,7 +51,7 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 		SSLKey:                    kingpin.Flag("sslkey", "Path to the SSL key used to secure the Portainer instance").String(),
 		Rollback:                  kingpin.Flag("rollback", "Rollback the database store to the previous version").Bool(),
 		SnapshotInterval:          kingpin.Flag("snapshot-interval", "Duration between each environment snapshot job").String(),
-		AdminPassword:             kingpin.Flag("admin-password", "Hashed admin password").String(),
+		AdminPassword:             kingpin.Flag("admin-password", "Set admin password with provided hash").String(),
 		AdminPasswordFile:         kingpin.Flag("admin-password-file", "Path to the file containing the password for the admin user").String(),
 		Labels:                    pairs(kingpin.Flag("hide-label", "Hide containers with a specific label in the UI").Short('l')),
 		Logo:                      kingpin.Flag("logo", "URL for the logo displayed in the UI").String(),

api/cli/defaults.go

@@ -18,8 +18,6 @@ const (
 	defaultHTTPDisabled        = "false"
 	defaultHTTPEnabled         = "false"
 	defaultSSL                 = "false"
-	defaultSSLCertPath         = "/certs/portainer.crt"
-	defaultSSLKeyPath          = "/certs/portainer.key"
 	defaultBaseURL             = "/"
 	defaultSecretKeyName       = "portainer"
 )

api/cli/defaults_windows.go

@@ -15,8 +15,6 @@ const (
 	defaultHTTPDisabled        = "false"
 	defaultHTTPEnabled         = "false"
 	defaultSSL                 = "false"
-	defaultSSLCertPath         = "C:\\certs\\portainer.crt"
-	defaultSSLKeyPath          = "C:\\certs\\portainer.key"
 	defaultSnapshotInterval    = "5m"
 	defaultBaseURL             = "/"
 	defaultSecretKeyName       = "portainer"

api/cmd/portainer/log.go

@@ -1,41 +1,19 @@
 package main
 
 import (
-	"fmt"
 	"log"
-	"strings"
 
 	"github.com/sirupsen/logrus"
 )
 
-type portainerFormatter struct {
-	logrus.TextFormatter
-}
-
-func (f *portainerFormatter) Format(entry *logrus.Entry) ([]byte, error) {
-	var levelColor int
-	switch entry.Level {
-	case logrus.DebugLevel, logrus.TraceLevel:
-		levelColor = 31 // gray
-	case logrus.WarnLevel:
-		levelColor = 33 // yellow
-	case logrus.ErrorLevel, logrus.FatalLevel, logrus.PanicLevel:
-		levelColor = 31 // red
-	default:
-		levelColor = 36 // blue
-	}
-	return []byte(fmt.Sprintf("\x1b[%dm%s\x1b[0m %s %s\n", levelColor, strings.ToUpper(entry.Level.String()), entry.Time.Format(f.TimestampFormat), entry.Message)), nil
-}
-
 func configureLogger() {
 	logger := logrus.New() // logger is to implicitly substitute stdlib's log
 	log.SetOutput(logger.Writer())
 
-	formatter := &logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true}
-	formatterLogrus := &portainerFormatter{logrus.TextFormatter{DisableTimestamp: false, DisableLevelTruncation: true, TimestampFormat: "2006/01/02 15:04:05", FullTimestamp: true}}
+	formatter := &logrus.TextFormatter{DisableTimestamp: false, DisableLevelTruncation: true}
 
 	logger.SetFormatter(formatter)
-	logrus.SetFormatter(formatterLogrus)
+	logrus.SetFormatter(formatter)
 
 	logger.SetLevel(logrus.DebugLevel)
 	logrus.SetLevel(logrus.DebugLevel)

api/cmd/portainer/main.go

@@ -4,10 +4,6 @@ import (
 	"context"
 	"crypto/sha256"
 	"fmt"
-	docker2 "github.com/portainer/portainer/api/orchestrators/docker"
-	kubernetes2 "github.com/portainer/portainer/api/orchestrators/kubernetes"
-	kubecli "github.com/portainer/portainer/api/orchestrators/kubernetes/cli"
-	"github.com/portainer/portainer/api/orchestrators/snapshot"
 	"log"
 	"os"
 	"path"
@@ -27,6 +23,7 @@ import (
 	"github.com/portainer/portainer/api/database/boltdb"
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/datastore"
+	"github.com/portainer/portainer/api/docker"
 	"github.com/portainer/portainer/api/exec"
 	"github.com/portainer/portainer/api/filesystem"
 	"github.com/portainer/portainer/api/git"
@@ -37,8 +34,11 @@ import (
 	kubeproxy "github.com/portainer/portainer/api/http/proxy/factory/kubernetes"
 	"github.com/portainer/portainer/api/internal/authorization"
 	"github.com/portainer/portainer/api/internal/edge"
+	"github.com/portainer/portainer/api/internal/snapshot"
 	"github.com/portainer/portainer/api/internal/ssl"
 	"github.com/portainer/portainer/api/jwt"
+	"github.com/portainer/portainer/api/kubernetes"
+	kubecli "github.com/portainer/portainer/api/kubernetes/cli"
 	"github.com/portainer/portainer/api/ldap"
 	"github.com/portainer/portainer/api/oauth"
 	"github.com/portainer/portainer/api/scheduler"
@@ -208,7 +208,7 @@ func initGitService() portainer.GitService {
 	return git.NewService()
 }
 
-func initSSLService(addr, dataPath, certPath, keyPath string, fileService portainer.FileService, dataStore dataservices.DataStore, shutdownTrigger context.CancelFunc) (*ssl.Service, error) {
+func initSSLService(addr, certPath, keyPath string, fileService portainer.FileService, dataStore dataservices.DataStore, shutdownTrigger context.CancelFunc) (*ssl.Service, error) {
 	slices := strings.Split(addr, ":")
 	host := slices[0]
 	if host == "" {
@@ -225,16 +225,19 @@ func initSSLService(addr, dataPath, certPath, keyPath string, fileService portai
 	return sslService, nil
 }
 
-func initDockerClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService) *docker2.ClientFactory {
-	return docker2.NewClientFactory(signatureService, reverseTunnelService)
+func initDockerClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService) *docker.ClientFactory {
+	return docker.NewClientFactory(signatureService, reverseTunnelService)
 }
 
 func initKubernetesClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService, instanceID string, dataStore dataservices.DataStore) *kubecli.ClientFactory {
 	return kubecli.NewClientFactory(signatureService, reverseTunnelService, instanceID, dataStore)
 }
 
-func initSnapshotService(snapshotIntervalFromFlag string, dataStore dataservices.DataStore, dockerClientFactory *docker2.ClientFactory, kubernetesClientFactory *kubecli.ClientFactory, shutdownCtx context.Context) (portainer.SnapshotService, error) {
-	snapshotService, err := snapshot.NewService(snapshotIntervalFromFlag, dataStore, dockerClientFactory, kubernetesClientFactory, shutdownCtx)
+func initSnapshotService(snapshotIntervalFromFlag string, dataStore dataservices.DataStore, dockerClientFactory *docker.ClientFactory, kubernetesClientFactory *kubecli.ClientFactory, shutdownCtx context.Context) (portainer.SnapshotService, error) {
+	dockerSnapshotter := docker.NewSnapshotter(dockerClientFactory)
+	kubernetesSnapshotter := kubernetes.NewSnapshotter(kubernetesClientFactory)
+
+	snapshotService, err := snapshot.NewService(snapshotIntervalFromFlag, dataStore, dockerSnapshotter, kubernetesSnapshotter, shutdownCtx)
 	if err != nil {
 		return nil, err
 	}
@@ -275,6 +278,12 @@ func updateSettingsFromFlags(dataStore dataservices.DataStore, flags *portainer.
 		settings.BlackListedLabels = *flags.Labels
 	}
 
+	if agentKey, ok := os.LookupEnv("AGENT_SECRET"); ok {
+		settings.AgentSecret = agentKey
+	} else {
+		settings.AgentSecret = ""
+	}
+
 	err = dataStore.Settings().UpdateSettings(settings)
 	if err != nil {
 		return err
@@ -372,7 +381,7 @@ func initKeyPair(fileService portainer.FileService, signatureService portainer.D
 }
 
 func createTLSSecuredEndpoint(flags *portainer.CLIFlags, dataStore dataservices.DataStore, snapshotService portainer.SnapshotService) error {
-	tlsConfiguration := database.TLSConfiguration{
+	tlsConfiguration := portainer.TLSConfiguration{
 		TLS:           *flags.TLS,
 		TLSSkipVerify: *flags.TLSSkipVerify,
 	}
@@ -385,11 +394,34 @@ func createTLSSecuredEndpoint(flags *portainer.CLIFlags, dataStore dataservices.
 		tlsConfiguration.TLS = true
 	}
 
-	endpoint := dataStore.Endpoint().NewDefault()
-	endpoint.Name = "primary"
-	endpoint.URL = *flags.EndpointURL
-	endpoint.GroupID = portainer.EndpointGroupID(1)
-	endpoint.Type = portainer.DockerEnvironment
+	endpointID := dataStore.Endpoint().GetNextIdentifier()
+	endpoint := &portainer.Endpoint{
+		ID:                 portainer.EndpointID(endpointID),
+		Name:               "primary",
+		URL:                *flags.EndpointURL,
+		GroupID:            portainer.EndpointGroupID(1),
+		Type:               portainer.DockerEnvironment,
+		TLSConfig:          tlsConfiguration,
+		UserAccessPolicies: portainer.UserAccessPolicies{},
+		TeamAccessPolicies: portainer.TeamAccessPolicies{},
+		TagIDs:             []portainer.TagID{},
+		Status:             portainer.EndpointStatusUp,
+		Snapshots:          []portainer.DockerSnapshot{},
+		Kubernetes:         portainer.KubernetesDefault(),
+
+		SecuritySettings: portainer.EndpointSecuritySettings{
+			AllowVolumeBrowserForRegularUsers: false,
+			EnableHostManagementFeatures:      false,
+
+			AllowSysctlSettingForRegularUsers:         true,
+			AllowBindMountsForRegularUsers:            true,
+			AllowPrivilegedModeForRegularUsers:        true,
+			AllowHostNamespaceForRegularUsers:         true,
+			AllowContainerCapabilitiesForRegularUsers: true,
+			AllowDeviceMappingForRegularUsers:         true,
+			AllowStackManagementForRegularUsers:       true,
+		},
+	}
 
 	if strings.HasPrefix(endpoint.URL, "tcp://") {
 		tlsConfig, err := crypto.CreateTLSConfigurationFromDisk(tlsConfiguration.TLSCACertPath, tlsConfiguration.TLSCertPath, tlsConfiguration.TLSKeyPath, tlsConfiguration.TLSSkipVerify)
@@ -423,11 +455,34 @@ func createUnsecuredEndpoint(endpointURL string, dataStore dataservices.DataStor
 		}
 	}
 
-	endpoint := dataStore.Endpoint().NewDefault()
-	endpoint.Name = "primary"
-	endpoint.URL = endpointURL
-	endpoint.GroupID = portainer.EndpointGroupID(1)
-	endpoint.Type = portainer.DockerEnvironment
+	endpointID := dataStore.Endpoint().GetNextIdentifier()
+	endpoint := &portainer.Endpoint{
+		ID:                 portainer.EndpointID(endpointID),
+		Name:               "primary",
+		URL:                endpointURL,
+		GroupID:            portainer.EndpointGroupID(1),
+		Type:               portainer.DockerEnvironment,
+		TLSConfig:          portainer.TLSConfiguration{},
+		UserAccessPolicies: portainer.UserAccessPolicies{},
+		TeamAccessPolicies: portainer.TeamAccessPolicies{},
+		TagIDs:             []portainer.TagID{},
+		Status:             portainer.EndpointStatusUp,
+		Snapshots:          []portainer.DockerSnapshot{},
+		Kubernetes:         portainer.KubernetesDefault(),
+
+		SecuritySettings: portainer.EndpointSecuritySettings{
+			AllowVolumeBrowserForRegularUsers: false,
+			EnableHostManagementFeatures:      false,
+
+			AllowSysctlSettingForRegularUsers:         true,
+			AllowBindMountsForRegularUsers:            true,
+			AllowPrivilegedModeForRegularUsers:        true,
+			AllowHostNamespaceForRegularUsers:         true,
+			AllowContainerCapabilitiesForRegularUsers: true,
+			AllowDeviceMappingForRegularUsers:         true,
+			AllowStackManagementForRegularUsers:       true,
+		},
+	}
 
 	err := snapshotService.SnapshotEndpoint(endpoint)
 	if err != nil {
@@ -519,7 +574,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 	cryptoService := initCryptoService()
 	digitalSignatureService := initDigitalSignatureService()
 
-	sslService, err := initSSLService(*flags.AddrHTTPS, *flags.Data, *flags.SSLCert, *flags.SSLKey, fileService, dataStore, shutdownTrigger)
+	sslService, err := initSSLService(*flags.AddrHTTPS, *flags.SSLCert, *flags.SSLKey, fileService, dataStore, shutdownTrigger)
 	if err != nil {
 		logrus.Fatal(err)
 	}
@@ -550,7 +605,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 
 	kubernetesTokenCacheManager := kubeproxy.NewTokenCacheManager()
 
-	kubeConfigService := kubernetes2.NewKubeConfigCAService(*flags.AddrHTTPS, sslSettings.CertPath)
+	kubeClusterAccessService := kubernetes.NewKubeClusterAccessService(*flags.BaseURL, *flags.AddrHTTPS, sslSettings.CertPath)
 
 	proxyManager := proxy.NewManager(dataStore, digitalSignatureService, reverseTunnelService, dockerClientFactory, kubernetesClientFactory, kubernetesTokenCacheManager)
 
@@ -657,7 +712,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		OpenAMTService:              openAMTService,
 		ProxyManager:                proxyManager,
 		KubernetesTokenCacheManager: kubernetesTokenCacheManager,
-		KubeConfigService:           kubeConfigService,
+		KubeClusterAccessService:    kubeClusterAccessService,
 		SignatureService:            digitalSignatureService,
 		SnapshotService:             snapshotService,
 		SSLService:                  sslService,
@@ -667,7 +722,6 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		ShutdownCtx:                 shutdownCtx,
 		ShutdownTrigger:             shutdownTrigger,
 		StackDeployer:               stackDeployer,
-		BaseURL:                     *flags.BaseURL,
 	}
 }
 

api/cmd/portainer/main_test.go

@@ -21,7 +21,7 @@ func (m mockKingpinSetting) SetValue(value kingpin.Value) {
 func Test_enableFeaturesFromFlags(t *testing.T) {
 	is := assert.New(t)
 
-	_, store, teardown := datastore.MustNewTestStore(true)
+	_, store, teardown := datastore.MustNewTestStore(true, true)
 	defer teardown()
 
 	tests := []struct {
@@ -76,7 +76,7 @@ func Test_optionalFeature(t *testing.T) {
 
 	is := assert.New(t)
 
-	_, store, teardown := datastore.MustNewTestStore(true)
+	_, store, teardown := datastore.MustNewTestStore(true, true)
 	defer teardown()
 
 	// Enable the test feature

api/connection.go

@@ -1,4 +1,4 @@
-package database
+package portainer
 
 import (
 	"io"
@@ -29,6 +29,7 @@ type Connection interface {
 	GetNextIdentifier(bucketName string) int
 	CreateObject(bucketName string, fn func(uint64) (int, interface{})) error
 	CreateObjectWithId(bucketName string, id int, obj interface{}) error
+	CreateObjectWithStringId(bucketName string, id []byte, obj interface{}) error
 	CreateObjectWithSetSequence(bucketName string, id int, obj interface{}) error
 	GetAll(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error
 	GetAllWithJsoniter(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error

api/crypto/hash.go

@@ -9,11 +9,11 @@ type Service struct{}
 
 // Hash hashes a string using the bcrypt algorithm
 func (*Service) Hash(data string) (string, error) {
-	hash, err := bcrypt.GenerateFromPassword([]byte(data), bcrypt.DefaultCost)
+	bytes, err := bcrypt.GenerateFromPassword([]byte(data), bcrypt.DefaultCost)
 	if err != nil {
-		return "", nil
+		return "", err
 	}
-	return string(hash), nil
+	return string(bytes), err
 }
 
 // CompareHashAndData compares a hash to clear data and returns an error if the comparison fails.

api/crypto/hash_test.go

@@ -0,0 +1,53 @@
+package crypto
+
+import (
+	"testing"
+)
+
+func TestService_Hash(t *testing.T) {
+	var s = &Service{}
+
+	type args struct {
+		hash string
+		data string
+	}
+	tests := []struct {
+		name   string
+		args   args
+		expect bool
+	}{
+		{
+			name: "Empty",
+			args: args{
+				hash: "",
+				data: "",
+			},
+			expect: false,
+		},
+		{
+			name: "Matching",
+			args: args{
+				hash: "$2a$10$6BFGd94oYx8k0bFNO6f33uPUpcpAJyg8UVX.akLe9EthF/ZBTXqcy",
+				data: "Passw0rd!",
+			},
+			expect: true,
+		},
+		{
+			name: "Not matching",
+			args: args{
+				hash: "$2a$10$ltKrUZ7492xyutHOb0/XweevU4jyw7QO66rP32jTVOMb3EX3JxA/a",
+				data: "Passw0rd!",
+			},
+			expect: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+
+			err := s.CompareHashAndData(tt.args.hash, tt.args.data)
+			if (err != nil) == tt.expect {
+				t.Errorf("Service.CompareHashAndData() = %v", err)
+			}
+		})
+	}
+}

api/database/boltdb/db.go

@@ -10,9 +10,9 @@ import (
 	"path"
 	"time"
 
-	"github.com/boltdb/bolt"
 	dserrors "github.com/portainer/portainer/api/dataservices/errors"
 	"github.com/sirupsen/logrus"
+	bolt "go.etcd.io/bbolt"
 )
 
 const (
@@ -161,7 +161,7 @@ func (connection *DbConnection) ExportRaw(filename string) error {
 		return fmt.Errorf("stat on %s failed: %s", databasePath, err)
 	}
 
-	b, err := connection.exportJson(databasePath)
+	b, err := connection.ExportJson(databasePath, true)
 	if err != nil {
 		return err
 	}
@@ -181,10 +181,7 @@ func (connection *DbConnection) ConvertToKey(v int) []byte {
 func (connection *DbConnection) SetServiceName(bucketName string) error {
 	return connection.Batch(func(tx *bolt.Tx) error {
 		_, err := tx.CreateBucketIfNotExists([]byte(bucketName))
-		if err != nil {
-			return err
-		}
-		return nil
+		return err
 	})
 }
 
@@ -314,6 +311,19 @@ func (connection *DbConnection) CreateObjectWithId(bucketName string, id int, ob
 	})
 }
 
+// CreateObjectWithStringId creates a new object in the bucket, using the specified id
+func (connection *DbConnection) CreateObjectWithStringId(bucketName string, id []byte, obj interface{}) error {
+	return connection.Batch(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(bucketName))
+		data, err := connection.MarshalObject(obj)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(id, data)
+	})
+}
+
 // CreateObjectWithSetSequence creates a new object in the bucket, using the specified id, and sets the bucket sequence
 // avoid this :)
 func (connection *DbConnection) CreateObjectWithSetSequence(bucketName string, id int, obj interface{}) error {
@@ -406,7 +416,10 @@ func (connection *DbConnection) RestoreMetadata(s map[string]interface{}) error
 		}
 
 		err = connection.Batch(func(tx *bolt.Tx) error {
-			bucket := tx.Bucket([]byte(bucketName))
+			bucket, err := tx.CreateBucketIfNotExists([]byte(bucketName))
+			if err != nil {
+				return err
+			}
 			return bucket.SetSequence(uint64(id))
 		})
 	}

api/database/boltdb/export.go

@@ -4,13 +4,34 @@ import (
 	"encoding/json"
 	"time"
 
-	"github.com/boltdb/bolt"
 	"github.com/sirupsen/logrus"
+	bolt "go.etcd.io/bbolt"
 )
 
+func backupMetadata(connection *bolt.DB) (map[string]interface{}, error) {
+	buckets := map[string]interface{}{}
+
+	err := connection.View(func(tx *bolt.Tx) error {
+		err := tx.ForEach(func(name []byte, bucket *bolt.Bucket) error {
+			bucketName := string(name)
+			bucket = tx.Bucket([]byte(bucketName))
+			seqId := bucket.Sequence()
+			buckets[bucketName] = int(seqId)
+			return nil
+		})
+
+		return err
+	})
+
+	return buckets, err
+}
+
+// ExportJSON creates a JSON representation from a DbConnection. You can include
+// the database's metadata or ignore it. Ensure the database is closed before
+// using this function
 // inspired by github.com/konoui/boltdb-exporter (which has no license)
 // but very much simplified, based on how we use boltdb
-func (c *DbConnection) exportJson(databasePath string) ([]byte, error) {
+func (c *DbConnection) ExportJson(databasePath string, metadata bool) ([]byte, error) {
 	logrus.WithField("databasePath", databasePath).Infof("exportJson")
 
 	connection, err := bolt.Open(databasePath, 0600, &bolt.Options{Timeout: 1 * time.Second, ReadOnly: true})
@@ -20,6 +41,13 @@ func (c *DbConnection) exportJson(databasePath string) ([]byte, error) {
 	defer connection.Close()
 
 	backup := make(map[string]interface{})
+	if metadata {
+		meta, err := backupMetadata(connection)
+		if err != nil {
+			logrus.WithError(err).Errorf("Failed exporting metadata: %v", err)
+		}
+		backup["__metadata"] = meta
+	}
 
 	err = connection.View(func(tx *bolt.Tx) error {
 		err = tx.ForEach(func(name []byte, bucket *bolt.Bucket) error {
@@ -45,15 +73,20 @@ func (c *DbConnection) exportJson(databasePath string) ([]byte, error) {
 			}
 			if bucketName == "version" {
 				backup[bucketName] = version
+				return nil
 			}
 			if len(list) > 0 {
 				if bucketName == "ssl" ||
 					bucketName == "settings" ||
 					bucketName == "tunnel_server" {
-					backup[bucketName] = list[0]
+					backup[bucketName] = nil
+					if len(list) > 0 {
+						backup[bucketName] = list[0]
+					}
 					return nil
 				}
 				backup[bucketName] = list
+				return nil
 			}
 
 			return nil

api/database/database.go

@@ -3,11 +3,12 @@ package database
 import (
 	"fmt"
 
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/database/boltdb"
 )
 
 // NewDatabase should use config options to return a connection to the requested database
-func NewDatabase(storeType, storePath string, encryptionKey []byte) (connection Connection, err error) {
+func NewDatabase(storeType, storePath string, encryptionKey []byte) (connection portainer.Connection, err error) {
 	switch storeType {
 	case "boltdb":
 		return &boltdb.DbConnection{

api/database/types.go

@@ -1,41 +0,0 @@
-package database
-
-//TODO: as the dependencies get simpler, these should move to their respective dataservices
-
-// EndpointID represents an environment(endpoint) identifier
-type EndpointID int
-
-// UserAccessPolicies represent the association of an access policy and a user
-type UserAccessPolicies map[UserID]AccessPolicy
-
-// TeamAccessPolicies represent the association of an access policy and a team
-type TeamAccessPolicies map[TeamID]AccessPolicy
-
-// TLSConfiguration represents a TLS configuration
-type TLSConfiguration struct {
-	// Use TLS
-	TLS bool `json:"TLS" example:"true"`
-	// Skip the verification of the server TLS certificate
-	TLSSkipVerify bool `json:"TLSSkipVerify" example:"false"`
-	// Path to the TLS CA certificate file
-	TLSCACertPath string `json:"TLSCACert,omitempty" example:"/data/tls/ca.pem"`
-	// Path to the TLS client certificate file
-	TLSCertPath string `json:"TLSCert,omitempty" example:"/data/tls/cert.pem"`
-	// Path to the TLS client key file
-	TLSKeyPath string `json:"TLSKey,omitempty" example:"/data/tls/key.pem"`
-}
-
-// TeamID represents a team identifier
-type TeamID int
-
-// UserID represents a user identifier
-type UserID int
-
-// AccessPolicy represent a policy that can be associated to a user or team
-type AccessPolicy struct {
-	// Role identifier. Reference the role that will be associated to this access policy
-	RoleID RoleID `json:"RoleId" example:"1"`
-}
-
-// RoleID represents a role identifier
-type RoleID int

api/dataservices/apikeyrepository/apikeyrepository.go

@@ -3,7 +3,6 @@ package apikeyrepository
 import (
 	"bytes"
 	"fmt"
-	"github.com/portainer/portainer/api/database"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices/errors"
@@ -17,11 +16,11 @@ const (
 
 // Service represents a service for managing api-key data.
 type Service struct {
-	connection database.Connection
+	connection portainer.Connection
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection database.Connection) (*Service, error) {
+func NewService(connection portainer.Connection) (*Service, error) {
 	err := connection.SetServiceName(BucketName)
 	if err != nil {
 		return nil, err
@@ -33,7 +32,7 @@ func NewService(connection database.Connection) (*Service, error) {
 }
 
 // GetAPIKeysByUserID returns a slice containing all the APIKeys a user has access to.
-func (service *Service) GetAPIKeysByUserID(userID database.UserID) ([]portainer.APIKey, error) {
+func (service *Service) GetAPIKeysByUserID(userID portainer.UserID) ([]portainer.APIKey, error) {
 	var result = make([]portainer.APIKey, 0)
 
 	err := service.connection.GetAll(

api/dataservices/customtemplate/customtemplate.go

@@ -2,7 +2,6 @@ package customtemplate
 
 import (
 	"fmt"
-	"github.com/portainer/portainer/api/database"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/sirupsen/logrus"
@@ -15,7 +14,7 @@ const (
 
 // Service represents a service for managing custom template data.
 type Service struct {
-	connection database.Connection
+	connection portainer.Connection
 }
 
 func (service *Service) BucketName() string {
@@ -23,7 +22,7 @@ func (service *Service) BucketName() string {
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection database.Connection) (*Service, error) {
+func NewService(connection portainer.Connection) (*Service, error) {
 	err := connection.SetServiceName(BucketName)
 	if err != nil {
 		return nil, err

api/dataservices/dockerhub/dockerhub.go

@@ -1,7 +1,7 @@
 package dockerhub
 
 import (
-	portainer "github.com/portainer/portainer/api/database"
+	portainer "github.com/portainer/portainer/api"
 )
 
 const (
@@ -10,8 +10,6 @@ const (
 	dockerHubKey = "DOCKERHUB"
 )
 
-// TODO: does this get used anymore?
-
 // Service represents a service for managing Dockerhub data.
 type Service struct {
 	connection portainer.Connection
@@ -34,8 +32,8 @@ func NewService(connection portainer.Connection) (*Service, error) {
 }
 
 // DockerHub returns the DockerHub object.
-func (service *Service) DockerHub() (*DockerHub, error) {
-	var dockerhub DockerHub
+func (service *Service) DockerHub() (*portainer.DockerHub, error) {
+	var dockerhub portainer.DockerHub
 
 	err := service.connection.GetObject(BucketName, []byte(dockerHubKey), &dockerhub)
 	if err != nil {
@@ -46,16 +44,6 @@ func (service *Service) DockerHub() (*DockerHub, error) {
 }
 
 // UpdateDockerHub updates a DockerHub object.
-func (service *Service) UpdateDockerHub(dockerhub *DockerHub) error {
+func (service *Service) UpdateDockerHub(dockerhub *portainer.DockerHub) error {
 	return service.connection.UpdateObject(BucketName, []byte(dockerHubKey), dockerhub)
 }
-
-// DockerHub represents all the required information to connect and use the Docker Hub
-type DockerHub struct {
-	// Is authentication against DockerHub enabled
-	Authentication bool `json:"Authentication" example:"true"`
-	// Username used to authenticate against the DockerHub
-	Username string `json:"Username" example:"user"`
-	// Password used to authenticate against the DockerHub
-	Password string `json:"Password,omitempty" example:"passwd"`
-}

api/dataservices/edgegroup/edgegroup.go

@@ -2,7 +2,6 @@ package edgegroup
 
 import (
 	"fmt"
-	"github.com/portainer/portainer/api/database"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/sirupsen/logrus"
@@ -15,7 +14,7 @@ const (
 
 // Service represents a service for managing Edge group data.
 type Service struct {
-	connection database.Connection
+	connection portainer.Connection
 }
 
 func (service *Service) BucketName() string {
@@ -23,7 +22,7 @@ func (service *Service) BucketName() string {
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection database.Connection) (*Service, error) {
+func NewService(connection portainer.Connection) (*Service, error) {
 	err := connection.SetServiceName(BucketName)
 	if err != nil {
 		return nil, err

api/dataservices/edgejob/edgejob.go

@@ -2,7 +2,8 @@ package edgejob
 
 import (
 	"fmt"
-	"github.com/portainer/portainer/api/database"
+
+	portainer "github.com/portainer/portainer/api"
 	"github.com/sirupsen/logrus"
 )
 
@@ -13,7 +14,7 @@ const (
 
 // Service represents a service for managing edge jobs data.
 type Service struct {
-	connection database.Connection
+	connection portainer.Connection
 }
 
 func (service *Service) BucketName() string {
@@ -21,7 +22,7 @@ func (service *Service) BucketName() string {
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection database.Connection) (*Service, error) {
+func NewService(connection portainer.Connection) (*Service, error) {
 	err := connection.SetServiceName(BucketName)
 	if err != nil {
 		return nil, err
@@ -33,29 +34,29 @@ func NewService(connection database.Connection) (*Service, error) {
 }
 
 // EdgeJobs returns a list of Edge jobs
-func (service *Service) EdgeJobs() ([]EdgeJob, error) {
-	var edgeJobs = make([]EdgeJob, 0)
+func (service *Service) EdgeJobs() ([]portainer.EdgeJob, error) {
+	var edgeJobs = make([]portainer.EdgeJob, 0)
 
 	err := service.connection.GetAll(
 		BucketName,
-		&EdgeJob{},
+		&portainer.EdgeJob{},
 		func(obj interface{}) (interface{}, error) {
 			//var tag portainer.Tag
-			job, ok := obj.(*EdgeJob)
+			job, ok := obj.(*portainer.EdgeJob)
 			if !ok {
 				logrus.WithField("obj", obj).Errorf("Failed to convert to EdgeJob object")
 				return nil, fmt.Errorf("Failed to convert to EdgeJob object: %s", obj)
 			}
 			edgeJobs = append(edgeJobs, *job)
-			return &EdgeJob{}, nil
+			return &portainer.EdgeJob{}, nil
 		})
 
 	return edgeJobs, err
 }
 
 // EdgeJob returns an Edge job by ID
-func (service *Service) EdgeJob(ID EdgeJobID) (*EdgeJob, error) {
-	var edgeJob EdgeJob
+func (service *Service) EdgeJob(ID portainer.EdgeJobID) (*portainer.EdgeJob, error) {
+	var edgeJob portainer.EdgeJob
 	identifier := service.connection.ConvertToKey(int(ID))
 
 	err := service.connection.GetObject(BucketName, identifier, &edgeJob)
@@ -66,25 +67,25 @@ func (service *Service) EdgeJob(ID EdgeJobID) (*EdgeJob, error) {
 	return &edgeJob, nil
 }
 
-// CreateEdgeJob creates a new Edge job
-func (service *Service) Create(edgeJob *EdgeJob) error {
-	return service.connection.CreateObject(
+// Create creates a new EdgeJob
+func (service *Service) Create(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error {
+	edgeJob.ID = ID
+
+	return service.connection.CreateObjectWithId(
 		BucketName,
-		func(id uint64) (int, interface{}) {
-			edgeJob.ID = EdgeJobID(id)
-			return int(edgeJob.ID), edgeJob
-		},
+		int(edgeJob.ID),
+		edgeJob,
 	)
 }
 
 // UpdateEdgeJob updates an Edge job by ID
-func (service *Service) UpdateEdgeJob(ID EdgeJobID, edgeJob *EdgeJob) error {
+func (service *Service) UpdateEdgeJob(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error {
 	identifier := service.connection.ConvertToKey(int(ID))
 	return service.connection.UpdateObject(BucketName, identifier, edgeJob)
 }
 
 // DeleteEdgeJob deletes an Edge job
-func (service *Service) DeleteEdgeJob(ID EdgeJobID) error {
+func (service *Service) DeleteEdgeJob(ID portainer.EdgeJobID) error {
 	identifier := service.connection.ConvertToKey(int(ID))
 	return service.connection.DeleteObject(BucketName, identifier)
 }
@@ -93,28 +94,3 @@ func (service *Service) DeleteEdgeJob(ID EdgeJobID) error {
 func (service *Service) GetNextIdentifier() int {
 	return service.connection.GetNextIdentifier(BucketName)
 }
-
-// EdgeJob represents a job that can run on Edge environments(endpoints).
-type EdgeJob struct {
-	// EdgeJob Identifier
-	ID             EdgeJobID                                   `json:"Id" example:"1"`
-	Created        int64                                       `json:"Created"`
-	CronExpression string                                      `json:"CronExpression"`
-	Endpoints      map[database.EndpointID]EdgeJobEndpointMeta `json:"Endpoints"`
-	Name           string                                      `json:"Name"`
-	ScriptPath     string                                      `json:"ScriptPath"`
-	Recurring      bool                                        `json:"Recurring"`
-	Version        int                                         `json:"Version"`
-}
-
-// EdgeJobEndpointMeta represents a meta data object for an Edge job and Environment(Endpoint) relation
-type EdgeJobEndpointMeta struct {
-	LogsStatus  EdgeJobLogsStatus
-	CollectLogs bool
-}
-
-// EdgeJobID represents an Edge job identifier
-type EdgeJobID int
-
-// EdgeJobLogsStatus represent status of logs collection job
-type EdgeJobLogsStatus int

api/dataservices/edgestack/edgestack.go

@@ -2,7 +2,6 @@ package edgestack
 
 import (
 	"fmt"
-	"github.com/portainer/portainer/api/database"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/sirupsen/logrus"
@@ -15,7 +14,7 @@ const (
 
 // Service represents a service for managing Edge stack data.
 type Service struct {
-	connection database.Connection
+	connection portainer.Connection
 }
 
 func (service *Service) BucketName() string {
@@ -23,7 +22,7 @@ func (service *Service) BucketName() string {
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection database.Connection) (*Service, error) {
+func NewService(connection portainer.Connection) (*Service, error) {
 	err := connection.SetServiceName(BucketName)
 	if err != nil {
 		return nil, err

api/dataservices/endpoint/endpoint.go

@@ -1,9 +1,10 @@
 package endpoint
 
 import (
+	"errors"
 	"fmt"
+
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/database"
 	"github.com/sirupsen/logrus"
 )
 
@@ -14,7 +15,7 @@ const (
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection database.Connection
+	connection portainer.Connection
 }
 
 func (service *Service) BucketName() string {
@@ -22,7 +23,7 @@ func (service *Service) BucketName() string {
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection database.Connection) (*Service, error) {
+func NewService(connection portainer.Connection) (*Service, error) {
 	err := connection.SetServiceName(BucketName)
 	if err != nil {
 		return nil, err
@@ -34,7 +35,7 @@ func NewService(connection database.Connection) (*Service, error) {
 }
 
 // Endpoint returns an environment(endpoint) by ID.
-func (service *Service) Endpoint(ID database.EndpointID) (*portainer.Endpoint, error) {
+func (service *Service) Endpoint(ID portainer.EndpointID) (*portainer.Endpoint, error) {
 	var endpoint portainer.Endpoint
 	identifier := service.connection.ConvertToKey(int(ID))
 
@@ -47,13 +48,13 @@ func (service *Service) Endpoint(ID database.EndpointID) (*portainer.Endpoint, e
 }
 
 // UpdateEndpoint updates an environment(endpoint).
-func (service *Service) UpdateEndpoint(ID database.EndpointID, endpoint *portainer.Endpoint) error {
+func (service *Service) UpdateEndpoint(ID portainer.EndpointID, endpoint *portainer.Endpoint) error {
 	identifier := service.connection.ConvertToKey(int(ID))
 	return service.connection.UpdateObject(BucketName, identifier, endpoint)
 }
 
 // DeleteEndpoint deletes an environment(endpoint).
-func (service *Service) DeleteEndpoint(ID database.EndpointID) error {
+func (service *Service) DeleteEndpoint(ID portainer.EndpointID) error {
 	identifier := service.connection.ConvertToKey(int(ID))
 	return service.connection.DeleteObject(BucketName, identifier)
 }
@@ -80,58 +81,19 @@ func (service *Service) Endpoints() ([]portainer.Endpoint, error) {
 
 // CreateEndpoint assign an ID to a new environment(endpoint) and saves it.
 func (service *Service) Create(endpoint *portainer.Endpoint) error {
-	if int(endpoint.ID) == 0 {
-		// TODO: hopefully this can become the only path
-		endpoint.ID = database.EndpointID(service.getNextIdentifier())
-	}
 	return service.connection.CreateObjectWithSetSequence(BucketName, int(endpoint.ID), endpoint)
 }
 
+// CreateEndpoint assign an ID to a new environment(endpoint) and saves it.
+func (service *Service) CreateWithCallback(endpoint *portainer.Endpoint, fn func(id uint64) (int, interface{})) error {
+	if endpoint.ID > 0 {
+		return errors.New("the endpoint must not have an ID")
+	}
+
+	return service.connection.CreateObject(BucketName, fn)
+}
+
 // GetNextIdentifier returns the next identifier for an environment(endpoint).
-func (service *Service) getNextIdentifier() int {
+func (service *Service) GetNextIdentifier() int {
 	return service.connection.GetNextIdentifier(BucketName)
 }
-
-func (service *Service) NewDefault() *portainer.Endpoint {
-	return &portainer.Endpoint{
-		//ID:                 0,
-		//Name:               "primary",
-		//URL:                *flags.EndpointURL,
-		//GroupID:            portainer.EndpointGroupID(1),
-		//Type:               portainer.DockerEnvironment,
-		TLSConfig: database.TLSConfiguration{
-			TLS: false,
-		},
-		UserAccessPolicies: database.UserAccessPolicies{},
-		TeamAccessPolicies: database.TeamAccessPolicies{},
-		TagIDs:             []portainer.TagID{},
-		Status:             portainer.EndpointStatusUp,
-		Snapshots:          []portainer.DockerSnapshot{},
-		Kubernetes:         kubernetesDefault(),
-
-		SecuritySettings: portainer.EndpointSecuritySettings{
-			AllowVolumeBrowserForRegularUsers: false,
-			EnableHostManagementFeatures:      false,
-
-			AllowSysctlSettingForRegularUsers:         true,
-			AllowBindMountsForRegularUsers:            true,
-			AllowPrivilegedModeForRegularUsers:        true,
-			AllowHostNamespaceForRegularUsers:         true,
-			AllowContainerCapabilitiesForRegularUsers: true,
-			AllowDeviceMappingForRegularUsers:         true,
-			AllowStackManagementForRegularUsers:       true,
-		},
-	}
-}
-
-func kubernetesDefault() portainer.KubernetesData {
-	return portainer.KubernetesData{
-		Configuration: portainer.KubernetesConfiguration{
-			UseLoadBalancer:  false,
-			UseServerMetrics: false,
-			StorageClasses:   []portainer.KubernetesStorageClassConfig{},
-			IngressClasses:   []portainer.KubernetesIngressClassConfig{},
-		},
-		Snapshots: []portainer.KubernetesSnapshot{},
-	}
-}

api/dataservices/endpointgroup/endpointgroup.go

@@ -2,7 +2,6 @@ package endpointgroup
 
 import (
 	"fmt"
-	"github.com/portainer/portainer/api/database"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/sirupsen/logrus"
@@ -15,7 +14,7 @@ const (
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection database.Connection
+	connection portainer.Connection
 }
 
 func (service *Service) BucketName() string {
@@ -23,7 +22,7 @@ func (service *Service) BucketName() string {
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection database.Connection) (*Service, error) {
+func NewService(connection portainer.Connection) (*Service, error) {
 	err := connection.SetServiceName(BucketName)
 	if err != nil {
 		return nil, err

api/dataservices/endpointrelation/endpointrelation.go

@@ -2,8 +2,8 @@ package endpointrelation
 
 import (
 	"fmt"
+
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/database"
 	"github.com/sirupsen/logrus"
 )
 
@@ -14,7 +14,7 @@ const (
 
 // Service represents a service for managing environment(endpoint) relation data.
 type Service struct {
-	connection database.Connection
+	connection portainer.Connection
 }
 
 func (service *Service) BucketName() string {
@@ -22,7 +22,7 @@ func (service *Service) BucketName() string {
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection database.Connection) (*Service, error) {
+func NewService(connection portainer.Connection) (*Service, error) {
 	err := connection.SetServiceName(BucketName)
 	if err != nil {
 		return nil, err
@@ -54,7 +54,7 @@ func (service *Service) EndpointRelations() ([]portainer.EndpointRelation, error
 }
 
 // EndpointRelation returns a Environment(Endpoint) relation object by EndpointID
-func (service *Service) EndpointRelation(endpointID database.EndpointID) (*portainer.EndpointRelation, error) {
+func (service *Service) EndpointRelation(endpointID portainer.EndpointID) (*portainer.EndpointRelation, error) {
 	var endpointRelation portainer.EndpointRelation
 	identifier := service.connection.ConvertToKey(int(endpointID))
 
@@ -72,13 +72,13 @@ func (service *Service) Create(endpointRelation *portainer.EndpointRelation) err
 }
 
 // UpdateEndpointRelation updates an Environment(Endpoint) relation object
-func (service *Service) UpdateEndpointRelation(EndpointID database.EndpointID, endpointRelation *portainer.EndpointRelation) error {
+func (service *Service) UpdateEndpointRelation(EndpointID portainer.EndpointID, endpointRelation *portainer.EndpointRelation) error {
 	identifier := service.connection.ConvertToKey(int(EndpointID))
 	return service.connection.UpdateObject(BucketName, identifier, endpointRelation)
 }
 
 // DeleteEndpointRelation deletes an Environment(Endpoint) relation object
-func (service *Service) DeleteEndpointRelation(EndpointID database.EndpointID) error {
+func (service *Service) DeleteEndpointRelation(EndpointID portainer.EndpointID) error {
 	identifier := service.connection.ConvertToKey(int(EndpointID))
 	return service.connection.DeleteObject(BucketName, identifier)
 }

api/dataservices/extension/extension.go

@@ -2,7 +2,6 @@ package extension
 
 import (
 	"fmt"
-	"github.com/portainer/portainer/api/database"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/sirupsen/logrus"
@@ -15,7 +14,7 @@ const (
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection database.Connection
+	connection portainer.Connection
 }
 
 func (service *Service) BucketName() string {
@@ -23,7 +22,7 @@ func (service *Service) BucketName() string {
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection database.Connection) (*Service, error) {
+func NewService(connection portainer.Connection) (*Service, error) {
 	err := connection.SetServiceName(BucketName)
 	if err != nil {
 		return nil, err

api/dataservices/fdoprofile/fdoprofile.go

@@ -2,7 +2,6 @@ package fdoprofile
 
 import (
 	"fmt"
-	"github.com/portainer/portainer/api/database"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/sirupsen/logrus"
@@ -15,7 +14,7 @@ const (
 
 // Service represents a service for managingFDO Profiles data.
 type Service struct {
-	connection database.Connection
+	connection portainer.Connection
 }
 
 func (service *Service) BucketName() string {
@@ -23,7 +22,7 @@ func (service *Service) BucketName() string {
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection database.Connection) (*Service, error) {
+func NewService(connection portainer.Connection) (*Service, error) {
 	err := connection.SetServiceName(BucketName)
 	if err != nil {
 		return nil, err

api/dataservices/helmuserrepository/helmuserrepository.go

@@ -2,7 +2,6 @@ package helmuserrepository
 
 import (
 	"fmt"
-	"github.com/portainer/portainer/api/database"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/sirupsen/logrus"
@@ -15,7 +14,7 @@ const (
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection database.Connection
+	connection portainer.Connection
 }
 
 func (service *Service) BucketName() string {
@@ -23,7 +22,7 @@ func (service *Service) BucketName() string {
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection database.Connection) (*Service, error) {
+func NewService(connection portainer.Connection) (*Service, error) {
 	err := connection.SetServiceName(BucketName)
 	if err != nil {
 		return nil, err
@@ -55,7 +54,7 @@ func (service *Service) HelmUserRepositories() ([]portainer.HelmUserRepository,
 }
 
 // HelmUserRepositoryByUserID return an array containing all the HelmUserRepository objects where the specified userID is present.
-func (service *Service) HelmUserRepositoryByUserID(userID database.UserID) ([]portainer.HelmUserRepository, error) {
+func (service *Service) HelmUserRepositoryByUserID(userID portainer.UserID) ([]portainer.HelmUserRepository, error) {
 	var result = make([]portainer.HelmUserRepository, 0)
 
 	err := service.connection.GetAll(

api/dataservices/interface.go

@@ -3,9 +3,6 @@ package dataservices
 // 	"github.com/portainer/portainer/api/dataservices"
 
 import (
-	"github.com/portainer/portainer/api/database"
-	"github.com/portainer/portainer/api/dataservices/edgejob"
-	"github.com/portainer/portainer/api/dataservices/registry"
 	"io"
 	"time"
 
@@ -75,11 +72,11 @@ type (
 
 	// EdgeJobService represents a service to manage Edge jobs
 	EdgeJobService interface {
-		EdgeJobs() ([]edgejob.EdgeJob, error)
-		EdgeJob(ID edgejob.EdgeJobID) (*edgejob.EdgeJob, error)
-		Create(edgeJob *edgejob.EdgeJob) error
-		UpdateEdgeJob(ID edgejob.EdgeJobID, edgeJob *edgejob.EdgeJob) error
-		DeleteEdgeJob(ID edgejob.EdgeJobID) error
+		EdgeJobs() ([]portainer.EdgeJob, error)
+		EdgeJob(ID portainer.EdgeJobID) (*portainer.EdgeJob, error)
+		Create(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error
+		UpdateEdgeJob(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error
+		DeleteEdgeJob(ID portainer.EdgeJobID) error
 		GetNextIdentifier() int
 		BucketName() string
 	}
@@ -97,13 +94,13 @@ type (
 
 	// EndpointService represents a service for managing environment(endpoint) data
 	EndpointService interface {
-		Endpoint(ID database.EndpointID) (*portainer.Endpoint, error)
+		Endpoint(ID portainer.EndpointID) (*portainer.Endpoint, error)
 		Endpoints() ([]portainer.Endpoint, error)
 		Create(endpoint *portainer.Endpoint) error
-		UpdateEndpoint(ID database.EndpointID, endpoint *portainer.Endpoint) error
-		DeleteEndpoint(ID database.EndpointID) error
-		//GetNextIdentifier() int
-		NewDefault() *portainer.Endpoint
+		CreateWithCallback(endpoint *portainer.Endpoint, fn func(uint64) (int, interface{})) error
+		UpdateEndpoint(ID portainer.EndpointID, endpoint *portainer.Endpoint) error
+		DeleteEndpoint(ID portainer.EndpointID) error
+		GetNextIdentifier() int
 		BucketName() string
 	}
 
@@ -120,10 +117,10 @@ type (
 	// EndpointRelationService represents a service for managing environment(endpoint) relations data
 	EndpointRelationService interface {
 		EndpointRelations() ([]portainer.EndpointRelation, error)
-		EndpointRelation(EndpointID database.EndpointID) (*portainer.EndpointRelation, error)
+		EndpointRelation(EndpointID portainer.EndpointID) (*portainer.EndpointRelation, error)
 		Create(endpointRelation *portainer.EndpointRelation) error
-		UpdateEndpointRelation(EndpointID database.EndpointID, endpointRelation *portainer.EndpointRelation) error
-		DeleteEndpointRelation(EndpointID database.EndpointID) error
+		UpdateEndpointRelation(EndpointID portainer.EndpointID, endpointRelation *portainer.EndpointRelation) error
+		DeleteEndpointRelation(EndpointID portainer.EndpointID) error
 		BucketName() string
 	}
 
@@ -141,7 +138,7 @@ type (
 	// HelmUserRepositoryService represents a service to manage HelmUserRepositories
 	HelmUserRepositoryService interface {
 		HelmUserRepositories() ([]portainer.HelmUserRepository, error)
-		HelmUserRepositoryByUserID(userID database.UserID) ([]portainer.HelmUserRepository, error)
+		HelmUserRepositoryByUserID(userID portainer.UserID) ([]portainer.HelmUserRepository, error)
 		Create(record *portainer.HelmUserRepository) error
 		UpdateHelmUserRepository(ID portainer.HelmUserRepositoryID, repository *portainer.HelmUserRepository) error
 		DeleteHelmUserRepository(ID portainer.HelmUserRepositoryID) error
@@ -159,11 +156,11 @@ type (
 
 	// RegistryService represents a service for managing registry data
 	RegistryService interface {
-		Registry(ID registry.RegistryID) (*registry.Registry, error)
-		Registries() ([]registry.Registry, error)
-		Create(registry *registry.Registry) error
-		UpdateRegistry(ID registry.RegistryID, registry *registry.Registry) error
-		DeleteRegistry(ID registry.RegistryID) error
+		Registry(ID portainer.RegistryID) (*portainer.Registry, error)
+		Registries() ([]portainer.Registry, error)
+		Create(registry *portainer.Registry) error
+		UpdateRegistry(ID portainer.RegistryID, registry *portainer.Registry) error
+		DeleteRegistry(ID portainer.RegistryID) error
 		BucketName() string
 	}
 
@@ -180,10 +177,10 @@ type (
 
 	// RoleService represents a service for managing user roles
 	RoleService interface {
-		Role(ID database.RoleID) (*portainer.Role, error)
+		Role(ID portainer.RoleID) (*portainer.Role, error)
 		Roles() ([]portainer.Role, error)
 		Create(role *portainer.Role) error
-		UpdateRole(ID database.RoleID, role *portainer.Role) error
+		UpdateRole(ID portainer.RoleID, role *portainer.Role) error
 		BucketName() string
 	}
 
@@ -193,7 +190,7 @@ type (
 		GetAPIKey(keyID portainer.APIKeyID) (*portainer.APIKey, error)
 		UpdateAPIKey(key *portainer.APIKey) error
 		DeleteAPIKey(ID portainer.APIKeyID) error
-		GetAPIKeysByUserID(userID database.UserID) ([]portainer.APIKey, error)
+		GetAPIKeysByUserID(userID portainer.UserID) ([]portainer.APIKey, error)
 		GetAPIKeyByDigest(digest []byte) (*portainer.APIKey, error)
 	}
 
@@ -239,12 +236,12 @@ type (
 
 	// TeamService represents a service for managing user data
 	TeamService interface {
-		Team(ID database.TeamID) (*portainer.Team, error)
+		Team(ID portainer.TeamID) (*portainer.Team, error)
 		TeamByName(name string) (*portainer.Team, error)
 		Teams() ([]portainer.Team, error)
 		Create(team *portainer.Team) error
-		UpdateTeam(ID database.TeamID, team *portainer.Team) error
-		DeleteTeam(ID database.TeamID) error
+		UpdateTeam(ID portainer.TeamID, team *portainer.Team) error
+		DeleteTeam(ID portainer.TeamID) error
 		BucketName() string
 	}
 
@@ -252,13 +249,13 @@ type (
 	TeamMembershipService interface {
 		TeamMembership(ID portainer.TeamMembershipID) (*portainer.TeamMembership, error)
 		TeamMemberships() ([]portainer.TeamMembership, error)
-		TeamMembershipsByUserID(userID database.UserID) ([]portainer.TeamMembership, error)
-		TeamMembershipsByTeamID(teamID database.TeamID) ([]portainer.TeamMembership, error)
+		TeamMembershipsByUserID(userID portainer.UserID) ([]portainer.TeamMembership, error)
+		TeamMembershipsByTeamID(teamID portainer.TeamID) ([]portainer.TeamMembership, error)
 		Create(membership *portainer.TeamMembership) error
 		UpdateTeamMembership(ID portainer.TeamMembershipID, membership *portainer.TeamMembership) error
 		DeleteTeamMembership(ID portainer.TeamMembershipID) error
-		DeleteTeamMembershipByUserID(userID database.UserID) error
-		DeleteTeamMembershipByTeamID(teamID database.TeamID) error
+		DeleteTeamMembershipByUserID(userID portainer.UserID) error
+		DeleteTeamMembershipByTeamID(teamID portainer.TeamID) error
 		BucketName() string
 	}
 
@@ -271,13 +268,13 @@ type (
 
 	// UserService represents a service for managing user data
 	UserService interface {
-		User(ID database.UserID) (*portainer.User, error)
+		User(ID portainer.UserID) (*portainer.User, error)
 		UserByUsername(username string) (*portainer.User, error)
 		Users() ([]portainer.User, error)
 		UsersByRole(role portainer.UserRole) ([]portainer.User, error)
 		Create(user *portainer.User) error
-		UpdateUser(ID database.UserID, user *portainer.User) error
-		DeleteUser(ID database.UserID) error
+		UpdateUser(ID portainer.UserID, user *portainer.User) error
+		DeleteUser(ID portainer.UserID) error
 		BucketName() string
 	}
 

api/dataservices/registry/registry.go

@@ -2,8 +2,8 @@ package registry
 
 import (
 	"fmt"
-	"github.com/portainer/portainer/api/database"
 
+	portainer "github.com/portainer/portainer/api"
 	"github.com/sirupsen/logrus"
 )
 
@@ -14,7 +14,7 @@ const (
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection database.Connection
+	connection portainer.Connection
 }
 
 func (service *Service) BucketName() string {
@@ -22,7 +22,7 @@ func (service *Service) BucketName() string {
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection database.Connection) (*Service, error) {
+func NewService(connection portainer.Connection) (*Service, error) {
 	err := connection.SetServiceName(BucketName)
 	if err != nil {
 		return nil, err
@@ -34,8 +34,8 @@ func NewService(connection database.Connection) (*Service, error) {
 }
 
 // Registry returns an registry by ID.
-func (service *Service) Registry(ID RegistryID) (*Registry, error) {
-	var registry Registry
+func (service *Service) Registry(ID portainer.RegistryID) (*portainer.Registry, error) {
+	var registry portainer.Registry
 	identifier := service.connection.ConvertToKey(int(ID))
 
 	err := service.connection.GetObject(BucketName, identifier, &registry)
@@ -47,128 +47,44 @@ func (service *Service) Registry(ID RegistryID) (*Registry, error) {
 }
 
 // Registries returns an array containing all the registries.
-func (service *Service) Registries() ([]Registry, error) {
-	var registries = make([]Registry, 0)
+func (service *Service) Registries() ([]portainer.Registry, error) {
+	var registries = make([]portainer.Registry, 0)
 
 	err := service.connection.GetAll(
 		BucketName,
-		&Registry{},
+		&portainer.Registry{},
 		func(obj interface{}) (interface{}, error) {
-			registry, ok := obj.(*Registry)
+			registry, ok := obj.(*portainer.Registry)
 			if !ok {
 				logrus.WithField("obj", obj).Errorf("Failed to convert to Registry object")
 				return nil, fmt.Errorf("Failed to convert to Registry object: %s", obj)
 			}
 			registries = append(registries, *registry)
-			return &Registry{}, nil
+			return &portainer.Registry{}, nil
 		})
 
 	return registries, err
 }
 
 // CreateRegistry creates a new registry.
-func (service *Service) Create(registry *Registry) error {
+func (service *Service) Create(registry *portainer.Registry) error {
 	return service.connection.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
-			registry.ID = RegistryID(id)
+			registry.ID = portainer.RegistryID(id)
 			return int(registry.ID), registry
 		},
 	)
 }
 
 // UpdateRegistry updates an registry.
-func (service *Service) UpdateRegistry(ID RegistryID, registry *Registry) error {
+func (service *Service) UpdateRegistry(ID portainer.RegistryID, registry *portainer.Registry) error {
 	identifier := service.connection.ConvertToKey(int(ID))
 	return service.connection.UpdateObject(BucketName, identifier, registry)
 }
 
 // DeleteRegistry deletes an registry.
-func (service *Service) DeleteRegistry(ID RegistryID) error {
+func (service *Service) DeleteRegistry(ID portainer.RegistryID) error {
 	identifier := service.connection.ConvertToKey(int(ID))
 	return service.connection.DeleteObject(BucketName, identifier)
 }
-
-// Registry represents a Docker registry with all the info required to connect to it
-type Registry struct {
-	// Registry Identifier
-	ID RegistryID `json:"Id" example:"1"`
-	// Registry Type (1 - Quay, 2 - Azure, 3 - Custom, 4 - Gitlab, 5 - ProGet, 6 - DockerHub, 7 - ECR)
-	Type RegistryType `json:"Type" enums:"1,2,3,4,5,6,7"`
-	// Registry Name
-	Name string `json:"Name" example:"my-registry"`
-	// URL or IP address of the Docker registry
-	URL string `json:"URL" example:"registry.mydomain.tld:2375"`
-	// Base URL, introduced for ProGet registry
-	BaseURL string `json:"BaseURL" example:"registry.mydomain.tld:2375"`
-	// Is authentication against this registry enabled
-	Authentication bool `json:"Authentication" example:"true"`
-	// Username or AccessKeyID used to authenticate against this registry
-	Username string `json:"Username" example:"registry user"`
-	// Password or SecretAccessKey used to authenticate against this registry
-	Password                string                           `json:"Password,omitempty" example:"registry_password"`
-	ManagementConfiguration *RegistryManagementConfiguration `json:"ManagementConfiguration"`
-	Gitlab                  GitlabRegistryData               `json:"Gitlab"`
-	Quay                    QuayRegistryData                 `json:"Quay"`
-	Ecr                     EcrData                          `json:"Ecr"`
-	RegistryAccesses        RegistryAccesses                 `json:"RegistryAccesses"`
-
-	// Deprecated fields
-	// Deprecated in DBVersion == 31
-	UserAccessPolicies database.UserAccessPolicies `json:"UserAccessPolicies"`
-	// Deprecated in DBVersion == 31
-	TeamAccessPolicies database.TeamAccessPolicies `json:"TeamAccessPolicies"`
-
-	// Deprecated in DBVersion == 18
-	AuthorizedUsers []database.UserID `json:"AuthorizedUsers"`
-	// Deprecated in DBVersion == 18
-	AuthorizedTeams []database.TeamID `json:"AuthorizedTeams"`
-
-	// Stores temporary access token
-	AccessToken       string `json:"AccessToken,omitempty"`
-	AccessTokenExpiry int64  `json:"AccessTokenExpiry,omitempty"`
-}
-
-type RegistryAccesses map[database.EndpointID]RegistryAccessPolicies
-
-type RegistryAccessPolicies struct {
-	UserAccessPolicies database.UserAccessPolicies `json:"UserAccessPolicies"`
-	TeamAccessPolicies database.TeamAccessPolicies `json:"TeamAccessPolicies"`
-	Namespaces         []string                    `json:"Namespaces"`
-}
-
-// RegistryID represents a registry identifier
-type RegistryID int
-
-// RegistryManagementConfiguration represents a configuration that can be used to query the registry API via the registry management extension.
-type RegistryManagementConfiguration struct {
-	Type              RegistryType              `json:"Type"`
-	Authentication    bool                      `json:"Authentication"`
-	Username          string                    `json:"Username"`
-	Password          string                    `json:"Password"`
-	TLSConfig         database.TLSConfiguration `json:"TLSConfig"`
-	Ecr               EcrData                   `json:"Ecr"`
-	AccessToken       string                    `json:"AccessToken,omitempty"`
-	AccessTokenExpiry int64                     `json:"AccessTokenExpiry,omitempty"`
-}
-
-// RegistryType represents a type of registry
-type RegistryType int
-
-// GitlabRegistryData represents data required for gitlab registry to work
-type GitlabRegistryData struct {
-	ProjectID   int    `json:"ProjectId"`
-	InstanceURL string `json:"InstanceURL"`
-	ProjectPath string `json:"ProjectPath"`
-}
-
-// QuayRegistryData represents data required for Quay registry to work
-type QuayRegistryData struct {
-	UseOrganisation  bool   `json:"UseOrganisation"`
-	OrganisationName string `json:"OrganisationName"`
-}
-
-// EcrData represents data required for ECR registry
-type EcrData struct {
-	Region string `json:"Region" example:"ap-southeast-2"`
-}

api/dataservices/resourcecontrol/resourcecontrol.go

@@ -2,7 +2,6 @@ package resourcecontrol
 
 import (
 	"fmt"
-	"github.com/portainer/portainer/api/database"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/sirupsen/logrus"
@@ -15,7 +14,7 @@ const (
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection database.Connection
+	connection portainer.Connection
 }
 
 func (service *Service) BucketName() string {
@@ -23,7 +22,7 @@ func (service *Service) BucketName() string {
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection database.Connection) (*Service, error) {
+func NewService(connection portainer.Connection) (*Service, error) {
 	err := connection.SetServiceName(BucketName)
 	if err != nil {
 		return nil, err

api/dataservices/role/role.go

@@ -2,7 +2,6 @@ package role
 
 import (
 	"fmt"
-	"github.com/portainer/portainer/api/database"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/sirupsen/logrus"
@@ -15,7 +14,7 @@ const (
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection database.Connection
+	connection portainer.Connection
 }
 
 func (service *Service) BucketName() string {
@@ -23,7 +22,7 @@ func (service *Service) BucketName() string {
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection database.Connection) (*Service, error) {
+func NewService(connection portainer.Connection) (*Service, error) {
 	err := connection.SetServiceName(BucketName)
 	if err != nil {
 		return nil, err
@@ -35,7 +34,7 @@ func NewService(connection database.Connection) (*Service, error) {
 }
 
 // Role returns a Role by ID
-func (service *Service) Role(ID database.RoleID) (*portainer.Role, error) {
+func (service *Service) Role(ID portainer.RoleID) (*portainer.Role, error) {
 	var set portainer.Role
 	identifier := service.connection.ConvertToKey(int(ID))
 
@@ -72,14 +71,14 @@ func (service *Service) Create(role *portainer.Role) error {
 	return service.connection.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
-			role.ID = database.RoleID(id)
+			role.ID = portainer.RoleID(id)
 			return int(role.ID), role
 		},
 	)
 }
 
 // UpdateRole updates a role.
-func (service *Service) UpdateRole(ID database.RoleID, role *portainer.Role) error {
+func (service *Service) UpdateRole(ID portainer.RoleID, role *portainer.Role) error {
 	identifier := service.connection.ConvertToKey(int(ID))
 	return service.connection.UpdateObject(BucketName, identifier, role)
 }

api/dataservices/schedule/schedule.go

@@ -2,8 +2,8 @@ package schedule
 
 import (
 	"fmt"
-	"github.com/portainer/portainer/api/database"
 
+	portainer "github.com/portainer/portainer/api"
 	"github.com/sirupsen/logrus"
 )
 
@@ -14,7 +14,7 @@ const (
 
 // Service represents a service for managing schedule data.
 type Service struct {
-	connection database.Connection
+	connection portainer.Connection
 }
 
 func (service *Service) BucketName() string {
@@ -22,7 +22,7 @@ func (service *Service) BucketName() string {
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection database.Connection) (*Service, error) {
+func NewService(connection portainer.Connection) (*Service, error) {
 	err := connection.SetServiceName(BucketName)
 	if err != nil {
 		return nil, err
@@ -34,8 +34,8 @@ func NewService(connection database.Connection) (*Service, error) {
 }
 
 // Schedule returns a schedule by ID.
-func (service *Service) Schedule(ID ScheduleID) (*Schedule, error) {
-	var schedule Schedule
+func (service *Service) Schedule(ID portainer.ScheduleID) (*portainer.Schedule, error) {
+	var schedule portainer.Schedule
 	identifier := service.connection.ConvertToKey(int(ID))
 
 	err := service.connection.GetObject(BucketName, identifier, &schedule)
@@ -47,32 +47,32 @@ func (service *Service) Schedule(ID ScheduleID) (*Schedule, error) {
 }
 
 // UpdateSchedule updates a schedule.
-func (service *Service) UpdateSchedule(ID ScheduleID, schedule *Schedule) error {
+func (service *Service) UpdateSchedule(ID portainer.ScheduleID, schedule *portainer.Schedule) error {
 	identifier := service.connection.ConvertToKey(int(ID))
 	return service.connection.UpdateObject(BucketName, identifier, schedule)
 }
 
 // DeleteSchedule deletes a schedule.
-func (service *Service) DeleteSchedule(ID ScheduleID) error {
+func (service *Service) DeleteSchedule(ID portainer.ScheduleID) error {
 	identifier := service.connection.ConvertToKey(int(ID))
 	return service.connection.DeleteObject(BucketName, identifier)
 }
 
 // Schedules return a array containing all the schedules.
-func (service *Service) Schedules() ([]Schedule, error) {
-	var schedules = make([]Schedule, 0)
+func (service *Service) Schedules() ([]portainer.Schedule, error) {
+	var schedules = make([]portainer.Schedule, 0)
 
 	err := service.connection.GetAll(
 		BucketName,
-		&Schedule{},
+		&portainer.Schedule{},
 		func(obj interface{}) (interface{}, error) {
-			schedule, ok := obj.(*Schedule)
+			schedule, ok := obj.(*portainer.Schedule)
 			if !ok {
 				logrus.WithField("obj", obj).Errorf("Failed to convert to Schedule object")
 				return nil, fmt.Errorf("Failed to convert to Schedule object: %s", obj)
 			}
 			schedules = append(schedules, *schedule)
-			return &Schedule{}, nil
+			return &portainer.Schedule{}, nil
 		})
 
 	return schedules, err
@@ -80,14 +80,14 @@ func (service *Service) Schedules() ([]Schedule, error) {
 
 // SchedulesByJobType return a array containing all the schedules
 // with the specified JobType.
-func (service *Service) SchedulesByJobType(jobType JobType) ([]Schedule, error) {
-	var schedules = make([]Schedule, 0)
+func (service *Service) SchedulesByJobType(jobType portainer.JobType) ([]portainer.Schedule, error) {
+	var schedules = make([]portainer.Schedule, 0)
 
 	err := service.connection.GetAll(
 		BucketName,
-		&Schedule{},
+		&portainer.Schedule{},
 		func(obj interface{}) (interface{}, error) {
-			schedule, ok := obj.(*Schedule)
+			schedule, ok := obj.(*portainer.Schedule)
 			if !ok {
 				logrus.WithField("obj", obj).Errorf("Failed to convert to Schedule object")
 				return nil, fmt.Errorf("Failed to convert to Schedule object: %s", obj)
@@ -95,14 +95,14 @@ func (service *Service) SchedulesByJobType(jobType JobType) ([]Schedule, error)
 			if schedule.JobType == jobType {
 				schedules = append(schedules, *schedule)
 			}
-			return &Schedule{}, nil
+			return &portainer.Schedule{}, nil
 		})
 
 	return schedules, err
 }
 
 // Create assign an ID to a new schedule and saves it.
-func (service *Service) CreateSchedule(schedule *Schedule) error {
+func (service *Service) CreateSchedule(schedule *portainer.Schedule) error {
 	return service.connection.CreateObjectWithSetSequence(BucketName, int(schedule.ID), schedule)
 }
 
@@ -110,43 +110,3 @@ func (service *Service) CreateSchedule(schedule *Schedule) error {
 func (service *Service) GetNextIdentifier() int {
 	return service.connection.GetNextIdentifier(BucketName)
 }
-
-// Schedule represents a scheduled job.
-// It only contains a pointer to one of the JobRunner implementations
-// based on the JobType.
-// NOTE: The Recurring option is only used by ScriptExecutionJob at the moment
-// Deprecated in favor of EdgeJob
-type Schedule struct {
-	// Schedule Identifier
-	ID             ScheduleID `json:"Id" example:"1"`
-	Name           string
-	CronExpression string
-	Recurring      bool
-	Created        int64
-	JobType        JobType
-	EdgeSchedule   *EdgeSchedule
-}
-
-// EdgeSchedule represents a scheduled job that can run on Edge environments(endpoints).
-// Deprecated in favor of EdgeJob
-type EdgeSchedule struct {
-	// EdgeSchedule Identifier
-	ID             ScheduleID            `json:"Id" example:"1"`
-	CronExpression string                `json:"CronExpression"`
-	Script         string                `json:"Script"`
-	Version        int                   `json:"Version"`
-	Endpoints      []database.EndpointID `json:"Endpoints"`
-}
-
-// ScheduleID represents a schedule identifier.
-// Deprecated in favor of EdgeJob
-type ScheduleID int
-
-// JobType represents a job type
-type JobType int
-
-const (
-	_ JobType = iota
-	// SnapshotJobType is a system job used to create environment(endpoint) snapshots
-	SnapshotJobType = 2
-)

api/dataservices/settings/settings.go

@@ -1,8 +1,9 @@
 package settings
 
 import (
+	"sync"
+
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/database"
 )
 
 const (
@@ -13,7 +14,31 @@ const (
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection database.Connection
+	connection portainer.Connection
+	cache      *portainer.Settings
+	mu         sync.RWMutex
+}
+
+func cloneSettings(src *portainer.Settings) *portainer.Settings {
+	if src == nil {
+		return nil
+	}
+
+	c := *src
+
+	if c.BlackListedLabels != nil {
+		c.BlackListedLabels = make([]portainer.Pair, len(src.BlackListedLabels))
+		copy(c.BlackListedLabels, src.BlackListedLabels)
+	}
+
+	if src.FeatureFlagSettings != nil {
+		c.FeatureFlagSettings = make(map[portainer.Feature]bool)
+		for k, v := range src.FeatureFlagSettings {
+			c.FeatureFlagSettings[k] = v
+		}
+	}
+
+	return &c
 }
 
 func (service *Service) BucketName() string {
@@ -21,7 +46,7 @@ func (service *Service) BucketName() string {
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection database.Connection) (*Service, error) {
+func NewService(connection portainer.Connection) (*Service, error) {
 	err := connection.SetServiceName(BucketName)
 	if err != nil {
 		return nil, err
@@ -34,6 +59,18 @@ func NewService(connection database.Connection) (*Service, error) {
 
 // Settings retrieve the settings object.
 func (service *Service) Settings() (*portainer.Settings, error) {
+	service.mu.RLock()
+	if service.cache != nil {
+		s := cloneSettings(service.cache)
+		service.mu.RUnlock()
+
+		return s, nil
+	}
+	service.mu.RUnlock()
+
+	service.mu.Lock()
+	defer service.mu.Unlock()
+
 	var settings portainer.Settings
 
 	err := service.connection.GetObject(BucketName, []byte(settingsKey), &settings)
@@ -41,12 +78,24 @@ func (service *Service) Settings() (*portainer.Settings, error) {
 		return nil, err
 	}
 
+	service.cache = cloneSettings(&settings)
+
 	return &settings, nil
 }
 
 // UpdateSettings persists a Settings object.
 func (service *Service) UpdateSettings(settings *portainer.Settings) error {
-	return service.connection.UpdateObject(BucketName, []byte(settingsKey), settings)
+	service.mu.Lock()
+	defer service.mu.Unlock()
+
+	err := service.connection.UpdateObject(BucketName, []byte(settingsKey), settings)
+	if err != nil {
+		return err
+	}
+
+	service.cache = cloneSettings(settings)
+
+	return nil
 }
 
 func (service *Service) IsFeatureFlagEnabled(feature portainer.Feature) bool {
@@ -62,3 +111,9 @@ func (service *Service) IsFeatureFlagEnabled(feature portainer.Feature) bool {
 
 	return false
 }
+
+func (service *Service) InvalidateCache() {
+	service.mu.Lock()
+	service.cache = nil
+	service.mu.Unlock()
+}

api/dataservices/ssl/ssl.go

@@ -2,7 +2,6 @@ package ssl
 
 import (
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/database"
 )
 
 const (
@@ -13,7 +12,7 @@ const (
 
 // Service represents a service for managing ssl data.
 type Service struct {
-	connection database.Connection
+	connection portainer.Connection
 }
 
 func (service *Service) BucketName() string {
@@ -21,7 +20,7 @@ func (service *Service) BucketName() string {
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection database.Connection) (*Service, error) {
+func NewService(connection portainer.Connection) (*Service, error) {
 	err := connection.SetServiceName(BucketName)
 	if err != nil {
 		return nil, err

api/dataservices/stack/stack.go

@@ -2,7 +2,6 @@ package stack
 
 import (
 	"fmt"
-	"github.com/portainer/portainer/api/database"
 	"strings"
 
 	"github.com/sirupsen/logrus"
@@ -18,7 +17,7 @@ const (
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection database.Connection
+	connection portainer.Connection
 }
 
 func (service *Service) BucketName() string {
@@ -26,7 +25,7 @@ func (service *Service) BucketName() string {
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection database.Connection) (*Service, error) {
+func NewService(connection portainer.Connection) (*Service, error) {
 	err := connection.SetServiceName(BucketName)
 	if err != nil {
 		return nil, err

api/dataservices/stack/tests/stack_test.go

@@ -29,7 +29,7 @@ func TestService_StackByWebhookID(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping test in short mode. Normally takes ~1s to run.")
 	}
-	_, store, teardown := datastore.MustNewTestStore(true)
+	_, store, teardown := datastore.MustNewTestStore(true, true)
 	defer teardown()
 
 	b := stackBuilder{t: t, store: store}
@@ -87,7 +87,7 @@ func Test_RefreshableStacks(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping test in short mode. Normally takes ~1s to run.")
 	}
-	_, store, teardown := datastore.MustNewTestStore(true)
+	_, store, teardown := datastore.MustNewTestStore(true, true)
 	defer teardown()
 
 	staticStack := portainer.Stack{ID: 1}

api/dataservices/tag/tag.go

@@ -2,7 +2,6 @@ package tag
 
 import (
 	"fmt"
-	"github.com/portainer/portainer/api/database"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/sirupsen/logrus"
@@ -15,7 +14,7 @@ const (
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection database.Connection
+	connection portainer.Connection
 }
 
 func (service *Service) BucketName() string {
@@ -23,7 +22,7 @@ func (service *Service) BucketName() string {
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection database.Connection) (*Service, error) {
+func NewService(connection portainer.Connection) (*Service, error) {
 	err := connection.SetServiceName(BucketName)
 	if err != nil {
 		return nil, err

api/dataservices/team/team.go

@@ -2,7 +2,6 @@ package team
 
 import (
 	"fmt"
-	"github.com/portainer/portainer/api/database"
 	"strings"
 
 	"github.com/portainer/portainer/api/dataservices/errors"
@@ -18,7 +17,7 @@ const (
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection database.Connection
+	connection portainer.Connection
 }
 
 func (service *Service) BucketName() string {
@@ -26,7 +25,7 @@ func (service *Service) BucketName() string {
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection database.Connection) (*Service, error) {
+func NewService(connection portainer.Connection) (*Service, error) {
 	err := connection.SetServiceName(BucketName)
 	if err != nil {
 		return nil, err
@@ -38,7 +37,7 @@ func NewService(connection database.Connection) (*Service, error) {
 }
 
 // Team returns a Team by ID
-func (service *Service) Team(ID database.TeamID) (*portainer.Team, error) {
+func (service *Service) Team(ID portainer.TeamID) (*portainer.Team, error) {
 	var team portainer.Team
 	identifier := service.connection.ConvertToKey(int(ID))
 
@@ -101,7 +100,7 @@ func (service *Service) Teams() ([]portainer.Team, error) {
 }
 
 // UpdateTeam saves a Team.
-func (service *Service) UpdateTeam(ID database.TeamID, team *portainer.Team) error {
+func (service *Service) UpdateTeam(ID portainer.TeamID, team *portainer.Team) error {
 	identifier := service.connection.ConvertToKey(int(ID))
 	return service.connection.UpdateObject(BucketName, identifier, team)
 }
@@ -111,14 +110,14 @@ func (service *Service) Create(team *portainer.Team) error {
 	return service.connection.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
-			team.ID = database.TeamID(id)
+			team.ID = portainer.TeamID(id)
 			return int(team.ID), team
 		},
 	)
 }
 
 // DeleteTeam deletes a Team.
-func (service *Service) DeleteTeam(ID database.TeamID) error {
+func (service *Service) DeleteTeam(ID portainer.TeamID) error {
 	identifier := service.connection.ConvertToKey(int(ID))
 	return service.connection.DeleteObject(BucketName, identifier)
 }

api/dataservices/team/tests/team_test.go

@@ -10,7 +10,7 @@ import (
 
 func Test_teamByName(t *testing.T) {
 	t.Run("When store is empty should return ErrObjectNotFound", func(t *testing.T) {
-		_, store, teardown := datastore.MustNewTestStore(true)
+		_, store, teardown := datastore.MustNewTestStore(true, true)
 		defer teardown()
 
 		_, err := store.Team().TeamByName("name")
@@ -19,7 +19,7 @@ func Test_teamByName(t *testing.T) {
 	})
 
 	t.Run("When there is no object with the same name should return ErrObjectNotFound", func(t *testing.T) {
-		_, store, teardown := datastore.MustNewTestStore(true)
+		_, store, teardown := datastore.MustNewTestStore(true, true)
 		defer teardown()
 
 		teamBuilder := teamBuilder{
@@ -35,7 +35,7 @@ func Test_teamByName(t *testing.T) {
 	})
 
 	t.Run("When there is an object with the same name should return the object", func(t *testing.T) {
-		_, store, teardown := datastore.MustNewTestStore(true)
+		_, store, teardown := datastore.MustNewTestStore(true, true)
 		defer teardown()
 
 		teamBuilder := teamBuilder{

api/dataservices/team/tests/utils.go

@@ -1,7 +1,6 @@
 package tests
 
 import (
-	"github.com/portainer/portainer/api/database"
 	"testing"
 
 	portainer "github.com/portainer/portainer/api"
@@ -18,7 +17,7 @@ type teamBuilder struct {
 func (b *teamBuilder) createNew(name string) *portainer.Team {
 	b.count++
 	team := &portainer.Team{
-		ID:   database.TeamID(b.count),
+		ID:   portainer.TeamID(b.count),
 		Name: name,
 	}
 

api/dataservices/teammembership/teammembership.go

@@ -2,7 +2,6 @@ package teammembership
 
 import (
 	"fmt"
-	"github.com/portainer/portainer/api/database"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/sirupsen/logrus"
@@ -15,7 +14,7 @@ const (
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection database.Connection
+	connection portainer.Connection
 }
 
 func (service *Service) BucketName() string {
@@ -23,7 +22,7 @@ func (service *Service) BucketName() string {
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection database.Connection) (*Service, error) {
+func NewService(connection portainer.Connection) (*Service, error) {
 	err := connection.SetServiceName(BucketName)
 	if err != nil {
 		return nil, err
@@ -68,7 +67,7 @@ func (service *Service) TeamMemberships() ([]portainer.TeamMembership, error) {
 }
 
 // TeamMembershipsByUserID return an array containing all the TeamMembership objects where the specified userID is present.
-func (service *Service) TeamMembershipsByUserID(userID database.UserID) ([]portainer.TeamMembership, error) {
+func (service *Service) TeamMembershipsByUserID(userID portainer.UserID) ([]portainer.TeamMembership, error) {
 	var memberships = make([]portainer.TeamMembership, 0)
 
 	err := service.connection.GetAll(
@@ -90,7 +89,7 @@ func (service *Service) TeamMembershipsByUserID(userID database.UserID) ([]porta
 }
 
 // TeamMembershipsByTeamID return an array containing all the TeamMembership objects where the specified teamID is present.
-func (service *Service) TeamMembershipsByTeamID(teamID database.TeamID) ([]portainer.TeamMembership, error) {
+func (service *Service) TeamMembershipsByTeamID(teamID portainer.TeamID) ([]portainer.TeamMembership, error) {
 	var memberships = make([]portainer.TeamMembership, 0)
 
 	err := service.connection.GetAll(
@@ -135,7 +134,7 @@ func (service *Service) DeleteTeamMembership(ID portainer.TeamMembershipID) erro
 }
 
 // DeleteTeamMembershipByUserID deletes all the TeamMembership object associated to a UserID.
-func (service *Service) DeleteTeamMembershipByUserID(userID database.UserID) error {
+func (service *Service) DeleteTeamMembershipByUserID(userID portainer.UserID) error {
 	return service.connection.DeleteAllObjects(
 		BucketName,
 		func(obj interface{}) (id int, ok bool) {
@@ -153,7 +152,7 @@ func (service *Service) DeleteTeamMembershipByUserID(userID database.UserID) err
 }
 
 // DeleteTeamMembershipByTeamID deletes all the TeamMembership object associated to a TeamID.
-func (service *Service) DeleteTeamMembershipByTeamID(teamID database.TeamID) error {
+func (service *Service) DeleteTeamMembershipByTeamID(teamID portainer.TeamID) error {
 	return service.connection.DeleteAllObjects(
 		BucketName,
 		func(obj interface{}) (id int, ok bool) {

api/dataservices/tunnelserver/tunnelserver.go

@@ -2,7 +2,6 @@ package tunnelserver
 
 import (
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/database"
 )
 
 const (
@@ -13,7 +12,7 @@ const (
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection database.Connection
+	connection portainer.Connection
 }
 
 func (service *Service) BucketName() string {
@@ -21,7 +20,7 @@ func (service *Service) BucketName() string {
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection database.Connection) (*Service, error) {
+func NewService(connection portainer.Connection) (*Service, error) {
 	err := connection.SetServiceName(BucketName)
 	if err != nil {
 		return nil, err

api/dataservices/user/user.go

@@ -2,7 +2,6 @@ package user
 
 import (
 	"fmt"
-	"github.com/portainer/portainer/api/database"
 	"strings"
 
 	"github.com/portainer/portainer/api/dataservices/errors"
@@ -18,7 +17,7 @@ const (
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection database.Connection
+	connection portainer.Connection
 }
 
 func (service *Service) BucketName() string {
@@ -26,7 +25,7 @@ func (service *Service) BucketName() string {
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection database.Connection) (*Service, error) {
+func NewService(connection portainer.Connection) (*Service, error) {
 	err := connection.SetServiceName(BucketName)
 	if err != nil {
 		return nil, err
@@ -38,7 +37,7 @@ func NewService(connection database.Connection) (*Service, error) {
 }
 
 // User returns a user by ID
-func (service *Service) User(ID database.UserID) (*portainer.User, error) {
+func (service *Service) User(ID portainer.UserID) (*portainer.User, error) {
 	var user portainer.User
 	identifier := service.connection.ConvertToKey(int(ID))
 
@@ -122,7 +121,7 @@ func (service *Service) UsersByRole(role portainer.UserRole) ([]portainer.User,
 }
 
 // UpdateUser saves a user.
-func (service *Service) UpdateUser(ID database.UserID, user *portainer.User) error {
+func (service *Service) UpdateUser(ID portainer.UserID, user *portainer.User) error {
 	identifier := service.connection.ConvertToKey(int(ID))
 	user.Username = strings.ToLower(user.Username)
 	return service.connection.UpdateObject(BucketName, identifier, user)
@@ -133,7 +132,7 @@ func (service *Service) Create(user *portainer.User) error {
 	return service.connection.CreateObject(
 		BucketName,
 		func(id uint64) (int, interface{}) {
-			user.ID = database.UserID(id)
+			user.ID = portainer.UserID(id)
 			user.Username = strings.ToLower(user.Username)
 
 			return int(user.ID), user
@@ -142,7 +141,7 @@ func (service *Service) Create(user *portainer.User) error {
 }
 
 // DeleteUser deletes a user.
-func (service *Service) DeleteUser(ID database.UserID) error {
+func (service *Service) DeleteUser(ID portainer.UserID) error {
 	identifier := service.connection.ConvertToKey(int(ID))
 	return service.connection.DeleteObject(BucketName, identifier)
 }

api/dataservices/version/version.go

@@ -1,7 +1,6 @@
 package version
 
 import (
-	"github.com/portainer/portainer/api/database"
 	"strconv"
 
 	portainer "github.com/portainer/portainer/api"
@@ -18,7 +17,7 @@ const (
 
 // Service represents a service to manage stored versions.
 type Service struct {
-	connection database.Connection
+	connection portainer.Connection
 }
 
 func (service *Service) BucketName() string {
@@ -26,7 +25,7 @@ func (service *Service) BucketName() string {
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection database.Connection) (*Service, error) {
+func NewService(connection portainer.Connection) (*Service, error) {
 	err := connection.SetServiceName(BucketName)
 	if err != nil {
 		return nil, err

api/dataservices/webhook/webhook.go

@@ -2,7 +2,6 @@ package webhook
 
 import (
 	"fmt"
-	"github.com/portainer/portainer/api/database"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices/errors"
@@ -16,7 +15,7 @@ const (
 
 // Service represents a service for managing webhook data.
 type Service struct {
-	connection database.Connection
+	connection portainer.Connection
 }
 
 func (service *Service) BucketName() string {
@@ -24,7 +23,7 @@ func (service *Service) BucketName() string {
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection database.Connection) (*Service, error) {
+func NewService(connection portainer.Connection) (*Service, error) {
 	err := connection.SetServiceName(BucketName)
 	if err != nil {
 		return nil, err

api/datastore/backup.go

@@ -69,6 +69,11 @@ func getBackupRestoreOptions(backupDir string) *BackupOptions {
 	}
 }
 
+// Backup current database with default options
+func (store *Store) Backup() (string, error) {
+	return store.backupWithOptions(nil)
+}
+
 func (store *Store) setupOptions(options *BackupOptions) *BackupOptions {
 	if options == nil {
 		options = &BackupOptions{}

api/datastore/backup_test.go

@@ -10,7 +10,7 @@ import (
 )
 
 func TestCreateBackupFolders(t *testing.T) {
-	_, store, teardown := MustNewTestStore(false)
+	_, store, teardown := MustNewTestStore(false, true)
 	defer teardown()
 
 	connection := store.GetConnection()
@@ -27,7 +27,7 @@ func TestCreateBackupFolders(t *testing.T) {
 }
 
 func TestStoreCreation(t *testing.T) {
-	_, store, teardown := MustNewTestStore(true)
+	_, store, teardown := MustNewTestStore(true, true)
 	defer teardown()
 
 	if store == nil {
@@ -40,7 +40,7 @@ func TestStoreCreation(t *testing.T) {
 }
 
 func TestBackup(t *testing.T) {
-	_, store, teardown := MustNewTestStore(true)
+	_, store, teardown := MustNewTestStore(true, true)
 	connection := store.GetConnection()
 	defer teardown()
 
@@ -67,7 +67,7 @@ func TestBackup(t *testing.T) {
 }
 
 func TestRemoveWithOptions(t *testing.T) {
-	_, store, teardown := MustNewTestStore(true)
+	_, store, teardown := MustNewTestStore(true, true)
 	defer teardown()
 
 	t.Run("successfully removes file if existent", func(t *testing.T) {
@@ -86,7 +86,7 @@ func TestRemoveWithOptions(t *testing.T) {
 
 		err = store.removeWithOptions(options)
 		if err != nil {
-			t.Errorf("RemoveWithOptions should successfully remove file; err=%w", err)
+			t.Errorf("RemoveWithOptions should successfully remove file; err=%v", err)
 		}
 
 		if isFileExist(f.Name()) {

api/datastore/datastore.go

@@ -2,7 +2,6 @@ package datastore
 
 import (
 	"fmt"
-	"github.com/portainer/portainer/api/database"
 	"io"
 	"os"
 	"path"
@@ -30,7 +29,7 @@ func (store *Store) edition() portainer.SoftwareEdition {
 }
 
 // NewStore initializes a new Store and the associated services
-func NewStore(storePath string, fileService portainer.FileService, connection database.Connection) *Store {
+func NewStore(storePath string, fileService portainer.FileService, connection portainer.Connection) *Store {
 	return &Store{
 		fileService: fileService,
 		connection:  connection,

api/datastore/datastore_test.go

@@ -2,9 +2,6 @@ package datastore
 
 import (
 	"fmt"
-	"github.com/portainer/portainer/api/database"
-	"github.com/portainer/portainer/api/dataservices/registry"
-	schedule2 "github.com/portainer/portainer/api/dataservices/schedule"
 	"runtime"
 	"strings"
 	"testing"
@@ -30,7 +27,7 @@ const (
 // TestStoreFull an eventually comprehensive set of tests for the Store.
 // The idea is what we write to the store, we should read back.
 func TestStoreFull(t *testing.T) {
-	_, store, teardown := MustNewTestStore(true)
+	_, store, teardown := MustNewTestStore(true, true)
 	defer teardown()
 
 	testCases := map[string]func(t *testing.T){
@@ -86,7 +83,7 @@ func (store *Store) testEnvironments(t *testing.T) {
 	store.CreateEndpointRelation(id)
 }
 
-func newEndpoint(endpointType portainer.EndpointType, id database.EndpointID, name, URL string, TLS bool) *portainer.Endpoint {
+func newEndpoint(endpointType portainer.EndpointType, id portainer.EndpointID, name, URL string, TLS bool) *portainer.Endpoint {
 	endpoint := &portainer.Endpoint{
 		ID:        id,
 		Name:      name,
@@ -94,11 +91,11 @@ func newEndpoint(endpointType portainer.EndpointType, id database.EndpointID, na
 		Type:      endpointType,
 		GroupID:   portainer.EndpointGroupID(1),
 		PublicURL: "",
-		TLSConfig: database.TLSConfiguration{
+		TLSConfig: portainer.TLSConfiguration{
 			TLS: false,
 		},
-		UserAccessPolicies: database.UserAccessPolicies{},
-		TeamAccessPolicies: database.TeamAccessPolicies{},
+		UserAccessPolicies: portainer.UserAccessPolicies{},
+		TeamAccessPolicies: portainer.TeamAccessPolicies{},
 		TagIDs:             []portainer.TagID{},
 		Status:             portainer.EndpointStatusUp,
 		Snapshots:          []portainer.DockerSnapshot{},
@@ -106,7 +103,7 @@ func newEndpoint(endpointType portainer.EndpointType, id database.EndpointID, na
 	}
 
 	if TLS {
-		endpoint.TLSConfig = database.TLSConfiguration{
+		endpoint.TLSConfig = portainer.TLSConfiguration{
 			TLS:           true,
 			TLSSkipVerify: true,
 		}
@@ -130,11 +127,11 @@ func setEndpointAuthorizations(endpoint *portainer.Endpoint) {
 	}
 }
 
-func (store *Store) CreateEndpoint(t *testing.T, name string, endpointType portainer.EndpointType, URL string, tls bool) database.EndpointID {
+func (store *Store) CreateEndpoint(t *testing.T, name string, endpointType portainer.EndpointType, URL string, tls bool) portainer.EndpointID {
 	is := assert.New(t)
 
 	var expectedEndpoint *portainer.Endpoint
-	id := database.EndpointID(store.Endpoint().GetNextIdentifier())
+	id := portainer.EndpointID(store.Endpoint().GetNextIdentifier())
 
 	switch endpointType {
 	case portainer.DockerEnvironment:
@@ -177,10 +174,10 @@ func (store *Store) CreateEndpoint(t *testing.T, name string, endpointType porta
 	return endpoint.ID
 }
 
-func (store *Store) CreateEndpointRelation(id database.EndpointID) {
+func (store *Store) CreateEndpointRelation(id portainer.EndpointID) {
 	relation := &portainer.EndpointRelation{
 		EndpointID: id,
-		EdgeStacks: map[portainer.EdgeStackID]bool{},
+		EdgeStacks: map[portainer.EdgeStackID]portainer.EdgeStackStatus{},
 	}
 
 	store.EndpointRelation().Create(relation)
@@ -331,17 +328,17 @@ func (store *Store) testRegistries(t *testing.T) {
 	regService := store.RegistryService
 	is.NotNil(regService, "RegistryService shouldn't be nil")
 
-	reg1 := &registry.Registry{
+	reg1 := &portainer.Registry{
 		ID:   1,
 		Type: portainer.DockerHubRegistry,
 		Name: "Dockerhub Registry Test",
 	}
 
-	reg2 := &registry.Registry{
+	reg2 := &portainer.Registry{
 		ID:   2,
 		Type: portainer.GitlabRegistry,
 		Name: "Gitlab Registry Test",
-		Gitlab: registry.GitlabRegistryData{
+		Gitlab: portainer.GitlabRegistryData{
 			ProjectID:   12345,
 			InstanceURL: "http://gitlab.com/12345",
 			ProjectPath: "mytestproject",
@@ -375,8 +372,8 @@ func (store *Store) testSchedules(t *testing.T) {
 	is := assert.New(t)
 
 	schedule := store.ScheduleService
-	s := &schedule2.Schedule{
-		ID:             schedule2.ScheduleID(schedule.GetNextIdentifier()),
+	s := &portainer.Schedule{
+		ID:             portainer.ScheduleID(schedule.GetNextIdentifier()),
 		Name:           "My Custom Schedule 1",
 		CronExpression: "*/5 * * * * portainer /bin/sh -c echo 'hello world'",
 	}

api/datastore/init.go

@@ -3,7 +3,6 @@ package datastore
 import (
 	"github.com/gofrs/uuid"
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/database"
 )
 
 // Init creates the default data set.
@@ -23,23 +22,18 @@ func (store *Store) Init() error {
 		return err
 	}
 
-	err = store.checkOrCreateDefaultData()
-	if err != nil {
-		return err
-	}
-
-	return nil
+	return store.checkOrCreateDefaultData()
 }
 
 func (store *Store) checkOrCreateInstanceID() error {
-	instanceID, err := store.VersionService.InstanceID()
+	_, err := store.VersionService.InstanceID()
 	if store.IsErrObjectNotFound(err) {
 		uid, err := uuid.NewV4()
 		if err != nil {
 			return err
 		}
 
-		instanceID = uid.String()
+		instanceID := uid.String()
 		return store.VersionService.StoreInstanceID(instanceID)
 	}
 	return err
@@ -56,7 +50,7 @@ func (store *Store) checkOrCreateDefaultSettings() error {
 			LDAPSettings: portainer.LDAPSettings{
 				AnonymousMode:   true,
 				AutoCreateUsers: true,
-				TLSConfig:       database.TLSConfiguration{},
+				TLSConfig:       portainer.TLSConfiguration{},
 				SearchSettings: []portainer.LDAPSearchSettings{
 					{},
 				},
@@ -113,8 +107,8 @@ func (store *Store) checkOrCreateDefaultData() error {
 			Name:               "Unassigned",
 			Description:        "Unassigned environments",
 			Labels:             []portainer.Pair{},
-			UserAccessPolicies: database.UserAccessPolicies{},
-			TeamAccessPolicies: database.TeamAccessPolicies{},
+			UserAccessPolicies: portainer.UserAccessPolicies{},
+			TeamAccessPolicies: portainer.TeamAccessPolicies{},
 			TagIDs:             []portainer.TagID{},
 		}
 

api/datastore/log/log.go

@@ -28,10 +28,20 @@ func (slog *ScopedLog) Debug(message string) {
 	slog.print(DEBUG, fmt.Sprintf("[message: %s]", message))
 }
 
+func (slog *ScopedLog) Debugf(message string, vars ...interface{}) {
+	message = fmt.Sprintf(message, vars...)
+	slog.print(DEBUG, fmt.Sprintf("[message: %s]", message))
+}
+
 func (slog *ScopedLog) Info(message string) {
 	slog.print(INFO, fmt.Sprintf("[message: %s]", message))
 }
 
+func (slog *ScopedLog) Infof(message string, vars ...interface{}) {
+	message = fmt.Sprintf(message, vars...)
+	slog.print(INFO, fmt.Sprintf("[message: %s]", message))
+}
+
 func (slog *ScopedLog) Error(message string, err error) {
 	slog.print(ERROR, fmt.Sprintf("[message: %s] [error: %s]", message, err))
 }

api/datastore/migrate_data.go

@@ -9,6 +9,7 @@ import (
 	plog "github.com/portainer/portainer/api/datastore/log"
 	"github.com/portainer/portainer/api/datastore/migrator"
 	"github.com/portainer/portainer/api/internal/authorization"
+	"github.com/sirupsen/logrus"
 
 	werrors "github.com/pkg/errors"
 	portainer "github.com/portainer/portainer/api"
@@ -24,6 +25,14 @@ func (store *Store) MigrateData() error {
 		return err
 	}
 
+	// Backup Database
+	backupPath, err := store.Backup()
+	if err != nil {
+		return werrors.Wrap(err, "while backing up db before migration")
+	}
+
+	store.SettingsService.InvalidateCache()
+
 	migratorParams := &migrator.MigratorParameters{
 		DatabaseVersion:         version,
 		EndpointGroupService:    store.EndpointGroupService,
@@ -46,7 +55,27 @@ func (store *Store) MigrateData() error {
 		AuthorizationService:    authorization.NewService(store),
 	}
 
-	return store.connectionMigrateData(migratorParams)
+	// restore on error
+	err = store.connectionMigrateData(migratorParams)
+	if err != nil {
+		logrus.Errorf("While DB migration %v. Restoring DB", err)
+		// Restore options
+		options := BackupOptions{
+			BackupPath: backupPath,
+		}
+		err := store.restoreWithOptions(&options)
+		if err != nil {
+			logrus.Fatalf(
+				"Failed restoring the backup. portainer database file needs to restored manually by "+
+					"replacing %s database file with recent backup %s. Error %v",
+				store.databasePath(),
+				options.BackupPath,
+				err,
+			)
+		}
+	}
+
+	return err
 }
 
 // FailSafeMigrate backup and restore DB if migration fail

api/datastore/migrate_data_test.go

@@ -1,13 +1,19 @@
 package datastore
 
 import (
+	"bytes"
+	"encoding/json"
 	"fmt"
+	"io"
 	"log"
+	"os"
 	"path/filepath"
 	"strings"
 	"testing"
 
+	"github.com/google/go-cmp/cmp"
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/database/boltdb"
 )
 
 // testVersion is a helper which tests current store version against wanted version
@@ -22,8 +28,32 @@ func testVersion(store *Store, versionWant int, t *testing.T) {
 }
 
 func TestMigrateData(t *testing.T) {
+	snapshotTests := []struct {
+		testName string
+		srcPath  string
+		wantPath string
+	}{
+		{
+			testName: "migrate version 24 to 35",
+			srcPath:  "test_data/input_24.json",
+			wantPath: "test_data/output_35.json",
+		},
+	}
+	for _, test := range snapshotTests {
+		t.Run(test.testName, func(t *testing.T) {
+			err := migrateDBTestHelper(t, test.srcPath, test.wantPath)
+			if err != nil {
+				t.Errorf(
+					"Failed migrating mock database %v: %v",
+					test.srcPath,
+					err,
+				)
+			}
+		})
+	}
+
 	t.Run("MigrateData for New Store & Re-Open Check", func(t *testing.T) {
-		newStore, store, teardown := MustNewTestStore(false)
+		newStore, store, teardown := MustNewTestStore(false, true)
 		defer teardown()
 
 		if !newStore {
@@ -50,7 +80,7 @@ func TestMigrateData(t *testing.T) {
 		{version: 21, expectedVersion: portainer.DBVersion},
 	}
 	for _, tc := range tests {
-		_, store, teardown := MustNewTestStore(true)
+		_, store, teardown := MustNewTestStore(true, true)
 		defer teardown()
 
 		// Setup data
@@ -75,7 +105,7 @@ func TestMigrateData(t *testing.T) {
 	}
 
 	t.Run("Error in MigrateData should restore backup before MigrateData", func(t *testing.T) {
-		_, store, teardown := MustNewTestStore(false)
+		_, store, teardown := MustNewTestStore(false, true)
 		defer teardown()
 
 		version := 17
@@ -87,7 +117,7 @@ func TestMigrateData(t *testing.T) {
 	})
 
 	t.Run("MigrateData should create backup file upon update", func(t *testing.T) {
-		_, store, teardown := MustNewTestStore(false)
+		_, store, teardown := MustNewTestStore(false, true)
 		defer teardown()
 		store.VersionService.StoreDBVersion(0)
 
@@ -101,7 +131,7 @@ func TestMigrateData(t *testing.T) {
 	})
 
 	t.Run("MigrateData should fail to create backup if database file is set to updating", func(t *testing.T) {
-		_, store, teardown := MustNewTestStore(false)
+		_, store, teardown := MustNewTestStore(false, true)
 		defer teardown()
 
 		store.VersionService.StoreIsUpdating(true)
@@ -116,7 +146,7 @@ func TestMigrateData(t *testing.T) {
 	})
 
 	t.Run("MigrateData should not create backup on startup if portainer version matches db", func(t *testing.T) {
-		_, store, teardown := MustNewTestStore(false)
+		_, store, teardown := MustNewTestStore(false, true)
 		defer teardown()
 
 		store.MigrateData()
@@ -131,7 +161,7 @@ func TestMigrateData(t *testing.T) {
 }
 
 func Test_getBackupRestoreOptions(t *testing.T) {
-	_, store, teardown := MustNewTestStore(false)
+	_, store, teardown := MustNewTestStore(false, true)
 	defer teardown()
 
 	options := getBackupRestoreOptions(store.commonBackupDir())
@@ -150,7 +180,7 @@ func Test_getBackupRestoreOptions(t *testing.T) {
 func TestRollback(t *testing.T) {
 	t.Run("Rollback should restore upgrade after backup", func(t *testing.T) {
 		version := 21
-		_, store, teardown := MustNewTestStore(false)
+		_, store, teardown := MustNewTestStore(false, true)
 		defer teardown()
 		store.VersionService.StoreDBVersion(version)
 
@@ -185,3 +215,250 @@ func isFileExist(path string) bool {
 	}
 	return len(matches) > 0
 }
+
+// migrateDBTestHelper loads a json representation of a bolt database from srcPath,
+// parses it into a database, runs a migration on that database, and then
+// compares it with an expected output database.
+func migrateDBTestHelper(t *testing.T, srcPath, wantPath string) error {
+	srcJSON, err := os.ReadFile(srcPath)
+	if err != nil {
+		t.Fatalf("failed loading source JSON file %v: %v", srcPath, err)
+	}
+
+	// Parse source json to db.
+	_, store, teardown := MustNewTestStore(true, false)
+	defer teardown()
+	err = importJSON(t, bytes.NewReader(srcJSON), store)
+	if err != nil {
+		return err
+	}
+
+	// Run the actual migrations on our input database.
+	err = store.MigrateData()
+	if err != nil {
+		return err
+	}
+
+	// Assert that our database connection is using bolt so we can call
+	// exportJson rather than ExportRaw. The exportJson function allows us to
+	// strip out the metadata which we don't want for our tests.
+	// TODO: update connection interface in CE to allow us to use ExportRaw and pass meta false
+	err = store.connection.Close()
+	if err != nil {
+		t.Fatalf("err closing bolt connection: %v", err)
+	}
+	con, ok := store.connection.(*boltdb.DbConnection)
+	if !ok {
+		t.Fatalf("backing database is not using boltdb, but the migrations test requires it")
+	}
+
+	// Convert database back to json.
+	databasePath := con.GetDatabaseFilePath()
+	if _, err := os.Stat(databasePath); err != nil {
+		return fmt.Errorf("stat on %s failed: %s", databasePath, err)
+	}
+
+	gotJSON, err := con.ExportJson(databasePath, false)
+	if err != nil {
+		t.Logf(
+			"failed re-exporting database %s to JSON: %v",
+			databasePath,
+			err,
+		)
+	}
+
+	wantJSON, err := os.ReadFile(wantPath)
+	if err != nil {
+		t.Fatalf("failed loading want JSON file %v: %v", wantPath, err)
+	}
+
+	// Compare the result we got with the one we wanted.
+	if diff := cmp.Diff(wantJSON, gotJSON); diff != "" {
+		gotPath := filepath.Join(os.TempDir(), "portainer-migrator-test-fail.json")
+		os.WriteFile(
+			gotPath,
+			gotJSON,
+			0600,
+		)
+		t.Errorf(
+			"migrate data from %s to %s failed\nwrote migrated input to %s\nmismatch (-want +got):\n%s",
+			srcPath,
+			wantPath,
+			gotPath,
+			diff,
+		)
+	}
+	return nil
+}
+
+// importJSON reads input JSON and commits it to a portainer datastore.Store.
+// Errors are logged with the testing package.
+func importJSON(t *testing.T, r io.Reader, store *Store) error {
+	objects := make(map[string]interface{})
+
+	// Parse json into map of objects.
+	d := json.NewDecoder(r)
+	d.UseNumber()
+	err := d.Decode(&objects)
+	if err != nil {
+		return err
+	}
+
+	// Get database connection from store.
+	con := store.connection
+
+	for k, v := range objects {
+		switch k {
+		case "version":
+			versions, ok := v.(map[string]interface{})
+			if !ok {
+				t.Logf("failed casting %s to map[string]interface{}", k)
+			}
+
+			dbVersion, ok := versions["DB_VERSION"]
+			if !ok {
+				t.Logf("failed getting DB_VERSION from %s", k)
+			}
+
+			numDBVersion, ok := dbVersion.(json.Number)
+			if !ok {
+				t.Logf("failed parsing DB_VERSION as json number from %s", k)
+			}
+
+			intDBVersion, err := numDBVersion.Int64()
+			if err != nil {
+				t.Logf("failed casting %v to int: %v", numDBVersion, intDBVersion)
+			}
+
+			err = con.CreateObjectWithStringId(
+				k,
+				[]byte("DB_VERSION"),
+				int(intDBVersion),
+			)
+			if err != nil {
+				t.Logf("failed writing DB_VERSION in %s: %v", k, err)
+			}
+
+			instanceID, ok := versions["INSTANCE_ID"]
+			if !ok {
+				t.Logf("failed getting INSTANCE_ID from %s", k)
+			}
+
+			err = con.CreateObjectWithStringId(
+				k,
+				[]byte("INSTANCE_ID"),
+				instanceID,
+			)
+			if err != nil {
+				t.Logf("failed writing INSTANCE_ID in %s: %v", k, err)
+			}
+
+		case "dockerhub":
+			obj, ok := v.([]interface{})
+			if !ok {
+				t.Logf("failed to cast %s to []interface{}", k)
+			}
+			err := con.CreateObjectWithStringId(
+				k,
+				[]byte("DOCKERHUB"),
+				obj[0],
+			)
+			if err != nil {
+				t.Logf("failed writing DOCKERHUB in %s: %v", k, err)
+			}
+
+		case "ssl":
+			obj, ok := v.(map[string]interface{})
+			if !ok {
+				t.Logf("failed to case %s to map[string]interface{}", k)
+			}
+			err := con.CreateObjectWithStringId(
+				k,
+				[]byte("SSL"),
+				obj,
+			)
+			if err != nil {
+				t.Logf("failed writing SSL in %s: %v", k, err)
+			}
+
+		case "settings":
+			obj, ok := v.(map[string]interface{})
+			if !ok {
+				t.Logf("failed to case %s to map[string]interface{}", k)
+			}
+			err := con.CreateObjectWithStringId(
+				k,
+				[]byte("SETTINGS"),
+				obj,
+			)
+			if err != nil {
+				t.Logf("failed writing SETTINGS in %s: %v", k, err)
+			}
+
+		case "tunnel_server":
+			obj, ok := v.(map[string]interface{})
+			if !ok {
+				t.Logf("failed to case %s to map[string]interface{}", k)
+			}
+			err := con.CreateObjectWithStringId(
+				k,
+				[]byte("INFO"),
+				obj,
+			)
+			if err != nil {
+				t.Logf("failed writing INFO in %s: %v", k, err)
+			}
+		case "templates":
+			continue
+
+		default:
+			objlist, ok := v.([]interface{})
+			if !ok {
+				t.Logf("failed to cast %s to []interface{}", k)
+			}
+
+			for _, obj := range objlist {
+				value, ok := obj.(map[string]interface{})
+				if !ok {
+					t.Logf("failed to cast %v to map[string]interface{}", obj)
+				} else {
+					var ok bool
+					var id interface{}
+					switch k {
+					case "endpoint_relations":
+						// TODO: need to make into an int, then do that weird
+						// stringification
+						id, ok = value["EndpointID"]
+					default:
+						id, ok = value["Id"]
+					}
+					if !ok {
+						// endpoint_relations: EndpointID
+						t.Logf("missing Id field: %s", k)
+						id = "error"
+					}
+					n, ok := id.(json.Number)
+					if !ok {
+						t.Logf("failed to cast %v to json.Number in %s", id, k)
+					} else {
+						key, err := n.Int64()
+						if err != nil {
+							t.Logf("failed to cast %v to int in %s", n, k)
+						} else {
+							err := con.CreateObjectWithId(
+								k,
+								int(key),
+								value,
+							)
+							if err != nil {
+								t.Logf("failed writing %v in %s: %v", key, k, err)
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+
+	return nil
+}

api/datastore/migrate_dbversion29_test.go

@@ -1,9 +1,9 @@
 package datastore
 
 import (
-	portainer "github.com/portainer/portainer/api/database"
 	"testing"
 
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/datastore/migrator"
 	"github.com/portainer/portainer/api/internal/authorization"
 )
@@ -33,7 +33,7 @@ func setup(store *Store) error {
 }
 
 func TestMigrateSettings(t *testing.T) {
-	_, store, teardown := MustNewTestStore(false)
+	_, store, teardown := MustNewTestStore(false, true)
 	defer teardown()
 
 	err := setup(store)

api/datastore/migrate_dbversion33_test.go

@@ -10,7 +10,7 @@ import (
 )
 
 func TestMigrateStackEntryPoint(t *testing.T) {
-	_, store, teardown := MustNewTestStore(false)
+	_, store, teardown := MustNewTestStore(false, true)
 	defer teardown()
 
 	stackService := store.Stack()

api/datastore/migrator/migrate_ce.go

@@ -1,16 +1,38 @@
 package migrator
 
 import (
-	"fmt"
+	"errors"
+	"reflect"
+	"runtime"
 
 	werrors "github.com/pkg/errors"
 	portainer "github.com/portainer/portainer/api"
 )
 
+type migration struct {
+	dbversion int
+	migrate   func() error
+}
+
 func migrationError(err error, context string) error {
 	return werrors.Wrap(err, "failed in "+context)
 }
 
+func newMigration(dbversion int, migrate func() error) migration {
+	return migration{
+		dbversion: dbversion,
+		migrate:   migrate,
+	}
+}
+
+func dbTooOldError() error {
+	return errors.New("migrating from less than Portainer 1.21.0 is not supported, please contact Portainer support.")
+}
+
+func GetFunctionName(i interface{}) string {
+	return runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name()
+}
+
 // Migrate checks the database version and migrate the existing data to the most recent data model.
 func (m *Migrator) Migrate() error {
 	// set DB to updating status
@@ -19,175 +41,90 @@ func (m *Migrator) Migrate() error {
 		return migrationError(err, "StoreIsUpdating")
 	}
 
-	if m.currentDBVersion < 17 {
-		return migrationError(err, "migrating from less than Portainer 1.21.0 is not supported, please contact Portainer support.")
+	migrations := []migration{
+		// Portainer < 1.21.0
+		newMigration(17, dbTooOldError),
+
+		// Portainer 1.21.0
+		newMigration(18, m.updateUsersToDBVersion18),
+		newMigration(18, m.updateEndpointsToDBVersion18),
+		newMigration(18, m.updateEndpointGroupsToDBVersion18),
+		newMigration(18, m.updateRegistriesToDBVersion18),
+
+		// 1.22.0
+		newMigration(19, m.updateSettingsToDBVersion19),
+
+		// 1.22.1
+		newMigration(20, m.updateUsersToDBVersion20),
+		newMigration(20, m.updateSettingsToDBVersion20),
+		newMigration(20, m.updateSchedulesToDBVersion20),
+
+		// Portainer 1.23.0
+		// DBVersion 21 is missing as it was shipped as via hotfix 1.22.2
+		newMigration(22, m.updateResourceControlsToDBVersion22),
+		newMigration(22, m.updateUsersAndRolesToDBVersion22),
+
+		// Portainer 1.24.0
+		newMigration(23, m.updateTagsToDBVersion23),
+		newMigration(23, m.updateEndpointsAndEndpointGroupsToDBVersion23),
+
+		// Portainer 1.24.1
+		newMigration(24, m.updateSettingsToDB24),
+
+		// Portainer 2.0.0
+		newMigration(25, m.updateSettingsToDB25),
+		newMigration(25, m.updateStacksToDB24), // yes this looks odd. Don't be tempted to move it
+
+		// Portainer 2.1.0
+		newMigration(26, m.updateEndpointSettingsToDB25),
+
+		// Portainer 2.2.0
+		newMigration(27, m.updateStackResourceControlToDB27),
+
+		// Portainer 2.6.0
+		newMigration(30, m.migrateDBVersionToDB30),
+
+		// Portainer 2.9.0
+		newMigration(32, m.migrateDBVersionToDB32),
+
+		// Portainer 2.9.1, 2.9.2
+		newMigration(33, m.migrateDBVersionToDB33),
+
+		// Portainer 2.10
+		newMigration(34, m.migrateDBVersionToDB34),
+
+		// Portainer 2.9.3 (yep out of order, but 2.10 is EE only)
+		newMigration(35, m.migrateDBVersionToDB35),
+
+		newMigration(36, m.migrateDBVersionToDB36),
+
+		// Portainer 2.13
+		newMigration(40, m.migrateDBVersionToDB40),
 	}
 
-	// Portainer 1.21.0
-	if m.currentDBVersion < 18 {
-		err := m.updateUsersToDBVersion18()
-		if err != nil {
-			return migrationError(err, "updateUsersToDBVersion18")
-		}
+	var lastDbVersion int
+	for _, migration := range migrations {
+		if m.currentDBVersion < migration.dbversion {
 
-		err = m.updateEndpointsToDBVersion18()
-		if err != nil {
-			return migrationError(err, "updateEndpointsToDBVersion18")
-		}
+			// Print the next line only when the version changes
+			if migration.dbversion > lastDbVersion {
+				migrateLog.Infof("Migrating DB to version %d", migration.dbversion)
+			}
 
-		err = m.updateEndpointGroupsToDBVersion18()
-		if err != nil {
-			return migrationError(err, "updateEndpointGroupsToDBVersion18")
-		}
-
-		err = m.updateRegistriesToDBVersion18()
-		if err != nil {
-			return migrationError(err, "updateRegistriesToDBVersion18")
-		}
-	}
-
-	// Portainer 1.22.0
-	if m.currentDBVersion < 19 {
-		err := m.updateSettingsToDBVersion19()
-		if err != nil {
-			return migrationError(err, "updateSettingsToDBVersion19")
-		}
-	}
-
-	// Portainer 1.22.1
-	if m.currentDBVersion < 20 {
-		err := m.updateUsersToDBVersion20()
-		if err != nil {
-			return migrationError(err, "updateUsersToDBVersion20")
-		}
-
-		err = m.updateSettingsToDBVersion20()
-		if err != nil {
-			return migrationError(err, "updateSettingsToDBVersion20")
-		}
-
-		err = m.updateSchedulesToDBVersion20()
-		if err != nil {
-			return migrationError(err, "updateSchedulesToDBVersion20")
-		}
-	}
-
-	// Portainer 1.23.0
-	// DBVersion 21 is missing as it was shipped as via hotfix 1.22.2
-	if m.currentDBVersion < 22 {
-		err := m.updateResourceControlsToDBVersion22()
-		if err != nil {
-			return migrationError(err, "updateResourceControlsToDBVersion22")
-		}
-
-		err = m.updateUsersAndRolesToDBVersion22()
-		if err != nil {
-			return migrationError(err, "updateUsersAndRolesToDBVersion22")
-		}
-	}
-
-	// Portainer 1.24.0
-	if m.currentDBVersion < 23 {
-		migrateLog.Info("Migrating to DB 23")
-		err := m.updateTagsToDBVersion23()
-		if err != nil {
-			return migrationError(err, "updateTagsToDBVersion23")
-		}
-
-		err = m.updateEndpointsAndEndpointGroupsToDBVersion23()
-		if err != nil {
-			return migrationError(err, "updateEndpointsAndEndpointGroupsToDBVersion23")
-		}
-	}
-
-	// Portainer 1.24.1
-	if m.currentDBVersion < 24 {
-		migrateLog.Info("Migrating to DB 24")
-		err := m.updateSettingsToDB24()
-		if err != nil {
-			return migrationError(err, "updateSettingsToDB24")
-		}
-	}
-
-	// Portainer 2.0.0
-	if m.currentDBVersion < 25 {
-		migrateLog.Info("Migrating to DB 25")
-		err := m.updateSettingsToDB25()
-		if err != nil {
-			return migrationError(err, "updateSettingsToDB25")
-		}
-
-		err = m.updateStacksToDB24()
-		if err != nil {
-			return migrationError(err, "updateStacksToDB24")
-		}
-	}
-
-	// Portainer 2.1.0
-	if m.currentDBVersion < 26 {
-		migrateLog.Info("Migrating to DB 26")
-		err := m.updateEndpointSettingsToDB25()
-		if err != nil {
-			return migrationError(err, "updateEndpointSettingsToDB25")
-		}
-	}
-
-	// Portainer 2.2.0
-	if m.currentDBVersion < 27 {
-		migrateLog.Info("Migrating to DB 27")
-		err := m.updateStackResourceControlToDB27()
-		if err != nil {
-			return migrationError(err, "updateStackResourceControlToDB27")
-		}
-	}
-
-	// Portainer 2.6.0
-	if m.currentDBVersion < 30 {
-		migrateLog.Info("Migrating to DB 30")
-		err := m.migrateDBVersionToDB30()
-		if err != nil {
-			return migrationError(err, "migrateDBVersionToDB30")
-		}
-	}
-
-	// Portainer 2.9.0
-	if m.currentDBVersion < 32 {
-		err := m.migrateDBVersionToDB32()
-		if err != nil {
-			return migrationError(err, "migrateDBVersionToDB32")
-		}
-	}
-
-	// Portainer 2.9.1, 2.9.2
-	if m.currentDBVersion < 33 {
-		migrateLog.Info("Migrating to DB 33")
-		err := m.migrateDBVersionToDB33()
-		if err != nil {
-			return migrationError(err, "migrateDBVersionToDB33")
-		}
-	}
-
-	// Portainer 2.10
-	if m.currentDBVersion < 34 {
-		migrateLog.Info("Migrating to DB 34")
-		if err := m.migrateDBVersionToDB34(); err != nil {
-			return migrationError(err, "migrateDBVersionToDB34")
-		}
-	}
-
-	// Portainer 2.9.3 (yep out of order, but 2.10 is EE only)
-	if m.currentDBVersion < 35 {
-		migrateLog.Info("Migrating to DB 35")
-		if err := m.migrateDBVersionToDB35(); err != nil {
-			return migrationError(err, "migrateDBVersionToDB35")
+			err := migration.migrate()
+			if err != nil {
+				return migrationError(err, GetFunctionName(migration.migrate))
+			}
 		}
+		lastDbVersion = migration.dbversion
 	}
 
+	migrateLog.Infof("Setting DB version to %d", portainer.DBVersion)
 	err = m.versionService.StoreDBVersion(portainer.DBVersion)
 	if err != nil {
 		return migrationError(err, "StoreDBVersion")
 	}
-	migrateLog.Info(fmt.Sprintf("Updated DB version to %d", portainer.DBVersion))
+	migrateLog.Infof("Updated DB version to %d", portainer.DBVersion)
 
 	// reset DB updating status
 	return m.versionService.StoreIsUpdating(false)

api/datastore/migrator/migrate_dbversion17.go

@@ -2,10 +2,10 @@ package migrator
 
 import (
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/database"
 )
 
 func (m *Migrator) updateUsersToDBVersion18() error {
+	migrateLog.Info("- updating users")
 	legacyUsers, err := m.userService.Users()
 	if err != nil {
 		return err
@@ -40,22 +40,23 @@ func (m *Migrator) updateUsersToDBVersion18() error {
 }
 
 func (m *Migrator) updateEndpointsToDBVersion18() error {
+	migrateLog.Info("- updating endpoints")
 	legacyEndpoints, err := m.endpointService.Endpoints()
 	if err != nil {
 		return err
 	}
 
 	for _, endpoint := range legacyEndpoints {
-		endpoint.UserAccessPolicies = make(database.UserAccessPolicies)
+		endpoint.UserAccessPolicies = make(portainer.UserAccessPolicies)
 		for _, userID := range endpoint.AuthorizedUsers {
-			endpoint.UserAccessPolicies[userID] = database.AccessPolicy{
+			endpoint.UserAccessPolicies[userID] = portainer.AccessPolicy{
 				RoleID: 4,
 			}
 		}
 
-		endpoint.TeamAccessPolicies = make(database.TeamAccessPolicies)
+		endpoint.TeamAccessPolicies = make(portainer.TeamAccessPolicies)
 		for _, teamID := range endpoint.AuthorizedTeams {
-			endpoint.TeamAccessPolicies[teamID] = database.AccessPolicy{
+			endpoint.TeamAccessPolicies[teamID] = portainer.AccessPolicy{
 				RoleID: 4,
 			}
 		}
@@ -70,22 +71,23 @@ func (m *Migrator) updateEndpointsToDBVersion18() error {
 }
 
 func (m *Migrator) updateEndpointGroupsToDBVersion18() error {
+	migrateLog.Info("- updating endpoint groups")
 	legacyEndpointGroups, err := m.endpointGroupService.EndpointGroups()
 	if err != nil {
 		return err
 	}
 
 	for _, endpointGroup := range legacyEndpointGroups {
-		endpointGroup.UserAccessPolicies = make(database.UserAccessPolicies)
+		endpointGroup.UserAccessPolicies = make(portainer.UserAccessPolicies)
 		for _, userID := range endpointGroup.AuthorizedUsers {
-			endpointGroup.UserAccessPolicies[userID] = database.AccessPolicy{
+			endpointGroup.UserAccessPolicies[userID] = portainer.AccessPolicy{
 				RoleID: 4,
 			}
 		}
 
-		endpointGroup.TeamAccessPolicies = make(database.TeamAccessPolicies)
+		endpointGroup.TeamAccessPolicies = make(portainer.TeamAccessPolicies)
 		for _, teamID := range endpointGroup.AuthorizedTeams {
-			endpointGroup.TeamAccessPolicies[teamID] = database.AccessPolicy{
+			endpointGroup.TeamAccessPolicies[teamID] = portainer.AccessPolicy{
 				RoleID: 4,
 			}
 		}
@@ -100,20 +102,21 @@ func (m *Migrator) updateEndpointGroupsToDBVersion18() error {
 }
 
 func (m *Migrator) updateRegistriesToDBVersion18() error {
+	migrateLog.Info("- updating registries")
 	legacyRegistries, err := m.registryService.Registries()
 	if err != nil {
 		return err
 	}
 
 	for _, registry := range legacyRegistries {
-		registry.UserAccessPolicies = make(database.UserAccessPolicies)
+		registry.UserAccessPolicies = make(portainer.UserAccessPolicies)
 		for _, userID := range registry.AuthorizedUsers {
-			registry.UserAccessPolicies[userID] = database.AccessPolicy{}
+			registry.UserAccessPolicies[userID] = portainer.AccessPolicy{}
 		}
 
-		registry.TeamAccessPolicies = make(database.TeamAccessPolicies)
+		registry.TeamAccessPolicies = make(portainer.TeamAccessPolicies)
 		for _, teamID := range registry.AuthorizedTeams {
-			registry.TeamAccessPolicies[teamID] = database.AccessPolicy{}
+			registry.TeamAccessPolicies[teamID] = portainer.AccessPolicy{}
 		}
 
 		err = m.registryService.UpdateRegistry(registry.ID, &registry)

api/datastore/migrator/migrate_dbversion18.go

@@ -3,6 +3,7 @@ package migrator
 import portainer "github.com/portainer/portainer/api"
 
 func (m *Migrator) updateSettingsToDBVersion19() error {
+	migrateLog.Info("- updating settings")
 	legacySettings, err := m.settingsService.Settings()
 	if err != nil {
 		return err

api/datastore/migrator/migrate_dbversion19.go

@@ -7,6 +7,7 @@ import (
 const scheduleScriptExecutionJobType = 1
 
 func (m *Migrator) updateUsersToDBVersion20() error {
+	migrateLog.Info("- updating user authentication")
 	return m.authorizationService.UpdateUsersAuthorizations()
 }
 
@@ -22,6 +23,7 @@ func (m *Migrator) updateSettingsToDBVersion20() error {
 }
 
 func (m *Migrator) updateSchedulesToDBVersion20() error {
+	migrateLog.Info("- updating schedules")
 	legacySchedules, err := m.scheduleService.Schedules()
 	if err != nil {
 		return err

api/datastore/migrator/migrate_dbversion20.go

@@ -1,11 +1,12 @@
 package migrator
 
 import (
-	"github.com/portainer/portainer/api/database"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/internal/authorization"
 )
 
 func (m *Migrator) updateResourceControlsToDBVersion22() error {
+	migrateLog.Info("- updating resource controls")
 	legacyResourceControls, err := m.resourceControlService.ResourceControls()
 	if err != nil {
 		return err
@@ -24,6 +25,7 @@ func (m *Migrator) updateResourceControlsToDBVersion22() error {
 }
 
 func (m *Migrator) updateUsersAndRolesToDBVersion22() error {
+	migrateLog.Info("- updating users and roles")
 	legacyUsers, err := m.userService.Users()
 	if err != nil {
 		return err
@@ -42,7 +44,7 @@ func (m *Migrator) updateUsersAndRolesToDBVersion22() error {
 		}
 	}
 
-	endpointAdministratorRole, err := m.roleService.Role(database.RoleID(1))
+	endpointAdministratorRole, err := m.roleService.Role(portainer.RoleID(1))
 	if err != nil {
 		return err
 	}
@@ -51,7 +53,7 @@ func (m *Migrator) updateUsersAndRolesToDBVersion22() error {
 
 	err = m.roleService.UpdateRole(endpointAdministratorRole.ID, endpointAdministratorRole)
 
-	helpDeskRole, err := m.roleService.Role(database.RoleID(2))
+	helpDeskRole, err := m.roleService.Role(portainer.RoleID(2))
 	if err != nil {
 		return err
 	}
@@ -60,7 +62,7 @@ func (m *Migrator) updateUsersAndRolesToDBVersion22() error {
 
 	err = m.roleService.UpdateRole(helpDeskRole.ID, helpDeskRole)
 
-	standardUserRole, err := m.roleService.Role(database.RoleID(3))
+	standardUserRole, err := m.roleService.Role(portainer.RoleID(3))
 	if err != nil {
 		return err
 	}
@@ -69,7 +71,7 @@ func (m *Migrator) updateUsersAndRolesToDBVersion22() error {
 
 	err = m.roleService.UpdateRole(standardUserRole.ID, standardUserRole)
 
-	readOnlyUserRole, err := m.roleService.Role(database.RoleID(4))
+	readOnlyUserRole, err := m.roleService.Role(portainer.RoleID(4))
 	if err != nil {
 		return err
 	}

api/datastore/migrator/migrate_dbversion22.go

@@ -1,12 +1,9 @@
 package migrator
 
-import (
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/database"
-)
+import portainer "github.com/portainer/portainer/api"
 
 func (m *Migrator) updateTagsToDBVersion23() error {
-	migrateLog.Info("Updating tags")
+	migrateLog.Info("- Updating tags")
 	tags, err := m.tagService.Tags()
 	if err != nil {
 		return err
@@ -14,7 +11,7 @@ func (m *Migrator) updateTagsToDBVersion23() error {
 
 	for _, tag := range tags {
 		tag.EndpointGroups = make(map[portainer.EndpointGroupID]bool)
-		tag.Endpoints = make(map[database.EndpointID]bool)
+		tag.Endpoints = make(map[portainer.EndpointID]bool)
 		err = m.tagService.UpdateTag(tag.ID, &tag)
 		if err != nil {
 			return err
@@ -24,7 +21,7 @@ func (m *Migrator) updateTagsToDBVersion23() error {
 }
 
 func (m *Migrator) updateEndpointsAndEndpointGroupsToDBVersion23() error {
-	migrateLog.Info("Updating endpoints and endpoint groups")
+	migrateLog.Info("- updating endpoints and endpoint groups")
 	tags, err := m.tagService.Tags()
 	if err != nil {
 		return err
@@ -57,7 +54,7 @@ func (m *Migrator) updateEndpointsAndEndpointGroupsToDBVersion23() error {
 
 		relation := &portainer.EndpointRelation{
 			EndpointID: endpoint.ID,
-			EdgeStacks: map[portainer.EdgeStackID]bool{},
+			EdgeStacks: map[portainer.EdgeStackID]portainer.EdgeStackStatus{},
 		}
 
 		err = m.endpointRelationService.Create(relation)

api/datastore/migrator/migrate_dbversion23.go

@@ -3,7 +3,7 @@ package migrator
 import portainer "github.com/portainer/portainer/api"
 
 func (m *Migrator) updateSettingsToDB24() error {
-	migrateLog.Info("Updating Settings")
+	migrateLog.Info("- updating Settings")
 
 	legacySettings, err := m.settingsService.Settings()
 	if err != nil {
@@ -18,7 +18,7 @@ func (m *Migrator) updateSettingsToDB24() error {
 }
 
 func (m *Migrator) updateStacksToDB24() error {
-	migrateLog.Info("Updating stacks")
+	migrateLog.Info("- updating stacks")
 	stacks, err := m.stackService.Stacks()
 	if err != nil {
 		return err

api/datastore/migrator/migrate_dbversion24.go

@@ -5,7 +5,7 @@ import (
 )
 
 func (m *Migrator) updateSettingsToDB25() error {
-	migrateLog.Info("Updating settings")
+	migrateLog.Info("- updating settings")
 
 	legacySettings, err := m.settingsService.Settings()
 	if err != nil {

api/datastore/migrator/migrate_dbversion25.go

@@ -5,7 +5,7 @@ import (
 )
 
 func (m *Migrator) updateEndpointSettingsToDB25() error {
-	migrateLog.Info("Updating endpoint settings")
+	migrateLog.Info("- updating endpoint settings")
 	settings, err := m.settingsService.Settings()
 	if err != nil {
 		return err

api/datastore/migrator/migrate_dbversion26.go

@@ -7,7 +7,7 @@ import (
 )
 
 func (m *Migrator) updateStackResourceControlToDB27() error {
-	migrateLog.Info("Updating stack resource controls")
+	migrateLog.Info("- updating stack resource controls")
 	resourceControls, err := m.resourceControlService.ResourceControls()
 	if err != nil {
 		return err

api/datastore/migrator/migrate_dbversion29.go

@@ -1,7 +1,7 @@
 package migrator
 
 func (m *Migrator) migrateDBVersionToDB30() error {
-	migrateLog.Info("Updating legacy settings")
+	migrateLog.Info("- updating legacy settings")
 	if err := m.MigrateSettingsToDB30(); err != nil {
 		return err
 	}

api/datastore/migrator/migrate_dbversion31.go

@@ -2,40 +2,34 @@ package migrator
 
 import (
 	"fmt"
-	"github.com/portainer/portainer/api/database"
-	"github.com/portainer/portainer/api/dataservices/errors"
-	registry2 "github.com/portainer/portainer/api/dataservices/registry"
-	snapshotutils "github.com/portainer/portainer/api/orchestrators/snapshot"
 	"log"
 
+	"github.com/portainer/portainer/api/dataservices/errors"
+
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/internal/endpointutils"
+	snapshotutils "github.com/portainer/portainer/api/internal/snapshot"
 )
 
 func (m *Migrator) migrateDBVersionToDB32() error {
-	migrateLog.Info("Updating registries")
 	err := m.updateRegistriesToDB32()
 	if err != nil {
 		return err
 	}
 
-	migrateLog.Info("Updating dockerhub")
 	err = m.updateDockerhubToDB32()
 	if err != nil {
 		return err
 	}
 
-	migrateLog.Info("Updating resource controls")
 	if err := m.updateVolumeResourceControlToDB32(); err != nil {
 		return err
 	}
 
-	migrateLog.Info("Updating kubeconfig expiry")
 	if err := m.kubeconfigExpiryToDB32(); err != nil {
 		return err
 	}
 
-	migrateLog.Info("Setting default helm repository url")
 	if err := m.helmRepositoryURLToDB32(); err != nil {
 		return err
 	}
@@ -44,6 +38,7 @@ func (m *Migrator) migrateDBVersionToDB32() error {
 }
 
 func (m *Migrator) updateRegistriesToDB32() error {
+	migrateLog.Info("- updating registries")
 	registries, err := m.registryService.Registries()
 	if err != nil {
 		return err
@@ -56,25 +51,25 @@ func (m *Migrator) updateRegistriesToDB32() error {
 
 	for _, registry := range registries {
 
-		registry.RegistryAccesses = registry2.RegistryAccesses{}
+		registry.RegistryAccesses = portainer.RegistryAccesses{}
 
 		for _, endpoint := range endpoints {
 
-			filteredUserAccessPolicies := database.UserAccessPolicies{}
+			filteredUserAccessPolicies := portainer.UserAccessPolicies{}
 			for userId, registryPolicy := range registry.UserAccessPolicies {
 				if _, found := endpoint.UserAccessPolicies[userId]; found {
 					filteredUserAccessPolicies[userId] = registryPolicy
 				}
 			}
 
-			filteredTeamAccessPolicies := database.TeamAccessPolicies{}
+			filteredTeamAccessPolicies := portainer.TeamAccessPolicies{}
 			for teamId, registryPolicy := range registry.TeamAccessPolicies {
 				if _, found := endpoint.TeamAccessPolicies[teamId]; found {
 					filteredTeamAccessPolicies[teamId] = registryPolicy
 				}
 			}
 
-			registry.RegistryAccesses[endpoint.ID] = registry2.RegistryAccessPolicies{
+			registry.RegistryAccesses[endpoint.ID] = portainer.RegistryAccessPolicies{
 				UserAccessPolicies: filteredUserAccessPolicies,
 				TeamAccessPolicies: filteredTeamAccessPolicies,
 				Namespaces:         []string{},
@@ -86,6 +81,7 @@ func (m *Migrator) updateRegistriesToDB32() error {
 }
 
 func (m *Migrator) updateDockerhubToDB32() error {
+	migrateLog.Info("- updating dockerhub")
 	dockerhub, err := m.dockerhubService.DockerHub()
 	if err == errors.ErrObjectNotFound {
 		return nil
@@ -97,14 +93,14 @@ func (m *Migrator) updateDockerhubToDB32() error {
 		return nil
 	}
 
-	registry := &registry2.Registry{
+	registry := &portainer.Registry{
 		Type:             portainer.DockerHubRegistry,
 		Name:             "Dockerhub (authenticated - migrated)",
 		URL:              "docker.io",
 		Authentication:   true,
 		Username:         dockerhub.Username,
 		Password:         dockerhub.Password,
-		RegistryAccesses: registry2.RegistryAccesses{},
+		RegistryAccesses: portainer.RegistryAccesses{},
 	}
 
 	// The following code will make this function idempotent.
@@ -124,7 +120,7 @@ func (m *Migrator) updateDockerhubToDB32() error {
 				migrated = true
 			} else {
 				// delete subsequent duplicates
-				m.registryService.DeleteRegistry(registry2.RegistryID(r.ID))
+				m.registryService.DeleteRegistry(portainer.RegistryID(r.ID))
 			}
 		}
 	}
@@ -144,25 +140,25 @@ func (m *Migrator) updateDockerhubToDB32() error {
 			endpoint.Type != portainer.AgentOnKubernetesEnvironment &&
 			endpoint.Type != portainer.EdgeAgentOnKubernetesEnvironment {
 
-			userAccessPolicies := database.UserAccessPolicies{}
+			userAccessPolicies := portainer.UserAccessPolicies{}
 			for userId := range endpoint.UserAccessPolicies {
 				if _, found := endpoint.UserAccessPolicies[userId]; found {
-					userAccessPolicies[userId] = database.AccessPolicy{
+					userAccessPolicies[userId] = portainer.AccessPolicy{
 						RoleID: 0,
 					}
 				}
 			}
 
-			teamAccessPolicies := database.TeamAccessPolicies{}
+			teamAccessPolicies := portainer.TeamAccessPolicies{}
 			for teamId := range endpoint.TeamAccessPolicies {
 				if _, found := endpoint.TeamAccessPolicies[teamId]; found {
-					teamAccessPolicies[teamId] = database.AccessPolicy{
+					teamAccessPolicies[teamId] = portainer.AccessPolicy{
 						RoleID: 0,
 					}
 				}
 			}
 
-			registry.RegistryAccesses[endpoint.ID] = registry2.RegistryAccessPolicies{
+			registry.RegistryAccesses[endpoint.ID] = portainer.RegistryAccessPolicies{
 				UserAccessPolicies: userAccessPolicies,
 				TeamAccessPolicies: teamAccessPolicies,
 				Namespaces:         []string{},
@@ -174,6 +170,7 @@ func (m *Migrator) updateDockerhubToDB32() error {
 }
 
 func (m *Migrator) updateVolumeResourceControlToDB32() error {
+	migrateLog.Info("- updating resource controls")
 	endpoints, err := m.endpointService.Endpoints()
 	if err != nil {
 		return fmt.Errorf("failed fetching environments: %w", err)
@@ -266,6 +263,7 @@ func findResourcesToUpdateForDB32(dockerID string, volumesData map[string]interf
 }
 
 func (m *Migrator) kubeconfigExpiryToDB32() error {
+	migrateLog.Info("- updating kubeconfig expiry")
 	settings, err := m.settingsService.Settings()
 	if err != nil {
 		return err
@@ -275,6 +273,7 @@ func (m *Migrator) kubeconfigExpiryToDB32() error {
 }
 
 func (m *Migrator) helmRepositoryURLToDB32() error {
+	migrateLog.Info("- setting default helm repository URL")
 	settings, err := m.settingsService.Settings()
 	if err != nil {
 		return err

api/datastore/migrator/migrate_dbversion32.go

@@ -1,8 +1,11 @@
 package migrator
 
-import portainer "github.com/portainer/portainer/api"
+import (
+	portainer "github.com/portainer/portainer/api"
+)
 
 func (m *Migrator) migrateDBVersionToDB33() error {
+	migrateLog.Info("- updating settings")
 	if err := m.migrateSettingsToDB33(); err != nil {
 		return err
 	}
@@ -16,7 +19,7 @@ func (m *Migrator) migrateSettingsToDB33() error {
 		return err
 	}
 
-	migrateLog.Info("Setting default kubectl shell image")
+	migrateLog.Info("- setting default kubectl shell image")
 	settings.KubectlShellImage = portainer.DefaultKubectlShellImage
 	return m.settingsService.UpdateSettings(settings)
 }

api/datastore/migrator/migrate_dbversion33.go

@@ -1,9 +1,11 @@
 package migrator
 
-import "github.com/portainer/portainer/api/dataservices"
+import (
+	"github.com/portainer/portainer/api/dataservices"
+)
 
 func (m *Migrator) migrateDBVersionToDB34() error {
-	migrateLog.Info("Migrating stacks")
+	migrateLog.Info("- updating stacks")
 	err := MigrateStackEntryPoint(m.stackService)
 	if err != nil {
 		return err

api/datastore/migrator/migrate_dbversion34.go

@@ -3,7 +3,7 @@ package migrator
 func (m *Migrator) migrateDBVersionToDB35() error {
 	// These should have been migrated already, but due to an earlier bug and a bunch of duplicates,
 	// calling it again will now fix the issue as the function has been repaired.
-	migrateLog.Info("Updating dockerhub registries")
+	migrateLog.Info("- updating dockerhub registries")
 	err := m.updateDockerhubToDB32()
 	if err != nil {
 		return err

api/datastore/migrator/migrate_dbversion35.go

@@ -0,0 +1,36 @@
+package migrator
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/internal/authorization"
+)
+
+func (m *Migrator) migrateDBVersionToDB36() error {
+	migrateLog.Info("Updating user authorizations")
+	if err := m.migrateUsersToDB36(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (m *Migrator) migrateUsersToDB36() error {
+	users, err := m.userService.Users()
+	if err != nil {
+		return err
+	}
+
+	for _, user := range users {
+		currentAuthorizations := authorization.DefaultPortainerAuthorizations()
+		currentAuthorizations[portainer.OperationPortainerUserListToken] = true
+		currentAuthorizations[portainer.OperationPortainerUserCreateToken] = true
+		currentAuthorizations[portainer.OperationPortainerUserRevokeToken] = true
+		user.PortainerAuthorizations = currentAuthorizations
+		err = m.userService.UpdateUser(user.ID, &user)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

api/datastore/migrator/migrate_dbversion40.go

@@ -0,0 +1,31 @@
+package migrator
+
+import "github.com/portainer/portainer/api/internal/endpointutils"
+
+func (m *Migrator) migrateDBVersionToDB40() error {
+	if err := m.trustCurrentEdgeEndpointsDB40(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (m *Migrator) trustCurrentEdgeEndpointsDB40() error {
+	migrateLog.Info("- trusting current edge endpoints")
+	endpoints, err := m.endpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, endpoint := range endpoints {
+		if endpointutils.IsEdgeEndpoint(&endpoint) {
+			endpoint.UserTrusted = true
+			err = m.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}

api/datastore/services.go

@@ -2,7 +2,6 @@ package datastore
 
 import (
 	"encoding/json"
-	"github.com/portainer/portainer/api/database"
 	"io/ioutil"
 	"strconv"
 
@@ -40,7 +39,7 @@ import (
 // Store defines the implementation of portainer.DataStore using
 // BoltDB as the storage system.
 type Store struct {
-	connection database.Connection
+	connection portainer.Connection
 
 	fileService               portainer.FileService
 	CustomTemplateService     *customtemplate.Service
@@ -349,17 +348,17 @@ func (store *Store) Webhook() dataservices.WebhookService {
 type storeExport struct {
 	CustomTemplate     []portainer.CustomTemplate     `json:"customtemplates,omitempty"`
 	EdgeGroup          []portainer.EdgeGroup          `json:"edgegroups,omitempty"`
-	EdgeJob            []edgejob.EdgeJob              `json:"edgejobs,omitempty"`
+	EdgeJob            []portainer.EdgeJob            `json:"edgejobs,omitempty"`
 	EdgeStack          []portainer.EdgeStack          `json:"edge_stack,omitempty"`
 	Endpoint           []portainer.Endpoint           `json:"endpoints,omitempty"`
 	EndpointGroup      []portainer.EndpointGroup      `json:"endpoint_groups,omitempty"`
 	EndpointRelation   []portainer.EndpointRelation   `json:"endpoint_relations,omitempty"`
 	Extensions         []portainer.Extension          `json:"extension,omitempty"`
 	HelmUserRepository []portainer.HelmUserRepository `json:"helm_user_repository,omitempty"`
-	Registry           []registry.Registry            `json:"registries,omitempty"`
+	Registry           []portainer.Registry           `json:"registries,omitempty"`
 	ResourceControl    []portainer.ResourceControl    `json:"resource_control,omitempty"`
 	Role               []portainer.Role               `json:"roles,omitempty"`
-	Schedules          []schedule.Schedule            `json:"schedules,omitempty"`
+	Schedules          []portainer.Schedule           `json:"schedules,omitempty"`
 	Settings           portainer.Settings             `json:"settings,omitempty"`
 	SSLSettings        portainer.SSLSettings          `json:"ssl,omitempty"`
 	Stack              []portainer.Stack              `json:"stacks,omitempty"`

api/datastore/test_data/output_35.json

@@ -0,0 +1,808 @@
+{
+  "dockerhub": [
+    {
+      "Authentication": false,
+      "Username": ""
+    }
+  ],
+  "endpoint_groups": [
+    {
+      "AuthorizedTeams": null,
+      "AuthorizedUsers": null,
+      "Description": "Unassigned endpoints",
+      "Id": 1,
+      "Labels": [],
+      "Name": "Unassigned",
+      "TagIds": [],
+      "Tags": null,
+      "TeamAccessPolicies": {},
+      "UserAccessPolicies": {}
+    }
+  ],
+  "endpoint_relations": [
+    {
+      "EdgeStacks": {},
+      "EndpointID": 1
+    }
+  ],
+  "endpoints": [
+    {
+      "AuthorizedTeams": null,
+      "AuthorizedUsers": null,
+      "AzureCredentials": {
+        "ApplicationID": "",
+        "AuthenticationKey": "",
+        "TenantID": ""
+      },
+      "ComposeSyntaxMaxVersion": "",
+      "EdgeCheckinInterval": 0,
+      "EdgeKey": "",
+      "GroupId": 1,
+      "Id": 1,
+      "IsEdgeDevice": false,
+      "Kubernetes": {
+        "Configuration": {
+          "IngressClasses": null,
+          "RestrictDefaultNamespace": false,
+          "StorageClasses": null,
+          "UseLoadBalancer": false,
+          "UseServerMetrics": false
+        },
+        "Snapshots": null
+      },
+      "LastCheckInDate": 0,
+      "Name": "local",
+      "PublicURL": "",
+      "QueryDate": 0,
+      "SecuritySettings": {
+        "allowBindMountsForRegularUsers": true,
+        "allowContainerCapabilitiesForRegularUsers": true,
+        "allowDeviceMappingForRegularUsers": true,
+        "allowHostNamespaceForRegularUsers": true,
+        "allowPrivilegedModeForRegularUsers": true,
+        "allowStackManagementForRegularUsers": true,
+        "allowSysctlSettingForRegularUsers": false,
+        "allowVolumeBrowserForRegularUsers": false,
+        "enableHostManagementFeatures": false
+      },
+      "Snapshots": [
+        {
+          "DockerSnapshotRaw": {
+            "Containers": null,
+            "Images": null,
+            "Info": null,
+            "Networks": null,
+            "Version": null,
+            "Volumes": null
+          },
+          "DockerVersion": "20.10.13",
+          "HealthyContainerCount": 0,
+          "ImageCount": 9,
+          "NodeCount": 0,
+          "RunningContainerCount": 5,
+          "ServiceCount": 0,
+          "StackCount": 2,
+          "StoppedContainerCount": 0,
+          "Swarm": false,
+          "Time": 1648610112,
+          "TotalCPU": 8,
+          "TotalMemory": 25098706944,
+          "UnhealthyContainerCount": 0,
+          "VolumeCount": 10
+        }
+      ],
+      "Status": 1,
+      "TLSConfig": {
+        "TLS": false,
+        "TLSSkipVerify": false
+      },
+      "TagIds": [],
+      "Tags": null,
+      "TeamAccessPolicies": {},
+      "Type": 1,
+      "URL": "unix:///var/run/docker.sock",
+      "UserAccessPolicies": {},
+      "UserTrusted": false
+    }
+  ],
+  "registries": [
+    {
+      "Authentication": true,
+      "AuthorizedTeams": null,
+      "AuthorizedUsers": null,
+      "BaseURL": "",
+      "Ecr": {
+        "Region": ""
+      },
+      "Gitlab": {
+        "InstanceURL": "",
+        "ProjectId": 0,
+        "ProjectPath": ""
+      },
+      "Id": 1,
+      "ManagementConfiguration": null,
+      "Name": "canister.io",
+      "Password": "MjWbx8A6YK7cw7",
+      "Quay": {
+        "OrganisationName": "",
+        "UseOrganisation": false
+      },
+      "RegistryAccesses": {
+        "1": {
+          "Namespaces": [],
+          "TeamAccessPolicies": {},
+          "UserAccessPolicies": {}
+        }
+      },
+      "TeamAccessPolicies": {},
+      "Type": 3,
+      "URL": "cloud.canister.io:5000",
+      "UserAccessPolicies": {},
+      "Username": "prabhatkhera"
+    }
+  ],
+  "resource_control": [
+    {
+      "AdministratorsOnly": false,
+      "Id": 2,
+      "Public": true,
+      "ResourceId": "762gbwaj8r4gcsdy8ld1u4why",
+      "SubResourceIds": [],
+      "System": false,
+      "TeamAccesses": [],
+      "Type": 5,
+      "UserAccesses": []
+    },
+    {
+      "AdministratorsOnly": false,
+      "Id": 3,
+      "Public": true,
+      "ResourceId": "1_alpine",
+      "SubResourceIds": [],
+      "System": false,
+      "TeamAccesses": [],
+      "Type": 6,
+      "UserAccesses": []
+    },
+    {
+      "AdministratorsOnly": false,
+      "Id": 4,
+      "Public": true,
+      "ResourceId": "1_redis",
+      "SubResourceIds": [],
+      "System": false,
+      "TeamAccesses": [],
+      "Type": 6,
+      "UserAccesses": []
+    },
+    {
+      "AdministratorsOnly": false,
+      "Id": 5,
+      "Public": false,
+      "ResourceId": "1_nginx",
+      "SubResourceIds": [],
+      "System": false,
+      "TeamAccesses": [
+        {
+          "AccessLevel": 1,
+          "TeamId": 1
+        }
+      ],
+      "Type": 6,
+      "UserAccesses": []
+    }
+  ],
+  "roles": [
+    {
+      "Authorizations": {
+        "DockerAgentBrowseDelete": true,
+        "DockerAgentBrowseGet": true,
+        "DockerAgentBrowseList": true,
+        "DockerAgentBrowsePut": true,
+        "DockerAgentBrowseRename": true,
+        "DockerAgentHostInfo": true,
+        "DockerAgentList": true,
+        "DockerAgentPing": true,
+        "DockerAgentUndefined": true,
+        "DockerBuildCancel": true,
+        "DockerBuildPrune": true,
+        "DockerConfigCreate": true,
+        "DockerConfigDelete": true,
+        "DockerConfigInspect": true,
+        "DockerConfigList": true,
+        "DockerConfigUpdate": true,
+        "DockerContainerArchive": true,
+        "DockerContainerArchiveInfo": true,
+        "DockerContainerAttach": true,
+        "DockerContainerAttachWebsocket": true,
+        "DockerContainerChanges": true,
+        "DockerContainerCreate": true,
+        "DockerContainerDelete": true,
+        "DockerContainerExec": true,
+        "DockerContainerExport": true,
+        "DockerContainerInspect": true,
+        "DockerContainerKill": true,
+        "DockerContainerList": true,
+        "DockerContainerLogs": true,
+        "DockerContainerPause": true,
+        "DockerContainerPrune": true,
+        "DockerContainerPutContainerArchive": true,
+        "DockerContainerRename": true,
+        "DockerContainerResize": true,
+        "DockerContainerRestart": true,
+        "DockerContainerStart": true,
+        "DockerContainerStats": true,
+        "DockerContainerStop": true,
+        "DockerContainerTop": true,
+        "DockerContainerUnpause": true,
+        "DockerContainerUpdate": true,
+        "DockerContainerWait": true,
+        "DockerDistributionInspect": true,
+        "DockerEvents": true,
+        "DockerExecInspect": true,
+        "DockerExecResize": true,
+        "DockerExecStart": true,
+        "DockerImageBuild": true,
+        "DockerImageCommit": true,
+        "DockerImageCreate": true,
+        "DockerImageDelete": true,
+        "DockerImageGet": true,
+        "DockerImageGetAll": true,
+        "DockerImageHistory": true,
+        "DockerImageInspect": true,
+        "DockerImageList": true,
+        "DockerImageLoad": true,
+        "DockerImagePrune": true,
+        "DockerImagePush": true,
+        "DockerImageSearch": true,
+        "DockerImageTag": true,
+        "DockerInfo": true,
+        "DockerNetworkConnect": true,
+        "DockerNetworkCreate": true,
+        "DockerNetworkDelete": true,
+        "DockerNetworkDisconnect": true,
+        "DockerNetworkInspect": true,
+        "DockerNetworkList": true,
+        "DockerNetworkPrune": true,
+        "DockerNodeDelete": true,
+        "DockerNodeInspect": true,
+        "DockerNodeList": true,
+        "DockerNodeUpdate": true,
+        "DockerPing": true,
+        "DockerPluginCreate": true,
+        "DockerPluginDelete": true,
+        "DockerPluginDisable": true,
+        "DockerPluginEnable": true,
+        "DockerPluginInspect": true,
+        "DockerPluginList": true,
+        "DockerPluginPrivileges": true,
+        "DockerPluginPull": true,
+        "DockerPluginPush": true,
+        "DockerPluginSet": true,
+        "DockerPluginUpgrade": true,
+        "DockerSecretCreate": true,
+        "DockerSecretDelete": true,
+        "DockerSecretInspect": true,
+        "DockerSecretList": true,
+        "DockerSecretUpdate": true,
+        "DockerServiceCreate": true,
+        "DockerServiceDelete": true,
+        "DockerServiceInspect": true,
+        "DockerServiceList": true,
+        "DockerServiceLogs": true,
+        "DockerServiceUpdate": true,
+        "DockerSessionStart": true,
+        "DockerSwarmInit": true,
+        "DockerSwarmInspect": true,
+        "DockerSwarmJoin": true,
+        "DockerSwarmLeave": true,
+        "DockerSwarmUnlock": true,
+        "DockerSwarmUnlockKey": true,
+        "DockerSwarmUpdate": true,
+        "DockerSystem": true,
+        "DockerTaskInspect": true,
+        "DockerTaskList": true,
+        "DockerTaskLogs": true,
+        "DockerUndefined": true,
+        "DockerVersion": true,
+        "DockerVolumeCreate": true,
+        "DockerVolumeDelete": true,
+        "DockerVolumeInspect": true,
+        "DockerVolumeList": true,
+        "DockerVolumePrune": true,
+        "EndpointResourcesAccess": true,
+        "IntegrationStoridgeAdmin": true,
+        "PortainerResourceControlCreate": true,
+        "PortainerResourceControlUpdate": true,
+        "PortainerStackCreate": true,
+        "PortainerStackDelete": true,
+        "PortainerStackFile": true,
+        "PortainerStackInspect": true,
+        "PortainerStackList": true,
+        "PortainerStackMigrate": true,
+        "PortainerStackUpdate": true,
+        "PortainerWebhookCreate": true,
+        "PortainerWebhookDelete": true,
+        "PortainerWebhookList": true,
+        "PortainerWebsocketExec": true
+      },
+      "Description": "Full control of all resources in an endpoint",
+      "Id": 1,
+      "Name": "Endpoint administrator",
+      "Priority": 1
+    },
+    {
+      "Authorizations": {
+        "DockerAgentHostInfo": true,
+        "DockerAgentList": true,
+        "DockerAgentPing": true,
+        "DockerConfigInspect": true,
+        "DockerConfigList": true,
+        "DockerContainerArchiveInfo": true,
+        "DockerContainerChanges": true,
+        "DockerContainerInspect": true,
+        "DockerContainerList": true,
+        "DockerContainerLogs": true,
+        "DockerContainerStats": true,
+        "DockerContainerTop": true,
+        "DockerDistributionInspect": true,
+        "DockerEvents": true,
+        "DockerImageGet": true,
+        "DockerImageGetAll": true,
+        "DockerImageHistory": true,
+        "DockerImageInspect": true,
+        "DockerImageList": true,
+        "DockerImageSearch": true,
+        "DockerInfo": true,
+        "DockerNetworkInspect": true,
+        "DockerNetworkList": true,
+        "DockerNodeInspect": true,
+        "DockerNodeList": true,
+        "DockerPing": true,
+        "DockerPluginList": true,
+        "DockerSecretInspect": true,
+        "DockerSecretList": true,
+        "DockerServiceInspect": true,
+        "DockerServiceList": true,
+        "DockerServiceLogs": true,
+        "DockerSwarmInspect": true,
+        "DockerSystem": true,
+        "DockerTaskInspect": true,
+        "DockerTaskList": true,
+        "DockerTaskLogs": true,
+        "DockerVersion": true,
+        "DockerVolumeInspect": true,
+        "DockerVolumeList": true,
+        "EndpointResourcesAccess": true,
+        "PortainerStackFile": true,
+        "PortainerStackInspect": true,
+        "PortainerStackList": true,
+        "PortainerWebhookList": true
+      },
+      "Description": "Read-only access of all resources in an endpoint",
+      "Id": 2,
+      "Name": "Helpdesk",
+      "Priority": 2
+    },
+    {
+      "Authorizations": {
+        "DockerAgentHostInfo": true,
+        "DockerAgentList": true,
+        "DockerAgentPing": true,
+        "DockerAgentUndefined": true,
+        "DockerBuildCancel": true,
+        "DockerBuildPrune": true,
+        "DockerConfigCreate": true,
+        "DockerConfigDelete": true,
+        "DockerConfigInspect": true,
+        "DockerConfigList": true,
+        "DockerConfigUpdate": true,
+        "DockerContainerArchive": true,
+        "DockerContainerArchiveInfo": true,
+        "DockerContainerAttach": true,
+        "DockerContainerAttachWebsocket": true,
+        "DockerContainerChanges": true,
+        "DockerContainerCreate": true,
+        "DockerContainerDelete": true,
+        "DockerContainerExec": true,
+        "DockerContainerExport": true,
+        "DockerContainerInspect": true,
+        "DockerContainerKill": true,
+        "DockerContainerList": true,
+        "DockerContainerLogs": true,
+        "DockerContainerPause": true,
+        "DockerContainerPutContainerArchive": true,
+        "DockerContainerRename": true,
+        "DockerContainerResize": true,
+        "DockerContainerRestart": true,
+        "DockerContainerStart": true,
+        "DockerContainerStats": true,
+        "DockerContainerStop": true,
+        "DockerContainerTop": true,
+        "DockerContainerUnpause": true,
+        "DockerContainerUpdate": true,
+        "DockerContainerWait": true,
+        "DockerDistributionInspect": true,
+        "DockerEvents": true,
+        "DockerExecInspect": true,
+        "DockerExecResize": true,
+        "DockerExecStart": true,
+        "DockerImageBuild": true,
+        "DockerImageCommit": true,
+        "DockerImageCreate": true,
+        "DockerImageDelete": true,
+        "DockerImageGet": true,
+        "DockerImageGetAll": true,
+        "DockerImageHistory": true,
+        "DockerImageInspect": true,
+        "DockerImageList": true,
+        "DockerImageLoad": true,
+        "DockerImagePush": true,
+        "DockerImageSearch": true,
+        "DockerImageTag": true,
+        "DockerInfo": true,
+        "DockerNetworkConnect": true,
+        "DockerNetworkCreate": true,
+        "DockerNetworkDelete": true,
+        "DockerNetworkDisconnect": true,
+        "DockerNetworkInspect": true,
+        "DockerNetworkList": true,
+        "DockerNodeDelete": true,
+        "DockerNodeInspect": true,
+        "DockerNodeList": true,
+        "DockerNodeUpdate": true,
+        "DockerPing": true,
+        "DockerPluginCreate": true,
+        "DockerPluginDelete": true,
+        "DockerPluginDisable": true,
+        "DockerPluginEnable": true,
+        "DockerPluginInspect": true,
+        "DockerPluginList": true,
+        "DockerPluginPrivileges": true,
+        "DockerPluginPull": true,
+        "DockerPluginPush": true,
+        "DockerPluginSet": true,
+        "DockerPluginUpgrade": true,
+        "DockerSecretCreate": true,
+        "DockerSecretDelete": true,
+        "DockerSecretInspect": true,
+        "DockerSecretList": true,
+        "DockerSecretUpdate": true,
+        "DockerServiceCreate": true,
+        "DockerServiceDelete": true,
+        "DockerServiceInspect": true,
+        "DockerServiceList": true,
+        "DockerServiceLogs": true,
+        "DockerServiceUpdate": true,
+        "DockerSessionStart": true,
+        "DockerSwarmInit": true,
+        "DockerSwarmInspect": true,
+        "DockerSwarmJoin": true,
+        "DockerSwarmLeave": true,
+        "DockerSwarmUnlock": true,
+        "DockerSwarmUnlockKey": true,
+        "DockerSwarmUpdate": true,
+        "DockerSystem": true,
+        "DockerTaskInspect": true,
+        "DockerTaskList": true,
+        "DockerTaskLogs": true,
+        "DockerUndefined": true,
+        "DockerVersion": true,
+        "DockerVolumeCreate": true,
+        "DockerVolumeDelete": true,
+        "DockerVolumeInspect": true,
+        "DockerVolumeList": true,
+        "PortainerResourceControlUpdate": true,
+        "PortainerStackCreate": true,
+        "PortainerStackDelete": true,
+        "PortainerStackFile": true,
+        "PortainerStackInspect": true,
+        "PortainerStackList": true,
+        "PortainerStackMigrate": true,
+        "PortainerStackUpdate": true,
+        "PortainerWebhookCreate": true,
+        "PortainerWebhookList": true,
+        "PortainerWebsocketExec": true
+      },
+      "Description": "Full control of assigned resources in an endpoint",
+      "Id": 3,
+      "Name": "Standard user",
+      "Priority": 3
+    },
+    {
+      "Authorizations": {
+        "DockerAgentHostInfo": true,
+        "DockerAgentList": true,
+        "DockerAgentPing": true,
+        "DockerConfigInspect": true,
+        "DockerConfigList": true,
+        "DockerContainerArchiveInfo": true,
+        "DockerContainerChanges": true,
+        "DockerContainerInspect": true,
+        "DockerContainerList": true,
+        "DockerContainerLogs": true,
+        "DockerContainerStats": true,
+        "DockerContainerTop": true,
+        "DockerDistributionInspect": true,
+        "DockerEvents": true,
+        "DockerImageGet": true,
+        "DockerImageGetAll": true,
+        "DockerImageHistory": true,
+        "DockerImageInspect": true,
+        "DockerImageList": true,
+        "DockerImageSearch": true,
+        "DockerInfo": true,
+        "DockerNetworkInspect": true,
+        "DockerNetworkList": true,
+        "DockerNodeInspect": true,
+        "DockerNodeList": true,
+        "DockerPing": true,
+        "DockerPluginList": true,
+        "DockerSecretInspect": true,
+        "DockerSecretList": true,
+        "DockerServiceInspect": true,
+        "DockerServiceList": true,
+        "DockerServiceLogs": true,
+        "DockerSwarmInspect": true,
+        "DockerSystem": true,
+        "DockerTaskInspect": true,
+        "DockerTaskList": true,
+        "DockerTaskLogs": true,
+        "DockerVersion": true,
+        "DockerVolumeInspect": true,
+        "DockerVolumeList": true,
+        "PortainerStackFile": true,
+        "PortainerStackInspect": true,
+        "PortainerStackList": true,
+        "PortainerWebhookList": true
+      },
+      "Description": "Read-only access of assigned resources in an endpoint",
+      "Id": 4,
+      "Name": "Read-only user",
+      "Priority": 4
+    }
+  ],
+  "schedules": [
+    {
+      "Created": 1648608136,
+      "CronExpression": "@every 5m",
+      "EdgeSchedule": null,
+      "EndpointSyncJob": null,
+      "Id": 1,
+      "JobType": 2,
+      "Name": "system_snapshot",
+      "Recurring": true,
+      "ScriptExecutionJob": null,
+      "SnapshotJob": {}
+    }
+  ],
+  "settings": {
+    "AgentSecret": "",
+    "AllowBindMountsForRegularUsers": true,
+    "AllowContainerCapabilitiesForRegularUsers": true,
+    "AllowDeviceMappingForRegularUsers": true,
+    "AllowHostNamespaceForRegularUsers": true,
+    "AllowPrivilegedModeForRegularUsers": true,
+    "AllowStackManagementForRegularUsers": true,
+    "AllowVolumeBrowserForRegularUsers": false,
+    "AuthenticationMethod": 1,
+    "BlackListedLabels": [],
+    "DisplayDonationHeader": false,
+    "DisplayExternalContributors": false,
+    "EdgeAgentCheckinInterval": 5,
+    "EdgePortainerUrl": "",
+    "EnableEdgeComputeFeatures": false,
+    "EnableHostManagementFeatures": false,
+    "EnableTelemetry": true,
+    "EnforceEdgeID": false,
+    "FeatureFlagSettings": null,
+    "HelmRepositoryURL": "https://charts.bitnami.com/bitnami",
+    "KubeconfigExpiry": "0",
+    "KubectlShellImage": "portainer/kubectl-shell",
+    "LDAPSettings": {
+      "AnonymousMode": true,
+      "AutoCreateUsers": true,
+      "GroupSearchSettings": [
+        {
+          "GroupAttribute": "",
+          "GroupBaseDN": "",
+          "GroupFilter": ""
+        }
+      ],
+      "ReaderDN": "",
+      "SearchSettings": [
+        {
+          "BaseDN": "",
+          "Filter": "",
+          "UserNameAttribute": ""
+        }
+      ],
+      "StartTLS": false,
+      "TLSConfig": {
+        "TLS": false,
+        "TLSSkipVerify": false
+      },
+      "URL": ""
+    },
+    "LogoURL": "",
+    "OAuthSettings": {
+      "AccessTokenURI": "",
+      "AuthorizationURI": "",
+      "ClientID": "",
+      "DefaultTeamID": 0,
+      "KubeSecretKey": null,
+      "LogoutURI": "",
+      "OAuthAutoCreateUsers": false,
+      "RedirectURI": "",
+      "ResourceURI": "",
+      "SSO": false,
+      "Scopes": "",
+      "UserIdentifier": ""
+    },
+    "SnapshotInterval": "5m",
+    "TemplatesURL": "https://raw.githubusercontent.com/portainer/templates/master/templates-2.0.json",
+    "TrustOnFirstConnect": false,
+    "UserSessionTimeout": "8h",
+    "fdoConfiguration": {
+      "enabled": false,
+      "ownerPassword": "",
+      "ownerURL": "",
+      "ownerUsername": ""
+    },
+    "openAMTConfiguration": {
+      "certFileContent": "",
+      "certFileName": "",
+      "certFilePassword": "",
+      "domainName": "",
+      "enabled": false,
+      "mpsPassword": "",
+      "mpsServer": "",
+      "mpsToken": "",
+      "mpsUser": ""
+    }
+  },
+  "ssl": {
+    "certPath": "",
+    "httpEnabled": true,
+    "keyPath": "",
+    "selfSigned": false
+  },
+  "stacks": [
+    {
+      "AdditionalFiles": null,
+      "AutoUpdate": null,
+      "CreatedBy": "",
+      "CreationDate": 0,
+      "EndpointId": 1,
+      "EntryPoint": "docker/alpine37-compose.yml",
+      "Env": [],
+      "FromAppTemplate": false,
+      "GitConfig": null,
+      "Id": 2,
+      "IsComposeFormat": false,
+      "Name": "alpine",
+      "Namespace": "",
+      "ProjectPath": "/home/prabhat/portainer/data/ce1.25/compose/2",
+      "ResourceControl": null,
+      "Status": 1,
+      "SwarmId": "s3fd604zdba7z13tbq2x6lyue",
+      "Type": 1,
+      "UpdateDate": 0,
+      "UpdatedBy": ""
+    },
+    {
+      "AdditionalFiles": null,
+      "AutoUpdate": null,
+      "CreatedBy": "",
+      "CreationDate": 0,
+      "EndpointId": 1,
+      "EntryPoint": "docker-compose.yml",
+      "Env": [],
+      "FromAppTemplate": false,
+      "GitConfig": null,
+      "Id": 5,
+      "IsComposeFormat": false,
+      "Name": "redis",
+      "Namespace": "",
+      "ProjectPath": "/home/prabhat/portainer/data/ce1.25/compose/5",
+      "ResourceControl": null,
+      "Status": 1,
+      "SwarmId": "",
+      "Type": 2,
+      "UpdateDate": 0,
+      "UpdatedBy": ""
+    },
+    {
+      "AdditionalFiles": null,
+      "AutoUpdate": null,
+      "CreatedBy": "",
+      "CreationDate": 0,
+      "EndpointId": 1,
+      "EntryPoint": "docker-compose.yml",
+      "Env": [],
+      "FromAppTemplate": false,
+      "GitConfig": null,
+      "Id": 6,
+      "IsComposeFormat": false,
+      "Name": "nginx",
+      "Namespace": "",
+      "ProjectPath": "/home/prabhat/portainer/data/ce1.25/compose/6",
+      "ResourceControl": null,
+      "Status": 1,
+      "SwarmId": "",
+      "Type": 2,
+      "UpdateDate": 0,
+      "UpdatedBy": ""
+    }
+  ],
+  "teams": [
+    {
+      "Id": 1,
+      "Name": "hello"
+    }
+  ],
+  "tunnel_server": {
+    "PrivateKeySeed": "IvX6ZPRuWtLS5zyg"
+  },
+  "users": [
+    {
+      "EndpointAuthorizations": null,
+      "Id": 1,
+      "Password": "$2a$10$siRDprr/5uUFAU8iom3Sr./WXQkN2dhSNjAC471pkJaALkghS762a",
+      "PortainerAuthorizations": {
+        "PortainerDockerHubInspect": true,
+        "PortainerEndpointGroupList": true,
+        "PortainerEndpointInspect": true,
+        "PortainerEndpointList": true,
+        "PortainerMOTD": true,
+        "PortainerRegistryInspect": true,
+        "PortainerRegistryList": true,
+        "PortainerTeamList": true,
+        "PortainerTemplateInspect": true,
+        "PortainerTemplateList": true,
+        "PortainerUserCreateToken": true,
+        "PortainerUserInspect": true,
+        "PortainerUserList": true,
+        "PortainerUserListToken": true,
+        "PortainerUserMemberships": true,
+        "PortainerUserRevokeToken": true
+      },
+      "Role": 1,
+      "TokenIssueAt": 0,
+      "UserTheme": "",
+      "Username": "admin"
+    },
+    {
+      "EndpointAuthorizations": null,
+      "Id": 2,
+      "Password": "$2a$10$WpCAW8mSt6FRRp1GkynbFOGSZnHR6E5j9cETZ8HiMlw06hVlDW/Li",
+      "PortainerAuthorizations": {
+        "PortainerDockerHubInspect": true,
+        "PortainerEndpointGroupList": true,
+        "PortainerEndpointInspect": true,
+        "PortainerEndpointList": true,
+        "PortainerMOTD": true,
+        "PortainerRegistryInspect": true,
+        "PortainerRegistryList": true,
+        "PortainerTeamList": true,
+        "PortainerTemplateInspect": true,
+        "PortainerTemplateList": true,
+        "PortainerUserCreateToken": true,
+        "PortainerUserInspect": true,
+        "PortainerUserList": true,
+        "PortainerUserListToken": true,
+        "PortainerUserMemberships": true,
+        "PortainerUserRevokeToken": true
+      },
+      "Role": 1,
+      "TokenIssueAt": 0,
+      "UserTheme": "",
+      "Username": "prabhat"
+    }
+  ],
+  "version": {
+    "DB_UPDATING": "false",
+    "DB_VERSION": "35",
+    "INSTANCE_ID": "null"
+  }
+}

api/datastore/teststore.go

@@ -14,12 +14,12 @@ import (
 
 var errTempDir = errors.New("can't create a temp dir")
 
-func (store *Store) GetConnection() database.Connection {
+func (store *Store) GetConnection() portainer.Connection {
 	return store.connection
 }
 
-func MustNewTestStore(init bool) (bool, *Store, func()) {
-	newStore, store, teardown, err := NewTestStore(init)
+func MustNewTestStore(init, secure bool) (bool, *Store, func()) {
+	newStore, store, teardown, err := NewTestStore(init, secure)
 	if err != nil {
 		if !errors.Is(err, errTempDir) {
 			teardown()
@@ -30,7 +30,7 @@ func MustNewTestStore(init bool) (bool, *Store, func()) {
 	return newStore, store, teardown
 }
 
-func NewTestStore(init bool) (bool, *Store, func(), error) {
+func NewTestStore(init, secure bool) (bool, *Store, func(), error) {
 	// Creates unique temp directory in a concurrency friendly manner.
 	storePath, err := ioutil.TempDir("", "test-store")
 	if err != nil {
@@ -42,7 +42,12 @@ func NewTestStore(init bool) (bool, *Store, func(), error) {
 		return false, nil, nil, err
 	}
 
-	connection, err := database.NewDatabase("boltdb", storePath, []byte("apassphrasewhichneedstobe32bytes"))
+	secretKey := []byte("apassphrasewhichneedstobe32bytes")
+	if !secure {
+		secretKey = nil
+	}
+
+	connection, err := database.NewDatabase("boltdb", storePath, secretKey)
 	if err != nil {
 		panic(err)
 	}

api/datastore/validate/validate.go

@@ -0,0 +1,15 @@
+package validate
+
+import (
+	"github.com/go-playground/validator/v10"
+	portainer "github.com/portainer/portainer/api"
+)
+
+var validate *validator.Validate
+
+func ValidateLDAPSettings(ldp *portainer.LDAPSettings) error {
+	validate = validator.New()
+	registerValidationMethods(validate)
+
+	return validate.Struct(ldp)
+}

api/datastore/validate/validate_test.go

@@ -0,0 +1,61 @@
+package validate
+
+import (
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+)
+
+func TestValidateLDAPSettings(t *testing.T) {
+
+	tests := []struct {
+		name    string
+		ldap    portainer.LDAPSettings
+		wantErr bool
+	}{
+		{
+			name:    "Empty LDAP Settings",
+			ldap:    portainer.LDAPSettings{},
+			wantErr: true,
+		},
+		{
+			name: "With URL",
+			ldap: portainer.LDAPSettings{
+				AnonymousMode: true,
+				URL:           "192.168.0.1:323",
+			},
+			wantErr: false,
+		},
+		{
+			name: "Validate URL and URLs",
+			ldap: portainer.LDAPSettings{
+				AnonymousMode: true,
+				URL:           "192.168.0.1:323",
+			},
+			wantErr: false,
+		},
+		{
+			name: "validate client ldap",
+			ldap: portainer.LDAPSettings{
+				AnonymousMode: false,
+				ReaderDN:      "CN=LDAP API Service Account",
+				Password:      "Qu**dfUUU**",
+				URL:           "aukdc15.pgc.co:389",
+				TLSConfig: portainer.TLSConfiguration{
+					TLS:           false,
+					TLSSkipVerify: false,
+				},
+			},
+			wantErr: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := ValidateLDAPSettings(&tt.ldap)
+			if (err == nil) == tt.wantErr {
+				t.Errorf("No error expected but got %s", err)
+			}
+		})
+	}
+}

api/datastore/validate/validationMethods.go

@@ -0,0 +1,17 @@
+package validate
+
+import (
+	"github.com/go-playground/validator/v10"
+)
+
+func registerValidationMethods(v *validator.Validate) {
+	v.RegisterValidation("validate_bool", ValidateBool)
+}
+
+/**
+ * Validation methods below are being used for custom validation
+ */
+func ValidateBool(fl validator.FieldLevel) bool {
+	_, ok := fl.Field().Interface().(bool)
+	return ok
+}

api/docker/snapshot.go

@@ -12,7 +12,30 @@ import (
 	"github.com/portainer/portainer/api"
 )
 
-func CreateSnapshot(cli *client.Client, endpoint *portainer.Endpoint) (*portainer.DockerSnapshot, error) {
+// Snapshotter represents a service used to create environment(endpoint) snapshots
+type Snapshotter struct {
+	clientFactory *ClientFactory
+}
+
+// NewSnapshotter returns a new Snapshotter instance
+func NewSnapshotter(clientFactory *ClientFactory) *Snapshotter {
+	return &Snapshotter{
+		clientFactory: clientFactory,
+	}
+}
+
+// CreateSnapshot creates a snapshot of a specific Docker environment(endpoint)
+func (snapshotter *Snapshotter) CreateSnapshot(endpoint *portainer.Endpoint) (*portainer.DockerSnapshot, error) {
+	cli, err := snapshotter.clientFactory.CreateClient(endpoint, "", nil)
+	if err != nil {
+		return nil, err
+	}
+	defer cli.Close()
+
+	return snapshot(cli, endpoint)
+}
+
+func snapshot(cli *client.Client, endpoint *portainer.Endpoint) (*portainer.DockerSnapshot, error) {
 	_, err := cli.Ping(context.Background())
 	if err != nil {
 		return nil, err

api/exec/exectest/kubernetes_mocks.go

@@ -2,7 +2,6 @@ package exectest
 
 import (
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/database"
 )
 
 type kubernetesMockDeployer struct{}
@@ -11,11 +10,11 @@ func NewKubernetesDeployer() portainer.KubernetesDeployer {
 	return &kubernetesMockDeployer{}
 }
 
-func (deployer *kubernetesMockDeployer) Deploy(userID database.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) {
+func (deployer *kubernetesMockDeployer) Deploy(userID portainer.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) {
 	return "", nil
 }
 
-func (deployer *kubernetesMockDeployer) Remove(userID database.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) {
+func (deployer *kubernetesMockDeployer) Remove(userID portainer.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) {
 	return "", nil
 }
 

api/exec/kubernetes_deploy.go

@@ -3,19 +3,20 @@ package exec
 import (
 	"bytes"
 	"fmt"
-	"github.com/portainer/portainer/api/database"
-	"github.com/portainer/portainer/api/orchestrators/kubernetes/cli"
+	"os"
 	"os/exec"
 	"path"
 	"runtime"
 	"strings"
 
 	"github.com/pkg/errors"
-	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/http/proxy"
 	"github.com/portainer/portainer/api/http/proxy/factory"
 	"github.com/portainer/portainer/api/http/proxy/factory/kubernetes"
+	"github.com/portainer/portainer/api/kubernetes/cli"
+
+	portainer "github.com/portainer/portainer/api"
 )
 
 // KubernetesDeployer represents a service to deploy resources inside a Kubernetes environment(endpoint).
@@ -42,7 +43,7 @@ func NewKubernetesDeployer(kubernetesTokenCacheManager *kubernetes.TokenCacheMan
 	}
 }
 
-func (deployer *KubernetesDeployer) getToken(userID database.UserID, endpoint *portainer.Endpoint, setLocalAdminToken bool) (string, error) {
+func (deployer *KubernetesDeployer) getToken(userID portainer.UserID, endpoint *portainer.Endpoint, setLocalAdminToken bool) (string, error) {
 	kubeCLI, err := deployer.kubernetesClientFactory.GetKubeClient(endpoint)
 	if err != nil {
 		return "", err
@@ -76,16 +77,16 @@ func (deployer *KubernetesDeployer) getToken(userID database.UserID, endpoint *p
 }
 
 // Deploy upserts Kubernetes resources defined in manifest(s)
-func (deployer *KubernetesDeployer) Deploy(userID database.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) {
+func (deployer *KubernetesDeployer) Deploy(userID portainer.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) {
 	return deployer.command("apply", userID, endpoint, manifestFiles, namespace)
 }
 
 // Remove deletes Kubernetes resources defined in manifest(s)
-func (deployer *KubernetesDeployer) Remove(userID database.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) {
+func (deployer *KubernetesDeployer) Remove(userID portainer.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) {
 	return deployer.command("delete", userID, endpoint, manifestFiles, namespace)
 }
 
-func (deployer *KubernetesDeployer) command(operation string, userID database.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) {
+func (deployer *KubernetesDeployer) command(operation string, userID portainer.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) {
 	token, err := deployer.getToken(userID, endpoint, endpoint.Type == portainer.KubernetesLocalEnvironment)
 	if err != nil {
 		return "", errors.Wrap(err, "failed generating a user token")
@@ -123,6 +124,8 @@ func (deployer *KubernetesDeployer) command(operation string, userID database.Us
 
 	var stderr bytes.Buffer
 	cmd := exec.Command(command, args...)
+	cmd.Env = os.Environ()
+	cmd.Env = append(cmd.Env, "POD_NAMESPACE=default")
 	cmd.Stderr = &stderr
 
 	output, err := cmd.Output()