Merge branch 'release/1.19.0'

* feat(home): add the ability to refresh endpoint information

* style(home): update refresh confirmation message

.codeclimate.yml

@@ -12,7 +12,8 @@ engines:
     enabled: true
     config:
       languages:
-      - javascript
+        javascript:
+          mass_threshold: 80
   eslint:
     enabled: true
     config:

.codefresh/codefresh_branch.yml

@@ -13,14 +13,28 @@ steps:
     image: portainer/angular-builder:latest
     working_directory: ${{build_backend}}
     commands:
-      - npm install -g bower grunt grunt-cli && npm install
-      - bower install --allow-root
-      - grunt build-webapp
+      - yarn
+      - yarn grunt build-webapp
       - mv api/cmd/portainer/portainer dist/
 
+  get_docker_version:
+    image: alpine:3.7
+    working_directory: ${{build_frontend}}
+    commands:
+      - cf_export DOCKER_VERSION=`cat gruntfile.js | grep -m 1 'shippedDockerVersion' | cut -d\' -f2`
+
+  download_docker_binary:
+    image: busybox
+    working_directory: ${{build_frontend}}
+    commands:
+      - echo ${{DOCKER_VERSION}}
+      - wget -O /tmp/docker-binaries.tgz https://download.docker.com/linux/static/stable/x86_64/docker-${{DOCKER_VERSION}}.tgz
+      - tar -xf /tmp/docker-binaries.tgz -C /tmp
+      - mv /tmp/docker/docker dist/
+
   build_image:
     type: build
-    working_directory: ${{build_frontend}}
+    working_directory: ${{download_docker_binary}}
     dockerfile: ./build/linux/Dockerfile
     image_name: portainer/portainer
     tag: ${{CF_BRANCH}}
@@ -30,7 +44,3 @@ steps:
     candidate: '${{build_image}}'
     tag: '${{CF_BRANCH}}'
     registry: dockerhub
-    when:
-      branch:
-        only:
-          - develop

.codefresh/codefresh_pullrequest.yml

@@ -0,0 +1,46 @@
+version: '1.0'
+steps:
+
+  build_backend:
+    image: portainer/golang-builder:ci
+    working_directory: ${{main_clone}}
+    commands:
+      - mkdir -p /go/src/github.com/${{CF_REPO_OWNER}}
+      - ln -s /codefresh/volume/${{CF_REPO_NAME}}/api /go/src/github.com/${{CF_REPO_OWNER}}/${{CF_REPO_NAME}}
+      - /build.sh api/cmd/portainer
+
+  build_frontend:
+    image: portainer/angular-builder:latest
+    working_directory: ${{build_backend}}
+    commands:
+      - yarn
+      - yarn grunt build-webapp
+      - mv api/cmd/portainer/portainer dist/
+
+  get_docker_version:
+    image: alpine:3.7
+    working_directory: ${{build_frontend}}
+    commands:
+      - cf_export DOCKER_VERSION=`cat gruntfile.js | grep -m 1 'shippedDockerVersion' | cut -d\' -f2`
+
+  download_docker_binary:
+    image: busybox
+    working_directory: ${{build_frontend}}
+    commands:
+      - echo ${{DOCKER_VERSION}}
+      - wget -O /tmp/docker-binaries.tgz https://download.docker.com/linux/static/stable/x86_64/docker-${{DOCKER_VERSION}}.tgz
+      - tar -xf /tmp/docker-binaries.tgz -C /tmp
+      - mv /tmp/docker/docker dist/
+
+  build_image:
+    type: build
+    working_directory: ${{download_docker_binary}}
+    dockerfile: ./build/linux/Dockerfile
+    image_name: portainer/portainer
+    tag: ${{CF_BRANCH}}
+
+  push_image:
+    type: push
+    candidate: '${{build_image}}'
+    tag: 'pr${{CF_PULL_REQUEST_NUMBER}}'
+    registry: dockerhub

.github/ISSUE_TEMPLATE/Bug_report.md

@@ -0,0 +1,47 @@
+---
+name: Bug report
+about: Create a bug report
+
+---
+
+<!--
+
+Thanks for reporting a bug for Portainer !
+
+Do you need help or have a question? Come chat with us on Slack http://portainer.io/slack/ or gitter https://gitter.im/portainer/Lobby.
+
+Before opening a new issue, make sure that we do not have any duplicates
+already open. You can ensure this by searching the issue list for this
+repository. If there is a duplicate, please close your issue and add a comment
+to the existing issue instead.
+
+Also, be sure to check our FAQ and documentation first: https://portainer.readthedocs.io
+-->
+
+**Bug description**
+
+A clear and concise description of what the bug is.
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+Briefly describe what you were expecting.
+
+**Steps to reproduce the issue:**
+
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Technical details:**
+
+* Portainer version:
+* Docker version (managed by Portainer):
+* Platform (windows/linux):
+* Command used to start Portainer (`docker run -p 9000:9000 portainer/portainer`):
+* Browser:
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/Custom.md

@@ -0,0 +1,15 @@
+---
+name: Question
+about: Ask us a question about Portainer usage or deployment
+
+---
+
+<!--
+
+Do you need help or have a question? Come chat with us on Slack http://portainer.io/slack/ or gitter https://gitter.im/portainer/Lobby.
+
+Also, be sure to check our FAQ and documentation first: https://portainer.readthedocs.io
+-->
+
+**Question**:
+How can I deploy Portainer on... ?

.github/ISSUE_TEMPLATE/Feature_request.md

@@ -0,0 +1,31 @@
+---
+name: Feature request
+about: Suggest a feature/enhancement that should be added in Portainer
+
+---
+
+<!--
+
+Thanks for opening a feature request for Portainer !
+
+Do you need help or have a question? Come chat with us on Slack http://portainer.io/slack/ or gitter https://gitter.im/portainer/Lobby.
+
+Before opening a new issue, make sure that we do not have any duplicates
+already open. You can ensure this by searching the issue list for this
+repository. If there is a duplicate, please close your issue and add a comment
+to the existing issue instead.
+
+Also, be sure to check our FAQ and documentation first: https://portainer.readthedocs.io
+-->
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

CODE_OF_CONDUCT.md

@@ -0,0 +1,46 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at anthony.lapenna@portainer.io. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

CONTRIBUTING.md

@@ -30,9 +30,6 @@ You can have a use Github filters to list these issues:
 * intermediate labeled issues: https://github.com/portainer/portainer/labels/exp%2Fintermediate
 * advanced labeled issues: https://github.com/portainer/portainer/labels/exp%2Fadvanced
 
-### Linting
-
-Please check your code using `grunt lint` before submitting your pull requests.
 
 ### Commit Message Format
 

LICENSE

@@ -1,4 +1,4 @@
-Portainer: Copyright (c) 2016 Portainer.io
+Copyright (c) 2018 Portainer.io
 
 This software is provided 'as-is', without any express or implied
 warranty.  In no event will the authors be held liable for any damages
@@ -14,46 +14,4 @@ freely, subject to the following restrictions:
    appreciated but is not required.
 2. Altered source versions must be plainly marked as such, and must not be
    misrepresented as being the original software.
-3. This notice may not be removed or altered from any source distribution.
-
-Portainer contains code which was originally under this license:
-
-UI For Docker: Copyright (c) 2013-2016 Michael Crosby (crosbymichael.com), Kevan Ahlquist (kevanahlquist.com), Anthony Lapenna (portainer.io)
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
-rdash-angular: Copyright (c) [2014] [Elliot Hesp]
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+3. This notice may not be removed or altered from any source distribution.

README.md

@@ -1,30 +1,39 @@
 
 <p align="center">
-  <img title="portainer" src='http://portainer.io/images/logo_alt.png' />
+  <img title="portainer" src='https://portainer.io/images/logo_alt.png' />
 </p>
 
 [![Docker Pulls](https://img.shields.io/docker/pulls/portainer/portainer.svg)](https://hub.docker.com/r/portainer/portainer/)
 [![Microbadger](https://images.microbadger.com/badges/image/portainer/portainer.svg)](http://microbadger.com/images/portainer/portainer "Image size")
 [![Documentation Status](https://readthedocs.org/projects/portainer/badge/?version=stable)](http://portainer.readthedocs.io/en/stable/?badge=stable)
 [![Codefresh build status]( https://g.codefresh.io/api/badges/build?repoOwner=portainer&repoName=portainer&branch=develop&pipelineName=portainer-ci&accountName=deviantony&type=cf-1)]( https://g.codefresh.io/repositories/portainer/portainer/builds?filter=trigger:build;branch:develop;service:5922a08a3a1aab000116fcc6~portainer-ci)
-[![Slack](http://portainer.io/slack/badge.svg)](http://portainer.io/slack/)
+[![Code Climate](https://codeclimate.com/github/portainer/portainer/badges/gpa.svg)](https://codeclimate.com/github/portainer/portainer)
+[![Slack](https://portainer.io/slack/badge.svg)](https://portainer.io/slack/)
 [![Gitter](https://badges.gitter.im/portainer/Lobby.svg)](https://gitter.im/portainer/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
 [![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=YHXZJQNJQ36H6)
 
-**_Portainer_** is a lightweight management UI which allows you to **easily** manage your Docker host or Swarm cluster.
+**_Portainer_** is a lightweight management UI which allows you to **easily** manage your different Docker environments (Docker hosts or Swarm clusters).
 
-**_Portainer_** is meant to be as **simple** to deploy as it is to use. It consists of a single container that can run on any Docker engine (Docker for Linux and Docker for Windows are supported).
+**_Portainer_** is meant to be as **simple** to deploy as it is to use. It consists of a single container that can run on any Docker engine (can be deployed as Linux container or a Windows native container).
 
-**_Portainer_** allows you to manage your Docker containers, images, volumes, networks and more ! It is compatible with the *standalone Docker* engine and with *Docker Swarm*.
+**_Portainer_** allows you to manage your Docker containers, images, volumes, networks and more ! It is compatible with the *standalone Docker* engine and with *Docker Swarm mode*.
 
 ## Demo
 
-<img src="http://portainer.io/images/screenshots/portainer.gif" width="77%"/>
+<img src="https://portainer.io/images/screenshots/portainer.gif" width="77%"/>
 
 You can try out the public demo instance: http://demo.portainer.io/ (login with the username **admin** and the password **tryportainer**).
 
 Please note that the public demo cluster is **reset every 15min**.
 
+Alternatively, you can deploy a copy of the demo stack inside a [play-with-docker (PWD)](https://labs.play-with-docker.com) playground:
+
+- Browse [PWD/?stack=portainer-demo/play-with-docker/docker-stack.yml](http://play-with-docker.com/?stack=https://raw.githubusercontent.com/portainer/portainer-demo/master/play-with-docker/docker-stack.yml)
+- Sign in with your [Docker ID](https://docs.docker.com/docker-id)
+- Follow [these](https://github.com/portainer/portainer-demo/blob/master/play-with-docker/docker-stack.yml#L5-L8) steps.
+
+Unlike the public demo, the playground sessions are deleted after 4 hours. Apart from that, all the settings are same, including default credentials.
+
 ## Getting started
 
 * [Deploy Portainer](https://portainer.readthedocs.io/en/latest/deployment.html)
@@ -34,8 +43,8 @@ Please note that the public demo cluster is **reset every 15min**.
 
 * Issues: https://github.com/portainer/portainer/issues
 * FAQ: https://portainer.readthedocs.io/en/latest/faq.html
+* Slack (chat): https://portainer.io/slack/
 * Gitter (chat): https://gitter.im/portainer/Lobby
-* Slack: http://portainer.io/slack/
 
 ## Reporting bugs and contributing
 
@@ -47,8 +56,18 @@ Please note that the public demo cluster is **reset every 15min**.
 **_Portainer_** has full support for the following Docker versions:
 
 * Docker 1.10 to the latest version
-* Docker Swarm >= 1.2.3
+* Standalone Docker Swarm >= 1.2.3 _(**NOTE:** Use of Standalone Docker Swarm is being discouraged since the introduction of built-in Swarm Mode in Docker. While older versions of Portainer had support for Standalone Docker Swarm, Portainer 1.17.0 and newer **do not** support it. However, the built-in Swarm Mode of Docker is fully supported.)_
 
 Partial support for the following Docker versions (some features may not be available):
 
 * Docker 1.9
+
+## Licensing
+
+Portainer is licensed under the zlib license. See [LICENSE](./LICENSE) for reference.
+
+Portainer also contains the following code, which is licensed under the [MIT license](https://opensource.org/licenses/MIT):
+
+UI For Docker: Copyright (c) 2013-2016 Michael Crosby (crosbymichael.com), Kevan Ahlquist (kevanahlquist.com), Anthony Lapenna (portainer.io)
+
+rdash-angular: Copyright (c) [2014] [Elliot Hesp]

api/archive/tar.go

@@ -0,0 +1,36 @@
+package archive
+
+import (
+	"archive/tar"
+	"bytes"
+)
+
+// TarFileInBuffer will create a tar archive containing a single file named via fileName and using the content
+// specified in fileContent. Returns the archive as a byte array.
+func TarFileInBuffer(fileContent []byte, fileName string) ([]byte, error) {
+	var buffer bytes.Buffer
+	tarWriter := tar.NewWriter(&buffer)
+
+	header := &tar.Header{
+		Name: fileName,
+		Mode: 0600,
+		Size: int64(len(fileContent)),
+	}
+
+	err := tarWriter.WriteHeader(header)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = tarWriter.Write(fileContent)
+	if err != nil {
+		return nil, err
+	}
+
+	err = tarWriter.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	return buffer.Bytes(), nil
+}

api/bolt/datastore.go

@@ -2,68 +2,68 @@ package bolt
 
 import (
 	"log"
-	"os"
+	"path"
 	"time"
 
 	"github.com/boltdb/bolt"
 	"github.com/portainer/portainer"
+	"github.com/portainer/portainer/bolt/dockerhub"
+	"github.com/portainer/portainer/bolt/endpoint"
+	"github.com/portainer/portainer/bolt/endpointgroup"
+	"github.com/portainer/portainer/bolt/migrator"
+	"github.com/portainer/portainer/bolt/registry"
+	"github.com/portainer/portainer/bolt/resourcecontrol"
+	"github.com/portainer/portainer/bolt/settings"
+	"github.com/portainer/portainer/bolt/stack"
+	"github.com/portainer/portainer/bolt/tag"
+	"github.com/portainer/portainer/bolt/team"
+	"github.com/portainer/portainer/bolt/teammembership"
+	"github.com/portainer/portainer/bolt/template"
+	"github.com/portainer/portainer/bolt/user"
+	"github.com/portainer/portainer/bolt/version"
+)
+
+const (
+	databaseFileName = "portainer.db"
 )
 
 // Store defines the implementation of portainer.DataStore using
 // BoltDB as the storage system.
 type Store struct {
-	// Path where is stored the BoltDB database.
-	Path string
-
-	// Services
-	UserService            *UserService
-	TeamService            *TeamService
-	TeamMembershipService  *TeamMembershipService
-	EndpointService        *EndpointService
-	ResourceControlService *ResourceControlService
-	VersionService         *VersionService
-	SettingsService        *SettingsService
-
-	db                    *bolt.DB
-	checkForDataMigration bool
+	path                   string
+	db                     *bolt.DB
+	checkForDataMigration  bool
+	fileService            portainer.FileService
+	DockerHubService       *dockerhub.Service
+	EndpointGroupService   *endpointgroup.Service
+	EndpointService        *endpoint.Service
+	RegistryService        *registry.Service
+	ResourceControlService *resourcecontrol.Service
+	SettingsService        *settings.Service
+	StackService           *stack.Service
+	TagService             *tag.Service
+	TeamMembershipService  *teammembership.Service
+	TeamService            *team.Service
+	TemplateService        *template.Service
+	UserService            *user.Service
+	VersionService         *version.Service
 }
 
-const (
-	databaseFileName          = "portainer.db"
-	versionBucketName         = "version"
-	userBucketName            = "users"
-	teamBucketName            = "teams"
-	teamMembershipBucketName  = "team_membership"
-	endpointBucketName        = "endpoints"
-	resourceControlBucketName = "resource_control"
-	settingsBucketName        = "settings"
-)
-
 // NewStore initializes a new Store and the associated services
-func NewStore(storePath string) (*Store, error) {
+func NewStore(storePath string, fileService portainer.FileService) (*Store, error) {
 	store := &Store{
-		Path:                   storePath,
-		UserService:            &UserService{},
-		TeamService:            &TeamService{},
-		TeamMembershipService:  &TeamMembershipService{},
-		EndpointService:        &EndpointService{},
-		ResourceControlService: &ResourceControlService{},
-		VersionService:         &VersionService{},
-		SettingsService:        &SettingsService{},
+		path:        storePath,
+		fileService: fileService,
 	}
-	store.UserService.store = store
-	store.TeamService.store = store
-	store.TeamMembershipService.store = store
-	store.EndpointService.store = store
-	store.ResourceControlService.store = store
-	store.VersionService.store = store
-	store.SettingsService.store = store
 
-	_, err := os.Stat(storePath + "/" + databaseFileName)
-	if err != nil && os.IsNotExist(err) {
-		store.checkForDataMigration = false
-	} else if err != nil {
+	databasePath := path.Join(storePath, databaseFileName)
+	databaseFileExists, err := fileService.FileExists(databasePath)
+	if err != nil {
 		return nil, err
+	}
+
+	if !databaseFileExists {
+		store.checkForDataMigration = false
 	} else {
 		store.checkForDataMigration = true
 	}
@@ -73,43 +73,37 @@ func NewStore(storePath string) (*Store, error) {
 
 // Open opens and initializes the BoltDB database.
 func (store *Store) Open() error {
-	path := store.Path + "/" + databaseFileName
-	db, err := bolt.Open(path, 0600, &bolt.Options{Timeout: 1 * time.Second})
+	databasePath := path.Join(store.path, databaseFileName)
+	db, err := bolt.Open(databasePath, 0600, &bolt.Options{Timeout: 1 * time.Second})
 	if err != nil {
 		return err
 	}
 	store.db = db
-	return db.Update(func(tx *bolt.Tx) error {
-		_, err := tx.CreateBucketIfNotExists([]byte(versionBucketName))
-		if err != nil {
-			return err
+
+	return store.initServices()
+}
+
+// Init creates the default data set.
+func (store *Store) Init() error {
+	groups, err := store.EndpointGroupService.EndpointGroups()
+	if err != nil {
+		return err
+	}
+
+	if len(groups) == 0 {
+		unassignedGroup := &portainer.EndpointGroup{
+			Name:            "Unassigned",
+			Description:     "Unassigned endpoints",
+			Labels:          []portainer.Pair{},
+			AuthorizedUsers: []portainer.UserID{},
+			AuthorizedTeams: []portainer.TeamID{},
+			Tags:            []string{},
 		}
-		_, err = tx.CreateBucketIfNotExists([]byte(userBucketName))
-		if err != nil {
-			return err
-		}
-		_, err = tx.CreateBucketIfNotExists([]byte(teamBucketName))
-		if err != nil {
-			return err
-		}
-		_, err = tx.CreateBucketIfNotExists([]byte(endpointBucketName))
-		if err != nil {
-			return err
-		}
-		_, err = tx.CreateBucketIfNotExists([]byte(resourceControlBucketName))
-		if err != nil {
-			return err
-		}
-		_, err = tx.CreateBucketIfNotExists([]byte(teamMembershipBucketName))
-		if err != nil {
-			return err
-		}
-		_, err = tx.CreateBucketIfNotExists([]byte(settingsBucketName))
-		if err != nil {
-			return err
-		}
-		return nil
-	})
+
+		return store.EndpointGroupService.CreateEndpointGroup(unassignedGroup)
+	}
+
+	return nil
 }
 
 // Close closes the BoltDB database.
@@ -123,28 +117,120 @@ func (store *Store) Close() error {
 // MigrateData automatically migrate the data based on the DBVersion.
 func (store *Store) MigrateData() error {
 	if !store.checkForDataMigration {
-		err := store.VersionService.StoreDBVersion(portainer.DBVersion)
-		if err != nil {
-			return err
-		}
-		return nil
+		return store.VersionService.StoreDBVersion(portainer.DBVersion)
 	}
 
 	version, err := store.VersionService.DBVersion()
-	if err == portainer.ErrDBVersionNotFound {
+	if err == portainer.ErrObjectNotFound {
 		version = 0
 	} else if err != nil {
 		return err
 	}
 
 	if version < portainer.DBVersion {
+		migratorParams := &migrator.Parameters{
+			DB:                     store.db,
+			DatabaseVersion:        version,
+			EndpointGroupService:   store.EndpointGroupService,
+			EndpointService:        store.EndpointService,
+			ResourceControlService: store.ResourceControlService,
+			SettingsService:        store.SettingsService,
+			StackService:           store.StackService,
+			UserService:            store.UserService,
+			VersionService:         store.VersionService,
+			FileService:            store.fileService,
+		}
+		migrator := migrator.NewMigrator(migratorParams)
+
 		log.Printf("Migrating database from version %v to %v.\n", version, portainer.DBVersion)
-		migrator := NewMigrator(store, version)
 		err = migrator.Migrate()
 		if err != nil {
+			log.Printf("An error occurred during database migration: %s\n", err)
 			return err
 		}
 	}
 
 	return nil
 }
+
+func (store *Store) initServices() error {
+	dockerhubService, err := dockerhub.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.DockerHubService = dockerhubService
+
+	endpointgroupService, err := endpointgroup.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.EndpointGroupService = endpointgroupService
+
+	endpointService, err := endpoint.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.EndpointService = endpointService
+
+	registryService, err := registry.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.RegistryService = registryService
+
+	resourcecontrolService, err := resourcecontrol.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.ResourceControlService = resourcecontrolService
+
+	settingsService, err := settings.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.SettingsService = settingsService
+
+	stackService, err := stack.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.StackService = stackService
+
+	tagService, err := tag.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.TagService = tagService
+
+	teammembershipService, err := teammembership.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.TeamMembershipService = teammembershipService
+
+	teamService, err := team.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.TeamService = teamService
+
+	templateService, err := template.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.TemplateService = templateService
+
+	userService, err := user.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.UserService = userService
+
+	versionService, err := version.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.VersionService = versionService
+
+	return nil
+}

api/bolt/dockerhub/dockerhub.go

@@ -0,0 +1,48 @@
+package dockerhub
+
+import (
+	"github.com/portainer/portainer"
+	"github.com/portainer/portainer/bolt/internal"
+
+	"github.com/boltdb/bolt"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName   = "dockerhub"
+	dockerHubKey = "DOCKERHUB"
+)
+
+// Service represents a service for managing Dockerhub data.
+type Service struct {
+	db *bolt.DB
+}
+
+// NewService creates a new instance of a service.
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		db: db,
+	}, nil
+}
+
+// DockerHub returns the DockerHub object.
+func (service *Service) DockerHub() (*portainer.DockerHub, error) {
+	var dockerhub portainer.DockerHub
+
+	err := internal.GetObject(service.db, BucketName, []byte(dockerHubKey), &dockerhub)
+	if err != nil {
+		return nil, err
+	}
+
+	return &dockerhub, nil
+}
+
+// UpdateDockerHub updates a DockerHub object.
+func (service *Service) UpdateDockerHub(dockerhub *portainer.DockerHub) error {
+	return internal.UpdateObject(service.db, BucketName, []byte(dockerHubKey), dockerhub)
+}

api/bolt/endpoint/endpoint.go

@@ -0,0 +1,146 @@
+package endpoint
+
+import (
+	"github.com/portainer/portainer"
+	"github.com/portainer/portainer/bolt/internal"
+
+	"github.com/boltdb/bolt"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "endpoints"
+)
+
+// Service represents a service for managing endpoint data.
+type Service struct {
+	db *bolt.DB
+}
+
+// NewService creates a new instance of a service.
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		db: db,
+	}, nil
+}
+
+// Endpoint returns an endpoint by ID.
+func (service *Service) Endpoint(ID portainer.EndpointID) (*portainer.Endpoint, error) {
+	var endpoint portainer.Endpoint
+	identifier := internal.Itob(int(ID))
+
+	err := internal.GetObject(service.db, BucketName, identifier, &endpoint)
+	if err != nil {
+		return nil, err
+	}
+
+	return &endpoint, nil
+}
+
+// UpdateEndpoint updates an endpoint.
+func (service *Service) UpdateEndpoint(ID portainer.EndpointID, endpoint *portainer.Endpoint) error {
+	identifier := internal.Itob(int(ID))
+	return internal.UpdateObject(service.db, BucketName, identifier, endpoint)
+}
+
+// DeleteEndpoint deletes an endpoint.
+func (service *Service) DeleteEndpoint(ID portainer.EndpointID) error {
+	identifier := internal.Itob(int(ID))
+	return internal.DeleteObject(service.db, BucketName, identifier)
+}
+
+// Endpoints return an array containing all the endpoints.
+func (service *Service) Endpoints() ([]portainer.Endpoint, error) {
+	var endpoints = make([]portainer.Endpoint, 0)
+
+	err := service.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var endpoint portainer.Endpoint
+			err := internal.UnmarshalObject(v, &endpoint)
+			if err != nil {
+				return err
+			}
+			endpoints = append(endpoints, endpoint)
+		}
+
+		return nil
+	})
+
+	return endpoints, err
+}
+
+// CreateEndpoint assign an ID to a new endpoint and saves it.
+func (service *Service) CreateEndpoint(endpoint *portainer.Endpoint) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		// We manually manage sequences for endpoints
+		err := bucket.SetSequence(uint64(endpoint.ID))
+		if err != nil {
+			return err
+		}
+
+		data, err := internal.MarshalObject(endpoint)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(endpoint.ID)), data)
+	})
+}
+
+// GetNextIdentifier returns the next identifier for an endpoint.
+func (service *Service) GetNextIdentifier() int {
+	return internal.GetNextIdentifier(service.db, BucketName)
+}
+
+// Synchronize creates, updates and deletes endpoints inside a single transaction.
+func (service *Service) Synchronize(toCreate, toUpdate, toDelete []*portainer.Endpoint) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		for _, endpoint := range toCreate {
+			id, _ := bucket.NextSequence()
+			endpoint.ID = portainer.EndpointID(id)
+
+			data, err := internal.MarshalObject(endpoint)
+			if err != nil {
+				return err
+			}
+
+			err = bucket.Put(internal.Itob(int(endpoint.ID)), data)
+			if err != nil {
+				return err
+			}
+		}
+
+		for _, endpoint := range toUpdate {
+			data, err := internal.MarshalObject(endpoint)
+			if err != nil {
+				return err
+			}
+
+			err = bucket.Put(internal.Itob(int(endpoint.ID)), data)
+			if err != nil {
+				return err
+			}
+		}
+
+		for _, endpoint := range toDelete {
+			err := bucket.Delete(internal.Itob(int(endpoint.ID)))
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+}

api/bolt/endpoint_service.go

@@ -1,154 +0,0 @@
-package bolt
-
-import (
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/internal"
-
-	"github.com/boltdb/bolt"
-)
-
-// EndpointService represents a service for managing users.
-type EndpointService struct {
-	store *Store
-}
-
-// Endpoint returns an endpoint by ID.
-func (service *EndpointService) Endpoint(ID portainer.EndpointID) (*portainer.Endpoint, error) {
-	var data []byte
-	err := service.store.db.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(endpointBucketName))
-		value := bucket.Get(internal.Itob(int(ID)))
-		if value == nil {
-			return portainer.ErrEndpointNotFound
-		}
-
-		data = make([]byte, len(value))
-		copy(data, value)
-		return nil
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	var endpoint portainer.Endpoint
-	err = internal.UnmarshalEndpoint(data, &endpoint)
-	if err != nil {
-		return nil, err
-	}
-	return &endpoint, nil
-}
-
-// Endpoints return an array containing all the endpoints.
-func (service *EndpointService) Endpoints() ([]portainer.Endpoint, error) {
-	var endpoints = make([]portainer.Endpoint, 0)
-	err := service.store.db.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(endpointBucketName))
-
-		cursor := bucket.Cursor()
-		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
-			var endpoint portainer.Endpoint
-			err := internal.UnmarshalEndpoint(v, &endpoint)
-			if err != nil {
-				return err
-			}
-			endpoints = append(endpoints, endpoint)
-		}
-
-		return nil
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	return endpoints, nil
-}
-
-// Synchronize creates, updates and deletes endpoints inside a single transaction.
-func (service *EndpointService) Synchronize(toCreate, toUpdate, toDelete []*portainer.Endpoint) error {
-	return service.store.db.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(endpointBucketName))
-
-		for _, endpoint := range toCreate {
-			err := storeNewEndpoint(endpoint, bucket)
-			if err != nil {
-				return err
-			}
-		}
-
-		for _, endpoint := range toUpdate {
-			err := marshalAndStoreEndpoint(endpoint, bucket)
-			if err != nil {
-				return err
-			}
-		}
-
-		for _, endpoint := range toDelete {
-			err := bucket.Delete(internal.Itob(int(endpoint.ID)))
-			if err != nil {
-				return err
-			}
-		}
-
-		return nil
-	})
-}
-
-// CreateEndpoint assign an ID to a new endpoint and saves it.
-func (service *EndpointService) CreateEndpoint(endpoint *portainer.Endpoint) error {
-	return service.store.db.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(endpointBucketName))
-		err := storeNewEndpoint(endpoint, bucket)
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-}
-
-// UpdateEndpoint updates an endpoint.
-func (service *EndpointService) UpdateEndpoint(ID portainer.EndpointID, endpoint *portainer.Endpoint) error {
-	data, err := internal.MarshalEndpoint(endpoint)
-	if err != nil {
-		return err
-	}
-
-	return service.store.db.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(endpointBucketName))
-		err = bucket.Put(internal.Itob(int(ID)), data)
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-}
-
-// DeleteEndpoint deletes an endpoint.
-func (service *EndpointService) DeleteEndpoint(ID portainer.EndpointID) error {
-	return service.store.db.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(endpointBucketName))
-		err := bucket.Delete(internal.Itob(int(ID)))
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-}
-
-func marshalAndStoreEndpoint(endpoint *portainer.Endpoint, bucket *bolt.Bucket) error {
-	data, err := internal.MarshalEndpoint(endpoint)
-	if err != nil {
-		return err
-	}
-
-	err = bucket.Put(internal.Itob(int(endpoint.ID)), data)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func storeNewEndpoint(endpoint *portainer.Endpoint, bucket *bolt.Bucket) error {
-	id, _ := bucket.NextSequence()
-	endpoint.ID = portainer.EndpointID(id)
-	return marshalAndStoreEndpoint(endpoint, bucket)
-}

api/bolt/endpointgroup/endpointgroup.go

@@ -0,0 +1,95 @@
+package endpointgroup
+
+import (
+	"github.com/portainer/portainer"
+	"github.com/portainer/portainer/bolt/internal"
+
+	"github.com/boltdb/bolt"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "endpoint_groups"
+)
+
+// Service represents a service for managing endpoint data.
+type Service struct {
+	db *bolt.DB
+}
+
+// NewService creates a new instance of a service.
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		db: db,
+	}, nil
+}
+
+// EndpointGroup returns an endpoint group by ID.
+func (service *Service) EndpointGroup(ID portainer.EndpointGroupID) (*portainer.EndpointGroup, error) {
+	var endpointGroup portainer.EndpointGroup
+	identifier := internal.Itob(int(ID))
+
+	err := internal.GetObject(service.db, BucketName, identifier, &endpointGroup)
+	if err != nil {
+		return nil, err
+	}
+
+	return &endpointGroup, nil
+}
+
+// UpdateEndpointGroup updates an endpoint group.
+func (service *Service) UpdateEndpointGroup(ID portainer.EndpointGroupID, endpointGroup *portainer.EndpointGroup) error {
+	identifier := internal.Itob(int(ID))
+	return internal.UpdateObject(service.db, BucketName, identifier, endpointGroup)
+}
+
+// DeleteEndpointGroup deletes an endpoint group.
+func (service *Service) DeleteEndpointGroup(ID portainer.EndpointGroupID) error {
+	identifier := internal.Itob(int(ID))
+	return internal.DeleteObject(service.db, BucketName, identifier)
+}
+
+// EndpointGroups return an array containing all the endpoint groups.
+func (service *Service) EndpointGroups() ([]portainer.EndpointGroup, error) {
+	var endpointGroups = make([]portainer.EndpointGroup, 0)
+
+	err := service.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var endpointGroup portainer.EndpointGroup
+			err := internal.UnmarshalObject(v, &endpointGroup)
+			if err != nil {
+				return err
+			}
+			endpointGroups = append(endpointGroups, endpointGroup)
+		}
+
+		return nil
+	})
+
+	return endpointGroups, err
+}
+
+// CreateEndpointGroup assign an ID to a new endpoint group and saves it.
+func (service *Service) CreateEndpointGroup(endpointGroup *portainer.EndpointGroup) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		id, _ := bucket.NextSequence()
+		endpointGroup.ID = portainer.EndpointGroupID(id)
+
+		data, err := internal.MarshalObject(endpointGroup)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(endpointGroup.ID)), data)
+	})
+}

api/bolt/internal/db.go

@@ -0,0 +1,94 @@
+package internal
+
+import (
+	"encoding/binary"
+
+	"github.com/boltdb/bolt"
+	"github.com/portainer/portainer"
+)
+
+// Itob returns an 8-byte big endian representation of v.
+// This function is typically used for encoding integer IDs to byte slices
+// so that they can be used as BoltDB keys.
+func Itob(v int) []byte {
+	b := make([]byte, 8)
+	binary.BigEndian.PutUint64(b, uint64(v))
+	return b
+}
+
+// CreateBucket is a generic function used to create a bucket inside a bolt database.
+func CreateBucket(db *bolt.DB, bucketName string) error {
+	return db.Update(func(tx *bolt.Tx) error {
+		_, err := tx.CreateBucketIfNotExists([]byte(bucketName))
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+// GetObject is a generic function used to retrieve an unmarshalled object from a bolt database.
+func GetObject(db *bolt.DB, bucketName string, key []byte, object interface{}) error {
+	var data []byte
+
+	err := db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(bucketName))
+
+		value := bucket.Get(key)
+		if value == nil {
+			return portainer.ErrObjectNotFound
+		}
+
+		data = make([]byte, len(value))
+		copy(data, value)
+
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	return UnmarshalObject(data, object)
+}
+
+// UpdateObject is a generic function used to update an object inside a bolt database.
+func UpdateObject(db *bolt.DB, bucketName string, key []byte, object interface{}) error {
+	return db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(bucketName))
+
+		data, err := MarshalObject(object)
+		if err != nil {
+			return err
+		}
+
+		err = bucket.Put(key, data)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+// DeleteObject is a generic function used to delete an object inside a bolt database.
+func DeleteObject(db *bolt.DB, bucketName string, key []byte) error {
+	return db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(bucketName))
+		return bucket.Delete(key)
+	})
+}
+
+// GetNextIdentifier is a generic function that returns the specified bucket identifier incremented by 1.
+func GetNextIdentifier(db *bolt.DB, bucketName string) int {
+	var identifier int
+
+	db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(bucketName))
+		id := bucket.Sequence()
+		identifier = int(id)
+		return nil
+	})
+
+	identifier++
+	return identifier
+}

api/bolt/internal/internal.go

@@ -1,77 +0,0 @@
-package internal
-
-import (
-	"github.com/portainer/portainer"
-
-	"encoding/binary"
-	"encoding/json"
-)
-
-// MarshalUser encodes a user to binary format.
-func MarshalUser(user *portainer.User) ([]byte, error) {
-	return json.Marshal(user)
-}
-
-// UnmarshalUser decodes a user from a binary data.
-func UnmarshalUser(data []byte, user *portainer.User) error {
-	return json.Unmarshal(data, user)
-}
-
-// MarshalTeam encodes a team to binary format.
-func MarshalTeam(team *portainer.Team) ([]byte, error) {
-	return json.Marshal(team)
-}
-
-// UnmarshalTeam decodes a team from a binary data.
-func UnmarshalTeam(data []byte, team *portainer.Team) error {
-	return json.Unmarshal(data, team)
-}
-
-// MarshalTeamMembership encodes a team membership to binary format.
-func MarshalTeamMembership(membership *portainer.TeamMembership) ([]byte, error) {
-	return json.Marshal(membership)
-}
-
-// UnmarshalTeamMembership decodes a team membership from a binary data.
-func UnmarshalTeamMembership(data []byte, membership *portainer.TeamMembership) error {
-	return json.Unmarshal(data, membership)
-}
-
-// MarshalEndpoint encodes an endpoint to binary format.
-func MarshalEndpoint(endpoint *portainer.Endpoint) ([]byte, error) {
-	return json.Marshal(endpoint)
-}
-
-// UnmarshalEndpoint decodes an endpoint from a binary data.
-func UnmarshalEndpoint(data []byte, endpoint *portainer.Endpoint) error {
-	return json.Unmarshal(data, endpoint)
-}
-
-// MarshalResourceControl encodes a resource control object to binary format.
-func MarshalResourceControl(rc *portainer.ResourceControl) ([]byte, error) {
-	return json.Marshal(rc)
-}
-
-// UnmarshalResourceControl decodes a resource control object from a binary data.
-func UnmarshalResourceControl(data []byte, rc *portainer.ResourceControl) error {
-	return json.Unmarshal(data, rc)
-}
-
-// MarshalSettings encodes a settings object to binary format.
-func MarshalSettings(settings *portainer.Settings) ([]byte, error) {
-	return json.Marshal(settings)
-}
-
-// UnmarshalSettings decodes a settings object from a binary data.
-func UnmarshalSettings(data []byte, settings *portainer.Settings) error {
-	return json.Unmarshal(data, settings)
-}
-
-// Itob returns an 8-byte big endian representation of v.
-// This function is typically used for encoding integer IDs to byte slices
-// so that they can be used as BoltDB keys.
-func Itob(v int) []byte {
-	b := make([]byte, 8)
-	binary.BigEndian.PutUint64(b, uint64(v))
-	return b
-}

api/bolt/internal/json.go

@@ -0,0 +1,15 @@
+package internal
+
+import (
+	"encoding/json"
+)
+
+// MarshalObject encodes an object to binary format
+func MarshalObject(object interface{}) ([]byte, error) {
+	return json.Marshal(object)
+}
+
+// UnmarshalObject decodes an object from binary data
+func UnmarshalObject(data []byte, object interface{}) error {
+	return json.Unmarshal(data, object)
+}

api/bolt/migrator.go

@@ -1,55 +0,0 @@
-package bolt
-
-import "github.com/portainer/portainer"
-
-// Migrator defines a service to migrate data after a Portainer version update.
-type Migrator struct {
-	UserService            *UserService
-	EndpointService        *EndpointService
-	ResourceControlService *ResourceControlService
-	VersionService         *VersionService
-	CurrentDBVersion       int
-	store                  *Store
-}
-
-// NewMigrator creates a new Migrator.
-func NewMigrator(store *Store, version int) *Migrator {
-	return &Migrator{
-		UserService:            store.UserService,
-		EndpointService:        store.EndpointService,
-		ResourceControlService: store.ResourceControlService,
-		VersionService:         store.VersionService,
-		CurrentDBVersion:       version,
-		store:                  store,
-	}
-}
-
-// Migrate checks the database version and migrate the existing data to the most recent data model.
-func (m *Migrator) Migrate() error {
-
-	// Portainer < 1.12
-	if m.CurrentDBVersion == 0 {
-		err := m.updateAdminUserToDBVersion1()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Portainer 1.12.x
-	if m.CurrentDBVersion == 1 {
-		err := m.updateResourceControlsToDBVersion2()
-		if err != nil {
-			return err
-		}
-		err = m.updateEndpointsToDBVersion2()
-		if err != nil {
-			return err
-		}
-	}
-
-	err := m.VersionService.StoreDBVersion(portainer.DBVersion)
-	if err != nil {
-		return err
-	}
-	return nil
-}

api/bolt/migrator/migrate_dbversion0.go

@@ -1,19 +1,20 @@
-package bolt
+package migrator
 
 import (
 	"github.com/boltdb/bolt"
 	"github.com/portainer/portainer"
+	"github.com/portainer/portainer/bolt/user"
 )
 
 func (m *Migrator) updateAdminUserToDBVersion1() error {
-	u, err := m.UserService.UserByUsername("admin")
+	u, err := m.userService.UserByUsername("admin")
 	if err == nil {
 		admin := &portainer.User{
 			Username: "admin",
 			Password: u.Password,
 			Role:     portainer.AdministratorRole,
 		}
-		err = m.UserService.CreateUser(admin)
+		err = m.userService.CreateUser(admin)
 		if err != nil {
 			return err
 		}
@@ -21,19 +22,15 @@ func (m *Migrator) updateAdminUserToDBVersion1() error {
 		if err != nil {
 			return err
 		}
-	} else if err != nil && err != portainer.ErrUserNotFound {
+	} else if err != nil && err != portainer.ErrObjectNotFound {
 		return err
 	}
 	return nil
 }
 
 func (m *Migrator) removeLegacyAdminUser() error {
-	return m.store.db.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(userBucketName))
-		err := bucket.Delete([]byte("admin"))
-		if err != nil {
-			return err
-		}
-		return nil
+	return m.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(user.BucketName))
+		return bucket.Delete([]byte("admin"))
 	})
 }

api/bolt/migrator/migrate_dbversion1.go

@@ -1,4 +1,4 @@
-package bolt
+package migrator
 
 import (
 	"github.com/boltdb/bolt"
@@ -16,7 +16,7 @@ func (m *Migrator) updateResourceControlsToDBVersion2() error {
 		resourceControl.SubResourceIDs = []string{}
 		resourceControl.TeamAccesses = []portainer.TeamResourceAccess{}
 
-		owner, err := m.UserService.User(resourceControl.OwnerID)
+		owner, err := m.userService.User(resourceControl.OwnerID)
 		if err != nil {
 			return err
 		}
@@ -33,7 +33,7 @@ func (m *Migrator) updateResourceControlsToDBVersion2() error {
 			resourceControl.UserAccesses = []portainer.UserResourceAccess{userAccess}
 		}
 
-		err = m.ResourceControlService.CreateResourceControl(&resourceControl)
+		err = m.resourceControlService.CreateResourceControl(&resourceControl)
 		if err != nil {
 			return err
 		}
@@ -43,14 +43,14 @@ func (m *Migrator) updateResourceControlsToDBVersion2() error {
 }
 
 func (m *Migrator) updateEndpointsToDBVersion2() error {
-	legacyEndpoints, err := m.EndpointService.Endpoints()
+	legacyEndpoints, err := m.endpointService.Endpoints()
 	if err != nil {
 		return err
 	}
 
 	for _, endpoint := range legacyEndpoints {
 		endpoint.AuthorizedTeams = []portainer.TeamID{}
-		err = m.EndpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+		err = m.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
 		if err != nil {
 			return err
 		}
@@ -61,12 +61,12 @@ func (m *Migrator) updateEndpointsToDBVersion2() error {
 
 func (m *Migrator) retrieveLegacyResourceControls() ([]portainer.ResourceControl, error) {
 	legacyResourceControls := make([]portainer.ResourceControl, 0)
-	err := m.store.db.View(func(tx *bolt.Tx) error {
+	err := m.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte("containerResourceControl"))
 		cursor := bucket.Cursor()
 		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
 			var resourceControl portainer.ResourceControl
-			err := internal.UnmarshalResourceControl(v, &resourceControl)
+			err := internal.UnmarshalObject(v, &resourceControl)
 			if err != nil {
 				return err
 			}
@@ -78,7 +78,7 @@ func (m *Migrator) retrieveLegacyResourceControls() ([]portainer.ResourceControl
 		cursor = bucket.Cursor()
 		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
 			var resourceControl portainer.ResourceControl
-			err := internal.UnmarshalResourceControl(v, &resourceControl)
+			err := internal.UnmarshalObject(v, &resourceControl)
 			if err != nil {
 				return err
 			}
@@ -90,7 +90,7 @@ func (m *Migrator) retrieveLegacyResourceControls() ([]portainer.ResourceControl
 		cursor = bucket.Cursor()
 		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
 			var resourceControl portainer.ResourceControl
-			err := internal.UnmarshalResourceControl(v, &resourceControl)
+			err := internal.UnmarshalObject(v, &resourceControl)
 			if err != nil {
 				return err
 			}

api/bolt/migrator/migrate_dbversion10.go

@@ -0,0 +1,28 @@
+package migrator
+
+import "github.com/portainer/portainer"
+
+func (m *Migrator) updateEndpointsToVersion11() error {
+	legacyEndpoints, err := m.endpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, endpoint := range legacyEndpoints {
+		if endpoint.Type == portainer.AgentOnDockerEnvironment {
+			endpoint.TLSConfig.TLS = true
+			endpoint.TLSConfig.TLSSkipVerify = true
+		} else {
+			if endpoint.TLSConfig.TLSSkipVerify && !endpoint.TLSConfig.TLS {
+				endpoint.TLSConfig.TLSSkipVerify = false
+			}
+		}
+
+		err = m.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

api/bolt/migrator/migrate_dbversion11.go

@@ -0,0 +1,127 @@
+package migrator
+
+import (
+	"strconv"
+	"strings"
+
+	"github.com/boltdb/bolt"
+	"github.com/portainer/portainer"
+	"github.com/portainer/portainer/bolt/internal"
+	"github.com/portainer/portainer/bolt/stack"
+)
+
+func (m *Migrator) updateEndpointsToVersion12() error {
+	legacyEndpoints, err := m.endpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, endpoint := range legacyEndpoints {
+		endpoint.Tags = []string{}
+
+		err = m.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (m *Migrator) updateEndpointGroupsToVersion12() error {
+	legacyEndpointGroups, err := m.endpointGroupService.EndpointGroups()
+	if err != nil {
+		return err
+	}
+
+	for _, group := range legacyEndpointGroups {
+		group.Tags = []string{}
+
+		err = m.endpointGroupService.UpdateEndpointGroup(group.ID, &group)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+type legacyStack struct {
+	ID          string               `json:"Id"`
+	Name        string               `json:"Name"`
+	EndpointID  portainer.EndpointID `json:"EndpointId"`
+	SwarmID     string               `json:"SwarmId"`
+	EntryPoint  string               `json:"EntryPoint"`
+	Env         []portainer.Pair     `json:"Env"`
+	ProjectPath string
+}
+
+func (m *Migrator) updateStacksToVersion12() error {
+	legacyStacks, err := m.retrieveLegacyStacks()
+	if err != nil {
+		return err
+	}
+
+	for _, legacyStack := range legacyStacks {
+		err := m.convertLegacyStack(&legacyStack)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (m *Migrator) convertLegacyStack(s *legacyStack) error {
+	stackID := m.stackService.GetNextIdentifier()
+	stack := &portainer.Stack{
+		ID:         portainer.StackID(stackID),
+		Name:       s.Name,
+		Type:       portainer.DockerSwarmStack,
+		SwarmID:    s.SwarmID,
+		EndpointID: 0,
+		EntryPoint: s.EntryPoint,
+		Env:        s.Env,
+	}
+
+	stack.ProjectPath = strings.Replace(s.ProjectPath, s.ID, strconv.Itoa(stackID), 1)
+	err := m.fileService.Rename(s.ProjectPath, stack.ProjectPath)
+	if err != nil {
+		return err
+	}
+
+	err = m.deleteLegacyStack(s.ID)
+	if err != nil {
+		return err
+	}
+
+	return m.stackService.CreateStack(stack)
+}
+
+func (m *Migrator) deleteLegacyStack(legacyID string) error {
+	return m.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(stack.BucketName))
+		return bucket.Delete([]byte(legacyID))
+	})
+}
+
+func (m *Migrator) retrieveLegacyStacks() ([]legacyStack, error) {
+	var legacyStacks = make([]legacyStack, 0)
+	err := m.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(stack.BucketName))
+		cursor := bucket.Cursor()
+
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var stack legacyStack
+			err := internal.UnmarshalObject(v, &stack)
+			if err != nil {
+				return err
+			}
+			legacyStacks = append(legacyStacks, stack)
+		}
+
+		return nil
+	})
+
+	return legacyStacks, err
+}

api/bolt/migrator/migrate_dbversion12.go

@@ -0,0 +1,17 @@
+package migrator
+
+import "github.com/portainer/portainer"
+
+func (m *Migrator) updateSettingsToVersion13() error {
+	legacySettings, err := m.settingsService.Settings()
+	if err != nil {
+		return err
+	}
+
+	legacySettings.LDAPSettings.AutoCreateUsers = false
+	legacySettings.LDAPSettings.GroupSearchSettings = []portainer.LDAPGroupSearchSettings{
+		portainer.LDAPGroupSearchSettings{},
+	}
+
+	return m.settingsService.UpdateSettings(legacySettings)
+}

api/bolt/migrator/migrate_dbversion2.go

@@ -0,0 +1,20 @@
+package migrator
+
+import "github.com/portainer/portainer"
+
+func (m *Migrator) updateSettingsToDBVersion3() error {
+	legacySettings, err := m.settingsService.Settings()
+	if err != nil {
+		return err
+	}
+
+	legacySettings.AuthenticationMethod = portainer.AuthenticationInternal
+	legacySettings.LDAPSettings = portainer.LDAPSettings{
+		TLSConfig: portainer.TLSConfiguration{},
+		SearchSettings: []portainer.LDAPSearchSettings{
+			portainer.LDAPSearchSettings{},
+		},
+	}
+
+	return m.settingsService.UpdateSettings(legacySettings)
+}

api/bolt/migrator/migrate_dbversion3.go

@@ -0,0 +1,28 @@
+package migrator
+
+import "github.com/portainer/portainer"
+
+func (m *Migrator) updateEndpointsToDBVersion4() error {
+	legacyEndpoints, err := m.endpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, endpoint := range legacyEndpoints {
+		endpoint.TLSConfig = portainer.TLSConfiguration{}
+		if endpoint.TLS {
+			endpoint.TLSConfig.TLS = true
+			endpoint.TLSConfig.TLSSkipVerify = false
+			endpoint.TLSConfig.TLSCACertPath = endpoint.TLSCACertPath
+			endpoint.TLSConfig.TLSCertPath = endpoint.TLSCertPath
+			endpoint.TLSConfig.TLSKeyPath = endpoint.TLSKeyPath
+		}
+
+		err = m.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

api/bolt/migrator/migrate_dbversion4.go

@@ -0,0 +1,11 @@
+package migrator
+
+func (m *Migrator) updateSettingsToVersion5() error {
+	legacySettings, err := m.settingsService.Settings()
+	if err != nil {
+		return err
+	}
+
+	legacySettings.AllowBindMountsForRegularUsers = true
+	return m.settingsService.UpdateSettings(legacySettings)
+}

api/bolt/migrator/migrate_dbversion5.go

@@ -0,0 +1,11 @@
+package migrator
+
+func (m *Migrator) updateSettingsToVersion6() error {
+	legacySettings, err := m.settingsService.Settings()
+	if err != nil {
+		return err
+	}
+
+	legacySettings.AllowPrivilegedModeForRegularUsers = true
+	return m.settingsService.UpdateSettings(legacySettings)
+}

api/bolt/migrator/migrate_dbversion6.go

@@ -0,0 +1,11 @@
+package migrator
+
+func (m *Migrator) updateSettingsToVersion7() error {
+	legacySettings, err := m.settingsService.Settings()
+	if err != nil {
+		return err
+	}
+	legacySettings.DisplayDonationHeader = true
+
+	return m.settingsService.UpdateSettings(legacySettings)
+}

api/bolt/migrator/migrate_dbversion7.go

@@ -0,0 +1,20 @@
+package migrator
+
+import "github.com/portainer/portainer"
+
+func (m *Migrator) updateEndpointsToVersion8() error {
+	legacyEndpoints, err := m.endpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, endpoint := range legacyEndpoints {
+		endpoint.Extensions = []portainer.EndpointExtension{}
+		err = m.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

api/bolt/migrator/migrate_dbversion8.go

@@ -0,0 +1,20 @@
+package migrator
+
+import "github.com/portainer/portainer"
+
+func (m *Migrator) updateEndpointsToVersion9() error {
+	legacyEndpoints, err := m.endpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, endpoint := range legacyEndpoints {
+		endpoint.GroupID = portainer.EndpointGroupID(1)
+		err = m.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

api/bolt/migrator/migrate_dbversion9.go

@@ -0,0 +1,20 @@
+package migrator
+
+import "github.com/portainer/portainer"
+
+func (m *Migrator) updateEndpointsToVersion10() error {
+	legacyEndpoints, err := m.endpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, endpoint := range legacyEndpoints {
+		endpoint.Type = portainer.DockerEnvironment
+		err = m.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

api/bolt/migrator/migrator.go

@@ -0,0 +1,182 @@
+package migrator
+
+import (
+	"github.com/boltdb/bolt"
+	"github.com/portainer/portainer"
+	"github.com/portainer/portainer/bolt/endpoint"
+	"github.com/portainer/portainer/bolt/endpointgroup"
+	"github.com/portainer/portainer/bolt/resourcecontrol"
+	"github.com/portainer/portainer/bolt/settings"
+	"github.com/portainer/portainer/bolt/stack"
+	"github.com/portainer/portainer/bolt/user"
+	"github.com/portainer/portainer/bolt/version"
+)
+
+type (
+	// Migrator defines a service to migrate data after a Portainer version update.
+	Migrator struct {
+		currentDBVersion       int
+		db                     *bolt.DB
+		endpointGroupService   *endpointgroup.Service
+		endpointService        *endpoint.Service
+		resourceControlService *resourcecontrol.Service
+		settingsService        *settings.Service
+		stackService           *stack.Service
+		userService            *user.Service
+		versionService         *version.Service
+		fileService            portainer.FileService
+	}
+
+	// Parameters represents the required parameters to create a new Migrator instance.
+	Parameters struct {
+		DB                     *bolt.DB
+		DatabaseVersion        int
+		EndpointGroupService   *endpointgroup.Service
+		EndpointService        *endpoint.Service
+		ResourceControlService *resourcecontrol.Service
+		SettingsService        *settings.Service
+		StackService           *stack.Service
+		UserService            *user.Service
+		VersionService         *version.Service
+		FileService            portainer.FileService
+	}
+)
+
+// NewMigrator creates a new Migrator.
+func NewMigrator(parameters *Parameters) *Migrator {
+	return &Migrator{
+		db:                     parameters.DB,
+		currentDBVersion:       parameters.DatabaseVersion,
+		endpointGroupService:   parameters.EndpointGroupService,
+		endpointService:        parameters.EndpointService,
+		resourceControlService: parameters.ResourceControlService,
+		settingsService:        parameters.SettingsService,
+		stackService:           parameters.StackService,
+		userService:            parameters.UserService,
+		versionService:         parameters.VersionService,
+		fileService:            parameters.FileService,
+	}
+}
+
+// Migrate checks the database version and migrate the existing data to the most recent data model.
+func (m *Migrator) Migrate() error {
+
+	// Portainer < 1.12
+	if m.currentDBVersion < 1 {
+		err := m.updateAdminUserToDBVersion1()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.12.x
+	if m.currentDBVersion < 2 {
+		err := m.updateResourceControlsToDBVersion2()
+		if err != nil {
+			return err
+		}
+		err = m.updateEndpointsToDBVersion2()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.13.x
+	if m.currentDBVersion < 3 {
+		err := m.updateSettingsToDBVersion3()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.14.0
+	if m.currentDBVersion < 4 {
+		err := m.updateEndpointsToDBVersion4()
+		if err != nil {
+			return err
+		}
+	}
+
+	// https://github.com/portainer/portainer/issues/1235
+	if m.currentDBVersion < 5 {
+		err := m.updateSettingsToVersion5()
+		if err != nil {
+			return err
+		}
+	}
+
+	// https://github.com/portainer/portainer/issues/1236
+	if m.currentDBVersion < 6 {
+		err := m.updateSettingsToVersion6()
+		if err != nil {
+			return err
+		}
+	}
+
+	// https://github.com/portainer/portainer/issues/1449
+	if m.currentDBVersion < 7 {
+		err := m.updateSettingsToVersion7()
+		if err != nil {
+			return err
+		}
+	}
+
+	if m.currentDBVersion < 8 {
+		err := m.updateEndpointsToVersion8()
+		if err != nil {
+			return err
+		}
+	}
+
+	// https: //github.com/portainer/portainer/issues/1396
+	if m.currentDBVersion < 9 {
+		err := m.updateEndpointsToVersion9()
+		if err != nil {
+			return err
+		}
+	}
+
+	// https://github.com/portainer/portainer/issues/461
+	if m.currentDBVersion < 10 {
+		err := m.updateEndpointsToVersion10()
+		if err != nil {
+			return err
+		}
+	}
+
+	// https://github.com/portainer/portainer/issues/1906
+	if m.currentDBVersion < 11 {
+		err := m.updateEndpointsToVersion11()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.18.0
+	if m.currentDBVersion < 12 {
+		err := m.updateEndpointsToVersion12()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateEndpointGroupsToVersion12()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateStacksToVersion12()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.19.0
+	if m.currentDBVersion < 13 {
+		err := m.updateSettingsToVersion13()
+		if err != nil {
+			return err
+		}
+	}
+
+	return m.versionService.StoreDBVersion(portainer.DBVersion)
+}

api/bolt/registry/registry.go

@@ -0,0 +1,95 @@
+package registry
+
+import (
+	"github.com/portainer/portainer"
+	"github.com/portainer/portainer/bolt/internal"
+
+	"github.com/boltdb/bolt"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "registries"
+)
+
+// Service represents a service for managing endpoint data.
+type Service struct {
+	db *bolt.DB
+}
+
+// NewService creates a new instance of a service.
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		db: db,
+	}, nil
+}
+
+// Registry returns an registry by ID.
+func (service *Service) Registry(ID portainer.RegistryID) (*portainer.Registry, error) {
+	var registry portainer.Registry
+	identifier := internal.Itob(int(ID))
+
+	err := internal.GetObject(service.db, BucketName, identifier, &registry)
+	if err != nil {
+		return nil, err
+	}
+
+	return &registry, nil
+}
+
+// Registries returns an array containing all the registries.
+func (service *Service) Registries() ([]portainer.Registry, error) {
+	var registries = make([]portainer.Registry, 0)
+
+	err := service.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var registry portainer.Registry
+			err := internal.UnmarshalObject(v, &registry)
+			if err != nil {
+				return err
+			}
+			registries = append(registries, registry)
+		}
+
+		return nil
+	})
+
+	return registries, err
+}
+
+// CreateRegistry creates a new registry.
+func (service *Service) CreateRegistry(registry *portainer.Registry) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		id, _ := bucket.NextSequence()
+		registry.ID = portainer.RegistryID(id)
+
+		data, err := internal.MarshalObject(registry)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(registry.ID)), data)
+	})
+}
+
+// UpdateRegistry updates an registry.
+func (service *Service) UpdateRegistry(ID portainer.RegistryID, registry *portainer.Registry) error {
+	identifier := internal.Itob(int(ID))
+	return internal.UpdateObject(service.db, BucketName, identifier, registry)
+}
+
+// DeleteRegistry deletes an registry.
+func (service *Service) DeleteRegistry(ID portainer.RegistryID) error {
+	identifier := internal.Itob(int(ID))
+	return internal.DeleteObject(service.db, BucketName, identifier)
+}

api/bolt/resource_control_service.go

@@ -1,148 +0,0 @@
-package bolt
-
-import (
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/internal"
-
-	"github.com/boltdb/bolt"
-)
-
-// ResourceControlService represents a service for managing resource controls.
-type ResourceControlService struct {
-	store *Store
-}
-
-// ResourceControl returns a ResourceControl object by ID
-func (service *ResourceControlService) ResourceControl(ID portainer.ResourceControlID) (*portainer.ResourceControl, error) {
-	var data []byte
-	err := service.store.db.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(resourceControlBucketName))
-		value := bucket.Get(internal.Itob(int(ID)))
-		if value == nil {
-			return portainer.ErrResourceControlNotFound
-		}
-
-		data = make([]byte, len(value))
-		copy(data, value)
-		return nil
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	var resourceControl portainer.ResourceControl
-	err = internal.UnmarshalResourceControl(data, &resourceControl)
-	if err != nil {
-		return nil, err
-	}
-	return &resourceControl, nil
-}
-
-// ResourceControlByResourceID returns a ResourceControl object by checking if the resourceID is equal
-// to the main ResourceID or in SubResourceIDs
-func (service *ResourceControlService) ResourceControlByResourceID(resourceID string) (*portainer.ResourceControl, error) {
-	var resourceControl *portainer.ResourceControl
-
-	err := service.store.db.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(resourceControlBucketName))
-		cursor := bucket.Cursor()
-		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
-			var rc portainer.ResourceControl
-			err := internal.UnmarshalResourceControl(v, &rc)
-			if err != nil {
-				return err
-			}
-			if rc.ResourceID == resourceID {
-				resourceControl = &rc
-			}
-			for _, subResourceID := range rc.SubResourceIDs {
-				if subResourceID == resourceID {
-					resourceControl = &rc
-				}
-			}
-		}
-
-		if resourceControl == nil {
-			return portainer.ErrResourceControlNotFound
-		}
-		return nil
-	})
-	if err != nil {
-		return nil, err
-	}
-	return resourceControl, nil
-}
-
-// ResourceControls returns all the ResourceControl objects
-func (service *ResourceControlService) ResourceControls() ([]portainer.ResourceControl, error) {
-	var rcs = make([]portainer.ResourceControl, 0)
-	err := service.store.db.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(resourceControlBucketName))
-
-		cursor := bucket.Cursor()
-		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
-			var resourceControl portainer.ResourceControl
-			err := internal.UnmarshalResourceControl(v, &resourceControl)
-			if err != nil {
-				return err
-			}
-			rcs = append(rcs, resourceControl)
-		}
-
-		return nil
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	return rcs, nil
-}
-
-// CreateResourceControl creates a new ResourceControl object
-func (service *ResourceControlService) CreateResourceControl(resourceControl *portainer.ResourceControl) error {
-	return service.store.db.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(resourceControlBucketName))
-		id, _ := bucket.NextSequence()
-		resourceControl.ID = portainer.ResourceControlID(id)
-		data, err := internal.MarshalResourceControl(resourceControl)
-		if err != nil {
-			return err
-		}
-
-		err = bucket.Put(internal.Itob(int(resourceControl.ID)), data)
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-}
-
-// UpdateResourceControl saves a ResourceControl object.
-func (service *ResourceControlService) UpdateResourceControl(ID portainer.ResourceControlID, resourceControl *portainer.ResourceControl) error {
-	data, err := internal.MarshalResourceControl(resourceControl)
-	if err != nil {
-		return err
-	}
-
-	return service.store.db.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(resourceControlBucketName))
-		err = bucket.Put(internal.Itob(int(ID)), data)
-
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-}
-
-// DeleteResourceControl deletes a ResourceControl object by ID
-func (service *ResourceControlService) DeleteResourceControl(ID portainer.ResourceControlID) error {
-	return service.store.db.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(resourceControlBucketName))
-		err := bucket.Delete(internal.Itob(int(ID)))
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-}

api/bolt/resourcecontrol/resourcecontrol.go

@@ -0,0 +1,134 @@
+package resourcecontrol
+
+import (
+	"github.com/portainer/portainer"
+	"github.com/portainer/portainer/bolt/internal"
+
+	"github.com/boltdb/bolt"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "resource_control"
+)
+
+// Service represents a service for managing endpoint data.
+type Service struct {
+	db *bolt.DB
+}
+
+// NewService creates a new instance of a service.
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		db: db,
+	}, nil
+}
+
+// ResourceControl returns a ResourceControl object by ID
+func (service *Service) ResourceControl(ID portainer.ResourceControlID) (*portainer.ResourceControl, error) {
+	var resourceControl portainer.ResourceControl
+	identifier := internal.Itob(int(ID))
+
+	err := internal.GetObject(service.db, BucketName, identifier, &resourceControl)
+	if err != nil {
+		return nil, err
+	}
+
+	return &resourceControl, nil
+}
+
+// ResourceControlByResourceID returns a ResourceControl object by checking if the resourceID is equal
+// to the main ResourceID or in SubResourceIDs
+func (service *Service) ResourceControlByResourceID(resourceID string) (*portainer.ResourceControl, error) {
+	var resourceControl *portainer.ResourceControl
+
+	err := service.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+		cursor := bucket.Cursor()
+
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var rc portainer.ResourceControl
+			err := internal.UnmarshalObject(v, &rc)
+			if err != nil {
+				return err
+			}
+
+			if rc.ResourceID == resourceID {
+				resourceControl = &rc
+				break
+			}
+
+			for _, subResourceID := range rc.SubResourceIDs {
+				if subResourceID == resourceID {
+					resourceControl = &rc
+					break
+				}
+			}
+		}
+
+		if resourceControl == nil {
+			return portainer.ErrObjectNotFound
+		}
+
+		return nil
+	})
+
+	return resourceControl, err
+}
+
+// ResourceControls returns all the ResourceControl objects
+func (service *Service) ResourceControls() ([]portainer.ResourceControl, error) {
+	var rcs = make([]portainer.ResourceControl, 0)
+
+	err := service.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var resourceControl portainer.ResourceControl
+			err := internal.UnmarshalObject(v, &resourceControl)
+			if err != nil {
+				return err
+			}
+			rcs = append(rcs, resourceControl)
+		}
+
+		return nil
+	})
+
+	return rcs, err
+}
+
+// CreateResourceControl creates a new ResourceControl object
+func (service *Service) CreateResourceControl(resourceControl *portainer.ResourceControl) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		id, _ := bucket.NextSequence()
+		resourceControl.ID = portainer.ResourceControlID(id)
+
+		data, err := internal.MarshalObject(resourceControl)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(resourceControl.ID)), data)
+	})
+}
+
+// UpdateResourceControl saves a ResourceControl object.
+func (service *Service) UpdateResourceControl(ID portainer.ResourceControlID, resourceControl *portainer.ResourceControl) error {
+	identifier := internal.Itob(int(ID))
+	return internal.UpdateObject(service.db, BucketName, identifier, resourceControl)
+}
+
+// DeleteResourceControl deletes a ResourceControl object by ID
+func (service *Service) DeleteResourceControl(ID portainer.ResourceControlID) error {
+	identifier := internal.Itob(int(ID))
+	return internal.DeleteObject(service.db, BucketName, identifier)
+}

api/bolt/settings/settings.go

@@ -0,0 +1,48 @@
+package settings
+
+import (
+	"github.com/portainer/portainer"
+	"github.com/portainer/portainer/bolt/internal"
+
+	"github.com/boltdb/bolt"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName  = "settings"
+	settingsKey = "SETTINGS"
+)
+
+// Service represents a service for managing endpoint data.
+type Service struct {
+	db *bolt.DB
+}
+
+// NewService creates a new instance of a service.
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		db: db,
+	}, nil
+}
+
+// Settings retrieve the settings object.
+func (service *Service) Settings() (*portainer.Settings, error) {
+	var settings portainer.Settings
+
+	err := internal.GetObject(service.db, BucketName, []byte(settingsKey), &settings)
+	if err != nil {
+		return nil, err
+	}
+
+	return &settings, nil
+}
+
+// UpdateSettings persists a Settings object.
+func (service *Service) UpdateSettings(settings *portainer.Settings) error {
+	return internal.UpdateObject(service.db, BucketName, []byte(settingsKey), settings)
+}

api/bolt/settings_service.go

@@ -1,61 +0,0 @@
-package bolt
-
-import (
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/internal"
-
-	"github.com/boltdb/bolt"
-)
-
-// SettingsService represents a service to manage application settings.
-type SettingsService struct {
-	store *Store
-}
-
-const (
-	dbSettingsKey = "SETTINGS"
-)
-
-// Settings retrieve the settings object.
-func (service *SettingsService) Settings() (*portainer.Settings, error) {
-	var data []byte
-	err := service.store.db.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(settingsBucketName))
-		value := bucket.Get([]byte(dbSettingsKey))
-		if value == nil {
-			return portainer.ErrSettingsNotFound
-		}
-
-		data = make([]byte, len(value))
-		copy(data, value)
-		return nil
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	var settings portainer.Settings
-	err = internal.UnmarshalSettings(data, &settings)
-	if err != nil {
-		return nil, err
-	}
-	return &settings, nil
-}
-
-// StoreSettings persists a Settings object.
-func (service *SettingsService) StoreSettings(settings *portainer.Settings) error {
-	return service.store.db.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(settingsBucketName))
-
-		data, err := internal.MarshalSettings(settings)
-		if err != nil {
-			return err
-		}
-
-		err = bucket.Put([]byte(dbSettingsKey), data)
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-}

api/bolt/stack/stack.go

@@ -0,0 +1,134 @@
+package stack
+
+import (
+	"github.com/portainer/portainer"
+	"github.com/portainer/portainer/bolt/internal"
+
+	"github.com/boltdb/bolt"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "stacks"
+)
+
+// Service represents a service for managing endpoint data.
+type Service struct {
+	db *bolt.DB
+}
+
+// NewService creates a new instance of a service.
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		db: db,
+	}, nil
+}
+
+// Stack returns a stack object by ID.
+func (service *Service) Stack(ID portainer.StackID) (*portainer.Stack, error) {
+	var stack portainer.Stack
+	identifier := internal.Itob(int(ID))
+
+	err := internal.GetObject(service.db, BucketName, identifier, &stack)
+	if err != nil {
+		return nil, err
+	}
+
+	return &stack, nil
+}
+
+// StackByName returns a stack object by name.
+func (service *Service) StackByName(name string) (*portainer.Stack, error) {
+	var stack *portainer.Stack
+
+	err := service.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+		cursor := bucket.Cursor()
+
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var t portainer.Stack
+			err := internal.UnmarshalObject(v, &t)
+			if err != nil {
+				return err
+			}
+
+			if t.Name == name {
+				stack = &t
+				break
+			}
+		}
+
+		if stack == nil {
+			return portainer.ErrObjectNotFound
+		}
+
+		return nil
+	})
+
+	return stack, err
+}
+
+// Stacks returns an array containing all the stacks.
+func (service *Service) Stacks() ([]portainer.Stack, error) {
+	var stacks = make([]portainer.Stack, 0)
+
+	err := service.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var stack portainer.Stack
+			err := internal.UnmarshalObject(v, &stack)
+			if err != nil {
+				return err
+			}
+			stacks = append(stacks, stack)
+		}
+
+		return nil
+	})
+
+	return stacks, err
+}
+
+// GetNextIdentifier returns the next identifier for a stack.
+func (service *Service) GetNextIdentifier() int {
+	return internal.GetNextIdentifier(service.db, BucketName)
+}
+
+// CreateStack creates a new stack.
+func (service *Service) CreateStack(stack *portainer.Stack) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		// We manually manage sequences for stacks
+		err := bucket.SetSequence(uint64(stack.ID))
+		if err != nil {
+			return err
+		}
+
+		data, err := internal.MarshalObject(stack)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(stack.ID)), data)
+	})
+}
+
+// UpdateStack updates a stack.
+func (service *Service) UpdateStack(ID portainer.StackID, stack *portainer.Stack) error {
+	identifier := internal.Itob(int(ID))
+	return internal.UpdateObject(service.db, BucketName, identifier, stack)
+}
+
+// DeleteStack deletes a stack.
+func (service *Service) DeleteStack(ID portainer.StackID) error {
+	identifier := internal.Itob(int(ID))
+	return internal.DeleteObject(service.db, BucketName, identifier)
+}

api/bolt/tag/tag.go

@@ -0,0 +1,76 @@
+package tag
+
+import (
+	"github.com/portainer/portainer"
+	"github.com/portainer/portainer/bolt/internal"
+
+	"github.com/boltdb/bolt"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "tags"
+)
+
+// Service represents a service for managing endpoint data.
+type Service struct {
+	db *bolt.DB
+}
+
+// NewService creates a new instance of a service.
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		db: db,
+	}, nil
+}
+
+// Tags return an array containing all the tags.
+func (service *Service) Tags() ([]portainer.Tag, error) {
+	var tags = make([]portainer.Tag, 0)
+
+	err := service.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var tag portainer.Tag
+			err := internal.UnmarshalObject(v, &tag)
+			if err != nil {
+				return err
+			}
+			tags = append(tags, tag)
+		}
+
+		return nil
+	})
+
+	return tags, err
+}
+
+// CreateTag creates a new tag.
+func (service *Service) CreateTag(tag *portainer.Tag) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		id, _ := bucket.NextSequence()
+		tag.ID = portainer.TagID(id)
+
+		data, err := internal.MarshalObject(tag)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(tag.ID)), data)
+	})
+}
+
+// DeleteTag deletes a tag.
+func (service *Service) DeleteTag(ID portainer.TagID) error {
+	identifier := internal.Itob(int(ID))
+	return internal.DeleteObject(service.db, BucketName, identifier)
+}

api/bolt/team/team.go

@@ -0,0 +1,126 @@
+package team
+
+import (
+	"github.com/portainer/portainer"
+	"github.com/portainer/portainer/bolt/internal"
+
+	"github.com/boltdb/bolt"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "teams"
+)
+
+// Service represents a service for managing endpoint data.
+type Service struct {
+	db *bolt.DB
+}
+
+// NewService creates a new instance of a service.
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		db: db,
+	}, nil
+}
+
+// Team returns a Team by ID
+func (service *Service) Team(ID portainer.TeamID) (*portainer.Team, error) {
+	var team portainer.Team
+	identifier := internal.Itob(int(ID))
+
+	err := internal.GetObject(service.db, BucketName, identifier, &team)
+	if err != nil {
+		return nil, err
+	}
+
+	return &team, nil
+}
+
+// TeamByName returns a team by name.
+func (service *Service) TeamByName(name string) (*portainer.Team, error) {
+	var team *portainer.Team
+
+	err := service.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var t portainer.Team
+			err := internal.UnmarshalObject(v, &t)
+			if err != nil {
+				return err
+			}
+
+			if t.Name == name {
+				team = &t
+				break
+			}
+		}
+
+		if team == nil {
+			return portainer.ErrObjectNotFound
+		}
+
+		return nil
+	})
+
+	return team, err
+}
+
+// Teams return an array containing all the teams.
+func (service *Service) Teams() ([]portainer.Team, error) {
+	var teams = make([]portainer.Team, 0)
+
+	err := service.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var team portainer.Team
+			err := internal.UnmarshalObject(v, &team)
+			if err != nil {
+				return err
+			}
+			teams = append(teams, team)
+		}
+
+		return nil
+	})
+
+	return teams, err
+}
+
+// UpdateTeam saves a Team.
+func (service *Service) UpdateTeam(ID portainer.TeamID, team *portainer.Team) error {
+	identifier := internal.Itob(int(ID))
+	return internal.UpdateObject(service.db, BucketName, identifier, team)
+}
+
+// CreateTeam creates a new Team.
+func (service *Service) CreateTeam(team *portainer.Team) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		id, _ := bucket.NextSequence()
+		team.ID = portainer.TeamID(id)
+
+		data, err := internal.MarshalObject(team)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(team.ID)), data)
+	})
+}
+
+// DeleteTeam deletes a Team.
+func (service *Service) DeleteTeam(ID portainer.TeamID) error {
+	identifier := internal.Itob(int(ID))
+	return internal.DeleteObject(service.db, BucketName, identifier)
+}

api/bolt/team_membership_service.go

@@ -1,217 +0,0 @@
-package bolt
-
-import (
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/internal"
-
-	"github.com/boltdb/bolt"
-)
-
-// TeamMembershipService represents a service for managing TeamMembership objects.
-type TeamMembershipService struct {
-	store *Store
-}
-
-// TeamMembership returns a TeamMembership object by ID
-func (service *TeamMembershipService) TeamMembership(ID portainer.TeamMembershipID) (*portainer.TeamMembership, error) {
-	var data []byte
-	err := service.store.db.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(teamMembershipBucketName))
-		value := bucket.Get(internal.Itob(int(ID)))
-		if value == nil {
-			return portainer.ErrTeamMembershipNotFound
-		}
-
-		data = make([]byte, len(value))
-		copy(data, value)
-		return nil
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	var membership portainer.TeamMembership
-	err = internal.UnmarshalTeamMembership(data, &membership)
-	if err != nil {
-		return nil, err
-	}
-	return &membership, nil
-}
-
-// TeamMemberships return an array containing all the TeamMembership objects.
-func (service *TeamMembershipService) TeamMemberships() ([]portainer.TeamMembership, error) {
-	var memberships = make([]portainer.TeamMembership, 0)
-	err := service.store.db.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(teamMembershipBucketName))
-
-		cursor := bucket.Cursor()
-		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
-			var membership portainer.TeamMembership
-			err := internal.UnmarshalTeamMembership(v, &membership)
-			if err != nil {
-				return err
-			}
-			memberships = append(memberships, membership)
-		}
-
-		return nil
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	return memberships, nil
-}
-
-// TeamMembershipsByUserID return an array containing all the TeamMembership objects where the specified userID is present.
-func (service *TeamMembershipService) TeamMembershipsByUserID(userID portainer.UserID) ([]portainer.TeamMembership, error) {
-	var memberships = make([]portainer.TeamMembership, 0)
-	err := service.store.db.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(teamMembershipBucketName))
-
-		cursor := bucket.Cursor()
-		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
-			var membership portainer.TeamMembership
-			err := internal.UnmarshalTeamMembership(v, &membership)
-			if err != nil {
-				return err
-			}
-			if membership.UserID == userID {
-				memberships = append(memberships, membership)
-			}
-		}
-
-		return nil
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	return memberships, nil
-}
-
-// TeamMembershipsByTeamID return an array containing all the TeamMembership objects where the specified teamID is present.
-func (service *TeamMembershipService) TeamMembershipsByTeamID(teamID portainer.TeamID) ([]portainer.TeamMembership, error) {
-	var memberships = make([]portainer.TeamMembership, 0)
-	err := service.store.db.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(teamMembershipBucketName))
-
-		cursor := bucket.Cursor()
-		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
-			var membership portainer.TeamMembership
-			err := internal.UnmarshalTeamMembership(v, &membership)
-			if err != nil {
-				return err
-			}
-			if membership.TeamID == teamID {
-				memberships = append(memberships, membership)
-			}
-		}
-
-		return nil
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	return memberships, nil
-}
-
-// UpdateTeamMembership saves a TeamMembership object.
-func (service *TeamMembershipService) UpdateTeamMembership(ID portainer.TeamMembershipID, membership *portainer.TeamMembership) error {
-	data, err := internal.MarshalTeamMembership(membership)
-	if err != nil {
-		return err
-	}
-
-	return service.store.db.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(teamMembershipBucketName))
-		err = bucket.Put(internal.Itob(int(ID)), data)
-
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-}
-
-// CreateTeamMembership creates a new TeamMembership object.
-func (service *TeamMembershipService) CreateTeamMembership(membership *portainer.TeamMembership) error {
-	return service.store.db.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(teamMembershipBucketName))
-
-		id, _ := bucket.NextSequence()
-		membership.ID = portainer.TeamMembershipID(id)
-
-		data, err := internal.MarshalTeamMembership(membership)
-		if err != nil {
-			return err
-		}
-
-		err = bucket.Put(internal.Itob(int(membership.ID)), data)
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-}
-
-// DeleteTeamMembership deletes a TeamMembership object.
-func (service *TeamMembershipService) DeleteTeamMembership(ID portainer.TeamMembershipID) error {
-	return service.store.db.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(teamMembershipBucketName))
-		err := bucket.Delete(internal.Itob(int(ID)))
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-}
-
-// DeleteTeamMembershipByUserID deletes all the TeamMembership object associated to a UserID.
-func (service *TeamMembershipService) DeleteTeamMembershipByUserID(userID portainer.UserID) error {
-	return service.store.db.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(teamMembershipBucketName))
-
-		cursor := bucket.Cursor()
-		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
-			var membership portainer.TeamMembership
-			err := internal.UnmarshalTeamMembership(v, &membership)
-			if err != nil {
-				return err
-			}
-			if membership.UserID == userID {
-				err := bucket.Delete(internal.Itob(int(membership.ID)))
-				if err != nil {
-					return err
-				}
-			}
-		}
-
-		return nil
-	})
-}
-
-// DeleteTeamMembershipByTeamID deletes all the TeamMembership object associated to a TeamID.
-func (service *TeamMembershipService) DeleteTeamMembershipByTeamID(teamID portainer.TeamID) error {
-	return service.store.db.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(teamMembershipBucketName))
-
-		cursor := bucket.Cursor()
-		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
-			var membership portainer.TeamMembership
-			err := internal.UnmarshalTeamMembership(v, &membership)
-			if err != nil {
-				return err
-			}
-			if membership.TeamID == teamID {
-				err := bucket.Delete(internal.Itob(int(membership.ID)))
-				if err != nil {
-					return err
-				}
-			}
-		}
-
-		return nil
-	})
-}

api/bolt/team_service.go

@@ -1,144 +0,0 @@
-package bolt
-
-import (
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/internal"
-
-	"github.com/boltdb/bolt"
-)
-
-// TeamService represents a service for managing teams.
-type TeamService struct {
-	store *Store
-}
-
-// Team returns a Team by ID
-func (service *TeamService) Team(ID portainer.TeamID) (*portainer.Team, error) {
-	var data []byte
-	err := service.store.db.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(teamBucketName))
-		value := bucket.Get(internal.Itob(int(ID)))
-		if value == nil {
-			return portainer.ErrTeamNotFound
-		}
-
-		data = make([]byte, len(value))
-		copy(data, value)
-		return nil
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	var team portainer.Team
-	err = internal.UnmarshalTeam(data, &team)
-	if err != nil {
-		return nil, err
-	}
-	return &team, nil
-}
-
-// TeamByName returns a team by name.
-func (service *TeamService) TeamByName(name string) (*portainer.Team, error) {
-	var team *portainer.Team
-
-	err := service.store.db.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(teamBucketName))
-		cursor := bucket.Cursor()
-		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
-			var t portainer.Team
-			err := internal.UnmarshalTeam(v, &t)
-			if err != nil {
-				return err
-			}
-			if t.Name == name {
-				team = &t
-			}
-		}
-
-		if team == nil {
-			return portainer.ErrTeamNotFound
-		}
-		return nil
-	})
-	if err != nil {
-		return nil, err
-	}
-	return team, nil
-}
-
-// Teams return an array containing all the teams.
-func (service *TeamService) Teams() ([]portainer.Team, error) {
-	var teams = make([]portainer.Team, 0)
-	err := service.store.db.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(teamBucketName))
-
-		cursor := bucket.Cursor()
-		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
-			var team portainer.Team
-			err := internal.UnmarshalTeam(v, &team)
-			if err != nil {
-				return err
-			}
-			teams = append(teams, team)
-		}
-
-		return nil
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	return teams, nil
-}
-
-// UpdateTeam saves a Team.
-func (service *TeamService) UpdateTeam(ID portainer.TeamID, team *portainer.Team) error {
-	data, err := internal.MarshalTeam(team)
-	if err != nil {
-		return err
-	}
-
-	return service.store.db.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(teamBucketName))
-		err = bucket.Put(internal.Itob(int(ID)), data)
-
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-}
-
-// CreateTeam creates a new Team.
-func (service *TeamService) CreateTeam(team *portainer.Team) error {
-	return service.store.db.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(teamBucketName))
-
-		id, _ := bucket.NextSequence()
-		team.ID = portainer.TeamID(id)
-
-		data, err := internal.MarshalTeam(team)
-		if err != nil {
-			return err
-		}
-
-		err = bucket.Put(internal.Itob(int(team.ID)), data)
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-}
-
-// DeleteTeam deletes a Team.
-func (service *TeamService) DeleteTeam(ID portainer.TeamID) error {
-	return service.store.db.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(teamBucketName))
-		err := bucket.Delete(internal.Itob(int(ID)))
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-}

api/bolt/teammembership/teammembership.go

@@ -0,0 +1,197 @@
+package teammembership
+
+import (
+	"github.com/portainer/portainer"
+	"github.com/portainer/portainer/bolt/internal"
+
+	"github.com/boltdb/bolt"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "team_membership"
+)
+
+// Service represents a service for managing endpoint data.
+type Service struct {
+	db *bolt.DB
+}
+
+// NewService creates a new instance of a service.
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		db: db,
+	}, nil
+}
+
+// TeamMembership returns a TeamMembership object by ID
+func (service *Service) TeamMembership(ID portainer.TeamMembershipID) (*portainer.TeamMembership, error) {
+	var membership portainer.TeamMembership
+	identifier := internal.Itob(int(ID))
+
+	err := internal.GetObject(service.db, BucketName, identifier, &membership)
+	if err != nil {
+		return nil, err
+	}
+
+	return &membership, nil
+}
+
+// TeamMemberships return an array containing all the TeamMembership objects.
+func (service *Service) TeamMemberships() ([]portainer.TeamMembership, error) {
+	var memberships = make([]portainer.TeamMembership, 0)
+
+	err := service.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var membership portainer.TeamMembership
+			err := internal.UnmarshalObject(v, &membership)
+			if err != nil {
+				return err
+			}
+			memberships = append(memberships, membership)
+		}
+
+		return nil
+	})
+
+	return memberships, err
+}
+
+// TeamMembershipsByUserID return an array containing all the TeamMembership objects where the specified userID is present.
+func (service *Service) TeamMembershipsByUserID(userID portainer.UserID) ([]portainer.TeamMembership, error) {
+	var memberships = make([]portainer.TeamMembership, 0)
+
+	err := service.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var membership portainer.TeamMembership
+			err := internal.UnmarshalObject(v, &membership)
+			if err != nil {
+				return err
+			}
+
+			if membership.UserID == userID {
+				memberships = append(memberships, membership)
+			}
+		}
+
+		return nil
+	})
+
+	return memberships, err
+}
+
+// TeamMembershipsByTeamID return an array containing all the TeamMembership objects where the specified teamID is present.
+func (service *Service) TeamMembershipsByTeamID(teamID portainer.TeamID) ([]portainer.TeamMembership, error) {
+	var memberships = make([]portainer.TeamMembership, 0)
+
+	err := service.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var membership portainer.TeamMembership
+			err := internal.UnmarshalObject(v, &membership)
+			if err != nil {
+				return err
+			}
+
+			if membership.TeamID == teamID {
+				memberships = append(memberships, membership)
+			}
+		}
+
+		return nil
+	})
+
+	return memberships, err
+}
+
+// UpdateTeamMembership saves a TeamMembership object.
+func (service *Service) UpdateTeamMembership(ID portainer.TeamMembershipID, membership *portainer.TeamMembership) error {
+	identifier := internal.Itob(int(ID))
+	return internal.UpdateObject(service.db, BucketName, identifier, membership)
+}
+
+// CreateTeamMembership creates a new TeamMembership object.
+func (service *Service) CreateTeamMembership(membership *portainer.TeamMembership) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		id, _ := bucket.NextSequence()
+		membership.ID = portainer.TeamMembershipID(id)
+
+		data, err := internal.MarshalObject(membership)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(membership.ID)), data)
+	})
+}
+
+// DeleteTeamMembership deletes a TeamMembership object.
+func (service *Service) DeleteTeamMembership(ID portainer.TeamMembershipID) error {
+	identifier := internal.Itob(int(ID))
+	return internal.DeleteObject(service.db, BucketName, identifier)
+}
+
+// DeleteTeamMembershipByUserID deletes all the TeamMembership object associated to a UserID.
+func (service *Service) DeleteTeamMembershipByUserID(userID portainer.UserID) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var membership portainer.TeamMembership
+			err := internal.UnmarshalObject(v, &membership)
+			if err != nil {
+				return err
+			}
+
+			if membership.UserID == userID {
+				err := bucket.Delete(internal.Itob(int(membership.ID)))
+				if err != nil {
+					return err
+				}
+			}
+		}
+
+		return nil
+	})
+}
+
+// DeleteTeamMembershipByTeamID deletes all the TeamMembership object associated to a TeamID.
+func (service *Service) DeleteTeamMembershipByTeamID(teamID portainer.TeamID) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var membership portainer.TeamMembership
+			err := internal.UnmarshalObject(v, &membership)
+			if err != nil {
+				return err
+			}
+
+			if membership.TeamID == teamID {
+				err := bucket.Delete(internal.Itob(int(membership.ID)))
+				if err != nil {
+					return err
+				}
+			}
+		}
+
+		return nil
+	})
+}

api/bolt/template/template.go

@@ -0,0 +1,95 @@
+package template
+
+import (
+	"github.com/portainer/portainer"
+	"github.com/portainer/portainer/bolt/internal"
+
+	"github.com/boltdb/bolt"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "templates"
+)
+
+// Service represents a service for managing endpoint data.
+type Service struct {
+	db *bolt.DB
+}
+
+// NewService creates a new instance of a service.
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		db: db,
+	}, nil
+}
+
+// Templates return an array containing all the templates.
+func (service *Service) Templates() ([]portainer.Template, error) {
+	var templates = make([]portainer.Template, 0)
+
+	err := service.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var template portainer.Template
+			err := internal.UnmarshalObject(v, &template)
+			if err != nil {
+				return err
+			}
+			templates = append(templates, template)
+		}
+
+		return nil
+	})
+
+	return templates, err
+}
+
+// Template returns a template by ID.
+func (service *Service) Template(ID portainer.TemplateID) (*portainer.Template, error) {
+	var template portainer.Template
+	identifier := internal.Itob(int(ID))
+
+	err := internal.GetObject(service.db, BucketName, identifier, &template)
+	if err != nil {
+		return nil, err
+	}
+
+	return &template, nil
+}
+
+// CreateTemplate creates a new template.
+func (service *Service) CreateTemplate(template *portainer.Template) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		id, _ := bucket.NextSequence()
+		template.ID = portainer.TemplateID(id)
+
+		data, err := internal.MarshalObject(template)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(template.ID)), data)
+	})
+}
+
+// UpdateTemplate saves a template.
+func (service *Service) UpdateTemplate(ID portainer.TemplateID, template *portainer.Template) error {
+	identifier := internal.Itob(int(ID))
+	return internal.UpdateObject(service.db, BucketName, identifier, template)
+}
+
+// DeleteTemplate deletes a template.
+func (service *Service) DeleteTemplate(ID portainer.TemplateID) error {
+	identifier := internal.Itob(int(ID))
+	return internal.DeleteObject(service.db, BucketName, identifier)
+}

api/bolt/user/user.go

@@ -0,0 +1,149 @@
+package user
+
+import (
+	"github.com/portainer/portainer"
+	"github.com/portainer/portainer/bolt/internal"
+
+	"github.com/boltdb/bolt"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "users"
+)
+
+// Service represents a service for managing endpoint data.
+type Service struct {
+	db *bolt.DB
+}
+
+// NewService creates a new instance of a service.
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		db: db,
+	}, nil
+}
+
+// User returns a user by ID
+func (service *Service) User(ID portainer.UserID) (*portainer.User, error) {
+	var user portainer.User
+	identifier := internal.Itob(int(ID))
+
+	err := internal.GetObject(service.db, BucketName, identifier, &user)
+	if err != nil {
+		return nil, err
+	}
+
+	return &user, nil
+}
+
+// UserByUsername returns a user by username.
+func (service *Service) UserByUsername(username string) (*portainer.User, error) {
+	var user *portainer.User
+
+	err := service.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+		cursor := bucket.Cursor()
+
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var u portainer.User
+			err := internal.UnmarshalObject(v, &u)
+			if err != nil {
+				return err
+			}
+
+			if u.Username == username {
+				user = &u
+				break
+			}
+		}
+
+		if user == nil {
+			return portainer.ErrObjectNotFound
+		}
+		return nil
+	})
+
+	return user, err
+}
+
+// Users return an array containing all the users.
+func (service *Service) Users() ([]portainer.User, error) {
+	var users = make([]portainer.User, 0)
+
+	err := service.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var user portainer.User
+			err := internal.UnmarshalObject(v, &user)
+			if err != nil {
+				return err
+			}
+			users = append(users, user)
+		}
+
+		return nil
+	})
+
+	return users, err
+}
+
+// UsersByRole return an array containing all the users with the specified role.
+func (service *Service) UsersByRole(role portainer.UserRole) ([]portainer.User, error) {
+	var users = make([]portainer.User, 0)
+	err := service.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var user portainer.User
+			err := internal.UnmarshalObject(v, &user)
+			if err != nil {
+				return err
+			}
+
+			if user.Role == role {
+				users = append(users, user)
+			}
+		}
+		return nil
+	})
+
+	return users, err
+}
+
+// UpdateUser saves a user.
+func (service *Service) UpdateUser(ID portainer.UserID, user *portainer.User) error {
+	identifier := internal.Itob(int(ID))
+	return internal.UpdateObject(service.db, BucketName, identifier, user)
+}
+
+// CreateUser creates a new user.
+func (service *Service) CreateUser(user *portainer.User) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		id, _ := bucket.NextSequence()
+		user.ID = portainer.UserID(id)
+
+		data, err := internal.MarshalObject(user)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(user.ID)), data)
+	})
+}
+
+// DeleteUser deletes a user.
+func (service *Service) DeleteUser(ID portainer.UserID) error {
+	identifier := internal.Itob(int(ID))
+	return internal.DeleteObject(service.db, BucketName, identifier)
+}

api/bolt/user_service.go

@@ -1,170 +0,0 @@
-package bolt
-
-import (
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/internal"
-
-	"github.com/boltdb/bolt"
-)
-
-// UserService represents a service for managing users.
-type UserService struct {
-	store *Store
-}
-
-// User returns a user by ID
-func (service *UserService) User(ID portainer.UserID) (*portainer.User, error) {
-	var data []byte
-	err := service.store.db.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(userBucketName))
-		value := bucket.Get(internal.Itob(int(ID)))
-		if value == nil {
-			return portainer.ErrUserNotFound
-		}
-
-		data = make([]byte, len(value))
-		copy(data, value)
-		return nil
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	var user portainer.User
-	err = internal.UnmarshalUser(data, &user)
-	if err != nil {
-		return nil, err
-	}
-	return &user, nil
-}
-
-// UserByUsername returns a user by username.
-func (service *UserService) UserByUsername(username string) (*portainer.User, error) {
-	var user *portainer.User
-
-	err := service.store.db.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(userBucketName))
-		cursor := bucket.Cursor()
-		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
-			var u portainer.User
-			err := internal.UnmarshalUser(v, &u)
-			if err != nil {
-				return err
-			}
-			if u.Username == username {
-				user = &u
-			}
-		}
-
-		if user == nil {
-			return portainer.ErrUserNotFound
-		}
-		return nil
-	})
-	if err != nil {
-		return nil, err
-	}
-	return user, nil
-}
-
-// Users return an array containing all the users.
-func (service *UserService) Users() ([]portainer.User, error) {
-	var users = make([]portainer.User, 0)
-	err := service.store.db.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(userBucketName))
-
-		cursor := bucket.Cursor()
-		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
-			var user portainer.User
-			err := internal.UnmarshalUser(v, &user)
-			if err != nil {
-				return err
-			}
-			users = append(users, user)
-		}
-
-		return nil
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	return users, nil
-}
-
-// UsersByRole return an array containing all the users with the specified role.
-func (service *UserService) UsersByRole(role portainer.UserRole) ([]portainer.User, error) {
-	var users = make([]portainer.User, 0)
-	err := service.store.db.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(userBucketName))
-
-		cursor := bucket.Cursor()
-		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
-			var user portainer.User
-			err := internal.UnmarshalUser(v, &user)
-			if err != nil {
-				return err
-			}
-			if user.Role == role {
-				users = append(users, user)
-			}
-		}
-		return nil
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	return users, nil
-}
-
-// UpdateUser saves a user.
-func (service *UserService) UpdateUser(ID portainer.UserID, user *portainer.User) error {
-	data, err := internal.MarshalUser(user)
-	if err != nil {
-		return err
-	}
-
-	return service.store.db.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(userBucketName))
-		err = bucket.Put(internal.Itob(int(ID)), data)
-
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-}
-
-// CreateUser creates a new user.
-func (service *UserService) CreateUser(user *portainer.User) error {
-	return service.store.db.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(userBucketName))
-
-		id, _ := bucket.NextSequence()
-		user.ID = portainer.UserID(id)
-
-		data, err := internal.MarshalUser(user)
-		if err != nil {
-			return err
-		}
-
-		err = bucket.Put(internal.Itob(int(user.ID)), data)
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-}
-
-// DeleteUser deletes a user.
-func (service *UserService) DeleteUser(ID portainer.UserID) error {
-	return service.store.db.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(userBucketName))
-		err := bucket.Delete(internal.Itob(int(ID)))
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-}

api/bolt/version/version.go

@@ -0,0 +1,66 @@
+package version
+
+import (
+	"strconv"
+
+	"github.com/boltdb/bolt"
+	"github.com/portainer/portainer"
+	"github.com/portainer/portainer/bolt/internal"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "version"
+	versionKey = "DB_VERSION"
+)
+
+// Service represents a service to manage stored versions.
+type Service struct {
+	db *bolt.DB
+}
+
+// NewService creates a new instance of a service.
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		db: db,
+	}, nil
+}
+
+// DBVersion retrieves the stored database version.
+func (service *Service) DBVersion() (int, error) {
+	var data []byte
+
+	err := service.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		value := bucket.Get([]byte(versionKey))
+		if value == nil {
+			return portainer.ErrObjectNotFound
+		}
+
+		data = make([]byte, len(value))
+		copy(data, value)
+
+		return nil
+	})
+	if err != nil {
+		return 0, err
+	}
+
+	return strconv.Atoi(string(data))
+}
+
+// StoreDBVersion store the database version.
+func (service *Service) StoreDBVersion(version int) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		data := []byte(strconv.Itoa(version))
+		return bucket.Put([]byte(versionKey), data)
+	})
+}

api/bolt/version_service.go

@@ -1,58 +0,0 @@
-package bolt
-
-import (
-	"strconv"
-
-	"github.com/portainer/portainer"
-
-	"github.com/boltdb/bolt"
-)
-
-// VersionService represents a service to manage stored versions.
-type VersionService struct {
-	store *Store
-}
-
-const (
-	dBVersionKey = "DB_VERSION"
-)
-
-// DBVersion retrieves the stored database version.
-func (service *VersionService) DBVersion() (int, error) {
-	var data []byte
-	err := service.store.db.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(versionBucketName))
-		value := bucket.Get([]byte(dBVersionKey))
-		if value == nil {
-			return portainer.ErrDBVersionNotFound
-		}
-
-		data = make([]byte, len(value))
-		copy(data, value)
-		return nil
-	})
-	if err != nil {
-		return 0, err
-	}
-
-	dbVersion, err := strconv.Atoi(string(data))
-	if err != nil {
-		return 0, err
-	}
-
-	return dbVersion, nil
-}
-
-// StoreDBVersion store the database version.
-func (service *VersionService) StoreDBVersion(version int) error {
-	return service.store.db.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(versionBucketName))
-
-		data := []byte(strconv.Itoa(version))
-		err := bucket.Put([]byte(dBVersionKey), data)
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-}

api/cli/cli.go

@@ -1,12 +1,12 @@
 package cli
 
 import (
-	"log"
 	"time"
 
 	"github.com/portainer/portainer"
 
 	"os"
+	"path/filepath"
 	"strings"
 
 	"gopkg.in/alecthomas/kingpin.v2"
@@ -16,12 +16,15 @@ import (
 type Service struct{}
 
 const (
-	errInvalidEndpointProtocol    = portainer.Error("Invalid endpoint protocol: Portainer only supports unix:// or tcp://")
-	errSocketNotFound             = portainer.Error("Unable to locate Unix socket")
-	errEndpointsFileNotFound      = portainer.Error("Unable to locate external endpoints file")
-	errInvalidSyncInterval        = portainer.Error("Invalid synchronization interval")
-	errEndpointExcludeExternal    = portainer.Error("Cannot use the -H flag mutually with --external-endpoints")
-	errNoAuthExcludeAdminPassword = portainer.Error("Cannot use --no-auth with --admin-password")
+	errInvalidEndpointProtocol       = portainer.Error("Invalid endpoint protocol: Portainer only supports unix://, npipe:// or tcp://")
+	errSocketOrNamedPipeNotFound     = portainer.Error("Unable to locate Unix socket or named pipe")
+	errEndpointsFileNotFound         = portainer.Error("Unable to locate external endpoints file")
+	errTemplateFileNotFound          = portainer.Error("Unable to locate template file on disk")
+	errInvalidSyncInterval           = portainer.Error("Invalid synchronization interval")
+	errInvalidSnapshotInterval       = portainer.Error("Invalid snapshot interval")
+	errEndpointExcludeExternal       = portainer.Error("Cannot use the -H flag mutually with --external-endpoints")
+	errNoAuthExcludeAdminPassword    = portainer.Error("Cannot use --no-auth with --admin-password or --admin-password-file")
+	errAdminPassExcludeAdminPassFile = portainer.Error("Cannot use --admin-password with --admin-password-file")
 )
 
 // ParseFlags parse the CLI flags and return a portainer.Flags struct
@@ -29,40 +32,58 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 	kingpin.Version(version)
 
 	flags := &portainer.CLIFlags{
-		Endpoint:          kingpin.Flag("host", "Dockerd endpoint").Short('H').String(),
-		ExternalEndpoints: kingpin.Flag("external-endpoints", "Path to a file defining available endpoints").String(),
-		SyncInterval:      kingpin.Flag("sync-interval", "Duration between each synchronization via the external endpoints source").Default(defaultSyncInterval).String(),
 		Addr:              kingpin.Flag("bind", "Address and port to serve Portainer").Default(defaultBindAddress).Short('p').String(),
 		Assets:            kingpin.Flag("assets", "Path to the assets").Default(defaultAssetsDirectory).Short('a').String(),
 		Data:              kingpin.Flag("data", "Path to the folder where the data is stored").Default(defaultDataDirectory).Short('d').String(),
+		EndpointURL:       kingpin.Flag("host", "Endpoint URL").Short('H').String(),
+		ExternalEndpoints: kingpin.Flag("external-endpoints", "Path to a file defining available endpoints").String(),
 		NoAuth:            kingpin.Flag("no-auth", "Disable authentication").Default(defaultNoAuth).Bool(),
-		NoAnalytics:       kingpin.Flag("no-analytics", "Disable Analytics in app").Default(defaultNoAuth).Bool(),
-		TLSVerify:         kingpin.Flag("tlsverify", "TLS support").Default(defaultTLSVerify).Bool(),
+		NoAnalytics:       kingpin.Flag("no-analytics", "Disable Analytics in app").Default(defaultNoAnalytics).Bool(),
+		TLS:               kingpin.Flag("tlsverify", "TLS support").Default(defaultTLS).Bool(),
+		TLSSkipVerify:     kingpin.Flag("tlsskipverify", "Disable TLS server verification").Default(defaultTLSSkipVerify).Bool(),
 		TLSCacert:         kingpin.Flag("tlscacert", "Path to the CA").Default(defaultTLSCACertPath).String(),
 		TLSCert:           kingpin.Flag("tlscert", "Path to the TLS certificate file").Default(defaultTLSCertPath).String(),
 		TLSKey:            kingpin.Flag("tlskey", "Path to the TLS key").Default(defaultTLSKeyPath).String(),
 		SSL:               kingpin.Flag("ssl", "Secure Portainer instance using SSL").Default(defaultSSL).Bool(),
 		SSLCert:           kingpin.Flag("sslcert", "Path to the SSL certificate used to secure the Portainer instance").Default(defaultSSLCertPath).String(),
 		SSLKey:            kingpin.Flag("sslkey", "Path to the SSL key used to secure the Portainer instance").Default(defaultSSLKeyPath).String(),
+		SyncInterval:      kingpin.Flag("sync-interval", "Duration between each synchronization via the external endpoints source").Default(defaultSyncInterval).String(),
+		Snapshot:          kingpin.Flag("snapshot", "Start a background job to create endpoint snapshots").Default(defaultSnapshot).Bool(),
+		SnapshotInterval:  kingpin.Flag("snapshot-interval", "Duration between each endpoint snapshot job").Default(defaultSnapshotInterval).String(),
 		AdminPassword:     kingpin.Flag("admin-password", "Hashed admin password").String(),
-		// Deprecated flags
-		Labels:    pairs(kingpin.Flag("hide-label", "Hide containers with a specific label in the UI").Short('l')),
-		Logo:      kingpin.Flag("logo", "URL for the logo displayed in the UI").String(),
-		Templates: kingpin.Flag("templates", "URL to the templates (apps) definitions").Short('t').String(),
+		AdminPasswordFile: kingpin.Flag("admin-password-file", "Path to the file containing the password for the admin user").String(),
+		Labels:            pairs(kingpin.Flag("hide-label", "Hide containers with a specific label in the UI").Short('l')),
+		Logo:              kingpin.Flag("logo", "URL for the logo displayed in the UI").String(),
+		Templates:         kingpin.Flag("templates", "URL to the templates definitions.").Short('t').String(),
+		TemplateFile:      kingpin.Flag("template-file", "Path to the templates (app) definitions on the filesystem").Default(defaultTemplateFile).String(),
 	}
 
 	kingpin.Parse()
+
+	if !filepath.IsAbs(*flags.Assets) {
+		ex, err := os.Executable()
+		if err != nil {
+			panic(err)
+		}
+		*flags.Assets = filepath.Join(filepath.Dir(ex), *flags.Assets)
+	}
+
 	return flags, nil
 }
 
 // ValidateFlags validates the values of the flags.
 func (*Service) ValidateFlags(flags *portainer.CLIFlags) error {
 
-	if *flags.Endpoint != "" && *flags.ExternalEndpoints != "" {
+	if *flags.EndpointURL != "" && *flags.ExternalEndpoints != "" {
 		return errEndpointExcludeExternal
 	}
 
-	err := validateEndpoint(*flags.Endpoint)
+	err := validateTemplateFile(*flags.TemplateFile)
+	if err != nil {
+		return err
+	}
+
+	err = validateEndpointURL(*flags.EndpointURL)
 	if err != nil {
 		return err
 	}
@@ -77,26 +98,34 @@ func (*Service) ValidateFlags(flags *portainer.CLIFlags) error {
 		return err
 	}
 
-	if *flags.NoAuth && (*flags.AdminPassword != "") {
+	err = validateSnapshotInterval(*flags.SnapshotInterval)
+	if err != nil {
+		return err
+	}
+
+	if *flags.NoAuth && (*flags.AdminPassword != "" || *flags.AdminPasswordFile != "") {
 		return errNoAuthExcludeAdminPassword
 	}
 
-	displayDeprecationWarnings(*flags.Templates, *flags.Logo, *flags.Labels)
+	if *flags.AdminPassword != "" && *flags.AdminPasswordFile != "" {
+		return errAdminPassExcludeAdminPassFile
+	}
 
 	return nil
 }
 
-func validateEndpoint(endpoint string) error {
-	if endpoint != "" {
-		if !strings.HasPrefix(endpoint, "unix://") && !strings.HasPrefix(endpoint, "tcp://") {
+func validateEndpointURL(endpointURL string) error {
+	if endpointURL != "" {
+		if !strings.HasPrefix(endpointURL, "unix://") && !strings.HasPrefix(endpointURL, "tcp://") && !strings.HasPrefix(endpointURL, "npipe://") {
 			return errInvalidEndpointProtocol
 		}
 
-		if strings.HasPrefix(endpoint, "unix://") {
-			socketPath := strings.TrimPrefix(endpoint, "unix://")
+		if strings.HasPrefix(endpointURL, "unix://") || strings.HasPrefix(endpointURL, "npipe://") {
+			socketPath := strings.TrimPrefix(endpointURL, "unix://")
+			socketPath = strings.TrimPrefix(socketPath, "npipe://")
 			if _, err := os.Stat(socketPath); err != nil {
 				if os.IsNotExist(err) {
-					return errSocketNotFound
+					return errSocketOrNamedPipeNotFound
 				}
 				return err
 			}
@@ -117,6 +146,16 @@ func validateExternalEndpoints(externalEndpoints string) error {
 	return nil
 }
 
+func validateTemplateFile(templateFile string) error {
+	if _, err := os.Stat(templateFile); err != nil {
+		if os.IsNotExist(err) {
+			return errTemplateFileNotFound
+		}
+		return err
+	}
+	return nil
+}
+
 func validateSyncInterval(syncInterval string) error {
 	if syncInterval != defaultSyncInterval {
 		_, err := time.ParseDuration(syncInterval)
@@ -127,14 +166,12 @@ func validateSyncInterval(syncInterval string) error {
 	return nil
 }
 
-func displayDeprecationWarnings(templates, logo string, labels []portainer.Pair) {
-	if templates != "" {
-		log.Println("Warning: the --templates / -t flag is deprecated and will be removed in future versions.")
-	}
-	if logo != "" {
-		log.Println("Warning: the --logo flag is deprecated and will be removed in future versions.")
-	}
-	if labels != nil {
-		log.Println("Warning: the --hide-label / -l flag is deprecated and will be removed in future versions.")
+func validateSnapshotInterval(snapshotInterval string) error {
+	if snapshotInterval != defaultSnapshotInterval {
+		_, err := time.ParseDuration(snapshotInterval)
+		if err != nil {
+			return errInvalidSnapshotInterval
+		}
 	}
+	return nil
 }

api/cli/defaults.go

@@ -3,17 +3,21 @@
 package cli
 
 const (
-	defaultBindAddress     = ":9000"
-	defaultDataDirectory   = "/data"
-	defaultAssetsDirectory = "."
-	defaultNoAuth          = "false"
-	defaultNoAnalytics     = "false"
-	defaultTLSVerify       = "false"
-	defaultTLSCACertPath   = "/certs/ca.pem"
-	defaultTLSCertPath     = "/certs/cert.pem"
-	defaultTLSKeyPath      = "/certs/key.pem"
-	defaultSSL             = "false"
-	defaultSSLCertPath     = "/certs/portainer.crt"
-	defaultSSLKeyPath      = "/certs/portainer.key"
-	defaultSyncInterval    = "60s"
+	defaultBindAddress      = ":9000"
+	defaultDataDirectory    = "/data"
+	defaultAssetsDirectory  = "./"
+	defaultNoAuth           = "false"
+	defaultNoAnalytics      = "false"
+	defaultTLS              = "false"
+	defaultTLSSkipVerify    = "false"
+	defaultTLSCACertPath    = "/certs/ca.pem"
+	defaultTLSCertPath      = "/certs/cert.pem"
+	defaultTLSKeyPath       = "/certs/key.pem"
+	defaultSSL              = "false"
+	defaultSSLCertPath      = "/certs/portainer.crt"
+	defaultSSLKeyPath       = "/certs/portainer.key"
+	defaultSyncInterval     = "60s"
+	defaultSnapshot         = "true"
+	defaultSnapshotInterval = "5m"
+	defaultTemplateFile     = "/templates.json"
 )

api/cli/defaults_windows.go

@@ -1,17 +1,21 @@
 package cli
 
 const (
-	defaultBindAddress     = ":9000"
-	defaultDataDirectory   = "C:\\data"
-	defaultAssetsDirectory = "."
-	defaultNoAuth          = "false"
-	defaultNoAnalytics     = "false"
-	defaultTLSVerify       = "false"
-	defaultTLSCACertPath   = "C:\\certs\\ca.pem"
-	defaultTLSCertPath     = "C:\\certs\\cert.pem"
-	defaultTLSKeyPath      = "C:\\certs\\key.pem"
-	defaultSSL             = "false"
-	defaultSSLCertPath     = "C:\\certs\\portainer.crt"
-	defaultSSLKeyPath      = "C:\\certs\\portainer.key"
-	defaultSyncInterval    = "60s"
+	defaultBindAddress      = ":9000"
+	defaultDataDirectory    = "C:\\data"
+	defaultAssetsDirectory  = "./"
+	defaultNoAuth           = "false"
+	defaultNoAnalytics      = "false"
+	defaultTLS              = "false"
+	defaultTLSSkipVerify    = "false"
+	defaultTLSCACertPath    = "C:\\certs\\ca.pem"
+	defaultTLSCertPath      = "C:\\certs\\cert.pem"
+	defaultTLSKeyPath       = "C:\\certs\\key.pem"
+	defaultSSL              = "false"
+	defaultSSLCertPath      = "C:\\certs\\portainer.crt"
+	defaultSSLKeyPath       = "C:\\certs\\portainer.key"
+	defaultSyncInterval     = "60s"
+	defaultSnapshot         = "true"
+	defaultSnapshotInterval = "5m"
+	defaultTemplateFile     = "/templates.json"
 )

api/cmd/portainer/main.go

@@ -1,14 +1,23 @@
 package main // import "github.com/portainer/portainer"
 
 import (
+	"encoding/json"
+	"strings"
+
 	"github.com/portainer/portainer"
 	"github.com/portainer/portainer/bolt"
 	"github.com/portainer/portainer/cli"
 	"github.com/portainer/portainer/cron"
 	"github.com/portainer/portainer/crypto"
-	"github.com/portainer/portainer/file"
+	"github.com/portainer/portainer/docker"
+	"github.com/portainer/portainer/exec"
+	"github.com/portainer/portainer/filesystem"
+	"github.com/portainer/portainer/git"
 	"github.com/portainer/portainer/http"
+	"github.com/portainer/portainer/http/client"
 	"github.com/portainer/portainer/jwt"
+	"github.com/portainer/portainer/ldap"
+	"github.com/portainer/portainer/libcompose"
 
 	"log"
 )
@@ -28,15 +37,15 @@ func initCLI() *portainer.CLIFlags {
 }
 
 func initFileService(dataStorePath string) portainer.FileService {
-	fileService, err := file.NewService(dataStorePath, "")
+	fileService, err := filesystem.NewService(dataStorePath, "")
 	if err != nil {
 		log.Fatal(err)
 	}
 	return fileService
 }
 
-func initStore(dataStorePath string) *bolt.Store {
-	store, err := bolt.NewStore(dataStorePath)
+func initStore(dataStorePath string, fileService portainer.FileService) *bolt.Store {
+	store, err := bolt.NewStore(dataStorePath, fileService)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -46,6 +55,11 @@ func initStore(dataStorePath string) *bolt.Store {
 		log.Fatal(err)
 	}
 
+	err = store.Init()
+	if err != nil {
+		log.Fatal(err)
+	}
+
 	err = store.MigrateData()
 	if err != nil {
 		log.Fatal(err)
@@ -53,6 +67,14 @@ func initStore(dataStorePath string) *bolt.Store {
 	return store
 }
 
+func initComposeStackManager(dataStorePath string) portainer.ComposeStackManager {
+	return libcompose.NewComposeStackManager(dataStorePath)
+}
+
+func initSwarmStackManager(assetsPath string, dataStorePath string, signatureService portainer.DigitalSignatureService, fileService portainer.FileService) (portainer.SwarmStackManager, error) {
+	return exec.NewSwarmStackManager(assetsPath, dataStorePath, signatureService, fileService)
+}
+
 func initJWTService(authenticationEnabled bool) portainer.JWTService {
 	if authenticationEnabled {
 		jwtService, err := jwt.NewService()
@@ -64,45 +86,96 @@ func initJWTService(authenticationEnabled bool) portainer.JWTService {
 	return nil
 }
 
+func initDigitalSignatureService() portainer.DigitalSignatureService {
+	return &crypto.ECDSAService{}
+}
+
 func initCryptoService() portainer.CryptoService {
 	return &crypto.Service{}
 }
 
-func initEndpointWatcher(endpointService portainer.EndpointService, externalEnpointFile string, syncInterval string) bool {
-	authorizeEndpointMgmt := true
-	if externalEnpointFile != "" {
-		authorizeEndpointMgmt = false
-		log.Println("Using external endpoint definition. Endpoint management via the API will be disabled.")
-		endpointWatcher := cron.NewWatcher(endpointService, syncInterval)
-		err := endpointWatcher.WatchEndpointFile(externalEnpointFile)
-		if err != nil {
-			log.Fatal(err)
-		}
-	}
-	return authorizeEndpointMgmt
+func initLDAPService() portainer.LDAPService {
+	return &ldap.Service{}
 }
 
-func initStatus(authorizeEndpointMgmt bool, flags *portainer.CLIFlags) *portainer.Status {
+func initGitService() portainer.GitService {
+	return &git.Service{}
+}
+
+func initClientFactory(signatureService portainer.DigitalSignatureService) *docker.ClientFactory {
+	return docker.NewClientFactory(signatureService)
+}
+
+func initSnapshotter(clientFactory *docker.ClientFactory) portainer.Snapshotter {
+	return docker.NewSnapshotter(clientFactory)
+}
+
+func initJobScheduler(endpointService portainer.EndpointService, snapshotter portainer.Snapshotter, flags *portainer.CLIFlags) (portainer.JobScheduler, error) {
+	jobScheduler := cron.NewJobScheduler(endpointService, snapshotter)
+
+	if *flags.ExternalEndpoints != "" {
+		log.Println("Using external endpoint definition. Endpoint management via the API will be disabled.")
+		err := jobScheduler.ScheduleEndpointSyncJob(*flags.ExternalEndpoints, *flags.SyncInterval)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if *flags.Snapshot {
+		err := jobScheduler.ScheduleSnapshotJob(*flags.SnapshotInterval)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return jobScheduler, nil
+}
+
+func initStatus(endpointManagement, snapshot bool, flags *portainer.CLIFlags) *portainer.Status {
 	return &portainer.Status{
 		Analytics:          !*flags.NoAnalytics,
 		Authentication:     !*flags.NoAuth,
-		EndpointManagement: authorizeEndpointMgmt,
+		EndpointManagement: endpointManagement,
+		Snapshot:           snapshot,
 		Version:            portainer.APIVersion,
 	}
 }
 
+func initDockerHub(dockerHubService portainer.DockerHubService) error {
+	_, err := dockerHubService.DockerHub()
+	if err == portainer.ErrObjectNotFound {
+		dockerhub := &portainer.DockerHub{
+			Authentication: false,
+			Username:       "",
+			Password:       "",
+		}
+		return dockerHubService.UpdateDockerHub(dockerhub)
+	} else if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func initSettings(settingsService portainer.SettingsService, flags *portainer.CLIFlags) error {
 	_, err := settingsService.Settings()
-	if err == portainer.ErrSettingsNotFound {
+	if err == portainer.ErrObjectNotFound {
 		settings := &portainer.Settings{
-			LogoURL:                     *flags.Logo,
-			DisplayExternalContributors: true,
-		}
-
-		if *flags.Templates != "" {
-			settings.TemplatesURL = *flags.Templates
-		} else {
-			settings.TemplatesURL = portainer.DefaultTemplatesURL
+			LogoURL:              *flags.Logo,
+			AuthenticationMethod: portainer.AuthenticationInternal,
+			LDAPSettings: portainer.LDAPSettings{
+				AutoCreateUsers: true,
+				TLSConfig:       portainer.TLSConfiguration{},
+				SearchSettings: []portainer.LDAPSearchSettings{
+					portainer.LDAPSearchSettings{},
+				},
+				GroupSearchSettings: []portainer.LDAPGroupSearchSettings{
+					portainer.LDAPGroupSearchSettings{},
+				},
+			},
+			AllowBindMountsForRegularUsers:     true,
+			AllowPrivilegedModeForRegularUsers: true,
+			SnapshotInterval:                   *flags.SnapshotInterval,
 		}
 
 		if *flags.Labels != nil {
@@ -111,7 +184,7 @@ func initSettings(settingsService portainer.SettingsService, flags *portainer.CL
 			settings.BlackListedLabels = make([]portainer.Pair, 0)
 		}
 
-		return settingsService.StoreSettings(settings)
+		return settingsService.UpdateSettings(settings)
 	} else if err != nil {
 		return err
 	}
@@ -119,6 +192,58 @@ func initSettings(settingsService portainer.SettingsService, flags *portainer.CL
 	return nil
 }
 
+func initTemplates(templateService portainer.TemplateService, fileService portainer.FileService, templateURL, templateFile string) error {
+
+	existingTemplates, err := templateService.Templates()
+	if err != nil {
+		return err
+	}
+
+	if len(existingTemplates) != 0 {
+		log.Printf("Templates already registered inside the database. Skipping template import.")
+		return nil
+	}
+
+	var templatesJSON []byte
+	if templateURL == "" {
+		return loadTemplatesFromFile(fileService, templateService, templateFile)
+	}
+
+	templatesJSON, err = client.Get(templateURL)
+	if err != nil {
+		log.Println("Unable to retrieve templates via HTTP")
+		return err
+	}
+
+	return unmarshalAndPersistTemplates(templateService, templatesJSON)
+}
+
+func loadTemplatesFromFile(fileService portainer.FileService, templateService portainer.TemplateService, templateFile string) error {
+	templatesJSON, err := fileService.GetFileContent(templateFile)
+	if err != nil {
+		log.Println("Unable to retrieve template via filesystem")
+		return err
+	}
+	return unmarshalAndPersistTemplates(templateService, templatesJSON)
+}
+
+func unmarshalAndPersistTemplates(templateService portainer.TemplateService, templateData []byte) error {
+	var templates []portainer.Template
+	err := json.Unmarshal(templateData, &templates)
+	if err != nil {
+		log.Println("Unable to parse templates file. Please review your template definition file.")
+		return err
+	}
+
+	for _, template := range templates {
+		err := templateService.CreateTemplate(&template)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func retrieveFirstEndpointFromDatabase(endpointService portainer.EndpointService) *portainer.Endpoint {
 	endpoints, err := endpointService.Endpoints()
 	if err != nil {
@@ -127,63 +252,247 @@ func retrieveFirstEndpointFromDatabase(endpointService portainer.EndpointService
 	return &endpoints[0]
 }
 
+func loadAndParseKeyPair(fileService portainer.FileService, signatureService portainer.DigitalSignatureService) error {
+	private, public, err := fileService.LoadKeyPair()
+	if err != nil {
+		return err
+	}
+	return signatureService.ParseKeyPair(private, public)
+}
+
+func generateAndStoreKeyPair(fileService portainer.FileService, signatureService portainer.DigitalSignatureService) error {
+	private, public, err := signatureService.GenerateKeyPair()
+	if err != nil {
+		return err
+	}
+	privateHeader, publicHeader := signatureService.PEMHeaders()
+	return fileService.StoreKeyPair(private, public, privateHeader, publicHeader)
+}
+
+func initKeyPair(fileService portainer.FileService, signatureService portainer.DigitalSignatureService) error {
+	existingKeyPair, err := fileService.KeyPairFilesExist()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if existingKeyPair {
+		return loadAndParseKeyPair(fileService, signatureService)
+	}
+	return generateAndStoreKeyPair(fileService, signatureService)
+}
+
+func createTLSSecuredEndpoint(flags *portainer.CLIFlags, endpointService portainer.EndpointService, snapshotter portainer.Snapshotter) error {
+	tlsConfiguration := portainer.TLSConfiguration{
+		TLS:           *flags.TLS,
+		TLSSkipVerify: *flags.TLSSkipVerify,
+	}
+
+	if *flags.TLS {
+		tlsConfiguration.TLSCACertPath = *flags.TLSCacert
+		tlsConfiguration.TLSCertPath = *flags.TLSCert
+		tlsConfiguration.TLSKeyPath = *flags.TLSKey
+	} else if !*flags.TLS && *flags.TLSSkipVerify {
+		tlsConfiguration.TLS = true
+	}
+
+	endpointID := endpointService.GetNextIdentifier()
+	endpoint := &portainer.Endpoint{
+		ID:              portainer.EndpointID(endpointID),
+		Name:            "primary",
+		URL:             *flags.EndpointURL,
+		GroupID:         portainer.EndpointGroupID(1),
+		Type:            portainer.DockerEnvironment,
+		TLSConfig:       tlsConfiguration,
+		AuthorizedUsers: []portainer.UserID{},
+		AuthorizedTeams: []portainer.TeamID{},
+		Extensions:      []portainer.EndpointExtension{},
+		Tags:            []string{},
+		Status:          portainer.EndpointStatusUp,
+		Snapshots:       []portainer.Snapshot{},
+	}
+
+	if strings.HasPrefix(endpoint.URL, "tcp://") {
+		tlsConfig, err := crypto.CreateTLSConfigurationFromDisk(tlsConfiguration.TLSCACertPath, tlsConfiguration.TLSCertPath, tlsConfiguration.TLSKeyPath, tlsConfiguration.TLSSkipVerify)
+		if err != nil {
+			return err
+		}
+
+		agentOnDockerEnvironment, err := client.ExecutePingOperation(endpoint.URL, tlsConfig)
+		if err != nil {
+			return err
+		}
+
+		if agentOnDockerEnvironment {
+			endpoint.Type = portainer.AgentOnDockerEnvironment
+		}
+	}
+
+	return snapshotAndPersistEndpoint(endpoint, endpointService, snapshotter)
+}
+
+func createUnsecuredEndpoint(endpointURL string, endpointService portainer.EndpointService, snapshotter portainer.Snapshotter) error {
+	if strings.HasPrefix(endpointURL, "tcp://") {
+		_, err := client.ExecutePingOperation(endpointURL, nil)
+		if err != nil {
+			return err
+		}
+	}
+
+	endpointID := endpointService.GetNextIdentifier()
+	endpoint := &portainer.Endpoint{
+		ID:              portainer.EndpointID(endpointID),
+		Name:            "primary",
+		URL:             endpointURL,
+		GroupID:         portainer.EndpointGroupID(1),
+		Type:            portainer.DockerEnvironment,
+		TLSConfig:       portainer.TLSConfiguration{},
+		AuthorizedUsers: []portainer.UserID{},
+		AuthorizedTeams: []portainer.TeamID{},
+		Extensions:      []portainer.EndpointExtension{},
+		Tags:            []string{},
+		Status:          portainer.EndpointStatusUp,
+		Snapshots:       []portainer.Snapshot{},
+	}
+
+	return snapshotAndPersistEndpoint(endpoint, endpointService, snapshotter)
+}
+
+func snapshotAndPersistEndpoint(endpoint *portainer.Endpoint, endpointService portainer.EndpointService, snapshotter portainer.Snapshotter) error {
+	snapshot, err := snapshotter.CreateSnapshot(endpoint)
+	endpoint.Status = portainer.EndpointStatusUp
+	if err != nil {
+		log.Printf("http error: endpoint snapshot error (endpoint=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
+	}
+
+	if snapshot != nil {
+		endpoint.Snapshots = []portainer.Snapshot{*snapshot}
+	}
+
+	return endpointService.CreateEndpoint(endpoint)
+}
+
+func initEndpoint(flags *portainer.CLIFlags, endpointService portainer.EndpointService, snapshotter portainer.Snapshotter) error {
+	if *flags.EndpointURL == "" {
+		return nil
+	}
+
+	endpoints, err := endpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	if len(endpoints) > 0 {
+		log.Println("Instance already has defined endpoints. Skipping the endpoint defined via CLI.")
+		return nil
+	}
+
+	if *flags.TLS || *flags.TLSSkipVerify {
+		return createTLSSecuredEndpoint(flags, endpointService, snapshotter)
+	}
+	return createUnsecuredEndpoint(*flags.EndpointURL, endpointService, snapshotter)
+}
+
 func main() {
 	flags := initCLI()
 
 	fileService := initFileService(*flags.Data)
 
-	store := initStore(*flags.Data)
+	store := initStore(*flags.Data, fileService)
 	defer store.Close()
 
 	jwtService := initJWTService(!*flags.NoAuth)
 
+	ldapService := initLDAPService()
+
+	gitService := initGitService()
+
 	cryptoService := initCryptoService()
 
-	authorizeEndpointMgmt := initEndpointWatcher(store.EndpointService, *flags.ExternalEndpoints, *flags.SyncInterval)
+	digitalSignatureService := initDigitalSignatureService()
 
-	err := initSettings(store.SettingsService, flags)
+	err := initKeyPair(fileService, digitalSignatureService)
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	applicationStatus := initStatus(authorizeEndpointMgmt, flags)
+	clientFactory := initClientFactory(digitalSignatureService)
 
-	if *flags.Endpoint != "" {
-		var endpoints []portainer.Endpoint
-		endpoints, err := store.EndpointService.Endpoints()
+	snapshotter := initSnapshotter(clientFactory)
+
+	jobScheduler, err := initJobScheduler(store.EndpointService, snapshotter, flags)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	jobScheduler.Start()
+
+	endpointManagement := true
+	if *flags.ExternalEndpoints != "" {
+		endpointManagement = false
+	}
+
+	swarmStackManager, err := initSwarmStackManager(*flags.Assets, *flags.Data, digitalSignatureService, fileService)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	composeStackManager := initComposeStackManager(*flags.Data)
+
+	err = initTemplates(store.TemplateService, fileService, *flags.Templates, *flags.TemplateFile)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	err = initSettings(store.SettingsService, flags)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	err = initDockerHub(store.DockerHubService)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	applicationStatus := initStatus(endpointManagement, *flags.Snapshot, flags)
+
+	err = initEndpoint(flags, store.EndpointService, snapshotter)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	adminPasswordHash := ""
+	if *flags.AdminPasswordFile != "" {
+		content, err := fileService.GetFileContent(*flags.AdminPasswordFile)
 		if err != nil {
 			log.Fatal(err)
 		}
-		if len(endpoints) == 0 {
-			endpoint := &portainer.Endpoint{
-				Name:            "primary",
-				URL:             *flags.Endpoint,
-				TLS:             *flags.TLSVerify,
-				TLSCACertPath:   *flags.TLSCacert,
-				TLSCertPath:     *flags.TLSCert,
-				TLSKeyPath:      *flags.TLSKey,
-				AuthorizedUsers: []portainer.UserID{},
-				AuthorizedTeams: []portainer.TeamID{},
+		adminPasswordHash, err = cryptoService.Hash(string(content))
+		if err != nil {
+			log.Fatal(err)
+		}
+	} else if *flags.AdminPassword != "" {
+		adminPasswordHash = *flags.AdminPassword
+	}
+
+	if adminPasswordHash != "" {
+		users, err := store.UserService.UsersByRole(portainer.AdministratorRole)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		if len(users) == 0 {
+			log.Printf("Creating admin user with password hash %s", adminPasswordHash)
+			user := &portainer.User{
+				Username: "admin",
+				Role:     portainer.AdministratorRole,
+				Password: adminPasswordHash,
 			}
-			err = store.EndpointService.CreateEndpoint(endpoint)
+			err := store.UserService.CreateUser(user)
 			if err != nil {
 				log.Fatal(err)
 			}
 		} else {
-			log.Println("Instance already has defined endpoints. Skipping the endpoint defined via CLI.")
-		}
-	}
-
-	if *flags.AdminPassword != "" {
-		log.Printf("Creating admin user with password hash %s", *flags.AdminPassword)
-		user := &portainer.User{
-			Username: "admin",
-			Role:     portainer.AdministratorRole,
-			Password: *flags.AdminPassword,
-		}
-		err := store.UserService.CreateUser(user)
-		if err != nil {
-			log.Fatal(err)
+			log.Println("Instance already has an administrator user defined. Skipping admin password related flags.")
 		}
 	}
 
@@ -192,22 +501,35 @@ func main() {
 		BindAddress:            *flags.Addr,
 		AssetsPath:             *flags.Assets,
 		AuthDisabled:           *flags.NoAuth,
-		EndpointManagement:     authorizeEndpointMgmt,
+		EndpointManagement:     endpointManagement,
 		UserService:            store.UserService,
 		TeamService:            store.TeamService,
 		TeamMembershipService:  store.TeamMembershipService,
 		EndpointService:        store.EndpointService,
+		EndpointGroupService:   store.EndpointGroupService,
 		ResourceControlService: store.ResourceControlService,
 		SettingsService:        store.SettingsService,
+		RegistryService:        store.RegistryService,
+		DockerHubService:       store.DockerHubService,
+		StackService:           store.StackService,
+		TagService:             store.TagService,
+		TemplateService:        store.TemplateService,
+		SwarmStackManager:      swarmStackManager,
+		ComposeStackManager:    composeStackManager,
 		CryptoService:          cryptoService,
 		JWTService:             jwtService,
 		FileService:            fileService,
+		LDAPService:            ldapService,
+		GitService:             gitService,
+		SignatureService:       digitalSignatureService,
+		JobScheduler:           jobScheduler,
+		Snapshotter:            snapshotter,
 		SSL:                    *flags.SSL,
 		SSLCert:                *flags.SSLCert,
 		SSLKey:                 *flags.SSLKey,
 	}
 
-	log.Printf("Starting Portainer on %s", *flags.Addr)
+	log.Printf("Starting Portainer %s on %s", portainer.APIVersion, *flags.Addr)
 	err = server.Start()
 	if err != nil {
 		log.Fatal(err)

api/cron/job_endpoint_snapshot.go

@@ -0,0 +1,60 @@
+package cron
+
+import (
+	"log"
+
+	"github.com/portainer/portainer"
+)
+
+type (
+	endpointSnapshotJob struct {
+		endpointService portainer.EndpointService
+		snapshotter     portainer.Snapshotter
+	}
+)
+
+func newEndpointSnapshotJob(endpointService portainer.EndpointService, snapshotter portainer.Snapshotter) endpointSnapshotJob {
+	return endpointSnapshotJob{
+		endpointService: endpointService,
+		snapshotter:     snapshotter,
+	}
+}
+
+func (job endpointSnapshotJob) Snapshot() error {
+
+	endpoints, err := job.endpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, endpoint := range endpoints {
+		if endpoint.Type == portainer.AzureEnvironment {
+			continue
+		}
+
+		snapshot, err := job.snapshotter.CreateSnapshot(&endpoint)
+		endpoint.Status = portainer.EndpointStatusUp
+		if err != nil {
+			log.Printf("cron error: endpoint snapshot error (endpoint=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
+			endpoint.Status = portainer.EndpointStatusDown
+		}
+
+		if snapshot != nil {
+			endpoint.Snapshots = []portainer.Snapshot{*snapshot}
+		}
+
+		err = job.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (job endpointSnapshotJob) Run() {
+	err := job.Snapshot()
+	if err != nil {
+		log.Printf("cron error: snapshot job error (err=%s)\n", err)
+	}
+}

api/cron/job_endpoint_sync.go

@@ -4,7 +4,6 @@ import (
 	"encoding/json"
 	"io/ioutil"
 	"log"
-	"os"
 	"strings"
 
 	"github.com/portainer/portainer"
@@ -12,7 +11,6 @@ import (
 
 type (
 	endpointSyncJob struct {
-		logger           *log.Logger
 		endpointService  portainer.EndpointService
 		endpointFilePath string
 	}
@@ -22,6 +20,16 @@ type (
 		endpointsToUpdate []*portainer.Endpoint
 		endpointsToDelete []*portainer.Endpoint
 	}
+
+	fileEndpoint struct {
+		Name          string `json:"Name"`
+		URL           string `json:"URL"`
+		TLS           bool   `json:"TLS,omitempty"`
+		TLSSkipVerify bool   `json:"TLSSkipVerify,omitempty"`
+		TLSCACert     string `json:"TLSCACert,omitempty"`
+		TLSCert       string `json:"TLSCert,omitempty"`
+		TLSKey        string `json:"TLSKey,omitempty"`
+	}
 )
 
 const (
@@ -31,15 +39,14 @@ const (
 
 func newEndpointSyncJob(endpointFilePath string, endpointService portainer.EndpointService) endpointSyncJob {
 	return endpointSyncJob{
-		logger:           log.New(os.Stderr, "", log.LstdFlags),
 		endpointService:  endpointService,
 		endpointFilePath: endpointFilePath,
 	}
 }
 
-func endpointSyncError(err error, logger *log.Logger) bool {
+func endpointSyncError(err error) bool {
 	if err != nil {
-		logger.Printf("Endpoint synchronization error: %s", err)
+		log.Printf("cron error: synchronization job error (err=%s)\n", err)
 		return true
 	}
 	return false
@@ -55,6 +62,28 @@ func isValidEndpoint(endpoint *portainer.Endpoint) bool {
 	return false
 }
 
+func convertFileEndpoints(fileEndpoints []fileEndpoint) []portainer.Endpoint {
+	convertedEndpoints := make([]portainer.Endpoint, 0)
+
+	for _, e := range fileEndpoints {
+		endpoint := portainer.Endpoint{
+			Name:      e.Name,
+			URL:       e.URL,
+			TLSConfig: portainer.TLSConfiguration{},
+		}
+		if e.TLS {
+			endpoint.TLSConfig.TLS = true
+			endpoint.TLSConfig.TLSSkipVerify = e.TLSSkipVerify
+			endpoint.TLSConfig.TLSCACertPath = e.TLSCACert
+			endpoint.TLSConfig.TLSCertPath = e.TLSCert
+			endpoint.TLSConfig.TLSKeyPath = e.TLSKey
+		}
+		convertedEndpoints = append(convertedEndpoints, endpoint)
+	}
+
+	return convertedEndpoints
+}
+
 func endpointExists(endpoint *portainer.Endpoint, endpoints []portainer.Endpoint) int {
 	for idx, v := range endpoints {
 		if endpoint.Name == v.Name && isValidEndpoint(&v) {
@@ -66,22 +95,25 @@ func endpointExists(endpoint *portainer.Endpoint, endpoints []portainer.Endpoint
 
 func mergeEndpointIfRequired(original, updated *portainer.Endpoint) *portainer.Endpoint {
 	var endpoint *portainer.Endpoint
-	if original.URL != updated.URL || original.TLS != updated.TLS ||
-		(updated.TLS && original.TLSCACertPath != updated.TLSCACertPath) ||
-		(updated.TLS && original.TLSCertPath != updated.TLSCertPath) ||
-		(updated.TLS && original.TLSKeyPath != updated.TLSKeyPath) {
+	if original.URL != updated.URL || original.TLSConfig.TLS != updated.TLSConfig.TLS ||
+		(updated.TLSConfig.TLS && original.TLSConfig.TLSSkipVerify != updated.TLSConfig.TLSSkipVerify) ||
+		(updated.TLSConfig.TLS && original.TLSConfig.TLSCACertPath != updated.TLSConfig.TLSCACertPath) ||
+		(updated.TLSConfig.TLS && original.TLSConfig.TLSCertPath != updated.TLSConfig.TLSCertPath) ||
+		(updated.TLSConfig.TLS && original.TLSConfig.TLSKeyPath != updated.TLSConfig.TLSKeyPath) {
 		endpoint = original
 		endpoint.URL = updated.URL
-		if updated.TLS {
-			endpoint.TLS = true
-			endpoint.TLSCACertPath = updated.TLSCACertPath
-			endpoint.TLSCertPath = updated.TLSCertPath
-			endpoint.TLSKeyPath = updated.TLSKeyPath
+		if updated.TLSConfig.TLS {
+			endpoint.TLSConfig.TLS = true
+			endpoint.TLSConfig.TLSSkipVerify = updated.TLSConfig.TLSSkipVerify
+			endpoint.TLSConfig.TLSCACertPath = updated.TLSConfig.TLSCACertPath
+			endpoint.TLSConfig.TLSCertPath = updated.TLSConfig.TLSCertPath
+			endpoint.TLSConfig.TLSKeyPath = updated.TLSConfig.TLSKeyPath
 		} else {
-			endpoint.TLS = false
-			endpoint.TLSCACertPath = ""
-			endpoint.TLSCertPath = ""
-			endpoint.TLSKeyPath = ""
+			endpoint.TLSConfig.TLS = false
+			endpoint.TLSConfig.TLSSkipVerify = false
+			endpoint.TLSConfig.TLSCACertPath = ""
+			endpoint.TLSConfig.TLSCertPath = ""
+			endpoint.TLSConfig.TLSKeyPath = ""
 		}
 	}
 	return endpoint
@@ -105,25 +137,23 @@ func (job endpointSyncJob) prepareSyncData(storedEndpoints, fileEndpoints []port
 		if fidx != -1 {
 			endpoint := mergeEndpointIfRequired(&storedEndpoints[idx], &fileEndpoints[fidx])
 			if endpoint != nil {
-				job.logger.Printf("New definition for a stored endpoint found in file, updating database. [name: %v] [url: %v]\n", endpoint.Name, endpoint.URL)
+				log.Printf("New definition for a stored endpoint found in file, updating database. [name: %v] [url: %v]\n", endpoint.Name, endpoint.URL)
 				endpointsToUpdate = append(endpointsToUpdate, endpoint)
-			} else {
-				job.logger.Printf("No change detected for a stored endpoint. [name: %v] [url: %v]\n", storedEndpoints[idx].Name, storedEndpoints[idx].URL)
 			}
 		} else {
-			job.logger.Printf("Stored endpoint not found in file (definition might be invalid), removing from database. [name: %v] [url: %v]", storedEndpoints[idx].Name, storedEndpoints[idx].URL)
+			log.Printf("Stored endpoint not found in file (definition might be invalid), removing from database. [name: %v] [url: %v]", storedEndpoints[idx].Name, storedEndpoints[idx].URL)
 			endpointsToDelete = append(endpointsToDelete, &storedEndpoints[idx])
 		}
 	}
 
 	for idx, endpoint := range fileEndpoints {
-		if endpoint.Name == "" || endpoint.URL == "" {
-			job.logger.Printf("Invalid file endpoint definition, skipping. [name: %v] [url: %v]", endpoint.Name, endpoint.URL)
+		if !isValidEndpoint(&endpoint) {
+			log.Printf("Invalid file endpoint definition, skipping. [name: %v] [url: %v]", endpoint.Name, endpoint.URL)
 			continue
 		}
 		sidx := endpointExists(&fileEndpoints[idx], storedEndpoints)
 		if sidx == -1 {
-			job.logger.Printf("File endpoint not found in database, adding to database. [name: %v] [url: %v]", fileEndpoints[idx].Name, fileEndpoints[idx].URL)
+			log.Printf("File endpoint not found in database, adding to database. [name: %v] [url: %v]", fileEndpoints[idx].Name, fileEndpoints[idx].URL)
 			endpointsToCreate = append(endpointsToCreate, &fileEndpoints[idx])
 		}
 	}
@@ -137,13 +167,13 @@ func (job endpointSyncJob) prepareSyncData(storedEndpoints, fileEndpoints []port
 
 func (job endpointSyncJob) Sync() error {
 	data, err := ioutil.ReadFile(job.endpointFilePath)
-	if endpointSyncError(err, job.logger) {
+	if endpointSyncError(err) {
 		return err
 	}
 
-	var fileEndpoints []portainer.Endpoint
+	var fileEndpoints []fileEndpoint
 	err = json.Unmarshal(data, &fileEndpoints)
-	if endpointSyncError(err, job.logger) {
+	if endpointSyncError(err) {
 		return err
 	}
 
@@ -152,23 +182,25 @@ func (job endpointSyncJob) Sync() error {
 	}
 
 	storedEndpoints, err := job.endpointService.Endpoints()
-	if endpointSyncError(err, job.logger) {
+	if endpointSyncError(err) {
 		return err
 	}
 
-	sync := job.prepareSyncData(storedEndpoints, fileEndpoints)
+	convertedFileEndpoints := convertFileEndpoints(fileEndpoints)
+
+	sync := job.prepareSyncData(storedEndpoints, convertedFileEndpoints)
 	if sync.requireSync() {
 		err = job.endpointService.Synchronize(sync.endpointsToCreate, sync.endpointsToUpdate, sync.endpointsToDelete)
-		if endpointSyncError(err, job.logger) {
+		if endpointSyncError(err) {
 			return err
 		}
-		job.logger.Printf("Endpoint synchronization ended. [created: %v] [updated: %v] [deleted: %v]", len(sync.endpointsToCreate), len(sync.endpointsToUpdate), len(sync.endpointsToDelete))
+		log.Printf("Endpoint synchronization ended. [created: %v] [updated: %v] [deleted: %v]", len(sync.endpointsToCreate), len(sync.endpointsToUpdate), len(sync.endpointsToDelete))
 	}
 	return nil
 }
 
 func (job endpointSyncJob) Run() {
-	job.logger.Println("Endpoint synchronization job started.")
+	log.Println("cron: synchronization job started")
 	err := job.Sync()
-	endpointSyncError(err, job.logger)
+	endpointSyncError(err)
 }

api/cron/scheduler.go

@@ -0,0 +1,86 @@
+package cron
+
+import (
+	"log"
+
+	"github.com/portainer/portainer"
+	"github.com/robfig/cron"
+)
+
+// JobScheduler represents a service for managing crons.
+type JobScheduler struct {
+	cron            *cron.Cron
+	endpointService portainer.EndpointService
+	snapshotter     portainer.Snapshotter
+
+	endpointFilePath     string
+	endpointSyncInterval string
+}
+
+// NewJobScheduler initializes a new service.
+func NewJobScheduler(endpointService portainer.EndpointService, snapshotter portainer.Snapshotter) *JobScheduler {
+	return &JobScheduler{
+		cron:            cron.New(),
+		endpointService: endpointService,
+		snapshotter:     snapshotter,
+	}
+}
+
+// ScheduleEndpointSyncJob schedules a cron job to synchronize the endpoints from a file
+func (scheduler *JobScheduler) ScheduleEndpointSyncJob(endpointFilePath string, interval string) error {
+
+	scheduler.endpointFilePath = endpointFilePath
+	scheduler.endpointSyncInterval = interval
+
+	job := newEndpointSyncJob(endpointFilePath, scheduler.endpointService)
+
+	err := job.Sync()
+	if err != nil {
+		return err
+	}
+
+	return scheduler.cron.AddJob("@every "+interval, job)
+}
+
+// ScheduleSnapshotJob schedules a cron job to create endpoint snapshots
+func (scheduler *JobScheduler) ScheduleSnapshotJob(interval string) error {
+	job := newEndpointSnapshotJob(scheduler.endpointService, scheduler.snapshotter)
+
+	err := job.Snapshot()
+	if err != nil {
+		return err
+	}
+
+	return scheduler.cron.AddJob("@every "+interval, job)
+}
+
+// UpdateSnapshotJob will update the schedules to match the new snapshot interval
+func (scheduler *JobScheduler) UpdateSnapshotJob(interval string) {
+	// TODO: the cron library do not support removing/updating schedules.
+	// As a work-around we need to re-create the cron and reschedule the jobs.
+	// We should update the library.
+	jobs := scheduler.cron.Entries()
+	scheduler.cron.Stop()
+
+	scheduler.cron = cron.New()
+
+	for _, job := range jobs {
+		switch job.Job.(type) {
+		case endpointSnapshotJob:
+			scheduler.ScheduleSnapshotJob(interval)
+		case endpointSyncJob:
+			scheduler.ScheduleEndpointSyncJob(scheduler.endpointFilePath, scheduler.endpointSyncInterval)
+		default:
+			log.Println("Unsupported job")
+		}
+	}
+
+	scheduler.cron.Start()
+}
+
+// Start starts the scheduled jobs
+func (scheduler *JobScheduler) Start() {
+	if len(scheduler.cron.Entries()) > 0 {
+		scheduler.cron.Start()
+	}
+}

api/cron/watcher.go

@@ -1,40 +0,0 @@
-package cron
-
-import (
-	"github.com/portainer/portainer"
-	"github.com/robfig/cron"
-)
-
-// Watcher represents a service for managing crons.
-type Watcher struct {
-	Cron            *cron.Cron
-	EndpointService portainer.EndpointService
-	syncInterval    string
-}
-
-// NewWatcher initializes a new service.
-func NewWatcher(endpointService portainer.EndpointService, syncInterval string) *Watcher {
-	return &Watcher{
-		Cron:            cron.New(),
-		EndpointService: endpointService,
-		syncInterval:    syncInterval,
-	}
-}
-
-// WatchEndpointFile starts a cron job to synchronize the endpoints from a file
-func (watcher *Watcher) WatchEndpointFile(endpointFilePath string) error {
-	job := newEndpointSyncJob(endpointFilePath, watcher.EndpointService)
-
-	err := job.Sync()
-	if err != nil {
-		return err
-	}
-
-	err = watcher.Cron.AddJob("@every "+watcher.syncInterval, job)
-	if err != nil {
-		return err
-	}
-
-	watcher.Cron.Start()
-	return nil
-}

api/crypto/ecdsa.go

@@ -0,0 +1,125 @@
+package crypto
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/md5"
+	"crypto/rand"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/hex"
+	"math/big"
+)
+
+const (
+	// PrivateKeyPemHeader represents the header that is appended to the PEM file when
+	// storing the private key.
+	PrivateKeyPemHeader = "EC PRIVATE KEY"
+	// PublicKeyPemHeader represents the header that is appended to the PEM file when
+	// storing the public key.
+	PublicKeyPemHeader = "ECDSA PUBLIC KEY"
+)
+
+// ECDSAService is a service used to create digital signatures when communicating with
+// an agent based environment. It will automatically generates a key pair using ECDSA or
+// can also reuse an existing ECDSA key pair.
+type ECDSAService struct {
+	privateKey    *ecdsa.PrivateKey
+	publicKey     *ecdsa.PublicKey
+	encodedPubKey string
+}
+
+// EncodedPublicKey returns the encoded version of the public that can be used
+// to be shared with other services. It's the hexadecimal encoding of the public key
+// content.
+func (service *ECDSAService) EncodedPublicKey() string {
+	return service.encodedPubKey
+}
+
+// PEMHeaders returns the ECDSA PEM headers.
+func (service *ECDSAService) PEMHeaders() (string, string) {
+	return PrivateKeyPemHeader, PublicKeyPemHeader
+}
+
+// ParseKeyPair parses existing private/public key pair content and associate
+// the parsed keys to the service.
+func (service *ECDSAService) ParseKeyPair(private, public []byte) error {
+	privateKey, err := x509.ParseECPrivateKey(private)
+	if err != nil {
+		return err
+	}
+
+	service.privateKey = privateKey
+
+	encodedKey := hex.EncodeToString(public)
+	service.encodedPubKey = encodedKey
+
+	publicKey, err := x509.ParsePKIXPublicKey(public)
+	if err != nil {
+		return err
+	}
+
+	service.publicKey = publicKey.(*ecdsa.PublicKey)
+
+	return nil
+}
+
+// GenerateKeyPair will create a new key pair using ECDSA.
+func (service *ECDSAService) GenerateKeyPair() ([]byte, []byte, error) {
+	pubkeyCurve := elliptic.P256()
+
+	privatekey, err := ecdsa.GenerateKey(pubkeyCurve, rand.Reader)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	service.privateKey = privatekey
+	service.publicKey = &privatekey.PublicKey
+
+	private, err := x509.MarshalECPrivateKey(service.privateKey)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	public, err := x509.MarshalPKIXPublicKey(service.publicKey)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	encodedKey := hex.EncodeToString(public)
+	service.encodedPubKey = encodedKey
+
+	return private, public, nil
+}
+
+// Sign creates a signature from a message.
+// It automatically hash the message using MD5 and creates a signature from
+// that hash.
+// It then encodes the generated signature in base64.
+func (service *ECDSAService) Sign(message string) (string, error) {
+	digest := md5.New()
+	digest.Write([]byte(message))
+	hash := digest.Sum(nil)
+
+	r := big.NewInt(0)
+	s := big.NewInt(0)
+
+	r, s, err := ecdsa.Sign(rand.Reader, service.privateKey, hash)
+	if err != nil {
+		return "", err
+	}
+
+	keyBytes := service.privateKey.Params().BitSize / 8
+
+	rBytes := r.Bytes()
+	rBytesPadded := make([]byte, keyBytes)
+	copy(rBytesPadded[keyBytes-len(rBytes):], rBytes)
+
+	sBytes := s.Bytes()
+	sBytesPadded := make([]byte, keyBytes)
+	copy(sBytesPadded[keyBytes-len(sBytes):], sBytes)
+
+	signature := append(rBytesPadded, sBytesPadded...)
+
+	return base64.RawStdEncoding.EncodeToString(signature), nil
+}

api/crypto/tls.go

@@ -6,21 +6,54 @@ import (
 	"io/ioutil"
 )
 
-// CreateTLSConfiguration initializes a tls.Config using a CA certificate, a certificate and a key
-func CreateTLSConfiguration(caCertPath, certPath, keyPath string) (*tls.Config, error) {
-	cert, err := tls.LoadX509KeyPair(certPath, keyPath)
-	if err != nil {
-		return nil, err
+// CreateTLSConfigurationFromBytes initializes a tls.Config using a CA certificate, a certificate and a key
+// loaded from memory.
+func CreateTLSConfigurationFromBytes(caCert, cert, key []byte, skipClientVerification, skipServerVerification bool) (*tls.Config, error) {
+	config := &tls.Config{}
+	config.InsecureSkipVerify = skipServerVerification
+
+	if !skipClientVerification {
+		certificate, err := tls.X509KeyPair(cert, key)
+		if err != nil {
+			return nil, err
+		}
+		config.Certificates = []tls.Certificate{certificate}
 	}
-	caCert, err := ioutil.ReadFile(caCertPath)
-	if err != nil {
-		return nil, err
-	}
-	caCertPool := x509.NewCertPool()
-	caCertPool.AppendCertsFromPEM(caCert)
-	config := &tls.Config{
-		Certificates: []tls.Certificate{cert},
-		RootCAs:      caCertPool,
+
+	if !skipServerVerification {
+		caCertPool := x509.NewCertPool()
+		caCertPool.AppendCertsFromPEM(caCert)
+		config.RootCAs = caCertPool
 	}
+
+	return config, nil
+}
+
+// CreateTLSConfigurationFromDisk initializes a tls.Config using a CA certificate, a certificate and a key
+// loaded from disk.
+func CreateTLSConfigurationFromDisk(caCertPath, certPath, keyPath string, skipServerVerification bool) (*tls.Config, error) {
+	config := &tls.Config{}
+	config.InsecureSkipVerify = skipServerVerification
+
+	if certPath != "" && keyPath != "" {
+		cert, err := tls.LoadX509KeyPair(certPath, keyPath)
+		if err != nil {
+			return nil, err
+		}
+
+		config.Certificates = []tls.Certificate{cert}
+	}
+
+	if !skipServerVerification && caCertPath != "" {
+		caCert, err := ioutil.ReadFile(caCertPath)
+		if err != nil {
+			return nil, err
+		}
+
+		caCertPool := x509.NewCertPool()
+		caCertPool.AppendCertsFromPEM(caCert)
+		config.RootCAs = caCertPool
+	}
+
 	return config, nil
 }

api/docker/client.go

@@ -0,0 +1,103 @@
+package docker
+
+import (
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/docker/docker/client"
+	"github.com/portainer/portainer"
+	"github.com/portainer/portainer/crypto"
+)
+
+const (
+	unsupportedEnvironmentType = portainer.Error("Environment not supported")
+)
+
+// ClientFactory is used to create Docker clients
+type ClientFactory struct {
+	signatureService portainer.DigitalSignatureService
+}
+
+// NewClientFactory returns a new instance of a ClientFactory
+func NewClientFactory(signatureService portainer.DigitalSignatureService) *ClientFactory {
+	return &ClientFactory{
+		signatureService: signatureService,
+	}
+}
+
+// CreateClient is a generic function to create a Docker client based on
+// a specific endpoint configuration
+func (factory *ClientFactory) CreateClient(endpoint *portainer.Endpoint) (*client.Client, error) {
+	if endpoint.Type == portainer.AzureEnvironment {
+		return nil, unsupportedEnvironmentType
+	} else if endpoint.Type == portainer.AgentOnDockerEnvironment {
+		return createAgentClient(endpoint, factory.signatureService)
+	}
+
+	if strings.HasPrefix(endpoint.URL, "unix://") || strings.HasPrefix(endpoint.URL, "npipe://") {
+		return createLocalClient(endpoint)
+	}
+	return createTCPClient(endpoint)
+}
+
+func createLocalClient(endpoint *portainer.Endpoint) (*client.Client, error) {
+	return client.NewClientWithOpts(
+		client.WithHost(endpoint.URL),
+		client.WithVersion(portainer.SupportedDockerAPIVersion),
+	)
+}
+
+func createTCPClient(endpoint *portainer.Endpoint) (*client.Client, error) {
+	httpCli, err := httpClient(endpoint)
+	if err != nil {
+		return nil, err
+	}
+
+	return client.NewClientWithOpts(
+		client.WithHost(endpoint.URL),
+		client.WithVersion(portainer.SupportedDockerAPIVersion),
+		client.WithHTTPClient(httpCli),
+	)
+}
+
+func createAgentClient(endpoint *portainer.Endpoint, signatureService portainer.DigitalSignatureService) (*client.Client, error) {
+	httpCli, err := httpClient(endpoint)
+	if err != nil {
+		return nil, err
+	}
+
+	signature, err := signatureService.Sign(portainer.PortainerAgentSignatureMessage)
+	if err != nil {
+		return nil, err
+	}
+
+	headers := map[string]string{
+		portainer.PortainerAgentPublicKeyHeader: signatureService.EncodedPublicKey(),
+		portainer.PortainerAgentSignatureHeader: signature,
+	}
+
+	return client.NewClientWithOpts(
+		client.WithHost(endpoint.URL),
+		client.WithVersion(portainer.SupportedDockerAPIVersion),
+		client.WithHTTPClient(httpCli),
+		client.WithHTTPHeaders(headers),
+	)
+}
+
+func httpClient(endpoint *portainer.Endpoint) (*http.Client, error) {
+	transport := &http.Transport{}
+
+	if endpoint.TLSConfig.TLS {
+		tlsConfig, err := crypto.CreateTLSConfigurationFromDisk(endpoint.TLSConfig.TLSCACertPath, endpoint.TLSConfig.TLSCertPath, endpoint.TLSConfig.TLSKeyPath, endpoint.TLSConfig.TLSSkipVerify)
+		if err != nil {
+			return nil, err
+		}
+		transport.TLSClientConfig = tlsConfig
+	}
+
+	return &http.Client{
+		Timeout:   time.Second * 10,
+		Transport: transport,
+	}, nil
+}

api/docker/snapshot.go

@@ -0,0 +1,135 @@
+package docker
+
+import (
+	"context"
+	"time"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/client"
+	"github.com/portainer/portainer"
+)
+
+func snapshot(cli *client.Client) (*portainer.Snapshot, error) {
+	_, err := cli.Ping(context.Background())
+	if err != nil {
+		return nil, err
+	}
+
+	snapshot := &portainer.Snapshot{
+		StackCount: 0,
+	}
+
+	err = snapshotInfo(snapshot, cli)
+	if err != nil {
+		return nil, err
+	}
+
+	if snapshot.Swarm {
+		err = snapshotSwarmServices(snapshot, cli)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	err = snapshotContainers(snapshot, cli)
+	if err != nil {
+		return nil, err
+	}
+
+	err = snapshotImages(snapshot, cli)
+	if err != nil {
+		return nil, err
+	}
+
+	err = snapshotVolumes(snapshot, cli)
+	if err != nil {
+		return nil, err
+	}
+
+	snapshot.Time = time.Now().Unix()
+	return snapshot, nil
+}
+
+func snapshotInfo(snapshot *portainer.Snapshot, cli *client.Client) error {
+	info, err := cli.Info(context.Background())
+	if err != nil {
+		return err
+	}
+
+	snapshot.Swarm = info.Swarm.ControlAvailable
+	snapshot.DockerVersion = info.ServerVersion
+	snapshot.TotalCPU = info.NCPU
+	snapshot.TotalMemory = info.MemTotal
+	return nil
+}
+
+func snapshotSwarmServices(snapshot *portainer.Snapshot, cli *client.Client) error {
+	stacks := make(map[string]struct{})
+
+	services, err := cli.ServiceList(context.Background(), types.ServiceListOptions{})
+	if err != nil {
+		return err
+	}
+
+	for _, service := range services {
+		for k, v := range service.Spec.Labels {
+			if k == "com.docker.stack.namespace" {
+				stacks[v] = struct{}{}
+			}
+		}
+	}
+
+	snapshot.ServiceCount = len(services)
+	snapshot.StackCount += len(stacks)
+	return nil
+}
+
+func snapshotContainers(snapshot *portainer.Snapshot, cli *client.Client) error {
+	containers, err := cli.ContainerList(context.Background(), types.ContainerListOptions{All: true})
+	if err != nil {
+		return err
+	}
+
+	runningContainers := 0
+	stoppedContainers := 0
+	stacks := make(map[string]struct{})
+	for _, container := range containers {
+		if container.State == "exited" {
+			stoppedContainers++
+		} else if container.State == "running" {
+			runningContainers++
+		}
+
+		for k, v := range container.Labels {
+			if k == "com.docker.compose.project" {
+				stacks[v] = struct{}{}
+			}
+		}
+	}
+
+	snapshot.RunningContainerCount = runningContainers
+	snapshot.StoppedContainerCount = stoppedContainers
+	snapshot.StackCount += len(stacks)
+	return nil
+}
+
+func snapshotImages(snapshot *portainer.Snapshot, cli *client.Client) error {
+	images, err := cli.ImageList(context.Background(), types.ImageListOptions{})
+	if err != nil {
+		return err
+	}
+
+	snapshot.ImageCount = len(images)
+	return nil
+}
+
+func snapshotVolumes(snapshot *portainer.Snapshot, cli *client.Client) error {
+	volumes, err := cli.VolumeList(context.Background(), filters.Args{})
+	if err != nil {
+		return err
+	}
+
+	snapshot.VolumeCount = len(volumes.Volumes)
+	return nil
+}

api/docker/snapshotter.go

@@ -0,0 +1,27 @@
+package docker
+
+import (
+	"github.com/portainer/portainer"
+)
+
+// Snapshotter represents a service used to create endpoint snapshots
+type Snapshotter struct {
+	clientFactory *ClientFactory
+}
+
+// NewSnapshotter returns a new Snapshotter instance
+func NewSnapshotter(clientFactory *ClientFactory) *Snapshotter {
+	return &Snapshotter{
+		clientFactory: clientFactory,
+	}
+}
+
+// CreateSnapshot creates a snapshot of a specific endpoint
+func (snapshotter *Snapshotter) CreateSnapshot(endpoint *portainer.Endpoint) (*portainer.Snapshot, error) {
+	cli, err := snapshotter.clientFactory.CreateClient(endpoint)
+	if err != nil {
+		return nil, err
+	}
+
+	return snapshot(cli)
+}

api/errors.go

@@ -4,52 +4,71 @@ package portainer
 const (
 	ErrUnauthorized           = Error("Unauthorized")
 	ErrResourceAccessDenied   = Error("Access denied to resource")
-	ErrResourceNotFound       = Error("Unable to find resource")
-	ErrUnsupportedDockerAPI   = Error("Unsupported Docker API response")
+	ErrObjectNotFound         = Error("Object not found inside the database")
 	ErrMissingSecurityContext = Error("Unable to find security details in request context")
 )
 
 // User errors.
 const (
-	ErrUserNotFound            = Error("User not found")
-	ErrUserAlreadyExists       = Error("User already exists")
-	ErrInvalidUsername         = Error("Invalid username. White spaces are not allowed.")
-	ErrAdminAlreadyInitialized = Error("Admin user already initialized")
+	ErrUserAlreadyExists          = Error("User already exists")
+	ErrInvalidUsername            = Error("Invalid username. White spaces are not allowed")
+	ErrAdminAlreadyInitialized    = Error("An administrator user already exists")
+	ErrAdminCannotRemoveSelf      = Error("Cannot remove your own user account. Contact another administrator")
+	ErrCannotRemoveLastLocalAdmin = Error("Cannot remove the last local administrator account")
 )
 
 // Team errors.
 const (
-	ErrTeamNotFound      = Error("Team not found")
 	ErrTeamAlreadyExists = Error("Team already exists")
 )
 
 // TeamMembership errors.
 const (
-	ErrTeamMembershipNotFound      = Error("Team membership not found")
-	ErrTeamMembershipAlreadyExists = Error("Team membership already exists for this user and team.")
+	ErrTeamMembershipAlreadyExists = Error("Team membership already exists for this user and team")
 )
 
 // ResourceControl errors.
 const (
-	ErrResourceControlNotFound      = Error("Resource control not found")
 	ErrResourceControlAlreadyExists = Error("A resource control is already applied on this resource")
 	ErrInvalidResourceControlType   = Error("Unsupported resource control type")
 )
 
 // Endpoint errors.
 const (
-	ErrEndpointNotFound     = Error("Endpoint not found")
 	ErrEndpointAccessDenied = Error("Access denied to endpoint")
 )
 
-// Version errors.
+// Azure environment errors
 const (
-	ErrDBVersionNotFound = Error("DB version not found")
+	ErrAzureInvalidCredentials = Error("Invalid Azure credentials")
 )
 
-// Settings errors.
+// Endpoint group errors.
 const (
-	ErrSettingsNotFound = Error("Settings not found")
+	ErrCannotRemoveDefaultGroup = Error("Cannot remove the default endpoint group")
+)
+
+// Registry errors.
+const (
+	ErrRegistryAlreadyExists = Error("A registry is already defined for this URL")
+)
+
+// Stack errors
+const (
+	ErrStackAlreadyExists              = Error("A stack already exists with this name")
+	ErrComposeFileNotFoundInRepository = Error("Unable to find a Compose file in the repository")
+	ErrStackNotExternal                = Error("Not an external stack")
+)
+
+// Tag errors
+const (
+	ErrTagAlreadyExists = Error("A tag already exists with this name")
+)
+
+// Endpoint extensions error
+const (
+	ErrEndpointExtensionNotSupported      = Error("This extension is not supported")
+	ErrEndpointExtensionAlreadyAssociated = Error("This extension is already associated to the endpoint")
 )
 
 // Crypto errors.

api/exec/swarm_stack.go

@@ -0,0 +1,178 @@
+package exec
+
+import (
+	"bytes"
+	"encoding/json"
+	"os"
+	"os/exec"
+	"path"
+	"runtime"
+
+	"github.com/portainer/portainer"
+)
+
+// SwarmStackManager represents a service for managing stacks.
+type SwarmStackManager struct {
+	binaryPath       string
+	dataPath         string
+	signatureService portainer.DigitalSignatureService
+	fileService      portainer.FileService
+}
+
+// NewSwarmStackManager initializes a new SwarmStackManager service.
+// It also updates the configuration of the Docker CLI binary.
+func NewSwarmStackManager(binaryPath, dataPath string, signatureService portainer.DigitalSignatureService, fileService portainer.FileService) (*SwarmStackManager, error) {
+	manager := &SwarmStackManager{
+		binaryPath:       binaryPath,
+		dataPath:         dataPath,
+		signatureService: signatureService,
+		fileService:      fileService,
+	}
+
+	err := manager.updateDockerCLIConfiguration(dataPath)
+	if err != nil {
+		return nil, err
+	}
+
+	return manager, nil
+}
+
+// Login executes the docker login command against a list of registries (including DockerHub).
+func (manager *SwarmStackManager) Login(dockerhub *portainer.DockerHub, registries []portainer.Registry, endpoint *portainer.Endpoint) {
+	command, args := prepareDockerCommandAndArgs(manager.binaryPath, manager.dataPath, endpoint)
+	for _, registry := range registries {
+		if registry.Authentication {
+			registryArgs := append(args, "login", "--username", registry.Username, "--password", registry.Password, registry.URL)
+			runCommandAndCaptureStdErr(command, registryArgs, nil, "")
+		}
+	}
+
+	if dockerhub.Authentication {
+		dockerhubArgs := append(args, "login", "--username", dockerhub.Username, "--password", dockerhub.Password)
+		runCommandAndCaptureStdErr(command, dockerhubArgs, nil, "")
+	}
+}
+
+// Logout executes the docker logout command.
+func (manager *SwarmStackManager) Logout(endpoint *portainer.Endpoint) error {
+	command, args := prepareDockerCommandAndArgs(manager.binaryPath, manager.dataPath, endpoint)
+	args = append(args, "logout")
+	return runCommandAndCaptureStdErr(command, args, nil, "")
+}
+
+// Deploy executes the docker stack deploy command.
+func (manager *SwarmStackManager) Deploy(stack *portainer.Stack, prune bool, endpoint *portainer.Endpoint) error {
+	stackFilePath := path.Join(stack.ProjectPath, stack.EntryPoint)
+	command, args := prepareDockerCommandAndArgs(manager.binaryPath, manager.dataPath, endpoint)
+
+	if prune {
+		args = append(args, "stack", "deploy", "--prune", "--with-registry-auth", "--compose-file", stackFilePath, stack.Name)
+	} else {
+		args = append(args, "stack", "deploy", "--with-registry-auth", "--compose-file", stackFilePath, stack.Name)
+	}
+
+	env := make([]string, 0)
+	for _, envvar := range stack.Env {
+		env = append(env, envvar.Name+"="+envvar.Value)
+	}
+
+	stackFolder := path.Dir(stackFilePath)
+	return runCommandAndCaptureStdErr(command, args, env, stackFolder)
+}
+
+// Remove executes the docker stack rm command.
+func (manager *SwarmStackManager) Remove(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
+	command, args := prepareDockerCommandAndArgs(manager.binaryPath, manager.dataPath, endpoint)
+	args = append(args, "stack", "rm", stack.Name)
+	return runCommandAndCaptureStdErr(command, args, nil, "")
+}
+
+func runCommandAndCaptureStdErr(command string, args []string, env []string, workingDir string) error {
+	var stderr bytes.Buffer
+	cmd := exec.Command(command, args...)
+	cmd.Stderr = &stderr
+	cmd.Dir = workingDir
+
+	if env != nil {
+		cmd.Env = os.Environ()
+		cmd.Env = append(cmd.Env, env...)
+	}
+
+	err := cmd.Run()
+	if err != nil {
+		return portainer.Error(stderr.String())
+	}
+
+	return nil
+}
+
+func prepareDockerCommandAndArgs(binaryPath, dataPath string, endpoint *portainer.Endpoint) (string, []string) {
+	// Assume Linux as a default
+	command := path.Join(binaryPath, "docker")
+
+	if runtime.GOOS == "windows" {
+		command = path.Join(binaryPath, "docker.exe")
+	}
+
+	args := make([]string, 0)
+	args = append(args, "--config", dataPath)
+	args = append(args, "-H", endpoint.URL)
+
+	if endpoint.TLSConfig.TLS {
+		args = append(args, "--tls")
+
+		if !endpoint.TLSConfig.TLSSkipVerify {
+			args = append(args, "--tlsverify", "--tlscacert", endpoint.TLSConfig.TLSCACertPath)
+		}
+
+		if endpoint.TLSConfig.TLSCertPath != "" && endpoint.TLSConfig.TLSKeyPath != "" {
+			args = append(args, "--tlscert", endpoint.TLSConfig.TLSCertPath, "--tlskey", endpoint.TLSConfig.TLSKeyPath)
+		}
+	}
+
+	return command, args
+}
+
+func (manager *SwarmStackManager) updateDockerCLIConfiguration(dataPath string) error {
+	configFilePath := path.Join(dataPath, "config.json")
+	config, err := manager.retrieveConfigurationFromDisk(configFilePath)
+	if err != nil {
+		return err
+	}
+
+	signature, err := manager.signatureService.Sign(portainer.PortainerAgentSignatureMessage)
+	if err != nil {
+		return err
+	}
+
+	if config["HttpHeaders"] == nil {
+		config["HttpHeaders"] = make(map[string]interface{})
+	}
+	headersObject := config["HttpHeaders"].(map[string]interface{})
+	headersObject["X-PortainerAgent-ManagerOperation"] = "1"
+	headersObject["X-PortainerAgent-Signature"] = signature
+	headersObject["X-PortainerAgent-PublicKey"] = manager.signatureService.EncodedPublicKey()
+
+	err = manager.fileService.WriteJSONToFile(configFilePath, config)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (manager *SwarmStackManager) retrieveConfigurationFromDisk(path string) (map[string]interface{}, error) {
+	var config map[string]interface{}
+
+	raw, err := manager.fileService.GetFileContent(path)
+	if err != nil {
+		return make(map[string]interface{}), nil
+	}
+
+	err = json.Unmarshal(raw, &config)
+	if err != nil {
+		return nil, err
+	}
+
+	return config, nil
+}

api/file/file.go

@@ -1,143 +0,0 @@
-package file
-
-import (
-	"github.com/portainer/portainer"
-
-	"io"
-	"os"
-	"path"
-	"strconv"
-)
-
-const (
-	// TLSStorePath represents the subfolder where TLS files are stored in the file store folder.
-	TLSStorePath = "tls"
-	// TLSCACertFile represents the name on disk for a TLS CA file.
-	TLSCACertFile = "ca.pem"
-	// TLSCertFile represents the name on disk for a TLS certificate file.
-	TLSCertFile = "cert.pem"
-	// TLSKeyFile represents the name on disk for a TLS key file.
-	TLSKeyFile = "key.pem"
-)
-
-// Service represents a service for managing files and directories.
-type Service struct {
-	dataStorePath string
-	fileStorePath string
-}
-
-// NewService initializes a new service. It creates a data directory and a directory to store files
-// inside this directory if they don't exist.
-func NewService(dataStorePath, fileStorePath string) (*Service, error) {
-	service := &Service{
-		dataStorePath: dataStorePath,
-		fileStorePath: path.Join(dataStorePath, fileStorePath),
-	}
-
-	// Checking if a mount directory exists is broken with Go on Windows.
-	// This will need to be reviewed after the issue has been fixed in Go.
-	// See: https://github.com/portainer/portainer/issues/474
-	// err := createDirectoryIfNotExist(dataStorePath, 0755)
-	// if err != nil {
-	// 	return nil, err
-	// }
-
-	err := service.createDirectoryInStoreIfNotExist(TLSStorePath)
-	if err != nil {
-		return nil, err
-	}
-
-	return service, nil
-}
-
-// StoreTLSFile creates a subfolder in the TLSStorePath and stores a new file with the content from r.
-func (service *Service) StoreTLSFile(endpointID portainer.EndpointID, fileType portainer.TLSFileType, r io.Reader) error {
-	ID := strconv.Itoa(int(endpointID))
-	endpointStorePath := path.Join(TLSStorePath, ID)
-	err := service.createDirectoryInStoreIfNotExist(endpointStorePath)
-	if err != nil {
-		return err
-	}
-
-	var fileName string
-	switch fileType {
-	case portainer.TLSFileCA:
-		fileName = TLSCACertFile
-	case portainer.TLSFileCert:
-		fileName = TLSCertFile
-	case portainer.TLSFileKey:
-		fileName = TLSKeyFile
-	default:
-		return portainer.ErrUndefinedTLSFileType
-	}
-
-	tlsFilePath := path.Join(endpointStorePath, fileName)
-	err = service.createFileInStore(tlsFilePath, r)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-// GetPathForTLSFile returns the absolute path to a specific TLS file for an endpoint.
-func (service *Service) GetPathForTLSFile(endpointID portainer.EndpointID, fileType portainer.TLSFileType) (string, error) {
-	var fileName string
-	switch fileType {
-	case portainer.TLSFileCA:
-		fileName = TLSCACertFile
-	case portainer.TLSFileCert:
-		fileName = TLSCertFile
-	case portainer.TLSFileKey:
-		fileName = TLSKeyFile
-	default:
-		return "", portainer.ErrUndefinedTLSFileType
-	}
-	ID := strconv.Itoa(int(endpointID))
-	return path.Join(service.fileStorePath, TLSStorePath, ID, fileName), nil
-}
-
-// DeleteTLSFiles deletes a folder containing the TLS files for an endpoint.
-func (service *Service) DeleteTLSFiles(endpointID portainer.EndpointID) error {
-	ID := strconv.Itoa(int(endpointID))
-	endpointPath := path.Join(service.fileStorePath, TLSStorePath, ID)
-	err := os.RemoveAll(endpointPath)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-// createDirectoryInStoreIfNotExist creates a new directory in the file store if it doesn't exists on the file system.
-func (service *Service) createDirectoryInStoreIfNotExist(name string) error {
-	path := path.Join(service.fileStorePath, name)
-	return createDirectoryIfNotExist(path, 0700)
-}
-
-// createDirectoryIfNotExist creates a directory if it doesn't exists on the file system.
-func createDirectoryIfNotExist(path string, mode uint32) error {
-	_, err := os.Stat(path)
-	if os.IsNotExist(err) {
-		err = os.Mkdir(path, os.FileMode(mode))
-		if err != nil {
-			return err
-		}
-	} else if err != nil {
-		return err
-	}
-	return nil
-}
-
-// createFile creates a new file in the file store with the content from r.
-func (service *Service) createFileInStore(filePath string, r io.Reader) error {
-	path := path.Join(service.fileStorePath, filePath)
-	out, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
-	if err != nil {
-		return err
-	}
-	defer out.Close()
-	_, err = io.Copy(out, r)
-	if err != nil {
-		return err
-	}
-	return nil
-}

api/filesystem/filesystem.go

@@ -0,0 +1,320 @@
+package filesystem
+
+import (
+	"bytes"
+	"encoding/json"
+	"encoding/pem"
+	"io/ioutil"
+
+	"github.com/portainer/portainer"
+
+	"io"
+	"os"
+	"path"
+)
+
+const (
+	// TLSStorePath represents the subfolder where TLS files are stored in the file store folder.
+	TLSStorePath = "tls"
+	// LDAPStorePath represents the subfolder where LDAP TLS files are stored in the TLSStorePath.
+	LDAPStorePath = "ldap"
+	// TLSCACertFile represents the name on disk for a TLS CA file.
+	TLSCACertFile = "ca.pem"
+	// TLSCertFile represents the name on disk for a TLS certificate file.
+	TLSCertFile = "cert.pem"
+	// TLSKeyFile represents the name on disk for a TLS key file.
+	TLSKeyFile = "key.pem"
+	// ComposeStorePath represents the subfolder where compose files are stored in the file store folder.
+	ComposeStorePath = "compose"
+	// ComposeFileDefaultName represents the default name of a compose file.
+	ComposeFileDefaultName = "docker-compose.yml"
+	// PrivateKeyFile represents the name on disk of the file containing the private key.
+	PrivateKeyFile = "portainer.key"
+	// PublicKeyFile represents the name on disk of the file containing the public key.
+	PublicKeyFile = "portainer.pub"
+)
+
+// Service represents a service for managing files and directories.
+type Service struct {
+	dataStorePath string
+	fileStorePath string
+}
+
+// NewService initializes a new service. It creates a data directory and a directory to store files
+// inside this directory if they don't exist.
+func NewService(dataStorePath, fileStorePath string) (*Service, error) {
+	service := &Service{
+		dataStorePath: dataStorePath,
+		fileStorePath: path.Join(dataStorePath, fileStorePath),
+	}
+
+	err := os.MkdirAll(dataStorePath, 0755)
+	if err != nil {
+		return nil, err
+	}
+
+	err = service.createDirectoryInStore(TLSStorePath)
+	if err != nil {
+		return nil, err
+	}
+
+	err = service.createDirectoryInStore(ComposeStorePath)
+	if err != nil {
+		return nil, err
+	}
+
+	return service, nil
+}
+
+// RemoveDirectory removes a directory on the filesystem.
+func (service *Service) RemoveDirectory(directoryPath string) error {
+	return os.RemoveAll(directoryPath)
+}
+
+// GetStackProjectPath returns the absolute path on the FS for a stack based
+// on its identifier.
+func (service *Service) GetStackProjectPath(stackIdentifier string) string {
+	return path.Join(service.fileStorePath, ComposeStorePath, stackIdentifier)
+}
+
+// StoreStackFileFromBytes creates a subfolder in the ComposeStorePath and stores a new file from bytes.
+// It returns the path to the folder where the file is stored.
+func (service *Service) StoreStackFileFromBytes(stackIdentifier, fileName string, data []byte) (string, error) {
+	stackStorePath := path.Join(ComposeStorePath, stackIdentifier)
+	err := service.createDirectoryInStore(stackStorePath)
+	if err != nil {
+		return "", err
+	}
+
+	composeFilePath := path.Join(stackStorePath, fileName)
+	r := bytes.NewReader(data)
+
+	err = service.createFileInStore(composeFilePath, r)
+	if err != nil {
+		return "", err
+	}
+
+	return path.Join(service.fileStorePath, stackStorePath), nil
+}
+
+// StoreTLSFileFromBytes creates a folder in the TLSStorePath and stores a new file from bytes.
+// It returns the path to the newly created file.
+func (service *Service) StoreTLSFileFromBytes(folder string, fileType portainer.TLSFileType, data []byte) (string, error) {
+	storePath := path.Join(TLSStorePath, folder)
+	err := service.createDirectoryInStore(storePath)
+	if err != nil {
+		return "", err
+	}
+
+	var fileName string
+	switch fileType {
+	case portainer.TLSFileCA:
+		fileName = TLSCACertFile
+	case portainer.TLSFileCert:
+		fileName = TLSCertFile
+	case portainer.TLSFileKey:
+		fileName = TLSKeyFile
+	default:
+		return "", portainer.ErrUndefinedTLSFileType
+	}
+
+	tlsFilePath := path.Join(storePath, fileName)
+	r := bytes.NewReader(data)
+	err = service.createFileInStore(tlsFilePath, r)
+	if err != nil {
+		return "", err
+	}
+	return path.Join(service.fileStorePath, tlsFilePath), nil
+}
+
+// GetPathForTLSFile returns the absolute path to a specific TLS file for an endpoint.
+func (service *Service) GetPathForTLSFile(folder string, fileType portainer.TLSFileType) (string, error) {
+	var fileName string
+	switch fileType {
+	case portainer.TLSFileCA:
+		fileName = TLSCACertFile
+	case portainer.TLSFileCert:
+		fileName = TLSCertFile
+	case portainer.TLSFileKey:
+		fileName = TLSKeyFile
+	default:
+		return "", portainer.ErrUndefinedTLSFileType
+	}
+	return path.Join(service.fileStorePath, TLSStorePath, folder, fileName), nil
+}
+
+// DeleteTLSFiles deletes a folder in the TLS store path.
+func (service *Service) DeleteTLSFiles(folder string) error {
+	storePath := path.Join(service.fileStorePath, TLSStorePath, folder)
+	err := os.RemoveAll(storePath)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// DeleteTLSFile deletes a specific TLS file from a folder.
+func (service *Service) DeleteTLSFile(folder string, fileType portainer.TLSFileType) error {
+	var fileName string
+	switch fileType {
+	case portainer.TLSFileCA:
+		fileName = TLSCACertFile
+	case portainer.TLSFileCert:
+		fileName = TLSCertFile
+	case portainer.TLSFileKey:
+		fileName = TLSKeyFile
+	default:
+		return portainer.ErrUndefinedTLSFileType
+	}
+
+	filePath := path.Join(service.fileStorePath, TLSStorePath, folder, fileName)
+
+	err := os.Remove(filePath)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// GetFileContent returns the content of a file as bytes.
+func (service *Service) GetFileContent(filePath string) ([]byte, error) {
+	content, err := ioutil.ReadFile(filePath)
+	if err != nil {
+		return nil, err
+	}
+
+	return content, nil
+}
+
+// Rename renames a file or directory
+func (service *Service) Rename(oldPath, newPath string) error {
+	return os.Rename(oldPath, newPath)
+}
+
+// WriteJSONToFile writes JSON to the specified file.
+func (service *Service) WriteJSONToFile(path string, content interface{}) error {
+	jsonContent, err := json.Marshal(content)
+	if err != nil {
+		return err
+	}
+
+	return ioutil.WriteFile(path, jsonContent, 0644)
+}
+
+// FileExists checks for the existence of the specified file.
+func (service *Service) FileExists(filePath string) (bool, error) {
+	if _, err := os.Stat(filePath); err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return false, err
+	}
+	return true, nil
+}
+
+// KeyPairFilesExist checks for the existence of the key files.
+func (service *Service) KeyPairFilesExist() (bool, error) {
+	privateKeyPath := path.Join(service.dataStorePath, PrivateKeyFile)
+	exists, err := service.FileExists(privateKeyPath)
+	if err != nil {
+		return false, err
+	}
+	if !exists {
+		return false, nil
+	}
+
+	publicKeyPath := path.Join(service.dataStorePath, PublicKeyFile)
+	exists, err = service.FileExists(publicKeyPath)
+	if err != nil {
+		return false, err
+	}
+	if !exists {
+		return false, nil
+	}
+
+	return true, nil
+}
+
+// StoreKeyPair store the specified keys content as PEM files on disk.
+func (service *Service) StoreKeyPair(private, public []byte, privatePEMHeader, publicPEMHeader string) error {
+	err := service.createPEMFileInStore(private, privatePEMHeader, PrivateKeyFile)
+	if err != nil {
+		return err
+	}
+
+	err = service.createPEMFileInStore(public, publicPEMHeader, PublicKeyFile)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// LoadKeyPair retrieve the content of both key files on disk.
+func (service *Service) LoadKeyPair() ([]byte, []byte, error) {
+	privateKey, err := service.getContentFromPEMFile(PrivateKeyFile)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	publicKey, err := service.getContentFromPEMFile(PublicKeyFile)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return privateKey, publicKey, nil
+}
+
+// createDirectoryInStore creates a new directory in the file store
+func (service *Service) createDirectoryInStore(name string) error {
+	path := path.Join(service.fileStorePath, name)
+	return os.MkdirAll(path, 0700)
+}
+
+// createFile creates a new file in the file store with the content from r.
+func (service *Service) createFileInStore(filePath string, r io.Reader) error {
+	path := path.Join(service.fileStorePath, filePath)
+
+	out, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	_, err = io.Copy(out, r)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (service *Service) createPEMFileInStore(content []byte, fileType, filePath string) error {
+	path := path.Join(service.fileStorePath, filePath)
+	block := &pem.Block{Type: fileType, Bytes: content}
+
+	out, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	err = pem.Encode(out, block)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (service *Service) getContentFromPEMFile(filePath string) ([]byte, error) {
+	path := path.Join(service.fileStorePath, filePath)
+
+	fileContent, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	block, _ := pem.Decode(fileContent)
+	return block.Bytes, nil
+}

api/git/git.go

@@ -0,0 +1,46 @@
+package git
+
+import (
+	"net/url"
+	"strings"
+
+	"gopkg.in/src-d/go-git.v4"
+	"gopkg.in/src-d/go-git.v4/plumbing"
+)
+
+// Service represents a service for managing Git.
+type Service struct{}
+
+// NewService initializes a new service.
+func NewService(dataStorePath string) (*Service, error) {
+	service := &Service{}
+
+	return service, nil
+}
+
+// ClonePublicRepository clones a public git repository using the specified URL in the specified
+// destination folder.
+func (service *Service) ClonePublicRepository(repositoryURL, referenceName string, destination string) error {
+	return cloneRepository(repositoryURL, referenceName, destination)
+}
+
+// ClonePrivateRepositoryWithBasicAuth clones a private git repository using the specified URL in the specified
+// destination folder. It will use the specified username and password for basic HTTP authentication.
+func (service *Service) ClonePrivateRepositoryWithBasicAuth(repositoryURL, referenceName string, destination, username, password string) error {
+	credentials := username + ":" + url.PathEscape(password)
+	repositoryURL = strings.Replace(repositoryURL, "://", "://"+credentials+"@", 1)
+	return cloneRepository(repositoryURL, referenceName, destination)
+}
+
+func cloneRepository(repositoryURL, referenceName string, destination string) error {
+	options := &git.CloneOptions{
+		URL: repositoryURL,
+	}
+
+	if referenceName != "" {
+		options.ReferenceName = plumbing.ReferenceName(referenceName)
+	}
+
+	_, err := git.PlainClone(destination, false, options)
+	return err
+}

api/http/client/client.go

@@ -0,0 +1,121 @@
+package client
+
+import (
+	"crypto/tls"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/portainer/portainer"
+)
+
+// HTTPClient represents a client to send HTTP requests.
+type HTTPClient struct {
+	*http.Client
+}
+
+// NewHTTPClient is used to build a new HTTPClient.
+func NewHTTPClient() *HTTPClient {
+	return &HTTPClient{
+		&http.Client{
+			Timeout: time.Second * 5,
+		},
+	}
+}
+
+// AzureAuthenticationResponse represents an Azure API authentication response.
+type AzureAuthenticationResponse struct {
+	AccessToken string `json:"access_token"`
+	ExpiresOn   string `json:"expires_on"`
+}
+
+// ExecuteAzureAuthenticationRequest is used to execute an authentication request
+// against the Azure API. It re-uses the same http.Client.
+func (client *HTTPClient) ExecuteAzureAuthenticationRequest(credentials *portainer.AzureCredentials) (*AzureAuthenticationResponse, error) {
+	loginURL := fmt.Sprintf("https://login.microsoftonline.com/%s/oauth2/token", credentials.TenantID)
+	params := url.Values{
+		"grant_type":    {"client_credentials"},
+		"client_id":     {credentials.ApplicationID},
+		"client_secret": {credentials.AuthenticationKey},
+		"resource":      {"https://management.azure.com/"},
+	}
+
+	response, err := client.PostForm(loginURL, params)
+	if err != nil {
+		return nil, err
+	}
+
+	if response.StatusCode != http.StatusOK {
+		return nil, portainer.ErrAzureInvalidCredentials
+	}
+
+	var token AzureAuthenticationResponse
+	err = json.NewDecoder(response.Body).Decode(&token)
+	if err != nil {
+		return nil, err
+	}
+
+	return &token, nil
+}
+
+// Get executes a simple HTTP GET to the specified URL and returns
+// the content of the response body.
+func Get(url string) ([]byte, error) {
+	client := &http.Client{
+		Timeout: time.Second * 3,
+	}
+
+	response, err := client.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	defer response.Body.Close()
+
+	body, err := ioutil.ReadAll(response.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	return body, nil
+}
+
+// ExecutePingOperation will send a SystemPing operation HTTP request to a Docker environment
+// using the specified host and optional TLS configuration.
+// It uses a new Http.Client for each operation.
+func ExecutePingOperation(host string, tlsConfig *tls.Config) (bool, error) {
+	transport := &http.Transport{}
+
+	scheme := "http"
+	if tlsConfig != nil {
+		transport.TLSClientConfig = tlsConfig
+		scheme = "https"
+	}
+
+	client := &http.Client{
+		Timeout:   time.Second * 3,
+		Transport: transport,
+	}
+
+	target := strings.Replace(host, "tcp://", scheme+"://", 1)
+	return pingOperation(client, target)
+}
+
+func pingOperation(client *http.Client, target string) (bool, error) {
+	pingOperationURL := target + "/_ping"
+
+	response, err := client.Get(pingOperationURL)
+	if err != nil {
+		return false, err
+	}
+
+	agentOnDockerEnvironment := false
+	if response.Header.Get(portainer.PortainerAgentHeader) != "" {
+		agentOnDockerEnvironment = true
+	}
+
+	return agentOnDockerEnvironment, nil
+}

api/http/error/error.go

@@ -4,27 +4,38 @@ import (
 	"encoding/json"
 	"log"
 	"net/http"
-	"strings"
 )
 
-// errorResponse is a generic response for sending a error.
-type errorResponse struct {
-	Err string `json:"err,omitempty"`
-}
-
-// WriteErrorResponse writes an error message to the response and logger.
-func WriteErrorResponse(w http.ResponseWriter, err error, code int, logger *log.Logger) {
-	if logger != nil {
-		logger.Printf("http error: %s (code=%d)", err, code)
+type (
+	// LoggerHandler defines a HTTP handler that includes a HandlerError return pointer
+	LoggerHandler func(http.ResponseWriter, *http.Request) *HandlerError
+	// HandlerError represents an error raised inside a HTTP handler
+	HandlerError struct {
+		StatusCode int
+		Message    string
+		Err        error
 	}
+	errorResponse struct {
+		Err string `json:"err,omitempty"`
+	}
+)
 
-	w.WriteHeader(code)
-	json.NewEncoder(w).Encode(&errorResponse{Err: err.Error()})
+func (handler LoggerHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
+	err := handler(rw, r)
+	if err != nil {
+		writeErrorResponse(rw, err)
+	}
 }
 
-// WriteMethodNotAllowedResponse writes an error message to the response and sets the Allow header.
-func WriteMethodNotAllowedResponse(w http.ResponseWriter, allowedMethods []string) {
-	w.Header().Set("Allow", strings.Join(allowedMethods, ", "))
-	w.WriteHeader(http.StatusMethodNotAllowed)
-	json.NewEncoder(w).Encode(&errorResponse{Err: http.StatusText(http.StatusMethodNotAllowed)})
+func writeErrorResponse(rw http.ResponseWriter, err *HandlerError) {
+	log.Printf("http error: %s (err=%s) (code=%d)\n", err.Message, err.Err, err.StatusCode)
+	rw.Header().Set("Content-Type", "application/json")
+	rw.WriteHeader(err.StatusCode)
+	json.NewEncoder(rw).Encode(&errorResponse{Err: err.Message})
+}
+
+// WriteError is a convenience function that creates a new HandlerError before calling writeErrorResponse.
+// For use outside of the standard http handlers.
+func WriteError(rw http.ResponseWriter, code int, message string, err error) {
+	writeErrorResponse(rw, &HandlerError{code, message, err})
 }

api/http/handler/auth.go

@@ -1,112 +0,0 @@
-package handler
-
-import (
-	"github.com/portainer/portainer"
-
-	"encoding/json"
-	"log"
-	"net/http"
-	"os"
-
-	"github.com/asaskevich/govalidator"
-	"github.com/gorilla/mux"
-	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/security"
-)
-
-// AuthHandler represents an HTTP API handler for managing authentication.
-type AuthHandler struct {
-	*mux.Router
-	Logger        *log.Logger
-	authDisabled  bool
-	UserService   portainer.UserService
-	CryptoService portainer.CryptoService
-	JWTService    portainer.JWTService
-}
-
-const (
-	// ErrInvalidCredentialsFormat is an error raised when credentials format is not valid
-	ErrInvalidCredentialsFormat = portainer.Error("Invalid credentials format")
-	// ErrInvalidCredentials is an error raised when credentials for a user are invalid
-	ErrInvalidCredentials = portainer.Error("Invalid credentials")
-	// ErrAuthDisabled is an error raised when trying to access the authentication endpoints
-	// when the server has been started with the --no-auth flag
-	ErrAuthDisabled = portainer.Error("Authentication is disabled")
-)
-
-// NewAuthHandler returns a new instance of AuthHandler.
-func NewAuthHandler(bouncer *security.RequestBouncer, authDisabled bool) *AuthHandler {
-	h := &AuthHandler{
-		Router:       mux.NewRouter(),
-		Logger:       log.New(os.Stderr, "", log.LstdFlags),
-		authDisabled: authDisabled,
-	}
-	h.Handle("/auth",
-		bouncer.PublicAccess(http.HandlerFunc(h.handlePostAuth)))
-
-	return h
-}
-
-func (handler *AuthHandler) handlePostAuth(w http.ResponseWriter, r *http.Request) {
-	if r.Method != http.MethodPost {
-		httperror.WriteMethodNotAllowedResponse(w, []string{http.MethodPost})
-		return
-	}
-
-	if handler.authDisabled {
-		httperror.WriteErrorResponse(w, ErrAuthDisabled, http.StatusServiceUnavailable, handler.Logger)
-		return
-	}
-
-	var req postAuthRequest
-	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
-		httperror.WriteErrorResponse(w, ErrInvalidJSON, http.StatusBadRequest, handler.Logger)
-		return
-	}
-
-	_, err := govalidator.ValidateStruct(req)
-	if err != nil {
-		httperror.WriteErrorResponse(w, ErrInvalidCredentialsFormat, http.StatusBadRequest, handler.Logger)
-		return
-	}
-
-	var username = req.Username
-	var password = req.Password
-
-	u, err := handler.UserService.UserByUsername(username)
-	if err == portainer.ErrUserNotFound {
-		httperror.WriteErrorResponse(w, err, http.StatusNotFound, handler.Logger)
-		return
-	} else if err != nil {
-		httperror.WriteErrorResponse(w, err, http.StatusInternalServerError, handler.Logger)
-		return
-	}
-
-	err = handler.CryptoService.CompareHashAndData(u.Password, password)
-	if err != nil {
-		httperror.WriteErrorResponse(w, ErrInvalidCredentials, http.StatusUnprocessableEntity, handler.Logger)
-		return
-	}
-
-	tokenData := &portainer.TokenData{
-		ID:       u.ID,
-		Username: u.Username,
-		Role:     u.Role,
-	}
-	token, err := handler.JWTService.GenerateToken(tokenData)
-	if err != nil {
-		httperror.WriteErrorResponse(w, err, http.StatusInternalServerError, handler.Logger)
-		return
-	}
-
-	encodeJSON(w, &postAuthResponse{JWT: token}, handler.Logger)
-}
-
-type postAuthRequest struct {
-	Username string `valid:"required"`
-	Password string `valid:"required"`
-}
-
-type postAuthResponse struct {
-	JWT string `json:"jwt"`
-}

api/http/handler/auth/authenticate.go

@@ -0,0 +1,187 @@
+package auth
+
+import (
+	"log"
+	"net/http"
+	"strings"
+
+	"github.com/asaskevich/govalidator"
+	"github.com/portainer/portainer"
+	httperror "github.com/portainer/portainer/http/error"
+	"github.com/portainer/portainer/http/request"
+	"github.com/portainer/portainer/http/response"
+)
+
+type authenticatePayload struct {
+	Username string
+	Password string
+}
+
+type authenticateResponse struct {
+	JWT string `json:"jwt"`
+}
+
+func (payload *authenticatePayload) Validate(r *http.Request) error {
+	if govalidator.IsNull(payload.Username) {
+		return portainer.Error("Invalid username")
+	}
+	if govalidator.IsNull(payload.Password) {
+		return portainer.Error("Invalid password")
+	}
+	return nil
+}
+
+func (handler *Handler) authenticate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	if handler.authDisabled {
+		return &httperror.HandlerError{http.StatusServiceUnavailable, "Cannot authenticate user. Portainer was started with the --no-auth flag", ErrAuthDisabled}
+	}
+
+	var payload authenticatePayload
+	err := request.DecodeAndValidateJSONPayload(r, &payload)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
+	}
+
+	settings, err := handler.SettingsService.Settings()
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve settings from the database", err}
+	}
+
+	u, err := handler.UserService.UserByUsername(payload.Username)
+	if err != nil && err != portainer.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a user with the specified username from the database", err}
+	}
+
+	if err == portainer.ErrObjectNotFound && settings.AuthenticationMethod == portainer.AuthenticationInternal {
+		return &httperror.HandlerError{http.StatusUnprocessableEntity, "Invalid credentials", portainer.ErrUnauthorized}
+	}
+
+	if settings.AuthenticationMethod == portainer.AuthenticationLDAP {
+		if u == nil && settings.LDAPSettings.AutoCreateUsers {
+			return handler.authenticateLDAPAndCreateUser(w, payload.Username, payload.Password, &settings.LDAPSettings)
+		} else if u == nil && !settings.LDAPSettings.AutoCreateUsers {
+			return &httperror.HandlerError{http.StatusUnprocessableEntity, "Invalid credentials", portainer.ErrUnauthorized}
+		}
+		return handler.authenticateLDAP(w, u, payload.Password, &settings.LDAPSettings)
+	}
+
+	return handler.authenticateInternal(w, u, payload.Password)
+}
+
+func (handler *Handler) authenticateLDAP(w http.ResponseWriter, user *portainer.User, password string, ldapSettings *portainer.LDAPSettings) *httperror.HandlerError {
+	err := handler.LDAPService.AuthenticateUser(user.Username, password, ldapSettings)
+	if err != nil {
+		return handler.authenticateInternal(w, user, password)
+	}
+
+	err = handler.addUserIntoTeams(user, ldapSettings)
+	if err != nil {
+		log.Printf("Warning: unable to automatically add user into teams: %s\n", err.Error())
+	}
+
+	return handler.writeToken(w, user)
+}
+
+func (handler *Handler) authenticateInternal(w http.ResponseWriter, user *portainer.User, password string) *httperror.HandlerError {
+	err := handler.CryptoService.CompareHashAndData(user.Password, password)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusUnprocessableEntity, "Invalid credentials", portainer.ErrUnauthorized}
+	}
+
+	return handler.writeToken(w, user)
+}
+
+func (handler *Handler) authenticateLDAPAndCreateUser(w http.ResponseWriter, username, password string, ldapSettings *portainer.LDAPSettings) *httperror.HandlerError {
+	err := handler.LDAPService.AuthenticateUser(username, password, ldapSettings)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusUnprocessableEntity, "Invalid credentials", err}
+	}
+
+	user := &portainer.User{
+		Username: username,
+		Role:     portainer.StandardUserRole,
+	}
+
+	err = handler.UserService.CreateUser(user)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist user inside the database", err}
+	}
+
+	err = handler.addUserIntoTeams(user, ldapSettings)
+	if err != nil {
+		log.Printf("Warning: unable to automatically add user into teams: %s\n", err.Error())
+	}
+
+	return handler.writeToken(w, user)
+}
+
+func (handler *Handler) writeToken(w http.ResponseWriter, user *portainer.User) *httperror.HandlerError {
+	tokenData := &portainer.TokenData{
+		ID:       user.ID,
+		Username: user.Username,
+		Role:     user.Role,
+	}
+
+	token, err := handler.JWTService.GenerateToken(tokenData)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to generate JWT token", err}
+	}
+
+	return response.JSON(w, &authenticateResponse{JWT: token})
+}
+
+func (handler *Handler) addUserIntoTeams(user *portainer.User, settings *portainer.LDAPSettings) error {
+	teams, err := handler.TeamService.Teams()
+	if err != nil {
+		return err
+	}
+
+	userGroups, err := handler.LDAPService.GetUserGroups(user.Username, settings)
+	if err != nil {
+		return err
+	}
+
+	userMemberships, err := handler.TeamMembershipService.TeamMembershipsByUserID(user.ID)
+	if err != nil {
+		return err
+	}
+
+	for _, team := range teams {
+		if teamExists(team.Name, userGroups) {
+
+			if teamMembershipExists(team.ID, userMemberships) {
+				continue
+			}
+
+			membership := &portainer.TeamMembership{
+				UserID: user.ID,
+				TeamID: team.ID,
+				Role:   portainer.TeamMember,
+			}
+
+			err := handler.TeamMembershipService.CreateTeamMembership(membership)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func teamExists(teamName string, ldapGroups []string) bool {
+	for _, group := range ldapGroups {
+		if strings.ToLower(group) == strings.ToLower(teamName) {
+			return true
+		}
+	}
+	return false
+}
+
+func teamMembershipExists(teamID portainer.TeamID, memberships []portainer.TeamMembership) bool {
+	for _, membership := range memberships {
+		if membership.TeamID == teamID {
+			return true
+		}
+	}
+	return false
+}

api/http/handler/auth/handler.go

@@ -0,0 +1,43 @@
+package auth
+
+import (
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"github.com/portainer/portainer"
+	httperror "github.com/portainer/portainer/http/error"
+	"github.com/portainer/portainer/http/security"
+)
+
+const (
+	// ErrInvalidCredentials is an error raised when credentials for a user are invalid
+	ErrInvalidCredentials = portainer.Error("Invalid credentials")
+	// ErrAuthDisabled is an error raised when trying to access the authentication endpoints
+	// when the server has been started with the --no-auth flag
+	ErrAuthDisabled = portainer.Error("Authentication is disabled")
+)
+
+// Handler is the HTTP handler used to handle authentication operations.
+type Handler struct {
+	*mux.Router
+	authDisabled          bool
+	UserService           portainer.UserService
+	CryptoService         portainer.CryptoService
+	JWTService            portainer.JWTService
+	LDAPService           portainer.LDAPService
+	SettingsService       portainer.SettingsService
+	TeamService           portainer.TeamService
+	TeamMembershipService portainer.TeamMembershipService
+}
+
+// NewHandler creates a handler to manage authentication operations.
+func NewHandler(bouncer *security.RequestBouncer, rateLimiter *security.RateLimiter, authDisabled bool) *Handler {
+	h := &Handler{
+		Router:       mux.NewRouter(),
+		authDisabled: authDisabled,
+	}
+	h.Handle("/auth",
+		rateLimiter.LimitAccess(bouncer.PublicAccess(httperror.LoggerHandler(h.authenticate)))).Methods(http.MethodPost)
+
+	return h
+}

api/http/handler/docker.go

@@ -1,94 +0,0 @@
-package handler
-
-import (
-	"strconv"
-
-	"github.com/portainer/portainer"
-	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/proxy"
-	"github.com/portainer/portainer/http/security"
-
-	"log"
-	"net/http"
-	"os"
-
-	"github.com/gorilla/mux"
-)
-
-// DockerHandler represents an HTTP API handler for proxying requests to the Docker API.
-type DockerHandler struct {
-	*mux.Router
-	Logger                *log.Logger
-	EndpointService       portainer.EndpointService
-	TeamMembershipService portainer.TeamMembershipService
-	ProxyManager          *proxy.Manager
-}
-
-// NewDockerHandler returns a new instance of DockerHandler.
-func NewDockerHandler(bouncer *security.RequestBouncer) *DockerHandler {
-	h := &DockerHandler{
-		Router: mux.NewRouter(),
-		Logger: log.New(os.Stderr, "", log.LstdFlags),
-	}
-	h.PathPrefix("/{id}/").Handler(
-		bouncer.AuthenticatedAccess(http.HandlerFunc(h.proxyRequestsToDockerAPI)))
-	return h
-}
-
-func (handler *DockerHandler) checkEndpointAccessControl(endpoint *portainer.Endpoint, userID portainer.UserID) bool {
-	for _, authorizedUserID := range endpoint.AuthorizedUsers {
-		if authorizedUserID == userID {
-			return true
-		}
-	}
-
-	memberships, _ := handler.TeamMembershipService.TeamMembershipsByUserID(userID)
-	for _, authorizedTeamID := range endpoint.AuthorizedTeams {
-		for _, membership := range memberships {
-			if membership.TeamID == authorizedTeamID {
-				return true
-			}
-		}
-	}
-	return false
-}
-
-func (handler *DockerHandler) proxyRequestsToDockerAPI(w http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
-	id := vars["id"]
-
-	parsedID, err := strconv.Atoi(id)
-	if err != nil {
-		httperror.WriteErrorResponse(w, err, http.StatusBadRequest, handler.Logger)
-		return
-	}
-
-	endpointID := portainer.EndpointID(parsedID)
-	endpoint, err := handler.EndpointService.Endpoint(endpointID)
-	if err != nil {
-		httperror.WriteErrorResponse(w, err, http.StatusInternalServerError, handler.Logger)
-		return
-	}
-
-	tokenData, err := security.RetrieveTokenData(r)
-	if err != nil {
-		httperror.WriteErrorResponse(w, err, http.StatusInternalServerError, handler.Logger)
-		return
-	}
-	if tokenData.Role != portainer.AdministratorRole && !handler.checkEndpointAccessControl(endpoint, tokenData.ID) {
-		httperror.WriteErrorResponse(w, portainer.ErrEndpointAccessDenied, http.StatusForbidden, handler.Logger)
-		return
-	}
-
-	var proxy http.Handler
-	proxy = handler.ProxyManager.GetProxy(string(endpointID))
-	if proxy == nil {
-		proxy, err = handler.ProxyManager.CreateAndRegisterProxy(endpoint)
-		if err != nil {
-			httperror.WriteErrorResponse(w, err, http.StatusBadRequest, handler.Logger)
-			return
-		}
-	}
-
-	http.StripPrefix("/"+id, proxy).ServeHTTP(w, r)
-}

api/http/handler/dockerhub/dockerhub_inspect.go

@@ -0,0 +1,19 @@
+package dockerhub
+
+import (
+	"net/http"
+
+	httperror "github.com/portainer/portainer/http/error"
+	"github.com/portainer/portainer/http/response"
+)
+
+// GET request on /api/dockerhub
+func (handler *Handler) dockerhubInspect(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	dockerhub, err := handler.DockerHubService.DockerHub()
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve DockerHub details from the database", err}
+	}
+
+	hideFields(dockerhub)
+	return response.JSON(w, dockerhub)
+}

api/http/handler/dockerhub/dockerhub_update.go

@@ -0,0 +1,52 @@
+package dockerhub
+
+import (
+	"net/http"
+
+	"github.com/asaskevich/govalidator"
+	"github.com/portainer/portainer"
+	httperror "github.com/portainer/portainer/http/error"
+	"github.com/portainer/portainer/http/request"
+	"github.com/portainer/portainer/http/response"
+)
+
+type dockerhubUpdatePayload struct {
+	Authentication bool
+	Username       string
+	Password       string
+}
+
+func (payload *dockerhubUpdatePayload) Validate(r *http.Request) error {
+	if payload.Authentication && (govalidator.IsNull(payload.Username) || govalidator.IsNull(payload.Password)) {
+		return portainer.Error("Invalid credentials. Username and password must be specified when authentication is enabled")
+	}
+	return nil
+}
+
+// PUT request on /api/dockerhub
+func (handler *Handler) dockerhubUpdate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	var payload dockerhubUpdatePayload
+	err := request.DecodeAndValidateJSONPayload(r, &payload)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
+	}
+
+	dockerhub := &portainer.DockerHub{
+		Authentication: false,
+		Username:       "",
+		Password:       "",
+	}
+
+	if payload.Authentication {
+		dockerhub.Authentication = true
+		dockerhub.Username = payload.Username
+		dockerhub.Password = payload.Password
+	}
+
+	err = handler.DockerHubService.UpdateDockerHub(dockerhub)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the Dockerhub changes inside the database", err}
+	}
+
+	return response.Empty(w)
+}

api/http/handler/dockerhub/handler.go

@@ -0,0 +1,33 @@
+package dockerhub
+
+import (
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"github.com/portainer/portainer"
+	httperror "github.com/portainer/portainer/http/error"
+	"github.com/portainer/portainer/http/security"
+)
+
+func hideFields(dockerHub *portainer.DockerHub) {
+	dockerHub.Password = ""
+}
+
+// Handler is the HTTP handler used to handle DockerHub operations.
+type Handler struct {
+	*mux.Router
+	DockerHubService portainer.DockerHubService
+}
+
+// NewHandler creates a handler to manage Dockerhub operations.
+func NewHandler(bouncer *security.RequestBouncer) *Handler {
+	h := &Handler{
+		Router: mux.NewRouter(),
+	}
+	h.Handle("/dockerhub",
+		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.dockerhubInspect))).Methods(http.MethodGet)
+	h.Handle("/dockerhub",
+		bouncer.AdministratorAccess(httperror.LoggerHandler(h.dockerhubUpdate))).Methods(http.MethodPut)
+
+	return h
+}

api/http/handler/endpoint.go

@@ -1,356 +0,0 @@
-package handler
-
-import (
-	"github.com/portainer/portainer"
-	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/proxy"
-	"github.com/portainer/portainer/http/security"
-
-	"encoding/json"
-	"log"
-	"net/http"
-	"os"
-	"strconv"
-
-	"github.com/asaskevich/govalidator"
-	"github.com/gorilla/mux"
-)
-
-// EndpointHandler represents an HTTP API handler for managing Docker endpoints.
-type EndpointHandler struct {
-	*mux.Router
-	Logger                      *log.Logger
-	authorizeEndpointManagement bool
-	EndpointService             portainer.EndpointService
-	FileService                 portainer.FileService
-	ProxyManager                *proxy.Manager
-}
-
-const (
-	// ErrEndpointManagementDisabled is an error raised when trying to access the endpoints management endpoints
-	// when the server has been started with the --external-endpoints flag
-	ErrEndpointManagementDisabled = portainer.Error("Endpoint management is disabled")
-)
-
-// NewEndpointHandler returns a new instance of EndpointHandler.
-func NewEndpointHandler(bouncer *security.RequestBouncer, authorizeEndpointManagement bool) *EndpointHandler {
-	h := &EndpointHandler{
-		Router: mux.NewRouter(),
-		Logger: log.New(os.Stderr, "", log.LstdFlags),
-		authorizeEndpointManagement: authorizeEndpointManagement,
-	}
-	h.Handle("/endpoints",
-		bouncer.AdministratorAccess(http.HandlerFunc(h.handlePostEndpoints))).Methods(http.MethodPost)
-	h.Handle("/endpoints",
-		bouncer.RestrictedAccess(http.HandlerFunc(h.handleGetEndpoints))).Methods(http.MethodGet)
-	h.Handle("/endpoints/{id}",
-		bouncer.AdministratorAccess(http.HandlerFunc(h.handleGetEndpoint))).Methods(http.MethodGet)
-	h.Handle("/endpoints/{id}",
-		bouncer.AdministratorAccess(http.HandlerFunc(h.handlePutEndpoint))).Methods(http.MethodPut)
-	h.Handle("/endpoints/{id}/access",
-		bouncer.AdministratorAccess(http.HandlerFunc(h.handlePutEndpointAccess))).Methods(http.MethodPut)
-	h.Handle("/endpoints/{id}",
-		bouncer.AdministratorAccess(http.HandlerFunc(h.handleDeleteEndpoint))).Methods(http.MethodDelete)
-
-	return h
-}
-
-// handleGetEndpoints handles GET requests on /endpoints
-func (handler *EndpointHandler) handleGetEndpoints(w http.ResponseWriter, r *http.Request) {
-	securityContext, err := security.RetrieveRestrictedRequestContext(r)
-	if err != nil {
-		httperror.WriteErrorResponse(w, err, http.StatusInternalServerError, handler.Logger)
-		return
-	}
-
-	endpoints, err := handler.EndpointService.Endpoints()
-	if err != nil {
-		httperror.WriteErrorResponse(w, err, http.StatusInternalServerError, handler.Logger)
-		return
-	}
-
-	filteredEndpoints, err := security.FilterEndpoints(endpoints, securityContext)
-	if err != nil {
-		httperror.WriteErrorResponse(w, err, http.StatusInternalServerError, handler.Logger)
-		return
-	}
-
-	encodeJSON(w, filteredEndpoints, handler.Logger)
-}
-
-// handlePostEndpoints handles POST requests on /endpoints
-func (handler *EndpointHandler) handlePostEndpoints(w http.ResponseWriter, r *http.Request) {
-	if !handler.authorizeEndpointManagement {
-		httperror.WriteErrorResponse(w, ErrEndpointManagementDisabled, http.StatusServiceUnavailable, handler.Logger)
-		return
-	}
-
-	var req postEndpointsRequest
-	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
-		httperror.WriteErrorResponse(w, ErrInvalidJSON, http.StatusBadRequest, handler.Logger)
-		return
-	}
-
-	_, err := govalidator.ValidateStruct(req)
-	if err != nil {
-		httperror.WriteErrorResponse(w, ErrInvalidRequestFormat, http.StatusBadRequest, handler.Logger)
-		return
-	}
-
-	endpoint := &portainer.Endpoint{
-		Name:            req.Name,
-		URL:             req.URL,
-		PublicURL:       req.PublicURL,
-		TLS:             req.TLS,
-		AuthorizedUsers: []portainer.UserID{},
-		AuthorizedTeams: []portainer.TeamID{},
-	}
-
-	err = handler.EndpointService.CreateEndpoint(endpoint)
-	if err != nil {
-		httperror.WriteErrorResponse(w, err, http.StatusInternalServerError, handler.Logger)
-		return
-	}
-
-	if req.TLS {
-		caCertPath, _ := handler.FileService.GetPathForTLSFile(endpoint.ID, portainer.TLSFileCA)
-		endpoint.TLSCACertPath = caCertPath
-		certPath, _ := handler.FileService.GetPathForTLSFile(endpoint.ID, portainer.TLSFileCert)
-		endpoint.TLSCertPath = certPath
-		keyPath, _ := handler.FileService.GetPathForTLSFile(endpoint.ID, portainer.TLSFileKey)
-		endpoint.TLSKeyPath = keyPath
-		err = handler.EndpointService.UpdateEndpoint(endpoint.ID, endpoint)
-		if err != nil {
-			httperror.WriteErrorResponse(w, err, http.StatusInternalServerError, handler.Logger)
-			return
-		}
-	}
-
-	encodeJSON(w, &postEndpointsResponse{ID: int(endpoint.ID)}, handler.Logger)
-}
-
-type postEndpointsRequest struct {
-	Name      string `valid:"required"`
-	URL       string `valid:"required"`
-	PublicURL string `valid:"-"`
-	TLS       bool
-}
-
-type postEndpointsResponse struct {
-	ID int `json:"Id"`
-}
-
-// handleGetEndpoint handles GET requests on /endpoints/:id
-func (handler *EndpointHandler) handleGetEndpoint(w http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
-	id := vars["id"]
-
-	endpointID, err := strconv.Atoi(id)
-	if err != nil {
-		httperror.WriteErrorResponse(w, err, http.StatusBadRequest, handler.Logger)
-		return
-	}
-
-	endpoint, err := handler.EndpointService.Endpoint(portainer.EndpointID(endpointID))
-	if err == portainer.ErrEndpointNotFound {
-		httperror.WriteErrorResponse(w, err, http.StatusNotFound, handler.Logger)
-		return
-	} else if err != nil {
-		httperror.WriteErrorResponse(w, err, http.StatusInternalServerError, handler.Logger)
-		return
-	}
-
-	encodeJSON(w, endpoint, handler.Logger)
-}
-
-// handlePutEndpointAccess handles PUT requests on /endpoints/:id/access
-func (handler *EndpointHandler) handlePutEndpointAccess(w http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
-	id := vars["id"]
-
-	endpointID, err := strconv.Atoi(id)
-	if err != nil {
-		httperror.WriteErrorResponse(w, err, http.StatusBadRequest, handler.Logger)
-		return
-	}
-
-	var req putEndpointAccessRequest
-	if err = json.NewDecoder(r.Body).Decode(&req); err != nil {
-		httperror.WriteErrorResponse(w, ErrInvalidJSON, http.StatusBadRequest, handler.Logger)
-		return
-	}
-
-	_, err = govalidator.ValidateStruct(req)
-	if err != nil {
-		httperror.WriteErrorResponse(w, ErrInvalidRequestFormat, http.StatusBadRequest, handler.Logger)
-		return
-	}
-
-	endpoint, err := handler.EndpointService.Endpoint(portainer.EndpointID(endpointID))
-	if err == portainer.ErrEndpointNotFound {
-		httperror.WriteErrorResponse(w, err, http.StatusNotFound, handler.Logger)
-		return
-	} else if err != nil {
-		httperror.WriteErrorResponse(w, err, http.StatusInternalServerError, handler.Logger)
-		return
-	}
-
-	if req.AuthorizedUsers != nil {
-		authorizedUserIDs := []portainer.UserID{}
-		for _, value := range req.AuthorizedUsers {
-			authorizedUserIDs = append(authorizedUserIDs, portainer.UserID(value))
-		}
-		endpoint.AuthorizedUsers = authorizedUserIDs
-	}
-
-	if req.AuthorizedTeams != nil {
-		authorizedTeamIDs := []portainer.TeamID{}
-		for _, value := range req.AuthorizedTeams {
-			authorizedTeamIDs = append(authorizedTeamIDs, portainer.TeamID(value))
-		}
-		endpoint.AuthorizedTeams = authorizedTeamIDs
-	}
-
-	err = handler.EndpointService.UpdateEndpoint(endpoint.ID, endpoint)
-	if err != nil {
-		httperror.WriteErrorResponse(w, err, http.StatusInternalServerError, handler.Logger)
-		return
-	}
-}
-
-type putEndpointAccessRequest struct {
-	AuthorizedUsers []int `valid:"-"`
-	AuthorizedTeams []int `valid:"-"`
-}
-
-// handlePutEndpoint handles PUT requests on /endpoints/:id
-func (handler *EndpointHandler) handlePutEndpoint(w http.ResponseWriter, r *http.Request) {
-	if !handler.authorizeEndpointManagement {
-		httperror.WriteErrorResponse(w, ErrEndpointManagementDisabled, http.StatusServiceUnavailable, handler.Logger)
-		return
-	}
-
-	vars := mux.Vars(r)
-	id := vars["id"]
-
-	endpointID, err := strconv.Atoi(id)
-	if err != nil {
-		httperror.WriteErrorResponse(w, err, http.StatusBadRequest, handler.Logger)
-		return
-	}
-
-	var req putEndpointsRequest
-	if err = json.NewDecoder(r.Body).Decode(&req); err != nil {
-		httperror.WriteErrorResponse(w, ErrInvalidJSON, http.StatusBadRequest, handler.Logger)
-		return
-	}
-
-	_, err = govalidator.ValidateStruct(req)
-	if err != nil {
-		httperror.WriteErrorResponse(w, ErrInvalidRequestFormat, http.StatusBadRequest, handler.Logger)
-		return
-	}
-
-	endpoint, err := handler.EndpointService.Endpoint(portainer.EndpointID(endpointID))
-	if err == portainer.ErrEndpointNotFound {
-		httperror.WriteErrorResponse(w, err, http.StatusNotFound, handler.Logger)
-		return
-	} else if err != nil {
-		httperror.WriteErrorResponse(w, err, http.StatusInternalServerError, handler.Logger)
-		return
-	}
-
-	if req.Name != "" {
-		endpoint.Name = req.Name
-	}
-
-	if req.URL != "" {
-		endpoint.URL = req.URL
-	}
-
-	if req.PublicURL != "" {
-		endpoint.PublicURL = req.PublicURL
-	}
-
-	if req.TLS {
-		endpoint.TLS = true
-		caCertPath, _ := handler.FileService.GetPathForTLSFile(endpoint.ID, portainer.TLSFileCA)
-		endpoint.TLSCACertPath = caCertPath
-		certPath, _ := handler.FileService.GetPathForTLSFile(endpoint.ID, portainer.TLSFileCert)
-		endpoint.TLSCertPath = certPath
-		keyPath, _ := handler.FileService.GetPathForTLSFile(endpoint.ID, portainer.TLSFileKey)
-		endpoint.TLSKeyPath = keyPath
-	} else {
-		endpoint.TLS = false
-		endpoint.TLSCACertPath = ""
-		endpoint.TLSCertPath = ""
-		endpoint.TLSKeyPath = ""
-		err = handler.FileService.DeleteTLSFiles(endpoint.ID)
-		if err != nil {
-			httperror.WriteErrorResponse(w, err, http.StatusInternalServerError, handler.Logger)
-			return
-		}
-	}
-
-	_, err = handler.ProxyManager.CreateAndRegisterProxy(endpoint)
-	if err != nil {
-		httperror.WriteErrorResponse(w, err, http.StatusInternalServerError, handler.Logger)
-		return
-	}
-
-	err = handler.EndpointService.UpdateEndpoint(endpoint.ID, endpoint)
-	if err != nil {
-		httperror.WriteErrorResponse(w, err, http.StatusInternalServerError, handler.Logger)
-		return
-	}
-}
-
-type putEndpointsRequest struct {
-	Name      string `valid:"-"`
-	URL       string `valid:"-"`
-	PublicURL string `valid:"-"`
-	TLS       bool   `valid:"-"`
-}
-
-// handleDeleteEndpoint handles DELETE requests on /endpoints/:id
-func (handler *EndpointHandler) handleDeleteEndpoint(w http.ResponseWriter, r *http.Request) {
-	if !handler.authorizeEndpointManagement {
-		httperror.WriteErrorResponse(w, ErrEndpointManagementDisabled, http.StatusServiceUnavailable, handler.Logger)
-		return
-	}
-
-	vars := mux.Vars(r)
-	id := vars["id"]
-
-	endpointID, err := strconv.Atoi(id)
-	if err != nil {
-		httperror.WriteErrorResponse(w, err, http.StatusBadRequest, handler.Logger)
-		return
-	}
-
-	endpoint, err := handler.EndpointService.Endpoint(portainer.EndpointID(endpointID))
-
-	if err == portainer.ErrEndpointNotFound {
-		httperror.WriteErrorResponse(w, err, http.StatusNotFound, handler.Logger)
-		return
-	} else if err != nil {
-		httperror.WriteErrorResponse(w, err, http.StatusInternalServerError, handler.Logger)
-		return
-	}
-
-	handler.ProxyManager.DeleteProxy(string(endpointID))
-
-	err = handler.EndpointService.DeleteEndpoint(portainer.EndpointID(endpointID))
-	if err != nil {
-		httperror.WriteErrorResponse(w, err, http.StatusInternalServerError, handler.Logger)
-		return
-	}
-
-	if endpoint.TLS {
-		err = handler.FileService.DeleteTLSFiles(portainer.EndpointID(endpointID))
-		if err != nil {
-			httperror.WriteErrorResponse(w, err, http.StatusInternalServerError, handler.Logger)
-			return
-		}
-	}
-}

api/http/handler/endpointgroups/endpointgroup_create.go

@@ -0,0 +1,66 @@
+package endpointgroups
+
+import (
+	"net/http"
+
+	"github.com/asaskevich/govalidator"
+	"github.com/portainer/portainer"
+	httperror "github.com/portainer/portainer/http/error"
+	"github.com/portainer/portainer/http/request"
+	"github.com/portainer/portainer/http/response"
+)
+
+type endpointGroupCreatePayload struct {
+	Name                string
+	Description         string
+	AssociatedEndpoints []portainer.EndpointID
+	Tags                []string
+}
+
+func (payload *endpointGroupCreatePayload) Validate(r *http.Request) error {
+	if govalidator.IsNull(payload.Name) {
+		return portainer.Error("Invalid endpoint group name")
+	}
+	if payload.Tags == nil {
+		payload.Tags = []string{}
+	}
+	return nil
+}
+
+// POST request on /api/endpoint_groups
+func (handler *Handler) endpointGroupCreate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	var payload endpointGroupCreatePayload
+	err := request.DecodeAndValidateJSONPayload(r, &payload)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
+	}
+
+	endpointGroup := &portainer.EndpointGroup{
+		Name:            payload.Name,
+		Description:     payload.Description,
+		AuthorizedUsers: []portainer.UserID{},
+		AuthorizedTeams: []portainer.TeamID{},
+		Tags:            payload.Tags,
+	}
+
+	err = handler.EndpointGroupService.CreateEndpointGroup(endpointGroup)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the endpoint group inside the database", err}
+	}
+
+	endpoints, err := handler.EndpointService.Endpoints()
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoints from the database", err}
+	}
+
+	for _, endpoint := range endpoints {
+		if endpoint.GroupID == portainer.EndpointGroupID(1) {
+			err = handler.checkForGroupAssignment(endpoint, endpointGroup.ID, payload.AssociatedEndpoints)
+			if err != nil {
+				return &httperror.HandlerError{http.StatusInternalServerError, "Unable to update endpoint", err}
+			}
+		}
+	}
+
+	return response.JSON(w, endpointGroup)
+}

api/http/handler/endpointgroups/endpointgroup_delete.go

@@ -0,0 +1,51 @@
+package endpointgroups
+
+import (
+	"net/http"
+
+	"github.com/portainer/portainer"
+	httperror "github.com/portainer/portainer/http/error"
+	"github.com/portainer/portainer/http/request"
+	"github.com/portainer/portainer/http/response"
+)
+
+// DELETE request on /api/endpoint_groups/:id
+func (handler *Handler) endpointGroupDelete(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	endpointGroupID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint group identifier route variable", err}
+	}
+
+	if endpointGroupID == 1 {
+		return &httperror.HandlerError{http.StatusForbidden, "Unable to remove the default 'Unassigned' group", portainer.ErrCannotRemoveDefaultGroup}
+	}
+
+	_, err = handler.EndpointGroupService.EndpointGroup(portainer.EndpointGroupID(endpointGroupID))
+	if err == portainer.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint group with the specified identifier inside the database", err}
+	} else if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint group with the specified identifier inside the database", err}
+	}
+
+	err = handler.EndpointGroupService.DeleteEndpointGroup(portainer.EndpointGroupID(endpointGroupID))
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to remove the endpoint group from the database", err}
+	}
+
+	endpoints, err := handler.EndpointService.Endpoints()
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoints from the database", err}
+	}
+
+	for _, endpoint := range endpoints {
+		if endpoint.GroupID == portainer.EndpointGroupID(endpointGroupID) {
+			endpoint.GroupID = portainer.EndpointGroupID(1)
+			err = handler.EndpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+			if err != nil {
+				return &httperror.HandlerError{http.StatusInternalServerError, "Unable to update endpoint", err}
+			}
+		}
+	}
+
+	return response.Empty(w)
+}

api/http/handler/endpointgroups/endpointgroup_inspect.go

@@ -0,0 +1,27 @@
+package endpointgroups
+
+import (
+	"net/http"
+
+	"github.com/portainer/portainer"
+	httperror "github.com/portainer/portainer/http/error"
+	"github.com/portainer/portainer/http/request"
+	"github.com/portainer/portainer/http/response"
+)
+
+// GET request on /api/endpoint_groups/:id
+func (handler *Handler) endpointGroupInspect(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	endpointGroupID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint group identifier route variable", err}
+	}
+
+	endpointGroup, err := handler.EndpointGroupService.EndpointGroup(portainer.EndpointGroupID(endpointGroupID))
+	if err == portainer.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint group with the specified identifier inside the database", err}
+	} else if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint group with the specified identifier inside the database", err}
+	}
+
+	return response.JSON(w, endpointGroup)
+}

api/http/handler/endpointgroups/endpointgroup_list.go

@@ -0,0 +1,25 @@
+package endpointgroups
+
+import (
+	"net/http"
+
+	httperror "github.com/portainer/portainer/http/error"
+	"github.com/portainer/portainer/http/response"
+	"github.com/portainer/portainer/http/security"
+)
+
+// GET request on /api/endpoint_groups
+func (handler *Handler) endpointGroupList(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	endpointGroups, err := handler.EndpointGroupService.EndpointGroups()
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoint groups from the database", err}
+	}
+
+	securityContext, err := security.RetrieveRestrictedRequestContext(r)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve info from request context", err}
+	}
+
+	endpointGroups = security.FilterEndpointGroups(endpointGroups, securityContext)
+	return response.JSON(w, endpointGroups)
+}

api/http/handler/endpointgroups/endpointgroup_update.go

@@ -0,0 +1,73 @@
+package endpointgroups
+
+import (
+	"net/http"
+
+	"github.com/portainer/portainer"
+	httperror "github.com/portainer/portainer/http/error"
+	"github.com/portainer/portainer/http/request"
+	"github.com/portainer/portainer/http/response"
+)
+
+type endpointGroupUpdatePayload struct {
+	Name                string
+	Description         string
+	AssociatedEndpoints []portainer.EndpointID
+	Tags                []string
+}
+
+func (payload *endpointGroupUpdatePayload) Validate(r *http.Request) error {
+	return nil
+}
+
+// PUT request on /api/endpoint_groups/:id
+func (handler *Handler) endpointGroupUpdate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	endpointGroupID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint group identifier route variable", err}
+	}
+
+	var payload endpointGroupUpdatePayload
+	err = request.DecodeAndValidateJSONPayload(r, &payload)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
+	}
+
+	endpointGroup, err := handler.EndpointGroupService.EndpointGroup(portainer.EndpointGroupID(endpointGroupID))
+	if err == portainer.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint group with the specified identifier inside the database", err}
+	} else if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint group with the specified identifier inside the database", err}
+	}
+
+	if payload.Name != "" {
+		endpointGroup.Name = payload.Name
+	}
+
+	if payload.Description != "" {
+		endpointGroup.Description = payload.Description
+	}
+
+	if payload.Tags != nil {
+		endpointGroup.Tags = payload.Tags
+	}
+
+	err = handler.EndpointGroupService.UpdateEndpointGroup(endpointGroup.ID, endpointGroup)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint group changes inside the database", err}
+	}
+
+	endpoints, err := handler.EndpointService.Endpoints()
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoints from the database", err}
+	}
+
+	for _, endpoint := range endpoints {
+		err = handler.updateEndpointGroup(endpoint, portainer.EndpointGroupID(endpointGroupID), payload.AssociatedEndpoints)
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to update endpoint", err}
+		}
+	}
+
+	return response.JSON(w, endpointGroup)
+}

api/http/handler/endpointgroups/endpointgroup_update_access.go

@@ -0,0 +1,63 @@
+package endpointgroups
+
+import (
+	"net/http"
+
+	"github.com/portainer/portainer"
+	httperror "github.com/portainer/portainer/http/error"
+	"github.com/portainer/portainer/http/request"
+	"github.com/portainer/portainer/http/response"
+)
+
+type endpointGroupUpdateAccessPayload struct {
+	AuthorizedUsers []int
+	AuthorizedTeams []int
+}
+
+func (payload *endpointGroupUpdateAccessPayload) Validate(r *http.Request) error {
+	return nil
+}
+
+// PUT request on /api/endpoint_groups/:id/access
+func (handler *Handler) endpointGroupUpdateAccess(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	endpointGroupID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint group identifier route variable", err}
+	}
+
+	var payload endpointGroupUpdateAccessPayload
+	err = request.DecodeAndValidateJSONPayload(r, &payload)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
+	}
+
+	endpointGroup, err := handler.EndpointGroupService.EndpointGroup(portainer.EndpointGroupID(endpointGroupID))
+	if err == portainer.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint group with the specified identifier inside the database", err}
+	} else if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint group with the specified identifier inside the database", err}
+	}
+
+	if payload.AuthorizedUsers != nil {
+		authorizedUserIDs := []portainer.UserID{}
+		for _, value := range payload.AuthorizedUsers {
+			authorizedUserIDs = append(authorizedUserIDs, portainer.UserID(value))
+		}
+		endpointGroup.AuthorizedUsers = authorizedUserIDs
+	}
+
+	if payload.AuthorizedTeams != nil {
+		authorizedTeamIDs := []portainer.TeamID{}
+		for _, value := range payload.AuthorizedTeams {
+			authorizedTeamIDs = append(authorizedTeamIDs, portainer.TeamID(value))
+		}
+		endpointGroup.AuthorizedTeams = authorizedTeamIDs
+	}
+
+	err = handler.EndpointGroupService.UpdateEndpointGroup(endpointGroup.ID, endpointGroup)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint group changes inside the database", err}
+	}
+
+	return response.JSON(w, endpointGroup)
+}

api/http/handler/endpointgroups/handler.go

@@ -0,0 +1,69 @@
+package endpointgroups
+
+import (
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"github.com/portainer/portainer"
+	httperror "github.com/portainer/portainer/http/error"
+	"github.com/portainer/portainer/http/security"
+)
+
+// Handler is the HTTP handler used to handle endpoint group operations.
+type Handler struct {
+	*mux.Router
+	EndpointService      portainer.EndpointService
+	EndpointGroupService portainer.EndpointGroupService
+}
+
+// NewHandler creates a handler to manage endpoint group operations.
+func NewHandler(bouncer *security.RequestBouncer) *Handler {
+	h := &Handler{
+		Router: mux.NewRouter(),
+	}
+	h.Handle("/endpoint_groups",
+		bouncer.AdministratorAccess(httperror.LoggerHandler(h.endpointGroupCreate))).Methods(http.MethodPost)
+	h.Handle("/endpoint_groups",
+		bouncer.RestrictedAccess(httperror.LoggerHandler(h.endpointGroupList))).Methods(http.MethodGet)
+	h.Handle("/endpoint_groups/{id}",
+		bouncer.AdministratorAccess(httperror.LoggerHandler(h.endpointGroupInspect))).Methods(http.MethodGet)
+	h.Handle("/endpoint_groups/{id}",
+		bouncer.AdministratorAccess(httperror.LoggerHandler(h.endpointGroupUpdate))).Methods(http.MethodPut)
+	h.Handle("/endpoint_groups/{id}/access",
+		bouncer.AdministratorAccess(httperror.LoggerHandler(h.endpointGroupUpdateAccess))).Methods(http.MethodPut)
+	h.Handle("/endpoint_groups/{id}",
+		bouncer.AdministratorAccess(httperror.LoggerHandler(h.endpointGroupDelete))).Methods(http.MethodDelete)
+
+	return h
+}
+
+func (handler *Handler) checkForGroupUnassignment(endpoint portainer.Endpoint, associatedEndpoints []portainer.EndpointID) error {
+	for _, id := range associatedEndpoints {
+		if id == endpoint.ID {
+			return nil
+		}
+	}
+
+	endpoint.GroupID = portainer.EndpointGroupID(1)
+	return handler.EndpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+}
+
+func (handler *Handler) checkForGroupAssignment(endpoint portainer.Endpoint, groupID portainer.EndpointGroupID, associatedEndpoints []portainer.EndpointID) error {
+	for _, id := range associatedEndpoints {
+
+		if id == endpoint.ID {
+			endpoint.GroupID = groupID
+			return handler.EndpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+		}
+	}
+	return nil
+}
+
+func (handler *Handler) updateEndpointGroup(endpoint portainer.Endpoint, groupID portainer.EndpointGroupID, associatedEndpoints []portainer.EndpointID) error {
+	if endpoint.GroupID == groupID {
+		return handler.checkForGroupUnassignment(endpoint, associatedEndpoints)
+	} else if endpoint.GroupID == portainer.EndpointGroupID(1) {
+		return handler.checkForGroupAssignment(endpoint, groupID, associatedEndpoints)
+	}
+	return nil
+}

api/http/handler/endpointproxy/handler.go

@@ -0,0 +1,32 @@
+package endpointproxy
+
+import (
+	"github.com/gorilla/mux"
+	"github.com/portainer/portainer"
+	httperror "github.com/portainer/portainer/http/error"
+	"github.com/portainer/portainer/http/proxy"
+	"github.com/portainer/portainer/http/security"
+)
+
+// Handler is the HTTP handler used to proxy requests to external APIs.
+type Handler struct {
+	*mux.Router
+	requestBouncer  *security.RequestBouncer
+	EndpointService portainer.EndpointService
+	ProxyManager    *proxy.Manager
+}
+
+// NewHandler creates a handler to proxy requests to external APIs.
+func NewHandler(bouncer *security.RequestBouncer) *Handler {
+	h := &Handler{
+		Router:         mux.NewRouter(),
+		requestBouncer: bouncer,
+	}
+	h.PathPrefix("/{id}/azure").Handler(
+		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.proxyRequestsToAzureAPI)))
+	h.PathPrefix("/{id}/docker").Handler(
+		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.proxyRequestsToDockerAPI)))
+	h.PathPrefix("/{id}/extensions/storidge").Handler(
+		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.proxyRequestsToStoridgeAPI)))
+	return h
+}

api/http/handler/endpointproxy/proxy_azure.go

@@ -0,0 +1,43 @@
+package endpointproxy
+
+import (
+	"strconv"
+
+	"github.com/portainer/portainer"
+	httperror "github.com/portainer/portainer/http/error"
+	"github.com/portainer/portainer/http/request"
+
+	"net/http"
+)
+
+func (handler *Handler) proxyRequestsToAzureAPI(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint identifier route variable", err}
+	}
+
+	endpoint, err := handler.EndpointService.Endpoint(portainer.EndpointID(endpointID))
+	if err == portainer.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
+	} else if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint with the specified identifier inside the database", err}
+	}
+
+	err = handler.requestBouncer.EndpointAccess(r, endpoint)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access endpoint", portainer.ErrEndpointAccessDenied}
+	}
+
+	var proxy http.Handler
+	proxy = handler.ProxyManager.GetProxy(string(endpointID))
+	if proxy == nil {
+		proxy, err = handler.ProxyManager.CreateAndRegisterProxy(endpoint)
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to create proxy", err}
+		}
+	}
+
+	id := strconv.Itoa(endpointID)
+	http.StripPrefix("/"+id+"/azure", proxy).ServeHTTP(w, r)
+	return nil
+}

api/http/handler/endpointproxy/proxy_docker.go

@@ -0,0 +1,43 @@
+package endpointproxy
+
+import (
+	"strconv"
+
+	"github.com/portainer/portainer"
+	httperror "github.com/portainer/portainer/http/error"
+	"github.com/portainer/portainer/http/request"
+
+	"net/http"
+)
+
+func (handler *Handler) proxyRequestsToDockerAPI(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint identifier route variable", err}
+	}
+
+	endpoint, err := handler.EndpointService.Endpoint(portainer.EndpointID(endpointID))
+	if err == portainer.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
+	} else if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint with the specified identifier inside the database", err}
+	}
+
+	err = handler.requestBouncer.EndpointAccess(r, endpoint)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access endpoint", portainer.ErrEndpointAccessDenied}
+	}
+
+	var proxy http.Handler
+	proxy = handler.ProxyManager.GetProxy(string(endpointID))
+	if proxy == nil {
+		proxy, err = handler.ProxyManager.CreateAndRegisterProxy(endpoint)
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to create proxy", err}
+		}
+	}
+
+	id := strconv.Itoa(endpointID)
+	http.StripPrefix("/"+id+"/docker", proxy).ServeHTTP(w, r)
+	return nil
+}

api/http/handler/endpointproxy/proxy_storidge.go

@@ -0,0 +1,56 @@
+package endpointproxy
+
+import (
+	"strconv"
+
+	"github.com/portainer/portainer"
+	httperror "github.com/portainer/portainer/http/error"
+	"github.com/portainer/portainer/http/request"
+
+	"net/http"
+)
+
+func (handler *Handler) proxyRequestsToStoridgeAPI(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint identifier route variable", err}
+	}
+
+	endpoint, err := handler.EndpointService.Endpoint(portainer.EndpointID(endpointID))
+	if err == portainer.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
+	} else if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint with the specified identifier inside the database", err}
+	}
+
+	err = handler.requestBouncer.EndpointAccess(r, endpoint)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access endpoint", portainer.ErrEndpointAccessDenied}
+	}
+
+	var storidgeExtension *portainer.EndpointExtension
+	for _, extension := range endpoint.Extensions {
+		if extension.Type == portainer.StoridgeEndpointExtension {
+			storidgeExtension = &extension
+		}
+	}
+
+	if storidgeExtension == nil {
+		return &httperror.HandlerError{http.StatusServiceUnavailable, "Storidge extension not supported on this endpoint", portainer.ErrEndpointExtensionNotSupported}
+	}
+
+	proxyExtensionKey := string(endpoint.ID) + "_" + string(portainer.StoridgeEndpointExtension)
+
+	var proxy http.Handler
+	proxy = handler.ProxyManager.GetExtensionProxy(proxyExtensionKey)
+	if proxy == nil {
+		proxy, err = handler.ProxyManager.CreateAndRegisterExtensionProxy(proxyExtensionKey, storidgeExtension.URL)
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to create extension proxy", err}
+		}
+	}
+
+	id := strconv.Itoa(endpointID)
+	http.StripPrefix("/"+id+"/extensions/storidge", proxy).ServeHTTP(w, r)
+	return nil
+}

api/http/handler/endpoints/endpoint_create.go

@@ -0,0 +1,338 @@
+package endpoints
+
+import (
+	"log"
+	"net/http"
+	"runtime"
+	"strconv"
+
+	"github.com/portainer/portainer"
+	"github.com/portainer/portainer/crypto"
+	"github.com/portainer/portainer/http/client"
+	httperror "github.com/portainer/portainer/http/error"
+	"github.com/portainer/portainer/http/request"
+	"github.com/portainer/portainer/http/response"
+)
+
+type endpointCreatePayload struct {
+	Name                   string
+	URL                    string
+	EndpointType           int
+	PublicURL              string
+	GroupID                int
+	TLS                    bool
+	TLSSkipVerify          bool
+	TLSSkipClientVerify    bool
+	TLSCACertFile          []byte
+	TLSCertFile            []byte
+	TLSKeyFile             []byte
+	AzureApplicationID     string
+	AzureTenantID          string
+	AzureAuthenticationKey string
+	Tags                   []string
+}
+
+func (payload *endpointCreatePayload) Validate(r *http.Request) error {
+	name, err := request.RetrieveMultiPartFormValue(r, "Name", false)
+	if err != nil {
+		return portainer.Error("Invalid stack name")
+	}
+	payload.Name = name
+
+	endpointType, err := request.RetrieveNumericMultiPartFormValue(r, "EndpointType", false)
+	if err != nil || endpointType == 0 {
+		return portainer.Error("Invalid endpoint type value. Value must be one of: 1 (Docker environment), 2 (Agent environment) or 3 (Azure environment)")
+	}
+	payload.EndpointType = endpointType
+
+	groupID, _ := request.RetrieveNumericMultiPartFormValue(r, "GroupID", true)
+	if groupID == 0 {
+		groupID = 1
+	}
+	payload.GroupID = groupID
+
+	var tags []string
+	err = request.RetrieveMultiPartFormJSONValue(r, "Tags", &tags, true)
+	if err != nil {
+		return portainer.Error("Invalid Tags parameter")
+	}
+	payload.Tags = tags
+	if payload.Tags == nil {
+		payload.Tags = make([]string, 0)
+	}
+
+	useTLS, _ := request.RetrieveBooleanMultiPartFormValue(r, "TLS", true)
+	payload.TLS = useTLS
+
+	if payload.TLS {
+		skipTLSServerVerification, _ := request.RetrieveBooleanMultiPartFormValue(r, "TLSSkipVerify", true)
+		payload.TLSSkipVerify = skipTLSServerVerification
+		skipTLSClientVerification, _ := request.RetrieveBooleanMultiPartFormValue(r, "TLSSkipClientVerify", true)
+		payload.TLSSkipClientVerify = skipTLSClientVerification
+
+		if !payload.TLSSkipVerify {
+			caCert, err := request.RetrieveMultiPartFormFile(r, "TLSCACertFile")
+			if err != nil {
+				return portainer.Error("Invalid CA certificate file. Ensure that the file is uploaded correctly")
+			}
+			payload.TLSCACertFile = caCert
+		}
+
+		if !payload.TLSSkipClientVerify {
+			cert, err := request.RetrieveMultiPartFormFile(r, "TLSCertFile")
+			if err != nil {
+				return portainer.Error("Invalid certificate file. Ensure that the file is uploaded correctly")
+			}
+			payload.TLSCertFile = cert
+
+			key, err := request.RetrieveMultiPartFormFile(r, "TLSKeyFile")
+			if err != nil {
+				return portainer.Error("Invalid key file. Ensure that the file is uploaded correctly")
+			}
+			payload.TLSKeyFile = key
+		}
+	}
+
+	switch portainer.EndpointType(payload.EndpointType) {
+	case portainer.AzureEnvironment:
+		azureApplicationID, err := request.RetrieveMultiPartFormValue(r, "AzureApplicationID", false)
+		if err != nil {
+			return portainer.Error("Invalid Azure application ID")
+		}
+		payload.AzureApplicationID = azureApplicationID
+
+		azureTenantID, err := request.RetrieveMultiPartFormValue(r, "AzureTenantID", false)
+		if err != nil {
+			return portainer.Error("Invalid Azure tenant ID")
+		}
+		payload.AzureTenantID = azureTenantID
+
+		azureAuthenticationKey, err := request.RetrieveMultiPartFormValue(r, "AzureAuthenticationKey", false)
+		if err != nil {
+			return portainer.Error("Invalid Azure authentication key")
+		}
+		payload.AzureAuthenticationKey = azureAuthenticationKey
+	default:
+		url, err := request.RetrieveMultiPartFormValue(r, "URL", true)
+		if err != nil {
+			return portainer.Error("Invalid endpoint URL")
+		}
+		payload.URL = url
+
+		publicURL, _ := request.RetrieveMultiPartFormValue(r, "PublicURL", true)
+		payload.PublicURL = publicURL
+	}
+
+	return nil
+}
+
+// POST request on /api/endpoints
+func (handler *Handler) endpointCreate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	if !handler.authorizeEndpointManagement {
+		return &httperror.HandlerError{http.StatusServiceUnavailable, "Endpoint management is disabled", ErrEndpointManagementDisabled}
+	}
+
+	payload := &endpointCreatePayload{}
+	err := payload.Validate(r)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
+	}
+
+	endpoint, endpointCreationError := handler.createEndpoint(payload)
+	if endpointCreationError != nil {
+		return endpointCreationError
+	}
+
+	return response.JSON(w, endpoint)
+}
+
+func (handler *Handler) createEndpoint(payload *endpointCreatePayload) (*portainer.Endpoint, *httperror.HandlerError) {
+	if portainer.EndpointType(payload.EndpointType) == portainer.AzureEnvironment {
+		return handler.createAzureEndpoint(payload)
+	}
+
+	if payload.TLS {
+		return handler.createTLSSecuredEndpoint(payload)
+	}
+	return handler.createUnsecuredEndpoint(payload)
+}
+
+func (handler *Handler) createAzureEndpoint(payload *endpointCreatePayload) (*portainer.Endpoint, *httperror.HandlerError) {
+	credentials := portainer.AzureCredentials{
+		ApplicationID:     payload.AzureApplicationID,
+		TenantID:          payload.AzureTenantID,
+		AuthenticationKey: payload.AzureAuthenticationKey,
+	}
+
+	httpClient := client.NewHTTPClient()
+	_, err := httpClient.ExecuteAzureAuthenticationRequest(&credentials)
+	if err != nil {
+		return nil, &httperror.HandlerError{http.StatusInternalServerError, "Unable to authenticate against Azure", err}
+	}
+
+	endpointID := handler.EndpointService.GetNextIdentifier()
+	endpoint := &portainer.Endpoint{
+		ID:               portainer.EndpointID(endpointID),
+		Name:             payload.Name,
+		URL:              payload.URL,
+		Type:             portainer.AzureEnvironment,
+		GroupID:          portainer.EndpointGroupID(payload.GroupID),
+		PublicURL:        payload.PublicURL,
+		AuthorizedUsers:  []portainer.UserID{},
+		AuthorizedTeams:  []portainer.TeamID{},
+		Extensions:       []portainer.EndpointExtension{},
+		AzureCredentials: credentials,
+		Tags:             payload.Tags,
+		Status:           portainer.EndpointStatusUp,
+		Snapshots:        []portainer.Snapshot{},
+	}
+
+	err = handler.EndpointService.CreateEndpoint(endpoint)
+	if err != nil {
+		return nil, &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint inside the database", err}
+	}
+
+	return endpoint, nil
+}
+
+func (handler *Handler) createUnsecuredEndpoint(payload *endpointCreatePayload) (*portainer.Endpoint, *httperror.HandlerError) {
+	endpointType := portainer.DockerEnvironment
+
+	if payload.URL == "" {
+		payload.URL = "unix:///var/run/docker.sock"
+		if runtime.GOOS == "windows" {
+			payload.URL = "npipe:////./pipe/docker_engine"
+		}
+	} else {
+		agentOnDockerEnvironment, err := client.ExecutePingOperation(payload.URL, nil)
+		if err != nil {
+			return nil, &httperror.HandlerError{http.StatusInternalServerError, "Unable to ping Docker environment", err}
+		}
+		if agentOnDockerEnvironment {
+			endpointType = portainer.AgentOnDockerEnvironment
+		}
+	}
+
+	endpointID := handler.EndpointService.GetNextIdentifier()
+	endpoint := &portainer.Endpoint{
+		ID:        portainer.EndpointID(endpointID),
+		Name:      payload.Name,
+		URL:       payload.URL,
+		Type:      endpointType,
+		GroupID:   portainer.EndpointGroupID(payload.GroupID),
+		PublicURL: payload.PublicURL,
+		TLSConfig: portainer.TLSConfiguration{
+			TLS: false,
+		},
+		AuthorizedUsers: []portainer.UserID{},
+		AuthorizedTeams: []portainer.TeamID{},
+		Extensions:      []portainer.EndpointExtension{},
+		Tags:            payload.Tags,
+		Status:          portainer.EndpointStatusUp,
+		Snapshots:       []portainer.Snapshot{},
+	}
+
+	err := handler.snapshotAndPersistEndpoint(endpoint)
+	if err != nil {
+		return nil, err
+	}
+
+	return endpoint, nil
+}
+
+func (handler *Handler) createTLSSecuredEndpoint(payload *endpointCreatePayload) (*portainer.Endpoint, *httperror.HandlerError) {
+	tlsConfig, err := crypto.CreateTLSConfigurationFromBytes(payload.TLSCACertFile, payload.TLSCertFile, payload.TLSKeyFile, payload.TLSSkipClientVerify, payload.TLSSkipVerify)
+	if err != nil {
+		return nil, &httperror.HandlerError{http.StatusInternalServerError, "Unable to create TLS configuration", err}
+	}
+
+	agentOnDockerEnvironment, err := client.ExecutePingOperation(payload.URL, tlsConfig)
+	if err != nil {
+		return nil, &httperror.HandlerError{http.StatusInternalServerError, "Unable to ping Docker environment", err}
+	}
+
+	endpointType := portainer.DockerEnvironment
+	if agentOnDockerEnvironment {
+		endpointType = portainer.AgentOnDockerEnvironment
+	}
+
+	endpointID := handler.EndpointService.GetNextIdentifier()
+	endpoint := &portainer.Endpoint{
+		ID:        portainer.EndpointID(endpointID),
+		Name:      payload.Name,
+		URL:       payload.URL,
+		Type:      endpointType,
+		GroupID:   portainer.EndpointGroupID(payload.GroupID),
+		PublicURL: payload.PublicURL,
+		TLSConfig: portainer.TLSConfiguration{
+			TLS:           payload.TLS,
+			TLSSkipVerify: payload.TLSSkipVerify,
+		},
+		AuthorizedUsers: []portainer.UserID{},
+		AuthorizedTeams: []portainer.TeamID{},
+		Extensions:      []portainer.EndpointExtension{},
+		Tags:            payload.Tags,
+		Status:          portainer.EndpointStatusUp,
+		Snapshots:       []portainer.Snapshot{},
+	}
+
+	filesystemError := handler.storeTLSFiles(endpoint, payload)
+	if err != nil {
+		return nil, filesystemError
+	}
+
+	endpointCreationError := handler.snapshotAndPersistEndpoint(endpoint)
+	if endpointCreationError != nil {
+		return nil, endpointCreationError
+	}
+
+	return endpoint, nil
+}
+
+func (handler *Handler) snapshotAndPersistEndpoint(endpoint *portainer.Endpoint) *httperror.HandlerError {
+	snapshot, err := handler.Snapshotter.CreateSnapshot(endpoint)
+	endpoint.Status = portainer.EndpointStatusUp
+	if err != nil {
+		log.Printf("http error: endpoint snapshot error (endpoint=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
+		endpoint.Status = portainer.EndpointStatusDown
+	}
+
+	if snapshot != nil {
+		endpoint.Snapshots = []portainer.Snapshot{*snapshot}
+	}
+
+	err = handler.EndpointService.CreateEndpoint(endpoint)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint inside the database", err}
+	}
+
+	return nil
+}
+
+func (handler *Handler) storeTLSFiles(endpoint *portainer.Endpoint, payload *endpointCreatePayload) *httperror.HandlerError {
+	folder := strconv.Itoa(int(endpoint.ID))
+
+	if !payload.TLSSkipVerify {
+		caCertPath, err := handler.FileService.StoreTLSFileFromBytes(folder, portainer.TLSFileCA, payload.TLSCACertFile)
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist TLS CA certificate file on disk", err}
+		}
+		endpoint.TLSConfig.TLSCACertPath = caCertPath
+	}
+
+	if !payload.TLSSkipClientVerify {
+		certPath, err := handler.FileService.StoreTLSFileFromBytes(folder, portainer.TLSFileCert, payload.TLSCertFile)
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist TLS certificate file on disk", err}
+		}
+		endpoint.TLSConfig.TLSCertPath = certPath
+
+		keyPath, err := handler.FileService.StoreTLSFileFromBytes(folder, portainer.TLSFileKey, payload.TLSKeyFile)
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist TLS key file on disk", err}
+		}
+		endpoint.TLSConfig.TLSKeyPath = keyPath
+	}
+
+	return nil
+}

api/http/handler/endpoints/endpoint_delete.go

@@ -0,0 +1,48 @@
+package endpoints
+
+import (
+	"net/http"
+	"strconv"
+
+	"github.com/portainer/portainer"
+	httperror "github.com/portainer/portainer/http/error"
+	"github.com/portainer/portainer/http/request"
+	"github.com/portainer/portainer/http/response"
+)
+
+// DELETE request on /api/endpoints/:id
+func (handler *Handler) endpointDelete(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	if !handler.authorizeEndpointManagement {
+		return &httperror.HandlerError{http.StatusServiceUnavailable, "Endpoint management is disabled", ErrEndpointManagementDisabled}
+	}
+
+	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint identifier route variable", err}
+	}
+
+	endpoint, err := handler.EndpointService.Endpoint(portainer.EndpointID(endpointID))
+	if err == portainer.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
+	} else if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint with the specified identifier inside the database", err}
+	}
+
+	if endpoint.TLSConfig.TLS {
+		folder := strconv.Itoa(endpointID)
+		err = handler.FileService.DeleteTLSFiles(folder)
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to remove TLS files from disk", err}
+		}
+	}
+
+	err = handler.EndpointService.DeleteEndpoint(portainer.EndpointID(endpointID))
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to remove endpoint from the database", err}
+	}
+
+	handler.ProxyManager.DeleteProxy(string(endpointID))
+	handler.ProxyManager.DeleteExtensionProxies(string(endpointID))
+
+	return response.Empty(w)
+}

api/http/handler/endpoints/endpoint_extension_add.go

@@ -0,0 +1,73 @@
+package endpoints
+
+import (
+	"net/http"
+
+	"github.com/asaskevich/govalidator"
+	"github.com/portainer/portainer"
+	httperror "github.com/portainer/portainer/http/error"
+	"github.com/portainer/portainer/http/request"
+	"github.com/portainer/portainer/http/response"
+)
+
+type endpointExtensionAddPayload struct {
+	Type int
+	URL  string
+}
+
+func (payload *endpointExtensionAddPayload) Validate(r *http.Request) error {
+	if payload.Type != 1 {
+		return portainer.Error("Invalid type value. Value must be one of: 1 (Storidge)")
+	}
+	if payload.Type == 1 && govalidator.IsNull(payload.URL) {
+		return portainer.Error("Invalid extension URL")
+	}
+	return nil
+}
+
+// POST request on /api/endpoints/:id/extensions
+func (handler *Handler) endpointExtensionAdd(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint identifier route variable", err}
+	}
+
+	endpoint, err := handler.EndpointService.Endpoint(portainer.EndpointID(endpointID))
+	if err == portainer.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
+	} else if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint with the specified identifier inside the database", err}
+	}
+
+	var payload endpointExtensionAddPayload
+	err = request.DecodeAndValidateJSONPayload(r, &payload)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
+	}
+
+	extensionType := portainer.EndpointExtensionType(payload.Type)
+
+	var extension *portainer.EndpointExtension
+	for _, ext := range endpoint.Extensions {
+		if ext.Type == extensionType {
+			extension = &ext
+		}
+	}
+
+	if extension != nil {
+		extension.URL = payload.URL
+	} else {
+		extension = &portainer.EndpointExtension{
+			Type: extensionType,
+			URL:  payload.URL,
+		}
+		endpoint.Extensions = append(endpoint.Extensions, *extension)
+	}
+
+	err = handler.EndpointService.UpdateEndpoint(endpoint.ID, endpoint)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint changes inside the database", err}
+	}
+
+	return response.JSON(w, extension)
+}

api/http/handler/endpoints/endpoint_extension_remove.go

@@ -0,0 +1,43 @@
+package endpoints
+
+import (
+	"net/http"
+
+	"github.com/portainer/portainer"
+	httperror "github.com/portainer/portainer/http/error"
+	"github.com/portainer/portainer/http/request"
+	"github.com/portainer/portainer/http/response"
+)
+
+// DELETE request on /api/endpoints/:id/extensions/:extensionType
+func (handler *Handler) endpointExtensionRemove(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint identifier route variable", err}
+	}
+
+	endpoint, err := handler.EndpointService.Endpoint(portainer.EndpointID(endpointID))
+	if err == portainer.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
+	} else if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint with the specified identifier inside the database", err}
+	}
+
+	extensionType, err := request.RetrieveNumericRouteVariableValue(r, "extensionType")
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid extension type route variable", err}
+	}
+
+	for idx, ext := range endpoint.Extensions {
+		if ext.Type == portainer.EndpointExtensionType(extensionType) {
+			endpoint.Extensions = append(endpoint.Extensions[:idx], endpoint.Extensions[idx+1:]...)
+		}
+	}
+
+	err = handler.EndpointService.UpdateEndpoint(endpoint.ID, endpoint)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint changes inside the database", err}
+	}
+
+	return response.Empty(w)
+}

api/http/handler/endpoints/endpoint_inspect.go

@@ -0,0 +1,34 @@
+package endpoints
+
+import (
+	"net/http"
+
+	"github.com/portainer/portainer"
+	httperror "github.com/portainer/portainer/http/error"
+	"github.com/portainer/portainer/http/request"
+	"github.com/portainer/portainer/http/response"
+)
+
+// GET request on /api/endpoints/:id
+func (handler *Handler) endpointInspect(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint identifier route variable", err}
+	}
+
+	endpoint, err := handler.EndpointService.Endpoint(portainer.EndpointID(endpointID))
+	if err == portainer.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
+	} else if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint with the specified identifier inside the database", err}
+	}
+
+	err = handler.requestBouncer.EndpointAccess(r, endpoint)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access endpoint", portainer.ErrEndpointAccessDenied}
+	}
+
+	hideFields(endpoint)
+
+	return response.JSON(w, endpoint)
+}

api/http/handler/endpoints/endpoint_list.go

@@ -0,0 +1,34 @@
+package endpoints
+
+import (
+	"net/http"
+
+	httperror "github.com/portainer/portainer/http/error"
+	"github.com/portainer/portainer/http/response"
+	"github.com/portainer/portainer/http/security"
+)
+
+// GET request on /api/endpoints
+func (handler *Handler) endpointList(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	endpoints, err := handler.EndpointService.Endpoints()
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoints from the database", err}
+	}
+
+	endpointGroups, err := handler.EndpointGroupService.EndpointGroups()
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoint groups from the database", err}
+	}
+
+	securityContext, err := security.RetrieveRestrictedRequestContext(r)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve info from request context", err}
+	}
+
+	filteredEndpoints := security.FilterEndpoints(endpoints, endpointGroups, securityContext)
+
+	for _, endpoint := range filteredEndpoints {
+		hideFields(&endpoint)
+	}
+	return response.JSON(w, filteredEndpoints)
+}

api/http/handler/endpoints/endpoint_snapshot.go

@@ -0,0 +1,42 @@
+package endpoints
+
+import (
+	"log"
+	"net/http"
+
+	"github.com/portainer/portainer"
+	httperror "github.com/portainer/portainer/http/error"
+	"github.com/portainer/portainer/http/response"
+)
+
+// POST request on /api/endpoints/snapshot
+func (handler *Handler) endpointSnapshot(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	endpoints, err := handler.EndpointService.Endpoints()
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoints from the database", err}
+	}
+
+	for _, endpoint := range endpoints {
+		if endpoint.Type == portainer.AzureEnvironment {
+			continue
+		}
+
+		snapshot, err := handler.Snapshotter.CreateSnapshot(&endpoint)
+		endpoint.Status = portainer.EndpointStatusUp
+		if err != nil {
+			log.Printf("http error: endpoint snapshot error (endpoint=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
+			endpoint.Status = portainer.EndpointStatusDown
+		}
+
+		if snapshot != nil {
+			endpoint.Snapshots = []portainer.Snapshot{*snapshot}
+		}
+
+		err = handler.EndpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint changes inside the database", err}
+		}
+	}
+
+	return response.Empty(w)
+}

api/http/handler/endpoints/endpoint_update.go

@@ -0,0 +1,142 @@
+package endpoints
+
+import (
+	"net/http"
+	"strconv"
+
+	"github.com/portainer/portainer"
+	"github.com/portainer/portainer/http/client"
+	httperror "github.com/portainer/portainer/http/error"
+	"github.com/portainer/portainer/http/request"
+	"github.com/portainer/portainer/http/response"
+)
+
+type endpointUpdatePayload struct {
+	Name                   string
+	URL                    string
+	PublicURL              string
+	GroupID                int
+	TLS                    bool
+	TLSSkipVerify          bool
+	TLSSkipClientVerify    bool
+	AzureApplicationID     string
+	AzureTenantID          string
+	AzureAuthenticationKey string
+	Tags                   []string
+}
+
+func (payload *endpointUpdatePayload) Validate(r *http.Request) error {
+	return nil
+}
+
+// PUT request on /api/endpoints/:id
+func (handler *Handler) endpointUpdate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	if !handler.authorizeEndpointManagement {
+		return &httperror.HandlerError{http.StatusServiceUnavailable, "Endpoint management is disabled", ErrEndpointManagementDisabled}
+	}
+
+	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint identifier route variable", err}
+	}
+
+	var payload endpointUpdatePayload
+	err = request.DecodeAndValidateJSONPayload(r, &payload)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
+	}
+
+	endpoint, err := handler.EndpointService.Endpoint(portainer.EndpointID(endpointID))
+	if err == portainer.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
+	} else if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint with the specified identifier inside the database", err}
+	}
+
+	if payload.Name != "" {
+		endpoint.Name = payload.Name
+	}
+
+	if payload.URL != "" {
+		endpoint.URL = payload.URL
+	}
+
+	if payload.PublicURL != "" {
+		endpoint.PublicURL = payload.PublicURL
+	}
+
+	if payload.GroupID != 0 {
+		endpoint.GroupID = portainer.EndpointGroupID(payload.GroupID)
+	}
+
+	if payload.Tags != nil {
+		endpoint.Tags = payload.Tags
+	}
+
+	if endpoint.Type == portainer.AzureEnvironment {
+		credentials := endpoint.AzureCredentials
+		if payload.AzureApplicationID != "" {
+			credentials.ApplicationID = payload.AzureApplicationID
+		}
+		if payload.AzureTenantID != "" {
+			credentials.TenantID = payload.AzureTenantID
+		}
+		if payload.AzureAuthenticationKey != "" {
+			credentials.AuthenticationKey = payload.AzureAuthenticationKey
+		}
+
+		httpClient := client.NewHTTPClient()
+		_, authErr := httpClient.ExecuteAzureAuthenticationRequest(&credentials)
+		if authErr != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to authenticate against Azure", authErr}
+		}
+		endpoint.AzureCredentials = credentials
+	}
+
+	folder := strconv.Itoa(endpointID)
+	if payload.TLS {
+		endpoint.TLSConfig.TLS = true
+		endpoint.TLSConfig.TLSSkipVerify = payload.TLSSkipVerify
+		if !payload.TLSSkipVerify {
+			caCertPath, _ := handler.FileService.GetPathForTLSFile(folder, portainer.TLSFileCA)
+			endpoint.TLSConfig.TLSCACertPath = caCertPath
+		} else {
+			endpoint.TLSConfig.TLSCACertPath = ""
+			handler.FileService.DeleteTLSFile(folder, portainer.TLSFileCA)
+		}
+
+		if !payload.TLSSkipClientVerify {
+			certPath, _ := handler.FileService.GetPathForTLSFile(folder, portainer.TLSFileCert)
+			endpoint.TLSConfig.TLSCertPath = certPath
+			keyPath, _ := handler.FileService.GetPathForTLSFile(folder, portainer.TLSFileKey)
+			endpoint.TLSConfig.TLSKeyPath = keyPath
+		} else {
+			endpoint.TLSConfig.TLSCertPath = ""
+			handler.FileService.DeleteTLSFile(folder, portainer.TLSFileCert)
+			endpoint.TLSConfig.TLSKeyPath = ""
+			handler.FileService.DeleteTLSFile(folder, portainer.TLSFileKey)
+		}
+	} else {
+		endpoint.TLSConfig.TLS = false
+		endpoint.TLSConfig.TLSSkipVerify = false
+		endpoint.TLSConfig.TLSCACertPath = ""
+		endpoint.TLSConfig.TLSCertPath = ""
+		endpoint.TLSConfig.TLSKeyPath = ""
+		err = handler.FileService.DeleteTLSFiles(folder)
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to remove TLS files from disk", err}
+		}
+	}
+
+	_, err = handler.ProxyManager.CreateAndRegisterProxy(endpoint)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to register HTTP proxy for the endpoint", err}
+	}
+
+	err = handler.EndpointService.UpdateEndpoint(endpoint.ID, endpoint)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint changes inside the database", err}
+	}
+
+	return response.JSON(w, endpoint)
+}

api/http/handler/endpoints/endpoint_update_access.go

@@ -0,0 +1,67 @@
+package endpoints
+
+import (
+	"net/http"
+
+	"github.com/portainer/portainer"
+	httperror "github.com/portainer/portainer/http/error"
+	"github.com/portainer/portainer/http/request"
+	"github.com/portainer/portainer/http/response"
+)
+
+type endpointUpdateAccessPayload struct {
+	AuthorizedUsers []int
+	AuthorizedTeams []int
+}
+
+func (payload *endpointUpdateAccessPayload) Validate(r *http.Request) error {
+	return nil
+}
+
+// PUT request on /api/endpoints/:id/access
+func (handler *Handler) endpointUpdateAccess(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	if !handler.authorizeEndpointManagement {
+		return &httperror.HandlerError{http.StatusServiceUnavailable, "Endpoint management is disabled", ErrEndpointManagementDisabled}
+	}
+
+	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint identifier route variable", err}
+	}
+
+	var payload endpointUpdateAccessPayload
+	err = request.DecodeAndValidateJSONPayload(r, &payload)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
+	}
+
+	endpoint, err := handler.EndpointService.Endpoint(portainer.EndpointID(endpointID))
+	if err == portainer.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
+	} else if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint with the specified identifier inside the database", err}
+	}
+
+	if payload.AuthorizedUsers != nil {
+		authorizedUserIDs := []portainer.UserID{}
+		for _, value := range payload.AuthorizedUsers {
+			authorizedUserIDs = append(authorizedUserIDs, portainer.UserID(value))
+		}
+		endpoint.AuthorizedUsers = authorizedUserIDs
+	}
+
+	if payload.AuthorizedTeams != nil {
+		authorizedTeamIDs := []portainer.TeamID{}
+		for _, value := range payload.AuthorizedTeams {
+			authorizedTeamIDs = append(authorizedTeamIDs, portainer.TeamID(value))
+		}
+		endpoint.AuthorizedTeams = authorizedTeamIDs
+	}
+
+	err = handler.EndpointService.UpdateEndpoint(endpoint.ID, endpoint)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint changes inside the database", err}
+	}
+
+	return response.JSON(w, endpoint)
+}

api/http/handler/endpoints/handler.go

@@ -0,0 +1,64 @@
+package endpoints
+
+import (
+	"github.com/portainer/portainer"
+	httperror "github.com/portainer/portainer/http/error"
+	"github.com/portainer/portainer/http/proxy"
+	"github.com/portainer/portainer/http/security"
+
+	"net/http"
+
+	"github.com/gorilla/mux"
+)
+
+const (
+	// ErrEndpointManagementDisabled is an error raised when trying to access the endpoints management endpoints
+	// when the server has been started with the --external-endpoints flag
+	ErrEndpointManagementDisabled = portainer.Error("Endpoint management is disabled")
+)
+
+func hideFields(endpoint *portainer.Endpoint) {
+	endpoint.AzureCredentials = portainer.AzureCredentials{}
+}
+
+// Handler is the HTTP handler used to handle endpoint operations.
+type Handler struct {
+	*mux.Router
+	authorizeEndpointManagement bool
+	requestBouncer              *security.RequestBouncer
+	EndpointService             portainer.EndpointService
+	EndpointGroupService        portainer.EndpointGroupService
+	FileService                 portainer.FileService
+	ProxyManager                *proxy.Manager
+	Snapshotter                 portainer.Snapshotter
+}
+
+// NewHandler creates a handler to manage endpoint operations.
+func NewHandler(bouncer *security.RequestBouncer, authorizeEndpointManagement bool) *Handler {
+	h := &Handler{
+		Router: mux.NewRouter(),
+		authorizeEndpointManagement: authorizeEndpointManagement,
+		requestBouncer:              bouncer,
+	}
+
+	h.Handle("/endpoints",
+		bouncer.AdministratorAccess(httperror.LoggerHandler(h.endpointCreate))).Methods(http.MethodPost)
+	h.Handle("/endpoints/snapshot",
+		bouncer.AdministratorAccess(httperror.LoggerHandler(h.endpointSnapshot))).Methods(http.MethodPost)
+	h.Handle("/endpoints",
+		bouncer.RestrictedAccess(httperror.LoggerHandler(h.endpointList))).Methods(http.MethodGet)
+	h.Handle("/endpoints/{id}",
+		bouncer.RestrictedAccess(httperror.LoggerHandler(h.endpointInspect))).Methods(http.MethodGet)
+	h.Handle("/endpoints/{id}",
+		bouncer.AdministratorAccess(httperror.LoggerHandler(h.endpointUpdate))).Methods(http.MethodPut)
+	h.Handle("/endpoints/{id}/access",
+		bouncer.AdministratorAccess(httperror.LoggerHandler(h.endpointUpdateAccess))).Methods(http.MethodPut)
+	h.Handle("/endpoints/{id}",
+		bouncer.AdministratorAccess(httperror.LoggerHandler(h.endpointDelete))).Methods(http.MethodDelete)
+	h.Handle("/endpoints/{id}/extensions",
+		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.endpointExtensionAdd))).Methods(http.MethodPost)
+	h.Handle("/endpoints/{id}/extensions/{extensionType}",
+		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.endpointExtensionRemove))).Methods(http.MethodDelete)
+
+	return h
+}

api/http/handler/file.go

@@ -1,61 +0,0 @@
-package handler
-
-import (
-	"os"
-
-	"github.com/portainer/portainer"
-	httperror "github.com/portainer/portainer/http/error"
-
-	"log"
-	"net/http"
-	"path"
-	"strings"
-)
-
-// FileHandler represents an HTTP API handler for managing static files.
-type FileHandler struct {
-	http.Handler
-	Logger             *log.Logger
-	allowedDirectories map[string]bool
-}
-
-// NewFileHandler returns a new instance of FileHandler.
-func NewFileHandler(assetPath string) *FileHandler {
-	h := &FileHandler{
-		Handler: http.FileServer(http.Dir(assetPath)),
-		Logger:  log.New(os.Stderr, "", log.LstdFlags),
-		allowedDirectories: map[string]bool{
-			"/":       true,
-			"/css":    true,
-			"/js":     true,
-			"/images": true,
-			"/fonts":  true,
-		},
-	}
-	return h
-}
-
-func isHTML(acceptContent []string) bool {
-	for _, accept := range acceptContent {
-		if strings.Contains(accept, "text/html") {
-			return true
-		}
-	}
-	return false
-}
-
-func (handler *FileHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	requestDirectory := path.Dir(r.URL.Path)
-	if !handler.allowedDirectories[requestDirectory] {
-		httperror.WriteErrorResponse(w, portainer.ErrResourceNotFound, http.StatusNotFound, handler.Logger)
-		return
-	}
-
-	if !isHTML(r.Header["Accept"]) {
-		w.Header().Set("Cache-Control", "max-age=31536000")
-	} else {
-		w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate")
-	}
-
-	handler.Handler.ServeHTTP(w, r)
-}