chore(version): bump version number

This reverts commit 1fbe6a12f1.

.babelrc

@@ -0,0 +1,12 @@
+{
+  "plugins": ["lodash", "angularjs-annotate"],
+  "presets": [
+    [
+      "@babel/preset-env",
+      {
+        "modules": false,
+        "useBuiltIns": "usage"
+      }
+    ]
+  ]
+}

.codeclimate.yml

@@ -1,5 +1,42 @@
----
-engines:
+version: "2"
+checks:
+  argument-count:
+    enabled: true
+    config:
+      threshold: 4
+  complex-logic:
+    enabled: true
+    config:
+      threshold: 4
+  file-lines:
+    enabled: true
+    config:
+      threshold: 300
+  method-complexity:
+    enabled: false
+  method-count:
+    enabled: true
+    config:
+      threshold: 20
+  method-lines:
+    enabled: true
+    config:
+      threshold: 50
+  nested-control-flow:
+    enabled: true
+    config:
+      threshold: 4
+  return-statements:
+    enabled: false
+  similar-code:
+    enabled: true
+    config:
+      threshold: #language-specific defaults. overrides affect all languages.
+  identical-code:
+    enabled: true
+    config:
+      threshold: #language-specific defaults. overrides affect all languages.
+plugins:
   gofmt:
     enabled: true
   golint:
@@ -16,14 +53,10 @@ engines:
           mass_threshold: 80
   eslint:
     enabled: true
+    channel: "eslint-5"
     config:
       config: .eslintrc.yml
   fixme:
     enabled: true
-ratings:
-  paths:
-  - "**.css"
-  - "**.js"
-  - "**.go"
-exclude_paths:
+exclude_patterns:
 - test/

.codefresh/codefresh_branch.yml

@@ -1,46 +0,0 @@
-version: '1.0'
-steps:
-
-  build_backend:
-    image: portainer/golang-builder:ci
-    working_directory: ${{main_clone}}
-    commands:
-      - mkdir -p /go/src/github.com/${{CF_REPO_OWNER}}
-      - ln -s /codefresh/volume/${{CF_REPO_NAME}}/api /go/src/github.com/${{CF_REPO_OWNER}}/${{CF_REPO_NAME}}
-      - /build.sh api/cmd/portainer
-
-  build_frontend:
-    image: portainer/angular-builder:latest
-    working_directory: ${{build_backend}}
-    commands:
-      - yarn
-      - yarn grunt build-webapp
-      - mv api/cmd/portainer/portainer dist/
-
-  get_docker_version:
-    image: alpine
-    working_directory: ${{build_frontend}}
-    commands:
-      - cf_export DOCKER_VERSION=`cat gruntfile.js | grep -m 1 'shippedDockerVersion' | cut -d\' -f2`
-
-  download_docker_binary:
-    image: busybox
-    working_directory: ${{build_frontend}}
-    commands:
-      - echo ${{DOCKER_VERSION}}
-      - wget -O /tmp/docker-binaries.tgz https://download.docker.com/linux/static/stable/x86_64/docker-${{DOCKER_VERSION}}.tgz
-      - tar -xf /tmp/docker-binaries.tgz -C /tmp
-      - mv /tmp/docker/docker dist/
-
-  build_image:
-    type: build
-    working_directory: ${{download_docker_binary}}
-    dockerfile: ./build/linux/Dockerfile
-    image_name: portainer/portainer
-    tag: ${{CF_BRANCH}}
-
-  push_image:
-    type: push
-    candidate: '${{build_image}}'
-    tag: '${{CF_BRANCH}}'
-    registry: dockerhub

.codefresh/codefresh_pullrequest.yml

@@ -1,46 +0,0 @@
-version: '1.0'
-steps:
-
-  build_backend:
-    image: portainer/golang-builder:ci
-    working_directory: ${{main_clone}}
-    commands:
-      - mkdir -p /go/src/github.com/${{CF_REPO_OWNER}}
-      - ln -s /codefresh/volume/${{CF_REPO_NAME}}/api /go/src/github.com/${{CF_REPO_OWNER}}/${{CF_REPO_NAME}}
-      - /build.sh api/cmd/portainer
-
-  build_frontend:
-    image: portainer/angular-builder:latest
-    working_directory: ${{build_backend}}
-    commands:
-      - yarn
-      - yarn grunt build-webapp
-      - mv api/cmd/portainer/portainer dist/
-
-  get_docker_version:
-    image: alpine
-    working_directory: ${{build_frontend}}
-    commands:
-      - cf_export DOCKER_VERSION=`cat gruntfile.js | grep -m 1 'shippedDockerVersion' | cut -d\' -f2`
-
-  download_docker_binary:
-    image: busybox
-    working_directory: ${{build_frontend}}
-    commands:
-      - echo ${{DOCKER_VERSION}}
-      - wget -O /tmp/docker-binaries.tgz https://download.docker.com/linux/static/stable/x86_64/docker-${{DOCKER_VERSION}}.tgz
-      - tar -xf /tmp/docker-binaries.tgz -C /tmp
-      - mv /tmp/docker/docker dist/
-
-  build_image:
-    type: build
-    working_directory: ${{download_docker_binary}}
-    dockerfile: ./build/linux/Dockerfile
-    image_name: portainer/portainer
-    tag: ${{CF_BRANCH}}
-
-  push_image:
-    type: push
-    candidate: '${{build_image}}'
-    tag: 'pr${{CF_PULL_REQUEST_NUMBER}}'
-    registry: dockerhub

.eslintignore

@@ -0,0 +1,3 @@
+node_modules/
+dist/
+test/

.eslintrc.yml

@@ -1,284 +1,292 @@
 env:
   browser: true
   jquery: true
+  node: true
+  es6: true
+
+globals:
+  angular: true
+  __CONFIG_GA_ID: true
 
-# globals:
-#   angular: true
-#   $: true
-#   _: true
-#   moment: true
-#   filesize: true
-#   splitargs: true
 extends:
   - 'eslint:recommended'
 
-# http://eslint.org/docs/rules/
+parserOptions:
+  ecmaVersion: 2018
+  sourceType: module
+  ecmaFeatures:
+    modules: true
+
+# # http://eslint.org/docs/rules/
 rules:
-  # Possible Errors
-  no-await-in-loop: off
-  no-cond-assign: error
-  no-console: off
-  no-constant-condition: error
-  no-control-regex: error
-  no-debugger: error
-  no-dupe-args: error
-  no-dupe-keys: error
-  no-duplicate-case: error
-  no-empty-character-class: error
-  no-empty: error
-  no-ex-assign: error
-  no-extra-boolean-cast: error
-  no-extra-parens: off
-  no-extra-semi: error
-  no-func-assign: error
-  no-inner-declarations:
-    - error
-    - functions
-  no-invalid-regexp: error
-  no-irregular-whitespace: error
-  no-negated-in-lhs: error
-  no-obj-calls: error
-  no-prototype-builtins: off
-  no-regex-spaces: error
-  no-sparse-arrays: error
-  no-template-curly-in-string: off
-  no-unexpected-multiline: error
-  no-unreachable: error
-  no-unsafe-finally: off
-  no-unsafe-negation: off
-  use-isnan: error
-  valid-jsdoc: off
-  valid-typeof: error
+#   # Possible Errors
+#   no-await-in-loop: off
+#   no-cond-assign: error
+#   no-console: off
+#   no-constant-condition: error
+#   no-control-regex: error
+#   no-debugger: error
+#   no-dupe-args: error
+#   no-dupe-keys: error
+#   no-duplicate-case: error
+#   no-empty-character-class: error
+  no-empty: warn
+#   no-ex-assign: error
+#   no-extra-boolean-cast: error
+#   no-extra-parens: off
+#   no-extra-semi: error
+#   no-func-assign: error
+#   no-inner-declarations:
+#     - error
+#     - functions
+#   no-invalid-regexp: error
+#   no-irregular-whitespace: error
+#   no-negated-in-lhs: error
+#   no-obj-calls: error
+#   no-prototype-builtins: off
+#   no-regex-spaces: error
+#   no-sparse-arrays: error
+#   no-template-curly-in-string: off
+#   no-unexpected-multiline: error
+#   no-unreachable: error
+#   no-unsafe-finally: off
+#   no-unsafe-negation: off
+#   use-isnan: error
+#   valid-jsdoc: off
+#   valid-typeof: error
 
-  # Best Practices
-  accessor-pairs: error
-  array-callback-return: off
-  block-scoped-var: off
-  class-methods-use-this: off
-  complexity:
-    - error
-    - 6
-  consistent-return: off
-  curly: off
-  default-case: off
-  dot-location: off
-  dot-notation: off
-  eqeqeq: error
-  guard-for-in: error
-  no-alert: error
-  no-caller: error
-  no-case-declarations: error
-  no-div-regex: error
-  no-else-return: off
-  no-empty-function: off
-  no-empty-pattern: error
-  no-eq-null: error
-  no-eval: error
-  no-extend-native: error
-  no-extra-bind: error
-  no-extra-label: off
-  no-fallthrough: error
-  no-floating-decimal: off
-  no-global-assign: off
-  no-implicit-coercion: off
-  no-implied-eval: error
-  no-invalid-this: off
-  no-iterator: error
-  no-labels:
-    - error
-    - allowLoop: true
-      allowSwitch: true
-  no-lone-blocks: error
-  no-loop-func: error
-  no-magic-number: off
-  no-multi-spaces: off
-  no-multi-str: off
-  no-native-reassign: error
-  no-new-func: error
-  no-new-wrappers: error
-  no-new: error
-  no-octal-escape: error
-  no-octal: error
-  no-param-reassign: off
-  no-proto: error
-  no-redeclare: error
-  no-restricted-properties: off
-  no-return-assign: error
-  no-return-await: off
-  no-script-url: error
-  no-self-assign: off
-  no-self-compare: error
-  no-sequences: off
-  no-throw-literal: off
-  no-unmodified-loop-condition: off
-  no-unused-expressions: error
-  no-unused-labels: off
-  no-useless-call: error
-  no-useless-concat: error
+#   # Best Practices
+#   accessor-pairs: error
+#   array-callback-return: off
+#   block-scoped-var: off
+#   class-methods-use-this: off
+#   complexity:
+#     - error
+#     - 6
+#   consistent-return: off
+#   curly: off
+#   default-case: off
+#   dot-location: off
+#   dot-notation: off
+#   eqeqeq: error
+#   guard-for-in: error
+#   no-alert: error
+#   no-caller: error
+#   no-case-declarations: error
+#   no-div-regex: error
+#   no-else-return: off
+  no-empty-function: warn
+#   no-empty-pattern: error
+#   no-eq-null: error
+#   no-eval: error
+#   no-extend-native: error
+#   no-extra-bind: error
+#   no-extra-label: off
+#   no-fallthrough: error
+#   no-floating-decimal: off
+#   no-global-assign: off
+#   no-implicit-coercion: off
+#   no-implied-eval: error
+#   no-invalid-this: off
+#   no-iterator: error
+#   no-labels:
+#     - error
+#     - allowLoop: true
+#       allowSwitch: true
+#   no-lone-blocks: error
+#   no-loop-func: error
+#   no-magic-number: off
+#   no-multi-spaces: off
+#   no-multi-str: off
+#   no-native-reassign: error
+#   no-new-func: error
+#   no-new-wrappers: error
+#   no-new: error
+#   no-octal-escape: error
+#   no-octal: error
+#   no-param-reassign: off
+#   no-proto: error
+#   no-redeclare: error
+#   no-restricted-properties: off
+#   no-return-assign: error
+#   no-return-await: off
+#   no-script-url: error
+#   no-self-assign: off
+#   no-self-compare: error
+#   no-sequences: off
+#   no-throw-literal: off
+#   no-unmodified-loop-condition: off
+#   no-unused-expressions: error
+#   no-unused-labels: off
+#   no-useless-call: error
+#   no-useless-concat: error
   no-useless-escape: off
-  no-useless-return: off
-  no-void: error
-  no-warning-comments: off
-  no-with: error
-  prefer-promise-reject-errors: off
-  radix: error
-  require-await: off
-  vars-on-top: off
-  wrap-iife: error
-  yoda: off
+#   no-useless-return: off
+#   no-void: error
+#   no-warning-comments: off
+#   no-with: error
+#   prefer-promise-reject-errors: off
+#   radix: error
+#   require-await: off
+#   vars-on-top: off
+#   wrap-iife: error
+#   yoda: off
 
-  # Strict
-  strict: off
+#   # Strict
+#   strict: off
 
-  # Variables
-  init-declarations: off
-  no-catch-shadow: error
-  no-delete-var: error
-  no-label-var: error
-  no-restricted-globals: off
-  no-shadow-restricted-names: error
-  no-shadow: off
-  no-undef-init: error
-  no-undef: off
-  no-undefined: off
-  no-unused-vars: off
-  no-use-before-define: off
+#   # Variables
+#   init-declarations: off
+#   no-catch-shadow: error
+#   no-delete-var: error
+#   no-label-var: error
+#   no-restricted-globals: off
+#   no-shadow-restricted-names: error
+#   no-shadow: off
+#   no-undef-init: error
+#   no-undef: off
+#   no-undefined: off
+#   no-unused-vars:
+#     - warn
+#     -
+#       vars: local
+#   no-use-before-define: off
 
-  # Node.js and CommonJS
-  callback-return: error
-  global-require: error
-  handle-callback-err: error
-  no-mixed-requires: off
-  no-new-require: off
-  no-path-concat: error
-  no-process-env: off
-  no-process-exit: error
-  no-restricted-modules: off
-  no-sync: off
+#   # Node.js and CommonJS
+#   callback-return: error
+#   global-require: error
+#   handle-callback-err: error
+#   no-mixed-requires: off
+#   no-new-require: off
+#   no-path-concat: error
+#   no-process-env: off
+#   no-process-exit: error
+#   no-restricted-modules: off
+#   no-sync: off
 
-  # Stylistic Issues
-  array-bracket-spacing: off
-  block-spacing: off
-  brace-style: off
-  camelcase: off
-  capitalized-comments: off
-  comma-dangle:
-    - error
-    - never
-  comma-spacing: off
-  comma-style: off
-  computed-property-spacing: off
-  consistent-this: off
-  eol-last: off
-  func-call-spacing: off
-  func-name-matching: off
-  func-names: off
-  func-style: off
-  id-length: off
-  id-match: off
-  indent: off
-  jsx-quotes: off
-  key-spacing: off
-  keyword-spacing: off
-  line-comment-position: off
-  linebreak-style:
-    - error
-    - unix
-  lines-around-comment: off
-  lines-around-directive: off
-  max-depth: off
-  max-len: off
-  max-nested-callbacks: off
-  max-params: off
-  max-statements-per-line: off
-  max-statements:
-    - error
-    - 30
-  multiline-ternary: off
-  new-cap: off
-  new-parens: off
-  newline-after-var: off
-  newline-before-return: off
-  newline-per-chained-call: off
-  no-array-constructor: off
-  no-bitwise: off
-  no-continue: off
-  no-inline-comments: off
-  no-lonely-if: off
-  no-mixed-operators: off
-  no-mixed-spaces-and-tabs: off
-  no-multi-assign: off
-  no-multiple-empty-lines: off
-  no-negated-condition: off
-  no-nested-ternary: off
-  no-new-object: off
-  no-plusplus: off
-  no-restricted-syntax: off
-  no-spaced-func: off
-  no-tabs: off
-  no-ternary: off
-  no-trailing-spaces: off
-  no-underscore-dangle: off
-  no-unneeded-ternary: off
-  object-curly-newline: off
-  object-curly-spacing: off
-  object-property-newline: off
-  one-var-declaration-per-line: off
-  one-var: off
-  operator-assignment: off
-  operator-linebreak: off
-  padded-blocks: off
-  quote-props: off
-  quotes:
-    - error
-    - single
-  require-jsdoc: off
-  semi-spacing: off
-  semi:
-    - error
-    - always
-  sort-keys: off
-  sort-vars: off
-  space-before-blocks: off
-  space-before-function-paren: off
-  space-in-parens: off
-  space-infix-ops: off
-  space-unary-ops: off
-  spaced-comment: off
-  template-tag-spacing: off
-  unicode-bom: off
-  wrap-regex: off
+#   # Stylistic Issues
+#   array-bracket-spacing: off
+#   block-spacing: off
+#   brace-style: off
+#   camelcase: off
+#   capitalized-comments: off
+#   comma-dangle:
+#     - error
+#     - never
+#   comma-spacing: off
+#   comma-style: off
+#   computed-property-spacing: off
+#   consistent-this: off
+#   eol-last: off
+#   func-call-spacing: off
+#   func-name-matching: off
+#   func-names: off
+#   func-style: off
+#   id-length: off
+#   id-match: off
+#   indent: off
+#   jsx-quotes: off
+#   key-spacing: off
+#   keyword-spacing: off
+#   line-comment-position: off
+#   linebreak-style:
+#     - error
+#     - unix
+#   lines-around-comment: off
+#   lines-around-directive: off
+#   max-depth: off
+#   max-len: off
+#   max-nested-callbacks: off
+#   max-params: off
+#   max-statements-per-line: off
+#   max-statements:
+#     - error
+#     - 30
+#   multiline-ternary: off
+#   new-cap: off
+#   new-parens: off
+#   newline-after-var: off
+#   newline-before-return: off
+#   newline-per-chained-call: off
+#   no-array-constructor: off
+#   no-bitwise: off
+#   no-continue: off
+#   no-inline-comments: off
+#   no-lonely-if: off
+#   no-mixed-operators: off
+#   no-mixed-spaces-and-tabs: off
+#   no-multi-assign: off
+#   no-multiple-empty-lines: off
+#   no-negated-condition: off
+#   no-nested-ternary: off
+#   no-new-object: off
+#   no-plusplus: off
+#   no-restricted-syntax: off
+#   no-spaced-func: off
+#   no-tabs: off
+#   no-ternary: off
+#   no-trailing-spaces: off
+#   no-underscore-dangle: off
+#   no-unneeded-ternary: off
+#   object-curly-newline: off
+#   object-curly-spacing: off
+#   object-property-newline: off
+#   one-var-declaration-per-line: off
+#   one-var: off
+#   operator-assignment: off
+#   operator-linebreak: off
+#   padded-blocks: off
+#   quote-props: off
+#   quotes:
+#     - error
+#     - single
+#   require-jsdoc: off
+#   semi-spacing: off
+#   semi:
+#     - error
+#     - always
+#   sort-keys: off
+#   sort-vars: off
+#   space-before-blocks: off
+#   space-before-function-paren: off
+#   space-in-parens: off
+#   space-infix-ops: off
+#   space-unary-ops: off
+#   spaced-comment: off
+#   template-tag-spacing: off
+#   unicode-bom: off
+#   wrap-regex: off
 
-  # ECMAScript 6
-  arrow-body-style: off
-  arrow-parens: off
-  arrow-spacing: off
-  constructor-super: off
-  generator-star-spacing: off
-  no-class-assign: off
-  no-confusing-arrow: off
-  no-const-assign: off
-  no-dupe-class-members: off
-  no-duplicate-imports: off
-  no-new-symbol: off
-  no-restricted-imports: off
-  no-this-before-super: off
-  no-useless-computed-key: off
-  no-useless-constructor: off
-  no-useless-rename: off
-  no-var: off
-  object-shorthand: off
-  prefer-arrow-callback: off
-  prefer-const: off
-  prefer-destructuring: off
-  prefer-numeric-literals: off
-  prefer-rest-params: off
-  prefer-reflect: off
-  prefer-spread: off
-  prefer-template: off
-  require-yield: off
-  rest-spread-spacing: off
-  sort-imports: off
-  symbol-description: off
-  template-curly-spacing: off
-  yield-star-spacing: off
+#   # ECMAScript 6
+#   arrow-body-style: off
+#   arrow-parens: off
+#   arrow-spacing: off
+#   constructor-super: off
+#   generator-star-spacing: off
+#   no-class-assign: off
+#   no-confusing-arrow: off
+#   no-const-assign: off
+#   no-dupe-class-members: off
+#   no-duplicate-imports: off
+#   no-new-symbol: off
+#   no-restricted-imports: off
+#   no-this-before-super: off
+#   no-useless-computed-key: off
+#   no-useless-constructor: off
+#   no-useless-rename: off
+#   no-var: off
+#   object-shorthand: off
+#   prefer-arrow-callback: off
+#   prefer-const: off
+#   prefer-destructuring: off
+#   prefer-numeric-literals: off
+#   prefer-rest-params: off
+#   prefer-reflect: off
+#   prefer-spread: off
+#   prefer-template: off
+#   require-yield: off
+#   rest-spread-spacing: off
+#   sort-imports: off
+#   symbol-description: off
+#   template-curly-spacing: off
+#   yield-star-spacing: off

.github/ISSUE_TEMPLATE/Bug_report.md

@@ -8,7 +8,9 @@ about: Create a bug report
 
 Thanks for reporting a bug for Portainer !
 
-Do you need help or have a question? Come chat with us on Slack http://portainer.io/slack/ or gitter https://gitter.im/portainer/Lobby.
+You can find more information about Portainer support framework policy here: https://www.portainer.io/2019/04/portainer-support-policy/
+
+Do you need help or have a question? Come chat with us on Slack http://portainer.io/slack/.
 
 Before opening a new issue, make sure that we do not have any duplicates
 already open. You can ensure this by searching the issue list for this
@@ -19,24 +21,18 @@ Also, be sure to check our FAQ and documentation first: https://portainer.readth
 -->
 
 **Bug description**
-
 A clear and concise description of what the bug is.
 
 **Expected behavior**
 A clear and concise description of what you expected to happen.
 
-Briefly describe what you were expecting.
-
 **Steps to reproduce the issue:**
-
-Steps to reproduce the behavior:
 1. Go to '...'
 2. Click on '....'
 3. Scroll down to '....'
 4. See error
 
 **Technical details:**
-
 * Portainer version:
 * Docker version (managed by Portainer):
 * Platform (windows/linux):

.github/ISSUE_TEMPLATE/Custom.md

@@ -6,7 +6,9 @@ about: Ask us a question about Portainer usage or deployment
 
 <!--
 
-Do you need help or have a question? Come chat with us on Slack http://portainer.io/slack/ or gitter https://gitter.im/portainer/Lobby.
+You can find more information about Portainer support framework policy here: https://www.portainer.io/2019/04/portainer-support-policy/
+
+Do you need help or have a question? Come chat with us on Slack http://portainer.io/slack/
 
 Also, be sure to check our FAQ and documentation first: https://portainer.readthedocs.io
 -->

.github/ISSUE_TEMPLATE/Feature_request.md

@@ -8,7 +8,7 @@ about: Suggest a feature/enhancement that should be added in Portainer
 
 Thanks for opening a feature request for Portainer !
 
-Do you need help or have a question? Come chat with us on Slack http://portainer.io/slack/ or gitter https://gitter.im/portainer/Lobby.
+Do you need help or have a question? Come chat with us on Slack http://portainer.io/slack/
 
 Before opening a new issue, make sure that we do not have any duplicates
 already open. You can ensure this by searching the issue list for this

.github/stale.yml

@@ -0,0 +1,55 @@
+# Config for Stalebot, limited to only `issues`
+only: issues
+
+# Issues config
+issues:
+  daysUntilStale: 60
+  daysUntilClose: 7
+
+  # Limit the number of actions per hour, from 1-30. Default is 30
+  limitPerRun: 30
+
+  # Issues with these labels will never be considered stale
+  exemptLabels:
+    - kind/enhancement
+    - kind/feature
+    - kind/question
+    - kind/style
+    - kind/workaround
+    - bug/need-confirmation
+    - bug/confirmed
+    - status/discuss
+  
+  # Only issues with all of these labels are checked if stale. Defaults to `[]` (disabled)
+  onlyLabels: []
+
+  # Set to true to ignore issues in a project (defaults to false)
+  exemptProjects: true
+  # Set to true to ignore issues in a milestone (defaults to false)
+  exemptMilestones: true
+  # Set to true to ignore issues with an assignee (defaults to false)
+  exemptAssignees: true
+
+  # Label to use when marking an issue as stale
+  staleLabel: status/stale
+
+  # Comment to post when marking an issue as stale. Set to `false` to disable
+  markComment: >
+    This issue has been marked as stale as it has not had recent activity, 
+    it will be closed if no further activity occurs in the next 7 days.
+    If you believe that it has been incorrectly labelled as stale, 
+    leave a comment and the label will be removed.
+
+  # Comment to post when removing the stale label.
+  # unmarkComment: >
+  #   Your comment here.
+
+  # Comment to post when closing a stale issue. Set to `false` to disable
+  closeComment: >
+    Since no further activity has appeared on this issue it will be closed.
+    If you believe that it has been incorrectly closed, leave a comment
+    and mention @itsconquest. One of our staff will then review the issue.
+    
+    Note - If it is an old bug report, make sure that it is reproduceable in the
+    latest version of Portainer as it may have already been fixed.
+    

.gitignore

@@ -4,3 +4,4 @@ dist
 portainer-checksum.txt
 api/cmd/portainer/portainer*
 .tmp
+.vscode

CONTRIBUTING.md

@@ -2,7 +2,7 @@
 
 Some basic conventions for contributing to this project.
 
-### General
+## General
 
 Please make sure that there aren't existing pull requests attempting to address the issue mentioned. Likewise, please check for issues related to update, as someone else may be working on the issue in a branch or fork.
 
@@ -13,7 +13,7 @@ When creating a new branch, prefix it with the *type* of the change (see section
 
 For example, if you work on a bugfix for the issue #361, you could name the branch `fix361-template-selection`.
 
-### Issues open to contribution
+## Issues open to contribution
 
 Want to contribute but don't know where to start?
 
@@ -24,14 +24,14 @@ Some of the open issues are labeled with prefix `exp/`, this is used to mark the
 either AngularJS or Golang
 * **advanced**: a task that require a deep understanding of the project codebase
 
-You can have a use Github filters to list these issues:
+You can use Github filters to list these issues:
 
 * beginner labeled issues: https://github.com/portainer/portainer/labels/exp%2Fbeginner
 * intermediate labeled issues: https://github.com/portainer/portainer/labels/exp%2Fintermediate
 * advanced labeled issues: https://github.com/portainer/portainer/labels/exp%2Fadvanced
 
 
-### Commit Message Format
+## Commit Message Format
 
 Each commit message should include a **type**, a **scope** and a **subject**:
 
@@ -47,7 +47,7 @@ Lines should not exceed 100 characters. This allows the message to be easier to
  #269 style(dashboard): update dashboard with new layout
 ```
 
-#### Type
+### Type
 
 Must be one of the following:
 
@@ -61,16 +61,30 @@ Must be one of the following:
 * **chore**: Changes to the build process or auxiliary tools and libraries such as documentation
   generation
 
-#### Scope
+### Scope
 
 The scope could be anything specifying place of the commit change. For example `networks`,
 `containers`, `images` etc...
 You can use the **area** label tag associated on the issue here (for `area/containers` use `containers` as a scope...)
 
-#### Subject
+### Subject
 
 The subject contains succinct description of the change:
 
 * use the imperative, present tense: "change" not "changed" nor "changes"
 * don't capitalize first letter
 * no dot (.) at the end
+
+## Contribution process
+
+Our contribution process is described below. Some of the steps can be visualized inside Github via specific `status/` labels, such as `status/1-functional-review` or `status/2-technical-review`.
+
+### Bug report
+
+![portainer_bugreport_workflow](https://user-images.githubusercontent.com/5485061/45727219-50190a00-bbf5-11e8-9fe8-3a563bb8d5d7.png)
+
+### Feature request
+
+The feature request process is similar to the bug report process but has an extra functional validation before the technical validation as well as a documentation validation before the testing phase.
+
+![portainer_featurerequest_workflow](https://user-images.githubusercontent.com/5485061/45727229-5ad39f00-bbf5-11e8-9550-16ba66c50615.png)

README.md

@@ -1,27 +1,20 @@
-
 <p align="center">
-  <img title="portainer" src='https://portainer.io/images/logo_alt.png' />
+  <img title="portainer" src='https://github.com/portainer/portainer/blob/develop/assets/images/logo_alt.png?raw=true' />
 </p>
 
 [![Docker Pulls](https://img.shields.io/docker/pulls/portainer/portainer.svg)](https://hub.docker.com/r/portainer/portainer/)
 [![Microbadger](https://images.microbadger.com/badges/image/portainer/portainer.svg)](http://microbadger.com/images/portainer/portainer "Image size")
 [![Documentation Status](https://readthedocs.org/projects/portainer/badge/?version=stable)](http://portainer.readthedocs.io/en/stable/?badge=stable)
-[![Codefresh build status]( https://g.codefresh.io/api/badges/build?repoOwner=portainer&repoName=portainer&branch=develop&pipelineName=portainer-ci&accountName=deviantony&type=cf-1)]( https://g.codefresh.io/repositories/portainer/portainer/builds?filter=trigger:build;branch:develop;service:5922a08a3a1aab000116fcc6~portainer-ci)
+[![Build Status](https://portainer.visualstudio.com/Portainer%20CI/_apis/build/status/Portainer%20CI?branchName=develop)](https://portainer.visualstudio.com/Portainer%20CI/_build/latest?definitionId=3&branchName=develop)
 [![Code Climate](https://codeclimate.com/github/portainer/portainer/badges/gpa.svg)](https://codeclimate.com/github/portainer/portainer)
-[![Slack](https://portainer.io/slack/badge.svg)](https://portainer.io/slack/)
-[![Gitter](https://badges.gitter.im/portainer/Lobby.svg)](https://gitter.im/portainer/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
 [![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=YHXZJQNJQ36H6)
 
 **_Portainer_** is a lightweight management UI which allows you to **easily** manage your different Docker environments (Docker hosts or Swarm clusters).
-
-**_Portainer_** is meant to be as **simple** to deploy as it is to use. It consists of a single container that can run on any Docker engine (can be deployed as Linux container or a Windows native container).
-
-**_Portainer_** allows you to manage your Docker containers, images, volumes, networks and more ! It is compatible with the *standalone Docker* engine and with *Docker Swarm mode*.
+**_Portainer_** is meant to be as **simple** to deploy as it is to use. It consists of a single container that can run on any Docker engine (can be deployed as Linux container or a Windows native container, supports other platforms too).
+**_Portainer_** allows you to manage all your Docker resources (containers, images, volumes, networks and more) ! It is compatible with the *standalone Docker* engine and with *Docker Swarm mode*.
 
 ## Demo
 
-<img src="https://portainer.io/images/screenshots/portainer.gif" width="77%"/>
-
 You can try out the public demo instance: http://demo.portainer.io/ (login with the username **admin** and the password **tryportainer**).
 
 Please note that the public demo cluster is **reset every 15min**.
@@ -41,16 +34,21 @@ Unlike the public demo, the playground sessions are deleted after 4 hours. Apart
 
 ## Getting help
 
+**NOTE**: You can find more information about Portainer support framework policy here: https://www.portainer.io/2019/04/portainer-support-policy/
+
 * Issues: https://github.com/portainer/portainer/issues
 * FAQ: https://portainer.readthedocs.io/en/latest/faq.html
 * Slack (chat): https://portainer.io/slack/
-* Gitter (chat): https://gitter.im/portainer/Lobby
 
 ## Reporting bugs and contributing
 
 * Want to report a bug or request a feature? Please open [an issue](https://github.com/portainer/portainer/issues/new).
 * Want to help us build **_portainer_**? Follow our [contribution guidelines](https://portainer.readthedocs.io/en/latest/contribute.html) to build it  locally and make a pull request. We need all the help we can get!
 
+## Security
+
+* Here at Portainer, we believe in [responsible disclosure](https://en.wikipedia.org/wiki/Responsible_disclosure) of security issues. If you have found a security issue, please report it to <security@portainer.io>.
+
 ## Limitations
 
 **_Portainer_** has full support for the following Docker versions:

api/archive/tar.go

@@ -7,13 +7,13 @@ import (
 
 // TarFileInBuffer will create a tar archive containing a single file named via fileName and using the content
 // specified in fileContent. Returns the archive as a byte array.
-func TarFileInBuffer(fileContent []byte, fileName string) ([]byte, error) {
+func TarFileInBuffer(fileContent []byte, fileName string, mode int64) ([]byte, error) {
 	var buffer bytes.Buffer
 	tarWriter := tar.NewWriter(&buffer)
 
 	header := &tar.Header{
 		Name: fileName,
-		Mode: 0600,
+		Mode: mode,
 		Size: int64(len(fileContent)),
 	}
 

api/archive/zip.go

@@ -0,0 +1,48 @@
+package archive
+
+import (
+	"archive/zip"
+	"bytes"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+)
+
+// UnzipArchive will unzip an archive from bytes into the dest destination folder on disk
+func UnzipArchive(archiveData []byte, dest string) error {
+	zipReader, err := zip.NewReader(bytes.NewReader(archiveData), int64(len(archiveData)))
+	if err != nil {
+		return err
+	}
+
+	for _, zipFile := range zipReader.File {
+
+		f, err := zipFile.Open()
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+
+		data, err := ioutil.ReadAll(f)
+		if err != nil {
+			return err
+		}
+
+		fpath := filepath.Join(dest, zipFile.Name)
+
+		outFile, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, zipFile.Mode())
+		if err != nil {
+			return err
+		}
+
+		_, err = io.Copy(outFile, bytes.NewReader(data))
+		if err != nil {
+			return err
+		}
+
+		outFile.Close()
+	}
+
+	return nil
+}

api/authorizations.go

@@ -0,0 +1,419 @@
+package portainer
+
+// AuthorizationService represents a service used to
+// update authorizations associated to a user or team.
+type AuthorizationService struct {
+	endpointService       EndpointService
+	endpointGroupService  EndpointGroupService
+	registryService       RegistryService
+	roleService           RoleService
+	teamMembershipService TeamMembershipService
+	userService           UserService
+}
+
+// AuthorizationServiceParameters are the required parameters
+// used to create a new AuthorizationService.
+type AuthorizationServiceParameters struct {
+	EndpointService       EndpointService
+	EndpointGroupService  EndpointGroupService
+	RegistryService       RegistryService
+	RoleService           RoleService
+	TeamMembershipService TeamMembershipService
+	UserService           UserService
+}
+
+// NewAuthorizationService returns a point to a new AuthorizationService instance.
+func NewAuthorizationService(parameters *AuthorizationServiceParameters) *AuthorizationService {
+	return &AuthorizationService{
+		endpointService:       parameters.EndpointService,
+		endpointGroupService:  parameters.EndpointGroupService,
+		registryService:       parameters.RegistryService,
+		roleService:           parameters.RoleService,
+		teamMembershipService: parameters.TeamMembershipService,
+		userService:           parameters.UserService,
+	}
+}
+
+// DefaultPortainerAuthorizations returns the default Portainer authorizations used by non-admin users.
+func DefaultPortainerAuthorizations() Authorizations {
+	return map[Authorization]bool{
+		OperationPortainerDockerHubInspect:        true,
+		OperationPortainerEndpointGroupList:       true,
+		OperationPortainerEndpointList:            true,
+		OperationPortainerEndpointInspect:         true,
+		OperationPortainerEndpointExtensionAdd:    true,
+		OperationPortainerEndpointExtensionRemove: true,
+		OperationPortainerExtensionList:           true,
+		OperationPortainerMOTD:                    true,
+		OperationPortainerRegistryList:            true,
+		OperationPortainerRegistryInspect:         true,
+		OperationPortainerTeamList:                true,
+		OperationPortainerTemplateList:            true,
+		OperationPortainerTemplateInspect:         true,
+		OperationPortainerUserList:                true,
+		OperationPortainerUserInspect:             true,
+		OperationPortainerUserMemberships:         true,
+	}
+}
+
+// UpdateVolumeBrowsingAuthorizations will update all the volume browsing authorizations for each role (except endpoint administrator)
+// based on the specified removeAuthorizations parameter. If removeAuthorizations is set to true, all
+// the authorizations will be dropped for the each role. If removeAuthorizations is set to false, the authorizations
+// will be reset based for each role.
+func (service AuthorizationService) UpdateVolumeBrowsingAuthorizations(remove bool) error {
+	roles, err := service.roleService.Roles()
+	if err != nil {
+		return err
+	}
+
+	for _, role := range roles {
+		// all roles except endpoint administrator
+		if role.ID != RoleID(1) {
+			updateRoleVolumeBrowsingAuthorizations(&role, remove)
+
+			err := service.roleService.UpdateRole(role.ID, &role)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func updateRoleVolumeBrowsingAuthorizations(role *Role, removeAuthorizations bool) {
+	if !removeAuthorizations {
+		delete(role.Authorizations, OperationDockerAgentBrowseDelete)
+		delete(role.Authorizations, OperationDockerAgentBrowseGet)
+		delete(role.Authorizations, OperationDockerAgentBrowseList)
+		delete(role.Authorizations, OperationDockerAgentBrowsePut)
+		delete(role.Authorizations, OperationDockerAgentBrowseRename)
+		return
+	}
+
+	role.Authorizations[OperationDockerAgentBrowseGet] = true
+	role.Authorizations[OperationDockerAgentBrowseList] = true
+
+	// Standard-user
+	if role.ID == RoleID(3) {
+		role.Authorizations[OperationDockerAgentBrowseDelete] = true
+		role.Authorizations[OperationDockerAgentBrowsePut] = true
+		role.Authorizations[OperationDockerAgentBrowseRename] = true
+	}
+}
+
+// RemoveTeamAccessPolicies will remove all existing access policies associated to the specified team
+func (service *AuthorizationService) RemoveTeamAccessPolicies(teamID TeamID) error {
+	endpoints, err := service.endpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, endpoint := range endpoints {
+		for policyTeamID := range endpoint.TeamAccessPolicies {
+			if policyTeamID == teamID {
+				delete(endpoint.TeamAccessPolicies, policyTeamID)
+
+				err := service.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+				if err != nil {
+					return err
+				}
+
+				break
+			}
+		}
+	}
+
+	endpointGroups, err := service.endpointGroupService.EndpointGroups()
+	if err != nil {
+		return err
+	}
+
+	for _, endpointGroup := range endpointGroups {
+		for policyTeamID := range endpointGroup.TeamAccessPolicies {
+			if policyTeamID == teamID {
+				delete(endpointGroup.TeamAccessPolicies, policyTeamID)
+
+				err := service.endpointGroupService.UpdateEndpointGroup(endpointGroup.ID, &endpointGroup)
+				if err != nil {
+					return err
+				}
+
+				break
+			}
+		}
+	}
+
+	registries, err := service.registryService.Registries()
+	if err != nil {
+		return err
+	}
+
+	for _, registry := range registries {
+		for policyTeamID := range registry.TeamAccessPolicies {
+			if policyTeamID == teamID {
+				delete(registry.TeamAccessPolicies, policyTeamID)
+
+				err := service.registryService.UpdateRegistry(registry.ID, &registry)
+				if err != nil {
+					return err
+				}
+
+				break
+			}
+		}
+	}
+
+	return nil
+}
+
+// RemoveUserAccessPolicies will remove all existing access policies associated to the specified user
+func (service *AuthorizationService) RemoveUserAccessPolicies(userID UserID) error {
+	endpoints, err := service.endpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, endpoint := range endpoints {
+		for policyUserID := range endpoint.UserAccessPolicies {
+			if policyUserID == userID {
+				delete(endpoint.UserAccessPolicies, policyUserID)
+
+				err := service.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+				if err != nil {
+					return err
+				}
+
+				break
+			}
+		}
+	}
+
+	endpointGroups, err := service.endpointGroupService.EndpointGroups()
+	if err != nil {
+		return err
+	}
+
+	for _, endpointGroup := range endpointGroups {
+		for policyUserID := range endpointGroup.UserAccessPolicies {
+			if policyUserID == userID {
+				delete(endpointGroup.UserAccessPolicies, policyUserID)
+
+				err := service.endpointGroupService.UpdateEndpointGroup(endpointGroup.ID, &endpointGroup)
+				if err != nil {
+					return err
+				}
+
+				break
+			}
+		}
+	}
+
+	registries, err := service.registryService.Registries()
+	if err != nil {
+		return err
+	}
+
+	for _, registry := range registries {
+		for policyUserID := range registry.UserAccessPolicies {
+			if policyUserID == userID {
+				delete(registry.UserAccessPolicies, policyUserID)
+
+				err := service.registryService.UpdateRegistry(registry.ID, &registry)
+				if err != nil {
+					return err
+				}
+
+				break
+			}
+		}
+	}
+
+	return nil
+}
+
+// UpdateUsersAuthorizations will trigger an update of the authorizations for all the users.
+func (service *AuthorizationService) UpdateUsersAuthorizations() error {
+	users, err := service.userService.Users()
+	if err != nil {
+		return err
+	}
+
+	for _, user := range users {
+		err := service.updateUserAuthorizations(user.ID)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (service *AuthorizationService) updateUserAuthorizations(userID UserID) error {
+	user, err := service.userService.User(userID)
+	if err != nil {
+		return err
+	}
+
+	endpointAuthorizations, err := service.getAuthorizations(user)
+	if err != nil {
+		return err
+	}
+
+	user.EndpointAuthorizations = endpointAuthorizations
+
+	return service.userService.UpdateUser(userID, user)
+}
+
+func (service *AuthorizationService) getAuthorizations(user *User) (EndpointAuthorizations, error) {
+	endpointAuthorizations := EndpointAuthorizations{}
+	if user.Role == AdministratorRole {
+		return endpointAuthorizations, nil
+	}
+
+	userMemberships, err := service.teamMembershipService.TeamMembershipsByUserID(user.ID)
+	if err != nil {
+		return endpointAuthorizations, err
+	}
+
+	endpoints, err := service.endpointService.Endpoints()
+	if err != nil {
+		return endpointAuthorizations, err
+	}
+
+	endpointGroups, err := service.endpointGroupService.EndpointGroups()
+	if err != nil {
+		return endpointAuthorizations, err
+	}
+
+	roles, err := service.roleService.Roles()
+	if err != nil {
+		return endpointAuthorizations, err
+	}
+
+	endpointAuthorizations = getUserEndpointAuthorizations(user, endpoints, endpointGroups, roles, userMemberships)
+
+	return endpointAuthorizations, nil
+}
+
+func getUserEndpointAuthorizations(user *User, endpoints []Endpoint, endpointGroups []EndpointGroup, roles []Role, userMemberships []TeamMembership) EndpointAuthorizations {
+	endpointAuthorizations := make(EndpointAuthorizations)
+
+	groupUserAccessPolicies := map[EndpointGroupID]UserAccessPolicies{}
+	groupTeamAccessPolicies := map[EndpointGroupID]TeamAccessPolicies{}
+	for _, endpointGroup := range endpointGroups {
+		groupUserAccessPolicies[endpointGroup.ID] = endpointGroup.UserAccessPolicies
+		groupTeamAccessPolicies[endpointGroup.ID] = endpointGroup.TeamAccessPolicies
+	}
+
+	for _, endpoint := range endpoints {
+		authorizations := getAuthorizationsFromUserEndpointPolicy(user, &endpoint, roles)
+		if len(authorizations) > 0 {
+			endpointAuthorizations[endpoint.ID] = authorizations
+			continue
+		}
+
+		authorizations = getAuthorizationsFromUserEndpointGroupPolicy(user, &endpoint, roles, groupUserAccessPolicies)
+		if len(authorizations) > 0 {
+			endpointAuthorizations[endpoint.ID] = authorizations
+			continue
+		}
+
+		authorizations = getAuthorizationsFromTeamEndpointPolicies(userMemberships, &endpoint, roles)
+		if len(authorizations) > 0 {
+			endpointAuthorizations[endpoint.ID] = authorizations
+			continue
+		}
+
+		authorizations = getAuthorizationsFromTeamEndpointGroupPolicies(userMemberships, &endpoint, roles, groupTeamAccessPolicies)
+		if len(authorizations) > 0 {
+			endpointAuthorizations[endpoint.ID] = authorizations
+		}
+	}
+
+	return endpointAuthorizations
+}
+
+func getAuthorizationsFromUserEndpointPolicy(user *User, endpoint *Endpoint, roles []Role) Authorizations {
+	policyRoles := make([]RoleID, 0)
+
+	policy, ok := endpoint.UserAccessPolicies[user.ID]
+	if ok {
+		policyRoles = append(policyRoles, policy.RoleID)
+	}
+
+	return getAuthorizationsFromRoles(policyRoles, roles)
+}
+
+func getAuthorizationsFromUserEndpointGroupPolicy(user *User, endpoint *Endpoint, roles []Role, groupAccessPolicies map[EndpointGroupID]UserAccessPolicies) Authorizations {
+	policyRoles := make([]RoleID, 0)
+
+	policy, ok := groupAccessPolicies[endpoint.GroupID][user.ID]
+	if ok {
+		policyRoles = append(policyRoles, policy.RoleID)
+	}
+
+	return getAuthorizationsFromRoles(policyRoles, roles)
+}
+
+func getAuthorizationsFromTeamEndpointPolicies(memberships []TeamMembership, endpoint *Endpoint, roles []Role) Authorizations {
+	policyRoles := make([]RoleID, 0)
+
+	for _, membership := range memberships {
+		policy, ok := endpoint.TeamAccessPolicies[membership.TeamID]
+		if ok {
+			policyRoles = append(policyRoles, policy.RoleID)
+		}
+	}
+
+	return getAuthorizationsFromRoles(policyRoles, roles)
+}
+
+func getAuthorizationsFromTeamEndpointGroupPolicies(memberships []TeamMembership, endpoint *Endpoint, roles []Role, groupAccessPolicies map[EndpointGroupID]TeamAccessPolicies) Authorizations {
+	policyRoles := make([]RoleID, 0)
+
+	for _, membership := range memberships {
+		policy, ok := groupAccessPolicies[endpoint.GroupID][membership.TeamID]
+		if ok {
+			policyRoles = append(policyRoles, policy.RoleID)
+		}
+	}
+
+	return getAuthorizationsFromRoles(policyRoles, roles)
+}
+
+func getAuthorizationsFromRoles(roleIdentifiers []RoleID, roles []Role) Authorizations {
+	var roleAuthorizations []Authorizations
+	for _, id := range roleIdentifiers {
+		for _, role := range roles {
+			if role.ID == id {
+				roleAuthorizations = append(roleAuthorizations, role.Authorizations)
+				break
+			}
+		}
+	}
+
+	processedAuthorizations := make(Authorizations)
+	if len(roleAuthorizations) > 0 {
+		processedAuthorizations = roleAuthorizations[0]
+		for idx, authorizations := range roleAuthorizations {
+			if idx == 0 {
+				continue
+			}
+			processedAuthorizations = mergeAuthorizations(processedAuthorizations, authorizations)
+		}
+	}
+
+	return processedAuthorizations
+}
+
+func mergeAuthorizations(a, b Authorizations) Authorizations {
+	c := make(map[Authorization]bool)
+
+	for k := range b {
+		if _, ok := a[k]; ok {
+			c[k] = true
+		}
+	}
+	return c
+}

api/bolt/datastore.go

@@ -5,21 +5,28 @@ import (
 	"path"
 	"time"
 
+	"github.com/portainer/portainer/api/bolt/tunnelserver"
+
 	"github.com/boltdb/bolt"
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/dockerhub"
-	"github.com/portainer/portainer/bolt/endpoint"
-	"github.com/portainer/portainer/bolt/endpointgroup"
-	"github.com/portainer/portainer/bolt/migrator"
-	"github.com/portainer/portainer/bolt/registry"
-	"github.com/portainer/portainer/bolt/resourcecontrol"
-	"github.com/portainer/portainer/bolt/settings"
-	"github.com/portainer/portainer/bolt/stack"
-	"github.com/portainer/portainer/bolt/tag"
-	"github.com/portainer/portainer/bolt/team"
-	"github.com/portainer/portainer/bolt/teammembership"
-	"github.com/portainer/portainer/bolt/user"
-	"github.com/portainer/portainer/bolt/version"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/dockerhub"
+	"github.com/portainer/portainer/api/bolt/endpoint"
+	"github.com/portainer/portainer/api/bolt/endpointgroup"
+	"github.com/portainer/portainer/api/bolt/extension"
+	"github.com/portainer/portainer/api/bolt/migrator"
+	"github.com/portainer/portainer/api/bolt/registry"
+	"github.com/portainer/portainer/api/bolt/resourcecontrol"
+	"github.com/portainer/portainer/api/bolt/role"
+	"github.com/portainer/portainer/api/bolt/schedule"
+	"github.com/portainer/portainer/api/bolt/settings"
+	"github.com/portainer/portainer/api/bolt/stack"
+	"github.com/portainer/portainer/api/bolt/tag"
+	"github.com/portainer/portainer/api/bolt/team"
+	"github.com/portainer/portainer/api/bolt/teammembership"
+	"github.com/portainer/portainer/api/bolt/template"
+	"github.com/portainer/portainer/api/bolt/user"
+	"github.com/portainer/portainer/api/bolt/version"
+	"github.com/portainer/portainer/api/bolt/webhook"
 )
 
 const (
@@ -33,9 +40,11 @@ type Store struct {
 	db                     *bolt.DB
 	checkForDataMigration  bool
 	fileService            portainer.FileService
+	RoleService            *role.Service
 	DockerHubService       *dockerhub.Service
 	EndpointGroupService   *endpointgroup.Service
 	EndpointService        *endpoint.Service
+	ExtensionService       *extension.Service
 	RegistryService        *registry.Service
 	ResourceControlService *resourcecontrol.Service
 	SettingsService        *settings.Service
@@ -43,8 +52,12 @@ type Store struct {
 	TagService             *tag.Service
 	TeamMembershipService  *teammembership.Service
 	TeamService            *team.Service
+	TemplateService        *template.Service
+	TunnelServerService    *tunnelserver.Service
 	UserService            *user.Service
 	VersionService         *version.Service
+	WebhookService         *webhook.Service
+	ScheduleService        *schedule.Service
 }
 
 // NewStore initializes a new Store and the associated services
@@ -81,29 +94,6 @@ func (store *Store) Open() error {
 	return store.initServices()
 }
 
-// Init creates the default data set.
-func (store *Store) Init() error {
-	groups, err := store.EndpointGroupService.EndpointGroups()
-	if err != nil {
-		return err
-	}
-
-	if len(groups) == 0 {
-		unassignedGroup := &portainer.EndpointGroup{
-			Name:            "Unassigned",
-			Description:     "Unassigned endpoints",
-			Labels:          []portainer.Pair{},
-			AuthorizedUsers: []portainer.UserID{},
-			AuthorizedTeams: []portainer.TeamID{},
-			Tags:            []string{},
-		}
-
-		return store.EndpointGroupService.CreateEndpointGroup(unassignedGroup)
-	}
-
-	return nil
-}
-
 // Close closes the BoltDB database.
 func (store *Store) Close() error {
 	if store.db != nil {
@@ -131,9 +121,15 @@ func (store *Store) MigrateData() error {
 			DatabaseVersion:        version,
 			EndpointGroupService:   store.EndpointGroupService,
 			EndpointService:        store.EndpointService,
+			ExtensionService:       store.ExtensionService,
+			RegistryService:        store.RegistryService,
 			ResourceControlService: store.ResourceControlService,
+			RoleService:            store.RoleService,
+			ScheduleService:        store.ScheduleService,
 			SettingsService:        store.SettingsService,
 			StackService:           store.StackService,
+			TeamMembershipService:  store.TeamMembershipService,
+			TemplateService:        store.TemplateService,
 			UserService:            store.UserService,
 			VersionService:         store.VersionService,
 			FileService:            store.fileService,
@@ -152,6 +148,12 @@ func (store *Store) MigrateData() error {
 }
 
 func (store *Store) initServices() error {
+	authorizationsetService, err := role.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.RoleService = authorizationsetService
+
 	dockerhubService, err := dockerhub.NewService(store.db)
 	if err != nil {
 		return err
@@ -170,6 +172,12 @@ func (store *Store) initServices() error {
 	}
 	store.EndpointService = endpointService
 
+	extensionService, err := extension.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.ExtensionService = extensionService
+
 	registryService, err := registry.NewService(store.db)
 	if err != nil {
 		return err
@@ -212,6 +220,18 @@ func (store *Store) initServices() error {
 	}
 	store.TeamService = teamService
 
+	templateService, err := template.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.TemplateService = templateService
+
+	tunnelServerService, err := tunnelserver.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.TunnelServerService = tunnelServerService
+
 	userService, err := user.NewService(store.db)
 	if err != nil {
 		return err
@@ -224,5 +244,17 @@ func (store *Store) initServices() error {
 	}
 	store.VersionService = versionService
 
+	webhookService, err := webhook.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.WebhookService = webhookService
+
+	scheduleService, err := schedule.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.ScheduleService = scheduleService
+
 	return nil
 }

api/bolt/dockerhub/dockerhub.go

@@ -1,8 +1,8 @@
 package dockerhub
 
 import (
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/internal"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
 )

api/bolt/endpoint/endpoint.go

@@ -1,10 +1,9 @@
 package endpoint
 
 import (
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/internal"
-
 	"github.com/boltdb/bolt"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
 )
 
 const (
@@ -64,7 +63,7 @@ func (service *Service) Endpoints() ([]portainer.Endpoint, error) {
 		cursor := bucket.Cursor()
 		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
 			var endpoint portainer.Endpoint
-			err := internal.UnmarshalObject(v, &endpoint)
+			err := internal.UnmarshalObjectWithJsoniter(v, &endpoint)
 			if err != nil {
 				return err
 			}
@@ -82,8 +81,11 @@ func (service *Service) CreateEndpoint(endpoint *portainer.Endpoint) error {
 	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
-		id, _ := bucket.NextSequence()
-		endpoint.ID = portainer.EndpointID(id)
+		// We manually manage sequences for endpoints
+		err := bucket.SetSequence(uint64(endpoint.ID))
+		if err != nil {
+			return err
+		}
 
 		data, err := internal.MarshalObject(endpoint)
 		if err != nil {
@@ -94,6 +96,11 @@ func (service *Service) CreateEndpoint(endpoint *portainer.Endpoint) error {
 	})
 }
 
+// GetNextIdentifier returns the next identifier for an endpoint.
+func (service *Service) GetNextIdentifier() int {
+	return internal.GetNextIdentifier(service.db, BucketName)
+}
+
 // Synchronize creates, updates and deletes endpoints inside a single transaction.
 func (service *Service) Synchronize(toCreate, toUpdate, toDelete []*portainer.Endpoint) error {
 	return service.db.Update(func(tx *bolt.Tx) error {

api/bolt/endpointgroup/endpointgroup.go

@@ -1,8 +1,8 @@
 package endpointgroup
 
 import (
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/internal"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
 )

api/bolt/extension/extension.go

@@ -0,0 +1,86 @@
+package extension
+
+import (
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
+
+	"github.com/boltdb/bolt"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "extension"
+)
+
+// Service represents a service for managing endpoint data.
+type Service struct {
+	db *bolt.DB
+}
+
+// NewService creates a new instance of a service.
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		db: db,
+	}, nil
+}
+
+// Extension returns a extension by ID
+func (service *Service) Extension(ID portainer.ExtensionID) (*portainer.Extension, error) {
+	var extension portainer.Extension
+	identifier := internal.Itob(int(ID))
+
+	err := internal.GetObject(service.db, BucketName, identifier, &extension)
+	if err != nil {
+		return nil, err
+	}
+
+	return &extension, nil
+}
+
+// Extensions return an array containing all the extensions.
+func (service *Service) Extensions() ([]portainer.Extension, error) {
+	var extensions = make([]portainer.Extension, 0)
+
+	err := service.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var extension portainer.Extension
+			err := internal.UnmarshalObject(v, &extension)
+			if err != nil {
+				return err
+			}
+			extensions = append(extensions, extension)
+		}
+
+		return nil
+	})
+
+	return extensions, err
+}
+
+// Persist persists a extension inside the database.
+func (service *Service) Persist(extension *portainer.Extension) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		data, err := internal.MarshalObject(extension)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(extension.ID)), data)
+	})
+}
+
+// DeleteExtension deletes a Extension.
+func (service *Service) DeleteExtension(ID portainer.ExtensionID) error {
+	identifier := internal.Itob(int(ID))
+	return internal.DeleteObject(service.db, BucketName, identifier)
+}

api/bolt/init.go

@@ -0,0 +1,423 @@
+package bolt
+
+import portainer "github.com/portainer/portainer/api"
+
+// Init creates the default data set.
+func (store *Store) Init() error {
+	groups, err := store.EndpointGroupService.EndpointGroups()
+	if err != nil {
+		return err
+	}
+
+	if len(groups) == 0 {
+		unassignedGroup := &portainer.EndpointGroup{
+			Name:               "Unassigned",
+			Description:        "Unassigned endpoints",
+			Labels:             []portainer.Pair{},
+			UserAccessPolicies: portainer.UserAccessPolicies{},
+			TeamAccessPolicies: portainer.TeamAccessPolicies{},
+			Tags:               []string{},
+		}
+
+		err = store.EndpointGroupService.CreateEndpointGroup(unassignedGroup)
+		if err != nil {
+			return err
+		}
+	}
+
+	roles, err := store.RoleService.Roles()
+	if err != nil {
+		return err
+	}
+
+	if len(roles) == 0 {
+		environmentAdministratorRole := &portainer.Role{
+			Name:        "Endpoint administrator",
+			Description: "Full control of all resources in an endpoint",
+			Authorizations: map[portainer.Authorization]bool{
+				portainer.OperationDockerContainerArchiveInfo:         true,
+				portainer.OperationDockerContainerList:                true,
+				portainer.OperationDockerContainerExport:              true,
+				portainer.OperationDockerContainerChanges:             true,
+				portainer.OperationDockerContainerInspect:             true,
+				portainer.OperationDockerContainerTop:                 true,
+				portainer.OperationDockerContainerLogs:                true,
+				portainer.OperationDockerContainerStats:               true,
+				portainer.OperationDockerContainerAttachWebsocket:     true,
+				portainer.OperationDockerContainerArchive:             true,
+				portainer.OperationDockerContainerCreate:              true,
+				portainer.OperationDockerContainerPrune:               true,
+				portainer.OperationDockerContainerKill:                true,
+				portainer.OperationDockerContainerPause:               true,
+				portainer.OperationDockerContainerUnpause:             true,
+				portainer.OperationDockerContainerRestart:             true,
+				portainer.OperationDockerContainerStart:               true,
+				portainer.OperationDockerContainerStop:                true,
+				portainer.OperationDockerContainerWait:                true,
+				portainer.OperationDockerContainerResize:              true,
+				portainer.OperationDockerContainerAttach:              true,
+				portainer.OperationDockerContainerExec:                true,
+				portainer.OperationDockerContainerRename:              true,
+				portainer.OperationDockerContainerUpdate:              true,
+				portainer.OperationDockerContainerPutContainerArchive: true,
+				portainer.OperationDockerContainerDelete:              true,
+				portainer.OperationDockerImageList:                    true,
+				portainer.OperationDockerImageSearch:                  true,
+				portainer.OperationDockerImageGetAll:                  true,
+				portainer.OperationDockerImageGet:                     true,
+				portainer.OperationDockerImageHistory:                 true,
+				portainer.OperationDockerImageInspect:                 true,
+				portainer.OperationDockerImageLoad:                    true,
+				portainer.OperationDockerImageCreate:                  true,
+				portainer.OperationDockerImagePrune:                   true,
+				portainer.OperationDockerImagePush:                    true,
+				portainer.OperationDockerImageTag:                     true,
+				portainer.OperationDockerImageDelete:                  true,
+				portainer.OperationDockerImageCommit:                  true,
+				portainer.OperationDockerImageBuild:                   true,
+				portainer.OperationDockerNetworkList:                  true,
+				portainer.OperationDockerNetworkInspect:               true,
+				portainer.OperationDockerNetworkCreate:                true,
+				portainer.OperationDockerNetworkConnect:               true,
+				portainer.OperationDockerNetworkDisconnect:            true,
+				portainer.OperationDockerNetworkPrune:                 true,
+				portainer.OperationDockerNetworkDelete:                true,
+				portainer.OperationDockerVolumeList:                   true,
+				portainer.OperationDockerVolumeInspect:                true,
+				portainer.OperationDockerVolumeCreate:                 true,
+				portainer.OperationDockerVolumePrune:                  true,
+				portainer.OperationDockerVolumeDelete:                 true,
+				portainer.OperationDockerExecInspect:                  true,
+				portainer.OperationDockerExecStart:                    true,
+				portainer.OperationDockerExecResize:                   true,
+				portainer.OperationDockerSwarmInspect:                 true,
+				portainer.OperationDockerSwarmUnlockKey:               true,
+				portainer.OperationDockerSwarmInit:                    true,
+				portainer.OperationDockerSwarmJoin:                    true,
+				portainer.OperationDockerSwarmLeave:                   true,
+				portainer.OperationDockerSwarmUpdate:                  true,
+				portainer.OperationDockerSwarmUnlock:                  true,
+				portainer.OperationDockerNodeList:                     true,
+				portainer.OperationDockerNodeInspect:                  true,
+				portainer.OperationDockerNodeUpdate:                   true,
+				portainer.OperationDockerNodeDelete:                   true,
+				portainer.OperationDockerServiceList:                  true,
+				portainer.OperationDockerServiceInspect:               true,
+				portainer.OperationDockerServiceLogs:                  true,
+				portainer.OperationDockerServiceCreate:                true,
+				portainer.OperationDockerServiceUpdate:                true,
+				portainer.OperationDockerServiceDelete:                true,
+				portainer.OperationDockerSecretList:                   true,
+				portainer.OperationDockerSecretInspect:                true,
+				portainer.OperationDockerSecretCreate:                 true,
+				portainer.OperationDockerSecretUpdate:                 true,
+				portainer.OperationDockerSecretDelete:                 true,
+				portainer.OperationDockerConfigList:                   true,
+				portainer.OperationDockerConfigInspect:                true,
+				portainer.OperationDockerConfigCreate:                 true,
+				portainer.OperationDockerConfigUpdate:                 true,
+				portainer.OperationDockerConfigDelete:                 true,
+				portainer.OperationDockerTaskList:                     true,
+				portainer.OperationDockerTaskInspect:                  true,
+				portainer.OperationDockerTaskLogs:                     true,
+				portainer.OperationDockerPluginList:                   true,
+				portainer.OperationDockerPluginPrivileges:             true,
+				portainer.OperationDockerPluginInspect:                true,
+				portainer.OperationDockerPluginPull:                   true,
+				portainer.OperationDockerPluginCreate:                 true,
+				portainer.OperationDockerPluginEnable:                 true,
+				portainer.OperationDockerPluginDisable:                true,
+				portainer.OperationDockerPluginPush:                   true,
+				portainer.OperationDockerPluginUpgrade:                true,
+				portainer.OperationDockerPluginSet:                    true,
+				portainer.OperationDockerPluginDelete:                 true,
+				portainer.OperationDockerSessionStart:                 true,
+				portainer.OperationDockerDistributionInspect:          true,
+				portainer.OperationDockerBuildPrune:                   true,
+				portainer.OperationDockerBuildCancel:                  true,
+				portainer.OperationDockerPing:                         true,
+				portainer.OperationDockerInfo:                         true,
+				portainer.OperationDockerVersion:                      true,
+				portainer.OperationDockerEvents:                       true,
+				portainer.OperationDockerSystem:                       true,
+				portainer.OperationDockerUndefined:                    true,
+				portainer.OperationDockerAgentPing:                    true,
+				portainer.OperationDockerAgentList:                    true,
+				portainer.OperationDockerAgentHostInfo:                true,
+				portainer.OperationDockerAgentBrowseDelete:            true,
+				portainer.OperationDockerAgentBrowseGet:               true,
+				portainer.OperationDockerAgentBrowseList:              true,
+				portainer.OperationDockerAgentBrowsePut:               true,
+				portainer.OperationDockerAgentBrowseRename:            true,
+				portainer.OperationDockerAgentUndefined:               true,
+				portainer.OperationPortainerResourceControlCreate:     true,
+				portainer.OperationPortainerResourceControlUpdate:     true,
+				portainer.OperationPortainerResourceControlDelete:     true,
+				portainer.OperationPortainerStackList:                 true,
+				portainer.OperationPortainerStackInspect:              true,
+				portainer.OperationPortainerStackFile:                 true,
+				portainer.OperationPortainerStackCreate:               true,
+				portainer.OperationPortainerStackMigrate:              true,
+				portainer.OperationPortainerStackUpdate:               true,
+				portainer.OperationPortainerStackDelete:               true,
+				portainer.OperationPortainerWebsocketExec:             true,
+				portainer.OperationPortainerWebhookList:               true,
+				portainer.OperationPortainerWebhookCreate:             true,
+				portainer.OperationPortainerWebhookDelete:             true,
+				portainer.OperationIntegrationStoridgeAdmin:           true,
+				portainer.EndpointResourcesAccess:                     true,
+			},
+		}
+
+		err = store.RoleService.CreateRole(environmentAdministratorRole)
+		if err != nil {
+			return err
+		}
+
+		environmentReadOnlyUserRole := &portainer.Role{
+			Name:        "Helpdesk",
+			Description: "Read-only access of all resources in an endpoint",
+			Authorizations: map[portainer.Authorization]bool{
+				portainer.OperationDockerContainerArchiveInfo: true,
+				portainer.OperationDockerContainerList:        true,
+				portainer.OperationDockerContainerChanges:     true,
+				portainer.OperationDockerContainerInspect:     true,
+				portainer.OperationDockerContainerTop:         true,
+				portainer.OperationDockerContainerLogs:        true,
+				portainer.OperationDockerContainerStats:       true,
+				portainer.OperationDockerImageList:            true,
+				portainer.OperationDockerImageSearch:          true,
+				portainer.OperationDockerImageGetAll:          true,
+				portainer.OperationDockerImageGet:             true,
+				portainer.OperationDockerImageHistory:         true,
+				portainer.OperationDockerImageInspect:         true,
+				portainer.OperationDockerNetworkList:          true,
+				portainer.OperationDockerNetworkInspect:       true,
+				portainer.OperationDockerVolumeList:           true,
+				portainer.OperationDockerVolumeInspect:        true,
+				portainer.OperationDockerSwarmInspect:         true,
+				portainer.OperationDockerNodeList:             true,
+				portainer.OperationDockerNodeInspect:          true,
+				portainer.OperationDockerServiceList:          true,
+				portainer.OperationDockerServiceInspect:       true,
+				portainer.OperationDockerServiceLogs:          true,
+				portainer.OperationDockerSecretList:           true,
+				portainer.OperationDockerSecretInspect:        true,
+				portainer.OperationDockerConfigList:           true,
+				portainer.OperationDockerConfigInspect:        true,
+				portainer.OperationDockerTaskList:             true,
+				portainer.OperationDockerTaskInspect:          true,
+				portainer.OperationDockerTaskLogs:             true,
+				portainer.OperationDockerPluginList:           true,
+				portainer.OperationDockerDistributionInspect:  true,
+				portainer.OperationDockerPing:                 true,
+				portainer.OperationDockerInfo:                 true,
+				portainer.OperationDockerVersion:              true,
+				portainer.OperationDockerEvents:               true,
+				portainer.OperationDockerSystem:               true,
+				portainer.OperationDockerAgentPing:            true,
+				portainer.OperationDockerAgentList:            true,
+				portainer.OperationDockerAgentHostInfo:        true,
+				portainer.OperationPortainerStackList:         true,
+				portainer.OperationPortainerStackInspect:      true,
+				portainer.OperationPortainerStackFile:         true,
+				portainer.OperationPortainerWebhookList:       true,
+				portainer.EndpointResourcesAccess:             true,
+			},
+		}
+
+		err = store.RoleService.CreateRole(environmentReadOnlyUserRole)
+		if err != nil {
+			return err
+		}
+
+		standardUserRole := &portainer.Role{
+			Name:        "Standard user",
+			Description: "Full control of assigned resources in an endpoint",
+			Authorizations: map[portainer.Authorization]bool{
+				portainer.OperationDockerContainerArchiveInfo:         true,
+				portainer.OperationDockerContainerList:                true,
+				portainer.OperationDockerContainerExport:              true,
+				portainer.OperationDockerContainerChanges:             true,
+				portainer.OperationDockerContainerInspect:             true,
+				portainer.OperationDockerContainerTop:                 true,
+				portainer.OperationDockerContainerLogs:                true,
+				portainer.OperationDockerContainerStats:               true,
+				portainer.OperationDockerContainerAttachWebsocket:     true,
+				portainer.OperationDockerContainerArchive:             true,
+				portainer.OperationDockerContainerCreate:              true,
+				portainer.OperationDockerContainerKill:                true,
+				portainer.OperationDockerContainerPause:               true,
+				portainer.OperationDockerContainerUnpause:             true,
+				portainer.OperationDockerContainerRestart:             true,
+				portainer.OperationDockerContainerStart:               true,
+				portainer.OperationDockerContainerStop:                true,
+				portainer.OperationDockerContainerWait:                true,
+				portainer.OperationDockerContainerResize:              true,
+				portainer.OperationDockerContainerAttach:              true,
+				portainer.OperationDockerContainerExec:                true,
+				portainer.OperationDockerContainerRename:              true,
+				portainer.OperationDockerContainerUpdate:              true,
+				portainer.OperationDockerContainerPutContainerArchive: true,
+				portainer.OperationDockerContainerDelete:              true,
+				portainer.OperationDockerImageList:                    true,
+				portainer.OperationDockerImageSearch:                  true,
+				portainer.OperationDockerImageGetAll:                  true,
+				portainer.OperationDockerImageGet:                     true,
+				portainer.OperationDockerImageHistory:                 true,
+				portainer.OperationDockerImageInspect:                 true,
+				portainer.OperationDockerImageLoad:                    true,
+				portainer.OperationDockerImageCreate:                  true,
+				portainer.OperationDockerImagePush:                    true,
+				portainer.OperationDockerImageTag:                     true,
+				portainer.OperationDockerImageDelete:                  true,
+				portainer.OperationDockerImageCommit:                  true,
+				portainer.OperationDockerImageBuild:                   true,
+				portainer.OperationDockerNetworkList:                  true,
+				portainer.OperationDockerNetworkInspect:               true,
+				portainer.OperationDockerNetworkCreate:                true,
+				portainer.OperationDockerNetworkConnect:               true,
+				portainer.OperationDockerNetworkDisconnect:            true,
+				portainer.OperationDockerNetworkDelete:                true,
+				portainer.OperationDockerVolumeList:                   true,
+				portainer.OperationDockerVolumeInspect:                true,
+				portainer.OperationDockerVolumeCreate:                 true,
+				portainer.OperationDockerVolumeDelete:                 true,
+				portainer.OperationDockerExecInspect:                  true,
+				portainer.OperationDockerExecStart:                    true,
+				portainer.OperationDockerExecResize:                   true,
+				portainer.OperationDockerSwarmInspect:                 true,
+				portainer.OperationDockerSwarmUnlockKey:               true,
+				portainer.OperationDockerSwarmInit:                    true,
+				portainer.OperationDockerSwarmJoin:                    true,
+				portainer.OperationDockerSwarmLeave:                   true,
+				portainer.OperationDockerSwarmUpdate:                  true,
+				portainer.OperationDockerSwarmUnlock:                  true,
+				portainer.OperationDockerNodeList:                     true,
+				portainer.OperationDockerNodeInspect:                  true,
+				portainer.OperationDockerNodeUpdate:                   true,
+				portainer.OperationDockerNodeDelete:                   true,
+				portainer.OperationDockerServiceList:                  true,
+				portainer.OperationDockerServiceInspect:               true,
+				portainer.OperationDockerServiceLogs:                  true,
+				portainer.OperationDockerServiceCreate:                true,
+				portainer.OperationDockerServiceUpdate:                true,
+				portainer.OperationDockerServiceDelete:                true,
+				portainer.OperationDockerSecretList:                   true,
+				portainer.OperationDockerSecretInspect:                true,
+				portainer.OperationDockerSecretCreate:                 true,
+				portainer.OperationDockerSecretUpdate:                 true,
+				portainer.OperationDockerSecretDelete:                 true,
+				portainer.OperationDockerConfigList:                   true,
+				portainer.OperationDockerConfigInspect:                true,
+				portainer.OperationDockerConfigCreate:                 true,
+				portainer.OperationDockerConfigUpdate:                 true,
+				portainer.OperationDockerConfigDelete:                 true,
+				portainer.OperationDockerTaskList:                     true,
+				portainer.OperationDockerTaskInspect:                  true,
+				portainer.OperationDockerTaskLogs:                     true,
+				portainer.OperationDockerPluginList:                   true,
+				portainer.OperationDockerPluginPrivileges:             true,
+				portainer.OperationDockerPluginInspect:                true,
+				portainer.OperationDockerPluginPull:                   true,
+				portainer.OperationDockerPluginCreate:                 true,
+				portainer.OperationDockerPluginEnable:                 true,
+				portainer.OperationDockerPluginDisable:                true,
+				portainer.OperationDockerPluginPush:                   true,
+				portainer.OperationDockerPluginUpgrade:                true,
+				portainer.OperationDockerPluginSet:                    true,
+				portainer.OperationDockerPluginDelete:                 true,
+				portainer.OperationDockerSessionStart:                 true,
+				portainer.OperationDockerDistributionInspect:          true,
+				portainer.OperationDockerBuildPrune:                   true,
+				portainer.OperationDockerBuildCancel:                  true,
+				portainer.OperationDockerPing:                         true,
+				portainer.OperationDockerInfo:                         true,
+				portainer.OperationDockerVersion:                      true,
+				portainer.OperationDockerEvents:                       true,
+				portainer.OperationDockerSystem:                       true,
+				portainer.OperationDockerUndefined:                    true,
+				portainer.OperationDockerAgentPing:                    true,
+				portainer.OperationDockerAgentList:                    true,
+				portainer.OperationDockerAgentHostInfo:                true,
+				portainer.OperationDockerAgentUndefined:               true,
+				portainer.OperationPortainerResourceControlCreate:     true,
+				portainer.OperationPortainerResourceControlUpdate:     true,
+				portainer.OperationPortainerResourceControlDelete:     true,
+				portainer.OperationPortainerStackList:                 true,
+				portainer.OperationPortainerStackInspect:              true,
+				portainer.OperationPortainerStackFile:                 true,
+				portainer.OperationPortainerStackCreate:               true,
+				portainer.OperationPortainerStackMigrate:              true,
+				portainer.OperationPortainerStackUpdate:               true,
+				portainer.OperationPortainerStackDelete:               true,
+				portainer.OperationPortainerWebsocketExec:             true,
+				portainer.OperationPortainerWebhookList:               true,
+				portainer.OperationPortainerWebhookCreate:             true,
+			},
+		}
+
+		err = store.RoleService.CreateRole(standardUserRole)
+		if err != nil {
+			return err
+		}
+
+		readOnlyUserRole := &portainer.Role{
+			Name:        "Read-only user",
+			Description: "Read-only access of assigned resources in an endpoint",
+			Authorizations: map[portainer.Authorization]bool{
+				portainer.OperationDockerContainerArchiveInfo: true,
+				portainer.OperationDockerContainerList:        true,
+				portainer.OperationDockerContainerChanges:     true,
+				portainer.OperationDockerContainerInspect:     true,
+				portainer.OperationDockerContainerTop:         true,
+				portainer.OperationDockerContainerLogs:        true,
+				portainer.OperationDockerContainerStats:       true,
+				portainer.OperationDockerImageList:            true,
+				portainer.OperationDockerImageSearch:          true,
+				portainer.OperationDockerImageGetAll:          true,
+				portainer.OperationDockerImageGet:             true,
+				portainer.OperationDockerImageHistory:         true,
+				portainer.OperationDockerImageInspect:         true,
+				portainer.OperationDockerNetworkList:          true,
+				portainer.OperationDockerNetworkInspect:       true,
+				portainer.OperationDockerVolumeList:           true,
+				portainer.OperationDockerVolumeInspect:        true,
+				portainer.OperationDockerSwarmInspect:         true,
+				portainer.OperationDockerNodeList:             true,
+				portainer.OperationDockerNodeInspect:          true,
+				portainer.OperationDockerServiceList:          true,
+				portainer.OperationDockerServiceInspect:       true,
+				portainer.OperationDockerServiceLogs:          true,
+				portainer.OperationDockerSecretList:           true,
+				portainer.OperationDockerSecretInspect:        true,
+				portainer.OperationDockerConfigList:           true,
+				portainer.OperationDockerConfigInspect:        true,
+				portainer.OperationDockerTaskList:             true,
+				portainer.OperationDockerTaskInspect:          true,
+				portainer.OperationDockerTaskLogs:             true,
+				portainer.OperationDockerPluginList:           true,
+				portainer.OperationDockerDistributionInspect:  true,
+				portainer.OperationDockerPing:                 true,
+				portainer.OperationDockerInfo:                 true,
+				portainer.OperationDockerVersion:              true,
+				portainer.OperationDockerEvents:               true,
+				portainer.OperationDockerSystem:               true,
+				portainer.OperationDockerAgentPing:            true,
+				portainer.OperationDockerAgentList:            true,
+				portainer.OperationDockerAgentHostInfo:        true,
+				portainer.OperationPortainerStackList:         true,
+				portainer.OperationPortainerStackInspect:      true,
+				portainer.OperationPortainerStackFile:         true,
+				portainer.OperationPortainerWebhookList:       true,
+			},
+		}
+
+		err = store.RoleService.CreateRole(readOnlyUserRole)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

api/bolt/internal/db.go

@@ -4,7 +4,7 @@ import (
 	"encoding/binary"
 
 	"github.com/boltdb/bolt"
-	"github.com/portainer/portainer"
+	"github.com/portainer/portainer/api"
 )
 
 // Itob returns an 8-byte big endian representation of v.

api/bolt/internal/json.go

@@ -2,6 +2,8 @@ package internal
 
 import (
 	"encoding/json"
+
+	jsoniter "github.com/json-iterator/go"
 )
 
 // MarshalObject encodes an object to binary format
@@ -13,3 +15,11 @@ func MarshalObject(object interface{}) ([]byte, error) {
 func UnmarshalObject(data []byte, object interface{}) error {
 	return json.Unmarshal(data, object)
 }
+
+// UnmarshalObjectWithJsoniter decodes an object from binary data
+// using the jsoniter library. It is mainly used to accelerate endpoint
+// decoding at the moment.
+func UnmarshalObjectWithJsoniter(data []byte, object interface{}) error {
+	var jsoni = jsoniter.ConfigCompatibleWithStandardLibrary
+	return jsoni.Unmarshal(data, &object)
+}

api/bolt/migrator/migrate_dbversion0.go

@@ -2,8 +2,8 @@ package migrator
 
 import (
 	"github.com/boltdb/bolt"
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/user"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/user"
 )
 
 func (m *Migrator) updateAdminUserToDBVersion1() error {

api/bolt/migrator/migrate_dbversion1.go

@@ -2,8 +2,8 @@ package migrator
 
 import (
 	"github.com/boltdb/bolt"
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/internal"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
 )
 
 func (m *Migrator) updateResourceControlsToDBVersion2() error {

api/bolt/migrator/migrate_dbversion10.go

@@ -1,6 +1,6 @@
 package migrator
 
-import "github.com/portainer/portainer"
+import "github.com/portainer/portainer/api"
 
 func (m *Migrator) updateEndpointsToVersion11() error {
 	legacyEndpoints, err := m.endpointService.Endpoints()

api/bolt/migrator/migrate_dbversion11.go

@@ -5,9 +5,9 @@ import (
 	"strings"
 
 	"github.com/boltdb/bolt"
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/internal"
-	"github.com/portainer/portainer/bolt/stack"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
+	"github.com/portainer/portainer/api/bolt/stack"
 )
 
 func (m *Migrator) updateEndpointsToVersion12() error {

api/bolt/migrator/migrate_dbversion12.go

@@ -0,0 +1,17 @@
+package migrator
+
+import "github.com/portainer/portainer/api"
+
+func (m *Migrator) updateSettingsToVersion13() error {
+	legacySettings, err := m.settingsService.Settings()
+	if err != nil {
+		return err
+	}
+
+	legacySettings.LDAPSettings.AutoCreateUsers = false
+	legacySettings.LDAPSettings.GroupSearchSettings = []portainer.LDAPGroupSearchSettings{
+		portainer.LDAPGroupSearchSettings{},
+	}
+
+	return m.settingsService.UpdateSettings(legacySettings)
+}

api/bolt/migrator/migrate_dbversion13.go

@@ -0,0 +1,19 @@
+package migrator
+
+func (m *Migrator) updateResourceControlsToDBVersion14() error {
+	resourceControls, err := m.resourceControlService.ResourceControls()
+	if err != nil {
+		return err
+	}
+
+	for _, resourceControl := range resourceControls {
+		if resourceControl.AdministratorsOnly == true {
+			err = m.resourceControlService.DeleteResourceControl(resourceControl.ID)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}

api/bolt/migrator/migrate_dbversion14.go

@@ -0,0 +1,35 @@
+package migrator
+
+import (
+	"strings"
+
+	"github.com/portainer/portainer/api"
+)
+
+func (m *Migrator) updateSettingsToDBVersion15() error {
+	legacySettings, err := m.settingsService.Settings()
+	if err != nil {
+		return err
+	}
+
+	legacySettings.EnableHostManagementFeatures = false
+	return m.settingsService.UpdateSettings(legacySettings)
+}
+
+func (m *Migrator) updateTemplatesToVersion15() error {
+	legacyTemplates, err := m.templateService.Templates()
+	if err != nil {
+		return err
+	}
+
+	for _, template := range legacyTemplates {
+		template.Logo = strings.Replace(template.Logo, "https://portainer.io/images", portainer.AssetsServerURL, -1)
+
+		err = m.templateService.UpdateTemplate(template.ID, &template)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

api/bolt/migrator/migrate_dbversion15.go

@@ -0,0 +1,14 @@
+package migrator
+
+func (m *Migrator) updateSettingsToDBVersion16() error {
+	legacySettings, err := m.settingsService.Settings()
+	if err != nil {
+		return err
+	}
+
+	if legacySettings.SnapshotInterval == "" {
+		legacySettings.SnapshotInterval = "5m"
+	}
+
+	return m.settingsService.UpdateSettings(legacySettings)
+}

api/bolt/migrator/migrate_dbversion16.go

@@ -0,0 +1,19 @@
+package migrator
+
+func (m *Migrator) updateExtensionsToDBVersion17() error {
+	legacyExtensions, err := m.extensionService.Extensions()
+	if err != nil {
+		return err
+	}
+
+	for _, extension := range legacyExtensions {
+		extension.License.Valid = true
+
+		err = m.extensionService.Persist(&extension)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

api/bolt/migrator/migrate_dbversion17.go

@@ -0,0 +1,125 @@
+package migrator
+
+import (
+	portainer "github.com/portainer/portainer/api"
+)
+
+func (m *Migrator) updateUsersToDBVersion18() error {
+	legacyUsers, err := m.userService.Users()
+	if err != nil {
+		return err
+	}
+
+	for _, user := range legacyUsers {
+		user.PortainerAuthorizations = map[portainer.Authorization]bool{
+			portainer.OperationPortainerDockerHubInspect:        true,
+			portainer.OperationPortainerEndpointGroupList:       true,
+			portainer.OperationPortainerEndpointList:            true,
+			portainer.OperationPortainerEndpointInspect:         true,
+			portainer.OperationPortainerEndpointExtensionAdd:    true,
+			portainer.OperationPortainerEndpointExtensionRemove: true,
+			portainer.OperationPortainerExtensionList:           true,
+			portainer.OperationPortainerMOTD:                    true,
+			portainer.OperationPortainerRegistryList:            true,
+			portainer.OperationPortainerRegistryInspect:         true,
+			portainer.OperationPortainerTeamList:                true,
+			portainer.OperationPortainerTemplateList:            true,
+			portainer.OperationPortainerTemplateInspect:         true,
+			portainer.OperationPortainerUserList:                true,
+			portainer.OperationPortainerUserMemberships:         true,
+		}
+
+		err = m.userService.UpdateUser(user.ID, &user)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (m *Migrator) updateEndpointsToDBVersion18() error {
+	legacyEndpoints, err := m.endpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, endpoint := range legacyEndpoints {
+		endpoint.UserAccessPolicies = make(portainer.UserAccessPolicies)
+		for _, userID := range endpoint.AuthorizedUsers {
+			endpoint.UserAccessPolicies[userID] = portainer.AccessPolicy{
+				RoleID: 4,
+			}
+		}
+
+		endpoint.TeamAccessPolicies = make(portainer.TeamAccessPolicies)
+		for _, teamID := range endpoint.AuthorizedTeams {
+			endpoint.TeamAccessPolicies[teamID] = portainer.AccessPolicy{
+				RoleID: 4,
+			}
+		}
+
+		err = m.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (m *Migrator) updateEndpointGroupsToDBVersion18() error {
+	legacyEndpointGroups, err := m.endpointGroupService.EndpointGroups()
+	if err != nil {
+		return err
+	}
+
+	for _, endpointGroup := range legacyEndpointGroups {
+		endpointGroup.UserAccessPolicies = make(portainer.UserAccessPolicies)
+		for _, userID := range endpointGroup.AuthorizedUsers {
+			endpointGroup.UserAccessPolicies[userID] = portainer.AccessPolicy{
+				RoleID: 4,
+			}
+		}
+
+		endpointGroup.TeamAccessPolicies = make(portainer.TeamAccessPolicies)
+		for _, teamID := range endpointGroup.AuthorizedTeams {
+			endpointGroup.TeamAccessPolicies[teamID] = portainer.AccessPolicy{
+				RoleID: 4,
+			}
+		}
+
+		err = m.endpointGroupService.UpdateEndpointGroup(endpointGroup.ID, &endpointGroup)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (m *Migrator) updateRegistriesToDBVersion18() error {
+	legacyRegistries, err := m.registryService.Registries()
+	if err != nil {
+		return err
+	}
+
+	for _, registry := range legacyRegistries {
+		registry.UserAccessPolicies = make(portainer.UserAccessPolicies)
+		for _, userID := range registry.AuthorizedUsers {
+			registry.UserAccessPolicies[userID] = portainer.AccessPolicy{}
+		}
+
+		registry.TeamAccessPolicies = make(portainer.TeamAccessPolicies)
+		for _, teamID := range registry.AuthorizedTeams {
+			registry.TeamAccessPolicies[teamID] = portainer.AccessPolicy{}
+		}
+
+		err = m.registryService.UpdateRegistry(registry.ID, &registry)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

api/bolt/migrator/migrate_dbversion18.go

@@ -0,0 +1,16 @@
+package migrator
+
+import portainer "github.com/portainer/portainer/api"
+
+func (m *Migrator) updateSettingsToDBVersion19() error {
+	legacySettings, err := m.settingsService.Settings()
+	if err != nil {
+		return err
+	}
+
+	if legacySettings.EdgeAgentCheckinInterval == 0 {
+		legacySettings.EdgeAgentCheckinInterval = portainer.DefaultEdgeAgentCheckinIntervalInSeconds
+	}
+
+	return m.settingsService.UpdateSettings(legacySettings)
+}

api/bolt/migrator/migrate_dbversion19.go

@@ -0,0 +1,67 @@
+package migrator
+
+import (
+	"strings"
+
+	portainer "github.com/portainer/portainer/api"
+)
+
+func (m *Migrator) updateUsersToDBVersion20() error {
+	authorizationServiceParameters := &portainer.AuthorizationServiceParameters{
+		EndpointService:       m.endpointService,
+		EndpointGroupService:  m.endpointGroupService,
+		RegistryService:       m.registryService,
+		RoleService:           m.roleService,
+		TeamMembershipService: m.teamMembershipService,
+		UserService:           m.userService,
+	}
+
+	authorizationService := portainer.NewAuthorizationService(authorizationServiceParameters)
+	return authorizationService.UpdateUsersAuthorizations()
+}
+
+func (m *Migrator) updateSettingsToDBVersion20() error {
+	legacySettings, err := m.settingsService.Settings()
+	if err != nil {
+		return err
+	}
+
+	legacySettings.AllowVolumeBrowserForRegularUsers = false
+
+	return m.settingsService.UpdateSettings(legacySettings)
+}
+
+func (m *Migrator) updateSchedulesToDBVersion20() error {
+	legacySchedules, err := m.scheduleService.Schedules()
+	if err != nil {
+		return err
+	}
+
+	for _, schedule := range legacySchedules {
+		if schedule.JobType == portainer.ScriptExecutionJobType {
+			if schedule.CronExpression == "0 0 * * *" {
+				schedule.CronExpression = "0 * * * *"
+			} else if schedule.CronExpression == "0 0 0/2 * *" {
+				schedule.CronExpression = "0 */2 * * *"
+			} else if schedule.CronExpression == "0 0 0 * *" {
+				schedule.CronExpression = "0 0 * * *"
+			} else {
+				revisedCronExpression := strings.Split(schedule.CronExpression, " ")
+				if len(revisedCronExpression) == 5 {
+					continue
+				}
+
+				revisedCronExpression = revisedCronExpression[1:]
+				schedule.CronExpression = strings.Join(revisedCronExpression, " ")
+			}
+
+			err := m.scheduleService.UpdateSchedule(schedule.ID, &schedule)
+			if err != nil {
+				return err
+			}
+		}
+
+	}
+
+	return nil
+}

api/bolt/migrator/migrate_dbversion2.go

@@ -1,6 +1,6 @@
 package migrator
 
-import "github.com/portainer/portainer"
+import "github.com/portainer/portainer/api"
 
 func (m *Migrator) updateSettingsToDBVersion3() error {
 	legacySettings, err := m.settingsService.Settings()

api/bolt/migrator/migrate_dbversion20.go

@@ -0,0 +1,22 @@
+package migrator
+
+import (
+	portainer "github.com/portainer/portainer/api"
+)
+
+func (m *Migrator) updateUsersToDBVersion21() error {
+	legacyUsers, err := m.userService.Users()
+	if err != nil {
+		return err
+	}
+
+	for _, user := range legacyUsers {
+		user.PortainerAuthorizations = portainer.DefaultPortainerAuthorizations()
+		err = m.userService.UpdateUser(user.ID, &user)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

api/bolt/migrator/migrate_dbversion3.go

@@ -1,6 +1,6 @@
 package migrator
 
-import "github.com/portainer/portainer"
+import "github.com/portainer/portainer/api"
 
 func (m *Migrator) updateEndpointsToDBVersion4() error {
 	legacyEndpoints, err := m.endpointService.Endpoints()

api/bolt/migrator/migrate_dbversion7.go

@@ -1,6 +1,6 @@
 package migrator
 
-import "github.com/portainer/portainer"
+import "github.com/portainer/portainer/api"
 
 func (m *Migrator) updateEndpointsToVersion8() error {
 	legacyEndpoints, err := m.endpointService.Endpoints()

api/bolt/migrator/migrate_dbversion8.go

@@ -1,6 +1,6 @@
 package migrator
 
-import "github.com/portainer/portainer"
+import "github.com/portainer/portainer/api"
 
 func (m *Migrator) updateEndpointsToVersion9() error {
 	legacyEndpoints, err := m.endpointService.Endpoints()

api/bolt/migrator/migrate_dbversion9.go

@@ -1,6 +1,6 @@
 package migrator
 
-import "github.com/portainer/portainer"
+import "github.com/portainer/portainer/api"
 
 func (m *Migrator) updateEndpointsToVersion10() error {
 	legacyEndpoints, err := m.endpointService.Endpoints()

api/bolt/migrator/migrator.go

@@ -2,14 +2,20 @@ package migrator
 
 import (
 	"github.com/boltdb/bolt"
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/endpoint"
-	"github.com/portainer/portainer/bolt/endpointgroup"
-	"github.com/portainer/portainer/bolt/resourcecontrol"
-	"github.com/portainer/portainer/bolt/settings"
-	"github.com/portainer/portainer/bolt/stack"
-	"github.com/portainer/portainer/bolt/user"
-	"github.com/portainer/portainer/bolt/version"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/endpoint"
+	"github.com/portainer/portainer/api/bolt/endpointgroup"
+	"github.com/portainer/portainer/api/bolt/extension"
+	"github.com/portainer/portainer/api/bolt/registry"
+	"github.com/portainer/portainer/api/bolt/resourcecontrol"
+	"github.com/portainer/portainer/api/bolt/role"
+	"github.com/portainer/portainer/api/bolt/schedule"
+	"github.com/portainer/portainer/api/bolt/settings"
+	"github.com/portainer/portainer/api/bolt/stack"
+	"github.com/portainer/portainer/api/bolt/teammembership"
+	"github.com/portainer/portainer/api/bolt/template"
+	"github.com/portainer/portainer/api/bolt/user"
+	"github.com/portainer/portainer/api/bolt/version"
 )
 
 type (
@@ -19,9 +25,15 @@ type (
 		db                     *bolt.DB
 		endpointGroupService   *endpointgroup.Service
 		endpointService        *endpoint.Service
+		extensionService       *extension.Service
+		registryService        *registry.Service
 		resourceControlService *resourcecontrol.Service
+		roleService            *role.Service
+		scheduleService        *schedule.Service
 		settingsService        *settings.Service
 		stackService           *stack.Service
+		teamMembershipService  *teammembership.Service
+		templateService        *template.Service
 		userService            *user.Service
 		versionService         *version.Service
 		fileService            portainer.FileService
@@ -33,9 +45,15 @@ type (
 		DatabaseVersion        int
 		EndpointGroupService   *endpointgroup.Service
 		EndpointService        *endpoint.Service
+		ExtensionService       *extension.Service
+		RegistryService        *registry.Service
 		ResourceControlService *resourcecontrol.Service
+		RoleService            *role.Service
+		ScheduleService        *schedule.Service
 		SettingsService        *settings.Service
 		StackService           *stack.Service
+		TeamMembershipService  *teammembership.Service
+		TemplateService        *template.Service
 		UserService            *user.Service
 		VersionService         *version.Service
 		FileService            portainer.FileService
@@ -49,8 +67,14 @@ func NewMigrator(parameters *Parameters) *Migrator {
 		currentDBVersion:       parameters.DatabaseVersion,
 		endpointGroupService:   parameters.EndpointGroupService,
 		endpointService:        parameters.EndpointService,
+		extensionService:       parameters.ExtensionService,
+		registryService:        parameters.RegistryService,
 		resourceControlService: parameters.ResourceControlService,
+		roleService:            parameters.RoleService,
+		scheduleService:        parameters.ScheduleService,
 		settingsService:        parameters.SettingsService,
+		teamMembershipService:  parameters.TeamMembershipService,
+		templateService:        parameters.TemplateService,
 		stackService:           parameters.StackService,
 		userService:            parameters.UserService,
 		versionService:         parameters.VersionService,
@@ -170,5 +194,106 @@ func (m *Migrator) Migrate() error {
 		}
 	}
 
+	// Portainer 1.19.0
+	if m.currentDBVersion < 13 {
+		err := m.updateSettingsToVersion13()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.19.2
+	if m.currentDBVersion < 14 {
+		err := m.updateResourceControlsToDBVersion14()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.20.0
+	if m.currentDBVersion < 15 {
+		err := m.updateSettingsToDBVersion15()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateTemplatesToVersion15()
+		if err != nil {
+			return err
+		}
+	}
+
+	if m.currentDBVersion < 16 {
+		err := m.updateSettingsToDBVersion16()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.20.1
+	if m.currentDBVersion < 17 {
+		err := m.updateExtensionsToDBVersion17()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.21.0
+	if m.currentDBVersion < 18 {
+		err := m.updateUsersToDBVersion18()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateEndpointsToDBVersion18()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateEndpointGroupsToDBVersion18()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateRegistriesToDBVersion18()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.22.0
+	if m.currentDBVersion < 19 {
+		err := m.updateSettingsToDBVersion19()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.22.1
+	if m.currentDBVersion < 20 {
+		err := m.updateUsersToDBVersion20()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateSettingsToDBVersion20()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateSchedulesToDBVersion20()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.22.2
+	if m.currentDBVersion < 21 {
+		err := m.updateUsersToDBVersion21()
+		if err != nil {
+			return err
+		}
+	}
+
 	return m.versionService.StoreDBVersion(portainer.DBVersion)
 }

api/bolt/registry/registry.go

@@ -1,8 +1,8 @@
 package registry
 
 import (
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/internal"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
 )

api/bolt/resourcecontrol/resourcecontrol.go

@@ -1,8 +1,8 @@
 package resourcecontrol
 
 import (
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/internal"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
 )

api/bolt/role/role.go

@@ -0,0 +1,89 @@
+package role
+
+import (
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
+
+	"github.com/boltdb/bolt"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "roles"
+)
+
+// Service represents a service for managing endpoint data.
+type Service struct {
+	db *bolt.DB
+}
+
+// NewService creates a new instance of a service.
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		db: db,
+	}, nil
+}
+
+// Role returns a Role by ID
+func (service *Service) Role(ID portainer.RoleID) (*portainer.Role, error) {
+	var set portainer.Role
+	identifier := internal.Itob(int(ID))
+
+	err := internal.GetObject(service.db, BucketName, identifier, &set)
+	if err != nil {
+		return nil, err
+	}
+
+	return &set, nil
+}
+
+// Roles return an array containing all the sets.
+func (service *Service) Roles() ([]portainer.Role, error) {
+	var sets = make([]portainer.Role, 0)
+
+	err := service.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var set portainer.Role
+			err := internal.UnmarshalObject(v, &set)
+			if err != nil {
+				return err
+			}
+			sets = append(sets, set)
+		}
+
+		return nil
+	})
+
+	return sets, err
+}
+
+// CreateRole creates a new Role.
+func (service *Service) CreateRole(role *portainer.Role) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		id, _ := bucket.NextSequence()
+		role.ID = portainer.RoleID(id)
+
+		data, err := internal.MarshalObject(role)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(role.ID)), data)
+	})
+}
+
+// UpdateRole updates a role.
+func (service *Service) UpdateRole(ID portainer.RoleID, role *portainer.Role) error {
+	identifier := internal.Itob(int(ID))
+	return internal.UpdateObject(service.db, BucketName, identifier, role)
+}

api/bolt/schedule/schedule.go

@@ -0,0 +1,129 @@
+package schedule
+
+import (
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
+
+	"github.com/boltdb/bolt"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "schedules"
+)
+
+// Service represents a service for managing schedule data.
+type Service struct {
+	db *bolt.DB
+}
+
+// NewService creates a new instance of a service.
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		db: db,
+	}, nil
+}
+
+// Schedule returns a schedule by ID.
+func (service *Service) Schedule(ID portainer.ScheduleID) (*portainer.Schedule, error) {
+	var schedule portainer.Schedule
+	identifier := internal.Itob(int(ID))
+
+	err := internal.GetObject(service.db, BucketName, identifier, &schedule)
+	if err != nil {
+		return nil, err
+	}
+
+	return &schedule, nil
+}
+
+// UpdateSchedule updates a schedule.
+func (service *Service) UpdateSchedule(ID portainer.ScheduleID, schedule *portainer.Schedule) error {
+	identifier := internal.Itob(int(ID))
+	return internal.UpdateObject(service.db, BucketName, identifier, schedule)
+}
+
+// DeleteSchedule deletes a schedule.
+func (service *Service) DeleteSchedule(ID portainer.ScheduleID) error {
+	identifier := internal.Itob(int(ID))
+	return internal.DeleteObject(service.db, BucketName, identifier)
+}
+
+// Schedules return a array containing all the schedules.
+func (service *Service) Schedules() ([]portainer.Schedule, error) {
+	var schedules = make([]portainer.Schedule, 0)
+
+	err := service.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var schedule portainer.Schedule
+			err := internal.UnmarshalObject(v, &schedule)
+			if err != nil {
+				return err
+			}
+			schedules = append(schedules, schedule)
+		}
+
+		return nil
+	})
+
+	return schedules, err
+}
+
+// SchedulesByJobType return a array containing all the schedules
+// with the specified JobType.
+func (service *Service) SchedulesByJobType(jobType portainer.JobType) ([]portainer.Schedule, error) {
+	var schedules = make([]portainer.Schedule, 0)
+
+	err := service.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var schedule portainer.Schedule
+			err := internal.UnmarshalObject(v, &schedule)
+			if err != nil {
+				return err
+			}
+			if schedule.JobType == jobType {
+				schedules = append(schedules, schedule)
+			}
+		}
+
+		return nil
+	})
+
+	return schedules, err
+}
+
+// CreateSchedule assign an ID to a new schedule and saves it.
+func (service *Service) CreateSchedule(schedule *portainer.Schedule) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		// We manually manage sequences for schedules
+		err := bucket.SetSequence(uint64(schedule.ID))
+		if err != nil {
+			return err
+		}
+
+		data, err := internal.MarshalObject(schedule)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(schedule.ID)), data)
+	})
+}
+
+// GetNextIdentifier returns the next identifier for a schedule.
+func (service *Service) GetNextIdentifier() int {
+	return internal.GetNextIdentifier(service.db, BucketName)
+}

api/bolt/settings/settings.go

@@ -1,8 +1,8 @@
 package settings
 
 import (
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/internal"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
 )

api/bolt/stack/stack.go

@@ -1,8 +1,8 @@
 package stack
 
 import (
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/internal"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
 )

api/bolt/tag/tag.go

@@ -1,8 +1,8 @@
 package tag
 
 import (
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/internal"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
 )

api/bolt/team/team.go

@@ -1,8 +1,8 @@
 package team
 
 import (
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/internal"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
 )

api/bolt/teammembership/teammembership.go

@@ -1,8 +1,8 @@
 package teammembership
 
 import (
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/internal"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
 )

api/bolt/template/template.go

@@ -0,0 +1,95 @@
+package template
+
+import (
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
+
+	"github.com/boltdb/bolt"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "templates"
+)
+
+// Service represents a service for managing endpoint data.
+type Service struct {
+	db *bolt.DB
+}
+
+// NewService creates a new instance of a service.
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		db: db,
+	}, nil
+}
+
+// Templates return an array containing all the templates.
+func (service *Service) Templates() ([]portainer.Template, error) {
+	var templates = make([]portainer.Template, 0)
+
+	err := service.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var template portainer.Template
+			err := internal.UnmarshalObject(v, &template)
+			if err != nil {
+				return err
+			}
+			templates = append(templates, template)
+		}
+
+		return nil
+	})
+
+	return templates, err
+}
+
+// Template returns a template by ID.
+func (service *Service) Template(ID portainer.TemplateID) (*portainer.Template, error) {
+	var template portainer.Template
+	identifier := internal.Itob(int(ID))
+
+	err := internal.GetObject(service.db, BucketName, identifier, &template)
+	if err != nil {
+		return nil, err
+	}
+
+	return &template, nil
+}
+
+// CreateTemplate creates a new template.
+func (service *Service) CreateTemplate(template *portainer.Template) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		id, _ := bucket.NextSequence()
+		template.ID = portainer.TemplateID(id)
+
+		data, err := internal.MarshalObject(template)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(template.ID)), data)
+	})
+}
+
+// UpdateTemplate saves a template.
+func (service *Service) UpdateTemplate(ID portainer.TemplateID, template *portainer.Template) error {
+	identifier := internal.Itob(int(ID))
+	return internal.UpdateObject(service.db, BucketName, identifier, template)
+}
+
+// DeleteTemplate deletes a template.
+func (service *Service) DeleteTemplate(ID portainer.TemplateID) error {
+	identifier := internal.Itob(int(ID))
+	return internal.DeleteObject(service.db, BucketName, identifier)
+}

api/bolt/tunnelserver/tunnelserver.go

@@ -0,0 +1,48 @@
+package tunnelserver
+
+import (
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
+
+	"github.com/boltdb/bolt"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "tunnel_server"
+	infoKey    = "INFO"
+)
+
+// Service represents a service for managing endpoint data.
+type Service struct {
+	db *bolt.DB
+}
+
+// NewService creates a new instance of a service.
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		db: db,
+	}, nil
+}
+
+// Info retrieve the TunnelServerInfo object.
+func (service *Service) Info() (*portainer.TunnelServerInfo, error) {
+	var info portainer.TunnelServerInfo
+
+	err := internal.GetObject(service.db, BucketName, []byte(infoKey), &info)
+	if err != nil {
+		return nil, err
+	}
+
+	return &info, nil
+}
+
+// UpdateInfo persists a TunnelServerInfo object.
+func (service *Service) UpdateInfo(settings *portainer.TunnelServerInfo) error {
+	return internal.UpdateObject(service.db, BucketName, []byte(infoKey), settings)
+}

api/bolt/user/user.go

@@ -1,8 +1,8 @@
 package user
 
 import (
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/internal"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
 )

api/bolt/version/version.go

@@ -4,8 +4,8 @@ import (
 	"strconv"
 
 	"github.com/boltdb/bolt"
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/internal"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
 )
 
 const (

api/bolt/webhook/webhook.go

@@ -0,0 +1,151 @@
+package webhook
+
+import (
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
+
+	"github.com/boltdb/bolt"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "webhooks"
+)
+
+// Service represents a service for managing webhook data.
+type Service struct {
+	db *bolt.DB
+}
+
+// NewService creates a new instance of a service.
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		db: db,
+	}, nil
+}
+
+//Webhooks returns an array of all webhooks
+func (service *Service) Webhooks() ([]portainer.Webhook, error) {
+	var webhooks = make([]portainer.Webhook, 0)
+
+	err := service.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var webhook portainer.Webhook
+			err := internal.UnmarshalObject(v, &webhook)
+			if err != nil {
+				return err
+			}
+			webhooks = append(webhooks, webhook)
+		}
+
+		return nil
+	})
+
+	return webhooks, err
+}
+
+// Webhook returns a webhook by ID.
+func (service *Service) Webhook(ID portainer.WebhookID) (*portainer.Webhook, error) {
+	var webhook portainer.Webhook
+	identifier := internal.Itob(int(ID))
+
+	err := internal.GetObject(service.db, BucketName, identifier, &webhook)
+	if err != nil {
+		return nil, err
+	}
+
+	return &webhook, nil
+}
+
+// WebhookByResourceID returns a webhook by the ResourceID it is associated with.
+func (service *Service) WebhookByResourceID(ID string) (*portainer.Webhook, error) {
+	var webhook *portainer.Webhook
+
+	err := service.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+		cursor := bucket.Cursor()
+
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var w portainer.Webhook
+			err := internal.UnmarshalObject(v, &w)
+			if err != nil {
+				return err
+			}
+
+			if w.ResourceID == ID {
+				webhook = &w
+				break
+			}
+		}
+
+		if webhook == nil {
+			return portainer.ErrObjectNotFound
+		}
+
+		return nil
+	})
+
+	return webhook, err
+}
+
+// WebhookByToken returns a webhook by the random token it is associated with.
+func (service *Service) WebhookByToken(token string) (*portainer.Webhook, error) {
+	var webhook *portainer.Webhook
+
+	err := service.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+		cursor := bucket.Cursor()
+
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var w portainer.Webhook
+			err := internal.UnmarshalObject(v, &w)
+			if err != nil {
+				return err
+			}
+
+			if w.Token == token {
+				webhook = &w
+				break
+			}
+		}
+
+		if webhook == nil {
+			return portainer.ErrObjectNotFound
+		}
+
+		return nil
+	})
+
+	return webhook, err
+}
+
+// DeleteWebhook deletes a webhook.
+func (service *Service) DeleteWebhook(ID portainer.WebhookID) error {
+	identifier := internal.Itob(int(ID))
+	return internal.DeleteObject(service.db, BucketName, identifier)
+}
+
+// CreateWebhook assign an ID to a new webhook and saves it.
+func (service *Service) CreateWebhook(webhook *portainer.Webhook) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		id, _ := bucket.NextSequence()
+		webhook.ID = portainer.WebhookID(id)
+
+		data, err := internal.MarshalObject(webhook)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(webhook.ID)), data)
+	})
+}

api/chisel/key.go

@@ -0,0 +1,24 @@
+package chisel
+
+import (
+	"encoding/base64"
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+// GenerateEdgeKey will generate a key that can be used by an Edge agent to register with a Portainer instance.
+// The key represents the following data in this particular format:
+// portainer_instance_url|tunnel_server_addr|tunnel_server_fingerprint|endpoint_ID
+// The key returned by this function is a base64 encoded version of the data.
+func (service *Service) GenerateEdgeKey(url, host string, endpointIdentifier int) string {
+	keyInformation := []string{
+		url,
+		fmt.Sprintf("%s:%s", host, service.serverPort),
+		service.serverFingerprint,
+		strconv.Itoa(endpointIdentifier),
+	}
+
+	key := strings.Join(keyInformation, "|")
+	return base64.RawStdEncoding.EncodeToString([]byte(key))
+}

api/chisel/schedules.go

@@ -0,0 +1,47 @@
+package chisel
+
+import (
+	"strconv"
+
+	portainer "github.com/portainer/portainer/api"
+)
+
+// AddSchedule register a schedule inside the tunnel details associated to an endpoint.
+func (service *Service) AddSchedule(endpointID portainer.EndpointID, schedule *portainer.EdgeSchedule) {
+	tunnel := service.GetTunnelDetails(endpointID)
+
+	existingScheduleIndex := -1
+	for idx, existingSchedule := range tunnel.Schedules {
+		if existingSchedule.ID == schedule.ID {
+			existingScheduleIndex = idx
+			break
+		}
+	}
+
+	if existingScheduleIndex == -1 {
+		tunnel.Schedules = append(tunnel.Schedules, *schedule)
+	} else {
+		tunnel.Schedules[existingScheduleIndex] = *schedule
+	}
+
+	key := strconv.Itoa(int(endpointID))
+	service.tunnelDetailsMap.Set(key, tunnel)
+}
+
+// RemoveSchedule will remove the specified schedule from each tunnel it was registered with.
+func (service *Service) RemoveSchedule(scheduleID portainer.ScheduleID) {
+	for item := range service.tunnelDetailsMap.IterBuffered() {
+		tunnelDetails := item.Val.(*portainer.TunnelDetails)
+
+		updatedSchedules := make([]portainer.EdgeSchedule, 0)
+		for _, schedule := range tunnelDetails.Schedules {
+			if schedule.ID == scheduleID {
+				continue
+			}
+			updatedSchedules = append(updatedSchedules, schedule)
+		}
+
+		tunnelDetails.Schedules = updatedSchedules
+		service.tunnelDetailsMap.Set(item.Key, tunnelDetails)
+	}
+}

api/chisel/service.go

@@ -0,0 +1,191 @@
+package chisel
+
+import (
+	"fmt"
+	"log"
+	"strconv"
+	"time"
+
+	"github.com/dchest/uniuri"
+
+	cmap "github.com/orcaman/concurrent-map"
+
+	chserver "github.com/jpillora/chisel/server"
+	portainer "github.com/portainer/portainer/api"
+)
+
+const (
+	tunnelCleanupInterval = 10 * time.Second
+	requiredTimeout       = 15 * time.Second
+	activeTimeout         = 4*time.Minute + 30*time.Second
+)
+
+// Service represents a service to manage the state of multiple reverse tunnels.
+// It is used to start a reverse tunnel server and to manage the connection status of each tunnel
+// connected to the tunnel server.
+type Service struct {
+	serverFingerprint   string
+	serverPort          string
+	tunnelDetailsMap    cmap.ConcurrentMap
+	endpointService     portainer.EndpointService
+	tunnelServerService portainer.TunnelServerService
+	snapshotter         portainer.Snapshotter
+	chiselServer        *chserver.Server
+}
+
+// NewService returns a pointer to a new instance of Service
+func NewService(endpointService portainer.EndpointService, tunnelServerService portainer.TunnelServerService) *Service {
+	return &Service{
+		tunnelDetailsMap:    cmap.New(),
+		endpointService:     endpointService,
+		tunnelServerService: tunnelServerService,
+	}
+}
+
+// StartTunnelServer starts a tunnel server on the specified addr and port.
+// It uses a seed to generate a new private/public key pair. If the seed cannot
+// be found inside the database, it will generate a new one randomly and persist it.
+// It starts the tunnel status verification process in the background.
+// The snapshotter is used in the tunnel status verification process.
+func (service *Service) StartTunnelServer(addr, port string, snapshotter portainer.Snapshotter) error {
+	keySeed, err := service.retrievePrivateKeySeed()
+	if err != nil {
+		return err
+	}
+
+	config := &chserver.Config{
+		Reverse: true,
+		KeySeed: keySeed,
+	}
+
+	chiselServer, err := chserver.NewServer(config)
+	if err != nil {
+		return err
+	}
+
+	service.serverFingerprint = chiselServer.GetFingerprint()
+	service.serverPort = port
+
+	err = chiselServer.Start(addr, port)
+	if err != nil {
+		return err
+	}
+	service.chiselServer = chiselServer
+
+	// TODO: work-around Chisel default behavior.
+	// By default, Chisel will allow anyone to connect if no user exists.
+	username, password := generateRandomCredentials()
+	err = service.chiselServer.AddUser(username, password, "127.0.0.1")
+	if err != nil {
+		return err
+	}
+
+	service.snapshotter = snapshotter
+	go service.startTunnelVerificationLoop()
+
+	return nil
+}
+
+func (service *Service) retrievePrivateKeySeed() (string, error) {
+	var serverInfo *portainer.TunnelServerInfo
+
+	serverInfo, err := service.tunnelServerService.Info()
+	if err == portainer.ErrObjectNotFound {
+		keySeed := uniuri.NewLen(16)
+
+		serverInfo = &portainer.TunnelServerInfo{
+			PrivateKeySeed: keySeed,
+		}
+
+		err := service.tunnelServerService.UpdateInfo(serverInfo)
+		if err != nil {
+			return "", err
+		}
+	} else if err != nil {
+		return "", err
+	}
+
+	return serverInfo.PrivateKeySeed, nil
+}
+
+func (service *Service) startTunnelVerificationLoop() {
+	log.Printf("[DEBUG] [chisel, monitoring] [check_interval_seconds: %f] [message: starting tunnel management process]", tunnelCleanupInterval.Seconds())
+	ticker := time.NewTicker(tunnelCleanupInterval)
+	stopSignal := make(chan struct{})
+
+	for {
+		select {
+		case <-ticker.C:
+			service.checkTunnels()
+		case <-stopSignal:
+			ticker.Stop()
+			return
+		}
+	}
+}
+
+func (service *Service) checkTunnels() {
+	for item := range service.tunnelDetailsMap.IterBuffered() {
+		tunnel := item.Val.(*portainer.TunnelDetails)
+
+		if tunnel.LastActivity.IsZero() || tunnel.Status == portainer.EdgeAgentIdle {
+			continue
+		}
+
+		elapsed := time.Since(tunnel.LastActivity)
+		log.Printf("[DEBUG] [chisel,monitoring] [endpoint_id: %s] [status: %s] [status_time_seconds: %f] [message: endpoint tunnel monitoring]", item.Key, tunnel.Status, elapsed.Seconds())
+
+		if tunnel.Status == portainer.EdgeAgentManagementRequired && elapsed.Seconds() < requiredTimeout.Seconds() {
+			continue
+		} else if tunnel.Status == portainer.EdgeAgentManagementRequired && elapsed.Seconds() > requiredTimeout.Seconds() {
+			log.Printf("[DEBUG] [chisel,monitoring] [endpoint_id: %s] [status: %s] [status_time_seconds: %f] [timeout_seconds: %f] [message: REQUIRED state timeout exceeded]", item.Key, tunnel.Status, elapsed.Seconds(), requiredTimeout.Seconds())
+		}
+
+		if tunnel.Status == portainer.EdgeAgentActive && elapsed.Seconds() < activeTimeout.Seconds() {
+			continue
+		} else if tunnel.Status == portainer.EdgeAgentActive && elapsed.Seconds() > activeTimeout.Seconds() {
+			log.Printf("[DEBUG] [chisel,monitoring] [endpoint_id: %s] [status: %s] [status_time_seconds: %f] [timeout_seconds: %f] [message: ACTIVE state timeout exceeded]", item.Key, tunnel.Status, elapsed.Seconds(), activeTimeout.Seconds())
+
+			endpointID, err := strconv.Atoi(item.Key)
+			if err != nil {
+				log.Printf("[ERROR] [chisel,snapshot,conversion] Invalid endpoint identifier (id: %s): %s", item.Key, err)
+			}
+
+			err = service.snapshotEnvironment(portainer.EndpointID(endpointID), tunnel.Port)
+			if err != nil {
+				log.Printf("[ERROR] [snapshot] Unable to snapshot Edge endpoint (id: %s): %s", item.Key, err)
+			}
+		}
+
+		if len(tunnel.Schedules) > 0 {
+			endpointID, err := strconv.Atoi(item.Key)
+			if err != nil {
+				log.Printf("[ERROR] [chisel,conversion] Invalid endpoint identifier (id: %s): %s", item.Key, err)
+				continue
+			}
+
+			service.SetTunnelStatusToIdle(portainer.EndpointID(endpointID))
+		} else {
+			service.tunnelDetailsMap.Remove(item.Key)
+		}
+
+	}
+}
+
+func (service *Service) snapshotEnvironment(endpointID portainer.EndpointID, tunnelPort int) error {
+	endpoint, err := service.endpointService.Endpoint(portainer.EndpointID(endpointID))
+	if err != nil {
+		return err
+	}
+
+	endpointURL := endpoint.URL
+	endpoint.URL = fmt.Sprintf("tcp://localhost:%d", tunnelPort)
+	snapshot, err := service.snapshotter.CreateSnapshot(endpoint)
+	if err != nil {
+		return err
+	}
+
+	endpoint.Snapshots = []portainer.Snapshot{*snapshot}
+	endpoint.URL = endpointURL
+	return service.endpointService.UpdateEndpoint(endpoint.ID, endpoint)
+}

api/chisel/tunnel.go

@@ -0,0 +1,144 @@
+package chisel
+
+import (
+	"encoding/base64"
+	"fmt"
+	"math/rand"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/portainer/libcrypto"
+
+	"github.com/dchest/uniuri"
+	portainer "github.com/portainer/portainer/api"
+)
+
+const (
+	minAvailablePort = 49152
+	maxAvailablePort = 65535
+)
+
+// getUnusedPort is used to generate an unused random port in the dynamic port range.
+// Dynamic ports (also called private ports) are 49152 to 65535.
+func (service *Service) getUnusedPort() int {
+	port := randomInt(minAvailablePort, maxAvailablePort)
+
+	for item := range service.tunnelDetailsMap.IterBuffered() {
+		tunnel := item.Val.(*portainer.TunnelDetails)
+		if tunnel.Port == port {
+			return service.getUnusedPort()
+		}
+	}
+
+	return port
+}
+
+func randomInt(min, max int) int {
+	return min + rand.Intn(max-min)
+}
+
+// GetTunnelDetails returns information about the tunnel associated to an endpoint.
+func (service *Service) GetTunnelDetails(endpointID portainer.EndpointID) *portainer.TunnelDetails {
+	key := strconv.Itoa(int(endpointID))
+
+	if item, ok := service.tunnelDetailsMap.Get(key); ok {
+		tunnelDetails := item.(*portainer.TunnelDetails)
+		return tunnelDetails
+	}
+
+	schedules := make([]portainer.EdgeSchedule, 0)
+	return &portainer.TunnelDetails{
+		Status:      portainer.EdgeAgentIdle,
+		Port:        0,
+		Schedules:   schedules,
+		Credentials: "",
+	}
+}
+
+// SetTunnelStatusToActive update the status of the tunnel associated to the specified endpoint.
+// It sets the status to ACTIVE.
+func (service *Service) SetTunnelStatusToActive(endpointID portainer.EndpointID) {
+	tunnel := service.GetTunnelDetails(endpointID)
+	tunnel.Status = portainer.EdgeAgentActive
+	tunnel.Credentials = ""
+	tunnel.LastActivity = time.Now()
+
+	key := strconv.Itoa(int(endpointID))
+	service.tunnelDetailsMap.Set(key, tunnel)
+}
+
+// SetTunnelStatusToIdle update the status of the tunnel associated to the specified endpoint.
+// It sets the status to IDLE.
+// It removes any existing credentials associated to the tunnel.
+func (service *Service) SetTunnelStatusToIdle(endpointID portainer.EndpointID) {
+	tunnel := service.GetTunnelDetails(endpointID)
+
+	tunnel.Status = portainer.EdgeAgentIdle
+	tunnel.Port = 0
+	tunnel.LastActivity = time.Now()
+
+	credentials := tunnel.Credentials
+	if credentials != "" {
+		tunnel.Credentials = ""
+		service.chiselServer.DeleteUser(strings.Split(credentials, ":")[0])
+	}
+
+	key := strconv.Itoa(int(endpointID))
+	service.tunnelDetailsMap.Set(key, tunnel)
+}
+
+// SetTunnelStatusToRequired update the status of the tunnel associated to the specified endpoint.
+// It sets the status to REQUIRED.
+// If no port is currently associated to the tunnel, it will associate a random unused port to the tunnel
+// and generate temporary credentials that can be used to establish a reverse tunnel on that port.
+// Credentials are encrypted using the Edge ID associated to the endpoint.
+func (service *Service) SetTunnelStatusToRequired(endpointID portainer.EndpointID) error {
+	tunnel := service.GetTunnelDetails(endpointID)
+
+	if tunnel.Port == 0 {
+		endpoint, err := service.endpointService.Endpoint(endpointID)
+		if err != nil {
+			return err
+		}
+
+		tunnel.Status = portainer.EdgeAgentManagementRequired
+		tunnel.Port = service.getUnusedPort()
+		tunnel.LastActivity = time.Now()
+
+		username, password := generateRandomCredentials()
+		authorizedRemote := fmt.Sprintf("^R:0.0.0.0:%d$", tunnel.Port)
+		err = service.chiselServer.AddUser(username, password, authorizedRemote)
+		if err != nil {
+			return err
+		}
+
+		credentials, err := encryptCredentials(username, password, endpoint.EdgeID)
+		if err != nil {
+			return err
+		}
+		tunnel.Credentials = credentials
+
+		key := strconv.Itoa(int(endpointID))
+		service.tunnelDetailsMap.Set(key, tunnel)
+	}
+
+	return nil
+}
+
+func generateRandomCredentials() (string, string) {
+	username := uniuri.NewLen(8)
+	password := uniuri.NewLen(8)
+	return username, password
+}
+
+func encryptCredentials(username, password, key string) (string, error) {
+	credentials := fmt.Sprintf("%s:%s", username, password)
+
+	encryptedCredentials, err := libcrypto.Encrypt([]byte(credentials), []byte(key))
+	if err != nil {
+		return "", err
+	}
+
+	return base64.RawStdEncoding.EncodeToString(encryptedCredentials), nil
+}

api/cli/cli.go

@@ -3,7 +3,7 @@ package cli
 import (
 	"time"
 
-	"github.com/portainer/portainer"
+	"github.com/portainer/portainer/api"
 
 	"os"
 	"path/filepath"
@@ -16,10 +16,12 @@ import (
 type Service struct{}
 
 const (
-	errInvalidEndpointProtocol       = portainer.Error("Invalid endpoint protocol: Portainer only supports unix:// or tcp://")
-	errSocketNotFound                = portainer.Error("Unable to locate Unix socket")
+	errInvalidEndpointProtocol       = portainer.Error("Invalid endpoint protocol: Portainer only supports unix://, npipe:// or tcp://")
+	errSocketOrNamedPipeNotFound     = portainer.Error("Unable to locate Unix socket or named pipe")
 	errEndpointsFileNotFound         = portainer.Error("Unable to locate external endpoints file")
+	errTemplateFileNotFound          = portainer.Error("Unable to locate template file on disk")
 	errInvalidSyncInterval           = portainer.Error("Invalid synchronization interval")
+	errInvalidSnapshotInterval       = portainer.Error("Invalid snapshot interval")
 	errEndpointExcludeExternal       = portainer.Error("Cannot use the -H flag mutually with --external-endpoints")
 	errNoAuthExcludeAdminPassword    = portainer.Error("Cannot use --no-auth with --admin-password or --admin-password-file")
 	errAdminPassExcludeAdminPassFile = portainer.Error("Cannot use --admin-password with --admin-password-file")
@@ -31,6 +33,8 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 
 	flags := &portainer.CLIFlags{
 		Addr:              kingpin.Flag("bind", "Address and port to serve Portainer").Default(defaultBindAddress).Short('p').String(),
+		TunnelAddr:        kingpin.Flag("tunnel-addr", "Address to serve the tunnel server").Default(defaultTunnelServerAddress).String(),
+		TunnelPort:        kingpin.Flag("tunnel-port", "Port to serve the tunnel server").Default(defaultTunnelServerPort).String(),
 		Assets:            kingpin.Flag("assets", "Path to the assets").Default(defaultAssetsDirectory).Short('a').String(),
 		Data:              kingpin.Flag("data", "Path to the folder where the data is stored").Default(defaultDataDirectory).Short('d').String(),
 		EndpointURL:       kingpin.Flag("host", "Endpoint URL").Short('H').String(),
@@ -46,11 +50,14 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 		SSLCert:           kingpin.Flag("sslcert", "Path to the SSL certificate used to secure the Portainer instance").Default(defaultSSLCertPath).String(),
 		SSLKey:            kingpin.Flag("sslkey", "Path to the SSL key used to secure the Portainer instance").Default(defaultSSLKeyPath).String(),
 		SyncInterval:      kingpin.Flag("sync-interval", "Duration between each synchronization via the external endpoints source").Default(defaultSyncInterval).String(),
+		Snapshot:          kingpin.Flag("snapshot", "Start a background job to create endpoint snapshots").Default(defaultSnapshot).Bool(),
+		SnapshotInterval:  kingpin.Flag("snapshot-interval", "Duration between each endpoint snapshot job").Default(defaultSnapshotInterval).String(),
 		AdminPassword:     kingpin.Flag("admin-password", "Hashed admin password").String(),
 		AdminPasswordFile: kingpin.Flag("admin-password-file", "Path to the file containing the password for the admin user").String(),
 		Labels:            pairs(kingpin.Flag("hide-label", "Hide containers with a specific label in the UI").Short('l')),
 		Logo:              kingpin.Flag("logo", "URL for the logo displayed in the UI").String(),
-		Templates:         kingpin.Flag("templates", "URL to the templates (apps) definitions").Short('t').String(),
+		Templates:         kingpin.Flag("templates", "URL to the templates definitions.").Short('t').String(),
+		TemplateFile:      kingpin.Flag("template-file", "Path to the templates (app) definitions on the filesystem").Default(defaultTemplateFile).String(),
 	}
 
 	kingpin.Parse()
@@ -73,7 +80,12 @@ func (*Service) ValidateFlags(flags *portainer.CLIFlags) error {
 		return errEndpointExcludeExternal
 	}
 
-	err := validateEndpointURL(*flags.EndpointURL)
+	err := validateTemplateFile(*flags.TemplateFile)
+	if err != nil {
+		return err
+	}
+
+	err = validateEndpointURL(*flags.EndpointURL)
 	if err != nil {
 		return err
 	}
@@ -88,6 +100,11 @@ func (*Service) ValidateFlags(flags *portainer.CLIFlags) error {
 		return err
 	}
 
+	err = validateSnapshotInterval(*flags.SnapshotInterval)
+	if err != nil {
+		return err
+	}
+
 	if *flags.NoAuth && (*flags.AdminPassword != "" || *flags.AdminPasswordFile != "") {
 		return errNoAuthExcludeAdminPassword
 	}
@@ -101,15 +118,16 @@ func (*Service) ValidateFlags(flags *portainer.CLIFlags) error {
 
 func validateEndpointURL(endpointURL string) error {
 	if endpointURL != "" {
-		if !strings.HasPrefix(endpointURL, "unix://") && !strings.HasPrefix(endpointURL, "tcp://") {
+		if !strings.HasPrefix(endpointURL, "unix://") && !strings.HasPrefix(endpointURL, "tcp://") && !strings.HasPrefix(endpointURL, "npipe://") {
 			return errInvalidEndpointProtocol
 		}
 
-		if strings.HasPrefix(endpointURL, "unix://") {
+		if strings.HasPrefix(endpointURL, "unix://") || strings.HasPrefix(endpointURL, "npipe://") {
 			socketPath := strings.TrimPrefix(endpointURL, "unix://")
+			socketPath = strings.TrimPrefix(socketPath, "npipe://")
 			if _, err := os.Stat(socketPath); err != nil {
 				if os.IsNotExist(err) {
-					return errSocketNotFound
+					return errSocketOrNamedPipeNotFound
 				}
 				return err
 			}
@@ -130,6 +148,16 @@ func validateExternalEndpoints(externalEndpoints string) error {
 	return nil
 }
 
+func validateTemplateFile(templateFile string) error {
+	if _, err := os.Stat(templateFile); err != nil {
+		if os.IsNotExist(err) {
+			return errTemplateFileNotFound
+		}
+		return err
+	}
+	return nil
+}
+
 func validateSyncInterval(syncInterval string) error {
 	if syncInterval != defaultSyncInterval {
 		_, err := time.ParseDuration(syncInterval)
@@ -139,3 +167,13 @@ func validateSyncInterval(syncInterval string) error {
 	}
 	return nil
 }
+
+func validateSnapshotInterval(snapshotInterval string) error {
+	if snapshotInterval != defaultSnapshotInterval {
+		_, err := time.ParseDuration(snapshotInterval)
+		if err != nil {
+			return errInvalidSnapshotInterval
+		}
+	}
+	return nil
+}

api/cli/defaults.go

@@ -3,18 +3,23 @@
 package cli
 
 const (
-	defaultBindAddress     = ":9000"
-	defaultDataDirectory   = "/data"
-	defaultAssetsDirectory = "./"
-	defaultNoAuth          = "false"
-	defaultNoAnalytics     = "false"
-	defaultTLS             = "false"
-	defaultTLSSkipVerify   = "false"
-	defaultTLSCACertPath   = "/certs/ca.pem"
-	defaultTLSCertPath     = "/certs/cert.pem"
-	defaultTLSKeyPath      = "/certs/key.pem"
-	defaultSSL             = "false"
-	defaultSSLCertPath     = "/certs/portainer.crt"
-	defaultSSLKeyPath      = "/certs/portainer.key"
-	defaultSyncInterval    = "60s"
+	defaultBindAddress         = ":9000"
+	defaultTunnelServerAddress = "0.0.0.0"
+	defaultTunnelServerPort    = "8000"
+	defaultDataDirectory       = "/data"
+	defaultAssetsDirectory     = "./"
+	defaultNoAuth              = "false"
+	defaultNoAnalytics         = "false"
+	defaultTLS                 = "false"
+	defaultTLSSkipVerify       = "false"
+	defaultTLSCACertPath       = "/certs/ca.pem"
+	defaultTLSCertPath         = "/certs/cert.pem"
+	defaultTLSKeyPath          = "/certs/key.pem"
+	defaultSSL                 = "false"
+	defaultSSLCertPath         = "/certs/portainer.crt"
+	defaultSSLKeyPath          = "/certs/portainer.key"
+	defaultSyncInterval        = "60s"
+	defaultSnapshot            = "true"
+	defaultSnapshotInterval    = "5m"
+	defaultTemplateFile        = "/templates.json"
 )

api/cli/defaults_windows.go

@@ -1,18 +1,23 @@
 package cli
 
 const (
-	defaultBindAddress     = ":9000"
-	defaultDataDirectory   = "C:\\data"
-	defaultAssetsDirectory = "./"
-	defaultNoAuth          = "false"
-	defaultNoAnalytics     = "false"
-	defaultTLS             = "false"
-	defaultTLSSkipVerify   = "false"
-	defaultTLSCACertPath   = "C:\\certs\\ca.pem"
-	defaultTLSCertPath     = "C:\\certs\\cert.pem"
-	defaultTLSKeyPath      = "C:\\certs\\key.pem"
-	defaultSSL             = "false"
-	defaultSSLCertPath     = "C:\\certs\\portainer.crt"
-	defaultSSLKeyPath      = "C:\\certs\\portainer.key"
-	defaultSyncInterval    = "60s"
+	defaultBindAddress         = ":9000"
+	defaultTunnelServerAddress = "0.0.0.0"
+	defaultTunnelServerPort    = "8000"
+	defaultDataDirectory       = "C:\\data"
+	defaultAssetsDirectory     = "./"
+	defaultNoAuth              = "false"
+	defaultNoAnalytics         = "false"
+	defaultTLS                 = "false"
+	defaultTLSSkipVerify       = "false"
+	defaultTLSCACertPath       = "C:\\certs\\ca.pem"
+	defaultTLSCertPath         = "C:\\certs\\cert.pem"
+	defaultTLSKeyPath          = "C:\\certs\\key.pem"
+	defaultSSL                 = "false"
+	defaultSSLCertPath         = "C:\\certs\\portainer.crt"
+	defaultSSLKeyPath          = "C:\\certs\\portainer.key"
+	defaultSyncInterval        = "60s"
+	defaultSnapshot            = "true"
+	defaultSnapshotInterval    = "5m"
+	defaultTemplateFile        = "/templates.json"
 )

api/cli/pairlist.go

@@ -1,7 +1,7 @@
 package cli
 
 import (
-	"github.com/portainer/portainer"
+	"github.com/portainer/portainer/api"
 
 	"fmt"
 	"gopkg.in/alecthomas/kingpin.v2"

api/cmd/portainer/main.go

@@ -1,23 +1,28 @@
-package main // import "github.com/portainer/portainer"
+package main
 
 import (
-	"strings"
-
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt"
-	"github.com/portainer/portainer/cli"
-	"github.com/portainer/portainer/cron"
-	"github.com/portainer/portainer/crypto"
-	"github.com/portainer/portainer/exec"
-	"github.com/portainer/portainer/filesystem"
-	"github.com/portainer/portainer/git"
-	"github.com/portainer/portainer/http"
-	"github.com/portainer/portainer/http/client"
-	"github.com/portainer/portainer/jwt"
-	"github.com/portainer/portainer/ldap"
-	"github.com/portainer/portainer/libcompose"
-
+	"encoding/json"
 	"log"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/portainer/portainer/api/chisel"
+
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt"
+	"github.com/portainer/portainer/api/cli"
+	"github.com/portainer/portainer/api/cron"
+	"github.com/portainer/portainer/api/crypto"
+	"github.com/portainer/portainer/api/docker"
+	"github.com/portainer/portainer/api/exec"
+	"github.com/portainer/portainer/api/filesystem"
+	"github.com/portainer/portainer/api/git"
+	"github.com/portainer/portainer/api/http"
+	"github.com/portainer/portainer/api/http/client"
+	"github.com/portainer/portainer/api/jwt"
+	"github.com/portainer/portainer/api/ldap"
+	"github.com/portainer/portainer/api/libcompose"
 )
 
 func initCLI() *portainer.CLIFlags {
@@ -65,12 +70,12 @@ func initStore(dataStorePath string, fileService portainer.FileService) *bolt.St
 	return store
 }
 
-func initComposeStackManager(dataStorePath string) portainer.ComposeStackManager {
-	return libcompose.NewComposeStackManager(dataStorePath)
+func initComposeStackManager(dataStorePath string, reverseTunnelService portainer.ReverseTunnelService) portainer.ComposeStackManager {
+	return libcompose.NewComposeStackManager(dataStorePath, reverseTunnelService)
 }
 
-func initSwarmStackManager(assetsPath string, dataStorePath string, signatureService portainer.DigitalSignatureService, fileService portainer.FileService) (portainer.SwarmStackManager, error) {
-	return exec.NewSwarmStackManager(assetsPath, dataStorePath, signatureService, fileService)
+func initSwarmStackManager(assetsPath string, dataStorePath string, signatureService portainer.DigitalSignatureService, fileService portainer.FileService, reverseTunnelService portainer.ReverseTunnelService) (portainer.SwarmStackManager, error) {
+	return exec.NewSwarmStackManager(assetsPath, dataStorePath, signatureService, fileService, reverseTunnelService)
 }
 
 func initJWTService(authenticationEnabled bool) portainer.JWTService {
@@ -85,7 +90,7 @@ func initJWTService(authenticationEnabled bool) portainer.JWTService {
 }
 
 func initDigitalSignatureService() portainer.DigitalSignatureService {
-	return &crypto.ECDSAService{}
+	return crypto.NewECDSAService(os.Getenv("AGENT_SECRET"))
 }
 
 func initCryptoService() portainer.CryptoService {
@@ -97,28 +102,136 @@ func initLDAPService() portainer.LDAPService {
 }
 
 func initGitService() portainer.GitService {
-	return &git.Service{}
+	return git.NewService()
 }
 
-func initEndpointWatcher(endpointService portainer.EndpointService, externalEnpointFile string, syncInterval string) bool {
-	authorizeEndpointMgmt := true
-	if externalEnpointFile != "" {
-		authorizeEndpointMgmt = false
-		log.Println("Using external endpoint definition. Endpoint management via the API will be disabled.")
-		endpointWatcher := cron.NewWatcher(endpointService, syncInterval)
-		err := endpointWatcher.WatchEndpointFile(externalEnpointFile)
-		if err != nil {
-			log.Fatal(err)
-		}
+func initClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService) *docker.ClientFactory {
+	return docker.NewClientFactory(signatureService, reverseTunnelService)
+}
+
+func initSnapshotter(clientFactory *docker.ClientFactory) portainer.Snapshotter {
+	return docker.NewSnapshotter(clientFactory)
+}
+
+func initJobScheduler() portainer.JobScheduler {
+	return cron.NewJobScheduler()
+}
+
+func loadSnapshotSystemSchedule(jobScheduler portainer.JobScheduler, snapshotter portainer.Snapshotter, scheduleService portainer.ScheduleService, endpointService portainer.EndpointService, settingsService portainer.SettingsService) error {
+	settings, err := settingsService.Settings()
+	if err != nil {
+		return err
 	}
-	return authorizeEndpointMgmt
+
+	schedules, err := scheduleService.SchedulesByJobType(portainer.SnapshotJobType)
+	if err != nil {
+		return err
+	}
+
+	var snapshotSchedule *portainer.Schedule
+	if len(schedules) == 0 {
+		snapshotJob := &portainer.SnapshotJob{}
+		snapshotSchedule = &portainer.Schedule{
+			ID:             portainer.ScheduleID(scheduleService.GetNextIdentifier()),
+			Name:           "system_snapshot",
+			CronExpression: "@every " + settings.SnapshotInterval,
+			Recurring:      true,
+			JobType:        portainer.SnapshotJobType,
+			SnapshotJob:    snapshotJob,
+			Created:        time.Now().Unix(),
+		}
+	} else {
+		snapshotSchedule = &schedules[0]
+	}
+
+	snapshotJobContext := cron.NewSnapshotJobContext(endpointService, snapshotter)
+	snapshotJobRunner := cron.NewSnapshotJobRunner(snapshotSchedule, snapshotJobContext)
+
+	err = jobScheduler.ScheduleJob(snapshotJobRunner)
+	if err != nil {
+		return err
+	}
+
+	if len(schedules) == 0 {
+		return scheduleService.CreateSchedule(snapshotSchedule)
+	}
+	return nil
 }
 
-func initStatus(authorizeEndpointMgmt bool, flags *portainer.CLIFlags) *portainer.Status {
+func loadEndpointSyncSystemSchedule(jobScheduler portainer.JobScheduler, scheduleService portainer.ScheduleService, endpointService portainer.EndpointService, flags *portainer.CLIFlags) error {
+	if *flags.ExternalEndpoints == "" {
+		return nil
+	}
+
+	log.Println("Using external endpoint definition. Endpoint management via the API will be disabled.")
+
+	schedules, err := scheduleService.SchedulesByJobType(portainer.EndpointSyncJobType)
+	if err != nil {
+		return err
+	}
+
+	if len(schedules) != 0 {
+		return nil
+	}
+
+	endpointSyncJob := &portainer.EndpointSyncJob{}
+
+	endpointSyncSchedule := &portainer.Schedule{
+		ID:              portainer.ScheduleID(scheduleService.GetNextIdentifier()),
+		Name:            "system_endpointsync",
+		CronExpression:  "@every " + *flags.SyncInterval,
+		Recurring:       true,
+		JobType:         portainer.EndpointSyncJobType,
+		EndpointSyncJob: endpointSyncJob,
+		Created:         time.Now().Unix(),
+	}
+
+	endpointSyncJobContext := cron.NewEndpointSyncJobContext(endpointService, *flags.ExternalEndpoints)
+	endpointSyncJobRunner := cron.NewEndpointSyncJobRunner(endpointSyncSchedule, endpointSyncJobContext)
+
+	err = jobScheduler.ScheduleJob(endpointSyncJobRunner)
+	if err != nil {
+		return err
+	}
+
+	return scheduleService.CreateSchedule(endpointSyncSchedule)
+}
+
+func loadSchedulesFromDatabase(jobScheduler portainer.JobScheduler, jobService portainer.JobService, scheduleService portainer.ScheduleService, endpointService portainer.EndpointService, fileService portainer.FileService, reverseTunnelService portainer.ReverseTunnelService) error {
+	schedules, err := scheduleService.Schedules()
+	if err != nil {
+		return err
+	}
+
+	for _, schedule := range schedules {
+
+		if schedule.JobType == portainer.ScriptExecutionJobType {
+			jobContext := cron.NewScriptExecutionJobContext(jobService, endpointService, fileService)
+			jobRunner := cron.NewScriptExecutionJobRunner(&schedule, jobContext)
+
+			err = jobScheduler.ScheduleJob(jobRunner)
+			if err != nil {
+				return err
+			}
+		}
+
+		if schedule.EdgeSchedule != nil {
+			for _, endpointID := range schedule.EdgeSchedule.Endpoints {
+				reverseTunnelService.AddSchedule(endpointID, schedule.EdgeSchedule)
+			}
+		}
+
+	}
+
+	return nil
+}
+
+func initStatus(endpointManagement, snapshot bool, flags *portainer.CLIFlags) *portainer.Status {
 	return &portainer.Status{
 		Analytics:          !*flags.NoAnalytics,
 		Authentication:     !*flags.NoAuth,
-		EndpointManagement: authorizeEndpointMgmt,
+		EndpointManagement: endpointManagement,
+		Snapshot:           snapshot,
 		Version:            portainer.APIVersion,
 	}
 }
@@ -143,23 +256,29 @@ func initSettings(settingsService portainer.SettingsService, flags *portainer.CL
 	_, err := settingsService.Settings()
 	if err == portainer.ErrObjectNotFound {
 		settings := &portainer.Settings{
-			LogoURL:                     *flags.Logo,
-			DisplayExternalContributors: false,
-			AuthenticationMethod:        portainer.AuthenticationInternal,
+			LogoURL:              *flags.Logo,
+			AuthenticationMethod: portainer.AuthenticationInternal,
 			LDAPSettings: portainer.LDAPSettings{
-				TLSConfig: portainer.TLSConfiguration{},
+				AutoCreateUsers: true,
+				TLSConfig:       portainer.TLSConfiguration{},
 				SearchSettings: []portainer.LDAPSearchSettings{
 					portainer.LDAPSearchSettings{},
 				},
+				GroupSearchSettings: []portainer.LDAPGroupSearchSettings{
+					portainer.LDAPGroupSearchSettings{},
+				},
 			},
+			OAuthSettings:                      portainer.OAuthSettings{},
 			AllowBindMountsForRegularUsers:     true,
 			AllowPrivilegedModeForRegularUsers: true,
+			AllowVolumeBrowserForRegularUsers:  false,
+			EnableHostManagementFeatures:       false,
+			SnapshotInterval:                   *flags.SnapshotInterval,
+			EdgeAgentCheckinInterval:           portainer.DefaultEdgeAgentCheckinIntervalInSeconds,
 		}
 
 		if *flags.Templates != "" {
 			settings.TemplatesURL = *flags.Templates
-		} else {
-			settings.TemplatesURL = portainer.DefaultTemplatesURL
 		}
 
 		if *flags.Labels != nil {
@@ -176,6 +295,45 @@ func initSettings(settingsService portainer.SettingsService, flags *portainer.CL
 	return nil
 }
 
+func initTemplates(templateService portainer.TemplateService, fileService portainer.FileService, templateURL, templateFile string) error {
+	if templateURL != "" {
+		log.Printf("Portainer started with the --templates flag. Using external templates, template management will be disabled.")
+		return nil
+	}
+
+	existingTemplates, err := templateService.Templates()
+	if err != nil {
+		return err
+	}
+
+	if len(existingTemplates) != 0 {
+		log.Printf("Templates already registered inside the database. Skipping template import.")
+		return nil
+	}
+
+	templatesJSON, err := fileService.GetFileContent(templateFile)
+	if err != nil {
+		log.Println("Unable to retrieve template definitions via filesystem")
+		return err
+	}
+
+	var templates []portainer.Template
+	err = json.Unmarshal(templatesJSON, &templates)
+	if err != nil {
+		log.Println("Unable to parse templates file. Please review your template definition file.")
+		return err
+	}
+
+	for _, template := range templates {
+		err := templateService.CreateTemplate(&template)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func retrieveFirstEndpointFromDatabase(endpointService portainer.EndpointService) *portainer.Endpoint {
 	endpoints, err := endpointService.Endpoints()
 	if err != nil {
@@ -213,7 +371,7 @@ func initKeyPair(fileService portainer.FileService, signatureService portainer.D
 	return generateAndStoreKeyPair(fileService, signatureService)
 }
 
-func createTLSSecuredEndpoint(flags *portainer.CLIFlags, endpointService portainer.EndpointService) error {
+func createTLSSecuredEndpoint(flags *portainer.CLIFlags, endpointService portainer.EndpointService, snapshotter portainer.Snapshotter) error {
 	tlsConfiguration := portainer.TLSConfiguration{
 		TLS:           *flags.TLS,
 		TLSSkipVerify: *flags.TLSSkipVerify,
@@ -227,16 +385,20 @@ func createTLSSecuredEndpoint(flags *portainer.CLIFlags, endpointService portain
 		tlsConfiguration.TLS = true
 	}
 
+	endpointID := endpointService.GetNextIdentifier()
 	endpoint := &portainer.Endpoint{
-		Name:            "primary",
-		URL:             *flags.EndpointURL,
-		GroupID:         portainer.EndpointGroupID(1),
-		Type:            portainer.DockerEnvironment,
-		TLSConfig:       tlsConfiguration,
-		AuthorizedUsers: []portainer.UserID{},
-		AuthorizedTeams: []portainer.TeamID{},
-		Extensions:      []portainer.EndpointExtension{},
-		Tags:            []string{},
+		ID:                 portainer.EndpointID(endpointID),
+		Name:               "primary",
+		URL:                *flags.EndpointURL,
+		GroupID:            portainer.EndpointGroupID(1),
+		Type:               portainer.DockerEnvironment,
+		TLSConfig:          tlsConfiguration,
+		UserAccessPolicies: portainer.UserAccessPolicies{},
+		TeamAccessPolicies: portainer.TeamAccessPolicies{},
+		Extensions:         []portainer.EndpointExtension{},
+		Tags:               []string{},
+		Status:             portainer.EndpointStatusUp,
+		Snapshots:          []portainer.Snapshot{},
 	}
 
 	if strings.HasPrefix(endpoint.URL, "tcp://") {
@@ -255,10 +417,10 @@ func createTLSSecuredEndpoint(flags *portainer.CLIFlags, endpointService portain
 		}
 	}
 
-	return endpointService.CreateEndpoint(endpoint)
+	return snapshotAndPersistEndpoint(endpoint, endpointService, snapshotter)
 }
 
-func createUnsecuredEndpoint(endpointURL string, endpointService portainer.EndpointService) error {
+func createUnsecuredEndpoint(endpointURL string, endpointService portainer.EndpointService, snapshotter portainer.Snapshotter) error {
 	if strings.HasPrefix(endpointURL, "tcp://") {
 		_, err := client.ExecutePingOperation(endpointURL, nil)
 		if err != nil {
@@ -266,22 +428,40 @@ func createUnsecuredEndpoint(endpointURL string, endpointService portainer.Endpo
 		}
 	}
 
+	endpointID := endpointService.GetNextIdentifier()
 	endpoint := &portainer.Endpoint{
-		Name:            "primary",
-		URL:             endpointURL,
-		GroupID:         portainer.EndpointGroupID(1),
-		Type:            portainer.DockerEnvironment,
-		TLSConfig:       portainer.TLSConfiguration{},
-		AuthorizedUsers: []portainer.UserID{},
-		AuthorizedTeams: []portainer.TeamID{},
-		Extensions:      []portainer.EndpointExtension{},
-		Tags:            []string{},
+		ID:                 portainer.EndpointID(endpointID),
+		Name:               "primary",
+		URL:                endpointURL,
+		GroupID:            portainer.EndpointGroupID(1),
+		Type:               portainer.DockerEnvironment,
+		TLSConfig:          portainer.TLSConfiguration{},
+		UserAccessPolicies: portainer.UserAccessPolicies{},
+		TeamAccessPolicies: portainer.TeamAccessPolicies{},
+		Extensions:         []portainer.EndpointExtension{},
+		Tags:               []string{},
+		Status:             portainer.EndpointStatusUp,
+		Snapshots:          []portainer.Snapshot{},
+	}
+
+	return snapshotAndPersistEndpoint(endpoint, endpointService, snapshotter)
+}
+
+func snapshotAndPersistEndpoint(endpoint *portainer.Endpoint, endpointService portainer.EndpointService, snapshotter portainer.Snapshotter) error {
+	snapshot, err := snapshotter.CreateSnapshot(endpoint)
+	endpoint.Status = portainer.EndpointStatusUp
+	if err != nil {
+		log.Printf("http error: endpoint snapshot error (endpoint=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
+	}
+
+	if snapshot != nil {
+		endpoint.Snapshots = []portainer.Snapshot{*snapshot}
 	}
 
 	return endpointService.CreateEndpoint(endpoint)
 }
 
-func initEndpoint(flags *portainer.CLIFlags, endpointService portainer.EndpointService) error {
+func initEndpoint(flags *portainer.CLIFlags, endpointService portainer.EndpointService, snapshotter portainer.Snapshotter) error {
 	if *flags.EndpointURL == "" {
 		return nil
 	}
@@ -297,9 +477,54 @@ func initEndpoint(flags *portainer.CLIFlags, endpointService portainer.EndpointS
 	}
 
 	if *flags.TLS || *flags.TLSSkipVerify {
-		return createTLSSecuredEndpoint(flags, endpointService)
+		return createTLSSecuredEndpoint(flags, endpointService, snapshotter)
+	}
+	return createUnsecuredEndpoint(*flags.EndpointURL, endpointService, snapshotter)
+}
+
+func initJobService(dockerClientFactory *docker.ClientFactory) portainer.JobService {
+	return docker.NewJobService(dockerClientFactory)
+}
+
+func initExtensionManager(fileService portainer.FileService, extensionService portainer.ExtensionService) (portainer.ExtensionManager, error) {
+	extensionManager := exec.NewExtensionManager(fileService, extensionService)
+
+	extensions, err := extensionService.Extensions()
+	if err != nil {
+		return nil, err
+	}
+
+	for _, extension := range extensions {
+		err := extensionManager.EnableExtension(&extension, extension.License.LicenseKey)
+		if err != nil {
+			log.Printf("Unable to enable extension: %s [extension: %s]", err.Error(), extension.Name)
+			extension.Enabled = false
+			extension.License.Valid = false
+		}
+
+		err = extensionService.Persist(&extension)
+		if err != nil {
+			return nil, err
+		}
+
+	}
+
+	return extensionManager, nil
+}
+
+func terminateIfNoAdminCreated(userService portainer.UserService) {
+	timer1 := time.NewTimer(5 * time.Minute)
+	<-timer1.C
+
+	users, err := userService.UsersByRole(portainer.AdministratorRole)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if len(users) == 0 {
+		log.Fatal("No administrator account was created after 5 min. Shutting down the Portainer instance for security reasons.")
+		return
 	}
-	return createUnsecuredEndpoint(*flags.EndpointURL, endpointService)
 }
 
 func main() {
@@ -312,41 +537,83 @@ func main() {
 
 	jwtService := initJWTService(!*flags.NoAuth)
 
-	cryptoService := initCryptoService()
-
-	digitalSignatureService := initDigitalSignatureService()
-
 	ldapService := initLDAPService()
 
 	gitService := initGitService()
 
-	authorizeEndpointMgmt := initEndpointWatcher(store.EndpointService, *flags.ExternalEndpoints, *flags.SyncInterval)
+	cryptoService := initCryptoService()
+
+	digitalSignatureService := initDigitalSignatureService()
 
 	err := initKeyPair(fileService, digitalSignatureService)
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	swarmStackManager, err := initSwarmStackManager(*flags.Assets, *flags.Data, digitalSignatureService, fileService)
+	extensionManager, err := initExtensionManager(fileService, store.ExtensionService)
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	composeStackManager := initComposeStackManager(*flags.Data)
+	reverseTunnelService := chisel.NewService(store.EndpointService, store.TunnelServerService)
+
+	clientFactory := initClientFactory(digitalSignatureService, reverseTunnelService)
+
+	jobService := initJobService(clientFactory)
+
+	snapshotter := initSnapshotter(clientFactory)
+
+	endpointManagement := true
+	if *flags.ExternalEndpoints != "" {
+		endpointManagement = false
+	}
+
+	swarmStackManager, err := initSwarmStackManager(*flags.Assets, *flags.Data, digitalSignatureService, fileService, reverseTunnelService)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	composeStackManager := initComposeStackManager(*flags.Data, reverseTunnelService)
+
+	err = initTemplates(store.TemplateService, fileService, *flags.Templates, *flags.TemplateFile)
+	if err != nil {
+		log.Fatal(err)
+	}
 
 	err = initSettings(store.SettingsService, flags)
 	if err != nil {
 		log.Fatal(err)
 	}
 
+	jobScheduler := initJobScheduler()
+
+	err = loadSchedulesFromDatabase(jobScheduler, jobService, store.ScheduleService, store.EndpointService, fileService, reverseTunnelService)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	err = loadEndpointSyncSystemSchedule(jobScheduler, store.ScheduleService, store.EndpointService, flags)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if *flags.Snapshot {
+		err = loadSnapshotSystemSchedule(jobScheduler, snapshotter, store.ScheduleService, store.EndpointService, store.SettingsService)
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	jobScheduler.Start()
+
 	err = initDockerHub(store.DockerHubService)
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	applicationStatus := initStatus(authorizeEndpointMgmt, flags)
+	applicationStatus := initStatus(endpointManagement, *flags.Snapshot, flags)
 
-	err = initEndpoint(flags, store.EndpointService)
+	err = initEndpoint(flags, store.EndpointService, snapshotter)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -357,7 +624,7 @@ func main() {
 		if err != nil {
 			log.Fatal(err)
 		}
-		adminPasswordHash, err = cryptoService.Hash(content)
+		adminPasswordHash, err = cryptoService.Hash(strings.TrimSuffix(string(content), "\n"))
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -374,9 +641,10 @@ func main() {
 		if len(users) == 0 {
 			log.Printf("Creating admin user with password hash %s", adminPasswordHash)
 			user := &portainer.User{
-				Username: "admin",
-				Role:     portainer.AdministratorRole,
-				Password: adminPasswordHash,
+				Username:                "admin",
+				Role:                    portainer.AdministratorRole,
+				Password:                adminPasswordHash,
+				PortainerAuthorizations: portainer.DefaultPortainerAuthorizations(),
 			}
 			err := store.UserService.CreateUser(user)
 			if err != nil {
@@ -387,34 +655,54 @@ func main() {
 		}
 	}
 
+	if !*flags.NoAuth {
+		go terminateIfNoAdminCreated(store.UserService)
+	}
+
+	err = reverseTunnelService.StartTunnelServer(*flags.TunnelAddr, *flags.TunnelPort, snapshotter)
+	if err != nil {
+		log.Fatal(err)
+	}
+
 	var server portainer.Server = &http.Server{
+		ReverseTunnelService:   reverseTunnelService,
 		Status:                 applicationStatus,
 		BindAddress:            *flags.Addr,
 		AssetsPath:             *flags.Assets,
 		AuthDisabled:           *flags.NoAuth,
-		EndpointManagement:     authorizeEndpointMgmt,
+		EndpointManagement:     endpointManagement,
+		RoleService:            store.RoleService,
 		UserService:            store.UserService,
 		TeamService:            store.TeamService,
 		TeamMembershipService:  store.TeamMembershipService,
 		EndpointService:        store.EndpointService,
 		EndpointGroupService:   store.EndpointGroupService,
+		ExtensionService:       store.ExtensionService,
 		ResourceControlService: store.ResourceControlService,
 		SettingsService:        store.SettingsService,
 		RegistryService:        store.RegistryService,
 		DockerHubService:       store.DockerHubService,
 		StackService:           store.StackService,
+		ScheduleService:        store.ScheduleService,
 		TagService:             store.TagService,
+		TemplateService:        store.TemplateService,
+		WebhookService:         store.WebhookService,
 		SwarmStackManager:      swarmStackManager,
 		ComposeStackManager:    composeStackManager,
+		ExtensionManager:       extensionManager,
 		CryptoService:          cryptoService,
 		JWTService:             jwtService,
 		FileService:            fileService,
 		LDAPService:            ldapService,
 		GitService:             gitService,
 		SignatureService:       digitalSignatureService,
+		JobScheduler:           jobScheduler,
+		Snapshotter:            snapshotter,
 		SSL:                    *flags.SSL,
 		SSLCert:                *flags.SSLCert,
 		SSLKey:                 *flags.SSLKey,
+		DockerClientFactory:    clientFactory,
+		JobService:             jobService,
 	}
 
 	log.Printf("Starting Portainer %s on %s", portainer.APIVersion, *flags.Addr)

api/cron/job_endpoint_sync.go

@@ -4,52 +4,98 @@ import (
 	"encoding/json"
 	"io/ioutil"
 	"log"
-	"os"
 	"strings"
 
-	"github.com/portainer/portainer"
+	"github.com/portainer/portainer/api"
 )
 
-type (
-	endpointSyncJob struct {
-		logger           *log.Logger
-		endpointService  portainer.EndpointService
-		endpointFilePath string
-	}
+// EndpointSyncJobRunner is used to run a EndpointSyncJob
+type EndpointSyncJobRunner struct {
+	schedule *portainer.Schedule
+	context  *EndpointSyncJobContext
+}
 
-	synchronization struct {
-		endpointsToCreate []*portainer.Endpoint
-		endpointsToUpdate []*portainer.Endpoint
-		endpointsToDelete []*portainer.Endpoint
-	}
+// EndpointSyncJobContext represents the context of execution of a EndpointSyncJob
+type EndpointSyncJobContext struct {
+	endpointService  portainer.EndpointService
+	endpointFilePath string
+}
 
-	fileEndpoint struct {
-		Name          string `json:"Name"`
-		URL           string `json:"URL"`
-		TLS           bool   `json:"TLS,omitempty"`
-		TLSSkipVerify bool   `json:"TLSSkipVerify,omitempty"`
-		TLSCACert     string `json:"TLSCACert,omitempty"`
-		TLSCert       string `json:"TLSCert,omitempty"`
-		TLSKey        string `json:"TLSKey,omitempty"`
-	}
-)
-
-const (
-	// ErrEmptyEndpointArray is an error raised when the external endpoint source array is empty.
-	ErrEmptyEndpointArray = portainer.Error("External endpoint source is empty")
-)
-
-func newEndpointSyncJob(endpointFilePath string, endpointService portainer.EndpointService) endpointSyncJob {
-	return endpointSyncJob{
-		logger:           log.New(os.Stderr, "", log.LstdFlags),
+// NewEndpointSyncJobContext returns a new context that can be used to execute a EndpointSyncJob
+func NewEndpointSyncJobContext(endpointService portainer.EndpointService, endpointFilePath string) *EndpointSyncJobContext {
+	return &EndpointSyncJobContext{
 		endpointService:  endpointService,
 		endpointFilePath: endpointFilePath,
 	}
 }
 
-func endpointSyncError(err error, logger *log.Logger) bool {
+// NewEndpointSyncJobRunner returns a new runner that can be scheduled
+func NewEndpointSyncJobRunner(schedule *portainer.Schedule, context *EndpointSyncJobContext) *EndpointSyncJobRunner {
+	return &EndpointSyncJobRunner{
+		schedule: schedule,
+		context:  context,
+	}
+}
+
+type synchronization struct {
+	endpointsToCreate []*portainer.Endpoint
+	endpointsToUpdate []*portainer.Endpoint
+	endpointsToDelete []*portainer.Endpoint
+}
+
+type fileEndpoint struct {
+	Name          string `json:"Name"`
+	URL           string `json:"URL"`
+	TLS           bool   `json:"TLS,omitempty"`
+	TLSSkipVerify bool   `json:"TLSSkipVerify,omitempty"`
+	TLSCACert     string `json:"TLSCACert,omitempty"`
+	TLSCert       string `json:"TLSCert,omitempty"`
+	TLSKey        string `json:"TLSKey,omitempty"`
+}
+
+// GetSchedule returns the schedule associated to the runner
+func (runner *EndpointSyncJobRunner) GetSchedule() *portainer.Schedule {
+	return runner.schedule
+}
+
+// Run triggers the execution of the endpoint synchronization process.
+func (runner *EndpointSyncJobRunner) Run() {
+	data, err := ioutil.ReadFile(runner.context.endpointFilePath)
+	if endpointSyncError(err) {
+		return
+	}
+
+	var fileEndpoints []fileEndpoint
+	err = json.Unmarshal(data, &fileEndpoints)
+	if endpointSyncError(err) {
+		return
+	}
+
+	if len(fileEndpoints) == 0 {
+		log.Println("background job error (endpoint synchronization). External endpoint source is empty")
+		return
+	}
+
+	storedEndpoints, err := runner.context.endpointService.Endpoints()
+	if endpointSyncError(err) {
+		return
+	}
+
+	convertedFileEndpoints := convertFileEndpoints(fileEndpoints)
+
+	sync := prepareSyncData(storedEndpoints, convertedFileEndpoints)
+	if sync.requireSync() {
+		err = runner.context.endpointService.Synchronize(sync.endpointsToCreate, sync.endpointsToUpdate, sync.endpointsToDelete)
+		if endpointSyncError(err) {
+			return
+		}
+		log.Printf("Endpoint synchronization ended. [created: %v] [updated: %v] [deleted: %v]", len(sync.endpointsToCreate), len(sync.endpointsToUpdate), len(sync.endpointsToDelete))
+	}
+}
+
+func endpointSyncError(err error) bool {
 	if err != nil {
-		logger.Printf("Endpoint synchronization error: %s", err)
+		log.Printf("background job error (endpoint synchronization). Unable to synchronize endpoints (err=%s)\n", err)
 		return true
 	}
 	return false
@@ -129,8 +175,7 @@ func (sync synchronization) requireSync() bool {
 	return false
 }
 
-// TMP: endpointSyncJob method to access logger, should be generic
-func (job endpointSyncJob) prepareSyncData(storedEndpoints, fileEndpoints []portainer.Endpoint) *synchronization {
+func prepareSyncData(storedEndpoints, fileEndpoints []portainer.Endpoint) *synchronization {
 	endpointsToCreate := make([]*portainer.Endpoint, 0)
 	endpointsToUpdate := make([]*portainer.Endpoint, 0)
 	endpointsToDelete := make([]*portainer.Endpoint, 0)
@@ -140,23 +185,23 @@ func (job endpointSyncJob) prepareSyncData(storedEndpoints, fileEndpoints []port
 		if fidx != -1 {
 			endpoint := mergeEndpointIfRequired(&storedEndpoints[idx], &fileEndpoints[fidx])
 			if endpoint != nil {
-				job.logger.Printf("New definition for a stored endpoint found in file, updating database. [name: %v] [url: %v]\n", endpoint.Name, endpoint.URL)
+				log.Printf("New definition for a stored endpoint found in file, updating database. [name: %v] [url: %v]\n", endpoint.Name, endpoint.URL)
 				endpointsToUpdate = append(endpointsToUpdate, endpoint)
 			}
 		} else {
-			job.logger.Printf("Stored endpoint not found in file (definition might be invalid), removing from database. [name: %v] [url: %v]", storedEndpoints[idx].Name, storedEndpoints[idx].URL)
+			log.Printf("Stored endpoint not found in file (definition might be invalid), removing from database. [name: %v] [url: %v]", storedEndpoints[idx].Name, storedEndpoints[idx].URL)
 			endpointsToDelete = append(endpointsToDelete, &storedEndpoints[idx])
 		}
 	}
 
 	for idx, endpoint := range fileEndpoints {
 		if !isValidEndpoint(&endpoint) {
-			job.logger.Printf("Invalid file endpoint definition, skipping. [name: %v] [url: %v]", endpoint.Name, endpoint.URL)
+			log.Printf("Invalid file endpoint definition, skipping. [name: %v] [url: %v]", endpoint.Name, endpoint.URL)
 			continue
 		}
 		sidx := endpointExists(&fileEndpoints[idx], storedEndpoints)
 		if sidx == -1 {
-			job.logger.Printf("File endpoint not found in database, adding to database. [name: %v] [url: %v]", fileEndpoints[idx].Name, fileEndpoints[idx].URL)
+			log.Printf("File endpoint not found in database, adding to database. [name: %v] [url: %v]", fileEndpoints[idx].Name, fileEndpoints[idx].URL)
 			endpointsToCreate = append(endpointsToCreate, &fileEndpoints[idx])
 		}
 	}
@@ -167,43 +212,3 @@ func (job endpointSyncJob) prepareSyncData(storedEndpoints, fileEndpoints []port
 		endpointsToDelete: endpointsToDelete,
 	}
 }
-
-func (job endpointSyncJob) Sync() error {
-	data, err := ioutil.ReadFile(job.endpointFilePath)
-	if endpointSyncError(err, job.logger) {
-		return err
-	}
-
-	var fileEndpoints []fileEndpoint
-	err = json.Unmarshal(data, &fileEndpoints)
-	if endpointSyncError(err, job.logger) {
-		return err
-	}
-
-	if len(fileEndpoints) == 0 {
-		return ErrEmptyEndpointArray
-	}
-
-	storedEndpoints, err := job.endpointService.Endpoints()
-	if endpointSyncError(err, job.logger) {
-		return err
-	}
-
-	convertedFileEndpoints := convertFileEndpoints(fileEndpoints)
-
-	sync := job.prepareSyncData(storedEndpoints, convertedFileEndpoints)
-	if sync.requireSync() {
-		err = job.endpointService.Synchronize(sync.endpointsToCreate, sync.endpointsToUpdate, sync.endpointsToDelete)
-		if endpointSyncError(err, job.logger) {
-			return err
-		}
-		job.logger.Printf("Endpoint synchronization ended. [created: %v] [updated: %v] [deleted: %v]", len(sync.endpointsToCreate), len(sync.endpointsToUpdate), len(sync.endpointsToDelete))
-	}
-	return nil
-}
-
-func (job endpointSyncJob) Run() {
-	job.logger.Println("Endpoint synchronization job started.")
-	err := job.Sync()
-	endpointSyncError(err, job.logger)
-}

api/cron/job_script_execution.go

@@ -0,0 +1,96 @@
+package cron
+
+import (
+	"log"
+	"time"
+
+	"github.com/portainer/portainer/api"
+)
+
+// ScriptExecutionJobRunner is used to run a ScriptExecutionJob
+type ScriptExecutionJobRunner struct {
+	schedule     *portainer.Schedule
+	context      *ScriptExecutionJobContext
+	executedOnce bool
+}
+
+// ScriptExecutionJobContext represents the context of execution of a ScriptExecutionJob
+type ScriptExecutionJobContext struct {
+	jobService      portainer.JobService
+	endpointService portainer.EndpointService
+	fileService     portainer.FileService
+}
+
+// NewScriptExecutionJobContext returns a new context that can be used to execute a ScriptExecutionJob
+func NewScriptExecutionJobContext(jobService portainer.JobService, endpointService portainer.EndpointService, fileService portainer.FileService) *ScriptExecutionJobContext {
+	return &ScriptExecutionJobContext{
+		jobService:      jobService,
+		endpointService: endpointService,
+		fileService:     fileService,
+	}
+}
+
+// NewScriptExecutionJobRunner returns a new runner that can be scheduled
+func NewScriptExecutionJobRunner(schedule *portainer.Schedule, context *ScriptExecutionJobContext) *ScriptExecutionJobRunner {
+	return &ScriptExecutionJobRunner{
+		schedule:     schedule,
+		context:      context,
+		executedOnce: false,
+	}
+}
+
+// Run triggers the execution of the job.
+// It will iterate through all the endpoints specified in the context to
+// execute the script associated to the job.
+func (runner *ScriptExecutionJobRunner) Run() {
+	if !runner.schedule.Recurring && runner.executedOnce {
+		return
+	}
+	runner.executedOnce = true
+
+	scriptFile, err := runner.context.fileService.GetFileContent(runner.schedule.ScriptExecutionJob.ScriptPath)
+	if err != nil {
+		log.Printf("scheduled job error (script execution). Unable to retrieve script file (err=%s)\n", err)
+		return
+	}
+
+	targets := make([]*portainer.Endpoint, 0)
+	for _, endpointID := range runner.schedule.ScriptExecutionJob.Endpoints {
+		endpoint, err := runner.context.endpointService.Endpoint(endpointID)
+		if err != nil {
+			log.Printf("scheduled job error (script execution). Unable to retrieve information about endpoint (id=%d) (err=%s)\n", endpointID, err)
+			return
+		}
+
+		targets = append(targets, endpoint)
+	}
+
+	runner.executeAndRetry(targets, scriptFile, 0)
+}
+
+func (runner *ScriptExecutionJobRunner) executeAndRetry(endpoints []*portainer.Endpoint, script []byte, retryCount int) {
+	retryTargets := make([]*portainer.Endpoint, 0)
+
+	for _, endpoint := range endpoints {
+		err := runner.context.jobService.ExecuteScript(endpoint, "", runner.schedule.ScriptExecutionJob.Image, script, runner.schedule)
+		if err == portainer.ErrUnableToPingEndpoint {
+			retryTargets = append(retryTargets, endpoint)
+		} else if err != nil {
+			log.Printf("scheduled job error (script execution). Unable to execute script (endpoint=%s) (err=%s)\n", endpoint.Name, err)
+		}
+	}
+
+	retryCount++
+	if retryCount >= runner.schedule.ScriptExecutionJob.RetryCount {
+		return
+	}
+
+	time.Sleep(time.Duration(runner.schedule.ScriptExecutionJob.RetryInterval) * time.Second)
+
+	runner.executeAndRetry(retryTargets, script, retryCount)
+}
+
+// GetSchedule returns the schedule associated to the runner
+func (runner *ScriptExecutionJobRunner) GetSchedule() *portainer.Schedule {
+	return runner.schedule
+}

api/cron/job_snapshot.go

@@ -0,0 +1,85 @@
+package cron
+
+import (
+	"log"
+
+	"github.com/portainer/portainer/api"
+)
+
+// SnapshotJobRunner is used to run a SnapshotJob
+type SnapshotJobRunner struct {
+	schedule *portainer.Schedule
+	context  *SnapshotJobContext
+}
+
+// SnapshotJobContext represents the context of execution of a SnapshotJob
+type SnapshotJobContext struct {
+	endpointService portainer.EndpointService
+	snapshotter     portainer.Snapshotter
+}
+
+// NewSnapshotJobContext returns a new context that can be used to execute a SnapshotJob
+func NewSnapshotJobContext(endpointService portainer.EndpointService, snapshotter portainer.Snapshotter) *SnapshotJobContext {
+	return &SnapshotJobContext{
+		endpointService: endpointService,
+		snapshotter:     snapshotter,
+	}
+}
+
+// NewSnapshotJobRunner returns a new runner that can be scheduled
+func NewSnapshotJobRunner(schedule *portainer.Schedule, context *SnapshotJobContext) *SnapshotJobRunner {
+	return &SnapshotJobRunner{
+		schedule: schedule,
+		context:  context,
+	}
+}
+
+// GetSchedule returns the schedule associated to the runner
+func (runner *SnapshotJobRunner) GetSchedule() *portainer.Schedule {
+	return runner.schedule
+}
+
+// Run triggers the execution of the schedule.
+// It will iterate through all the endpoints available in the database to
+// create a snapshot of each one of them.
+// As a snapshot can be a long process, to avoid any concurrency issue we
+// retrieve the latest version of the endpoint right after a snapshot.
+func (runner *SnapshotJobRunner) Run() {
+	go func() {
+		endpoints, err := runner.context.endpointService.Endpoints()
+		if err != nil {
+			log.Printf("background schedule error (endpoint snapshot). Unable to retrieve endpoint list (err=%s)\n", err)
+			return
+		}
+
+		for _, endpoint := range endpoints {
+			if endpoint.Type == portainer.AzureEnvironment || endpoint.Type == portainer.EdgeAgentEnvironment {
+				continue
+			}
+
+			snapshot, snapshotError := runner.context.snapshotter.CreateSnapshot(&endpoint)
+
+			latestEndpointReference, err := runner.context.endpointService.Endpoint(endpoint.ID)
+			if latestEndpointReference == nil {
+				log.Printf("background schedule error (endpoint snapshot). Endpoint not found inside the database anymore (endpoint=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
+				continue
+			}
+
+			latestEndpointReference.Status = portainer.EndpointStatusUp
+			if snapshotError != nil {
+				log.Printf("background schedule error (endpoint snapshot). Unable to create snapshot (endpoint=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, snapshotError)
+				latestEndpointReference.Status = portainer.EndpointStatusDown
+			}
+
+			if snapshot != nil {
+				latestEndpointReference.Snapshots = []portainer.Snapshot{*snapshot}
+			}
+
+			err = runner.context.endpointService.UpdateEndpoint(latestEndpointReference.ID, latestEndpointReference)
+			if err != nil {
+				log.Printf("background schedule error (endpoint snapshot). Unable to update endpoint (endpoint=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
+				return
+			}
+		}
+	}()
+}

api/cron/scheduler.go

@@ -0,0 +1,116 @@
+package cron
+
+import (
+	"github.com/portainer/portainer/api"
+	"github.com/robfig/cron"
+)
+
+// JobScheduler represents a service for managing crons
+type JobScheduler struct {
+	cron *cron.Cron
+}
+
+// NewJobScheduler initializes a new service
+func NewJobScheduler() *JobScheduler {
+	return &JobScheduler{
+		cron: cron.New(),
+	}
+}
+
+// ScheduleJob schedules the execution of a job via a runner
+func (scheduler *JobScheduler) ScheduleJob(runner portainer.JobRunner) error {
+	_, err := scheduler.cron.AddJob(runner.GetSchedule().CronExpression, runner)
+	return err
+}
+
+// UpdateSystemJobSchedule updates the first occurence of the specified
+// scheduled job based on the specified job type.
+// It does so by re-creating a new cron
+// and adding all the existing jobs. It will then re-schedule the new job
+// with the update cron expression passed in parameter.
+// NOTE: the cron library do not support updating schedules directly
+// hence the work-around
+func (scheduler *JobScheduler) UpdateSystemJobSchedule(jobType portainer.JobType, newCronExpression string) error {
+	cronEntries := scheduler.cron.Entries()
+	newCron := cron.New()
+
+	for _, entry := range cronEntries {
+		if entry.Job.(portainer.JobRunner).GetSchedule().JobType == jobType {
+			_, err := newCron.AddJob(newCronExpression, entry.Job)
+			if err != nil {
+				return err
+			}
+			continue
+		}
+
+		newCron.Schedule(entry.Schedule, entry.Job)
+	}
+
+	scheduler.cron.Stop()
+	scheduler.cron = newCron
+	scheduler.cron.Start()
+	return nil
+}
+
+// UpdateJobSchedule updates a specific scheduled job by re-creating a new cron
+// and adding all the existing jobs. It will then re-schedule the new job
+// via the specified JobRunner parameter.
+// NOTE: the cron library do not support updating schedules directly
+// hence the work-around
+func (scheduler *JobScheduler) UpdateJobSchedule(runner portainer.JobRunner) error {
+	cronEntries := scheduler.cron.Entries()
+	newCron := cron.New()
+
+	for _, entry := range cronEntries {
+
+		if entry.Job.(portainer.JobRunner).GetSchedule().ID == runner.GetSchedule().ID {
+
+			var jobRunner cron.Job = runner
+			if entry.Job.(portainer.JobRunner).GetSchedule().JobType == portainer.SnapshotJobType {
+				jobRunner = entry.Job
+			}
+
+			_, err := newCron.AddJob(runner.GetSchedule().CronExpression, jobRunner)
+			if err != nil {
+				return err
+			}
+			continue
+		}
+
+		newCron.Schedule(entry.Schedule, entry.Job)
+	}
+
+	scheduler.cron.Stop()
+	scheduler.cron = newCron
+	scheduler.cron.Start()
+	return nil
+}
+
+// UnscheduleJob remove a scheduled job by re-creating a new cron
+// and adding all the existing jobs except for the one specified via scheduleID.
+// NOTE: the cron library do not support removing schedules directly
+// hence the work-around
+func (scheduler *JobScheduler) UnscheduleJob(scheduleID portainer.ScheduleID) {
+	cronEntries := scheduler.cron.Entries()
+	newCron := cron.New()
+
+	for _, entry := range cronEntries {
+
+		if entry.Job.(portainer.JobRunner).GetSchedule().ID == scheduleID {
+			continue
+		}
+
+		newCron.Schedule(entry.Schedule, entry.Job)
+	}
+
+	scheduler.cron.Stop()
+	scheduler.cron = newCron
+	scheduler.cron.Start()
+}
+
+// Start starts the scheduled jobs
+func (scheduler *JobScheduler) Start() {
+	if len(scheduler.cron.Entries()) > 0 {
+		scheduler.cron.Start()
+	}
+}

api/cron/watcher.go

@@ -1,40 +0,0 @@
-package cron
-
-import (
-	"github.com/portainer/portainer"
-	"github.com/robfig/cron"
-)
-
-// Watcher represents a service for managing crons.
-type Watcher struct {
-	Cron            *cron.Cron
-	EndpointService portainer.EndpointService
-	syncInterval    string
-}
-
-// NewWatcher initializes a new service.
-func NewWatcher(endpointService portainer.EndpointService, syncInterval string) *Watcher {
-	return &Watcher{
-		Cron:            cron.New(),
-		EndpointService: endpointService,
-		syncInterval:    syncInterval,
-	}
-}
-
-// WatchEndpointFile starts a cron job to synchronize the endpoints from a file
-func (watcher *Watcher) WatchEndpointFile(endpointFilePath string) error {
-	job := newEndpointSyncJob(endpointFilePath, watcher.EndpointService)
-
-	err := job.Sync()
-	if err != nil {
-		return err
-	}
-
-	err = watcher.Cron.AddJob("@every "+watcher.syncInterval, job)
-	if err != nil {
-		return err
-	}
-
-	watcher.Cron.Start()
-	return nil
-}

api/crypto/ecdsa.go

@@ -3,12 +3,13 @@ package crypto
 import (
 	"crypto/ecdsa"
 	"crypto/elliptic"
-	"crypto/md5"
 	"crypto/rand"
 	"crypto/x509"
 	"encoding/base64"
 	"encoding/hex"
 	"math/big"
+
+	"github.com/portainer/libcrypto"
 )
 
 const (
@@ -27,6 +28,15 @@ type ECDSAService struct {
 	privateKey    *ecdsa.PrivateKey
 	publicKey     *ecdsa.PublicKey
 	encodedPubKey string
+	secret        string
+}
+
+// NewECDSAService returns a pointer to a ECDSAService.
+// An optional secret can be specified
+func NewECDSAService(secret string) *ECDSAService {
+	return &ECDSAService{
+		secret: secret,
+	}
 }
 
 // EncodedPublicKey returns the encoded version of the public that can be used
@@ -92,14 +102,18 @@ func (service *ECDSAService) GenerateKeyPair() ([]byte, []byte, error) {
 	return private, public, nil
 }
 
-// Sign creates a signature from a message.
-// It automatically hash the message using MD5 and creates a signature from
+// CreateSignature creates a digital signature.
+// It automatically hash a specific message using MD5 and creates a signature from
 // that hash.
+// If a secret is associated to the service, it will be used instead of the specified
+// message.
 // It then encodes the generated signature in base64.
-func (service *ECDSAService) Sign(message string) (string, error) {
-	digest := md5.New()
-	digest.Write([]byte(message))
-	hash := digest.Sum(nil)
+func (service *ECDSAService) CreateSignature(message string) (string, error) {
+	if service.secret != "" {
+		message = service.secret
+	}
+
+	hash := libcrypto.HashFromBytes([]byte(message))
 
 	r := big.NewInt(0)
 	s := big.NewInt(0)

api/docker/client.go

@@ -0,0 +1,136 @@
+package docker
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/docker/docker/client"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/crypto"
+)
+
+const (
+	unsupportedEnvironmentType  = portainer.Error("Environment not supported")
+	defaultDockerRequestTimeout = 60
+)
+
+// ClientFactory is used to create Docker clients
+type ClientFactory struct {
+	signatureService     portainer.DigitalSignatureService
+	reverseTunnelService portainer.ReverseTunnelService
+}
+
+// NewClientFactory returns a new instance of a ClientFactory
+func NewClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService) *ClientFactory {
+	return &ClientFactory{
+		signatureService:     signatureService,
+		reverseTunnelService: reverseTunnelService,
+	}
+}
+
+// CreateClient is a generic function to create a Docker client based on
+// a specific endpoint configuration. The nodeName parameter can be used
+// with an agent enabled endpoint to target a specific node in an agent cluster.
+func (factory *ClientFactory) CreateClient(endpoint *portainer.Endpoint, nodeName string) (*client.Client, error) {
+	if endpoint.Type == portainer.AzureEnvironment {
+		return nil, unsupportedEnvironmentType
+	} else if endpoint.Type == portainer.AgentOnDockerEnvironment {
+		return createAgentClient(endpoint, factory.signatureService, nodeName)
+	} else if endpoint.Type == portainer.EdgeAgentEnvironment {
+		return createEdgeClient(endpoint, factory.reverseTunnelService, nodeName)
+	}
+
+	if strings.HasPrefix(endpoint.URL, "unix://") || strings.HasPrefix(endpoint.URL, "npipe://") {
+		return createLocalClient(endpoint)
+	}
+	return createTCPClient(endpoint)
+}
+
+func createLocalClient(endpoint *portainer.Endpoint) (*client.Client, error) {
+	return client.NewClientWithOpts(
+		client.WithHost(endpoint.URL),
+		client.WithVersion(portainer.SupportedDockerAPIVersion),
+	)
+}
+
+func createTCPClient(endpoint *portainer.Endpoint) (*client.Client, error) {
+	httpCli, err := httpClient(endpoint)
+	if err != nil {
+		return nil, err
+	}
+
+	return client.NewClientWithOpts(
+		client.WithHost(endpoint.URL),
+		client.WithVersion(portainer.SupportedDockerAPIVersion),
+		client.WithHTTPClient(httpCli),
+	)
+}
+
+func createEdgeClient(endpoint *portainer.Endpoint, reverseTunnelService portainer.ReverseTunnelService, nodeName string) (*client.Client, error) {
+	httpCli, err := httpClient(endpoint)
+	if err != nil {
+		return nil, err
+	}
+
+	headers := map[string]string{}
+	if nodeName != "" {
+		headers[portainer.PortainerAgentTargetHeader] = nodeName
+	}
+
+	tunnel := reverseTunnelService.GetTunnelDetails(endpoint.ID)
+	endpointURL := fmt.Sprintf("http://localhost:%d", tunnel.Port)
+
+	return client.NewClientWithOpts(
+		client.WithHost(endpointURL),
+		client.WithVersion(portainer.SupportedDockerAPIVersion),
+		client.WithHTTPClient(httpCli),
+		client.WithHTTPHeaders(headers),
+	)
+}
+
+func createAgentClient(endpoint *portainer.Endpoint, signatureService portainer.DigitalSignatureService, nodeName string) (*client.Client, error) {
+	httpCli, err := httpClient(endpoint)
+	if err != nil {
+		return nil, err
+	}
+
+	signature, err := signatureService.CreateSignature(portainer.PortainerAgentSignatureMessage)
+	if err != nil {
+		return nil, err
+	}
+
+	headers := map[string]string{
+		portainer.PortainerAgentPublicKeyHeader: signatureService.EncodedPublicKey(),
+		portainer.PortainerAgentSignatureHeader: signature,
+	}
+
+	if nodeName != "" {
+		headers[portainer.PortainerAgentTargetHeader] = nodeName
+	}
+
+	return client.NewClientWithOpts(
+		client.WithHost(endpoint.URL),
+		client.WithVersion(portainer.SupportedDockerAPIVersion),
+		client.WithHTTPClient(httpCli),
+		client.WithHTTPHeaders(headers),
+	)
+}
+
+func httpClient(endpoint *portainer.Endpoint) (*http.Client, error) {
+	transport := &http.Transport{}
+
+	if endpoint.TLSConfig.TLS {
+		tlsConfig, err := crypto.CreateTLSConfigurationFromDisk(endpoint.TLSConfig.TLSCACertPath, endpoint.TLSConfig.TLSCertPath, endpoint.TLSConfig.TLSKeyPath, endpoint.TLSConfig.TLSSkipVerify)
+		if err != nil {
+			return nil, err
+		}
+		transport.TLSClientConfig = tlsConfig
+	}
+
+	return &http.Client{
+		Transport: transport,
+		Timeout:   defaultDockerRequestTimeout * time.Second,
+	}, nil
+}

api/docker/job.go

@@ -0,0 +1,115 @@
+package docker
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"io/ioutil"
+	"strconv"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/network"
+	"github.com/docker/docker/api/types/strslice"
+	"github.com/docker/docker/client"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/archive"
+)
+
+// JobService represents a service that handles the execution of jobs
+type JobService struct {
+	dockerClientFactory *ClientFactory
+}
+
+// NewJobService returns a pointer to a new job service
+func NewJobService(dockerClientFactory *ClientFactory) *JobService {
+	return &JobService{
+		dockerClientFactory: dockerClientFactory,
+	}
+}
+
+// ExecuteScript will leverage a privileged container to execute a script against the specified endpoint/nodename.
+// It will copy the script content specified as a parameter inside a container based on the specified image and execute it.
+func (service *JobService) ExecuteScript(endpoint *portainer.Endpoint, nodeName, image string, script []byte, schedule *portainer.Schedule) error {
+	buffer, err := archive.TarFileInBuffer(script, "script.sh", 0700)
+	if err != nil {
+		return err
+	}
+
+	cli, err := service.dockerClientFactory.CreateClient(endpoint, nodeName)
+	if err != nil {
+		return err
+	}
+	defer cli.Close()
+
+	_, err = cli.Ping(context.Background())
+	if err != nil {
+		return portainer.ErrUnableToPingEndpoint
+	}
+
+	err = pullImage(cli, image)
+	if err != nil {
+		return err
+	}
+
+	containerConfig := &container.Config{
+		AttachStdin:  true,
+		AttachStdout: true,
+		AttachStderr: true,
+		Tty:          true,
+		WorkingDir:   "/tmp",
+		Image:        image,
+		Labels: map[string]string{
+			"io.portainer.job.endpoint": strconv.Itoa(int(endpoint.ID)),
+		},
+		Cmd: strslice.StrSlice([]string{"sh", "/tmp/script.sh"}),
+	}
+
+	if schedule != nil {
+		containerConfig.Labels["io.portainer.schedule.id"] = strconv.Itoa(int(schedule.ID))
+	}
+
+	hostConfig := &container.HostConfig{
+		Binds:       []string{"/:/host", "/etc:/etc:ro", "/usr:/usr:ro", "/run:/run:ro", "/sbin:/sbin:ro", "/var:/var:ro"},
+		NetworkMode: "host",
+		Privileged:  true,
+	}
+
+	networkConfig := &network.NetworkingConfig{}
+
+	body, err := cli.ContainerCreate(context.Background(), containerConfig, hostConfig, networkConfig, "")
+	if err != nil {
+		return err
+	}
+
+	if schedule != nil {
+		err = cli.ContainerRename(context.Background(), body.ID, schedule.Name+"_"+body.ID)
+		if err != nil {
+			return err
+		}
+	}
+
+	copyOptions := types.CopyToContainerOptions{}
+	err = cli.CopyToContainer(context.Background(), body.ID, "/tmp", bytes.NewReader(buffer), copyOptions)
+	if err != nil {
+		return err
+	}
+
+	startOptions := types.ContainerStartOptions{}
+	return cli.ContainerStart(context.Background(), body.ID, startOptions)
+}
+
+func pullImage(cli *client.Client, image string) error {
+	imageReadCloser, err := cli.ImagePull(context.Background(), image, types.ImagePullOptions{})
+	if err != nil {
+		return err
+	}
+	defer imageReadCloser.Close()
+
+	_, err = io.Copy(ioutil.Discard, imageReadCloser)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

api/docker/snapshot.go

@@ -0,0 +1,189 @@
+package docker
+
+import (
+	"context"
+	"log"
+	"time"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/client"
+	"github.com/portainer/portainer/api"
+)
+
+func snapshot(cli *client.Client, endpoint *portainer.Endpoint) (*portainer.Snapshot, error) {
+	_, err := cli.Ping(context.Background())
+	if err != nil {
+		return nil, err
+	}
+
+	snapshot := &portainer.Snapshot{
+		StackCount: 0,
+	}
+
+	err = snapshotInfo(snapshot, cli)
+	if err != nil {
+		log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot engine information] [endpoint: %s] [err: %s]", endpoint.Name, err)
+	}
+
+	if snapshot.Swarm {
+		err = snapshotSwarmServices(snapshot, cli)
+		if err != nil {
+			log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot Swarm services] [endpoint: %s] [err: %s]", endpoint.Name, err)
+		}
+
+		err = snapshotNodes(snapshot, cli)
+		if err != nil {
+			log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot Swarm nodes] [endpoint: %s] [err: %s]", endpoint.Name, err)
+		}
+	}
+
+	err = snapshotContainers(snapshot, cli)
+	if err != nil {
+		log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot containers] [endpoint: %s] [err: %s]", endpoint.Name, err)
+	}
+
+	err = snapshotImages(snapshot, cli)
+	if err != nil {
+		log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot images] [endpoint: %s] [err: %s]", endpoint.Name, err)
+	}
+
+	err = snapshotVolumes(snapshot, cli)
+	if err != nil {
+		log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot volumes] [endpoint: %s] [err: %s]", endpoint.Name, err)
+	}
+
+	err = snapshotNetworks(snapshot, cli)
+	if err != nil {
+		log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot networks] [endpoint: %s] [err: %s]", endpoint.Name, err)
+	}
+
+	err = snapshotVersion(snapshot, cli)
+	if err != nil {
+		log.Printf("[WARN] [docker,snapshot] [message: unable to snapshot engine version] [endpoint: %s] [err: %s]", endpoint.Name, err)
+	}
+
+	snapshot.Time = time.Now().Unix()
+	return snapshot, nil
+}
+
+func snapshotInfo(snapshot *portainer.Snapshot, cli *client.Client) error {
+	info, err := cli.Info(context.Background())
+	if err != nil {
+		return err
+	}
+
+	snapshot.Swarm = info.Swarm.ControlAvailable
+	snapshot.DockerVersion = info.ServerVersion
+	snapshot.TotalCPU = info.NCPU
+	snapshot.TotalMemory = info.MemTotal
+	snapshot.SnapshotRaw.Info = info
+	return nil
+}
+
+func snapshotNodes(snapshot *portainer.Snapshot, cli *client.Client) error {
+	nodes, err := cli.NodeList(context.Background(), types.NodeListOptions{})
+	if err != nil {
+		return err
+	}
+	var nanoCpus int64
+	var totalMem int64
+	for _, node := range nodes {
+		nanoCpus += node.Description.Resources.NanoCPUs
+		totalMem += node.Description.Resources.MemoryBytes
+	}
+	snapshot.TotalCPU = int(nanoCpus / 1e9)
+	snapshot.TotalMemory = totalMem
+	return nil
+}
+
+func snapshotSwarmServices(snapshot *portainer.Snapshot, cli *client.Client) error {
+	stacks := make(map[string]struct{})
+
+	services, err := cli.ServiceList(context.Background(), types.ServiceListOptions{})
+	if err != nil {
+		return err
+	}
+
+	for _, service := range services {
+		for k, v := range service.Spec.Labels {
+			if k == "com.docker.stack.namespace" {
+				stacks[v] = struct{}{}
+			}
+		}
+	}
+
+	snapshot.ServiceCount = len(services)
+	snapshot.StackCount += len(stacks)
+	return nil
+}
+
+func snapshotContainers(snapshot *portainer.Snapshot, cli *client.Client) error {
+	containers, err := cli.ContainerList(context.Background(), types.ContainerListOptions{All: true})
+	if err != nil {
+		return err
+	}
+
+	runningContainers := 0
+	stoppedContainers := 0
+	stacks := make(map[string]struct{})
+	for _, container := range containers {
+		if container.State == "exited" {
+			stoppedContainers++
+		} else if container.State == "running" {
+			runningContainers++
+		}
+
+		for k, v := range container.Labels {
+			if k == "com.docker.compose.project" {
+				stacks[v] = struct{}{}
+			}
+		}
+	}
+
+	snapshot.RunningContainerCount = runningContainers
+	snapshot.StoppedContainerCount = stoppedContainers
+	snapshot.StackCount += len(stacks)
+	snapshot.SnapshotRaw.Containers = containers
+	return nil
+}
+
+func snapshotImages(snapshot *portainer.Snapshot, cli *client.Client) error {
+	images, err := cli.ImageList(context.Background(), types.ImageListOptions{})
+	if err != nil {
+		return err
+	}
+
+	snapshot.ImageCount = len(images)
+	snapshot.SnapshotRaw.Images = images
+	return nil
+}
+
+func snapshotVolumes(snapshot *portainer.Snapshot, cli *client.Client) error {
+	volumes, err := cli.VolumeList(context.Background(), filters.Args{})
+	if err != nil {
+		return err
+	}
+
+	snapshot.VolumeCount = len(volumes.Volumes)
+	snapshot.SnapshotRaw.Volumes = volumes
+	return nil
+}
+
+func snapshotNetworks(snapshot *portainer.Snapshot, cli *client.Client) error {
+	networks, err := cli.NetworkList(context.Background(), types.NetworkListOptions{})
+	if err != nil {
+		return err
+	}
+	snapshot.SnapshotRaw.Networks = networks
+	return nil
+}
+
+func snapshotVersion(snapshot *portainer.Snapshot, cli *client.Client) error {
+	version, err := cli.ServerVersion(context.Background())
+	if err != nil {
+		return err
+	}
+	snapshot.SnapshotRaw.Version = version
+	return nil
+}

api/docker/snapshotter.go

@@ -0,0 +1,28 @@
+package docker
+
+import (
+	"github.com/portainer/portainer/api"
+)
+
+// Snapshotter represents a service used to create endpoint snapshots
+type Snapshotter struct {
+	clientFactory *ClientFactory
+}
+
+// NewSnapshotter returns a new Snapshotter instance
+func NewSnapshotter(clientFactory *ClientFactory) *Snapshotter {
+	return &Snapshotter{
+		clientFactory: clientFactory,
+	}
+}
+
+// CreateSnapshot creates a snapshot of a specific endpoint
+func (snapshotter *Snapshotter) CreateSnapshot(endpoint *portainer.Endpoint) (*portainer.Snapshot, error) {
+	cli, err := snapshotter.clientFactory.CreateClient(endpoint, "")
+	if err != nil {
+		return nil, err
+	}
+	defer cli.Close()
+
+	return snapshot(cli, endpoint)
+}

api/errors.go

@@ -4,16 +4,18 @@ package portainer
 const (
 	ErrUnauthorized           = Error("Unauthorized")
 	ErrResourceAccessDenied   = Error("Access denied to resource")
+	ErrAuthorizationRequired  = Error("Authorization required for this operation")
 	ErrObjectNotFound         = Error("Object not found inside the database")
 	ErrMissingSecurityContext = Error("Unable to find security details in request context")
 )
 
 // User errors.
 const (
-	ErrUserAlreadyExists       = Error("User already exists")
-	ErrInvalidUsername         = Error("Invalid username. White spaces are not allowed")
-	ErrAdminAlreadyInitialized = Error("An administrator user already exists")
-	ErrAdminCannotRemoveSelf   = Error("Cannot remove your own user account. Contact another administrator")
+	ErrUserAlreadyExists          = Error("User already exists")
+	ErrInvalidUsername            = Error("Invalid username. White spaces are not allowed")
+	ErrAdminAlreadyInitialized    = Error("An administrator user already exists")
+	ErrAdminCannotRemoveSelf      = Error("Cannot remove your own user account. Contact another administrator")
+	ErrCannotRemoveLastLocalAdmin = Error("Cannot remove the last local administrator account")
 )
 
 // Team errors.
@@ -87,8 +89,29 @@ const (
 	ErrUndefinedTLSFileType = Error("Undefined TLS file type")
 )
 
+// Extension errors.
+const (
+	ErrExtensionAlreadyEnabled = Error("This extension is already enabled")
+)
+
+// Docker errors.
+const (
+	ErrUnableToPingEndpoint = Error("Unable to communicate with the endpoint")
+)
+
+// Schedule errors.
+const (
+	ErrHostManagementFeaturesDisabled = Error("Host management features are disabled")
+)
+
 // Error represents an application error.
 type Error string
 
 // Error returns the error message.
 func (e Error) Error() string { return string(e) }
+
+// Webhook errors
+const (
+	ErrWebhookAlreadyExists   = Error("A webhook for this resource already exists")
+	ErrUnsupportedWebhookType = Error("Webhooks for this resource are not currently supported")
+)

api/exec/extension.go

@@ -0,0 +1,217 @@
+package exec
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"log"
+	"os/exec"
+	"path"
+	"runtime"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/orcaman/concurrent-map"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/http/client"
+)
+
+var extensionDownloadBaseURL = "https://portainer-io-assets.sfo2.digitaloceanspaces.com/extensions/"
+
+var extensionBinaryMap = map[portainer.ExtensionID]string{
+	portainer.RegistryManagementExtension:  "extension-registry-management",
+	portainer.OAuthAuthenticationExtension: "extension-oauth-authentication",
+	portainer.RBACExtension:                "extension-rbac",
+}
+
+// ExtensionManager represents a service used to
+// manage extension processes.
+type ExtensionManager struct {
+	processes        cmap.ConcurrentMap
+	fileService      portainer.FileService
+	extensionService portainer.ExtensionService
+}
+
+// NewExtensionManager returns a pointer to an ExtensionManager
+func NewExtensionManager(fileService portainer.FileService, extensionService portainer.ExtensionService) *ExtensionManager {
+	return &ExtensionManager{
+		processes:        cmap.New(),
+		fileService:      fileService,
+		extensionService: extensionService,
+	}
+}
+
+func processKey(ID portainer.ExtensionID) string {
+	return strconv.Itoa(int(ID))
+}
+
+func buildExtensionURL(extension *portainer.Extension) string {
+	extensionURL := extensionDownloadBaseURL
+	extensionURL += extensionBinaryMap[extension.ID]
+	extensionURL += "-" + runtime.GOOS + "-" + runtime.GOARCH
+	extensionURL += "-" + extension.Version
+	extensionURL += ".zip"
+	return extensionURL
+}
+
+func buildExtensionPath(binaryPath string, extension *portainer.Extension) string {
+
+	extensionFilename := extensionBinaryMap[extension.ID]
+	extensionFilename += "-" + runtime.GOOS + "-" + runtime.GOARCH
+	extensionFilename += "-" + extension.Version
+
+	if runtime.GOOS == "windows" {
+		extensionFilename += ".exe"
+	}
+
+	extensionPath := path.Join(
+		binaryPath,
+		extensionFilename)
+
+	return extensionPath
+}
+
+// FetchExtensionDefinitions will fetch the list of available
+// extension definitions from the official Portainer assets server
+func (manager *ExtensionManager) FetchExtensionDefinitions() ([]portainer.Extension, error) {
+	extensionData, err := client.Get(portainer.ExtensionDefinitionsURL, 30)
+	if err != nil {
+		return nil, err
+	}
+
+	var extensions []portainer.Extension
+	err = json.Unmarshal(extensionData, &extensions)
+	if err != nil {
+		return nil, err
+	}
+
+	return extensions, nil
+}
+
+// EnableExtension will check for the existence of the extension binary on the filesystem
+// first. If it does not exist, it will download it from the official Portainer assets server.
+// After installing the binary on the filesystem, it will execute the binary in license check
+// mode to validate the extension license. If the license is valid, it will then start
+// the extension process and register it in the processes map.
+func (manager *ExtensionManager) EnableExtension(extension *portainer.Extension, licenseKey string) error {
+	extensionBinaryPath := buildExtensionPath(manager.fileService.GetBinaryFolder(), extension)
+	extensionBinaryExists, err := manager.fileService.FileExists(extensionBinaryPath)
+	if err != nil {
+		return err
+	}
+
+	if !extensionBinaryExists {
+		err := manager.downloadExtension(extension)
+		if err != nil {
+			return err
+		}
+	}
+
+	licenseDetails, err := validateLicense(extensionBinaryPath, licenseKey)
+	if err != nil {
+		return err
+	}
+
+	extension.License = portainer.LicenseInformation{
+		LicenseKey: licenseKey,
+		Company:    licenseDetails[0],
+		Expiration: licenseDetails[1],
+		Valid:      true,
+	}
+	extension.Version = licenseDetails[2]
+
+	return manager.startExtensionProcess(extension, extensionBinaryPath)
+}
+
+// DisableExtension will retrieve the process associated to the extension
+// from the processes map and kill the process. It will then remove the process
+// from the processes map and remove the binary associated to the extension
+// from the filesystem
+func (manager *ExtensionManager) DisableExtension(extension *portainer.Extension) error {
+	process, ok := manager.processes.Get(processKey(extension.ID))
+	if !ok {
+		return nil
+	}
+
+	err := process.(*exec.Cmd).Process.Kill()
+	if err != nil {
+		return err
+	}
+
+	manager.processes.Remove(processKey(extension.ID))
+
+	extensionBinaryPath := buildExtensionPath(manager.fileService.GetBinaryFolder(), extension)
+	return manager.fileService.RemoveDirectory(extensionBinaryPath)
+}
+
+// UpdateExtension will download the new extension binary from the official Portainer assets
+// server, disable the previous extension via DisableExtension, trigger a license check
+// and then start the extension process and add it to the processes map
+func (manager *ExtensionManager) UpdateExtension(extension *portainer.Extension, version string) error {
+	oldVersion := extension.Version
+
+	extension.Version = version
+	err := manager.downloadExtension(extension)
+	if err != nil {
+		return err
+	}
+
+	extension.Version = oldVersion
+	err = manager.DisableExtension(extension)
+	if err != nil {
+		return err
+	}
+
+	extension.Version = version
+	extensionBinaryPath := buildExtensionPath(manager.fileService.GetBinaryFolder(), extension)
+
+	licenseDetails, err := validateLicense(extensionBinaryPath, extension.License.LicenseKey)
+	if err != nil {
+		return err
+	}
+
+	extension.Version = licenseDetails[2]
+
+	return manager.startExtensionProcess(extension, extensionBinaryPath)
+}
+
+func (manager *ExtensionManager) downloadExtension(extension *portainer.Extension) error {
+	extensionURL := buildExtensionURL(extension)
+
+	data, err := client.Get(extensionURL, 30)
+	if err != nil {
+		return err
+	}
+
+	return manager.fileService.ExtractExtensionArchive(data)
+}
+
+func validateLicense(binaryPath, licenseKey string) ([]string, error) {
+	licenseCheckProcess := exec.Command(binaryPath, "-license", licenseKey, "-check")
+	cmdOutput := &bytes.Buffer{}
+	licenseCheckProcess.Stdout = cmdOutput
+
+	err := licenseCheckProcess.Run()
+	if err != nil {
+		log.Printf("[DEBUG] [exec,extension] [message: unable to run extension process] [err: %s]", err)
+		return nil, errors.New("Invalid extension license key")
+	}
+
+	output := string(cmdOutput.Bytes())
+
+	return strings.Split(output, "|"), nil
+}
+
+func (manager *ExtensionManager) startExtensionProcess(extension *portainer.Extension, binaryPath string) error {
+	extensionProcess := exec.Command(binaryPath, "-license", extension.License.LicenseKey)
+	err := extensionProcess.Start()
+	if err != nil {
+		return err
+	}
+
+	time.Sleep(3 * time.Second)
+
+	manager.processes.Set(processKey(extension.ID), extensionProcess)
+	return nil
+}

api/exec/swarm_stack.go

@@ -3,30 +3,33 @@ package exec
 import (
 	"bytes"
 	"encoding/json"
+	"fmt"
 	"os"
 	"os/exec"
 	"path"
 	"runtime"
 
-	"github.com/portainer/portainer"
+	"github.com/portainer/portainer/api"
 )
 
 // SwarmStackManager represents a service for managing stacks.
 type SwarmStackManager struct {
-	binaryPath       string
-	dataPath         string
-	signatureService portainer.DigitalSignatureService
-	fileService      portainer.FileService
+	binaryPath           string
+	dataPath             string
+	signatureService     portainer.DigitalSignatureService
+	fileService          portainer.FileService
+	reverseTunnelService portainer.ReverseTunnelService
 }
 
 // NewSwarmStackManager initializes a new SwarmStackManager service.
 // It also updates the configuration of the Docker CLI binary.
-func NewSwarmStackManager(binaryPath, dataPath string, signatureService portainer.DigitalSignatureService, fileService portainer.FileService) (*SwarmStackManager, error) {
+func NewSwarmStackManager(binaryPath, dataPath string, signatureService portainer.DigitalSignatureService, fileService portainer.FileService, reverseTunnelService portainer.ReverseTunnelService) (*SwarmStackManager, error) {
 	manager := &SwarmStackManager{
-		binaryPath:       binaryPath,
-		dataPath:         dataPath,
-		signatureService: signatureService,
-		fileService:      fileService,
+		binaryPath:           binaryPath,
+		dataPath:             dataPath,
+		signatureService:     signatureService,
+		fileService:          fileService,
+		reverseTunnelService: reverseTunnelService,
 	}
 
 	err := manager.updateDockerCLIConfiguration(dataPath)
@@ -39,7 +42,7 @@ func NewSwarmStackManager(binaryPath, dataPath string, signatureService portaine
 
 // Login executes the docker login command against a list of registries (including DockerHub).
 func (manager *SwarmStackManager) Login(dockerhub *portainer.DockerHub, registries []portainer.Registry, endpoint *portainer.Endpoint) {
-	command, args := prepareDockerCommandAndArgs(manager.binaryPath, manager.dataPath, endpoint)
+	command, args := manager.prepareDockerCommandAndArgs(manager.binaryPath, manager.dataPath, endpoint)
 	for _, registry := range registries {
 		if registry.Authentication {
 			registryArgs := append(args, "login", "--username", registry.Username, "--password", registry.Password, registry.URL)
@@ -55,7 +58,7 @@ func (manager *SwarmStackManager) Login(dockerhub *portainer.DockerHub, registri
 
 // Logout executes the docker logout command.
 func (manager *SwarmStackManager) Logout(endpoint *portainer.Endpoint) error {
-	command, args := prepareDockerCommandAndArgs(manager.binaryPath, manager.dataPath, endpoint)
+	command, args := manager.prepareDockerCommandAndArgs(manager.binaryPath, manager.dataPath, endpoint)
 	args = append(args, "logout")
 	return runCommandAndCaptureStdErr(command, args, nil, "")
 }
@@ -63,7 +66,7 @@ func (manager *SwarmStackManager) Logout(endpoint *portainer.Endpoint) error {
 // Deploy executes the docker stack deploy command.
 func (manager *SwarmStackManager) Deploy(stack *portainer.Stack, prune bool, endpoint *portainer.Endpoint) error {
 	stackFilePath := path.Join(stack.ProjectPath, stack.EntryPoint)
-	command, args := prepareDockerCommandAndArgs(manager.binaryPath, manager.dataPath, endpoint)
+	command, args := manager.prepareDockerCommandAndArgs(manager.binaryPath, manager.dataPath, endpoint)
 
 	if prune {
 		args = append(args, "stack", "deploy", "--prune", "--with-registry-auth", "--compose-file", stackFilePath, stack.Name)
@@ -82,7 +85,7 @@ func (manager *SwarmStackManager) Deploy(stack *portainer.Stack, prune bool, end
 
 // Remove executes the docker stack rm command.
 func (manager *SwarmStackManager) Remove(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
-	command, args := prepareDockerCommandAndArgs(manager.binaryPath, manager.dataPath, endpoint)
+	command, args := manager.prepareDockerCommandAndArgs(manager.binaryPath, manager.dataPath, endpoint)
 	args = append(args, "stack", "rm", stack.Name)
 	return runCommandAndCaptureStdErr(command, args, nil, "")
 }
@@ -106,7 +109,7 @@ func runCommandAndCaptureStdErr(command string, args []string, env []string, wor
 	return nil
 }
 
-func prepareDockerCommandAndArgs(binaryPath, dataPath string, endpoint *portainer.Endpoint) (string, []string) {
+func (manager *SwarmStackManager) prepareDockerCommandAndArgs(binaryPath, dataPath string, endpoint *portainer.Endpoint) (string, []string) {
 	// Assume Linux as a default
 	command := path.Join(binaryPath, "docker")
 
@@ -116,7 +119,14 @@ func prepareDockerCommandAndArgs(binaryPath, dataPath string, endpoint *portaine
 
 	args := make([]string, 0)
 	args = append(args, "--config", dataPath)
-	args = append(args, "-H", endpoint.URL)
+
+	endpointURL := endpoint.URL
+	if endpoint.Type == portainer.EdgeAgentEnvironment {
+		tunnel := manager.reverseTunnelService.GetTunnelDetails(endpoint.ID)
+		endpointURL = fmt.Sprintf("tcp://localhost:%d", tunnel.Port)
+	}
+
+	args = append(args, "-H", endpointURL)
 
 	if endpoint.TLSConfig.TLS {
 		args = append(args, "--tls")
@@ -140,7 +150,7 @@ func (manager *SwarmStackManager) updateDockerCLIConfiguration(dataPath string)
 		return err
 	}
 
-	signature, err := manager.signatureService.Sign(portainer.PortainerAgentSignatureMessage)
+	signature, err := manager.signatureService.CreateSignature(portainer.PortainerAgentSignatureMessage)
 	if err != nil {
 		return err
 	}
@@ -169,7 +179,7 @@ func (manager *SwarmStackManager) retrieveConfigurationFromDisk(path string) (ma
 		return make(map[string]interface{}), nil
 	}
 
-	err = json.Unmarshal([]byte(raw), &config)
+	err = json.Unmarshal(raw, &config)
 	if err != nil {
 		return nil, err
 	}

api/filesystem/filesystem.go

@@ -6,7 +6,8 @@ import (
 	"encoding/pem"
 	"io/ioutil"
 
-	"github.com/portainer/portainer"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/archive"
 
 	"io"
 	"os"
@@ -32,6 +33,13 @@ const (
 	PrivateKeyFile = "portainer.key"
 	// PublicKeyFile represents the name on disk of the file containing the public key.
 	PublicKeyFile = "portainer.pub"
+	// BinaryStorePath represents the subfolder where binaries are stored in the file store folder.
+	BinaryStorePath = "bin"
+	// ScheduleStorePath represents the subfolder where schedule files are stored.
+	ScheduleStorePath = "schedules"
+	// ExtensionRegistryManagementStorePath represents the subfolder where files related to the
+	// registry management extension are stored.
+	ExtensionRegistryManagementStorePath = "extensions"
 )
 
 // Service represents a service for managing files and directories.
@@ -63,9 +71,30 @@ func NewService(dataStorePath, fileStorePath string) (*Service, error) {
 		return nil, err
 	}
 
+	err = service.createDirectoryInStore(BinaryStorePath)
+	if err != nil {
+		return nil, err
+	}
+
 	return service, nil
 }
 
+// GetBinaryFolder returns the full path to the binary store on the filesystem
+func (service *Service) GetBinaryFolder() string {
+	return path.Join(service.fileStorePath, BinaryStorePath)
+}
+
+// ExtractExtensionArchive extracts the content of an extension archive
+// specified as raw data into the binary store on the filesystem
+func (service *Service) ExtractExtensionArchive(data []byte) error {
+	err := archive.UnzipArchive(data, path.Join(service.fileStorePath, BinaryStorePath))
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // RemoveDirectory removes a directory on the filesystem.
 func (service *Service) RemoveDirectory(directoryPath string) error {
 	return os.RemoveAll(directoryPath)
@@ -97,6 +126,27 @@ func (service *Service) StoreStackFileFromBytes(stackIdentifier, fileName string
 	return path.Join(service.fileStorePath, stackStorePath), nil
 }
 
+// StoreRegistryManagementFileFromBytes creates a subfolder in the
+// ExtensionRegistryManagementStorePath and stores a new file from bytes.
+// It returns the path to the folder where the file is stored.
+func (service *Service) StoreRegistryManagementFileFromBytes(folder, fileName string, data []byte) (string, error) {
+	extensionStorePath := path.Join(ExtensionRegistryManagementStorePath, folder)
+	err := service.createDirectoryInStore(extensionStorePath)
+	if err != nil {
+		return "", err
+	}
+
+	file := path.Join(extensionStorePath, fileName)
+	r := bytes.NewReader(data)
+
+	err = service.createFileInStore(file, r)
+	if err != nil {
+		return "", err
+	}
+
+	return path.Join(service.fileStorePath, file), nil
+}
+
 // StoreTLSFileFromBytes creates a folder in the TLSStorePath and stores a new file from bytes.
 // It returns the path to the newly created file.
 func (service *Service) StoreTLSFileFromBytes(folder string, fileType portainer.TLSFileType, data []byte) (string, error) {
@@ -176,14 +226,14 @@ func (service *Service) DeleteTLSFile(folder string, fileType portainer.TLSFileT
 	return nil
 }
 
-// GetFileContent returns a string content from file.
-func (service *Service) GetFileContent(filePath string) (string, error) {
+// GetFileContent returns the content of a file as bytes.
+func (service *Service) GetFileContent(filePath string) ([]byte, error) {
 	content, err := ioutil.ReadFile(filePath)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 
-	return string(content), nil
+	return content, nil
 }
 
 // Rename renames a file or directory
@@ -318,3 +368,32 @@ func (service *Service) getContentFromPEMFile(filePath string) ([]byte, error) {
 	block, _ := pem.Decode(fileContent)
 	return block.Bytes, nil
 }
+
+// GetScheduleFolder returns the absolute path on the filesystem for a schedule based
+// on its identifier.
+func (service *Service) GetScheduleFolder(identifier string) string {
+	return path.Join(service.fileStorePath, ScheduleStorePath, identifier)
+}
+
+// StoreScheduledJobFileFromBytes creates a subfolder in the ScheduleStorePath and stores a new file from bytes.
+// It returns the path to the folder where the file is stored.
+func (service *Service) StoreScheduledJobFileFromBytes(identifier string, data []byte) (string, error) {
+	scheduleStorePath := path.Join(ScheduleStorePath, identifier)
+	err := service.createDirectoryInStore(scheduleStorePath)
+	if err != nil {
+		return "", err
+	}
+
+	filePath := path.Join(scheduleStorePath, createScheduledJobFileName(identifier))
+	r := bytes.NewReader(data)
+	err = service.createFileInStore(filePath, r)
+	if err != nil {
+		return "", err
+	}
+
+	return path.Join(service.fileStorePath, filePath), nil
+}
+
+func createScheduledJobFileName(identifier string) string {
+	return "job_" + identifier + ".sh"
+}

api/git/git.go

@@ -1,39 +1,62 @@
 package git
 
 import (
+	"crypto/tls"
+	"net/http"
 	"net/url"
 	"strings"
+	"time"
 
 	"gopkg.in/src-d/go-git.v4"
+	"gopkg.in/src-d/go-git.v4/plumbing"
+	"gopkg.in/src-d/go-git.v4/plumbing/transport/client"
+	githttp "gopkg.in/src-d/go-git.v4/plumbing/transport/http"
 )
 
 // Service represents a service for managing Git.
-type Service struct{}
+type Service struct {
+	httpsCli *http.Client
+}
 
 // NewService initializes a new service.
-func NewService(dataStorePath string) (*Service, error) {
-	service := &Service{}
+func NewService() *Service {
+	httpsCli := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		},
+		Timeout: 300 * time.Second,
+	}
 
-	return service, nil
+	client.InstallProtocol("https", githttp.NewClient(httpsCli))
+
+	return &Service{
+		httpsCli: httpsCli,
+	}
 }
 
 // ClonePublicRepository clones a public git repository using the specified URL in the specified
 // destination folder.
-func (service *Service) ClonePublicRepository(repositoryURL, destination string) error {
-	return cloneRepository(repositoryURL, destination)
+func (service *Service) ClonePublicRepository(repositoryURL, referenceName string, destination string) error {
+	return cloneRepository(repositoryURL, referenceName, destination)
 }
 
 // ClonePrivateRepositoryWithBasicAuth clones a private git repository using the specified URL in the specified
 // destination folder. It will use the specified username and password for basic HTTP authentication.
-func (service *Service) ClonePrivateRepositoryWithBasicAuth(repositoryURL, destination, username, password string) error {
+func (service *Service) ClonePrivateRepositoryWithBasicAuth(repositoryURL, referenceName string, destination, username, password string) error {
 	credentials := username + ":" + url.PathEscape(password)
 	repositoryURL = strings.Replace(repositoryURL, "://", "://"+credentials+"@", 1)
-	return cloneRepository(repositoryURL, destination)
+	return cloneRepository(repositoryURL, referenceName, destination)
 }
 
-func cloneRepository(repositoryURL, destination string) error {
-	_, err := git.PlainClone(destination, false, &git.CloneOptions{
+func cloneRepository(repositoryURL, referenceName, destination string) error {
+	options := &git.CloneOptions{
 		URL: repositoryURL,
-	})
+	}
+
+	if referenceName != "" {
+		options.ReferenceName = plumbing.ReferenceName(referenceName)
+	}
+
+	_, err := git.PlainClone(destination, false, options)
 	return err
 }

api/http/client/client.go

@@ -4,12 +4,18 @@ import (
 	"crypto/tls"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 	"strings"
 	"time"
 
-	"github.com/portainer/portainer"
+	"github.com/portainer/portainer/api"
+)
+
+const (
+	errInvalidResponseStatus = portainer.Error("Invalid response status (expecting 200)")
+	defaultHTTPTimeout       = 5
 )
 
 // HTTPClient represents a client to send HTTP requests.
@@ -21,7 +27,7 @@ type HTTPClient struct {
 func NewHTTPClient() *HTTPClient {
 	return &HTTPClient{
 		&http.Client{
-			Timeout: time.Second * 5,
+			Timeout: time.Second * time.Duration(defaultHTTPTimeout),
 		},
 	}
 }
@@ -61,6 +67,37 @@ func (client *HTTPClient) ExecuteAzureAuthenticationRequest(credentials *portain
 	return &token, nil
 }
 
+// Get executes a simple HTTP GET to the specified URL and returns
+// the content of the response body. Timeout can be specified via the timeout parameter,
+// will default to defaultHTTPTimeout if set to 0.
+func Get(url string, timeout int) ([]byte, error) {
+
+	if timeout == 0 {
+		timeout = defaultHTTPTimeout
+	}
+
+	client := &http.Client{
+		Timeout: time.Second * time.Duration(timeout),
+	}
+
+	response, err := client.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	defer response.Body.Close()
+
+	if response.StatusCode != http.StatusOK {
+		return nil, errInvalidResponseStatus
+	}
+
+	body, err := ioutil.ReadAll(response.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	return body, nil
+}
+
 // ExecutePingOperation will send a SystemPing operation HTTP request to a Docker environment
 // using the specified host and optional TLS configuration.
 // It uses a new Http.Client for each operation.

api/http/error/error.go

@@ -1,41 +0,0 @@
-package error
-
-import (
-	"encoding/json"
-	"log"
-	"net/http"
-)
-
-type (
-	// LoggerHandler defines a HTTP handler that includes a HandlerError return pointer
-	LoggerHandler func(http.ResponseWriter, *http.Request) *HandlerError
-	// HandlerError represents an error raised inside a HTTP handler
-	HandlerError struct {
-		StatusCode int
-		Message    string
-		Err        error
-	}
-	errorResponse struct {
-		Err string `json:"err,omitempty"`
-	}
-)
-
-func (handler LoggerHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
-	err := handler(rw, r)
-	if err != nil {
-		writeErrorResponse(rw, err)
-	}
-}
-
-func writeErrorResponse(rw http.ResponseWriter, err *HandlerError) {
-	log.Printf("http error: %s (err=%s) (code=%d)\n", err.Message, err.Err, err.StatusCode)
-	rw.Header().Set("Content-Type", "application/json")
-	rw.WriteHeader(err.StatusCode)
-	json.NewEncoder(rw).Encode(&errorResponse{Err: err.Message})
-}
-
-// WriteError is a convenience function that creates a new HandlerError before calling writeErrorResponse.
-// For use outside of the standard http handlers.
-func WriteError(rw http.ResponseWriter, code int, message string, err error) {
-	writeErrorResponse(rw, &HandlerError{code, message, err})
-}

api/http/handler/auth/authenticate.go

@@ -1,13 +1,15 @@
 package auth
 
 import (
+	"log"
 	"net/http"
+	"strings"
 
 	"github.com/asaskevich/govalidator"
-	"github.com/portainer/portainer"
-	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/request"
-	"github.com/portainer/portainer/http/response"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	"github.com/portainer/portainer/api"
 )
 
 type authenticatePayload struct {
@@ -40,36 +42,91 @@ func (handler *Handler) authenticate(w http.ResponseWriter, r *http.Request) *ht
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
 	}
 
-	u, err := handler.UserService.UserByUsername(payload.Username)
-	if err == portainer.ErrObjectNotFound {
-		return &httperror.HandlerError{http.StatusBadRequest, "Invalid credentials", ErrInvalidCredentials}
-	} else if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a user with the specified username from the database", err}
-	}
-
 	settings, err := handler.SettingsService.Settings()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve settings from the database", err}
 	}
 
-	if settings.AuthenticationMethod == portainer.AuthenticationLDAP && u.ID != 1 {
-		err = handler.LDAPService.AuthenticateUser(payload.Username, payload.Password, &settings.LDAPSettings)
-		if err != nil {
-			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to authenticate user via LDAP/AD", err}
-		}
-	} else {
-		err = handler.CryptoService.CompareHashAndData(u.Password, payload.Password)
-		if err != nil {
-			return &httperror.HandlerError{http.StatusUnprocessableEntity, "Invalid credentials", ErrInvalidCredentials}
-		}
+	u, err := handler.UserService.UserByUsername(payload.Username)
+	if err != nil && err != portainer.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a user with the specified username from the database", err}
 	}
 
+	if err == portainer.ErrObjectNotFound && (settings.AuthenticationMethod == portainer.AuthenticationInternal || settings.AuthenticationMethod == portainer.AuthenticationOAuth) {
+		return &httperror.HandlerError{http.StatusUnprocessableEntity, "Invalid credentials", portainer.ErrUnauthorized}
+	}
+
+	if settings.AuthenticationMethod == portainer.AuthenticationLDAP {
+		if u == nil && settings.LDAPSettings.AutoCreateUsers {
+			return handler.authenticateLDAPAndCreateUser(w, payload.Username, payload.Password, &settings.LDAPSettings)
+		} else if u == nil && !settings.LDAPSettings.AutoCreateUsers {
+			return &httperror.HandlerError{http.StatusUnprocessableEntity, "Invalid credentials", portainer.ErrUnauthorized}
+		}
+		return handler.authenticateLDAP(w, u, payload.Password, &settings.LDAPSettings)
+	}
+
+	return handler.authenticateInternal(w, u, payload.Password)
+}
+
+func (handler *Handler) authenticateLDAP(w http.ResponseWriter, user *portainer.User, password string, ldapSettings *portainer.LDAPSettings) *httperror.HandlerError {
+	err := handler.LDAPService.AuthenticateUser(user.Username, password, ldapSettings)
+	if err != nil {
+		return handler.authenticateInternal(w, user, password)
+	}
+
+	err = handler.addUserIntoTeams(user, ldapSettings)
+	if err != nil {
+		log.Printf("Warning: unable to automatically add user into teams: %s\n", err.Error())
+	}
+
+	return handler.writeToken(w, user)
+}
+
+func (handler *Handler) authenticateInternal(w http.ResponseWriter, user *portainer.User, password string) *httperror.HandlerError {
+	err := handler.CryptoService.CompareHashAndData(user.Password, password)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusUnprocessableEntity, "Invalid credentials", portainer.ErrUnauthorized}
+	}
+
+	return handler.writeToken(w, user)
+}
+
+func (handler *Handler) authenticateLDAPAndCreateUser(w http.ResponseWriter, username, password string, ldapSettings *portainer.LDAPSettings) *httperror.HandlerError {
+	err := handler.LDAPService.AuthenticateUser(username, password, ldapSettings)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusUnprocessableEntity, "Invalid credentials", err}
+	}
+
+	user := &portainer.User{
+		Username:                username,
+		Role:                    portainer.StandardUserRole,
+		PortainerAuthorizations: portainer.DefaultPortainerAuthorizations(),
+	}
+
+	err = handler.UserService.CreateUser(user)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist user inside the database", err}
+	}
+
+	err = handler.addUserIntoTeams(user, ldapSettings)
+	if err != nil {
+		log.Printf("Warning: unable to automatically add user into teams: %s\n", err.Error())
+	}
+
+	return handler.writeToken(w, user)
+}
+
+func (handler *Handler) writeToken(w http.ResponseWriter, user *portainer.User) *httperror.HandlerError {
 	tokenData := &portainer.TokenData{
-		ID:       u.ID,
-		Username: u.Username,
-		Role:     u.Role,
+		ID:       user.ID,
+		Username: user.Username,
+		Role:     user.Role,
 	}
 
+	return handler.persistAndWriteToken(w, tokenData)
+}
+
+func (handler *Handler) persistAndWriteToken(w http.ResponseWriter, tokenData *portainer.TokenData) *httperror.HandlerError {
 	token, err := handler.JWTService.GenerateToken(tokenData)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to generate JWT token", err}
@@ -77,3 +134,59 @@ func (handler *Handler) authenticate(w http.ResponseWriter, r *http.Request) *ht
 
 	return response.JSON(w, &authenticateResponse{JWT: token})
 }
+
+func (handler *Handler) addUserIntoTeams(user *portainer.User, settings *portainer.LDAPSettings) error {
+	teams, err := handler.TeamService.Teams()
+	if err != nil {
+		return err
+	}
+
+	userGroups, err := handler.LDAPService.GetUserGroups(user.Username, settings)
+	if err != nil {
+		return err
+	}
+
+	userMemberships, err := handler.TeamMembershipService.TeamMembershipsByUserID(user.ID)
+	if err != nil {
+		return err
+	}
+
+	for _, team := range teams {
+		if teamExists(team.Name, userGroups) {
+
+			if teamMembershipExists(team.ID, userMemberships) {
+				continue
+			}
+
+			membership := &portainer.TeamMembership{
+				UserID: user.ID,
+				TeamID: team.ID,
+				Role:   portainer.TeamMember,
+			}
+
+			err := handler.TeamMembershipService.CreateTeamMembership(membership)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func teamExists(teamName string, ldapGroups []string) bool {
+	for _, group := range ldapGroups {
+		if strings.ToLower(group) == strings.ToLower(teamName) {
+			return true
+		}
+	}
+	return false
+}
+
+func teamMembershipExists(teamID portainer.TeamID, memberships []portainer.TeamMembership) bool {
+	for _, membership := range memberships {
+		if membership.TeamID == teamID {
+			return true
+		}
+	}
+	return false
+}

api/http/handler/auth/authenticate_oauth.go

@@ -0,0 +1,139 @@
+package auth
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"log"
+	"net/http"
+
+	"github.com/asaskevich/govalidator"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/portainer/api"
+)
+
+type oauthPayload struct {
+	Code string
+}
+
+func (payload *oauthPayload) Validate(r *http.Request) error {
+	if govalidator.IsNull(payload.Code) {
+		return portainer.Error("Invalid OAuth authorization code")
+	}
+	return nil
+}
+
+func (handler *Handler) authenticateThroughExtension(code, licenseKey string, settings *portainer.OAuthSettings) (string, error) {
+	extensionURL := handler.ProxyManager.GetExtensionURL(portainer.OAuthAuthenticationExtension)
+
+	encodedConfiguration, err := json.Marshal(settings)
+	if err != nil {
+		return "", nil
+	}
+
+	req, err := http.NewRequest("GET", extensionURL+"/validate", nil)
+	if err != nil {
+		return "", err
+	}
+
+	client := &http.Client{}
+	req.Header.Set("X-OAuth-Config", string(encodedConfiguration))
+	req.Header.Set("X-OAuth-Code", code)
+	req.Header.Set("X-PortainerExtension-License", licenseKey)
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", err
+	}
+
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+
+	type extensionResponse struct {
+		Username string `json:"Username,omitempty"`
+		Err      string `json:"err,omitempty"`
+		Details  string `json:"details,omitempty"`
+	}
+
+	var extResp extensionResponse
+	err = json.Unmarshal(body, &extResp)
+	if err != nil {
+		return "", err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return "", portainer.Error(extResp.Err + ":" + extResp.Details)
+	}
+
+	return extResp.Username, nil
+}
+
+func (handler *Handler) validateOAuth(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	var payload oauthPayload
+	err := request.DecodeAndValidateJSONPayload(r, &payload)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
+	}
+
+	settings, err := handler.SettingsService.Settings()
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve settings from the database", err}
+	}
+
+	if settings.AuthenticationMethod != 3 {
+		return &httperror.HandlerError{http.StatusForbidden, "OAuth authentication is not enabled", portainer.Error("OAuth authentication is not enabled")}
+	}
+
+	extension, err := handler.ExtensionService.Extension(portainer.OAuthAuthenticationExtension)
+	if err == portainer.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusNotFound, "Oauth authentication extension is not enabled", err}
+	} else if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a extension with the specified identifier inside the database", err}
+	}
+
+	username, err := handler.authenticateThroughExtension(payload.Code, extension.License.LicenseKey, &settings.OAuthSettings)
+	if err != nil {
+		log.Printf("[DEBUG] - OAuth authentication error: %s", err)
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to authenticate through OAuth", portainer.ErrUnauthorized}
+	}
+
+	user, err := handler.UserService.UserByUsername(username)
+	if err != nil && err != portainer.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a user with the specified username from the database", err}
+	}
+
+	if user == nil && !settings.OAuthSettings.OAuthAutoCreateUsers {
+		return &httperror.HandlerError{http.StatusForbidden, "Account not created beforehand in Portainer and automatic user provisioning not enabled", portainer.ErrUnauthorized}
+	}
+
+	if user == nil {
+		user = &portainer.User{
+			Username:                username,
+			Role:                    portainer.StandardUserRole,
+			PortainerAuthorizations: portainer.DefaultPortainerAuthorizations(),
+		}
+
+		err = handler.UserService.CreateUser(user)
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist user inside the database", err}
+		}
+
+		if settings.OAuthSettings.DefaultTeamID != 0 {
+			membership := &portainer.TeamMembership{
+				UserID: user.ID,
+				TeamID: settings.OAuthSettings.DefaultTeamID,
+				Role:   portainer.TeamMember,
+			}
+
+			err = handler.TeamMembershipService.CreateTeamMembership(membership)
+			if err != nil {
+				return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist team membership inside the database", err}
+			}
+		}
+	}
+
+	return handler.writeToken(w, user)
+}

api/http/handler/auth/handler.go

@@ -4,9 +4,10 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
-	"github.com/portainer/portainer"
-	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/security"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/http/proxy"
+	"github.com/portainer/portainer/api/http/security"
 )
 
 const (
@@ -20,12 +21,19 @@ const (
 // Handler is the HTTP handler used to handle authentication operations.
 type Handler struct {
 	*mux.Router
-	authDisabled    bool
-	UserService     portainer.UserService
-	CryptoService   portainer.CryptoService
-	JWTService      portainer.JWTService
-	LDAPService     portainer.LDAPService
-	SettingsService portainer.SettingsService
+	authDisabled          bool
+	UserService           portainer.UserService
+	CryptoService         portainer.CryptoService
+	JWTService            portainer.JWTService
+	LDAPService           portainer.LDAPService
+	SettingsService       portainer.SettingsService
+	TeamService           portainer.TeamService
+	TeamMembershipService portainer.TeamMembershipService
+	ExtensionService      portainer.ExtensionService
+	EndpointService       portainer.EndpointService
+	EndpointGroupService  portainer.EndpointGroupService
+	RoleService           portainer.RoleService
+	ProxyManager          *proxy.Manager
 }
 
 // NewHandler creates a handler to manage authentication operations.
@@ -34,6 +42,9 @@ func NewHandler(bouncer *security.RequestBouncer, rateLimiter *security.RateLimi
 		Router:       mux.NewRouter(),
 		authDisabled: authDisabled,
 	}
+
+	h.Handle("/auth/oauth/validate",
+		rateLimiter.LimitAccess(bouncer.PublicAccess(httperror.LoggerHandler(h.validateOAuth)))).Methods(http.MethodPost)
 	h.Handle("/auth",
 		rateLimiter.LimitAccess(bouncer.PublicAccess(httperror.LoggerHandler(h.authenticate)))).Methods(http.MethodPost)
 

api/http/handler/dockerhub/dockerhub_inspect.go

@@ -3,8 +3,8 @@ package dockerhub
 import (
 	"net/http"
 
-	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/response"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/response"
 )
 
 // GET request on /api/dockerhub

api/http/handler/dockerhub/dockerhub_update.go

@@ -4,10 +4,10 @@ import (
 	"net/http"
 
 	"github.com/asaskevich/govalidator"
-	"github.com/portainer/portainer"
-	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/request"
-	"github.com/portainer/portainer/http/response"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	"github.com/portainer/portainer/api"
 )
 
 type dockerhubUpdatePayload struct {

api/http/handler/dockerhub/handler.go

@@ -4,9 +4,9 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
-	"github.com/portainer/portainer"
-	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/security"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/http/security"
 )
 
 func hideFields(dockerHub *portainer.DockerHub) {
@@ -25,9 +25,9 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler {
 		Router: mux.NewRouter(),
 	}
 	h.Handle("/dockerhub",
-		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.dockerhubInspect))).Methods(http.MethodGet)
+		bouncer.RestrictedAccess(httperror.LoggerHandler(h.dockerhubInspect))).Methods(http.MethodGet)
 	h.Handle("/dockerhub",
-		bouncer.AdministratorAccess(httperror.LoggerHandler(h.dockerhubUpdate))).Methods(http.MethodPut)
+		bouncer.AdminAccess(httperror.LoggerHandler(h.dockerhubUpdate))).Methods(http.MethodPut)
 
 	return h
 }

api/http/handler/endpointgroups/endpointgroup_create.go

@@ -4,10 +4,10 @@ import (
 	"net/http"
 
 	"github.com/asaskevich/govalidator"
-	"github.com/portainer/portainer"
-	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/request"
-	"github.com/portainer/portainer/http/response"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	"github.com/portainer/portainer/api"
 )
 
 type endpointGroupCreatePayload struct {
@@ -36,11 +36,11 @@ func (handler *Handler) endpointGroupCreate(w http.ResponseWriter, r *http.Reque
 	}
 
 	endpointGroup := &portainer.EndpointGroup{
-		Name:            payload.Name,
-		Description:     payload.Description,
-		AuthorizedUsers: []portainer.UserID{},
-		AuthorizedTeams: []portainer.TeamID{},
-		Tags:            payload.Tags,
+		Name:               payload.Name,
+		Description:        payload.Description,
+		UserAccessPolicies: portainer.UserAccessPolicies{},
+		TeamAccessPolicies: portainer.TeamAccessPolicies{},
+		Tags:               payload.Tags,
 	}
 
 	err = handler.EndpointGroupService.CreateEndpointGroup(endpointGroup)
@@ -53,11 +53,17 @@ func (handler *Handler) endpointGroupCreate(w http.ResponseWriter, r *http.Reque
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoints from the database", err}
 	}
 
-	for _, endpoint := range endpoints {
-		if endpoint.GroupID == portainer.EndpointGroupID(1) {
-			err = handler.checkForGroupAssignment(endpoint, endpointGroup.ID, payload.AssociatedEndpoints)
-			if err != nil {
-				return &httperror.HandlerError{http.StatusInternalServerError, "Unable to update endpoint", err}
+	for _, id := range payload.AssociatedEndpoints {
+		for _, endpoint := range endpoints {
+			if endpoint.ID == id {
+				endpoint.GroupID = endpointGroup.ID
+
+				err := handler.EndpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+				if err != nil {
+					return &httperror.HandlerError{http.StatusInternalServerError, "Unable to update endpoint", err}
+				}
+
+				break
 			}
 		}
 	}

api/http/handler/endpointgroups/endpointgroup_delete.go

@@ -3,10 +3,10 @@ package endpointgroups
 import (
 	"net/http"
 
-	"github.com/portainer/portainer"
-	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/request"
-	"github.com/portainer/portainer/http/response"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	"github.com/portainer/portainer/api"
 )
 
 // DELETE request on /api/endpoint_groups/:id
@@ -37,8 +37,10 @@ func (handler *Handler) endpointGroupDelete(w http.ResponseWriter, r *http.Reque
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoints from the database", err}
 	}
 
+	updateAuthorizations := false
 	for _, endpoint := range endpoints {
 		if endpoint.GroupID == portainer.EndpointGroupID(endpointGroupID) {
+			updateAuthorizations = true
 			endpoint.GroupID = portainer.EndpointGroupID(1)
 			err = handler.EndpointService.UpdateEndpoint(endpoint.ID, &endpoint)
 			if err != nil {
@@ -47,5 +49,12 @@ func (handler *Handler) endpointGroupDelete(w http.ResponseWriter, r *http.Reque
 		}
 	}
 
+	if updateAuthorizations {
+		err = handler.AuthorizationService.UpdateUsersAuthorizations()
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to update user authorizations", err}
+		}
+	}
+
 	return response.Empty(w)
 }

api/http/handler/endpointgroups/endpointgroup_endpoint_add.go

@@ -0,0 +1,46 @@
+package endpointgroups
+
+import (
+	"net/http"
+
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	"github.com/portainer/portainer/api"
+)
+
+// PUT request on /api/endpoint_groups/:id/endpoints/:endpointId
+func (handler *Handler) endpointGroupAddEndpoint(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	endpointGroupID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint group identifier route variable", err}
+	}
+
+	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "endpointId")
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint identifier route variable", err}
+	}
+
+	endpointGroup, err := handler.EndpointGroupService.EndpointGroup(portainer.EndpointGroupID(endpointGroupID))
+	if err == portainer.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint group with the specified identifier inside the database", err}
+	} else if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint group with the specified identifier inside the database", err}
+	}
+
+	endpoint, err := handler.EndpointService.Endpoint(portainer.EndpointID(endpointID))
+	if err == portainer.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
+	} else if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint with the specified identifier inside the database", err}
+	}
+
+	endpoint.GroupID = endpointGroup.ID
+
+	err = handler.EndpointService.UpdateEndpoint(endpoint.ID, endpoint)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint changes inside the database", err}
+	}
+
+	return response.Empty(w)
+}

api/http/handler/endpointgroups/endpointgroup_endpoint_delete.go

@@ -0,0 +1,46 @@
+package endpointgroups
+
+import (
+	"net/http"
+
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	"github.com/portainer/portainer/api"
+)
+
+// DELETE request on /api/endpoint_groups/:id/endpoints/:endpointId
+func (handler *Handler) endpointGroupDeleteEndpoint(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	endpointGroupID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint group identifier route variable", err}
+	}
+
+	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "endpointId")
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint identifier route variable", err}
+	}
+
+	_, err = handler.EndpointGroupService.EndpointGroup(portainer.EndpointGroupID(endpointGroupID))
+	if err == portainer.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint group with the specified identifier inside the database", err}
+	} else if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint group with the specified identifier inside the database", err}
+	}
+
+	endpoint, err := handler.EndpointService.Endpoint(portainer.EndpointID(endpointID))
+	if err == portainer.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
+	} else if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint with the specified identifier inside the database", err}
+	}
+
+	endpoint.GroupID = portainer.EndpointGroupID(1)
+
+	err = handler.EndpointService.UpdateEndpoint(endpoint.ID, endpoint)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint changes inside the database", err}
+	}
+
+	return response.Empty(w)
+}

api/http/handler/endpointgroups/endpointgroup_inspect.go

@@ -3,10 +3,10 @@ package endpointgroups
 import (
 	"net/http"
 
-	"github.com/portainer/portainer"
-	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/request"
-	"github.com/portainer/portainer/http/response"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	"github.com/portainer/portainer/api"
 )
 
 // GET request on /api/endpoint_groups/:id

api/http/handler/endpointgroups/endpointgroup_list.go

@@ -3,9 +3,9 @@ package endpointgroups
 import (
 	"net/http"
 
-	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/response"
-	"github.com/portainer/portainer/http/security"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/response"
+	"github.com/portainer/portainer/api/http/security"
 )
 
 // GET request on /api/endpoint_groups

api/http/handler/endpointgroups/endpointgroup_update.go

@@ -2,18 +2,20 @@ package endpointgroups
 
 import (
 	"net/http"
+	"reflect"
 
-	"github.com/portainer/portainer"
-	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/request"
-	"github.com/portainer/portainer/http/response"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	"github.com/portainer/portainer/api"
 )
 
 type endpointGroupUpdatePayload struct {
-	Name                string
-	Description         string
-	AssociatedEndpoints []portainer.EndpointID
-	Tags                []string
+	Name               string
+	Description        string
+	Tags               []string
+	UserAccessPolicies portainer.UserAccessPolicies
+	TeamAccessPolicies portainer.TeamAccessPolicies
 }
 
 func (payload *endpointGroupUpdatePayload) Validate(r *http.Request) error {
@@ -52,20 +54,26 @@ func (handler *Handler) endpointGroupUpdate(w http.ResponseWriter, r *http.Reque
 		endpointGroup.Tags = payload.Tags
 	}
 
+	updateAuthorizations := false
+	if payload.UserAccessPolicies != nil && !reflect.DeepEqual(payload.UserAccessPolicies, endpointGroup.UserAccessPolicies) {
+		endpointGroup.UserAccessPolicies = payload.UserAccessPolicies
+		updateAuthorizations = true
+	}
+
+	if payload.TeamAccessPolicies != nil && !reflect.DeepEqual(payload.TeamAccessPolicies, endpointGroup.TeamAccessPolicies) {
+		endpointGroup.TeamAccessPolicies = payload.TeamAccessPolicies
+		updateAuthorizations = true
+	}
+
 	err = handler.EndpointGroupService.UpdateEndpointGroup(endpointGroup.ID, endpointGroup)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint group changes inside the database", err}
 	}
 
-	endpoints, err := handler.EndpointService.Endpoints()
-	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoints from the database", err}
-	}
-
-	for _, endpoint := range endpoints {
-		err = handler.updateEndpointGroup(endpoint, portainer.EndpointGroupID(endpointGroupID), payload.AssociatedEndpoints)
+	if updateAuthorizations {
+		err = handler.AuthorizationService.UpdateUsersAuthorizations()
 		if err != nil {
-			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to update endpoint", err}
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to update user authorizations", err}
 		}
 	}
 

api/http/handler/endpointgroups/endpointgroup_update_access.go

@@ -1,63 +0,0 @@
-package endpointgroups
-
-import (
-	"net/http"
-
-	"github.com/portainer/portainer"
-	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/request"
-	"github.com/portainer/portainer/http/response"
-)
-
-type endpointGroupUpdateAccessPayload struct {
-	AuthorizedUsers []int
-	AuthorizedTeams []int
-}
-
-func (payload *endpointGroupUpdateAccessPayload) Validate(r *http.Request) error {
-	return nil
-}
-
-// PUT request on /api/endpoint_groups/:id/access
-func (handler *Handler) endpointGroupUpdateAccess(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	endpointGroupID, err := request.RetrieveNumericRouteVariableValue(r, "id")
-	if err != nil {
-		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint group identifier route variable", err}
-	}
-
-	var payload endpointGroupUpdateAccessPayload
-	err = request.DecodeAndValidateJSONPayload(r, &payload)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
-	}
-
-	endpointGroup, err := handler.EndpointGroupService.EndpointGroup(portainer.EndpointGroupID(endpointGroupID))
-	if err == portainer.ErrObjectNotFound {
-		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint group with the specified identifier inside the database", err}
-	} else if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint group with the specified identifier inside the database", err}
-	}
-
-	if payload.AuthorizedUsers != nil {
-		authorizedUserIDs := []portainer.UserID{}
-		for _, value := range payload.AuthorizedUsers {
-			authorizedUserIDs = append(authorizedUserIDs, portainer.UserID(value))
-		}
-		endpointGroup.AuthorizedUsers = authorizedUserIDs
-	}
-
-	if payload.AuthorizedTeams != nil {
-		authorizedTeamIDs := []portainer.TeamID{}
-		for _, value := range payload.AuthorizedTeams {
-			authorizedTeamIDs = append(authorizedTeamIDs, portainer.TeamID(value))
-		}
-		endpointGroup.AuthorizedTeams = authorizedTeamIDs
-	}
-
-	err = handler.EndpointGroupService.UpdateEndpointGroup(endpointGroup.ID, endpointGroup)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint group changes inside the database", err}
-	}
-
-	return response.JSON(w, endpointGroup)
-}

api/http/handler/endpointgroups/handler.go

@@ -4,9 +4,9 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
-	"github.com/portainer/portainer"
-	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/security"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/http/security"
 )
 
 // Handler is the HTTP handler used to handle endpoint group operations.
@@ -14,6 +14,7 @@ type Handler struct {
 	*mux.Router
 	EndpointService      portainer.EndpointService
 	EndpointGroupService portainer.EndpointGroupService
+	AuthorizationService *portainer.AuthorizationService
 }
 
 // NewHandler creates a handler to manage endpoint group operations.
@@ -22,48 +23,18 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler {
 		Router: mux.NewRouter(),
 	}
 	h.Handle("/endpoint_groups",
-		bouncer.AdministratorAccess(httperror.LoggerHandler(h.endpointGroupCreate))).Methods(http.MethodPost)
+		bouncer.AdminAccess(httperror.LoggerHandler(h.endpointGroupCreate))).Methods(http.MethodPost)
 	h.Handle("/endpoint_groups",
 		bouncer.RestrictedAccess(httperror.LoggerHandler(h.endpointGroupList))).Methods(http.MethodGet)
 	h.Handle("/endpoint_groups/{id}",
-		bouncer.AdministratorAccess(httperror.LoggerHandler(h.endpointGroupInspect))).Methods(http.MethodGet)
+		bouncer.AdminAccess(httperror.LoggerHandler(h.endpointGroupInspect))).Methods(http.MethodGet)
 	h.Handle("/endpoint_groups/{id}",
-		bouncer.AdministratorAccess(httperror.LoggerHandler(h.endpointGroupUpdate))).Methods(http.MethodPut)
-	h.Handle("/endpoint_groups/{id}/access",
-		bouncer.AdministratorAccess(httperror.LoggerHandler(h.endpointGroupUpdateAccess))).Methods(http.MethodPut)
+		bouncer.AdminAccess(httperror.LoggerHandler(h.endpointGroupUpdate))).Methods(http.MethodPut)
 	h.Handle("/endpoint_groups/{id}",
-		bouncer.AdministratorAccess(httperror.LoggerHandler(h.endpointGroupDelete))).Methods(http.MethodDelete)
-
+		bouncer.AdminAccess(httperror.LoggerHandler(h.endpointGroupDelete))).Methods(http.MethodDelete)
+	h.Handle("/endpoint_groups/{id}/endpoints/{endpointId}",
+		bouncer.AdminAccess(httperror.LoggerHandler(h.endpointGroupAddEndpoint))).Methods(http.MethodPut)
+	h.Handle("/endpoint_groups/{id}/endpoints/{endpointId}",
+		bouncer.AdminAccess(httperror.LoggerHandler(h.endpointGroupDeleteEndpoint))).Methods(http.MethodDelete)
 	return h
 }
-
-func (handler *Handler) checkForGroupUnassignment(endpoint portainer.Endpoint, associatedEndpoints []portainer.EndpointID) error {
-	for _, id := range associatedEndpoints {
-		if id == endpoint.ID {
-			return nil
-		}
-	}
-
-	endpoint.GroupID = portainer.EndpointGroupID(1)
-	return handler.EndpointService.UpdateEndpoint(endpoint.ID, &endpoint)
-}
-
-func (handler *Handler) checkForGroupAssignment(endpoint portainer.Endpoint, groupID portainer.EndpointGroupID, associatedEndpoints []portainer.EndpointID) error {
-	for _, id := range associatedEndpoints {
-
-		if id == endpoint.ID {
-			endpoint.GroupID = groupID
-			return handler.EndpointService.UpdateEndpoint(endpoint.ID, &endpoint)
-		}
-	}
-	return nil
-}
-
-func (handler *Handler) updateEndpointGroup(endpoint portainer.Endpoint, groupID portainer.EndpointGroupID, associatedEndpoints []portainer.EndpointID) error {
-	if endpoint.GroupID == groupID {
-		return handler.checkForGroupUnassignment(endpoint, associatedEndpoints)
-	} else if endpoint.GroupID == portainer.EndpointGroupID(1) {
-		return handler.checkForGroupAssignment(endpoint, groupID, associatedEndpoints)
-	}
-	return nil
-}

api/http/handler/endpointproxy/handler.go

@@ -2,18 +2,20 @@ package endpointproxy
 
 import (
 	"github.com/gorilla/mux"
-	"github.com/portainer/portainer"
-	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/proxy"
-	"github.com/portainer/portainer/http/security"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/http/proxy"
+	"github.com/portainer/portainer/api/http/security"
 )
 
 // Handler is the HTTP handler used to proxy requests to external APIs.
 type Handler struct {
 	*mux.Router
-	requestBouncer  *security.RequestBouncer
-	EndpointService portainer.EndpointService
-	ProxyManager    *proxy.Manager
+	requestBouncer       *security.RequestBouncer
+	EndpointService      portainer.EndpointService
+	SettingsService      portainer.SettingsService
+	ProxyManager         *proxy.Manager
+	ReverseTunnelService portainer.ReverseTunnelService
 }
 
 // NewHandler creates a handler to proxy requests to external APIs.
@@ -26,7 +28,7 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler {
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.proxyRequestsToAzureAPI)))
 	h.PathPrefix("/{id}/docker").Handler(
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.proxyRequestsToDockerAPI)))
-	h.PathPrefix("/{id}/extensions/storidge").Handler(
+	h.PathPrefix("/{id}/storidge").Handler(
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.proxyRequestsToStoridgeAPI)))
 	return h
 }