feat(edge): update deployment instructions

Release 1.21.0

.babelrc

@@ -0,0 +1,12 @@
+{
+  "plugins": ["lodash", "angularjs-annotate"],
+  "presets": [
+    [
+      "@babel/preset-env",
+      {
+        "modules": false,
+        "useBuiltIns": "usage"
+      }
+    ]
+  ]
+}

.codeclimate.yml

@@ -1,5 +1,42 @@
----
-engines:
+version: "2"
+checks:
+  argument-count:
+    enabled: true
+    config:
+      threshold: 4
+  complex-logic:
+    enabled: true
+    config:
+      threshold: 4
+  file-lines:
+    enabled: true
+    config:
+      threshold: 300
+  method-complexity:
+    enabled: false
+  method-count:
+    enabled: true
+    config:
+      threshold: 20
+  method-lines:
+    enabled: true
+    config:
+      threshold: 50
+  nested-control-flow:
+    enabled: true
+    config:
+      threshold: 4
+  return-statements:
+    enabled: false
+  similar-code:
+    enabled: true
+    config:
+      threshold: #language-specific defaults. overrides affect all languages.
+  identical-code:
+    enabled: true
+    config:
+      threshold: #language-specific defaults. overrides affect all languages.
+plugins:
   gofmt:
     enabled: true
   golint:
@@ -16,14 +53,10 @@ engines:
           mass_threshold: 80
   eslint:
     enabled: true
+    channel: "eslint-5"
     config:
       config: .eslintrc.yml
   fixme:
     enabled: true
-ratings:
-  paths:
-  - "**.css"
-  - "**.js"
-  - "**.go"
-exclude_paths:
+exclude_patterns:
 - test/

.codefresh/codefresh_branch.yml

@@ -1,46 +0,0 @@
-version: '1.0'
-steps:
-
-  build_backend:
-    image: portainer/golang-builder:ci
-    working_directory: ${{main_clone}}
-    commands:
-      - mkdir -p /go/src/github.com/${{CF_REPO_OWNER}}
-      - ln -s /codefresh/volume/${{CF_REPO_NAME}}/api /go/src/github.com/${{CF_REPO_OWNER}}/${{CF_REPO_NAME}}
-      - /build.sh api/cmd/portainer
-
-  build_frontend:
-    image: portainer/angular-builder:latest
-    working_directory: ${{build_backend}}
-    commands:
-      - yarn
-      - yarn grunt build-webapp
-      - mv api/cmd/portainer/portainer dist/
-
-  get_docker_version:
-    image: alpine
-    working_directory: ${{build_frontend}}
-    commands:
-      - cf_export DOCKER_VERSION=`cat gruntfile.js | grep -m 1 'shippedDockerVersion' | cut -d\' -f2`
-
-  download_docker_binary:
-    image: busybox
-    working_directory: ${{build_frontend}}
-    commands:
-      - echo ${{DOCKER_VERSION}}
-      - wget -O /tmp/docker-binaries.tgz https://download.docker.com/linux/static/stable/x86_64/docker-${{DOCKER_VERSION}}.tgz
-      - tar -xf /tmp/docker-binaries.tgz -C /tmp
-      - mv /tmp/docker/docker dist/
-
-  build_image:
-    type: build
-    working_directory: ${{download_docker_binary}}
-    dockerfile: ./build/linux/Dockerfile
-    image_name: portainer/portainer
-    tag: ${{CF_BRANCH}}
-
-  push_image:
-    type: push
-    candidate: '${{build_image}}'
-    tag: '${{CF_BRANCH}}'
-    registry: dockerhub

.codefresh/codefresh_pullrequest.yml

@@ -1,46 +0,0 @@
-version: '1.0'
-steps:
-
-  build_backend:
-    image: portainer/golang-builder:ci
-    working_directory: ${{main_clone}}
-    commands:
-      - mkdir -p /go/src/github.com/${{CF_REPO_OWNER}}
-      - ln -s /codefresh/volume/${{CF_REPO_NAME}}/api /go/src/github.com/${{CF_REPO_OWNER}}/${{CF_REPO_NAME}}
-      - /build.sh api/cmd/portainer
-
-  build_frontend:
-    image: portainer/angular-builder:latest
-    working_directory: ${{build_backend}}
-    commands:
-      - yarn
-      - yarn grunt build-webapp
-      - mv api/cmd/portainer/portainer dist/
-
-  get_docker_version:
-    image: alpine
-    working_directory: ${{build_frontend}}
-    commands:
-      - cf_export DOCKER_VERSION=`cat gruntfile.js | grep -m 1 'shippedDockerVersion' | cut -d\' -f2`
-
-  download_docker_binary:
-    image: busybox
-    working_directory: ${{build_frontend}}
-    commands:
-      - echo ${{DOCKER_VERSION}}
-      - wget -O /tmp/docker-binaries.tgz https://download.docker.com/linux/static/stable/x86_64/docker-${{DOCKER_VERSION}}.tgz
-      - tar -xf /tmp/docker-binaries.tgz -C /tmp
-      - mv /tmp/docker/docker dist/
-
-  build_image:
-    type: build
-    working_directory: ${{download_docker_binary}}
-    dockerfile: ./build/linux/Dockerfile
-    image_name: portainer/portainer
-    tag: ${{CF_BRANCH}}
-
-  push_image:
-    type: push
-    candidate: '${{build_image}}'
-    tag: 'pr${{CF_PULL_REQUEST_NUMBER}}'
-    registry: dockerhub

.eslintignore

@@ -0,0 +1,3 @@
+node_modules/
+dist/
+test/

.eslintrc.yml

@@ -1,284 +1,292 @@
 env:
   browser: true
   jquery: true
+  node: true
+  es6: true
+
+globals:
+  angular: true
+  __CONFIG_GA_ID: true
 
-# globals:
-#   angular: true
-#   $: true
-#   _: true
-#   moment: true
-#   filesize: true
-#   splitargs: true
 extends:
   - 'eslint:recommended'
 
-# http://eslint.org/docs/rules/
+parserOptions:
+  ecmaVersion: 2018
+  sourceType: module
+  ecmaFeatures:
+    modules: true
+
+# # http://eslint.org/docs/rules/
 rules:
-  # Possible Errors
-  no-await-in-loop: off
-  no-cond-assign: error
-  no-console: off
-  no-constant-condition: error
-  no-control-regex: error
-  no-debugger: error
-  no-dupe-args: error
-  no-dupe-keys: error
-  no-duplicate-case: error
-  no-empty-character-class: error
-  no-empty: error
-  no-ex-assign: error
-  no-extra-boolean-cast: error
-  no-extra-parens: off
-  no-extra-semi: error
-  no-func-assign: error
-  no-inner-declarations:
-    - error
-    - functions
-  no-invalid-regexp: error
-  no-irregular-whitespace: error
-  no-negated-in-lhs: error
-  no-obj-calls: error
-  no-prototype-builtins: off
-  no-regex-spaces: error
-  no-sparse-arrays: error
-  no-template-curly-in-string: off
-  no-unexpected-multiline: error
-  no-unreachable: error
-  no-unsafe-finally: off
-  no-unsafe-negation: off
-  use-isnan: error
-  valid-jsdoc: off
-  valid-typeof: error
+#   # Possible Errors
+#   no-await-in-loop: off
+#   no-cond-assign: error
+#   no-console: off
+#   no-constant-condition: error
+#   no-control-regex: error
+#   no-debugger: error
+#   no-dupe-args: error
+#   no-dupe-keys: error
+#   no-duplicate-case: error
+#   no-empty-character-class: error
+  no-empty: warn
+#   no-ex-assign: error
+#   no-extra-boolean-cast: error
+#   no-extra-parens: off
+#   no-extra-semi: error
+#   no-func-assign: error
+#   no-inner-declarations:
+#     - error
+#     - functions
+#   no-invalid-regexp: error
+#   no-irregular-whitespace: error
+#   no-negated-in-lhs: error
+#   no-obj-calls: error
+#   no-prototype-builtins: off
+#   no-regex-spaces: error
+#   no-sparse-arrays: error
+#   no-template-curly-in-string: off
+#   no-unexpected-multiline: error
+#   no-unreachable: error
+#   no-unsafe-finally: off
+#   no-unsafe-negation: off
+#   use-isnan: error
+#   valid-jsdoc: off
+#   valid-typeof: error
 
-  # Best Practices
-  accessor-pairs: error
-  array-callback-return: off
-  block-scoped-var: off
-  class-methods-use-this: off
-  complexity:
-    - error
-    - 6
-  consistent-return: off
-  curly: off
-  default-case: off
-  dot-location: off
-  dot-notation: off
-  eqeqeq: error
-  guard-for-in: error
-  no-alert: error
-  no-caller: error
-  no-case-declarations: error
-  no-div-regex: error
-  no-else-return: off
-  no-empty-function: off
-  no-empty-pattern: error
-  no-eq-null: error
-  no-eval: error
-  no-extend-native: error
-  no-extra-bind: error
-  no-extra-label: off
-  no-fallthrough: error
-  no-floating-decimal: off
-  no-global-assign: off
-  no-implicit-coercion: off
-  no-implied-eval: error
-  no-invalid-this: off
-  no-iterator: error
-  no-labels:
-    - error
-    - allowLoop: true
-      allowSwitch: true
-  no-lone-blocks: error
-  no-loop-func: error
-  no-magic-number: off
-  no-multi-spaces: off
-  no-multi-str: off
-  no-native-reassign: error
-  no-new-func: error
-  no-new-wrappers: error
-  no-new: error
-  no-octal-escape: error
-  no-octal: error
-  no-param-reassign: off
-  no-proto: error
-  no-redeclare: error
-  no-restricted-properties: off
-  no-return-assign: error
-  no-return-await: off
-  no-script-url: error
-  no-self-assign: off
-  no-self-compare: error
-  no-sequences: off
-  no-throw-literal: off
-  no-unmodified-loop-condition: off
-  no-unused-expressions: error
-  no-unused-labels: off
-  no-useless-call: error
-  no-useless-concat: error
+#   # Best Practices
+#   accessor-pairs: error
+#   array-callback-return: off
+#   block-scoped-var: off
+#   class-methods-use-this: off
+#   complexity:
+#     - error
+#     - 6
+#   consistent-return: off
+#   curly: off
+#   default-case: off
+#   dot-location: off
+#   dot-notation: off
+#   eqeqeq: error
+#   guard-for-in: error
+#   no-alert: error
+#   no-caller: error
+#   no-case-declarations: error
+#   no-div-regex: error
+#   no-else-return: off
+  no-empty-function: warn
+#   no-empty-pattern: error
+#   no-eq-null: error
+#   no-eval: error
+#   no-extend-native: error
+#   no-extra-bind: error
+#   no-extra-label: off
+#   no-fallthrough: error
+#   no-floating-decimal: off
+#   no-global-assign: off
+#   no-implicit-coercion: off
+#   no-implied-eval: error
+#   no-invalid-this: off
+#   no-iterator: error
+#   no-labels:
+#     - error
+#     - allowLoop: true
+#       allowSwitch: true
+#   no-lone-blocks: error
+#   no-loop-func: error
+#   no-magic-number: off
+#   no-multi-spaces: off
+#   no-multi-str: off
+#   no-native-reassign: error
+#   no-new-func: error
+#   no-new-wrappers: error
+#   no-new: error
+#   no-octal-escape: error
+#   no-octal: error
+#   no-param-reassign: off
+#   no-proto: error
+#   no-redeclare: error
+#   no-restricted-properties: off
+#   no-return-assign: error
+#   no-return-await: off
+#   no-script-url: error
+#   no-self-assign: off
+#   no-self-compare: error
+#   no-sequences: off
+#   no-throw-literal: off
+#   no-unmodified-loop-condition: off
+#   no-unused-expressions: error
+#   no-unused-labels: off
+#   no-useless-call: error
+#   no-useless-concat: error
   no-useless-escape: off
-  no-useless-return: off
-  no-void: error
-  no-warning-comments: off
-  no-with: error
-  prefer-promise-reject-errors: off
-  radix: error
-  require-await: off
-  vars-on-top: off
-  wrap-iife: error
-  yoda: off
+#   no-useless-return: off
+#   no-void: error
+#   no-warning-comments: off
+#   no-with: error
+#   prefer-promise-reject-errors: off
+#   radix: error
+#   require-await: off
+#   vars-on-top: off
+#   wrap-iife: error
+#   yoda: off
 
-  # Strict
-  strict: off
+#   # Strict
+#   strict: off
 
-  # Variables
-  init-declarations: off
-  no-catch-shadow: error
-  no-delete-var: error
-  no-label-var: error
-  no-restricted-globals: off
-  no-shadow-restricted-names: error
-  no-shadow: off
-  no-undef-init: error
-  no-undef: off
-  no-undefined: off
-  no-unused-vars: off
-  no-use-before-define: off
+#   # Variables
+#   init-declarations: off
+#   no-catch-shadow: error
+#   no-delete-var: error
+#   no-label-var: error
+#   no-restricted-globals: off
+#   no-shadow-restricted-names: error
+#   no-shadow: off
+#   no-undef-init: error
+#   no-undef: off
+#   no-undefined: off
+#   no-unused-vars:
+#     - warn
+#     -
+#       vars: local
+#   no-use-before-define: off
 
-  # Node.js and CommonJS
-  callback-return: error
-  global-require: error
-  handle-callback-err: error
-  no-mixed-requires: off
-  no-new-require: off
-  no-path-concat: error
-  no-process-env: off
-  no-process-exit: error
-  no-restricted-modules: off
-  no-sync: off
+#   # Node.js and CommonJS
+#   callback-return: error
+#   global-require: error
+#   handle-callback-err: error
+#   no-mixed-requires: off
+#   no-new-require: off
+#   no-path-concat: error
+#   no-process-env: off
+#   no-process-exit: error
+#   no-restricted-modules: off
+#   no-sync: off
 
-  # Stylistic Issues
-  array-bracket-spacing: off
-  block-spacing: off
-  brace-style: off
-  camelcase: off
-  capitalized-comments: off
-  comma-dangle:
-    - error
-    - never
-  comma-spacing: off
-  comma-style: off
-  computed-property-spacing: off
-  consistent-this: off
-  eol-last: off
-  func-call-spacing: off
-  func-name-matching: off
-  func-names: off
-  func-style: off
-  id-length: off
-  id-match: off
-  indent: off
-  jsx-quotes: off
-  key-spacing: off
-  keyword-spacing: off
-  line-comment-position: off
-  linebreak-style:
-    - error
-    - unix
-  lines-around-comment: off
-  lines-around-directive: off
-  max-depth: off
-  max-len: off
-  max-nested-callbacks: off
-  max-params: off
-  max-statements-per-line: off
-  max-statements:
-    - error
-    - 30
-  multiline-ternary: off
-  new-cap: off
-  new-parens: off
-  newline-after-var: off
-  newline-before-return: off
-  newline-per-chained-call: off
-  no-array-constructor: off
-  no-bitwise: off
-  no-continue: off
-  no-inline-comments: off
-  no-lonely-if: off
-  no-mixed-operators: off
-  no-mixed-spaces-and-tabs: off
-  no-multi-assign: off
-  no-multiple-empty-lines: off
-  no-negated-condition: off
-  no-nested-ternary: off
-  no-new-object: off
-  no-plusplus: off
-  no-restricted-syntax: off
-  no-spaced-func: off
-  no-tabs: off
-  no-ternary: off
-  no-trailing-spaces: off
-  no-underscore-dangle: off
-  no-unneeded-ternary: off
-  object-curly-newline: off
-  object-curly-spacing: off
-  object-property-newline: off
-  one-var-declaration-per-line: off
-  one-var: off
-  operator-assignment: off
-  operator-linebreak: off
-  padded-blocks: off
-  quote-props: off
-  quotes:
-    - error
-    - single
-  require-jsdoc: off
-  semi-spacing: off
-  semi:
-    - error
-    - always
-  sort-keys: off
-  sort-vars: off
-  space-before-blocks: off
-  space-before-function-paren: off
-  space-in-parens: off
-  space-infix-ops: off
-  space-unary-ops: off
-  spaced-comment: off
-  template-tag-spacing: off
-  unicode-bom: off
-  wrap-regex: off
+#   # Stylistic Issues
+#   array-bracket-spacing: off
+#   block-spacing: off
+#   brace-style: off
+#   camelcase: off
+#   capitalized-comments: off
+#   comma-dangle:
+#     - error
+#     - never
+#   comma-spacing: off
+#   comma-style: off
+#   computed-property-spacing: off
+#   consistent-this: off
+#   eol-last: off
+#   func-call-spacing: off
+#   func-name-matching: off
+#   func-names: off
+#   func-style: off
+#   id-length: off
+#   id-match: off
+#   indent: off
+#   jsx-quotes: off
+#   key-spacing: off
+#   keyword-spacing: off
+#   line-comment-position: off
+#   linebreak-style:
+#     - error
+#     - unix
+#   lines-around-comment: off
+#   lines-around-directive: off
+#   max-depth: off
+#   max-len: off
+#   max-nested-callbacks: off
+#   max-params: off
+#   max-statements-per-line: off
+#   max-statements:
+#     - error
+#     - 30
+#   multiline-ternary: off
+#   new-cap: off
+#   new-parens: off
+#   newline-after-var: off
+#   newline-before-return: off
+#   newline-per-chained-call: off
+#   no-array-constructor: off
+#   no-bitwise: off
+#   no-continue: off
+#   no-inline-comments: off
+#   no-lonely-if: off
+#   no-mixed-operators: off
+#   no-mixed-spaces-and-tabs: off
+#   no-multi-assign: off
+#   no-multiple-empty-lines: off
+#   no-negated-condition: off
+#   no-nested-ternary: off
+#   no-new-object: off
+#   no-plusplus: off
+#   no-restricted-syntax: off
+#   no-spaced-func: off
+#   no-tabs: off
+#   no-ternary: off
+#   no-trailing-spaces: off
+#   no-underscore-dangle: off
+#   no-unneeded-ternary: off
+#   object-curly-newline: off
+#   object-curly-spacing: off
+#   object-property-newline: off
+#   one-var-declaration-per-line: off
+#   one-var: off
+#   operator-assignment: off
+#   operator-linebreak: off
+#   padded-blocks: off
+#   quote-props: off
+#   quotes:
+#     - error
+#     - single
+#   require-jsdoc: off
+#   semi-spacing: off
+#   semi:
+#     - error
+#     - always
+#   sort-keys: off
+#   sort-vars: off
+#   space-before-blocks: off
+#   space-before-function-paren: off
+#   space-in-parens: off
+#   space-infix-ops: off
+#   space-unary-ops: off
+#   spaced-comment: off
+#   template-tag-spacing: off
+#   unicode-bom: off
+#   wrap-regex: off
 
-  # ECMAScript 6
-  arrow-body-style: off
-  arrow-parens: off
-  arrow-spacing: off
-  constructor-super: off
-  generator-star-spacing: off
-  no-class-assign: off
-  no-confusing-arrow: off
-  no-const-assign: off
-  no-dupe-class-members: off
-  no-duplicate-imports: off
-  no-new-symbol: off
-  no-restricted-imports: off
-  no-this-before-super: off
-  no-useless-computed-key: off
-  no-useless-constructor: off
-  no-useless-rename: off
-  no-var: off
-  object-shorthand: off
-  prefer-arrow-callback: off
-  prefer-const: off
-  prefer-destructuring: off
-  prefer-numeric-literals: off
-  prefer-rest-params: off
-  prefer-reflect: off
-  prefer-spread: off
-  prefer-template: off
-  require-yield: off
-  rest-spread-spacing: off
-  sort-imports: off
-  symbol-description: off
-  template-curly-spacing: off
-  yield-star-spacing: off
+#   # ECMAScript 6
+#   arrow-body-style: off
+#   arrow-parens: off
+#   arrow-spacing: off
+#   constructor-super: off
+#   generator-star-spacing: off
+#   no-class-assign: off
+#   no-confusing-arrow: off
+#   no-const-assign: off
+#   no-dupe-class-members: off
+#   no-duplicate-imports: off
+#   no-new-symbol: off
+#   no-restricted-imports: off
+#   no-this-before-super: off
+#   no-useless-computed-key: off
+#   no-useless-constructor: off
+#   no-useless-rename: off
+#   no-var: off
+#   object-shorthand: off
+#   prefer-arrow-callback: off
+#   prefer-const: off
+#   prefer-destructuring: off
+#   prefer-numeric-literals: off
+#   prefer-rest-params: off
+#   prefer-reflect: off
+#   prefer-spread: off
+#   prefer-template: off
+#   require-yield: off
+#   rest-spread-spacing: off
+#   sort-imports: off
+#   symbol-description: off
+#   template-curly-spacing: off
+#   yield-star-spacing: off

.github/ISSUE_TEMPLATE/Bug_report.md

@@ -8,7 +8,9 @@ about: Create a bug report
 
 Thanks for reporting a bug for Portainer !
 
-Do you need help or have a question? Come chat with us on Slack http://portainer.io/slack/ or gitter https://gitter.im/portainer/Lobby.
+You can find more information about Portainer support framework policy here: https://www.portainer.io/2019/04/portainer-support-policy/
+
+Do you need help or have a question? Come chat with us on Slack http://portainer.io/slack/.
 
 Before opening a new issue, make sure that we do not have any duplicates
 already open. You can ensure this by searching the issue list for this

.github/ISSUE_TEMPLATE/Custom.md

@@ -6,7 +6,9 @@ about: Ask us a question about Portainer usage or deployment
 
 <!--
 
-Do you need help or have a question? Come chat with us on Slack http://portainer.io/slack/ or gitter https://gitter.im/portainer/Lobby.
+You can find more information about Portainer support framework policy here: https://www.portainer.io/2019/04/portainer-support-policy/
+
+Do you need help or have a question? Come chat with us on Slack http://portainer.io/slack/
 
 Also, be sure to check our FAQ and documentation first: https://portainer.readthedocs.io
 -->

.github/ISSUE_TEMPLATE/Feature_request.md

@@ -8,7 +8,7 @@ about: Suggest a feature/enhancement that should be added in Portainer
 
 Thanks for opening a feature request for Portainer !
 
-Do you need help or have a question? Come chat with us on Slack http://portainer.io/slack/ or gitter https://gitter.im/portainer/Lobby.
+Do you need help or have a question? Come chat with us on Slack http://portainer.io/slack/
 
 Before opening a new issue, make sure that we do not have any duplicates
 already open. You can ensure this by searching the issue list for this

.gitignore

@@ -4,3 +4,4 @@ dist
 portainer-checksum.txt
 api/cmd/portainer/portainer*
 .tmp
+.vscode

CONTRIBUTING.md

@@ -2,7 +2,7 @@
 
 Some basic conventions for contributing to this project.
 
-### General
+## General
 
 Please make sure that there aren't existing pull requests attempting to address the issue mentioned. Likewise, please check for issues related to update, as someone else may be working on the issue in a branch or fork.
 
@@ -13,7 +13,7 @@ When creating a new branch, prefix it with the *type* of the change (see section
 
 For example, if you work on a bugfix for the issue #361, you could name the branch `fix361-template-selection`.
 
-### Issues open to contribution
+## Issues open to contribution
 
 Want to contribute but don't know where to start?
 
@@ -24,14 +24,14 @@ Some of the open issues are labeled with prefix `exp/`, this is used to mark the
 either AngularJS or Golang
 * **advanced**: a task that require a deep understanding of the project codebase
 
-You can have a use Github filters to list these issues:
+You can use Github filters to list these issues:
 
 * beginner labeled issues: https://github.com/portainer/portainer/labels/exp%2Fbeginner
 * intermediate labeled issues: https://github.com/portainer/portainer/labels/exp%2Fintermediate
 * advanced labeled issues: https://github.com/portainer/portainer/labels/exp%2Fadvanced
 
 
-### Commit Message Format
+## Commit Message Format
 
 Each commit message should include a **type**, a **scope** and a **subject**:
 
@@ -47,7 +47,7 @@ Lines should not exceed 100 characters. This allows the message to be easier to
  #269 style(dashboard): update dashboard with new layout
 ```
 
-#### Type
+### Type
 
 Must be one of the following:
 
@@ -61,16 +61,30 @@ Must be one of the following:
 * **chore**: Changes to the build process or auxiliary tools and libraries such as documentation
   generation
 
-#### Scope
+### Scope
 
 The scope could be anything specifying place of the commit change. For example `networks`,
 `containers`, `images` etc...
 You can use the **area** label tag associated on the issue here (for `area/containers` use `containers` as a scope...)
 
-#### Subject
+### Subject
 
 The subject contains succinct description of the change:
 
 * use the imperative, present tense: "change" not "changed" nor "changes"
 * don't capitalize first letter
 * no dot (.) at the end
+
+## Contribution process
+
+Our contribution process is described below. Some of the steps can be visualized inside Github via specific `status/` labels, such as `status/1-functional-review` or `status/2-technical-review`.
+
+### Bug report
+
+![portainer_bugreport_workflow](https://user-images.githubusercontent.com/5485061/45727219-50190a00-bbf5-11e8-9fe8-3a563bb8d5d7.png)
+
+### Feature request
+
+The feature request process is similar to the bug report process but has an extra functional validation before the technical validation as well as a documentation validation before the testing phase.
+
+![portainer_featurerequest_workflow](https://user-images.githubusercontent.com/5485061/45727229-5ad39f00-bbf5-11e8-9550-16ba66c50615.png)

README.md

@@ -1,27 +1,20 @@
-
 <p align="center">
-  <img title="portainer" src='https://portainer.io/images/logo_alt.png' />
+  <img title="portainer" src='https://github.com/portainer/portainer/blob/develop/assets/images/logo_alt.png?raw=true' />
 </p>
 
 [![Docker Pulls](https://img.shields.io/docker/pulls/portainer/portainer.svg)](https://hub.docker.com/r/portainer/portainer/)
 [![Microbadger](https://images.microbadger.com/badges/image/portainer/portainer.svg)](http://microbadger.com/images/portainer/portainer "Image size")
 [![Documentation Status](https://readthedocs.org/projects/portainer/badge/?version=stable)](http://portainer.readthedocs.io/en/stable/?badge=stable)
-[![Codefresh build status]( https://g.codefresh.io/api/badges/build?repoOwner=portainer&repoName=portainer&branch=develop&pipelineName=portainer-ci&accountName=deviantony&type=cf-1)]( https://g.codefresh.io/repositories/portainer/portainer/builds?filter=trigger:build;branch:develop;service:5922a08a3a1aab000116fcc6~portainer-ci)
+[![Build Status](https://portainer.visualstudio.com/Portainer%20CI/_apis/build/status/Portainer%20CI?branchName=develop)](https://portainer.visualstudio.com/Portainer%20CI/_build/latest?definitionId=3&branchName=develop)
 [![Code Climate](https://codeclimate.com/github/portainer/portainer/badges/gpa.svg)](https://codeclimate.com/github/portainer/portainer)
-[![Slack](https://portainer.io/slack/badge.svg)](https://portainer.io/slack/)
-[![Gitter](https://badges.gitter.im/portainer/Lobby.svg)](https://gitter.im/portainer/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
 [![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=YHXZJQNJQ36H6)
 
 **_Portainer_** is a lightweight management UI which allows you to **easily** manage your different Docker environments (Docker hosts or Swarm clusters).
-
-**_Portainer_** is meant to be as **simple** to deploy as it is to use. It consists of a single container that can run on any Docker engine (can be deployed as Linux container or a Windows native container).
-
-**_Portainer_** allows you to manage your Docker containers, images, volumes, networks and more ! It is compatible with the *standalone Docker* engine and with *Docker Swarm mode*.
+**_Portainer_** is meant to be as **simple** to deploy as it is to use. It consists of a single container that can run on any Docker engine (can be deployed as Linux container or a Windows native container, supports other platforms too).
+**_Portainer_** allows you to manage your all your Docker resources (containers, images, volumes, networks and more) ! It is compatible with the *standalone Docker* engine and with *Docker Swarm mode*.
 
 ## Demo
 
-<img src="https://portainer.io/images/screenshots/portainer.gif" width="77%"/>
-
 You can try out the public demo instance: http://demo.portainer.io/ (login with the username **admin** and the password **tryportainer**).
 
 Please note that the public demo cluster is **reset every 15min**.
@@ -41,10 +34,11 @@ Unlike the public demo, the playground sessions are deleted after 4 hours. Apart
 
 ## Getting help
 
+**NOTE**: You can find more information about Portainer support framework policy here: https://www.portainer.io/2019/04/portainer-support-policy/
+
 * Issues: https://github.com/portainer/portainer/issues
 * FAQ: https://portainer.readthedocs.io/en/latest/faq.html
 * Slack (chat): https://portainer.io/slack/
-* Gitter (chat): https://gitter.im/portainer/Lobby
 
 ## Reporting bugs and contributing
 

api/archive/tar.go

@@ -7,13 +7,13 @@ import (
 
 // TarFileInBuffer will create a tar archive containing a single file named via fileName and using the content
 // specified in fileContent. Returns the archive as a byte array.
-func TarFileInBuffer(fileContent []byte, fileName string) ([]byte, error) {
+func TarFileInBuffer(fileContent []byte, fileName string, mode int64) ([]byte, error) {
 	var buffer bytes.Buffer
 	tarWriter := tar.NewWriter(&buffer)
 
 	header := &tar.Header{
 		Name: fileName,
-		Mode: 0600,
+		Mode: mode,
 		Size: int64(len(fileContent)),
 	}
 

api/archive/zip.go

@@ -0,0 +1,48 @@
+package archive
+
+import (
+	"archive/zip"
+	"bytes"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+)
+
+// UnzipArchive will unzip an archive from bytes into the dest destination folder on disk
+func UnzipArchive(archiveData []byte, dest string) error {
+	zipReader, err := zip.NewReader(bytes.NewReader(archiveData), int64(len(archiveData)))
+	if err != nil {
+		return err
+	}
+
+	for _, zipFile := range zipReader.File {
+
+		f, err := zipFile.Open()
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+
+		data, err := ioutil.ReadAll(f)
+		if err != nil {
+			return err
+		}
+
+		fpath := filepath.Join(dest, zipFile.Name)
+
+		outFile, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, zipFile.Mode())
+		if err != nil {
+			return err
+		}
+
+		_, err = io.Copy(outFile, bytes.NewReader(data))
+		if err != nil {
+			return err
+		}
+
+		outFile.Close()
+	}
+
+	return nil
+}

api/bolt/datastore.go

@@ -6,20 +6,25 @@ import (
 	"time"
 
 	"github.com/boltdb/bolt"
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/dockerhub"
-	"github.com/portainer/portainer/bolt/endpoint"
-	"github.com/portainer/portainer/bolt/endpointgroup"
-	"github.com/portainer/portainer/bolt/migrator"
-	"github.com/portainer/portainer/bolt/registry"
-	"github.com/portainer/portainer/bolt/resourcecontrol"
-	"github.com/portainer/portainer/bolt/settings"
-	"github.com/portainer/portainer/bolt/stack"
-	"github.com/portainer/portainer/bolt/tag"
-	"github.com/portainer/portainer/bolt/team"
-	"github.com/portainer/portainer/bolt/teammembership"
-	"github.com/portainer/portainer/bolt/user"
-	"github.com/portainer/portainer/bolt/version"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/dockerhub"
+	"github.com/portainer/portainer/api/bolt/endpoint"
+	"github.com/portainer/portainer/api/bolt/endpointgroup"
+	"github.com/portainer/portainer/api/bolt/extension"
+	"github.com/portainer/portainer/api/bolt/migrator"
+	"github.com/portainer/portainer/api/bolt/registry"
+	"github.com/portainer/portainer/api/bolt/resourcecontrol"
+	"github.com/portainer/portainer/api/bolt/role"
+	"github.com/portainer/portainer/api/bolt/schedule"
+	"github.com/portainer/portainer/api/bolt/settings"
+	"github.com/portainer/portainer/api/bolt/stack"
+	"github.com/portainer/portainer/api/bolt/tag"
+	"github.com/portainer/portainer/api/bolt/team"
+	"github.com/portainer/portainer/api/bolt/teammembership"
+	"github.com/portainer/portainer/api/bolt/template"
+	"github.com/portainer/portainer/api/bolt/user"
+	"github.com/portainer/portainer/api/bolt/version"
+	"github.com/portainer/portainer/api/bolt/webhook"
 )
 
 const (
@@ -33,9 +38,11 @@ type Store struct {
 	db                     *bolt.DB
 	checkForDataMigration  bool
 	fileService            portainer.FileService
+	RoleService            *role.Service
 	DockerHubService       *dockerhub.Service
 	EndpointGroupService   *endpointgroup.Service
 	EndpointService        *endpoint.Service
+	ExtensionService       *extension.Service
 	RegistryService        *registry.Service
 	ResourceControlService *resourcecontrol.Service
 	SettingsService        *settings.Service
@@ -43,8 +50,11 @@ type Store struct {
 	TagService             *tag.Service
 	TeamMembershipService  *teammembership.Service
 	TeamService            *team.Service
+	TemplateService        *template.Service
 	UserService            *user.Service
 	VersionService         *version.Service
+	WebhookService         *webhook.Service
+	ScheduleService        *schedule.Service
 }
 
 // NewStore initializes a new Store and the associated services
@@ -81,29 +91,6 @@ func (store *Store) Open() error {
 	return store.initServices()
 }
 
-// Init creates the default data set.
-func (store *Store) Init() error {
-	groups, err := store.EndpointGroupService.EndpointGroups()
-	if err != nil {
-		return err
-	}
-
-	if len(groups) == 0 {
-		unassignedGroup := &portainer.EndpointGroup{
-			Name:            "Unassigned",
-			Description:     "Unassigned endpoints",
-			Labels:          []portainer.Pair{},
-			AuthorizedUsers: []portainer.UserID{},
-			AuthorizedTeams: []portainer.TeamID{},
-			Tags:            []string{},
-		}
-
-		return store.EndpointGroupService.CreateEndpointGroup(unassignedGroup)
-	}
-
-	return nil
-}
-
 // Close closes the BoltDB database.
 func (store *Store) Close() error {
 	if store.db != nil {
@@ -131,9 +118,12 @@ func (store *Store) MigrateData() error {
 			DatabaseVersion:        version,
 			EndpointGroupService:   store.EndpointGroupService,
 			EndpointService:        store.EndpointService,
+			ExtensionService:       store.ExtensionService,
+			RegistryService:        store.RegistryService,
 			ResourceControlService: store.ResourceControlService,
 			SettingsService:        store.SettingsService,
 			StackService:           store.StackService,
+			TemplateService:        store.TemplateService,
 			UserService:            store.UserService,
 			VersionService:         store.VersionService,
 			FileService:            store.fileService,
@@ -152,6 +142,12 @@ func (store *Store) MigrateData() error {
 }
 
 func (store *Store) initServices() error {
+	authorizationsetService, err := role.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.RoleService = authorizationsetService
+
 	dockerhubService, err := dockerhub.NewService(store.db)
 	if err != nil {
 		return err
@@ -170,6 +166,12 @@ func (store *Store) initServices() error {
 	}
 	store.EndpointService = endpointService
 
+	extensionService, err := extension.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.ExtensionService = extensionService
+
 	registryService, err := registry.NewService(store.db)
 	if err != nil {
 		return err
@@ -212,6 +214,12 @@ func (store *Store) initServices() error {
 	}
 	store.TeamService = teamService
 
+	templateService, err := template.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.TemplateService = templateService
+
 	userService, err := user.NewService(store.db)
 	if err != nil {
 		return err
@@ -224,5 +232,17 @@ func (store *Store) initServices() error {
 	}
 	store.VersionService = versionService
 
+	webhookService, err := webhook.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.WebhookService = webhookService
+
+	scheduleService, err := schedule.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.ScheduleService = scheduleService
+
 	return nil
 }

api/bolt/dockerhub/dockerhub.go

@@ -1,8 +1,8 @@
 package dockerhub
 
 import (
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/internal"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
 )

api/bolt/endpoint/endpoint.go

@@ -1,8 +1,8 @@
 package endpoint
 
 import (
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/internal"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
 )
@@ -82,8 +82,11 @@ func (service *Service) CreateEndpoint(endpoint *portainer.Endpoint) error {
 	return service.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(BucketName))
 
-		id, _ := bucket.NextSequence()
-		endpoint.ID = portainer.EndpointID(id)
+		// We manually manage sequences for endpoints
+		err := bucket.SetSequence(uint64(endpoint.ID))
+		if err != nil {
+			return err
+		}
 
 		data, err := internal.MarshalObject(endpoint)
 		if err != nil {
@@ -94,6 +97,11 @@ func (service *Service) CreateEndpoint(endpoint *portainer.Endpoint) error {
 	})
 }
 
+// GetNextIdentifier returns the next identifier for an endpoint.
+func (service *Service) GetNextIdentifier() int {
+	return internal.GetNextIdentifier(service.db, BucketName)
+}
+
 // Synchronize creates, updates and deletes endpoints inside a single transaction.
 func (service *Service) Synchronize(toCreate, toUpdate, toDelete []*portainer.Endpoint) error {
 	return service.db.Update(func(tx *bolt.Tx) error {

api/bolt/endpointgroup/endpointgroup.go

@@ -1,8 +1,8 @@
 package endpointgroup
 
 import (
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/internal"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
 )

api/bolt/extension/extension.go

@@ -0,0 +1,86 @@
+package extension
+
+import (
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
+
+	"github.com/boltdb/bolt"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "extension"
+)
+
+// Service represents a service for managing endpoint data.
+type Service struct {
+	db *bolt.DB
+}
+
+// NewService creates a new instance of a service.
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		db: db,
+	}, nil
+}
+
+// Extension returns a extension by ID
+func (service *Service) Extension(ID portainer.ExtensionID) (*portainer.Extension, error) {
+	var extension portainer.Extension
+	identifier := internal.Itob(int(ID))
+
+	err := internal.GetObject(service.db, BucketName, identifier, &extension)
+	if err != nil {
+		return nil, err
+	}
+
+	return &extension, nil
+}
+
+// Extensions return an array containing all the extensions.
+func (service *Service) Extensions() ([]portainer.Extension, error) {
+	var extensions = make([]portainer.Extension, 0)
+
+	err := service.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var extension portainer.Extension
+			err := internal.UnmarshalObject(v, &extension)
+			if err != nil {
+				return err
+			}
+			extensions = append(extensions, extension)
+		}
+
+		return nil
+	})
+
+	return extensions, err
+}
+
+// Persist persists a extension inside the database.
+func (service *Service) Persist(extension *portainer.Extension) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		data, err := internal.MarshalObject(extension)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(extension.ID)), data)
+	})
+}
+
+// DeleteExtension deletes a Extension.
+func (service *Service) DeleteExtension(ID portainer.ExtensionID) error {
+	identifier := internal.Itob(int(ID))
+	return internal.DeleteObject(service.db, BucketName, identifier)
+}

api/bolt/init.go

@@ -0,0 +1,432 @@
+package bolt
+
+import portainer "github.com/portainer/portainer/api"
+
+// Init creates the default data set.
+func (store *Store) Init() error {
+	groups, err := store.EndpointGroupService.EndpointGroups()
+	if err != nil {
+		return err
+	}
+
+	if len(groups) == 0 {
+		unassignedGroup := &portainer.EndpointGroup{
+			Name:               "Unassigned",
+			Description:        "Unassigned endpoints",
+			Labels:             []portainer.Pair{},
+			UserAccessPolicies: portainer.UserAccessPolicies{},
+			TeamAccessPolicies: portainer.TeamAccessPolicies{},
+			Tags:               []string{},
+		}
+
+		err = store.EndpointGroupService.CreateEndpointGroup(unassignedGroup)
+		if err != nil {
+			return err
+		}
+	}
+
+	roles, err := store.RoleService.Roles()
+	if err != nil {
+		return err
+	}
+
+	if len(roles) == 0 {
+		environmentAdministratorRole := &portainer.Role{
+			Name:        "Endpoint administrator",
+			Description: "Full control of all resources in an endpoint",
+			Authorizations: map[portainer.Authorization]bool{
+				portainer.OperationDockerContainerArchiveInfo:         true,
+				portainer.OperationDockerContainerList:                true,
+				portainer.OperationDockerContainerExport:              true,
+				portainer.OperationDockerContainerChanges:             true,
+				portainer.OperationDockerContainerInspect:             true,
+				portainer.OperationDockerContainerTop:                 true,
+				portainer.OperationDockerContainerLogs:                true,
+				portainer.OperationDockerContainerStats:               true,
+				portainer.OperationDockerContainerAttachWebsocket:     true,
+				portainer.OperationDockerContainerArchive:             true,
+				portainer.OperationDockerContainerCreate:              true,
+				portainer.OperationDockerContainerPrune:               true,
+				portainer.OperationDockerContainerKill:                true,
+				portainer.OperationDockerContainerPause:               true,
+				portainer.OperationDockerContainerUnpause:             true,
+				portainer.OperationDockerContainerRestart:             true,
+				portainer.OperationDockerContainerStart:               true,
+				portainer.OperationDockerContainerStop:                true,
+				portainer.OperationDockerContainerWait:                true,
+				portainer.OperationDockerContainerResize:              true,
+				portainer.OperationDockerContainerAttach:              true,
+				portainer.OperationDockerContainerExec:                true,
+				portainer.OperationDockerContainerRename:              true,
+				portainer.OperationDockerContainerUpdate:              true,
+				portainer.OperationDockerContainerPutContainerArchive: true,
+				portainer.OperationDockerContainerDelete:              true,
+				portainer.OperationDockerImageList:                    true,
+				portainer.OperationDockerImageSearch:                  true,
+				portainer.OperationDockerImageGetAll:                  true,
+				portainer.OperationDockerImageGet:                     true,
+				portainer.OperationDockerImageHistory:                 true,
+				portainer.OperationDockerImageInspect:                 true,
+				portainer.OperationDockerImageLoad:                    true,
+				portainer.OperationDockerImageCreate:                  true,
+				portainer.OperationDockerImagePrune:                   true,
+				portainer.OperationDockerImagePush:                    true,
+				portainer.OperationDockerImageTag:                     true,
+				portainer.OperationDockerImageDelete:                  true,
+				portainer.OperationDockerImageCommit:                  true,
+				portainer.OperationDockerImageBuild:                   true,
+				portainer.OperationDockerNetworkList:                  true,
+				portainer.OperationDockerNetworkInspect:               true,
+				portainer.OperationDockerNetworkCreate:                true,
+				portainer.OperationDockerNetworkConnect:               true,
+				portainer.OperationDockerNetworkDisconnect:            true,
+				portainer.OperationDockerNetworkPrune:                 true,
+				portainer.OperationDockerNetworkDelete:                true,
+				portainer.OperationDockerVolumeList:                   true,
+				portainer.OperationDockerVolumeInspect:                true,
+				portainer.OperationDockerVolumeCreate:                 true,
+				portainer.OperationDockerVolumePrune:                  true,
+				portainer.OperationDockerVolumeDelete:                 true,
+				portainer.OperationDockerExecInspect:                  true,
+				portainer.OperationDockerExecStart:                    true,
+				portainer.OperationDockerExecResize:                   true,
+				portainer.OperationDockerSwarmInspect:                 true,
+				portainer.OperationDockerSwarmUnlockKey:               true,
+				portainer.OperationDockerSwarmInit:                    true,
+				portainer.OperationDockerSwarmJoin:                    true,
+				portainer.OperationDockerSwarmLeave:                   true,
+				portainer.OperationDockerSwarmUpdate:                  true,
+				portainer.OperationDockerSwarmUnlock:                  true,
+				portainer.OperationDockerNodeList:                     true,
+				portainer.OperationDockerNodeInspect:                  true,
+				portainer.OperationDockerNodeUpdate:                   true,
+				portainer.OperationDockerNodeDelete:                   true,
+				portainer.OperationDockerServiceList:                  true,
+				portainer.OperationDockerServiceInspect:               true,
+				portainer.OperationDockerServiceLogs:                  true,
+				portainer.OperationDockerServiceCreate:                true,
+				portainer.OperationDockerServiceUpdate:                true,
+				portainer.OperationDockerServiceDelete:                true,
+				portainer.OperationDockerSecretList:                   true,
+				portainer.OperationDockerSecretInspect:                true,
+				portainer.OperationDockerSecretCreate:                 true,
+				portainer.OperationDockerSecretUpdate:                 true,
+				portainer.OperationDockerSecretDelete:                 true,
+				portainer.OperationDockerConfigList:                   true,
+				portainer.OperationDockerConfigInspect:                true,
+				portainer.OperationDockerConfigCreate:                 true,
+				portainer.OperationDockerConfigUpdate:                 true,
+				portainer.OperationDockerConfigDelete:                 true,
+				portainer.OperationDockerTaskList:                     true,
+				portainer.OperationDockerTaskInspect:                  true,
+				portainer.OperationDockerTaskLogs:                     true,
+				portainer.OperationDockerPluginList:                   true,
+				portainer.OperationDockerPluginPrivileges:             true,
+				portainer.OperationDockerPluginInspect:                true,
+				portainer.OperationDockerPluginPull:                   true,
+				portainer.OperationDockerPluginCreate:                 true,
+				portainer.OperationDockerPluginEnable:                 true,
+				portainer.OperationDockerPluginDisable:                true,
+				portainer.OperationDockerPluginPush:                   true,
+				portainer.OperationDockerPluginUpgrade:                true,
+				portainer.OperationDockerPluginSet:                    true,
+				portainer.OperationDockerPluginDelete:                 true,
+				portainer.OperationDockerSessionStart:                 true,
+				portainer.OperationDockerDistributionInspect:          true,
+				portainer.OperationDockerBuildPrune:                   true,
+				portainer.OperationDockerBuildCancel:                  true,
+				portainer.OperationDockerPing:                         true,
+				portainer.OperationDockerInfo:                         true,
+				portainer.OperationDockerVersion:                      true,
+				portainer.OperationDockerEvents:                       true,
+				portainer.OperationDockerSystem:                       true,
+				portainer.OperationDockerUndefined:                    true,
+				portainer.OperationDockerAgentPing:                    true,
+				portainer.OperationDockerAgentList:                    true,
+				portainer.OperationDockerAgentHostInfo:                true,
+				portainer.OperationDockerAgentBrowseDelete:            true,
+				portainer.OperationDockerAgentBrowseGet:               true,
+				portainer.OperationDockerAgentBrowseList:              true,
+				portainer.OperationDockerAgentBrowsePut:               true,
+				portainer.OperationDockerAgentBrowseRename:            true,
+				portainer.OperationDockerAgentUndefined:               true,
+				portainer.OperationPortainerResourceControlCreate:     true,
+				portainer.OperationPortainerResourceControlUpdate:     true,
+				portainer.OperationPortainerResourceControlDelete:     true,
+				portainer.OperationPortainerStackList:                 true,
+				portainer.OperationPortainerStackInspect:              true,
+				portainer.OperationPortainerStackFile:                 true,
+				portainer.OperationPortainerStackCreate:               true,
+				portainer.OperationPortainerStackMigrate:              true,
+				portainer.OperationPortainerStackUpdate:               true,
+				portainer.OperationPortainerStackDelete:               true,
+				portainer.OperationPortainerWebsocketExec:             true,
+				portainer.OperationPortainerWebhookList:               true,
+				portainer.OperationPortainerWebhookCreate:             true,
+				portainer.OperationPortainerWebhookDelete:             true,
+				portainer.OperationIntegrationStoridgeAdmin:           true,
+				portainer.EndpointResourcesAccess:                     true,
+			},
+		}
+
+		err = store.RoleService.CreateRole(environmentAdministratorRole)
+		if err != nil {
+			return err
+		}
+
+		environmentReadOnlyUserRole := &portainer.Role{
+			Name:        "Helpdesk",
+			Description: "Read-only access of all resources in an endpoint",
+			Authorizations: map[portainer.Authorization]bool{
+				portainer.OperationDockerContainerArchiveInfo: true,
+				portainer.OperationDockerContainerList:        true,
+				portainer.OperationDockerContainerChanges:     true,
+				portainer.OperationDockerContainerInspect:     true,
+				portainer.OperationDockerContainerTop:         true,
+				portainer.OperationDockerContainerLogs:        true,
+				portainer.OperationDockerContainerStats:       true,
+				portainer.OperationDockerImageList:            true,
+				portainer.OperationDockerImageSearch:          true,
+				portainer.OperationDockerImageGetAll:          true,
+				portainer.OperationDockerImageGet:             true,
+				portainer.OperationDockerImageHistory:         true,
+				portainer.OperationDockerImageInspect:         true,
+				portainer.OperationDockerNetworkList:          true,
+				portainer.OperationDockerNetworkInspect:       true,
+				portainer.OperationDockerVolumeList:           true,
+				portainer.OperationDockerVolumeInspect:        true,
+				portainer.OperationDockerSwarmInspect:         true,
+				portainer.OperationDockerNodeList:             true,
+				portainer.OperationDockerNodeInspect:          true,
+				portainer.OperationDockerServiceList:          true,
+				portainer.OperationDockerServiceInspect:       true,
+				portainer.OperationDockerServiceLogs:          true,
+				portainer.OperationDockerSecretList:           true,
+				portainer.OperationDockerSecretInspect:        true,
+				portainer.OperationDockerConfigList:           true,
+				portainer.OperationDockerConfigInspect:        true,
+				portainer.OperationDockerTaskList:             true,
+				portainer.OperationDockerTaskInspect:          true,
+				portainer.OperationDockerTaskLogs:             true,
+				portainer.OperationDockerPluginList:           true,
+				portainer.OperationDockerDistributionInspect:  true,
+				portainer.OperationDockerPing:                 true,
+				portainer.OperationDockerInfo:                 true,
+				portainer.OperationDockerVersion:              true,
+				portainer.OperationDockerEvents:               true,
+				portainer.OperationDockerSystem:               true,
+				portainer.OperationDockerAgentPing:            true,
+				portainer.OperationDockerAgentList:            true,
+				portainer.OperationDockerAgentHostInfo:        true,
+				portainer.OperationDockerAgentBrowseGet:       true,
+				portainer.OperationDockerAgentBrowseList:      true,
+				portainer.OperationPortainerStackList:         true,
+				portainer.OperationPortainerStackInspect:      true,
+				portainer.OperationPortainerStackFile:         true,
+				portainer.OperationPortainerWebhookList:       true,
+				portainer.EndpointResourcesAccess:             true,
+			},
+		}
+
+		err = store.RoleService.CreateRole(environmentReadOnlyUserRole)
+		if err != nil {
+			return err
+		}
+
+		standardUserRole := &portainer.Role{
+			Name:        "Standard user",
+			Description: "Full control of assigned resources in an endpoint",
+			Authorizations: map[portainer.Authorization]bool{
+				portainer.OperationDockerContainerArchiveInfo:         true,
+				portainer.OperationDockerContainerList:                true,
+				portainer.OperationDockerContainerExport:              true,
+				portainer.OperationDockerContainerChanges:             true,
+				portainer.OperationDockerContainerInspect:             true,
+				portainer.OperationDockerContainerTop:                 true,
+				portainer.OperationDockerContainerLogs:                true,
+				portainer.OperationDockerContainerStats:               true,
+				portainer.OperationDockerContainerAttachWebsocket:     true,
+				portainer.OperationDockerContainerArchive:             true,
+				portainer.OperationDockerContainerCreate:              true,
+				portainer.OperationDockerContainerKill:                true,
+				portainer.OperationDockerContainerPause:               true,
+				portainer.OperationDockerContainerUnpause:             true,
+				portainer.OperationDockerContainerRestart:             true,
+				portainer.OperationDockerContainerStart:               true,
+				portainer.OperationDockerContainerStop:                true,
+				portainer.OperationDockerContainerWait:                true,
+				portainer.OperationDockerContainerResize:              true,
+				portainer.OperationDockerContainerAttach:              true,
+				portainer.OperationDockerContainerExec:                true,
+				portainer.OperationDockerContainerRename:              true,
+				portainer.OperationDockerContainerUpdate:              true,
+				portainer.OperationDockerContainerPutContainerArchive: true,
+				portainer.OperationDockerContainerDelete:              true,
+				portainer.OperationDockerImageList:                    true,
+				portainer.OperationDockerImageSearch:                  true,
+				portainer.OperationDockerImageGetAll:                  true,
+				portainer.OperationDockerImageGet:                     true,
+				portainer.OperationDockerImageHistory:                 true,
+				portainer.OperationDockerImageInspect:                 true,
+				portainer.OperationDockerImageLoad:                    true,
+				portainer.OperationDockerImageCreate:                  true,
+				portainer.OperationDockerImagePush:                    true,
+				portainer.OperationDockerImageTag:                     true,
+				portainer.OperationDockerImageDelete:                  true,
+				portainer.OperationDockerImageCommit:                  true,
+				portainer.OperationDockerImageBuild:                   true,
+				portainer.OperationDockerNetworkList:                  true,
+				portainer.OperationDockerNetworkInspect:               true,
+				portainer.OperationDockerNetworkCreate:                true,
+				portainer.OperationDockerNetworkConnect:               true,
+				portainer.OperationDockerNetworkDisconnect:            true,
+				portainer.OperationDockerNetworkDelete:                true,
+				portainer.OperationDockerVolumeList:                   true,
+				portainer.OperationDockerVolumeInspect:                true,
+				portainer.OperationDockerVolumeCreate:                 true,
+				portainer.OperationDockerVolumeDelete:                 true,
+				portainer.OperationDockerExecInspect:                  true,
+				portainer.OperationDockerExecStart:                    true,
+				portainer.OperationDockerExecResize:                   true,
+				portainer.OperationDockerSwarmInspect:                 true,
+				portainer.OperationDockerSwarmUnlockKey:               true,
+				portainer.OperationDockerSwarmInit:                    true,
+				portainer.OperationDockerSwarmJoin:                    true,
+				portainer.OperationDockerSwarmLeave:                   true,
+				portainer.OperationDockerSwarmUpdate:                  true,
+				portainer.OperationDockerSwarmUnlock:                  true,
+				portainer.OperationDockerNodeList:                     true,
+				portainer.OperationDockerNodeInspect:                  true,
+				portainer.OperationDockerNodeUpdate:                   true,
+				portainer.OperationDockerNodeDelete:                   true,
+				portainer.OperationDockerServiceList:                  true,
+				portainer.OperationDockerServiceInspect:               true,
+				portainer.OperationDockerServiceLogs:                  true,
+				portainer.OperationDockerServiceCreate:                true,
+				portainer.OperationDockerServiceUpdate:                true,
+				portainer.OperationDockerServiceDelete:                true,
+				portainer.OperationDockerSecretList:                   true,
+				portainer.OperationDockerSecretInspect:                true,
+				portainer.OperationDockerSecretCreate:                 true,
+				portainer.OperationDockerSecretUpdate:                 true,
+				portainer.OperationDockerSecretDelete:                 true,
+				portainer.OperationDockerConfigList:                   true,
+				portainer.OperationDockerConfigInspect:                true,
+				portainer.OperationDockerConfigCreate:                 true,
+				portainer.OperationDockerConfigUpdate:                 true,
+				portainer.OperationDockerConfigDelete:                 true,
+				portainer.OperationDockerTaskList:                     true,
+				portainer.OperationDockerTaskInspect:                  true,
+				portainer.OperationDockerTaskLogs:                     true,
+				portainer.OperationDockerPluginList:                   true,
+				portainer.OperationDockerPluginPrivileges:             true,
+				portainer.OperationDockerPluginInspect:                true,
+				portainer.OperationDockerPluginPull:                   true,
+				portainer.OperationDockerPluginCreate:                 true,
+				portainer.OperationDockerPluginEnable:                 true,
+				portainer.OperationDockerPluginDisable:                true,
+				portainer.OperationDockerPluginPush:                   true,
+				portainer.OperationDockerPluginUpgrade:                true,
+				portainer.OperationDockerPluginSet:                    true,
+				portainer.OperationDockerPluginDelete:                 true,
+				portainer.OperationDockerSessionStart:                 true,
+				portainer.OperationDockerDistributionInspect:          true,
+				portainer.OperationDockerBuildPrune:                   true,
+				portainer.OperationDockerBuildCancel:                  true,
+				portainer.OperationDockerPing:                         true,
+				portainer.OperationDockerInfo:                         true,
+				portainer.OperationDockerVersion:                      true,
+				portainer.OperationDockerEvents:                       true,
+				portainer.OperationDockerSystem:                       true,
+				portainer.OperationDockerUndefined:                    true,
+				portainer.OperationDockerAgentPing:                    true,
+				portainer.OperationDockerAgentList:                    true,
+				portainer.OperationDockerAgentHostInfo:                true,
+				portainer.OperationDockerAgentBrowseDelete:            true,
+				portainer.OperationDockerAgentBrowseGet:               true,
+				portainer.OperationDockerAgentBrowseList:              true,
+				portainer.OperationDockerAgentBrowsePut:               true,
+				portainer.OperationDockerAgentBrowseRename:            true,
+				portainer.OperationDockerAgentUndefined:               true,
+				portainer.OperationPortainerResourceControlCreate:     true,
+				portainer.OperationPortainerResourceControlUpdate:     true,
+				portainer.OperationPortainerResourceControlDelete:     true,
+				portainer.OperationPortainerStackList:                 true,
+				portainer.OperationPortainerStackInspect:              true,
+				portainer.OperationPortainerStackFile:                 true,
+				portainer.OperationPortainerStackCreate:               true,
+				portainer.OperationPortainerStackMigrate:              true,
+				portainer.OperationPortainerStackUpdate:               true,
+				portainer.OperationPortainerStackDelete:               true,
+				portainer.OperationPortainerWebsocketExec:             true,
+				portainer.OperationPortainerWebhookList:               true,
+				portainer.OperationPortainerWebhookCreate:             true,
+			},
+		}
+
+		err = store.RoleService.CreateRole(standardUserRole)
+		if err != nil {
+			return err
+		}
+
+		readOnlyUserRole := &portainer.Role{
+			Name:        "Read-only user",
+			Description: "Read-only access of assigned resources in an endpoint",
+			Authorizations: map[portainer.Authorization]bool{
+				portainer.OperationDockerContainerArchiveInfo: true,
+				portainer.OperationDockerContainerList:        true,
+				portainer.OperationDockerContainerChanges:     true,
+				portainer.OperationDockerContainerInspect:     true,
+				portainer.OperationDockerContainerTop:         true,
+				portainer.OperationDockerContainerLogs:        true,
+				portainer.OperationDockerContainerStats:       true,
+				portainer.OperationDockerImageList:            true,
+				portainer.OperationDockerImageSearch:          true,
+				portainer.OperationDockerImageGetAll:          true,
+				portainer.OperationDockerImageGet:             true,
+				portainer.OperationDockerImageHistory:         true,
+				portainer.OperationDockerImageInspect:         true,
+				portainer.OperationDockerNetworkList:          true,
+				portainer.OperationDockerNetworkInspect:       true,
+				portainer.OperationDockerVolumeList:           true,
+				portainer.OperationDockerVolumeInspect:        true,
+				portainer.OperationDockerSwarmInspect:         true,
+				portainer.OperationDockerNodeList:             true,
+				portainer.OperationDockerNodeInspect:          true,
+				portainer.OperationDockerServiceList:          true,
+				portainer.OperationDockerServiceInspect:       true,
+				portainer.OperationDockerServiceLogs:          true,
+				portainer.OperationDockerSecretList:           true,
+				portainer.OperationDockerSecretInspect:        true,
+				portainer.OperationDockerConfigList:           true,
+				portainer.OperationDockerConfigInspect:        true,
+				portainer.OperationDockerTaskList:             true,
+				portainer.OperationDockerTaskInspect:          true,
+				portainer.OperationDockerTaskLogs:             true,
+				portainer.OperationDockerPluginList:           true,
+				portainer.OperationDockerDistributionInspect:  true,
+				portainer.OperationDockerPing:                 true,
+				portainer.OperationDockerInfo:                 true,
+				portainer.OperationDockerVersion:              true,
+				portainer.OperationDockerEvents:               true,
+				portainer.OperationDockerSystem:               true,
+				portainer.OperationDockerAgentPing:            true,
+				portainer.OperationDockerAgentList:            true,
+				portainer.OperationDockerAgentHostInfo:        true,
+				portainer.OperationDockerAgentBrowseGet:       true,
+				portainer.OperationDockerAgentBrowseList:      true,
+				portainer.OperationPortainerStackList:         true,
+				portainer.OperationPortainerStackInspect:      true,
+				portainer.OperationPortainerStackFile:         true,
+				portainer.OperationPortainerWebhookList:       true,
+			},
+		}
+
+		err = store.RoleService.CreateRole(readOnlyUserRole)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

api/bolt/internal/db.go

@@ -4,7 +4,7 @@ import (
 	"encoding/binary"
 
 	"github.com/boltdb/bolt"
-	"github.com/portainer/portainer"
+	"github.com/portainer/portainer/api"
 )
 
 // Itob returns an 8-byte big endian representation of v.

api/bolt/migrator/migrate_dbversion0.go

@@ -2,8 +2,8 @@ package migrator
 
 import (
 	"github.com/boltdb/bolt"
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/user"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/user"
 )
 
 func (m *Migrator) updateAdminUserToDBVersion1() error {

api/bolt/migrator/migrate_dbversion1.go

@@ -2,8 +2,8 @@ package migrator
 
 import (
 	"github.com/boltdb/bolt"
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/internal"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
 )
 
 func (m *Migrator) updateResourceControlsToDBVersion2() error {

api/bolt/migrator/migrate_dbversion10.go

@@ -1,6 +1,6 @@
 package migrator
 
-import "github.com/portainer/portainer"
+import "github.com/portainer/portainer/api"
 
 func (m *Migrator) updateEndpointsToVersion11() error {
 	legacyEndpoints, err := m.endpointService.Endpoints()

api/bolt/migrator/migrate_dbversion11.go

@@ -5,9 +5,9 @@ import (
 	"strings"
 
 	"github.com/boltdb/bolt"
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/internal"
-	"github.com/portainer/portainer/bolt/stack"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
+	"github.com/portainer/portainer/api/bolt/stack"
 )
 
 func (m *Migrator) updateEndpointsToVersion12() error {

api/bolt/migrator/migrate_dbversion12.go

@@ -0,0 +1,17 @@
+package migrator
+
+import "github.com/portainer/portainer/api"
+
+func (m *Migrator) updateSettingsToVersion13() error {
+	legacySettings, err := m.settingsService.Settings()
+	if err != nil {
+		return err
+	}
+
+	legacySettings.LDAPSettings.AutoCreateUsers = false
+	legacySettings.LDAPSettings.GroupSearchSettings = []portainer.LDAPGroupSearchSettings{
+		portainer.LDAPGroupSearchSettings{},
+	}
+
+	return m.settingsService.UpdateSettings(legacySettings)
+}

api/bolt/migrator/migrate_dbversion13.go

@@ -0,0 +1,19 @@
+package migrator
+
+func (m *Migrator) updateResourceControlsToDBVersion14() error {
+	resourceControls, err := m.resourceControlService.ResourceControls()
+	if err != nil {
+		return err
+	}
+
+	for _, resourceControl := range resourceControls {
+		if resourceControl.AdministratorsOnly == true {
+			err = m.resourceControlService.DeleteResourceControl(resourceControl.ID)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}

api/bolt/migrator/migrate_dbversion14.go

@@ -0,0 +1,35 @@
+package migrator
+
+import (
+	"strings"
+
+	"github.com/portainer/portainer/api"
+)
+
+func (m *Migrator) updateSettingsToDBVersion15() error {
+	legacySettings, err := m.settingsService.Settings()
+	if err != nil {
+		return err
+	}
+
+	legacySettings.EnableHostManagementFeatures = false
+	return m.settingsService.UpdateSettings(legacySettings)
+}
+
+func (m *Migrator) updateTemplatesToVersion15() error {
+	legacyTemplates, err := m.templateService.Templates()
+	if err != nil {
+		return err
+	}
+
+	for _, template := range legacyTemplates {
+		template.Logo = strings.Replace(template.Logo, "https://portainer.io/images", portainer.AssetsServerURL, -1)
+
+		err = m.templateService.UpdateTemplate(template.ID, &template)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

api/bolt/migrator/migrate_dbversion15.go

@@ -0,0 +1,14 @@
+package migrator
+
+func (m *Migrator) updateSettingsToDBVersion16() error {
+	legacySettings, err := m.settingsService.Settings()
+	if err != nil {
+		return err
+	}
+
+	if legacySettings.SnapshotInterval == "" {
+		legacySettings.SnapshotInterval = "5m"
+	}
+
+	return m.settingsService.UpdateSettings(legacySettings)
+}

api/bolt/migrator/migrate_dbversion16.go

@@ -0,0 +1,19 @@
+package migrator
+
+func (m *Migrator) updateExtensionsToDBVersion17() error {
+	legacyExtensions, err := m.extensionService.Extensions()
+	if err != nil {
+		return err
+	}
+
+	for _, extension := range legacyExtensions {
+		extension.License.Valid = true
+
+		err = m.extensionService.Persist(&extension)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

api/bolt/migrator/migrate_dbversion17.go

@@ -0,0 +1,125 @@
+package migrator
+
+import (
+	portainer "github.com/portainer/portainer/api"
+)
+
+func (m *Migrator) updateUsersToDBVersion18() error {
+	legacyUsers, err := m.userService.Users()
+	if err != nil {
+		return err
+	}
+
+	for _, user := range legacyUsers {
+		user.PortainerAuthorizations = map[portainer.Authorization]bool{
+			portainer.OperationPortainerDockerHubInspect:        true,
+			portainer.OperationPortainerEndpointGroupList:       true,
+			portainer.OperationPortainerEndpointList:            true,
+			portainer.OperationPortainerEndpointInspect:         true,
+			portainer.OperationPortainerEndpointExtensionAdd:    true,
+			portainer.OperationPortainerEndpointExtensionRemove: true,
+			portainer.OperationPortainerExtensionList:           true,
+			portainer.OperationPortainerMOTD:                    true,
+			portainer.OperationPortainerRegistryList:            true,
+			portainer.OperationPortainerRegistryInspect:         true,
+			portainer.OperationPortainerTeamList:                true,
+			portainer.OperationPortainerTemplateList:            true,
+			portainer.OperationPortainerTemplateInspect:         true,
+			portainer.OperationPortainerUserList:                true,
+			portainer.OperationPortainerUserMemberships:         true,
+		}
+
+		err = m.userService.UpdateUser(user.ID, &user)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (m *Migrator) updateEndpointsToDBVersion18() error {
+	legacyEndpoints, err := m.endpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, endpoint := range legacyEndpoints {
+		endpoint.UserAccessPolicies = make(portainer.UserAccessPolicies)
+		for _, userID := range endpoint.AuthorizedUsers {
+			endpoint.UserAccessPolicies[userID] = portainer.AccessPolicy{
+				RoleID: 4,
+			}
+		}
+
+		endpoint.TeamAccessPolicies = make(portainer.TeamAccessPolicies)
+		for _, teamID := range endpoint.AuthorizedTeams {
+			endpoint.TeamAccessPolicies[teamID] = portainer.AccessPolicy{
+				RoleID: 4,
+			}
+		}
+
+		err = m.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (m *Migrator) updateEndpointGroupsToDBVersion18() error {
+	legacyEndpointGroups, err := m.endpointGroupService.EndpointGroups()
+	if err != nil {
+		return err
+	}
+
+	for _, endpointGroup := range legacyEndpointGroups {
+		endpointGroup.UserAccessPolicies = make(portainer.UserAccessPolicies)
+		for _, userID := range endpointGroup.AuthorizedUsers {
+			endpointGroup.UserAccessPolicies[userID] = portainer.AccessPolicy{
+				RoleID: 4,
+			}
+		}
+
+		endpointGroup.TeamAccessPolicies = make(portainer.TeamAccessPolicies)
+		for _, teamID := range endpointGroup.AuthorizedTeams {
+			endpointGroup.TeamAccessPolicies[teamID] = portainer.AccessPolicy{
+				RoleID: 4,
+			}
+		}
+
+		err = m.endpointGroupService.UpdateEndpointGroup(endpointGroup.ID, &endpointGroup)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (m *Migrator) updateRegistriesToDBVersion18() error {
+	legacyRegistries, err := m.registryService.Registries()
+	if err != nil {
+		return err
+	}
+
+	for _, registry := range legacyRegistries {
+		registry.UserAccessPolicies = make(portainer.UserAccessPolicies)
+		for _, userID := range registry.AuthorizedUsers {
+			registry.UserAccessPolicies[userID] = portainer.AccessPolicy{}
+		}
+
+		registry.TeamAccessPolicies = make(portainer.TeamAccessPolicies)
+		for _, teamID := range registry.AuthorizedTeams {
+			registry.TeamAccessPolicies[teamID] = portainer.AccessPolicy{}
+		}
+
+		err = m.registryService.UpdateRegistry(registry.ID, &registry)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

api/bolt/migrator/migrate_dbversion2.go

@@ -1,6 +1,6 @@
 package migrator
 
-import "github.com/portainer/portainer"
+import "github.com/portainer/portainer/api"
 
 func (m *Migrator) updateSettingsToDBVersion3() error {
 	legacySettings, err := m.settingsService.Settings()

api/bolt/migrator/migrate_dbversion3.go

@@ -1,6 +1,6 @@
 package migrator
 
-import "github.com/portainer/portainer"
+import "github.com/portainer/portainer/api"
 
 func (m *Migrator) updateEndpointsToDBVersion4() error {
 	legacyEndpoints, err := m.endpointService.Endpoints()

api/bolt/migrator/migrate_dbversion7.go

@@ -1,6 +1,6 @@
 package migrator
 
-import "github.com/portainer/portainer"
+import "github.com/portainer/portainer/api"
 
 func (m *Migrator) updateEndpointsToVersion8() error {
 	legacyEndpoints, err := m.endpointService.Endpoints()

api/bolt/migrator/migrate_dbversion8.go

@@ -1,6 +1,6 @@
 package migrator
 
-import "github.com/portainer/portainer"
+import "github.com/portainer/portainer/api"
 
 func (m *Migrator) updateEndpointsToVersion9() error {
 	legacyEndpoints, err := m.endpointService.Endpoints()

api/bolt/migrator/migrate_dbversion9.go

@@ -1,6 +1,6 @@
 package migrator
 
-import "github.com/portainer/portainer"
+import "github.com/portainer/portainer/api"
 
 func (m *Migrator) updateEndpointsToVersion10() error {
 	legacyEndpoints, err := m.endpointService.Endpoints()

api/bolt/migrator/migrator.go

@@ -2,14 +2,17 @@ package migrator
 
 import (
 	"github.com/boltdb/bolt"
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/endpoint"
-	"github.com/portainer/portainer/bolt/endpointgroup"
-	"github.com/portainer/portainer/bolt/resourcecontrol"
-	"github.com/portainer/portainer/bolt/settings"
-	"github.com/portainer/portainer/bolt/stack"
-	"github.com/portainer/portainer/bolt/user"
-	"github.com/portainer/portainer/bolt/version"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/endpoint"
+	"github.com/portainer/portainer/api/bolt/endpointgroup"
+	"github.com/portainer/portainer/api/bolt/extension"
+	"github.com/portainer/portainer/api/bolt/registry"
+	"github.com/portainer/portainer/api/bolt/resourcecontrol"
+	"github.com/portainer/portainer/api/bolt/settings"
+	"github.com/portainer/portainer/api/bolt/stack"
+	"github.com/portainer/portainer/api/bolt/template"
+	"github.com/portainer/portainer/api/bolt/user"
+	"github.com/portainer/portainer/api/bolt/version"
 )
 
 type (
@@ -19,9 +22,12 @@ type (
 		db                     *bolt.DB
 		endpointGroupService   *endpointgroup.Service
 		endpointService        *endpoint.Service
+		extensionService       *extension.Service
+		registryService        *registry.Service
 		resourceControlService *resourcecontrol.Service
 		settingsService        *settings.Service
 		stackService           *stack.Service
+		templateService        *template.Service
 		userService            *user.Service
 		versionService         *version.Service
 		fileService            portainer.FileService
@@ -33,9 +39,12 @@ type (
 		DatabaseVersion        int
 		EndpointGroupService   *endpointgroup.Service
 		EndpointService        *endpoint.Service
+		ExtensionService       *extension.Service
+		RegistryService        *registry.Service
 		ResourceControlService *resourcecontrol.Service
 		SettingsService        *settings.Service
 		StackService           *stack.Service
+		TemplateService        *template.Service
 		UserService            *user.Service
 		VersionService         *version.Service
 		FileService            portainer.FileService
@@ -49,8 +58,11 @@ func NewMigrator(parameters *Parameters) *Migrator {
 		currentDBVersion:       parameters.DatabaseVersion,
 		endpointGroupService:   parameters.EndpointGroupService,
 		endpointService:        parameters.EndpointService,
+		extensionService:       parameters.ExtensionService,
+		registryService:        parameters.RegistryService,
 		resourceControlService: parameters.ResourceControlService,
 		settingsService:        parameters.SettingsService,
+		templateService:        parameters.TemplateService,
 		stackService:           parameters.StackService,
 		userService:            parameters.UserService,
 		versionService:         parameters.VersionService,
@@ -170,5 +182,72 @@ func (m *Migrator) Migrate() error {
 		}
 	}
 
+	// Portainer 1.19.0
+	if m.currentDBVersion < 13 {
+		err := m.updateSettingsToVersion13()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.19.2
+	if m.currentDBVersion < 14 {
+		err := m.updateResourceControlsToDBVersion14()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.20.0
+	if m.currentDBVersion < 15 {
+		err := m.updateSettingsToDBVersion15()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateTemplatesToVersion15()
+		if err != nil {
+			return err
+		}
+	}
+
+	if m.currentDBVersion < 16 {
+		err := m.updateSettingsToDBVersion16()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.20.1
+	if m.currentDBVersion < 17 {
+		err := m.updateExtensionsToDBVersion17()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Portainer 1.21.0
+	if m.currentDBVersion < 18 {
+		err := m.updateUsersToDBVersion18()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateEndpointsToDBVersion18()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateEndpointGroupsToDBVersion18()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateRegistriesToDBVersion18()
+		if err != nil {
+			return err
+		}
+	}
+
 	return m.versionService.StoreDBVersion(portainer.DBVersion)
 }

api/bolt/registry/registry.go

@@ -1,8 +1,8 @@
 package registry
 
 import (
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/internal"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
 )

api/bolt/resourcecontrol/resourcecontrol.go

@@ -1,8 +1,8 @@
 package resourcecontrol
 
 import (
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/internal"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
 )

api/bolt/role/role.go

@@ -0,0 +1,83 @@
+package role
+
+import (
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
+
+	"github.com/boltdb/bolt"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "roles"
+)
+
+// Service represents a service for managing endpoint data.
+type Service struct {
+	db *bolt.DB
+}
+
+// NewService creates a new instance of a service.
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		db: db,
+	}, nil
+}
+
+// Role returns a Role by ID
+func (service *Service) Role(ID portainer.RoleID) (*portainer.Role, error) {
+	var set portainer.Role
+	identifier := internal.Itob(int(ID))
+
+	err := internal.GetObject(service.db, BucketName, identifier, &set)
+	if err != nil {
+		return nil, err
+	}
+
+	return &set, nil
+}
+
+// Roles return an array containing all the sets.
+func (service *Service) Roles() ([]portainer.Role, error) {
+	var sets = make([]portainer.Role, 0)
+
+	err := service.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var set portainer.Role
+			err := internal.UnmarshalObject(v, &set)
+			if err != nil {
+				return err
+			}
+			sets = append(sets, set)
+		}
+
+		return nil
+	})
+
+	return sets, err
+}
+
+// CreateRole creates a new Role.
+func (service *Service) CreateRole(set *portainer.Role) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		id, _ := bucket.NextSequence()
+		set.ID = portainer.RoleID(id)
+
+		data, err := internal.MarshalObject(set)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(set.ID)), data)
+	})
+}

api/bolt/schedule/schedule.go

@@ -0,0 +1,129 @@
+package schedule
+
+import (
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
+
+	"github.com/boltdb/bolt"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "schedules"
+)
+
+// Service represents a service for managing schedule data.
+type Service struct {
+	db *bolt.DB
+}
+
+// NewService creates a new instance of a service.
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		db: db,
+	}, nil
+}
+
+// Schedule returns a schedule by ID.
+func (service *Service) Schedule(ID portainer.ScheduleID) (*portainer.Schedule, error) {
+	var schedule portainer.Schedule
+	identifier := internal.Itob(int(ID))
+
+	err := internal.GetObject(service.db, BucketName, identifier, &schedule)
+	if err != nil {
+		return nil, err
+	}
+
+	return &schedule, nil
+}
+
+// UpdateSchedule updates a schedule.
+func (service *Service) UpdateSchedule(ID portainer.ScheduleID, schedule *portainer.Schedule) error {
+	identifier := internal.Itob(int(ID))
+	return internal.UpdateObject(service.db, BucketName, identifier, schedule)
+}
+
+// DeleteSchedule deletes a schedule.
+func (service *Service) DeleteSchedule(ID portainer.ScheduleID) error {
+	identifier := internal.Itob(int(ID))
+	return internal.DeleteObject(service.db, BucketName, identifier)
+}
+
+// Schedules return a array containing all the schedules.
+func (service *Service) Schedules() ([]portainer.Schedule, error) {
+	var schedules = make([]portainer.Schedule, 0)
+
+	err := service.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var schedule portainer.Schedule
+			err := internal.UnmarshalObject(v, &schedule)
+			if err != nil {
+				return err
+			}
+			schedules = append(schedules, schedule)
+		}
+
+		return nil
+	})
+
+	return schedules, err
+}
+
+// SchedulesByJobType return a array containing all the schedules
+// with the specified JobType.
+func (service *Service) SchedulesByJobType(jobType portainer.JobType) ([]portainer.Schedule, error) {
+	var schedules = make([]portainer.Schedule, 0)
+
+	err := service.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var schedule portainer.Schedule
+			err := internal.UnmarshalObject(v, &schedule)
+			if err != nil {
+				return err
+			}
+			if schedule.JobType == jobType {
+				schedules = append(schedules, schedule)
+			}
+		}
+
+		return nil
+	})
+
+	return schedules, err
+}
+
+// CreateSchedule assign an ID to a new schedule and saves it.
+func (service *Service) CreateSchedule(schedule *portainer.Schedule) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		// We manually manage sequences for schedules
+		err := bucket.SetSequence(uint64(schedule.ID))
+		if err != nil {
+			return err
+		}
+
+		data, err := internal.MarshalObject(schedule)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(schedule.ID)), data)
+	})
+}
+
+// GetNextIdentifier returns the next identifier for a schedule.
+func (service *Service) GetNextIdentifier() int {
+	return internal.GetNextIdentifier(service.db, BucketName)
+}

api/bolt/settings/settings.go

@@ -1,8 +1,8 @@
 package settings
 
 import (
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/internal"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
 )

api/bolt/stack/stack.go

@@ -1,8 +1,8 @@
 package stack
 
 import (
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/internal"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
 )

api/bolt/tag/tag.go

@@ -1,8 +1,8 @@
 package tag
 
 import (
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/internal"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
 )

api/bolt/team/team.go

@@ -1,8 +1,8 @@
 package team
 
 import (
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/internal"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
 )

api/bolt/teammembership/teammembership.go

@@ -1,8 +1,8 @@
 package teammembership
 
 import (
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/internal"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
 )

api/bolt/template/template.go

@@ -0,0 +1,95 @@
+package template
+
+import (
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
+
+	"github.com/boltdb/bolt"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "templates"
+)
+
+// Service represents a service for managing endpoint data.
+type Service struct {
+	db *bolt.DB
+}
+
+// NewService creates a new instance of a service.
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		db: db,
+	}, nil
+}
+
+// Templates return an array containing all the templates.
+func (service *Service) Templates() ([]portainer.Template, error) {
+	var templates = make([]portainer.Template, 0)
+
+	err := service.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var template portainer.Template
+			err := internal.UnmarshalObject(v, &template)
+			if err != nil {
+				return err
+			}
+			templates = append(templates, template)
+		}
+
+		return nil
+	})
+
+	return templates, err
+}
+
+// Template returns a template by ID.
+func (service *Service) Template(ID portainer.TemplateID) (*portainer.Template, error) {
+	var template portainer.Template
+	identifier := internal.Itob(int(ID))
+
+	err := internal.GetObject(service.db, BucketName, identifier, &template)
+	if err != nil {
+		return nil, err
+	}
+
+	return &template, nil
+}
+
+// CreateTemplate creates a new template.
+func (service *Service) CreateTemplate(template *portainer.Template) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		id, _ := bucket.NextSequence()
+		template.ID = portainer.TemplateID(id)
+
+		data, err := internal.MarshalObject(template)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(template.ID)), data)
+	})
+}
+
+// UpdateTemplate saves a template.
+func (service *Service) UpdateTemplate(ID portainer.TemplateID, template *portainer.Template) error {
+	identifier := internal.Itob(int(ID))
+	return internal.UpdateObject(service.db, BucketName, identifier, template)
+}
+
+// DeleteTemplate deletes a template.
+func (service *Service) DeleteTemplate(ID portainer.TemplateID) error {
+	identifier := internal.Itob(int(ID))
+	return internal.DeleteObject(service.db, BucketName, identifier)
+}

api/bolt/user/user.go

@@ -1,8 +1,8 @@
 package user
 
 import (
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/internal"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
 
 	"github.com/boltdb/bolt"
 )

api/bolt/version/version.go

@@ -4,8 +4,8 @@ import (
 	"strconv"
 
 	"github.com/boltdb/bolt"
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt/internal"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
 )
 
 const (

api/bolt/webhook/webhook.go

@@ -0,0 +1,151 @@
+package webhook
+
+import (
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
+
+	"github.com/boltdb/bolt"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "webhooks"
+)
+
+// Service represents a service for managing webhook data.
+type Service struct {
+	db *bolt.DB
+}
+
+// NewService creates a new instance of a service.
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		db: db,
+	}, nil
+}
+
+//Webhooks returns an array of all webhooks
+func (service *Service) Webhooks() ([]portainer.Webhook, error) {
+	var webhooks = make([]portainer.Webhook, 0)
+
+	err := service.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var webhook portainer.Webhook
+			err := internal.UnmarshalObject(v, &webhook)
+			if err != nil {
+				return err
+			}
+			webhooks = append(webhooks, webhook)
+		}
+
+		return nil
+	})
+
+	return webhooks, err
+}
+
+// Webhook returns a webhook by ID.
+func (service *Service) Webhook(ID portainer.WebhookID) (*portainer.Webhook, error) {
+	var webhook portainer.Webhook
+	identifier := internal.Itob(int(ID))
+
+	err := internal.GetObject(service.db, BucketName, identifier, &webhook)
+	if err != nil {
+		return nil, err
+	}
+
+	return &webhook, nil
+}
+
+// WebhookByResourceID returns a webhook by the ResourceID it is associated with.
+func (service *Service) WebhookByResourceID(ID string) (*portainer.Webhook, error) {
+	var webhook *portainer.Webhook
+
+	err := service.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+		cursor := bucket.Cursor()
+
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var w portainer.Webhook
+			err := internal.UnmarshalObject(v, &w)
+			if err != nil {
+				return err
+			}
+
+			if w.ResourceID == ID {
+				webhook = &w
+				break
+			}
+		}
+
+		if webhook == nil {
+			return portainer.ErrObjectNotFound
+		}
+
+		return nil
+	})
+
+	return webhook, err
+}
+
+// WebhookByToken returns a webhook by the random token it is associated with.
+func (service *Service) WebhookByToken(token string) (*portainer.Webhook, error) {
+	var webhook *portainer.Webhook
+
+	err := service.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+		cursor := bucket.Cursor()
+
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var w portainer.Webhook
+			err := internal.UnmarshalObject(v, &w)
+			if err != nil {
+				return err
+			}
+
+			if w.Token == token {
+				webhook = &w
+				break
+			}
+		}
+
+		if webhook == nil {
+			return portainer.ErrObjectNotFound
+		}
+
+		return nil
+	})
+
+	return webhook, err
+}
+
+// DeleteWebhook deletes a webhook.
+func (service *Service) DeleteWebhook(ID portainer.WebhookID) error {
+	identifier := internal.Itob(int(ID))
+	return internal.DeleteObject(service.db, BucketName, identifier)
+}
+
+// CreateWebhook assign an ID to a new webhook and saves it.
+func (service *Service) CreateWebhook(webhook *portainer.Webhook) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		id, _ := bucket.NextSequence()
+		webhook.ID = portainer.WebhookID(id)
+
+		data, err := internal.MarshalObject(webhook)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(webhook.ID)), data)
+	})
+}

api/chisel/server.go

@@ -0,0 +1,43 @@
+package chisel
+
+import (
+	chserver "github.com/jpillora/chisel/server"
+)
+
+type Server struct {
+	address     string
+	port        string
+	fingerprint string
+}
+
+func NewServer(address string, port string) *Server {
+	return &Server{
+		address: address,
+		port:    port,
+	}
+}
+
+// Start starts the reverse tunnel server
+func (server *Server) Start() error {
+
+	// TODO: keyseed management (persistence)
+	// + auth management
+	// Consider multiple users for auth?
+	config := &chserver.Config{
+		Reverse: true,
+		KeySeed: "keyseedexample",
+		Auth:    "agent@randomstring",
+	}
+
+	chiselServer, err := chserver.NewServer(config)
+	if err != nil {
+		return err
+	}
+
+	server.fingerprint = chiselServer.GetFingerprint()
+	return chiselServer.Start(server.address, server.port)
+}
+
+func (server *Server) GetFingerprint() string {
+	return server.fingerprint
+}

api/cli/cli.go

@@ -3,7 +3,7 @@ package cli
 import (
 	"time"
 
-	"github.com/portainer/portainer"
+	"github.com/portainer/portainer/api"
 
 	"os"
 	"path/filepath"
@@ -16,10 +16,12 @@ import (
 type Service struct{}
 
 const (
-	errInvalidEndpointProtocol       = portainer.Error("Invalid endpoint protocol: Portainer only supports unix:// or tcp://")
-	errSocketNotFound                = portainer.Error("Unable to locate Unix socket")
+	errInvalidEndpointProtocol       = portainer.Error("Invalid endpoint protocol: Portainer only supports unix://, npipe:// or tcp://")
+	errSocketOrNamedPipeNotFound     = portainer.Error("Unable to locate Unix socket or named pipe")
 	errEndpointsFileNotFound         = portainer.Error("Unable to locate external endpoints file")
+	errTemplateFileNotFound          = portainer.Error("Unable to locate template file on disk")
 	errInvalidSyncInterval           = portainer.Error("Invalid synchronization interval")
+	errInvalidSnapshotInterval       = portainer.Error("Invalid snapshot interval")
 	errEndpointExcludeExternal       = portainer.Error("Cannot use the -H flag mutually with --external-endpoints")
 	errNoAuthExcludeAdminPassword    = portainer.Error("Cannot use --no-auth with --admin-password or --admin-password-file")
 	errAdminPassExcludeAdminPassFile = portainer.Error("Cannot use --admin-password with --admin-password-file")
@@ -31,6 +33,8 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 
 	flags := &portainer.CLIFlags{
 		Addr:              kingpin.Flag("bind", "Address and port to serve Portainer").Default(defaultBindAddress).Short('p').String(),
+		TunnelAddr:        kingpin.Flag("tunnel-addr", "Address to serve the tunnel server").Default(defaultTunnelServerAddress).String(),
+		TunnelPort:        kingpin.Flag("tunnel-port", "Port to serve the tunnel server").Default(defaultTunnelServerPort).String(),
 		Assets:            kingpin.Flag("assets", "Path to the assets").Default(defaultAssetsDirectory).Short('a').String(),
 		Data:              kingpin.Flag("data", "Path to the folder where the data is stored").Default(defaultDataDirectory).Short('d').String(),
 		EndpointURL:       kingpin.Flag("host", "Endpoint URL").Short('H').String(),
@@ -46,11 +50,14 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 		SSLCert:           kingpin.Flag("sslcert", "Path to the SSL certificate used to secure the Portainer instance").Default(defaultSSLCertPath).String(),
 		SSLKey:            kingpin.Flag("sslkey", "Path to the SSL key used to secure the Portainer instance").Default(defaultSSLKeyPath).String(),
 		SyncInterval:      kingpin.Flag("sync-interval", "Duration between each synchronization via the external endpoints source").Default(defaultSyncInterval).String(),
+		Snapshot:          kingpin.Flag("snapshot", "Start a background job to create endpoint snapshots").Default(defaultSnapshot).Bool(),
+		SnapshotInterval:  kingpin.Flag("snapshot-interval", "Duration between each endpoint snapshot job").Default(defaultSnapshotInterval).String(),
 		AdminPassword:     kingpin.Flag("admin-password", "Hashed admin password").String(),
 		AdminPasswordFile: kingpin.Flag("admin-password-file", "Path to the file containing the password for the admin user").String(),
 		Labels:            pairs(kingpin.Flag("hide-label", "Hide containers with a specific label in the UI").Short('l')),
 		Logo:              kingpin.Flag("logo", "URL for the logo displayed in the UI").String(),
-		Templates:         kingpin.Flag("templates", "URL to the templates (apps) definitions").Short('t').String(),
+		Templates:         kingpin.Flag("templates", "URL to the templates definitions.").Short('t').String(),
+		TemplateFile:      kingpin.Flag("template-file", "Path to the templates (app) definitions on the filesystem").Default(defaultTemplateFile).String(),
 	}
 
 	kingpin.Parse()
@@ -73,7 +80,12 @@ func (*Service) ValidateFlags(flags *portainer.CLIFlags) error {
 		return errEndpointExcludeExternal
 	}
 
-	err := validateEndpointURL(*flags.EndpointURL)
+	err := validateTemplateFile(*flags.TemplateFile)
+	if err != nil {
+		return err
+	}
+
+	err = validateEndpointURL(*flags.EndpointURL)
 	if err != nil {
 		return err
 	}
@@ -88,6 +100,11 @@ func (*Service) ValidateFlags(flags *portainer.CLIFlags) error {
 		return err
 	}
 
+	err = validateSnapshotInterval(*flags.SnapshotInterval)
+	if err != nil {
+		return err
+	}
+
 	if *flags.NoAuth && (*flags.AdminPassword != "" || *flags.AdminPasswordFile != "") {
 		return errNoAuthExcludeAdminPassword
 	}
@@ -101,15 +118,16 @@ func (*Service) ValidateFlags(flags *portainer.CLIFlags) error {
 
 func validateEndpointURL(endpointURL string) error {
 	if endpointURL != "" {
-		if !strings.HasPrefix(endpointURL, "unix://") && !strings.HasPrefix(endpointURL, "tcp://") {
+		if !strings.HasPrefix(endpointURL, "unix://") && !strings.HasPrefix(endpointURL, "tcp://") && !strings.HasPrefix(endpointURL, "npipe://") {
 			return errInvalidEndpointProtocol
 		}
 
-		if strings.HasPrefix(endpointURL, "unix://") {
+		if strings.HasPrefix(endpointURL, "unix://") || strings.HasPrefix(endpointURL, "npipe://") {
 			socketPath := strings.TrimPrefix(endpointURL, "unix://")
+			socketPath = strings.TrimPrefix(socketPath, "npipe://")
 			if _, err := os.Stat(socketPath); err != nil {
 				if os.IsNotExist(err) {
-					return errSocketNotFound
+					return errSocketOrNamedPipeNotFound
 				}
 				return err
 			}
@@ -130,6 +148,16 @@ func validateExternalEndpoints(externalEndpoints string) error {
 	return nil
 }
 
+func validateTemplateFile(templateFile string) error {
+	if _, err := os.Stat(templateFile); err != nil {
+		if os.IsNotExist(err) {
+			return errTemplateFileNotFound
+		}
+		return err
+	}
+	return nil
+}
+
 func validateSyncInterval(syncInterval string) error {
 	if syncInterval != defaultSyncInterval {
 		_, err := time.ParseDuration(syncInterval)
@@ -139,3 +167,13 @@ func validateSyncInterval(syncInterval string) error {
 	}
 	return nil
 }
+
+func validateSnapshotInterval(snapshotInterval string) error {
+	if snapshotInterval != defaultSnapshotInterval {
+		_, err := time.ParseDuration(snapshotInterval)
+		if err != nil {
+			return errInvalidSnapshotInterval
+		}
+	}
+	return nil
+}

api/cli/defaults.go

@@ -3,18 +3,23 @@
 package cli
 
 const (
-	defaultBindAddress     = ":9000"
-	defaultDataDirectory   = "/data"
-	defaultAssetsDirectory = "./"
-	defaultNoAuth          = "false"
-	defaultNoAnalytics     = "false"
-	defaultTLS             = "false"
-	defaultTLSSkipVerify   = "false"
-	defaultTLSCACertPath   = "/certs/ca.pem"
-	defaultTLSCertPath     = "/certs/cert.pem"
-	defaultTLSKeyPath      = "/certs/key.pem"
-	defaultSSL             = "false"
-	defaultSSLCertPath     = "/certs/portainer.crt"
-	defaultSSLKeyPath      = "/certs/portainer.key"
-	defaultSyncInterval    = "60s"
+	defaultBindAddress         = ":9000"
+	defaultTunnelServerAddress = "0.0.0.0"
+	defaultTunnelServerPort    = "8000"
+	defaultDataDirectory       = "/data"
+	defaultAssetsDirectory     = "./"
+	defaultNoAuth              = "false"
+	defaultNoAnalytics         = "false"
+	defaultTLS                 = "false"
+	defaultTLSSkipVerify       = "false"
+	defaultTLSCACertPath       = "/certs/ca.pem"
+	defaultTLSCertPath         = "/certs/cert.pem"
+	defaultTLSKeyPath          = "/certs/key.pem"
+	defaultSSL                 = "false"
+	defaultSSLCertPath         = "/certs/portainer.crt"
+	defaultSSLKeyPath          = "/certs/portainer.key"
+	defaultSyncInterval        = "60s"
+	defaultSnapshot            = "true"
+	defaultSnapshotInterval    = "5m"
+	defaultTemplateFile        = "/templates.json"
 )

api/cli/defaults_windows.go

@@ -1,18 +1,23 @@
 package cli
 
 const (
-	defaultBindAddress     = ":9000"
-	defaultDataDirectory   = "C:\\data"
-	defaultAssetsDirectory = "./"
-	defaultNoAuth          = "false"
-	defaultNoAnalytics     = "false"
-	defaultTLS             = "false"
-	defaultTLSSkipVerify   = "false"
-	defaultTLSCACertPath   = "C:\\certs\\ca.pem"
-	defaultTLSCertPath     = "C:\\certs\\cert.pem"
-	defaultTLSKeyPath      = "C:\\certs\\key.pem"
-	defaultSSL             = "false"
-	defaultSSLCertPath     = "C:\\certs\\portainer.crt"
-	defaultSSLKeyPath      = "C:\\certs\\portainer.key"
-	defaultSyncInterval    = "60s"
+	defaultBindAddress         = ":9000"
+	defaultTunnelServerAddress = "0.0.0.0"
+	defaultTunnelServerPort    = "8000"
+	defaultDataDirectory       = "C:\\data"
+	defaultAssetsDirectory     = "./"
+	defaultNoAuth              = "false"
+	defaultNoAnalytics         = "false"
+	defaultTLS                 = "false"
+	defaultTLSSkipVerify       = "false"
+	defaultTLSCACertPath       = "C:\\certs\\ca.pem"
+	defaultTLSCertPath         = "C:\\certs\\cert.pem"
+	defaultTLSKeyPath          = "C:\\certs\\key.pem"
+	defaultSSL                 = "false"
+	defaultSSLCertPath         = "C:\\certs\\portainer.crt"
+	defaultSSLKeyPath          = "C:\\certs\\portainer.key"
+	defaultSyncInterval        = "60s"
+	defaultSnapshot            = "true"
+	defaultSnapshotInterval    = "5m"
+	defaultTemplateFile        = "/templates.json"
 )

api/cli/pairlist.go

@@ -1,7 +1,7 @@
 package cli
 
 import (
-	"github.com/portainer/portainer"
+	"github.com/portainer/portainer/api"
 
 	"fmt"
 	"gopkg.in/alecthomas/kingpin.v2"

api/cmd/portainer/main.go

@@ -1,23 +1,27 @@
-package main // import "github.com/portainer/portainer"
+package main
 
 import (
-	"strings"
-
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/bolt"
-	"github.com/portainer/portainer/cli"
-	"github.com/portainer/portainer/cron"
-	"github.com/portainer/portainer/crypto"
-	"github.com/portainer/portainer/exec"
-	"github.com/portainer/portainer/filesystem"
-	"github.com/portainer/portainer/git"
-	"github.com/portainer/portainer/http"
-	"github.com/portainer/portainer/http/client"
-	"github.com/portainer/portainer/jwt"
-	"github.com/portainer/portainer/ldap"
-	"github.com/portainer/portainer/libcompose"
-
+	"encoding/json"
 	"log"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt"
+	"github.com/portainer/portainer/api/chisel"
+	"github.com/portainer/portainer/api/cli"
+	"github.com/portainer/portainer/api/cron"
+	"github.com/portainer/portainer/api/crypto"
+	"github.com/portainer/portainer/api/docker"
+	"github.com/portainer/portainer/api/exec"
+	"github.com/portainer/portainer/api/filesystem"
+	"github.com/portainer/portainer/api/git"
+	"github.com/portainer/portainer/api/http"
+	"github.com/portainer/portainer/api/http/client"
+	"github.com/portainer/portainer/api/jwt"
+	"github.com/portainer/portainer/api/ldap"
+	"github.com/portainer/portainer/api/libcompose"
 )
 
 func initCLI() *portainer.CLIFlags {
@@ -85,7 +89,7 @@ func initJWTService(authenticationEnabled bool) portainer.JWTService {
 }
 
 func initDigitalSignatureService() portainer.DigitalSignatureService {
-	return &crypto.ECDSAService{}
+	return crypto.NewECDSAService(os.Getenv("AGENT_SECRET"))
 }
 
 func initCryptoService() portainer.CryptoService {
@@ -100,25 +104,126 @@ func initGitService() portainer.GitService {
 	return &git.Service{}
 }
 
-func initEndpointWatcher(endpointService portainer.EndpointService, externalEnpointFile string, syncInterval string) bool {
-	authorizeEndpointMgmt := true
-	if externalEnpointFile != "" {
-		authorizeEndpointMgmt = false
-		log.Println("Using external endpoint definition. Endpoint management via the API will be disabled.")
-		endpointWatcher := cron.NewWatcher(endpointService, syncInterval)
-		err := endpointWatcher.WatchEndpointFile(externalEnpointFile)
-		if err != nil {
-			log.Fatal(err)
-		}
-	}
-	return authorizeEndpointMgmt
+func initClientFactory(signatureService portainer.DigitalSignatureService) *docker.ClientFactory {
+	return docker.NewClientFactory(signatureService)
 }
 
-func initStatus(authorizeEndpointMgmt bool, flags *portainer.CLIFlags) *portainer.Status {
+func initSnapshotter(clientFactory *docker.ClientFactory) portainer.Snapshotter {
+	return docker.NewSnapshotter(clientFactory)
+}
+
+func initJobScheduler() portainer.JobScheduler {
+	return cron.NewJobScheduler()
+}
+
+func loadSnapshotSystemSchedule(jobScheduler portainer.JobScheduler, snapshotter portainer.Snapshotter, scheduleService portainer.ScheduleService, endpointService portainer.EndpointService, settingsService portainer.SettingsService) error {
+	settings, err := settingsService.Settings()
+	if err != nil {
+		return err
+	}
+
+	schedules, err := scheduleService.SchedulesByJobType(portainer.SnapshotJobType)
+	if err != nil {
+		return err
+	}
+
+	var snapshotSchedule *portainer.Schedule
+	if len(schedules) == 0 {
+		snapshotJob := &portainer.SnapshotJob{}
+		snapshotSchedule = &portainer.Schedule{
+			ID:             portainer.ScheduleID(scheduleService.GetNextIdentifier()),
+			Name:           "system_snapshot",
+			CronExpression: "@every " + settings.SnapshotInterval,
+			Recurring:      true,
+			JobType:        portainer.SnapshotJobType,
+			SnapshotJob:    snapshotJob,
+			Created:        time.Now().Unix(),
+		}
+	} else {
+		snapshotSchedule = &schedules[0]
+	}
+
+	snapshotJobContext := cron.NewSnapshotJobContext(endpointService, snapshotter)
+	snapshotJobRunner := cron.NewSnapshotJobRunner(snapshotSchedule, snapshotJobContext)
+
+	err = jobScheduler.ScheduleJob(snapshotJobRunner)
+	if err != nil {
+		return err
+	}
+
+	if len(schedules) == 0 {
+		return scheduleService.CreateSchedule(snapshotSchedule)
+	}
+	return nil
+}
+
+func loadEndpointSyncSystemSchedule(jobScheduler portainer.JobScheduler, scheduleService portainer.ScheduleService, endpointService portainer.EndpointService, flags *portainer.CLIFlags) error {
+	if *flags.ExternalEndpoints == "" {
+		return nil
+	}
+
+	log.Println("Using external endpoint definition. Endpoint management via the API will be disabled.")
+
+	schedules, err := scheduleService.SchedulesByJobType(portainer.EndpointSyncJobType)
+	if err != nil {
+		return err
+	}
+
+	if len(schedules) != 0 {
+		return nil
+	}
+
+	endpointSyncJob := &portainer.EndpointSyncJob{}
+
+	endpointSyncSchedule := &portainer.Schedule{
+		ID:              portainer.ScheduleID(scheduleService.GetNextIdentifier()),
+		Name:            "system_endpointsync",
+		CronExpression:  "@every " + *flags.SyncInterval,
+		Recurring:       true,
+		JobType:         portainer.EndpointSyncJobType,
+		EndpointSyncJob: endpointSyncJob,
+		Created:         time.Now().Unix(),
+	}
+
+	endpointSyncJobContext := cron.NewEndpointSyncJobContext(endpointService, *flags.ExternalEndpoints)
+	endpointSyncJobRunner := cron.NewEndpointSyncJobRunner(endpointSyncSchedule, endpointSyncJobContext)
+
+	err = jobScheduler.ScheduleJob(endpointSyncJobRunner)
+	if err != nil {
+		return err
+	}
+
+	return scheduleService.CreateSchedule(endpointSyncSchedule)
+}
+
+func loadSchedulesFromDatabase(jobScheduler portainer.JobScheduler, jobService portainer.JobService, scheduleService portainer.ScheduleService, endpointService portainer.EndpointService, fileService portainer.FileService) error {
+	schedules, err := scheduleService.Schedules()
+	if err != nil {
+		return err
+	}
+
+	for _, schedule := range schedules {
+
+		if schedule.JobType == portainer.ScriptExecutionJobType {
+			jobContext := cron.NewScriptExecutionJobContext(jobService, endpointService, fileService)
+			jobRunner := cron.NewScriptExecutionJobRunner(&schedule, jobContext)
+
+			err = jobScheduler.ScheduleJob(jobRunner)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func initStatus(endpointManagement, snapshot bool, flags *portainer.CLIFlags) *portainer.Status {
 	return &portainer.Status{
 		Analytics:          !*flags.NoAnalytics,
 		Authentication:     !*flags.NoAuth,
-		EndpointManagement: authorizeEndpointMgmt,
+		EndpointManagement: endpointManagement,
+		Snapshot:           snapshot,
 		Version:            portainer.APIVersion,
 	}
 }
@@ -143,23 +248,27 @@ func initSettings(settingsService portainer.SettingsService, flags *portainer.CL
 	_, err := settingsService.Settings()
 	if err == portainer.ErrObjectNotFound {
 		settings := &portainer.Settings{
-			LogoURL:                     *flags.Logo,
-			DisplayExternalContributors: false,
-			AuthenticationMethod:        portainer.AuthenticationInternal,
+			LogoURL:              *flags.Logo,
+			AuthenticationMethod: portainer.AuthenticationInternal,
 			LDAPSettings: portainer.LDAPSettings{
-				TLSConfig: portainer.TLSConfiguration{},
+				AutoCreateUsers: true,
+				TLSConfig:       portainer.TLSConfiguration{},
 				SearchSettings: []portainer.LDAPSearchSettings{
 					portainer.LDAPSearchSettings{},
 				},
+				GroupSearchSettings: []portainer.LDAPGroupSearchSettings{
+					portainer.LDAPGroupSearchSettings{},
+				},
 			},
+			OAuthSettings:                      portainer.OAuthSettings{},
 			AllowBindMountsForRegularUsers:     true,
 			AllowPrivilegedModeForRegularUsers: true,
+			EnableHostManagementFeatures:       false,
+			SnapshotInterval:                   *flags.SnapshotInterval,
 		}
 
 		if *flags.Templates != "" {
 			settings.TemplatesURL = *flags.Templates
-		} else {
-			settings.TemplatesURL = portainer.DefaultTemplatesURL
 		}
 
 		if *flags.Labels != nil {
@@ -176,6 +285,45 @@ func initSettings(settingsService portainer.SettingsService, flags *portainer.CL
 	return nil
 }
 
+func initTemplates(templateService portainer.TemplateService, fileService portainer.FileService, templateURL, templateFile string) error {
+	if templateURL != "" {
+		log.Printf("Portainer started with the --templates flag. Using external templates, template management will be disabled.")
+		return nil
+	}
+
+	existingTemplates, err := templateService.Templates()
+	if err != nil {
+		return err
+	}
+
+	if len(existingTemplates) != 0 {
+		log.Printf("Templates already registered inside the database. Skipping template import.")
+		return nil
+	}
+
+	templatesJSON, err := fileService.GetFileContent(templateFile)
+	if err != nil {
+		log.Println("Unable to retrieve template definitions via filesystem")
+		return err
+	}
+
+	var templates []portainer.Template
+	err = json.Unmarshal(templatesJSON, &templates)
+	if err != nil {
+		log.Println("Unable to parse templates file. Please review your template definition file.")
+		return err
+	}
+
+	for _, template := range templates {
+		err := templateService.CreateTemplate(&template)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func retrieveFirstEndpointFromDatabase(endpointService portainer.EndpointService) *portainer.Endpoint {
 	endpoints, err := endpointService.Endpoints()
 	if err != nil {
@@ -213,7 +361,7 @@ func initKeyPair(fileService portainer.FileService, signatureService portainer.D
 	return generateAndStoreKeyPair(fileService, signatureService)
 }
 
-func createTLSSecuredEndpoint(flags *portainer.CLIFlags, endpointService portainer.EndpointService) error {
+func createTLSSecuredEndpoint(flags *portainer.CLIFlags, endpointService portainer.EndpointService, snapshotter portainer.Snapshotter) error {
 	tlsConfiguration := portainer.TLSConfiguration{
 		TLS:           *flags.TLS,
 		TLSSkipVerify: *flags.TLSSkipVerify,
@@ -227,16 +375,20 @@ func createTLSSecuredEndpoint(flags *portainer.CLIFlags, endpointService portain
 		tlsConfiguration.TLS = true
 	}
 
+	endpointID := endpointService.GetNextIdentifier()
 	endpoint := &portainer.Endpoint{
-		Name:            "primary",
-		URL:             *flags.EndpointURL,
-		GroupID:         portainer.EndpointGroupID(1),
-		Type:            portainer.DockerEnvironment,
-		TLSConfig:       tlsConfiguration,
-		AuthorizedUsers: []portainer.UserID{},
-		AuthorizedTeams: []portainer.TeamID{},
-		Extensions:      []portainer.EndpointExtension{},
-		Tags:            []string{},
+		ID:                 portainer.EndpointID(endpointID),
+		Name:               "primary",
+		URL:                *flags.EndpointURL,
+		GroupID:            portainer.EndpointGroupID(1),
+		Type:               portainer.DockerEnvironment,
+		TLSConfig:          tlsConfiguration,
+		UserAccessPolicies: portainer.UserAccessPolicies{},
+		TeamAccessPolicies: portainer.TeamAccessPolicies{},
+		Extensions:         []portainer.EndpointExtension{},
+		Tags:               []string{},
+		Status:             portainer.EndpointStatusUp,
+		Snapshots:          []portainer.Snapshot{},
 	}
 
 	if strings.HasPrefix(endpoint.URL, "tcp://") {
@@ -255,10 +407,10 @@ func createTLSSecuredEndpoint(flags *portainer.CLIFlags, endpointService portain
 		}
 	}
 
-	return endpointService.CreateEndpoint(endpoint)
+	return snapshotAndPersistEndpoint(endpoint, endpointService, snapshotter)
 }
 
-func createUnsecuredEndpoint(endpointURL string, endpointService portainer.EndpointService) error {
+func createUnsecuredEndpoint(endpointURL string, endpointService portainer.EndpointService, snapshotter portainer.Snapshotter) error {
 	if strings.HasPrefix(endpointURL, "tcp://") {
 		_, err := client.ExecutePingOperation(endpointURL, nil)
 		if err != nil {
@@ -266,22 +418,40 @@ func createUnsecuredEndpoint(endpointURL string, endpointService portainer.Endpo
 		}
 	}
 
+	endpointID := endpointService.GetNextIdentifier()
 	endpoint := &portainer.Endpoint{
-		Name:            "primary",
-		URL:             endpointURL,
-		GroupID:         portainer.EndpointGroupID(1),
-		Type:            portainer.DockerEnvironment,
-		TLSConfig:       portainer.TLSConfiguration{},
-		AuthorizedUsers: []portainer.UserID{},
-		AuthorizedTeams: []portainer.TeamID{},
-		Extensions:      []portainer.EndpointExtension{},
-		Tags:            []string{},
+		ID:                 portainer.EndpointID(endpointID),
+		Name:               "primary",
+		URL:                endpointURL,
+		GroupID:            portainer.EndpointGroupID(1),
+		Type:               portainer.DockerEnvironment,
+		TLSConfig:          portainer.TLSConfiguration{},
+		UserAccessPolicies: portainer.UserAccessPolicies{},
+		TeamAccessPolicies: portainer.TeamAccessPolicies{},
+		Extensions:         []portainer.EndpointExtension{},
+		Tags:               []string{},
+		Status:             portainer.EndpointStatusUp,
+		Snapshots:          []portainer.Snapshot{},
+	}
+
+	return snapshotAndPersistEndpoint(endpoint, endpointService, snapshotter)
+}
+
+func snapshotAndPersistEndpoint(endpoint *portainer.Endpoint, endpointService portainer.EndpointService, snapshotter portainer.Snapshotter) error {
+	snapshot, err := snapshotter.CreateSnapshot(endpoint)
+	endpoint.Status = portainer.EndpointStatusUp
+	if err != nil {
+		log.Printf("http error: endpoint snapshot error (endpoint=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
+	}
+
+	if snapshot != nil {
+		endpoint.Snapshots = []portainer.Snapshot{*snapshot}
 	}
 
 	return endpointService.CreateEndpoint(endpoint)
 }
 
-func initEndpoint(flags *portainer.CLIFlags, endpointService portainer.EndpointService) error {
+func initEndpoint(flags *portainer.CLIFlags, endpointService portainer.EndpointService, snapshotter portainer.Snapshotter) error {
 	if *flags.EndpointURL == "" {
 		return nil
 	}
@@ -297,9 +467,49 @@ func initEndpoint(flags *portainer.CLIFlags, endpointService portainer.EndpointS
 	}
 
 	if *flags.TLS || *flags.TLSSkipVerify {
-		return createTLSSecuredEndpoint(flags, endpointService)
+		return createTLSSecuredEndpoint(flags, endpointService, snapshotter)
+	}
+	return createUnsecuredEndpoint(*flags.EndpointURL, endpointService, snapshotter)
+}
+
+func initJobService(dockerClientFactory *docker.ClientFactory) portainer.JobService {
+	return docker.NewJobService(dockerClientFactory)
+}
+
+func initExtensionManager(fileService portainer.FileService, extensionService portainer.ExtensionService) (portainer.ExtensionManager, error) {
+	extensionManager := exec.NewExtensionManager(fileService, extensionService)
+
+	extensions, err := extensionService.Extensions()
+	if err != nil {
+		return nil, err
+	}
+
+	for _, extension := range extensions {
+		err := extensionManager.EnableExtension(&extension, extension.License.LicenseKey)
+		if err != nil {
+			log.Printf("Unable to enable extension: %s [extension: %s]", err.Error(), extension.Name)
+			extension.Enabled = false
+			extension.License.Valid = false
+			extensionService.Persist(&extension)
+		}
+	}
+
+	return extensionManager, nil
+}
+
+func terminateIfNoAdminCreated(userService portainer.UserService) {
+	timer1 := time.NewTimer(5 * time.Minute)
+	<-timer1.C
+
+	users, err := userService.UsersByRole(portainer.AdministratorRole)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if len(users) == 0 {
+		log.Fatal("No administrator account was created after 5 min. Shutting down the Portainer instance for security reasons.")
+		return
 	}
-	return createUnsecuredEndpoint(*flags.EndpointURL, endpointService)
 }
 
 func main() {
@@ -312,21 +522,35 @@ func main() {
 
 	jwtService := initJWTService(!*flags.NoAuth)
 
-	cryptoService := initCryptoService()
-
-	digitalSignatureService := initDigitalSignatureService()
-
 	ldapService := initLDAPService()
 
 	gitService := initGitService()
 
-	authorizeEndpointMgmt := initEndpointWatcher(store.EndpointService, *flags.ExternalEndpoints, *flags.SyncInterval)
+	cryptoService := initCryptoService()
+
+	digitalSignatureService := initDigitalSignatureService()
 
 	err := initKeyPair(fileService, digitalSignatureService)
 	if err != nil {
 		log.Fatal(err)
 	}
 
+	extensionManager, err := initExtensionManager(fileService, store.ExtensionService)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	clientFactory := initClientFactory(digitalSignatureService)
+
+	jobService := initJobService(clientFactory)
+
+	snapshotter := initSnapshotter(clientFactory)
+
+	endpointManagement := true
+	if *flags.ExternalEndpoints != "" {
+		endpointManagement = false
+	}
+
 	swarmStackManager, err := initSwarmStackManager(*flags.Assets, *flags.Data, digitalSignatureService, fileService)
 	if err != nil {
 		log.Fatal(err)
@@ -334,19 +558,45 @@ func main() {
 
 	composeStackManager := initComposeStackManager(*flags.Data)
 
+	err = initTemplates(store.TemplateService, fileService, *flags.Templates, *flags.TemplateFile)
+	if err != nil {
+		log.Fatal(err)
+	}
+
 	err = initSettings(store.SettingsService, flags)
 	if err != nil {
 		log.Fatal(err)
 	}
 
+	jobScheduler := initJobScheduler()
+
+	err = loadSchedulesFromDatabase(jobScheduler, jobService, store.ScheduleService, store.EndpointService, fileService)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	err = loadEndpointSyncSystemSchedule(jobScheduler, store.ScheduleService, store.EndpointService, flags)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if *flags.Snapshot {
+		err = loadSnapshotSystemSchedule(jobScheduler, snapshotter, store.ScheduleService, store.EndpointService, store.SettingsService)
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	jobScheduler.Start()
+
 	err = initDockerHub(store.DockerHubService)
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	applicationStatus := initStatus(authorizeEndpointMgmt, flags)
+	applicationStatus := initStatus(endpointManagement, *flags.Snapshot, flags)
 
-	err = initEndpoint(flags, store.EndpointService)
+	err = initEndpoint(flags, store.EndpointService, snapshotter)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -357,7 +607,7 @@ func main() {
 		if err != nil {
 			log.Fatal(err)
 		}
-		adminPasswordHash, err = cryptoService.Hash(content)
+		adminPasswordHash, err = cryptoService.Hash(string(content))
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -377,6 +627,23 @@ func main() {
 				Username: "admin",
 				Role:     portainer.AdministratorRole,
 				Password: adminPasswordHash,
+				PortainerAuthorizations: map[portainer.Authorization]bool{
+					portainer.OperationPortainerDockerHubInspect:        true,
+					portainer.OperationPortainerEndpointGroupList:       true,
+					portainer.OperationPortainerEndpointList:            true,
+					portainer.OperationPortainerEndpointInspect:         true,
+					portainer.OperationPortainerEndpointExtensionAdd:    true,
+					portainer.OperationPortainerEndpointExtensionRemove: true,
+					portainer.OperationPortainerExtensionList:           true,
+					portainer.OperationPortainerMOTD:                    true,
+					portainer.OperationPortainerRegistryList:            true,
+					portainer.OperationPortainerRegistryInspect:         true,
+					portainer.OperationPortainerTeamList:                true,
+					portainer.OperationPortainerTemplateList:            true,
+					portainer.OperationPortainerTemplateInspect:         true,
+					portainer.OperationPortainerUserList:                true,
+					portainer.OperationPortainerUserMemberships:         true,
+				},
 			}
 			err := store.UserService.CreateUser(user)
 			if err != nil {
@@ -387,34 +654,55 @@ func main() {
 		}
 	}
 
+	if !*flags.NoAuth {
+		go terminateIfNoAdminCreated(store.UserService)
+	}
+
+	var tunnelServer portainer.TunnelServer = chisel.NewServer(*flags.TunnelAddr, *flags.TunnelPort)
+	err = tunnelServer.Start()
+	if err != nil {
+		log.Fatal(err)
+	}
+
 	var server portainer.Server = &http.Server{
-		Status:                 applicationStatus,
-		BindAddress:            *flags.Addr,
-		AssetsPath:             *flags.Assets,
-		AuthDisabled:           *flags.NoAuth,
-		EndpointManagement:     authorizeEndpointMgmt,
-		UserService:            store.UserService,
-		TeamService:            store.TeamService,
-		TeamMembershipService:  store.TeamMembershipService,
-		EndpointService:        store.EndpointService,
-		EndpointGroupService:   store.EndpointGroupService,
-		ResourceControlService: store.ResourceControlService,
-		SettingsService:        store.SettingsService,
-		RegistryService:        store.RegistryService,
-		DockerHubService:       store.DockerHubService,
-		StackService:           store.StackService,
-		TagService:             store.TagService,
-		SwarmStackManager:      swarmStackManager,
-		ComposeStackManager:    composeStackManager,
-		CryptoService:          cryptoService,
-		JWTService:             jwtService,
-		FileService:            fileService,
-		LDAPService:            ldapService,
-		GitService:             gitService,
-		SignatureService:       digitalSignatureService,
-		SSL:                    *flags.SSL,
-		SSLCert:                *flags.SSLCert,
-		SSLKey:                 *flags.SSLKey,
+		TunnelServerFingerprint: tunnelServer.GetFingerprint(),
+		Status:                  applicationStatus,
+		BindAddress:             *flags.Addr,
+		AssetsPath:              *flags.Assets,
+		AuthDisabled:            *flags.NoAuth,
+		EndpointManagement:      endpointManagement,
+		RoleService:             store.RoleService,
+		UserService:             store.UserService,
+		TeamService:             store.TeamService,
+		TeamMembershipService:   store.TeamMembershipService,
+		EndpointService:         store.EndpointService,
+		EndpointGroupService:    store.EndpointGroupService,
+		ExtensionService:        store.ExtensionService,
+		ResourceControlService:  store.ResourceControlService,
+		SettingsService:         store.SettingsService,
+		RegistryService:         store.RegistryService,
+		DockerHubService:        store.DockerHubService,
+		StackService:            store.StackService,
+		ScheduleService:         store.ScheduleService,
+		TagService:              store.TagService,
+		TemplateService:         store.TemplateService,
+		WebhookService:          store.WebhookService,
+		SwarmStackManager:       swarmStackManager,
+		ComposeStackManager:     composeStackManager,
+		ExtensionManager:        extensionManager,
+		CryptoService:           cryptoService,
+		JWTService:              jwtService,
+		FileService:             fileService,
+		LDAPService:             ldapService,
+		GitService:              gitService,
+		SignatureService:        digitalSignatureService,
+		JobScheduler:            jobScheduler,
+		Snapshotter:             snapshotter,
+		SSL:                     *flags.SSL,
+		SSLCert:                 *flags.SSLCert,
+		SSLKey:                  *flags.SSLKey,
+		DockerClientFactory:     clientFactory,
+		JobService:              jobService,
 	}
 
 	log.Printf("Starting Portainer %s on %s", portainer.APIVersion, *flags.Addr)

api/cron/job_endpoint_sync.go

@@ -4,52 +4,98 @@ import (
 	"encoding/json"
 	"io/ioutil"
 	"log"
-	"os"
 	"strings"
 
-	"github.com/portainer/portainer"
+	"github.com/portainer/portainer/api"
 )
 
-type (
-	endpointSyncJob struct {
-		logger           *log.Logger
-		endpointService  portainer.EndpointService
-		endpointFilePath string
-	}
+// EndpointSyncJobRunner is used to run a EndpointSyncJob
+type EndpointSyncJobRunner struct {
+	schedule *portainer.Schedule
+	context  *EndpointSyncJobContext
+}
 
-	synchronization struct {
-		endpointsToCreate []*portainer.Endpoint
-		endpointsToUpdate []*portainer.Endpoint
-		endpointsToDelete []*portainer.Endpoint
-	}
+// EndpointSyncJobContext represents the context of execution of a EndpointSyncJob
+type EndpointSyncJobContext struct {
+	endpointService  portainer.EndpointService
+	endpointFilePath string
+}
 
-	fileEndpoint struct {
-		Name          string `json:"Name"`
-		URL           string `json:"URL"`
-		TLS           bool   `json:"TLS,omitempty"`
-		TLSSkipVerify bool   `json:"TLSSkipVerify,omitempty"`
-		TLSCACert     string `json:"TLSCACert,omitempty"`
-		TLSCert       string `json:"TLSCert,omitempty"`
-		TLSKey        string `json:"TLSKey,omitempty"`
-	}
-)
-
-const (
-	// ErrEmptyEndpointArray is an error raised when the external endpoint source array is empty.
-	ErrEmptyEndpointArray = portainer.Error("External endpoint source is empty")
-)
-
-func newEndpointSyncJob(endpointFilePath string, endpointService portainer.EndpointService) endpointSyncJob {
-	return endpointSyncJob{
-		logger:           log.New(os.Stderr, "", log.LstdFlags),
+// NewEndpointSyncJobContext returns a new context that can be used to execute a EndpointSyncJob
+func NewEndpointSyncJobContext(endpointService portainer.EndpointService, endpointFilePath string) *EndpointSyncJobContext {
+	return &EndpointSyncJobContext{
 		endpointService:  endpointService,
 		endpointFilePath: endpointFilePath,
 	}
 }
 
-func endpointSyncError(err error, logger *log.Logger) bool {
+// NewEndpointSyncJobRunner returns a new runner that can be scheduled
+func NewEndpointSyncJobRunner(schedule *portainer.Schedule, context *EndpointSyncJobContext) *EndpointSyncJobRunner {
+	return &EndpointSyncJobRunner{
+		schedule: schedule,
+		context:  context,
+	}
+}
+
+type synchronization struct {
+	endpointsToCreate []*portainer.Endpoint
+	endpointsToUpdate []*portainer.Endpoint
+	endpointsToDelete []*portainer.Endpoint
+}
+
+type fileEndpoint struct {
+	Name          string `json:"Name"`
+	URL           string `json:"URL"`
+	TLS           bool   `json:"TLS,omitempty"`
+	TLSSkipVerify bool   `json:"TLSSkipVerify,omitempty"`
+	TLSCACert     string `json:"TLSCACert,omitempty"`
+	TLSCert       string `json:"TLSCert,omitempty"`
+	TLSKey        string `json:"TLSKey,omitempty"`
+}
+
+// GetSchedule returns the schedule associated to the runner
+func (runner *EndpointSyncJobRunner) GetSchedule() *portainer.Schedule {
+	return runner.schedule
+}
+
+// Run triggers the execution of the endpoint synchronization process.
+func (runner *EndpointSyncJobRunner) Run() {
+	data, err := ioutil.ReadFile(runner.context.endpointFilePath)
+	if endpointSyncError(err) {
+		return
+	}
+
+	var fileEndpoints []fileEndpoint
+	err = json.Unmarshal(data, &fileEndpoints)
+	if endpointSyncError(err) {
+		return
+	}
+
+	if len(fileEndpoints) == 0 {
+		log.Println("background job error (endpoint synchronization). External endpoint source is empty")
+		return
+	}
+
+	storedEndpoints, err := runner.context.endpointService.Endpoints()
+	if endpointSyncError(err) {
+		return
+	}
+
+	convertedFileEndpoints := convertFileEndpoints(fileEndpoints)
+
+	sync := prepareSyncData(storedEndpoints, convertedFileEndpoints)
+	if sync.requireSync() {
+		err = runner.context.endpointService.Synchronize(sync.endpointsToCreate, sync.endpointsToUpdate, sync.endpointsToDelete)
+		if endpointSyncError(err) {
+			return
+		}
+		log.Printf("Endpoint synchronization ended. [created: %v] [updated: %v] [deleted: %v]", len(sync.endpointsToCreate), len(sync.endpointsToUpdate), len(sync.endpointsToDelete))
+	}
+}
+
+func endpointSyncError(err error) bool {
 	if err != nil {
-		logger.Printf("Endpoint synchronization error: %s", err)
+		log.Printf("background job error (endpoint synchronization). Unable to synchronize endpoints (err=%s)\n", err)
 		return true
 	}
 	return false
@@ -129,8 +175,7 @@ func (sync synchronization) requireSync() bool {
 	return false
 }
 
-// TMP: endpointSyncJob method to access logger, should be generic
-func (job endpointSyncJob) prepareSyncData(storedEndpoints, fileEndpoints []portainer.Endpoint) *synchronization {
+func prepareSyncData(storedEndpoints, fileEndpoints []portainer.Endpoint) *synchronization {
 	endpointsToCreate := make([]*portainer.Endpoint, 0)
 	endpointsToUpdate := make([]*portainer.Endpoint, 0)
 	endpointsToDelete := make([]*portainer.Endpoint, 0)
@@ -140,23 +185,23 @@ func (job endpointSyncJob) prepareSyncData(storedEndpoints, fileEndpoints []port
 		if fidx != -1 {
 			endpoint := mergeEndpointIfRequired(&storedEndpoints[idx], &fileEndpoints[fidx])
 			if endpoint != nil {
-				job.logger.Printf("New definition for a stored endpoint found in file, updating database. [name: %v] [url: %v]\n", endpoint.Name, endpoint.URL)
+				log.Printf("New definition for a stored endpoint found in file, updating database. [name: %v] [url: %v]\n", endpoint.Name, endpoint.URL)
 				endpointsToUpdate = append(endpointsToUpdate, endpoint)
 			}
 		} else {
-			job.logger.Printf("Stored endpoint not found in file (definition might be invalid), removing from database. [name: %v] [url: %v]", storedEndpoints[idx].Name, storedEndpoints[idx].URL)
+			log.Printf("Stored endpoint not found in file (definition might be invalid), removing from database. [name: %v] [url: %v]", storedEndpoints[idx].Name, storedEndpoints[idx].URL)
 			endpointsToDelete = append(endpointsToDelete, &storedEndpoints[idx])
 		}
 	}
 
 	for idx, endpoint := range fileEndpoints {
 		if !isValidEndpoint(&endpoint) {
-			job.logger.Printf("Invalid file endpoint definition, skipping. [name: %v] [url: %v]", endpoint.Name, endpoint.URL)
+			log.Printf("Invalid file endpoint definition, skipping. [name: %v] [url: %v]", endpoint.Name, endpoint.URL)
 			continue
 		}
 		sidx := endpointExists(&fileEndpoints[idx], storedEndpoints)
 		if sidx == -1 {
-			job.logger.Printf("File endpoint not found in database, adding to database. [name: %v] [url: %v]", fileEndpoints[idx].Name, fileEndpoints[idx].URL)
+			log.Printf("File endpoint not found in database, adding to database. [name: %v] [url: %v]", fileEndpoints[idx].Name, fileEndpoints[idx].URL)
 			endpointsToCreate = append(endpointsToCreate, &fileEndpoints[idx])
 		}
 	}
@@ -167,43 +212,3 @@ func (job endpointSyncJob) prepareSyncData(storedEndpoints, fileEndpoints []port
 		endpointsToDelete: endpointsToDelete,
 	}
 }
-
-func (job endpointSyncJob) Sync() error {
-	data, err := ioutil.ReadFile(job.endpointFilePath)
-	if endpointSyncError(err, job.logger) {
-		return err
-	}
-
-	var fileEndpoints []fileEndpoint
-	err = json.Unmarshal(data, &fileEndpoints)
-	if endpointSyncError(err, job.logger) {
-		return err
-	}
-
-	if len(fileEndpoints) == 0 {
-		return ErrEmptyEndpointArray
-	}
-
-	storedEndpoints, err := job.endpointService.Endpoints()
-	if endpointSyncError(err, job.logger) {
-		return err
-	}
-
-	convertedFileEndpoints := convertFileEndpoints(fileEndpoints)
-
-	sync := job.prepareSyncData(storedEndpoints, convertedFileEndpoints)
-	if sync.requireSync() {
-		err = job.endpointService.Synchronize(sync.endpointsToCreate, sync.endpointsToUpdate, sync.endpointsToDelete)
-		if endpointSyncError(err, job.logger) {
-			return err
-		}
-		job.logger.Printf("Endpoint synchronization ended. [created: %v] [updated: %v] [deleted: %v]", len(sync.endpointsToCreate), len(sync.endpointsToUpdate), len(sync.endpointsToDelete))
-	}
-	return nil
-}
-
-func (job endpointSyncJob) Run() {
-	job.logger.Println("Endpoint synchronization job started.")
-	err := job.Sync()
-	endpointSyncError(err, job.logger)
-}

api/cron/job_script_execution.go

@@ -0,0 +1,96 @@
+package cron
+
+import (
+	"log"
+	"time"
+
+	"github.com/portainer/portainer/api"
+)
+
+// ScriptExecutionJobRunner is used to run a ScriptExecutionJob
+type ScriptExecutionJobRunner struct {
+	schedule     *portainer.Schedule
+	context      *ScriptExecutionJobContext
+	executedOnce bool
+}
+
+// ScriptExecutionJobContext represents the context of execution of a ScriptExecutionJob
+type ScriptExecutionJobContext struct {
+	jobService      portainer.JobService
+	endpointService portainer.EndpointService
+	fileService     portainer.FileService
+}
+
+// NewScriptExecutionJobContext returns a new context that can be used to execute a ScriptExecutionJob
+func NewScriptExecutionJobContext(jobService portainer.JobService, endpointService portainer.EndpointService, fileService portainer.FileService) *ScriptExecutionJobContext {
+	return &ScriptExecutionJobContext{
+		jobService:      jobService,
+		endpointService: endpointService,
+		fileService:     fileService,
+	}
+}
+
+// NewScriptExecutionJobRunner returns a new runner that can be scheduled
+func NewScriptExecutionJobRunner(schedule *portainer.Schedule, context *ScriptExecutionJobContext) *ScriptExecutionJobRunner {
+	return &ScriptExecutionJobRunner{
+		schedule:     schedule,
+		context:      context,
+		executedOnce: false,
+	}
+}
+
+// Run triggers the execution of the job.
+// It will iterate through all the endpoints specified in the context to
+// execute the script associated to the job.
+func (runner *ScriptExecutionJobRunner) Run() {
+	if !runner.schedule.Recurring && runner.executedOnce {
+		return
+	}
+	runner.executedOnce = true
+
+	scriptFile, err := runner.context.fileService.GetFileContent(runner.schedule.ScriptExecutionJob.ScriptPath)
+	if err != nil {
+		log.Printf("scheduled job error (script execution). Unable to retrieve script file (err=%s)\n", err)
+		return
+	}
+
+	targets := make([]*portainer.Endpoint, 0)
+	for _, endpointID := range runner.schedule.ScriptExecutionJob.Endpoints {
+		endpoint, err := runner.context.endpointService.Endpoint(endpointID)
+		if err != nil {
+			log.Printf("scheduled job error (script execution). Unable to retrieve information about endpoint (id=%d) (err=%s)\n", endpointID, err)
+			return
+		}
+
+		targets = append(targets, endpoint)
+	}
+
+	runner.executeAndRetry(targets, scriptFile, 0)
+}
+
+func (runner *ScriptExecutionJobRunner) executeAndRetry(endpoints []*portainer.Endpoint, script []byte, retryCount int) {
+	retryTargets := make([]*portainer.Endpoint, 0)
+
+	for _, endpoint := range endpoints {
+		err := runner.context.jobService.ExecuteScript(endpoint, "", runner.schedule.ScriptExecutionJob.Image, script, runner.schedule)
+		if err == portainer.ErrUnableToPingEndpoint {
+			retryTargets = append(retryTargets, endpoint)
+		} else if err != nil {
+			log.Printf("scheduled job error (script execution). Unable to execute script (endpoint=%s) (err=%s)\n", endpoint.Name, err)
+		}
+	}
+
+	retryCount++
+	if retryCount >= runner.schedule.ScriptExecutionJob.RetryCount {
+		return
+	}
+
+	time.Sleep(time.Duration(runner.schedule.ScriptExecutionJob.RetryInterval) * time.Second)
+
+	runner.executeAndRetry(retryTargets, script, retryCount)
+}
+
+// GetSchedule returns the schedule associated to the runner
+func (runner *ScriptExecutionJobRunner) GetSchedule() *portainer.Schedule {
+	return runner.schedule
+}

api/cron/job_snapshot.go

@@ -0,0 +1,85 @@
+package cron
+
+import (
+	"log"
+
+	"github.com/portainer/portainer/api"
+)
+
+// SnapshotJobRunner is used to run a SnapshotJob
+type SnapshotJobRunner struct {
+	schedule *portainer.Schedule
+	context  *SnapshotJobContext
+}
+
+// SnapshotJobContext represents the context of execution of a SnapshotJob
+type SnapshotJobContext struct {
+	endpointService portainer.EndpointService
+	snapshotter     portainer.Snapshotter
+}
+
+// NewSnapshotJobContext returns a new context that can be used to execute a SnapshotJob
+func NewSnapshotJobContext(endpointService portainer.EndpointService, snapshotter portainer.Snapshotter) *SnapshotJobContext {
+	return &SnapshotJobContext{
+		endpointService: endpointService,
+		snapshotter:     snapshotter,
+	}
+}
+
+// NewSnapshotJobRunner returns a new runner that can be scheduled
+func NewSnapshotJobRunner(schedule *portainer.Schedule, context *SnapshotJobContext) *SnapshotJobRunner {
+	return &SnapshotJobRunner{
+		schedule: schedule,
+		context:  context,
+	}
+}
+
+// GetSchedule returns the schedule associated to the runner
+func (runner *SnapshotJobRunner) GetSchedule() *portainer.Schedule {
+	return runner.schedule
+}
+
+// Run triggers the execution of the schedule.
+// It will iterate through all the endpoints available in the database to
+// create a snapshot of each one of them.
+// As a snapshot can be a long process, to avoid any concurrency issue we
+// retrieve the latest version of the endpoint right after a snapshot.
+func (runner *SnapshotJobRunner) Run() {
+	go func() {
+		endpoints, err := runner.context.endpointService.Endpoints()
+		if err != nil {
+			log.Printf("background schedule error (endpoint snapshot). Unable to retrieve endpoint list (err=%s)\n", err)
+			return
+		}
+
+		for _, endpoint := range endpoints {
+			if endpoint.Type == portainer.AzureEnvironment {
+				continue
+			}
+
+			snapshot, snapshotError := runner.context.snapshotter.CreateSnapshot(&endpoint)
+
+			latestEndpointReference, err := runner.context.endpointService.Endpoint(endpoint.ID)
+			if latestEndpointReference == nil {
+				log.Printf("background schedule error (endpoint snapshot). Endpoint not found inside the database anymore (endpoint=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
+				continue
+			}
+
+			latestEndpointReference.Status = portainer.EndpointStatusUp
+			if snapshotError != nil {
+				log.Printf("background schedule error (endpoint snapshot). Unable to create snapshot (endpoint=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, snapshotError)
+				latestEndpointReference.Status = portainer.EndpointStatusDown
+			}
+
+			if snapshot != nil {
+				latestEndpointReference.Snapshots = []portainer.Snapshot{*snapshot}
+			}
+
+			err = runner.context.endpointService.UpdateEndpoint(latestEndpointReference.ID, latestEndpointReference)
+			if err != nil {
+				log.Printf("background schedule error (endpoint snapshot). Unable to update endpoint (endpoint=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
+				return
+			}
+		}
+	}()
+}

api/cron/scheduler.go

@@ -0,0 +1,115 @@
+package cron
+
+import (
+	"github.com/portainer/portainer/api"
+	"github.com/robfig/cron"
+)
+
+// JobScheduler represents a service for managing crons
+type JobScheduler struct {
+	cron *cron.Cron
+}
+
+// NewJobScheduler initializes a new service
+func NewJobScheduler() *JobScheduler {
+	return &JobScheduler{
+		cron: cron.New(),
+	}
+}
+
+// ScheduleJob schedules the execution of a job via a runner
+func (scheduler *JobScheduler) ScheduleJob(runner portainer.JobRunner) error {
+	return scheduler.cron.AddJob(runner.GetSchedule().CronExpression, runner)
+}
+
+// UpdateSystemJobSchedule updates the first occurence of the specified
+// scheduled job based on the specified job type.
+// It does so by re-creating a new cron
+// and adding all the existing jobs. It will then re-schedule the new job
+// with the update cron expression passed in parameter.
+// NOTE: the cron library do not support updating schedules directly
+// hence the work-around
+func (scheduler *JobScheduler) UpdateSystemJobSchedule(jobType portainer.JobType, newCronExpression string) error {
+	cronEntries := scheduler.cron.Entries()
+	newCron := cron.New()
+
+	for _, entry := range cronEntries {
+		if entry.Job.(portainer.JobRunner).GetSchedule().JobType == jobType {
+			err := newCron.AddJob(newCronExpression, entry.Job)
+			if err != nil {
+				return err
+			}
+			continue
+		}
+
+		newCron.Schedule(entry.Schedule, entry.Job)
+	}
+
+	scheduler.cron.Stop()
+	scheduler.cron = newCron
+	scheduler.cron.Start()
+	return nil
+}
+
+// UpdateJobSchedule updates a specific scheduled job by re-creating a new cron
+// and adding all the existing jobs. It will then re-schedule the new job
+// via the specified JobRunner parameter.
+// NOTE: the cron library do not support updating schedules directly
+// hence the work-around
+func (scheduler *JobScheduler) UpdateJobSchedule(runner portainer.JobRunner) error {
+	cronEntries := scheduler.cron.Entries()
+	newCron := cron.New()
+
+	for _, entry := range cronEntries {
+
+		if entry.Job.(portainer.JobRunner).GetSchedule().ID == runner.GetSchedule().ID {
+
+			var jobRunner cron.Job = runner
+			if entry.Job.(portainer.JobRunner).GetSchedule().JobType == portainer.SnapshotJobType {
+				jobRunner = entry.Job
+			}
+
+			err := newCron.AddJob(runner.GetSchedule().CronExpression, jobRunner)
+			if err != nil {
+				return err
+			}
+			continue
+		}
+
+		newCron.Schedule(entry.Schedule, entry.Job)
+	}
+
+	scheduler.cron.Stop()
+	scheduler.cron = newCron
+	scheduler.cron.Start()
+	return nil
+}
+
+// UnscheduleJob remove a scheduled job by re-creating a new cron
+// and adding all the existing jobs except for the one specified via scheduleID.
+// NOTE: the cron library do not support removing schedules directly
+// hence the work-around
+func (scheduler *JobScheduler) UnscheduleJob(scheduleID portainer.ScheduleID) {
+	cronEntries := scheduler.cron.Entries()
+	newCron := cron.New()
+
+	for _, entry := range cronEntries {
+
+		if entry.Job.(portainer.JobRunner).GetSchedule().ID == scheduleID {
+			continue
+		}
+
+		newCron.Schedule(entry.Schedule, entry.Job)
+	}
+
+	scheduler.cron.Stop()
+	scheduler.cron = newCron
+	scheduler.cron.Start()
+}
+
+// Start starts the scheduled jobs
+func (scheduler *JobScheduler) Start() {
+	if len(scheduler.cron.Entries()) > 0 {
+		scheduler.cron.Start()
+	}
+}

api/cron/watcher.go

@@ -1,40 +0,0 @@
-package cron
-
-import (
-	"github.com/portainer/portainer"
-	"github.com/robfig/cron"
-)
-
-// Watcher represents a service for managing crons.
-type Watcher struct {
-	Cron            *cron.Cron
-	EndpointService portainer.EndpointService
-	syncInterval    string
-}
-
-// NewWatcher initializes a new service.
-func NewWatcher(endpointService portainer.EndpointService, syncInterval string) *Watcher {
-	return &Watcher{
-		Cron:            cron.New(),
-		EndpointService: endpointService,
-		syncInterval:    syncInterval,
-	}
-}
-
-// WatchEndpointFile starts a cron job to synchronize the endpoints from a file
-func (watcher *Watcher) WatchEndpointFile(endpointFilePath string) error {
-	job := newEndpointSyncJob(endpointFilePath, watcher.EndpointService)
-
-	err := job.Sync()
-	if err != nil {
-		return err
-	}
-
-	err = watcher.Cron.AddJob("@every "+watcher.syncInterval, job)
-	if err != nil {
-		return err
-	}
-
-	watcher.Cron.Start()
-	return nil
-}

api/crypto/ecdsa.go

@@ -3,7 +3,6 @@ package crypto
 import (
 	"crypto/ecdsa"
 	"crypto/elliptic"
-	"crypto/md5"
 	"crypto/rand"
 	"crypto/x509"
 	"encoding/base64"
@@ -27,6 +26,15 @@ type ECDSAService struct {
 	privateKey    *ecdsa.PrivateKey
 	publicKey     *ecdsa.PublicKey
 	encodedPubKey string
+	secret        string
+}
+
+// NewECDSAService returns a pointer to a ECDSAService.
+// An optional secret can be specified
+func NewECDSAService(secret string) *ECDSAService {
+	return &ECDSAService{
+		secret: secret,
+	}
 }
 
 // EncodedPublicKey returns the encoded version of the public that can be used
@@ -92,14 +100,18 @@ func (service *ECDSAService) GenerateKeyPair() ([]byte, []byte, error) {
 	return private, public, nil
 }
 
-// Sign creates a signature from a message.
-// It automatically hash the message using MD5 and creates a signature from
+// CreateSignature creates a digital signature.
+// It automatically hash a specific message using MD5 and creates a signature from
 // that hash.
+// If a secret is associated to the service, it will be used instead of the specified
+// message.
 // It then encodes the generated signature in base64.
-func (service *ECDSAService) Sign(message string) (string, error) {
-	digest := md5.New()
-	digest.Write([]byte(message))
-	hash := digest.Sum(nil)
+func (service *ECDSAService) CreateSignature(message string) (string, error) {
+	if service.secret != "" {
+		message = service.secret
+	}
+
+	hash := HashFromBytes([]byte(message))
 
 	r := big.NewInt(0)
 	s := big.NewInt(0)

api/crypto/md5.go

@@ -0,0 +1,10 @@
+package crypto
+
+import "crypto/md5"
+
+// HashFromBytes returns the hash of the specified data
+func HashFromBytes(data []byte) []byte {
+	digest := md5.New()
+	digest.Write(data)
+	return digest.Sum(nil)
+}

api/docker/client.go

@@ -0,0 +1,108 @@
+package docker
+
+import (
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/docker/docker/client"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/crypto"
+)
+
+const (
+	unsupportedEnvironmentType = portainer.Error("Environment not supported")
+)
+
+// ClientFactory is used to create Docker clients
+type ClientFactory struct {
+	signatureService portainer.DigitalSignatureService
+}
+
+// NewClientFactory returns a new instance of a ClientFactory
+func NewClientFactory(signatureService portainer.DigitalSignatureService) *ClientFactory {
+	return &ClientFactory{
+		signatureService: signatureService,
+	}
+}
+
+// CreateClient is a generic function to create a Docker client based on
+// a specific endpoint configuration. The nodeName parameter can be used
+// with an agent enabled endpoint to target a specific node in an agent cluster.
+func (factory *ClientFactory) CreateClient(endpoint *portainer.Endpoint, nodeName string) (*client.Client, error) {
+	if endpoint.Type == portainer.AzureEnvironment {
+		return nil, unsupportedEnvironmentType
+	} else if endpoint.Type == portainer.AgentOnDockerEnvironment {
+		return createAgentClient(endpoint, factory.signatureService, nodeName)
+	}
+
+	if strings.HasPrefix(endpoint.URL, "unix://") || strings.HasPrefix(endpoint.URL, "npipe://") {
+		return createLocalClient(endpoint)
+	}
+	return createTCPClient(endpoint)
+}
+
+func createLocalClient(endpoint *portainer.Endpoint) (*client.Client, error) {
+	return client.NewClientWithOpts(
+		client.WithHost(endpoint.URL),
+		client.WithVersion(portainer.SupportedDockerAPIVersion),
+	)
+}
+
+func createTCPClient(endpoint *portainer.Endpoint) (*client.Client, error) {
+	httpCli, err := httpClient(endpoint)
+	if err != nil {
+		return nil, err
+	}
+
+	return client.NewClientWithOpts(
+		client.WithHost(endpoint.URL),
+		client.WithVersion(portainer.SupportedDockerAPIVersion),
+		client.WithHTTPClient(httpCli),
+	)
+}
+
+func createAgentClient(endpoint *portainer.Endpoint, signatureService portainer.DigitalSignatureService, nodeName string) (*client.Client, error) {
+	httpCli, err := httpClient(endpoint)
+	if err != nil {
+		return nil, err
+	}
+
+	signature, err := signatureService.CreateSignature(portainer.PortainerAgentSignatureMessage)
+	if err != nil {
+		return nil, err
+	}
+
+	headers := map[string]string{
+		portainer.PortainerAgentPublicKeyHeader: signatureService.EncodedPublicKey(),
+		portainer.PortainerAgentSignatureHeader: signature,
+	}
+
+	if nodeName != "" {
+		headers[portainer.PortainerAgentTargetHeader] = nodeName
+	}
+
+	return client.NewClientWithOpts(
+		client.WithHost(endpoint.URL),
+		client.WithVersion(portainer.SupportedDockerAPIVersion),
+		client.WithHTTPClient(httpCli),
+		client.WithHTTPHeaders(headers),
+	)
+}
+
+func httpClient(endpoint *portainer.Endpoint) (*http.Client, error) {
+	transport := &http.Transport{}
+
+	if endpoint.TLSConfig.TLS {
+		tlsConfig, err := crypto.CreateTLSConfigurationFromDisk(endpoint.TLSConfig.TLSCACertPath, endpoint.TLSConfig.TLSCertPath, endpoint.TLSConfig.TLSKeyPath, endpoint.TLSConfig.TLSSkipVerify)
+		if err != nil {
+			return nil, err
+		}
+		transport.TLSClientConfig = tlsConfig
+	}
+
+	return &http.Client{
+		Transport: transport,
+		Timeout:   30 * time.Second,
+	}, nil
+}

api/docker/job.go

@@ -0,0 +1,115 @@
+package docker
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"io/ioutil"
+	"strconv"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/network"
+	"github.com/docker/docker/api/types/strslice"
+	"github.com/docker/docker/client"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/archive"
+)
+
+// JobService represents a service that handles the execution of jobs
+type JobService struct {
+	dockerClientFactory *ClientFactory
+}
+
+// NewJobService returns a pointer to a new job service
+func NewJobService(dockerClientFactory *ClientFactory) *JobService {
+	return &JobService{
+		dockerClientFactory: dockerClientFactory,
+	}
+}
+
+// ExecuteScript will leverage a privileged container to execute a script against the specified endpoint/nodename.
+// It will copy the script content specified as a parameter inside a container based on the specified image and execute it.
+func (service *JobService) ExecuteScript(endpoint *portainer.Endpoint, nodeName, image string, script []byte, schedule *portainer.Schedule) error {
+	buffer, err := archive.TarFileInBuffer(script, "script.sh", 0700)
+	if err != nil {
+		return err
+	}
+
+	cli, err := service.dockerClientFactory.CreateClient(endpoint, nodeName)
+	if err != nil {
+		return err
+	}
+	defer cli.Close()
+
+	_, err = cli.Ping(context.Background())
+	if err != nil {
+		return portainer.ErrUnableToPingEndpoint
+	}
+
+	err = pullImage(cli, image)
+	if err != nil {
+		return err
+	}
+
+	containerConfig := &container.Config{
+		AttachStdin:  true,
+		AttachStdout: true,
+		AttachStderr: true,
+		Tty:          true,
+		WorkingDir:   "/tmp",
+		Image:        image,
+		Labels: map[string]string{
+			"io.portainer.job.endpoint": strconv.Itoa(int(endpoint.ID)),
+		},
+		Cmd: strslice.StrSlice([]string{"sh", "/tmp/script.sh"}),
+	}
+
+	if schedule != nil {
+		containerConfig.Labels["io.portainer.schedule.id"] = strconv.Itoa(int(schedule.ID))
+	}
+
+	hostConfig := &container.HostConfig{
+		Binds:       []string{"/:/host", "/etc:/etc:ro", "/usr:/usr:ro", "/run:/run:ro", "/sbin:/sbin:ro", "/var:/var:ro"},
+		NetworkMode: "host",
+		Privileged:  true,
+	}
+
+	networkConfig := &network.NetworkingConfig{}
+
+	body, err := cli.ContainerCreate(context.Background(), containerConfig, hostConfig, networkConfig, "")
+	if err != nil {
+		return err
+	}
+
+	if schedule != nil {
+		err = cli.ContainerRename(context.Background(), body.ID, schedule.Name+"_"+body.ID)
+		if err != nil {
+			return err
+		}
+	}
+
+	copyOptions := types.CopyToContainerOptions{}
+	err = cli.CopyToContainer(context.Background(), body.ID, "/tmp", bytes.NewReader(buffer), copyOptions)
+	if err != nil {
+		return err
+	}
+
+	startOptions := types.ContainerStartOptions{}
+	return cli.ContainerStart(context.Background(), body.ID, startOptions)
+}
+
+func pullImage(cli *client.Client, image string) error {
+	imageReadCloser, err := cli.ImagePull(context.Background(), image, types.ImagePullOptions{})
+	if err != nil {
+		return err
+	}
+	defer imageReadCloser.Close()
+
+	_, err = io.Copy(ioutil.Discard, imageReadCloser)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

api/docker/snapshot.go

@@ -0,0 +1,188 @@
+package docker
+
+import (
+	"context"
+	"time"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/client"
+	"github.com/portainer/portainer/api"
+)
+
+func snapshot(cli *client.Client) (*portainer.Snapshot, error) {
+	_, err := cli.Ping(context.Background())
+	if err != nil {
+		return nil, err
+	}
+
+	snapshot := &portainer.Snapshot{
+		StackCount: 0,
+	}
+
+	err = snapshotInfo(snapshot, cli)
+	if err != nil {
+		return nil, err
+	}
+
+	if snapshot.Swarm {
+		err = snapshotSwarmServices(snapshot, cli)
+		if err != nil {
+			return nil, err
+		}
+
+		err = snapshotNodes(snapshot, cli)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	err = snapshotContainers(snapshot, cli)
+	if err != nil {
+		return nil, err
+	}
+
+	err = snapshotImages(snapshot, cli)
+	if err != nil {
+		return nil, err
+	}
+
+	err = snapshotVolumes(snapshot, cli)
+	if err != nil {
+		return nil, err
+	}
+
+	err = snapshotNetworks(snapshot, cli)
+	if err != nil {
+		return nil, err
+	}
+
+	err = snapshotVersion(snapshot, cli)
+	if err != nil {
+		return nil, err
+	}
+
+	snapshot.Time = time.Now().Unix()
+	return snapshot, nil
+}
+
+func snapshotInfo(snapshot *portainer.Snapshot, cli *client.Client) error {
+	info, err := cli.Info(context.Background())
+	if err != nil {
+		return err
+	}
+
+	snapshot.Swarm = info.Swarm.ControlAvailable
+	snapshot.DockerVersion = info.ServerVersion
+	snapshot.TotalCPU = info.NCPU
+	snapshot.TotalMemory = info.MemTotal
+	snapshot.SnapshotRaw.Info = info
+	return nil
+}
+
+func snapshotNodes(snapshot *portainer.Snapshot, cli *client.Client) error {
+	nodes, err := cli.NodeList(context.Background(), types.NodeListOptions{})
+	if err != nil {
+		return err
+	}
+	var nanoCpus int64
+	var totalMem int64
+	for _, node := range nodes {
+		nanoCpus += node.Description.Resources.NanoCPUs
+		totalMem += node.Description.Resources.MemoryBytes
+	}
+	snapshot.TotalCPU = int(nanoCpus / 1e9)
+	snapshot.TotalMemory = totalMem
+	return nil
+}
+
+func snapshotSwarmServices(snapshot *portainer.Snapshot, cli *client.Client) error {
+	stacks := make(map[string]struct{})
+
+	services, err := cli.ServiceList(context.Background(), types.ServiceListOptions{})
+	if err != nil {
+		return err
+	}
+
+	for _, service := range services {
+		for k, v := range service.Spec.Labels {
+			if k == "com.docker.stack.namespace" {
+				stacks[v] = struct{}{}
+			}
+		}
+	}
+
+	snapshot.ServiceCount = len(services)
+	snapshot.StackCount += len(stacks)
+	return nil
+}
+
+func snapshotContainers(snapshot *portainer.Snapshot, cli *client.Client) error {
+	containers, err := cli.ContainerList(context.Background(), types.ContainerListOptions{All: true})
+	if err != nil {
+		return err
+	}
+
+	runningContainers := 0
+	stoppedContainers := 0
+	stacks := make(map[string]struct{})
+	for _, container := range containers {
+		if container.State == "exited" {
+			stoppedContainers++
+		} else if container.State == "running" {
+			runningContainers++
+		}
+
+		for k, v := range container.Labels {
+			if k == "com.docker.compose.project" {
+				stacks[v] = struct{}{}
+			}
+		}
+	}
+
+	snapshot.RunningContainerCount = runningContainers
+	snapshot.StoppedContainerCount = stoppedContainers
+	snapshot.StackCount += len(stacks)
+	snapshot.SnapshotRaw.Containers = containers
+	return nil
+}
+
+func snapshotImages(snapshot *portainer.Snapshot, cli *client.Client) error {
+	images, err := cli.ImageList(context.Background(), types.ImageListOptions{})
+	if err != nil {
+		return err
+	}
+
+	snapshot.ImageCount = len(images)
+	snapshot.SnapshotRaw.Images = images
+	return nil
+}
+
+func snapshotVolumes(snapshot *portainer.Snapshot, cli *client.Client) error {
+	volumes, err := cli.VolumeList(context.Background(), filters.Args{})
+	if err != nil {
+		return err
+	}
+
+	snapshot.VolumeCount = len(volumes.Volumes)
+	snapshot.SnapshotRaw.Volumes = volumes
+	return nil
+}
+
+func snapshotNetworks(snapshot *portainer.Snapshot, cli *client.Client) error {
+	networks, err := cli.NetworkList(context.Background(), types.NetworkListOptions{})
+	if err != nil {
+		return err
+	}
+	snapshot.SnapshotRaw.Networks = networks
+	return nil
+}
+
+func snapshotVersion(snapshot *portainer.Snapshot, cli *client.Client) error {
+	version, err := cli.ServerVersion(context.Background())
+	if err != nil {
+		return err
+	}
+	snapshot.SnapshotRaw.Version = version
+	return nil
+}

api/docker/snapshotter.go

@@ -0,0 +1,28 @@
+package docker
+
+import (
+	"github.com/portainer/portainer/api"
+)
+
+// Snapshotter represents a service used to create endpoint snapshots
+type Snapshotter struct {
+	clientFactory *ClientFactory
+}
+
+// NewSnapshotter returns a new Snapshotter instance
+func NewSnapshotter(clientFactory *ClientFactory) *Snapshotter {
+	return &Snapshotter{
+		clientFactory: clientFactory,
+	}
+}
+
+// CreateSnapshot creates a snapshot of a specific endpoint
+func (snapshotter *Snapshotter) CreateSnapshot(endpoint *portainer.Endpoint) (*portainer.Snapshot, error) {
+	cli, err := snapshotter.clientFactory.CreateClient(endpoint, "")
+	if err != nil {
+		return nil, err
+	}
+	defer cli.Close()
+
+	return snapshot(cli)
+}

api/errors.go

@@ -4,16 +4,18 @@ package portainer
 const (
 	ErrUnauthorized           = Error("Unauthorized")
 	ErrResourceAccessDenied   = Error("Access denied to resource")
+	ErrAuthorizationRequired  = Error("Authorization required for this operation")
 	ErrObjectNotFound         = Error("Object not found inside the database")
 	ErrMissingSecurityContext = Error("Unable to find security details in request context")
 )
 
 // User errors.
 const (
-	ErrUserAlreadyExists       = Error("User already exists")
-	ErrInvalidUsername         = Error("Invalid username. White spaces are not allowed")
-	ErrAdminAlreadyInitialized = Error("An administrator user already exists")
-	ErrAdminCannotRemoveSelf   = Error("Cannot remove your own user account. Contact another administrator")
+	ErrUserAlreadyExists          = Error("User already exists")
+	ErrInvalidUsername            = Error("Invalid username. White spaces are not allowed")
+	ErrAdminAlreadyInitialized    = Error("An administrator user already exists")
+	ErrAdminCannotRemoveSelf      = Error("Cannot remove your own user account. Contact another administrator")
+	ErrCannotRemoveLastLocalAdmin = Error("Cannot remove the last local administrator account")
 )
 
 // Team errors.
@@ -87,8 +89,29 @@ const (
 	ErrUndefinedTLSFileType = Error("Undefined TLS file type")
 )
 
+// Extension errors.
+const (
+	ErrExtensionAlreadyEnabled = Error("This extension is already enabled")
+)
+
+// Docker errors.
+const (
+	ErrUnableToPingEndpoint = Error("Unable to communicate with the endpoint")
+)
+
+// Schedule errors.
+const (
+	ErrHostManagementFeaturesDisabled = Error("Host management features are disabled")
+)
+
 // Error represents an application error.
 type Error string
 
 // Error returns the error message.
 func (e Error) Error() string { return string(e) }
+
+// Webhook errors
+const (
+	ErrWebhookAlreadyExists   = Error("A webhook for this resource already exists")
+	ErrUnsupportedWebhookType = Error("Webhooks for this resource are not currently supported")
+)

api/exec/extension.go

@@ -0,0 +1,215 @@
+package exec
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"os/exec"
+	"path"
+	"runtime"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/orcaman/concurrent-map"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/http/client"
+)
+
+var extensionDownloadBaseURL = "https://portainer-io-assets.sfo2.digitaloceanspaces.com/extensions/"
+
+var extensionBinaryMap = map[portainer.ExtensionID]string{
+	portainer.RegistryManagementExtension:  "extension-registry-management",
+	portainer.OAuthAuthenticationExtension: "extension-oauth-authentication",
+	portainer.RBACExtension:                "extension-rbac",
+}
+
+// ExtensionManager represents a service used to
+// manage extension processes.
+type ExtensionManager struct {
+	processes        cmap.ConcurrentMap
+	fileService      portainer.FileService
+	extensionService portainer.ExtensionService
+}
+
+// NewExtensionManager returns a pointer to an ExtensionManager
+func NewExtensionManager(fileService portainer.FileService, extensionService portainer.ExtensionService) *ExtensionManager {
+	return &ExtensionManager{
+		processes:        cmap.New(),
+		fileService:      fileService,
+		extensionService: extensionService,
+	}
+}
+
+func processKey(ID portainer.ExtensionID) string {
+	return strconv.Itoa(int(ID))
+}
+
+func buildExtensionURL(extension *portainer.Extension) string {
+	extensionURL := extensionDownloadBaseURL
+	extensionURL += extensionBinaryMap[extension.ID]
+	extensionURL += "-" + runtime.GOOS + "-" + runtime.GOARCH
+	extensionURL += "-" + extension.Version
+	extensionURL += ".zip"
+	return extensionURL
+}
+
+func buildExtensionPath(binaryPath string, extension *portainer.Extension) string {
+
+	extensionFilename := extensionBinaryMap[extension.ID]
+	extensionFilename += "-" + runtime.GOOS + "-" + runtime.GOARCH
+	extensionFilename += "-" + extension.Version
+
+	if runtime.GOOS == "windows" {
+		extensionFilename += ".exe"
+	}
+
+	extensionPath := path.Join(
+		binaryPath,
+		extensionFilename)
+
+	return extensionPath
+}
+
+// FetchExtensionDefinitions will fetch the list of available
+// extension definitions from the official Portainer assets server
+func (manager *ExtensionManager) FetchExtensionDefinitions() ([]portainer.Extension, error) {
+	extensionData, err := client.Get(portainer.ExtensionDefinitionsURL, 30)
+	if err != nil {
+		return nil, err
+	}
+
+	var extensions []portainer.Extension
+	err = json.Unmarshal(extensionData, &extensions)
+	if err != nil {
+		return nil, err
+	}
+
+	return extensions, nil
+}
+
+// EnableExtension will check for the existence of the extension binary on the filesystem
+// first. If it does not exist, it will download it from the official Portainer assets server.
+// After installing the binary on the filesystem, it will execute the binary in license check
+// mode to validate the extension license. If the license is valid, it will then start
+// the extension process and register it in the processes map.
+func (manager *ExtensionManager) EnableExtension(extension *portainer.Extension, licenseKey string) error {
+	extensionBinaryPath := buildExtensionPath(manager.fileService.GetBinaryFolder(), extension)
+	extensionBinaryExists, err := manager.fileService.FileExists(extensionBinaryPath)
+	if err != nil {
+		return err
+	}
+
+	if !extensionBinaryExists {
+		err := manager.downloadExtension(extension)
+		if err != nil {
+			return err
+		}
+	}
+
+	licenseDetails, err := validateLicense(extensionBinaryPath, licenseKey)
+	if err != nil {
+		return err
+	}
+
+	extension.License = portainer.LicenseInformation{
+		LicenseKey: licenseKey,
+		Company:    licenseDetails[0],
+		Expiration: licenseDetails[1],
+		Valid:      true,
+	}
+	extension.Version = licenseDetails[2]
+
+	return manager.startExtensionProcess(extension, extensionBinaryPath)
+}
+
+// DisableExtension will retrieve the process associated to the extension
+// from the processes map and kill the process. It will then remove the process
+// from the processes map and remove the binary associated to the extension
+// from the filesystem
+func (manager *ExtensionManager) DisableExtension(extension *portainer.Extension) error {
+	process, ok := manager.processes.Get(processKey(extension.ID))
+	if !ok {
+		return nil
+	}
+
+	err := process.(*exec.Cmd).Process.Kill()
+	if err != nil {
+		return err
+	}
+
+	manager.processes.Remove(processKey(extension.ID))
+
+	extensionBinaryPath := buildExtensionPath(manager.fileService.GetBinaryFolder(), extension)
+	return manager.fileService.RemoveDirectory(extensionBinaryPath)
+}
+
+// UpdateExtension will download the new extension binary from the official Portainer assets
+// server, disable the previous extension via DisableExtension, trigger a license check
+// and then start the extension process and add it to the processes map
+func (manager *ExtensionManager) UpdateExtension(extension *portainer.Extension, version string) error {
+	oldVersion := extension.Version
+
+	extension.Version = version
+	err := manager.downloadExtension(extension)
+	if err != nil {
+		return err
+	}
+
+	extension.Version = oldVersion
+	err = manager.DisableExtension(extension)
+	if err != nil {
+		return err
+	}
+
+	extension.Version = version
+	extensionBinaryPath := buildExtensionPath(manager.fileService.GetBinaryFolder(), extension)
+
+	licenseDetails, err := validateLicense(extensionBinaryPath, extension.License.LicenseKey)
+	if err != nil {
+		return err
+	}
+
+	extension.Version = licenseDetails[2]
+
+	return manager.startExtensionProcess(extension, extensionBinaryPath)
+}
+
+func (manager *ExtensionManager) downloadExtension(extension *portainer.Extension) error {
+	extensionURL := buildExtensionURL(extension)
+
+	data, err := client.Get(extensionURL, 30)
+	if err != nil {
+		return err
+	}
+
+	return manager.fileService.ExtractExtensionArchive(data)
+}
+
+func validateLicense(binaryPath, licenseKey string) ([]string, error) {
+	licenseCheckProcess := exec.Command(binaryPath, "-license", licenseKey, "-check")
+	cmdOutput := &bytes.Buffer{}
+	licenseCheckProcess.Stdout = cmdOutput
+
+	err := licenseCheckProcess.Run()
+	if err != nil {
+		return nil, errors.New("Invalid extension license key")
+	}
+
+	output := string(cmdOutput.Bytes())
+
+	return strings.Split(output, "|"), nil
+}
+
+func (manager *ExtensionManager) startExtensionProcess(extension *portainer.Extension, binaryPath string) error {
+	extensionProcess := exec.Command(binaryPath, "-license", extension.License.LicenseKey)
+	err := extensionProcess.Start()
+	if err != nil {
+		return err
+	}
+
+	time.Sleep(3 * time.Second)
+
+	manager.processes.Set(processKey(extension.ID), extensionProcess)
+	return nil
+}

api/exec/swarm_stack.go

@@ -8,7 +8,7 @@ import (
 	"path"
 	"runtime"
 
-	"github.com/portainer/portainer"
+	"github.com/portainer/portainer/api"
 )
 
 // SwarmStackManager represents a service for managing stacks.
@@ -140,7 +140,7 @@ func (manager *SwarmStackManager) updateDockerCLIConfiguration(dataPath string)
 		return err
 	}
 
-	signature, err := manager.signatureService.Sign(portainer.PortainerAgentSignatureMessage)
+	signature, err := manager.signatureService.CreateSignature(portainer.PortainerAgentSignatureMessage)
 	if err != nil {
 		return err
 	}
@@ -169,7 +169,7 @@ func (manager *SwarmStackManager) retrieveConfigurationFromDisk(path string) (ma
 		return make(map[string]interface{}), nil
 	}
 
-	err = json.Unmarshal([]byte(raw), &config)
+	err = json.Unmarshal(raw, &config)
 	if err != nil {
 		return nil, err
 	}

api/filesystem/filesystem.go

@@ -6,7 +6,8 @@ import (
 	"encoding/pem"
 	"io/ioutil"
 
-	"github.com/portainer/portainer"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/archive"
 
 	"io"
 	"os"
@@ -32,6 +33,13 @@ const (
 	PrivateKeyFile = "portainer.key"
 	// PublicKeyFile represents the name on disk of the file containing the public key.
 	PublicKeyFile = "portainer.pub"
+	// BinaryStorePath represents the subfolder where binaries are stored in the file store folder.
+	BinaryStorePath = "bin"
+	// ScheduleStorePath represents the subfolder where schedule files are stored.
+	ScheduleStorePath = "schedules"
+	// ExtensionRegistryManagementStorePath represents the subfolder where files related to the
+	// registry management extension are stored.
+	ExtensionRegistryManagementStorePath = "extensions"
 )
 
 // Service represents a service for managing files and directories.
@@ -63,9 +71,30 @@ func NewService(dataStorePath, fileStorePath string) (*Service, error) {
 		return nil, err
 	}
 
+	err = service.createDirectoryInStore(BinaryStorePath)
+	if err != nil {
+		return nil, err
+	}
+
 	return service, nil
 }
 
+// GetBinaryFolder returns the full path to the binary store on the filesystem
+func (service *Service) GetBinaryFolder() string {
+	return path.Join(service.fileStorePath, BinaryStorePath)
+}
+
+// ExtractExtensionArchive extracts the content of an extension archive
+// specified as raw data into the binary store on the filesystem
+func (service *Service) ExtractExtensionArchive(data []byte) error {
+	err := archive.UnzipArchive(data, path.Join(service.fileStorePath, BinaryStorePath))
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // RemoveDirectory removes a directory on the filesystem.
 func (service *Service) RemoveDirectory(directoryPath string) error {
 	return os.RemoveAll(directoryPath)
@@ -97,6 +126,27 @@ func (service *Service) StoreStackFileFromBytes(stackIdentifier, fileName string
 	return path.Join(service.fileStorePath, stackStorePath), nil
 }
 
+// StoreRegistryManagementFileFromBytes creates a subfolder in the
+// ExtensionRegistryManagementStorePath and stores a new file from bytes.
+// It returns the path to the folder where the file is stored.
+func (service *Service) StoreRegistryManagementFileFromBytes(folder, fileName string, data []byte) (string, error) {
+	extensionStorePath := path.Join(ExtensionRegistryManagementStorePath, folder)
+	err := service.createDirectoryInStore(extensionStorePath)
+	if err != nil {
+		return "", err
+	}
+
+	file := path.Join(extensionStorePath, fileName)
+	r := bytes.NewReader(data)
+
+	err = service.createFileInStore(file, r)
+	if err != nil {
+		return "", err
+	}
+
+	return path.Join(service.fileStorePath, file), nil
+}
+
 // StoreTLSFileFromBytes creates a folder in the TLSStorePath and stores a new file from bytes.
 // It returns the path to the newly created file.
 func (service *Service) StoreTLSFileFromBytes(folder string, fileType portainer.TLSFileType, data []byte) (string, error) {
@@ -176,14 +226,14 @@ func (service *Service) DeleteTLSFile(folder string, fileType portainer.TLSFileT
 	return nil
 }
 
-// GetFileContent returns a string content from file.
-func (service *Service) GetFileContent(filePath string) (string, error) {
+// GetFileContent returns the content of a file as bytes.
+func (service *Service) GetFileContent(filePath string) ([]byte, error) {
 	content, err := ioutil.ReadFile(filePath)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 
-	return string(content), nil
+	return content, nil
 }
 
 // Rename renames a file or directory
@@ -318,3 +368,32 @@ func (service *Service) getContentFromPEMFile(filePath string) ([]byte, error) {
 	block, _ := pem.Decode(fileContent)
 	return block.Bytes, nil
 }
+
+// GetScheduleFolder returns the absolute path on the filesystem for a schedule based
+// on its identifier.
+func (service *Service) GetScheduleFolder(identifier string) string {
+	return path.Join(service.fileStorePath, ScheduleStorePath, identifier)
+}
+
+// StoreScheduledJobFileFromBytes creates a subfolder in the ScheduleStorePath and stores a new file from bytes.
+// It returns the path to the folder where the file is stored.
+func (service *Service) StoreScheduledJobFileFromBytes(identifier string, data []byte) (string, error) {
+	scheduleStorePath := path.Join(ScheduleStorePath, identifier)
+	err := service.createDirectoryInStore(scheduleStorePath)
+	if err != nil {
+		return "", err
+	}
+
+	filePath := path.Join(scheduleStorePath, createScheduledJobFileName(identifier))
+	r := bytes.NewReader(data)
+	err = service.createFileInStore(filePath, r)
+	if err != nil {
+		return "", err
+	}
+
+	return path.Join(service.fileStorePath, filePath), nil
+}
+
+func createScheduledJobFileName(identifier string) string {
+	return "job_" + identifier + ".sh"
+}

api/git/git.go

@@ -5,6 +5,7 @@ import (
 	"strings"
 
 	"gopkg.in/src-d/go-git.v4"
+	"gopkg.in/src-d/go-git.v4/plumbing"
 )
 
 // Service represents a service for managing Git.
@@ -19,21 +20,27 @@ func NewService(dataStorePath string) (*Service, error) {
 
 // ClonePublicRepository clones a public git repository using the specified URL in the specified
 // destination folder.
-func (service *Service) ClonePublicRepository(repositoryURL, destination string) error {
-	return cloneRepository(repositoryURL, destination)
+func (service *Service) ClonePublicRepository(repositoryURL, referenceName string, destination string) error {
+	return cloneRepository(repositoryURL, referenceName, destination)
 }
 
 // ClonePrivateRepositoryWithBasicAuth clones a private git repository using the specified URL in the specified
 // destination folder. It will use the specified username and password for basic HTTP authentication.
-func (service *Service) ClonePrivateRepositoryWithBasicAuth(repositoryURL, destination, username, password string) error {
+func (service *Service) ClonePrivateRepositoryWithBasicAuth(repositoryURL, referenceName string, destination, username, password string) error {
 	credentials := username + ":" + url.PathEscape(password)
 	repositoryURL = strings.Replace(repositoryURL, "://", "://"+credentials+"@", 1)
-	return cloneRepository(repositoryURL, destination)
+	return cloneRepository(repositoryURL, referenceName, destination)
 }
 
-func cloneRepository(repositoryURL, destination string) error {
-	_, err := git.PlainClone(destination, false, &git.CloneOptions{
+func cloneRepository(repositoryURL, referenceName string, destination string) error {
+	options := &git.CloneOptions{
 		URL: repositoryURL,
-	})
+	}
+
+	if referenceName != "" {
+		options.ReferenceName = plumbing.ReferenceName(referenceName)
+	}
+
+	_, err := git.PlainClone(destination, false, options)
 	return err
 }

api/http/client/client.go

@@ -4,12 +4,18 @@ import (
 	"crypto/tls"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 	"strings"
 	"time"
 
-	"github.com/portainer/portainer"
+	"github.com/portainer/portainer/api"
+)
+
+const (
+	errInvalidResponseStatus = portainer.Error("Invalid response status (expecting 200)")
+	defaultHTTPTimeout       = 5
 )
 
 // HTTPClient represents a client to send HTTP requests.
@@ -21,7 +27,7 @@ type HTTPClient struct {
 func NewHTTPClient() *HTTPClient {
 	return &HTTPClient{
 		&http.Client{
-			Timeout: time.Second * 5,
+			Timeout: time.Second * time.Duration(defaultHTTPTimeout),
 		},
 	}
 }
@@ -61,6 +67,37 @@ func (client *HTTPClient) ExecuteAzureAuthenticationRequest(credentials *portain
 	return &token, nil
 }
 
+// Get executes a simple HTTP GET to the specified URL and returns
+// the content of the response body. Timeout can be specified via the timeout parameter,
+// will default to defaultHTTPTimeout if set to 0.
+func Get(url string, timeout int) ([]byte, error) {
+
+	if timeout == 0 {
+		timeout = defaultHTTPTimeout
+	}
+
+	client := &http.Client{
+		Timeout: time.Second * time.Duration(timeout),
+	}
+
+	response, err := client.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	defer response.Body.Close()
+
+	if response.StatusCode != http.StatusOK {
+		return nil, errInvalidResponseStatus
+	}
+
+	body, err := ioutil.ReadAll(response.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	return body, nil
+}
+
 // ExecutePingOperation will send a SystemPing operation HTTP request to a Docker environment
 // using the specified host and optional TLS configuration.
 // It uses a new Http.Client for each operation.

api/http/error/error.go

@@ -1,41 +0,0 @@
-package error
-
-import (
-	"encoding/json"
-	"log"
-	"net/http"
-)
-
-type (
-	// LoggerHandler defines a HTTP handler that includes a HandlerError return pointer
-	LoggerHandler func(http.ResponseWriter, *http.Request) *HandlerError
-	// HandlerError represents an error raised inside a HTTP handler
-	HandlerError struct {
-		StatusCode int
-		Message    string
-		Err        error
-	}
-	errorResponse struct {
-		Err string `json:"err,omitempty"`
-	}
-)
-
-func (handler LoggerHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
-	err := handler(rw, r)
-	if err != nil {
-		writeErrorResponse(rw, err)
-	}
-}
-
-func writeErrorResponse(rw http.ResponseWriter, err *HandlerError) {
-	log.Printf("http error: %s (err=%s) (code=%d)\n", err.Message, err.Err, err.StatusCode)
-	rw.Header().Set("Content-Type", "application/json")
-	rw.WriteHeader(err.StatusCode)
-	json.NewEncoder(rw).Encode(&errorResponse{Err: err.Message})
-}
-
-// WriteError is a convenience function that creates a new HandlerError before calling writeErrorResponse.
-// For use outside of the standard http handlers.
-func WriteError(rw http.ResponseWriter, code int, message string, err error) {
-	writeErrorResponse(rw, &HandlerError{code, message, err})
-}

api/http/handler/auth/authenticate.go

@@ -1,13 +1,15 @@
 package auth
 
 import (
+	"log"
 	"net/http"
+	"strings"
 
 	"github.com/asaskevich/govalidator"
-	"github.com/portainer/portainer"
-	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/request"
-	"github.com/portainer/portainer/http/response"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	"github.com/portainer/portainer/api"
 )
 
 type authenticatePayload struct {
@@ -40,36 +42,152 @@ func (handler *Handler) authenticate(w http.ResponseWriter, r *http.Request) *ht
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
 	}
 
-	u, err := handler.UserService.UserByUsername(payload.Username)
-	if err == portainer.ErrObjectNotFound {
-		return &httperror.HandlerError{http.StatusBadRequest, "Invalid credentials", ErrInvalidCredentials}
-	} else if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a user with the specified username from the database", err}
-	}
-
 	settings, err := handler.SettingsService.Settings()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve settings from the database", err}
 	}
 
-	if settings.AuthenticationMethod == portainer.AuthenticationLDAP && u.ID != 1 {
-		err = handler.LDAPService.AuthenticateUser(payload.Username, payload.Password, &settings.LDAPSettings)
-		if err != nil {
-			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to authenticate user via LDAP/AD", err}
-		}
-	} else {
-		err = handler.CryptoService.CompareHashAndData(u.Password, payload.Password)
-		if err != nil {
-			return &httperror.HandlerError{http.StatusUnprocessableEntity, "Invalid credentials", ErrInvalidCredentials}
-		}
+	u, err := handler.UserService.UserByUsername(payload.Username)
+	if err != nil && err != portainer.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a user with the specified username from the database", err}
 	}
 
+	if err == portainer.ErrObjectNotFound && settings.AuthenticationMethod == portainer.AuthenticationInternal {
+		return &httperror.HandlerError{http.StatusUnprocessableEntity, "Invalid credentials", portainer.ErrUnauthorized}
+	}
+
+	if settings.AuthenticationMethod == portainer.AuthenticationLDAP {
+		if u == nil && settings.LDAPSettings.AutoCreateUsers {
+			return handler.authenticateLDAPAndCreateUser(w, payload.Username, payload.Password, &settings.LDAPSettings)
+		} else if u == nil && !settings.LDAPSettings.AutoCreateUsers {
+			return &httperror.HandlerError{http.StatusUnprocessableEntity, "Invalid credentials", portainer.ErrUnauthorized}
+		}
+		return handler.authenticateLDAP(w, u, payload.Password, &settings.LDAPSettings)
+	}
+
+	return handler.authenticateInternal(w, u, payload.Password)
+}
+
+func (handler *Handler) authenticateLDAP(w http.ResponseWriter, user *portainer.User, password string, ldapSettings *portainer.LDAPSettings) *httperror.HandlerError {
+	err := handler.LDAPService.AuthenticateUser(user.Username, password, ldapSettings)
+	if err != nil {
+		return handler.authenticateInternal(w, user, password)
+	}
+
+	err = handler.addUserIntoTeams(user, ldapSettings)
+	if err != nil {
+		log.Printf("Warning: unable to automatically add user into teams: %s\n", err.Error())
+	}
+
+	return handler.writeToken(w, user)
+}
+
+func (handler *Handler) authenticateInternal(w http.ResponseWriter, user *portainer.User, password string) *httperror.HandlerError {
+	err := handler.CryptoService.CompareHashAndData(user.Password, password)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusUnprocessableEntity, "Invalid credentials", portainer.ErrUnauthorized}
+	}
+
+	return handler.writeToken(w, user)
+}
+
+func (handler *Handler) authenticateLDAPAndCreateUser(w http.ResponseWriter, username, password string, ldapSettings *portainer.LDAPSettings) *httperror.HandlerError {
+	err := handler.LDAPService.AuthenticateUser(username, password, ldapSettings)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusUnprocessableEntity, "Invalid credentials", err}
+	}
+
+	user := &portainer.User{
+		Username: username,
+		Role:     portainer.StandardUserRole,
+		PortainerAuthorizations: map[portainer.Authorization]bool{
+			portainer.OperationPortainerDockerHubInspect:        true,
+			portainer.OperationPortainerEndpointGroupList:       true,
+			portainer.OperationPortainerEndpointList:            true,
+			portainer.OperationPortainerEndpointInspect:         true,
+			portainer.OperationPortainerEndpointExtensionAdd:    true,
+			portainer.OperationPortainerEndpointExtensionRemove: true,
+			portainer.OperationPortainerExtensionList:           true,
+			portainer.OperationPortainerMOTD:                    true,
+			portainer.OperationPortainerRegistryList:            true,
+			portainer.OperationPortainerRegistryInspect:         true,
+			portainer.OperationPortainerTeamList:                true,
+			portainer.OperationPortainerTemplateList:            true,
+			portainer.OperationPortainerTemplateInspect:         true,
+			portainer.OperationPortainerUserList:                true,
+			portainer.OperationPortainerUserMemberships:         true,
+		},
+	}
+
+	err = handler.UserService.CreateUser(user)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist user inside the database", err}
+	}
+
+	err = handler.addUserIntoTeams(user, ldapSettings)
+	if err != nil {
+		log.Printf("Warning: unable to automatically add user into teams: %s\n", err.Error())
+	}
+
+	return handler.writeToken(w, user)
+}
+
+func (handler *Handler) writeToken(w http.ResponseWriter, user *portainer.User) *httperror.HandlerError {
 	tokenData := &portainer.TokenData{
-		ID:       u.ID,
-		Username: u.Username,
-		Role:     u.Role,
+		ID:                      user.ID,
+		Username:                user.Username,
+		Role:                    user.Role,
+		PortainerAuthorizations: user.PortainerAuthorizations,
 	}
 
+	_, err := handler.ExtensionService.Extension(portainer.RBACExtension)
+	if err == portainer.ErrObjectNotFound {
+		return handler.persistAndWriteToken(w, tokenData)
+	} else if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a extension with the specified identifier inside the database", err}
+	}
+
+	endpointAuthorizations, err := handler.getAuthorizations(user)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve authorizations associated to the user", err}
+	}
+	tokenData.EndpointAuthorizations = endpointAuthorizations
+
+	return handler.persistAndWriteToken(w, tokenData)
+}
+
+func (handler *Handler) getAuthorizations(user *portainer.User) (portainer.EndpointAuthorizations, error) {
+	endpointAuthorizations := portainer.EndpointAuthorizations{}
+	if user.Role == portainer.AdministratorRole {
+		return endpointAuthorizations, nil
+	}
+
+	userMemberships, err := handler.TeamMembershipService.TeamMembershipsByUserID(user.ID)
+	if err != nil {
+		return endpointAuthorizations, err
+	}
+
+	endpoints, err := handler.EndpointService.Endpoints()
+	if err != nil {
+		return endpointAuthorizations, err
+	}
+
+	endpointGroups, err := handler.EndpointGroupService.EndpointGroups()
+	if err != nil {
+		return endpointAuthorizations, err
+	}
+
+	roles, err := handler.RoleService.Roles()
+	if err != nil {
+		return endpointAuthorizations, err
+	}
+
+	endpointAuthorizations = getUserEndpointAuthorizations(user, endpoints, endpointGroups, roles, userMemberships)
+
+	return endpointAuthorizations, nil
+}
+
+func (handler *Handler) persistAndWriteToken(w http.ResponseWriter, tokenData *portainer.TokenData) *httperror.HandlerError {
 	token, err := handler.JWTService.GenerateToken(tokenData)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to generate JWT token", err}
@@ -77,3 +195,59 @@ func (handler *Handler) authenticate(w http.ResponseWriter, r *http.Request) *ht
 
 	return response.JSON(w, &authenticateResponse{JWT: token})
 }
+
+func (handler *Handler) addUserIntoTeams(user *portainer.User, settings *portainer.LDAPSettings) error {
+	teams, err := handler.TeamService.Teams()
+	if err != nil {
+		return err
+	}
+
+	userGroups, err := handler.LDAPService.GetUserGroups(user.Username, settings)
+	if err != nil {
+		return err
+	}
+
+	userMemberships, err := handler.TeamMembershipService.TeamMembershipsByUserID(user.ID)
+	if err != nil {
+		return err
+	}
+
+	for _, team := range teams {
+		if teamExists(team.Name, userGroups) {
+
+			if teamMembershipExists(team.ID, userMemberships) {
+				continue
+			}
+
+			membership := &portainer.TeamMembership{
+				UserID: user.ID,
+				TeamID: team.ID,
+				Role:   portainer.TeamMember,
+			}
+
+			err := handler.TeamMembershipService.CreateTeamMembership(membership)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func teamExists(teamName string, ldapGroups []string) bool {
+	for _, group := range ldapGroups {
+		if strings.ToLower(group) == strings.ToLower(teamName) {
+			return true
+		}
+	}
+	return false
+}
+
+func teamMembershipExists(teamID portainer.TeamID, memberships []portainer.TeamMembership) bool {
+	for _, membership := range memberships {
+		if membership.TeamID == teamID {
+			return true
+		}
+	}
+	return false
+}

api/http/handler/auth/authenticate_oauth.go

@@ -0,0 +1,155 @@
+package auth
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"log"
+	"net/http"
+
+	"github.com/asaskevich/govalidator"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/portainer/api"
+)
+
+type oauthPayload struct {
+	Code string
+}
+
+func (payload *oauthPayload) Validate(r *http.Request) error {
+	if govalidator.IsNull(payload.Code) {
+		return portainer.Error("Invalid OAuth authorization code")
+	}
+	return nil
+}
+
+func (handler *Handler) authenticateThroughExtension(code, licenseKey string, settings *portainer.OAuthSettings) (string, error) {
+	extensionURL := handler.ProxyManager.GetExtensionURL(portainer.OAuthAuthenticationExtension)
+
+	encodedConfiguration, err := json.Marshal(settings)
+	if err != nil {
+		return "", nil
+	}
+
+	req, err := http.NewRequest("GET", extensionURL+"/validate", nil)
+	if err != nil {
+		return "", err
+	}
+
+	client := &http.Client{}
+	req.Header.Set("X-OAuth-Config", string(encodedConfiguration))
+	req.Header.Set("X-OAuth-Code", code)
+	req.Header.Set("X-PortainerExtension-License", licenseKey)
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", err
+	}
+
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+
+	type extensionResponse struct {
+		Username string `json:"Username,omitempty"`
+		Err      string `json:"err,omitempty"`
+		Details  string `json:"details,omitempty"`
+	}
+
+	var extResp extensionResponse
+	err = json.Unmarshal(body, &extResp)
+	if err != nil {
+		return "", err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return "", portainer.Error(extResp.Err + ":" + extResp.Details)
+	}
+
+	return extResp.Username, nil
+}
+
+func (handler *Handler) validateOAuth(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	var payload oauthPayload
+	err := request.DecodeAndValidateJSONPayload(r, &payload)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
+	}
+
+	settings, err := handler.SettingsService.Settings()
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve settings from the database", err}
+	}
+
+	if settings.AuthenticationMethod != 3 {
+		return &httperror.HandlerError{http.StatusForbidden, "OAuth authentication is not enabled", portainer.Error("OAuth authentication is not enabled")}
+	}
+
+	extension, err := handler.ExtensionService.Extension(portainer.OAuthAuthenticationExtension)
+	if err == portainer.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusNotFound, "Oauth authentication extension is not enabled", err}
+	} else if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a extension with the specified identifier inside the database", err}
+	}
+
+	username, err := handler.authenticateThroughExtension(payload.Code, extension.License.LicenseKey, &settings.OAuthSettings)
+	if err != nil {
+		log.Printf("[DEBUG] - OAuth authentication error: %s", err)
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to authenticate through OAuth", portainer.ErrUnauthorized}
+	}
+
+	user, err := handler.UserService.UserByUsername(username)
+	if err != nil && err != portainer.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a user with the specified username from the database", err}
+	}
+
+	if user == nil && !settings.OAuthSettings.OAuthAutoCreateUsers {
+		return &httperror.HandlerError{http.StatusForbidden, "Account not created beforehand in Portainer and automatic user provisioning not enabled", portainer.ErrUnauthorized}
+	}
+
+	if user == nil {
+		user = &portainer.User{
+			Username: username,
+			Role:     portainer.StandardUserRole,
+			PortainerAuthorizations: map[portainer.Authorization]bool{
+				portainer.OperationPortainerDockerHubInspect:        true,
+				portainer.OperationPortainerEndpointGroupList:       true,
+				portainer.OperationPortainerEndpointList:            true,
+				portainer.OperationPortainerEndpointInspect:         true,
+				portainer.OperationPortainerEndpointExtensionAdd:    true,
+				portainer.OperationPortainerEndpointExtensionRemove: true,
+				portainer.OperationPortainerExtensionList:           true,
+				portainer.OperationPortainerMOTD:                    true,
+				portainer.OperationPortainerRegistryList:            true,
+				portainer.OperationPortainerRegistryInspect:         true,
+				portainer.OperationPortainerTeamList:                true,
+				portainer.OperationPortainerTemplateList:            true,
+				portainer.OperationPortainerTemplateInspect:         true,
+				portainer.OperationPortainerUserList:                true,
+				portainer.OperationPortainerUserMemberships:         true,
+			},
+		}
+
+		err = handler.UserService.CreateUser(user)
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist user inside the database", err}
+		}
+
+		if settings.OAuthSettings.DefaultTeamID != 0 {
+			membership := &portainer.TeamMembership{
+				UserID: user.ID,
+				TeamID: settings.OAuthSettings.DefaultTeamID,
+				Role:   portainer.TeamMember,
+			}
+
+			err = handler.TeamMembershipService.CreateTeamMembership(membership)
+			if err != nil {
+				return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist team membership inside the database", err}
+			}
+		}
+	}
+
+	return handler.writeToken(w, user)
+}

api/http/handler/auth/authorization.go

@@ -0,0 +1,122 @@
+package auth
+
+import portainer "github.com/portainer/portainer/api"
+
+func getUserEndpointAuthorizations(user *portainer.User, endpoints []portainer.Endpoint, endpointGroups []portainer.EndpointGroup, roles []portainer.Role, userMemberships []portainer.TeamMembership) portainer.EndpointAuthorizations {
+	endpointAuthorizations := make(portainer.EndpointAuthorizations)
+
+	groupUserAccessPolicies := map[portainer.EndpointGroupID]portainer.UserAccessPolicies{}
+	groupTeamAccessPolicies := map[portainer.EndpointGroupID]portainer.TeamAccessPolicies{}
+	for _, endpointGroup := range endpointGroups {
+		groupUserAccessPolicies[endpointGroup.ID] = endpointGroup.UserAccessPolicies
+		groupTeamAccessPolicies[endpointGroup.ID] = endpointGroup.TeamAccessPolicies
+	}
+
+	for _, endpoint := range endpoints {
+		authorizations := getAuthorizationsFromUserEndpointPolicy(user, &endpoint, roles)
+		if len(authorizations) > 0 {
+			endpointAuthorizations[endpoint.ID] = authorizations
+			continue
+		}
+
+		authorizations = getAuthorizationsFromUserEndpointGroupPolicy(user, &endpoint, roles, groupUserAccessPolicies)
+		if len(authorizations) > 0 {
+			endpointAuthorizations[endpoint.ID] = authorizations
+			continue
+		}
+
+		authorizations = getAuthorizationsFromTeamEndpointPolicies(userMemberships, &endpoint, roles)
+		if len(authorizations) > 0 {
+			endpointAuthorizations[endpoint.ID] = authorizations
+			continue
+		}
+
+		endpointAuthorizations[endpoint.ID] = getAuthorizationsFromTeamEndpointGroupPolicies(userMemberships, &endpoint, roles, groupTeamAccessPolicies)
+	}
+
+	return endpointAuthorizations
+}
+
+func getAuthorizationsFromUserEndpointPolicy(user *portainer.User, endpoint *portainer.Endpoint, roles []portainer.Role) portainer.Authorizations {
+	policyRoles := make([]portainer.RoleID, 0)
+
+	policy, ok := endpoint.UserAccessPolicies[user.ID]
+	if ok {
+		policyRoles = append(policyRoles, policy.RoleID)
+	}
+
+	return getAuthorizationsFromRoles(policyRoles, roles)
+}
+
+func getAuthorizationsFromUserEndpointGroupPolicy(user *portainer.User, endpoint *portainer.Endpoint, roles []portainer.Role, groupAccessPolicies map[portainer.EndpointGroupID]portainer.UserAccessPolicies) portainer.Authorizations {
+	policyRoles := make([]portainer.RoleID, 0)
+
+	policy, ok := groupAccessPolicies[endpoint.GroupID][user.ID]
+	if ok {
+		policyRoles = append(policyRoles, policy.RoleID)
+	}
+
+	return getAuthorizationsFromRoles(policyRoles, roles)
+}
+
+func getAuthorizationsFromTeamEndpointPolicies(memberships []portainer.TeamMembership, endpoint *portainer.Endpoint, roles []portainer.Role) portainer.Authorizations {
+	policyRoles := make([]portainer.RoleID, 0)
+
+	for _, membership := range memberships {
+		policy, ok := endpoint.TeamAccessPolicies[membership.TeamID]
+		if ok {
+			policyRoles = append(policyRoles, policy.RoleID)
+		}
+	}
+
+	return getAuthorizationsFromRoles(policyRoles, roles)
+}
+
+func getAuthorizationsFromTeamEndpointGroupPolicies(memberships []portainer.TeamMembership, endpoint *portainer.Endpoint, roles []portainer.Role, groupAccessPolicies map[portainer.EndpointGroupID]portainer.TeamAccessPolicies) portainer.Authorizations {
+	policyRoles := make([]portainer.RoleID, 0)
+
+	for _, membership := range memberships {
+		policy, ok := groupAccessPolicies[endpoint.GroupID][membership.TeamID]
+		if ok {
+			policyRoles = append(policyRoles, policy.RoleID)
+		}
+	}
+
+	return getAuthorizationsFromRoles(policyRoles, roles)
+}
+
+func getAuthorizationsFromRoles(roleIdentifiers []portainer.RoleID, roles []portainer.Role) portainer.Authorizations {
+	var roleAuthorizations []portainer.Authorizations
+	for _, id := range roleIdentifiers {
+		for _, role := range roles {
+			if role.ID == id {
+				roleAuthorizations = append(roleAuthorizations, role.Authorizations)
+				break
+			}
+		}
+	}
+
+	processedAuthorizations := make(portainer.Authorizations)
+	if len(roleAuthorizations) > 0 {
+		processedAuthorizations = roleAuthorizations[0]
+		for idx, authorizations := range roleAuthorizations {
+			if idx == 0 {
+				continue
+			}
+			processedAuthorizations = mergeAuthorizations(processedAuthorizations, authorizations)
+		}
+	}
+
+	return processedAuthorizations
+}
+
+func mergeAuthorizations(a, b portainer.Authorizations) portainer.Authorizations {
+	c := make(map[portainer.Authorization]bool)
+
+	for k := range b {
+		if _, ok := a[k]; ok {
+			c[k] = true
+		}
+	}
+	return c
+}

api/http/handler/auth/handler.go

@@ -4,9 +4,10 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
-	"github.com/portainer/portainer"
-	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/security"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/http/proxy"
+	"github.com/portainer/portainer/api/http/security"
 )
 
 const (
@@ -20,12 +21,19 @@ const (
 // Handler is the HTTP handler used to handle authentication operations.
 type Handler struct {
 	*mux.Router
-	authDisabled    bool
-	UserService     portainer.UserService
-	CryptoService   portainer.CryptoService
-	JWTService      portainer.JWTService
-	LDAPService     portainer.LDAPService
-	SettingsService portainer.SettingsService
+	authDisabled          bool
+	UserService           portainer.UserService
+	CryptoService         portainer.CryptoService
+	JWTService            portainer.JWTService
+	LDAPService           portainer.LDAPService
+	SettingsService       portainer.SettingsService
+	TeamService           portainer.TeamService
+	TeamMembershipService portainer.TeamMembershipService
+	ExtensionService      portainer.ExtensionService
+	EndpointService       portainer.EndpointService
+	EndpointGroupService  portainer.EndpointGroupService
+	RoleService           portainer.RoleService
+	ProxyManager          *proxy.Manager
 }
 
 // NewHandler creates a handler to manage authentication operations.
@@ -34,6 +42,9 @@ func NewHandler(bouncer *security.RequestBouncer, rateLimiter *security.RateLimi
 		Router:       mux.NewRouter(),
 		authDisabled: authDisabled,
 	}
+
+	h.Handle("/auth/oauth/validate",
+		rateLimiter.LimitAccess(bouncer.PublicAccess(httperror.LoggerHandler(h.validateOAuth)))).Methods(http.MethodPost)
 	h.Handle("/auth",
 		rateLimiter.LimitAccess(bouncer.PublicAccess(httperror.LoggerHandler(h.authenticate)))).Methods(http.MethodPost)
 

api/http/handler/dockerhub/dockerhub_inspect.go

@@ -3,8 +3,8 @@ package dockerhub
 import (
 	"net/http"
 
-	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/response"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/response"
 )
 
 // GET request on /api/dockerhub

api/http/handler/dockerhub/dockerhub_update.go

@@ -4,10 +4,10 @@ import (
 	"net/http"
 
 	"github.com/asaskevich/govalidator"
-	"github.com/portainer/portainer"
-	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/request"
-	"github.com/portainer/portainer/http/response"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	"github.com/portainer/portainer/api"
 )
 
 type dockerhubUpdatePayload struct {

api/http/handler/dockerhub/handler.go

@@ -4,9 +4,9 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
-	"github.com/portainer/portainer"
-	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/security"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/http/security"
 )
 
 func hideFields(dockerHub *portainer.DockerHub) {
@@ -25,9 +25,9 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler {
 		Router: mux.NewRouter(),
 	}
 	h.Handle("/dockerhub",
-		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.dockerhubInspect))).Methods(http.MethodGet)
+		bouncer.AuthorizedAccess(httperror.LoggerHandler(h.dockerhubInspect))).Methods(http.MethodGet)
 	h.Handle("/dockerhub",
-		bouncer.AdministratorAccess(httperror.LoggerHandler(h.dockerhubUpdate))).Methods(http.MethodPut)
+		bouncer.AuthorizedAccess(httperror.LoggerHandler(h.dockerhubUpdate))).Methods(http.MethodPut)
 
 	return h
 }

api/http/handler/endpointgroups/endpointgroup_create.go

@@ -4,10 +4,10 @@ import (
 	"net/http"
 
 	"github.com/asaskevich/govalidator"
-	"github.com/portainer/portainer"
-	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/request"
-	"github.com/portainer/portainer/http/response"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	"github.com/portainer/portainer/api"
 )
 
 type endpointGroupCreatePayload struct {
@@ -36,11 +36,11 @@ func (handler *Handler) endpointGroupCreate(w http.ResponseWriter, r *http.Reque
 	}
 
 	endpointGroup := &portainer.EndpointGroup{
-		Name:            payload.Name,
-		Description:     payload.Description,
-		AuthorizedUsers: []portainer.UserID{},
-		AuthorizedTeams: []portainer.TeamID{},
-		Tags:            payload.Tags,
+		Name:               payload.Name,
+		Description:        payload.Description,
+		UserAccessPolicies: portainer.UserAccessPolicies{},
+		TeamAccessPolicies: portainer.TeamAccessPolicies{},
+		Tags:               payload.Tags,
 	}
 
 	err = handler.EndpointGroupService.CreateEndpointGroup(endpointGroup)

api/http/handler/endpointgroups/endpointgroup_delete.go

@@ -3,10 +3,10 @@ package endpointgroups
 import (
 	"net/http"
 
-	"github.com/portainer/portainer"
-	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/request"
-	"github.com/portainer/portainer/http/response"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	"github.com/portainer/portainer/api"
 )
 
 // DELETE request on /api/endpoint_groups/:id

api/http/handler/endpointgroups/endpointgroup_inspect.go

@@ -3,10 +3,10 @@ package endpointgroups
 import (
 	"net/http"
 
-	"github.com/portainer/portainer"
-	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/request"
-	"github.com/portainer/portainer/http/response"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	"github.com/portainer/portainer/api"
 )
 
 // GET request on /api/endpoint_groups/:id

api/http/handler/endpointgroups/endpointgroup_list.go

@@ -3,9 +3,9 @@ package endpointgroups
 import (
 	"net/http"
 
-	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/response"
-	"github.com/portainer/portainer/http/security"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/response"
+	"github.com/portainer/portainer/api/http/security"
 )
 
 // GET request on /api/endpoint_groups

api/http/handler/endpointgroups/endpointgroup_update.go

@@ -3,10 +3,10 @@ package endpointgroups
 import (
 	"net/http"
 
-	"github.com/portainer/portainer"
-	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/request"
-	"github.com/portainer/portainer/http/response"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	"github.com/portainer/portainer/api"
 )
 
 type endpointGroupUpdatePayload struct {
@@ -14,6 +14,8 @@ type endpointGroupUpdatePayload struct {
 	Description         string
 	AssociatedEndpoints []portainer.EndpointID
 	Tags                []string
+	UserAccessPolicies  portainer.UserAccessPolicies
+	TeamAccessPolicies  portainer.TeamAccessPolicies
 }
 
 func (payload *endpointGroupUpdatePayload) Validate(r *http.Request) error {
@@ -52,20 +54,30 @@ func (handler *Handler) endpointGroupUpdate(w http.ResponseWriter, r *http.Reque
 		endpointGroup.Tags = payload.Tags
 	}
 
+	if payload.UserAccessPolicies != nil {
+		endpointGroup.UserAccessPolicies = payload.UserAccessPolicies
+	}
+
+	if payload.TeamAccessPolicies != nil {
+		endpointGroup.TeamAccessPolicies = payload.TeamAccessPolicies
+	}
+
 	err = handler.EndpointGroupService.UpdateEndpointGroup(endpointGroup.ID, endpointGroup)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint group changes inside the database", err}
 	}
 
-	endpoints, err := handler.EndpointService.Endpoints()
-	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoints from the database", err}
-	}
-
-	for _, endpoint := range endpoints {
-		err = handler.updateEndpointGroup(endpoint, portainer.EndpointGroupID(endpointGroupID), payload.AssociatedEndpoints)
+	if payload.AssociatedEndpoints != nil {
+		endpoints, err := handler.EndpointService.Endpoints()
 		if err != nil {
-			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to update endpoint", err}
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoints from the database", err}
+		}
+
+		for _, endpoint := range endpoints {
+			err = handler.updateEndpointGroup(endpoint, portainer.EndpointGroupID(endpointGroupID), payload.AssociatedEndpoints)
+			if err != nil {
+				return &httperror.HandlerError{http.StatusInternalServerError, "Unable to update endpoint", err}
+			}
 		}
 	}
 

api/http/handler/endpointgroups/endpointgroup_update_access.go

@@ -1,63 +0,0 @@
-package endpointgroups
-
-import (
-	"net/http"
-
-	"github.com/portainer/portainer"
-	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/request"
-	"github.com/portainer/portainer/http/response"
-)
-
-type endpointGroupUpdateAccessPayload struct {
-	AuthorizedUsers []int
-	AuthorizedTeams []int
-}
-
-func (payload *endpointGroupUpdateAccessPayload) Validate(r *http.Request) error {
-	return nil
-}
-
-// PUT request on /api/endpoint_groups/:id/access
-func (handler *Handler) endpointGroupUpdateAccess(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	endpointGroupID, err := request.RetrieveNumericRouteVariableValue(r, "id")
-	if err != nil {
-		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint group identifier route variable", err}
-	}
-
-	var payload endpointGroupUpdateAccessPayload
-	err = request.DecodeAndValidateJSONPayload(r, &payload)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
-	}
-
-	endpointGroup, err := handler.EndpointGroupService.EndpointGroup(portainer.EndpointGroupID(endpointGroupID))
-	if err == portainer.ErrObjectNotFound {
-		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint group with the specified identifier inside the database", err}
-	} else if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint group with the specified identifier inside the database", err}
-	}
-
-	if payload.AuthorizedUsers != nil {
-		authorizedUserIDs := []portainer.UserID{}
-		for _, value := range payload.AuthorizedUsers {
-			authorizedUserIDs = append(authorizedUserIDs, portainer.UserID(value))
-		}
-		endpointGroup.AuthorizedUsers = authorizedUserIDs
-	}
-
-	if payload.AuthorizedTeams != nil {
-		authorizedTeamIDs := []portainer.TeamID{}
-		for _, value := range payload.AuthorizedTeams {
-			authorizedTeamIDs = append(authorizedTeamIDs, portainer.TeamID(value))
-		}
-		endpointGroup.AuthorizedTeams = authorizedTeamIDs
-	}
-
-	err = handler.EndpointGroupService.UpdateEndpointGroup(endpointGroup.ID, endpointGroup)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint group changes inside the database", err}
-	}
-
-	return response.JSON(w, endpointGroup)
-}

api/http/handler/endpointgroups/handler.go

@@ -4,9 +4,9 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
-	"github.com/portainer/portainer"
-	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/security"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/http/security"
 )
 
 // Handler is the HTTP handler used to handle endpoint group operations.
@@ -22,17 +22,15 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler {
 		Router: mux.NewRouter(),
 	}
 	h.Handle("/endpoint_groups",
-		bouncer.AdministratorAccess(httperror.LoggerHandler(h.endpointGroupCreate))).Methods(http.MethodPost)
+		bouncer.AuthorizedAccess(httperror.LoggerHandler(h.endpointGroupCreate))).Methods(http.MethodPost)
 	h.Handle("/endpoint_groups",
-		bouncer.RestrictedAccess(httperror.LoggerHandler(h.endpointGroupList))).Methods(http.MethodGet)
+		bouncer.AuthorizedAccess(httperror.LoggerHandler(h.endpointGroupList))).Methods(http.MethodGet)
 	h.Handle("/endpoint_groups/{id}",
-		bouncer.AdministratorAccess(httperror.LoggerHandler(h.endpointGroupInspect))).Methods(http.MethodGet)
+		bouncer.AuthorizedAccess(httperror.LoggerHandler(h.endpointGroupInspect))).Methods(http.MethodGet)
 	h.Handle("/endpoint_groups/{id}",
-		bouncer.AdministratorAccess(httperror.LoggerHandler(h.endpointGroupUpdate))).Methods(http.MethodPut)
-	h.Handle("/endpoint_groups/{id}/access",
-		bouncer.AdministratorAccess(httperror.LoggerHandler(h.endpointGroupUpdateAccess))).Methods(http.MethodPut)
+		bouncer.AuthorizedAccess(httperror.LoggerHandler(h.endpointGroupUpdate))).Methods(http.MethodPut)
 	h.Handle("/endpoint_groups/{id}",
-		bouncer.AdministratorAccess(httperror.LoggerHandler(h.endpointGroupDelete))).Methods(http.MethodDelete)
+		bouncer.AuthorizedAccess(httperror.LoggerHandler(h.endpointGroupDelete))).Methods(http.MethodDelete)
 
 	return h
 }

api/http/handler/endpointproxy/handler.go

@@ -2,10 +2,10 @@ package endpointproxy
 
 import (
 	"github.com/gorilla/mux"
-	"github.com/portainer/portainer"
-	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/proxy"
-	"github.com/portainer/portainer/http/security"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/http/proxy"
+	"github.com/portainer/portainer/api/http/security"
 )
 
 // Handler is the HTTP handler used to proxy requests to external APIs.
@@ -23,10 +23,10 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler {
 		requestBouncer: bouncer,
 	}
 	h.PathPrefix("/{id}/azure").Handler(
-		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.proxyRequestsToAzureAPI)))
+		bouncer.RestrictedAccess(httperror.LoggerHandler(h.proxyRequestsToAzureAPI)))
 	h.PathPrefix("/{id}/docker").Handler(
-		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.proxyRequestsToDockerAPI)))
-	h.PathPrefix("/{id}/extensions/storidge").Handler(
-		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.proxyRequestsToStoridgeAPI)))
+		bouncer.RestrictedAccess(httperror.LoggerHandler(h.proxyRequestsToDockerAPI)))
+	h.PathPrefix("/{id}/storidge").Handler(
+		bouncer.RestrictedAccess(httperror.LoggerHandler(h.proxyRequestsToStoridgeAPI)))
 	return h
 }

api/http/handler/endpointproxy/proxy_azure.go

@@ -3,9 +3,9 @@ package endpointproxy
 import (
 	"strconv"
 
-	"github.com/portainer/portainer"
-	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/request"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/portainer/api"
 
 	"net/http"
 )
@@ -23,9 +23,9 @@ func (handler *Handler) proxyRequestsToAzureAPI(w http.ResponseWriter, r *http.R
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint with the specified identifier inside the database", err}
 	}
 
-	err = handler.requestBouncer.EndpointAccess(r, endpoint)
+	err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint, false)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access endpoint", portainer.ErrEndpointAccessDenied}
+		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access endpoint", err}
 	}
 
 	var proxy http.Handler

api/http/handler/endpointproxy/proxy_docker.go

@@ -1,11 +1,12 @@
 package endpointproxy
 
 import (
+	"errors"
 	"strconv"
 
-	"github.com/portainer/portainer"
-	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/request"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/portainer/api"
 
 	"net/http"
 )
@@ -23,9 +24,13 @@ func (handler *Handler) proxyRequestsToDockerAPI(w http.ResponseWriter, r *http.
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint with the specified identifier inside the database", err}
 	}
 
-	err = handler.requestBouncer.EndpointAccess(r, endpoint)
+	if endpoint.Type != 4 && endpoint.Status == portainer.EndpointStatusDown {
+		return &httperror.HandlerError{http.StatusServiceUnavailable, "Unable to query endpoint", errors.New("Endpoint is down")}
+	}
+
+	err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint, true)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access endpoint", portainer.ErrEndpointAccessDenied}
+		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access endpoint", err}
 	}
 
 	var proxy http.Handler

api/http/handler/endpointproxy/proxy_storidge.go

@@ -1,11 +1,13 @@
 package endpointproxy
 
+// TODO: legacy extension management
+
 import (
 	"strconv"
 
-	"github.com/portainer/portainer"
-	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/request"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/portainer/api"
 
 	"net/http"
 )
@@ -23,9 +25,9 @@ func (handler *Handler) proxyRequestsToStoridgeAPI(w http.ResponseWriter, r *htt
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint with the specified identifier inside the database", err}
 	}
 
-	err = handler.requestBouncer.EndpointAccess(r, endpoint)
+	err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint, false)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access endpoint", portainer.ErrEndpointAccessDenied}
+		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access endpoint", err}
 	}
 
 	var storidgeExtension *portainer.EndpointExtension
@@ -39,18 +41,18 @@ func (handler *Handler) proxyRequestsToStoridgeAPI(w http.ResponseWriter, r *htt
 		return &httperror.HandlerError{http.StatusServiceUnavailable, "Storidge extension not supported on this endpoint", portainer.ErrEndpointExtensionNotSupported}
 	}
 
-	proxyExtensionKey := string(endpoint.ID) + "_" + string(portainer.StoridgeEndpointExtension)
+	proxyExtensionKey := strconv.Itoa(endpointID) + "_" + strconv.Itoa(int(portainer.StoridgeEndpointExtension)) + "_" + storidgeExtension.URL
 
 	var proxy http.Handler
-	proxy = handler.ProxyManager.GetExtensionProxy(proxyExtensionKey)
+	proxy = handler.ProxyManager.GetLegacyExtensionProxy(proxyExtensionKey)
 	if proxy == nil {
-		proxy, err = handler.ProxyManager.CreateAndRegisterExtensionProxy(proxyExtensionKey, storidgeExtension.URL)
+		proxy, err = handler.ProxyManager.CreateLegacyExtensionProxy(proxyExtensionKey, storidgeExtension.URL)
 		if err != nil {
 			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to create extension proxy", err}
 		}
 	}
 
 	id := strconv.Itoa(endpointID)
-	http.StripPrefix("/"+id+"/extensions/storidge", proxy).ServeHTTP(w, r)
+	http.StripPrefix("/"+id+"/storidge", proxy).ServeHTTP(w, r)
 	return nil
 }

api/http/handler/endpoints/endpoint_create.go

@@ -1,16 +1,20 @@
 package endpoints
 
 import (
+	"encoding/base64"
+	"log"
+	"math/rand"
 	"net/http"
+	"runtime"
 	"strconv"
 	"strings"
 
-	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/crypto"
-	"github.com/portainer/portainer/http/client"
-	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/request"
-	"github.com/portainer/portainer/http/response"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/crypto"
+	"github.com/portainer/portainer/api/http/client"
 )
 
 type endpointCreatePayload struct {
@@ -34,13 +38,13 @@ type endpointCreatePayload struct {
 func (payload *endpointCreatePayload) Validate(r *http.Request) error {
 	name, err := request.RetrieveMultiPartFormValue(r, "Name", false)
 	if err != nil {
-		return portainer.Error("Invalid stack name")
+		return portainer.Error("Invalid endpoint name")
 	}
 	payload.Name = name
 
 	endpointType, err := request.RetrieveNumericMultiPartFormValue(r, "EndpointType", false)
 	if err != nil || endpointType == 0 {
-		return portainer.Error("Invalid endpoint type value. Value must be one of: 1 (Docker environment), 2 (Agent environment) or 3 (Azure environment)")
+		return portainer.Error("Invalid endpoint type value. Value must be one of: 1 (Docker environment), 2 (Agent environment), 3 (Azure environment) or 4 (Edge Agent environment)")
 	}
 	payload.EndpointType = endpointType
 
@@ -56,6 +60,9 @@ func (payload *endpointCreatePayload) Validate(r *http.Request) error {
 		return portainer.Error("Invalid Tags parameter")
 	}
 	payload.Tags = tags
+	if payload.Tags == nil {
+		payload.Tags = make([]string, 0)
+	}
 
 	useTLS, _ := request.RetrieveBooleanMultiPartFormValue(r, "TLS", true)
 	payload.TLS = useTLS
@@ -67,7 +74,7 @@ func (payload *endpointCreatePayload) Validate(r *http.Request) error {
 		payload.TLSSkipClientVerify = skipTLSClientVerification
 
 		if !payload.TLSSkipVerify {
-			caCert, err := request.RetrieveMultiPartFormFile(r, "TLSCACertFile")
+			caCert, _, err := request.RetrieveMultiPartFormFile(r, "TLSCACertFile")
 			if err != nil {
 				return portainer.Error("Invalid CA certificate file. Ensure that the file is uploaded correctly")
 			}
@@ -75,13 +82,13 @@ func (payload *endpointCreatePayload) Validate(r *http.Request) error {
 		}
 
 		if !payload.TLSSkipClientVerify {
-			cert, err := request.RetrieveMultiPartFormFile(r, "TLSCertFile")
+			cert, _, err := request.RetrieveMultiPartFormFile(r, "TLSCertFile")
 			if err != nil {
 				return portainer.Error("Invalid certificate file. Ensure that the file is uploaded correctly")
 			}
 			payload.TLSCertFile = cert
 
-			key, err := request.RetrieveMultiPartFormFile(r, "TLSKeyFile")
+			key, _, err := request.RetrieveMultiPartFormFile(r, "TLSKeyFile")
 			if err != nil {
 				return portainer.Error("Invalid key file. Ensure that the file is uploaded correctly")
 			}
@@ -109,7 +116,7 @@ func (payload *endpointCreatePayload) Validate(r *http.Request) error {
 		}
 		payload.AzureAuthenticationKey = azureAuthenticationKey
 	default:
-		url, err := request.RetrieveMultiPartFormValue(r, "URL", false)
+		url, err := request.RetrieveMultiPartFormValue(r, "URL", true)
 		if err != nil {
 			return portainer.Error("Invalid endpoint URL")
 		}
@@ -145,6 +152,8 @@ func (handler *Handler) endpointCreate(w http.ResponseWriter, r *http.Request) *
 func (handler *Handler) createEndpoint(payload *endpointCreatePayload) (*portainer.Endpoint, *httperror.HandlerError) {
 	if portainer.EndpointType(payload.EndpointType) == portainer.AzureEnvironment {
 		return handler.createAzureEndpoint(payload)
+	} else if portainer.EndpointType(payload.EndpointType) == portainer.EdgeAgentEnvironment {
+		return handler.createEdgeAgentEndpoint(payload)
 	}
 
 	if payload.TLS {
@@ -166,17 +175,21 @@ func (handler *Handler) createAzureEndpoint(payload *endpointCreatePayload) (*po
 		return nil, &httperror.HandlerError{http.StatusInternalServerError, "Unable to authenticate against Azure", err}
 	}
 
+	endpointID := handler.EndpointService.GetNextIdentifier()
 	endpoint := &portainer.Endpoint{
-		Name:             payload.Name,
-		URL:              payload.URL,
-		Type:             portainer.AzureEnvironment,
-		GroupID:          portainer.EndpointGroupID(payload.GroupID),
-		PublicURL:        payload.PublicURL,
-		AuthorizedUsers:  []portainer.UserID{},
-		AuthorizedTeams:  []portainer.TeamID{},
-		Extensions:       []portainer.EndpointExtension{},
-		AzureCredentials: credentials,
-		Tags:             payload.Tags,
+		ID:                 portainer.EndpointID(endpointID),
+		Name:               payload.Name,
+		URL:                "https://management.azure.com",
+		Type:               portainer.AzureEnvironment,
+		GroupID:            portainer.EndpointGroupID(payload.GroupID),
+		PublicURL:          payload.PublicURL,
+		UserAccessPolicies: portainer.UserAccessPolicies{},
+		TeamAccessPolicies: portainer.TeamAccessPolicies{},
+		Extensions:         []portainer.EndpointExtension{},
+		AzureCredentials:   credentials,
+		Tags:               payload.Tags,
+		Status:             portainer.EndpointStatusUp,
+		Snapshots:          []portainer.Snapshot{},
 	}
 
 	err = handler.EndpointService.CreateEndpoint(endpoint)
@@ -187,10 +200,72 @@ func (handler *Handler) createAzureEndpoint(payload *endpointCreatePayload) (*po
 	return endpoint, nil
 }
 
+// TODO: relocate in a service
+// must be unique (e.g. not used / referenced)
+func randomInt(min, max int) int {
+	// should be randomize at service creation time?
+	// if not seeded, will always get same port order
+	// might not be a problem and maybe not required
+	//rand.Seed(time.Now().UnixNano())
+
+	return min + rand.Intn(max-min)
+}
+
+func (handler *Handler) createEdgeAgentEndpoint(payload *endpointCreatePayload) (*portainer.Endpoint, *httperror.HandlerError) {
+	endpointType := portainer.EdgeAgentEnvironment
+	endpointID := handler.EndpointService.GetNextIdentifier()
+
+	// get random port
+	// Dynamic ports (also called private ports) are 49152 to 65535.
+	// TODO: register this port somewhere
+	portnumber := randomInt(49152, 65535)
+
+	keyInformation := []string{
+		strings.TrimPrefix(payload.URL, "tcp://"),
+		"8000",
+		handler.TunnelServerFingerprint,
+		strconv.Itoa(portnumber),
+		"agent:randomstring",
+	}
+
+	key := strings.Join(keyInformation, "|")
+	encodedKey := base64.RawStdEncoding.EncodeToString([]byte(key))
+
+	endpoint := &portainer.Endpoint{
+		ID:      portainer.EndpointID(endpointID),
+		Name:    payload.Name,
+		URL:     "tcp://localhost:" + strconv.Itoa(portnumber),
+		Type:    endpointType,
+		GroupID: portainer.EndpointGroupID(payload.GroupID),
+		TLSConfig: portainer.TLSConfiguration{
+			TLS: false,
+		},
+		AuthorizedUsers: []portainer.UserID{},
+		AuthorizedTeams: []portainer.TeamID{},
+		Extensions:      []portainer.EndpointExtension{},
+		Tags:            payload.Tags,
+		Status:          portainer.EndpointStatusUp,
+		Snapshots:       []portainer.Snapshot{},
+		EdgeKey:         string(encodedKey),
+	}
+
+	err := handler.EndpointService.CreateEndpoint(endpoint)
+	if err != nil {
+		return nil, &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint inside the database", err}
+	}
+
+	return endpoint, nil
+}
+
 func (handler *Handler) createUnsecuredEndpoint(payload *endpointCreatePayload) (*portainer.Endpoint, *httperror.HandlerError) {
 	endpointType := portainer.DockerEnvironment
 
-	if !strings.HasPrefix(payload.URL, "unix://") {
+	if payload.URL == "" {
+		payload.URL = "unix:///var/run/docker.sock"
+		if runtime.GOOS == "windows" {
+			payload.URL = "npipe:////./pipe/docker_engine"
+		}
+	} else {
 		agentOnDockerEnvironment, err := client.ExecutePingOperation(payload.URL, nil)
 		if err != nil {
 			return nil, &httperror.HandlerError{http.StatusInternalServerError, "Unable to ping Docker environment", err}
@@ -200,7 +275,9 @@ func (handler *Handler) createUnsecuredEndpoint(payload *endpointCreatePayload)
 		}
 	}
 
+	endpointID := handler.EndpointService.GetNextIdentifier()
 	endpoint := &portainer.Endpoint{
+		ID:        portainer.EndpointID(endpointID),
 		Name:      payload.Name,
 		URL:       payload.URL,
 		Type:      endpointType,
@@ -209,15 +286,17 @@ func (handler *Handler) createUnsecuredEndpoint(payload *endpointCreatePayload)
 		TLSConfig: portainer.TLSConfiguration{
 			TLS: false,
 		},
-		AuthorizedUsers: []portainer.UserID{},
-		AuthorizedTeams: []portainer.TeamID{},
-		Extensions:      []portainer.EndpointExtension{},
-		Tags:            payload.Tags,
+		UserAccessPolicies: portainer.UserAccessPolicies{},
+		TeamAccessPolicies: portainer.TeamAccessPolicies{},
+		Extensions:         []portainer.EndpointExtension{},
+		Tags:               payload.Tags,
+		Status:             portainer.EndpointStatusUp,
+		Snapshots:          []portainer.Snapshot{},
 	}
 
-	err := handler.EndpointService.CreateEndpoint(endpoint)
+	err := handler.snapshotAndPersistEndpoint(endpoint)
 	if err != nil {
-		return nil, &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint inside the database", err}
+		return nil, err
 	}
 
 	return endpoint, nil
@@ -239,7 +318,9 @@ func (handler *Handler) createTLSSecuredEndpoint(payload *endpointCreatePayload)
 		endpointType = portainer.AgentOnDockerEnvironment
 	}
 
+	endpointID := handler.EndpointService.GetNextIdentifier()
 	endpoint := &portainer.Endpoint{
+		ID:        portainer.EndpointID(endpointID),
 		Name:      payload.Name,
 		URL:       payload.URL,
 		Type:      endpointType,
@@ -249,38 +330,53 @@ func (handler *Handler) createTLSSecuredEndpoint(payload *endpointCreatePayload)
 			TLS:           payload.TLS,
 			TLSSkipVerify: payload.TLSSkipVerify,
 		},
-		AuthorizedUsers: []portainer.UserID{},
-		AuthorizedTeams: []portainer.TeamID{},
-		Extensions:      []portainer.EndpointExtension{},
-		Tags:            payload.Tags,
-	}
-
-	err = handler.EndpointService.CreateEndpoint(endpoint)
-	if err != nil {
-		return nil, &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint inside the database", err}
+		UserAccessPolicies: portainer.UserAccessPolicies{},
+		TeamAccessPolicies: portainer.TeamAccessPolicies{},
+		Extensions:         []portainer.EndpointExtension{},
+		Tags:               payload.Tags,
+		Status:             portainer.EndpointStatusUp,
+		Snapshots:          []portainer.Snapshot{},
 	}
 
 	filesystemError := handler.storeTLSFiles(endpoint, payload)
 	if err != nil {
-		handler.EndpointService.DeleteEndpoint(endpoint.ID)
 		return nil, filesystemError
 	}
 
-	err = handler.EndpointService.UpdateEndpoint(endpoint.ID, endpoint)
-	if err != nil {
-		return nil, &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint changes inside the database", err}
+	endpointCreationError := handler.snapshotAndPersistEndpoint(endpoint)
+	if endpointCreationError != nil {
+		return nil, endpointCreationError
 	}
 
 	return endpoint, nil
 }
 
+func (handler *Handler) snapshotAndPersistEndpoint(endpoint *portainer.Endpoint) *httperror.HandlerError {
+	snapshot, err := handler.Snapshotter.CreateSnapshot(endpoint)
+	endpoint.Status = portainer.EndpointStatusUp
+	if err != nil {
+		log.Printf("http error: endpoint snapshot error (endpoint=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
+		endpoint.Status = portainer.EndpointStatusDown
+	}
+
+	if snapshot != nil {
+		endpoint.Snapshots = []portainer.Snapshot{*snapshot}
+	}
+
+	err = handler.EndpointService.CreateEndpoint(endpoint)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint inside the database", err}
+	}
+
+	return nil
+}
+
 func (handler *Handler) storeTLSFiles(endpoint *portainer.Endpoint, payload *endpointCreatePayload) *httperror.HandlerError {
 	folder := strconv.Itoa(int(endpoint.ID))
 
 	if !payload.TLSSkipVerify {
 		caCertPath, err := handler.FileService.StoreTLSFileFromBytes(folder, portainer.TLSFileCA, payload.TLSCACertFile)
 		if err != nil {
-			handler.EndpointService.DeleteEndpoint(endpoint.ID)
 			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist TLS CA certificate file on disk", err}
 		}
 		endpoint.TLSConfig.TLSCACertPath = caCertPath
@@ -289,14 +385,12 @@ func (handler *Handler) storeTLSFiles(endpoint *portainer.Endpoint, payload *end
 	if !payload.TLSSkipClientVerify {
 		certPath, err := handler.FileService.StoreTLSFileFromBytes(folder, portainer.TLSFileCert, payload.TLSCertFile)
 		if err != nil {
-			handler.EndpointService.DeleteEndpoint(endpoint.ID)
 			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist TLS certificate file on disk", err}
 		}
 		endpoint.TLSConfig.TLSCertPath = certPath
 
 		keyPath, err := handler.FileService.StoreTLSFileFromBytes(folder, portainer.TLSFileKey, payload.TLSKeyFile)
 		if err != nil {
-			handler.EndpointService.DeleteEndpoint(endpoint.ID)
 			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist TLS key file on disk", err}
 		}
 		endpoint.TLSConfig.TLSKeyPath = keyPath

api/http/handler/endpoints/endpoint_delete.go

@@ -4,10 +4,10 @@ import (
 	"net/http"
 	"strconv"
 
-	"github.com/portainer/portainer"
-	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/request"
-	"github.com/portainer/portainer/http/response"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	"github.com/portainer/portainer/api"
 )
 
 // DELETE request on /api/endpoints/:id
@@ -42,7 +42,6 @@ func (handler *Handler) endpointDelete(w http.ResponseWriter, r *http.Request) *
 	}
 
 	handler.ProxyManager.DeleteProxy(string(endpointID))
-	handler.ProxyManager.DeleteExtensionProxies(string(endpointID))
 
 	return response.Empty(w)
 }

api/http/handler/endpoints/endpoint_extension_add.go

@@ -1,13 +1,15 @@
 package endpoints
 
+// TODO: legacy extension management
+
 import (
 	"net/http"
 
 	"github.com/asaskevich/govalidator"
-	"github.com/portainer/portainer"
-	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/request"
-	"github.com/portainer/portainer/http/response"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	"github.com/portainer/portainer/api"
 )
 
 type endpointExtensionAddPayload struct {
@@ -48,9 +50,9 @@ func (handler *Handler) endpointExtensionAdd(w http.ResponseWriter, r *http.Requ
 	extensionType := portainer.EndpointExtensionType(payload.Type)
 
 	var extension *portainer.EndpointExtension
-	for _, ext := range endpoint.Extensions {
-		if ext.Type == extensionType {
-			extension = &ext
+	for idx := range endpoint.Extensions {
+		if endpoint.Extensions[idx].Type == extensionType {
+			extension = &endpoint.Extensions[idx]
 		}
 	}
 

api/http/handler/endpoints/endpoint_extension_remove.go

@@ -1,12 +1,14 @@
 package endpoints
 
+// TODO: legacy extension management
+
 import (
 	"net/http"
 
-	"github.com/portainer/portainer"
-	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/request"
-	"github.com/portainer/portainer/http/response"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	"github.com/portainer/portainer/api"
 )
 
 // DELETE request on /api/endpoints/:id/extensions/:extensionType

api/http/handler/endpoints/endpoint_inspect.go

@@ -3,10 +3,10 @@ package endpoints
 import (
 	"net/http"
 
-	"github.com/portainer/portainer"
-	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/request"
-	"github.com/portainer/portainer/http/response"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	"github.com/portainer/portainer/api"
 )
 
 // GET request on /api/endpoints/:id
@@ -23,5 +23,12 @@ func (handler *Handler) endpointInspect(w http.ResponseWriter, r *http.Request)
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint with the specified identifier inside the database", err}
 	}
 
+	err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint, false)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access endpoint", err}
+	}
+
+	hideFields(endpoint)
+
 	return response.JSON(w, endpoint)
 }

api/http/handler/endpoints/endpoint_job.go

@@ -0,0 +1,111 @@
+package endpoints
+
+import (
+	"errors"
+	"net/http"
+
+	"github.com/asaskevich/govalidator"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	"github.com/portainer/portainer/api"
+)
+
+type endpointJobFromFilePayload struct {
+	Image string
+	File  []byte
+}
+
+type endpointJobFromFileContentPayload struct {
+	Image       string
+	FileContent string
+}
+
+func (payload *endpointJobFromFilePayload) Validate(r *http.Request) error {
+	file, _, err := request.RetrieveMultiPartFormFile(r, "File")
+	if err != nil {
+		return portainer.Error("Invalid Script file. Ensure that the file is uploaded correctly")
+	}
+	payload.File = file
+
+	image, err := request.RetrieveMultiPartFormValue(r, "Image", false)
+	if err != nil {
+		return portainer.Error("Invalid image name")
+	}
+	payload.Image = image
+
+	return nil
+}
+
+func (payload *endpointJobFromFileContentPayload) Validate(r *http.Request) error {
+	if govalidator.IsNull(payload.FileContent) {
+		return portainer.Error("Invalid script file content")
+	}
+
+	if govalidator.IsNull(payload.Image) {
+		return portainer.Error("Invalid image name")
+	}
+
+	return nil
+}
+
+// POST request on /api/endpoints/:id/job?method&nodeName
+func (handler *Handler) endpointJob(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint identifier route variable", err}
+	}
+
+	method, err := request.RetrieveQueryParameter(r, "method", false)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid query parameter: method", err}
+	}
+
+	nodeName, _ := request.RetrieveQueryParameter(r, "nodeName", true)
+
+	endpoint, err := handler.EndpointService.Endpoint(portainer.EndpointID(endpointID))
+	if err == portainer.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
+	} else if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint with the specified identifier inside the database", err}
+	}
+
+	switch method {
+	case "file":
+		return handler.executeJobFromFile(w, r, endpoint, nodeName)
+	case "string":
+		return handler.executeJobFromFileContent(w, r, endpoint, nodeName)
+	}
+
+	return &httperror.HandlerError{http.StatusBadRequest, "Invalid value for query parameter: method. Value must be one of: string or file", errors.New(request.ErrInvalidQueryParameter)}
+}
+
+func (handler *Handler) executeJobFromFile(w http.ResponseWriter, r *http.Request, endpoint *portainer.Endpoint, nodeName string) *httperror.HandlerError {
+	payload := &endpointJobFromFilePayload{}
+	err := payload.Validate(r)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
+	}
+
+	err = handler.JobService.ExecuteScript(endpoint, nodeName, payload.Image, payload.File, nil)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Failed executing job", err}
+	}
+
+	return response.Empty(w)
+}
+
+func (handler *Handler) executeJobFromFileContent(w http.ResponseWriter, r *http.Request, endpoint *portainer.Endpoint, nodeName string) *httperror.HandlerError {
+	var payload endpointJobFromFileContentPayload
+	err := request.DecodeAndValidateJSONPayload(r, &payload)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
+	}
+
+	err = handler.JobService.ExecuteScript(endpoint, nodeName, payload.Image, []byte(payload.FileContent), nil)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Failed executing job", err}
+	}
+
+	return response.Empty(w)
+}

api/http/handler/endpoints/endpoint_list.go

@@ -3,9 +3,9 @@ package endpoints
 import (
 	"net/http"
 
-	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/response"
-	"github.com/portainer/portainer/http/security"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/response"
+	"github.com/portainer/portainer/api/http/security"
 )
 
 // GET request on /api/endpoints
@@ -27,8 +27,9 @@ func (handler *Handler) endpointList(w http.ResponseWriter, r *http.Request) *ht
 
 	filteredEndpoints := security.FilterEndpoints(endpoints, endpointGroups, securityContext)
 
-	for _, endpoint := range filteredEndpoints {
-		hideFields(&endpoint)
+	for idx := range filteredEndpoints {
+		hideFields(&filteredEndpoints[idx])
 	}
+
 	return response.JSON(w, filteredEndpoints)
 }

api/http/handler/endpoints/endpoint_snapshot.go

@@ -0,0 +1,52 @@
+package endpoints
+
+import (
+	"net/http"
+
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	"github.com/portainer/portainer/api"
+)
+
+// POST request on /api/endpoints/:id/snapshot
+func (handler *Handler) endpointSnapshot(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint identifier route variable", err}
+	}
+
+	endpoint, err := handler.EndpointService.Endpoint(portainer.EndpointID(endpointID))
+	if err == portainer.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
+	} else if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint with the specified identifier inside the database", err}
+	}
+
+	if endpoint.Type == portainer.AzureEnvironment {
+		return &httperror.HandlerError{http.StatusBadRequest, "Snapshots not supported for Azure endpoints", err}
+	}
+
+	snapshot, snapshotError := handler.Snapshotter.CreateSnapshot(endpoint)
+
+	latestEndpointReference, err := handler.EndpointService.Endpoint(endpoint.ID)
+	if latestEndpointReference == nil {
+		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
+	}
+
+	latestEndpointReference.Status = portainer.EndpointStatusUp
+	if snapshotError != nil {
+		latestEndpointReference.Status = portainer.EndpointStatusDown
+	}
+
+	if snapshot != nil {
+		latestEndpointReference.Snapshots = []portainer.Snapshot{*snapshot}
+	}
+
+	err = handler.EndpointService.UpdateEndpoint(latestEndpointReference.ID, latestEndpointReference)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint changes inside the database", err}
+	}
+
+	return response.Empty(w)
+}