Compare commits

..

98 Commits

Author SHA1 Message Date
Oscar Zhou
b284d7094a fix(stack): filter out orphan stacks that have same name as normal stacks [EE-6791] (#11471) 2024-04-03 09:53:36 +13:00
LP B
7bb54bcbe6 fix(app): replace fields removed by Docker 25 and 26 (#11469)
* fix(app/volume): make optional Container and ContainerConfig fields removed in docker 26

* fix(app/image): use image.Size instead of image.VirtualSize removed in Docker 25
2024-03-29 13:57:18 +01:00
cmeng
b3c489366f fix(edge-stack): avoid reference of undefined EE-6914 (#11465) 2024-03-27 16:02:25 +13:00
cmeng
5eca761883 feat(version): bump to 2.20.1 EE-6933 (#11459) 2024-03-27 15:41:45 +13:00
andres-portainer
bea8acce1f fix(kubernetes): avoid a deadlock EE-6901 (#11446) 2024-03-25 14:19:33 -03:00
Matt Hook
6a3eda4bce fix(doclinks): fix help link paths [EE-6861] (#11417) 2024-03-19 11:46:55 +13:00
Matt Hook
889c36f64a fix(docs): fix all remaining webhook app links [EE-6861] (#11392) 2024-03-18 16:28:43 +13:00
Matt Hook
c8fb3adda3 fix(kube): fix edit application webhook link [EE-6861] (#11390) 2024-03-18 10:21:20 +13:00
cmeng
f15be1d92a fix(stack): prepopulate when creating template from stack EE-6853 (#11379) 2024-03-18 09:36:04 +13:00
Oscar Zhou
d9ae249ffe chore(template/git): sync frontend code from ee (#11343) 2024-03-18 08:55:26 +13:00
Matt Hook
04de06c07f fix(docs): make all doc links versioned [EE-6861] (#11381) 2024-03-15 16:57:42 +13:00
Matt Hook
59d53940fe fix(stacks): update swagger stacks doc description [EE-6860] (#11383) 2024-03-15 16:47:05 +13:00
cmeng
db16888379 fix(container): make blank string as valid value EE-6852 (#11372) 2024-03-15 09:01:42 +13:00
Prabhat Khera
8880876bcd fix(auth): make createAccessToken api backward compatible [EE-6818] (#11327)
* fix(auth): make createAccessToken api backward compatible [EE-6818]

* fix(api): api error message [EE-6818]

* fix messages
2024-03-14 09:02:25 +13:00
Ali
bfe5a49263 fix(app): only show special message when limits change for existing app resource limit [EE-6837] (#11368)
Co-authored-by: testa113 <testa113>
2024-03-14 08:45:53 +13:00
cmeng
6e11c10bab fix(csrf): disable csrf secure cookie EE-6787 (#11299) 2024-03-13 11:22:18 +13:00
LP B
cb9ab3b375 fix(app): views not loading when quickly navigating in app (#11279) 2024-03-12 15:16:19 +01:00
Chaim Lev-Ari
b13dac0f6d fix(docker): apply private uac to edge admin [EE-6788] (#11284) 2024-03-12 09:59:39 +02:00
cmeng
0144a98b3b fix(edge-stack): deploy button is disabled EE-6819 (#11354) 2024-03-12 17:19:45 +13:00
Prabhat Khera
64a08c59e9 address review commets (#11361) 2024-03-12 11:32:03 +13:00
Ali
1090c82beb fix(app): on create don't mention previous values [EE-6837] (#11351)
Co-authored-by: testa113 <testa113>
2024-03-11 16:43:45 +13:00
Prabhat Khera
6094dc115b fix(container): autocomplete off for create container form [EE-6761] (#11337)
* autocomplete off doe create container form

* address review commets

* remove auto complete off from forms
2024-03-11 13:38:59 +13:00
Prabhat Khera
30513695b5 fix(kube): stackname in daemonsets and statefulsets app [EE-6670] (#11353) 2024-03-11 10:04:55 +13:00
Chaim Lev-Ari
dd2be9fb1e refactor(tests): wrap tests explicitly with provider [EE-6686] (#11276) 2024-03-10 14:22:05 +02:00
Chaim Lev-Ari
e265b8b67c fix(kube/config): validate change window start [EE-6830] (#11328) 2024-03-10 09:42:29 +02:00
Matt Hook
cc1ce9412a fix(exec): improve alignment of help icon [EE-6816] (#11340) 2024-03-08 14:03:01 +13:00
Prabhat Khera
8eb8df2b30 fix(kube-stacks): change wordings [EE-6670] (#11335) 2024-03-08 12:15:27 +13:00
Ali
c0bd2dfdaf fix(matomo): stop oauth link event [EE-6779] (#11333) 2024-03-08 10:17:26 +13:00
Matt Hook
bf65a38d5a fix(exec): fix alignment and text size and alignment [EE-6816] (#11324) 2024-03-07 12:57:53 +13:00
cmeng
0ea21f2317 fix(menu): edge compute menu not clickable EE-6804 (#11320) 2024-03-07 12:11:59 +13:00
Prabhat Khera
b5f839a920 fix(stacks): make stackName kube stack specific field [EE-6670] (#11316)
* fix(stacks): make stackName kube stack specific field [EE-6670]

* fix wordings
2024-03-07 11:31:28 +13:00
Prabhat Khera
29025e7dd4 fix(UI): axios progress bar loading issue [EE-6781] (#11290) 2024-03-07 11:30:23 +13:00
Ali
692981b615 fix(time window): show errors for component [EE-6800] (#11318)
Co-authored-by: testa113 <testa113>
2024-03-07 09:03:26 +13:00
Chaim Lev-Ari
d6545b6af5 fix(kube/setup): add a11y labels [EE-6747] (#11308) 2024-03-06 14:57:03 +02:00
Matt Hook
6bbf62fe64 fix(contexthelp): remove extra slash from contexthelp docs link [EE-6780] (#11312) 2024-03-06 16:38:19 +13:00
Matt Hook
6b3ddf11d4 fix(helm): remove helm insights from the stack datatable [EE-6803] (#11313) 2024-03-06 16:36:48 +13:00
Dakota Walsh
77c9124e8a fix(datatable): title size EE-6774 (#11273) 2024-03-06 08:01:45 +13:00
Chaim Lev-Ari
2c3dcdd14e fix(docker/images): export image [EE-6807] (#11305) 2024-03-05 19:30:45 +02:00
matias-portainer
ec913b45d6 fix(edge/templates): get correct default value for selectType env vars EE-6796 (#11293) 2024-03-04 10:35:19 -03:00
Matt Hook
51c672af21 fix(kube): update doc links to match new menu structure [EE-6759] (#11266) 2024-03-01 15:37:32 +13:00
Matt Hook
ff178641be fix(help): add versioned doc links to support LTS/STS docs [EE-6780] (#11282) 2024-03-01 15:36:19 +13:00
cmeng
a43454076b fix(edge-stacks): take not-found stack as removed EE-6758 (#11249) 2024-03-01 11:50:27 +13:00
cmeng
a7eaa0f3fa fix(container): get old container info correctly EE-6716 (#11215) 2024-03-01 09:14:26 +13:00
cmeng
8ad11fc88f fix(stack): more space for add button EE-6773 (#11258) 2024-03-01 09:11:46 +13:00
Chaim Lev-Ari
43a95874f4 fix(auth): prevent unauthorized redirect on page load [EE-6777] (#11265) 2024-02-29 09:41:29 +02:00
Chaim Lev-Ari
b4f4c3212a feat(kube): add a11y props for smoke tests [EE-6747] (#11262) 2024-02-29 09:26:10 +02:00
Chaim Lev-Ari
d44f57ed6f fix(ci): prevent tests from running twice [EE-6728] (#11196) 2024-02-29 08:11:46 +02:00
Chaim Lev-Ari
eba08cdca0 fix(docker): hide write buttons for non authorized [EE-6775] (#11261) 2024-02-27 12:36:47 +02:00
Prabhat Khera
de3a3f88a0 fix(ui): autocomplete on edge custom template and stacks [EE-6761] (#11269) 2024-02-27 20:15:56 +13:00
Matt Hook
f6b2c879bc fix(kube): make app autorefresh and show system settings stay [EE-6771] (#11256) 2024-02-27 11:18:28 +13:00
Prabhat Khera
f5fbcd4d9d fix(stack): auto complete dropdown in docker stacks [EE-6761] (#11254) 2024-02-26 11:43:18 +13:00
Ali
f8b68a809f fix(app): parse nan in validation check [EE-6714] (#11247) 2024-02-26 09:20:59 +13:00
Oscar Zhou
6258c02353 fix(edge/template): validate app template env vars [EE-6743] (#11234) 2024-02-26 09:00:03 +13:00
Chaim Lev-Ari
0fd20277c1 fix(docker): prevent non admins from passing security settings [EE-6765] (#11239) 2024-02-25 11:57:19 +02:00
cmeng
988064a542 fix(stack): make web editor readonly for git template EE-6706 (#11183) 2024-02-23 13:28:20 +13:00
Matt Hook
380b23a9f5 fix(dependancies): update compose and runc [EE-6744] (#11243) 2024-02-23 11:48:49 +13:00
Prabhat Khera
158b43194c fix(ui): turn autocomplete off for git deployment [EE-6761] (#11241) 2024-02-23 08:44:00 +13:00
Ali
1bbe98379a fix(app): NaN validation for autoscaling [EE-6714] (#11238) 2024-02-22 17:36:41 +13:00
Matt Hook
8f9b265f5a fix(helm) tighten up helm requests [EE-6722] (#11233) 2024-02-22 11:35:01 +13:00
Ali
1cdd3fdfe2 fix(input): allow clearing number inputs [EE-6714] (#11187) 2024-02-21 10:43:28 +13:00
Ali
4e95139909 fix(inputlist): update warning style [EE-6737] (#11222) 2024-02-21 08:29:14 +13:00
Matt Hook
704d75596d fix(libhttp): capitalize http error responses for better display [EE-6698] (#11109) 2024-02-21 07:51:29 +13:00
Chaim Lev-Ari
a8938779bf fix(ui): check for authorization [EE-6733] (#11207) 2024-02-20 11:06:05 +02:00
Chaim Lev-Ari
bb6f4e026a fix(kube/apps): move namespace selector in apps view [EE-6612] (#11069) 2024-02-20 10:14:11 +02:00
Ali
b64166ff25 fix(app): remove insight from helm [EE-6693] (#11214)
Co-authored-by: testa113 <testa113>
2024-02-20 17:25:22 +13:00
Ali
bac1c28fa9 fix(app): set values in react autoscaling form section [EE-6740] (#11220) 2024-02-20 09:35:32 +13:00
Prabhat Khera
a17da6d2cd fix(git): update stack name for git stacks [EE-6670] (#11218) 2024-02-20 09:23:50 +13:00
Chaim Lev-Ari
24c2baf6cc feat(a11y): add labels and roles [EE-6717] (#11209) 2024-02-19 16:37:21 +02:00
Oscar Zhou
22b4d029fd fix(edge/template): custom template git fields not pre-filled [EE-6695] (#11113) 2024-02-19 08:39:16 +13:00
Ali
b126472ec7 fix(app): update app type when changing data access policy [EE-6719] (#11210)
Co-authored-by: testa113 <testa113>
2024-02-19 08:08:17 +13:00
Ali
a46fa3b2c4 fix(app): avoid duplicate env requests [EE-6727] (#11193)
Co-authored-by: testa113 <testa113>
2024-02-16 14:02:02 +13:00
Prabhat Khera
a374157d6f fix(ui): update search placeholder [EE-6667] (#11191)
* update search placeholder

* remove box selector description
2024-02-16 12:34:10 +13:00
Matt Hook
861ed662e2 fix(namespace): fix default namespace quota [EE-6700] (#11184) 2024-02-16 08:17:10 +13:00
Chaim Lev-Ari
99b89a8ec5 chore(eslint): add rule to check imports [EE-6730] (#11200) 2024-02-15 17:45:54 +02:00
Chaim Lev-Ari
95750c2339 fix(auth): export hasAuthorizations [EE-6595] (#11198) 2024-02-15 14:05:45 +02:00
Chaim Lev-Ari
165d6165dc feat(ui): restrict views by role [EE-6595] (#11071) 2024-02-15 13:29:55 +02:00
Chaim Lev-Ari
fe6ed55cab feat(edge/stacks): add app templates to deploy types [EE-6632] (#11070) 2024-02-15 09:00:57 +02:00
Chaim Lev-Ari
edea9e3481 feat(auth): add useIsEdgeAdmin hook [EE-6627] (#11101) 2024-02-14 19:50:26 -03:00
Ali
c08b5af85a fix(insight): split insight from input [EE-6693] (#11177)
Co-authored-by: testa113 <testa113>
2024-02-15 10:46:02 +13:00
Prabhat Khera
ed861044a7 Revert "fix(logs): add NOCOLOR option for use when exporting to greylog etc […" (#11178)
This reverts commit aca6d33548.
2024-02-15 06:26:22 +13:00
Chaim Lev-Ari
a83321ebe6 feat(ui): write tests [EE-6685] (#11082) 2024-02-14 17:25:32 +02:00
Ali
513cd9c9b3 fix(configs): correct 'external' display in tables [EE-6649] (#11111)
Co-authored-by: testa113 <testa113>
2024-02-14 11:48:05 +13:00
Ali
dc94bf141e fix(stacks): add app form stacks input [EE-6693] (#11105) 2024-02-14 09:01:02 +13:00
Dakota Walsh
24471a9ae1 fix(restore): add S3 teaser [EE-6675] (#11096) 2024-02-14 08:40:34 +13:00
Matt Hook
aca6d33548 fix(logs): add NOCOLOR option for use when exporting to greylog etc [EE-6696] (#11107) 2024-02-14 07:54:47 +13:00
Ali
ca77b85c65 fix(kube-owner): owner labels from resources created via manifest [EE-6647] (#11103)
Co-authored-by: testa113 <testa113>
2024-02-12 15:30:59 +13:00
Prabhat Khera
1fd4291630 fix(ui): stackname auto fill on create from manifest screen [EE-6688] (#11100)
* fix(ui): stackname auto fill on create from manifest screen [EE-6688]

* address review comment
2024-02-12 10:54:24 +13:00
Ali
08dd7f6d2a fix(auth): isAdmin redirect for wizard [EE-6669] (#11075)
Co-authored-by: testa113 <testa113>
2024-02-12 08:04:44 +13:00
Prabhat Khera
ce4b0e759c fix(ui): scroll issue [EE-6667 (#11085)
* Fix scroll issue

* fix minorissue

* address review comments

* add comment
2024-02-09 15:35:38 +13:00
Steven Kang
538e7a823b fix: pre-release build only after merging (#11098) 2024-02-09 15:26:39 +13:00
Matt Hook
956e8d3c59 fix(docs): fix swagger docs for webhook params [EE-6668] (#11089) 2024-02-09 14:44:29 +13:00
Prabhat Khera
1c5458f0d4 fix(kube): ingress path duplication issue [EE-6649] (#11087) 2024-02-09 07:49:57 +13:00
Prabhat Khera
f6085ffad7 fix stack name update issue (#11065) 2024-02-08 13:51:06 +13:00
Matt Hook
490bda2eaf fix(kube-apps): add helm insights, remove namespace insights panel [EE-6671] (#11078) 2024-02-08 11:18:48 +13:00
Prabhat Khera
d601d8eb7b fix(UI): some minor fixes [EE-6667] (#11062)
* minor tweeks for kubernetes settings

* address review comments
2024-02-06 12:17:35 +13:00
Steven Kang
b0564b9238 Pre-release as part of the CI (#11067)
* feat: add pre-release
* feat: add extension
* feat: fix typo
2024-02-05 18:29:12 +13:00
Prabhat Khera
8922585a70 keep labels on edit ingress, configmaps and secrets (#11063) 2024-02-05 16:30:31 +13:00
Ali
d7cf2284dc fix(r2a): don't set errors to undefined [EE-6665] (#11060)
Co-authored-by: testa113 <testa113>
2024-02-05 14:24:15 +13:00
4764 changed files with 101214 additions and 290192 deletions

View File

@@ -1,52 +0,0 @@
root = "."
testdata_dir = "testdata"
tmp_dir = ".tmp"
[build]
args_bin = []
bin = "./dist/portainer"
cmd = "SKIP_GO_GET=true make build-server"
delay = 1000
exclude_dir = []
exclude_file = []
exclude_regex = ["_test.go"]
exclude_unchanged = false
follow_symlink = false
full_bin = "./dist/portainer --log-level=DEBUG"
include_dir = ["api"]
include_ext = ["go"]
include_file = []
kill_delay = "0s"
log = "build-errors.log"
poll = false
poll_interval = 0
post_cmd = []
pre_cmd = []
rerun = false
rerun_delay = 500
send_interrupt = false
stop_on_error = false
[color]
app = ""
build = "yellow"
main = "magenta"
runner = "green"
watcher = "cyan"
[log]
main_only = false
silent = false
time = false
[misc]
clean_on_exit = false
[proxy]
app_port = 0
enabled = false
proxy_port = 0
[screen]
clear_on_rebuild = false
keep_scroll = true

3
.eslintignore Normal file
View File

@@ -0,0 +1,3 @@
node_modules/
dist/
test/

148
.eslintrc.yml Normal file
View File

@@ -0,0 +1,148 @@
env:
browser: true
jquery: true
node: true
es6: true
globals:
angular: true
extends:
- 'eslint:recommended'
- 'plugin:storybook/recommended'
- 'plugin:import/typescript'
- prettier
plugins:
- import
parserOptions:
ecmaVersion: 2018
sourceType: module
project: './tsconfig.json'
ecmaFeatures:
modules: true
rules:
no-console: error
no-alert: error
no-control-regex: 'off'
no-empty: warn
no-empty-function: warn
no-useless-escape: 'off'
import/named: error
import/order:
[
'error',
{
pathGroups:
[
{ pattern: '@@/**', group: 'internal', position: 'after' },
{ pattern: '@/**', group: 'internal' },
{ pattern: '{Kubernetes,Portainer,Agent,Azure,Docker}/**', group: 'internal' },
],
groups: ['builtin', 'external', 'internal', 'parent', 'sibling', 'index'],
pathGroupsExcludedImportTypes: ['internal'],
},
]
no-restricted-imports:
- error
- patterns:
- group:
- '@/react/test-utils/*'
message: 'These utils are just for test files'
settings:
'import/resolver':
alias:
map:
- ['@@', './app/react/components']
- ['@', './app']
extensions: ['.js', '.ts', '.tsx']
typescript: true
node: true
overrides:
- files:
- app/**/*.ts{,x}
parserOptions:
project: './tsconfig.json'
parser: '@typescript-eslint/parser'
plugins:
- '@typescript-eslint'
- 'regex'
extends:
- airbnb
- airbnb-typescript
- 'plugin:eslint-comments/recommended'
- 'plugin:react-hooks/recommended'
- 'plugin:react/jsx-runtime'
- 'plugin:@typescript-eslint/recommended'
- 'plugin:@typescript-eslint/eslint-recommended'
- 'plugin:promise/recommended'
- 'plugin:storybook/recommended'
- prettier # should be last
settings:
react:
version: 'detect'
rules:
import/order:
[
'error',
{
pathGroups: [{ pattern: '@@/**', group: 'internal', position: 'after' }, { pattern: '@/**', group: 'internal' }],
groups: ['builtin', 'external', 'internal', 'parent', 'sibling', 'index'],
'newlines-between': 'always',
},
]
no-plusplus: off
func-style: [error, 'declaration']
import/prefer-default-export: off
no-use-before-define: 'off'
'@typescript-eslint/no-use-before-define': ['error', { functions: false, 'allowNamedExports': true }]
no-shadow: 'off'
'@typescript-eslint/no-shadow': off
jsx-a11y/no-autofocus: warn
react/forbid-prop-types: off
react/require-default-props: off
react/no-array-index-key: off
no-underscore-dangle: off
react/jsx-filename-extension: [0]
import/no-extraneous-dependencies: ['error', { devDependencies: true }]
'@typescript-eslint/explicit-module-boundary-types': off
'@typescript-eslint/no-unused-vars': 'error'
'@typescript-eslint/no-explicit-any': 'error'
'jsx-a11y/label-has-associated-control': ['error', { 'assert': 'either', controlComponents: ['Input', 'Checkbox'] }]
'react/function-component-definition': ['error', { 'namedComponents': 'function-declaration' }]
'react/jsx-no-bind': off
'no-await-in-loop': 'off'
'react/jsx-no-useless-fragment': ['error', { allowExpressions: true }]
'regex/invalid': ['error', [{ 'regex': '<Icon icon="(.*)"', 'message': 'Please directly import the `lucide-react` icon instead of using the string' }]]
'@typescript-eslint/no-restricted-imports':
- error
- patterns:
- group:
- '@/react/test-utils/*'
message: 'These utils are just for test files'
overrides: # allow props spreading for hoc files
- files:
- app/**/with*.ts{,x}
rules:
'react/jsx-props-no-spreading': off
- files:
- app/**/*.test.*
extends:
- 'plugin:vitest/recommended'
env:
'vitest/env': true
rules:
'react/jsx-no-constructed-context-values': off
'@typescript-eslint/no-restricted-imports': off
no-restricted-imports: off
- files:
- app/**/*.stories.*
rules:
'no-alert': off
'@typescript-eslint/no-restricted-imports': off
no-restricted-imports: off

View File

@@ -3,13 +3,13 @@ body:
attributes:
value: |
# Welcome!
Thanks for suggesting an idea for Portainer!
Before opening a new idea or feature request, make sure that we do not have any duplicates already open. You can ensure this by [searching this discussion category](https://github.com/orgs/portainer/discussions/categories/ideas). If there is a duplicate, please add a comment to the existing idea instead.
Before opening a new idea or feature request, make sure that we do not have any duplicates already open. You can ensure this by [searching this discussion cagetory](https://github.com/orgs/portainer/discussions/categories/ideas). If there is a duplicate, please add a comment to the existing idea instead.
Also, be sure to check our [knowledge base](https://portal.portainer.io/knowledge) and [documentation](https://docs.portainer.io) as they may point you toward a solution.
**DO NOT FILE DUPLICATE REQUESTS.**
- type: textarea

View File

@@ -2,17 +2,16 @@ name: Bug Report
description: Create a report to help us improve.
labels: kind/bug,bug/need-confirmation
body:
- type: markdown
attributes:
value: |
# Welcome!
The issue tracker is for reporting bugs. If you have an [idea for a new feature](https://github.com/orgs/portainer/discussions/categories/ideas) or a [general question about Portainer](https://github.com/orgs/portainer/discussions/categories/help) please post in our [GitHub Discussions](https://github.com/orgs/portainer/discussions).
You can also ask for help in our [community Slack channel](https://join.slack.com/t/portainer/shared_invite/zt-txh3ljab-52QHTyjCqbe5RibC2lcjKA).
Please note that we only provide support for current versions of Portainer. You can find a list of supported versions in our [lifecycle policy](https://docs.portainer.io/start/lifecycle).
**DO NOT FILE ISSUES FOR GENERAL SUPPORT QUESTIONS**.
- type: checkboxes
@@ -22,7 +21,7 @@ body:
options:
- label: Yes, I've searched similar issues on [GitHub](https://github.com/portainer/portainer/issues).
required: true
- label: Yes, I've checked whether this issue is covered in the Portainer [documentation](https://docs.portainer.io).
- label: Yes, I've checked whether this issue is covered in the Portainer [documentation](https://docs.portainer.io) or [knowledge base](https://portal.portainer.io/knowledge).
required: true
- type: markdown
@@ -44,7 +43,7 @@ body:
- type: textarea
attributes:
label: Problem Description
description: A clear and concise description of what the bug is.
description: A clear and concise description of what the bug is.
validations:
required: true
@@ -70,7 +69,7 @@ body:
1. Go to '...'
2. Click on '....'
3. Scroll down to '....'
4. See error
4. See error
validations:
required: true
@@ -91,33 +90,23 @@ body:
- type: dropdown
attributes:
label: Portainer version
description: We only provide support for current versions of Portainer as per the lifecycle policy linked above. If you are on an older version of Portainer we recommend [updating first](https://docs.portainer.io/start/upgrade) in case your bug has already been fixed.
description: We only provide support for the most recent version of Portainer and the previous 3 versions. If you are on an older version of Portainer we recommend [upgrading first](https://docs.portainer.io/start/upgrade) in case your bug has already been fixed.
multiple: false
options:
- '2.42.0'
- '2.41.1'
- '2.41.0'
- '2.40.0'
- '2.39.3'
- '2.39.2'
- '2.39.1'
- '2.39.0'
- '2.38.1'
- '2.38.0'
- '2.37.0'
- '2.36.0'
- '2.35.0'
- '2.34.0'
- '2.33.8'
- '2.33.7'
- '2.33.6'
- '2.33.5'
- '2.33.4'
- '2.33.3'
- '2.33.2'
- '2.33.1'
- '2.33.0'
- '2.19.4'
- '2.19.3'
- '2.19.2'
- '2.19.1'
- '2.19.0'
- '2.18.4'
- '2.18.3'
- '2.18.2'
- '2.18.1'
- '2.17.1'
- '2.17.0'
- '2.16.2'
- '2.16.1'
- '2.16.0'
validations:
required: true
@@ -155,7 +144,7 @@ body:
- type: input
attributes:
label: Browser
description: |
description: |
Enter your browser and version. Example: Google Chrome 114.0
validations:
required: false

View File

@@ -1,86 +0,0 @@
name: Build image
on:
push:
branches: [develop]
tags: ['v*']
workflow_dispatch: {}
env:
IMAGE: ghcr.io/vvzvlad/portainer-ce
jobs:
build:
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup pnpm
uses: pnpm/action-setup@v4
- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: 22
- name: Setup Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Resolve version
id: ver
run: echo "version=$(node -p "require('./package.json').version")" >> "$GITHUB_OUTPUT"
- name: Install client dependencies
# CI forces pnpm into --frozen-lockfile, which fails with
# ERR_PNPM_LOCKFILE_CONFIG_MISMATCH because the committed lockfile lacks
# the pnpmfileChecksum for the configDependencies in package.json.
# Reconcile the lockfile explicitly; the later frozen install in
# `make client-deps` then finds a matching lockfile. pnpm ignores the
# npm_config_frozen_lockfile env var, so an explicit flag is required.
run: pnpm install --no-frozen-lockfile
- name: Build client and server
env:
SKIP_GO_GET: "true"
CONTAINER_IMAGE_TAG: ${{ steps.ver.outputs.version }}
BUILDNUMBER: ${{ github.run_number }}
# Pin the embedded commit to the full SHA so it matches the image
# GIT_COMMIT build-arg and does not depend on the shallow checkout.
GIT_COMMIT_HASH: ${{ github.sha }}
# ENV=production selects webpack/webpack.production.js (minified bundle),
# matching the official CE image; the Makefile default is development.
run: make build-all ENV=production
- name: Ensure storybook directory exists
# make build-all does not produce dist/storybook, but alpine.Dockerfile
# has `COPY dist/storybook* /storybook/`; without a match the docker build fails.
run: mkdir -p dist/storybook
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Log in to GHCR
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push image (linux/amd64, alpine base)
uses: docker/build-push-action@v6
with:
context: .
file: build/linux/alpine.Dockerfile
platforms: linux/amd64
push: true
tags: |
${{ env.IMAGE }}:${{ steps.ver.outputs.version }}
${{ env.IMAGE }}:latest
build-args: |
GIT_COMMIT=${{ github.sha }}

176
.github/workflows/ci.yaml vendored Normal file
View File

@@ -0,0 +1,176 @@
name: ci
on:
workflow_dispatch:
push:
branches:
- 'develop'
- 'release/*'
pull_request:
branches:
- 'develop'
- 'release/*'
- 'feat/*'
- 'fix/*'
- 'refactor/*'
types:
- opened
- reopened
- synchronize
- ready_for_review
env:
DOCKER_HUB_REPO: portainerci/portainer-ce
EXTENSION_HUB_REPO: portainerci/portainer-docker-extension
GO_VERSION: 1.21.6
NODE_VERSION: 18.x
jobs:
build_images:
strategy:
matrix:
config:
- { platform: linux, arch: amd64, version: "" }
- { platform: linux, arch: arm64, version: "" }
- { platform: linux, arch: arm, version: "" }
- { platform: linux, arch: ppc64le, version: "" }
- { platform: linux, arch: s390x, version: "" }
- { platform: windows, arch: amd64, version: 1809 }
- { platform: windows, arch: amd64, version: ltsc2022 }
runs-on: ubuntu-latest
if: github.event.pull_request.draft == false
steps:
- name: '[preparation] checkout the current branch'
uses: actions/checkout@v4.1.1
with:
ref: ${{ github.event.inputs.branch }}
- name: '[preparation] set up golang'
uses: actions/setup-go@v5.0.0
with:
go-version: ${{ env.GO_VERSION }}
- name: '[preparation] set up node.js'
uses: actions/setup-node@v4.0.1
with:
node-version: ${{ env.NODE_VERSION }}
cache: 'yarn'
- name: '[preparation] set up qemu'
uses: docker/setup-qemu-action@v3.0.0
- name: '[preparation] set up docker context for buildx'
run: docker context create builders
- name: '[preparation] set up docker buildx'
uses: docker/setup-buildx-action@v3.0.0
with:
endpoint: builders
- name: '[preparation] docker login'
uses: docker/login-action@v3.0.0
with:
username: ${{ secrets.DOCKER_HUB_USERNAME }}
password: ${{ secrets.DOCKER_HUB_PASSWORD }}
- name: '[preparation] set the container image tag'
run: |
if [[ "${GITHUB_REF_NAME}" =~ ^release/.*$ ]]; then
# use the release branch name as the tag for release branches
# for instance, release/2.19 becomes 2.19
CONTAINER_IMAGE_TAG=$(echo $GITHUB_REF_NAME | cut -d "/" -f 2)
elif [ "${GITHUB_EVENT_NAME}" == "pull_request" ]; then
# use pr${{ github.event.number }} as the tag for pull requests
# for instance, pr123
CONTAINER_IMAGE_TAG="pr${{ github.event.number }}"
else
# replace / with - in the branch name
# for instance, feature/1.0.0 -> feature-1.0.0
CONTAINER_IMAGE_TAG=$(echo $GITHUB_REF_NAME | sed 's/\//-/g')
fi
echo "CONTAINER_IMAGE_TAG=${CONTAINER_IMAGE_TAG}-${{ matrix.config.platform }}${{ matrix.config.version }}-${{ matrix.config.arch }}" >> $GITHUB_ENV
- name: '[execution] build linux & windows portainer binaries'
run: |
export YARN_VERSION=$(yarn --version)
export WEBPACK_VERSION=$(yarn list webpack --depth=0 | grep webpack | awk -F@ '{print $2}')
export BUILDNUMBER=${GITHUB_RUN_NUMBER}
GIT_COMMIT_HASH_LONG=${{ github.sha }}
export GIT_COMMIT_HASH_SHORT={GIT_COMMIT_HASH_LONG:0:7}
NODE_ENV="testing"
if [[ "${GITHUB_REF_NAME}" =~ ^release/.*$ ]]; then
NODE_ENV="production"
fi
make build-all PLATFORM=${{ matrix.config.platform }} ARCH=${{ matrix.config.arch }} ENV=${NODE_ENV}
env:
CONTAINER_IMAGE_TAG: ${{ env.CONTAINER_IMAGE_TAG }}
- name: '[execution] build and push docker images'
run: |
if [ "${{ matrix.config.platform }}" == "windows" ]; then
mv dist/portainer dist/portainer.exe
docker buildx build --output=type=registry --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} --build-arg OSVERSION=${{ matrix.config.version }} -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}" -f build/${{ matrix.config.platform }}/Dockerfile .
else
docker buildx build --output=type=registry --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}" -f build/${{ matrix.config.platform }}/Dockerfile .
docker buildx build --output=type=registry --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-alpine" -f build/${{ matrix.config.platform }}/alpine.Dockerfile .
if [[ "${GITHUB_REF_NAME}" =~ ^release/.*$ ]]; then
docker buildx build --output=type=registry --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} -t "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}" -f build/${{ matrix.config.platform }}/Dockerfile .
docker buildx build --output=type=registry --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} -t "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-alpine" -f build/${{ matrix.config.platform }}/alpine.Dockerfile .
fi
fi
env:
CONTAINER_IMAGE_TAG: ${{ env.CONTAINER_IMAGE_TAG }}
build_manifests:
runs-on: ubuntu-latest
if: github.event.pull_request.draft == false
needs: [build_images]
steps:
- name: '[preparation] docker login'
uses: docker/login-action@v3.0.0
with:
username: ${{ secrets.DOCKER_HUB_USERNAME }}
password: ${{ secrets.DOCKER_HUB_PASSWORD }}
- name: '[preparation] set up docker context for buildx'
run: docker version && docker context create builders
- name: '[preparation] set up docker buildx'
uses: docker/setup-buildx-action@v3.0.0
with:
endpoint: builders
- name: '[execution] build and push manifests'
run: |
if [[ "${GITHUB_REF_NAME}" =~ ^release/.*$ ]]; then
# use the release branch name as the tag for release branches
# for instance, release/2.19 becomes 2.19
CONTAINER_IMAGE_TAG=$(echo $GITHUB_REF_NAME | cut -d "/" -f 2)
elif [ "${GITHUB_EVENT_NAME}" == "pull_request" ]; then
# use pr${{ github.event.number }} as the tag for pull requests
# for instance, pr123
CONTAINER_IMAGE_TAG="pr${{ github.event.number }}"
else
# replace / with - in the branch name
# for instance, feature/1.0.0 -> feature-1.0.0
CONTAINER_IMAGE_TAG=$(echo $GITHUB_REF_NAME | sed 's/\//-/g')
fi
docker buildx imagetools create -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}" \
"${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-amd64" \
"${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm64" \
"${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm" \
"${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-ppc64le" \
"${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-s390x" \
"${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-windows1809-amd64" \
"${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-windowsltsc2022-amd64"
docker buildx imagetools create -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-alpine" \
"${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-amd64-alpine" \
"${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm64-alpine" \
"${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm-alpine"
if [[ "${GITHUB_REF_NAME}" =~ ^release/.*$ ]]; then
docker buildx imagetools create -t "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}" \
"${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-amd64" \
"${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm64" \
"${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm" \
"${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-ppc64le" \
"${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-s390x"
docker buildx imagetools create -t "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-alpine" \
"${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-amd64-alpine" \
"${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm64-alpine" \
"${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm-alpine"
fi

15
.github/workflows/label-conflcts.yaml vendored Normal file
View File

@@ -0,0 +1,15 @@
on:
push:
branches:
- develop
- 'release/**'
jobs:
triage:
runs-on: ubuntu-latest
steps:
- uses: mschilde/auto-label-merge-conflicts@master
with:
CONFLICT_LABEL_NAME: 'has conflicts'
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
MAX_RETRIES: 10
WAIT_MS: 60000

55
.github/workflows/lint.yml vendored Normal file
View File

@@ -0,0 +1,55 @@
name: Lint
on:
push:
branches:
- master
- develop
- release/*
pull_request:
branches:
- master
- develop
- release/*
types:
- opened
- reopened
- synchronize
- ready_for_review
env:
GO_VERSION: 1.21.6
NODE_VERSION: 18.x
jobs:
run-linters:
name: Run linters
runs-on: ubuntu-latest
if: github.event.pull_request.draft == false
steps:
- uses: actions/checkout@v2
- uses: actions/setup-node@v2
with:
node-version: ${{ env.NODE_VERSION }}
cache: 'yarn'
- uses: actions/setup-go@v4
with:
go-version: ${{ env.GO_VERSION }}
- run: yarn --frozen-lockfile
- name: Run linters
uses: wearerequired/lint-action@v1
with:
eslint: true
eslint_extensions: ts,tsx,js,jsx
prettier: true
prettier_dir: app/
gofmt: true
gofmt_dir: api/
- name: Typecheck
uses: icrawl/action-tsc@v1
- name: GolangCI-Lint
uses: golangci/golangci-lint-action@v3
with:
version: v1.55.2
args: --timeout=10m -c .golangci.yaml

View File

@@ -0,0 +1,252 @@
name: Nightly Code Security Scan
on:
schedule:
- cron: '0 20 * * *'
workflow_dispatch:
env:
GO_VERSION: 1.21.6
jobs:
client-dependencies:
name: Client Dependency Check
runs-on: ubuntu-latest
if: >- # only run for develop branch
github.ref == 'refs/heads/develop'
outputs:
js: ${{ steps.set-matrix.outputs.js_result }}
steps:
- name: checkout repository
uses: actions/checkout@master
- name: scan vulnerabilities by Snyk
uses: snyk/actions/node@master
continue-on-error: true # To make sure that artifact upload gets called
env:
SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
with:
json: true
- name: upload scan result as develop artifact
uses: actions/upload-artifact@v3
with:
name: js-security-scan-develop-result
path: snyk.json
- name: develop scan report export to html
run: |
$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=snyk --path="/data/snyk.json" --output-type=table --export --export-filename="/data/js-result")
- name: upload html file as artifact
uses: actions/upload-artifact@v3
with:
name: html-js-result-${{github.run_id}}
path: js-result.html
- name: analyse vulnerabilities
id: set-matrix
run: |
result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=snyk --path="/data/snyk.json" --output-type=matrix)
echo "js_result=${result}" >> $GITHUB_OUTPUT
server-dependencies:
name: Server Dependency Check
runs-on: ubuntu-latest
if: >- # only run for develop branch
github.ref == 'refs/heads/develop'
outputs:
go: ${{ steps.set-matrix.outputs.go_result }}
steps:
- name: checkout repository
uses: actions/checkout@master
- name: install Go
uses: actions/setup-go@v3
with:
go-version: ${{ env.GO_VERSION }}
- name: download Go modules
run: cd ./api && go get -t -v -d ./...
- name: scan vulnerabilities by Snyk
continue-on-error: true # To make sure that artifact upload gets called
env:
SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
run: |
yarn global add snyk
snyk test --file=./go.mod --json-file-output=snyk.json 2>/dev/null || :
- name: upload scan result as develop artifact
uses: actions/upload-artifact@v3
with:
name: go-security-scan-develop-result
path: snyk.json
- name: develop scan report export to html
run: |
$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=snyk --path="/data/snyk.json" --output-type=table --export --export-filename="/data/go-result")
- name: upload html file as artifact
uses: actions/upload-artifact@v3
with:
name: html-go-result-${{github.run_id}}
path: go-result.html
- name: analyse vulnerabilities
id: set-matrix
run: |
result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=snyk --path="/data/snyk.json" --output-type=matrix)
echo "go_result=${result}" >> $GITHUB_OUTPUT
image-vulnerability:
name: Image Vulnerability Check
runs-on: ubuntu-latest
if: >-
github.ref == 'refs/heads/develop'
outputs:
image-trivy: ${{ steps.set-trivy-matrix.outputs.image_trivy_result }}
image-docker-scout: ${{ steps.set-docker-scout-matrix.outputs.image_docker_scout_result }}
steps:
- name: scan vulnerabilities by Trivy
uses: docker://docker.io/aquasec/trivy:latest
continue-on-error: true
with:
args: image --ignore-unfixed=true --vuln-type="os,library" --exit-code=1 --format="json" --output="image-trivy.json" --no-progress portainerci/portainer:develop
- name: upload Trivy image security scan result as artifact
uses: actions/upload-artifact@v3
with:
name: image-security-scan-develop-result
path: image-trivy.json
- name: develop Trivy scan report export to html
run: |
$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=trivy --path="/data/image-trivy.json" --output-type=table --export --export-filename="/data/image-trivy-result")
- name: upload html file as Trivy artifact
uses: actions/upload-artifact@v3
with:
name: html-image-result-${{github.run_id}}
path: image-trivy-result.html
- name: analyse vulnerabilities from Trivy
id: set-trivy-matrix
run: |
result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=trivy --path="/data/image-trivy.json" --output-type=matrix)
echo "image_trivy_result=${result}" >> $GITHUB_OUTPUT
- name: scan vulnerabilities by Docker Scout
uses: docker/scout-action@v1
continue-on-error: true
with:
command: cves
image: portainerci/portainer:develop
sarif-file: image-docker-scout.json
dockerhub-user: ${{ secrets.DOCKER_HUB_USERNAME }}
dockerhub-password: ${{ secrets.DOCKER_HUB_PASSWORD }}
- name: upload Docker Scout image security scan result as artifact
uses: actions/upload-artifact@v3
with:
name: image-security-scan-develop-result
path: image-docker-scout.json
- name: develop Docker Scout scan report export to html
run: |
$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=docker-scout --path="/data/image-docker-scout.json" --output-type=table --export --export-filename="/data/image-docker-scout-result")
- name: upload html file as Docker Scout artifact
uses: actions/upload-artifact@v3
with:
name: html-image-result-${{github.run_id}}
path: image-docker-scout-result.html
- name: analyse vulnerabilities from Docker Scout
id: set-docker-scout-matrix
run: |
result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=docker-scout --path="/data/image-docker-scout.json" --output-type=matrix)
echo "image_docker_scout_result=${result}" >> $GITHUB_OUTPUT
result-analysis:
name: Analyse Scan Results
needs: [client-dependencies, server-dependencies, image-vulnerability]
runs-on: ubuntu-latest
if: >-
github.ref == 'refs/heads/develop'
strategy:
matrix:
js: ${{fromJson(needs.client-dependencies.outputs.js)}}
go: ${{fromJson(needs.server-dependencies.outputs.go)}}
image-trivy: ${{fromJson(needs.image-vulnerability.outputs.image-trivy)}}
image-docker-scout: ${{fromJson(needs.image-vulnerability.outputs.image-docker-scout)}}
steps:
- name: display the results of js, Go, and image scan
run: |
echo "${{ matrix.js.status }}"
echo "${{ matrix.go.status }}"
echo "${{ matrix.image-trivy.status }}"
echo "${{ matrix.image-docker-scout.status }}"
echo "${{ matrix.js.summary }}"
echo "${{ matrix.go.summary }}"
echo "${{ matrix.image-trivy.summary }}"
echo "${{ matrix.image-docker-scout.summary }}"
- name: send message to Slack
if: >-
matrix.js.status == 'failure' ||
matrix.go.status == 'failure' ||
matrix.image-trivy.status == 'failure' ||
matrix.image-docker-scout.status == 'failure'
uses: slackapi/slack-github-action@v1.23.0
with:
payload: |
{
"blocks": [
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "Code Scanning Result (*${{ github.repository }}*)\n*<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|GitHub Actions Workflow URL>*"
}
}
],
"attachments": [
{
"color": "#FF0000",
"blocks": [
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "*JS dependency check*: *${{ matrix.js.status }}*\n${{ matrix.js.summary }}"
}
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "*Go dependency check*: *${{ matrix.go.status }}*\n${{ matrix.go.summary }}"
}
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "*Image Trivy vulnerability check*: *${{ matrix.image-trivy.status }}*\n${{ matrix.image-trivy.summary }}\n"
}
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "*Image Docker Scout vulnerability check*: *${{ matrix.image-docker-scout.status }}*\n${{ matrix.image-docker-scout.summary }}\n"
}
}
]
}
]
}
env:
SLACK_WEBHOOK_URL: ${{ secrets.SECURITY_SLACK_WEBHOOK_URL }}
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK

298
.github/workflows/pr-security.yml vendored Normal file
View File

@@ -0,0 +1,298 @@
name: PR Code Security Scan
on:
pull_request_review:
types:
- submitted
- edited
paths:
- 'package.json'
- 'go.mod'
- 'build/linux/Dockerfile'
- 'build/linux/alpine.Dockerfile'
- 'build/windows/Dockerfile'
- '.github/workflows/pr-security.yml'
env:
GO_VERSION: 1.21.6
NODE_VERSION: 18.x
jobs:
client-dependencies:
name: Client Dependency Check
runs-on: ubuntu-latest
if: >-
github.event.pull_request &&
github.event.review.body == '/scan' &&
github.event.pull_request.draft == false
outputs:
jsdiff: ${{ steps.set-diff-matrix.outputs.js_diff_result }}
steps:
- name: checkout repository
uses: actions/checkout@master
- name: scan vulnerabilities by Snyk
uses: snyk/actions/node@master
continue-on-error: true # To make sure that artifact upload gets called
env:
SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
with:
json: true
- name: upload scan result as pull-request artifact
uses: actions/upload-artifact@v3
with:
name: js-security-scan-feat-result
path: snyk.json
- name: download artifacts from develop branch built by nightly scan
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
mv ./snyk.json ./js-snyk-feature.json
(gh run download -n js-security-scan-develop-result -R ${{ github.repository }} 2>&1 >/dev/null) || :
if [[ -e ./snyk.json ]]; then
mv ./snyk.json ./js-snyk-develop.json
else
echo "null" > ./js-snyk-develop.json
fi
- name: pr vs develop scan report comparison export to html
run: |
$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=snyk --path="/data/js-snyk-feature.json" --compare-to="/data/js-snyk-develop.json" --output-type=table --export --export-filename="/data/js-result")
- name: upload html file as artifact
uses: actions/upload-artifact@v3
with:
name: html-js-result-compare-to-develop-${{github.run_id}}
path: js-result.html
- name: analyse different vulnerabilities against develop branch
id: set-diff-matrix
run: |
result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=snyk --path="/data/js-snyk-feature.json" --compare-to="/data/js-snyk-develop.json" --output-type=matrix)
echo "js_diff_result=${result}" >> $GITHUB_OUTPUT
server-dependencies:
name: Server Dependency Check
runs-on: ubuntu-latest
if: >-
github.event.pull_request &&
github.event.review.body == '/scan' &&
github.event.pull_request.draft == false
outputs:
godiff: ${{ steps.set-diff-matrix.outputs.go_diff_result }}
steps:
- name: checkout repository
uses: actions/checkout@master
- name: install Go
uses: actions/setup-go@v3
with:
go-version: ${{ env.GO_VERSION }}
- name: download Go modules
run: cd ./api && go get -t -v -d ./...
- name: scan vulnerabilities by Snyk
continue-on-error: true # To make sure that artifact upload gets called
env:
SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
run: |
yarn global add snyk
snyk test --file=./go.mod --json-file-output=snyk.json 2>/dev/null || :
- name: upload scan result as pull-request artifact
uses: actions/upload-artifact@v3
with:
name: go-security-scan-feature-result
path: snyk.json
- name: download artifacts from develop branch built by nightly scan
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
mv ./snyk.json ./go-snyk-feature.json
(gh run download -n go-security-scan-develop-result -R ${{ github.repository }} 2>&1 >/dev/null) || :
if [[ -e ./snyk.json ]]; then
mv ./snyk.json ./go-snyk-develop.json
else
echo "null" > ./go-snyk-develop.json
fi
- name: pr vs develop scan report comparison export to html
run: |
$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=snyk --path="/data/go-snyk-feature.json" --compare-to="/data/go-snyk-develop.json" --output-type=table --export --export-filename="/data/go-result")
- name: upload html file as artifact
uses: actions/upload-artifact@v3
with:
name: html-go-result-compare-to-develop-${{github.run_id}}
path: go-result.html
- name: analyse different vulnerabilities against develop branch
id: set-diff-matrix
run: |
result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=snyk --path="/data/go-snyk-feature.json" --compare-to="/data/go-snyk-develop.json" --output-type=matrix)
echo "go_diff_result=${result}" >> $GITHUB_OUTPUT
image-vulnerability:
name: Image Vulnerability Check
runs-on: ubuntu-latest
if: >-
github.event.pull_request &&
github.event.review.body == '/scan' &&
github.event.pull_request.draft == false
outputs:
imagediff-trivy: ${{ steps.set-diff-trivy-matrix.outputs.image_diff_trivy_result }}
imagediff-docker-scout: ${{ steps.set-diff-docker-scout-matrix.outputs.image_diff_docker_scout_result }}
steps:
- name: checkout code
uses: actions/checkout@master
- name: install Go
uses: actions/setup-go@v3
with:
go-version: ${{ env.GO_VERSION }}
- name: install Node.js
uses: actions/setup-node@v3
with:
node-version: ${{ env.NODE_VERSION }}
- name: Install packages
run: yarn --frozen-lockfile
- name: build
run: make build-all
- name: set up docker buildx
uses: docker/setup-buildx-action@v2
- name: build and compress image
uses: docker/build-push-action@v4
with:
context: .
file: build/linux/Dockerfile
tags: local-portainer:${{ github.sha }}
outputs: type=docker,dest=/tmp/local-portainer-image.tar
- name: load docker image
run: |
docker load --input /tmp/local-portainer-image.tar
- name: scan vulnerabilities by Trivy
uses: docker://docker.io/aquasec/trivy:latest
continue-on-error: true
with:
args: image --ignore-unfixed=true --vuln-type="os,library" --exit-code=1 --format="json" --output="image-trivy.json" --no-progress local-portainer:${{ github.sha }}
- name: upload Trivy image security scan result as artifact
uses: actions/upload-artifact@v3
with:
name: image-security-scan-feature-result
path: image-trivy.json
- name: download Trivy artifacts from develop branch built by nightly scan
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
mv ./image-trivy.json ./image-trivy-feature.json
(gh run download -n image-security-scan-develop-result -R ${{ github.repository }} 2>&1 >/dev/null) || :
if [[ -e ./image-trivy.json ]]; then
mv ./image-trivy.json ./image-trivy-develop.json
else
echo "null" > ./image-trivy-develop.json
fi
- name: pr vs develop Trivy scan report comparison export to html
run: |
$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=trivy --path="/data/image-trivy-feature.json" --compare-to="/data/image-trivy-develop.json" --output-type=table --export --export-filename="/data/image-trivy-result")
- name: upload html file as Trivy artifact
uses: actions/upload-artifact@v3
with:
name: html-image-result-compare-to-develop-${{github.run_id}}
path: image-trivy-result.html
- name: analyse different vulnerabilities against develop branch by Trivy
id: set-diff-trivy-matrix
run: |
result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=trivy --path="/data/image-trivy-feature.json" --compare-to="/data/image-trivy-develop.json" --output-type=matrix)
echo "image_diff_trivy_result=${result}" >> $GITHUB_OUTPUT
- name: scan vulnerabilities by Docker Scout
uses: docker/scout-action@v1
continue-on-error: true
with:
command: cves
image: local-portainer:${{ github.sha }}
sarif-file: image-docker-scout.json
dockerhub-user: ${{ secrets.DOCKER_HUB_USERNAME }}
dockerhub-password: ${{ secrets.DOCKER_HUB_PASSWORD }}
- name: upload Docker Scout image security scan result as artifact
uses: actions/upload-artifact@v3
with:
name: image-security-scan-feature-result
path: image-docker-scout.json
- name: download Docker Scout artifacts from develop branch built by nightly scan
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
mv ./image-docker-scout.json ./image-docker-scout-feature.json
(gh run download -n image-security-scan-develop-result -R ${{ github.repository }} 2>&1 >/dev/null) || :
if [[ -e ./image-docker-scout.json ]]; then
mv ./image-docker-scout.json ./image-docker-scout-develop.json
else
echo "null" > ./image-docker-scout-develop.json
fi
- name: pr vs develop Docker Scout scan report comparison export to html
run: |
$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=docker-scout --path="/data/image-docker-scout-feature.json" --compare-to="/data/image-docker-scout-develop.json" --output-type=table --export --export-filename="/data/image-docker-scout-result")
- name: upload html file as Docker Scout artifact
uses: actions/upload-artifact@v3
with:
name: html-image-result-compare-to-develop-${{github.run_id}}
path: image-docker-scout-result.html
- name: analyse different vulnerabilities against develop branch by Docker Scout
id: set-diff-docker-scout-matrix
run: |
result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=docker-scout --path="/data/image-docker-scout-feature.json" --compare-to="/data/image-docker-scout-develop.json" --output-type=matrix)
echo "image_diff_docker_scout_result=${result}" >> $GITHUB_OUTPUT
result-analysis:
name: Analyse Scan Result Against develop Branch
needs: [client-dependencies, server-dependencies, image-vulnerability]
runs-on: ubuntu-latest
if: >-
github.event.pull_request &&
github.event.review.body == '/scan' &&
github.event.pull_request.draft == false
strategy:
matrix:
jsdiff: ${{fromJson(needs.client-dependencies.outputs.jsdiff)}}
godiff: ${{fromJson(needs.server-dependencies.outputs.godiff)}}
imagediff-trivy: ${{fromJson(needs.image-vulnerability.outputs.imagediff-trivy)}}
imagediff-docker-scout: ${{fromJson(needs.image-vulnerability.outputs.imagediff-docker-scout)}}
steps:
- name: check job status of diff result
if: >-
matrix.jsdiff.status == 'failure' ||
matrix.godiff.status == 'failure' ||
matrix.imagediff-trivy.status == 'failure' ||
matrix.imagediff-docker-scout.status == 'failure'
run: |
echo "${{ matrix.jsdiff.status }}"
echo "${{ matrix.godiff.status }}"
echo "${{ matrix.imagediff-trivy.status }}"
echo "${{ matrix.imagediff-docker-scout.status }}"
echo "${{ matrix.jsdiff.summary }}"
echo "${{ matrix.godiff.summary }}"
echo "${{ matrix.imagediff-trivy.summary }}"
echo "${{ matrix.imagediff-docker-scout.summary }}"
exit 1

19
.github/workflows/rebase.yml vendored Normal file
View File

@@ -0,0 +1,19 @@
name: Automatic Rebase
on:
issue_comment:
types: [created]
jobs:
rebase:
name: Rebase
if: github.event.issue.pull_request != '' && contains(github.event.comment.body, '/rebase')
runs-on: ubuntu-latest
steps:
- name: Checkout the latest code
uses: actions/checkout@v2
with:
token: ${{ secrets.GITHUB_TOKEN }}
fetch-depth: 0 # otherwise, you will fail to push refs to dest repo
- name: Automatic Rebase
uses: cirrus-actions/rebase@1.4
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

28
.github/workflows/stale.yml vendored Normal file
View File

@@ -0,0 +1,28 @@
name: Close Stale Issues
on:
schedule:
- cron: '0 12 * * *'
workflow_dispatch:
jobs:
stale:
runs-on: ubuntu-latest
permissions:
issues: write
steps:
- uses: actions/stale@v8
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
# Issue Config
days-before-issue-stale: 60
days-before-issue-close: 7
stale-issue-label: 'status/stale'
exempt-all-issue-milestones: true # Do not stale issues in a milestone
exempt-issue-labels: kind/enhancement, kind/style, kind/workaround, kind/refactor, bug/need-confirmation, bug/confirmed, status/discuss
stale-issue-message: 'This issue has been marked as stale as it has not had recent activity, it will be closed if no further activity occurs in the next 7 days. If you believe that it has been incorrectly labelled as stale, leave a comment and the label will be removed.'
close-issue-message: 'Since no further activity has appeared on this issue it will be closed. If you believe that it has been incorrectly closed, leave a comment mentioning `portainer/support` and one of our staff will then review the issue. Note - If it is an old bug report, make sure that it is reproduceable in the latest version of Portainer as it may have already been fixed.'
# Pull Request Config
days-before-pr-stale: -1 # Do not stale pull request
days-before-pr-close: -1 # Do not close pull request

56
.github/workflows/test.yaml vendored Normal file
View File

@@ -0,0 +1,56 @@
name: Test
env:
GO_VERSION: 1.21.6
NODE_VERSION: 18.x
on:
pull_request:
branches:
- master
- develop
- release/*
types:
- opened
- reopened
- synchronize
- ready_for_review
push:
branches:
- master
- develop
- release/*
jobs:
test-client:
runs-on: ubuntu-latest
if: github.event.pull_request.draft == false
steps:
- uses: actions/checkout@v2
- uses: actions/setup-node@v2
with:
node-version: ${{ env.NODE_VERSION }}
cache: 'yarn'
- run: yarn --frozen-lockfile
- name: Run tests
run: make test-client ARGS="--maxWorkers=2 --minWorkers=1"
test-server:
strategy:
matrix:
config:
- { platform: linux, arch: amd64 }
- { platform: linux, arch: arm64 }
- { platform: windows, arch: amd64, version: 1809 }
- { platform: windows, arch: amd64, version: ltsc2022 }
runs-on: ubuntu-latest
if: github.event.pull_request.draft == false
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v3
with:
go-version: ${{ env.GO_VERSION }}
- name: Run tests
run: make test-server

View File

@@ -0,0 +1,39 @@
name: Validate OpenAPI specs
on:
pull_request:
branches:
- master
- develop
- 'release/*'
types:
- opened
- reopened
- synchronize
- ready_for_review
env:
GO_VERSION: 1.21.6
NODE_VERSION: 18.x
jobs:
openapi-spec:
runs-on: ubuntu-latest
if: github.event.pull_request.draft == false
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v3
with:
go-version: ${{ env.GO_VERSION }}
- name: Download golang modules
run: cd ./api && go get -t -v -d ./...
- uses: actions/setup-node@v3
with:
node-version: ${{ env.NODE_VERSION }}
cache: 'yarn'
- run: yarn --frozen-lockfile
- name: Validate OpenAPI Spec
run: make docs-validate

3
.gitignore vendored
View File

@@ -4,7 +4,6 @@ dist
portainer-checksum.txt
api/cmd/portainer/portainer*
storybook-static
debug-storybook.log
.tmp
**/.vscode/settings.json
**/.vscode/tasks.json
@@ -19,5 +18,3 @@ api/docs
.env
go.work.sum
.vitest

1
.godir Normal file
View File

@@ -0,0 +1 @@
portainer

View File

@@ -1,13 +0,0 @@
version: '2'
linters:
default: none
enable:
- forbidigo
settings:
forbidigo:
forbid:
- pattern: ^dataservices.DataStore.(EdgeGroup|EdgeJob|EdgeStack|EndpointRelation|Endpoint|GitCredential|Registry|ResourceControl|Role|Settings|Snapshot|SSLSettings|Stack|Tag|User)$
msg: Use a transaction instead
- pattern: ^(filepath|path)\.Join$
msg: Use filesystem.JoinPaths() from github.com/portainer/portainer/api/filesystem to prevent path traversal attacks
analyze-types: true

View File

@@ -1,126 +1,40 @@
version: '2'
run:
allow-parallel-runners: true
linters:
default: none
# Disable all linters, the defaults don't pass on our code yet
disable-all: true
# Enable these for now
enable:
- gocritic
- bodyclose
- copyloopvar
- unused
- depguard
- errcheck
- errorlint
- forbidigo
- gosimple
- govet
- ineffassign
- intrange
- perfsprint
- staticcheck
- unused
- mirror
- durationcheck
- errorlint
- govet
- usetesting
- zerologlint
- testifylint
- modernize
- unconvert
- unused
- zerologlint
- exptostd
settings:
staticcheck:
checks: ['all', '-ST1003', '-ST1005', '-ST1016', '-SA1019', '-QF1003']
depguard:
rules:
main:
files:
- '!**/*_test.go'
- '!**/base.go'
- '!**/base_tx.go'
deny:
- pkg: encoding/json
desc: use github.com/segmentio/encoding/json
- pkg: golang.org/x/exp
desc: exp is not allowed
- pkg: github.com/portainer/libcrypto
desc: use github.com/portainer/portainer/pkg/libcrypto
- pkg: github.com/portainer/libhttp
desc: use github.com/portainer/portainer/pkg/libhttp
- pkg: golang.org/x/crypto
desc: golang.org/x/crypto is not allowed because of FIPS mode
- pkg: github.com/ProtonMail/go-crypto/openpgp
desc: github.com/ProtonMail/go-crypto/openpgp is not allowed because of FIPS mode
- pkg: github.com/cosi-project/runtime
desc: github.com/cosi-project/runtime is not allowed because of FIPS mode
- pkg: gopkg.in/yaml.v2
desc: use go.yaml.in/yaml/v3 instead
- pkg: gopkg.in/yaml.v3
desc: use go.yaml.in/yaml/v3 instead
- pkg: github.com/golang-jwt/jwt/v4
desc: use github.com/golang-jwt/jwt/v5 instead
- pkg: github.com/mitchellh/mapstructure
desc: use github.com/go-viper/mapstructure/v2 instead
- pkg: gopkg.in/alecthomas/kingpin.v2
desc: use github.com/alecthomas/kingpin/v2 instead
- pkg: github.com/jcmturner/gokrb5$
desc: use github.com/jcmturner/gokrb5/v8 instead
- pkg: github.com/gofrs/uuid
desc: use github.com/google/uuid
- pkg: github.com/Masterminds/semver$
desc: use github.com/Masterminds/semver/v3
- pkg: github.com/blang/semver
desc: use github.com/Masterminds/semver/v3
- pkg: github.com/coreos/go-semver
desc: use github.com/Masterminds/semver/v3
- pkg: github.com/hashicorp/go-version
desc: use github.com/Masterminds/semver/v3
gocritic:
disable-all: true
enabled-checks:
- ruleguard
settings:
ruleguard:
rules: './analysis/ssrf.go,./analysis/git.go'
forbidigo:
forbid:
- pattern: ^tls\.Config$
msg: Use crypto.CreateTLSConfiguration() instead
- pattern: ^tls\.Config\.(InsecureSkipVerify|MinVersion|MaxVersion|CipherSuites|CurvePreferences)$
msg: Do not set this field directly, use crypto.CreateTLSConfiguration() instead
- pattern: ^object\.(Commit|Tag)\.Verify$
msg: 'Not allowed because of FIPS mode'
- pattern: ^(types\.SystemContext\.)?(DockerDaemonInsecureSkipTLSVerify|DockerInsecureSkipTLSVerify|OCIInsecureSkipTLSVerify)$
msg: 'Not allowed because of FIPS mode'
- pattern: ^git\.PlainClone(Context|WithOptions)?$
msg: Use git.CloneContext with NewNoSymlinkFS to prevent symlink traversal attacks
analyze-types: true
exclusions:
generated: lax
presets:
- comments
- common-false-positives
- legacy
- exportloopref
linters-settings:
depguard:
rules:
- path: pkg/libhttp/ssrf
linters:
- gocritic
text: ruleguard
- path: pkg/libhttp/ssrf/builder\.go
linters:
- forbidigo
paths:
- third_party$
- builtin$
- examples$
formatters:
enable:
- gofmt
exclusions:
generated: lax
paths:
- third_party$
- builtin$
- examples$
main:
deny:
- pkg: 'encoding/json'
desc: 'use github.com/segmentio/encoding/json'
- pkg: 'github.com/sirupsen/logrus'
desc: 'logging is allowed only by github.com/rs/zerolog'
- pkg: 'golang.org/x/exp'
desc: 'exp is not allowed'
- pkg: 'github.com/portainer/libcrypto'
desc: 'use github.com/portainer/portainer/pkg/libcrypto'
- pkg: 'github.com/portainer/libhttp'
desc: 'use github.com/portainer/portainer/pkg/libhttp'
files:
- '!**/*_test.go'
- '!**/base.go'
- '!**/base_tx.go'
# errorlint is causing a typecheck error for some reason. The go compiler will report these
# anyway, so ignore them from the linter
issues:
exclude-rules:
- path: ./
linters:
- typecheck

View File

@@ -1,4 +1,4 @@
#!/usr/bin/env sh
. "$(dirname -- "$0")/_/husky.sh"
cd $(dirname -- "$0") && pnpm lint-staged
yarn lint-staged

View File

@@ -1,5 +1,2 @@
dist
api/datastore/test_data
coverage
pnpm-lock.yaml
api/datastore/test_data

View File

@@ -5,18 +5,21 @@
"trailingComma": "es5",
"overrides": [
{
"files": ["*.html"],
"files": [
"*.html"
],
"options": {
"parser": "angular"
}
},
{
"files": ["*.{j,t}sx", "*.ts"],
"files": [
"*.{j,t}sx",
"*.ts"
],
"options": {
"printWidth": 80
}
}
],
"plugins": ["prettier-plugin-tailwindcss"],
"tailwindFunctions": ["clsx"]
}
]
}

View File

@@ -1,56 +1,30 @@
// This file has been automatically migrated to valid ESM format by Storybook.
import { fileURLToPath } from 'node:url';
import { createRequire } from 'node:module';
import path, { dirname } from 'path';
import { StorybookConfig } from '@storybook/react-webpack5';
import TsconfigPathsPlugin from 'tsconfig-paths-webpack-plugin';
import { Configuration } from 'webpack';
import postcss from 'postcss';
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const require = createRequire(import.meta.url);
const config: StorybookConfig = {
stories: ['../app/**/*.stories.@(ts|tsx)'],
addons: [
'@storybook/addon-links',
'@storybook/addon-webpack5-compiler-swc',
'@chromatic-com/storybook',
'@storybook/addon-essentials',
{
name: '@storybook/addon-styling-webpack',
name: '@storybook/addon-styling',
options: {
rules: [
{
test: /\.css$/,
sideEffects: true,
use: [
require.resolve('style-loader'),
{
loader: require.resolve('css-loader'),
options: {
importLoaders: 1,
modules: {
localIdentName: '[path][name]__[local]',
auto: true,
exportLocalsConvention: 'camelCaseOnly',
},
},
},
{
loader: require.resolve('postcss-loader'),
options: {
implementation: postcss,
},
},
],
cssLoaderOptions: {
importLoaders: 1,
modules: {
localIdentName: '[path][name]__[local]',
auto: true,
exportLocalsConvention: 'camelCaseOnly',
},
],
},
postCss: {
implementation: postcss,
},
},
},
'@storybook/addon-docs',
],
webpackFinal: (config) => {
const rules = config?.module?.rules || [];
@@ -93,7 +67,12 @@ const config: StorybookConfig = {
...config,
resolve: {
...config.resolve,
tsconfig: path.resolve(__dirname, '..', 'tsconfig.json'),
plugins: [
...(config.resolve?.plugins || []),
new TsconfigPathsPlugin({
extensions: config.resolve?.extensions,
}),
],
},
module: {
...config.module,
@@ -103,13 +82,12 @@ const config: StorybookConfig = {
},
staticDirs: ['./public'],
typescript: {
reactDocgen: 'react-docgen',
reactDocgen: 'react-docgen-typescript',
},
framework: {
name: '@storybook/react-webpack5',
options: {},
},
docs: {},
};
export default config;

View File

@@ -1,14 +1,14 @@
import { useEffect } from 'react';
import '../app/assets/css';
import React from 'react';
import { pushStateLocationPlugin, UIRouter } from '@uirouter/react';
import { initialize as initMSW, mswLoader } from 'msw-storybook-addon';
import { handlers } from '../app/setup-tests/server-handlers';
import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
import { Preview } from '@storybook/react-webpack5';
import { QueryClient, QueryClientProvider } from 'react-query';
initMSW(
{
onUnhandledRequest: ({ method, url }) => {
console.log(method, url);
if (url.startsWith('/api')) {
console.error(`Unhandled ${method} request to ${url}.
@@ -22,65 +22,31 @@ initMSW(
handlers
);
export const parameters = {
actions: { argTypesRegex: '^on[A-Z].*' },
controls: {
matchers: {
color: /(background|color)$/i,
date: /Date$/,
},
},
msw: {
handlers,
},
};
const testQueryClient = new QueryClient({
defaultOptions: { queries: { retry: false } },
});
const preview: Preview = {
globalTypes: {
theme: {
description: 'Portainer color theme',
toolbar: {
title: 'Theme',
icon: 'paintbrush',
items: [
{ value: 'light', title: 'Light', icon: 'sun' },
{ value: 'dark', title: 'Dark', icon: 'moon' },
{ value: 'highcontrast', title: 'High Contrast', icon: 'eye' },
],
dynamicTitle: true,
},
},
},
initialGlobals: {
theme: 'light',
},
decorators: (Story, context) => {
const theme = context.globals.theme;
export const decorators = [
(Story) => (
<QueryClientProvider client={testQueryClient}>
<UIRouter plugins={[pushStateLocationPlugin]}>
<Story />
</UIRouter>
</QueryClientProvider>
),
];
useEffect(() => {
if (theme === 'light') {
document.documentElement.removeAttribute('theme');
} else {
document.documentElement.setAttribute('theme', theme);
}
}, [theme]);
return (
<QueryClientProvider client={testQueryClient}>
<UIRouter plugins={[pushStateLocationPlugin]}>
<Story />
</UIRouter>
</QueryClientProvider>
);
},
loaders: [mswLoader],
parameters: {
options: {
storySort: {
order: ['Design System', 'Components', '*'],
},
},
controls: {
matchers: {
color: /(background|color)$/i,
date: /Date$/,
},
},
msw: {
handlers,
},
},
};
export default preview;
export const loaders = [mswLoader];

View File

@@ -2,26 +2,26 @@
/* tslint:disable */
/**
* Mock Service Worker.
* Mock Service Worker (2.0.11).
* @see https://github.com/mswjs/msw
* - Please do NOT modify this file.
* - Please do NOT serve this file on production.
*/
const PACKAGE_VERSION = '2.12.10';
const INTEGRITY_CHECKSUM = '4db4a41e972cec1b64cc569c66952d82';
const INTEGRITY_CHECKSUM = 'c5f7f8e188b673ea4e677df7ea3c5a39';
const IS_MOCKED_RESPONSE = Symbol('isMockedResponse');
const activeClientIds = new Set();
addEventListener('install', function () {
self.addEventListener('install', function () {
self.skipWaiting();
});
addEventListener('activate', function (event) {
self.addEventListener('activate', function (event) {
event.waitUntil(self.clients.claim());
});
addEventListener('message', async function (event) {
const clientId = Reflect.get(event.source || {}, 'id');
self.addEventListener('message', async function (event) {
const clientId = event.source.id;
if (!clientId || !self.clients) {
return;
@@ -48,10 +48,7 @@ addEventListener('message', async function (event) {
case 'INTEGRITY_CHECK_REQUEST': {
sendToClient(client, {
type: 'INTEGRITY_CHECK_RESPONSE',
payload: {
packageVersion: PACKAGE_VERSION,
checksum: INTEGRITY_CHECKSUM,
},
payload: INTEGRITY_CHECKSUM,
});
break;
}
@@ -61,16 +58,16 @@ addEventListener('message', async function (event) {
sendToClient(client, {
type: 'MOCKING_ENABLED',
payload: {
client: {
id: client.id,
frameType: client.frameType,
},
},
payload: true,
});
break;
}
case 'MOCK_DEACTIVATE': {
activeClientIds.delete(clientId);
break;
}
case 'CLIENT_CLOSED': {
activeClientIds.delete(clientId);
@@ -88,91 +85,72 @@ addEventListener('message', async function (event) {
}
});
addEventListener('fetch', function (event) {
const requestInterceptedAt = Date.now();
self.addEventListener('fetch', function (event) {
const { request } = event;
// Bypass navigation requests.
if (event.request.mode === 'navigate') {
if (request.mode === 'navigate') {
return;
}
// Opening the DevTools triggers the "only-if-cached" request
// that cannot be handled by the worker. Bypass such requests.
if (event.request.cache === 'only-if-cached' && event.request.mode !== 'same-origin') {
if (request.cache === 'only-if-cached' && request.mode !== 'same-origin') {
return;
}
// Bypass all requests when there are no active clients.
// Prevents the self-unregistered worked from handling requests
// after it's been terminated (still remains active until the next reload).
// after it's been deleted (still remains active until the next reload).
if (activeClientIds.size === 0) {
return;
}
// Generate unique request ID.
const requestId = crypto.randomUUID();
event.respondWith(handleRequest(event, requestId, requestInterceptedAt));
event.respondWith(handleRequest(event, requestId));
});
/**
* @param {FetchEvent} event
* @param {string} requestId
* @param {number} requestInterceptedAt
*/
async function handleRequest(event, requestId, requestInterceptedAt) {
async function handleRequest(event, requestId) {
const client = await resolveMainClient(event);
const requestCloneForEvents = event.request.clone();
const response = await getResponse(event, client, requestId, requestInterceptedAt);
const response = await getResponse(event, client, requestId);
// Send back the response clone for the "response:*" life-cycle events.
// Ensure MSW is active and ready to handle the message, otherwise
// this message will pend indefinitely.
if (client && activeClientIds.has(client.id)) {
const serializedRequest = await serializeRequest(requestCloneForEvents);
(async function () {
const responseClone = response.clone();
// Clone the response so both the client and the library could consume it.
const responseClone = response.clone();
sendToClient(
client,
{
type: 'RESPONSE',
payload: {
isMockedResponse: IS_MOCKED_RESPONSE in response,
request: {
id: requestId,
...serializedRequest,
},
response: {
sendToClient(
client,
{
type: 'RESPONSE',
payload: {
requestId,
isMockedResponse: IS_MOCKED_RESPONSE in response,
type: responseClone.type,
status: responseClone.status,
statusText: responseClone.statusText,
headers: Object.fromEntries(responseClone.headers.entries()),
body: responseClone.body,
headers: Object.fromEntries(responseClone.headers.entries()),
},
},
},
responseClone.body ? [serializedRequest.body, responseClone.body] : []
);
[responseClone.body]
);
})();
}
return response;
}
/**
* Resolve the main client for the given event.
* Client that issues a request doesn't necessarily equal the client
* that registered the worker. It's with the latter the worker should
* communicate with during the response resolving phase.
* @param {FetchEvent} event
* @returns {Promise<Client | undefined>}
*/
// Resolve the main client for the given event.
// Client that issues a request doesn't necessarily equal the client
// that registered the worker. It's with the latter the worker should
// communicate with during the response resolving phase.
async function resolveMainClient(event) {
const client = await self.clients.get(event.clientId);
if (activeClientIds.has(event.clientId)) {
return client;
}
if (client?.frameType === 'top-level') {
return client;
}
@@ -193,37 +171,20 @@ async function resolveMainClient(event) {
});
}
/**
* @param {FetchEvent} event
* @param {Client | undefined} client
* @param {string} requestId
* @param {number} requestInterceptedAt
* @returns {Promise<Response>}
*/
async function getResponse(event, client, requestId, requestInterceptedAt) {
async function getResponse(event, client, requestId) {
const { request } = event;
// Clone the request because it might've been already used
// (i.e. its body has been read and sent to the client).
const requestClone = event.request.clone();
const requestClone = request.clone();
function passthrough() {
// Cast the request headers to a new Headers instance
// so the headers can be manipulated with.
const headers = new Headers(requestClone.headers);
const headers = Object.fromEntries(requestClone.headers.entries());
// Remove the "accept" header value that marked this request as passthrough.
// This prevents request alteration and also keeps it compliant with the
// user-defined CORS policies.
const acceptHeader = headers.get('accept');
if (acceptHeader) {
const values = acceptHeader.split(',').map((value) => value.trim());
const filteredValues = values.filter((value) => value !== 'msw/passthrough');
if (filteredValues.length > 0) {
headers.set('accept', filteredValues.join(', '));
} else {
headers.delete('accept');
}
}
// Remove internal MSW request header so the passthrough request
// complies with any potential CORS preflight checks on the server.
// Some servers forbid unknown request headers.
delete headers['x-msw-intention'];
return fetch(requestClone, { headers });
}
@@ -241,19 +202,37 @@ async function getResponse(event, client, requestId, requestInterceptedAt) {
return passthrough();
}
// Bypass requests with the explicit bypass header.
// Such requests can be issued by "ctx.fetch()".
const mswIntention = request.headers.get('x-msw-intention');
if (['bypass', 'passthrough'].includes(mswIntention)) {
return passthrough();
}
// Notify the client that a request has been intercepted.
const serializedRequest = await serializeRequest(event.request);
const requestBuffer = await request.arrayBuffer();
const clientMessage = await sendToClient(
client,
{
type: 'REQUEST',
payload: {
id: requestId,
interceptedAt: requestInterceptedAt,
...serializedRequest,
url: request.url,
mode: request.mode,
method: request.method,
headers: Object.fromEntries(request.headers.entries()),
cache: request.cache,
credentials: request.credentials,
destination: request.destination,
integrity: request.integrity,
redirect: request.redirect,
referrer: request.referrer,
referrerPolicy: request.referrerPolicy,
body: requestBuffer,
keepalive: request.keepalive,
},
},
[serializedRequest.body]
[requestBuffer]
);
switch (clientMessage.type) {
@@ -261,7 +240,7 @@ async function getResponse(event, client, requestId, requestInterceptedAt) {
return respondWithMock(clientMessage.data);
}
case 'PASSTHROUGH': {
case 'MOCK_NOT_FOUND': {
return passthrough();
}
}
@@ -269,12 +248,6 @@ async function getResponse(event, client, requestId, requestInterceptedAt) {
return passthrough();
}
/**
* @param {Client} client
* @param {any} message
* @param {Array<Transferable>} transferrables
* @returns {Promise<any>}
*/
function sendToClient(client, message, transferrables = []) {
return new Promise((resolve, reject) => {
const channel = new MessageChannel();
@@ -287,15 +260,11 @@ function sendToClient(client, message, transferrables = []) {
resolve(event.data);
};
client.postMessage(message, [channel.port2, ...transferrables.filter(Boolean)]);
client.postMessage(message, [channel.port2].concat(transferrables.filter(Boolean)));
});
}
/**
* @param {Response} response
* @returns {Response}
*/
function respondWithMock(response) {
async function respondWithMock(response) {
// Setting response status code to 0 is a no-op.
// However, when responding with a "Response.error()", the produced Response
// instance will have status code set to 0. Since it's not possible to create
@@ -313,24 +282,3 @@ function respondWithMock(response) {
return mockedResponse;
}
/**
* @param {Request} request
*/
async function serializeRequest(request) {
return {
url: request.url,
mode: request.mode,
method: request.method,
headers: Object.fromEntries(request.headers.entries()),
cache: request.cache,
credentials: request.credentials,
destination: request.destination,
integrity: request.integrity,
redirect: request.redirect,
referrer: request.referrer,
referrerPolicy: request.referrerPolicy,
body: await request.arrayBuffer(),
keepalive: request.keepalive,
};
}

View File

@@ -0,0 +1,19 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"name": "Launch",
"type": "go",
"request": "launch",
"mode": "debug",
"program": "${workspaceRoot}/api/cmd/portainer",
"cwd": "${workspaceRoot}",
"env": {},
"showLog": true,
"args": ["--data", "${env:HOME}/portainer-data", "--assets", "${workspaceRoot}/dist"]
}
]
}

View File

@@ -0,0 +1,191 @@
{
// Place your portainer workspace snippets here. Each snippet is defined under a snippet name and has a scope, prefix, body and
// description. Add comma separated ids of the languages where the snippet is applicable in the scope field. If scope
// is left empty or omitted, the snippet gets applied to all languages. The prefix is what is
// used to trigger the snippet and the body will be expanded and inserted. Possible variables are:
// $1, $2 for tab stops, $0 for the final cursor position, and ${1:label}, ${2:another} for placeholders.
// Placeholders with the same ids are connected.
// Example:
// "Print to console": {
// "scope": "javascript,typescript",
// "prefix": "log",
// "body": [
// "console.log('$1');",
// "$2"
// ],
// "description": "Log output to console"
// }
"React Named Export Component": {
"prefix": "rnec",
"body": [
"export function $TM_FILENAME_BASE() {",
" return <div>$TM_FILENAME_BASE</div>;",
"}"
],
"description": "React Named Export Component"
},
"Component": {
"scope": "javascript",
"prefix": "mycomponent",
"description": "Dummy Angularjs Component",
"body": [
"import angular from 'angular';",
"import controller from './${TM_FILENAME_BASE}Controller'",
"",
"angular.module('portainer.${TM_DIRECTORY/.*\\/app\\/([^\\/]*)(\\/.*)?$/$1/}').component('$TM_FILENAME_BASE', {",
" templateUrl: './$TM_FILENAME_BASE.html',",
" controller,",
"});",
""
]
},
"Controller": {
"scope": "javascript",
"prefix": "mycontroller",
"body": [
"class ${TM_FILENAME_BASE/(.*)/${1:/capitalize}/} {",
"\t/* @ngInject */",
"\tconstructor($0) {",
"\t}",
"}",
"",
"export default ${TM_FILENAME_BASE/(.*)/${1:/capitalize}/};"
],
"description": "Dummy ES6+ controller"
},
"Service": {
"scope": "javascript",
"prefix": "myservice",
"description": "Dummy ES6+ service",
"body": [
"import angular from 'angular';",
"import PortainerError from 'Portainer/error';",
"",
"class $1 {",
" /* @ngInject */",
" constructor(\\$async, $0) {",
" this.\\$async = \\$async;",
"",
" this.getAsync = this.getAsync.bind(this);",
" this.getAllAsync = this.getAllAsync.bind(this);",
" this.createAsync = this.createAsync.bind(this);",
" this.updateAsync = this.updateAsync.bind(this);",
" this.deleteAsync = this.deleteAsync.bind(this);",
" }",
"",
" /**",
" * GET",
" */",
" async getAsync() {",
" try {",
"",
" } catch (err) {",
" throw new PortainerError('', err);",
" }",
" }",
"",
" async getAllAsync() {",
" try {",
"",
" } catch (err) {",
" throw new PortainerError('', err);",
" }",
" }",
"",
" get() {",
" if () {",
" return this.\\$async(this.getAsync);",
" }",
" return this.\\$async(this.getAllAsync);",
" }",
"",
" /**",
" * CREATE",
" */",
" async createAsync() {",
" try {",
"",
" } catch (err) {",
" throw new PortainerError('', err);",
" }",
" }",
"",
" create() {",
" return this.\\$async(this.createAsync);",
" }",
"",
" /**",
" * UPDATE",
" */",
" async updateAsync() {",
" try {",
"",
" } catch (err) {",
" throw new PortainerError('', err);",
" }",
" }",
"",
" update() {",
" return this.\\$async(this.updateAsync);",
" }",
"",
" /**",
" * DELETE",
" */",
" async deleteAsync() {",
" try {",
"",
" } catch (err) {",
" throw new PortainerError('', err);",
" }",
" }",
"",
" delete() {",
" return this.\\$async(this.deleteAsync);",
" }",
"}",
"",
"export default $1;",
"angular.module('portainer.${TM_DIRECTORY/.*\\/app\\/([^\\/]*)(\\/.*)?$/$1/}').service('$1', $1);"
]
},
"swagger-api-doc": {
"prefix": "swapi",
"scope": "go",
"description": "Snippet for a api doc",
"body": [
"// @id ",
"// @summary ",
"// @description ",
"// @description **Access policy**: ",
"// @tags ",
"// @security ApiKeyAuth",
"// @security jwt",
"// @accept json",
"// @produce json",
"// @param id path int true \"identifier\"",
"// @param body body Object true \"details\"",
"// @success 200 {object} portainer. \"Success\"",
"// @success 204 \"Success\"",
"// @failure 400 \"Invalid request\"",
"// @failure 403 \"Permission denied\"",
"// @failure 404 \" not found\"",
"// @failure 500 \"Server error\"",
"// @router /{id} [get]"
]
},
"analytics": {
"prefix": "nlt",
"body": ["analytics-on", "analytics-category=\"$1\"", "analytics-event=\"$2\""],
"description": "analytics"
},
"analytics-if": {
"prefix": "nltf",
"body": ["analytics-if=\"$1\""],
"description": "analytics"
},
"analytics-metadata": {
"prefix": "nltm",
"body": "analytics-properties=\"{ metadata: { $1 } }\""
}
}

View File

@@ -0,0 +1,8 @@
{
"go.lintTool": "golangci-lint",
"go.lintFlags": ["--fast", "-E", "exportloopref"],
"gopls": {
"build.expandWorkspaceToModule": false
},
"gitlens.advanced.blame.customArguments": ["--ignore-revs-file", ".git-blame-ignore-revs"]
}

View File

@@ -1,68 +0,0 @@
# Portainer Community Edition
Open-source container management platform with full Docker and Kubernetes support.
## Project Structure
For a detailed breakdown of frontend and backend directory layout, feature locations, and common development tasks, see [docs/guidelines/project-structure.md](../../docs/guidelines/project-structure.md).
## Frontend Guidelines
- [docs/guidelines/frontend-conventions.md](../../docs/guidelines/frontend-conventions.md) — component structure, React Query patterns, shared components, forms, theming
- [docs/guidelines/typescript-conventions.md](../../docs/guidelines/typescript-conventions.md) — types, anti-patterns, union types, named constants
- [docs/guidelines/frontend-unit-testing.md](../../docs/guidelines/frontend-unit-testing.md) — Vitest, React Testing Library
## Backend Guidelines
- [docs/guidelines/go-conventions.md](../../docs/guidelines/go-conventions.md) — error handling, naming, testing, code style
- [docs/guidelines/server-architecture.md](../../docs/guidelines/server-architecture.md) — Clean Architecture layers, transactions, CE/EE sharing patterns
- [docs/guidelines/logging.md](../../docs/guidelines/logging.md) — zerolog usage, log levels, message style
- [docs/guidelines/backend-code-reusability.md](../../docs/guidelines/backend-code-reusability.md) — how CE and EE share backend code
## Package Manager
- **PNPM** 10+ (for frontend)
- **Go** 1.26.1 (for backend)
## Build Commands
```bash
# Full build
make build # Build both client and server
make build-client # Build React/AngularJS frontend
make build-server # Build Go binary
make build-image # Build Docker image
# Development
make dev # Run both in dev mode
make dev-client # Start webpack-dev-server (port 8999)
make dev-server # Run containerized Go server
# Frontend
pnpm dev # Webpack dev server
pnpm build # Build frontend with webpack
pnpm typecheck # Run typecheck for frontend (with tsc)
pnpm lint # lint frontend (with eslint)
pnpm test # test frontend (with vitest)
pnpm format # format frontend (with prettier)
# Testing
make test # All tests (backend + frontend)
make test-server # Backend tests only
make lint # Lint all code
make format # Format code
```
## Development Servers
- Frontend: http://localhost:8999
- Backend: http://localhost:9000 (HTTP) / https://localhost:9443 (HTTPS)
## Local demo stand
To build an image from one or more feature branches and run it (e.g. to demo open
PRs together), see [docs/dev-stand.md](docs/dev-stand.md). **Read its Gotchas
first** — most importantly, build the image with `make build-image ENV=production`
(without it, `build-image` ships a development client bundle that the CSP blocks,
leaving the UI stuck forever on "Loading Portainer…"), and note that the admin
password must be simple/special-char-free but at least 12 characters long.

View File

@@ -8,19 +8,19 @@ In the interest of fostering an open and welcoming environment, we as contributo
Examples of behavior that contributes to creating a positive environment include:
- Using welcoming and inclusive language
- Being respectful of differing viewpoints and experiences
- Gracefully accepting constructive criticism
- Focusing on what is best for the community
- Showing empathy towards other community members
* Using welcoming and inclusive language
* Being respectful of differing viewpoints and experiences
* Gracefully accepting constructive criticism
* Focusing on what is best for the community
* Showing empathy towards other community members
Examples of unacceptable behavior by participants include:
- The use of sexualized language or imagery and unwelcome sexual attention or advances
- Trolling, insulting/derogatory comments, and personal or political attacks
- Public or private harassment
- Publishing others' private information, such as a physical or electronic address, without explicit permission
- Other conduct which could reasonably be considered inappropriate in a professional setting
* The use of sexualized language or imagery and unwelcome sexual attention or advances
* Trolling, insulting/derogatory comments, and personal or political attacks
* Public or private harassment
* Publishing others' private information, such as a physical or electronic address, without explicit permission
* Other conduct which could reasonably be considered inappropriate in a professional setting
## Our Responsibilities
@@ -34,7 +34,7 @@ This Code of Conduct applies both within project spaces and in public spaces whe
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at contribute@portainer.io. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at anthony.lapenna@portainer.io. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.

View File

@@ -77,7 +77,7 @@ The feature request process is similar to the bug report process but has an extr
## Build and run Portainer locally
Ensure you have Docker, Node.js, pnpm, and Golang installed in the correct versions.
Ensure you have Docker, Node.js, yarn, and Golang installed in the correct versions.
Install dependencies:
@@ -147,9 +147,7 @@ When adding a new route to an existing handler use the following as a template (
// @router /{id} [get]
```
explanation about each line can be found [here](https://github.com/swaggo/swag#api-operation)
After changing these annotations, regenerate the TypeScript API client and types — see [Generating API types](./README.md#generating-api-types).
explanation about each line can be found (here)[https://github.com/swaggo/swag#api-operation]
## Licensing

View File

@@ -1,50 +1,57 @@
# See: https://gist.github.com/asukakenji/f15ba7e588ac42795f421b48b8aede63
# For a list of valid GOOS and GOARCH values
# Note: these can be overriden on the command line e.g. `make PLATFORM=<platform> ARCH=<arch>`
PLATFORM=$(shell go env GOOS)
ARCH=$(shell go env GOARCH)
# build target, can be one of "production", "testing", "development"
ENV=development
WEBPACK_CONFIG=webpack/webpack.$(ENV).js
TAG=local
SWAG=go run github.com/swaggo/swag/cmd/swag@v1.16.6
GOTESTSUM_VERSION?=v1.13.0
GOTESTSUM=go run gotest.tools/gotestsum@$(GOTESTSUM_VERSION)
GOLANGCI_LINT_VERSION := $(shell cat $(shell git rev-parse --show-toplevel)/.golangci-version)
SWAG=go run github.com/swaggo/swag/cmd/swag@v1.16.2
GOTESTSUM=go run gotest.tools/gotestsum@latest
# Don't change anything below this line unless you know what you're doing
.DEFAULT_GOAL := help
##@ Building
.PHONY: all init-dist build-storybook build build-client build-server build-image devops
.PHONY: init-dist build-storybook build build-client build-server build-image devops
init-dist:
@mkdir -p dist
all: tidy deps build-server build-client ## Build the client, server and download external dependancies (doesn't build an image)
build-all: all ## Alias for the 'all' target (used by CI)
build-all: deps build-server build-client ## Build the client, server and download external dependancies (doesn't build an image)
build-client: init-dist ## Build the client
export NODE_ENV=$(ENV) && pnpm run build --config $(WEBPACK_CONFIG)
export NODE_ENV=$(ENV) && yarn build --config $(WEBPACK_CONFIG)
build-server: init-dist ## Build the server binary
./build/build_binary.sh "$(PLATFORM)" "$(ARCH)"
build-image: build-all ## Build the Portainer image locally
docker buildx build --load -t portainerci/portainer-ce:$(TAG) -f build/linux/Dockerfile .
docker buildx build --load -t portainerci/portainer:$(TAG) -f build/linux/Dockerfile .
build-storybook: ## Build and serve the storybook files
pnpm run storybook:build
yarn storybook:build
devops: clean deps build-client ## Build the everything target specifically for CI
echo "Building the devops binary..."
@./build/build_binary_azuredevops.sh "$(PLATFORM)" "$(ARCH)"
##@ Build dependencies
.PHONY: deps server-deps client-deps tidy
deps: server-deps client-deps ## Download all client and server build dependancies
## This is empty because the pipeline requires it but ce has no server deps
server-deps: init-dist ## Download dependant server binaries
@./build/download_binaries.sh $(PLATFORM) $(ARCH)
client-deps: ## Install client dependencies
pnpm install
yarn
tidy: ## Tidy up the go.mod file
@go mod tidy
cd api && go mod tidy
##@ Cleanup
.PHONY: clean
@@ -52,32 +59,29 @@ clean: ## Remove all build and download artifacts
@echo "Clearing the dist directory..."
@rm -rf dist/*
##@ Testing
.PHONY: test test-client test-server
test: test-server test-client ## Run all tests
test-client: ## Run client tests
pnpm run test $(ARGS) --coverage
TEST_PACKAGES?=./...
yarn test $(ARGS)
test-server: ## Run server tests
$(GOTESTSUM) --format pkgname-and-test-fails --format-hide-empty-pkg --hide-summary skipped -- -cover -covermode=atomic -coverprofile=coverage.out $(TEST_PACKAGES)
$(GOTESTSUM) --format pkgname-and-test-fails --format-hide-empty-pkg --hide-summary skipped -- -cover ./...
##@ Dev
.PHONY: dev dev-client dev-server
dev: ## Run both the client and server in development mode
dev: ## Run both the client and server in development mode
make dev-server
make dev-client
dev-client: ## Run the client in development mode
pnpm install && pnpm run dev
dev-client: ## Run the client in development mode
yarn dev
dev-server: build-server ## Run the server in development mode
@./dev/run_container.sh
dev-server-podman: build-server ## Run the server in development mode
@./dev/run_container_podman.sh
##@ Format
.PHONY: format format-client format-server
@@ -85,59 +89,36 @@ dev-server-podman: build-server ## Run the server in development mode
format: format-client format-server ## Format all code
format-client: ## Format client code
pnpm run format
yarn format
format-server: ## Format server code
go fmt ./...
##@ Lint
.PHONY: lint lint-client lint-server check-lint-version
.PHONY: lint lint-client lint-server
lint: lint-client lint-server ## Lint all code
lint-client: ## Lint client code
pnpm run lint
yarn lint
check-lint-version:
@installed=v$$(golangci-lint --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1); \
if [ "$$installed" = "v" ]; then \
echo "ERROR: golangci-lint not found, need $(GOLANGCI_LINT_VERSION)"; \
echo "Install: go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION)"; \
exit 1; \
elif [ "$$installed" != "$(GOLANGCI_LINT_VERSION)" ]; then \
echo "ERROR: golangci-lint $$installed installed, need $(GOLANGCI_LINT_VERSION)"; \
echo "Install: go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION)"; \
exit 1; \
fi
lint-server: tidy check-lint-version ## Lint server code
lint-server: ## Lint server code
golangci-lint run --timeout=10m -c .golangci.yaml
golangci-lint run --timeout=10m --new-from-rev=HEAD~ -c .golangci-forward.yaml
##@ Extension
.PHONY: dev-extension
dev-extension: build-server build-client ## Run the extension in development mode
make local -f build/docker-extension/Makefile
##@ Docs
.PHONY: docs-build docs-validate docs-clean docs-validate-clean
docs-build: init-dist ## Build docs
go mod download
cd api && $(SWAG) init -o "../dist/docs" -ot "yaml" -g ./http/handler/handler.go --parseDependency --parseInternal --parseDepth 2 -p pascalcase --markdownFiles ./ --overridesFile .swaggo
cd api && $(SWAG) init -o "../dist/docs" -ot "yaml" -g ./http/handler/handler.go --parseDependency --parseInternal --parseDepth 2 -p pascalcase --markdownFiles ./
docs-validate: docs-build ## Validate docs
pnpm swagger2openapi --warnOnly dist/docs/swagger.yaml -o dist/docs/openapi.yaml
pnpm swagger-cli validate dist/docs/openapi.yaml
.PHONY: docs-serve
docs-serve: docs-build ## Serve docs locally with Swagger UI on port 8080
docker run -p 8080:8080 \
-e SWAGGER_JSON=/foo/swagger.yaml \
-v $(PWD)/dist/docs:/foo \
swaggerapi/swagger-ui
.PHONY: generate-api
generate-api: docs-validate ## Generate API client and types from OpenAPI spec
pnpm generate-api
yarn swagger2openapi --warnOnly dist/docs/swagger.yaml -o dist/docs/openapi.yaml
yarn swagger-cli validate dist/docs/openapi.yaml
##@ Helpers
.PHONY: help

View File

@@ -8,9 +8,9 @@ Portainer consists of a single container that can run on any cluster. It can be
**Portainer Business Edition** builds on the open-source base and includes a range of advanced features and functions (like RBAC and Support) that are specific to the needs of business users.
- [Compare Portainer CE and Compare Portainer BE](https://www.portainer.io/features)
- [Compare Portainer CE and Compare Portainer BE](https://portainer.io/products)
- [Take3 – get 3 free nodes of Portainer Business for as long as you want them](https://www.portainer.io/take-3)
- [Portainer BE install guide](https://academy.portainer.io/install/)
- [Portainer BE install guide](https://install.portainer.io)
## Latest Version
@@ -20,19 +20,22 @@ Portainer CE is updated regularly. We aim to do an update release every couple o
## Getting started
- [Deploy Portainer](https://docs.portainer.io/start/install-ce)
- [Deploy Portainer](https://docs.portainer.io/start/install)
- [Documentation](https://docs.portainer.io)
- [Contribute to the project](https://docs.portainer.io/contribute/contribute)
## Features & Functions
View [this](https://www.portainer.io/features) table to see all of the Portainer CE functionality and compare to Portainer Business.
View [this](https://www.portainer.io/products) table to see all of the Portainer CE functionality and compare to Portainer Business.
- [Portainer CE for Docker / Docker Swarm](https://www.portainer.io/solutions/docker)
- [Portainer CE for Kubernetes](https://www.portainer.io/solutions/kubernetes-ui)
## Getting help
Portainer CE is an open source project and is supported by the community. You can buy a supported version of Portainer at portainer.io
Learn more about Portainer's community support channels [here.](https://www.portainer.io/resources/get-help/get-support)
Learn more about Portainer's community support channels [here.](https://www.portainer.io/get-support-for-portainer)
- Issues: https://github.com/portainer/portainer/issues
- Slack (chat): [https://portainer.io/slack](https://portainer.io/slack)
@@ -44,45 +47,19 @@ You can join the Portainer Community by visiting [https://www.portainer.io/join-
- Want to report a bug or request a feature? Please open [an issue](https://github.com/portainer/portainer/issues/new).
- Want to help us build **_portainer_**? Follow our [contribution guidelines](https://docs.portainer.io/contribute/contribute) to build it locally and make a pull request.
## Generating API types
The frontend consumes a TypeScript API client (SDK functions and request/response types) that is generated from the Go API's Swagger annotations. Regenerate it after any API change — a new endpoint, a changed request/response shape, or a removed endpoint:
```bash
make generate-api
```
This runs the following pipeline:
```
Go Swagger annotations
→ dist/docs/swagger.yaml (make docs-build, via swaggo/swag)
→ dist/docs/openapi.yaml (swagger2openapi + validation)
→ app/react/portainer/generated-api/portainer/ (hey-api/openapi-ts)
```
The generator is configured in [`openapi-ts.config.ts`](./openapi-ts.config.ts), which controls the output path, plugins, and tag filters (for example, `deprecated` endpoints and `edge_agent`-tagged routes are excluded).
The generated files live in `app/react/portainer/generated-api/portainer/` and must **not** be edited by hand — your changes would be overwritten on the next run. Import the generated SDK functions and types instead of writing direct HTTP calls:
- `@api/sdk.gen` — SDK functions
- `@api/types.gen` — request/response types
See [Adding api docs](./CONTRIBUTING.md#adding-api-docs) for how to annotate handlers so they are picked up by the generator.
## Security
For information about reporting security vulnerabilities, please see our [Security Policy](SECURITY.md).
- Here at Portainer, we believe in [responsible disclosure](https://en.wikipedia.org/wiki/Responsible_disclosure) of security issues. If you have found a security issue, please report it to <security@portainer.io>.
## Work for us
If you are a developer, and our code in this repo makes sense to you, we would love to hear from you. We are always on the hunt for awesome devs, either freelance or employed. Drop us a line to success@portainer.io with your details and/or visit our [careers page](https://apply.workable.com/portainer/).
If you are a developer, and our code in this repo makes sense to you, we would love to hear from you. We are always on the hunt for awesome devs, either freelance or employed. Drop us a line to info@portainer.io with your details and/or visit our [careers page](https://portainer.io/careers).
## Privacy
**To make sure we focus our development effort in the right places we need to know which features get used most often. To give us this information we use [Matomo Analytics](https://matomo.org/), which is hosted in Germany and is fully GDPR compliant.**
When Portainer first starts, you are given the option to DISABLE analytics. If you **don't** choose to disable it, we collect anonymous usage as per [our privacy policy](https://www.portainer.io/legal/privacy-policy). **Please note**, there is no personally identifiable information sent or stored at any time and we only use the data to help us improve Portainer.
When Portainer first starts, you are given the option to DISABLE analytics. If you **don't** choose to disable it, we collect anonymous usage as per [our privacy policy](https://www.portainer.io/privacy-policy). **Please note**, there is no personally identifiable information sent or stored at any time and we only use the data to help us improve Portainer.
## Limitations

View File

@@ -1,60 +0,0 @@
# Security Policy
## Supported Versions
Portainer maintains both Short-Term Support (STS) and Long-Term Support (LTS) versions in accordance with our official [Portainer Lifecycle Policy](https://docs.portainer.io/start/lifecycle).
| Version Type | Support Status |
| ------------------------ | ------------------------------------------- |
| LTS (Long-Term Support) | Supported for critical security fixes |
| STS (Short-Term Support) | Supported until the next STS or LTS release |
| Legacy / EOL | Not supported |
For a detailed breakdown of current versions and their specific End of Life (EOL) dates,
please refer to the [Portainer Lifecycle Policy](https://docs.portainer.io/start/lifecycle).
## Reporting a Vulnerability
The Portainer team takes the security of our products seriously. If you believe you have found a security vulnerability in any Portainer-owned repository, please report it to us responsibly.
**Please do not report security vulnerabilities via public GitHub issues.**
### Disclosure Process
1. **Report**: You can report in one of two ways:
- **GitHub**: Use the **Report a vulnerability** button on the **Security** tab of this repository.
- **Email**: Send your findings to security@portainer.io.
2. **Details**: To help us verify the issue, please include:
- A description of the vulnerability and its potential impact.
- Step-by-step instructions to reproduce the issue (e.g. proof-of-concept code, scripts, or screenshots).
- The version of the software and the environment in which it was found.
3. **Acknowledge**: We will acknowledge receipt of your report and provide an initial assessment.
4. **Resolution**: We will work to resolve the issue as quickly as possible. We request that you do not disclose the vulnerability publicly until we have released a fix and notified affected users.
## Our Commitment
If you follow the responsible disclosure process, we will:
- Respond to your report in a timely manner.
- Provide an estimated timeline for remediation.
- Notify you when the vulnerability has been patched.
- Give credit for the discovery (if desired) once the fix is public.
We will make every effort to promptly address any security weaknesses. Security advisories and fixes will be published through GitHub Security Advisories and other channels as needed.
Thank you for helping keep Portainer and our community secure.
## Resources
- [Contributing to Portainer](https://docs.portainer.io/contribute/contribute#contributing-to-the-portainer-ce-codebase)

View File

@@ -1,118 +0,0 @@
import {
Children,
useState,
useEffect,
useRef,
useContext,
createContext,
ReactNode,
} from 'react';
type MenuCtxType = {
isOpen: boolean;
setOpen: (v: boolean) => void;
menuRef: React.RefObject<HTMLDivElement>;
label: string;
setLabel: (v: string) => void;
};
const MenuCtx = createContext<MenuCtxType | null>(null);
export function Menu({ children }: { children?: ReactNode }) {
const [isOpen, setOpen] = useState(false);
const [label, setLabel] = useState('');
const menuRef = useRef<HTMLDivElement>(null);
useEffect(() => {
function handleDocDown(e: MouseEvent) {
const target = e.target as Node | null;
if (
isOpen &&
menuRef.current &&
target &&
!menuRef.current.contains(target)
) {
setOpen(false);
}
}
document.addEventListener('mousedown', handleDocDown);
return () => document.removeEventListener('mousedown', handleDocDown);
}, [isOpen]);
return (
<MenuCtx.Provider value={{ isOpen, setOpen, menuRef, label, setLabel }}>
<div ref={menuRef}>{children}</div>
</MenuCtx.Provider>
);
}
export function MenuButton({
children,
onClick: externalOnClick,
...props
}: {
children?: ReactNode;
onClick?: () => void;
[key: string]: unknown;
}) {
const ctx = useContext(MenuCtx);
useEffect(() => {
const firstText = Children.toArray(children).find(
(c) => typeof c === 'string'
);
if (firstText) ctx?.setLabel(firstText as string);
});
function handleClick() {
externalOnClick?.();
ctx?.setOpen(!ctx.isOpen);
}
return (
<button type="button" onClick={handleClick} {...props}>
{children}
</button>
);
}
export function MenuList({
children,
className,
}: {
children?: ReactNode;
className?: string;
}) {
const ctx = useContext(MenuCtx);
if (!ctx?.isOpen) return null;
return (
<div role="menu" aria-label={ctx.label || undefined} className={className}>
{children}
</div>
);
}
export function MenuItem({
children,
onSelect,
className,
}: {
children?: ReactNode;
onSelect?: () => void;
className?: string;
}) {
const ctx = useContext(MenuCtx);
function handleClick() {
onSelect?.();
ctx?.setOpen(false);
}
return (
// eslint-disable-next-line jsx-a11y/click-events-have-key-events, jsx-a11y/interactive-supports-focus
<div role="menuitem" onClick={handleClick} className={className}>
{children}
</div>
);
}

View File

@@ -1,18 +0,0 @@
//go:build ignore
package gorules
import "github.com/quasilyte/go-ruleguard/dsl"
// inMemoryCloneWithWorktree flags git clone calls that use memory.NewStorage() as
// the storer while also writing files to a real worktree. This holds all git objects
// in heap for the duration of the clone, which is unbounded for user-supplied repos.
func inMemoryCloneWithWorktree(m dsl.Matcher) {
m.Match(`git.CloneContext($_, memory.NewStorage(), $wt, $_)`).
Where(m["wt"].Text != "nil").
Report(`git.CloneContext with memory.NewStorage() holds all git objects in heap; use gogitfs.NewStorage with a filesystem storer instead`)
m.Match(`git.Clone(memory.NewStorage(), $wt, $_)`).
Where(m["wt"].Text != "nil").
Report(`git.Clone with memory.NewStorage() holds all git objects in heap; use gogitfs.NewStorage with a filesystem storer instead`)
}

View File

@@ -1,75 +0,0 @@
//go:build ignore
package gorules
import "github.com/quasilyte/go-ruleguard/dsl"
// unwrappedHTTPTransport flags any bare http.Transport composite literal.
// All transports must be created via ssrf.NewTransport or ssrf.NewInternalTransport,
// which clone http.DefaultTransport and handle SSRF protection internally.
func unwrappedHTTPTransport(m dsl.Matcher) {
m.Match(`$f(&http.Transport{$*_})`).
Report(`$f receives a bare *http.Transport; use ssrf.NewTransport(tlsConfig) or ssrf.NewInternalTransport(tlsConfig) instead`)
m.Match(`$_ := &http.Transport{$*_}`).
Report(`bare *http.Transport variable; use ssrf.NewTransport(tlsConfig) or ssrf.NewInternalTransport(tlsConfig) instead`)
m.Match(`$_.Transport = &http.Transport{$*_}`).
Report(`bare *http.Transport field assignment; use ssrf.NewTransport(tlsConfig) or ssrf.NewInternalTransport(tlsConfig) instead`)
}
// helmGetterTransport flags getter.WithTransport calls that receive a bare *http.Transport.
// Helm v4 installs its own transport and bypasses http.DefaultTransport, so the transport
// passed here must be created via ssrf.NewTransport.
func helmGetterTransport(m dsl.Matcher) {
m.Match(`getter.WithTransport(&http.Transport{$*_})`).
Report(`getter.WithTransport called with a bare *http.Transport; use ssrf.NewTransport(tlsConfig) as Helm v4 bypasses http.DefaultTransport`)
}
// cloneDefaultTransport flags direct clones of *http.Transport outside main.go.
// The one legitimate clone is in main.go where http.DefaultTransport is globally
// wrapped with SSRF protection at server startup.
func cloneDefaultTransport(m dsl.Matcher) {
m.Match(`$_.(*http.Transport).Clone()`).
Where(!m.File().Name.Matches(`^main\.go$`)).
Report(`cloning *http.Transport directly is forbidden; use ssrf.NewTransport(tlsConfig) or ssrf.NewInternalTransport(tlsConfig) instead`)
}
// internalTransportMisuse flags calls to NewInternalTransport outside the proxy
// factory files where Chisel-tunnel and in-cluster K8s destinations are valid exemptions.
func internalTransportMisuse(m dsl.Matcher) {
m.Match(`ssrf.NewInternalTransport($*_)`).
Where(
!(m.File().PkgPath.Matches(`proxy/factory`) &&
m.File().Name.Matches(`^(docker|agent|local_transport|edge_transport|docker_unix|docker_windows)\.go$`))).
Report(`NewInternalTransport bypasses SSRF validation; only valid in the proxy factory files for local sockets and internally-routed endpoints`)
}
// dialerOverride flags direct assignments to any of the dialer fields on a transport.
// The only valid assignments are in docker_unix.go and docker_windows.go where a
// custom dialer is required for unix sockets and named pipes.
func dialerOverride(m dsl.Matcher) {
m.Match(`$_.DialContext = $*_`).
Where(
!(m.File().PkgPath.Matches(`proxy/factory`) &&
m.File().Name.Matches(`^(docker_unix|docker_windows)\.go$`))).
Report(`direct DialContext assignment replaces the transport dialer; use ssrf.NewTransport or ssrf.NewInternalTransport instead`)
m.Match(`$_.Dial = $*_`).
Where(
!(m.File().PkgPath.Matches(`proxy/factory`) &&
m.File().Name.Matches(`^(docker_unix|docker_windows)\.go$`))).
Report(`direct Dial assignment replaces the transport dialer; use ssrf.NewTransport or ssrf.NewInternalTransport instead`)
m.Match(`$_.DialTLSContext = $*_`).
Where(
!(m.File().PkgPath.Matches(`proxy/factory`) &&
m.File().Name.Matches(`^(docker_unix|docker_windows)\.go$`))).
Report(`direct DialTLSContext assignment replaces the transport dialer; use ssrf.NewTransport or ssrf.NewInternalTransport instead`)
m.Match(`$_.DialTLS = $*_`).
Where(
!(m.File().PkgPath.Matches(`proxy/factory`) &&
m.File().Name.Matches(`^(docker_unix|docker_windows)\.go$`))).
Report(`direct DialTLS assignment replaces the transport dialer; use ssrf.NewTransport or ssrf.NewInternalTransport instead`)
}

View File

@@ -1,5 +0,0 @@
//go:build tools
package gorules
import _ "github.com/quasilyte/go-ruleguard/dsl"

View File

@@ -1 +0,0 @@
replace k8s.io/apimachinery/pkg/apis/meta/v1.Duration string

View File

@@ -19,22 +19,24 @@ const RedirectReasonAdminInitTimeout string = "AdminInitTimeout"
type Monitor struct {
timeout time.Duration
datastore dataservices.DataStore
shutdownCtx context.Context
cancellationFunc context.CancelFunc
mu sync.RWMutex
adminInitDisabled bool
}
// New creates a monitor that when started will wait for the timeout duration and then shutdown the application unless it has been initialized.
func New(timeout time.Duration, datastore dataservices.DataStore) *Monitor {
func New(timeout time.Duration, datastore dataservices.DataStore, shutdownCtx context.Context) *Monitor {
return &Monitor{
timeout: timeout,
datastore: datastore,
shutdownCtx: shutdownCtx,
adminInitDisabled: false,
}
}
// Start starts the monitor. The monitor will stop when ctx is cancelled, or when Stop is called.
func (m *Monitor) Start(ctx context.Context) {
// Starts starts the monitor. Active monitor could be stopped or shuttted down by cancelling the shutdown context.
func (m *Monitor) Start() {
m.mu.Lock()
defer m.mu.Unlock()
@@ -42,7 +44,7 @@ func (m *Monitor) Start(ctx context.Context) {
return
}
cancellationCtx, cancellationFunc := context.WithCancel(ctx)
cancellationCtx, cancellationFunc := context.WithCancel(context.Background())
m.cancellationFunc = cancellationFunc
go func() {
@@ -67,6 +69,8 @@ func (m *Monitor) Start(ctx context.Context) {
}
case <-cancellationCtx.Done():
log.Debug().Msg("canceling initialization monitor")
case <-m.shutdownCtx.Done():
log.Debug().Msg("shutting down initialization monitor")
}
}()
}

View File

@@ -1,8 +1,8 @@
package adminmonitor
import (
"context"
"testing"
"testing/synctest"
"time"
portainer "github.com/portainer/portainer/api"
@@ -11,28 +11,21 @@ import (
)
func Test_stopWithoutStarting(t *testing.T) {
t.Parallel()
monitor := New(1*time.Minute, nil)
monitor := New(1*time.Minute, nil, nil)
monitor.Stop()
}
func Test_stopCouldBeCalledMultipleTimes(t *testing.T) {
t.Parallel()
monitor := New(1*time.Minute, nil)
monitor := New(1*time.Minute, nil, nil)
monitor.Stop()
monitor.Stop()
}
func Test_startOrStopCouldBeCalledMultipleTimesConcurrently(t *testing.T) {
t.Parallel()
synctest.Test(t, test_startOrStopCouldBeCalledMultipleTimesConcurrently)
}
monitor := New(1*time.Minute, nil, context.Background())
func test_startOrStopCouldBeCalledMultipleTimesConcurrently(t *testing.T) {
monitor := New(1*time.Minute, nil)
go monitor.Start(t.Context())
monitor.Start(t.Context())
go monitor.Start()
monitor.Start()
go monitor.Stop()
monitor.Stop()
@@ -41,9 +34,8 @@ func test_startOrStopCouldBeCalledMultipleTimesConcurrently(t *testing.T) {
}
func Test_canStopStartedMonitor(t *testing.T) {
t.Parallel()
monitor := New(1*time.Minute, nil)
monitor.Start(t.Context())
monitor := New(1*time.Minute, nil, context.Background())
monitor.Start()
assert.NotNil(t, monitor.cancellationFunc, "cancellation function is missing in started monitor")
monitor.Stop()
@@ -51,12 +43,11 @@ func Test_canStopStartedMonitor(t *testing.T) {
}
func Test_start_shouldDisableInstanceAfterTimeout_ifNotInitialized(t *testing.T) {
t.Parallel()
timeout := 10 * time.Millisecond
datastore := i.NewDatastore(i.WithUsers([]portainer.User{}))
monitor := New(timeout, datastore)
monitor.Start(t.Context())
monitor := New(timeout, datastore, context.Background())
monitor.Start()
<-time.After(20 * timeout)
assert.True(t, monitor.WasInstanceDisabled(), "monitor should have been timeout and instance is disabled")

View File

@@ -1,7 +1,6 @@
package agent
import (
"context"
"crypto/tls"
"errors"
"fmt"
@@ -11,24 +10,21 @@ import (
"time"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/url"
"github.com/portainer/portainer/pkg/libhttp/ssrf"
"github.com/rs/zerolog/log"
"github.com/portainer/portainer/api/internal/url"
)
// GetAgentVersionAndPlatform returns the agent version and platform
//
// it sends a ping to the agent and parses the version and platform from the headers
func GetAgentVersionAndPlatform(endpointUrl string, tlsConfig *tls.Config) (portainer.AgentPlatform, string, error) { //nolint:forbidigo
if err := ssrf.CheckURL(context.Background(), endpointUrl); err != nil {
return 0, "", err
func GetAgentVersionAndPlatform(endpointUrl string, tlsConfig *tls.Config) (portainer.AgentPlatform, string, error) {
httpCli := &http.Client{
Timeout: 3 * time.Second,
}
httpCli := &http.Client{Timeout: 3 * time.Second}
if tlsConfig != nil {
httpCli.Transport = ssrf.NewTransport(tlsConfig)
httpCli.Transport = &http.Transport{
TLSClientConfig: tlsConfig,
}
}
parsedURL, err := url.ParseURL(endpointUrl + "/ping")
@@ -48,10 +44,8 @@ func GetAgentVersionAndPlatform(endpointUrl string, tlsConfig *tls.Config) (port
return 0, "", err
}
_, _ = io.Copy(io.Discard, resp.Body)
if err := resp.Body.Close(); err != nil {
log.Warn().Err(err).Msg("failed to close response body")
}
io.Copy(io.Discard, resp.Body)
resp.Body.Close()
if resp.StatusCode != http.StatusNoContent {
return 0, "", fmt.Errorf("Failed request with status %d", resp.StatusCode)

View File

@@ -1,119 +0,0 @@
package agent
import (
"net/http"
"net/http/httptest"
"strconv"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/stretchr/testify/require"
)
func tlsServer(t *testing.T, handler http.HandlerFunc) *httptest.Server {
t.Helper()
srv := httptest.NewTLSServer(handler)
t.Cleanup(srv.Close)
return srv
}
func TestGetAgentVersionAndPlatform_Success(t *testing.T) {
t.Parallel()
srv := tlsServer(t, func(w http.ResponseWriter, r *http.Request) {
w.Header().Set(portainer.PortainerAgentHeader, "2.19.0")
w.Header().Set(portainer.HTTPResponseAgentPlatform, "1")
w.WriteHeader(http.StatusNoContent)
})
tlsCfg := srv.Client().Transport.(*http.Transport).TLSClientConfig
platform, version, err := GetAgentVersionAndPlatform(srv.URL, tlsCfg)
require.NoError(t, err)
require.Equal(t, portainer.AgentPlatformDocker, platform)
require.Equal(t, "2.19.0", version)
}
func TestGetAgentVersionAndPlatform_NonOKStatus(t *testing.T) {
t.Parallel()
srv := tlsServer(t, func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusUnauthorized)
})
tlsCfg := srv.Client().Transport.(*http.Transport).TLSClientConfig
_, _, err := GetAgentVersionAndPlatform(srv.URL, tlsCfg)
require.Error(t, err)
}
func TestGetAgentVersionAndPlatform_MissingVersionHeader(t *testing.T) {
t.Parallel()
srv := tlsServer(t, func(w http.ResponseWriter, r *http.Request) {
w.Header().Set(portainer.HTTPResponseAgentPlatform, "1")
w.WriteHeader(http.StatusNoContent)
})
tlsCfg := srv.Client().Transport.(*http.Transport).TLSClientConfig
_, _, err := GetAgentVersionAndPlatform(srv.URL, tlsCfg)
require.Error(t, err)
}
func TestGetAgentVersionAndPlatform_MissingPlatformHeader(t *testing.T) {
t.Parallel()
srv := tlsServer(t, func(w http.ResponseWriter, r *http.Request) {
w.Header().Set(portainer.PortainerAgentHeader, "2.19.0")
w.WriteHeader(http.StatusNoContent)
})
tlsCfg := srv.Client().Transport.(*http.Transport).TLSClientConfig
_, _, err := GetAgentVersionAndPlatform(srv.URL, tlsCfg)
require.Error(t, err)
}
func TestGetAgentVersionAndPlatform_InvalidPlatformZero(t *testing.T) {
t.Parallel()
srv := tlsServer(t, func(w http.ResponseWriter, r *http.Request) {
w.Header().Set(portainer.PortainerAgentHeader, "2.19.0")
w.Header().Set(portainer.HTTPResponseAgentPlatform, "0")
w.WriteHeader(http.StatusNoContent)
})
tlsCfg := srv.Client().Transport.(*http.Transport).TLSClientConfig
_, _, err := GetAgentVersionAndPlatform(srv.URL, tlsCfg)
require.Error(t, err)
}
func TestGetAgentVersionAndPlatform_NonNumericPlatform(t *testing.T) {
t.Parallel()
srv := tlsServer(t, func(w http.ResponseWriter, r *http.Request) {
w.Header().Set(portainer.PortainerAgentHeader, "2.19.0")
w.Header().Set(portainer.HTTPResponseAgentPlatform, "docker")
w.WriteHeader(http.StatusNoContent)
})
tlsCfg := srv.Client().Transport.(*http.Transport).TLSClientConfig
_, _, err := GetAgentVersionAndPlatform(srv.URL, tlsCfg)
require.Error(t, err)
}
func TestGetAgentVersionAndPlatform_PingPathAppended(t *testing.T) {
t.Parallel()
var gotPath string
srv := tlsServer(t, func(w http.ResponseWriter, r *http.Request) {
gotPath = r.URL.Path
w.Header().Set(portainer.PortainerAgentHeader, "2.19.0")
w.Header().Set(portainer.HTTPResponseAgentPlatform, strconv.Itoa(int(portainer.AgentPlatformKubernetes)))
w.WriteHeader(http.StatusNoContent)
})
tlsCfg := srv.Client().Transport.(*http.Transport).TLSClientConfig
_, _, err := GetAgentVersionAndPlatform(srv.URL, tlsCfg)
require.NoError(t, err)
require.Equal(t, "/ping", gotPath)
}

64
api/api-description.md Normal file
View File

@@ -0,0 +1,64 @@
Portainer API is an HTTP API served by Portainer. It is used by the Portainer UI and everything you can do with the UI can be done using the HTTP API.
Examples are available at https://documentation.portainer.io/api/api-examples/
You can find out more about Portainer at [http://portainer.io](http://portainer.io) and get some support on [Slack](http://portainer.io/slack/).
# Authentication
Most of the API environments(endpoints) require to be authenticated as well as some level of authorization to be used.
Portainer API uses JSON Web Token to manage authentication and thus requires you to provide a token in the **Authorization** header of each request
with the **Bearer** authentication mechanism.
Example:
```
Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6MSwidXNlcm5hbWUiOiJhZG1pbiIsInJvbGUiOjEsImV4cCI6MTQ5OTM3NjE1NH0.NJ6vE8FY1WG6jsRQzfMqeatJ4vh2TWAeeYfDhP71YEE
```
# Security
Each API environment(endpoint) has an associated access policy, it is documented in the description of each environment(endpoint).
Different access policies are available:
- Public access
- Authenticated access
- Restricted access
- Administrator access
### Public access
No authentication is required to access the environments(endpoints) with this access policy.
### Authenticated access
Authentication is required to access the environments(endpoints) with this access policy.
### Restricted access
Authentication is required to access the environments(endpoints) with this access policy.
Extra-checks might be added to ensure access to the resource is granted. Returned data might also be filtered.
### Administrator access
Authentication as well as an administrator role are required to access the environments(endpoints) with this access policy.
# Execute Docker requests
Portainer **DO NOT** expose specific environments(endpoints) to manage your Docker resources (create a container, remove a volume, etc...).
Instead, it acts as a reverse-proxy to the Docker HTTP API. This means that you can execute Docker requests **via** the Portainer HTTP API.
To do so, you can use the `/endpoints/{id}/docker` Portainer API environment(endpoint) (which is not documented below due to Swagger limitations). This environment(endpoint) has a restricted access policy so you still need to be authenticated to be able to query this environment(endpoint). Any query on this environment(endpoint) will be proxied to the Docker API of the associated environment(endpoint) (requests and responses objects are the same as documented in the Docker API).
# Private Registry
Using private registry, you will need to pass a based64 encoded JSON string ‘{"registryId":\<registryID value\>}’ inside the Request Header. The parameter name is "X-Registry-Auth".
\<registryID value\> - The registry ID where the repository was created.
Example:
```
eyJyZWdpc3RyeUlkIjoxfQ==
```
**NOTE**: You can find more information on how to query the Docker API in the [Docker official documentation](https://docs.docker.com/engine/api/v1.30/) as well as in [this Portainer example](https://documentation.portainer.io/api/api-examples/).

View File

@@ -1,61 +0,0 @@
The Portainer API is an HTTP API served by Portainer. It is used by the Portainer UI, and anything you can do in the UI can also be done via the HTTP API.
API examples are available in the [Portainer documentation](https://documentation.portainer.io/api/api-examples/)
You can find out more about Portainer [on our website](http://portainer.io) and get some support on [Slack](http://portainer.io/slack/).
# Authentication
Most of the API endpoints require authentication, as well as some level of authorization.
Portainer uses JSON Web Tokens to manage authentication. You must provide a token in the **Authorization** header of each request using the **Bearer** scheme.
Example:
```
Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6MSwidXNlcm5hbWUiOiJhZG1pbiIsInJvbGUiOjEsImV4cCI6MTQ5OTM3NjE1NH0.NJ6vE8FY1WG6jsRQzfMqeatJ4vh2TWAeeYfDhP71YEE
```
# Security
Each API endpoint has an associated access policy, documented in its description.
The following policies are available:
- Public access
- Authenticated access
- Restricted access
- Administrator access
### Public access
No authentication is required.
### Authenticated access
Authentication is required.
### Restricted access
Authentication is required. Additional checks may apply to verify access to the resource, and returned data may be filtered.
### Administrator access
Authentication and an administrator role are both required.
# Execute Docker requests
Portainer does not expose dedicated endpoints for managing Docker resources (create a container, remove a volume, etc).
Instead, it acts as a reverse-proxy to the Docker HTTP API, allowing you to execute Docker requests via the Portainer HTTP API.
To do so, use the `/endpoints/{id}/docker` endpoint. Note that this endpoint is not documented below due to Swagger limitations. It has a restricted access policy, so authentication is still required. Any request made to this endpoint is proxied to the Docker API of the associated environment - request and response objects are identical to those in the [Docker official documentation](https://docs.docker.com/engine/api).
# Private Registry
When using a private registry, include a Base64-encoded JSON string in the request header. The header parameter name is `X-Registry-Auth` and the value should encode the following structure: ‘{"registryId":\<registryId\>}’ where `<registryId>` is the ID of the registry where the repository was created.
Example encoded value:
```
eyJyZWdpc3RyeUlkIjoxfQ==
```

View File

@@ -3,50 +3,48 @@ package apikey
import (
"testing"
"github.com/portainer/portainer/api/internal/securecookie"
"github.com/stretchr/testify/assert"
)
func Test_generateRandomKey(t *testing.T) {
t.Parallel()
is := assert.New(t)
tests := []struct {
name string
wantLength int
name string
wantLenth int
}{
{
name: "Generate a random key of length 16",
wantLength: 16,
name: "Generate a random key of length 16",
wantLenth: 16,
},
{
name: "Generate a random key of length 32",
wantLength: 32,
name: "Generate a random key of length 32",
wantLenth: 32,
},
{
name: "Generate a random key of length 64",
wantLength: 64,
name: "Generate a random key of length 64",
wantLenth: 64,
},
{
name: "Generate a random key of length 128",
wantLength: 128,
name: "Generate a random key of length 128",
wantLenth: 128,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := GenerateRandomKey(tt.wantLength)
is.Len(got, tt.wantLength)
got := securecookie.GenerateRandomKey(tt.wantLenth)
is.Equal(tt.wantLenth, len(got))
})
}
t.Run("Generated keys are unique", func(t *testing.T) {
keys := make(map[string]bool)
for range 100 {
key := GenerateRandomKey(8)
for i := 0; i < 100; i++ {
key := securecookie.GenerateRandomKey(8)
_, ok := keys[string(key)]
is.False(ok)
keys[string(key)] = true
}
})

View File

@@ -1,79 +1,69 @@
package apikey
import (
portainer "github.com/portainer/portainer/api"
lru "github.com/hashicorp/golang-lru"
portainer "github.com/portainer/portainer/api"
)
const DefaultAPIKeyCacheSize = 1024
const defaultAPIKeyCacheSize = 1024
// entry is a tuple containing the user and API key associated to an API key digest
type entry[T any] struct {
user T
type entry struct {
user portainer.User
apiKey portainer.APIKey
}
type UserCompareFn[T any] func(T, portainer.UserID) bool
// ApiKeyCache is a concurrency-safe, in-memory cache which primarily exists for to reduce database roundtrips.
// apiKeyCache is a concurrency-safe, in-memory cache which primarily exists for to reduce database roundtrips.
// We store the api-key digest (keys) and the associated user and key-data (values) in the cache.
// This is required because HTTP requests will contain only the api-key digest in the x-api-key request header;
// digest value must be mapped to a portainer user (and respective key data) for validation.
// This cache is used to avoid multiple database queries to retrieve these user/key associated to the digest.
type ApiKeyCache[T any] struct {
type apiKeyCache struct {
// cache type [string]entry cache (key: string(digest), value: user/key entry)
// note: []byte keys are not supported by golang-lru Cache
cache *lru.Cache
userCmpFn UserCompareFn[T]
cache *lru.Cache
}
// NewAPIKeyCache creates a new cache for API keys
func NewAPIKeyCache[T any](cacheSize int, userCompareFn UserCompareFn[T]) *ApiKeyCache[T] {
func NewAPIKeyCache(cacheSize int) *apiKeyCache {
cache, _ := lru.New(cacheSize)
return &ApiKeyCache[T]{cache: cache, userCmpFn: userCompareFn}
return &apiKeyCache{cache: cache}
}
// Get returns the user/key associated to an api-key's digest
// This is required because HTTP requests will contain the digest of the API key in header,
// the digest value must be mapped to a portainer user.
func (c *ApiKeyCache[T]) Get(digest string) (T, portainer.APIKey, bool) {
func (c *apiKeyCache) Get(digest string) (portainer.User, portainer.APIKey, bool) {
val, ok := c.cache.Get(digest)
if !ok {
var t T
return t, portainer.APIKey{}, false
return portainer.User{}, portainer.APIKey{}, false
}
tuple := val.(entry[T])
tuple := val.(entry)
return tuple.user, tuple.apiKey, true
}
// Set persists a user/key entry to the cache
func (c *ApiKeyCache[T]) Set(digest string, user T, apiKey portainer.APIKey) {
c.cache.Add(digest, entry[T]{
func (c *apiKeyCache) Set(digest string, user portainer.User, apiKey portainer.APIKey) {
c.cache.Add(digest, entry{
user: user,
apiKey: apiKey,
})
}
// Delete evicts a digest's user/key entry key from the cache
func (c *ApiKeyCache[T]) Delete(digest string) {
func (c *apiKeyCache) Delete(digest string) {
c.cache.Remove(digest)
}
// InvalidateUserKeyCache loops through all the api-keys associated to a user and removes them from the cache
func (c *ApiKeyCache[T]) InvalidateUserKeyCache(userId portainer.UserID) bool {
func (c *apiKeyCache) InvalidateUserKeyCache(userId portainer.UserID) bool {
present := false
for _, k := range c.cache.Keys() {
user, _, _ := c.Get(k.(string))
if c.userCmpFn(user, userId) {
present = c.cache.Remove(k) || present
if user.ID == userId {
present = c.cache.Remove(k)
}
}
return present
}

View File

@@ -8,14 +8,13 @@ import (
)
func Test_apiKeyCacheGet(t *testing.T) {
t.Parallel()
is := assert.New(t)
keyCache := NewAPIKeyCache(10, compareUser)
keyCache := NewAPIKeyCache(10)
// pre-populate cache
keyCache.cache.Add(string("foo"), entry[portainer.User]{user: portainer.User{}, apiKey: portainer.APIKey{}})
keyCache.cache.Add(string(""), entry[portainer.User]{user: portainer.User{}, apiKey: portainer.APIKey{}})
keyCache.cache.Add(string("foo"), entry{user: portainer.User{}, apiKey: portainer.APIKey{}})
keyCache.cache.Add(string(""), entry{user: portainer.User{}, apiKey: portainer.APIKey{}})
tests := []struct {
digest string
@@ -36,7 +35,7 @@ func Test_apiKeyCacheGet(t *testing.T) {
}
for _, test := range tests {
t.Run(test.digest, func(t *testing.T) {
t.Run(string(test.digest), func(t *testing.T) {
_, _, found := keyCache.Get(test.digest)
is.Equal(test.found, found)
})
@@ -44,10 +43,9 @@ func Test_apiKeyCacheGet(t *testing.T) {
}
func Test_apiKeyCacheSet(t *testing.T) {
t.Parallel()
is := assert.New(t)
keyCache := NewAPIKeyCache(10, compareUser)
keyCache := NewAPIKeyCache(10)
// pre-populate cache
keyCache.Set("bar", portainer.User{ID: 2}, portainer.APIKey{})
@@ -59,24 +57,23 @@ func Test_apiKeyCacheSet(t *testing.T) {
val, ok := keyCache.cache.Get(string("bar"))
is.True(ok)
tuple := val.(entry[portainer.User])
tuple := val.(entry)
is.Equal(portainer.User{ID: 2}, tuple.user)
val, ok = keyCache.cache.Get(string("foo"))
is.True(ok)
tuple = val.(entry[portainer.User])
tuple = val.(entry)
is.Equal(portainer.User{ID: 3}, tuple.user)
}
func Test_apiKeyCacheDelete(t *testing.T) {
t.Parallel()
is := assert.New(t)
keyCache := NewAPIKeyCache(10, compareUser)
keyCache := NewAPIKeyCache(10)
t.Run("Delete an existing entry", func(t *testing.T) {
keyCache.cache.Add(string("foo"), entry[portainer.User]{user: portainer.User{ID: 1}, apiKey: portainer.APIKey{}})
keyCache.cache.Add(string("foo"), entry{user: portainer.User{ID: 1}, apiKey: portainer.APIKey{}})
keyCache.Delete("foo")
_, ok := keyCache.cache.Get(string("foo"))
@@ -90,7 +87,6 @@ func Test_apiKeyCacheDelete(t *testing.T) {
}
func Test_apiKeyCacheLRU(t *testing.T) {
t.Parallel()
is := assert.New(t)
tests := []struct {
@@ -132,7 +128,7 @@ func Test_apiKeyCacheLRU(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
keyCache := NewAPIKeyCache(test.cacheLen, compareUser)
keyCache := NewAPIKeyCache(test.cacheLen)
for _, key := range test.key {
keyCache.Set(key, portainer.User{ID: 1}, portainer.APIKey{})
@@ -152,13 +148,12 @@ func Test_apiKeyCacheLRU(t *testing.T) {
}
func Test_apiKeyCacheInvalidateUserKeyCache(t *testing.T) {
t.Parallel()
is := assert.New(t)
keyCache := NewAPIKeyCache(10, compareUser)
keyCache := NewAPIKeyCache(10)
t.Run("Removes users keys from cache", func(t *testing.T) {
keyCache.cache.Add(string("foo"), entry[portainer.User]{user: portainer.User{ID: 1}, apiKey: portainer.APIKey{}})
keyCache.cache.Add(string("foo"), entry{user: portainer.User{ID: 1}, apiKey: portainer.APIKey{}})
ok := keyCache.InvalidateUserKeyCache(1)
is.True(ok)
@@ -168,8 +163,8 @@ func Test_apiKeyCacheInvalidateUserKeyCache(t *testing.T) {
})
t.Run("Does not affect other keys", func(t *testing.T) {
keyCache.cache.Add(string("foo"), entry[portainer.User]{user: portainer.User{ID: 1}, apiKey: portainer.APIKey{}})
keyCache.cache.Add(string("bar"), entry[portainer.User]{user: portainer.User{ID: 2}, apiKey: portainer.APIKey{}})
keyCache.cache.Add(string("foo"), entry{user: portainer.User{ID: 1}, apiKey: portainer.APIKey{}})
keyCache.cache.Add(string("bar"), entry{user: portainer.User{ID: 2}, apiKey: portainer.APIKey{}})
ok := keyCache.InvalidateUserKeyCache(1)
is.True(ok)

View File

@@ -1,15 +1,14 @@
package apikey
import (
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"fmt"
"io"
"time"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/internal/securecookie"
"github.com/pkg/errors"
)
@@ -21,45 +20,30 @@ var ErrInvalidAPIKey = errors.New("Invalid API key")
type apiKeyService struct {
apiKeyRepository dataservices.APIKeyRepository
userRepository dataservices.UserService
cache *ApiKeyCache[portainer.User]
}
// GenerateRandomKey generates a random key of specified length
// source: https://github.com/gorilla/securecookie/blob/master/securecookie.go#L515
func GenerateRandomKey(length int) []byte {
k := make([]byte, length)
if _, err := io.ReadFull(rand.Reader, k); err != nil {
return nil
}
return k
}
func compareUser(u portainer.User, id portainer.UserID) bool {
return u.ID == id
cache *apiKeyCache
}
func NewAPIKeyService(apiKeyRepository dataservices.APIKeyRepository, userRepository dataservices.UserService) *apiKeyService {
return &apiKeyService{
apiKeyRepository: apiKeyRepository,
userRepository: userRepository,
cache: NewAPIKeyCache(DefaultAPIKeyCacheSize, compareUser),
cache: NewAPIKeyCache(defaultAPIKeyCacheSize),
}
}
// HashRaw computes a hash digest of provided raw API key.
func (a *apiKeyService) HashRaw(rawKey string) string {
hashDigest := sha256.Sum256([]byte(rawKey))
return base64.StdEncoding.EncodeToString(hashDigest[:])
}
// GenerateApiKey generates a raw API key for a user (for one-time display).
// The generated API key is stored in the cache and database.
func (a *apiKeyService) GenerateApiKey(user portainer.User, description string) (string, *portainer.APIKey, error) {
randKey := GenerateRandomKey(32)
randKey := securecookie.GenerateRandomKey(32)
encodedRawAPIKey := base64.StdEncoding.EncodeToString(randKey)
prefixedAPIKey := portainerAPIKeyPrefix + encodedRawAPIKey
hashDigest := a.HashRaw(prefixedAPIKey)
apiKey := &portainer.APIKey{
@@ -70,7 +54,8 @@ func (a *apiKeyService) GenerateApiKey(user portainer.User, description string)
Digest: hashDigest,
}
if err := a.apiKeyRepository.Create(apiKey); err != nil {
err := a.apiKeyRepository.Create(apiKey)
if err != nil {
return "", nil, errors.Wrap(err, "Unable to create API key")
}
@@ -93,6 +78,7 @@ func (a *apiKeyService) GetAPIKeys(userID portainer.UserID) ([]portainer.APIKey,
// GetDigestUserAndKey returns the user and api-key associated to a specified hash digest.
// A cache lookup is performed first; if the user/api-key is not found in the cache, respective database lookups are performed.
func (a *apiKeyService) GetDigestUserAndKey(digest string) (portainer.User, portainer.APIKey, error) {
// get api key from cache if possible
cachedUser, cachedKey, ok := a.cache.Get(digest)
if ok {
return cachedUser, cachedKey, nil
@@ -120,21 +106,20 @@ func (a *apiKeyService) UpdateAPIKey(apiKey *portainer.APIKey) error {
if err != nil {
return errors.Wrap(err, "Unable to retrieve API key")
}
a.cache.Set(apiKey.Digest, user, *apiKey)
return a.apiKeyRepository.Update(apiKey.ID, apiKey)
}
// DeleteAPIKey deletes an API key and removes the digest/api-key entry from the cache.
func (a *apiKeyService) DeleteAPIKey(apiKeyID portainer.APIKeyID) error {
// get api-key digest to remove from cache
apiKey, err := a.apiKeyRepository.Read(apiKeyID)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("Unable to retrieve API key: %d", apiKeyID))
}
// delete the user/api-key from cache
a.cache.Delete(apiKey.Digest)
return a.apiKeyRepository.Delete(apiKeyID)
}

View File

@@ -10,20 +10,17 @@ import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/datastore"
"github.com/stretchr/testify/assert"
"github.com/rs/zerolog/log"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func Test_SatisfiesAPIKeyServiceInterface(t *testing.T) {
t.Parallel()
is := assert.New(t)
is.Implements((*APIKeyService)(nil), NewAPIKeyService(nil, nil))
}
func Test_GenerateApiKey(t *testing.T) {
t.Parallel()
is := assert.New(t)
_, store := datastore.MustNewTestStore(t, true, true)
@@ -33,7 +30,7 @@ func Test_GenerateApiKey(t *testing.T) {
t.Run("Successfully generates API key", func(t *testing.T) {
desc := "test-1"
rawKey, apiKey, err := service.GenerateApiKey(portainer.User{ID: 1}, desc)
require.NoError(t, err)
is.NoError(err)
is.NotEmpty(rawKey)
is.NotEmpty(apiKey)
is.Equal(desc, apiKey.Description)
@@ -41,7 +38,7 @@ func Test_GenerateApiKey(t *testing.T) {
t.Run("Api key prefix is 7 chars", func(t *testing.T) {
rawKey, apiKey, err := service.GenerateApiKey(portainer.User{ID: 1}, "test-2")
require.NoError(t, err)
is.NoError(err)
is.Equal(rawKey[:7], apiKey.Prefix)
is.Len(apiKey.Prefix, 7)
@@ -49,7 +46,7 @@ func Test_GenerateApiKey(t *testing.T) {
t.Run("Api key has 'ptr_' as prefix", func(t *testing.T) {
rawKey, _, err := service.GenerateApiKey(portainer.User{ID: 1}, "test-x")
require.NoError(t, err)
is.NoError(err)
is.Equal(portainerAPIKeyPrefix, "ptr_")
is.True(strings.HasPrefix(rawKey, "ptr_"))
@@ -58,7 +55,7 @@ func Test_GenerateApiKey(t *testing.T) {
t.Run("Successfully caches API key", func(t *testing.T) {
user := portainer.User{ID: 1}
_, apiKey, err := service.GenerateApiKey(user, "test-3")
require.NoError(t, err)
is.NoError(err)
userFromCache, apiKeyFromCache, ok := service.cache.Get(apiKey.Digest)
is.True(ok)
@@ -68,7 +65,7 @@ func Test_GenerateApiKey(t *testing.T) {
t.Run("Decoded raw api-key digest matches generated digest", func(t *testing.T) {
rawKey, apiKey, err := service.GenerateApiKey(portainer.User{ID: 1}, "test-4")
require.NoError(t, err)
is.NoError(err)
generatedDigest := sha256.Sum256([]byte(rawKey))
@@ -77,7 +74,6 @@ func Test_GenerateApiKey(t *testing.T) {
}
func Test_GetAPIKey(t *testing.T) {
t.Parallel()
is := assert.New(t)
_, store := datastore.MustNewTestStore(t, true, true)
@@ -87,17 +83,16 @@ func Test_GetAPIKey(t *testing.T) {
t.Run("Successfully returns all API keys", func(t *testing.T) {
user := portainer.User{ID: 1}
_, apiKey, err := service.GenerateApiKey(user, "test-1")
require.NoError(t, err)
is.NoError(err)
apiKeyGot, err := service.GetAPIKey(apiKey.ID)
require.NoError(t, err)
is.NoError(err)
is.Equal(apiKey, apiKeyGot)
})
}
func Test_GetAPIKeys(t *testing.T) {
t.Parallel()
is := assert.New(t)
_, store := datastore.MustNewTestStore(t, true, true)
@@ -107,18 +102,17 @@ func Test_GetAPIKeys(t *testing.T) {
t.Run("Successfully returns all API keys", func(t *testing.T) {
user := portainer.User{ID: 1}
_, _, err := service.GenerateApiKey(user, "test-1")
require.NoError(t, err)
is.NoError(err)
_, _, err = service.GenerateApiKey(user, "test-2")
require.NoError(t, err)
is.NoError(err)
keys, err := service.GetAPIKeys(user.ID)
require.NoError(t, err)
is.NoError(err)
is.Len(keys, 2)
})
}
func Test_GetDigestUserAndKey(t *testing.T) {
t.Parallel()
is := assert.New(t)
_, store := datastore.MustNewTestStore(t, true, true)
@@ -128,10 +122,10 @@ func Test_GetDigestUserAndKey(t *testing.T) {
t.Run("Successfully returns user and api key associated to digest", func(t *testing.T) {
user := portainer.User{ID: 1}
_, apiKey, err := service.GenerateApiKey(user, "test-1")
require.NoError(t, err)
is.NoError(err)
userGot, apiKeyGot, err := service.GetDigestUserAndKey(apiKey.Digest)
require.NoError(t, err)
is.NoError(err)
is.Equal(user, userGot)
is.Equal(*apiKey, apiKeyGot)
})
@@ -139,10 +133,10 @@ func Test_GetDigestUserAndKey(t *testing.T) {
t.Run("Successfully caches user and api key associated to digest", func(t *testing.T) {
user := portainer.User{ID: 1}
_, apiKey, err := service.GenerateApiKey(user, "test-1")
require.NoError(t, err)
is.NoError(err)
userGot, apiKeyGot, err := service.GetDigestUserAndKey(apiKey.Digest)
require.NoError(t, err)
is.NoError(err)
is.Equal(user, userGot)
is.Equal(*apiKey, apiKeyGot)
@@ -154,7 +148,6 @@ func Test_GetDigestUserAndKey(t *testing.T) {
}
func Test_UpdateAPIKey(t *testing.T) {
t.Parallel()
is := assert.New(t)
_, store := datastore.MustNewTestStore(t, true, true)
@@ -163,19 +156,16 @@ func Test_UpdateAPIKey(t *testing.T) {
t.Run("Successfully updates the api-key LastUsed time", func(t *testing.T) {
user := portainer.User{ID: 1}
err := store.User().Create(&user)
require.NoError(t, err)
store.User().Create(&user)
_, apiKey, err := service.GenerateApiKey(user, "test-x")
require.NoError(t, err)
is.NoError(err)
apiKey.LastUsed = time.Now().UTC().Unix()
err = service.UpdateAPIKey(apiKey)
require.NoError(t, err)
is.NoError(err)
_, apiKeyGot, err := service.GetDigestUserAndKey(apiKey.Digest)
require.NoError(t, err)
is.NoError(err)
log.Debug().Str("wanted", fmt.Sprintf("%+v", apiKey)).Str("got", fmt.Sprintf("%+v", apiKeyGot)).Msg("")
@@ -184,7 +174,7 @@ func Test_UpdateAPIKey(t *testing.T) {
t.Run("Successfully updates api-key in cache upon api-key update", func(t *testing.T) {
_, apiKey, err := service.GenerateApiKey(portainer.User{ID: 1}, "test-x2")
require.NoError(t, err)
is.NoError(err)
_, apiKeyFromCache, ok := service.cache.Get(apiKey.Digest)
is.True(ok)
@@ -194,7 +184,7 @@ func Test_UpdateAPIKey(t *testing.T) {
is.NotEqual(*apiKey, apiKeyFromCache)
err = service.UpdateAPIKey(apiKey)
require.NoError(t, err)
is.NoError(err)
_, updatedAPIKeyFromCache, ok := service.cache.Get(apiKey.Digest)
is.True(ok)
@@ -203,7 +193,6 @@ func Test_UpdateAPIKey(t *testing.T) {
}
func Test_DeleteAPIKey(t *testing.T) {
t.Parallel()
is := assert.New(t)
_, store := datastore.MustNewTestStore(t, true, true)
@@ -213,30 +202,30 @@ func Test_DeleteAPIKey(t *testing.T) {
t.Run("Successfully updates the api-key", func(t *testing.T) {
user := portainer.User{ID: 1}
_, apiKey, err := service.GenerateApiKey(user, "test-1")
require.NoError(t, err)
is.NoError(err)
_, apiKeyGot, err := service.GetDigestUserAndKey(apiKey.Digest)
require.NoError(t, err)
is.NoError(err)
is.Equal(*apiKey, apiKeyGot)
err = service.DeleteAPIKey(apiKey.ID)
require.NoError(t, err)
is.NoError(err)
_, _, err = service.GetDigestUserAndKey(apiKey.Digest)
require.Error(t, err)
is.Error(err)
})
t.Run("Successfully removes api-key from cache upon deletion", func(t *testing.T) {
user := portainer.User{ID: 1}
_, apiKey, err := service.GenerateApiKey(user, "test-1")
require.NoError(t, err)
is.NoError(err)
_, apiKeyFromCache, ok := service.cache.Get(apiKey.Digest)
is.True(ok)
is.Equal(*apiKey, apiKeyFromCache)
err = service.DeleteAPIKey(apiKey.ID)
require.NoError(t, err)
is.NoError(err)
_, _, ok = service.cache.Get(apiKey.Digest)
is.False(ok)
@@ -244,7 +233,6 @@ func Test_DeleteAPIKey(t *testing.T) {
}
func Test_InvalidateUserKeyCache(t *testing.T) {
t.Parallel()
is := assert.New(t)
_, store := datastore.MustNewTestStore(t, true, true)
@@ -255,10 +243,10 @@ func Test_InvalidateUserKeyCache(t *testing.T) {
// generate api keys
user := portainer.User{ID: 1}
_, apiKey1, err := service.GenerateApiKey(user, "test-1")
require.NoError(t, err)
is.NoError(err)
_, apiKey2, err := service.GenerateApiKey(user, "test-2")
require.NoError(t, err)
is.NoError(err)
// verify api keys are present in cache
_, apiKeyFromCache, ok := service.cache.Get(apiKey1.Digest)
@@ -285,11 +273,11 @@ func Test_InvalidateUserKeyCache(t *testing.T) {
// generate keys for 2 users
user1 := portainer.User{ID: 1}
_, apiKey1, err := service.GenerateApiKey(user1, "test-1")
require.NoError(t, err)
is.NoError(err)
user2 := portainer.User{ID: 2}
_, apiKey2, err := service.GenerateApiKey(user2, "test-2")
require.NoError(t, err)
is.NoError(err)
// verify keys in cache
_, apiKeyFromCache, ok := service.cache.Get(apiKey1.Digest)

View File

@@ -17,15 +17,18 @@ func TarFileInBuffer(fileContent []byte, fileName string, mode int64) ([]byte, e
Size: int64(len(fileContent)),
}
if err := tarWriter.WriteHeader(header); err != nil {
err := tarWriter.WriteHeader(header)
if err != nil {
return nil, err
}
if _, err := tarWriter.Write(fileContent); err != nil {
_, err = tarWriter.Write(fileContent)
if err != nil {
return nil, err
}
if err := tarWriter.Close(); err != nil {
err = tarWriter.Close()
if err != nil {
return nil, err
}
@@ -40,7 +43,10 @@ type tarFileInBuffer struct {
func NewTarFileInBuffer() *tarFileInBuffer {
var b bytes.Buffer
return &tarFileInBuffer{b: &b, w: tar.NewWriter(&b)}
return &tarFileInBuffer{
b: &b,
w: tar.NewWriter(&b),
}
}
// Put puts a single file to tar archive buffer.
@@ -55,9 +61,11 @@ func (t *tarFileInBuffer) Put(fileContent []byte, fileName string, mode int64) e
return err
}
_, err := t.w.Write(fileContent)
if _, err := t.w.Write(fileContent); err != nil {
return err
}
return err
return nil
}
// Bytes returns the archive as a byte array.

View File

@@ -9,27 +9,23 @@ import (
"os"
"path/filepath"
"strings"
"github.com/portainer/portainer/api/filesystem"
"github.com/portainer/portainer/api/logs"
)
// TarGzDir creates a tar.gz archive and returns it's path.
// abosolutePath should be an absolute path to a directory.
// Archive name will be <directoryName>.tar.gz and will be placed next to the directory.
func TarGzDir(absolutePath string) (string, error) {
targzPath := filepath.Join(absolutePath, filepath.Base(absolutePath)+".tar.gz")
targzPath := filepath.Join(absolutePath, fmt.Sprintf("%s.tar.gz", filepath.Base(absolutePath)))
outFile, err := os.Create(targzPath)
if err != nil {
return "", err
}
defer logs.CloseAndLogErr(outFile)
defer outFile.Close()
zipWriter := gzip.NewWriter(outFile)
defer logs.CloseAndLogErr(zipWriter)
defer zipWriter.Close()
tarWriter := tar.NewWriter(zipWriter)
defer logs.CloseAndLogErr(tarWriter)
defer tarWriter.Close()
err = filepath.Walk(absolutePath, func(path string, info os.FileInfo, err error) error {
if err != nil {
@@ -90,7 +86,7 @@ func ExtractTarGz(r io.Reader, outputDirPath string) error {
if err != nil {
return err
}
defer logs.CloseAndLogErr(zipReader)
defer zipReader.Close()
tarReader := tar.NewReader(zipReader)
@@ -109,7 +105,7 @@ func ExtractTarGz(r io.Reader, outputDirPath string) error {
case tar.TypeDir:
// skip, dir will be created with a file
case tar.TypeReg:
p := filesystem.JoinPaths(outputDirPath, header.Name)
p := filepath.Clean(filepath.Join(outputDirPath, header.Name))
if err := os.MkdirAll(filepath.Dir(p), 0o744); err != nil {
return fmt.Errorf("Failed to extract dir %s", filepath.Dir(p))
}
@@ -120,7 +116,7 @@ func ExtractTarGz(r io.Reader, outputDirPath string) error {
if _, err := io.Copy(outFile, tarReader); err != nil {
return fmt.Errorf("Failed to extract file %s", header.Name)
}
logs.CloseAndLogErr(outFile)
outFile.Close()
default:
return fmt.Errorf("tar: unknown type: %v in %s",
header.Typeflag,

View File

@@ -1,108 +1,51 @@
package archive
import (
"archive/tar"
"compress/gzip"
"fmt"
"os"
"os/exec"
"path"
"path/filepath"
"testing"
"github.com/portainer/portainer/api/filesystem"
"github.com/rs/zerolog/log"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func listFiles(dir string) []string {
items := make([]string, 0)
if err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if path == dir {
return nil
}
items = append(items, path)
return nil
}); err != nil {
log.Warn().Err(err).Msg("failed to list files in directory")
}
})
return items
}
func Test_shouldCreateArchive(t *testing.T) {
t.Parallel()
func Test_shouldCreateArhive(t *testing.T) {
tmpdir := t.TempDir()
content := []byte("content")
err := os.WriteFile(filesystem.JoinPaths(tmpdir, "outer"), content, 0600)
require.NoError(t, err)
err = os.MkdirAll(filesystem.JoinPaths(tmpdir, "dir"), 0700)
require.NoError(t, err)
err = os.WriteFile(filesystem.JoinPaths(tmpdir, "dir", ".dotfile"), content, 0600)
require.NoError(t, err)
err = os.WriteFile(filesystem.JoinPaths(tmpdir, "dir", "inner"), content, 0600)
require.NoError(t, err)
os.WriteFile(path.Join(tmpdir, "outer"), content, 0600)
os.MkdirAll(path.Join(tmpdir, "dir"), 0700)
os.WriteFile(path.Join(tmpdir, "dir", ".dotfile"), content, 0600)
os.WriteFile(path.Join(tmpdir, "dir", "inner"), content, 0600)
gzPath, err := TarGzDir(tmpdir)
require.NoError(t, err)
assert.Equal(t, filesystem.JoinPaths(tmpdir, filepath.Base(tmpdir)+".tar.gz"), gzPath)
assert.Nil(t, err)
assert.Equal(t, filepath.Join(tmpdir, fmt.Sprintf("%s.tar.gz", filepath.Base(tmpdir))), gzPath)
extractionDir := t.TempDir()
cmd := exec.Command("tar", "-xzf", gzPath, "-C", extractionDir)
if err := cmd.Run(); err != nil {
err = cmd.Run()
if err != nil {
t.Fatal("Failed to extract archive: ", err)
}
extractedFiles := listFiles(extractionDir)
wasExtracted := func(p string) {
fullpath := filesystem.JoinPaths(extractionDir, p)
assert.Contains(t, extractedFiles, fullpath)
copyContent, err := os.ReadFile(fullpath)
require.NoError(t, err)
assert.Equal(t, content, copyContent)
}
wasExtracted("outer")
wasExtracted("dir/inner")
wasExtracted("dir/.dotfile")
}
func Test_shouldCreateArchive2(t *testing.T) {
t.Parallel()
tmpdir := t.TempDir()
content := []byte("content")
err := os.WriteFile(filesystem.JoinPaths(tmpdir, "outer"), content, 0600)
require.NoError(t, err)
err = os.MkdirAll(filesystem.JoinPaths(tmpdir, "dir"), 0700)
require.NoError(t, err)
err = os.WriteFile(filesystem.JoinPaths(tmpdir, "dir", ".dotfile"), content, 0600)
require.NoError(t, err)
err = os.WriteFile(filesystem.JoinPaths(tmpdir, "dir", "inner"), content, 0600)
require.NoError(t, err)
gzPath, err := TarGzDir(tmpdir)
require.NoError(t, err)
assert.Equal(t, filesystem.JoinPaths(tmpdir, filepath.Base(tmpdir)+".tar.gz"), gzPath)
extractionDir := t.TempDir()
r, _ := os.Open(gzPath)
if err := ExtractTarGz(r, extractionDir); err != nil {
t.Fatal("Failed to extract archive: ", err)
}
extractedFiles := listFiles(extractionDir)
wasExtracted := func(p string) {
fullpath := filesystem.JoinPaths(extractionDir, p)
fullpath := path.Join(extractionDir, p)
assert.Contains(t, extractedFiles, fullpath)
copyContent, _ := os.ReadFile(fullpath)
assert.Equal(t, content, copyContent)
@@ -113,56 +56,34 @@ func Test_shouldCreateArchive2(t *testing.T) {
wasExtracted("dir/.dotfile")
}
func TestExtractTarGzPathTraversal(t *testing.T) {
t.Parallel()
testDir := t.TempDir()
func Test_shouldCreateArhiveXXXXX(t *testing.T) {
tmpdir := t.TempDir()
content := []byte("content")
os.WriteFile(path.Join(tmpdir, "outer"), content, 0600)
os.MkdirAll(path.Join(tmpdir, "dir"), 0700)
os.WriteFile(path.Join(tmpdir, "dir", ".dotfile"), content, 0600)
os.WriteFile(path.Join(tmpdir, "dir", "inner"), content, 0600)
// Create an evil file with a path traversal attempt
tarPath := filesystem.JoinPaths(testDir, "evil.tar.gz")
gzPath, err := TarGzDir(tmpdir)
assert.Nil(t, err)
assert.Equal(t, filepath.Join(tmpdir, fmt.Sprintf("%s.tar.gz", filepath.Base(tmpdir))), gzPath)
evilFile, err := os.Create(tarPath)
require.NoError(t, err)
extractionDir := t.TempDir()
r, _ := os.Open(gzPath)
ExtractTarGz(r, extractionDir)
if err != nil {
t.Fatal("Failed to extract archive: ", err)
}
extractedFiles := listFiles(extractionDir)
gzWriter := gzip.NewWriter(evilFile)
tarWriter := tar.NewWriter(gzWriter)
content := []byte("evil content")
header := &tar.Header{
Name: "../evil.txt",
Mode: 0600,
Size: int64(len(content)),
Typeflag: tar.TypeReg,
wasExtracted := func(p string) {
fullpath := path.Join(extractionDir, p)
assert.Contains(t, extractedFiles, fullpath)
copyContent, _ := os.ReadFile(fullpath)
assert.Equal(t, content, copyContent)
}
err = tarWriter.WriteHeader(header)
require.NoError(t, err)
_, err = tarWriter.Write(content)
require.NoError(t, err)
err = tarWriter.Close()
require.NoError(t, err)
err = gzWriter.Close()
require.NoError(t, err)
err = evilFile.Close()
require.NoError(t, err)
// Attempt to extract the evil file
extractionDir := filesystem.JoinPaths(testDir, "extraction")
err = os.Mkdir(extractionDir, 0700)
require.NoError(t, err)
tarFile, err := os.Open(tarPath)
require.NoError(t, err)
// Check that the file didn't escape
err = ExtractTarGz(tarFile, extractionDir)
require.NoError(t, err)
require.NoFileExists(t, filesystem.JoinPaths(testDir, "evil.txt"))
err = tarFile.Close()
require.NoError(t, err)
wasExtracted("outer")
wasExtracted("dir/inner")
wasExtracted("dir/.dotfile")
}

View File

@@ -2,17 +2,60 @@ package archive
import (
"archive/zip"
"bytes"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"github.com/portainer/portainer/api/logs"
"github.com/pkg/errors"
)
// UnzipArchive will unzip an archive from bytes into the dest destination folder on disk
func UnzipArchive(archiveData []byte, dest string) error {
zipReader, err := zip.NewReader(bytes.NewReader(archiveData), int64(len(archiveData)))
if err != nil {
return err
}
for _, zipFile := range zipReader.File {
err := extractFileFromArchive(zipFile, dest)
if err != nil {
return err
}
}
return nil
}
func extractFileFromArchive(file *zip.File, dest string) error {
f, err := file.Open()
if err != nil {
return err
}
defer f.Close()
data, err := io.ReadAll(f)
if err != nil {
return err
}
fpath := filepath.Join(dest, file.Name)
outFile, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode())
if err != nil {
return err
}
_, err = io.Copy(outFile, bytes.NewReader(data))
if err != nil {
return err
}
return outFile.Close()
}
// UnzipFile will decompress a zip archive, moving all files and folders
// within the zip file (parameter 1) to an output directory (parameter 2).
func UnzipFile(src string, dest string) error {
@@ -20,7 +63,7 @@ func UnzipFile(src string, dest string) error {
if err != nil {
return err
}
defer logs.CloseAndLogErr(r)
defer r.Close()
for _, f := range r.File {
p := filepath.Join(dest, f.Name)
@@ -32,14 +75,12 @@ func UnzipFile(src string, dest string) error {
if f.FileInfo().IsDir() {
// Make Folder
if err := os.MkdirAll(p, os.ModePerm); err != nil {
return err
}
os.MkdirAll(p, os.ModePerm)
continue
}
if err := unzipFile(f, p); err != nil {
err = unzipFile(f, p)
if err != nil {
return err
}
}
@@ -52,20 +93,20 @@ func unzipFile(f *zip.File, p string) error {
if err := os.MkdirAll(filepath.Dir(p), os.ModePerm); err != nil {
return errors.Wrapf(err, "unzipFile: can't make a path %s", p)
}
outFile, err := os.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
if err != nil {
return errors.Wrapf(err, "unzipFile: can't create file %s", p)
}
defer logs.CloseAndLogErr(outFile)
defer outFile.Close()
rc, err := f.Open()
if err != nil {
return errors.Wrapf(err, "unzipFile: can't open zip file %s in the archive", f.Name)
}
defer logs.CloseAndLogErr(rc)
defer rc.Close()
if _, err = io.Copy(outFile, rc); err != nil {
_, err = io.Copy(outFile, rc)
if err != nil {
return errors.Wrapf(err, "unzipFile: can't copy an archived file content")
}

View File

@@ -1,16 +1,13 @@
package archive
import (
"path/filepath"
"testing"
"github.com/portainer/portainer/api/filesystem"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestUnzipFile(t *testing.T) {
t.Parallel()
dir := t.TempDir()
/*
Archive structure.
@@ -23,10 +20,10 @@ func TestUnzipFile(t *testing.T) {
err := UnzipFile("./testdata/sample_archive.zip", dir)
require.NoError(t, err)
assert.NoError(t, err)
archiveDir := dir + "/sample_archive"
assert.FileExists(t, filesystem.JoinPaths(archiveDir, "0.txt"))
assert.FileExists(t, filesystem.JoinPaths(archiveDir, "0", "1.txt"))
assert.FileExists(t, filesystem.JoinPaths(archiveDir, "0", "1", "2.txt"))
assert.FileExists(t, filepath.Join(archiveDir, "0.txt"))
assert.FileExists(t, filepath.Join(archiveDir, "0", "1.txt"))
assert.FileExists(t, filepath.Join(archiveDir, "0", "1", "2.txt"))
}

View File

@@ -3,19 +3,19 @@ package ecr
import (
"context"
"encoding/base64"
"errors"
"fmt"
"strings"
"time"
)
func (s *Service) GetEncodedAuthorizationToken(ctx context.Context) (token *string, expiry *time.Time, err error) {
getAuthorizationTokenOutput, err := s.client.GetAuthorizationToken(ctx, nil)
func (s *Service) GetEncodedAuthorizationToken() (token *string, expiry *time.Time, err error) {
getAuthorizationTokenOutput, err := s.client.GetAuthorizationToken(context.TODO(), nil)
if err != nil {
return
}
if len(getAuthorizationTokenOutput.AuthorizationData) == 0 {
err = errors.New("AuthorizationData is empty")
err = fmt.Errorf("AuthorizationData is empty")
return
}
@@ -27,8 +27,8 @@ func (s *Service) GetEncodedAuthorizationToken(ctx context.Context) (token *stri
return
}
func (s *Service) GetAuthorizationToken(ctx context.Context) (token *string, expiry *time.Time, err error) {
tokenEncodedStr, expiry, err := s.GetEncodedAuthorizationToken(ctx)
func (s *Service) GetAuthorizationToken() (token *string, expiry *time.Time, err error) {
tokenEncodedStr, expiry, err := s.GetEncodedAuthorizationToken()
if err != nil {
return
}
@@ -50,7 +50,7 @@ func (s *Service) ParseAuthorizationToken(token string) (username string, passwo
splitToken := strings.Split(token, ":")
if len(splitToken) < 2 {
err = errors.New("invalid ECR authorization token")
err = fmt.Errorf("invalid ECR authorization token")
return
}

View File

@@ -6,15 +6,6 @@ import (
"github.com/aws/aws-sdk-go-v2/service/ecr"
)
// Registry represents an ECR registry endpoint information.
// This struct is used to parse and validate ECR endpoint URLs.
type Registry struct {
ID string // AWS account ID (empty for accountless endpoints like "ecr-fips.us-west-1.amazonaws.com")
FIPS bool // Whether this is a FIPS endpoint (contains "-fips" in the URL)
Region string // AWS region (e.g., "us-east-1", "us-gov-west-1")
Public bool // Whether this is ecr-public.aws.com
}
type (
Service struct {
accessKey string

View File

@@ -1,70 +0,0 @@
package ecr
import (
"fmt"
"net/url"
"regexp"
"strings"
)
// ecrEndpointPattern matches all valid ECR endpoints including account-prefixed and accountless formats.
// Based on AWS ECR credential helper regex but extended to support accountless endpoints.
//
// Supported formats:
// - Account-prefixed: 123456789012.dkr.ecr-fips.us-east-1.amazonaws.com
// - Account-prefixed (hyphen): 123456789012.dkr-ecr-fips.us-west-1.on.aws
// - Accountless service: ecr-fips.us-west-1.amazonaws.com
// - Accountless API: ecr-fips.us-east-1.api.aws
// - Non-FIPS variants: All formats above without "-fips"
//
// Regex groups:
// - Group 1: Full account prefix (optional) - e.g., "123456789012.dkr." or "123456789012.dkr-"
// - Group 2: Account ID (optional) - e.g., "123456789012"
// - Group 3: FIPS flag (optional) - either "-fips" or empty string
// - Group 4: Region - e.g., "us-east-1", "us-gov-west-1"
// - Group 5: Domain suffix - e.g., "amazonaws.com", "api.aws"
var ecrEndpointPattern = regexp.MustCompile(
`^((\d{12})\.dkr[\.\-])?ecr(\-fips)?\.([a-zA-Z0-9][a-zA-Z0-9-_]*)\.(amazonaws\.(?:com(?:\.cn)?|eu)|api\.aws|on\.(?:aws|amazonwebservices\.com\.cn)|sc2s\.sgov\.gov|c2s\.ic\.gov|cloud\.adc-e\.uk|csp\.hci\.ic\.gov)$`,
)
// ParseECREndpoint parses an ECR registry URL and extracts registry information.
// This function replaces the AWS ECR credential helper library's ExtractRegistry function,
// which only supports account-prefixed endpoints.
//
// Reference: https://docs.aws.amazon.com/general/latest/gr/ecr.html
func ParseECREndpoint(urlStr string) (*Registry, error) {
// Normalize URL by adding https:// prefix if not present
if !strings.HasPrefix(urlStr, "https://") && !strings.HasPrefix(urlStr, "http://") {
urlStr = "https://" + urlStr
}
u, err := url.Parse(urlStr)
if err != nil {
return nil, fmt.Errorf("invalid URL: %w", err)
}
hostname := u.Hostname()
// Special case: ECR Public
// ECR Public uses a different domain and doesn't have FIPS variant
if hostname == "ecr-public.aws.com" {
return &Registry{
FIPS: false,
Public: true,
}, nil
}
// Parse standard ECR endpoints using regex
matches := ecrEndpointPattern.FindStringSubmatch(hostname)
if len(matches) == 0 {
return nil, fmt.Errorf("not a valid ECR endpoint: %s", hostname)
}
return &Registry{
ID: matches[2], // Account ID (may be empty for accountless endpoints)
FIPS: matches[3] == "-fips", // Check if "-fips" is present
Region: matches[4], // AWS region
Public: false,
}, nil
}

View File

@@ -1,254 +0,0 @@
package ecr
import (
"testing"
)
func TestParseECREndpoint(t *testing.T) {
t.Parallel()
tests := []struct {
name string
url string
want *Registry
wantError bool
}{
// Standard AWS Commercial - Account-prefixed FIPS
{
name: "account-prefixed FIPS us-east-1",
url: "123456789012.dkr.ecr-fips.us-east-1.amazonaws.com",
want: &Registry{
ID: "123456789012",
FIPS: true,
Region: "us-east-1",
Public: false,
},
},
{
name: "account-prefixed FIPS us-west-2",
url: "123456789012.dkr.ecr-fips.us-west-2.amazonaws.com",
want: &Registry{
ID: "123456789012",
FIPS: true,
Region: "us-west-2",
Public: false,
},
},
// Accountless FIPS service endpoints
{
name: "accountless FIPS us-west-1",
url: "ecr-fips.us-west-1.amazonaws.com",
want: &Registry{
ID: "",
FIPS: true,
Region: "us-west-1",
Public: false,
},
},
{
name: "accountless FIPS us-east-2",
url: "ecr-fips.us-east-2.amazonaws.com",
want: &Registry{
ID: "",
FIPS: true,
Region: "us-east-2",
Public: false,
},
},
// Accountless FIPS API endpoints
{
name: "accountless FIPS API us-west-1",
url: "ecr-fips.us-west-1.api.aws",
want: &Registry{
ID: "",
FIPS: true,
Region: "us-west-1",
Public: false,
},
},
{
name: "accountless FIPS API us-east-1",
url: "ecr-fips.us-east-1.api.aws",
want: &Registry{
ID: "",
FIPS: true,
Region: "us-east-1",
Public: false,
},
},
// on.aws domain with hyphen separator
{
name: "account-prefixed FIPS hyphen us-west-1",
url: "123456789012.dkr-ecr-fips.us-west-1.on.aws",
want: &Registry{
ID: "123456789012",
FIPS: true,
Region: "us-west-1",
Public: false,
},
},
{
name: "account-prefixed FIPS hyphen us-east-2",
url: "123456789012.dkr-ecr-fips.us-east-2.on.aws",
want: &Registry{
ID: "123456789012",
FIPS: true,
Region: "us-east-2",
Public: false,
},
},
// AWS GovCloud
{
name: "account-prefixed FIPS us-gov-east-1",
url: "123456789012.dkr.ecr-fips.us-gov-east-1.amazonaws.com",
want: &Registry{
ID: "123456789012",
FIPS: true,
Region: "us-gov-east-1",
Public: false,
},
},
{
name: "account-prefixed FIPS us-gov-west-1",
url: "123456789012.dkr.ecr-fips.us-gov-west-1.amazonaws.com",
want: &Registry{
ID: "123456789012",
FIPS: true,
Region: "us-gov-west-1",
Public: false,
},
},
{
name: "accountless FIPS us-gov-west-1",
url: "ecr-fips.us-gov-west-1.amazonaws.com",
want: &Registry{
ID: "",
FIPS: true,
Region: "us-gov-west-1",
Public: false,
},
},
{
name: "accountless FIPS API us-gov-east-1",
url: "ecr-fips.us-gov-east-1.api.aws",
want: &Registry{
ID: "",
FIPS: true,
Region: "us-gov-east-1",
Public: false,
},
},
// ECR Public
{
name: "ecr-public",
url: "ecr-public.aws.com",
want: &Registry{
ID: "",
FIPS: false,
Region: "",
Public: true,
},
},
// Non-FIPS endpoints (valid ECR but FIPS=false)
{
name: "account-prefixed non-FIPS us-east-1",
url: "123456789012.dkr.ecr.us-east-1.amazonaws.com",
want: &Registry{
ID: "123456789012",
FIPS: false,
Region: "us-east-1",
Public: false,
},
},
{
name: "accountless non-FIPS us-west-1",
url: "ecr.us-west-1.amazonaws.com",
want: &Registry{
ID: "",
FIPS: false,
Region: "us-west-1",
Public: false,
},
},
{
name: "accountless non-FIPS API us-east-2",
url: "ecr.us-east-2.api.aws",
want: &Registry{
ID: "",
FIPS: false,
Region: "us-east-2",
Public: false,
},
},
// URLs with https:// prefix
{
name: "with https prefix",
url: "https://ecr-fips.us-west-1.amazonaws.com",
want: &Registry{
ID: "",
FIPS: true,
Region: "us-west-1",
Public: false,
},
},
// Invalid endpoints
{
name: "not an ECR URL",
url: "not-an-ecr-url.com",
wantError: true,
},
{
name: "invalid account ID length",
url: "123.dkr.ecr-fips.us-east-1.amazonaws.com",
wantError: true,
},
{
name: "empty string",
url: "",
wantError: true,
},
{
name: "docker hub",
url: "docker.io",
wantError: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := ParseECREndpoint(tt.url)
if tt.wantError {
if err == nil {
t.Errorf("ParseECREndpoint() expected error but got none")
}
return
}
if err != nil {
t.Errorf("ParseECREndpoint() unexpected error: %v", err)
return
}
if got.ID != tt.want.ID {
t.Errorf("ParseECREndpoint() ID = %v, want %v", got.ID, tt.want.ID)
}
if got.FIPS != tt.want.FIPS {
t.Errorf("ParseECREndpoint() FIPS = %v, want %v", got.FIPS, tt.want.FIPS)
}
if got.Region != tt.want.Region {
t.Errorf("ParseECREndpoint() Region = %v, want %v", got.Region, tt.want.Region)
}
if got.Public != tt.want.Public {
t.Errorf("ParseECREndpoint() Public = %v, want %v", got.Public, tt.want.Public)
}
})
}
}

View File

@@ -12,7 +12,6 @@ import (
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/filesystem"
"github.com/portainer/portainer/api/http/offlinegate"
"github.com/portainer/portainer/api/logs"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
@@ -22,7 +21,6 @@ const rwxr__r__ os.FileMode = 0o744
var filesToBackup = []string{
"certs",
"chisel",
"compose",
"config.json",
"custom_templates",
@@ -32,13 +30,40 @@ var filesToBackup = []string{
"portainer.key",
"portainer.pub",
"tls",
"chisel",
}
// Creates a tar.gz system archive and encrypts it if password is not empty. Returns a path to the archive file.
func CreateBackupArchive(password string, gate *offlinegate.OfflineGate, datastore dataservices.DataStore, filestorePath string) (string, error) {
backupDirPath, err := backupDatabaseAndFilesystem(gate, datastore, filestorePath)
if err != nil {
return "", err
unlock := gate.Lock()
defer unlock()
backupDirPath := filepath.Join(filestorePath, "backup", time.Now().Format("2006-01-02_15-04-05"))
if err := os.MkdirAll(backupDirPath, rwxr__r__); err != nil {
return "", errors.Wrap(err, "Failed to create backup dir")
}
{
// new export
exportFilename := path.Join(backupDirPath, fmt.Sprintf("export-%d.json", time.Now().Unix()))
err := datastore.Export(exportFilename)
if err != nil {
log.Error().Err(err).Str("filename", exportFilename).Msg("failed to export")
} else {
log.Debug().Str("filename", exportFilename).Msg("file exported")
}
}
if err := backupDb(backupDirPath, datastore); err != nil {
return "", errors.Wrap(err, "Failed to backup database")
}
for _, filename := range filesToBackup {
err := filesystem.CopyPath(filepath.Join(filestorePath, filename), backupDirPath)
if err != nil {
return "", errors.Wrap(err, "Failed to create backup file")
}
}
archivePath, err := archive.TarGzDir(backupDirPath)
@@ -56,40 +81,8 @@ func CreateBackupArchive(password string, gate *offlinegate.OfflineGate, datasto
return archivePath, nil
}
func backupDatabaseAndFilesystem(gate *offlinegate.OfflineGate, datastore dataservices.DataStore, filestorePath string) (string, error) {
unlock := gate.Lock()
defer unlock()
backupDirPath := filepath.Join(filestorePath, "backup", time.Now().Format("2006-01-02_15-04-05"))
if err := os.MkdirAll(backupDirPath, rwxr__r__); err != nil {
return "", errors.Wrap(err, "Failed to create backup dir")
}
// new export
exportFilename := path.Join(backupDirPath, fmt.Sprintf("export-%d.json", time.Now().Unix()))
if err := datastore.Export(exportFilename); err != nil {
log.Error().Err(err).Str("filename", exportFilename).Msg("failed to export")
} else {
log.Debug().Str("filename", exportFilename).Msg("file exported")
}
if err := backupDb(backupDirPath, datastore); err != nil {
return "", errors.Wrap(err, "Failed to backup database")
}
for _, filename := range filesToBackup {
if err := filesystem.CopyPath(filepath.Join(filestorePath, filename), backupDirPath); err != nil {
return "", errors.Wrap(err, "Failed to create backup file")
}
}
return backupDirPath, nil
}
func backupDb(backupDirPath string, datastore dataservices.DataStore) error {
dbFileName := datastore.Connection().GetDatabaseFileName()
_, err := datastore.Backup(filepath.Join(backupDirPath, dbFileName))
_, err := datastore.Backup(filepath.Join(backupDirPath, "portainer.db"))
return err
}
@@ -98,13 +91,15 @@ func encrypt(path string, passphrase string) (string, error) {
if err != nil {
return "", err
}
defer logs.CloseAndLogErr(in)
defer in.Close()
outFileName := path + ".encrypted"
outFileName := fmt.Sprintf("%s.encrypted", path)
out, err := os.Create(outFileName)
if err != nil {
return "", err
}
return outFileName, crypto.AesEncrypt(in, out, []byte(passphrase))
err = crypto.AesEncrypt(in, out, []byte(passphrase))
return outFileName, err
}

View File

@@ -1,274 +0,0 @@
package backup
import (
"bytes"
"context"
"io"
"os"
"path/filepath"
"testing"
"github.com/portainer/portainer/api/archive"
"github.com/portainer/portainer/api/crypto"
"github.com/portainer/portainer/api/datastore"
"github.com/portainer/portainer/api/filesystem"
"github.com/portainer/portainer/api/http/offlinegate"
"github.com/portainer/portainer/pkg/fips"
"github.com/stretchr/testify/require"
)
func init() {
fips.InitFIPS(false)
}
func TestGetRestoreSourcePath_DBAtRoot(t *testing.T) {
t.Parallel()
dir := t.TempDir()
err := os.WriteFile(filesystem.JoinPaths(dir, "portainer.db"), []byte("db"), 0o600)
require.NoError(t, err)
result, err := getRestoreSourcePath(dir)
require.NoError(t, err)
require.Equal(t, dir, result)
}
func TestGetRestoreSourcePath_EncryptedDBAtRoot(t *testing.T) {
t.Parallel()
dir := t.TempDir()
err := os.WriteFile(filesystem.JoinPaths(dir, "portainer.edb"), []byte("db"), 0o600)
require.NoError(t, err)
result, err := getRestoreSourcePath(dir)
require.NoError(t, err)
require.Equal(t, dir, result)
}
func TestGetRestoreSourcePath_DBInSubdirectory(t *testing.T) {
t.Parallel()
dir := t.TempDir()
sub := filesystem.JoinPaths(dir, "backup-2024-01-01")
err := os.Mkdir(sub, 0o700)
require.NoError(t, err)
err = os.WriteFile(filesystem.JoinPaths(sub, "portainer.db"), []byte("db"), 0o600)
require.NoError(t, err)
result, err := getRestoreSourcePath(dir)
require.NoError(t, err)
require.Equal(t, sub, result)
}
func TestGetRestoreSourcePath_NoDBFile(t *testing.T) {
t.Parallel()
dir := t.TempDir()
err := os.WriteFile(filesystem.JoinPaths(dir, "other.file"), []byte("data"), 0o600)
require.NoError(t, err)
result, err := getRestoreSourcePath(dir)
require.NoError(t, err)
require.Equal(t, dir, result)
}
func TestGetRestoreSourcePath_EmptyDir(t *testing.T) {
t.Parallel()
dir := t.TempDir()
result, err := getRestoreSourcePath(dir)
require.NoError(t, err)
require.Equal(t, dir, result)
}
func TestEncryptDecrypt_RoundTrip(t *testing.T) {
t.Parallel()
dir := t.TempDir()
plaintext := []byte("sensitive portainer backup data")
srcPath := filesystem.JoinPaths(dir, "archive.tar.gz")
err := os.WriteFile(srcPath, plaintext, 0o600)
require.NoError(t, err)
encryptedPath, err := encrypt(srcPath, "mysecretpassword")
require.NoError(t, err)
require.Equal(t, srcPath+".encrypted", encryptedPath)
encryptedData, err := os.ReadFile(encryptedPath)
require.NoError(t, err)
decryptedReader, err := crypto.AesDecrypt(bytes.NewReader(encryptedData), []byte("mysecretpassword"))
require.NoError(t, err)
decrypted, err := io.ReadAll(decryptedReader)
require.NoError(t, err)
require.Equal(t, plaintext, decrypted)
}
func TestEncryptDecrypt_WrongPassword(t *testing.T) {
t.Parallel()
dir := t.TempDir()
srcPath := filesystem.JoinPaths(dir, "archive.tar.gz")
err := os.WriteFile(srcPath, []byte("data"), 0o600)
require.NoError(t, err)
encryptedPath, err := encrypt(srcPath, "correctpassword")
require.NoError(t, err)
encryptedData, err := os.ReadFile(encryptedPath)
require.NoError(t, err)
_, err = crypto.AesDecrypt(bytes.NewReader(encryptedData), []byte("wrongpassword"))
require.Error(t, err)
}
func TestCreateBackupArchive_NoPassword(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, true, false)
storePath := store.GetConnection().GetStorePath()
gate := offlinegate.NewOfflineGate()
archivePath, err := CreateBackupArchive("", gate, store, storePath)
require.NoError(t, err)
f, err := os.Open(archivePath)
require.NoError(t, err)
t.Cleanup(func() {
err := f.Close()
require.NoError(t, err)
})
extractDir := t.TempDir()
err = archive.ExtractTarGz(f, extractDir)
require.NoError(t, err)
dbFound := false
err = filepath.Walk(extractDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.Name() == "portainer.db" {
dbFound = true
}
return nil
})
require.NoError(t, err)
require.True(t, dbFound, "archive should contain portainer.db")
}
func TestCreateBackupArchive_WithPassword(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, true, false)
storePath := store.GetConnection().GetStorePath()
gate := offlinegate.NewOfflineGate()
archivePath, err := CreateBackupArchive("backup-secret", gate, store, storePath)
require.NoError(t, err)
require.Contains(t, archivePath, ".encrypted")
encryptedData, err := os.ReadFile(archivePath)
require.NoError(t, err)
decryptedReader, err := crypto.AesDecrypt(bytes.NewReader(encryptedData), []byte("backup-secret"))
require.NoError(t, err)
extractDir := t.TempDir()
err = archive.ExtractTarGz(decryptedReader, extractDir)
require.NoError(t, err)
dbFound := false
err = filepath.Walk(extractDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.Name() == "portainer.db" {
dbFound = true
}
return nil
})
require.NoError(t, err)
require.True(t, dbFound, "decrypted archive should contain portainer.db")
}
func TestRestoreArchive_NoPassword(t *testing.T) {
t.Parallel()
_, store1 := datastore.MustNewTestStore(t, true, false)
storePath1 := store1.GetConnection().GetStorePath()
gate := offlinegate.NewOfflineGate()
archivePath, err := CreateBackupArchive("", gate, store1, storePath1)
require.NoError(t, err)
archiveData, err := os.ReadFile(archivePath)
require.NoError(t, err)
_, store2 := datastore.MustNewTestStore(t, true, false)
storePath2 := store2.GetConnection().GetStorePath()
ctx, cancel := context.WithCancel(t.Context())
err = RestoreArchive(bytes.NewReader(archiveData), "", storePath2, gate, store2, cancel)
require.NoError(t, err)
require.ErrorIs(t, ctx.Err(), context.Canceled)
_, err = os.Stat(filesystem.JoinPaths(storePath2, "portainer.db"))
require.NoError(t, err)
}
func TestRestoreArchive_WithPassword(t *testing.T) {
t.Parallel()
_, store1 := datastore.MustNewTestStore(t, true, false)
storePath1 := store1.GetConnection().GetStorePath()
gate := offlinegate.NewOfflineGate()
archivePath, err := CreateBackupArchive("restore-secret", gate, store1, storePath1)
require.NoError(t, err)
archiveData, err := os.ReadFile(archivePath)
require.NoError(t, err)
_, store2 := datastore.MustNewTestStore(t, true, false)
storePath2 := store2.GetConnection().GetStorePath()
ctx, cancel := context.WithCancel(t.Context())
err = RestoreArchive(bytes.NewReader(archiveData), "restore-secret", storePath2, gate, store2, cancel)
require.NoError(t, err)
require.ErrorIs(t, ctx.Err(), context.Canceled)
_, err = os.Stat(filesystem.JoinPaths(storePath2, "portainer.db"))
require.NoError(t, err)
}
func TestRestoreArchive_WrongPassword(t *testing.T) {
t.Parallel()
_, store1 := datastore.MustNewTestStore(t, true, false)
storePath1 := store1.GetConnection().GetStorePath()
gate := offlinegate.NewOfflineGate()
archivePath, err := CreateBackupArchive("correct-password", gate, store1, storePath1)
require.NoError(t, err)
archiveData, err := os.ReadFile(archivePath)
require.NoError(t, err)
_, store2 := datastore.MustNewTestStore(t, true, false)
storePath2 := store2.GetConnection().GetStorePath()
_, cancel := context.WithCancel(t.Context())
err = RestoreArchive(bytes.NewReader(archiveData), "wrong-password", storePath2, gate, store2, cancel)
require.Error(t, err)
}

View File

@@ -16,8 +16,6 @@ import (
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/filesystem"
"github.com/portainer/portainer/api/http/offlinegate"
"github.com/rs/zerolog/log"
)
var filesToRestore = append(filesToBackup, "portainer.db")
@@ -28,25 +26,22 @@ func RestoreArchive(archive io.Reader, password string, filestorePath string, ga
if password != "" {
archive, err = decrypt(archive, password)
if err != nil {
return errors.Wrap(err, "failed to decrypt the archive. Please ensure the password is correct and try again")
return errors.Wrap(err, "failed to decrypt the archive")
}
}
restorePath := filepath.Join(filestorePath, "restore", time.Now().Format("20060102150405"))
defer func() {
if err := os.RemoveAll(filepath.Dir(restorePath)); err != nil {
log.Warn().Err(err).Msg("failed to clean up restore files")
}
}()
defer os.RemoveAll(filepath.Dir(restorePath))
if err := extractArchive(archive, restorePath); err != nil {
err = extractArchive(archive, restorePath)
if err != nil {
return errors.Wrap(err, "cannot extract files from the archive. Please ensure the password is correct and try again")
}
unlock := gate.Lock()
defer unlock()
if err := datastore.Close(); err != nil {
if err = datastore.Close(); err != nil {
return errors.Wrap(err, "Failed to stop db")
}
@@ -56,7 +51,7 @@ func RestoreArchive(archive io.Reader, password string, filestorePath string, ga
return errors.Wrap(err, "failed to restore from backup. Portainer database missing from backup file")
}
if err := restoreFiles(restorePath, filestorePath); err != nil {
if err = restoreFiles(restorePath, filestorePath); err != nil {
return errors.Wrap(err, "failed to restore the system state")
}
@@ -94,7 +89,8 @@ func getRestoreSourcePath(dir string) (string, error) {
func restoreFiles(srcDir string, destinationDir string) error {
for _, filename := range filesToRestore {
if err := filesystem.CopyPath(filepath.Join(srcDir, filename), destinationDir); err != nil {
err := filesystem.CopyPath(filepath.Join(srcDir, filename), destinationDir)
if err != nil {
return err
}
}
@@ -102,18 +98,14 @@ func restoreFiles(srcDir string, destinationDir string) error {
// TODO: This is very boltdb module specific once again due to the filename. Move to bolt module? Refactor for another day
// Prevent the possibility of having both databases. Remove any default new instance
if err := os.Remove(filepath.Join(destinationDir, boltdb.DatabaseFileName)); err != nil && !os.IsNotExist(err) {
return err
}
if err := os.Remove(filepath.Join(destinationDir, boltdb.EncryptedDatabaseFileName)); err != nil && !os.IsNotExist(err) {
return err
}
os.Remove(filepath.Join(destinationDir, boltdb.DatabaseFileName))
os.Remove(filepath.Join(destinationDir, boltdb.EncryptedDatabaseFileName))
// Now copy the database. It'll be either portainer.db or portainer.edb
// Note: CopyPath does not return an error if the source file doesn't exist
if err := filesystem.CopyPath(filepath.Join(srcDir, boltdb.EncryptedDatabaseFileName), destinationDir); err != nil {
err := filesystem.CopyPath(filepath.Join(srcDir, boltdb.EncryptedDatabaseFileName), destinationDir)
if err != nil {
return err
}

12
api/build/variables.go Normal file
View File

@@ -0,0 +1,12 @@
package build
import "runtime"
// Variables to be set during the build time
var BuildNumber string
var ImageTag string
var NodejsVersion string
var YarnVersion string
var WebpackVersion string
var GoVersion string = runtime.Version()
var GitCommit string

View File

@@ -54,8 +54,8 @@ func ecdsaGenerateKey(c elliptic.Curve, rand io.Reader) (*ecdsa.PrivateKey, erro
}
priv := new(ecdsa.PrivateKey)
priv.Curve = c
priv.PublicKey.Curve = c
priv.D = k
priv.X, priv.Y = c.ScalarBaseMult(k.Bytes())
priv.PublicKey.X, priv.PublicKey.Y = c.ScalarBaseMult(k.Bytes())
return priv, nil
}

View File

@@ -6,7 +6,6 @@ import (
)
func TestGenerateGo119CompatibleKey(t *testing.T) {
t.Parallel()
type args struct {
seed string
}

75
api/chisel/schedules.go Normal file
View File

@@ -0,0 +1,75 @@
package chisel
import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/internal/edge/cache"
)
// AddEdgeJob register an EdgeJob inside the tunnel details associated to an environment(endpoint).
func (service *Service) AddEdgeJob(endpoint *portainer.Endpoint, edgeJob *portainer.EdgeJob) {
if endpoint.Edge.AsyncMode {
return
}
service.mu.Lock()
tunnel := service.getTunnelDetails(endpoint.ID)
existingJobIndex := -1
for idx, existingJob := range tunnel.Jobs {
if existingJob.ID == edgeJob.ID {
existingJobIndex = idx
break
}
}
if existingJobIndex == -1 {
tunnel.Jobs = append(tunnel.Jobs, *edgeJob)
} else {
tunnel.Jobs[existingJobIndex] = *edgeJob
}
cache.Del(endpoint.ID)
service.mu.Unlock()
}
// RemoveEdgeJob will remove the specified Edge job from each tunnel it was registered with.
func (service *Service) RemoveEdgeJob(edgeJobID portainer.EdgeJobID) {
service.mu.Lock()
for endpointID, tunnel := range service.tunnelDetailsMap {
n := 0
for _, edgeJob := range tunnel.Jobs {
if edgeJob.ID != edgeJobID {
tunnel.Jobs[n] = edgeJob
n++
}
}
tunnel.Jobs = tunnel.Jobs[:n]
cache.Del(endpointID)
}
service.mu.Unlock()
}
func (service *Service) RemoveEdgeJobFromEndpoint(endpointID portainer.EndpointID, edgeJobID portainer.EdgeJobID) {
service.mu.Lock()
tunnel := service.getTunnelDetails(endpointID)
n := 0
for _, edgeJob := range tunnel.Jobs {
if edgeJob.ID != edgeJobID {
tunnel.Jobs[n] = edgeJob
n++
}
}
tunnel.Jobs = tunnel.Jobs[:n]
cache.Del(endpointID)
service.mu.Unlock()
}

View File

@@ -11,7 +11,6 @@ import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/http/proxy"
"github.com/portainer/portainer/pkg/schedule"
chserver "github.com/jpillora/chisel/server"
"github.com/jpillora/chisel/share/ccrypto"
@@ -20,6 +19,7 @@ import (
const (
tunnelCleanupInterval = 10 * time.Second
requiredTimeout = 15 * time.Second
activeTimeout = 4*time.Minute + 30*time.Second
pingTimeout = 3 * time.Second
)
@@ -28,54 +28,32 @@ const (
// It is used to start a reverse tunnel server and to manage the connection status of each tunnel
// connected to the tunnel server.
type Service struct {
serverFingerprint string
serverPort string
activeTunnels map[portainer.EndpointID]*portainer.TunnelDetails
edgeJobs map[portainer.EndpointID][]portainer.EdgeJob
dataStore dataservices.DataStore
snapshotService portainer.SnapshotService
chiselServer *chserver.Server
shutdownCtx context.Context
ProxyManager *proxy.Manager
mu sync.RWMutex
fileService portainer.FileService
defaultCheckinInterval int
serverFingerprint string
serverPort string
tunnelDetailsMap map[portainer.EndpointID]*portainer.TunnelDetails
dataStore dataservices.DataStore
snapshotService portainer.SnapshotService
chiselServer *chserver.Server
shutdownCtx context.Context
ProxyManager *proxy.Manager
mu sync.Mutex
fileService portainer.FileService
}
// NewService returns a pointer to a new instance of Service
func NewService(dataStore dataservices.DataStore, shutdownCtx context.Context, fileService portainer.FileService) *Service {
defaultCheckinInterval := portainer.DefaultEdgeAgentCheckinIntervalInSeconds
settings, err := dataStore.Settings().Settings()
if err == nil {
defaultCheckinInterval = settings.EdgeAgentCheckinInterval
} else {
log.Error().Err(err).Msg("unable to retrieve the settings from the database")
}
return &Service{
activeTunnels: make(map[portainer.EndpointID]*portainer.TunnelDetails),
edgeJobs: make(map[portainer.EndpointID][]portainer.EdgeJob),
dataStore: dataStore,
shutdownCtx: shutdownCtx,
fileService: fileService,
defaultCheckinInterval: defaultCheckinInterval,
tunnelDetailsMap: make(map[portainer.EndpointID]*portainer.TunnelDetails),
dataStore: dataStore,
shutdownCtx: shutdownCtx,
fileService: fileService,
}
}
// pingAgent ping the given agent so that the agent can keep the tunnel alive
func (service *Service) pingAgent(endpointID portainer.EndpointID) error {
endpoint, err := service.dataStore.Endpoint().Endpoint(endpointID)
if err != nil {
return err
}
tunnelAddr, err := service.TunnelAddr(endpoint)
if err != nil {
return err
}
requestURL := fmt.Sprintf("http://%s/ping", tunnelAddr)
tunnel := service.GetTunnelDetails(endpointID)
requestURL := fmt.Sprintf("http://127.0.0.1:%d/ping", tunnel.Port)
req, err := http.NewRequest(http.MethodHead, requestURL, nil)
if err != nil {
return err
@@ -90,55 +68,55 @@ func (service *Service) pingAgent(endpointID portainer.EndpointID) error {
return err
}
_, _ = io.Copy(io.Discard, resp.Body)
return resp.Body.Close()
io.Copy(io.Discard, resp.Body)
resp.Body.Close()
return nil
}
// KeepTunnelAlive keeps the tunnel of the given environment for maxAlive duration, or until ctx is done
func (service *Service) KeepTunnelAlive(endpointID portainer.EndpointID, ctx context.Context, maxAlive time.Duration) {
go service.keepTunnelAlive(endpointID, ctx, maxAlive)
}
go func() {
log.Debug().
Int("endpoint_id", int(endpointID)).
Float64("max_alive_minutes", maxAlive.Minutes()).
Msg("KeepTunnelAlive: start")
func (service *Service) keepTunnelAlive(endpointID portainer.EndpointID, ctx context.Context, maxAlive time.Duration) {
log.Debug().
Int("endpoint_id", int(endpointID)).
Float64("max_alive_minutes", maxAlive.Minutes()).
Msg("KeepTunnelAlive: start")
maxAliveTicker := time.NewTicker(maxAlive)
defer maxAliveTicker.Stop()
maxAliveTicker := time.NewTicker(maxAlive)
defer maxAliveTicker.Stop()
pingTicker := time.NewTicker(tunnelCleanupInterval)
defer pingTicker.Stop()
pingTicker := time.NewTicker(tunnelCleanupInterval)
defer pingTicker.Stop()
for {
select {
case <-pingTicker.C:
service.SetTunnelStatusToActive(endpointID)
err := service.pingAgent(endpointID)
if err != nil {
log.Debug().
Int("endpoint_id", int(endpointID)).
Err(err).
Msg("KeepTunnelAlive: ping agent")
}
case <-maxAliveTicker.C:
log.Debug().
Int("endpoint_id", int(endpointID)).
Float64("timeout_minutes", maxAlive.Minutes()).
Msg("KeepTunnelAlive: tunnel keep alive timeout")
for {
select {
case <-pingTicker.C:
service.UpdateLastActivity(endpointID)
if err := service.pingAgent(endpointID); err != nil {
return
case <-ctx.Done():
err := ctx.Err()
log.Debug().
Int("endpoint_id", int(endpointID)).
Err(err).
Msg("KeepTunnelAlive: ping agent")
Msg("KeepTunnelAlive: tunnel stop")
return
}
case <-maxAliveTicker.C:
log.Debug().
Int("endpoint_id", int(endpointID)).
Float64("timeout_minutes", maxAlive.Minutes()).
Msg("KeepTunnelAlive: tunnel keep alive timeout")
return
case <-ctx.Done():
err := ctx.Err()
log.Debug().
Int("endpoint_id", int(endpointID)).
Err(err).
Msg("KeepTunnelAlive: tunnel stop")
return
}
}
}()
}
// StartTunnelServer starts a tunnel server on the specified addr and port.
@@ -148,6 +126,7 @@ func (service *Service) keepTunnelAlive(endpointID portainer.EndpointID, ctx con
// The snapshotter is used in the tunnel status verification process.
func (service *Service) StartTunnelServer(addr, port string, snapshotService portainer.SnapshotService) error {
privateKeyFile, err := service.retrievePrivateKeyFile()
if err != nil {
return err
}
@@ -165,21 +144,21 @@ func (service *Service) StartTunnelServer(addr, port string, snapshotService por
service.serverFingerprint = chiselServer.GetFingerprint()
service.serverPort = port
if err := chiselServer.Start(addr, port); err != nil {
err = chiselServer.Start(addr, port)
if err != nil {
return err
}
service.chiselServer = chiselServer
// TODO: work-around Chisel default behavior.
// By default, Chisel will allow anyone to connect if no user exists.
username, password := generateRandomCredentials()
if err = service.chiselServer.AddUser(username, password, "127.0.0.1"); err != nil {
err = service.chiselServer.AddUser(username, password, "127.0.0.1")
if err != nil {
return err
}
service.snapshotService = snapshotService
go service.startTunnelVerificationLoop()
return nil
@@ -193,39 +172,37 @@ func (service *Service) StopTunnelServer() error {
func (service *Service) retrievePrivateKeyFile() (string, error) {
privateKeyFile := service.fileService.GetDefaultChiselPrivateKeyPath()
if exists, _ := service.fileService.FileExists(privateKeyFile); exists {
exist, _ := service.fileService.FileExists(privateKeyFile)
if !exist {
log.Debug().
Str("private-key", privateKeyFile).
Msg("Chisel private key file does not exist")
privateKey, err := ccrypto.GenerateKey("")
if err != nil {
log.Error().
Err(err).
Msg("Failed to generate chisel private key")
return "", err
}
err = service.fileService.StoreChiselPrivateKey(privateKey)
if err != nil {
log.Error().
Err(err).
Msg("Failed to save Chisel private key to disk")
return "", err
} else {
log.Info().
Str("private-key", privateKeyFile).
Msg("Generated a new Chisel private key file")
}
} else {
log.Info().
Str("private-key", privateKeyFile).
Msg("found Chisel private key file on disk")
return privateKeyFile, nil
Msg("Found Chisel private key file on disk")
}
log.Debug().
Str("private-key", privateKeyFile).
Msg("chisel private key file does not exist")
privateKey, err := ccrypto.GenerateKey("")
if err != nil {
log.Error().
Err(err).
Msg("failed to generate chisel private key")
return "", err
}
if err = service.fileService.StoreChiselPrivateKey(privateKey); err != nil {
log.Error().
Err(err).
Msg("failed to save Chisel private key to disk")
return "", err
}
log.Info().
Str("private-key", privateKeyFile).
Msg("generated a new Chisel private key file")
return privateKeyFile, nil
}
@@ -234,94 +211,81 @@ func (service *Service) startTunnelVerificationLoop() {
Float64("check_interval_seconds", tunnelCleanupInterval.Seconds()).
Msg("starting tunnel management process")
schedule.RunOnInterval(service.shutdownCtx, tunnelCleanupInterval, service.checkTunnels, func() {
log.Debug().Msg("shutting down tunnel service")
ticker := time.NewTicker(tunnelCleanupInterval)
if err := service.StopTunnelServer(); err != nil {
log.Debug().Err(err).Msg("stopped tunnel service")
}
})
}
for {
select {
case <-ticker.C:
service.checkTunnels()
case <-service.shutdownCtx.Done():
log.Debug().Msg("shutting down tunnel service")
// checkTunnels finds tunnels that need snapshots and processes them one at a time.
// For active tunnels missing an initial snapshot, it takes one without closing the tunnel.
// For tunnels idle past activeTimeout, it snapshots and closes them.
func (service *Service) checkTunnels() {
service.mu.RLock()
for endpointID, tunnel := range service.activeTunnels {
elapsed := time.Since(tunnel.LastActivity)
log.Debug().
Int("endpoint_id", int(endpointID)).
Float64("last_activity_seconds", elapsed.Seconds()).
Msg("environment tunnel monitoring")
tunnelPort := tunnel.Port
if !tunnel.HasSnapshot && elapsed < activeTimeout {
service.mu.RUnlock()
if endpointHasSnapshot(service.dataStore, endpointID) {
service.markSnapshotTaken(endpointID)
return
}
log.Debug().
Int("endpoint_id", int(endpointID)).
Msg("taking initial snapshot for active Edge environment")
if service.snapshotAndLog(endpointID, tunnelPort) {
service.markSnapshotTaken(endpointID)
if err := service.StopTunnelServer(); err != nil {
log.Debug().Err(err).Msg("stopped tunnel service")
}
ticker.Stop()
return
}
}
}
if tunnel.Status == portainer.EdgeAgentManagementRequired && elapsed < activeTimeout {
func (service *Service) checkTunnels() {
tunnels := make(map[portainer.EndpointID]portainer.TunnelDetails)
service.mu.Lock()
for key, tunnel := range service.tunnelDetailsMap {
if tunnel.LastActivity.IsZero() || tunnel.Status == portainer.EdgeAgentIdle {
continue
}
service.mu.RUnlock()
log.Debug().
Int("endpoint_id", int(endpointID)).
Float64("last_activity_seconds", elapsed.Seconds()).
Float64("timeout_seconds", activeTimeout.Seconds()).
Msg("last activity timeout exceeded")
service.snapshotAndLog(endpointID, tunnelPort)
service.close(endpointID)
return
}
service.mu.RUnlock()
}
func (service *Service) snapshotAndLog(endpointID portainer.EndpointID, tunnelPort int) bool {
if err := service.snapshotEnvironment(endpointID, tunnelPort); err != nil {
log.Error().
Int("endpoint_id", int(endpointID)).
Err(err).
Msg("unable to snapshot Edge environment")
if service.dataStore.IsErrObjectNotFound(err) {
service.close(endpointID)
if tunnel.Status == portainer.EdgeAgentManagementRequired && time.Since(tunnel.LastActivity) < requiredTimeout {
continue
}
return false
if tunnel.Status == portainer.EdgeAgentActive && time.Since(tunnel.LastActivity) < activeTimeout {
continue
}
tunnels[key] = *tunnel
}
service.mu.Unlock()
return true
}
for endpointID, tunnel := range tunnels {
elapsed := time.Since(tunnel.LastActivity)
log.Debug().
Int("endpoint_id", int(endpointID)).
Str("status", tunnel.Status).
Float64("status_time_seconds", elapsed.Seconds()).
Msg("environment tunnel monitoring")
func (service *Service) markSnapshotTaken(endpointID portainer.EndpointID) {
service.mu.Lock()
defer service.mu.Unlock()
if tunnel.Status == portainer.EdgeAgentManagementRequired && elapsed > requiredTimeout {
log.Debug().
Int("endpoint_id", int(endpointID)).
Str("status", tunnel.Status).
Float64("status_time_seconds", elapsed.Seconds()).
Float64("timeout_seconds", requiredTimeout.Seconds()).
Msg("REQUIRED state timeout exceeded")
}
if tun, ok := service.activeTunnels[endpointID]; ok {
tun.HasSnapshot = true
if tunnel.Status == portainer.EdgeAgentActive && elapsed > activeTimeout {
log.Debug().
Int("endpoint_id", int(endpointID)).
Str("status", tunnel.Status).
Float64("status_time_seconds", elapsed.Seconds()).
Float64("timeout_seconds", activeTimeout.Seconds()).
Msg("ACTIVE state timeout exceeded")
err := service.snapshotEnvironment(endpointID, tunnel.Port)
if err != nil {
log.Error().
Int("endpoint_id", int(endpointID)).
Err(err).
Msg("unable to snapshot Edge environment")
}
}
service.SetTunnelStatusToIdle(portainer.EndpointID(endpointID))
}
}

View File

@@ -1,58 +1,20 @@
package chisel
import (
"context"
"errors"
"net"
"net/http"
"testing"
"time"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/datastore"
"github.com/portainer/portainer/pkg/fips"
"github.com/stretchr/testify/require"
)
func init() {
fips.InitFIPS(false)
}
type mockSnapshotService struct {
snapshotFn func(endpoint *portainer.Endpoint) error
}
func (m *mockSnapshotService) Start(_ context.Context) {}
func (m *mockSnapshotService) SetSnapshotInterval(_ string) error { return nil }
func (m *mockSnapshotService) SnapshotEndpoint(endpoint *portainer.Endpoint) error {
if m.snapshotFn != nil {
return m.snapshotFn(endpoint)
}
return nil
}
func (m *mockSnapshotService) FillSnapshotData(_ *portainer.Endpoint, _ bool) error { return nil }
func newEdgeEndpoint(id portainer.EndpointID) *portainer.Endpoint {
return &portainer.Endpoint{
ID: id,
EdgeID: "test-edge-id",
Type: portainer.EdgeAgentOnDockerEnvironment,
UserTrusted: true,
}
}
func TestPingAgentPanic(t *testing.T) {
t.Parallel()
endpoint := newEdgeEndpoint(1)
endpointID := portainer.EndpointID(1)
_, store := datastore.MustNewTestStore(t, false, true)
s := NewService(store, nil, nil)
s := NewService(nil, nil, nil)
defer func() {
require.Nil(t, recover())
@@ -66,173 +28,12 @@ func TestPingAgentPanic(t *testing.T) {
ln, err := net.ListenTCP("tcp", &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0})
require.NoError(t, err)
srv := &http.Server{Handler: mux}
errCh := make(chan error)
go func() {
errCh <- srv.Serve(ln)
require.NoError(t, http.Serve(ln, mux))
}()
err = s.Open(endpoint)
require.NoError(t, err)
s.activeTunnels[endpoint.ID].Port = ln.Addr().(*net.TCPAddr).Port
s.getTunnelDetails(endpointID)
s.tunnelDetailsMap[endpointID].Port = ln.Addr().(*net.TCPAddr).Port
require.Error(t, s.pingAgent(endpoint.ID))
require.NoError(t, srv.Shutdown(t.Context()))
require.ErrorIs(t, <-errCh, http.ErrServerClosed)
}
func TestOpenDefaultsHasSnapshotToFalse(t *testing.T) {
t.Parallel()
endpoint := newEdgeEndpoint(1)
_, store := datastore.MustNewTestStore(t, false, true)
s := NewService(store, nil, nil)
err := s.Open(endpoint)
require.NoError(t, err)
require.False(t, s.activeTunnels[endpoint.ID].HasSnapshot)
}
func TestCheckTunnelsSetsHasSnapshotWhenSnapshotExists(t *testing.T) {
t.Parallel()
endpoint := newEdgeEndpoint(2)
_, store := datastore.MustNewTestStore(t, false, true)
err := store.Endpoint().Create(endpoint)
require.NoError(t, err)
snap := &portainer.Snapshot{
EndpointID: endpoint.ID,
Docker: &portainer.DockerSnapshot{},
}
err = store.Snapshot().Create(snap)
require.NoError(t, err)
s := NewService(store, nil, nil)
s.activeTunnels[endpoint.ID] = &portainer.TunnelDetails{
Status: portainer.EdgeAgentManagementRequired,
Port: 50003,
LastActivity: time.Now(),
}
s.checkTunnels()
require.NotNil(t, s.activeTunnels[endpoint.ID], "tunnel must remain open")
require.True(t, s.activeTunnels[endpoint.ID].HasSnapshot)
}
func TestCheckTunnelsSnapshotsActiveEnvironmentAndKeepsTunnelAlive(t *testing.T) {
t.Parallel()
endpoint := newEdgeEndpoint(3)
_, store := datastore.MustNewTestStore(t, false, true)
err := store.Endpoint().Create(endpoint)
require.NoError(t, err)
snapshotCalled := false
svc := &mockSnapshotService{
snapshotFn: func(_ *portainer.Endpoint) error {
snapshotCalled = true
return nil
},
}
s := NewService(store, nil, nil)
s.snapshotService = svc
s.activeTunnels[endpoint.ID] = &portainer.TunnelDetails{
Status: portainer.EdgeAgentManagementRequired,
Port: 50000,
LastActivity: time.Now(),
}
s.checkTunnels()
require.True(t, snapshotCalled)
require.NotNil(t, s.activeTunnels[endpoint.ID], "tunnel must remain open after snapshot")
require.True(t, s.activeTunnels[endpoint.ID].HasSnapshot)
}
func TestCheckTunnelsKeepsHasSnapshotFalseOnSnapshotFailure(t *testing.T) {
t.Parallel()
endpoint := newEdgeEndpoint(4)
_, store := datastore.MustNewTestStore(t, false, true)
err := store.Endpoint().Create(endpoint)
require.NoError(t, err)
svc := &mockSnapshotService{
snapshotFn: func(_ *portainer.Endpoint) error {
return errors.New("snapshot failed")
},
}
s := NewService(store, nil, nil)
s.snapshotService = svc
s.activeTunnels[endpoint.ID] = &portainer.TunnelDetails{
Status: portainer.EdgeAgentManagementRequired,
Port: 50001,
LastActivity: time.Now(),
}
s.checkTunnels()
require.NotNil(t, s.activeTunnels[endpoint.ID], "tunnel must remain open after failed snapshot")
require.False(t, s.activeTunnels[endpoint.ID].HasSnapshot, "HasSnapshot must stay false after failure")
}
func TestCheckTunnelsClosesStaleEntryForDeletedEndpoint(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
// Endpoint is not created in the store, simulates deletion while tunnel stays open.
s := NewService(store, nil, nil)
s.activeTunnels[1] = &portainer.TunnelDetails{
Status: portainer.EdgeAgentManagementRequired,
Port: 50010,
LastActivity: time.Now(),
}
s.checkTunnels()
require.Nil(t, s.activeTunnels[1], "stale tunnel for deleted endpoint must be removed immediately")
}
func TestCheckTunnelsClosesIdleTunnelAndSnapshots(t *testing.T) {
t.Parallel()
endpoint := newEdgeEndpoint(5)
_, store := datastore.MustNewTestStore(t, false, true)
err := store.Endpoint().Create(endpoint)
require.NoError(t, err)
snapshotCalled := false
svc := &mockSnapshotService{
snapshotFn: func(_ *portainer.Endpoint) error {
snapshotCalled = true
return nil
},
}
s := NewService(store, nil, nil)
s.snapshotService = svc
s.activeTunnels[endpoint.ID] = &portainer.TunnelDetails{
Status: portainer.EdgeAgentManagementRequired,
Port: 50002,
LastActivity: time.Now().Add(-(activeTimeout + time.Second)),
}
s.checkTunnels()
require.True(t, snapshotCalled)
require.Nil(t, s.activeTunnels[endpoint.ID], "tunnel must be closed after idle timeout")
require.Error(t, s.pingAgent(endpointID))
}

View File

@@ -4,20 +4,15 @@ import (
"encoding/base64"
"errors"
"fmt"
"net"
"math/rand"
"strings"
"time"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/internal/edge"
"github.com/portainer/portainer/api/internal/edge/cache"
"github.com/portainer/portainer/api/internal/endpointutils"
"github.com/portainer/portainer/pkg/libcrypto"
"github.com/portainer/portainer/pkg/librand"
"github.com/dchest/uniuri"
"github.com/rs/zerolog/log"
)
const (
@@ -25,209 +20,171 @@ const (
maxAvailablePort = 65535
)
var (
ErrNonEdgeEnv = errors.New("cannot open a tunnel for non-edge environments")
ErrAsyncEnv = errors.New("cannot open a tunnel for async edge environments")
ErrInvalidEnv = errors.New("cannot open a tunnel for an invalid environment")
)
// Open will mark the tunnel as REQUIRED so the agent opens it
func (s *Service) Open(endpoint *portainer.Endpoint) error {
if !endpointutils.IsEdgeEndpoint(endpoint) {
return ErrNonEdgeEnv
}
if endpoint.Edge.AsyncMode {
return ErrAsyncEnv
}
if endpoint.ID == 0 || endpoint.EdgeID == "" || !endpoint.UserTrusted {
return ErrInvalidEnv
}
s.mu.Lock()
defer s.mu.Unlock()
if _, ok := s.activeTunnels[endpoint.ID]; ok {
return nil
}
defer cache.Del(endpoint.ID)
tun := &portainer.TunnelDetails{
Status: portainer.EdgeAgentManagementRequired,
Port: s.getUnusedPort(),
LastActivity: time.Now(),
}
username, password := generateRandomCredentials()
if s.chiselServer != nil {
authorizedRemote := fmt.Sprintf("^R:0.0.0.0:%d$", tun.Port)
if err := s.chiselServer.AddUser(username, password, authorizedRemote); err != nil {
return err
}
}
credentials, err := encryptCredentials(username, password, endpoint.EdgeID)
if err != nil {
return err
}
tun.Credentials = credentials
s.activeTunnels[endpoint.ID] = tun
return nil
}
// close removes the tunnel from the map so the agent will close it.
// The lock is released before cleaning up the chisel user and proxy to avoid
// blocking Config/Open callers while DeleteUser interacts with chisel internals.
func (s *Service) close(endpointID portainer.EndpointID) {
s.mu.Lock()
tun, ok := s.activeTunnels[endpointID]
if !ok {
s.mu.Unlock()
return
}
delete(s.activeTunnels, endpointID)
cache.Del(endpointID)
s.mu.Unlock()
if s.chiselServer != nil {
user, _, _ := strings.Cut(tun.Credentials, ":")
s.chiselServer.DeleteUser(user)
}
if s.ProxyManager != nil {
s.ProxyManager.DeleteEndpointProxy(endpointID)
}
}
// Config returns the tunnel details needed for the agent to connect
func (s *Service) Config(endpointID portainer.EndpointID) portainer.TunnelDetails {
s.mu.RLock()
defer s.mu.RUnlock()
if tun, ok := s.activeTunnels[endpointID]; ok {
return *tun
}
return portainer.TunnelDetails{Status: portainer.EdgeAgentIdle}
}
// TunnelAddr returns the address of the local tunnel, including the port, it
// will block until the tunnel is ready
func (s *Service) TunnelAddr(endpoint *portainer.Endpoint) (string, error) {
if err := s.Open(endpoint); err != nil {
return "", err
}
tun := s.Config(endpoint.ID)
checkinInterval := time.Duration(s.tryEffectiveCheckinInterval(endpoint)) * time.Second
for t0 := time.Now(); ; {
if time.Since(t0) > 2*checkinInterval {
s.close(endpoint.ID)
return "", errors.New("unable to open the tunnel")
}
// Check if the tunnel is established
conn, err := net.DialTCP("tcp", nil, &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: tun.Port})
if err != nil {
time.Sleep(checkinInterval / 100)
continue
}
if err := conn.Close(); err != nil {
log.Warn().Err(err).Msg("failed to close tcp connection")
}
break
}
s.UpdateLastActivity(endpoint.ID)
return fmt.Sprintf("127.0.0.1:%d", tun.Port), nil
}
// tryEffectiveCheckinInterval avoids a potential deadlock by returning a
// previous known value after a timeout
func (s *Service) tryEffectiveCheckinInterval(endpoint *portainer.Endpoint) int {
ch := make(chan int, 1)
go func() {
ch <- edge.EffectiveCheckinInterval(s.dataStore, endpoint)
}()
select {
case <-time.After(50 * time.Millisecond):
s.mu.RLock()
defer s.mu.RUnlock()
return s.defaultCheckinInterval
case i := <-ch:
s.mu.Lock()
s.defaultCheckinInterval = i
s.mu.Unlock()
return i
}
}
// UpdateLastActivity sets the current timestamp to avoid the tunnel timeout
func (s *Service) UpdateLastActivity(endpointID portainer.EndpointID) {
s.mu.Lock()
defer s.mu.Unlock()
if tun, ok := s.activeTunnels[endpointID]; ok {
tun.LastActivity = time.Now()
}
}
// NOTE: it needs to be called with the lock acquired
// getUnusedPort is used to generate an unused random port in the dynamic port range.
// Dynamic ports (also called private ports) are 49152 to 65535.
func (service *Service) getUnusedPort() int {
port := randomInt(minAvailablePort, maxAvailablePort)
for _, tunnel := range service.activeTunnels {
for _, tunnel := range service.tunnelDetailsMap {
if tunnel.Port == port {
return service.getUnusedPort()
}
}
conn, err := net.DialTCP("tcp", nil, &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: port})
if err == nil {
if err := conn.Close(); err != nil {
log.Warn().Msg("failed to close tcp connection that checks if port is free")
}
log.Debug().
Int("port", port).
Msg("selected port is in use, trying a different one")
return service.getUnusedPort()
}
return port
}
func randomInt(min, max int) int {
return min + librand.Intn(max-min)
return min + rand.Intn(max-min)
}
// NOTE: it needs to be called with the lock acquired
func (service *Service) getTunnelDetails(endpointID portainer.EndpointID) *portainer.TunnelDetails {
if tunnel, ok := service.tunnelDetailsMap[endpointID]; ok {
return tunnel
}
tunnel := &portainer.TunnelDetails{
Status: portainer.EdgeAgentIdle,
}
service.tunnelDetailsMap[endpointID] = tunnel
cache.Del(endpointID)
return tunnel
}
// GetTunnelDetails returns information about the tunnel associated to an environment(endpoint).
func (service *Service) GetTunnelDetails(endpointID portainer.EndpointID) portainer.TunnelDetails {
service.mu.Lock()
defer service.mu.Unlock()
return *service.getTunnelDetails(endpointID)
}
// GetActiveTunnel retrieves an active tunnel which allows communicating with edge agent
func (service *Service) GetActiveTunnel(endpoint *portainer.Endpoint) (portainer.TunnelDetails, error) {
if endpoint.Edge.AsyncMode {
return portainer.TunnelDetails{}, errors.New("cannot open tunnel on async endpoint")
}
tunnel := service.GetTunnelDetails(endpoint.ID)
if tunnel.Status == portainer.EdgeAgentActive {
// update the LastActivity
service.SetTunnelStatusToActive(endpoint.ID)
}
if tunnel.Status == portainer.EdgeAgentIdle || tunnel.Status == portainer.EdgeAgentManagementRequired {
err := service.SetTunnelStatusToRequired(endpoint.ID)
if err != nil {
return portainer.TunnelDetails{}, fmt.Errorf("failed opening tunnel to endpoint: %w", err)
}
if endpoint.EdgeCheckinInterval == 0 {
settings, err := service.dataStore.Settings().Settings()
if err != nil {
return portainer.TunnelDetails{}, fmt.Errorf("failed fetching settings from db: %w", err)
}
endpoint.EdgeCheckinInterval = settings.EdgeAgentCheckinInterval
}
time.Sleep(2 * time.Duration(endpoint.EdgeCheckinInterval) * time.Second)
}
return service.GetTunnelDetails(endpoint.ID), nil
}
// SetTunnelStatusToActive update the status of the tunnel associated to the specified environment(endpoint).
// It sets the status to ACTIVE.
func (service *Service) SetTunnelStatusToActive(endpointID portainer.EndpointID) {
service.mu.Lock()
tunnel := service.getTunnelDetails(endpointID)
tunnel.Status = portainer.EdgeAgentActive
tunnel.Credentials = ""
tunnel.LastActivity = time.Now()
service.mu.Unlock()
cache.Del(endpointID)
}
// SetTunnelStatusToIdle update the status of the tunnel associated to the specified environment(endpoint).
// It sets the status to IDLE.
// It removes any existing credentials associated to the tunnel.
func (service *Service) SetTunnelStatusToIdle(endpointID portainer.EndpointID) {
service.mu.Lock()
tunnel := service.getTunnelDetails(endpointID)
tunnel.Status = portainer.EdgeAgentIdle
tunnel.Port = 0
tunnel.LastActivity = time.Now()
credentials := tunnel.Credentials
if credentials != "" {
tunnel.Credentials = ""
if service.chiselServer != nil {
service.chiselServer.DeleteUser(strings.Split(credentials, ":")[0])
}
}
service.ProxyManager.DeleteEndpointProxy(endpointID)
service.mu.Unlock()
cache.Del(endpointID)
}
// SetTunnelStatusToRequired update the status of the tunnel associated to the specified environment(endpoint).
// It sets the status to REQUIRED.
// If no port is currently associated to the tunnel, it will associate a random unused port to the tunnel
// and generate temporary credentials that can be used to establish a reverse tunnel on that port.
// Credentials are encrypted using the Edge ID associated to the environment(endpoint).
func (service *Service) SetTunnelStatusToRequired(endpointID portainer.EndpointID) error {
defer cache.Del(endpointID)
tunnel := service.getTunnelDetails(endpointID)
service.mu.Lock()
defer service.mu.Unlock()
if tunnel.Port == 0 {
endpoint, err := service.dataStore.Endpoint().Endpoint(endpointID)
if err != nil {
return err
}
tunnel.Status = portainer.EdgeAgentManagementRequired
tunnel.Port = service.getUnusedPort()
tunnel.LastActivity = time.Now()
username, password := generateRandomCredentials()
authorizedRemote := fmt.Sprintf("^R:0.0.0.0:%d$", tunnel.Port)
if service.chiselServer != nil {
err = service.chiselServer.AddUser(username, password, authorizedRemote)
if err != nil {
return err
}
}
credentials, err := encryptCredentials(username, password, endpoint.EdgeID)
if err != nil {
return err
}
tunnel.Credentials = credentials
}
return nil
}
func generateRandomCredentials() (string, string) {
username := uniuri.NewLen(8)
password := uniuri.NewLen(8)
return username, password
}
@@ -241,18 +198,3 @@ func encryptCredentials(username, password, key string) (string, error) {
return base64.RawStdEncoding.EncodeToString(encryptedCredentials), nil
}
func endpointHasSnapshot(dataStore dataservices.DataStore, endpointID portainer.EndpointID) bool {
var hasSnapshot bool
_ = dataStore.ViewTx(func(tx dataservices.DataStoreTx) error {
s, err := tx.Snapshot().Read(endpointID)
if err != nil {
return err
}
hasSnapshot = s.Docker != nil || s.Kubernetes != nil
return nil
})
return hasSnapshot
}

View File

@@ -1,80 +0,0 @@
package chisel
import (
"net"
"strings"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
)
type testSettingsService struct {
dataservices.SettingsService
}
func (s *testSettingsService) Settings() (*portainer.Settings, error) {
return &portainer.Settings{
EdgeAgentCheckinInterval: 1,
}, nil
}
type testStore struct {
dataservices.DataStore
}
func (s *testStore) Settings() dataservices.SettingsService {
return &testSettingsService{}
}
func TestGetUnusedPort(t *testing.T) {
t.Parallel()
testCases := []struct {
name string
existingTunnels map[portainer.EndpointID]*portainer.TunnelDetails
expectedError error
}{
{
name: "simple case",
},
{
name: "existing tunnels",
existingTunnels: map[portainer.EndpointID]*portainer.TunnelDetails{
portainer.EndpointID(1): {
Port: 53072,
},
portainer.EndpointID(2): {
Port: 63072,
},
},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
store := &testStore{}
s := NewService(store, nil, nil)
s.activeTunnels = tc.existingTunnels
port := s.getUnusedPort()
if port < 49152 || port > 65535 {
t.Fatalf("Expected port to be inbetween 49152 and 65535 but got %d", port)
}
for _, tun := range tc.existingTunnels {
if tun.Port == port {
t.Fatalf("returned port %d already has an existing tunnel", port)
}
}
conn, err := net.DialTCP("tcp", nil, &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: port})
if err == nil {
// Ignore error
_ = conn.Close()
t.Fatalf("expected port %d to be unused", port)
} else if !strings.Contains(err.Error(), "connection refused") {
t.Fatalf("unexpected error: %v", err)
}
})
}
}

View File

@@ -9,35 +9,46 @@ import (
portainer "github.com/portainer/portainer/api"
"github.com/alecthomas/kingpin/v2"
"github.com/rs/zerolog/log"
"gopkg.in/alecthomas/kingpin.v2"
)
// Service implements the CLIService interface
type Service struct{}
var (
ErrInvalidEndpointProtocol = errors.New("Invalid environment protocol: Portainer only supports unix://, npipe:// or tcp://")
ErrSocketOrNamedPipeNotFound = errors.New("Unable to locate Unix socket or named pipe")
ErrInvalidSnapshotInterval = errors.New("Invalid snapshot interval")
ErrAdminPassExcludeAdminPassFile = errors.New("Cannot use --admin-password with --admin-password-file")
errInvalidEndpointProtocol = errors.New("Invalid environment protocol: Portainer only supports unix://, npipe:// or tcp://")
errSocketOrNamedPipeNotFound = errors.New("Unable to locate Unix socket or named pipe")
errInvalidSnapshotInterval = errors.New("Invalid snapshot interval")
errAdminPassExcludeAdminPassFile = errors.New("Cannot use --admin-password with --admin-password-file")
)
func CLIFlags() *portainer.CLIFlags {
return &portainer.CLIFlags{
// ParseFlags parse the CLI flags and return a portainer.Flags struct
func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
kingpin.Version(version)
flags := &portainer.CLIFlags{
Addr: kingpin.Flag("bind", "Address and port to serve Portainer").Default(defaultBindAddress).Short('p').String(),
AddrHTTPS: kingpin.Flag("bind-https", "Address and port to serve Portainer via https").Default(defaultHTTPSBindAddress).String(),
TunnelAddr: kingpin.Flag("tunnel-addr", "Address to serve the tunnel server").Default(defaultTunnelServerAddress).String(),
TunnelPort: kingpin.Flag("tunnel-port", "Port to serve the tunnel server").Default(defaultTunnelServerPort).String(),
Assets: kingpin.Flag("assets", "Path to the assets").Default(defaultAssetsDirectory).Short('a').String(),
Data: kingpin.Flag("data", "Path to the folder where the data is stored").Default(defaultDataDirectory).Short('d').String(),
DemoEnvironment: kingpin.Flag("demo", "Demo environment").Bool(),
EndpointURL: kingpin.Flag("host", "Environment URL").Short('H').String(),
FeatureFlags: kingpin.Flag("feat", "List of feature flags").Envar(portainer.FeatureFlagEnvVar).Strings(),
FeatureFlags: kingpin.Flag("feat", "List of feature flags").Strings(),
EnableEdgeComputeFeatures: kingpin.Flag("edge-compute", "Enable Edge Compute features").Bool(),
NoAnalytics: kingpin.Flag("no-analytics", "Disable Analytics in app (deprecated)").Bool(),
TLS: kingpin.Flag("tlsverify", "TLS support").Default(defaultTLS).Bool(),
TLSSkipVerify: kingpin.Flag("tlsskipverify", "Disable TLS server verification").Default(defaultTLSSkipVerify).Bool(),
TLSCacert: kingpin.Flag("tlscacert", "Path to the CA").Default(defaultTLSCACertPath).String(),
TLSCert: kingpin.Flag("tlscert", "Path to the TLS certificate file").Default(defaultTLSCertPath).String(),
TLSKey: kingpin.Flag("tlskey", "Path to the TLS key").Default(defaultTLSKeyPath).String(),
HTTPDisabled: kingpin.Flag("http-disabled", "Serve portainer only on https").Default(defaultHTTPDisabled).Bool(),
HTTPEnabled: kingpin.Flag("http-enabled", "Serve portainer on http").Default(defaultHTTPEnabled).Bool(),
SSL: kingpin.Flag("ssl", "Secure Portainer instance using SSL (deprecated)").Default(defaultSSL).Bool(),
SSLCert: kingpin.Flag("sslcert", "Path to the SSL certificate used to secure the Portainer instance").String(),
SSLKey: kingpin.Flag("sslkey", "Path to the SSL key used to secure the Portainer instance").String(),
Rollback: kingpin.Flag("rollback", "Rollback the database to the previous backup").Bool(),
SnapshotInterval: kingpin.Flag("snapshot-interval", "Duration between each environment snapshot job").String(),
AdminPassword: kingpin.Flag("admin-password", "Set admin password with provided hash").String(),
@@ -51,65 +62,11 @@ func CLIFlags() *portainer.CLIFlags {
MaxBatchDelay: kingpin.Flag("max-batch-delay", "Maximum delay before a batch starts").Duration(),
SecretKeyName: kingpin.Flag("secret-key-name", "Secret key name for encryption and will be used as /run/secrets/<secret-key-name>.").Default(defaultSecretKeyName).String(),
LogLevel: kingpin.Flag("log-level", "Set the minimum logging level to show").Default("INFO").Enum("DEBUG", "INFO", "WARN", "ERROR"),
LogMode: kingpin.Flag("log-mode", "Set the logging output mode").Default("PRETTY").Enum("NOCOLOR", "PRETTY", "JSON"),
PullLimitCheckDisabled: kingpin.Flag("pull-limit-check-disabled", "Pull limit check").Envar(portainer.PullLimitCheckDisabledEnvVar).Default(defaultPullLimitCheckDisabled).Bool(),
TrustedOrigins: kingpin.Flag("trusted-origins", "List of trusted origins for CSRF protection. Separate multiple origins with a comma.").Envar(portainer.TrustedOriginsEnvVar).String(),
CSP: kingpin.Flag("csp", "Content Security Policy (CSP) header").Envar(portainer.CSPEnvVar).Default("true").Bool(),
CompactDB: kingpin.Flag("compact-db", "Enable database compaction on startup").Envar(portainer.CompactDBEnvVar).Default("false").Bool(),
NoSetupToken: kingpin.Flag("no-setup-token", "Disable the setup token requirement for admin initialization and restore on an uninitialized instance").Envar(portainer.NoSetupTokenEnvVar).Bool(),
SetupToken: kingpin.Flag("setup-token", "Set a custom setup token for admin initialization and restore on an uninitialized instance (overrides auto-generation)").Envar(portainer.SetupTokenEnvVar).String(),
LogMode: kingpin.Flag("log-mode", "Set the logging output mode").Default("PRETTY").Enum("PRETTY", "JSON"),
}
}
// ParseFlags parse the CLI flags and return a portainer.Flags struct
func (Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
kingpin.Version(version)
var hasSSLFlag, hasSSLCertFlag, hasSSLKeyFlag bool
sslFlag := kingpin.Flag(
"ssl",
"Secure Portainer instance using SSL (deprecated)",
).Default(defaultSSL).IsSetByUser(&hasSSLFlag)
ssl := sslFlag.Bool()
sslCertFlag := kingpin.Flag(
"sslcert",
"Path to the SSL certificate used to secure the Portainer instance",
).IsSetByUser(&hasSSLCertFlag)
sslCert := sslCertFlag.String()
sslKeyFlag := kingpin.Flag(
"sslkey",
"Path to the SSL key used to secure the Portainer instance",
).IsSetByUser(&hasSSLKeyFlag)
sslKey := sslKeyFlag.String()
flags := CLIFlags()
var hasTLSFlag, hasTLSCertFlag, hasTLSKeyFlag bool
tlsFlag := kingpin.Flag("tlsverify", "TLS support").Default(defaultTLS).IsSetByUser(&hasTLSFlag)
flags.TLS = tlsFlag.Bool()
tlsCertFlag := kingpin.Flag(
"tlscert",
"Path to the TLS certificate file",
).Default(defaultTLSCertPath).IsSetByUser(&hasTLSCertFlag)
flags.TLSCert = tlsCertFlag.String()
tlsKeyFlag := kingpin.Flag("tlskey", "Path to the TLS key").Default(defaultTLSKeyPath).IsSetByUser(&hasTLSKeyFlag)
flags.TLSKey = tlsKeyFlag.String()
flags.TLSCacert = kingpin.Flag("tlscacert", "Path to the CA").Default(defaultTLSCACertPath).String()
var hasKubectlShellImageFlag bool
kubectlShellImageFlag := kingpin.Flag(
"kubectl-shell-image",
"Kubectl shell image",
).Envar(portainer.KubectlShellImageEnvVar).
Default(portainer.DefaultKubectlShellImage).
IsSetByUser(&hasKubectlShellImageFlag)
flags.KubectlShellImage = kubectlShellImageFlag.String()
kingpin.Parse()
_, kubectlShellImageEnvVarSet := os.LookupEnv(portainer.KubectlShellImageEnvVar)
flags.KubectlShellImageSet = hasKubectlShellImageFlag || kubectlShellImageEnvVarSet
if !filepath.IsAbs(*flags.Assets) {
ex, err := os.Executable()
if err != nil {
@@ -119,58 +76,25 @@ func (Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
*flags.Assets = filepath.Join(filepath.Dir(ex), *flags.Assets)
}
// If the user didn't provide a tls flag remove the defaults to match previous behaviour
if !hasTLSFlag {
if !hasTLSCertFlag {
*flags.TLSCert = ""
}
if !hasTLSKeyFlag {
*flags.TLSKey = ""
}
}
if hasSSLFlag {
log.Warn().Msgf("the %q flag is deprecated. use %q instead.", sslFlag.Model().Name, tlsFlag.Model().Name)
if !hasTLSFlag {
flags.TLS = ssl
}
}
if hasSSLCertFlag {
log.Warn().Msgf("the %q flag is deprecated. use %q instead.", sslCertFlag.Model().Name, tlsCertFlag.Model().Name)
if !hasTLSCertFlag {
flags.TLSCert = sslCert
}
}
if hasSSLKeyFlag {
log.Warn().Msgf("the %q flag is deprecated. use %q instead.", sslKeyFlag.Model().Name, tlsKeyFlag.Model().Name)
if !hasTLSKeyFlag {
flags.TLSKey = sslKey
}
}
return flags, nil
}
// ValidateFlags validates the values of the flags.
func (Service) ValidateFlags(flags *portainer.CLIFlags) error {
func (*Service) ValidateFlags(flags *portainer.CLIFlags) error {
displayDeprecationWarnings(flags)
if err := ValidateEndpointURL(*flags.EndpointURL); err != nil {
err := validateEndpointURL(*flags.EndpointURL)
if err != nil {
return err
}
if err := ValidateSnapshotInterval(*flags.SnapshotInterval); err != nil {
err = validateSnapshotInterval(*flags.SnapshotInterval)
if err != nil {
return err
}
if *flags.AdminPassword != "" && *flags.AdminPasswordFile != "" {
return ErrAdminPassExcludeAdminPassFile
return errAdminPassExcludeAdminPassFile
}
return nil
@@ -180,24 +104,27 @@ func displayDeprecationWarnings(flags *portainer.CLIFlags) {
if *flags.NoAnalytics {
log.Warn().Msg("the --no-analytics flag has been kept to allow migration of instances running a previous version of Portainer with this flag enabled, to version 2.0 where enabling this flag will have no effect")
}
if *flags.SSL {
log.Warn().Msg("SSL is enabled by default and there is no need for the --ssl flag, it has been kept to allow migration of instances running a previous version of Portainer with this flag enabled")
}
}
func ValidateEndpointURL(endpointURL string) error {
func validateEndpointURL(endpointURL string) error {
if endpointURL == "" {
return nil
}
if !strings.HasPrefix(endpointURL, "unix://") && !strings.HasPrefix(endpointURL, "tcp://") && !strings.HasPrefix(endpointURL, "npipe://") {
return ErrInvalidEndpointProtocol
return errInvalidEndpointProtocol
}
if strings.HasPrefix(endpointURL, "unix://") || strings.HasPrefix(endpointURL, "npipe://") {
socketPath := strings.TrimPrefix(endpointURL, "unix://")
socketPath = strings.TrimPrefix(socketPath, "npipe://")
if _, err := os.Stat(socketPath); err != nil {
if os.IsNotExist(err) {
return ErrSocketOrNamedPipeNotFound
return errSocketOrNamedPipeNotFound
}
return err
@@ -207,13 +134,14 @@ func ValidateEndpointURL(endpointURL string) error {
return nil
}
func ValidateSnapshotInterval(snapshotInterval string) error {
func validateSnapshotInterval(snapshotInterval string) error {
if snapshotInterval == "" {
return nil
}
if _, err := time.ParseDuration(snapshotInterval); err != nil {
return ErrInvalidSnapshotInterval
_, err := time.ParseDuration(snapshotInterval)
if err != nil {
return errInvalidSnapshotInterval
}
return nil

View File

@@ -1,263 +0,0 @@
package cli
import (
"io"
"os"
"strings"
"testing"
portainer "github.com/portainer/portainer/api"
zerolog "github.com/rs/zerolog/log"
"github.com/stretchr/testify/require"
)
func TestOptionParser(t *testing.T) {
p := Service{}
require.NotNil(t, p)
a := os.Args
defer func() { os.Args = a }()
os.Args = []string{"portainer", "--edge-compute"}
opts, err := p.ParseFlags("2.34.5")
require.NoError(t, err)
require.False(t, *opts.HTTPDisabled)
require.True(t, *opts.EnableEdgeComputeFeatures)
}
func TestParseKubectlShellImageFlag(t *testing.T) {
tests := []struct {
name string
args []string
envVars map[string]string
expectedKubectlShellImageSet bool
expectedKubectlShellFlag string
}{
{
name: "no flag, no env var",
expectedKubectlShellImageSet: false,
expectedKubectlShellFlag: portainer.DefaultKubectlShellImage,
},
{
name: "explicit flag",
args: []string{"portainer", "--kubectl-shell-image=myimage:v2"},
expectedKubectlShellImageSet: true,
expectedKubectlShellFlag: "myimage:v2",
},
{
name: "env var",
envVars: map[string]string{portainer.KubectlShellImageEnvVar: "myimage:v3"},
expectedKubectlShellImageSet: true,
expectedKubectlShellFlag: "myimage:v3",
},
{
name: "both env var and flag set",
args: []string{"portainer", "--kubectl-shell-image=myimage:v2"},
envVars: map[string]string{portainer.KubectlShellImageEnvVar: "myimage:v3"},
expectedKubectlShellImageSet: true,
expectedKubectlShellFlag: "myimage:v2",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
if tc.args == nil {
tc.args = []string{"portainer"}
}
setOsArgs(t, tc.args)
for k, v := range tc.envVars {
t.Setenv(k, v)
}
flags, err := Service{}.ParseFlags("test-version")
require.NoError(t, err)
require.Equal(t, tc.expectedKubectlShellImageSet, flags.KubectlShellImageSet)
require.Equal(t, tc.expectedKubectlShellFlag, *flags.KubectlShellImage)
})
}
}
func TestParseTLSFlags(t *testing.T) {
testCases := []struct {
name string
args []string
expectedTLSFlag bool
expectedTLSCertFlag string
expectedTLSKeyFlag string
expectedLogMessages []string
}{
{
name: "no flags",
expectedTLSFlag: false,
expectedTLSCertFlag: "",
expectedTLSKeyFlag: "",
},
{
name: "only ssl flag",
args: []string{
"portainer",
"--ssl",
},
expectedTLSFlag: true,
expectedTLSCertFlag: "",
expectedTLSKeyFlag: "",
},
{
name: "only tls flag",
args: []string{
"portainer",
"--tlsverify",
},
expectedTLSFlag: true,
expectedTLSCertFlag: defaultTLSCertPath,
expectedTLSKeyFlag: defaultTLSKeyPath,
},
{
name: "partial ssl flags",
args: []string{
"portainer",
"--ssl",
"--sslcert=ssl-cert-flag-value",
},
expectedTLSFlag: true,
expectedTLSCertFlag: "ssl-cert-flag-value",
expectedTLSKeyFlag: "",
},
{
name: "partial tls flags",
args: []string{
"portainer",
"--tlsverify",
"--tlscert=tls-cert-flag-value",
},
expectedTLSFlag: true,
expectedTLSCertFlag: "tls-cert-flag-value",
expectedTLSKeyFlag: defaultTLSKeyPath,
},
{
name: "partial tls and ssl flags",
args: []string{
"portainer",
"--tlsverify",
"--tlscert=tls-cert-flag-value",
"--sslkey=ssl-key-flag-value",
},
expectedTLSFlag: true,
expectedTLSCertFlag: "tls-cert-flag-value",
expectedTLSKeyFlag: "ssl-key-flag-value",
},
{
name: "partial tls and ssl flags 2",
args: []string{
"portainer",
"--ssl",
"--tlscert=tls-cert-flag-value",
"--sslkey=ssl-key-flag-value",
},
expectedTLSFlag: true,
expectedTLSCertFlag: "tls-cert-flag-value",
expectedTLSKeyFlag: "ssl-key-flag-value",
},
{
name: "ssl flags",
args: []string{
"portainer",
"--ssl",
"--sslcert=ssl-cert-flag-value",
"--sslkey=ssl-key-flag-value",
},
expectedTLSFlag: true,
expectedTLSCertFlag: "ssl-cert-flag-value",
expectedTLSKeyFlag: "ssl-key-flag-value",
expectedLogMessages: []string{
"the \\\"ssl\\\" flag is deprecated. use \\\"tlsverify\\\" instead.",
"the \\\"sslcert\\\" flag is deprecated. use \\\"tlscert\\\" instead.",
"the \\\"sslkey\\\" flag is deprecated. use \\\"tlskey\\\" instead.",
},
},
{
name: "tls flags",
args: []string{
"portainer",
"--tlsverify",
"--tlscert=tls-cert-flag-value",
"--tlskey=tls-key-flag-value",
},
expectedTLSFlag: true,
expectedTLSCertFlag: "tls-cert-flag-value",
expectedTLSKeyFlag: "tls-key-flag-value",
},
{
name: "tls and ssl flags",
args: []string{
"portainer",
"--tlsverify",
"--tlscert=tls-cert-flag-value",
"--tlskey=tls-key-flag-value",
"--ssl",
"--sslcert=ssl-cert-flag-value",
"--sslkey=ssl-key-flag-value",
},
expectedTLSFlag: true,
expectedTLSCertFlag: "tls-cert-flag-value",
expectedTLSKeyFlag: "tls-key-flag-value",
expectedLogMessages: []string{
"the \\\"ssl\\\" flag is deprecated. use \\\"tlsverify\\\" instead.",
"the \\\"sslcert\\\" flag is deprecated. use \\\"tlscert\\\" instead.",
"the \\\"sslkey\\\" flag is deprecated. use \\\"tlskey\\\" instead.",
},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
var logOutput strings.Builder
setupLogOutput(t, &logOutput)
if tc.args == nil {
tc.args = []string{"portainer"}
}
setOsArgs(t, tc.args)
s := Service{}
flags, err := s.ParseFlags("test-version")
if err != nil {
t.Fatalf("error parsing flags: %v", err)
}
if flags.TLS == nil {
t.Fatal("TLS flag was nil")
}
require.Equal(t, tc.expectedTLSFlag, *flags.TLS, "tlsverify flag didn't match")
require.Equal(t, tc.expectedTLSCertFlag, *flags.TLSCert, "tlscert flag didn't match")
require.Equal(t, tc.expectedTLSKeyFlag, *flags.TLSKey, "tlskey flag didn't match")
for _, expectedLogMessage := range tc.expectedLogMessages {
require.Contains(t, logOutput.String(), expectedLogMessage, "Log didn't contain expected message")
}
})
}
}
func setOsArgs(t *testing.T, args []string) {
t.Helper()
previousArgs := os.Args
os.Args = args
t.Cleanup(func() {
os.Args = previousArgs
})
}
func setupLogOutput(t *testing.T, w io.Writer) {
t.Helper()
oldLogger := zerolog.Logger
zerolog.Logger = zerolog.Output(w)
t.Cleanup(func() {
zerolog.Logger = oldLogger
})
}

View File

@@ -19,5 +19,7 @@ func Confirm(message string) (bool, error) {
}
answer = strings.ReplaceAll(answer, "\n", "")
return strings.EqualFold(answer, "y") || strings.EqualFold(answer, "yes"), nil
answer = strings.ToLower(answer)
return answer == "y" || answer == "yes", nil
}

View File

@@ -1,23 +1,23 @@
//go:build !windows
// +build !windows
package cli
const (
defaultBindAddress = ":9000"
defaultHTTPSBindAddress = ":9443"
defaultTunnelServerAddress = "0.0.0.0"
defaultTunnelServerPort = "8000"
defaultDataDirectory = "/data"
defaultAssetsDirectory = "./"
defaultTLS = "false"
defaultTLSSkipVerify = "false"
defaultTLSCACertPath = "/certs/ca.pem"
defaultTLSCertPath = "/certs/cert.pem"
defaultTLSKeyPath = "/certs/key.pem"
defaultHTTPDisabled = "false"
defaultHTTPEnabled = "false"
defaultSSL = "false"
defaultBaseURL = "/"
defaultSecretKeyName = "portainer"
defaultPullLimitCheckDisabled = "false"
defaultBindAddress = ":9000"
defaultHTTPSBindAddress = ":9443"
defaultTunnelServerAddress = "0.0.0.0"
defaultTunnelServerPort = "8000"
defaultDataDirectory = "/data"
defaultAssetsDirectory = "./"
defaultTLS = "false"
defaultTLSSkipVerify = "false"
defaultTLSCACertPath = "/certs/ca.pem"
defaultTLSCertPath = "/certs/cert.pem"
defaultTLSKeyPath = "/certs/key.pem"
defaultHTTPDisabled = "false"
defaultHTTPEnabled = "false"
defaultSSL = "false"
defaultBaseURL = "/"
defaultSecretKeyName = "portainer"
)

View File

@@ -1,22 +1,21 @@
package cli
const (
defaultBindAddress = ":9000"
defaultHTTPSBindAddress = ":9443"
defaultTunnelServerAddress = "0.0.0.0"
defaultTunnelServerPort = "8000"
defaultDataDirectory = "C:\\data"
defaultAssetsDirectory = "./"
defaultTLS = "false"
defaultTLSSkipVerify = "false"
defaultTLSCACertPath = "C:\\certs\\ca.pem"
defaultTLSCertPath = "C:\\certs\\cert.pem"
defaultTLSKeyPath = "C:\\certs\\key.pem"
defaultHTTPDisabled = "false"
defaultHTTPEnabled = "false"
defaultSSL = "false"
defaultSnapshotInterval = "5m"
defaultBaseURL = "/"
defaultSecretKeyName = "portainer"
defaultPullLimitCheckDisabled = "false"
defaultBindAddress = ":9000"
defaultHTTPSBindAddress = ":9443"
defaultTunnelServerAddress = "0.0.0.0"
defaultTunnelServerPort = "8000"
defaultDataDirectory = "C:\\data"
defaultAssetsDirectory = "./"
defaultTLS = "false"
defaultTLSSkipVerify = "false"
defaultTLSCACertPath = "C:\\certs\\ca.pem"
defaultTLSCertPath = "C:\\certs\\cert.pem"
defaultTLSKeyPath = "C:\\certs\\key.pem"
defaultHTTPDisabled = "false"
defaultHTTPEnabled = "false"
defaultSSL = "false"
defaultSnapshotInterval = "5m"
defaultBaseURL = "/"
defaultSecretKeyName = "portainer"
)

View File

@@ -6,7 +6,7 @@ import (
"fmt"
"strings"
"github.com/alecthomas/kingpin/v2"
"gopkg.in/alecthomas/kingpin.v2"
)
type pairList []portainer.Pair

45
api/cli/pairlistbool.go Normal file
View File

@@ -0,0 +1,45 @@
package cli
import (
"strings"
portainer "github.com/portainer/portainer/api"
"gopkg.in/alecthomas/kingpin.v2"
)
type pairListBool []portainer.Pair
// Set implementation for a list of portainer.Pair
func (l *pairListBool) Set(value string) error {
p := new(portainer.Pair)
// default to true. example setting=true is equivalent to setting
parts := strings.SplitN(value, "=", 2)
if len(parts) != 2 {
p.Name = parts[0]
p.Value = "true"
} else {
p.Name = parts[0]
p.Value = parts[1]
}
*l = append(*l, *p)
return nil
}
// String implementation for a list of pair
func (l *pairListBool) String() string {
return ""
}
// IsCumulative implementation for a list of pair
func (l *pairListBool) IsCumulative() bool {
return true
}
func BoolPairs(s kingpin.Settings) (target *[]portainer.Pair) {
target = new([]portainer.Pair)
s.SetValue((*pairListBool)(target))
return
}

56
api/cmd/portainer/log.go Normal file
View File

@@ -0,0 +1,56 @@
package main
import (
"fmt"
stdlog "log"
"os"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/rs/zerolog/pkgerrors"
)
func configureLogger() {
zerolog.ErrorStackFieldName = "stack_trace"
zerolog.ErrorStackMarshaler = pkgerrors.MarshalStack
zerolog.TimeFieldFormat = zerolog.TimeFormatUnix
stdlog.SetFlags(0)
stdlog.SetOutput(log.Logger)
log.Logger = log.Logger.With().Caller().Stack().Logger()
}
func setLoggingLevel(level string) {
switch level {
case "ERROR":
zerolog.SetGlobalLevel(zerolog.ErrorLevel)
case "WARN":
zerolog.SetGlobalLevel(zerolog.WarnLevel)
case "INFO":
zerolog.SetGlobalLevel(zerolog.InfoLevel)
case "DEBUG":
zerolog.SetGlobalLevel(zerolog.DebugLevel)
}
}
func setLoggingMode(mode string) {
switch mode {
case "PRETTY":
log.Logger = log.Output(zerolog.ConsoleWriter{
Out: os.Stderr,
TimeFormat: "2006/01/02 03:04PM",
FormatMessage: formatMessage,
})
case "JSON":
log.Logger = log.Output(os.Stderr)
}
}
func formatMessage(i interface{}) string {
if i == nil {
return ""
}
return fmt.Sprintf("%s |", i)
}

View File

@@ -1,20 +1,17 @@
package main
import (
"cmp"
"context"
"crypto/sha256"
nethttp "net/http"
"os"
"path"
"strings"
"time"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/apikey"
"github.com/portainer/portainer/api/build"
"github.com/portainer/portainer/api/chisel"
"github.com/portainer/portainer/api/cli"
"github.com/portainer/portainer/api/containerautomation"
"github.com/portainer/portainer/api/crypto"
"github.com/portainer/portainer/api/database"
"github.com/portainer/portainer/api/database/boltdb"
@@ -22,17 +19,18 @@ import (
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/datastore"
"github.com/portainer/portainer/api/datastore/migrator"
"github.com/portainer/portainer/api/datastore/postinit"
"github.com/portainer/portainer/api/demo"
"github.com/portainer/portainer/api/docker"
dockerclient "github.com/portainer/portainer/api/docker/client"
"github.com/portainer/portainer/api/exec"
"github.com/portainer/portainer/api/filesystem"
"github.com/portainer/portainer/api/git"
"github.com/portainer/portainer/api/hostmanagement/openamt"
"github.com/portainer/portainer/api/http"
"github.com/portainer/portainer/api/http/proxy"
kubeproxy "github.com/portainer/portainer/api/http/proxy/factory/kubernetes"
"github.com/portainer/portainer/api/http/security/setuptoken"
"github.com/portainer/portainer/api/internal/authorization"
"github.com/portainer/portainer/api/internal/edge"
"github.com/portainer/portainer/api/internal/edge/edgestacks"
"github.com/portainer/portainer/api/internal/endpointutils"
"github.com/portainer/portainer/api/internal/snapshot"
@@ -42,40 +40,28 @@ import (
"github.com/portainer/portainer/api/kubernetes"
kubecli "github.com/portainer/portainer/api/kubernetes/cli"
"github.com/portainer/portainer/api/ldap"
"github.com/portainer/portainer/api/logs"
"github.com/portainer/portainer/api/oauth"
"github.com/portainer/portainer/api/pendingactions"
"github.com/portainer/portainer/api/pendingactions/actions"
"github.com/portainer/portainer/api/pendingactions/handlers"
"github.com/portainer/portainer/api/platform"
"github.com/portainer/portainer/api/scheduler"
"github.com/portainer/portainer/api/stacks/deployments"
"github.com/portainer/portainer/pkg/build"
"github.com/portainer/portainer/pkg/featureflags"
"github.com/portainer/portainer/pkg/fips"
"github.com/portainer/portainer/pkg/libhelm"
"github.com/portainer/portainer/pkg/libhttp/ssrf"
"github.com/portainer/portainer/pkg/libstack"
"github.com/portainer/portainer/pkg/libstack/compose"
libswarm "github.com/portainer/portainer/pkg/libstack/swarm"
"github.com/portainer/portainer/pkg/validate"
gogitclient "github.com/go-git/go-git/v5/plumbing/transport/client"
gogitraw "github.com/go-git/go-git/v5/plumbing/transport/git"
gogithttp "github.com/go-git/go-git/v5/plumbing/transport/http"
gogitssh "github.com/go-git/go-git/v5/plumbing/transport/ssh"
"github.com/google/uuid"
"github.com/gofrs/uuid"
"github.com/rs/zerolog/log"
)
func initCLI() *portainer.CLIFlags {
cliService := cli.Service{}
var cliService portainer.CLIService = &cli.Service{}
flags, err := cliService.ParseFlags(portainer.APIVersion)
if err != nil {
log.Fatal().Err(err).Msg("failed parsing flags")
}
if err := cliService.ValidateFlags(flags); err != nil {
err = cliService.ValidateFlags(flags)
if err != nil {
log.Fatal().Err(err).Msg("failed validating flags")
}
@@ -92,7 +78,7 @@ func initFileService(dataStorePath string) portainer.FileService {
}
func initDataStore(flags *portainer.CLIFlags, secretKey []byte, fileService portainer.FileService, shutdownCtx context.Context) dataservices.DataStore {
connection, err := database.NewDatabase("boltdb", *flags.Data, secretKey, *flags.CompactDB)
connection, err := database.NewDatabase("boltdb", *flags.Data, secretKey)
if err != nil {
log.Fatal().Err(err).Msg("failed creating database connection")
}
@@ -105,15 +91,15 @@ func initDataStore(flags *portainer.CLIFlags, secretKey []byte, fileService port
log.Fatal().Msg("failed creating database connection: expecting a boltdb database type but a different one was received")
}
store := datastore.NewStore(flags, fileService, connection)
store := datastore.NewStore(*flags.Data, fileService, connection)
isNew, err := store.Open()
if err != nil {
log.Fatal().Err(err).Msg("failed opening store")
}
if *flags.Rollback {
if err := store.Rollback(false); err != nil {
err := store.Rollback(false)
if err != nil {
log.Fatal().Err(err).Msg("failed rolling back")
}
@@ -122,17 +108,18 @@ func initDataStore(flags *portainer.CLIFlags, secretKey []byte, fileService port
}
// Init sets some defaults - it's basically a migration
if err := store.Init(); err != nil {
err = store.Init()
if err != nil {
log.Fatal().Err(err).Msg("failed initializing data store")
}
if isNew {
instanceId, err := uuid.NewRandom()
instanceId, err := uuid.NewV4()
if err != nil {
log.Fatal().Err(err).Msg("failed generating instance id")
}
migratorInstance := migrator.NewMigrator(&migrator.MigratorParameters{Flags: flags})
migratorInstance := migrator.NewMigrator(&migrator.MigratorParameters{})
migratorCount := migratorInstance.GetMigratorCountOfCurrentAPIVersion()
// from MigrateData
@@ -142,27 +129,28 @@ func initDataStore(flags *portainer.CLIFlags, secretKey []byte, fileService port
InstanceID: instanceId.String(),
MigratorCount: migratorCount,
}
store.VersionService.UpdateVersion(&v)
if err := store.VersionService.UpdateVersion(&v); err != nil {
log.Fatal().Err(err).Msg("failed to update version")
}
if err := updateSettingsFromFlags(store, flags); err != nil {
err = updateSettingsFromFlags(store, flags)
if err != nil {
log.Fatal().Err(err).Msg("failed updating settings from flags")
}
} else if err := store.MigrateData(); err != nil {
log.Fatal().Err(err).Msg("failed migration")
} else {
err = store.MigrateData()
if err != nil {
log.Fatal().Err(err).Msg("failed migration")
}
}
if err := updateSettingsFromFlags(store, flags); err != nil {
err = updateSettingsFromFlags(store, flags)
if err != nil {
log.Fatal().Err(err).Msg("failed updating settings from flags")
}
// this is for the db restore functionality - needs more tests.
go func() {
<-shutdownCtx.Done()
defer logs.CloseAndLogErr(connection)
defer connection.Close()
}()
return store
@@ -178,8 +166,32 @@ func checkDBSchemaServerVersionMatch(dbStore dataservices.DataStore, serverVersi
return v.SchemaVersion == serverVersion && v.Edition == serverEdition
}
func initKubernetesDeployer(kubernetesTokenCacheManager *kubeproxy.TokenCacheManager, kubernetesClientFactory *kubecli.ClientFactory, dataStore dataservices.DataStore, reverseTunnelService portainer.ReverseTunnelService, signatureService portainer.DigitalSignatureService, proxyManager *proxy.Manager) portainer.KubernetesDeployer {
return exec.NewKubernetesDeployer(kubernetesTokenCacheManager, kubernetesClientFactory, dataStore, reverseTunnelService, signatureService, proxyManager)
func initComposeStackManager(composeDeployer libstack.Deployer, proxyManager *proxy.Manager) portainer.ComposeStackManager {
composeWrapper, err := exec.NewComposeStackManager(composeDeployer, proxyManager)
if err != nil {
log.Fatal().Err(err).Msg("failed creating compose manager")
}
return composeWrapper
}
func initSwarmStackManager(
assetsPath string,
configPath string,
signatureService portainer.DigitalSignatureService,
fileService portainer.FileService,
reverseTunnelService portainer.ReverseTunnelService,
dataStore dataservices.DataStore,
) (portainer.SwarmStackManager, error) {
return exec.NewSwarmStackManager(assetsPath, configPath, signatureService, fileService, reverseTunnelService, dataStore)
}
func initKubernetesDeployer(kubernetesTokenCacheManager *kubeproxy.TokenCacheManager, kubernetesClientFactory *kubecli.ClientFactory, dataStore dataservices.DataStore, reverseTunnelService portainer.ReverseTunnelService, signatureService portainer.DigitalSignatureService, proxyManager *proxy.Manager, assetsPath string) portainer.KubernetesDeployer {
return exec.NewKubernetesDeployer(kubernetesTokenCacheManager, kubernetesClientFactory, dataStore, reverseTunnelService, signatureService, proxyManager, assetsPath)
}
func initHelmPackageManager(assetsPath string) (libhelm.HelmPackageManager, error) {
return libhelm.NewHelmPackageManager(libhelm.HelmConfig{BinaryPath: assetsPath})
}
func initAPIKeyService(datastore dataservices.DataStore) apikey.APIKeyService {
@@ -191,16 +203,36 @@ func initJWTService(userSessionTimeout string, dataStore dataservices.DataStore)
userSessionTimeout = portainer.DefaultUserSessionTimeout
}
return jwt.NewService(userSessionTimeout, dataStore)
jwtService, err := jwt.NewService(userSessionTimeout, dataStore)
if err != nil {
return nil, err
}
return jwtService, nil
}
func initDigitalSignatureService() portainer.DigitalSignatureService {
return crypto.NewECDSAService(os.Getenv("AGENT_SECRET"))
}
func initCryptoService() portainer.CryptoService {
return &crypto.Service{}
}
func initLDAPService() portainer.LDAPService {
return &ldap.Service{}
}
func initOAuthService() portainer.OAuthService {
return oauth.NewService()
}
func initGitService(ctx context.Context) portainer.GitService {
return git.NewService(ctx)
}
func initSSLService(addr, certPath, keyPath string, fileService portainer.FileService, dataStore dataservices.DataStore, shutdownTrigger context.CancelFunc) (*ssl.Service, error) {
slices := strings.Split(addr, ":")
host := slices[0]
if host == "" {
host = "0.0.0.0"
@@ -208,24 +240,34 @@ func initSSLService(addr, certPath, keyPath string, fileService portainer.FileSe
sslService := ssl.NewService(fileService, dataStore, shutdownTrigger)
if err := sslService.Init(host, certPath, keyPath); err != nil {
err := sslService.Init(host, certPath, keyPath)
if err != nil {
return nil, err
}
return sslService, nil
}
func initDockerClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService) *dockerclient.ClientFactory {
return dockerclient.NewClientFactory(signatureService, reverseTunnelService)
}
func initKubernetesClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService, dataStore dataservices.DataStore, instanceID, addrHTTPS, userSessionTimeout string) (*kubecli.ClientFactory, error) {
return kubecli.NewClientFactory(signatureService, reverseTunnelService, dataStore, instanceID, addrHTTPS, userSessionTimeout)
}
func initSnapshotService(
snapshotIntervalFromFlag string,
dataStore dataservices.DataStore,
dockerClientFactory *dockerclient.ClientFactory,
kubernetesClientFactory *kubecli.ClientFactory,
shutdownCtx context.Context,
pendingActionsService *pendingactions.PendingActionsService,
) (portainer.SnapshotService, error) {
dockerSnapshotter := docker.NewSnapshotter(dockerClientFactory)
kubernetesSnapshotter := kubernetes.NewSnapshotter(kubernetesClientFactory)
snapshotService, err := snapshot.NewService(snapshotIntervalFromFlag, dataStore, dockerSnapshotter, kubernetesSnapshotter, pendingActionsService)
snapshotService, err := snapshot.NewService(snapshotIntervalFromFlag, dataStore, dockerSnapshotter, kubernetesSnapshotter, shutdownCtx, pendingActionsService)
if err != nil {
return nil, err
}
@@ -233,32 +275,6 @@ func initSnapshotService(
return snapshotService, nil
}
func resolveSetupToken(tx dataservices.DataStoreTx, providedToken string) (string, error) {
admins, err := tx.User().UsersByRole(portainer.AdministratorRole)
if err != nil {
return "", err
}
if len(admins) > 0 {
return "", nil
}
if providedToken != "" {
log.Info().Msg("using custom setup token; admin initialization and backup restore require this token in the X-Setup-Token header")
return providedToken, nil
}
token, err := setuptoken.Generate()
if err != nil {
return "", err
}
log.Info().
Str("setup_token", token).
Msg("no administrator account configured; admin initialization and backup restore require this setup token in the X-Setup-Token header. Start with --no-setup-token to disable.")
return token, nil
}
func initStatus(instanceID string) *portainer.Status {
return &portainer.Status{
Version: portainer.APIVersion,
@@ -272,25 +288,34 @@ func updateSettingsFromFlags(dataStore dataservices.DataStore, flags *portainer.
return err
}
settings.SnapshotInterval = cmp.Or(*flags.SnapshotInterval, settings.SnapshotInterval)
settings.LogoURL = cmp.Or(*flags.Logo, settings.LogoURL)
settings.EnableEdgeComputeFeatures = cmp.Or(*flags.EnableEdgeComputeFeatures, settings.EnableEdgeComputeFeatures)
settings.TemplatesURL = cmp.Or(*flags.Templates, settings.TemplatesURL)
if *flags.SnapshotInterval != "" {
settings.SnapshotInterval = *flags.SnapshotInterval
}
if flags.KubectlShellImageSet {
settings.KubectlShellImage = *flags.KubectlShellImage
if *flags.Logo != "" {
settings.LogoURL = *flags.Logo
}
if *flags.EnableEdgeComputeFeatures {
settings.EnableEdgeComputeFeatures = *flags.EnableEdgeComputeFeatures
}
if *flags.Templates != "" {
settings.TemplatesURL = *flags.Templates
}
if *flags.Labels != nil {
settings.BlackListedLabels = *flags.Labels
}
settings.AgentSecret = ""
if agentKey, ok := os.LookupEnv("AGENT_SECRET"); ok {
settings.AgentSecret = agentKey
} else {
settings.AgentSecret = ""
}
if err := dataStore.Settings().UpdateSettings(settings); err != nil {
err = dataStore.Settings().UpdateSettings(settings)
if err != nil {
return err
}
@@ -313,7 +338,6 @@ func loadAndParseKeyPair(fileService portainer.FileService, signatureService por
if err != nil {
return err
}
return signatureService.ParseKeyPair(private, public)
}
@@ -322,9 +346,7 @@ func generateAndStoreKeyPair(fileService portainer.FileService, signatureService
if err != nil {
return err
}
privateHeader, publicHeader := signatureService.PEMHeaders()
return fileService.StoreKeyPair(private, public, privateHeader, publicHeader)
}
@@ -337,23 +359,11 @@ func initKeyPair(fileService portainer.FileService, signatureService portainer.D
if existingKeyPair {
return loadAndParseKeyPair(fileService, signatureService)
}
return generateAndStoreKeyPair(fileService, signatureService)
}
// dbSecretPath build the path to the file that contains the db encryption
// secret. Normally in Docker this is built from the static path inside
// /run/secrets for example: /run/secrets/<keyFilenameFlag> but for ease of
// use outside Docker it also accepts an absolute path
func dbSecretPath(keyFilenameFlag string) string {
if path.IsAbs(keyFilenameFlag) {
return keyFilenameFlag
}
return path.Join("/run/secrets", keyFilenameFlag)
}
func loadEncryptionSecretKey(keyfilename string) []byte {
content, err := os.ReadFile(keyfilename)
content, err := os.ReadFile(path.Join("/run/secrets", keyfilename))
if err != nil {
if os.IsNotExist(err) {
log.Info().Str("filename", keyfilename).Msg("encryption key file not present")
@@ -365,34 +375,19 @@ func loadEncryptionSecretKey(keyfilename string) []byte {
}
// return a 32 byte hash of the secret (required for AES)
// fips compliant version of this is not implemented in -ce
hash := sha256.Sum256(content)
return hash[:]
}
func buildServer(flags *portainer.CLIFlags, shutdownCtx context.Context, shutdownTrigger context.CancelFunc) portainer.Server {
func buildServer(flags *portainer.CLIFlags) portainer.Server {
shutdownCtx, shutdownTrigger := context.WithCancel(context.Background())
if flags.FeatureFlags != nil {
featureflags.Parse(*flags.FeatureFlags, portainer.SupportedFeatureFlags)
}
trustedOrigins := []string{}
if *flags.TrustedOrigins != "" {
// validate if the trusted origins are valid urls
for origin := range strings.SplitSeq(*flags.TrustedOrigins, ",") {
if !validate.IsTrustedOrigin(origin) {
log.Fatal().Str("trusted_origin", origin).Msg("invalid trusted origin: must be scheme://host or scheme://host:port (e.g. https://example.com)")
}
trustedOrigins = append(trustedOrigins, origin)
}
}
// -ce can not ever be run in FIPS mode
fips.InitFIPS(false)
fileService := initFileService(*flags.Data)
encryptionKey := loadEncryptionSecretKey(dbSecretPath(*flags.SecretKeyName))
encryptionKey := loadEncryptionSecretKey(*flags.SecretKeyName)
if encryptionKey == nil {
log.Info().Msg("proceeding without encryption key")
}
@@ -408,19 +403,6 @@ func buildServer(flags *portainer.CLIFlags, shutdownCtx context.Context, shutdow
log.Fatal().Msg("The database schema version does not align with the server version. Please consider reverting to the previous server version or addressing the database migration issue.")
}
if err := ssrf.Configure(dataStore.AllowList()); err != nil {
log.Fatal().Err(err).Msg("failed initializing ssrf service")
}
if !ssrf.WrapDefaultTransport() {
log.Fatal().Msg("failed to wrap default HTTP transport with SSRF protection")
}
gogithttp.DefaultClient = gogithttp.NewClient(&nethttp.Client{Transport: nethttp.DefaultTransport})
gogitclient.InstallProtocol("git", git.NewSSRFGitTransport(gogitraw.DefaultClient))
gogitclient.InstallProtocol("ssh", git.NewSSRFGitTransport(gogitssh.DefaultClient))
gogitclient.InstallProtocol("file", nil)
instanceID, err := dataStore.Version().InstanceID()
if err != nil {
log.Fatal().Err(err).Msg("failed getting instance id")
@@ -438,19 +420,21 @@ func buildServer(flags *portainer.CLIFlags, shutdownCtx context.Context, shutdow
log.Fatal().Err(err).Msg("failed initializing JWT service")
}
ldapService := ldap.Service{}
ldapService := initLDAPService()
oauthService := oauth.NewService()
oauthService := initOAuthService()
gitService := git.NewService(shutdownCtx)
gitService := initGitService(shutdownCtx)
cryptoService := crypto.Service{}
openAMTService := openamt.NewService()
signatureService := initDigitalSignatureService()
cryptoService := initCryptoService()
digitalSignatureService := initDigitalSignatureService()
edgeStacksService := edgestacks.NewService(dataStore)
sslService, err := initSSLService(*flags.AddrHTTPS, *flags.TLSCert, *flags.TLSKey, fileService, dataStore, shutdownTrigger)
sslService, err := initSSLService(*flags.AddrHTTPS, *flags.SSLCert, *flags.SSLKey, fileService, dataStore, shutdownTrigger)
if err != nil {
log.Fatal().Err(err).Msg("")
}
@@ -460,63 +444,77 @@ func buildServer(flags *portainer.CLIFlags, shutdownCtx context.Context, shutdow
log.Fatal().Err(err).Msg("failed to get SSL settings")
}
if err := initKeyPair(fileService, signatureService); err != nil {
err = initKeyPair(fileService, digitalSignatureService)
if err != nil {
log.Fatal().Err(err).Msg("failed initializing key pair")
}
reverseTunnelService := chisel.NewService(dataStore, shutdownCtx, fileService)
dockerClientFactory := dockerclient.NewClientFactory(signatureService, reverseTunnelService)
kubernetesClientFactory, err := kubecli.NewClientFactory(signatureService, reverseTunnelService, dataStore, instanceID, *flags.AddrHTTPS, settings.UserSessionTimeout)
if err != nil {
log.Fatal().Err(err).Msg("failed initializing Kubernetes Client Factory service")
}
dockerClientFactory := initDockerClientFactory(digitalSignatureService, reverseTunnelService)
kubernetesClientFactory, err := initKubernetesClientFactory(digitalSignatureService, reverseTunnelService, dataStore, instanceID, *flags.AddrHTTPS, settings.UserSessionTimeout)
authorizationService := authorization.NewService(dataStore)
authorizationService.K8sClientFactory = kubernetesClientFactory
pendingActionsService := pendingactions.NewService(dataStore, kubernetesClientFactory, authorizationService, shutdownCtx)
snapshotService, err := initSnapshotService(*flags.SnapshotInterval, dataStore, dockerClientFactory, kubernetesClientFactory, shutdownCtx, pendingActionsService)
if err != nil {
log.Fatal().Err(err).Msg("failed initializing snapshot service")
}
snapshotService.Start()
kubernetesTokenCacheManager := kubeproxy.NewTokenCacheManager()
kubeClusterAccessService := kubernetes.NewKubeClusterAccessService(*flags.BaseURL, *flags.AddrHTTPS, sslSettings.CertPath)
proxyManager := proxy.NewManager(kubernetesClientFactory)
proxyManager := proxy.NewManager(dataStore, digitalSignatureService, reverseTunnelService, dockerClientFactory, kubernetesClientFactory, kubernetesTokenCacheManager, gitService)
reverseTunnelService.ProxyManager = proxyManager
composeDeployer := compose.NewComposeDeployer()
dockerConfigPath := fileService.GetDockerConfigPath()
composeStackManager := exec.NewComposeStackManager(composeDeployer, proxyManager)
swarmStackManager := exec.NewSwarmStackManager(libswarm.NewSwarmDeployer(), proxyManager)
kubernetesDeployer := initKubernetesDeployer(kubernetesTokenCacheManager, kubernetesClientFactory, dataStore, reverseTunnelService, signatureService, proxyManager)
pendingActionsService := pendingactions.NewService(dataStore, kubernetesClientFactory)
pendingActionsService.RegisterHandler(actions.CleanNAPWithOverridePolicies, handlers.NewHandlerCleanNAPWithOverridePolicies(authorizationService, dataStore))
pendingActionsService.RegisterHandler(actions.DeletePortainerK8sRegistrySecrets, handlers.NewHandlerDeleteRegistrySecrets(authorizationService, dataStore, kubernetesClientFactory))
pendingActionsService.RegisterHandler(actions.PostInitMigrateEnvironment, handlers.NewHandlerPostInitMigrateEnvironment(authorizationService, dataStore, kubernetesClientFactory, dockerClientFactory, *flags.Assets, kubernetesDeployer))
snapshotService, err := initSnapshotService(*flags.SnapshotInterval, dataStore, dockerClientFactory, kubernetesClientFactory, pendingActionsService)
composeDeployer, err := compose.NewComposeDeployer(*flags.Assets, dockerConfigPath)
if err != nil {
log.Fatal().Err(err).Msg("failed initializing snapshot service")
log.Fatal().Err(err).Msg("failed initializing compose deployer")
}
snapshotService.Start(shutdownCtx)
composeStackManager := initComposeStackManager(composeDeployer, proxyManager)
proxyManager.NewProxyFactory(dataStore, signatureService, reverseTunnelService, dockerClientFactory, kubernetesClientFactory, kubernetesTokenCacheManager, gitService, snapshotService, jwtService)
swarmStackManager, err := initSwarmStackManager(*flags.Assets, dockerConfigPath, digitalSignatureService, fileService, reverseTunnelService, dataStore)
if err != nil {
log.Fatal().Err(err).Msg("failed initializing swarm stack manager")
}
helmPackageManager := libhelm.NewHelmPackageManager()
kubernetesDeployer := initKubernetesDeployer(kubernetesTokenCacheManager, kubernetesClientFactory, dataStore, reverseTunnelService, digitalSignatureService, proxyManager, *flags.Assets)
helmPackageManager, err := initHelmPackageManager(*flags.Assets)
if err != nil {
log.Fatal().Err(err).Msg("failed initializing helm package manager")
}
err = edge.LoadEdgeJobs(dataStore, reverseTunnelService)
if err != nil {
log.Fatal().Err(err).Msg("failed loading edge jobs from database")
}
applicationStatus := initStatus(instanceID)
demoService := demo.NewService()
if *flags.DemoEnvironment {
err := demoService.Init(dataStore, cryptoService)
if err != nil {
log.Fatal().Err(err).Msg("failed initializing demo environment")
}
}
// channel to control when the admin user is created
adminCreationDone := make(chan struct{}, 1)
go endpointutils.InitEndpoint(shutdownCtx, adminCreationDone, flags, dataStore, snapshotService)
adminPasswordHash := ""
if *flags.AdminPasswordFile != "" {
content, err := fileService.GetFileContent(*flags.AdminPasswordFile, "")
if err != nil {
@@ -539,14 +537,14 @@ func buildServer(flags *portainer.CLIFlags, shutdownCtx context.Context, shutdow
if len(users) == 0 {
log.Info().Msg("created admin user with the given password.")
user := &portainer.User{
Username: "admin",
Role: portainer.AdministratorRole,
Password: adminPasswordHash,
}
if err := dataStore.User().Create(user); err != nil {
err := dataStore.User().Create(user)
if err != nil {
log.Fatal().Err(err).Msg("failed creating admin user")
}
@@ -557,47 +555,21 @@ func buildServer(flags *portainer.CLIFlags, shutdownCtx context.Context, shutdow
}
}
setupToken := ""
if adminPasswordHash == "" && !*flags.NoSetupToken {
if err := dataStore.ViewTx(func(tx dataservices.DataStoreTx) error {
var txErr error
setupToken, txErr = resolveSetupToken(tx, *flags.SetupToken)
return txErr
}); err != nil {
log.Fatal().Err(err).Msg("failed initializing setup token")
}
}
if err := reverseTunnelService.StartTunnelServer(*flags.TunnelAddr, *flags.TunnelPort, snapshotService); err != nil {
err = reverseTunnelService.StartTunnelServer(*flags.TunnelAddr, *flags.TunnelPort, snapshotService)
if err != nil {
log.Fatal().Err(err).Msg("failed starting tunnel server")
}
scheduler := scheduler.NewScheduler(shutdownCtx)
stackDeployer := deployments.NewStackDeployer(swarmStackManager, composeStackManager, kubernetesDeployer, dockerClientFactory, dataStore)
if err := deployments.StartStackSchedules(scheduler, stackDeployer, dataStore, gitService); err != nil {
log.Fatal().Err(err).Msg("failed to start stack scheduler")
}
containerService := docker.NewContainerService(dockerClientFactory, dataStore)
containerAutomationService := containerautomation.NewService(shutdownCtx, scheduler, dataStore, dockerClientFactory, containerService, stackDeployer)
containerAutomationService.Start()
deployments.StartStackSchedules(scheduler, stackDeployer, dataStore, gitService)
sslDBSettings, err := dataStore.SSLSettings().Settings()
if err != nil {
log.Fatal().Msg("failed to fetch SSL settings from DB")
}
platformService := platform.NewService(dataStore)
upgradeService, err := upgrade.NewService(
*flags.Assets,
kubernetesClientFactory,
dockerClientFactory,
composeStackManager,
dataStore,
fileService,
stackDeployer,
)
upgradeService, err := upgrade.NewService(*flags.Assets, composeDeployer, kubernetesClientFactory)
if err != nil {
log.Fatal().Err(err).Msg("failed initializing upgrade service")
}
@@ -606,31 +578,21 @@ func buildServer(flags *portainer.CLIFlags, shutdownCtx context.Context, shutdow
// but some more complex migrations require access to a kubernetes or docker
// client. Therefore we run a separate migration process just before
// starting the server.
postInitMigrator := postinit.NewPostInitMigrator(
postInitMigrator := datastore.NewPostInitMigrator(
kubernetesClientFactory,
dockerClientFactory,
dataStore,
*flags.Assets,
kubernetesDeployer,
)
if err := postInitMigrator.PostInitMigrate(); err != nil {
log.Fatal().Err(err).Msg("failure during post init migrations")
}
if err := dataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
return recoverStaleDeployingStacks(tx)
}); err != nil {
log.Info().Err(err).
Msg("Error recovering stale deploying stacks")
}
return &http.Server{
AuthorizationService: authorizationService,
ReverseTunnelService: reverseTunnelService,
Status: applicationStatus,
BindAddress: *flags.Addr,
BindAddressHTTPS: *flags.AddrHTTPS,
CSP: *flags.CSP,
HTTPEnabled: sslDBSettings.HTTPEnabled,
AssetsPath: *flags.Assets,
DataStore: dataStore,
@@ -646,89 +608,48 @@ func buildServer(flags *portainer.CLIFlags, shutdownCtx context.Context, shutdow
LDAPService: ldapService,
OAuthService: oauthService,
GitService: gitService,
OpenAMTService: openAMTService,
ProxyManager: proxyManager,
KubernetesTokenCacheManager: kubernetesTokenCacheManager,
KubeClusterAccessService: kubeClusterAccessService,
SignatureService: signatureService,
SignatureService: digitalSignatureService,
SnapshotService: snapshotService,
SSLService: sslService,
DockerClientFactory: dockerClientFactory,
KubernetesClientFactory: kubernetesClientFactory,
Scheduler: scheduler,
ContainerAutomationService: containerAutomationService,
ShutdownCtx: shutdownCtx,
ShutdownTrigger: shutdownTrigger,
StackDeployer: stackDeployer,
DemoService: demoService,
UpgradeService: upgradeService,
AdminCreationDone: adminCreationDone,
PendingActionsService: pendingActionsService,
PlatformService: platformService,
PullLimitCheckDisabled: *flags.PullLimitCheckDisabled,
TrustedOrigins: trustedOrigins,
SetupToken: setupToken,
}
}
func main() {
logs.ConfigureLogger()
logs.SetLoggingMode("PRETTY")
configureLogger()
setLoggingMode("PRETTY")
flags := initCLI()
logs.SetLoggingLevel(*flags.LogLevel)
logs.SetLoggingMode(*flags.LogMode)
setLoggingLevel(*flags.LogLevel)
setLoggingMode(*flags.LogMode)
for {
shutdownCtx, shutdownTrigger := context.WithCancel(context.Background())
server := buildServer(flags, shutdownCtx, shutdownTrigger)
server := buildServer(flags)
log.Info().
Str("version", portainer.APIVersion).
Str("build_number", build.BuildNumber).
Str("image_tag", build.ImageTag).
Str("nodejs_version", build.NodejsVersion).
Str("pnpm_version", build.PnpmVersion).
Str("yarn_version", build.YarnVersion).
Str("webpack_version", build.WebpackVersion).
Str("go_version", build.GoVersion).
Msg("starting Portainer")
err := server.Start(shutdownCtx)
err := server.Start()
log.Info().Err(err).Msg("HTTP server exited")
}
}
// recoverStaleDeployingStacks resets any stack that was left in the Deploying state
// (e.g. because the server was restarted mid-deployment) to the Error state so the
// user can retry.
func recoverStaleDeployingStacks(tx dataservices.DataStoreTx) error {
stacks, err := tx.Stack().ReadAll(func(s portainer.Stack) bool {
return s.Status == portainer.StackStatusDeploying
})
if err != nil {
return err
}
for _, stack := range stacks {
stack.Status = portainer.StackStatusError
stack.DeploymentStatus = append(stack.DeploymentStatus, portainer.StackDeploymentStatus{
Status: portainer.StackStatusError,
Time: time.Now().Unix(),
Message: "Deployment interrupted by server restart",
})
if err := tx.Stack().Update(stack.ID, &stack); err != nil {
log.Warn().Err(err).
Int("stack_id", int(stack.ID)).
Str("context", "RecoverStaleDeployingStacks").
Msg("Unable to recover stale deploying stack")
continue
}
log.Debug().
Int("stack_id", int(stack.ID)).
Str("stack_name", stack.Name).
Str("context", "RecoverStaleDeployingStacks").
Msg("Recovered stale deploying stack to error state")
}
return nil
}

View File

@@ -1,159 +0,0 @@
package main
import (
"os"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/filesystem"
"github.com/portainer/portainer/api/internal/testhelpers"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func Test_resolveSetupToken(t *testing.T) {
t.Parallel()
t.Run("admin already exists — returns empty token", func(t *testing.T) {
admin := portainer.User{Role: portainer.AdministratorRole}
store := testhelpers.NewDatastore(testhelpers.WithUsers([]portainer.User{admin}))
token, err := resolveSetupToken(store, "")
require.NoError(t, err)
assert.Empty(t, token)
})
t.Run("no admin — generates a 64-char hex token", func(t *testing.T) {
store := testhelpers.NewDatastore(testhelpers.WithUsers([]portainer.User{}))
token, err := resolveSetupToken(store, "")
require.NoError(t, err)
assert.Len(t, token, 64)
token2, err := resolveSetupToken(store, "")
require.NoError(t, err)
assert.NotEqual(t, token, token2)
})
t.Run("no admin — uses provided token", func(t *testing.T) {
store := testhelpers.NewDatastore(testhelpers.WithUsers([]portainer.User{}))
token, err := resolveSetupToken(store, "mysecrettoken")
require.NoError(t, err)
assert.Equal(t, "mysecrettoken", token)
})
t.Run("admin already exists — ignores provided token", func(t *testing.T) {
admin := portainer.User{Role: portainer.AdministratorRole}
store := testhelpers.NewDatastore(testhelpers.WithUsers([]portainer.User{admin}))
token, err := resolveSetupToken(store, "mysecrettoken")
require.NoError(t, err)
assert.Empty(t, token)
})
}
const secretFileName = "secret.txt"
func createPasswordFile(t *testing.T, secretPath, password string) string {
err := os.WriteFile(secretPath, []byte(password), 0o600)
require.NoError(t, err)
return secretPath
}
func TestLoadEncryptionSecretKey(t *testing.T) {
t.Parallel()
tempDir := t.TempDir()
secretPath := filesystem.JoinPaths(tempDir, secretFileName)
// first pointing to file that does not exist, gives nil hash (no encryption)
encryptionKey := loadEncryptionSecretKey(secretPath)
require.Nil(t, encryptionKey)
// point to a directory instead of a file
encryptionKey = loadEncryptionSecretKey(tempDir)
require.Nil(t, encryptionKey)
password := "portainer@1234"
createPasswordFile(t, secretPath, password)
encryptionKey = loadEncryptionSecretKey(secretPath)
require.NotNil(t, encryptionKey)
// should be 32 bytes for aes256 encryption
require.Len(t, encryptionKey, 32)
}
func TestUpdateSettingsFromFlags_KubectlShellImage(t *testing.T) {
const existingImage = "existing-image:v1"
const newImage = "new-image:v2"
emptyString := ""
falseBool := false
var emptyLabels []portainer.Pair
tests := []struct {
name string
imageSet bool
flagImage string
expectedKubectlShellImage string
}{
{
name: "flag not set — DB image unchanged",
imageSet: false,
flagImage: portainer.DefaultKubectlShellImage,
expectedKubectlShellImage: existingImage,
},
{
name: "flag set — DB image updated",
imageSet: true,
flagImage: newImage,
expectedKubectlShellImage: newImage,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
store := testhelpers.NewDatastore(
testhelpers.WithSettingsService(&portainer.Settings{
KubectlShellImage: existingImage,
}),
testhelpers.WithSSLSettingsService(&portainer.SSLSettings{}),
)
flags := &portainer.CLIFlags{
SnapshotInterval: &emptyString,
Logo: &emptyString,
EnableEdgeComputeFeatures: &falseBool,
Templates: &emptyString,
Labels: &emptyLabels,
HTTPDisabled: &falseBool,
HTTPEnabled: &falseBool,
}
flags.KubectlShellImage = &tc.flagImage
flags.KubectlShellImageSet = tc.imageSet
err := updateSettingsFromFlags(store, flags)
require.NoError(t, err)
settings, err := store.Settings().Settings()
require.NoError(t, err)
require.Equal(t, tc.expectedKubectlShellImage, settings.KubectlShellImage)
})
}
}
func TestDBSecretPath(t *testing.T) {
t.Parallel()
tests := []struct {
keyFilenameFlag string
expected string
}{
{keyFilenameFlag: "secret.txt", expected: "/run/secrets/secret.txt"},
{keyFilenameFlag: "/tmp/secret.txt", expected: "/tmp/secret.txt"},
{keyFilenameFlag: "/run/secrets/secret.txt", expected: "/run/secrets/secret.txt"},
{keyFilenameFlag: "./secret.txt", expected: "/run/secrets/secret.txt"},
{keyFilenameFlag: "../secret.txt", expected: "/run/secret.txt"},
{keyFilenameFlag: "foo/bar/secret.txt", expected: "/run/secrets/foo/bar/secret.txt"},
}
for _, test := range tests {
assert.Equal(t, test.expected, dbSecretPath(test.keyFilenameFlag))
}
}

View File

@@ -1,148 +0,0 @@
// Package concurrent provides utilities for running multiple functions concurrently in Go.
// For example, many kubernetes calls can take a while to fulfill. Oftentimes in Portainer
// we need to get a list of objects from multiple kubernetes REST APIs. We can often call these
// apis concurrently to speed up the response time.
// This package provides a clean way to do just that.
//
// Examples:
// The ConfigMaps and Secrets function converted using concurrent.Run.
/*
// GetConfigMapsAndSecrets gets all the ConfigMaps AND all the Secrets for a
// given namespace in a k8s endpoint. The result is a list of both config maps
// and secrets. The IsSecret boolean property indicates if a given struct is a
// secret or configmap.
func (kcl *KubeClient) GetConfigMapsAndSecrets(namespace string) ([]models.K8sConfigMapOrSecret, error) {
// use closures to capture the current kube client and namespace by declaring wrapper functions
// that match the interface signature for concurrent.Func
listConfigMaps := func(ctx context.Context) (any, error) {
return kcl.cli.CoreV1().ConfigMaps(namespace).List(context.Background(), meta.ListOptions{})
}
listSecrets := func(ctx context.Context) (any, error) {
return kcl.cli.CoreV1().Secrets(namespace).List(context.Background(), meta.ListOptions{})
}
// run the functions concurrently and wait for results. We can also pass in a context to cancel.
// e.g. Deadline timer.
results, err := concurrent.Run(context.TODO(), listConfigMaps, listSecrets)
if err != nil {
return nil, err
}
var configMapList *core.ConfigMapList
var secretList *core.SecretList
for _, r := range results {
switch v := r.Result.(type) {
case *core.ConfigMapList:
configMapList = v
case *core.SecretList:
secretList = v
}
}
// TODO: Applications
var combined []models.K8sConfigMapOrSecret
for _, m := range configMapList.Items {
var cm models.K8sConfigMapOrSecret
cm.UID = string(m.UID)
cm.Name = m.Name
cm.Namespace = m.Namespace
cm.Annotations = m.Annotations
cm.Data = m.Data
cm.CreationDate = m.CreationTimestamp.Time.UTC().Format(time.RFC3339)
combined = append(combined, cm)
}
for _, s := range secretList.Items {
var secret models.K8sConfigMapOrSecret
secret.UID = string(s.UID)
secret.Name = s.Name
secret.Namespace = s.Namespace
secret.Annotations = s.Annotations
secret.Data = msbToMss(s.Data)
secret.CreationDate = s.CreationTimestamp.Time.UTC().Format(time.RFC3339)
secret.IsSecret = true
secret.SecretType = string(s.Type)
combined = append(combined, secret)
}
return combined, nil
}
*/
package concurrent
import (
"context"
"sync"
)
// Result contains the result and any error returned from running a client task function
type Result struct {
Result any // the result of running the task function
Err error // any error that occurred while running the task function
}
// Func is a function returns a result or error
type Func func(ctx context.Context) (any, error)
// Run runs a list of functions returns the results
func Run(ctx context.Context, maxConcurrency int, tasks ...Func) ([]Result, error) {
var wg sync.WaitGroup
resultsChan := make(chan Result, len(tasks))
taskChan := make(chan Func, len(tasks))
localCtx, cancelCtx := context.WithCancel(ctx)
defer cancelCtx()
runTask := func() {
defer wg.Done()
for fn := range taskChan {
result, err := fn(localCtx)
resultsChan <- Result{Result: result, Err: err}
}
}
// Set maxConcurrency to the number of tasks if zero or negative
if maxConcurrency <= 0 {
maxConcurrency = len(tasks)
}
// Start worker goroutines
for range maxConcurrency {
wg.Add(1)
go runTask()
}
// Add tasks to the task channel
for _, fn := range tasks {
taskChan <- fn
}
// Close the task channel to signal workers to stop when all tasks are done
close(taskChan)
// Wait for all workers to complete
wg.Wait()
close(resultsChan)
// Collect the results and cancel on error
results := make([]Result, 0, len(tasks))
for r := range resultsChan {
if r.Err != nil {
cancelCtx()
return nil, r.Err
}
results = append(results, r)
}
return results, nil
}

View File

@@ -1,149 +0,0 @@
package concurrent
import (
"context"
"errors"
"sync/atomic"
"testing"
"testing/synctest"
"time"
"github.com/stretchr/testify/require"
)
func TestRun_AllSucceed(t *testing.T) {
t.Parallel()
fn1 := func(ctx context.Context) (any, error) { return "one", nil }
fn2 := func(ctx context.Context) (any, error) { return "two", nil }
fn3 := func(ctx context.Context) (any, error) { return "three", nil }
results, err := Run(t.Context(), 0, fn1, fn2, fn3)
require.NoError(t, err)
require.Len(t, results, 3)
values := make([]string, 0, len(results))
for _, r := range results {
values = append(values, r.Result.(string))
}
require.ElementsMatch(t, []string{"one", "two", "three"}, values)
}
func TestRun_OneError(t *testing.T) {
t.Parallel()
sentinel := errors.New("task failed")
fn1 := func(ctx context.Context) (any, error) { return "ok", nil }
fn2 := func(ctx context.Context) (any, error) { return nil, sentinel }
_, err := Run(t.Context(), 0, fn1, fn2)
require.ErrorIs(t, err, sentinel)
}
func TestRun_NoTasks(t *testing.T) {
t.Parallel()
results, err := Run(t.Context(), 0)
require.NoError(t, err)
require.Empty(t, results)
}
func TestRun_MaxConcurrency(t *testing.T) {
t.Parallel()
const numTasks = 10
var peak atomic.Int32
var active atomic.Int32
task := func(ctx context.Context) (any, error) {
current := active.Add(1)
if current > peak.Load() {
peak.Store(current)
}
time.Sleep(10 * time.Millisecond)
active.Add(-1)
return nil, nil
}
tasks := make([]Func, numTasks)
for i := range tasks {
tasks[i] = task
}
synctest.Test(t, func(t *testing.T) {
results, err := Run(t.Context(), 3, tasks...)
require.NoError(t, err)
require.Len(t, results, numTasks)
require.LessOrEqual(t, peak.Load(), int32(3))
})
}
func TestRun_ZeroConcurrencyUsesAllTasks(t *testing.T) {
t.Parallel()
const numTasks = 5
var peak atomic.Int32
var active atomic.Int32
task := func(ctx context.Context) (any, error) {
current := active.Add(1)
if current > peak.Load() {
peak.Store(current)
}
time.Sleep(20 * time.Millisecond)
active.Add(-1)
return nil, nil
}
tasks := make([]Func, numTasks)
for i := range tasks {
tasks[i] = task
}
synctest.Test(t, func(t *testing.T) {
results, err := Run(t.Context(), 0, tasks...)
require.NoError(t, err)
require.Len(t, results, numTasks)
require.Equal(t, int32(numTasks), peak.Load())
})
}
func TestRun_ContextCancelledBeforeStart(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithCancel(t.Context())
cancel()
called := atomic.Bool{}
fn := func(ctx context.Context) (any, error) {
called.Store(true)
return nil, ctx.Err()
}
_, err := Run(ctx, 1, fn, fn, fn)
require.Error(t, err)
}
func TestRun_ContextPassedToTasks(t *testing.T) {
t.Parallel()
type key struct{}
ctx := context.WithValue(t.Context(), key{}, "testvalue")
fn := func(ctx context.Context) (any, error) {
return ctx.Value(key{}), nil
}
results, err := Run(ctx, 0, fn)
require.NoError(t, err)
require.Equal(t, "testvalue", results[0].Result)
}

View File

@@ -5,23 +5,22 @@ import (
)
type ReadTransaction interface {
GetObject(bucketName string, key []byte, object any) error
GetRawBytes(bucketName string, key []byte) ([]byte, error)
GetAll(bucketName string, obj any, append func(o any) (any, error)) error
GetAllWithKeyPrefix(bucketName string, keyPrefix []byte, obj any, append func(o any) (any, error)) error
KeyExists(bucketName string, key []byte) (bool, error)
GetObject(bucketName string, key []byte, object interface{}) error
GetAll(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error
GetAllWithJsoniter(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error
GetAllWithKeyPrefix(bucketName string, keyPrefix []byte, obj interface{}, append func(o interface{}) (interface{}, error)) error
}
type Transaction interface {
ReadTransaction
SetServiceName(bucketName string) error
UpdateObject(bucketName string, key []byte, object any) error
UpdateObject(bucketName string, key []byte, object interface{}) error
DeleteObject(bucketName string, key []byte) error
CreateObject(bucketName string, fn func(uint64) (int, any)) error
CreateObjectWithId(bucketName string, id int, obj any) error
CreateObjectWithStringId(bucketName string, id []byte, obj any) error
DeleteAllObjects(bucketName string, obj any, matching func(o any) (id int, ok bool)) error
CreateObject(bucketName string, fn func(uint64) (int, interface{})) error
CreateObjectWithId(bucketName string, id int, obj interface{}) error
CreateObjectWithStringId(bucketName string, id []byte, obj interface{}) error
DeleteAllObjects(bucketName string, obj interface{}, matching func(o interface{}) (id int, ok bool)) error
GetNextIdentifier(bucketName string) int
}
@@ -42,14 +41,13 @@ type Connection interface {
GetDatabaseFileName() string
GetDatabaseFilePath() string
GetStorePath() string
GetDatabaseFileSize() (int64, error)
IsEncryptedStore() bool
NeedsEncryptionMigration() (bool, error)
SetEncrypted(encrypted bool) error
SetEncrypted(encrypted bool)
BackupMetadata() (map[string]any, error)
RestoreMetadata(s map[string]any) error
BackupMetadata() (map[string]interface{}, error)
RestoreMetadata(s map[string]interface{}) error
UpdateObjectFunc(bucketName string, key []byte, object any, updateFn func()) error
ConvertToKey(v int) []byte

View File

@@ -1,201 +0,0 @@
package containerautomation
import (
"context"
"time"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/internal/endpointutils"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/filters"
"github.com/rs/zerolog/log"
)
const (
// retryWindow is the rolling window over which max restarts per container are counted.
retryWindow = 10 * time.Minute
// restartCooldown is the minimum delay between two restarts of the same container,
// giving its healthcheck time to recover before we try again.
restartCooldown = 60 * time.Second
// endpointTimeout bounds the container-list call for a single endpoint.
endpointTimeout = 30 * time.Second
// restartTimeoutBuffer is added on top of a container's stop-timeout to derive
// the deadline of its own restart context, leaving room for the engine to kill
// and start the container after the graceful stop window elapses.
restartTimeoutBuffer = 15 * time.Second
)
// retryState tracks restart accounting for a single container across ticks.
type retryState struct {
attempts int
windowStart time.Time
lastRestart time.Time
}
// retryPolicy holds the cooldown/window parameters applied to a container.
type retryPolicy struct {
maxRetries int
window time.Duration
cooldown time.Duration
}
// decideRestart is a pure function that decides whether an unhealthy container
// should be restarted now, given its current retry state and policy. It returns
// the decision and the updated state to persist.
//
// Rules, in order:
// - reset the window (and attempts) when the window has elapsed;
// - deny while still within the cooldown since the last restart;
// - deny once the max number of restarts in the current window is reached;
// - otherwise restart, incrementing the attempt counter.
func decideRestart(state retryState, policy retryPolicy, now time.Time) (bool, retryState) {
if state.windowStart.IsZero() || now.Sub(state.windowStart) >= policy.window {
state.windowStart = now
state.attempts = 0
}
if !state.lastRestart.IsZero() && now.Sub(state.lastRestart) < policy.cooldown {
return false, state
}
if state.attempts >= policy.maxRetries {
return false, state
}
state.attempts++
state.lastRestart = now
return true, state
}
// heal runs a single auto-heal pass over every reachable Docker endpoint.
// It is registered with the scheduler and guarded against overlapping ticks by
// the Service. Errors are logged per endpoint/container so one failure does not
// abort the whole pass; it always returns nil so the scheduler keeps the job.
func (s *Service) heal() error {
if !s.running.CompareAndSwap(false, true) {
log.Debug().Msg("auto-heal: previous run still in progress, skipping tick")
return nil
}
defer s.running.Store(false)
scope := s.scope()
endpoints, err := s.dataStore.Endpoint().Endpoints()
if err != nil {
log.Warn().Err(err).Msg("auto-heal: unable to list environments")
return nil
}
for i := range endpoints {
endpoint := &endpoints[i]
// M1 scope: native Docker endpoints only. Kubernetes is not applicable and
// Edge/async endpoints are not reachable synchronously from the scheduler.
if !endpointutils.IsDockerEndpoint(endpoint) || endpointutils.IsEdgeEndpoint(endpoint) {
continue
}
// Per-endpoint opt-out (M5): skip environments where automation is disabled,
// independently of the global switch. Zero value participates, so existing
// installs are unaffected.
if !AutomationEnabledForEndpoint(endpoint) {
log.Debug().Int("endpoint_id", int(endpoint.ID)).
Msg("auto-heal: automation disabled for this environment, skipping")
continue
}
s.healEndpoint(endpoint, scope)
}
// Drop retry state only for containers whose retry window has fully elapsed
// since their last restart. A container that briefly leaves the unhealthy
// filter (e.g. while "starting" after a restart) keeps its accounting, so the
// cooldown / max-retries storm guard survives flapping.
s.pruneRetries(time.Now())
return nil
}
// healEndpoint restarts the in-scope unhealthy containers of a single endpoint.
func (s *Service) healEndpoint(endpoint *portainer.Endpoint, scope string) {
endpointID := int(endpoint.ID)
// Swarm note (M1 limitation): we connect to the endpoint's primary node only
// (nodeName ""). Containers scheduled on other Swarm nodes are not healed here;
// per-node iteration is deferred to a later milestone.
clientTimeout := endpointTimeout
cli, err := s.clientFactory.CreateClient(endpoint, "", &clientTimeout)
if err != nil {
log.Warn().Err(err).Int("endpoint_id", endpointID).Msg("auto-heal: unable to create Docker client")
return
}
defer cli.Close()
listCtx, cancel := context.WithTimeout(s.baseCtx, endpointTimeout)
defer cancel()
// List running unhealthy containers only (All:false). Docker keeps
// Health.Status=="unhealthy" on stopped containers, so listing with All:true
// would let us "restart" (i.e. start) an intentionally-stopped container.
listFilters := filters.NewArgs(filters.Arg("health", "unhealthy"))
containers, err := cli.ContainerList(listCtx, container.ListOptions{All: false, Filters: listFilters})
if err != nil {
log.Warn().Err(err).Int("endpoint_id", endpointID).Msg("auto-heal: unable to list containers")
return
}
s.healContainers(cli, endpoint, scope, containers)
}
// healContainers applies the restart decision + heal-restart to each listed
// unhealthy container of an endpoint. It is split out from healEndpoint (which
// creates the client and lists the containers) so the restart loop can be
// exercised with a fake dockerClient in tests. cli is typed as the interface;
// the concrete *dockerclient.Client returned by CreateClient satisfies it.
func (s *Service) healContainers(cli dockerClient, endpoint *portainer.Endpoint, scope string, containers []container.Summary) {
endpointID := int(endpoint.ID)
for _, c := range containers {
if !InScope(scope, c.Labels) {
continue
}
policy := retryPolicy{
maxRetries: MaxRetries(c.Labels),
window: retryWindow,
cooldown: restartCooldown,
}
ok, newState := decideRestart(s.getRetry(c.ID), policy, time.Now())
s.setRetry(c.ID, newState)
if !ok {
log.Debug().Str("container_id", c.ID).Int("endpoint_id", endpointID).
Msg("auto-heal: restart skipped (cooldown or max retries reached)")
continue
}
timeout := StopTimeout(c.Labels)
// Each restart gets its own context, bounded by the container's stop-timeout
// plus a buffer, so one slow restart cannot starve the others and a hung
// engine call is bounded independently of the list deadline.
restartTimeout := time.Duration(timeout)*time.Second + restartTimeoutBuffer
restartCtx, restartCancel := context.WithTimeout(s.baseCtx, restartTimeout)
err := cli.ContainerRestart(restartCtx, c.ID, container.StopOptions{Timeout: &timeout})
restartCancel()
if err != nil {
log.Warn().Err(err).Str("container_id", c.ID).Int("endpoint_id", endpointID).
Msg("auto-heal: failed to restart unhealthy container")
continue
}
log.Info().Str("container_id", c.ID).Int("endpoint_id", endpointID).Int("attempt", newState.attempts).
Msg("auto-heal: restarted unhealthy container")
s.notifier.Notify(Event{
Kind: EventHealRestarted, EndpointID: endpointID, ContainerID: c.ID, ContainerName: containerName(c.Names),
Message: "restarted unhealthy container",
})
}
}

View File

@@ -1,137 +0,0 @@
package containerautomation
import (
"testing"
"time"
)
func TestDecideRestart(t *testing.T) {
policy := retryPolicy{
maxRetries: 3,
window: 10 * time.Minute,
cooldown: 60 * time.Second,
}
base := time.Date(2026, 6, 28, 12, 0, 0, 0, time.UTC)
t.Run("first restart on empty state", func(t *testing.T) {
ok, state := decideRestart(retryState{}, policy, base)
if !ok {
t.Fatal("expected restart on first unhealthy observation")
}
if state.attempts != 1 {
t.Errorf("attempts = %d, want 1", state.attempts)
}
if !state.windowStart.Equal(base) || !state.lastRestart.Equal(base) {
t.Error("windowStart/lastRestart should be set to now")
}
})
t.Run("blocked during cooldown", func(t *testing.T) {
_, state := decideRestart(retryState{}, policy, base)
ok, _ := decideRestart(state, policy, base.Add(30*time.Second))
if ok {
t.Error("expected restart to be blocked within cooldown")
}
})
t.Run("allowed after cooldown", func(t *testing.T) {
_, state := decideRestart(retryState{}, policy, base)
ok, state := decideRestart(state, policy, base.Add(61*time.Second))
if !ok {
t.Error("expected restart allowed after cooldown")
}
if state.attempts != 2 {
t.Errorf("attempts = %d, want 2", state.attempts)
}
})
t.Run("max retries enforced within window", func(t *testing.T) {
state := retryState{}
now := base
allowed := 0
for i := 0; i < 6; i++ {
ok, newState := decideRestart(state, policy, now)
state = newState
if ok {
allowed++
}
now = now.Add(policy.cooldown + time.Second)
}
if allowed != policy.maxRetries {
t.Errorf("allowed %d restarts, want %d (max per window)", allowed, policy.maxRetries)
}
})
t.Run("counter resets after window elapses", func(t *testing.T) {
state := retryState{attempts: 3, windowStart: base, lastRestart: base}
ok, newState := decideRestart(state, policy, base.Add(policy.window+time.Second))
if !ok {
t.Error("expected restart allowed once the window elapsed")
}
if newState.attempts != 1 {
t.Errorf("attempts = %d, want 1 after window reset", newState.attempts)
}
})
}
func TestPruneRetries(t *testing.T) {
now := time.Date(2026, 6, 28, 12, 0, 0, 0, time.UTC)
s := &Service{retries: map[string]retryState{
// within the window -> retained
"fresh": {attempts: 1, windowStart: now.Add(-time.Minute), lastRestart: now.Add(-time.Minute)},
// exactly at the window boundary -> pruned
"edge": {attempts: 2, windowStart: now.Add(-retryWindow), lastRestart: now.Add(-retryWindow)},
// long past the window -> pruned
"stale": {attempts: 3, windowStart: now.Add(-2 * retryWindow), lastRestart: now.Add(-2 * retryWindow)},
}}
s.pruneRetries(now)
if _, ok := s.retries["fresh"]; !ok {
t.Error("entry within the retry window should be retained")
}
if _, ok := s.retries["edge"]; ok {
t.Error("entry exactly at the window boundary should be pruned")
}
if _, ok := s.retries["stale"]; ok {
t.Error("entry past the retry window should be pruned")
}
}
// TestRetryStateSurvivesStartingTick locks in the F1 fix: a container that flaps
// through "starting" right after a restart (and so briefly drops out of the
// health=unhealthy filter) must keep its retry accounting across the tick where
// it is not observed, otherwise the cooldown / max-retries storm guard is
// defeated and the next unhealthy observation triggers an immediate restart.
func TestRetryStateSurvivesStartingTick(t *testing.T) {
policy := retryPolicy{maxRetries: 3, window: retryWindow, cooldown: restartCooldown}
const id = "flapper"
s := &Service{retries: make(map[string]retryState)}
t0 := time.Date(2026, 6, 28, 12, 0, 0, 0, time.UTC)
// Tick 1: container is unhealthy -> first restart.
ok, state := decideRestart(s.getRetry(id), policy, t0)
s.setRetry(id, state)
if !ok || state.attempts != 1 {
t.Fatalf("tick 1: ok=%v attempts=%d, want restart with attempts=1", ok, state.attempts)
}
// Tick 2 (t0+30s): the container is "starting" and not in the unhealthy list.
// Prune must NOT drop its state because the window has not elapsed.
s.pruneRetries(t0.Add(30 * time.Second))
if _, kept := s.retries[id]; !kept {
t.Fatal("tick 2: retry state was pruned while the container was 'starting'")
}
// Tick 3 (t0+45s): unhealthy again, still within the cooldown. The surviving
// state must block the restart and the attempt count must not be reset.
ok, state = decideRestart(s.getRetry(id), policy, t0.Add(45*time.Second))
s.setRetry(id, state)
if ok {
t.Error("tick 3: restart should be blocked by the surviving cooldown")
}
if state.attempts != 1 {
t.Errorf("tick 3: attempts = %d, want 1 (state survived, not reset)", state.attempts)
}
}

View File

@@ -1,597 +0,0 @@
package containerautomation
import (
"context"
"fmt"
"strings"
"time"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/docker/images"
"github.com/portainer/portainer/api/internal/endpointutils"
"github.com/portainer/portainer/api/stacks/deployments"
"github.com/portainer/portainer/api/stacks/stackutils"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/image"
"github.com/rs/zerolog/log"
)
const (
// statusCheckTimeout bounds a single container image-status resolution
// (container inspect + remote digest fetch).
statusCheckTimeout = 30 * time.Second
// recreateTimeout bounds a standalone recreate (pull + stop + create + start).
// Pulls can be slow, so it is generous.
recreateTimeout = 10 * time.Minute
// stackRedeployTimeout bounds a single stack redeploy-with-pull.
stackRedeployTimeout = 15 * time.Minute
)
// update runs a single auto-update pass over every reachable Docker endpoint.
// It is registered with the scheduler and guarded against overlapping ticks by
// the Service. Errors are logged per endpoint/container so one failure does not
// abort the whole pass; it always returns nil so the scheduler keeps the job.
func (s *Service) update() error {
if !s.updateRunning.CompareAndSwap(false, true) {
log.Debug().Msg("auto-update: previous run still in progress, skipping tick")
return nil
}
defer s.updateRunning.Store(false)
settings, err := s.dataStore.Settings().Settings()
if err != nil {
log.Warn().Err(err).Msg("auto-update: unable to read settings")
return nil
}
scope := ScopeLabeled
if settings.ContainerAutomation.AutoUpdate.Scope == ScopeAll {
scope = ScopeAll
}
opts := updateOptions{
cleanup: settings.ContainerAutomation.AutoUpdate.Cleanup,
rollback: settings.ContainerAutomation.AutoUpdate.RollbackOnFailure,
rollbackTimeout: parseRollbackTimeout(settings.ContainerAutomation.AutoUpdate.RollbackTimeout),
}
endpoints, err := s.dataStore.Endpoint().Endpoints()
if err != nil {
log.Warn().Err(err).Msg("auto-update: unable to list environments")
return nil
}
for i := range endpoints {
endpoint := &endpoints[i]
// Native Docker endpoints only: Kubernetes is not applicable and
// Edge/async endpoints are not reachable synchronously from the scheduler.
if !endpointutils.IsDockerEndpoint(endpoint) || endpointutils.IsEdgeEndpoint(endpoint) {
continue
}
// Per-endpoint opt-out (M5): skip environments where automation is disabled,
// independently of the global switch. Zero value participates, so existing
// installs are unaffected.
if !AutomationEnabledForEndpoint(endpoint) {
log.Debug().Int("endpoint_id", int(endpoint.ID)).
Msg("auto-update: automation disabled for this environment, skipping")
continue
}
s.updateEndpoint(endpoint, scope, opts)
}
// Drop rolled-back records whose cooldown has fully elapsed (mirrors auto-heal's
// pruneRetries), so the loop-guard map cannot grow unbounded.
s.pruneRolledBack(time.Now())
return nil
}
// updateOptions carries the per-pass auto-update toggles resolved from settings.
type updateOptions struct {
// cleanup removes the now-dangling old image after a confirmed-good update.
cleanup bool
// rollback enables the health gate + rollback of a failed standalone update.
rollback bool
// rollbackTimeout bounds how long the health gate waits before rolling back.
rollbackTimeout time.Duration
}
// parseRollbackTimeout resolves the configured rollback timeout, falling back to
// the default when empty or unparseable.
func parseRollbackTimeout(raw string) time.Duration {
d, err := time.ParseDuration(raw)
if err != nil || d <= 0 {
return defaultRollbackTimeout
}
return d
}
// updateEndpoint applies image updates to the in-scope, outdated containers of a
// single endpoint, routing each container to the standalone / stack / external
// apply path. Stack-managed candidates are grouped so each owning stack is
// redeployed at most once per tick.
func (s *Service) updateEndpoint(endpoint *portainer.Endpoint, scope string, opts updateOptions) {
endpointID := int(endpoint.ID)
// Swarm note (M4 limitation, mirrors auto-heal): we connect to the endpoint's
// primary node only (nodeName ""). Containers scheduled on other Swarm nodes
// are not updated here; stacks are redeployed cluster-wide by the swarm engine.
clientTimeout := endpointTimeout
cli, err := s.clientFactory.CreateClient(endpoint, "", &clientTimeout)
if err != nil {
log.Warn().Err(err).Int("endpoint_id", endpointID).Msg("auto-update: unable to create Docker client")
return
}
defer cli.Close()
listCtx, cancel := context.WithTimeout(s.baseCtx, endpointTimeout)
defer cancel()
// Running containers only: a stopped container has nothing to update now and
// would be started by a bare recreate.
containers, err := cli.ContainerList(listCtx, container.ListOptions{All: false})
if err != nil {
log.Warn().Err(err).Int("endpoint_id", endpointID).Msg("auto-update: unable to list containers")
return
}
// Collect the in-scope, outdated, non-monitor-only containers as candidates.
// An in-scope monitor-only container is still status-checked (keeping its badge
// cache warm) but never auto-applied. This only covers in-scope containers: in
// "labeled" scope a monitor-only container without the enable label is filtered
// out below before any status check, so its badge is not refreshed here.
var candidates []UpdateCandidate
for _, c := range containers {
if !InUpdateScope(scope, c.Labels) {
continue
}
// Resolve the image status. This also refreshes the package-level status
// cache that backs the badge, so in-scope monitor-only containers are still
// checked even though they are never auto-applied.
statusCtx, statusCancel := context.WithTimeout(s.baseCtx, statusCheckTimeout)
status, err := s.digestClient.ContainerImageStatus(statusCtx, c.ID, endpoint, "")
statusCancel()
if err != nil {
// Pull / registry-auth / network failure: leave the running container
// untouched, never recreate on a failed check.
log.Warn().Err(err).Str("container_id", c.ID).Int("endpoint_id", endpointID).
Msg("auto-update: image status check failed, leaving container untouched")
continue
}
if status != images.Outdated {
continue
}
// Monitor-only: detect-only, never auto-apply (status already cached above).
if IsMonitorOnly(c.Labels) {
log.Info().Str("container_id", c.ID).Int("endpoint_id", endpointID).
Msg("auto-update: outdated image detected but container is monitor-only, not applying")
continue
}
candidates = append(candidates, UpdateCandidate{ID: c.ID, Name: containerName(c.Names), ImageID: c.ImageID, Image: c.Image, Labels: c.Labels})
}
// Route and de-duplicate: one redeploy per stack per tick.
grouped := groupContainersForUpdate(candidates, s.stackLookupForEndpoint(endpoint.ID))
for _, ext := range grouped.External {
log.Debug().Str("container_id", ext.ID).Int("endpoint_id", endpointID).
Msg("auto-update: outdated externally-managed compose container, detect only")
}
for _, c := range grouped.Standalone {
s.updateStandalone(cli, endpoint, c, opts)
}
for _, st := range grouped.Stacks {
s.updateStack(cli, endpoint, st)
}
}
// stackLookupForEndpoint builds a compose-project-name -> Portainer compose stack
// resolver for a single endpoint. Only Docker Compose stacks on this endpoint
// match; a same-named swarm/kubernetes stack is treated as external (mirrors
// M3's resolveContainerUpdatePath).
func (s *Service) stackLookupForEndpoint(endpointID portainer.EndpointID) func(project string) *StackMatch {
stacks, err := s.dataStore.Stack().ReadAll()
if err != nil {
log.Warn().Err(err).Int("endpoint_id", int(endpointID)).
Msg("auto-update: unable to read stacks, treating compose containers as external")
return func(string) *StackMatch { return nil }
}
byName := make(map[string]*StackMatch)
for i := range stacks {
st := &stacks[i]
if st.EndpointID != endpointID || st.Type != portainer.DockerComposeStack {
continue
}
byName[st.Name] = &StackMatch{StackID: int(st.ID), IsGit: st.WorkflowID != 0}
}
return func(project string) *StackMatch {
return byName[project]
}
}
// updateStandalone recreates a standalone container with a re-pull of its image,
// then (when rollback is enabled and the container has a healthcheck) holds a
// health gate over the new container and rolls back to the previous image if it
// fails to become healthy. The old-image cleanup is deliberately ordered AFTER
// the health gate, so the rollback target is never removed before the update is
// confirmed good.
//
// Sequence: capture old image id + original ref + healthcheck -> recreate(pull)
// -> [health gate] -> on healthy: cleanup (if enabled); on unhealthy: rollback
// (never cleanup).
func (s *Service) updateStandalone(cli dockerClient, endpoint *portainer.Endpoint, c UpdateCandidate, opts updateOptions) {
endpointID := int(endpoint.ID)
// Loop-guard safety: the rolled-back map is keyed by endpoint+name (the only
// identifier that survives a recreate). An unnamed container cannot be recorded
// (recordRolledBack skips it), so with rollback enabled a container that keeps
// failing its health gate would update->rollback every tick with NO suppression.
// Skip the unnamed case when rollback is on so it cannot enter that
// unsuppressable loop; detection/badge refresh already happened upstream and is
// unaffected. (With rollback off there is no rollback to loop, so we proceed.)
if skipUnnamedForRollback(opts.rollback, c.Name) {
log.Info().Str("container_id", c.ID).Int("endpoint_id", endpointID).
Msg("auto-update: skipping unnamed standalone container, rollback is enabled but there is no stable name to key the loop guard")
return
}
// Update->rollback loop guard: if this container's update was rolled back
// recently and the remote still points at the SAME failed image, skip it until
// the cooldown elapses. A genuinely new upstream image (a changed remote digest)
// is not blocked.
rollbackMapKey := rollbackKey(endpoint.ID, c.Name)
if rec, ok := s.getRolledBack(rollbackMapKey); ok && s.shouldSkipRolledBack(rollbackMapKey, rec) {
log.Info().Str("container_id", c.ID).Str("container", c.Name).Str("image", rec.ref).Int("endpoint_id", endpointID).
Msg("auto-update: skipping update, a recent rollback failed on this image and the remote is unchanged (cooldown)")
return
}
// Capture the pre-update image identity for a possible rollback. The container
// list gives us the old image id; an inspect adds the original reference (re-tag
// target), whether a usable healthcheck exists, and the healthcheck start_period
// (which must be waited out before deciding). We only health-gate when rollback
// is enabled, the container has a healthcheck, we resolved both the old image id
// and its reference, and that reference is a proper tag (a digest-pinned or bare
// image id cannot be re-tagged, so the gate could never roll back).
oldImageID := c.ImageID
var originalRef string
var startPeriod time.Duration
healthGated := false
if opts.rollback {
// Bound the inspect like every other engine call so a hung/unreachable engine
// cannot block the whole sequential tick until shutdown.
inspectCtx, inspectCancel := context.WithTimeout(s.baseCtx, endpointTimeout)
inspect, err := cli.ContainerInspect(inspectCtx, c.ID)
inspectCancel()
if err != nil {
log.Warn().Err(err).Str("container_id", c.ID).Int("endpoint_id", endpointID).
Msg("auto-update: unable to inspect container before update, proceeding without a health gate")
} else {
originalRef = inspect.Config.Image
if oldImageID == "" {
oldImageID = inspect.Image
}
if hc := inspect.Config.Healthcheck; hc != nil {
startPeriod = hc.StartPeriod
}
switch {
case !hasHealthGate(inspect.Config.Healthcheck):
log.Info().Str("container_id", c.ID).Int("endpoint_id", endpointID).
Msg("auto-update: container has no healthcheck, updating without a rollback gate")
case oldImageID == "" || originalRef == "":
log.Info().Str("container_id", c.ID).Int("endpoint_id", endpointID).
Msg("auto-update: unable to resolve previous image identity, updating without a rollback gate")
case !isTagReference(originalRef):
log.Info().Str("container_id", c.ID).Str("image", originalRef).Int("endpoint_id", endpointID).
Msg("auto-update: health gate skipped, image is digest-pinned and cannot be rolled back")
default:
healthGated = true
}
}
}
ctx, cancel := context.WithTimeout(s.baseCtx, recreateTimeout)
defer cancel()
newContainer, err := s.containerService.Recreate(ctx, endpoint, c.ID, true, "", "")
if err != nil {
// Recreate preserves config and rolls back on a create failure; a pull or
// create failure leaves the original container running.
log.Warn().Err(err).Str("container_id", c.ID).Int("endpoint_id", endpointID).
Msg("auto-update: failed to recreate standalone container")
s.notifier.Notify(Event{
Kind: EventUpdateFailed, EndpointID: endpointID, ContainerID: c.ID, ContainerName: c.Name,
Message: "failed to recreate standalone container", Err: err,
})
return
}
log.Info().Str("container_id", c.ID).Int("endpoint_id", endpointID).
Msg("auto-update: recreated standalone container with updated image")
newImage := ""
if newContainer != nil {
newImage = newContainer.Config.Image
}
// Health gate: roll back if the new container does not become healthy in time.
// The old image is preserved (not cleaned up) until the gate confirms health,
// so the rollback target is still available. The "updated" event is held until
// the gate confirms health, so an observer never sees a misleading
// "updated" -> "rollback" sequence for the same container; on the rollback path
// only EventRollback (or update-failed) is emitted.
if healthGated {
switch s.healthGate(cli, newContainer.ID, opts.rollbackTimeout, startPeriod) {
case gateAborted:
// Server shutdown mid-gate: leave the new container in place, do not roll
// back and do not emit an event (we never observed a real failure).
return
case gateRollback:
s.rollback(cli, endpoint, newContainer.ID, oldImageID, originalRef, c.Name)
return
case gateHealthy:
// Confirmed healthy: fall through to emit "updated" and clean up.
}
}
// Emit "updated" now: either there was no gate (emitted right after recreate,
// as before), or the gate confirmed the new container is healthy.
s.notifier.Notify(Event{
Kind: EventUpdated, EndpointID: endpointID, ContainerID: newContainer.ID, ContainerName: c.Name,
Image: newImage, OldDigest: oldImageID, NewDigest: newContainer.Image,
Message: "updated standalone container",
})
if opts.cleanup && newContainer != nil && newContainer.Image != oldImageID {
s.cleanupOldImage(cli, endpoint, oldImageID)
}
}
// containerName returns a container's primary name without the leading slash, or
// "" when none is reported. The name is stable across a recreate (Recreate
// assigns a new container ID but preserves the name), so it keys the rolled-back
// loop-guard map.
func containerName(names []string) string {
if len(names) == 0 {
return ""
}
return strings.TrimPrefix(names[0], "/")
}
// skipUnnamedForRollback reports whether a standalone update must be skipped
// because rollback is enabled but the container has no stable name to key the
// loop guard. The rolled-back map is keyed by endpoint+name (the only identifier
// that survives a recreate); without a name the guard cannot record a failed
// target, so a repeatedly-failing update would loop update->rollback every tick
// with no suppression. When rollback is off there is nothing to loop, so an
// unnamed container is still allowed to update.
func skipUnnamedForRollback(rollback bool, name string) bool {
return rollback && name == ""
}
// rollbackKey identifies a standalone container in the rolled-back map by its
// endpoint and (recreate-stable) name. A recreate assigns a new container ID, so
// the ID cannot key state across an update; the name is preserved.
func rollbackKey(endpointID portainer.EndpointID, name string) string {
return fmt.Sprintf("%d/%s", int(endpointID), name)
}
// resolveRemoteDigest fetches the current remote image digest for a reference. It
// tells whether a rolled-back container's upstream target is still the same
// failed image (skip) or a new push (retry).
func (s *Service) resolveRemoteDigest(ctx context.Context, ref string) (string, error) {
img, err := images.ParseImage(images.ParseImageOptions{Name: ref})
if err != nil {
return "", err
}
dig, err := s.digestClient.RemoteDigest(ctx, img)
if err != nil {
return "", err
}
return dig.String(), nil
}
// recordRolledBack stores the failed target after a successful rollback so the
// next poll skips re-pulling the same broken image. The failed remote digest is
// resolved now (the registry is reachable, the image was just pulled); if it
// cannot be resolved the record is still stored with an empty digest and the
// guard skips conservatively until the cooldown elapses.
func (s *Service) recordRolledBack(endpoint *portainer.Endpoint, name, ref string) {
if name == "" {
// Without a stable key we cannot reliably match the container next tick.
log.Debug().Str("image", ref).Int("endpoint_id", int(endpoint.ID)).
Msg("auto-update: rolled-back container has no name, loop guard not recorded")
return
}
ctx, cancel := context.WithTimeout(s.baseCtx, statusCheckTimeout)
digest, err := s.resolveRemoteDigest(ctx, ref)
cancel()
if err != nil {
log.Debug().Err(err).Str("image", ref).Int("endpoint_id", int(endpoint.ID)).
Msg("auto-update: could not resolve failed remote digest, loop guard will skip conservatively until cooldown")
}
s.setRolledBack(rollbackKey(endpoint.ID, name), rolledBackTarget{ref: ref, digest: digest, at: time.Now()})
}
// shouldSkipRolledBack reports whether a standalone container must be skipped this
// tick to avoid the update->rollback loop, clearing the record once the skip no
// longer applies (cooldown elapsed or a new upstream image). It resolves the
// current remote digest so a genuinely new image is never blocked.
func (s *Service) shouldSkipRolledBack(key string, rec rolledBackTarget) bool {
now := time.Now()
// Fast paths that avoid a registry call: cooldown elapsed -> clear & proceed;
// no recorded digest -> skip conservatively while the cooldown is open.
if now.Sub(rec.at) >= updateRollbackCooldown {
s.clearRolledBack(key)
return false
}
if rec.digest == "" {
return true
}
ctx, cancel := context.WithTimeout(s.baseCtx, statusCheckTimeout)
currentDigest, err := s.resolveRemoteDigest(ctx, rec.ref)
cancel()
if err != nil {
// Cannot confirm the upstream target changed: stay conservative and skip to
// avoid re-entering the loop, until the cooldown elapses.
log.Debug().Err(err).Str("image", rec.ref).
Msg("auto-update: cannot resolve remote digest for a rolled-back container, skipping until cooldown")
return true
}
if decideUpdateSkip(rec, currentDigest, now, updateRollbackCooldown) {
return true
}
// New upstream image (changed digest): the failed target is gone, clear the
// record and let the update proceed.
s.clearRolledBack(key)
return false
}
// cleanupOldImage attempts a conservative removal of the previous image after a
// standalone update. The removal is NOT forced: Docker refuses to delete an
// image that still carries tags or is referenced by any container, so this only
// succeeds when the old image has become genuinely dangling (untagged and
// unused). It never touches a tagged image still in use.
func (s *Service) cleanupOldImage(cli dockerClient, endpoint *portainer.Endpoint, oldImageID string) {
if oldImageID == "" {
return
}
ctx, cancel := context.WithTimeout(s.baseCtx, endpointTimeout)
defer cancel()
if _, err := cli.ImageRemove(ctx, oldImageID, image.RemoveOptions{Force: false, PruneChildren: false}); err != nil {
log.Debug().Err(err).Str("image_id", oldImageID).Int("endpoint_id", int(endpoint.ID)).
Msg("auto-update: old image not removed (still tagged or in use)")
return
}
log.Info().Str("image_id", oldImageID).Int("endpoint_id", int(endpoint.ID)).
Msg("auto-update: removed dangling old image after update")
}
// updateStack applies an image update to a Portainer-managed compose stack so its
// containers are recreated by the stack engine and stay part of the stack. It is
// called at most once per stack per tick.
//
// - git stacks: detect-only here. A git stack's source of truth is its commit;
// this tick's trigger is an image-only update (same compose manifest, newer
// upstream digest), which the git redeploy path (RedeployWhenChanged) would
// short-circuit without applying — while still doing a real git fetch every
// tick. So we skip git stacks: the image update lands on the stack's next git
// change or via a manual "Update now", and we do not fetch git every tick.
// - file stacks: the deployer is driven directly with forcePullImage=true,
// applying the image update immediately.
//
// On a successful file-stack redeploy it emits one EventUpdated per member
// container that triggered the update (not a single aggregate stack event), each
// carrying the stack name and a best-effort post-redeploy new image id.
func (s *Service) updateStack(cli dockerClient, endpoint *portainer.Endpoint, st StackUpdate) {
if st.IsGit {
// Detect-only: leave git bookkeeping to the git redeploy path. Logged at
// debug so it does not repeat at info on every tick (it would otherwise
// fire for an unchanged git stack indefinitely).
log.Debug().Int("stack_id", st.StackID).Int("endpoint_id", int(endpoint.ID)).
Msg("auto-update: outdated git stack image detected, detect only (applied on next git change or manual update)")
return
}
ctx, cancel := context.WithTimeout(s.baseCtx, stackRedeployTimeout)
defer cancel()
stack, err := s.dataStore.Stack().Read(portainer.StackID(st.StackID))
if err != nil {
log.Warn().Err(err).Int("stack_id", st.StackID).Int("endpoint_id", int(endpoint.ID)).
Msg("auto-update: unable to read stack for redeploy")
return
}
// Resolve registries the same way the established userless/system redeploy does
// (RedeployWhenChanged): scope them to the stack author's access on the endpoint
// and refresh ECR tokens, so an ECR-backed stack authenticates with fresh
// credentials instead of the stale token a raw ReadAll() would pass.
registries, err := deployments.ResolveStackRegistries(s.dataStore, stack, endpoint.ID)
if err != nil {
log.Warn().Err(err).Int("stack_id", st.StackID).Int("endpoint_id", int(endpoint.ID)).
Msg("auto-update: unable to resolve registries for stack redeploy")
return
}
// prune=false (conservative: do not remove resources the user may rely on),
// forcePullImage=true (the whole point), forceRecreate=false.
if stackutils.IsRelativePathStack(stack) {
err = s.stackDeployer.DeployRemoteComposeStack(ctx, stack, endpoint, registries, false, true, false)
} else {
err = s.stackDeployer.DeployComposeStack(ctx, stack, endpoint, registries, false, true, false)
}
if err != nil {
log.Warn().Err(err).Int("stack_id", st.StackID).Int("endpoint_id", int(endpoint.ID)).
Msg("auto-update: failed to redeploy compose stack with re-pull")
return
}
log.Info().Int("stack_id", st.StackID).Int("endpoint_id", int(endpoint.ID)).
Msg("auto-update: redeployed compose stack with updated images")
// One notification PER updated container (the maintainer's requirement), each
// showing the container's stack name. The stack was redeployed as a whole, so the
// per-container new image id is not in hand; re-inspect each container by its
// (compose-stable) name to fill in the "new" digest best-effort. A failed inspect
// leaves NewDigest empty and the message falls back to "image updated" — never a
// blocked delivery.
for _, c := range st.Containers {
s.notifier.Notify(Event{
Kind: EventUpdated, EndpointID: int(endpoint.ID), StackID: st.StackID,
StackName: c.Labels[composeProjectLabel], ContainerName: c.Name,
Image: c.Image, OldDigest: c.ImageID, NewDigest: s.inspectImageID(cli, c.Name),
Message: "updated stack container",
})
}
}
// inspectImageID re-inspects a container by its (compose-stable) name after a stack
// redeploy to recover the new local image id for the update notification. It is
// best-effort: any failure (or an empty name) yields "", and the caller degrades the
// message to "image updated" rather than blocking delivery. The inspect is bounded
// like every other engine call so a hung engine cannot stall the tick.
func (s *Service) inspectImageID(cli dockerClient, containerName string) string {
if containerName == "" {
return ""
}
ctx, cancel := context.WithTimeout(s.baseCtx, endpointTimeout)
defer cancel()
inspect, err := cli.ContainerInspect(ctx, containerName)
if err != nil {
log.Debug().Err(err).Str("container", containerName).
Msg("auto-update: unable to inspect stack container for its new image id, notifying without it")
return ""
}
return inspect.Image
}

View File

@@ -1,148 +0,0 @@
package containerautomation
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/datastore"
"github.com/portainer/portainer/api/internal/testhelpers"
"github.com/docker/docker/api/types/container"
dockerclient "github.com/docker/docker/client"
"github.com/stretchr/testify/require"
)
// newStackInspectClient builds a Docker client wired to a test server that answers
// ContainerInspect by name, returning the given new image id. It is the seam the
// post-redeploy best-effort "new digest" re-inspect uses.
func newStackInspectClient(t *testing.T, newImageIDByName map[string]string) *dockerclient.Client {
t.Helper()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
for name, imageID := range newImageIDByName {
if strings.HasSuffix(r.URL.Path, "/containers/"+name+"/json") {
_ = json.NewEncoder(w).Encode(container.InspectResponse{
ContainerJSONBase: &container.ContainerJSONBase{ID: name, Image: imageID},
Config: &container.Config{},
})
return
}
}
http.Error(w, "not found", http.StatusNotFound)
}))
t.Cleanup(srv.Close)
cli, err := dockerclient.NewClientWithOpts(
dockerclient.WithHost(srv.URL),
dockerclient.WithHTTPClient(http.DefaultClient),
)
require.NoError(t, err)
return cli
}
// TestUpdateStackEmitsPerContainerEvents proves the maintainer's requirement: a
// (file) stack redeploy emits one EventUpdated PER updated member container, each
// carrying the compose stack name (from the container's label, not a Stack().Read)
// and a best-effort post-redeploy new image id — never a single aggregate stack
// event.
func TestUpdateStackEmitsPerContainerEvents(t *testing.T) {
_, store := datastore.MustNewTestStore(t, true, false)
// A stack author must exist for registry resolution; an admin resolves to the
// (empty) registry set without needing endpoint/team wiring.
require.NoError(t, store.User().Create(&portainer.User{ID: 1, Username: "auto", Role: portainer.AdministratorRole}))
endpoint := &portainer.Endpoint{ID: 1, Name: "nebula.lc"}
require.NoError(t, store.Endpoint().Create(endpoint))
require.NoError(t, store.Stack().Create(&portainer.Stack{
ID: 7, EndpointID: 1, Name: "cache-demo", Type: portainer.DockerComposeStack, CreatedBy: "auto",
}))
const (
oldEsphome = "sha256:59b94983c73a000000000000000000000000000000000000000000000000aaaa"
newEsphome = "sha256:2231ca5d676d000000000000000000000000000000000000000000000000bbbb"
oldOther = "sha256:1111111111110000000000000000000000000000000000000000000000000000"
newOther = "sha256:2222222222220000000000000000000000000000000000000000000000000000"
)
cli := newStackInspectClient(t, map[string]string{
"esphome": newEsphome,
"other": newOther,
})
rec := &recordingNotifier{}
s := &Service{
baseCtx: context.Background(),
dataStore: store,
stackDeployer: testhelpers.NewTestStackDeployer(),
notifier: rec,
}
st := StackUpdate{
StackID: 7,
IsGit: false,
Containers: []UpdateCandidate{
{Name: "esphome", ImageID: oldEsphome, Image: "esphome/esphome:latest", Labels: map[string]string{composeProjectLabel: "cache-demo"}},
{Name: "other", ImageID: oldOther, Image: "redis:7", Labels: map[string]string{composeProjectLabel: "cache-demo"}},
},
}
s.updateStack(cli, endpoint, st)
require.Len(t, rec.events, 2, "one EventUpdated per updated member container, not one aggregate stack event")
byContainer := map[string]Event{}
for _, e := range rec.events {
require.Equal(t, EventUpdated, e.Kind)
require.Equal(t, "cache-demo", e.StackName, "each per-container event carries the compose stack name")
require.Equal(t, 7, e.StackID)
byContainer[e.ContainerName] = e
}
esphome, ok := byContainer["esphome"]
require.True(t, ok, "expected a per-container event for esphome")
require.Equal(t, oldEsphome, esphome.OldDigest)
require.Equal(t, newEsphome, esphome.NewDigest, "the new image id is recovered by re-inspecting the container after redeploy")
other, ok := byContainer["other"]
require.True(t, ok, "expected a per-container event for other")
require.Equal(t, oldOther, other.OldDigest)
require.Equal(t, newOther, other.NewDigest)
}
// TestUpdateStackGitIsDetectOnly guards that a git stack stays detect-only: it is
// not redeployed and emits no notification (its image update lands on the next git
// change or a manual update).
func TestUpdateStackGitIsDetectOnly(t *testing.T) {
_, store := datastore.MustNewTestStore(t, true, false)
endpoint := &portainer.Endpoint{ID: 1, Name: "nebula.lc"}
require.NoError(t, store.Endpoint().Create(endpoint))
deployer := testhelpers.NewTestStackDeployer()
rec := &recordingNotifier{}
s := &Service{
baseCtx: context.Background(),
dataStore: store,
stackDeployer: deployer,
notifier: rec,
}
cli := newStackInspectClient(t, nil)
s.updateStack(cli, endpoint, StackUpdate{
StackID: 9, IsGit: true,
Containers: []UpdateCandidate{{Name: "esphome", Labels: map[string]string{composeProjectLabel: "cache-demo"}}},
})
require.Empty(t, rec.events, "a git stack is detect-only, no per-container notification")
require.Zero(t, deployer.DeployComposeCallCount, "a git stack must not be redeployed here")
}

View File

@@ -1,236 +0,0 @@
package containerautomation
import (
"context"
"strings"
"testing"
"time"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/datastore"
"github.com/portainer/portainer/api/docker/images"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/stretchr/testify/require"
)
const (
// 64-hex content-addressable image ids for the pre/post-update identities.
oldImageID = "sha256:1111111111111111111111111111111111111111111111111111111111111111"
newImageID = "sha256:2222222222222222222222222222222222222222222222222222222222222222"
)
// preUpdateInspect is the pre-update ContainerInspect the standalone path issues
// on the OLD container to capture its original image ref + healthcheck (so the
// rollback health gate is armed).
func preUpdateInspect(id, ref string) container.InspectResponse {
return container.InspectResponse{
ContainerJSONBase: &container.ContainerJSONBase{ID: id, Image: oldImageID},
Config: &container.Config{
Image: ref,
Healthcheck: &container.HealthConfig{Test: []string{"CMD", "true"}},
},
}
}
// healthInspect is the health-gate ContainerInspect on the NEW container,
// reporting the given health status.
func healthInspect(id string, status container.HealthStatus) container.InspectResponse {
return container.InspectResponse{
ContainerJSONBase: &container.ContainerJSONBase{
ID: id,
State: &container.State{Running: true, Health: &container.Health{Status: status}},
},
Config: &container.Config{},
}
}
func countPrefix(calls []string, prefix string) int {
n := 0
for _, c := range calls {
if strings.HasPrefix(c, prefix) {
n++
}
}
return n
}
// TestUpdateStandaloneHappyPathCleansUpAfterHealthGate locks in the happy-path
// wiring #20 calls out: pull/recreate -> health gate confirms healthy -> the
// "updated" event is emitted only AFTER health is confirmed -> the old-image
// cleanup runs strictly AFTER the healthy gate (so a rollback target could never
// be deleted before the update is confirmed good).
func TestUpdateStandaloneHappyPathCleansUpAfterHealthGate(t *testing.T) {
const (
oldID = "old-id"
newID = "new-id"
ref = "nginx:1.21"
)
seq := &callSeq{}
notif := &seqNotifier{seq: seq}
cli := newFakeDockerClient(seq)
cli.inspectByID[oldID] = preUpdateInspect(oldID, ref)
cli.inspectByID[newID] = healthInspect(newID, container.Healthy)
rec := &fakeRecreator{seq: seq, result: &types.ContainerJSON{
ContainerJSONBase: &container.ContainerJSONBase{ID: newID, Image: newImageID},
Config: &container.Config{Image: ref},
}}
s := &Service{
baseCtx: context.Background(),
containerService: rec,
notifier: notif,
rolledBack: map[string]rolledBackTarget{},
}
endpoint := &portainer.Endpoint{ID: 1}
c := UpdateCandidate{ID: oldID, Name: "web", ImageID: oldImageID, Image: ref}
opts := updateOptions{cleanup: true, rollback: true, rollbackTimeout: 60 * time.Second}
s.updateStandalone(cli, endpoint, c, opts)
calls := seq.snapshot()
iRecreate := seq.indexOf("recreate:" + oldID)
iGate := seq.indexOf("inspect:" + newID)
iUpdated := seq.indexOf("event:" + string(EventUpdated))
iCleanup := seq.indexOf("imageremove:" + oldImageID)
require.NotEqual(t, -1, iRecreate, "recreate must run")
require.NotEqual(t, -1, iGate, "the health gate must poll the new container")
require.NotEqual(t, -1, iUpdated, "an updated event must be emitted")
require.NotEqual(t, -1, iCleanup, "cleanup must remove the old image")
require.Less(t, iRecreate, iGate, "recreate must happen before the health gate")
require.Less(t, iGate, iUpdated, "the updated event must be held until health is confirmed")
require.Less(t, iUpdated, iCleanup, "cleanup must run strictly AFTER the healthy gate/updated event")
require.Equal(t, oldImageID, calls[iCleanup][len("imageremove:"):], "cleanup targets the OLD image, never the new/rollback target")
updated, n := notif.only(EventUpdated)
require.Equal(t, 1, n, "exactly one updated event")
require.Equal(t, newID, updated.ContainerID)
require.Equal(t, oldImageID, updated.OldDigest)
require.Equal(t, newImageID, updated.NewDigest)
_, rollbacks := notif.only(EventRollback)
require.Zero(t, rollbacks, "no rollback on the happy path")
}
// TestUpdateStandaloneRollbackPreservesTarget locks in the rollback-path wiring:
// a new container that fails the health gate is rolled back to the previous image
// (re-tag -> recreate on the old image with NO pull), EventRollback (not
// EventUpdated) is emitted, and the old-image cleanup NEVER runs — so the rollback
// target is never deleted early.
func TestUpdateStandaloneRollbackPreservesTarget(t *testing.T) {
_, store := datastore.MustNewTestStore(t, true, false)
const (
oldID = "old-id"
newID = "new-id"
// A registry ref that resolves to a fast connection-refused so the loop-guard's
// best-effort remote-digest resolution fails immediately (offline-safe) without
// blocking; the record is still stored with an empty digest.
ref = "localhost:1/web:v1"
)
seq := &callSeq{}
notif := &seqNotifier{seq: seq}
cli := newFakeDockerClient(seq)
cli.inspectByID[oldID] = preUpdateInspect(oldID, ref)
cli.inspectByID[newID] = healthInspect(newID, container.Unhealthy)
rec := &fakeRecreator{seq: seq, result: &types.ContainerJSON{
ContainerJSONBase: &container.ContainerJSONBase{ID: newID, Image: newImageID},
Config: &container.Config{Image: ref},
}}
s := &Service{
baseCtx: context.Background(),
containerService: rec,
digestClient: images.NewClientWithRegistry(images.NewRegistryClient(store), nil),
notifier: notif,
rolledBack: map[string]rolledBackTarget{},
}
endpoint := &portainer.Endpoint{ID: 1}
c := UpdateCandidate{ID: oldID, Name: "web", ImageID: oldImageID, Image: ref}
// cleanup enabled to prove it is NOT run on the rollback path.
opts := updateOptions{cleanup: true, rollback: true, rollbackTimeout: 60 * time.Second}
s.updateStandalone(cli, endpoint, c, opts)
calls := seq.snapshot()
iGate := seq.indexOf("inspect:" + newID)
iTag := seq.indexOf("imagetag:" + oldImageID + "->" + ref)
iRollbackRecreate := seq.indexOf("recreate:" + newID)
iRollbackEvent := seq.indexOf("event:" + string(EventRollback))
require.NotEqual(t, -1, iGate, "the health gate must poll the new container")
require.NotEqual(t, -1, iTag, "rollback must re-tag the previous image onto the original ref")
require.NotEqual(t, -1, iRollbackRecreate, "rollback must recreate on the previous image")
require.NotEqual(t, -1, iRollbackEvent, "a rollback event must be emitted")
require.Less(t, iGate, iTag, "rollback happens after the failed gate")
require.Less(t, iTag, iRollbackRecreate, "re-tag the old image before recreating on it")
require.Less(t, iRollbackRecreate, iRollbackEvent, "the rollback event follows the rollback recreate")
require.Zero(t, countPrefix(calls, "imageremove:"), "cleanup must NOT run on the rollback path (rollback target preserved)")
_, updates := notif.only(EventUpdated)
require.Zero(t, updates, "no updated event on the rollback path")
rollback, n := notif.only(EventRollback)
require.Equal(t, 1, n, "exactly one rollback event")
require.Equal(t, ref, rollback.Image, "the rollback event carries the restored original ref")
// The recreate seam saw two calls: the initial pull-recreate on the old container,
// then the rollback recreate on the new container WITHOUT a pull (resolves the
// re-tagged previous image).
require.Len(t, rec.calls, 2)
require.Equal(t, recreateCall{containerID: oldID, forcePullImage: true}, rec.calls[0])
require.Equal(t, recreateCall{containerID: newID, forcePullImage: false}, rec.calls[1])
}
// TestHealContainersRestartsUnhealthyThenRespectsCooldown locks in the auto-heal
// restart loop: an unhealthy container is restarted (restart precedes the heal
// event), and a second immediate pass is suppressed by the restart cooldown/loop
// guard so a flapping container is not restart-stormed.
func TestHealContainersRestartsUnhealthyThenRespectsCooldown(t *testing.T) {
seq := &callSeq{}
notif := &seqNotifier{seq: seq}
cli := newFakeDockerClient(seq)
s := &Service{
baseCtx: context.Background(),
notifier: notif,
retries: map[string]retryState{},
}
endpoint := &portainer.Endpoint{ID: 1}
containers := []container.Summary{{ID: "c1", Names: []string{"/web"}}}
// First pass: the unhealthy container is restarted and a heal event is emitted.
s.healContainers(cli, endpoint, ScopeAll, containers)
require.Equal(t, []string{"restart:c1", "event:" + string(EventHealRestarted)}, seq.snapshot(),
"the restart must precede the heal event")
require.Equal(t, 1, s.getRetry("c1").attempts, "the restart is accounted against the retry budget")
ev, n := notif.only(EventHealRestarted)
require.Equal(t, 1, n)
require.Equal(t, "c1", ev.ContainerID)
require.Equal(t, "web", ev.ContainerName)
// Second immediate pass: within the restart cooldown, the loop guard suppresses a
// second restart (no new restart call, no new event).
s.healContainers(cli, endpoint, ScopeAll, containers)
require.Equal(t, 1, countPrefix(seq.snapshot(), "restart:"), "no second restart within the cooldown")
_, n2 := notif.only(EventHealRestarted)
require.Equal(t, 1, n2, "the flap is suppressed by the restart cooldown")
}

View File

@@ -1,152 +0,0 @@
package containerautomation
import (
"context"
"fmt"
"sync"
portainer "github.com/portainer/portainer/api"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/image"
)
// callSeq is a shared, ordered recorder the daemon-path fakes write to, so a test
// can assert the SEQUENCE of operations across the docker client, the recreator
// and the notifier on a single timeline (e.g. cleanup strictly after the healthy
// gate, "updated" held until health is confirmed).
type callSeq struct {
mu sync.Mutex
calls []string
}
func (s *callSeq) record(entry string) {
s.mu.Lock()
defer s.mu.Unlock()
s.calls = append(s.calls, entry)
}
// snapshot returns a copy of the recorded calls in order.
func (s *callSeq) snapshot() []string {
s.mu.Lock()
defer s.mu.Unlock()
out := make([]string, len(s.calls))
copy(out, s.calls)
return out
}
// indexOf returns the position of the first recorded call equal to entry, or -1.
func (s *callSeq) indexOf(entry string) int {
for i, c := range s.snapshot() {
if c == entry {
return i
}
}
return -1
}
// fakeDockerClient is a programmable, call-recording implementation of the
// dockerClient seam. Inspect responses/errors are keyed by container id so the
// pre-update inspect (old id) and the health-gate poll (new id) can return
// different states within one update.
type fakeDockerClient struct {
seq *callSeq
inspectByID map[string]container.InspectResponse
inspectErrByID map[string]error
imageTagErr error
imageRemoveErr error
restartErrByID map[string]error
}
func newFakeDockerClient(seq *callSeq) *fakeDockerClient {
return &fakeDockerClient{
seq: seq,
inspectByID: map[string]container.InspectResponse{},
inspectErrByID: map[string]error{},
restartErrByID: map[string]error{},
}
}
func (f *fakeDockerClient) ContainerInspect(_ context.Context, containerID string) (container.InspectResponse, error) {
f.seq.record("inspect:" + containerID)
if err := f.inspectErrByID[containerID]; err != nil {
return container.InspectResponse{}, err
}
resp, ok := f.inspectByID[containerID]
if !ok {
return container.InspectResponse{}, fmt.Errorf("fake: no inspect programmed for %q", containerID)
}
return resp, nil
}
func (f *fakeDockerClient) ContainerRestart(_ context.Context, containerID string, _ container.StopOptions) error {
f.seq.record("restart:" + containerID)
return f.restartErrByID[containerID]
}
func (f *fakeDockerClient) ImageTag(_ context.Context, source, target string) error {
f.seq.record("imagetag:" + source + "->" + target)
return f.imageTagErr
}
func (f *fakeDockerClient) ImageRemove(_ context.Context, imageID string, _ image.RemoveOptions) ([]image.DeleteResponse, error) {
f.seq.record("imageremove:" + imageID)
if f.imageRemoveErr != nil {
return nil, f.imageRemoveErr
}
return []image.DeleteResponse{{Deleted: imageID}}, nil
}
// recreateCall records the salient arguments of a single Recreate invocation.
type recreateCall struct {
containerID string
forcePullImage bool
}
// fakeRecreator is a programmable, call-recording implementation of the
// containerRecreator seam. It returns the same result for every call; the
// standalone rollback path recreates a second time (on the previous image) and
// ignores that return value.
type fakeRecreator struct {
seq *callSeq
result *types.ContainerJSON
err error
calls []recreateCall
}
func (f *fakeRecreator) Recreate(_ context.Context, _ *portainer.Endpoint, containerID string, forcePullImage bool, _, _ string) (*types.ContainerJSON, error) {
f.seq.record("recreate:" + containerID)
f.calls = append(f.calls, recreateCall{containerID: containerID, forcePullImage: forcePullImage})
if f.err != nil {
return nil, f.err
}
return f.result, nil
}
// seqNotifier records each event onto the shared timeline (as "event:<kind>") and
// keeps the full events for content assertions.
type seqNotifier struct {
seq *callSeq
events []Event
}
func (n *seqNotifier) Notify(event Event) {
n.seq.record("event:" + string(event.Kind))
n.events = append(n.events, event)
}
// only returns the single recorded event of the given kind, requiring exactly one.
func (n *seqNotifier) only(kind EventKind) (Event, int) {
var found Event
count := 0
for _, e := range n.events {
if e.Kind == kind {
found = e
count++
}
}
return found, count
}

View File

@@ -1,257 +0,0 @@
package containerautomation
import "strconv"
const (
// Scope values shared by the auto-heal and auto-update global settings.
ScopeLabeled = "labeled"
ScopeAll = "all"
// Primary labels (with community aliases) controlling per-container auto-heal.
labelEnable = "io.portainer.autoheal.enable"
labelEnableAlias = "autoheal"
labelStopTimeout = "io.portainer.autoheal.stop-timeout"
labelStopTimeoutAlias = "autoheal.stop.timeout"
labelRetries = "io.portainer.autoheal.retries"
// Primary labels (with watchtower aliases) controlling per-container auto-update.
labelUpdateEnable = "io.portainer.update.enable"
labelUpdateEnableAlias = "com.centurylinklabs.watchtower.enable"
labelUpdateMonitorOnly = "io.portainer.update.monitor-only"
labelUpdateMonitorOnlyAlias = "com.centurylinklabs.watchtower.monitor-only"
// composeProjectLabel identifies the compose project a container belongs to.
composeProjectLabel = "com.docker.compose.project"
// Defaults used when a label is missing or holds an invalid value.
defaultStopTimeout = 10
defaultRetries = 3
)
// InScope reports whether a container is subject to auto-heal given the global
// scope and the container's labels.
//
// - "all": every container is in scope, unless it explicitly opts out with the
// enable label set to false.
// - "labeled" (default): only containers with the enable label set to true.
func InScope(scope string, labels map[string]string) bool {
enabled, present := boolLabel(labels, labelEnable, labelEnableAlias)
switch scope {
case ScopeAll:
if present && !enabled {
return false
}
return true
default: // ScopeLabeled
return present && enabled
}
}
// boolLabel resolves a boolean label (primary key first, alias second).
// It returns the parsed value and whether the label was present at all.
// Invalid values are treated as false but still count as "present".
func boolLabel(labels map[string]string, key, alias string) (value bool, present bool) {
raw, ok := labels[key]
if !ok {
raw, ok = labels[alias]
}
if !ok {
return false, false
}
parsed, err := strconv.ParseBool(raw)
if err != nil {
return false, true
}
return parsed, true
}
// InUpdateScope reports whether a container is subject to auto-update given the
// global scope and the container's labels. It mirrors InScope but reads the
// update enable label (io.portainer.update.enable / watchtower alias):
//
// - "all": every container is in scope, unless it explicitly opts out with the
// update enable label set to false.
// - "labeled" (default): only containers with the update enable label true.
func InUpdateScope(scope string, labels map[string]string) bool {
enabled, present := boolLabel(labels, labelUpdateEnable, labelUpdateEnableAlias)
switch scope {
case ScopeAll:
if present && !enabled {
return false
}
return true
default: // ScopeLabeled
return present && enabled
}
}
// IsMonitorOnly reports whether a container is flagged detect-only via the
// monitor-only label (io.portainer.update.monitor-only / watchtower alias).
// Such containers have their image status resolved (for the badge cache) but are
// never auto-applied.
func IsMonitorOnly(labels map[string]string) bool {
value, present := boolLabel(labels, labelUpdateMonitorOnly, labelUpdateMonitorOnlyAlias)
return present && value
}
// UpdateKind is the apply path resolved for an outdated container.
type UpdateKind string
const (
// UpdateStandalone: recreate-with-pull (no compose project).
UpdateStandalone UpdateKind = "standalone"
// UpdateStack: redeploy the owning Portainer compose stack with re-pull, so
// the container stays part of its stack.
UpdateStack UpdateKind = "stack"
// UpdateExternal: compose-managed but with no matching Portainer compose
// stack record; Portainer must not touch it (would detach it / drift).
UpdateExternal UpdateKind = "external"
)
// StackMatch is the Portainer Docker Compose stack a compose project resolves to.
type StackMatch struct {
StackID int
// IsGit routes file vs git redeploy at apply time.
IsGit bool
}
// UpdateRouting is the decision returned by resolveContainerUpdateRouting.
type UpdateRouting struct {
Kind UpdateKind
StackID int
IsGit bool
}
// resolveContainerUpdateRouting decides how a container's image update must be
// applied, given a lookup that resolves a compose project name to a matching
// Portainer Docker Compose stack (nil when none exists or it is not a compose
// stack). It is the Go equivalent of M3's TS resolveContainerUpdatePath: pure
// and side-effect free so it can be unit-tested without Docker or the datastore.
//
// - No compose project label -> standalone (recreate-with-pull).
// - Compose project matching a Portainer compose stack -> stack
// (redeploy-with-pull, keeps the container in its stack).
// - Compose project with no matching Portainer compose stack -> external
// (managed outside Portainer / a same-named stack of another type), left
// untouched to avoid detaching it or drifting.
func resolveContainerUpdateRouting(labels map[string]string, stackLookup func(project string) *StackMatch) UpdateRouting {
project := labels[composeProjectLabel]
if project == "" {
return UpdateRouting{Kind: UpdateStandalone}
}
match := stackLookup(project)
if match == nil {
return UpdateRouting{Kind: UpdateExternal}
}
return UpdateRouting{Kind: UpdateStack, StackID: match.StackID, IsGit: match.IsGit}
}
// UpdateCandidate is an outdated, in-scope container considered for auto-update.
type UpdateCandidate struct {
ID string
// Name is the container's primary name (no leading slash). It is stable across
// a recreate and keys the update->rollback loop guard.
Name string
// ImageID is the pre-update local image id ("sha256:..."), the "old" digest in a
// per-container update notification.
ImageID string
// Image is the container's image reference (e.g. "nginx:latest"), carried for the
// notification.
Image string
Labels map[string]string
}
// StackUpdate identifies a Portainer stack to redeploy once, together with the
// affected member containers so each updated container can emit its own
// notification (with the stack name) after the redeploy.
type StackUpdate struct {
StackID int
IsGit bool
// Containers are the outdated member containers that triggered this stack
// redeploy, threaded through from detection so a per-container notification can
// be emitted for each (name + old image id + image + labels/stack name).
Containers []UpdateCandidate
}
// GroupedUpdates partitions candidates into their apply paths, de-duplicating
// stack containers so each owning stack is redeployed at most once per tick
// (the overlap guard for stack fan-out). Pure and unit-testable, the Go analogue
// of M3's groupContainersForUpdate.
type GroupedUpdates struct {
Standalone []UpdateCandidate
External []UpdateCandidate
Stacks []StackUpdate
}
// groupContainersForUpdate routes each candidate and collapses stack candidates
// so a stack with several outdated containers is redeployed only once.
func groupContainersForUpdate(candidates []UpdateCandidate, stackLookup func(project string) *StackMatch) GroupedUpdates {
grouped := GroupedUpdates{}
// stackIndex maps a stack id to its slot in grouped.Stacks so a stack is
// redeployed once, while every member container is still collected for its own
// notification (rather than discarded at the collapse).
stackIndex := make(map[int]int)
for _, c := range candidates {
routing := resolveContainerUpdateRouting(c.Labels, stackLookup)
switch routing.Kind {
case UpdateStandalone:
grouped.Standalone = append(grouped.Standalone, c)
case UpdateExternal:
grouped.External = append(grouped.External, c)
case UpdateStack:
idx, ok := stackIndex[routing.StackID]
if !ok {
grouped.Stacks = append(grouped.Stacks, StackUpdate{StackID: routing.StackID, IsGit: routing.IsGit})
idx = len(grouped.Stacks) - 1
stackIndex[routing.StackID] = idx
}
grouped.Stacks[idx].Containers = append(grouped.Stacks[idx].Containers, c)
}
}
return grouped
}
// StopTimeout returns the per-container stop timeout (in seconds) from labels,
// falling back to the default when missing or invalid.
func StopTimeout(labels map[string]string) int {
return positiveIntLabel(labels, labelStopTimeout, labelStopTimeoutAlias, defaultStopTimeout)
}
// MaxRetries returns the per-container max restarts per window from labels,
// falling back to the default when missing or invalid.
func MaxRetries(labels map[string]string) int {
return positiveIntLabel(labels, labelRetries, "", defaultRetries)
}
// positiveIntLabel reads an integer label (primary first, optional alias second)
// and returns it when strictly positive, otherwise the provided default.
func positiveIntLabel(labels map[string]string, key, alias string, fallback int) int {
raw, ok := labels[key]
if !ok && alias != "" {
raw, ok = labels[alias]
}
if !ok {
return fallback
}
value, err := strconv.Atoi(raw)
if err != nil || value <= 0 {
return fallback
}
return value
}

View File

@@ -1,248 +0,0 @@
package containerautomation
import "testing"
func TestInScope(t *testing.T) {
tests := []struct {
name string
scope string
labels map[string]string
want bool
}{
{"labeled: no labels", ScopeLabeled, nil, false},
{"labeled: enable true (primary)", ScopeLabeled, map[string]string{labelEnable: "true"}, true},
{"labeled: enable true (alias)", ScopeLabeled, map[string]string{labelEnableAlias: "true"}, true},
{"labeled: enable false", ScopeLabeled, map[string]string{labelEnable: "false"}, false},
{"labeled: enable bad value", ScopeLabeled, map[string]string{labelEnable: "yepp"}, false},
{"labeled: primary wins over alias", ScopeLabeled, map[string]string{labelEnable: "true", labelEnableAlias: "false"}, true},
{"all: no labels", ScopeAll, nil, true},
{"all: enable true", ScopeAll, map[string]string{labelEnable: "true"}, true},
{"all: explicit opt-out", ScopeAll, map[string]string{labelEnable: "false"}, false},
{"all: opt-out via alias", ScopeAll, map[string]string{labelEnableAlias: "0"}, false},
{"all: bad value is not opt-out", ScopeAll, map[string]string{labelEnable: "nope"}, false},
{"unknown scope falls back to labeled", "weird", map[string]string{labelEnable: "true"}, true},
{"unknown scope, no label", "weird", nil, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := InScope(tt.scope, tt.labels); got != tt.want {
t.Errorf("InScope(%q, %v) = %v, want %v", tt.scope, tt.labels, got, tt.want)
}
})
}
}
func TestStopTimeout(t *testing.T) {
tests := []struct {
name string
labels map[string]string
want int
}{
{"default when missing", nil, defaultStopTimeout},
{"primary value", map[string]string{labelStopTimeout: "25"}, 25},
{"alias value", map[string]string{labelStopTimeoutAlias: "15"}, 15},
{"primary wins over alias", map[string]string{labelStopTimeout: "25", labelStopTimeoutAlias: "15"}, 25},
{"bad value falls back", map[string]string{labelStopTimeout: "abc"}, defaultStopTimeout},
{"zero falls back", map[string]string{labelStopTimeout: "0"}, defaultStopTimeout},
{"negative falls back", map[string]string{labelStopTimeout: "-5"}, defaultStopTimeout},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := StopTimeout(tt.labels); got != tt.want {
t.Errorf("StopTimeout(%v) = %d, want %d", tt.labels, got, tt.want)
}
})
}
}
func TestMaxRetries(t *testing.T) {
tests := []struct {
name string
labels map[string]string
want int
}{
{"default when missing", nil, defaultRetries},
{"explicit value", map[string]string{labelRetries: "5"}, 5},
{"bad value falls back", map[string]string{labelRetries: "lots"}, defaultRetries},
{"zero falls back", map[string]string{labelRetries: "0"}, defaultRetries},
{"no alias for retries", map[string]string{"autoheal.retries": "7"}, defaultRetries},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := MaxRetries(tt.labels); got != tt.want {
t.Errorf("MaxRetries(%v) = %d, want %d", tt.labels, got, tt.want)
}
})
}
}
func TestInUpdateScope(t *testing.T) {
tests := []struct {
name string
scope string
labels map[string]string
want bool
}{
{"labeled: no labels", ScopeLabeled, nil, false},
{"labeled: enable true (primary)", ScopeLabeled, map[string]string{labelUpdateEnable: "true"}, true},
{"labeled: enable true (watchtower alias)", ScopeLabeled, map[string]string{labelUpdateEnableAlias: "true"}, true},
{"labeled: enable false", ScopeLabeled, map[string]string{labelUpdateEnable: "false"}, false},
{"labeled: enable bad value", ScopeLabeled, map[string]string{labelUpdateEnable: "soon"}, false},
{"labeled: primary wins over alias", ScopeLabeled, map[string]string{labelUpdateEnable: "true", labelUpdateEnableAlias: "false"}, true},
{"all: no labels", ScopeAll, nil, true},
{"all: enable true", ScopeAll, map[string]string{labelUpdateEnable: "true"}, true},
{"all: explicit opt-out", ScopeAll, map[string]string{labelUpdateEnable: "false"}, false},
{"all: opt-out via watchtower alias", ScopeAll, map[string]string{labelUpdateEnableAlias: "0"}, false},
{"all: bad value is not opt-out", ScopeAll, map[string]string{labelUpdateEnable: "nope"}, false},
{"unknown scope falls back to labeled", "weird", map[string]string{labelUpdateEnable: "true"}, true},
{"unknown scope, no label", "weird", nil, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := InUpdateScope(tt.scope, tt.labels); got != tt.want {
t.Errorf("InUpdateScope(%q, %v) = %v, want %v", tt.scope, tt.labels, got, tt.want)
}
})
}
}
func TestIsMonitorOnly(t *testing.T) {
tests := []struct {
name string
labels map[string]string
want bool
}{
{"no labels", nil, false},
{"primary true", map[string]string{labelUpdateMonitorOnly: "true"}, true},
{"watchtower alias true", map[string]string{labelUpdateMonitorOnlyAlias: "true"}, true},
{"primary false", map[string]string{labelUpdateMonitorOnly: "false"}, false},
{"bad value", map[string]string{labelUpdateMonitorOnly: "maybe"}, false},
{"primary wins over alias", map[string]string{labelUpdateMonitorOnly: "true", labelUpdateMonitorOnlyAlias: "false"}, true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := IsMonitorOnly(tt.labels); got != tt.want {
t.Errorf("IsMonitorOnly(%v) = %v, want %v", tt.labels, got, tt.want)
}
})
}
}
func TestResolveContainerUpdateRouting(t *testing.T) {
// stackLookup resolves "my-stack" to compose stack 7 (git) and nothing else,
// mirroring how the job builds a per-endpoint compose-stack index.
stackLookup := func(project string) *StackMatch {
if project == "my-stack" {
return &StackMatch{StackID: 7, IsGit: true}
}
return nil
}
tests := []struct {
name string
labels map[string]string
want UpdateRouting
}{
{
name: "no compose label -> standalone",
labels: map[string]string{"foo": "bar"},
want: UpdateRouting{Kind: UpdateStandalone},
},
{
name: "empty compose label -> standalone",
labels: map[string]string{composeProjectLabel: ""},
want: UpdateRouting{Kind: UpdateStandalone},
},
{
name: "compose project matching a portainer compose stack -> stack",
labels: map[string]string{composeProjectLabel: "my-stack"},
want: UpdateRouting{Kind: UpdateStack, StackID: 7, IsGit: true},
},
{
name: "compose project with no matching stack -> external",
labels: map[string]string{composeProjectLabel: "other"},
want: UpdateRouting{Kind: UpdateExternal},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := resolveContainerUpdateRouting(tt.labels, stackLookup)
if got != tt.want {
t.Errorf("resolveContainerUpdateRouting(%v) = %+v, want %+v", tt.labels, got, tt.want)
}
})
}
}
func TestGroupContainersForUpdate(t *testing.T) {
// stackLookup: "web" -> compose stack 3 (file), "api" -> compose stack 4 (git).
stackLookup := func(project string) *StackMatch {
switch project {
case "web":
return &StackMatch{StackID: 3, IsGit: false}
case "api":
return &StackMatch{StackID: 4, IsGit: true}
default:
return nil
}
}
candidates := []UpdateCandidate{
{ID: "standalone-1"},
{ID: "web-a", Name: "web-a", Labels: map[string]string{composeProjectLabel: "web"}},
{ID: "web-b", Name: "web-b", Labels: map[string]string{composeProjectLabel: "web"}}, // same stack -> deduped redeploy, both kept as members
{ID: "api-a", Name: "api-a", Labels: map[string]string{composeProjectLabel: "api"}},
{ID: "ext-1", Labels: map[string]string{composeProjectLabel: "unknown"}},
}
grouped := groupContainersForUpdate(candidates, stackLookup)
if len(grouped.Standalone) != 1 || grouped.Standalone[0].ID != "standalone-1" {
t.Errorf("Standalone = %+v, want one entry standalone-1", grouped.Standalone)
}
if len(grouped.External) != 1 || grouped.External[0].ID != "ext-1" {
t.Errorf("External = %+v, want one entry ext-1", grouped.External)
}
// One redeploy per stack: web appears twice in input but once in output.
if len(grouped.Stacks) != 2 {
t.Fatalf("Stacks = %+v, want 2 deduped stacks", grouped.Stacks)
}
got := map[int]bool{}
for _, st := range grouped.Stacks {
got[st.StackID] = st.IsGit
}
if isGit, ok := got[3]; !ok || isGit {
t.Errorf("stack 3 = (%v, present=%v), want present file stack", isGit, ok)
}
if isGit, ok := got[4]; !ok || !isGit {
t.Errorf("stack 4 = (%v, present=%v), want present git stack", isGit, ok)
}
// The stack is redeployed once, but every member container is threaded through
// (not discarded at the collapse) so each can emit its own notification.
members := map[int][]string{}
for _, st := range grouped.Stacks {
for _, c := range st.Containers {
members[st.StackID] = append(members[st.StackID], c.Name)
}
}
if got := members[3]; len(got) != 2 || got[0] != "web-a" || got[1] != "web-b" {
t.Errorf("stack 3 members = %v, want [web-a web-b]", got)
}
if got := members[4]; len(got) != 1 || got[0] != "api-a" {
t.Errorf("stack 4 members = %v, want [api-a]", got)
}
}

View File

@@ -1,113 +0,0 @@
package containerautomation
import "github.com/rs/zerolog/log"
// EventKind enumerates the container-automation events surfaced to a Notifier.
// The set is intentionally small: it is the seam future milestones extend with
// real senders (Slack/email/webhook) without touching the daemon call sites.
type EventKind string
const (
// EventUpdated is emitted after a container/stack image was updated.
EventUpdated EventKind = "updated"
// EventRollback is emitted after a health-gated rollback to the previous image.
EventRollback EventKind = "rollback"
// EventUpdateFailed is emitted when an update (or its rollback) could not be applied.
EventUpdateFailed EventKind = "update-failed"
// EventHealRestarted is emitted after an unhealthy container was restarted.
EventHealRestarted EventKind = "heal-restarted"
)
// Event is a structured container-automation notification. Optional fields are
// left zero when not applicable to the event (e.g. StackID for a standalone
// update, ContainerID for a stack redeploy).
type Event struct {
Kind EventKind
EndpointID int
ContainerID string
// ContainerName is the human-readable container name (no leading slash), used
// by the webhook message. It may be empty for events keyed only by ID.
ContainerName string
StackID int
// StackName is the compose project (stack) name a container belongs to, sourced
// from its com.docker.compose.project label at detection time. It is set on a
// per-container update event for a stack member so the webhook can print a
// "Stack [name]" line without a StackID/Stack().Read round-trip; empty for
// standalone containers.
StackName string
Image string
// OldDigest and NewDigest carry the pre/post image identities for an update
// (image IDs, e.g. "sha256:59b9..."). They are threaded from the update call
// site where they are known and left empty otherwise; the webhook notifier
// short-forms them into the "old → new" part of the message.
OldDigest string
NewDigest string
Message string
// Err carries the underlying error for failure events; nil otherwise.
Err error
}
// Notifier receives container-automation events. CE has no generic notification
// subsystem, so the only implementation is logNotifier; this interface is the
// seam external senders plug into later.
type Notifier interface {
Notify(event Event)
}
// logNotifier is the default Notifier: it emits each event as a structured log
// line. It never blocks and never errors, so it is safe to call from the daemon
// hot path.
type logNotifier struct{}
// Notify logs the event with its kind and context fields. Failure events are
// logged at warn (with the error), the rest at info.
func (logNotifier) Notify(event Event) {
entry := log.Info()
if event.Kind == EventUpdateFailed {
entry = log.Warn()
if event.Err != nil {
entry = entry.Err(event.Err)
}
}
entry = entry.Str("event", string(event.Kind)).Int("endpoint_id", event.EndpointID)
if event.ContainerID != "" {
entry = entry.Str("container_id", event.ContainerID)
}
if event.StackID != 0 {
entry = entry.Int("stack_id", event.StackID)
}
if event.Image != "" {
entry = entry.Str("image", event.Image)
}
message := event.Message
if message == "" {
message = "container automation event"
}
entry.Msg("container automation: " + message)
}
// multiNotifier fans an event out to several notifiers in order. It is how the
// service composes the always-on logNotifier with the optional webhookNotifier
// without either implementation having to know about the other. Each notifier is
// itself non-blocking, so multiNotifier stays safe on the daemon hot path.
type multiNotifier []Notifier
// Notify forwards the event to every wrapped notifier. Each call is isolated by
// a recover() so one misbehaving notifier can neither abort the others nor let a
// panic reach the daemon hot path; logNotifier is kept first and unchanged.
func (m multiNotifier) Notify(event Event) {
for _, n := range m {
func() {
defer func() {
if r := recover(); r != nil {
log.Warn().Interface("panic", r).Msg("container automation: recovered from panic in notifier")
}
}()
n.Notify(event)
}()
}
}

View File

@@ -1,90 +0,0 @@
package containerautomation
import (
"errors"
"testing"
portainer "github.com/portainer/portainer/api"
)
// recordingNotifier captures emitted events for assertions in tests.
type recordingNotifier struct {
events []Event
}
func (r *recordingNotifier) Notify(event Event) {
r.events = append(r.events, event)
}
func TestLogNotifierDoesNotPanic(t *testing.T) {
n := logNotifier{}
// Every event kind, including a failure carrying an error, must log without
// panicking and without requiring any optional field.
n.Notify(Event{Kind: EventUpdated, EndpointID: 1, ContainerID: "abc", Image: "nginx:latest"})
n.Notify(Event{Kind: EventUpdated, EndpointID: 1, StackID: 7})
n.Notify(Event{Kind: EventRollback, EndpointID: 2, ContainerID: "def", Image: "nginx:1.0"})
n.Notify(Event{Kind: EventHealRestarted, EndpointID: 3, ContainerID: "ghi"})
n.Notify(Event{Kind: EventUpdateFailed, EndpointID: 4, ContainerID: "jkl", Err: errors.New("boom")})
n.Notify(Event{Kind: EventUpdateFailed, EndpointID: 4}) // failure without an error
n.Notify(Event{}) // zero value
}
func TestRecordingNotifierCapturesEvents(t *testing.T) {
r := &recordingNotifier{}
r.Notify(Event{Kind: EventUpdated, EndpointID: 1})
r.Notify(Event{Kind: EventRollback, EndpointID: 1})
if len(r.events) != 2 {
t.Fatalf("captured %d events, want 2", len(r.events))
}
if r.events[0].Kind != EventUpdated || r.events[1].Kind != EventRollback {
t.Errorf("unexpected event kinds: %v, %v", r.events[0].Kind, r.events[1].Kind)
}
}
// panicNotifier always panics, standing in for a misbehaving notifier.
type panicNotifier struct{}
func (panicNotifier) Notify(Event) {
panic("boom")
}
// TestMultiNotifierIsolatesPanics verifies a panicking notifier neither aborts
// the sibling notifiers nor lets the panic reach the caller.
func TestMultiNotifierIsolatesPanics(t *testing.T) {
before := &recordingNotifier{}
after := &recordingNotifier{}
m := multiNotifier{before, panicNotifier{}, after}
// Must not panic even though a wrapped notifier does.
m.Notify(Event{Kind: EventUpdated, EndpointID: 1})
if len(before.events) != 1 {
t.Errorf("notifier before the panicking one got %d events, want 1", len(before.events))
}
if len(after.events) != 1 {
t.Errorf("notifier after the panicking one got %d events, want 1 (panic must not abort the loop)", len(after.events))
}
}
func TestAutomationEnabledForEndpoint(t *testing.T) {
tests := []struct {
name string
endpoint *portainer.Endpoint
want bool
}{
{name: "nil endpoint is not enabled", endpoint: nil, want: false},
{name: "default (zero value) participates", endpoint: &portainer.Endpoint{}, want: true},
{name: "explicitly disabled opts out", endpoint: &portainer.Endpoint{ContainerAutomationDisabled: true}, want: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := AutomationEnabledForEndpoint(tt.endpoint); got != tt.want {
t.Errorf("AutomationEnabledForEndpoint() = %v, want %v", got, tt.want)
}
})
}
}

View File

@@ -1,386 +0,0 @@
package containerautomation
import (
"context"
"errors"
"regexp"
"time"
portainer "github.com/portainer/portainer/api"
"github.com/docker/docker/api/types/container"
"github.com/rs/zerolog/log"
"go.podman.io/image/v5/docker/reference"
)
const (
// defaultRollbackTimeout bounds how long the health gate waits for a freshly
// updated standalone container to become healthy before rolling back.
defaultRollbackTimeout = 120 * time.Second
// rollbackPollInterval is the delay between two health probes of the new
// container while the rollback window is open.
rollbackPollInterval = 3 * time.Second
// rollbackGateBuffer is added to the rollback timeout when deriving the inspect
// context deadline, leaving room for the final probe to complete after the
// decision deadline elapses.
rollbackGateBuffer = 10 * time.Second
// startPeriodBuffer is added to a container's healthcheck start_period when it
// is longer than the rollback timeout, so the gate waits through the whole
// start period (during which Docker reports "starting") plus a small grace
// before deciding. Without it a legitimately slow-starting container would be
// rolled back while it is still initializing normally.
startPeriodBuffer = 15 * time.Second
// maxConsecutiveInspectErrors is how many back-to-back inspect failures the
// health gate tolerates before declaring the update failed. A single transient
// Docker API blip must not trigger a false rollback, so the gate keeps polling
// and only gives up once the failures are clearly not transient.
maxConsecutiveInspectErrors = 3
// updateRollbackCooldown is how long a standalone container whose update was
// rolled back is skipped from updating to the SAME failed image again. It
// breaks the update->rollback loop: without it a persistently-unhealthy new
// image would be re-pulled and rolled back on every poll tick. A genuinely new
// upstream image (a changed remote digest) is not blocked; the cooldown only
// suppresses the exact target that just failed. It is generous because a broken
// upstream image is normally fixed by a new push, which lifts the skip at once.
updateRollbackCooldown = 24 * time.Hour
)
// rolledBackTarget records that a standalone container's update to a specific
// remote image was rolled back, so the same target is skipped until the cooldown
// elapses or the upstream digest changes.
type rolledBackTarget struct {
// ref is the container's original image reference (the re-tag target), used to
// re-resolve the current remote digest on later ticks.
ref string
// digest is the remote image digest that failed the health gate. A later tick
// resolving a DIFFERENT digest (a new upstream push) is allowed through; the
// same digest is skipped until the cooldown elapses. Empty when it could not be
// resolved at rollback time, in which case the guard skips conservatively.
digest string
// at is when the rollback happened; the cooldown is measured from it.
at time.Time
}
// decideUpdateSkip is the pure core of the update->rollback loop guard: given a
// recorded rolled-back target and the freshly-resolved current remote digest, it
// reports whether the standalone update must be skipped this tick. The skip holds
// only while the cooldown is open AND the remote still points at the same failed
// image; once the cooldown elapses the skip is lifted. An unknown recorded digest
// is skipped conservatively (we cannot prove the target changed). Mirrors the
// decideRestart pattern so it is unit-testable without Docker.
func decideUpdateSkip(rec rolledBackTarget, currentDigest string, now time.Time, cooldown time.Duration) bool {
if now.Sub(rec.at) >= cooldown {
return false
}
if rec.digest == "" {
return true
}
return currentDigest == rec.digest
}
// rollbackOutcome is the decision produced from a single health sample.
type rollbackOutcome int
const (
// rollbackContinue: still starting and before the deadline, keep polling.
rollbackContinue rollbackOutcome = iota
// rollbackHealthy: the new container is healthy, accept the update.
rollbackHealthy
// rollbackTrigger: the new container failed the health gate, roll back.
rollbackTrigger
)
// gateResult is the terminal outcome of healthGate. It is a tri-state because a
// shutdown mid-gate must be distinguished from a genuine failure: only a real
// unhealthy/not-running/deadline outcome may roll back.
type gateResult int
const (
// gateHealthy: the new container became healthy in time, accept the update.
gateHealthy gateResult = iota
// gateRollback: the new container failed the gate, roll back to the old image.
gateRollback
// gateAborted: the service base context was cancelled (server shutdown) while
// the gate was open. The new container is left running as-is; no rollback and
// no failure event, since we never observed an actual failure.
gateAborted
)
// imageIDReference matches a content-addressable image id carried verbatim in a
// container's Config.Image when it was started from a bare id (e.g.
// "sha256:ab12…"). Such an id is not a tag and cannot be re-tagged, so it must
// not enable the health gate. A full bare hex id (no algorithm prefix) is
// already rejected by reference.ParseNormalizedNamed; this catches the
// algorithm-prefixed digest form, which otherwise parses as a bogus tag.
var imageIDReference = regexp.MustCompile(`^[a-z0-9]+:[0-9a-f]{64}$`)
// containerHealth is the minimal health signal the gate polls. It is built from
// a container inspect but kept independent of the Docker SDK so the decision
// logic can be unit-tested without a Docker engine.
type containerHealth struct {
// Running reports whether the container is currently running. A container that
// has exited within the window is a failed update.
Running bool
// Status is the Docker health status: "starting", "healthy", "unhealthy" or
// "none"/"" when there is no healthcheck.
Status string
}
// decideRollback is a pure decision over a single health sample taken at time
// `now`, given the rollback `deadline`. It is the testable core of the health
// gate: callers feed it successive samples and act on the outcome.
//
// Rules, in order:
// - healthy -> accept the update (rollbackHealthy);
// - unhealthy -> roll back immediately (Docker only reports unhealthy after the
// configured retries fail, so it is a definitive signal);
// - not running (crashed/exited post-start) -> roll back;
// - still starting past the deadline -> roll back (never became healthy in time);
// - otherwise keep waiting (rollbackContinue).
func decideRollback(h containerHealth, now, deadline time.Time) rollbackOutcome {
switch h.Status {
case string(container.Healthy):
return rollbackHealthy
case string(container.Unhealthy):
return rollbackTrigger
}
if !h.Running {
return rollbackTrigger
}
if !now.Before(deadline) {
return rollbackTrigger
}
return rollbackContinue
}
// effectiveRollbackDeadline derives the health-gate deadline from the gate start
// time, the configured rollback timeout, and the container's healthcheck
// start_period. While a container is within its start_period Docker keeps
// reporting "starting" (it never reports unhealthy yet), so a start_period
// longer than the rollback timeout would otherwise trip a premature rollback
// while the container is initializing normally. The deadline is therefore the
// later of (start + timeout) and (start + start_period + buffer).
func effectiveRollbackDeadline(start time.Time, timeout, startPeriod time.Duration) time.Time {
window := timeout
if startPeriod > 0 {
if d := startPeriod + startPeriodBuffer; d > window {
window = d
}
}
return start.Add(window)
}
// inspectErrorTolerated reports whether the health gate should keep polling after
// `consecutive` back-to-back inspect failures rather than declaring the update
// failed. Up to maxConsecutiveInspectErrors transient errors are tolerated; the
// counter is reset by the caller on any successful inspect.
func inspectErrorTolerated(consecutive int) bool {
return consecutive <= maxConsecutiveInspectErrors
}
// hasHealthGate reports whether a container's healthcheck config yields a usable
// health signal. A nil config, an empty test, or an explicit {"NONE"} disable all
// mean Docker never reports healthy/unhealthy, so there is nothing to gate on.
func hasHealthGate(hc *container.HealthConfig) bool {
if hc == nil || len(hc.Test) == 0 {
return false
}
return hc.Test[0] != "NONE"
}
// isTagReference reports whether ref is a proper tag reference that the health
// gate can roll back. Rolling back re-tags the previous image id onto ref via
// ImageTag, which Docker rejects for a digest-pinned reference (repo@sha256:…)
// with "refusing to create a tag with a digest reference", and which is
// meaningless for a bare image id. Such containers are detected here so the gate
// is skipped instead of silently no-op'ing.
func isTagReference(ref string) bool {
if ref == "" {
return false
}
// Algorithm-prefixed image id (e.g. "sha256:<64 hex>"): a bare id, not a tag.
if imageIDReference.MatchString(ref) {
return false
}
named, err := reference.ParseNormalizedNamed(ref)
if err != nil {
// Unparseable (e.g. a full bare hex image id): not a usable tag target.
return false
}
// A digest-pinned reference (with or without a tag) cannot be re-tagged.
if _, ok := named.(reference.Canonical); ok {
return false
}
return true
}
// healthGate polls the new container's health until it becomes healthy, fails, or
// the rollback window elapses, returning the terminal gateResult.
//
// The polling context is derived from the service base context, so a server
// shutdown ends the wait. A shutdown is reported as gateAborted (leave the new
// container in place, do not roll back): we never observed a real failure, and a
// rollback derived from the cancelled context would itself fail and emit a
// misleading "rollback failed" event on every shutdown during a gate window.
//
// Transient inspect failures (a brief Docker API blip) are tolerated: the gate
// keeps polling and only declares the update failed after more than
// maxConsecutiveInspectErrors consecutive failures, resetting on any success.
//
// Scheduling note (known limitation): this poll runs inside the sequential update
// tick, so N unhealthy standalone containers with rollback enabled can each hold
// the tick for up to their rollback window, delaying other containers/endpoints
// in the same tick. The overlap guard in update() still prevents ticks from
// piling up; this is accepted rather than re-architected (no per-container
// goroutine) to keep the update path simple and ordered.
func (s *Service) healthGate(cli dockerClient, containerID string, timeout, startPeriod time.Duration) gateResult {
if timeout <= 0 {
timeout = defaultRollbackTimeout
}
deadline := effectiveRollbackDeadline(time.Now(), timeout, startPeriod)
ctx, cancel := context.WithDeadline(s.baseCtx, deadline.Add(rollbackGateBuffer))
defer cancel()
consecutiveErrors := 0
for {
inspect, err := cli.ContainerInspect(ctx, containerID)
if err != nil {
// Server shutdown cancelled the base context: abort without rolling back.
if errors.Is(ctx.Err(), context.Canceled) || errors.Is(s.baseCtx.Err(), context.Canceled) {
log.Debug().Str("container_id", containerID).
Msg("auto-update: health gate aborted due to shutdown")
return gateAborted
}
consecutiveErrors++
if !inspectErrorTolerated(consecutiveErrors) {
// Repeated failures: the container vanished or the engine is
// unreachable, treat as a failed update so the rollback can restore
// the previous image.
log.Warn().Err(err).Str("container_id", containerID).Int("consecutive_errors", consecutiveErrors).
Msg("auto-update: health gate inspect failed repeatedly, treating as unhealthy")
return gateRollback
}
// Tolerate a transient blip: keep polling until the data resolves or the
// deadline passes.
log.Debug().Err(err).Str("container_id", containerID).Int("consecutive_errors", consecutiveErrors).
Msg("auto-update: health gate inspect failed, retrying (transient)")
select {
case <-ctx.Done():
return s.gateDeadlineResult()
case <-time.After(rollbackPollInterval):
}
continue
}
consecutiveErrors = 0
h := containerHealth{Running: inspect.State != nil && inspect.State.Running}
if inspect.State != nil && inspect.State.Health != nil {
h.Status = string(inspect.State.Health.Status)
}
switch decideRollback(h, time.Now(), deadline) {
case rollbackHealthy:
return gateHealthy
case rollbackTrigger:
return gateRollback
}
select {
case <-ctx.Done():
return s.gateDeadlineResult()
case <-time.After(rollbackPollInterval):
}
}
}
// gateDeadlineResult maps a context-done gate exit to its outcome: a base-context
// cancellation (shutdown) aborts without rolling back, while a plain deadline
// (the container never became healthy in time) rolls back.
func (s *Service) gateDeadlineResult() gateResult {
if errors.Is(s.baseCtx.Err(), context.Canceled) {
log.Debug().Msg("auto-update: health gate aborted due to shutdown")
return gateAborted
}
return gateRollback
}
// rollback restores the previous image after a failed health-gated update. It
// re-tags the old image id back onto the container's original reference (which
// the new image currently owns), then recreates the new container on that
// reference with no pull, so Recreate's full config-preservation + create-failure
// rollback is reused while resolving to the old image.
//
// Side effect: re-tagging moves `originalRef` from the new image to the old one,
// leaving the new (unhealthy) image untagged/dangling. It is intentionally left
// in place (not pruned) so an operator can inspect why the update failed.
//
// If any step fails the previous image cannot be safely restored, so the
// (unhealthy) new container is left running rather than destroyed, and a loud
// failure notification is emitted.
func (s *Service) rollback(cli dockerClient, endpoint *portainer.Endpoint, newContainerID, oldImageID, originalRef, containerName string) {
endpointID := int(endpoint.ID)
log.Warn().Str("container_id", newContainerID).Str("image", originalRef).Int("endpoint_id", endpointID).
Msg("auto-update: new container failed the health gate, rolling back to the previous image")
ctx, cancel := context.WithTimeout(s.baseCtx, recreateTimeout)
defer cancel()
// Re-tag the previous image id back onto the original reference. After the
// update the reference points at the new image; this moves it back so Recreate
// resolves the old image without a pull.
if err := cli.ImageTag(ctx, oldImageID, originalRef); err != nil {
log.Error().Err(err).Str("image_id", oldImageID).Str("image", originalRef).Int("endpoint_id", endpointID).
Msg("auto-update: rollback failed to re-tag the previous image, leaving the unhealthy container in place")
s.notifier.Notify(Event{
Kind: EventUpdateFailed, EndpointID: endpointID, ContainerID: newContainerID, ContainerName: containerName,
Image: originalRef, Message: "rollback failed: could not re-tag previous image", Err: err,
})
return
}
if _, err := s.containerService.Recreate(ctx, endpoint, newContainerID, false, "", ""); err != nil {
log.Error().Err(err).Str("container_id", newContainerID).Str("image", originalRef).Int("endpoint_id", endpointID).
Msg("auto-update: rollback recreate failed, leaving the unhealthy container in place")
s.notifier.Notify(Event{
Kind: EventUpdateFailed, EndpointID: endpointID, ContainerID: newContainerID, ContainerName: containerName,
Image: originalRef, Message: "rollback failed: could not recreate on previous image", Err: err,
})
return
}
log.Warn().Str("container_id", newContainerID).Str("image", originalRef).Int("endpoint_id", endpointID).
Msg("auto-update: rolled back to the previous image after a failed update")
s.notifier.Notify(Event{
Kind: EventRollback, EndpointID: endpointID, ContainerID: newContainerID, ContainerName: containerName,
Image: originalRef, Message: "rolled back to previous image after failed health check",
})
// Record the failed target so the next poll does not immediately re-pull the
// same broken image and roll back again (the update->rollback loop). Recorded
// only after a SUCCESSFUL rollback; a changed remote digest later lifts the skip.
s.recordRolledBack(endpoint, containerName, originalRef)
}

View File

@@ -1,333 +0,0 @@
package containerautomation
import (
"testing"
"time"
"github.com/docker/docker/api/types/container"
)
func TestDecideRollback(t *testing.T) {
now := time.Date(2026, 6, 28, 12, 0, 0, 0, time.UTC)
deadline := now.Add(120 * time.Second)
tests := []struct {
name string
health containerHealth
at time.Time
want rollbackOutcome
}{
{
name: "healthy within the window accepts the update",
health: containerHealth{Running: true, Status: string(container.Healthy)},
at: now.Add(10 * time.Second),
want: rollbackHealthy,
},
{
name: "unhealthy triggers an immediate rollback",
health: containerHealth{Running: true, Status: string(container.Unhealthy)},
at: now.Add(10 * time.Second),
want: rollbackTrigger,
},
{
name: "still starting before the deadline keeps polling",
health: containerHealth{Running: true, Status: string(container.Starting)},
at: now.Add(10 * time.Second),
want: rollbackContinue,
},
{
name: "still starting past the deadline rolls back",
health: containerHealth{Running: true, Status: string(container.Starting)},
at: now.Add(121 * time.Second),
want: rollbackTrigger,
},
{
name: "starting exactly at the deadline rolls back",
health: containerHealth{Running: true, Status: string(container.Starting)},
at: deadline,
want: rollbackTrigger,
},
{
name: "exited container rolls back even before the deadline",
health: containerHealth{Running: false, Status: string(container.Starting)},
at: now.Add(5 * time.Second),
want: rollbackTrigger,
},
{
name: "unhealthy wins over a stopped state",
health: containerHealth{Running: false, Status: string(container.Unhealthy)},
at: now.Add(5 * time.Second),
want: rollbackTrigger,
},
{
name: "healthy wins even past the deadline",
health: containerHealth{Running: true, Status: string(container.Healthy)},
at: now.Add(200 * time.Second),
want: rollbackHealthy,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := decideRollback(tt.health, tt.at, deadline); got != tt.want {
t.Errorf("decideRollback() = %v, want %v", got, tt.want)
}
})
}
}
func TestEffectiveRollbackDeadline(t *testing.T) {
start := time.Date(2026, 6, 28, 12, 0, 0, 0, time.UTC)
timeout := 120 * time.Second
tests := []struct {
name string
startPeriod time.Duration
want time.Time
}{
{
name: "no start period uses the timeout",
startPeriod: 0,
want: start.Add(timeout),
},
{
name: "start period shorter than timeout uses the timeout",
startPeriod: 30 * time.Second,
want: start.Add(timeout),
},
{
name: "start period longer than timeout extends to start period plus buffer",
startPeriod: 300 * time.Second,
want: start.Add(300*time.Second + startPeriodBuffer),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := effectiveRollbackDeadline(start, timeout, tt.startPeriod); !got.Equal(tt.want) {
t.Errorf("effectiveRollbackDeadline() = %v, want %v", got, tt.want)
}
})
}
}
// TestDecideRollbackWithLongStartPeriod proves the F3 fix end to end at the
// decision layer: with a start_period longer than the configured rollback
// timeout, the start-period-aware deadline keeps a still-starting container
// alive while it is within the start period, and only rolls back after it.
func TestDecideRollbackWithLongStartPeriod(t *testing.T) {
start := time.Date(2026, 6, 28, 12, 0, 0, 0, time.UTC)
timeout := 60 * time.Second
startPeriod := 300 * time.Second
deadline := effectiveRollbackDeadline(start, timeout, startPeriod)
starting := containerHealth{Running: true, Status: string(container.Starting)}
// Past the bare timeout but still within the start period: keep waiting.
if got := decideRollback(starting, start.Add(120*time.Second), deadline); got != rollbackContinue {
t.Errorf("within start_period: decideRollback() = %v, want rollbackContinue", got)
}
// After the start period (plus buffer): roll back.
if got := decideRollback(starting, start.Add(330*time.Second), deadline); got != rollbackTrigger {
t.Errorf("after start_period: decideRollback() = %v, want rollbackTrigger", got)
}
}
func TestInspectErrorTolerated(t *testing.T) {
tests := []struct {
name string
consecutive int
want bool
}{
{name: "first transient error is tolerated", consecutive: 1, want: true},
{name: "second consecutive error is tolerated", consecutive: 2, want: true},
{name: "at the threshold is still tolerated", consecutive: maxConsecutiveInspectErrors, want: true},
{name: "beyond the threshold is a failure", consecutive: maxConsecutiveInspectErrors + 1, want: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := inspectErrorTolerated(tt.consecutive); got != tt.want {
t.Errorf("inspectErrorTolerated(%d) = %v, want %v", tt.consecutive, got, tt.want)
}
})
}
}
func TestIsTagReference(t *testing.T) {
const digest = "sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2"
tests := []struct {
name string
ref string
want bool
}{
{name: "tagged reference is rollbackable", ref: "nginx:1.21", want: true},
{name: "untagged reference (implicit latest) is rollbackable", ref: "nginx", want: true},
{name: "fully-qualified tagged reference is rollbackable", ref: "registry.example.com/team/app:v2", want: true},
{name: "digest-pinned reference cannot be re-tagged", ref: "nginx@" + digest, want: false},
{name: "tagged-and-digest-pinned reference cannot be re-tagged", ref: "nginx:1.21@" + digest, want: false},
{name: "algorithm-prefixed bare image id cannot be re-tagged", ref: digest, want: false},
{name: "full bare hex image id cannot be re-tagged", ref: "02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2", want: false},
{name: "empty reference is not rollbackable", ref: "", want: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := isTagReference(tt.ref); got != tt.want {
t.Errorf("isTagReference(%q) = %v, want %v", tt.ref, got, tt.want)
}
})
}
}
func TestSkipUnnamedForRollback(t *testing.T) {
tests := []struct {
name string
rollback bool
cName string
want bool
}{
{name: "rollback on, unnamed -> skip (unsuppressable loop otherwise)", rollback: true, cName: "", want: true},
{name: "rollback on, named -> proceed (guard can key it)", rollback: true, cName: "web", want: false},
{name: "rollback off, unnamed -> proceed (no rollback to loop)", rollback: false, cName: "", want: false},
{name: "rollback off, named -> proceed", rollback: false, cName: "web", want: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := skipUnnamedForRollback(tt.rollback, tt.cName); got != tt.want {
t.Errorf("skipUnnamedForRollback(%v, %q) = %v, want %v", tt.rollback, tt.cName, got, tt.want)
}
})
}
}
func TestHasHealthGate(t *testing.T) {
tests := []struct {
name string
hc *container.HealthConfig
want bool
}{
{name: "nil config has no gate", hc: nil, want: false},
{name: "empty test inherits, no usable gate", hc: &container.HealthConfig{Test: nil}, want: false},
{name: "explicit NONE disables the gate", hc: &container.HealthConfig{Test: []string{"NONE"}}, want: false},
{name: "CMD healthcheck yields a gate", hc: &container.HealthConfig{Test: []string{"CMD", "curl", "-f", "localhost"}}, want: true},
{name: "CMD-SHELL healthcheck yields a gate", hc: &container.HealthConfig{Test: []string{"CMD-SHELL", "exit 0"}}, want: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := hasHealthGate(tt.hc); got != tt.want {
t.Errorf("hasHealthGate() = %v, want %v", got, tt.want)
}
})
}
}
func TestParseRollbackTimeout(t *testing.T) {
tests := []struct {
name string
raw string
want time.Duration
}{
{name: "valid duration", raw: "90s", want: 90 * time.Second},
{name: "empty falls back to default", raw: "", want: defaultRollbackTimeout},
{name: "unparseable falls back to default", raw: "nope", want: defaultRollbackTimeout},
{name: "zero falls back to default", raw: "0s", want: defaultRollbackTimeout},
{name: "negative falls back to default", raw: "-5s", want: defaultRollbackTimeout},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := parseRollbackTimeout(tt.raw); got != tt.want {
t.Errorf("parseRollbackTimeout(%q) = %v, want %v", tt.raw, got, tt.want)
}
})
}
}
func TestDecideUpdateSkip(t *testing.T) {
now := time.Date(2026, 6, 28, 12, 0, 0, 0, time.UTC)
cooldown := 24 * time.Hour
tests := []struct {
name string
rec rolledBackTarget
currentDigest string
want bool
}{
{
name: "same digest within cooldown is skipped",
rec: rolledBackTarget{digest: "sha256:aaa", at: now.Add(-1 * time.Hour)},
currentDigest: "sha256:aaa",
want: true,
},
{
name: "new digest within cooldown is not skipped",
rec: rolledBackTarget{digest: "sha256:aaa", at: now.Add(-1 * time.Hour)},
currentDigest: "sha256:bbb",
want: false,
},
{
name: "same digest after cooldown is not skipped",
rec: rolledBackTarget{digest: "sha256:aaa", at: now.Add(-25 * time.Hour)},
currentDigest: "sha256:aaa",
want: false,
},
{
name: "unknown recorded digest is skipped conservatively within cooldown",
rec: rolledBackTarget{digest: "", at: now.Add(-1 * time.Hour)},
currentDigest: "sha256:aaa",
want: true,
},
{
name: "unknown recorded digest after cooldown is not skipped",
rec: rolledBackTarget{digest: "", at: now.Add(-25 * time.Hour)},
currentDigest: "sha256:aaa",
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := decideUpdateSkip(tt.rec, tt.currentDigest, now, cooldown); got != tt.want {
t.Errorf("decideUpdateSkip() = %v, want %v", got, tt.want)
}
})
}
}
// TestPruneRolledBack locks in the F8 fix: pruneRolledBack must iterate the
// rolledBack map and drop only entries whose cooldown has fully elapsed, keeping
// fresh ones, so the map cannot grow unbounded. It mirrors TestPruneRetries. The
// boundary is inclusive (production uses now.Sub(at) >= updateRollbackCooldown),
// so an entry exactly at the cooldown is pruned.
func TestPruneRolledBack(t *testing.T) {
now := time.Date(2026, 6, 28, 12, 0, 0, 0, time.UTC)
s := &Service{rolledBack: map[string]rolledBackTarget{
// within the cooldown -> retained
"fresh": {ref: "img:fresh", digest: "sha256:aaa", at: now.Add(-updateRollbackCooldown / 2)},
// exactly at the cooldown boundary -> pruned (>= is inclusive)
"edge": {ref: "img:edge", digest: "sha256:bbb", at: now.Add(-updateRollbackCooldown)},
// long past the cooldown -> pruned
"stale": {ref: "img:stale", digest: "sha256:ccc", at: now.Add(-2 * updateRollbackCooldown)},
}}
s.pruneRolledBack(now)
if _, ok := s.rolledBack["fresh"]; !ok {
t.Error("entry within the rollback cooldown should be retained")
}
if _, ok := s.rolledBack["edge"]; ok {
t.Error("entry exactly at the cooldown boundary should be pruned")
}
if _, ok := s.rolledBack["stale"]; ok {
t.Error("entry past the rollback cooldown should be pruned")
}
if len(s.rolledBack) != 1 {
t.Errorf("rolledBack length = %d, want 1", len(s.rolledBack))
}
}

View File

@@ -1,40 +0,0 @@
package containerautomation
import (
"context"
portainer "github.com/portainer/portainer/api"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/image"
)
// dockerClient is the minimal subset of the Docker SDK client
// (*github.com/docker/docker/client.Client) that the auto-update and auto-heal
// apply paths call. Threading this interface — instead of the concrete client —
// through the daemon paths lets their wiring/sequencing be exercised with a fake
// in tests, while the real client returned by ClientFactory.CreateClient
// satisfies it unchanged at the call sites.
type dockerClient interface {
// ContainerInspect backs the pre-update image-identity capture
// (updateStandalone), the health-gate poll (healthGate) and the post-redeploy
// new-image re-inspect (inspectImageID).
ContainerInspect(ctx context.Context, containerID string) (container.InspectResponse, error)
// ContainerRestart backs the auto-heal restart of an unhealthy container.
ContainerRestart(ctx context.Context, containerID string, options container.StopOptions) error
// ImageTag backs the rollback re-tag of the previous image onto the original ref.
ImageTag(ctx context.Context, source, target string) error
// ImageRemove backs the conservative cleanup of the dangling old image.
ImageRemove(ctx context.Context, imageID string, options image.RemoveOptions) ([]image.DeleteResponse, error)
}
// containerRecreator is the minimal seam over *docker.ContainerService used by the
// standalone update and rollback paths to recreate a container (pull + stop +
// create + start). The concrete *docker.ContainerService satisfies it; threading
// the interface lets the recreate step be faked so the surrounding sequencing
// (recreate -> health gate -> cleanup / rollback) is testable without a live
// engine.
type containerRecreator interface {
Recreate(ctx context.Context, endpoint *portainer.Endpoint, containerID string, forcePullImage bool, imageTag, nodeName string) (*types.ContainerJSON, error)
}

View File

@@ -1,321 +0,0 @@
// Package containerautomation provides native container automation that runs as
// background scheduler jobs. M1 implements auto-heal (restarting Docker
// containers whose healthcheck reports "unhealthy", replacing the
// willfarrell/autoheal sidecar); M4 adds auto-update (periodically detecting
// outdated images and applying updates, replacing the containrrr/watchtower
// sidecar).
package containerautomation
import (
"context"
"sync"
"sync/atomic"
"time"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/docker"
dockerclient "github.com/portainer/portainer/api/docker/client"
"github.com/portainer/portainer/api/docker/images"
"github.com/portainer/portainer/api/scheduler"
"github.com/portainer/portainer/api/stacks/deployments"
"github.com/rs/zerolog/log"
)
const (
// defaultCheckInterval is used when the configured auto-heal interval is empty or unparseable.
defaultCheckInterval = 30 * time.Second
// defaultPollInterval is used when the configured auto-update interval is empty or unparseable.
// It is conservative (hours) to stay within registry rate limits; the image-status cache is
// short-lived (keyed by the local imageID), so each poll re-checks the remote digest.
defaultPollInterval = 6 * time.Hour
)
// Service manages the lifecycle of the auto-heal and auto-update scheduler jobs
// and keeps the per-container retry state in memory across ticks.
type Service struct {
// baseCtx is the application shutdown context. It is the base for every
// per-operation timeout context, so a server shutdown cancels in-flight heal
// restarts and update redeploys instead of letting them run detached.
baseCtx context.Context
scheduler *scheduler.Scheduler
dataStore dataservices.DataStore
clientFactory *dockerclient.ClientFactory
// Dependencies used by the auto-update job (M4).
digestClient *images.DigestClient
// containerService is the recreate seam (satisfied by *docker.ContainerService);
// an interface so the standalone update/rollback recreate step can be faked in
// tests. See containerRecreator.
containerService containerRecreator
stackDeployer deployments.StackDeployer
// notifier receives automation events (update/rollback/failure/heal). The
// default is logNotifier; the field is the seam external senders plug into.
notifier Notifier
mu sync.Mutex
healJobID string
updateJobID string
// running guards against overlapping heal ticks.
running atomic.Bool
// updateRunning guards against overlapping update ticks.
updateRunning atomic.Bool
retryMu sync.Mutex
retries map[string]retryState
// rolledBackMu guards rolledBack.
rolledBackMu sync.Mutex
// rolledBack records standalone containers whose update was rolled back, keyed
// by endpoint+name, so the auto-update job does not immediately re-pull the
// same failed image and roll back again on the next tick (the update->rollback
// loop guard, mirroring the auto-heal retries map).
//
// This state is in-memory only and is NOT persisted: after a Portainer restart
// the map is empty, so at most one extra update->rollback cycle per restart is
// possible before the guard re-records the failed target. Persisting it would
// require a datastore schema (key + digest + timestamp) and is intentionally out
// of scope here; the cooldown-bounded single extra cycle is an acceptable
// trade-off against that complexity.
rolledBack map[string]rolledBackTarget
}
// NewService creates a new container automation service. Call Start to schedule
// the jobs according to the persisted settings. baseCtx is the application
// shutdown context: it bounds the job operation contexts so a shutdown cancels
// any in-flight heal/update. The stackDeployer and containerService are used by
// the auto-update job; they may be nil only in tests that do not exercise
// auto-update.
func NewService(
baseCtx context.Context,
scheduler *scheduler.Scheduler,
dataStore dataservices.DataStore,
clientFactory *dockerclient.ClientFactory,
containerService *docker.ContainerService,
stackDeployer deployments.StackDeployer,
) *Service {
if baseCtx == nil {
baseCtx = context.Background()
}
return &Service{
baseCtx: baseCtx,
scheduler: scheduler,
dataStore: dataStore,
clientFactory: clientFactory,
digestClient: images.NewClientWithRegistry(images.NewRegistryClient(dataStore), clientFactory),
containerService: containerService,
stackDeployer: stackDeployer,
// Compose the always-on log notifier with the optional webhook notifier.
// The webhook reads the current settings per-event from the datastore, so a
// URL change in the UI takes effect without a restart; logNotifier keeps the
// existing structured log output unchanged.
notifier: multiNotifier{logNotifier{}, newWebhookNotifier(dataStore)},
retries: make(map[string]retryState),
rolledBack: make(map[string]rolledBackTarget),
}
}
// AutomationEnabledForEndpoint reports whether container automation (auto-heal and
// auto-update) should run for an environment. It is the per-endpoint opt-out (M5)
// layered on top of the global switch: an environment participates unless it has
// been explicitly disabled. The zero value (not disabled) preserves the
// pre-M5 behavior for every existing environment.
func AutomationEnabledForEndpoint(endpoint *portainer.Endpoint) bool {
return endpoint != nil && !endpoint.ContainerAutomationDisabled
}
// Start schedules the enabled jobs according to the persisted settings.
func (s *Service) Start() {
s.mu.Lock()
defer s.mu.Unlock()
s.start()
}
// Reload re-applies the current settings: it stops the running jobs and starts
// fresh ones with the new intervals, or leaves them stopped if disabled. It is
// safe to call after a settings update.
//
// Note: stopping a job unschedules future ticks but does not interrupt a tick
// already in progress. An in-flight heal/update pass runs to completion on its
// original (pre-reload) context and is only cancelled by a server shutdown (via
// baseCtx); the new interval takes effect from the next scheduled tick. The
// overlap guards (running/updateRunning) and the per-map mutexes keep this safe
// against data races, so this is a deliberate behavioural nuance, not a bug.
func (s *Service) Reload() error {
s.mu.Lock()
defer s.mu.Unlock()
s.stop()
s.start()
return nil
}
// start (re)schedules the enabled jobs from settings. Caller must hold s.mu.
func (s *Service) start() {
settings, err := s.dataStore.Settings().Settings()
if err != nil {
log.Warn().Err(err).Msg("container automation: unable to read settings, jobs not scheduled")
return
}
s.startHeal(settings)
s.startUpdate(settings)
}
// startHeal schedules the auto-heal job if enabled. Caller must hold s.mu.
func (s *Service) startHeal(settings *portainer.Settings) {
if s.healJobID != "" {
return
}
autoHeal := settings.ContainerAutomation.AutoHeal
if !autoHeal.Enabled {
return
}
interval, err := time.ParseDuration(autoHeal.CheckInterval)
if err != nil || interval <= 0 {
log.Warn().Str("interval", autoHeal.CheckInterval).Dur("default", defaultCheckInterval).
Msg("auto-heal: invalid check interval, falling back to default")
interval = defaultCheckInterval
}
s.healJobID = s.scheduler.StartJobEvery(interval, s.heal)
log.Info().Dur("interval", interval).Msg("auto-heal: job scheduled")
}
// startUpdate schedules the auto-update job if enabled. Caller must hold s.mu.
func (s *Service) startUpdate(settings *portainer.Settings) {
if s.updateJobID != "" {
return
}
autoUpdate := settings.ContainerAutomation.AutoUpdate
if !autoUpdate.Enabled {
return
}
interval, err := time.ParseDuration(autoUpdate.PollInterval)
if err != nil || interval <= 0 {
log.Warn().Str("interval", autoUpdate.PollInterval).Dur("default", defaultPollInterval).
Msg("auto-update: invalid poll interval, falling back to default")
interval = defaultPollInterval
}
s.updateJobID = s.scheduler.StartJobEvery(interval, s.update)
log.Info().Dur("interval", interval).Msg("auto-update: job scheduled")
}
// stop cancels the running jobs, if any. Caller must hold s.mu.
func (s *Service) stop() {
if s.healJobID != "" {
if err := s.scheduler.StopJob(s.healJobID); err != nil {
log.Warn().Err(err).Msg("auto-heal: could not stop the job")
}
s.healJobID = ""
}
if s.updateJobID != "" {
if err := s.scheduler.StopJob(s.updateJobID); err != nil {
log.Warn().Err(err).Msg("auto-update: could not stop the job")
}
s.updateJobID = ""
}
}
// scope returns the configured auto-heal scope, defaulting to "labeled".
func (s *Service) scope() string {
settings, err := s.dataStore.Settings().Settings()
if err != nil {
return ScopeLabeled
}
if settings.ContainerAutomation.AutoHeal.Scope == ScopeAll {
return ScopeAll
}
return ScopeLabeled
}
// getRetry returns the retry state for a container (zero value if unknown).
func (s *Service) getRetry(containerID string) retryState {
s.retryMu.Lock()
defer s.retryMu.Unlock()
return s.retries[containerID]
}
// setRetry stores the retry state for a container.
func (s *Service) setRetry(containerID string, state retryState) {
s.retryMu.Lock()
defer s.retryMu.Unlock()
s.retries[containerID] = state
}
// getRolledBack returns the rolled-back target for a key and whether it exists.
func (s *Service) getRolledBack(key string) (rolledBackTarget, bool) {
s.rolledBackMu.Lock()
defer s.rolledBackMu.Unlock()
rec, ok := s.rolledBack[key]
return rec, ok
}
// setRolledBack records a rolled-back target for a key.
func (s *Service) setRolledBack(key string, rec rolledBackTarget) {
s.rolledBackMu.Lock()
defer s.rolledBackMu.Unlock()
s.rolledBack[key] = rec
}
// clearRolledBack drops the rolled-back record for a key (cooldown elapsed or a
// new upstream image lifted the skip).
func (s *Service) clearRolledBack(key string) {
s.rolledBackMu.Lock()
defer s.rolledBackMu.Unlock()
delete(s.rolledBack, key)
}
// pruneRolledBack drops rolled-back records whose cooldown has fully elapsed, so
// the map cannot grow unbounded. It mirrors pruneRetries.
func (s *Service) pruneRolledBack(now time.Time) {
s.rolledBackMu.Lock()
defer s.rolledBackMu.Unlock()
for key, rec := range s.rolledBack {
if now.Sub(rec.at) >= updateRollbackCooldown {
delete(s.rolledBack, key)
}
}
}
// pruneRetries drops retry state for containers whose retry window has fully
// elapsed since their last restart. A container is kept regardless of whether it
// appeared in the current tick: one that briefly leaves the unhealthy filter
// (e.g. while "starting" right after a restart) must not lose its accounting, or
// the cooldown / max-retries storm guard would be defeated. A container that has
// recovered and stayed quiet for longer than the window is cleaned up (fresh
// budget next incident, no unbounded growth).
func (s *Service) pruneRetries(now time.Time) {
s.retryMu.Lock()
defer s.retryMu.Unlock()
for id, state := range s.retries {
if now.Sub(state.lastRestart) >= retryWindow {
delete(s.retries, id)
}
}
}

View File

@@ -1,261 +0,0 @@
package containerautomation
import (
"context"
"fmt"
"net/http"
"net/url"
"strings"
"time"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/rs/zerolog/log"
)
const (
// webhookMessagePlaceholder is the token replaced in the configured webhook
// URL with the URL-encoded event message. When present, the notifier issues a
// GET on the substituted URL ("message in the address"); when absent, it POSTs
// the plain-text message as the request body.
webhookMessagePlaceholder = "{{message}}"
// webhookTimeout bounds each webhook HTTP call so a slow or unresponsive
// endpoint cannot pile up goroutines. The call already runs off the hot path.
webhookTimeout = 10 * time.Second
// shortDigestLen is how many leading hex characters of an image digest the
// message keeps (matches the maintainer's example, e.g. "59b94983c73a").
shortDigestLen = 12
)
// webhookNotifier delivers container-automation events to a user-configured HTTP
// endpoint. It reads the current webhook URL from the datastore on every event
// so a settings change takes effect without a restart, formats a human-readable
// message, and performs the HTTP call in a background goroutine so a slow or
// broken endpoint never delays or fails the daemon hot path.
type webhookNotifier struct {
dataStore dataservices.DataStore
client *http.Client
}
// newWebhookNotifier builds a webhookNotifier bound to the datastore. The HTTP
// client carries the per-call timeout so a request cannot hang indefinitely.
func newWebhookNotifier(dataStore dataservices.DataStore) webhookNotifier {
return webhookNotifier{
dataStore: dataStore,
client: &http.Client{Timeout: webhookTimeout},
}
}
// webhookURLForKind selects the configured webhook URL for an event kind: the
// update-family events (image update, rollback, update-failed) route to the
// update endpoint, and the auto-heal restart routes to the heal endpoint. This
// lets a user enable notifications for one mechanism without the other — an
// empty URL for a mechanism means "no webhook for that mechanism".
func webhookURLForKind(notification portainer.ContainerAutomationNotificationSettings, kind EventKind) string {
switch kind {
case EventUpdated, EventRollback, EventUpdateFailed:
return notification.UpdateWebhookURL
case EventHealRestarted:
return notification.HealWebhookURL
default:
return ""
}
}
// Notify reads the webhook URL for the event's mechanism (update vs heal) and,
// when set, dispatches the event in a background goroutine. Only the settings
// read and the empty-URL short-circuit run synchronously (they decide whether
// to spawn at all); message formatting — which itself reads Endpoint()/Stack()
// from the datastore — and the HTTP call both happen off the daemon hot path,
// under a single recover(). It never blocks the caller and never returns an
// error: the webhook is strictly best-effort. When the URL for the event's
// mechanism is empty, the event is skipped and the other mechanism is
// unaffected.
func (n webhookNotifier) Notify(event Event) {
settings, err := n.dataStore.Settings().Settings()
if err != nil {
log.Warn().Err(err).Msg("container automation webhook: unable to read settings, skipping notification")
return
}
webhookURL := strings.TrimSpace(webhookURLForKind(settings.ContainerAutomation.Notification, event.Kind))
if webhookURL == "" {
return
}
// Best-effort delivery: never block or fail the caller (the update/heal hot
// path). Everything below — the env/stack datastore reads in formatMessage and
// the bounded HTTP call — runs in its own goroutine, and any panic there is
// recovered so it can never crash the daemon.
go func() {
defer func() {
if r := recover(); r != nil {
log.Warn().Interface("panic", r).Msg("container automation webhook: recovered from panic during delivery")
}
}()
message := n.formatMessage(settings, event)
n.deliver(webhookURL, message)
}()
}
// deliver performs the HTTP call for a single event. It is always invoked from
// the Notify goroutine (which recovers any panic), so a broken endpoint can
// never block or crash the daemon.
func (n webhookNotifier) deliver(webhookURL, message string) {
ctx, cancel := context.WithTimeout(context.Background(), webhookTimeout)
defer cancel()
var (
req *http.Request
err error
)
if strings.Contains(webhookURL, webhookMessagePlaceholder) {
// Substitution mode: replace the placeholder with the URL-encoded message
// and GET the resulting address (the maintainer's "message in the URL").
target := strings.ReplaceAll(webhookURL, webhookMessagePlaceholder, url.QueryEscape(message))
req, err = http.NewRequestWithContext(ctx, http.MethodGet, target, nil)
} else {
// No placeholder: POST the plain-text message as the body, useful for
// generic POST-style webhooks.
req, err = http.NewRequestWithContext(ctx, http.MethodPost, webhookURL, strings.NewReader(message))
if err == nil {
req.Header.Set("Content-Type", "text/plain; charset=utf-8")
}
}
if err != nil {
log.Warn().Err(err).Msg("container automation webhook: unable to build request")
return
}
resp, err := n.client.Do(req)
if err != nil {
log.Warn().Err(err).Msg("container automation webhook: delivery failed")
return
}
defer resp.Body.Close()
if resp.StatusCode >= http.StatusBadRequest {
log.Warn().Int("status", resp.StatusCode).Msg("container automation webhook: endpoint returned an error status")
}
}
// formatMessage builds the human-readable message for an event. It resolves the
// environment name from the endpoint and the stack name from the stack via the
// datastore, mirroring the maintainer's example:
//
// Environment | nebula.lc
// Stack [cache-demo]
// Update [esphome]: 59b94983c73a → 2231ca5d676d
//
// The context line is the stack for stack-scoped events, otherwise the container;
// the action line is adapted per event kind (update / rollback / update-failed /
// auto-heal restart). Auto-heal renders as:
//
// Environment | nebula.lc
// Container [nginx]
// Auto-heal: restarted unhealthy container
func (n webhookNotifier) formatMessage(settings *portainer.Settings, event Event) string {
lines := []string{"Environment | " + n.environmentName(event.EndpointID)}
// Context line: the stack for stack-scoped events, otherwise the container. A
// per-container stack-member update carries StackName (from the compose label),
// preferred over a StackID/Stack().Read round-trip; the container itself still
// names the action line below.
switch {
case event.StackName != "":
lines = append(lines, fmt.Sprintf("Stack [%s]", event.StackName))
case event.StackID != 0:
lines = append(lines, fmt.Sprintf("Stack [%s]", n.stackName(event.StackID)))
case event.ContainerName != "":
lines = append(lines, fmt.Sprintf("Container [%s]", event.ContainerName))
}
// Subject for the action line: the container name when known, else the stack
// name, else a short container id.
subject := event.ContainerName
if subject == "" && event.StackID != 0 {
subject = n.stackName(event.StackID)
}
if subject == "" {
subject = shortDigest(event.ContainerID)
}
switch event.Kind {
case EventUpdated:
if event.OldDigest != "" && event.NewDigest != "" {
lines = append(lines, fmt.Sprintf("Update [%s]: %s → %s", subject, shortDigest(event.OldDigest), shortDigest(event.NewDigest)))
} else {
lines = append(lines, fmt.Sprintf("Update [%s]: image updated", subject))
}
case EventRollback:
lines = append(lines, fmt.Sprintf("Rollback [%s]: rolled back to previous image after failed health check", subject))
case EventUpdateFailed:
line := fmt.Sprintf("Update failed [%s]", subject)
if event.Message != "" {
line += ": " + event.Message
}
if event.Err != nil {
line += fmt.Sprintf(" (%s)", event.Err)
}
lines = append(lines, line)
case EventHealRestarted:
lines = append(lines, "Auto-heal: restarted unhealthy container")
default:
if event.Message != "" {
lines = append(lines, event.Message)
}
}
return strings.Join(lines, "\n")
}
// environmentName resolves an endpoint id to its display name, degrading to a
// "#<id>" placeholder when the endpoint cannot be read (deleted, or a zero id).
func (n webhookNotifier) environmentName(endpointID int) string {
if endpointID == 0 {
return "unknown"
}
endpoint, err := n.dataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
if err != nil || endpoint == nil {
return fmt.Sprintf("#%d", endpointID)
}
return endpoint.Name
}
// stackName resolves a stack id to its name, degrading to a "#<id>" placeholder
// when the stack cannot be read.
func (n webhookNotifier) stackName(stackID int) string {
stack, err := n.dataStore.Stack().Read(portainer.StackID(stackID))
if err != nil || stack == nil {
return fmt.Sprintf("#%d", stackID)
}
return stack.Name
}
// shortDigest trims an image id/digest to a short, human-friendly hex form
// (shortDigestLen chars), matching the maintainer's example. It drops a leading
// "sha256:" algorithm prefix so "sha256:59b94983c73a..." -> "59b94983c73a".
func shortDigest(s string) string {
if s == "" {
return ""
}
if i := strings.LastIndex(s, "sha256:"); i >= 0 {
s = s[i+len("sha256:"):]
}
if len(s) > shortDigestLen {
return s[:shortDigestLen]
}
return s
}

View File

@@ -1,437 +0,0 @@
package containerautomation
import (
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/datastore"
)
// newTestWebhookNotifier builds an initialized test datastore, sets both the
// update and heal webhook URLs to the same value (so the notifier fires for
// every event kind), and returns a webhookNotifier bound to it. Use
// newTestWebhookNotifierSplit to configure the two URLs independently.
func newTestWebhookNotifier(t *testing.T, webhookURL string) (webhookNotifier, *datastore.Store) {
t.Helper()
return newTestWebhookNotifierSplit(t, webhookURL, webhookURL)
}
// newTestWebhookNotifierSplit builds an initialized test datastore with the
// auto-update and auto-heal webhook URLs set independently, and returns a
// webhookNotifier bound to it.
func newTestWebhookNotifierSplit(t *testing.T, updateURL, healURL string) (webhookNotifier, *datastore.Store) {
t.Helper()
_, store := datastore.MustNewTestStore(t, true, false)
settings, err := store.Settings().Settings()
if err != nil {
t.Fatalf("read settings: %v", err)
}
settings.ContainerAutomation.Notification.UpdateWebhookURL = updateURL
settings.ContainerAutomation.Notification.HealWebhookURL = healURL
if err := store.Settings().UpdateSettings(settings); err != nil {
t.Fatalf("update settings: %v", err)
}
return newWebhookNotifier(store), store
}
func createEndpoint(t *testing.T, store *datastore.Store, id int, name string) {
t.Helper()
if err := store.Endpoint().Create(&portainer.Endpoint{ID: portainer.EndpointID(id), Name: name}); err != nil {
t.Fatalf("create endpoint: %v", err)
}
}
func createStack(t *testing.T, store *datastore.Store, id int, name string) {
t.Helper()
if err := store.Stack().Create(&portainer.Stack{ID: portainer.StackID(id), Name: name}); err != nil {
t.Fatalf("create stack: %v", err)
}
}
// TestWebhookNotifierGETPlaceholder verifies the placeholder is replaced with the
// URL-encoded message and the URL is fetched with GET.
func TestWebhookNotifierGETPlaceholder(t *testing.T) {
reqs := make(chan *http.Request, 1)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
reqs <- r
}))
defer srv.Close()
n, store := newTestWebhookNotifier(t, srv.URL+"/hook?msg="+webhookMessagePlaceholder)
createEndpoint(t, store, 1, "prod")
n.Notify(Event{Kind: EventHealRestarted, EndpointID: 1, ContainerName: "nginx"})
select {
case r := <-reqs:
if r.Method != http.MethodGet {
t.Errorf("method = %s, want GET", r.Method)
}
got := r.URL.Query().Get("msg")
want := "Environment | prod\nContainer [nginx]\nAuto-heal: restarted unhealthy container"
if got != want {
t.Errorf("decoded msg = %q, want %q", got, want)
}
// The raw query must be URL-encoded: no literal spaces/newlines on the wire.
if strings.ContainsAny(r.URL.RawQuery, " \n") {
t.Errorf("raw query is not URL-encoded: %q", r.URL.RawQuery)
}
case <-time.After(2 * time.Second):
t.Fatal("webhook GET was not received")
}
}
// TestWebhookNotifierPOSTFallback verifies that a URL without the placeholder is
// POSTed with the plain-text message as the body.
func TestWebhookNotifierPOSTFallback(t *testing.T) {
type captured struct {
method string
body string
}
ch := make(chan captured, 1)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
b, _ := io.ReadAll(r.Body)
ch <- captured{method: r.Method, body: string(b)}
}))
defer srv.Close()
n, store := newTestWebhookNotifier(t, srv.URL+"/hook")
createEndpoint(t, store, 2, "staging")
n.Notify(Event{Kind: EventHealRestarted, EndpointID: 2, ContainerName: "api"})
select {
case c := <-ch:
if c.method != http.MethodPost {
t.Errorf("method = %s, want POST", c.method)
}
want := "Environment | staging\nContainer [api]\nAuto-heal: restarted unhealthy container"
if c.body != want {
t.Errorf("body = %q, want %q", c.body, want)
}
case <-time.After(2 * time.Second):
t.Fatal("webhook POST was not received")
}
}
// TestWebhookNotifierEmptyURLNoCall verifies no HTTP call is made when the URL is
// empty.
func TestWebhookNotifierEmptyURLNoCall(t *testing.T) {
called := make(chan struct{}, 1)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
called <- struct{}{}
}))
defer srv.Close()
n, _ := newTestWebhookNotifier(t, "")
n.Notify(Event{Kind: EventHealRestarted, EndpointID: 1, ContainerName: "x"})
select {
case <-called:
t.Fatal("webhook should not be called when the URL is empty")
case <-time.After(300 * time.Millisecond):
// No call, as expected.
}
}
// waitForRequest returns the first request seen on ch, or fails after a short
// grace period.
func waitForRequest(t *testing.T, ch <-chan *http.Request, what string) *http.Request {
t.Helper()
select {
case r := <-ch:
return r
case <-time.After(2 * time.Second):
t.Fatalf("%s was not received", what)
return nil
}
}
// expectNoRequest asserts nothing arrives on ch within a short grace period.
func expectNoRequest(t *testing.T, ch <-chan *http.Request, what string) {
t.Helper()
select {
case <-ch:
t.Fatalf("%s should not have been called", what)
case <-time.After(300 * time.Millisecond):
// No call, as expected.
}
}
// TestWebhookNotifierUpdateEventRoutesToUpdateURL verifies an update-family event
// dispatches to the auto-update URL only; the heal URL is set but never called.
func TestWebhookNotifierUpdateEventRoutesToUpdateURL(t *testing.T) {
updateReqs := make(chan *http.Request, 1)
updateSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
updateReqs <- r
}))
defer updateSrv.Close()
healReqs := make(chan *http.Request, 1)
healSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
healReqs <- r
}))
defer healSrv.Close()
n, store := newTestWebhookNotifierSplit(t, updateSrv.URL+"/update", healSrv.URL+"/heal")
createEndpoint(t, store, 1, "prod")
for _, kind := range []EventKind{EventUpdated, EventRollback, EventUpdateFailed} {
n.Notify(Event{Kind: kind, EndpointID: 1, ContainerName: "c"})
r := waitForRequest(t, updateReqs, "update webhook for "+string(kind))
if r.URL.Path != "/update" {
t.Errorf("kind %s hit %q, want /update", kind, r.URL.Path)
}
}
expectNoRequest(t, healReqs, "heal webhook")
}
// TestWebhookNotifierHealEventRoutesToHealURL verifies a heal event dispatches to
// the auto-heal URL only; the update URL is set but never called.
func TestWebhookNotifierHealEventRoutesToHealURL(t *testing.T) {
updateReqs := make(chan *http.Request, 1)
updateSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
updateReqs <- r
}))
defer updateSrv.Close()
healReqs := make(chan *http.Request, 1)
healSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
healReqs <- r
}))
defer healSrv.Close()
n, store := newTestWebhookNotifierSplit(t, updateSrv.URL+"/update", healSrv.URL+"/heal")
createEndpoint(t, store, 1, "prod")
n.Notify(Event{Kind: EventHealRestarted, EndpointID: 1, ContainerName: "nginx"})
r := waitForRequest(t, healReqs, "heal webhook")
if r.URL.Path != "/heal" {
t.Errorf("heal event hit %q, want /heal", r.URL.Path)
}
expectNoRequest(t, updateReqs, "update webhook")
}
// TestWebhookNotifierEmptyUpdateURLSkipsUpdateOnly verifies that an empty
// auto-update URL suppresses update-family events while heal still fires.
func TestWebhookNotifierEmptyUpdateURLSkipsUpdateOnly(t *testing.T) {
healReqs := make(chan *http.Request, 1)
healSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
healReqs <- r
}))
defer healSrv.Close()
n, store := newTestWebhookNotifierSplit(t, "", healSrv.URL+"/heal")
createEndpoint(t, store, 1, "prod")
// Update-family event: no URL configured, so nothing is delivered.
n.Notify(Event{Kind: EventUpdated, EndpointID: 1, ContainerName: "c"})
expectNoRequest(t, healReqs, "heal webhook on an update event")
// Heal event: the heal URL is set, so it still fires.
n.Notify(Event{Kind: EventHealRestarted, EndpointID: 1, ContainerName: "nginx"})
waitForRequest(t, healReqs, "heal webhook")
}
// TestWebhookNotifierEmptyHealURLSkipsHealOnly verifies that an empty auto-heal
// URL suppresses heal events while update-family events still fire.
func TestWebhookNotifierEmptyHealURLSkipsHealOnly(t *testing.T) {
updateReqs := make(chan *http.Request, 1)
updateSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
updateReqs <- r
}))
defer updateSrv.Close()
n, store := newTestWebhookNotifierSplit(t, updateSrv.URL+"/update", "")
createEndpoint(t, store, 1, "prod")
// Heal event: no URL configured, so nothing is delivered.
n.Notify(Event{Kind: EventHealRestarted, EndpointID: 1, ContainerName: "nginx"})
expectNoRequest(t, updateReqs, "update webhook on a heal event")
// Update event: the update URL is set, so it still fires.
n.Notify(Event{Kind: EventUpdated, EndpointID: 1, ContainerName: "c"})
waitForRequest(t, updateReqs, "update webhook")
}
// TestWebhookNotifierFailingEndpointDoesNotBlock verifies that a broken endpoint
// neither blocks the caller nor panics.
func TestWebhookNotifierFailingEndpointDoesNotBlock(t *testing.T) {
// Start then immediately close a server so its address refuses connections.
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
deadURL := srv.URL
srv.Close()
n, store := newTestWebhookNotifier(t, deadURL+"/hook?msg="+webhookMessagePlaceholder)
createEndpoint(t, store, 1, "prod")
done := make(chan struct{})
go func() {
n.Notify(Event{Kind: EventUpdated, EndpointID: 1, ContainerName: "c"})
close(done)
}()
select {
case <-done:
// Notify returned promptly despite the failing endpoint.
case <-time.After(1 * time.Second):
t.Fatal("Notify blocked on a failing endpoint")
}
// Give the background delivery goroutine time to hit the error path; it must
// log-and-return, never panic.
time.Sleep(200 * time.Millisecond)
}
// TestFormatMessageStandaloneUpdate covers the maintainer's update format for a
// standalone container, with the old->new short digests.
func TestFormatMessageStandaloneUpdate(t *testing.T) {
n, store := newTestWebhookNotifier(t, "unused")
createEndpoint(t, store, 1, "nebula.lc")
settings, _ := store.Settings().Settings()
msg := n.formatMessage(settings, Event{
Kind: EventUpdated, EndpointID: 1, ContainerName: "esphome",
OldDigest: "sha256:59b94983c73aabcd", NewDigest: "sha256:2231ca5d676dabcd",
})
want := "Environment | nebula.lc\nContainer [esphome]\nUpdate [esphome]: 59b94983c73a → 2231ca5d676d"
if msg != want {
t.Errorf("got:\n%q\nwant:\n%q", msg, want)
}
}
// TestFormatMessageStackUpdate covers a stack-scoped update (no per-container
// digests): the context line is the stack name.
func TestFormatMessageStackUpdate(t *testing.T) {
n, store := newTestWebhookNotifier(t, "unused")
createEndpoint(t, store, 1, "nebula.lc")
createStack(t, store, 7, "cache-demo")
settings, _ := store.Settings().Settings()
msg := n.formatMessage(settings, Event{
Kind: EventUpdated, EndpointID: 1, StackID: 7,
})
want := "Environment | nebula.lc\nStack [cache-demo]\nUpdate [cache-demo]: image updated"
if msg != want {
t.Errorf("got:\n%q\nwant:\n%q", msg, want)
}
}
// TestFormatMessageStackMemberUpdate covers the per-container update of a
// stack-member container: the context line is the compose stack name (from
// StackName, no Stack().Read), the action line names the container with its
// old->new digests. This is the maintainer's target output.
func TestFormatMessageStackMemberUpdate(t *testing.T) {
n, store := newTestWebhookNotifier(t, "unused")
createEndpoint(t, store, 1, "nebula.lc")
settings, _ := store.Settings().Settings()
msg := n.formatMessage(settings, Event{
Kind: EventUpdated, EndpointID: 1, StackID: 7, StackName: "cache-demo",
ContainerName: "esphome",
OldDigest: "sha256:59b94983c73aabcd", NewDigest: "sha256:2231ca5d676dabcd",
})
want := "Environment | nebula.lc\nStack [cache-demo]\nUpdate [esphome]: 59b94983c73a → 2231ca5d676d"
if msg != want {
t.Errorf("got:\n%q\nwant:\n%q", msg, want)
}
}
// TestFormatMessageStackMemberUpdateNoNewDigest covers the best-effort fallback:
// when the post-redeploy new image id could not be recovered, the message still
// carries the stack and container and degrades the action line to "image updated"
// rather than blocking delivery.
func TestFormatMessageStackMemberUpdateNoNewDigest(t *testing.T) {
n, store := newTestWebhookNotifier(t, "unused")
createEndpoint(t, store, 1, "nebula.lc")
settings, _ := store.Settings().Settings()
msg := n.formatMessage(settings, Event{
Kind: EventUpdated, EndpointID: 1, StackID: 7, StackName: "cache-demo",
ContainerName: "esphome", OldDigest: "sha256:59b94983c73aabcd",
})
want := "Environment | nebula.lc\nStack [cache-demo]\nUpdate [esphome]: image updated"
if msg != want {
t.Errorf("got:\n%q\nwant:\n%q", msg, want)
}
}
// TestFormatMessageAutoHeal covers the auto-heal message design.
func TestFormatMessageAutoHeal(t *testing.T) {
n, store := newTestWebhookNotifier(t, "unused")
createEndpoint(t, store, 3, "prod")
settings, _ := store.Settings().Settings()
msg := n.formatMessage(settings, Event{
Kind: EventHealRestarted, EndpointID: 3, ContainerName: "nginx",
})
want := "Environment | prod\nContainer [nginx]\nAuto-heal: restarted unhealthy container"
if msg != want {
t.Errorf("got:\n%q\nwant:\n%q", msg, want)
}
}
// TestFormatMessageUnknownEndpoint verifies the "#<id>" fallback when the
// endpoint cannot be resolved.
func TestFormatMessageUnknownEndpoint(t *testing.T) {
n, store := newTestWebhookNotifier(t, "unused")
settings, _ := store.Settings().Settings()
msg := n.formatMessage(settings, Event{
Kind: EventHealRestarted, EndpointID: 99, ContainerName: "ghost",
})
want := "Environment | #99\nContainer [ghost]\nAuto-heal: restarted unhealthy container"
if msg != want {
t.Errorf("got:\n%q\nwant:\n%q", msg, want)
}
}
// TestShortDigest covers digest short-forming.
func TestShortDigest(t *testing.T) {
cases := map[string]string{
"sha256:59b94983c73a1122334455": "59b94983c73a",
"59b94983c73a1122334455": "59b94983c73a",
"short": "short",
"": "",
}
for in, want := range cases {
if got := shortDigest(in); got != want {
t.Errorf("shortDigest(%q) = %q, want %q", in, got, want)
}
}
}

View File

@@ -1,382 +1,55 @@
package crypto
import (
"bufio"
"bytes"
"crypto/aes"
"crypto/cipher"
"crypto/pbkdf2"
"crypto/rand"
"crypto/sha256"
"errors"
"fmt"
"io"
"strings"
"github.com/portainer/portainer/pkg/fips"
// Not allowed in FIPS mode
"golang.org/x/crypto/argon2" //nolint:depguard
"golang.org/x/crypto/scrypt" //nolint:depguard
"golang.org/x/crypto/scrypt"
)
const (
// AES GCM settings
aesGcmHeader = "AES256-GCM" // The encrypted file header
aesGcmBlockSize = 1024 * 1024 // 1MB block for aes gcm
// NOTE: has to go with what is considered to be a simplistic in that it omits any
// authentication of the encrypted data.
// Person with better knowledge is welcomed to improve it.
// sourced from https://golang.org/src/crypto/cipher/example_test.go
aesGcmFIPSHeader = "FIPS-AES256-GCM"
aesGcmFIPSBlockSize = 16 * 1024 * 1024 // 16MB block for aes gcm
var emptySalt []byte = make([]byte, 0)
// Argon2 settings
// Recommended settings lower memory hardware according to current OWASP recommendations
// Considering some people run portainer on a NAS I think it's prudent not to assume we're on server grade hardware
// https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html#argon2id
argon2MemoryCost = 12 * 1024
argon2TimeCost = 3
argon2Threads = 1
argon2KeyLength = 32
pbkdf2Iterations = 600_000 // use recommended iterations from https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html#pbkdf2 a little overkill for this use
pbkdf2SaltLength = 32
)
// AesEncrypt reads from input, encrypts with AES-256 and writes to output. passphrase is used to generate an encryption key
func AesEncrypt(input io.Reader, output io.Writer, passphrase []byte) error {
if fips.FIPSMode() {
if err := aesEncryptGCMFIPS(input, output, passphrase); err != nil {
return fmt.Errorf("error encrypting file: %w", err)
}
} else {
if err := aesEncryptGCM(input, output, passphrase); err != nil {
return fmt.Errorf("error encrypting file: %w", err)
}
}
return nil
}
// AesDecrypt reads from input, decrypts with AES-256 and returns the reader to read the decrypted content from
func AesDecrypt(input io.Reader, passphrase []byte) (io.Reader, error) {
return aesDecrypt(input, passphrase, fips.FIPSMode())
}
func aesDecrypt(input io.Reader, passphrase []byte, fipsMode bool) (io.Reader, error) {
// Read file header to determine how it was encrypted
inputReader := bufio.NewReader(input)
header, err := inputReader.Peek(len(aesGcmFIPSHeader))
if err != nil {
return nil, fmt.Errorf("error reading encrypted backup file header: %w", err)
}
if strings.HasPrefix(string(header), aesGcmFIPSHeader) {
if !fipsMode {
return nil, errors.New("fips encrypted file detected but fips mode is not enabled")
}
reader, err := aesDecryptGCMFIPS(inputReader, passphrase)
if err != nil {
return nil, fmt.Errorf("error decrypting file: %w", err)
}
return reader, nil
}
if strings.HasPrefix(string(header), aesGcmHeader) {
if fipsMode {
return nil, errors.New("fips mode is enabled but non-fips encrypted file detected")
}
reader, err := aesDecryptGCM(inputReader, passphrase)
if err != nil {
return nil, fmt.Errorf("error decrypting file: %w", err)
}
return reader, nil
}
// Use the previous decryption routine which has no header (to support older archives)
reader, err := aesDecryptOFB(inputReader, passphrase)
if err != nil {
return nil, fmt.Errorf("error decrypting legacy file backup: %w", err)
}
return reader, nil
}
// aesEncryptGCM reads from input, encrypts with AES-256 and writes to output. passphrase is used to generate an encryption key.
func aesEncryptGCM(input io.Reader, output io.Writer, passphrase []byte) error {
// Derive key using argon2 with a random salt
salt := make([]byte, 16) // 16 bytes salt
if _, err := io.ReadFull(rand.Reader, salt); err != nil {
return err
}
key := argon2.IDKey(passphrase, salt, argon2TimeCost, argon2MemoryCost, argon2Threads, 32)
block, err := aes.NewCipher(key)
if err != nil {
return err
}
aesgcm, err := cipher.NewGCM(block)
if err != nil {
return err
}
// Generate nonce
nonce, err := NewRandomNonce(aesgcm.NonceSize())
if err != nil {
return err
}
// write the header
if _, err := output.Write([]byte(aesGcmHeader)); err != nil {
return err
}
// Write nonce and salt to the output file
if _, err := output.Write(salt); err != nil {
return err
}
if _, err := output.Write(nonce.Value()); err != nil {
return err
}
// Buffer for reading plaintext blocks
buf := make([]byte, aesGcmBlockSize) // Adjust buffer size as needed
ciphertext := make([]byte, len(buf)+aesgcm.Overhead())
// Encrypt plaintext in blocks
for {
n, err := io.ReadFull(input, buf)
if n == 0 {
break // end of plaintext input
}
if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
return err
}
// Seal encrypts the plaintext using the nonce returning the updated slice.
ciphertext = aesgcm.Seal(ciphertext[:0], nonce.Value(), buf[:n], nil)
if _, err := output.Write(ciphertext); err != nil {
return err
}
if err := nonce.Increment(); err != nil {
return err
}
}
return nil
}
// aesDecryptGCM reads from input, decrypts with AES-256 and returns the reader to read the decrypted content from.
func aesDecryptGCM(input io.Reader, passphrase []byte) (io.Reader, error) {
// Reader & verify header
header := make([]byte, len(aesGcmHeader))
if _, err := io.ReadFull(input, header); err != nil {
return nil, err
}
if string(header) != aesGcmHeader {
return nil, errors.New("invalid header")
}
// Read salt
salt := make([]byte, 16) // Salt size
if _, err := io.ReadFull(input, salt); err != nil {
return nil, err
}
key := argon2.IDKey(passphrase, salt, argon2TimeCost, argon2MemoryCost, argon2Threads, 32)
// Initialize AES cipher block
block, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
// Create GCM mode with the cipher block
aesgcm, err := cipher.NewGCM(block)
if err != nil {
return nil, err
}
// Read nonce from the input reader
nonce := NewNonce(aesgcm.NonceSize())
if err := nonce.Read(input); err != nil {
return nil, err
}
// Initialize a buffer to store decrypted data
buf := bytes.Buffer{}
plaintext := make([]byte, aesGcmBlockSize)
// Decrypt the ciphertext in blocks
for {
// Read a block of ciphertext from the input reader
ciphertextBlock := make([]byte, aesGcmBlockSize+aesgcm.Overhead()) // Adjust block size as needed
n, err := io.ReadFull(input, ciphertextBlock)
if n == 0 {
break // end of ciphertext
}
if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
return nil, err
}
// Decrypt the block of ciphertext
plaintext, err = aesgcm.Open(plaintext[:0], nonce.Value(), ciphertextBlock[:n], nil)
if err != nil {
return nil, err
}
if _, err := buf.Write(plaintext); err != nil {
return nil, err
}
if err := nonce.Increment(); err != nil {
return nil, err
}
}
return &buf, nil
}
// aesEncryptGCMFIPS reads from input, encrypts with AES-256 in a fips compliant
// way and writes to output. passphrase is used to generate an encryption key.
func aesEncryptGCMFIPS(input io.Reader, output io.Writer, passphrase []byte) error {
salt := make([]byte, pbkdf2SaltLength)
if _, err := io.ReadFull(rand.Reader, salt); err != nil {
return err
}
key, err := pbkdf2.Key(sha256.New, string(passphrase), salt, pbkdf2Iterations, 32)
if err != nil {
return fmt.Errorf("error deriving key: %w", err)
}
block, err := aes.NewCipher(key)
if err != nil {
return err
}
// write the header
if _, err := output.Write([]byte(aesGcmFIPSHeader)); err != nil {
return err
}
// Write nonce and salt to the output file
if _, err := output.Write(salt); err != nil {
return err
}
// Buffer for reading plaintext blocks
buf := make([]byte, aesGcmFIPSBlockSize)
// Encrypt plaintext in blocks
for {
// new random nonce for each block
aesgcm, err := cipher.NewGCMWithRandomNonce(block)
if err != nil {
return fmt.Errorf("error creating gcm: %w", err)
}
n, err := io.ReadFull(input, buf)
if n == 0 {
break // end of plaintext input
}
if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
return err
}
// Seal encrypts the plaintext
ciphertext := aesgcm.Seal(nil, nil, buf[:n], nil)
if _, err := output.Write(ciphertext); err != nil {
return err
}
}
return nil
}
// aesDecryptGCMFIPS reads from input, decrypts with AES-256 in a fips compliant
// way and returns the reader to read the decrypted content from.
func aesDecryptGCMFIPS(input io.Reader, passphrase []byte) (io.Reader, error) {
// Reader & verify header
header := make([]byte, len(aesGcmFIPSHeader))
if _, err := io.ReadFull(input, header); err != nil {
return nil, err
}
if string(header) != aesGcmFIPSHeader {
return nil, errors.New("invalid header")
}
// Read salt
salt := make([]byte, pbkdf2SaltLength)
if _, err := io.ReadFull(input, salt); err != nil {
return nil, err
}
key, err := pbkdf2.Key(sha256.New, string(passphrase), salt, pbkdf2Iterations, 32)
if err != nil {
return nil, fmt.Errorf("error deriving key: %w", err)
}
// Initialize AES cipher block
block, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
// Initialize a buffer to store decrypted data
buf := bytes.Buffer{}
// Decrypt the ciphertext in blocks
for {
// Create GCM mode with the cipher block
aesgcm, err := cipher.NewGCMWithRandomNonce(block)
if err != nil {
return nil, err
}
// Read a block of ciphertext from the input reader
ciphertextBlock := make([]byte, aesGcmFIPSBlockSize+aesgcm.Overhead())
n, err := io.ReadFull(input, ciphertextBlock)
if n == 0 {
break // end of ciphertext
}
if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
return nil, err
}
// Decrypt the block of ciphertext
plaintext, err := aesgcm.Open(nil, nil, ciphertextBlock[:n], nil)
if err != nil {
return nil, err
}
if _, err := buf.Write(plaintext); err != nil {
return nil, err
}
}
return &buf, nil
}
// aesDecryptOFB reads from input, decrypts with AES-256 and returns the reader to a read decrypted content from.
// AesEncrypt reads from input, encrypts with AES-256 and writes to the output.
// passphrase is used to generate an encryption key.
// note: This function used to decrypt files that were encrypted without a header i.e. old archives
func aesDecryptOFB(input io.Reader, passphrase []byte) (io.Reader, error) {
func AesEncrypt(input io.Reader, output io.Writer, passphrase []byte) error {
// making a 32 bytes key that would correspond to AES-256
// don't necessarily need a salt, so just kept in empty
key, err := scrypt.Key(passphrase, nil, 32768, 8, 1, 32)
key, err := scrypt.Key(passphrase, emptySalt, 32768, 8, 1, 32)
if err != nil {
return err
}
block, err := aes.NewCipher(key)
if err != nil {
return err
}
// If the key is unique for each ciphertext, then it's ok to use a zero
// IV.
var iv [aes.BlockSize]byte
stream := cipher.NewOFB(block, iv[:])
writer := &cipher.StreamWriter{S: stream, W: output}
// Copy the input to the output, encrypting as we go.
if _, err := io.Copy(writer, input); err != nil {
return err
}
return nil
}
// AesDecrypt reads from input, decrypts with AES-256 and returns the reader to a read decrypted content from.
// passphrase is used to generate an encryption key.
func AesDecrypt(input io.Reader, passphrase []byte) (io.Reader, error) {
// making a 32 bytes key that would correspond to AES-256
// don't necessarily need a salt, so just kept in empty
key, err := scrypt.Key(passphrase, emptySalt, 32768, 8, 1, 32)
if err != nil {
return nil, err
}
@@ -386,25 +59,12 @@ func aesDecryptOFB(input io.Reader, passphrase []byte) (io.Reader, error) {
return nil, err
}
// If the key is unique for each ciphertext, then it's ok to use a zero IV.
// If the key is unique for each ciphertext, then it's ok to use a zero
// IV.
var iv [aes.BlockSize]byte
stream := cipher.NewOFB(block, iv[:])
reader := &cipher.StreamReader{S: stream, R: input}
return reader, nil
}
// HasEncryptedHeader checks if the data has an encrypted header, note that fips
// mode changes this behavior and so will only recognize data encrypted by the
// same mode (fips enabled or disabled)
func HasEncryptedHeader(data []byte) bool {
return hasEncryptedHeader(data, fips.FIPSMode())
}
func hasEncryptedHeader(data []byte, fipsMode bool) bool {
if fipsMode {
return bytes.HasPrefix(data, []byte(aesGcmFIPSHeader))
}
return bytes.HasPrefix(data, []byte(aesGcmHeader))
}

Some files were not shown because too many files have changed in this diff Show More