Compare commits

..

18 Commits

Author SHA1 Message Date
portainer-bot[bot]
bf5d5cab68 chore(version): bump 2.41.1 (#2606)
Co-authored-by: nickl-portainer <nicholas.loomans@portainer.io>
2026-05-11 02:39:40 +00:00
bernard-portainer
51ffea9c93 feat(homeView) add age sort option as default [C9S-150] (#2541) 2026-05-05 08:16:50 +12:00
LP B
eb0ee117a5 fix(api/workflows): kubernetes UAC (#2507) 2026-04-29 18:53:02 -03:00
Xing
c52767fb04 fix(test): isolate registry config in OCI client tests to fix env-dependent failures [C9S-119] (#2497)
Co-authored-by: nickl-portainer <nicholas.loomans@portainer.io>
2026-04-30 09:33:09 +12:00
RHCowan
8e39a16172 fix(agent): correct Podman container engine header in sync edge client [BE-12887] (#2498) (#2506) 2026-04-30 09:03:20 +12:00
LP B
e964be75db fix(api/workflows): move filterK8SStacks outside of transaction (#2504) 2026-04-29 17:57:02 +02:00
Cara Ryan
6776b01ac8 fix(home):CE group by health down discrepancies between headings and list [C9S-139] (#2484) 2026-04-28 15:42:56 +12:00
bernard-portainer
b96031965a fix(environmentlist) use nevironment card in home view [C9S-42] (#2483) 2026-04-28 15:38:15 +12:00
Cara Ryan
b2a2e5c222 feat(home): environment home page ui improvements to highlight groups [C9S-23] (#2453)
Signed-off-by: Bernard Setz <bernard.setz@portainer.io>
Co-authored-by: bernard-portainer <bernard.setz@portainer.io>
Co-authored-by: Ali <83188384+testA113@users.noreply.github.com>
Co-authored-by: Yajith Dayarathna <yajith.dayarathna@portainer.io>
Co-authored-by: Chaim Lev-Ari <chiptus@users.noreply.github.com>
Co-authored-by: andres-portainer <91705312+andres-portainer@users.noreply.github.com>
Co-authored-by: Josiah Clumont <josiah.clumont@portainer.io>
Co-authored-by: Dakota Walsh <101994734+dakota-portainer@users.noreply.github.com>
2026-04-28 13:39:48 +12:00
LP B
27285a94ac feat(api/gitops): list and filter kubernetes git workflows (#2465) 2026-04-27 15:19:17 -03:00
Chaim Lev-Ari
b3f01973ec fix(ui/sortable-list): remove 1 as page size option [BE-12900] (#2470) 2026-04-27 17:01:08 +03:00
Chaim Lev-Ari
17ffd62480 feat(gitops): show live git validity status in workflow overview [BE-12885] (#2467)
Co-authored-by: Claude <noreply@anthropic.com>
2026-04-27 13:11:52 +03:00
Chaim Lev-Ari
86f6aba362 fix(gitops): align list component with current design [BE-12888] (#2445) 2026-04-26 16:54:51 +03:00
Chaim Lev-Ari
718e11ccd0 fix(kube/stacks): allow empty stack name [BE-12889] (#2446) 2026-04-26 12:14:53 +03:00
Josiah Clumont
e68b0e80f1 feat(recommendations): completeness recommendations [C9S-18] (#2262) (#2454) 2026-04-24 14:55:15 +12:00
Ali
9a14f2acb7 feat(docker): add docker builder prune as option [C9S-128] (#2451) 2026-04-24 10:21:32 +12:00
Ali
01ff1486e0 fix(ui): use uuidv4 instead of cryptorandomuuid to support non-secure browsers [c9s-133] (#2433) 2026-04-24 08:41:48 +12:00
andres-portainer
b91f77a554 feat(gitops): introduce workflows view [BE-12807] (#2391) (#2428)
Co-authored-by: Chaim Lev-Ari <chiptus@users.noreply.github.com>
Co-authored-by: Chaim Lev-Ari <chaim.lev-ari@portainer.io>
2026-04-22 14:37:04 -03:00
1215 changed files with 20445 additions and 87070 deletions

View File

@@ -3,13 +3,13 @@ body:
attributes:
value: |
# Welcome!
Thanks for suggesting an idea for Portainer!
Before opening a new idea or feature request, make sure that we do not have any duplicates already open. You can ensure this by [searching this discussion category](https://github.com/orgs/portainer/discussions/categories/ideas). If there is a duplicate, please add a comment to the existing idea instead.
Also, be sure to check our [knowledge base](https://portal.portainer.io/knowledge) and [documentation](https://docs.portainer.io) as they may point you toward a solution.
**DO NOT FILE DUPLICATE REQUESTS.**
- type: textarea

View File

@@ -94,12 +94,7 @@ body:
description: We only provide support for current versions of Portainer as per the lifecycle policy linked above. If you are on an older version of Portainer we recommend [updating first](https://docs.portainer.io/start/upgrade) in case your bug has already been fixed.
multiple: false
options:
- '2.42.0'
- '2.41.1'
- '2.41.0'
- '2.40.0'
- '2.39.3'
- '2.39.2'
- '2.39.1'
- '2.39.0'
- '2.38.1'
@@ -108,7 +103,6 @@ body:
- '2.36.0'
- '2.35.0'
- '2.34.0'
- '2.33.8'
- '2.33.7'
- '2.33.6'
- '2.33.5'
@@ -117,7 +111,38 @@ body:
- '2.33.2'
- '2.33.1'
- '2.33.0'
- '2.32.0'
- '2.31.3'
- '2.31.2'
- '2.31.1'
- '2.31.0'
- '2.30.1'
- '2.30.0'
- '2.29.2'
- '2.29.1'
- '2.29.0'
- '2.28.1'
- '2.28.0'
- '2.27.9'
- '2.27.8'
- '2.27.7'
- '2.27.6'
- '2.27.5'
- '2.27.4'
- '2.27.3'
- '2.27.2'
- '2.27.1'
- '2.27.0'
- '2.26.1'
- '2.26.0'
- '2.25.1'
- '2.25.0'
- '2.24.1'
- '2.24.0'
- '2.23.0'
- '2.22.0'
- '2.21.5'
- '2.21.4'
validations:
required: true

View File

@@ -1,86 +0,0 @@
name: Build image
on:
push:
branches: [develop]
tags: ['v*']
workflow_dispatch: {}
env:
IMAGE: ghcr.io/vvzvlad/portainer-ce
jobs:
build:
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup pnpm
uses: pnpm/action-setup@v4
- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: 22
- name: Setup Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Resolve version
id: ver
run: echo "version=$(node -p "require('./package.json').version")" >> "$GITHUB_OUTPUT"
- name: Install client dependencies
# CI forces pnpm into --frozen-lockfile, which fails with
# ERR_PNPM_LOCKFILE_CONFIG_MISMATCH because the committed lockfile lacks
# the pnpmfileChecksum for the configDependencies in package.json.
# Reconcile the lockfile explicitly; the later frozen install in
# `make client-deps` then finds a matching lockfile. pnpm ignores the
# npm_config_frozen_lockfile env var, so an explicit flag is required.
run: pnpm install --no-frozen-lockfile
- name: Build client and server
env:
SKIP_GO_GET: "true"
CONTAINER_IMAGE_TAG: ${{ steps.ver.outputs.version }}
BUILDNUMBER: ${{ github.run_number }}
# Pin the embedded commit to the full SHA so it matches the image
# GIT_COMMIT build-arg and does not depend on the shallow checkout.
GIT_COMMIT_HASH: ${{ github.sha }}
# ENV=production selects webpack/webpack.production.js (minified bundle),
# matching the official CE image; the Makefile default is development.
run: make build-all ENV=production
- name: Ensure storybook directory exists
# make build-all does not produce dist/storybook, but alpine.Dockerfile
# has `COPY dist/storybook* /storybook/`; without a match the docker build fails.
run: mkdir -p dist/storybook
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Log in to GHCR
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push image (linux/amd64, alpine base)
uses: docker/build-push-action@v6
with:
context: .
file: build/linux/alpine.Dockerfile
platforms: linux/amd64
push: true
tags: |
${{ env.IMAGE }}:${{ steps.ver.outputs.version }}
${{ env.IMAGE }}:latest
build-args: |
GIT_COMMIT=${{ github.sha }}

1
.gitignore vendored
View File

@@ -4,7 +4,6 @@ dist
portainer-checksum.txt
api/cmd/portainer/portainer*
storybook-static
debug-storybook.log
.tmp
**/.vscode/settings.json
**/.vscode/tasks.json

View File

@@ -1,4 +1,4 @@
version: '2'
version: "2"
linters:
default: none
enable:

View File

@@ -1,11 +1,10 @@
version: '2'
version: "2"
run:
allow-parallel-runners: true
linters:
default: none
enable:
- gocritic
- bodyclose
- copyloopvar
- depguard
@@ -32,7 +31,7 @@ linters:
- exptostd
settings:
staticcheck:
checks: ['all', '-ST1003', '-ST1005', '-ST1016', '-SA1019', '-QF1003']
checks: ["all", "-ST1003", "-ST1005", "-ST1016", "-SA1019", "-QF1003"]
depguard:
rules:
main:
@@ -77,13 +76,6 @@ linters:
desc: use github.com/Masterminds/semver/v3
- pkg: github.com/hashicorp/go-version
desc: use github.com/Masterminds/semver/v3
gocritic:
disable-all: true
enabled-checks:
- ruleguard
settings:
ruleguard:
rules: './analysis/ssrf.go,./analysis/git.go'
forbidigo:
forbid:
- pattern: ^tls\.Config$
@@ -91,11 +83,9 @@ linters:
- pattern: ^tls\.Config\.(InsecureSkipVerify|MinVersion|MaxVersion|CipherSuites|CurvePreferences)$
msg: Do not set this field directly, use crypto.CreateTLSConfiguration() instead
- pattern: ^object\.(Commit|Tag)\.Verify$
msg: 'Not allowed because of FIPS mode'
msg: "Not allowed because of FIPS mode"
- pattern: ^(types\.SystemContext\.)?(DockerDaemonInsecureSkipTLSVerify|DockerInsecureSkipTLSVerify|OCIInsecureSkipTLSVerify)$
msg: 'Not allowed because of FIPS mode'
- pattern: ^git\.PlainClone(Context|WithOptions)?$
msg: Use git.CloneContext with NewNoSymlinkFS to prevent symlink traversal attacks
msg: "Not allowed because of FIPS mode"
analyze-types: true
exclusions:
generated: lax
@@ -103,14 +93,6 @@ linters:
- comments
- common-false-positives
- legacy
rules:
- path: pkg/libhttp/ssrf
linters:
- gocritic
text: ruleguard
- path: pkg/libhttp/ssrf/builder\.go
linters:
- forbidigo
paths:
- third_party$
- builtin$

View File

@@ -1,5 +1,3 @@
dist
api/datastore/test_data
coverage
pnpm-lock.yaml
coverage

View File

@@ -1,21 +1,15 @@
// This file has been automatically migrated to valid ESM format by Storybook.
import { fileURLToPath } from 'node:url';
import { createRequire } from 'node:module';
import path, { dirname } from 'path';
import path from 'path';
import { StorybookConfig } from '@storybook/react-webpack5';
import { Configuration } from 'webpack';
import postcss from 'postcss';
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const require = createRequire(import.meta.url);
const config: StorybookConfig = {
stories: ['../app/**/*.stories.@(ts|tsx)'],
addons: [
'@storybook/addon-links',
'@storybook/addon-essentials',
'@storybook/addon-webpack5-compiler-swc',
'@chromatic-com/storybook',
{
@@ -50,7 +44,6 @@ const config: StorybookConfig = {
],
},
},
'@storybook/addon-docs',
],
webpackFinal: (config) => {
const rules = config?.module?.rules || [];
@@ -103,7 +96,12 @@ const config: StorybookConfig = {
},
staticDirs: ['./public'],
typescript: {
reactDocgen: 'react-docgen',
reactDocgen: 'react-docgen-typescript',
reactDocgenTypescriptOptions: {
compilerOptions: {
outDir: path.resolve(__dirname, '..', 'dist/public'),
},
},
},
framework: {
name: '@storybook/react-webpack5',

View File

@@ -1,10 +1,9 @@
import { useEffect } from 'react';
import '../app/assets/css';
import { pushStateLocationPlugin, UIRouter } from '@uirouter/react';
import { initialize as initMSW, mswLoader } from 'msw-storybook-addon';
import { handlers } from '../app/setup-tests/server-handlers';
import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
import { Preview } from '@storybook/react-webpack5';
import { Preview } from '@storybook/react';
initMSW(
{
@@ -27,43 +26,13 @@ const testQueryClient = new QueryClient({
});
const preview: Preview = {
globalTypes: {
theme: {
description: 'Portainer color theme',
toolbar: {
title: 'Theme',
icon: 'paintbrush',
items: [
{ value: 'light', title: 'Light', icon: 'sun' },
{ value: 'dark', title: 'Dark', icon: 'moon' },
{ value: 'highcontrast', title: 'High Contrast', icon: 'eye' },
],
dynamicTitle: true,
},
},
},
initialGlobals: {
theme: 'light',
},
decorators: (Story, context) => {
const theme = context.globals.theme;
useEffect(() => {
if (theme === 'light') {
document.documentElement.removeAttribute('theme');
} else {
document.documentElement.setAttribute('theme', theme);
}
}, [theme]);
return (
<QueryClientProvider client={testQueryClient}>
<UIRouter plugins={[pushStateLocationPlugin]}>
<Story />
</UIRouter>
</QueryClientProvider>
);
},
decorators: (Story) => (
<QueryClientProvider client={testQueryClient}>
<UIRouter plugins={[pushStateLocationPlugin]}>
<Story />
</UIRouter>
</QueryClientProvider>
),
loaders: [mswLoader],
parameters: {
options: {

View File

@@ -1,6 +1,8 @@
/* eslint-disable */
/* tslint:disable */
import { v4 as uuidv4 } from 'uuid';
/**
* Mock Service Worker.
* @see https://github.com/mswjs/msw
@@ -109,7 +111,7 @@ addEventListener('fetch', function (event) {
return;
}
const requestId = crypto.randomUUID();
const requestId = uuidv4();
event.respondWith(handleRequest(event, requestId, requestInterceptedAt));
});

View File

@@ -8,19 +8,19 @@ In the interest of fostering an open and welcoming environment, we as contributo
Examples of behavior that contributes to creating a positive environment include:
- Using welcoming and inclusive language
- Being respectful of differing viewpoints and experiences
- Gracefully accepting constructive criticism
- Focusing on what is best for the community
- Showing empathy towards other community members
* Using welcoming and inclusive language
* Being respectful of differing viewpoints and experiences
* Gracefully accepting constructive criticism
* Focusing on what is best for the community
* Showing empathy towards other community members
Examples of unacceptable behavior by participants include:
- The use of sexualized language or imagery and unwelcome sexual attention or advances
- Trolling, insulting/derogatory comments, and personal or political attacks
- Public or private harassment
- Publishing others' private information, such as a physical or electronic address, without explicit permission
- Other conduct which could reasonably be considered inappropriate in a professional setting
* The use of sexualized language or imagery and unwelcome sexual attention or advances
* Trolling, insulting/derogatory comments, and personal or political attacks
* Public or private harassment
* Publishing others' private information, such as a physical or electronic address, without explicit permission
* Other conduct which could reasonably be considered inappropriate in a professional setting
## Our Responsibilities
@@ -34,7 +34,7 @@ This Code of Conduct applies both within project spaces and in public spaces whe
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at contribute@portainer.io. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at anthony.lapenna@portainer.io. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.

View File

@@ -147,9 +147,7 @@ When adding a new route to an existing handler use the following as a template (
// @router /{id} [get]
```
explanation about each line can be found [here](https://github.com/swaggo/swag#api-operation)
After changing these annotations, regenerate the TypeScript API client and types — see [Generating API types](./README.md#generating-api-types).
explanation about each line can be found (here)[https://github.com/swaggo/swag#api-operation]
## Licensing

View File

@@ -3,10 +3,9 @@ ENV=development
WEBPACK_CONFIG=webpack/webpack.$(ENV).js
TAG=local
SWAG=go run github.com/swaggo/swag/cmd/swag@v1.16.6
SWAG=go run github.com/swaggo/swag/cmd/swag@v1.16.2
GOTESTSUM_VERSION?=v1.13.0
GOTESTSUM=go run gotest.tools/gotestsum@$(GOTESTSUM_VERSION)
GOLANGCI_LINT_VERSION := $(shell cat $(shell git rev-parse --show-toplevel)/.golangci-version)
# Don't change anything below this line unless you know what you're doing
.DEFAULT_GOAL := help
@@ -37,8 +36,8 @@ build-storybook: ## Build and serve the storybook files
.PHONY: deps server-deps client-deps tidy
deps: server-deps client-deps ## Download all client and server build dependancies
## This is empty because the pipeline requires it but ce has no server deps
server-deps: init-dist ## Download dependant server binaries
@./build/download_binaries.sh $(PLATFORM) $(ARCH)
client-deps: ## Install client dependencies
pnpm install
@@ -91,25 +90,13 @@ format-server: ## Format server code
go fmt ./...
##@ Lint
.PHONY: lint lint-client lint-server check-lint-version
.PHONY: lint lint-client lint-server
lint: lint-client lint-server ## Lint all code
lint-client: ## Lint client code
pnpm run lint
check-lint-version:
@installed=v$$(golangci-lint --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1); \
if [ "$$installed" = "v" ]; then \
echo "ERROR: golangci-lint not found, need $(GOLANGCI_LINT_VERSION)"; \
echo "Install: go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION)"; \
exit 1; \
elif [ "$$installed" != "$(GOLANGCI_LINT_VERSION)" ]; then \
echo "ERROR: golangci-lint $$installed installed, need $(GOLANGCI_LINT_VERSION)"; \
echo "Install: go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION)"; \
exit 1; \
fi
lint-server: tidy check-lint-version ## Lint server code
lint-server: tidy ## Lint server code
golangci-lint run --timeout=10m -c .golangci.yaml
golangci-lint run --timeout=10m --new-from-rev=HEAD~ -c .golangci-forward.yaml
@@ -121,8 +108,8 @@ dev-extension: build-server build-client ## Run the extension in development mod
##@ Docs
.PHONY: docs-build docs-validate docs-clean docs-validate-clean
docs-build: init-dist ## Build docs
go mod download
cd api && $(SWAG) init -o "../dist/docs" -ot "yaml" -g ./http/handler/handler.go --parseDependency --parseInternal --parseDepth 2 -p pascalcase --markdownFiles ./ --overridesFile .swaggo
go mod download -x
cd api && $(SWAG) init -o "../dist/docs" -ot "yaml" -g ./http/handler/handler.go --parseDependency --parseInternal --parseDepth 2 -p pascalcase --markdownFiles ./
docs-validate: docs-build ## Validate docs
pnpm swagger2openapi --warnOnly dist/docs/swagger.yaml -o dist/docs/openapi.yaml
@@ -134,10 +121,6 @@ docs-serve: docs-build ## Serve docs locally with Swagger UI on port 8080
-e SWAGGER_JSON=/foo/swagger.yaml \
-v $(PWD)/dist/docs:/foo \
swaggerapi/swagger-ui
.PHONY: generate-api
generate-api: docs-validate ## Generate API client and types from OpenAPI spec
pnpm generate-api
##@ Helpers
.PHONY: help

View File

@@ -44,32 +44,6 @@ You can join the Portainer Community by visiting [https://www.portainer.io/join-
- Want to report a bug or request a feature? Please open [an issue](https://github.com/portainer/portainer/issues/new).
- Want to help us build **_portainer_**? Follow our [contribution guidelines](https://docs.portainer.io/contribute/contribute) to build it locally and make a pull request.
## Generating API types
The frontend consumes a TypeScript API client (SDK functions and request/response types) that is generated from the Go API's Swagger annotations. Regenerate it after any API change — a new endpoint, a changed request/response shape, or a removed endpoint:
```bash
make generate-api
```
This runs the following pipeline:
```
Go Swagger annotations
→ dist/docs/swagger.yaml (make docs-build, via swaggo/swag)
→ dist/docs/openapi.yaml (swagger2openapi + validation)
→ app/react/portainer/generated-api/portainer/ (hey-api/openapi-ts)
```
The generator is configured in [`openapi-ts.config.ts`](./openapi-ts.config.ts), which controls the output path, plugins, and tag filters (for example, `deprecated` endpoints and `edge_agent`-tagged routes are excluded).
The generated files live in `app/react/portainer/generated-api/portainer/` and must **not** be edited by hand — your changes would be overwritten on the next run. Import the generated SDK functions and types instead of writing direct HTTP calls:
- `@api/sdk.gen` — SDK functions
- `@api/types.gen` — request/response types
See [Adding api docs](./CONTRIBUTING.md#adding-api-docs) for how to annotate handlers so they are picked up by the generator.
## Security
For information about reporting security vulnerabilities, please see our [Security Policy](SECURITY.md).

View File

@@ -1,118 +0,0 @@
import {
Children,
useState,
useEffect,
useRef,
useContext,
createContext,
ReactNode,
} from 'react';
type MenuCtxType = {
isOpen: boolean;
setOpen: (v: boolean) => void;
menuRef: React.RefObject<HTMLDivElement>;
label: string;
setLabel: (v: string) => void;
};
const MenuCtx = createContext<MenuCtxType | null>(null);
export function Menu({ children }: { children?: ReactNode }) {
const [isOpen, setOpen] = useState(false);
const [label, setLabel] = useState('');
const menuRef = useRef<HTMLDivElement>(null);
useEffect(() => {
function handleDocDown(e: MouseEvent) {
const target = e.target as Node | null;
if (
isOpen &&
menuRef.current &&
target &&
!menuRef.current.contains(target)
) {
setOpen(false);
}
}
document.addEventListener('mousedown', handleDocDown);
return () => document.removeEventListener('mousedown', handleDocDown);
}, [isOpen]);
return (
<MenuCtx.Provider value={{ isOpen, setOpen, menuRef, label, setLabel }}>
<div ref={menuRef}>{children}</div>
</MenuCtx.Provider>
);
}
export function MenuButton({
children,
onClick: externalOnClick,
...props
}: {
children?: ReactNode;
onClick?: () => void;
[key: string]: unknown;
}) {
const ctx = useContext(MenuCtx);
useEffect(() => {
const firstText = Children.toArray(children).find(
(c) => typeof c === 'string'
);
if (firstText) ctx?.setLabel(firstText as string);
});
function handleClick() {
externalOnClick?.();
ctx?.setOpen(!ctx.isOpen);
}
return (
<button type="button" onClick={handleClick} {...props}>
{children}
</button>
);
}
export function MenuList({
children,
className,
}: {
children?: ReactNode;
className?: string;
}) {
const ctx = useContext(MenuCtx);
if (!ctx?.isOpen) return null;
return (
<div role="menu" aria-label={ctx.label || undefined} className={className}>
{children}
</div>
);
}
export function MenuItem({
children,
onSelect,
className,
}: {
children?: ReactNode;
onSelect?: () => void;
className?: string;
}) {
const ctx = useContext(MenuCtx);
function handleClick() {
onSelect?.();
ctx?.setOpen(false);
}
return (
// eslint-disable-next-line jsx-a11y/click-events-have-key-events, jsx-a11y/interactive-supports-focus
<div role="menuitem" onClick={handleClick} className={className}>
{children}
</div>
);
}

View File

@@ -1,18 +0,0 @@
//go:build ignore
package gorules
import "github.com/quasilyte/go-ruleguard/dsl"
// inMemoryCloneWithWorktree flags git clone calls that use memory.NewStorage() as
// the storer while also writing files to a real worktree. This holds all git objects
// in heap for the duration of the clone, which is unbounded for user-supplied repos.
func inMemoryCloneWithWorktree(m dsl.Matcher) {
m.Match(`git.CloneContext($_, memory.NewStorage(), $wt, $_)`).
Where(m["wt"].Text != "nil").
Report(`git.CloneContext with memory.NewStorage() holds all git objects in heap; use gogitfs.NewStorage with a filesystem storer instead`)
m.Match(`git.Clone(memory.NewStorage(), $wt, $_)`).
Where(m["wt"].Text != "nil").
Report(`git.Clone with memory.NewStorage() holds all git objects in heap; use gogitfs.NewStorage with a filesystem storer instead`)
}

View File

@@ -1,75 +0,0 @@
//go:build ignore
package gorules
import "github.com/quasilyte/go-ruleguard/dsl"
// unwrappedHTTPTransport flags any bare http.Transport composite literal.
// All transports must be created via ssrf.NewTransport or ssrf.NewInternalTransport,
// which clone http.DefaultTransport and handle SSRF protection internally.
func unwrappedHTTPTransport(m dsl.Matcher) {
m.Match(`$f(&http.Transport{$*_})`).
Report(`$f receives a bare *http.Transport; use ssrf.NewTransport(tlsConfig) or ssrf.NewInternalTransport(tlsConfig) instead`)
m.Match(`$_ := &http.Transport{$*_}`).
Report(`bare *http.Transport variable; use ssrf.NewTransport(tlsConfig) or ssrf.NewInternalTransport(tlsConfig) instead`)
m.Match(`$_.Transport = &http.Transport{$*_}`).
Report(`bare *http.Transport field assignment; use ssrf.NewTransport(tlsConfig) or ssrf.NewInternalTransport(tlsConfig) instead`)
}
// helmGetterTransport flags getter.WithTransport calls that receive a bare *http.Transport.
// Helm v4 installs its own transport and bypasses http.DefaultTransport, so the transport
// passed here must be created via ssrf.NewTransport.
func helmGetterTransport(m dsl.Matcher) {
m.Match(`getter.WithTransport(&http.Transport{$*_})`).
Report(`getter.WithTransport called with a bare *http.Transport; use ssrf.NewTransport(tlsConfig) as Helm v4 bypasses http.DefaultTransport`)
}
// cloneDefaultTransport flags direct clones of *http.Transport outside main.go.
// The one legitimate clone is in main.go where http.DefaultTransport is globally
// wrapped with SSRF protection at server startup.
func cloneDefaultTransport(m dsl.Matcher) {
m.Match(`$_.(*http.Transport).Clone()`).
Where(!m.File().Name.Matches(`^main\.go$`)).
Report(`cloning *http.Transport directly is forbidden; use ssrf.NewTransport(tlsConfig) or ssrf.NewInternalTransport(tlsConfig) instead`)
}
// internalTransportMisuse flags calls to NewInternalTransport outside the proxy
// factory files where Chisel-tunnel and in-cluster K8s destinations are valid exemptions.
func internalTransportMisuse(m dsl.Matcher) {
m.Match(`ssrf.NewInternalTransport($*_)`).
Where(
!(m.File().PkgPath.Matches(`proxy/factory`) &&
m.File().Name.Matches(`^(docker|agent|local_transport|edge_transport|docker_unix|docker_windows)\.go$`))).
Report(`NewInternalTransport bypasses SSRF validation; only valid in the proxy factory files for local sockets and internally-routed endpoints`)
}
// dialerOverride flags direct assignments to any of the dialer fields on a transport.
// The only valid assignments are in docker_unix.go and docker_windows.go where a
// custom dialer is required for unix sockets and named pipes.
func dialerOverride(m dsl.Matcher) {
m.Match(`$_.DialContext = $*_`).
Where(
!(m.File().PkgPath.Matches(`proxy/factory`) &&
m.File().Name.Matches(`^(docker_unix|docker_windows)\.go$`))).
Report(`direct DialContext assignment replaces the transport dialer; use ssrf.NewTransport or ssrf.NewInternalTransport instead`)
m.Match(`$_.Dial = $*_`).
Where(
!(m.File().PkgPath.Matches(`proxy/factory`) &&
m.File().Name.Matches(`^(docker_unix|docker_windows)\.go$`))).
Report(`direct Dial assignment replaces the transport dialer; use ssrf.NewTransport or ssrf.NewInternalTransport instead`)
m.Match(`$_.DialTLSContext = $*_`).
Where(
!(m.File().PkgPath.Matches(`proxy/factory`) &&
m.File().Name.Matches(`^(docker_unix|docker_windows)\.go$`))).
Report(`direct DialTLSContext assignment replaces the transport dialer; use ssrf.NewTransport or ssrf.NewInternalTransport instead`)
m.Match(`$_.DialTLS = $*_`).
Where(
!(m.File().PkgPath.Matches(`proxy/factory`) &&
m.File().Name.Matches(`^(docker_unix|docker_windows)\.go$`))).
Report(`direct DialTLS assignment replaces the transport dialer; use ssrf.NewTransport or ssrf.NewInternalTransport instead`)
}

View File

@@ -1,5 +0,0 @@
//go:build tools
package gorules
import _ "github.com/quasilyte/go-ruleguard/dsl"

View File

@@ -1 +0,0 @@
replace k8s.io/apimachinery/pkg/apis/meta/v1.Duration string

View File

@@ -1,7 +1,6 @@
package agent
import (
"context"
"crypto/tls"
"errors"
"fmt"
@@ -12,7 +11,6 @@ import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/url"
"github.com/portainer/portainer/pkg/libhttp/ssrf"
"github.com/rs/zerolog/log"
)
@@ -21,14 +19,10 @@ import (
//
// it sends a ping to the agent and parses the version and platform from the headers
func GetAgentVersionAndPlatform(endpointUrl string, tlsConfig *tls.Config) (portainer.AgentPlatform, string, error) { //nolint:forbidigo
if err := ssrf.CheckURL(context.Background(), endpointUrl); err != nil {
return 0, "", err
}
httpCli := &http.Client{Timeout: 3 * time.Second}
if tlsConfig != nil {
httpCli.Transport = ssrf.NewTransport(tlsConfig)
httpCli.Transport = &http.Transport{TLSClientConfig: tlsConfig}
}
parsedURL, err := url.ParseURL(endpointUrl + "/ping")

View File

@@ -1,119 +0,0 @@
package agent
import (
"net/http"
"net/http/httptest"
"strconv"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/stretchr/testify/require"
)
func tlsServer(t *testing.T, handler http.HandlerFunc) *httptest.Server {
t.Helper()
srv := httptest.NewTLSServer(handler)
t.Cleanup(srv.Close)
return srv
}
func TestGetAgentVersionAndPlatform_Success(t *testing.T) {
t.Parallel()
srv := tlsServer(t, func(w http.ResponseWriter, r *http.Request) {
w.Header().Set(portainer.PortainerAgentHeader, "2.19.0")
w.Header().Set(portainer.HTTPResponseAgentPlatform, "1")
w.WriteHeader(http.StatusNoContent)
})
tlsCfg := srv.Client().Transport.(*http.Transport).TLSClientConfig
platform, version, err := GetAgentVersionAndPlatform(srv.URL, tlsCfg)
require.NoError(t, err)
require.Equal(t, portainer.AgentPlatformDocker, platform)
require.Equal(t, "2.19.0", version)
}
func TestGetAgentVersionAndPlatform_NonOKStatus(t *testing.T) {
t.Parallel()
srv := tlsServer(t, func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusUnauthorized)
})
tlsCfg := srv.Client().Transport.(*http.Transport).TLSClientConfig
_, _, err := GetAgentVersionAndPlatform(srv.URL, tlsCfg)
require.Error(t, err)
}
func TestGetAgentVersionAndPlatform_MissingVersionHeader(t *testing.T) {
t.Parallel()
srv := tlsServer(t, func(w http.ResponseWriter, r *http.Request) {
w.Header().Set(portainer.HTTPResponseAgentPlatform, "1")
w.WriteHeader(http.StatusNoContent)
})
tlsCfg := srv.Client().Transport.(*http.Transport).TLSClientConfig
_, _, err := GetAgentVersionAndPlatform(srv.URL, tlsCfg)
require.Error(t, err)
}
func TestGetAgentVersionAndPlatform_MissingPlatformHeader(t *testing.T) {
t.Parallel()
srv := tlsServer(t, func(w http.ResponseWriter, r *http.Request) {
w.Header().Set(portainer.PortainerAgentHeader, "2.19.0")
w.WriteHeader(http.StatusNoContent)
})
tlsCfg := srv.Client().Transport.(*http.Transport).TLSClientConfig
_, _, err := GetAgentVersionAndPlatform(srv.URL, tlsCfg)
require.Error(t, err)
}
func TestGetAgentVersionAndPlatform_InvalidPlatformZero(t *testing.T) {
t.Parallel()
srv := tlsServer(t, func(w http.ResponseWriter, r *http.Request) {
w.Header().Set(portainer.PortainerAgentHeader, "2.19.0")
w.Header().Set(portainer.HTTPResponseAgentPlatform, "0")
w.WriteHeader(http.StatusNoContent)
})
tlsCfg := srv.Client().Transport.(*http.Transport).TLSClientConfig
_, _, err := GetAgentVersionAndPlatform(srv.URL, tlsCfg)
require.Error(t, err)
}
func TestGetAgentVersionAndPlatform_NonNumericPlatform(t *testing.T) {
t.Parallel()
srv := tlsServer(t, func(w http.ResponseWriter, r *http.Request) {
w.Header().Set(portainer.PortainerAgentHeader, "2.19.0")
w.Header().Set(portainer.HTTPResponseAgentPlatform, "docker")
w.WriteHeader(http.StatusNoContent)
})
tlsCfg := srv.Client().Transport.(*http.Transport).TLSClientConfig
_, _, err := GetAgentVersionAndPlatform(srv.URL, tlsCfg)
require.Error(t, err)
}
func TestGetAgentVersionAndPlatform_PingPathAppended(t *testing.T) {
t.Parallel()
var gotPath string
srv := tlsServer(t, func(w http.ResponseWriter, r *http.Request) {
gotPath = r.URL.Path
w.Header().Set(portainer.PortainerAgentHeader, "2.19.0")
w.Header().Set(portainer.HTTPResponseAgentPlatform, strconv.Itoa(int(portainer.AgentPlatformKubernetes)))
w.WriteHeader(http.StatusNoContent)
})
tlsCfg := srv.Client().Transport.(*http.Transport).TLSClientConfig
_, _, err := GetAgentVersionAndPlatform(srv.URL, tlsCfg)
require.NoError(t, err)
require.Equal(t, "/ping", gotPath)
}

64
api/api-description.md Normal file
View File

@@ -0,0 +1,64 @@
Portainer API is an HTTP API served by Portainer. It is used by the Portainer UI and everything you can do with the UI can be done using the HTTP API.
Examples are available at https://documentation.portainer.io/api/api-examples/
You can find out more about Portainer at [http://portainer.io](http://portainer.io) and get some support on [Slack](http://portainer.io/slack/).
# Authentication
Most of the API environments(endpoints) require to be authenticated as well as some level of authorization to be used.
Portainer API uses JSON Web Token to manage authentication and thus requires you to provide a token in the **Authorization** header of each request
with the **Bearer** authentication mechanism.
Example:
```
Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6MSwidXNlcm5hbWUiOiJhZG1pbiIsInJvbGUiOjEsImV4cCI6MTQ5OTM3NjE1NH0.NJ6vE8FY1WG6jsRQzfMqeatJ4vh2TWAeeYfDhP71YEE
```
# Security
Each API environment(endpoint) has an associated access policy, it is documented in the description of each environment(endpoint).
Different access policies are available:
- Public access
- Authenticated access
- Restricted access
- Administrator access
### Public access
No authentication is required to access the environments(endpoints) with this access policy.
### Authenticated access
Authentication is required to access the environments(endpoints) with this access policy.
### Restricted access
Authentication is required to access the environments(endpoints) with this access policy.
Extra-checks might be added to ensure access to the resource is granted. Returned data might also be filtered.
### Administrator access
Authentication as well as an administrator role are required to access the environments(endpoints) with this access policy.
# Execute Docker requests
Portainer **DO NOT** expose specific environments(endpoints) to manage your Docker resources (create a container, remove a volume, etc...).
Instead, it acts as a reverse-proxy to the Docker HTTP API. This means that you can execute Docker requests **via** the Portainer HTTP API.
To do so, you can use the `/endpoints/{id}/docker` Portainer API environment(endpoint) (which is not documented below due to Swagger limitations). This environment(endpoint) has a restricted access policy so you still need to be authenticated to be able to query this environment(endpoint). Any query on this environment(endpoint) will be proxied to the Docker API of the associated environment(endpoint) (requests and responses objects are the same as documented in the Docker API).
# Private Registry
Using private registry, you will need to pass a based64 encoded JSON string ‘{"registryId":\<registryID value\>}’ inside the Request Header. The parameter name is "X-Registry-Auth".
\<registryID value\> - The registry ID where the repository was created.
Example:
```
eyJyZWdpc3RyeUlkIjoxfQ==
```
**NOTE**: You can find more information on how to query the Docker API in the [Docker official documentation](https://docs.docker.com/engine/api/v1.30/) as well as in [this Portainer example](https://documentation.portainer.io/api/api-examples/).

View File

@@ -1,61 +0,0 @@
The Portainer API is an HTTP API served by Portainer. It is used by the Portainer UI, and anything you can do in the UI can also be done via the HTTP API.
API examples are available in the [Portainer documentation](https://documentation.portainer.io/api/api-examples/)
You can find out more about Portainer [on our website](http://portainer.io) and get some support on [Slack](http://portainer.io/slack/).
# Authentication
Most of the API endpoints require authentication, as well as some level of authorization.
Portainer uses JSON Web Tokens to manage authentication. You must provide a token in the **Authorization** header of each request using the **Bearer** scheme.
Example:
```
Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6MSwidXNlcm5hbWUiOiJhZG1pbiIsInJvbGUiOjEsImV4cCI6MTQ5OTM3NjE1NH0.NJ6vE8FY1WG6jsRQzfMqeatJ4vh2TWAeeYfDhP71YEE
```
# Security
Each API endpoint has an associated access policy, documented in its description.
The following policies are available:
- Public access
- Authenticated access
- Restricted access
- Administrator access
### Public access
No authentication is required.
### Authenticated access
Authentication is required.
### Restricted access
Authentication is required. Additional checks may apply to verify access to the resource, and returned data may be filtered.
### Administrator access
Authentication and an administrator role are both required.
# Execute Docker requests
Portainer does not expose dedicated endpoints for managing Docker resources (create a container, remove a volume, etc).
Instead, it acts as a reverse-proxy to the Docker HTTP API, allowing you to execute Docker requests via the Portainer HTTP API.
To do so, use the `/endpoints/{id}/docker` endpoint. Note that this endpoint is not documented below due to Swagger limitations. It has a restricted access policy, so authentication is still required. Any request made to this endpoint is proxied to the Docker API of the associated environment - request and response objects are identical to those in the [Docker official documentation](https://docs.docker.com/engine/api).
# Private Registry
When using a private registry, include a Base64-encoded JSON string in the request header. The header parameter name is `X-Registry-Auth` and the value should encode the following structure: ‘{"registryId":\<registryId\>}’ where `<registryId>` is the ID of the registry where the repository was created.
Example encoded value:
```
eyJyZWdpc3RyeUlkIjoxfQ==
```

View File

@@ -8,8 +8,8 @@ import (
"time"
)
func (s *Service) GetEncodedAuthorizationToken(ctx context.Context) (token *string, expiry *time.Time, err error) {
getAuthorizationTokenOutput, err := s.client.GetAuthorizationToken(ctx, nil)
func (s *Service) GetEncodedAuthorizationToken() (token *string, expiry *time.Time, err error) {
getAuthorizationTokenOutput, err := s.client.GetAuthorizationToken(context.TODO(), nil)
if err != nil {
return
}
@@ -27,8 +27,8 @@ func (s *Service) GetEncodedAuthorizationToken(ctx context.Context) (token *stri
return
}
func (s *Service) GetAuthorizationToken(ctx context.Context) (token *string, expiry *time.Time, err error) {
tokenEncodedStr, expiry, err := s.GetEncodedAuthorizationToken(ctx)
func (s *Service) GetAuthorizationToken() (token *string, expiry *time.Time, err error) {
tokenEncodedStr, expiry, err := s.GetEncodedAuthorizationToken()
if err != nil {
return
}

View File

@@ -1,274 +0,0 @@
package backup
import (
"bytes"
"context"
"io"
"os"
"path/filepath"
"testing"
"github.com/portainer/portainer/api/archive"
"github.com/portainer/portainer/api/crypto"
"github.com/portainer/portainer/api/datastore"
"github.com/portainer/portainer/api/filesystem"
"github.com/portainer/portainer/api/http/offlinegate"
"github.com/portainer/portainer/pkg/fips"
"github.com/stretchr/testify/require"
)
func init() {
fips.InitFIPS(false)
}
func TestGetRestoreSourcePath_DBAtRoot(t *testing.T) {
t.Parallel()
dir := t.TempDir()
err := os.WriteFile(filesystem.JoinPaths(dir, "portainer.db"), []byte("db"), 0o600)
require.NoError(t, err)
result, err := getRestoreSourcePath(dir)
require.NoError(t, err)
require.Equal(t, dir, result)
}
func TestGetRestoreSourcePath_EncryptedDBAtRoot(t *testing.T) {
t.Parallel()
dir := t.TempDir()
err := os.WriteFile(filesystem.JoinPaths(dir, "portainer.edb"), []byte("db"), 0o600)
require.NoError(t, err)
result, err := getRestoreSourcePath(dir)
require.NoError(t, err)
require.Equal(t, dir, result)
}
func TestGetRestoreSourcePath_DBInSubdirectory(t *testing.T) {
t.Parallel()
dir := t.TempDir()
sub := filesystem.JoinPaths(dir, "backup-2024-01-01")
err := os.Mkdir(sub, 0o700)
require.NoError(t, err)
err = os.WriteFile(filesystem.JoinPaths(sub, "portainer.db"), []byte("db"), 0o600)
require.NoError(t, err)
result, err := getRestoreSourcePath(dir)
require.NoError(t, err)
require.Equal(t, sub, result)
}
func TestGetRestoreSourcePath_NoDBFile(t *testing.T) {
t.Parallel()
dir := t.TempDir()
err := os.WriteFile(filesystem.JoinPaths(dir, "other.file"), []byte("data"), 0o600)
require.NoError(t, err)
result, err := getRestoreSourcePath(dir)
require.NoError(t, err)
require.Equal(t, dir, result)
}
func TestGetRestoreSourcePath_EmptyDir(t *testing.T) {
t.Parallel()
dir := t.TempDir()
result, err := getRestoreSourcePath(dir)
require.NoError(t, err)
require.Equal(t, dir, result)
}
func TestEncryptDecrypt_RoundTrip(t *testing.T) {
t.Parallel()
dir := t.TempDir()
plaintext := []byte("sensitive portainer backup data")
srcPath := filesystem.JoinPaths(dir, "archive.tar.gz")
err := os.WriteFile(srcPath, plaintext, 0o600)
require.NoError(t, err)
encryptedPath, err := encrypt(srcPath, "mysecretpassword")
require.NoError(t, err)
require.Equal(t, srcPath+".encrypted", encryptedPath)
encryptedData, err := os.ReadFile(encryptedPath)
require.NoError(t, err)
decryptedReader, err := crypto.AesDecrypt(bytes.NewReader(encryptedData), []byte("mysecretpassword"))
require.NoError(t, err)
decrypted, err := io.ReadAll(decryptedReader)
require.NoError(t, err)
require.Equal(t, plaintext, decrypted)
}
func TestEncryptDecrypt_WrongPassword(t *testing.T) {
t.Parallel()
dir := t.TempDir()
srcPath := filesystem.JoinPaths(dir, "archive.tar.gz")
err := os.WriteFile(srcPath, []byte("data"), 0o600)
require.NoError(t, err)
encryptedPath, err := encrypt(srcPath, "correctpassword")
require.NoError(t, err)
encryptedData, err := os.ReadFile(encryptedPath)
require.NoError(t, err)
_, err = crypto.AesDecrypt(bytes.NewReader(encryptedData), []byte("wrongpassword"))
require.Error(t, err)
}
func TestCreateBackupArchive_NoPassword(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, true, false)
storePath := store.GetConnection().GetStorePath()
gate := offlinegate.NewOfflineGate()
archivePath, err := CreateBackupArchive("", gate, store, storePath)
require.NoError(t, err)
f, err := os.Open(archivePath)
require.NoError(t, err)
t.Cleanup(func() {
err := f.Close()
require.NoError(t, err)
})
extractDir := t.TempDir()
err = archive.ExtractTarGz(f, extractDir)
require.NoError(t, err)
dbFound := false
err = filepath.Walk(extractDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.Name() == "portainer.db" {
dbFound = true
}
return nil
})
require.NoError(t, err)
require.True(t, dbFound, "archive should contain portainer.db")
}
func TestCreateBackupArchive_WithPassword(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, true, false)
storePath := store.GetConnection().GetStorePath()
gate := offlinegate.NewOfflineGate()
archivePath, err := CreateBackupArchive("backup-secret", gate, store, storePath)
require.NoError(t, err)
require.Contains(t, archivePath, ".encrypted")
encryptedData, err := os.ReadFile(archivePath)
require.NoError(t, err)
decryptedReader, err := crypto.AesDecrypt(bytes.NewReader(encryptedData), []byte("backup-secret"))
require.NoError(t, err)
extractDir := t.TempDir()
err = archive.ExtractTarGz(decryptedReader, extractDir)
require.NoError(t, err)
dbFound := false
err = filepath.Walk(extractDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.Name() == "portainer.db" {
dbFound = true
}
return nil
})
require.NoError(t, err)
require.True(t, dbFound, "decrypted archive should contain portainer.db")
}
func TestRestoreArchive_NoPassword(t *testing.T) {
t.Parallel()
_, store1 := datastore.MustNewTestStore(t, true, false)
storePath1 := store1.GetConnection().GetStorePath()
gate := offlinegate.NewOfflineGate()
archivePath, err := CreateBackupArchive("", gate, store1, storePath1)
require.NoError(t, err)
archiveData, err := os.ReadFile(archivePath)
require.NoError(t, err)
_, store2 := datastore.MustNewTestStore(t, true, false)
storePath2 := store2.GetConnection().GetStorePath()
ctx, cancel := context.WithCancel(t.Context())
err = RestoreArchive(bytes.NewReader(archiveData), "", storePath2, gate, store2, cancel)
require.NoError(t, err)
require.ErrorIs(t, ctx.Err(), context.Canceled)
_, err = os.Stat(filesystem.JoinPaths(storePath2, "portainer.db"))
require.NoError(t, err)
}
func TestRestoreArchive_WithPassword(t *testing.T) {
t.Parallel()
_, store1 := datastore.MustNewTestStore(t, true, false)
storePath1 := store1.GetConnection().GetStorePath()
gate := offlinegate.NewOfflineGate()
archivePath, err := CreateBackupArchive("restore-secret", gate, store1, storePath1)
require.NoError(t, err)
archiveData, err := os.ReadFile(archivePath)
require.NoError(t, err)
_, store2 := datastore.MustNewTestStore(t, true, false)
storePath2 := store2.GetConnection().GetStorePath()
ctx, cancel := context.WithCancel(t.Context())
err = RestoreArchive(bytes.NewReader(archiveData), "restore-secret", storePath2, gate, store2, cancel)
require.NoError(t, err)
require.ErrorIs(t, ctx.Err(), context.Canceled)
_, err = os.Stat(filesystem.JoinPaths(storePath2, "portainer.db"))
require.NoError(t, err)
}
func TestRestoreArchive_WrongPassword(t *testing.T) {
t.Parallel()
_, store1 := datastore.MustNewTestStore(t, true, false)
storePath1 := store1.GetConnection().GetStorePath()
gate := offlinegate.NewOfflineGate()
archivePath, err := CreateBackupArchive("correct-password", gate, store1, storePath1)
require.NoError(t, err)
archiveData, err := os.ReadFile(archivePath)
require.NoError(t, err)
_, store2 := datastore.MustNewTestStore(t, true, false)
storePath2 := store2.GetConnection().GetStorePath()
_, cancel := context.WithCancel(t.Context())
err = RestoreArchive(bytes.NewReader(archiveData), "wrong-password", storePath2, gate, store2, cancel)
require.Error(t, err)
}

View File

@@ -243,9 +243,8 @@ func (service *Service) startTunnelVerificationLoop() {
})
}
// checkTunnels finds tunnels that need snapshots and processes them one at a time.
// For active tunnels missing an initial snapshot, it takes one without closing the tunnel.
// For tunnels idle past activeTimeout, it snapshots and closes them.
// checkTunnels finds the first tunnel that has not had any activity recently
// and attempts to take a snapshot, then closes it and returns
func (service *Service) checkTunnels() {
service.mu.RLock()
@@ -256,32 +255,12 @@ func (service *Service) checkTunnels() {
Float64("last_activity_seconds", elapsed.Seconds()).
Msg("environment tunnel monitoring")
tunnelPort := tunnel.Port
if !tunnel.HasSnapshot && elapsed < activeTimeout {
service.mu.RUnlock()
if endpointHasSnapshot(service.dataStore, endpointID) {
service.markSnapshotTaken(endpointID)
return
}
log.Debug().
Int("endpoint_id", int(endpointID)).
Msg("taking initial snapshot for active Edge environment")
if service.snapshotAndLog(endpointID, tunnelPort) {
service.markSnapshotTaken(endpointID)
}
return
}
if tunnel.Status == portainer.EdgeAgentManagementRequired && elapsed < activeTimeout {
continue
}
tunnelPort := tunnel.Port
service.mu.RUnlock()
log.Debug().
@@ -290,7 +269,13 @@ func (service *Service) checkTunnels() {
Float64("timeout_seconds", activeTimeout.Seconds()).
Msg("last activity timeout exceeded")
service.snapshotAndLog(endpointID, tunnelPort)
if err := service.snapshotEnvironment(endpointID, tunnelPort); err != nil {
log.Error().
Int("endpoint_id", int(endpointID)).
Err(err).
Msg("unable to snapshot Edge environment")
}
service.close(endpointID)
return
@@ -299,32 +284,6 @@ func (service *Service) checkTunnels() {
service.mu.RUnlock()
}
func (service *Service) snapshotAndLog(endpointID portainer.EndpointID, tunnelPort int) bool {
if err := service.snapshotEnvironment(endpointID, tunnelPort); err != nil {
log.Error().
Int("endpoint_id", int(endpointID)).
Err(err).
Msg("unable to snapshot Edge environment")
if service.dataStore.IsErrObjectNotFound(err) {
service.close(endpointID)
}
return false
}
return true
}
func (service *Service) markSnapshotTaken(endpointID portainer.EndpointID) {
service.mu.Lock()
defer service.mu.Unlock()
if tun, ok := service.activeTunnels[endpointID]; ok {
tun.HasSnapshot = true
}
}
func (service *Service) snapshotEnvironment(endpointID portainer.EndpointID, tunnelPort int) error {
endpoint, err := service.dataStore.Endpoint().Endpoint(endpointID)
if err != nil {

View File

@@ -1,8 +1,6 @@
package chisel
import (
"context"
"errors"
"net"
"net/http"
"testing"
@@ -19,36 +17,14 @@ func init() {
fips.InitFIPS(false)
}
type mockSnapshotService struct {
snapshotFn func(endpoint *portainer.Endpoint) error
}
func (m *mockSnapshotService) Start(_ context.Context) {}
func (m *mockSnapshotService) SetSnapshotInterval(_ string) error { return nil }
func (m *mockSnapshotService) SnapshotEndpoint(endpoint *portainer.Endpoint) error {
if m.snapshotFn != nil {
return m.snapshotFn(endpoint)
}
return nil
}
func (m *mockSnapshotService) FillSnapshotData(_ *portainer.Endpoint, _ bool) error { return nil }
func newEdgeEndpoint(id portainer.EndpointID) *portainer.Endpoint {
return &portainer.Endpoint{
ID: id,
func TestPingAgentPanic(t *testing.T) {
t.Parallel()
endpoint := &portainer.Endpoint{
ID: 1,
EdgeID: "test-edge-id",
Type: portainer.EdgeAgentOnDockerEnvironment,
UserTrusted: true,
}
}
func TestPingAgentPanic(t *testing.T) {
t.Parallel()
endpoint := newEdgeEndpoint(1)
_, store := datastore.MustNewTestStore(t, false, true)
@@ -81,158 +57,3 @@ func TestPingAgentPanic(t *testing.T) {
require.NoError(t, srv.Shutdown(t.Context()))
require.ErrorIs(t, <-errCh, http.ErrServerClosed)
}
func TestOpenDefaultsHasSnapshotToFalse(t *testing.T) {
t.Parallel()
endpoint := newEdgeEndpoint(1)
_, store := datastore.MustNewTestStore(t, false, true)
s := NewService(store, nil, nil)
err := s.Open(endpoint)
require.NoError(t, err)
require.False(t, s.activeTunnels[endpoint.ID].HasSnapshot)
}
func TestCheckTunnelsSetsHasSnapshotWhenSnapshotExists(t *testing.T) {
t.Parallel()
endpoint := newEdgeEndpoint(2)
_, store := datastore.MustNewTestStore(t, false, true)
err := store.Endpoint().Create(endpoint)
require.NoError(t, err)
snap := &portainer.Snapshot{
EndpointID: endpoint.ID,
Docker: &portainer.DockerSnapshot{},
}
err = store.Snapshot().Create(snap)
require.NoError(t, err)
s := NewService(store, nil, nil)
s.activeTunnels[endpoint.ID] = &portainer.TunnelDetails{
Status: portainer.EdgeAgentManagementRequired,
Port: 50003,
LastActivity: time.Now(),
}
s.checkTunnels()
require.NotNil(t, s.activeTunnels[endpoint.ID], "tunnel must remain open")
require.True(t, s.activeTunnels[endpoint.ID].HasSnapshot)
}
func TestCheckTunnelsSnapshotsActiveEnvironmentAndKeepsTunnelAlive(t *testing.T) {
t.Parallel()
endpoint := newEdgeEndpoint(3)
_, store := datastore.MustNewTestStore(t, false, true)
err := store.Endpoint().Create(endpoint)
require.NoError(t, err)
snapshotCalled := false
svc := &mockSnapshotService{
snapshotFn: func(_ *portainer.Endpoint) error {
snapshotCalled = true
return nil
},
}
s := NewService(store, nil, nil)
s.snapshotService = svc
s.activeTunnels[endpoint.ID] = &portainer.TunnelDetails{
Status: portainer.EdgeAgentManagementRequired,
Port: 50000,
LastActivity: time.Now(),
}
s.checkTunnels()
require.True(t, snapshotCalled)
require.NotNil(t, s.activeTunnels[endpoint.ID], "tunnel must remain open after snapshot")
require.True(t, s.activeTunnels[endpoint.ID].HasSnapshot)
}
func TestCheckTunnelsKeepsHasSnapshotFalseOnSnapshotFailure(t *testing.T) {
t.Parallel()
endpoint := newEdgeEndpoint(4)
_, store := datastore.MustNewTestStore(t, false, true)
err := store.Endpoint().Create(endpoint)
require.NoError(t, err)
svc := &mockSnapshotService{
snapshotFn: func(_ *portainer.Endpoint) error {
return errors.New("snapshot failed")
},
}
s := NewService(store, nil, nil)
s.snapshotService = svc
s.activeTunnels[endpoint.ID] = &portainer.TunnelDetails{
Status: portainer.EdgeAgentManagementRequired,
Port: 50001,
LastActivity: time.Now(),
}
s.checkTunnels()
require.NotNil(t, s.activeTunnels[endpoint.ID], "tunnel must remain open after failed snapshot")
require.False(t, s.activeTunnels[endpoint.ID].HasSnapshot, "HasSnapshot must stay false after failure")
}
func TestCheckTunnelsClosesStaleEntryForDeletedEndpoint(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
// Endpoint is not created in the store, simulates deletion while tunnel stays open.
s := NewService(store, nil, nil)
s.activeTunnels[1] = &portainer.TunnelDetails{
Status: portainer.EdgeAgentManagementRequired,
Port: 50010,
LastActivity: time.Now(),
}
s.checkTunnels()
require.Nil(t, s.activeTunnels[1], "stale tunnel for deleted endpoint must be removed immediately")
}
func TestCheckTunnelsClosesIdleTunnelAndSnapshots(t *testing.T) {
t.Parallel()
endpoint := newEdgeEndpoint(5)
_, store := datastore.MustNewTestStore(t, false, true)
err := store.Endpoint().Create(endpoint)
require.NoError(t, err)
snapshotCalled := false
svc := &mockSnapshotService{
snapshotFn: func(_ *portainer.Endpoint) error {
snapshotCalled = true
return nil
},
}
s := NewService(store, nil, nil)
s.snapshotService = svc
s.activeTunnels[endpoint.ID] = &portainer.TunnelDetails{
Status: portainer.EdgeAgentManagementRequired,
Port: 50002,
LastActivity: time.Now().Add(-(activeTimeout + time.Second)),
}
s.checkTunnels()
require.True(t, snapshotCalled)
require.Nil(t, s.activeTunnels[endpoint.ID], "tunnel must be closed after idle timeout")
}

View File

@@ -9,7 +9,6 @@ import (
"time"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/internal/edge"
"github.com/portainer/portainer/api/internal/edge/cache"
"github.com/portainer/portainer/api/internal/endpointutils"
@@ -82,24 +81,17 @@ func (s *Service) Open(endpoint *portainer.Endpoint) error {
return nil
}
// close removes the tunnel from the map so the agent will close it.
// The lock is released before cleaning up the chisel user and proxy to avoid
// blocking Config/Open callers while DeleteUser interacts with chisel internals.
// close removes the tunnel from the map so the agent will close it
func (s *Service) close(endpointID portainer.EndpointID) {
s.mu.Lock()
defer s.mu.Unlock()
tun, ok := s.activeTunnels[endpointID]
if !ok {
s.mu.Unlock()
return
}
delete(s.activeTunnels, endpointID)
cache.Del(endpointID)
s.mu.Unlock()
if s.chiselServer != nil {
if len(tun.Credentials) > 0 && s.chiselServer != nil {
user, _, _ := strings.Cut(tun.Credentials, ":")
s.chiselServer.DeleteUser(user)
}
@@ -107,6 +99,10 @@ func (s *Service) close(endpointID portainer.EndpointID) {
if s.ProxyManager != nil {
s.ProxyManager.DeleteEndpointProxy(endpointID)
}
delete(s.activeTunnels, endpointID)
cache.Del(endpointID)
}
// Config returns the tunnel details needed for the agent to connect
@@ -241,18 +237,3 @@ func encryptCredentials(username, password, key string) (string, error) {
return base64.RawStdEncoding.EncodeToString(encryptedCredentials), nil
}
func endpointHasSnapshot(dataStore dataservices.DataStore, endpointID portainer.EndpointID) bool {
var hasSnapshot bool
_ = dataStore.ViewTx(func(tx dataservices.DataStoreTx) error {
s, err := tx.Snapshot().Read(endpointID)
if err != nil {
return err
}
hasSnapshot = s.Docker != nil || s.Kubernetes != nil
return nil
})
return hasSnapshot
}

View File

@@ -56,8 +56,6 @@ func CLIFlags() *portainer.CLIFlags {
TrustedOrigins: kingpin.Flag("trusted-origins", "List of trusted origins for CSRF protection. Separate multiple origins with a comma.").Envar(portainer.TrustedOriginsEnvVar).String(),
CSP: kingpin.Flag("csp", "Content Security Policy (CSP) header").Envar(portainer.CSPEnvVar).Default("true").Bool(),
CompactDB: kingpin.Flag("compact-db", "Enable database compaction on startup").Envar(portainer.CompactDBEnvVar).Default("false").Bool(),
NoSetupToken: kingpin.Flag("no-setup-token", "Disable the setup token requirement for admin initialization and restore on an uninitialized instance").Envar(portainer.NoSetupTokenEnvVar).Bool(),
SetupToken: kingpin.Flag("setup-token", "Set a custom setup token for admin initialization and restore on an uninitialized instance (overrides auto-generation)").Envar(portainer.SetupTokenEnvVar).String(),
}
}
@@ -96,20 +94,13 @@ func (Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
flags.TLSKey = tlsKeyFlag.String()
flags.TLSCacert = kingpin.Flag("tlscacert", "Path to the CA").Default(defaultTLSCACertPath).String()
var hasKubectlShellImageFlag bool
kubectlShellImageFlag := kingpin.Flag(
flags.KubectlShellImage = kingpin.Flag(
"kubectl-shell-image",
"Kubectl shell image",
).Envar(portainer.KubectlShellImageEnvVar).
Default(portainer.DefaultKubectlShellImage).
IsSetByUser(&hasKubectlShellImageFlag)
flags.KubectlShellImage = kubectlShellImageFlag.String()
).Envar(portainer.KubectlShellImageEnvVar).Default(portainer.DefaultKubectlShellImage).String()
kingpin.Parse()
_, kubectlShellImageEnvVarSet := os.LookupEnv(portainer.KubectlShellImageEnvVar)
flags.KubectlShellImageSet = hasKubectlShellImageFlag || kubectlShellImageEnvVarSet
if !filepath.IsAbs(*flags.Assets) {
ex, err := os.Executable()
if err != nil {

View File

@@ -6,7 +6,6 @@ import (
"strings"
"testing"
portainer "github.com/portainer/portainer/api"
zerolog "github.com/rs/zerolog/log"
"github.com/stretchr/testify/require"
)
@@ -27,59 +26,6 @@ func TestOptionParser(t *testing.T) {
require.True(t, *opts.EnableEdgeComputeFeatures)
}
func TestParseKubectlShellImageFlag(t *testing.T) {
tests := []struct {
name string
args []string
envVars map[string]string
expectedKubectlShellImageSet bool
expectedKubectlShellFlag string
}{
{
name: "no flag, no env var",
expectedKubectlShellImageSet: false,
expectedKubectlShellFlag: portainer.DefaultKubectlShellImage,
},
{
name: "explicit flag",
args: []string{"portainer", "--kubectl-shell-image=myimage:v2"},
expectedKubectlShellImageSet: true,
expectedKubectlShellFlag: "myimage:v2",
},
{
name: "env var",
envVars: map[string]string{portainer.KubectlShellImageEnvVar: "myimage:v3"},
expectedKubectlShellImageSet: true,
expectedKubectlShellFlag: "myimage:v3",
},
{
name: "both env var and flag set",
args: []string{"portainer", "--kubectl-shell-image=myimage:v2"},
envVars: map[string]string{portainer.KubectlShellImageEnvVar: "myimage:v3"},
expectedKubectlShellImageSet: true,
expectedKubectlShellFlag: "myimage:v2",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
if tc.args == nil {
tc.args = []string{"portainer"}
}
setOsArgs(t, tc.args)
for k, v := range tc.envVars {
t.Setenv(k, v)
}
flags, err := Service{}.ParseFlags("test-version")
require.NoError(t, err)
require.Equal(t, tc.expectedKubectlShellImageSet, flags.KubectlShellImageSet)
require.Equal(t, tc.expectedKubectlShellFlag, *flags.KubectlShellImage)
})
}
}
func TestParseTLSFlags(t *testing.T) {
testCases := []struct {
name string

View File

@@ -4,7 +4,6 @@ import (
"cmp"
"context"
"crypto/sha256"
nethttp "net/http"
"os"
"path"
"strings"
@@ -27,10 +26,10 @@ import (
"github.com/portainer/portainer/api/exec"
"github.com/portainer/portainer/api/filesystem"
"github.com/portainer/portainer/api/git"
"github.com/portainer/portainer/api/hostmanagement/openamt"
"github.com/portainer/portainer/api/http"
"github.com/portainer/portainer/api/http/proxy"
kubeproxy "github.com/portainer/portainer/api/http/proxy/factory/kubernetes"
"github.com/portainer/portainer/api/http/security/setuptoken"
"github.com/portainer/portainer/api/internal/authorization"
"github.com/portainer/portainer/api/internal/edge/edgestacks"
"github.com/portainer/portainer/api/internal/endpointutils"
@@ -53,15 +52,9 @@ import (
"github.com/portainer/portainer/pkg/featureflags"
"github.com/portainer/portainer/pkg/fips"
"github.com/portainer/portainer/pkg/libhelm"
"github.com/portainer/portainer/pkg/libhttp/ssrf"
"github.com/portainer/portainer/pkg/libstack/compose"
libswarm "github.com/portainer/portainer/pkg/libstack/swarm"
"github.com/portainer/portainer/pkg/validate"
gogitclient "github.com/go-git/go-git/v5/plumbing/transport/client"
gogitraw "github.com/go-git/go-git/v5/plumbing/transport/git"
gogithttp "github.com/go-git/go-git/v5/plumbing/transport/http"
gogitssh "github.com/go-git/go-git/v5/plumbing/transport/ssh"
"github.com/google/uuid"
"github.com/rs/zerolog/log"
)
@@ -232,32 +225,6 @@ func initSnapshotService(
return snapshotService, nil
}
func resolveSetupToken(tx dataservices.DataStoreTx, providedToken string) (string, error) {
admins, err := tx.User().UsersByRole(portainer.AdministratorRole)
if err != nil {
return "", err
}
if len(admins) > 0 {
return "", nil
}
if providedToken != "" {
log.Info().Msg("using custom setup token; admin initialization and backup restore require this token in the X-Setup-Token header")
return providedToken, nil
}
token, err := setuptoken.Generate()
if err != nil {
return "", err
}
log.Info().
Str("setup_token", token).
Msg("no administrator account configured; admin initialization and backup restore require this setup token in the X-Setup-Token header. Start with --no-setup-token to disable.")
return token, nil
}
func initStatus(instanceID string) *portainer.Status {
return &portainer.Status{
Version: portainer.APIVersion,
@@ -276,10 +243,6 @@ func updateSettingsFromFlags(dataStore dataservices.DataStore, flags *portainer.
settings.EnableEdgeComputeFeatures = cmp.Or(*flags.EnableEdgeComputeFeatures, settings.EnableEdgeComputeFeatures)
settings.TemplatesURL = cmp.Or(*flags.Templates, settings.TemplatesURL)
if flags.KubectlShellImageSet {
settings.KubectlShellImage = *flags.KubectlShellImage
}
if *flags.Labels != nil {
settings.BlackListedLabels = *flags.Labels
}
@@ -371,6 +334,7 @@ func loadEncryptionSecretKey(keyfilename string) []byte {
}
func buildServer(flags *portainer.CLIFlags, shutdownCtx context.Context, shutdownTrigger context.CancelFunc) portainer.Server {
if flags.FeatureFlags != nil {
featureflags.Parse(*flags.FeatureFlags, portainer.SupportedFeatureFlags)
}
@@ -407,19 +371,6 @@ func buildServer(flags *portainer.CLIFlags, shutdownCtx context.Context, shutdow
log.Fatal().Msg("The database schema version does not align with the server version. Please consider reverting to the previous server version or addressing the database migration issue.")
}
if err := ssrf.Configure(dataStore.AllowList()); err != nil {
log.Fatal().Err(err).Msg("failed initializing ssrf service")
}
if !ssrf.WrapDefaultTransport() {
log.Fatal().Msg("failed to wrap default HTTP transport with SSRF protection")
}
gogithttp.DefaultClient = gogithttp.NewClient(&nethttp.Client{Transport: nethttp.DefaultTransport})
gogitclient.InstallProtocol("git", git.NewSSRFGitTransport(gogitraw.DefaultClient))
gogitclient.InstallProtocol("ssh", git.NewSSRFGitTransport(gogitssh.DefaultClient))
gogitclient.InstallProtocol("file", nil)
instanceID, err := dataStore.Version().InstanceID()
if err != nil {
log.Fatal().Err(err).Msg("failed getting instance id")
@@ -443,6 +394,9 @@ func buildServer(flags *portainer.CLIFlags, shutdownCtx context.Context, shutdow
gitService := git.NewService(shutdownCtx)
// Setting insecureSkipVerify to true to preserve the old behaviour.
openAMTService := openamt.NewService(true)
cryptoService := crypto.Service{}
signatureService := initDigitalSignatureService()
@@ -483,11 +437,16 @@ func buildServer(flags *portainer.CLIFlags, shutdownCtx context.Context, shutdow
reverseTunnelService.ProxyManager = proxyManager
dockerConfigPath := fileService.GetDockerConfigPath()
composeDeployer := compose.NewComposeDeployer()
composeStackManager := exec.NewComposeStackManager(composeDeployer, proxyManager)
composeStackManager := exec.NewComposeStackManager(composeDeployer, proxyManager, dataStore)
swarmStackManager := exec.NewSwarmStackManager(libswarm.NewSwarmDeployer(), proxyManager)
swarmStackManager, err := exec.NewSwarmStackManager(*flags.Assets, dockerConfigPath, signatureService, fileService, reverseTunnelService, dataStore)
if err != nil {
log.Fatal().Err(err).Msg("failed initializing swarm stack manager")
}
kubernetesDeployer := initKubernetesDeployer(kubernetesTokenCacheManager, kubernetesClientFactory, dataStore, reverseTunnelService, signatureService, proxyManager)
@@ -556,17 +515,6 @@ func buildServer(flags *portainer.CLIFlags, shutdownCtx context.Context, shutdow
}
}
setupToken := ""
if adminPasswordHash == "" && !*flags.NoSetupToken {
if err := dataStore.ViewTx(func(tx dataservices.DataStoreTx) error {
var txErr error
setupToken, txErr = resolveSetupToken(tx, *flags.SetupToken)
return txErr
}); err != nil {
log.Fatal().Err(err).Msg("failed initializing setup token")
}
}
if err := reverseTunnelService.StartTunnelServer(*flags.TunnelAddr, *flags.TunnelPort, snapshotService); err != nil {
log.Fatal().Err(err).Msg("failed starting tunnel server")
}
@@ -641,6 +589,7 @@ func buildServer(flags *portainer.CLIFlags, shutdownCtx context.Context, shutdow
LDAPService: ldapService,
OAuthService: oauthService,
GitService: gitService,
OpenAMTService: openAMTService,
ProxyManager: proxyManager,
KubernetesTokenCacheManager: kubernetesTokenCacheManager,
KubeClusterAccessService: kubeClusterAccessService,
@@ -658,7 +607,6 @@ func buildServer(flags *portainer.CLIFlags, shutdownCtx context.Context, shutdow
PlatformService: platformService,
PullLimitCheckDisabled: *flags.PullLimitCheckDisabled,
TrustedOrigins: trustedOrigins,
SetupToken: setupToken,
}
}

View File

@@ -4,56 +4,16 @@ import (
"os"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/filesystem"
"github.com/portainer/portainer/api/internal/testhelpers"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func Test_resolveSetupToken(t *testing.T) {
t.Parallel()
t.Run("admin already exists — returns empty token", func(t *testing.T) {
admin := portainer.User{Role: portainer.AdministratorRole}
store := testhelpers.NewDatastore(testhelpers.WithUsers([]portainer.User{admin}))
token, err := resolveSetupToken(store, "")
require.NoError(t, err)
assert.Empty(t, token)
})
t.Run("no admin — generates a 64-char hex token", func(t *testing.T) {
store := testhelpers.NewDatastore(testhelpers.WithUsers([]portainer.User{}))
token, err := resolveSetupToken(store, "")
require.NoError(t, err)
assert.Len(t, token, 64)
token2, err := resolveSetupToken(store, "")
require.NoError(t, err)
assert.NotEqual(t, token, token2)
})
t.Run("no admin — uses provided token", func(t *testing.T) {
store := testhelpers.NewDatastore(testhelpers.WithUsers([]portainer.User{}))
token, err := resolveSetupToken(store, "mysecrettoken")
require.NoError(t, err)
assert.Equal(t, "mysecrettoken", token)
})
t.Run("admin already exists — ignores provided token", func(t *testing.T) {
admin := portainer.User{Role: portainer.AdministratorRole}
store := testhelpers.NewDatastore(testhelpers.WithUsers([]portainer.User{admin}))
token, err := resolveSetupToken(store, "mysecrettoken")
require.NoError(t, err)
assert.Empty(t, token)
})
}
const secretFileName = "secret.txt"
func createPasswordFile(t *testing.T, secretPath, password string) string {
err := os.WriteFile(secretPath, []byte(password), 0o600)
err := os.WriteFile(secretPath, []byte(password), 0600)
require.NoError(t, err)
return secretPath
}
@@ -80,65 +40,6 @@ func TestLoadEncryptionSecretKey(t *testing.T) {
require.Len(t, encryptionKey, 32)
}
func TestUpdateSettingsFromFlags_KubectlShellImage(t *testing.T) {
const existingImage = "existing-image:v1"
const newImage = "new-image:v2"
emptyString := ""
falseBool := false
var emptyLabels []portainer.Pair
tests := []struct {
name string
imageSet bool
flagImage string
expectedKubectlShellImage string
}{
{
name: "flag not set — DB image unchanged",
imageSet: false,
flagImage: portainer.DefaultKubectlShellImage,
expectedKubectlShellImage: existingImage,
},
{
name: "flag set — DB image updated",
imageSet: true,
flagImage: newImage,
expectedKubectlShellImage: newImage,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
store := testhelpers.NewDatastore(
testhelpers.WithSettingsService(&portainer.Settings{
KubectlShellImage: existingImage,
}),
testhelpers.WithSSLSettingsService(&portainer.SSLSettings{}),
)
flags := &portainer.CLIFlags{
SnapshotInterval: &emptyString,
Logo: &emptyString,
EnableEdgeComputeFeatures: &falseBool,
Templates: &emptyString,
Labels: &emptyLabels,
HTTPDisabled: &falseBool,
HTTPEnabled: &falseBool,
}
flags.KubectlShellImage = &tc.flagImage
flags.KubectlShellImageSet = tc.imageSet
err := updateSettingsFromFlags(store, flags)
require.NoError(t, err)
settings, err := store.Settings().Settings()
require.NoError(t, err)
require.Equal(t, tc.expectedKubectlShellImage, settings.KubectlShellImage)
})
}
}
func TestDBSecretPath(t *testing.T) {
t.Parallel()
tests := []struct {

View File

@@ -1,149 +0,0 @@
package concurrent
import (
"context"
"errors"
"sync/atomic"
"testing"
"testing/synctest"
"time"
"github.com/stretchr/testify/require"
)
func TestRun_AllSucceed(t *testing.T) {
t.Parallel()
fn1 := func(ctx context.Context) (any, error) { return "one", nil }
fn2 := func(ctx context.Context) (any, error) { return "two", nil }
fn3 := func(ctx context.Context) (any, error) { return "three", nil }
results, err := Run(t.Context(), 0, fn1, fn2, fn3)
require.NoError(t, err)
require.Len(t, results, 3)
values := make([]string, 0, len(results))
for _, r := range results {
values = append(values, r.Result.(string))
}
require.ElementsMatch(t, []string{"one", "two", "three"}, values)
}
func TestRun_OneError(t *testing.T) {
t.Parallel()
sentinel := errors.New("task failed")
fn1 := func(ctx context.Context) (any, error) { return "ok", nil }
fn2 := func(ctx context.Context) (any, error) { return nil, sentinel }
_, err := Run(t.Context(), 0, fn1, fn2)
require.ErrorIs(t, err, sentinel)
}
func TestRun_NoTasks(t *testing.T) {
t.Parallel()
results, err := Run(t.Context(), 0)
require.NoError(t, err)
require.Empty(t, results)
}
func TestRun_MaxConcurrency(t *testing.T) {
t.Parallel()
const numTasks = 10
var peak atomic.Int32
var active atomic.Int32
task := func(ctx context.Context) (any, error) {
current := active.Add(1)
if current > peak.Load() {
peak.Store(current)
}
time.Sleep(10 * time.Millisecond)
active.Add(-1)
return nil, nil
}
tasks := make([]Func, numTasks)
for i := range tasks {
tasks[i] = task
}
synctest.Test(t, func(t *testing.T) {
results, err := Run(t.Context(), 3, tasks...)
require.NoError(t, err)
require.Len(t, results, numTasks)
require.LessOrEqual(t, peak.Load(), int32(3))
})
}
func TestRun_ZeroConcurrencyUsesAllTasks(t *testing.T) {
t.Parallel()
const numTasks = 5
var peak atomic.Int32
var active atomic.Int32
task := func(ctx context.Context) (any, error) {
current := active.Add(1)
if current > peak.Load() {
peak.Store(current)
}
time.Sleep(20 * time.Millisecond)
active.Add(-1)
return nil, nil
}
tasks := make([]Func, numTasks)
for i := range tasks {
tasks[i] = task
}
synctest.Test(t, func(t *testing.T) {
results, err := Run(t.Context(), 0, tasks...)
require.NoError(t, err)
require.Len(t, results, numTasks)
require.Equal(t, int32(numTasks), peak.Load())
})
}
func TestRun_ContextCancelledBeforeStart(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithCancel(t.Context())
cancel()
called := atomic.Bool{}
fn := func(ctx context.Context) (any, error) {
called.Store(true)
return nil, ctx.Err()
}
_, err := Run(ctx, 1, fn, fn, fn)
require.Error(t, err)
}
func TestRun_ContextPassedToTasks(t *testing.T) {
t.Parallel()
type key struct{}
ctx := context.WithValue(t.Context(), key{}, "testvalue")
fn := func(ctx context.Context) (any, error) {
return ctx.Value(key{}), nil
}
results, err := Run(ctx, 0, fn)
require.NoError(t, err)
require.Equal(t, "testvalue", results[0].Result)
}

View File

@@ -46,7 +46,7 @@ type Connection interface {
IsEncryptedStore() bool
NeedsEncryptionMigration() (bool, error)
SetEncrypted(encrypted bool) error
SetEncrypted(encrypted bool)
BackupMetadata() (map[string]any, error)
RestoreMetadata(s map[string]any) error

View File

@@ -4,7 +4,6 @@ import (
"crypto/rand"
"errors"
"io"
"slices"
)
type Nonce struct {
@@ -46,7 +45,7 @@ func (n *Nonce) Value() []byte {
func (n *Nonce) Increment() error {
// Start incrementing from the least significant byte
for i := range slices.Backward(n.val) {
for i := len(n.val) - 1; i >= 0; i-- {
// Increment the current byte
n.val[i]++

View File

@@ -1,8 +1,6 @@
package boltdb
import (
"crypto/aes"
"crypto/cipher"
"encoding/binary"
"errors"
"fmt"
@@ -42,8 +40,6 @@ type DbConnection struct {
isEncrypted bool
Compact bool
gcm cipher.AEAD
*bolt.DB
}
@@ -79,28 +75,8 @@ func (connection *DbConnection) GetDatabaseFileSize() (int64, error) {
return file.Size(), nil
}
func (connection *DbConnection) SetEncrypted(flag bool) error {
func (connection *DbConnection) SetEncrypted(flag bool) {
connection.isEncrypted = flag
if !flag || connection.EncryptionKey == nil {
connection.gcm = nil
return nil
}
block, err := aes.NewCipher(connection.EncryptionKey)
if err != nil {
return fmt.Errorf("creating AES cipher for database encryption: %w", err)
}
gcm, err := cipher.NewGCMWithRandomNonce(block)
if err != nil {
return fmt.Errorf("creating GCM cipher for database encryption: %w", err)
}
connection.gcm = gcm
return nil
}
// Return true if the database is encrypted
@@ -124,9 +100,7 @@ func (connection *DbConnection) NeedsEncryptionMigration() (bool, error) {
// If we have a loaded encryption key, always set encrypted
if connection.EncryptionKey != nil {
if err := connection.SetEncrypted(true); err != nil {
return false, err
}
connection.SetEncrypted(true)
}
// Check for portainer.db

View File

@@ -131,7 +131,7 @@ func Test_NeedsEncryptionMigration(t *testing.T) {
}
if tc.key {
connection.EncryptionKey = secretToEncryptionKey("secret")
connection.EncryptionKey = []byte("secret")
}
result, err := connection.NeedsEncryptionMigration()
@@ -142,57 +142,6 @@ func Test_NeedsEncryptionMigration(t *testing.T) {
}
}
func TestSetEncrypted_InvalidKeyReturnsError(t *testing.T) {
t.Parallel()
conn := DbConnection{EncryptionKey: []byte("bad")}
err := conn.SetEncrypted(true)
require.Error(t, err)
require.Nil(t, conn.gcm)
}
func TestSetEncrypted_NilKeyDoesNotSetGCM(t *testing.T) {
t.Parallel()
conn := DbConnection{}
err := conn.SetEncrypted(true)
require.NoError(t, err)
require.Nil(t, conn.gcm)
}
func TestSetEncrypted_EnableThenDisableStopsEncryption(t *testing.T) {
t.Parallel()
key := secretToEncryptionKey(passphrase)
conn := DbConnection{EncryptionKey: key}
err := conn.SetEncrypted(true)
require.NoError(t, err)
require.NotNil(t, conn.gcm)
err = conn.SetEncrypted(false)
require.NoError(t, err)
require.Nil(t, conn.gcm)
// MarshalObject must return plaintext after encryption is disabled
data, err := conn.MarshalObject("hello")
require.NoError(t, err)
require.Equal(t, "hello", string(data))
}
func TestNeedsEncryptionMigration_InvalidKeyError(t *testing.T) {
t.Parallel()
conn := DbConnection{
Path: t.TempDir(),
EncryptionKey: []byte("bad"),
}
result, err := conn.NeedsEncryptionMigration()
require.Error(t, err)
require.False(t, result)
}
func TestDBCompaction(t *testing.T) {
t.Parallel()
db := &DbConnection{Path: t.TempDir()}

View File

@@ -2,6 +2,7 @@ package boltdb
import (
"bytes"
"crypto/aes"
"crypto/cipher"
"github.com/pkg/errors"
@@ -27,18 +28,18 @@ func (connection *DbConnection) MarshalObject(object any) ([]byte, error) {
}
}
if connection.gcm == nil {
if connection.getEncryptionKey() == nil {
return buf.Bytes(), nil
}
return encrypt(buf.Bytes(), connection.gcm), nil
return encrypt(buf.Bytes(), connection.getEncryptionKey())
}
// UnmarshalObject decodes an object from binary data
func (connection *DbConnection) UnmarshalObject(data []byte, object any) error {
var err error
if connection.gcm != nil {
data, err = decrypt(data, connection.gcm)
if connection.getEncryptionKey() != nil {
data, err = decrypt(data, connection.getEncryptionKey())
if err != nil {
return errors.Wrap(err, "Failed decrypting object")
}
@@ -58,23 +59,48 @@ func (connection *DbConnection) UnmarshalObject(data []byte, object any) error {
return err
}
func encrypt(plaintext []byte, gcm cipher.AEAD) []byte {
return gcm.Seal(nil, nil, plaintext, nil)
// mmm, don't have a KMS .... aes GCM seems the most likely from
// https://gist.github.com/atoponce/07d8d4c833873be2f68c34f9afc5a78a#symmetric-encryption
func encrypt(plaintext []byte, passphrase []byte) (encrypted []byte, err error) {
block, err := aes.NewCipher(passphrase)
if err != nil {
return encrypted, err
}
// NewGCMWithRandomNonce in go 1.24 handles setting up the nonce and adding it to the encrypted output
gcm, err := cipher.NewGCMWithRandomNonce(block)
if err != nil {
return encrypted, err
}
return gcm.Seal(nil, nil, plaintext, nil), nil
}
func decrypt(encrypted []byte, gcm cipher.AEAD) ([]byte, error) {
func decrypt(encrypted []byte, passphrase []byte) (plaintextByte []byte, err error) {
if string(encrypted) == "false" {
return []byte("false"), nil
}
if len(encrypted) < gcm.Overhead() {
block, err := aes.NewCipher(passphrase)
if err != nil {
return encrypted, errors.Wrap(err, "Error creating cypher block")
}
// NewGCMWithRandomNonce in go 1.24 handles reading the nonce from the encrypted input for us
gcm, err := cipher.NewGCMWithRandomNonce(block)
if err != nil {
return encrypted, errors.Wrap(err, "Error creating GCM")
}
if len(encrypted) < gcm.NonceSize() {
return encrypted, errEncryptedStringTooShort
}
plaintextByte, err := gcm.Open(nil, nil, encrypted, nil)
plaintextByte, err = gcm.Open(nil, nil, encrypted, nil)
if err != nil {
return encrypted, errors.Wrap(err, "Error decrypting text")
}
return plaintextByte, nil
return plaintextByte, err
}

View File

@@ -17,7 +17,7 @@ import (
)
const (
jsonobject = `{"LogoURL":"","BlackListedLabels":[],"AuthenticationMethod":1,"InternalAuthSettings": {"RequiredPasswordLength": 12}"LDAPSettings":{"AnonymousMode":true,"ReaderDN":"","URL":"","TLSConfig":{"TLS":false,"TLSSkipVerify":false},"StartTLS":false,"SearchSettings":[{"BaseDN":"","Filter":"","UserNameAttribute":""}],"GroupSearchSettings":[{"GroupBaseDN":"","GroupFilter":"","GroupAttribute":""}],"AutoCreateUsers":true},"OAuthSettings":{"ClientID":"","AccessTokenURI":"","AuthorizationURI":"","ResourceURI":"","RedirectURI":"","UserIdentifier":"","Scopes":"","OAuthAutoCreateUsers":false,"DefaultTeamID":0,"SSO":true,"LogoutURI":"","KubeSecretKey":"j0zLVtY/lAWBk62ByyF0uP80SOXaitsABP0TTJX8MhI="},"FeatureFlagSettings":{},"SnapshotInterval":"5m","TemplatesURL":"https://raw.githubusercontent.com/portainer/templates/master/templates-2.0.json","EdgeAgentCheckinInterval":5,"EnableEdgeComputeFeatures":false,"UserSessionTimeout":"8h","KubeconfigExpiry":"0","HelmRepositoryURL":"https://charts.bitnami.com/bitnami","KubectlShellImage":"portainer/kubectl-shell","DisplayDonationHeader":false,"DisplayExternalContributors":false,"EnableHostManagementFeatures":false,"AllowVolumeBrowserForRegularUsers":false,"AllowBindMountsForRegularUsers":false,"AllowPrivilegedModeForRegularUsers":false,"AllowHostNamespaceForRegularUsers":false,"AllowStackManagementForRegularUsers":false,"AllowDeviceMappingForRegularUsers":false,"AllowContainerCapabilitiesForRegularUsers":false}`
jsonobject = `{"LogoURL":"","BlackListedLabels":[],"AuthenticationMethod":1,"InternalAuthSettings": {"RequiredPasswordLength": 12}"LDAPSettings":{"AnonymousMode":true,"ReaderDN":"","URL":"","TLSConfig":{"TLS":false,"TLSSkipVerify":false},"StartTLS":false,"SearchSettings":[{"BaseDN":"","Filter":"","UserNameAttribute":""}],"GroupSearchSettings":[{"GroupBaseDN":"","GroupFilter":"","GroupAttribute":""}],"AutoCreateUsers":true},"OAuthSettings":{"ClientID":"","AccessTokenURI":"","AuthorizationURI":"","ResourceURI":"","RedirectURI":"","UserIdentifier":"","Scopes":"","OAuthAutoCreateUsers":false,"DefaultTeamID":0,"SSO":true,"LogoutURI":"","KubeSecretKey":"j0zLVtY/lAWBk62ByyF0uP80SOXaitsABP0TTJX8MhI="},"OpenAMTConfiguration":{"Enabled":false,"MPSServer":"","MPSUser":"","MPSPassword":"","MPSToken":"","CertFileContent":"","CertFileName":"","CertFilePassword":"","DomainName":""},"FeatureFlagSettings":{},"SnapshotInterval":"5m","TemplatesURL":"https://raw.githubusercontent.com/portainer/templates/master/templates-2.0.json","EdgeAgentCheckinInterval":5,"EnableEdgeComputeFeatures":false,"UserSessionTimeout":"8h","KubeconfigExpiry":"0","HelmRepositoryURL":"https://charts.bitnami.com/bitnami","KubectlShellImage":"portainer/kubectl-shell","DisplayDonationHeader":false,"DisplayExternalContributors":false,"EnableHostManagementFeatures":false,"AllowVolumeBrowserForRegularUsers":false,"AllowBindMountsForRegularUsers":false,"AllowPrivilegedModeForRegularUsers":false,"AllowHostNamespaceForRegularUsers":false,"AllowStackManagementForRegularUsers":false,"AllowDeviceMappingForRegularUsers":false,"AllowContainerCapabilitiesForRegularUsers":false}`
passphrase = "my secret key"
)
@@ -170,10 +170,7 @@ func Test_ObjectMarshallingEncrypted(t *testing.T) {
}
key := secretToEncryptionKey(passphrase)
conn := DbConnection{EncryptionKey: key}
err := conn.SetEncrypted(true)
require.NoError(t, err)
conn := DbConnection{EncryptionKey: key, isEncrypted: true}
for _, test := range tests {
t.Run(fmt.Sprintf("%s -> %s", test.object, test.expected), func(t *testing.T) {
@@ -235,16 +232,13 @@ func Test_NonceSources(t *testing.T) {
return plaintext, err
}
encryptNewFn := encrypt
decryptNewFn := decrypt
passphrase := make([]byte, 32)
_, err := io.ReadFull(rand.Reader, passphrase)
require.NoError(t, err)
block, err := aes.NewCipher(passphrase)
require.NoError(t, err)
gcm, err := cipher.NewGCMWithRandomNonce(block)
require.NoError(t, err)
junk := make([]byte, 1024)
_, err = io.ReadFull(rand.Reader, junk)
require.NoError(t, err)
@@ -269,12 +263,13 @@ func Test_NonceSources(t *testing.T) {
enc, err = encryptOldFn(plain, passphrase)
require.NoError(t, err)
dec, err = decrypt(enc, gcm)
dec, err = decryptNewFn(enc, passphrase)
require.NoError(t, err)
require.Equal(t, plain, dec)
enc = encrypt(plain, gcm)
enc, err = encryptNewFn(plain, passphrase)
require.NoError(t, err)
dec, err = decryptOldFn(enc, passphrase)
require.NoError(t, err)
@@ -282,110 +277,3 @@ func Test_NonceSources(t *testing.T) {
require.Equal(t, plain, dec)
}
}
func TestDecrypt_FalseStringBypassesDecryption(t *testing.T) {
t.Parallel()
key := secretToEncryptionKey(passphrase)
block, err := aes.NewCipher(key)
require.NoError(t, err)
gcm, err := cipher.NewGCMWithRandomNonce(block)
require.NoError(t, err)
result, err := decrypt([]byte("false"), gcm)
require.NoError(t, err)
require.Equal(t, []byte("false"), result)
}
func TestDecrypt_ShortDataReturnsError(t *testing.T) {
t.Parallel()
key := secretToEncryptionKey(passphrase)
block, err := aes.NewCipher(key)
require.NoError(t, err)
gcm, err := cipher.NewGCMWithRandomNonce(block)
require.NoError(t, err)
short := []byte("short")
result, err := decrypt(short, gcm)
require.ErrorIs(t, err, errEncryptedStringTooShort)
require.Equal(t, short, result)
}
func TestDecrypt_CorruptDataReturnsError(t *testing.T) {
t.Parallel()
key := secretToEncryptionKey(passphrase)
block, err := aes.NewCipher(key)
require.NoError(t, err)
gcm, err := cipher.NewGCMWithRandomNonce(block)
require.NoError(t, err)
// 30 bytes passes the length check but fails authentication
corrupted := make([]byte, 30)
_, err = io.ReadFull(rand.Reader, corrupted)
require.NoError(t, err)
result, err := decrypt(corrupted, gcm)
require.Error(t, err)
require.Equal(t, corrupted, result)
}
// BenchmarkEncryptCachedCipher measures the new approach: cipher created once and reused.
func BenchmarkEncryptCachedCipher(b *testing.B) {
key := secretToEncryptionKey(passphrase)
conn := DbConnection{EncryptionKey: key}
err := conn.SetEncrypted(true)
require.NoError(b, err)
data := []byte(jsonobject)
b.ResetTimer()
for b.Loop() {
_ = encrypt(data, conn.gcm)
}
}
// BenchmarkEncryptPerCallCipher measures the old approach: cipher created on every call.
func BenchmarkEncryptPerCallCipher(b *testing.B) {
key := secretToEncryptionKey(passphrase)
data := []byte(jsonobject)
b.ResetTimer()
for b.Loop() {
block, err := aes.NewCipher(key)
if err != nil {
b.Fatal(err)
}
gcm, err := cipher.NewGCMWithRandomNonce(block)
if err != nil {
b.Fatal(err)
}
_ = gcm.Seal(nil, nil, data, nil)
}
}
// BenchmarkEncryptCachedCipherParallel verifies the cached cipher is safe for concurrent use.
func BenchmarkEncryptCachedCipherParallel(b *testing.B) {
key := secretToEncryptionKey(passphrase)
conn := DbConnection{EncryptionKey: key}
err := conn.SetEncrypted(true)
require.NoError(b, err)
data := []byte(jsonobject)
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
_ = encrypt(data, conn.gcm)
}
})
}

View File

@@ -40,10 +40,10 @@ func (tx *DbTransaction) GetRawBytes(bucketName string, key []byte) ([]byte, err
return nil, fmt.Errorf("%w (bucket=%s, key=%s)", dserrors.ErrObjectNotFound, bucketName, keyToString(key))
}
if tx.conn.gcm != nil {
if tx.conn.getEncryptionKey() != nil {
var err error
if value, err = decrypt(value, tx.conn.gcm); err != nil {
if value, err = decrypt(value, tx.conn.getEncryptionKey()); err != nil {
return value, errors.Wrap(err, "Failed decrypting object")
}
}

View File

@@ -2,7 +2,6 @@ package boltdb
import (
"errors"
"strconv"
"testing"
portainer "github.com/portainer/portainer/api"
@@ -24,10 +23,10 @@ func TestTxs(t *testing.T) {
err := conn.Open()
require.NoError(t, err)
t.Cleanup(func() {
defer func() {
err := conn.Close()
require.NoError(t, err)
})
}()
// Error propagation
err = conn.UpdateTx(func(tx portainer.Transaction) error {
@@ -104,57 +103,3 @@ func TestTxs(t *testing.T) {
})
require.Error(t, err)
}
func BenchmarkGetAll(b *testing.B) {
const endpointBucket = "endpoints"
const n = 10000
conn := DbConnection{Path: b.TempDir()}
err := conn.Open()
require.NoError(b, err)
b.Cleanup(func() {
err := conn.Close()
require.NoError(b, err)
})
err = conn.UpdateTx(func(tx portainer.Transaction) error {
if err := tx.SetServiceName(endpointBucket); err != nil {
return err
}
for i := 1; i <= n; i++ {
ep := portainer.Endpoint{
ID: portainer.EndpointID(i),
Name: "env-" + strconv.Itoa(i),
Type: portainer.DockerEnvironment,
URL: "tcp://192.168.1." + strconv.Itoa(i%254+1) + ":2375",
PublicURL: "https://env-" + strconv.Itoa(i) + ".example.com",
GroupID: portainer.EndpointGroupID(i%10 + 1),
TagIDs: []portainer.TagID{portainer.TagID(i%5 + 1), portainer.TagID(i%3 + 1)},
LastCheckInDate: int64(i) * 1000,
EdgeID: "edge-" + strconv.Itoa(i),
}
if err := tx.CreateObjectWithId(endpointBucket, i, &ep); err != nil {
return err
}
}
return nil
})
require.NoError(b, err)
b.ResetTimer()
b.ReportAllocs()
for b.Loop() {
var collection []portainer.Endpoint
if err := conn.ViewTx(func(tx portainer.Transaction) error {
return tx.GetAll(endpointBucket, new(portainer.Endpoint), dataservices.AppendFn(&collection))
}); err != nil {
b.Fatal(err)
}
}
}

View File

@@ -1,131 +0,0 @@
package allowlist
import (
"fmt"
lru "github.com/hashicorp/golang-lru"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/pkg/libhttp/ssrf"
)
const (
BucketName = "allowlist"
)
type Service struct {
baseService dataservices.BaseDataService[portainer.AllowList, portainer.AllowListKey]
cache *lru.Cache
}
func (service *Service) BucketName() string {
return service.baseService.BucketName()
}
func NewService(connection portainer.Connection) (*Service, error) {
err := connection.SetServiceName(BucketName)
if err != nil {
return nil, err
}
service := &Service{
baseService: dataservices.BaseDataService[portainer.AllowList, portainer.AllowListKey]{
Bucket: BucketName,
Connection: connection,
},
}
err = service.populateCache()
return service, err
}
func (service *Service) populateCache() error {
allowListKeys := []portainer.AllowListKey{portainer.AllowListSSRF}
cache, err := lru.New(len(allowListKeys))
if err != nil {
return err
}
for _, k := range allowListKeys {
allowList, err := service.baseService.Read(k)
if dataservices.IsErrObjectNotFound(err) {
allowList = &portainer.AllowList{
ID: k,
Mode: portainer.SSRFModeOff,
Entries: []string{},
}
} else if err != nil {
return err
}
parsedAllowList := ssrf.ParseAllowedHosts(allowList.Entries)
parsedAllowList.Mode = allowList.Mode
cache.Add(k, &parsedAllowList)
}
service.cache = cache
return nil
}
func (service *Service) Tx(tx portainer.Transaction) *ServiceTx {
return &ServiceTx{
baseService: service.baseService.Tx(tx),
cache: service.cache,
}
}
func (service *Service) Read(id portainer.AllowListKey) (*portainer.AllowList, error) {
var result *portainer.AllowList
if err := service.baseService.Connection.ViewTx(func(tx portainer.Transaction) error {
var err error
result, err = service.Tx(tx).Read(id)
return err
}); err != nil {
return nil, err
}
return result, nil
}
func (service *Service) ReadAll() ([]portainer.AllowList, error) {
var result []portainer.AllowList
if err := service.baseService.Connection.ViewTx(func(tx portainer.Transaction) error {
var err error
result, err = service.Tx(tx).ReadAll()
return err
}); err != nil {
return nil, err
}
return result, nil
}
func (service *Service) ReadParsed(id portainer.AllowListKey) (*portainer.ParsedAllowList, error) {
allowListAny, ok := service.cache.Get(id)
if ok {
allowList, ok := allowListAny.(*portainer.ParsedAllowList)
if !ok {
return nil, fmt.Errorf("expected ParsedAllowList in cache but got %T", allowListAny)
}
return allowList, nil
}
var result *portainer.ParsedAllowList
err := service.baseService.Connection.ViewTx(func(tx portainer.Transaction) error {
var err error
result, err = service.Tx(tx).ReadParsed(id)
return err
})
return result, err
}
func (service *Service) Update(id portainer.AllowListKey, allowList *portainer.AllowList) error {
return service.baseService.Connection.UpdateTx(func(tx portainer.Transaction) error {
return service.Tx(tx).Update(id, allowList)
})
}

View File

@@ -1,89 +0,0 @@
package allowlist_test
import (
"net"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/datastore"
"github.com/stretchr/testify/require"
)
func TestAllowListReadEmpty(t *testing.T) {
t.Parallel()
_, ds := datastore.MustNewTestStore(t, false, false)
got, err := ds.AllowList().Read(portainer.AllowListSSRF)
expected := &portainer.AllowList{
ID: portainer.AllowListSSRF,
Mode: portainer.SSRFModeOff,
Entries: []string{},
}
require.NoError(t, err)
require.Equal(t, expected, got)
}
func TestAllowListUpdate(t *testing.T) {
t.Parallel()
_, ds := datastore.MustNewTestStore(t, false, false)
expected := &portainer.AllowList{
ID: portainer.AllowListSSRF,
Mode: portainer.SSRFModeEnforce,
Entries: []string{"example.com", "10.0.0.0/8"},
}
require.NoError(t, ds.AllowList().Update(portainer.AllowListSSRF, expected))
got, err := ds.AllowList().Read(portainer.AllowListSSRF)
require.NoError(t, err)
require.Equal(t, expected, got)
}
func TestAllowListReadAllEmpty(t *testing.T) {
t.Parallel()
_, ds := datastore.MustNewTestStore(t, false, false)
got, err := ds.AllowList().ReadAll()
require.NoError(t, err)
require.Equal(t, []portainer.AllowList{}, got)
}
func TestAllowListReadAllAfterUpdate(t *testing.T) {
t.Parallel()
_, ds := datastore.MustNewTestStore(t, false, false)
expected := portainer.AllowList{
ID: portainer.AllowListSSRF,
Mode: portainer.SSRFModeEnforce,
Entries: []string{"example.com", "10.0.0.0/8"},
}
require.NoError(t, ds.AllowList().Update(portainer.AllowListSSRF, &expected))
got, err := ds.AllowList().ReadAll()
require.NoError(t, err)
require.Equal(t, []portainer.AllowList{expected}, got)
}
func TestAllowListReadParsedAfterUpdate(t *testing.T) {
t.Parallel()
_, ds := datastore.MustNewTestStore(t, false, false)
require.NoError(t, ds.AllowList().Update(portainer.AllowListSSRF, &portainer.AllowList{
ID: portainer.AllowListSSRF,
Mode: portainer.SSRFModeEnforce,
Entries: []string{"example.com"},
}))
expected := &portainer.ParsedAllowList{
Mode: portainer.SSRFModeEnforce,
Nets: []*net.IPNet{},
Hosts: map[string]bool{
"example.com": true,
},
}
got, err := ds.AllowList().ReadParsed(portainer.AllowListSSRF)
require.NoError(t, err)
require.Equal(t, expected, got)
}

View File

@@ -1,77 +0,0 @@
package allowlist
import (
"fmt"
lru "github.com/hashicorp/golang-lru"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/pkg/libhttp/ssrf"
)
type ServiceTx struct {
baseService dataservices.BaseDataServiceTx[portainer.AllowList, portainer.AllowListKey]
cache *lru.Cache
}
func (service *ServiceTx) BucketName() string {
return service.baseService.BucketName()
}
func (service *ServiceTx) ReadParsed(id portainer.AllowListKey) (*portainer.ParsedAllowList, error) {
allowListAny, ok := service.cache.Get(id)
if ok {
allowList, ok := allowListAny.(*portainer.ParsedAllowList)
if !ok {
return nil, fmt.Errorf("expected ParsedAllowList in cache but got %T", allowListAny)
}
return allowList, nil
}
allowList, err := service.Read(id)
if err != nil {
return nil, err
}
parsed := ssrf.ParseAllowedHosts(allowList.Entries)
parsed.Mode = allowList.Mode
service.cache.Add(id, &parsed)
return &parsed, nil
}
func (service *ServiceTx) Read(id portainer.AllowListKey) (*portainer.AllowList, error) {
allowList, err := service.baseService.Read(id)
if dataservices.IsErrObjectNotFound(err) {
allowList = &portainer.AllowList{
ID: id,
Mode: portainer.SSRFModeOff,
Entries: []string{},
}
} else if err != nil {
return nil, err
}
return allowList, nil
}
func (service *ServiceTx) ReadAll() ([]portainer.AllowList, error) {
allowLists, err := service.baseService.ReadAll()
if err != nil && !dataservices.IsErrObjectNotFound(err) {
return nil, err
}
return allowLists, nil
}
func (service *ServiceTx) Update(id portainer.AllowListKey, allowList *portainer.AllowList) error {
if err := service.baseService.Update(id, allowList); err != nil {
return err
}
parsed := ssrf.ParseAllowedHosts(allowList.Entries)
parsed.Mode = allowList.Mode
service.cache.Add(id, &parsed)
return nil
}

View File

@@ -1,92 +0,0 @@
package allowlist_test
import (
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/datastore"
"github.com/stretchr/testify/require"
)
func TestAllowListReadTx(t *testing.T) {
t.Parallel()
_, ds := datastore.MustNewTestStore(t, false, false)
var got *portainer.AllowList
require.NoError(t, ds.ViewTx(func(tx dataservices.DataStoreTx) error {
var err error
got, err = tx.AllowList().Read(portainer.AllowListSSRF)
return err
}))
expected := &portainer.AllowList{
ID: portainer.AllowListSSRF,
Mode: portainer.SSRFModeOff,
Entries: []string{},
}
require.Equal(t, expected, got)
}
func TestAllowListReadAllEmptyTx(t *testing.T) {
t.Parallel()
_, ds := datastore.MustNewTestStore(t, false, false)
var got []portainer.AllowList
require.NoError(t, ds.ViewTx(func(tx dataservices.DataStoreTx) error {
var err error
got, err = tx.AllowList().ReadAll()
return err
}))
require.Equal(t, []portainer.AllowList{}, got)
}
func TestAllowListReadAllAfterUpdateTx(t *testing.T) {
t.Parallel()
_, ds := datastore.MustNewTestStore(t, false, false)
expected := portainer.AllowList{
ID: portainer.AllowListSSRF,
Mode: portainer.SSRFModeEnforce,
Entries: []string{"example.com"},
}
require.NoError(t, ds.UpdateTx(func(tx dataservices.DataStoreTx) error {
return tx.AllowList().Update(portainer.AllowListSSRF, &expected)
}))
var got []portainer.AllowList
require.NoError(t, ds.ViewTx(func(tx dataservices.DataStoreTx) error {
var err error
got, err = tx.AllowList().ReadAll()
return err
}))
require.Equal(t, []portainer.AllowList{expected}, got)
}
func TestAllowListUpdateTx(t *testing.T) {
t.Parallel()
_, ds := datastore.MustNewTestStore(t, false, false)
expected := &portainer.AllowList{
ID: portainer.AllowListSSRF,
Mode: portainer.SSRFModeEnforce,
Entries: []string{"example.com"},
}
require.NoError(t, ds.UpdateTx(func(tx dataservices.DataStoreTx) error {
return tx.AllowList().Update(portainer.AllowListSSRF, expected)
}))
var got *portainer.AllowList
require.NoError(t, ds.ViewTx(func(tx dataservices.DataStoreTx) error {
var err error
got, err = tx.AllowList().Read(portainer.AllowListSSRF)
return err
}))
require.Equal(t, expected, got)
}

View File

@@ -2,10 +2,13 @@ package apikeyrepository
import (
"errors"
"fmt"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
dserrors "github.com/portainer/portainer/api/dataservices/errors"
"github.com/rs/zerolog/log"
)
// BucketName represents the name of the bucket where this service stores data.
@@ -37,10 +40,19 @@ func (service *Service) GetAPIKeysByUserID(userID portainer.UserID) ([]portainer
err := service.Connection.GetAll(
BucketName,
&portainer.APIKey{},
dataservices.FilterFn(&result, func(record portainer.APIKey) bool {
return record.UserID == userID
}),
)
func(obj any) (any, error) {
record, ok := obj.(*portainer.APIKey)
if !ok {
log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to APIKey object")
return nil, fmt.Errorf("failed to convert to APIKey object: %s", obj)
}
if record.UserID == userID {
result = append(result, *record)
}
return &portainer.APIKey{}, nil
})
return result, err
}
@@ -48,18 +60,27 @@ func (service *Service) GetAPIKeysByUserID(userID portainer.UserID) ([]portainer
// GetAPIKeyByDigest returns the API key for the associated digest.
// Note: there is a 1-to-1 mapping of api-key and digest
func (service *Service) GetAPIKeyByDigest(digest string) (*portainer.APIKey, error) {
var found portainer.APIKey
var k *portainer.APIKey
stop := errors.New("ok")
err := service.Connection.GetAll(
BucketName,
&portainer.APIKey{},
dataservices.FirstFn(&found, func(key portainer.APIKey) bool {
return key.Digest == digest
}),
)
func(obj any) (any, error) {
key, ok := obj.(*portainer.APIKey)
if !ok {
log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to APIKey object")
return nil, fmt.Errorf("failed to convert to APIKey object: %s", obj)
}
if key.Digest == digest {
k = key
return nil, stop
}
if errors.Is(err, dataservices.ErrStop) {
return &found, nil
return &portainer.APIKey{}, nil
})
if errors.Is(err, stop) {
return k, nil
}
if err == nil {

View File

@@ -1,8 +1,11 @@
package edgestack
import (
"fmt"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/rs/zerolog/log"
)
type ServiceTx struct {
@@ -21,8 +24,17 @@ func (service ServiceTx) EdgeStacks() ([]portainer.EdgeStack, error) {
err := service.tx.GetAll(
BucketName,
&portainer.EdgeStack{},
dataservices.AppendFn(&stacks),
)
func(obj any) (any, error) {
stack, ok := obj.(*portainer.EdgeStack)
if !ok {
log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EdgeStack object")
return nil, fmt.Errorf("failed to convert to EdgeStack object: %s", obj)
}
stacks = append(stacks, *stack)
return &portainer.EdgeStack{}, nil
})
return stacks, err
}

View File

@@ -1,8 +1,6 @@
package edgestackstatus
import (
"encoding/binary"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
)
@@ -87,9 +85,5 @@ func (s *Service) Clear(edgeStackID portainer.EdgeStackID, relatedEnvironmentsID
}
func (s *Service) key(edgeStackID portainer.EdgeStackID, endpointID portainer.EndpointID) []byte {
k := make([]byte, 16)
binary.BigEndian.PutUint64(k[:8], uint64(edgeStackID))
binary.BigEndian.PutUint64(k[8:], uint64(endpointID))
return k
return append(s.conn.ConvertToKey(int(edgeStackID)), s.conn.ConvertToKey(int(endpointID))...)
}

View File

@@ -27,10 +27,7 @@ func AppendFn[T any](collection *[]T) func(obj any) (any, error) {
*collection = append(*collection, *element)
var zero T
*element = zero
return element, nil
return new(T), nil
}
}
@@ -47,10 +44,7 @@ func FilterFn[T any](collection *[]T, predicate func(T) bool) func(obj any) (any
*collection = append(*collection, *element)
}
var zero T
*element = zero
return element, nil
return new(T), nil
}
}
@@ -66,12 +60,9 @@ func FirstFn[T any](element *T, predicate func(T) bool) func(obj any) (any, erro
if predicate(*e) {
*element = *e
return e, ErrStop
return new(T), ErrStop
}
var zero T
*e = zero
return e, nil
return new(T), nil
}
}

View File

@@ -8,7 +8,6 @@ import (
type (
DataStoreTx interface {
IsErrObjectNotFound(err error) bool
AllowList() AllowListService
CustomTemplate() CustomTemplateService
EdgeGroup() EdgeGroupService
EdgeJob() EdgeJobService
@@ -25,7 +24,6 @@ type (
Settings() SettingsService
Snapshot() SnapshotService
SSLSettings() SSLSettingsService
Source() SourceService
Stack() StackService
Tag() TagService
TeamMembership() TeamMembershipService
@@ -34,7 +32,6 @@ type (
User() UserService
Version() VersionService
Webhook() WebhookService
Workflow() WorkflowService
PendingActions() PendingActionsService
}
@@ -54,15 +51,6 @@ type (
DataStoreTx
}
// AllowListService represents a service for managing the URL allow list
AllowListService interface {
Read(id portainer.AllowListKey) (*portainer.AllowList, error)
ReadAll() ([]portainer.AllowList, error)
ReadParsed(id portainer.AllowListKey) (*portainer.ParsedAllowList, error)
Update(id portainer.AllowListKey, allowList *portainer.AllowList) error
BucketName() string
}
// CustomTemplateService represents a service to manage custom templates
CustomTemplateService interface {
BaseCRUD[portainer.CustomTemplate, portainer.CustomTemplateID]
@@ -195,11 +183,6 @@ type (
BucketName() string
}
// SourceService represents a service for managing GitOps source data
SourceService interface {
BaseCRUD[portainer.Source, portainer.SourceID]
}
// StackService represents a service for managing stack data
StackService interface {
BaseCRUD[portainer.Stack, portainer.StackID]
@@ -262,9 +245,4 @@ type (
WebhookByResourceID(resourceID string) (*portainer.Webhook, error)
WebhookByToken(token string) (*portainer.Webhook, error)
}
// WorkflowService represents a service for managing GitOps workflow data
WorkflowService interface {
BaseCRUD[portainer.Workflow, portainer.WorkflowID]
}
)

View File

@@ -2,10 +2,13 @@ package resourcecontrol
import (
"errors"
"fmt"
"slices"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/rs/zerolog/log"
)
// BucketName represents the name of the bucket where this service stores data.
@@ -45,26 +48,35 @@ func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
// to the main ResourceID or in SubResourceIDs. It also performs a check on the resource type. Return nil
// if no ResourceControl was found.
func (service *Service) ResourceControlByResourceIDAndType(resourceID string, resourceType portainer.ResourceControlType) (*portainer.ResourceControl, error) {
var found portainer.ResourceControl
var resourceControl *portainer.ResourceControl
stop := errors.New("ok")
err := service.Connection.GetAll(
BucketName,
&portainer.ResourceControl{},
dataservices.FirstFn(&found, func(rc portainer.ResourceControl) bool {
return (rc.ResourceID == resourceID && rc.Type == resourceType) ||
slices.Contains(rc.SubResourceIDs, resourceID)
}),
)
func(obj any) (any, error) {
rc, ok := obj.(*portainer.ResourceControl)
if !ok {
log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to ResourceControl object")
return nil, fmt.Errorf("failed to convert to ResourceControl object: %s", obj)
}
if errors.Is(err, dataservices.ErrStop) {
return &found, nil
if rc.ResourceID == resourceID && rc.Type == resourceType {
resourceControl = rc
return nil, stop
}
if slices.Contains(rc.SubResourceIDs, resourceID) {
resourceControl = rc
return nil, stop
}
return &portainer.ResourceControl{}, nil
})
if errors.Is(err, stop) {
return resourceControl, nil
}
if err != nil {
return nil, err
}
return nil, nil
return nil, err
}
// CreateResourceControl creates a new ResourceControl object

View File

@@ -2,10 +2,13 @@ package resourcecontrol
import (
"errors"
"fmt"
"slices"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/rs/zerolog/log"
)
type ServiceTx struct {
@@ -16,26 +19,35 @@ type ServiceTx struct {
// to the main ResourceID or in SubResourceIDs. It also performs a check on the resource type. Return nil
// if no ResourceControl was found.
func (service ServiceTx) ResourceControlByResourceIDAndType(resourceID string, resourceType portainer.ResourceControlType) (*portainer.ResourceControl, error) {
var found portainer.ResourceControl
var resourceControl *portainer.ResourceControl
stop := errors.New("ok")
err := service.Tx.GetAll(
BucketName,
&portainer.ResourceControl{},
dataservices.FirstFn(&found, func(rc portainer.ResourceControl) bool {
return (rc.ResourceID == resourceID && rc.Type == resourceType) ||
slices.Contains(rc.SubResourceIDs, resourceID)
}),
)
func(obj any) (any, error) {
rc, ok := obj.(*portainer.ResourceControl)
if !ok {
log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to ResourceControl object")
return nil, fmt.Errorf("failed to convert to ResourceControl object: %s", obj)
}
if errors.Is(err, dataservices.ErrStop) {
return &found, nil
if rc.ResourceID == resourceID && rc.Type == resourceType {
resourceControl = rc
return nil, stop
}
if slices.Contains(rc.SubResourceIDs, resourceID) {
resourceControl = rc
return nil, stop
}
return &portainer.ResourceControl{}, nil
})
if errors.Is(err, stop) {
return resourceControl, nil
}
if err != nil {
return nil, err
}
return nil, nil
return nil, err
}
// CreateResourceControl creates a new ResourceControl object

View File

@@ -1,50 +0,0 @@
package source
import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
)
// BucketName represents the name of the bucket where this service stores data.
const BucketName = "sources"
// Service represents a service for managing GitOps source data.
type Service struct {
dataservices.BaseDataService[portainer.Source, portainer.SourceID]
}
// NewService creates a new instance of a service.
func NewService(connection portainer.Connection) (*Service, error) {
err := connection.SetServiceName(BucketName)
if err != nil {
return nil, err
}
return &Service{
BaseDataService: dataservices.BaseDataService[portainer.Source, portainer.SourceID]{
Bucket: BucketName,
Connection: connection,
},
}, nil
}
func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
return ServiceTx{
BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.Source, portainer.SourceID]{
Bucket: BucketName,
Connection: service.Connection,
Tx: tx,
},
}
}
// Create creates a new source.
func (service *Service) Create(source *portainer.Source) error {
return service.Connection.CreateObject(
BucketName,
func(id uint64) (int, any) {
source.ID = portainer.SourceID(id)
return int(source.ID), source
},
)
}

View File

@@ -1,21 +0,0 @@
package source
import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
)
type ServiceTx struct {
dataservices.BaseDataServiceTx[portainer.Source, portainer.SourceID]
}
// Create creates a new source.
func (service ServiceTx) Create(source *portainer.Source) error {
return service.Tx.CreateObject(
BucketName,
func(id uint64) (int, any) {
source.ID = portainer.SourceID(id)
return int(source.ID), source
},
)
}

View File

@@ -7,8 +7,6 @@ import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
dserrors "github.com/portainer/portainer/api/dataservices/errors"
"github.com/rs/zerolog/log"
)
// BucketName represents the name of the bucket where this service stores data.
@@ -83,21 +81,9 @@ func (service *Service) GetNextIdentifier() int {
// CreateStack creates a new stack.
func (service *Service) Create(stack *portainer.Stack) error {
if stack.GitConfig != nil {
log.Warn().Int("stackID", int(stack.ID)).Str("url", stack.GitConfig.URL).Msg("stack persisted with non-nil GitConfig; GitConfig is deprecated, use WorkflowID/Source instead")
}
return service.Connection.CreateObjectWithId(BucketName, int(stack.ID), stack)
}
func (service *Service) Update(ID portainer.StackID, stack *portainer.Stack) error {
if stack.GitConfig != nil {
log.Warn().Int("stackID", int(ID)).Str("url", stack.GitConfig.URL).Msg("stack persisted with non-nil GitConfig; GitConfig is deprecated, use WorkflowID/Source instead")
}
return service.BaseDataService.Update(ID, stack)
}
// StackByWebhookID returns a pointer to a stack object by webhook ID.
// It returns nil, errors.ErrObjectNotFound if there's no stack associated with the webhook ID.
func (service *Service) StackByWebhookID(id string) (*portainer.Stack, error) {
@@ -130,7 +116,7 @@ func (service *Service) RefreshableStacks() ([]portainer.Stack, error) {
BucketName,
&portainer.Stack{},
dataservices.FilterFn(&stacks, func(e portainer.Stack) bool {
return e.WorkflowID != 0 && e.AutoUpdate != nil && e.AutoUpdate.Interval != ""
return e.AutoUpdate != nil && e.AutoUpdate.Interval != ""
}),
)
}

View File

@@ -93,15 +93,14 @@ func Test_RefreshableStacks(t *testing.T) {
staticStack := portainer.Stack{ID: 1}
stackWithWebhook := portainer.Stack{ID: 2, AutoUpdate: &portainer.AutoUpdateSettings{Webhook: "webhook"}}
intervalNoWorkflow := portainer.Stack{ID: 3, AutoUpdate: &portainer.AutoUpdateSettings{Interval: "1m"}}
refreshableStack := portainer.Stack{ID: 4, WorkflowID: 1, AutoUpdate: &portainer.AutoUpdateSettings{Interval: "1m"}}
refreshableStack := portainer.Stack{ID: 3, AutoUpdate: &portainer.AutoUpdateSettings{Interval: "1m"}}
for _, stack := range []*portainer.Stack{&staticStack, &stackWithWebhook, &intervalNoWorkflow, &refreshableStack} {
for _, stack := range []*portainer.Stack{&staticStack, &stackWithWebhook, &refreshableStack} {
err := store.Stack().Create(stack)
require.NoError(t, err)
}
stacks, err := store.Stack().RefreshableStacks()
require.NoError(t, err)
require.ElementsMatch(t, []portainer.Stack{refreshableStack}, stacks)
assert.ElementsMatch(t, []portainer.Stack{refreshableStack}, stacks)
}

View File

@@ -7,8 +7,6 @@ import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
dserrors "github.com/portainer/portainer/api/dataservices/errors"
"github.com/rs/zerolog/log"
)
type ServiceTx struct {
@@ -58,21 +56,9 @@ func (service ServiceTx) GetNextIdentifier() int {
// CreateStack creates a new stack.
func (service ServiceTx) Create(stack *portainer.Stack) error {
if stack.GitConfig != nil {
log.Warn().Int("stackID", int(stack.ID)).Str("url", stack.GitConfig.URL).Msg("stack persisted with non-nil GitConfig; GitConfig is deprecated, use WorkflowID/Source instead")
}
return service.Tx.CreateObjectWithId(BucketName, int(stack.ID), stack)
}
func (service ServiceTx) Update(ID portainer.StackID, stack *portainer.Stack) error {
if stack.GitConfig != nil {
log.Warn().Int("stackID", int(ID)).Str("url", stack.GitConfig.URL).Msg("stack persisted with non-nil GitConfig; GitConfig is deprecated, use WorkflowID/Source instead")
}
return service.BaseDataServiceTx.Update(ID, stack)
}
// StackByWebhookID returns a pointer to a stack object by webhook ID.
// It returns nil, errors.ErrObjectNotFound if there's no stack associated with the webhook ID.
func (service ServiceTx) StackByWebhookID(id string) (*portainer.Stack, error) {
@@ -106,7 +92,7 @@ func (service ServiceTx) RefreshableStacks() ([]portainer.Stack, error) {
BucketName,
&portainer.Stack{},
dataservices.FilterFn(&stacks, func(e portainer.Stack) bool {
return e.WorkflowID != 0 && e.AutoUpdate != nil && e.AutoUpdate.Interval != ""
return e.AutoUpdate != nil && e.AutoUpdate.Interval != ""
}),
)
}

View File

@@ -1,46 +0,0 @@
package workflow
import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
)
const BucketName = "workflows"
type Service struct {
dataservices.BaseDataService[portainer.Workflow, portainer.WorkflowID]
}
func NewService(connection portainer.Connection) (*Service, error) {
err := connection.SetServiceName(BucketName)
if err != nil {
return nil, err
}
return &Service{
BaseDataService: dataservices.BaseDataService[portainer.Workflow, portainer.WorkflowID]{
Bucket: BucketName,
Connection: connection,
},
}, nil
}
func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
return ServiceTx{
BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.Workflow, portainer.WorkflowID]{
Bucket: BucketName,
Connection: service.Connection,
Tx: tx,
},
}
}
func (service *Service) Create(workflow *portainer.Workflow) error {
return service.Connection.CreateObject(
BucketName,
func(id uint64) (int, any) {
workflow.ID = portainer.WorkflowID(id)
return int(workflow.ID), workflow
},
)
}

View File

@@ -1,20 +0,0 @@
package workflow
import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
)
type ServiceTx struct {
dataservices.BaseDataServiceTx[portainer.Workflow, portainer.WorkflowID]
}
func (service ServiceTx) Create(workflow *portainer.Workflow) error {
return service.Tx.CreateObject(
BucketName,
func(id uint64) (int, any) {
workflow.ID = portainer.WorkflowID(id)
return int(workflow.ID), workflow
},
)
}

View File

@@ -130,8 +130,7 @@ func TestBackupDBFileUsesCorrectPath(t *testing.T) {
_, store := MustNewTestStore(t, true, false)
t.Run("backs up unencrypted db when encrypted flag is false", func(t *testing.T) {
err := store.connection.SetEncrypted(false)
require.NoError(t, err)
store.connection.SetEncrypted(false)
backupFilename, err := store.backupDBFile("")
require.NoError(t, err)

View File

@@ -35,9 +35,7 @@ func (store *Store) Open() (newStore bool, err error) {
// NeedsEncryptionMigration() sets encrypted=true as a side effect when a key exists.
// We need to set it back to false so GetDatabaseFilePath() returns the path to the
// actual unencrypted file (portainer.db) that we want to back up.
if err := store.connection.SetEncrypted(false); err != nil {
return false, err
}
store.connection.SetEncrypted(false)
// Use backupDBFile directly since connection isn't open yet
// and we don't want to trigger the close/open cycle of Backup()
@@ -126,10 +124,7 @@ func (store *Store) Rollback(force bool) error {
}
func (store *Store) encryptDB() error {
if err := store.connection.SetEncrypted(false); err != nil {
return err
}
store.connection.SetEncrypted(false)
if err := store.connection.Open(); err != nil {
return err
}

View File

@@ -72,16 +72,12 @@ func newEndpoint(endpointType portainer.EndpointType, id portainer.EndpointID, n
TLSConfig: portainer.TLSConfiguration{
TLS: false,
},
TagIDs: nil,
Status: portainer.EndpointStatusUp,
Snapshots: nil,
Kubernetes: portainer.KubernetesData{
Configuration: portainer.KubernetesConfiguration{
UseLoadBalancer: false,
UseServerMetrics: false,
EnableResourceOverCommit: true,
},
},
UserAccessPolicies: portainer.UserAccessPolicies{},
TeamAccessPolicies: portainer.TeamAccessPolicies{},
TagIDs: []portainer.TagID{},
Status: portainer.EndpointStatusUp,
Snapshots: []portainer.DockerSnapshot{},
Kubernetes: portainer.KubernetesDefault(),
}
if TLS {

View File

@@ -59,7 +59,6 @@ func (store *Store) checkOrCreateDefaultSettings() error {
KubectlShellImage: *store.flags.KubectlShellImage,
IsDockerDesktopExtension: isDDExtention,
EnforceEdgeID: true,
}
return store.SettingsService.UpdateSettings(defaultSettings)

View File

@@ -88,9 +88,6 @@ func (store *Store) newMigratorParameters(version *models.Version, flags *portai
EdgeGroupService: store.EdgeGroupService,
TunnelServerService: store.TunnelServerService,
PendingActionsService: store.PendingActionsService,
CustomTemplateService: store.CustomTemplateService,
SourceService: store.SourceService,
WorkflowService: store.WorkflowService,
}
}

View File

@@ -1,250 +0,0 @@
package migrator
import (
"fmt"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices/stack"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/rs/zerolog/log"
)
type legacyRepoConfig struct {
URL string
ReferenceName string
ConfigFilePath string
Authentication *legacyGitAuthentication
ConfigHash string
TLSSkipVerify bool
}
type legacyGitAuthentication struct {
Username string
Password string
Provider int `json:",omitempty"`
AuthorizationType int `json:",omitempty"`
}
func (lrc *legacyRepoConfig) toRepoConfig() *gittypes.RepoConfig {
if lrc == nil {
return nil
}
cfg := &gittypes.RepoConfig{
URL: lrc.URL,
ReferenceName: lrc.ReferenceName,
ConfigFilePath: lrc.ConfigFilePath,
ConfigHash: lrc.ConfigHash,
TLSSkipVerify: lrc.TLSSkipVerify,
}
if lrc.Authentication != nil {
cfg.Authentication = &gittypes.GitAuthentication{
Username: lrc.Authentication.Username,
Password: lrc.Authentication.Password,
Provider: gittypes.GitProvider(lrc.Authentication.Provider),
AuthorizationType: gittypes.GitCredentialAuthType(lrc.Authentication.AuthorizationType),
}
}
return cfg
}
type legacyStack struct {
ID int `json:"Id"`
GitConfig *legacyRepoConfig `json:"GitConfig"`
WorkflowID *int
}
// sourceDedupeKey is the identity used to detect duplicate Sources during migration.
// Two stacks sharing the same URL and credentials must reuse the same Source record.
type sourceDedupeKey struct {
url string
username string
password string
}
func gitSourceKey(cfg *gittypes.RepoConfig) sourceDedupeKey {
key := sourceDedupeKey{url: cfg.URL}
if cfg.Authentication != nil {
key.username = cfg.Authentication.Username
key.password = cfg.Authentication.Password
}
return key
}
func (m *Migrator) migrateGitConfigToSources_2_43_0() error {
log.Info().Msg("migrating git-backed stacks to Source+Workflow records")
var legacyStacks []legacyStack
err := m.stackService.Connection.GetAll(
stack.BucketName,
new(legacyStack),
func(obj any) (any, error) {
s, ok := obj.(*legacyStack)
if !ok {
return nil, fmt.Errorf("unexpected type reading stack bucket: %T", obj)
}
legacyStacks = append(legacyStacks, *s)
return new(legacyStack), nil
},
)
if err != nil {
return err
}
existingSources, err := m.sourceService.ReadAll()
if err != nil {
return err
}
sourcesByKey := make(map[sourceDedupeKey]portainer.SourceID, len(existingSources))
for _, src := range existingSources {
if src.Git != nil {
sourcesByKey[gitSourceKey(src.Git)] = src.ID
}
}
for _, ls := range legacyStacks {
if ls.GitConfig == nil || (ls.WorkflowID != nil && *ls.WorkflowID != 0) {
continue
}
cfg := ls.GitConfig.toRepoConfig()
cfg.URL = gittypes.SanitizeURL(cfg.URL)
key := gitSourceKey(cfg)
var newSrcID portainer.SourceID
if err := m.stackService.Connection.UpdateTx(func(tx portainer.Transaction) error {
srcID, exists := sourcesByKey[key]
if !exists {
src := &portainer.Source{
Name: gittypes.RepoName(cfg.URL),
Type: portainer.SourceTypeGit,
Git: cfg,
}
if err := m.sourceService.Tx(tx).Create(src); err != nil {
return fmt.Errorf("failed to create source for stack %d: %w", ls.ID, err)
}
srcID = src.ID
newSrcID = src.ID
}
liveStack, err := m.stackService.Tx(tx).Read(portainer.StackID(ls.ID))
if err != nil {
return fmt.Errorf("failed to read stack %d: %w", ls.ID, err)
}
wf := &portainer.Workflow{
Name: liveStack.Name,
Artifacts: []portainer.Artifact{{
StackID: portainer.StackID(ls.ID),
Files: []portainer.ArtifactFile{{
SourceID: srcID,
Path: cfg.ConfigFilePath,
Ref: cfg.ReferenceName,
Hash: cfg.ConfigHash,
}},
}},
}
if err := m.workflowService.Tx(tx).Create(wf); err != nil {
return fmt.Errorf("failed to create workflow for stack %d: %w", ls.ID, err)
}
liveStack.WorkflowID = wf.ID
liveStack.GitConfig = nil
return m.stackService.Tx(tx).Update(portainer.StackID(ls.ID), liveStack)
}); err != nil {
return fmt.Errorf("failed to migrate stack %d: %w", ls.ID, err)
}
if newSrcID != 0 {
sourcesByKey[key] = newSrcID
}
}
return nil
}
func (m *Migrator) migrateCustomTemplateGitConfigToSources_2_43_0() error {
log.Info().Msg("migrating git-backed custom templates to Source records")
templates, err := m.customTemplateService.ReadAll()
if err != nil {
return err
}
existingSources, err := m.sourceService.ReadAll()
if err != nil {
return err
}
sourcesByKey := make(map[sourceDedupeKey]portainer.SourceID, len(existingSources))
for _, src := range existingSources {
if src.Git != nil {
sourcesByKey[gitSourceKey(src.Git)] = src.ID
}
}
for i := range templates {
t := &templates[i]
if t.GitConfig == nil || t.Artifact != nil {
continue
}
cfg := &gittypes.RepoConfig{
URL: gittypes.SanitizeURL(t.GitConfig.URL),
Authentication: t.GitConfig.Authentication,
TLSSkipVerify: t.GitConfig.TLSSkipVerify,
}
key := gitSourceKey(cfg)
var newSrcID portainer.SourceID
if err := m.stackService.Connection.UpdateTx(func(tx portainer.Transaction) error {
srcID, exists := sourcesByKey[key]
if !exists {
src := &portainer.Source{
Name: gittypes.RepoName(cfg.URL),
Type: portainer.SourceTypeGit,
Git: cfg,
}
if err := m.sourceService.Tx(tx).Create(src); err != nil {
return fmt.Errorf("failed to create source for custom template %d: %w", t.ID, err)
}
srcID = src.ID
newSrcID = src.ID
}
t.Artifact = &portainer.Artifact{
Files: []portainer.ArtifactFile{{
SourceID: srcID,
Path: t.GitConfig.ConfigFilePath,
Ref: t.GitConfig.ReferenceName,
Hash: t.GitConfig.ConfigHash,
}},
}
t.GitConfig = nil
return m.customTemplateService.Tx(tx).Update(t.ID, t)
}); err != nil {
return fmt.Errorf("failed to migrate custom template %d: %w", t.ID, err)
}
if newSrcID != 0 {
sourcesByKey[key] = newSrcID
}
}
return nil
}

View File

@@ -1,462 +0,0 @@
package migrator
import (
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/database/boltdb"
"github.com/portainer/portainer/api/dataservices/customtemplate"
"github.com/portainer/portainer/api/dataservices/source"
"github.com/portainer/portainer/api/dataservices/stack"
"github.com/portainer/portainer/api/dataservices/workflow"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/portainer/portainer/api/logs"
"github.com/stretchr/testify/require"
)
func TestMigrateGitConfigToSources_2_43_0_GitStackMigrated(t *testing.T) {
t.Parallel()
conn := &boltdb.DbConnection{Path: t.TempDir()}
err := conn.Open()
require.NoError(t, err)
defer logs.CloseAndLogErr(conn)
stackSvc, err := stack.NewService(conn)
require.NoError(t, err)
sourceSvc, err := source.NewService(conn)
require.NoError(t, err)
workflowSvc, err := workflow.NewService(conn)
require.NoError(t, err)
m := NewMigrator(&MigratorParameters{
StackService: stackSvc,
SourceService: sourceSvc,
WorkflowService: workflowSvc,
})
gitStack := &portainer.Stack{
ID: 1,
Name: "git-stack",
GitConfig: &gittypes.RepoConfig{
URL: "https://github.com/example/repo",
ReferenceName: "refs/heads/main",
ConfigHash: "abc123",
},
}
err = conn.CreateObjectWithId(stack.BucketName, int(gitStack.ID), gitStack)
require.NoError(t, err)
err = m.migrateGitConfigToSources_2_43_0()
require.NoError(t, err)
migrated, err := stackSvc.Read(gitStack.ID)
require.NoError(t, err)
require.NotZero(t, migrated.WorkflowID)
require.Nil(t, migrated.GitConfig)
wf, err := workflowSvc.Read(migrated.WorkflowID)
require.NoError(t, err)
require.Len(t, wf.Artifacts, 1)
require.Len(t, wf.Artifacts[0].Files, 1)
src, err := sourceSvc.Read(wf.Artifacts[0].Files[0].SourceID)
require.NoError(t, err)
require.Equal(t, portainer.SourceTypeGit, src.Type)
require.Equal(t, gitStack.GitConfig.URL, src.Git.URL)
require.Equal(t, gitStack.GitConfig.ReferenceName, src.Git.ReferenceName)
}
func TestMigrateGitConfigToSources_2_43_0_NonGitStackUntouched(t *testing.T) {
t.Parallel()
conn := &boltdb.DbConnection{Path: t.TempDir()}
err := conn.Open()
require.NoError(t, err)
defer logs.CloseAndLogErr(conn)
stackSvc, err := stack.NewService(conn)
require.NoError(t, err)
sourceSvc, err := source.NewService(conn)
require.NoError(t, err)
workflowSvc, err := workflow.NewService(conn)
require.NoError(t, err)
m := NewMigrator(&MigratorParameters{
StackService: stackSvc,
SourceService: sourceSvc,
WorkflowService: workflowSvc,
})
plainStack := &portainer.Stack{
ID: 1,
Name: "plain-stack",
}
err = conn.CreateObjectWithId(stack.BucketName, int(plainStack.ID), plainStack)
require.NoError(t, err)
err = m.migrateGitConfigToSources_2_43_0()
require.NoError(t, err)
result, err := stackSvc.Read(plainStack.ID)
require.NoError(t, err)
require.Zero(t, result.WorkflowID)
require.Nil(t, result.GitConfig)
sources, err := sourceSvc.ReadAll()
require.NoError(t, err)
require.Empty(t, sources)
workflows, err := workflowSvc.ReadAll()
require.NoError(t, err)
require.Empty(t, workflows)
}
func TestMigrateGitConfigToSources_2_43_0_DuplicateSourcesDeduped(t *testing.T) {
t.Parallel()
conn := &boltdb.DbConnection{Path: t.TempDir()}
err := conn.Open()
require.NoError(t, err)
defer logs.CloseAndLogErr(conn)
stackSvc, err := stack.NewService(conn)
require.NoError(t, err)
sourceSvc, err := source.NewService(conn)
require.NoError(t, err)
workflowSvc, err := workflow.NewService(conn)
require.NoError(t, err)
m := NewMigrator(&MigratorParameters{
StackService: stackSvc,
SourceService: sourceSvc,
WorkflowService: workflowSvc,
})
sharedURL := "https://github.com/example/shared-repo"
stack1 := &portainer.Stack{
ID: 1,
Name: "stack-a",
GitConfig: &gittypes.RepoConfig{
URL: sharedURL,
ReferenceName: "refs/heads/main",
},
}
stack2 := &portainer.Stack{
ID: 2,
Name: "stack-b",
GitConfig: &gittypes.RepoConfig{
URL: sharedURL,
ReferenceName: "refs/heads/develop",
},
}
err = conn.CreateObjectWithId(stack.BucketName, int(stack1.ID), stack1)
require.NoError(t, err)
err = conn.CreateObjectWithId(stack.BucketName, int(stack2.ID), stack2)
require.NoError(t, err)
err = m.migrateGitConfigToSources_2_43_0()
require.NoError(t, err)
sources, err := sourceSvc.ReadAll()
require.NoError(t, err)
require.Len(t, sources, 1, "two stacks with the same URL must share one Source")
workflows, err := workflowSvc.ReadAll()
require.NoError(t, err)
require.Len(t, workflows, 2, "each stack must get its own Workflow")
sharedSourceID := sources[0].ID
for _, wf := range workflows {
require.Len(t, wf.Artifacts, 1)
require.Len(t, wf.Artifacts[0].Files, 1)
require.Equal(t, sharedSourceID, wf.Artifacts[0].Files[0].SourceID)
}
}
func TestMigrateGitConfigToSources_2_43_0_Idempotent(t *testing.T) {
t.Parallel()
conn := &boltdb.DbConnection{Path: t.TempDir()}
err := conn.Open()
require.NoError(t, err)
defer logs.CloseAndLogErr(conn)
stackSvc, err := stack.NewService(conn)
require.NoError(t, err)
sourceSvc, err := source.NewService(conn)
require.NoError(t, err)
workflowSvc, err := workflow.NewService(conn)
require.NoError(t, err)
m := NewMigrator(&MigratorParameters{
StackService: stackSvc,
SourceService: sourceSvc,
WorkflowService: workflowSvc,
})
gitStack := &portainer.Stack{
ID: 1,
Name: "git-stack",
GitConfig: &gittypes.RepoConfig{
URL: "https://github.com/example/repo",
},
}
err = conn.CreateObjectWithId(stack.BucketName, int(gitStack.ID), gitStack)
require.NoError(t, err)
err = m.migrateGitConfigToSources_2_43_0()
require.NoError(t, err)
// Second run must not create duplicate Source/Workflow records
err = m.migrateGitConfigToSources_2_43_0()
require.NoError(t, err)
sources, err := sourceSvc.ReadAll()
require.NoError(t, err)
require.Len(t, sources, 1)
workflows, err := workflowSvc.ReadAll()
require.NoError(t, err)
require.Len(t, workflows, 1)
}
func TestMigrateCustomTemplateGitConfigToSources_2_43_0_GitTemplateMigrated(t *testing.T) {
t.Parallel()
conn := &boltdb.DbConnection{Path: t.TempDir()}
err := conn.Open()
require.NoError(t, err)
defer logs.CloseAndLogErr(conn)
stackSvc, err := stack.NewService(conn)
require.NoError(t, err)
sourceSvc, err := source.NewService(conn)
require.NoError(t, err)
customTemplateSvc, err := customtemplate.NewService(conn)
require.NoError(t, err)
m := NewMigrator(&MigratorParameters{
StackService: stackSvc,
SourceService: sourceSvc,
CustomTemplateService: customTemplateSvc,
})
tmpl := &portainer.CustomTemplate{
ID: 1,
GitConfig: &gittypes.RepoConfig{
URL: "https://github.com/example/repo",
ReferenceName: "refs/heads/main",
ConfigFilePath: "docker-compose.yml",
ConfigHash: "abc123",
},
}
err = conn.CreateObjectWithId(customtemplate.BucketName, int(tmpl.ID), tmpl)
require.NoError(t, err)
err = m.migrateCustomTemplateGitConfigToSources_2_43_0()
require.NoError(t, err)
migrated, err := customTemplateSvc.Read(tmpl.ID)
require.NoError(t, err)
require.NotNil(t, migrated.Artifact)
require.Nil(t, migrated.GitConfig)
require.Len(t, migrated.Artifact.Files, 1)
require.Equal(t, "refs/heads/main", migrated.Artifact.Files[0].Ref)
require.Equal(t, "docker-compose.yml", migrated.Artifact.Files[0].Path)
require.Equal(t, "abc123", migrated.Artifact.Files[0].Hash)
src, err := sourceSvc.Read(migrated.Artifact.Files[0].SourceID)
require.NoError(t, err)
require.Equal(t, portainer.SourceTypeGit, src.Type)
require.Equal(t, "https://github.com/example/repo", src.Git.URL)
}
func TestMigrateCustomTemplateGitConfigToSources_2_43_0_NonGitTemplateUntouched(t *testing.T) {
t.Parallel()
conn := &boltdb.DbConnection{Path: t.TempDir()}
err := conn.Open()
require.NoError(t, err)
defer logs.CloseAndLogErr(conn)
stackSvc, err := stack.NewService(conn)
require.NoError(t, err)
sourceSvc, err := source.NewService(conn)
require.NoError(t, err)
customTemplateSvc, err := customtemplate.NewService(conn)
require.NoError(t, err)
m := NewMigrator(&MigratorParameters{
StackService: stackSvc,
SourceService: sourceSvc,
CustomTemplateService: customTemplateSvc,
})
tmpl := &portainer.CustomTemplate{ID: 1, Title: "plain-template"}
err = conn.CreateObjectWithId(customtemplate.BucketName, int(tmpl.ID), tmpl)
require.NoError(t, err)
err = m.migrateCustomTemplateGitConfigToSources_2_43_0()
require.NoError(t, err)
result, err := customTemplateSvc.Read(tmpl.ID)
require.NoError(t, err)
require.Nil(t, result.Artifact)
require.Nil(t, result.GitConfig)
sources, err := sourceSvc.ReadAll()
require.NoError(t, err)
require.Empty(t, sources)
}
func TestMigrateCustomTemplateGitConfigToSources_2_43_0_AlreadyMigratedSkipped(t *testing.T) {
t.Parallel()
conn := &boltdb.DbConnection{Path: t.TempDir()}
err := conn.Open()
require.NoError(t, err)
defer logs.CloseAndLogErr(conn)
stackSvc, err := stack.NewService(conn)
require.NoError(t, err)
sourceSvc, err := source.NewService(conn)
require.NoError(t, err)
customTemplateSvc, err := customtemplate.NewService(conn)
require.NoError(t, err)
m := NewMigrator(&MigratorParameters{
StackService: stackSvc,
SourceService: sourceSvc,
CustomTemplateService: customTemplateSvc,
})
// Template already has Artifact set (already migrated)
srcID := portainer.SourceID(99)
tmpl := &portainer.CustomTemplate{
ID: 1,
GitConfig: &gittypes.RepoConfig{
URL: "https://github.com/example/repo",
},
Artifact: &portainer.Artifact{
Files: []portainer.ArtifactFile{{SourceID: srcID}},
},
}
err = conn.CreateObjectWithId(customtemplate.BucketName, int(tmpl.ID), tmpl)
require.NoError(t, err)
err = m.migrateCustomTemplateGitConfigToSources_2_43_0()
require.NoError(t, err)
sources, err := sourceSvc.ReadAll()
require.NoError(t, err)
require.Empty(t, sources, "no new sources should be created for already-migrated templates")
}
func TestMigrateCustomTemplateGitConfigToSources_2_43_0_DuplicateSourcesDeduped(t *testing.T) {
t.Parallel()
conn := &boltdb.DbConnection{Path: t.TempDir()}
err := conn.Open()
require.NoError(t, err)
defer logs.CloseAndLogErr(conn)
stackSvc, err := stack.NewService(conn)
require.NoError(t, err)
sourceSvc, err := source.NewService(conn)
require.NoError(t, err)
customTemplateSvc, err := customtemplate.NewService(conn)
require.NoError(t, err)
m := NewMigrator(&MigratorParameters{
StackService: stackSvc,
SourceService: sourceSvc,
CustomTemplateService: customTemplateSvc,
})
sharedURL := "https://github.com/example/shared-repo"
tmpl1 := &portainer.CustomTemplate{
ID: 1,
Title: "template-a",
GitConfig: &gittypes.RepoConfig{
URL: sharedURL,
ReferenceName: "refs/heads/main",
},
}
tmpl2 := &portainer.CustomTemplate{
ID: 2,
Title: "template-b",
GitConfig: &gittypes.RepoConfig{
URL: sharedURL,
ReferenceName: "refs/heads/develop",
},
}
err = conn.CreateObjectWithId(customtemplate.BucketName, int(tmpl1.ID), tmpl1)
require.NoError(t, err)
err = conn.CreateObjectWithId(customtemplate.BucketName, int(tmpl2.ID), tmpl2)
require.NoError(t, err)
err = m.migrateCustomTemplateGitConfigToSources_2_43_0()
require.NoError(t, err)
sources, err := sourceSvc.ReadAll()
require.NoError(t, err)
require.Len(t, sources, 1, "two templates with the same URL must share one Source")
sharedSrcID := sources[0].ID
migrated1, err := customTemplateSvc.Read(tmpl1.ID)
require.NoError(t, err)
require.NotNil(t, migrated1.Artifact)
require.Equal(t, sharedSrcID, migrated1.Artifact.Files[0].SourceID)
migrated2, err := customTemplateSvc.Read(tmpl2.ID)
require.NoError(t, err)
require.NotNil(t, migrated2.Artifact)
require.Equal(t, sharedSrcID, migrated2.Artifact.Files[0].SourceID)
}
func TestMigrateCustomTemplateGitConfigToSources_2_43_0_Idempotent(t *testing.T) {
t.Parallel()
conn := &boltdb.DbConnection{Path: t.TempDir()}
err := conn.Open()
require.NoError(t, err)
defer logs.CloseAndLogErr(conn)
stackSvc, err := stack.NewService(conn)
require.NoError(t, err)
sourceSvc, err := source.NewService(conn)
require.NoError(t, err)
customTemplateSvc, err := customtemplate.NewService(conn)
require.NoError(t, err)
m := NewMigrator(&MigratorParameters{
StackService: stackSvc,
SourceService: sourceSvc,
CustomTemplateService: customTemplateSvc,
})
tmpl := &portainer.CustomTemplate{
ID: 1,
GitConfig: &gittypes.RepoConfig{
URL: "https://github.com/example/repo",
},
}
err = conn.CreateObjectWithId(customtemplate.BucketName, int(tmpl.ID), tmpl)
require.NoError(t, err)
err = m.migrateCustomTemplateGitConfigToSources_2_43_0()
require.NoError(t, err)
// Second run must not create duplicate Source records
err = m.migrateCustomTemplateGitConfigToSources_2_43_0()
require.NoError(t, err)
sources, err := sourceSvc.ReadAll()
require.NoError(t, err)
require.Len(t, sources, 1)
}

View File

@@ -5,7 +5,6 @@ import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/database/models"
"github.com/portainer/portainer/api/dataservices/customtemplate"
"github.com/portainer/portainer/api/dataservices/dockerhub"
"github.com/portainer/portainer/api/dataservices/edgegroup"
"github.com/portainer/portainer/api/dataservices/edgejob"
@@ -22,14 +21,12 @@ import (
"github.com/portainer/portainer/api/dataservices/schedule"
"github.com/portainer/portainer/api/dataservices/settings"
"github.com/portainer/portainer/api/dataservices/snapshot"
"github.com/portainer/portainer/api/dataservices/source"
"github.com/portainer/portainer/api/dataservices/stack"
"github.com/portainer/portainer/api/dataservices/tag"
"github.com/portainer/portainer/api/dataservices/teammembership"
"github.com/portainer/portainer/api/dataservices/tunnelserver"
"github.com/portainer/portainer/api/dataservices/user"
"github.com/portainer/portainer/api/dataservices/version"
"github.com/portainer/portainer/api/dataservices/workflow"
"github.com/portainer/portainer/api/internal/authorization"
"github.com/Masterminds/semver/v3"
@@ -67,9 +64,6 @@ type (
edgeGroupService *edgegroup.Service
TunnelServerService *tunnelserver.Service
pendingActionsService *pendingactions.Service
customTemplateService *customtemplate.Service
sourceService *source.Service
workflowService *workflow.Service
}
// MigratorParameters represents the required parameters to create a new Migrator instance.
@@ -100,9 +94,6 @@ type (
EdgeGroupService *edgegroup.Service
TunnelServerService *tunnelserver.Service
PendingActionsService *pendingactions.Service
CustomTemplateService *customtemplate.Service
SourceService *source.Service
WorkflowService *workflow.Service
}
)
@@ -135,9 +126,6 @@ func NewMigrator(parameters *MigratorParameters) *Migrator {
edgeGroupService: parameters.EdgeGroupService,
TunnelServerService: parameters.TunnelServerService,
pendingActionsService: parameters.PendingActionsService,
customTemplateService: parameters.CustomTemplateService,
sourceService: parameters.SourceService,
workflowService: parameters.WorkflowService,
}
migrator.initMigrations()
@@ -272,11 +260,6 @@ func (m *Migrator) initMigrations() {
m.addMigrations("2.40.0", m.migrateRegistryAccessSASecrets_2_40_0)
m.addMigrations("2.43.0",
m.migrateGitConfigToSources_2_43_0,
m.migrateCustomTemplateGitConfigToSources_2_43_0,
)
// WARNING: do not change migrations that have already been released!
// Add new migrations above...

View File

@@ -7,7 +7,6 @@ import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/database/models"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/dataservices/allowlist"
"github.com/portainer/portainer/api/dataservices/apikeyrepository"
"github.com/portainer/portainer/api/dataservices/customtemplate"
"github.com/portainer/portainer/api/dataservices/dockerhub"
@@ -27,7 +26,6 @@ import (
"github.com/portainer/portainer/api/dataservices/schedule"
"github.com/portainer/portainer/api/dataservices/settings"
"github.com/portainer/portainer/api/dataservices/snapshot"
"github.com/portainer/portainer/api/dataservices/source"
"github.com/portainer/portainer/api/dataservices/ssl"
"github.com/portainer/portainer/api/dataservices/stack"
"github.com/portainer/portainer/api/dataservices/tag"
@@ -37,7 +35,6 @@ import (
"github.com/portainer/portainer/api/dataservices/user"
"github.com/portainer/portainer/api/dataservices/version"
"github.com/portainer/portainer/api/dataservices/webhook"
"github.com/portainer/portainer/api/dataservices/workflow"
"github.com/rs/zerolog/log"
"github.com/segmentio/encoding/json"
@@ -52,7 +49,6 @@ type Store struct {
connection portainer.Connection
fileService portainer.FileService
AllowListService *allowlist.Service
CustomTemplateService *customtemplate.Service
DockerHubService *dockerhub.Service
EdgeGroupService *edgegroup.Service
@@ -71,7 +67,6 @@ type Store struct {
ScheduleService *schedule.Service
SettingsService *settings.Service
SnapshotService *snapshot.Service
SourceService *source.Service
SSLSettingsService *ssl.Service
StackService *stack.Service
TagService *tag.Service
@@ -81,17 +76,10 @@ type Store struct {
UserService *user.Service
VersionService *version.Service
WebhookService *webhook.Service
WorkflowService *workflow.Service
PendingActionsService *pendingactions.Service
}
func (store *Store) initServices() error {
allowListService, err := allowlist.NewService(store.connection)
if err != nil {
return err
}
store.AllowListService = allowListService
authorizationsetService, err := role.NewService(store.connection)
if err != nil {
return err
@@ -191,12 +179,6 @@ func (store *Store) initServices() error {
}
store.SnapshotService = snapshotService
sourceService, err := source.NewService(store.connection)
if err != nil {
return err
}
store.SourceService = sourceService
sslSettingsService, err := ssl.NewService(store.connection)
if err != nil {
return err
@@ -257,12 +239,6 @@ func (store *Store) initServices() error {
}
store.WebhookService = webhookService
workflowService, err := workflow.NewService(store.connection)
if err != nil {
return err
}
store.WorkflowService = workflowService
scheduleService, err := schedule.NewService(store.connection)
if err != nil {
return err
@@ -283,11 +259,6 @@ func (store *Store) PendingActions() dataservices.PendingActionsService {
return store.PendingActionsService
}
// AllowList gives access to the AllowList data management layer
func (store *Store) AllowList() dataservices.AllowListService {
return store.AllowListService
}
// CustomTemplate gives access to the CustomTemplate data management layer
func (store *Store) CustomTemplate() dataservices.CustomTemplateService {
return store.CustomTemplateService
@@ -361,11 +332,6 @@ func (store *Store) Snapshot() dataservices.SnapshotService {
return store.SnapshotService
}
// Source gives access to the Source data management layer
func (store *Store) Source() dataservices.SourceService {
return store.SourceService
}
// SSLSettings gives access to the SSL Settings data management layer
func (store *Store) SSLSettings() dataservices.SSLSettingsService {
return store.SSLSettingsService
@@ -411,11 +377,6 @@ func (store *Store) Webhook() dataservices.WebhookService {
return store.WebhookService
}
// Workflow gives access to the Workflow data management layer
func (store *Store) Workflow() dataservices.WorkflowService {
return store.WorkflowService
}
type storeExport struct {
CustomTemplate []portainer.CustomTemplate `json:"customtemplates,omitempty"`
EdgeGroup []portainer.EdgeGroup `json:"edgegroups,omitempty"`
@@ -433,7 +394,6 @@ type storeExport struct {
Settings portainer.Settings `json:"settings,omitzero"`
Snapshot []portainer.Snapshot `json:"snapshots,omitempty"`
SSLSettings portainer.SSLSettings `json:"ssl,omitzero"`
Source []portainer.Source `json:"sources,omitempty"`
Stack []portainer.Stack `json:"stacks,omitempty"`
Tag []portainer.Tag `json:"tags,omitempty"`
TeamMembership []portainer.TeamMembership `json:"team_membership,omitempty"`
@@ -442,7 +402,6 @@ type storeExport struct {
User []portainer.User `json:"users,omitempty"`
Version models.Version `json:"version,omitzero"`
Webhook []portainer.Webhook `json:"webhooks,omitempty"`
Workflow []portainer.Workflow `json:"workflows,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
@@ -577,14 +536,6 @@ func (store *Store) Export(filename string) (err error) {
backup.SSLSettings = *settings
}
if s, err := store.Source().ReadAll(); err != nil {
if !store.IsErrObjectNotFound(err) {
log.Error().Err(err).Msg("exporting Sources")
}
} else {
backup.Source = s
}
if t, err := store.Stack().ReadAll(); err != nil {
if !store.IsErrObjectNotFound(err) {
log.Error().Err(err).Msg("exporting Stacks")
@@ -641,14 +592,6 @@ func (store *Store) Export(filename string) (err error) {
backup.Webhook = webhooks
}
if w, err := store.Workflow().ReadAll(); err != nil {
if !store.IsErrObjectNotFound(err) {
log.Error().Err(err).Msg("exporting Workflows")
}
} else {
backup.Workflow = w
}
if version, err := store.Version().Version(); err != nil {
if !store.IsErrObjectNotFound(err) {
log.Error().Err(err).Msg("exporting Version")
@@ -667,7 +610,7 @@ func (store *Store) Export(filename string) (err error) {
return err
}
return os.WriteFile(filename, b, 0o600)
return os.WriteFile(filename, b, 0600)
}
func (store *Store) Import(filename string) (err error) {
@@ -767,18 +710,6 @@ func (store *Store) Import(filename string) (err error) {
}
}
for _, v := range backup.Source {
if err := store.Source().Update(v.ID, &v); err != nil {
log.Warn().Err(err).Msg("failed to update the source in the database")
}
}
for _, v := range backup.Workflow {
if err := store.Workflow().Update(v.ID, &v); err != nil {
log.Warn().Err(err).Msg("failed to update the workflow in the database")
}
}
for _, v := range backup.Stack {
if err := store.Stack().Update(v.ID, &v); err != nil {
log.Warn().Err(err).Msg("failed to update the stack in the database")

View File

@@ -14,10 +14,6 @@ func (tx *StoreTx) IsErrObjectNotFound(err error) bool {
return tx.store.IsErrObjectNotFound(err)
}
func (tx *StoreTx) AllowList() dataservices.AllowListService {
return tx.store.AllowListService.Tx(tx.tx)
}
func (tx *StoreTx) CustomTemplate() dataservices.CustomTemplateService {
return tx.store.CustomTemplateService.Tx(tx.tx)
}
@@ -78,10 +74,6 @@ func (tx *StoreTx) Snapshot() dataservices.SnapshotService {
return tx.store.SnapshotService.Tx(tx.tx)
}
func (tx *StoreTx) Source() dataservices.SourceService {
return tx.store.SourceService.Tx(tx.tx)
}
func (tx *StoreTx) SSLSettings() dataservices.SSLSettingsService {
return tx.store.SSLSettingsService.Tx(tx.tx)
}
@@ -110,7 +102,3 @@ func (tx *StoreTx) User() dataservices.UserService {
func (tx *StoreTx) Version() dataservices.VersionService { return nil }
func (tx *StoreTx) Webhook() dataservices.WebhookService { return nil }
func (tx *StoreTx) Workflow() dataservices.WorkflowService {
return tx.store.WorkflowService.Tx(tx.tx)
}

View File

@@ -1,5 +1,4 @@
{
"allowlist": null,
"api_key": null,
"customtemplates": null,
"dockerhub": [
@@ -34,7 +33,11 @@
],
"endpoints": [
{
"Agent": {},
"Agent": {
"Version": ""
},
"AuthorizedTeams": null,
"AuthorizedUsers": null,
"AzureCredentials": {
"ApplicationID": "",
"AuthenticationKey": "",
@@ -50,6 +53,7 @@
},
"EdgeCheckinInterval": 0,
"EdgeKey": "",
"Gpus": [],
"GroupId": 1,
"Heartbeat": false,
"Id": 1,
@@ -58,8 +62,10 @@
"AllowNoneIngressClass": false,
"EnableResourceOverCommit": false,
"IngressAvailabilityPerNamespace": true,
"IngressClasses": null,
"ResourceOverCommitPercentage": 0,
"RestrictDefaultNamespace": false,
"StorageClasses": null,
"UseLoadBalancer": false,
"UseServerMetrics": false
},
@@ -67,7 +73,8 @@
"IsServerIngressClassDetected": false,
"IsServerMetricsDetected": false,
"IsServerStorageDetected": false
}
},
"Snapshots": []
},
"LastCheckInDate": 0,
"Name": "local",
@@ -89,13 +96,18 @@
"allowVolumeBrowserForRegularUsers": false,
"enableHostManagementFeatures": false
},
"Snapshots": [],
"Status": 1,
"TLSConfig": {
"TLS": false,
"TLSSkipVerify": false
},
"TagIds": [],
"Tags": null,
"TeamAccessPolicies": {},
"Type": 1,
"URL": "unix:///var/run/docker.sock"
"URL": "unix:///var/run/docker.sock",
"UserAccessPolicies": {}
}
],
"extension": null,
@@ -595,7 +607,6 @@
"EnableEdgeComputeFeatures": false,
"EnforceEdgeID": false,
"FeatureFlagSettings": null,
"ForceSecureCookies": false,
"GlobalDeploymentOptions": {
"hideStacksFunctionality": false
},
@@ -604,7 +615,7 @@
"RequiredPasswordLength": 12
},
"KubeconfigExpiry": "0",
"KubectlShellImage": "portainer/kubectl-shell:2.43.0",
"KubectlShellImage": "portainer/kubectl-shell:2.41.1",
"LDAPSettings": {
"AnonymousMode": true,
"AutoCreateUsers": true,
@@ -649,7 +660,18 @@
"SnapshotInterval": "5m",
"TemplatesURL": "",
"TrustOnFirstConnect": false,
"UserSessionTimeout": "8h"
"UserSessionTimeout": "8h",
"openAMTConfiguration": {
"certFileContent": "",
"certFileName": "",
"certFilePassword": "",
"domainName": "",
"enabled": false,
"mpsPassword": "",
"mpsServer": "",
"mpsToken": "",
"mpsUser": ""
}
},
"snapshots": [
{
@@ -657,6 +679,8 @@
"ContainerCount": 0,
"DiagnosticsData": {},
"DockerSnapshotRaw": {
"Containers": null,
"Images": null,
"Info": {
"Architecture": "",
"CDISpecDirs": null,
@@ -732,6 +756,7 @@
"SystemTime": "",
"Warnings": null
},
"Networks": null,
"Version": {
"ApiVersion": "",
"Arch": "",
@@ -750,10 +775,12 @@
},
"DockerVersion": "20.10.13",
"GpuUseAll": false,
"GpuUseList": null,
"HealthyContainerCount": 0,
"ImageCount": 9,
"IsPodman": false,
"NodeCount": 0,
"PerformanceMetrics": null,
"RunningContainerCount": 5,
"ServiceCount": 0,
"StackCount": 2,
@@ -769,7 +796,6 @@
"Kubernetes": null
}
],
"sources": null,
"ssl": {
"certPath": "",
"httpEnabled": true,
@@ -921,8 +947,7 @@
}
],
"version": {
"VERSION": "{\"SchemaVersion\":\"2.43.0\",\"MigratorCount\":2,\"Edition\":1,\"InstanceID\":\"463d5c47-0ea5-4aca-85b1-405ceefee254\"}"
"VERSION": "{\"SchemaVersion\":\"2.41.1\",\"MigratorCount\":0,\"Edition\":1,\"InstanceID\":\"463d5c47-0ea5-4aca-85b1-405ceefee254\"}"
},
"webhooks": null,
"workflows": null
"webhooks": null
}

View File

@@ -11,8 +11,6 @@ import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/crypto"
"github.com/portainer/portainer/pkg/libhttp/ssrf"
"github.com/rs/zerolog/log"
"github.com/docker/docker/api/types/image"
@@ -90,7 +88,7 @@ func createTCPClient(endpoint *portainer.Endpoint, timeout *time.Duration) (*cli
client.WithHTTPClient(httpCli),
}
if endpoint.TLSConfig.TLS {
if nnTransport, ok := httpCli.Transport.(*NodeNameTransport); ok && nnTransport.TLSClientConfig != nil {
opts = append(opts, client.WithScheme("https"))
}
@@ -124,7 +122,7 @@ func createAgentClient(endpoint *portainer.Endpoint, endpointURL string, signatu
client.WithHTTPHeaders(headers),
}
if endpoint.TLSConfig.TLS {
if nnTransport, ok := httpCli.Transport.(*NodeNameTransport); ok && nnTransport.TLSClientConfig != nil {
opts = append(opts, client.WithScheme("https"))
}
@@ -186,20 +184,17 @@ func (t *NodeNameTransport) RoundTrip(req *http.Request) (*http.Response, error)
}
func httpClient(endpoint *portainer.Endpoint, timeout *time.Duration) (*http.Client, error) {
var transport *NodeNameTransport
transport := &NodeNameTransport{
Transport: &http.Transport{},
}
if endpoint.TLSConfig.TLS {
tlsConfig, err := crypto.CreateTLSConfigurationFromDisk(endpoint.TLSConfig)
if err != nil {
return nil, err
}
transport = &NodeNameTransport{
Transport: ssrf.NewTransport(tlsConfig),
}
} else {
transport = &NodeNameTransport{
Transport: ssrf.NewTransport(nil),
}
transport.TLSClientConfig = tlsConfig
}
clientTimeout := defaultDockerRequestTimeout

View File

@@ -5,10 +5,9 @@ import (
"strings"
"time"
portainer "github.com/portainer/portainer/api"
dockerclient "github.com/portainer/portainer/api/docker/client"
"github.com/docker/docker/api/types/image"
dockerclient "github.com/docker/docker/client"
"github.com/opencontainers/go-digest"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
@@ -16,33 +15,28 @@ import (
imagetypes "go.podman.io/image/v5/types"
)
const digestFetchTimeout = 5 * time.Second
// ClientFactory creates Docker clients for a given environment.
type ClientFactory interface {
CreateClient(endpoint *portainer.Endpoint, nodeName string, timeout *time.Duration) (*dockerclient.Client, error)
}
// RegistryAuthProvider looks up registry credentials for an image.
type RegistryAuthProvider interface {
RegistryAuth(image Image) (string, string, error)
// Options holds docker registry object options
type Options struct {
Auth imagetypes.DockerAuthConfig
Timeout time.Duration
}
type DigestClient struct {
clientFactory ClientFactory
clientFactory *dockerclient.ClientFactory
opts Options
sysCtx *imagetypes.SystemContext
registryClient RegistryAuthProvider
registryClient *RegistryClient
}
func NewClientWithRegistry(registryClient RegistryAuthProvider, clientFactory ClientFactory) *DigestClient {
func NewClientWithRegistry(registryClient *RegistryClient, clientFactory *dockerclient.ClientFactory) *DigestClient {
return &DigestClient{
clientFactory: clientFactory,
registryClient: registryClient,
}
}
func (c *DigestClient) RemoteDigest(ctx context.Context, image Image) (digest.Digest, error) {
ctx, cancel := context.WithTimeout(ctx, digestFetchTimeout)
func (c *DigestClient) RemoteDigest(image Image) (digest.Digest, error) {
ctx, cancel := c.timeoutContext()
defer cancel()
// Docker references with both a tag and digest are currently not supported
@@ -176,3 +170,14 @@ func ParseRepoTag(repoTag string) *Image {
return &image
}
func (c *DigestClient) timeoutContext() (context.Context, context.CancelFunc) {
ctx := context.Background()
var cancel context.CancelFunc = func() {}
if c.opts.Timeout > 0 {
ctx, cancel = context.WithTimeout(ctx, c.opts.Timeout)
}
return ctx, cancel
}

View File

@@ -24,22 +24,14 @@ func NewRegistryClient(dataStore dataservices.DataStore) *RegistryClient {
}
func (c *RegistryClient) RegistryAuth(image Image) (string, string, error) {
registry, err := cachedRegistry(image.Opts.Name)
registries, err := c.dataStore.Registry().ReadAll()
if err != nil {
var registries []portainer.Registry
err = c.dataStore.ViewTx(func(tx dataservices.DataStoreTx) error {
var err error
registries, err = tx.Registry().ReadAll()
return err
})
if err != nil {
return "", "", err
}
return "", "", err
}
registry, err = findBestMatchRegistry(image.Opts.Name, registries)
if err != nil {
return "", "", err
}
registry, err := findBestMatchRegistry(image.Opts.Name, registries)
if err != nil {
return "", "", err
}
if !registry.Authentication {
@@ -62,22 +54,14 @@ func (c *RegistryClient) CertainRegistryAuth(registry *portainer.Registry) (stri
}
func (c *RegistryClient) EncodedRegistryAuth(image Image) (string, error) {
registry, err := cachedRegistry(image.Opts.Name)
registries, err := c.dataStore.Registry().ReadAll()
if err != nil {
var registries []portainer.Registry
err = c.dataStore.ViewTx(func(tx dataservices.DataStoreTx) error {
var err error
registries, err = tx.Registry().ReadAll()
return err
})
if err != nil {
return "", err
}
return "", err
}
registry, err = findBestMatchRegistry(image.Opts.Name, registries)
if err != nil {
return "", err
}
registry, err := findBestMatchRegistry(image.Opts.Name, registries)
if err != nil {
return "", err
}
if !registry.Authentication {
@@ -137,7 +121,7 @@ func findBestMatchRegistry(repository string, registries []portainer.Registry) (
return nil, errors.New("no registries matched")
}
registriesCache.Set(repository, *match, 0)
registriesCache.Set(repository, match, 0)
return match, nil
}

View File

@@ -57,21 +57,6 @@ func TestFindBestMatchNeedAuthRegistry(t *testing.T) {
})
}
func TestFindBestMatchRegistryCachesResult(t *testing.T) {
t.Parallel()
repository := "caching-test/nginx:latest"
registries := []portainer.Registry{createNewRegistry("docker.io", "", true)}
r, err := findBestMatchRegistry(repository, registries)
require.NoError(t, err)
cached, err := cachedRegistry(repository)
require.NoError(t, err)
require.Equal(t, r.URL, cached.URL)
require.Equal(t, r.Authentication, cached.Authentication)
}
func createNewRegistry(domain, username string, auth bool) portainer.Registry {
registry := portainer.Registry{
URL: domain,

View File

@@ -16,7 +16,6 @@ import (
"github.com/patrickmn/go-cache"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
"golang.org/x/sync/errgroup"
)
// Status constants
@@ -29,11 +28,6 @@ const (
Error = Status("error")
)
const (
errorStatusCacheTTL = 5 * time.Minute
maxConcurrentStatusChecks = 8
)
var (
statusCache = cache.New(24*time.Hour, 24*time.Hour)
remoteDigestCache = cache.New(5*time.Second, 5*time.Second)
@@ -52,17 +46,13 @@ func (c *DigestClient) ContainersImageStatus(ctx context.Context, containers []t
}
statuses := make([]Status, len(containers))
g, ctx := errgroup.WithContext(ctx)
g.SetLimit(maxConcurrentStatusChecks)
containerStatus := func(ct types.Container) Status {
for i, ct := range containers {
var nodeName string
if swarmNodeId := ct.Labels[consts.SwarmNodeIDLabel]; swarmNodeId != "" {
if swarmNodeName, ok := swarmID2NameCache.Get(swarmNodeId); ok {
nodeName, _ = swarmNodeName.(string)
} else {
node, _, err := cli.NodeInspectWithRaw(ctx, swarmNodeId)
node, _, err := cli.NodeInspectWithRaw(ctx, ct.Labels[consts.SwarmNodeIDLabel])
if err != nil {
return Error
}
@@ -74,26 +64,23 @@ func (c *DigestClient) ContainersImageStatus(ctx context.Context, containers []t
s, err := c.ContainerImageStatus(ctx, ct.ID, endpoint, nodeName)
if err != nil {
statuses[i] = Error
log.Warn().Str("containerId", ct.ID).Err(err).Msg("error when fetching image status for container")
return Error
continue
}
return s
statuses[i] = s
if s == Outdated || s == Processing {
break
}
}
for i, ct := range containers {
g.Go(func() error {
statuses[i] = containerStatus(ct)
return nil
})
}
_ = g.Wait()
return AggregateImageStatus(statuses)
return FigureOut(statuses)
}
func AggregateImageStatus(statuses []Status) Status {
func FigureOut(statuses []Status) Status {
if allMatch(statuses, Skipped) {
return Skipped
}
@@ -154,7 +141,7 @@ func (c *DigestClient) ContainerImageStatus(ctx context.Context, containerID str
images = append(images, ParseRepoTags(imageInspect.RepoTags)...)
}
s, err := c.checkStatus(ctx, images, digs)
s, err := c.checkStatus(images, digs)
if err != nil {
log.Debug().Str("image", container.Image).Err(err).Msg("fetching a certain image status")
return Error, err
@@ -204,7 +191,7 @@ func (c *DigestClient) ServiceImageStatus(ctx context.Context, serviceID string,
return c.ContainersImageStatus(ctx, nonExistedOrStoppedContainers, endpoint), nil
}
func (c *DigestClient) checkStatus(ctx context.Context, images []*Image, digests []digest.Digest) (Status, error) {
func (c *DigestClient) checkStatus(images []*Image, digests []digest.Digest) (Status, error) {
if digests == nil {
digests = make([]digest.Digest, 0)
}
@@ -229,7 +216,7 @@ func (c *DigestClient) checkStatus(ctx context.Context, images []*Image, digests
remoteDigest, _ = rd.(digest.Digest)
}
if remoteDigest == "" {
remoteDigest, err = c.RemoteDigest(ctx, *img)
remoteDigest, err = c.RemoteDigest(*img)
if err != nil {
log.Error().Str("image", img.String()).Msg("error when fetch remote digest for image")
return Error, err
@@ -276,10 +263,6 @@ func CacheResourceImageStatus(resourceID string, status Status) {
statusCache.Set(resourceID, status, 0)
}
func CacheErrorImageStatus(resourceID string) {
statusCache.Set(resourceID, Error, errorStatusCacheTTL)
}
func CachedImageDigest(resourceID string) (Status, error) {
if s, ok := statusCache.Get(resourceID); ok {
return s.(Status), nil

View File

@@ -1,63 +0,0 @@
package images
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestAggregateImageStatus(t *testing.T) {
t.Parallel()
f := func(statuses []Status, expected Status) {
t.Helper()
require.Equal(t, expected, AggregateImageStatus(statuses))
}
f([]Status{Skipped, Skipped, Skipped}, Skipped)
f([]Status{Preparing, Preparing}, Preparing)
f([]Status{Updated, Outdated, Processing, Error}, Outdated)
f([]Status{Updated, Processing, Error}, Processing)
f([]Status{Updated, Error}, Error)
f([]Status{Updated, Updated}, Updated)
f([]Status{}, Updated)
f([]Status{Updated, Skipped}, Updated)
}
func TestCachedResourceImageStatusMiss(t *testing.T) {
t.Parallel()
_, err := CachedResourceImageStatus("status-test-miss-key")
require.Error(t, err)
}
func TestCachedResourceImageStatusHitAndEvict(t *testing.T) {
t.Parallel()
key := "status-test-hit-evict-key"
CacheResourceImageStatus(key, Updated)
s, err := CachedResourceImageStatus(key)
require.NoError(t, err)
require.Equal(t, Updated, s)
EvictImageStatus(key)
_, err = CachedResourceImageStatus(key)
require.Error(t, err)
}
func TestCacheErrorImageStatus(t *testing.T) {
t.Parallel()
key := "status-test-error-key"
CacheErrorImageStatus(key)
s, err := CachedResourceImageStatus(key)
require.NoError(t, err)
require.Equal(t, Error, s)
EvictImageStatus(key)
}

View File

@@ -1,79 +1,5 @@
package exec
import (
"fmt"
"regexp"
"strings"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/http/proxy"
"github.com/portainer/portainer/api/http/proxy/factory"
"github.com/portainer/portainer/api/internal/registryutils"
"github.com/docker/cli/cli/config/types"
"github.com/rs/zerolog/log"
)
import "regexp"
var stackNameNormalizeRegex = regexp.MustCompile("[^-_a-z0-9]+")
func normalizeStackName(name string) string {
return stackNameNormalizeRegex.ReplaceAllString(strings.ToLower(name), "")
}
// fetchEndpointProxy returns the Docker host URL for the given endpoint.
// For remote endpoints it creates a local proxy that handles TLS termination and
// Portainer agent header injection; for local unix/npipe sockets no proxy is needed.
func fetchEndpointProxy(proxyManager *proxy.Manager, endpoint *portainer.Endpoint) (string, *factory.ProxyServer, error) {
if strings.HasPrefix(endpoint.URL, "unix://") || strings.HasPrefix(endpoint.URL, "npipe://") {
return "", nil, nil
}
proxy, err := proxyManager.CreateAgentProxyServer(endpoint)
if err != nil {
return "", nil, err
}
return fmt.Sprintf("tcp://127.0.0.1:%d", proxy.Port), proxy, nil
}
// portainerRegistriesToAuthConfigs converts registries to Docker auth configs.
// Callers must ensure ECR tokens are valid before calling this function (e.g. via
// registryutils.RefreshAndPersistECRTokens with a real DataStoreTx). This function
// intentionally performs no DB writes to avoid write-lock contention when called inside
// an active BoltDB write transaction.
func portainerRegistriesToAuthConfigs(registries []portainer.Registry) []types.AuthConfig {
var authConfigs []types.AuthConfig
for _, r := range registries {
ac := types.AuthConfig{
Username: r.Username,
Password: r.Password,
ServerAddress: r.URL,
}
if r.Authentication {
var err error
ac.Username, ac.Password, err = getEffectiveRegUsernamePassword(&r)
if err != nil {
continue
}
}
authConfigs = append(authConfigs, ac)
}
return authConfigs
}
func getEffectiveRegUsernamePassword(registry *portainer.Registry) (string, string, error) {
username, password, err := registryutils.GetRegEffectiveCredential(registry)
if err != nil {
log.Warn().
Err(err).
Str("RegistryName", registry.Name).
Msg("Failed to get effective credential. Skip logging with this registry.")
}
return username, password, err
}

View File

@@ -6,25 +6,35 @@ import (
"io"
"os"
"path"
"strings"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/http/proxy"
"github.com/portainer/portainer/api/http/proxy/factory"
"github.com/portainer/portainer/api/internal/registryutils"
"github.com/portainer/portainer/api/logs"
"github.com/portainer/portainer/api/stacks/stackutils"
"github.com/portainer/portainer/pkg/libstack"
"github.com/docker/cli/cli/config/types"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
)
// ComposeStackManager is a wrapper for docker-compose binary
type ComposeStackManager struct {
deployer libstack.Deployer
proxyManager *proxy.Manager
dataStore dataservices.DataStore
}
// NewComposeStackManager returns a Compose stack manager
func NewComposeStackManager(deployer libstack.Deployer, proxyManager *proxy.Manager) *ComposeStackManager {
func NewComposeStackManager(deployer libstack.Deployer, proxyManager *proxy.Manager, dataStore dataservices.DataStore) *ComposeStackManager {
return &ComposeStackManager{
deployer: deployer,
proxyManager: proxyManager,
dataStore: dataStore,
}
}
@@ -35,9 +45,9 @@ func (manager *ComposeStackManager) ComposeSyntaxMaxVersion() string {
// Up builds, (re)creates and starts containers in the background. Wraps `docker-compose up -d` command
func (manager *ComposeStackManager) Up(ctx context.Context, stack *portainer.Stack, endpoint *portainer.Endpoint, options portainer.ComposeUpOptions) error {
url, proxy, err := fetchEndpointProxy(manager.proxyManager, endpoint)
url, proxy, err := manager.fetchEndpointProxy(endpoint)
if err != nil {
return fmt.Errorf("failed to fetch environment proxy: %w", err)
return errors.Wrap(err, "failed to fetch environment proxy")
}
if proxy != nil {
@@ -46,32 +56,30 @@ func (manager *ComposeStackManager) Up(ctx context.Context, stack *portainer.Sta
envFilePath, err := createEnvFile(stack)
if err != nil {
return fmt.Errorf("failed to create env file: %w", err)
return errors.Wrap(err, "failed to create env file")
}
filePaths := stackutils.GetStackFilePaths(stack, true)
if err = manager.deployer.Deploy(ctx, filePaths, libstack.DeployOptions{
err = manager.deployer.Deploy(ctx, filePaths, libstack.DeployOptions{
Options: libstack.Options{
WorkingDir: stack.ProjectPath,
EnvFilePath: envFilePath,
Host: url,
ProjectName: stack.Name,
Registries: portainerRegistriesToAuthConfigs(options.Registries),
Registries: portainerRegistriesToAuthConfigs(manager.dataStore, options.Registries),
},
ForceRecreate: options.ForceRecreate,
AbortOnContainerExit: options.AbortOnContainerExit,
RemoveOrphans: options.Prune,
}); err != nil {
return fmt.Errorf("failed to deploy a stack: %w", err)
}
return nil
})
return errors.Wrap(err, "failed to deploy a stack")
}
// Run runs a one-off command on a service. Wraps `docker-compose run` command
func (manager *ComposeStackManager) Run(ctx context.Context, stack *portainer.Stack, endpoint *portainer.Endpoint, serviceName string, options portainer.ComposeRunOptions) error {
url, proxy, err := fetchEndpointProxy(manager.proxyManager, endpoint)
url, proxy, err := manager.fetchEndpointProxy(endpoint)
if err != nil {
return fmt.Errorf("failed to fetch environment proxy: %w", err)
return errors.Wrap(err, "failed to fetch environment proxy")
}
if proxy != nil {
@@ -80,78 +88,86 @@ func (manager *ComposeStackManager) Run(ctx context.Context, stack *portainer.St
envFilePath, err := createEnvFile(stack)
if err != nil {
return fmt.Errorf("failed to create env file: %w", err)
return errors.Wrap(err, "failed to create env file")
}
filePaths := stackutils.GetStackFilePaths(stack, true)
if err = manager.deployer.Run(ctx, filePaths, serviceName, libstack.RunOptions{
err = manager.deployer.Run(ctx, filePaths, serviceName, libstack.RunOptions{
Options: libstack.Options{
WorkingDir: stack.ProjectPath,
EnvFilePath: envFilePath,
Host: url,
ProjectName: stack.Name,
Registries: portainerRegistriesToAuthConfigs(options.Registries),
Registries: portainerRegistriesToAuthConfigs(manager.dataStore, options.Registries),
},
Remove: options.Remove,
Args: options.Args,
Detached: options.Detached,
}); err != nil {
return fmt.Errorf("failed to deploy a stack: %w", err)
}
return nil
})
return errors.Wrap(err, "failed to deploy a stack")
}
// Down stops and removes containers, networks, images, and volumes
func (manager *ComposeStackManager) Down(ctx context.Context, stack *portainer.Stack, endpoint *portainer.Endpoint) error {
url, proxy, err := fetchEndpointProxy(manager.proxyManager, endpoint)
url, proxy, err := manager.fetchEndpointProxy(endpoint)
if err != nil {
return fmt.Errorf("failed to fetch environment proxy: %w", err)
return err
} else if proxy != nil {
defer proxy.Close()
}
if err = manager.deployer.Remove(ctx, stack.Name, nil, libstack.RemoveOptions{
err = manager.deployer.Remove(ctx, stack.Name, nil, libstack.RemoveOptions{
Options: libstack.Options{
WorkingDir: "",
Host: url,
},
}); err != nil {
return fmt.Errorf("failed to remove a stack: %w", err)
}
return nil
})
return errors.Wrap(err, "failed to remove a stack")
}
// Pull an image associated with a service defined in a docker-compose.yml or docker-stack.yml file,
// but does not start containers based on those images.
func (manager *ComposeStackManager) Pull(ctx context.Context, stack *portainer.Stack, endpoint *portainer.Endpoint, options portainer.ComposeOptions) error {
url, proxy, err := fetchEndpointProxy(manager.proxyManager, endpoint)
url, proxy, err := manager.fetchEndpointProxy(endpoint)
if err != nil {
return fmt.Errorf("failed to fetch environment proxy: %w", err)
return err
} else if proxy != nil {
defer proxy.Close()
}
envFilePath, err := createEnvFile(stack)
if err != nil {
return fmt.Errorf("failed to create env file: %w", err)
return errors.Wrap(err, "failed to create env file")
}
filePaths := stackutils.GetStackFilePaths(stack, true)
if err = manager.deployer.Pull(ctx, filePaths, libstack.Options{
err = manager.deployer.Pull(ctx, filePaths, libstack.Options{
WorkingDir: stack.ProjectPath,
EnvFilePath: envFilePath,
Host: url,
ProjectName: stack.Name,
Registries: portainerRegistriesToAuthConfigs(options.Registries),
}); err != nil {
return fmt.Errorf("failed to pull images of the stack: %w", err)
}
return nil
Registries: portainerRegistriesToAuthConfigs(manager.dataStore, options.Registries),
})
return errors.Wrap(err, "failed to pull images of the stack")
}
// NormalizeStackName returns a new stack name with unsupported characters replaced
func (manager *ComposeStackManager) NormalizeStackName(name string) string {
return normalizeStackName(name)
return stackNameNormalizeRegex.ReplaceAllString(strings.ToLower(name), "")
}
func (manager *ComposeStackManager) fetchEndpointProxy(endpoint *portainer.Endpoint) (string, *factory.ProxyServer, error) {
if strings.HasPrefix(endpoint.URL, "unix://") || strings.HasPrefix(endpoint.URL, "npipe://") {
return "", nil, nil
}
proxy, err := manager.proxyManager.CreateAgentProxyServer(endpoint)
if err != nil {
return "", nil, err
}
return fmt.Sprintf("tcp://127.0.0.1:%d", proxy.Port), proxy, nil
}
// createEnvFile creates a file that would hold both "in-place" and default environment variables.
@@ -162,7 +178,7 @@ func createEnvFile(stack *portainer.Stack) (string, error) {
}
envFilePath := path.Join(stack.ProjectPath, "stack.env")
envfile, err := os.OpenFile(envFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600)
envfile, err := os.OpenFile(envFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
return "", err
}
@@ -213,3 +229,49 @@ func copyConfigEnvVars(w io.Writer, envs []portainer.Pair) error {
return nil
}
func portainerRegistriesToAuthConfigs(tx dataservices.DataStoreTx, registries []portainer.Registry) []types.AuthConfig {
var authConfigs []types.AuthConfig
for _, r := range registries {
ac := types.AuthConfig{
Username: r.Username,
Password: r.Password,
ServerAddress: r.URL,
}
if r.Authentication {
var err error
ac.Username, ac.Password, err = getEffectiveRegUsernamePassword(tx, &r)
if err != nil {
continue
}
}
authConfigs = append(authConfigs, ac)
}
return authConfigs
}
func getEffectiveRegUsernamePassword(tx dataservices.DataStoreTx, registry *portainer.Registry) (string, string, error) {
if err := registryutils.EnsureRegTokenValid(tx, registry); err != nil {
log.Warn().
Err(err).
Str("RegistryName", registry.Name).
Msg("Failed to validate registry token. Skip logging with this registry.")
return "", "", err
}
username, password, err := registryutils.GetRegEffectiveCredential(registry)
if err != nil {
log.Warn().
Err(err).
Str("RegistryName", registry.Name).
Msg("Failed to get effective credential. Skip logging with this registry.")
}
return username, password, err
}

View File

@@ -48,7 +48,7 @@ func Test_UpAndDown(t *testing.T) {
deployer := compose.NewComposeDeployer()
w := NewComposeStackManager(deployer, nil)
w := NewComposeStackManager(deployer, nil, nil)
if err := w.Up(t.Context(), stack, endpoint, portainer.ComposeUpOptions{}); err != nil {
t.Fatalf("Error calling docker-compose up: %s", err)

View File

@@ -4,7 +4,6 @@ import (
"io"
"os"
"testing"
"time"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/filesystem"
@@ -96,74 +95,3 @@ func Test_createEnvFile_mergesDefultAndInplaceEnvVars(t *testing.T) {
assert.Equal(t, []byte("VAR1=VAL1\nVAR2=VAL2\n\nVAR1=NEW_VAL1\nVAR3=VAL3\n"), content)
}
func Test_portainerRegistriesToAuthConfigs(t *testing.T) {
t.Parallel()
t.Run("returns empty slice for empty input", func(t *testing.T) {
t.Parallel()
result := portainerRegistriesToAuthConfigs([]portainer.Registry{})
require.Nil(t, result)
})
t.Run("uses registry URL, username and password for non-authenticated registry", func(t *testing.T) {
t.Parallel()
registries := []portainer.Registry{
{URL: "registry.example.com", Username: "user", Password: "pass", Authentication: false},
}
result := portainerRegistriesToAuthConfigs(registries)
require.Len(t, result, 1)
require.Equal(t, "registry.example.com", result[0].ServerAddress)
require.Equal(t, "user", result[0].Username)
require.Equal(t, "pass", result[0].Password)
})
t.Run("uses username and password for authenticated non-ECR registry", func(t *testing.T) {
t.Parallel()
registries := []portainer.Registry{
{URL: "registry.example.com", Username: "user", Password: "pass", Authentication: true, Type: portainer.CustomRegistry},
}
result := portainerRegistriesToAuthConfigs(registries)
require.Len(t, result, 1)
require.Equal(t, "user", result[0].Username)
require.Equal(t, "pass", result[0].Password)
})
t.Run("parses ECR access token for authenticated ECR registry with valid token", func(t *testing.T) {
t.Parallel()
registries := []portainer.Registry{
{
URL: "123456789.dkr.ecr.us-east-1.amazonaws.com",
Username: "AKIAIOSFODNN7EXAMPLE",
Password: "secretkey",
Authentication: true,
Type: portainer.EcrRegistry,
Ecr: portainer.EcrData{Region: "us-east-1"},
AccessToken: "AWS:ecr-password",
AccessTokenExpiry: time.Now().Add(time.Hour).Unix(),
},
}
result := portainerRegistriesToAuthConfigs(registries)
require.Len(t, result, 1)
require.Equal(t, "AWS", result[0].Username)
require.Equal(t, "ecr-password", result[0].Password)
})
t.Run("includes valid registries and skips ones with credential errors", func(t *testing.T) {
t.Parallel()
registries := []portainer.Registry{
{URL: "valid.example.com", Username: "user", Password: "pass", Authentication: false},
{
URL: "123456789.dkr.ecr.us-east-1.amazonaws.com",
Authentication: true,
Type: portainer.EcrRegistry,
Ecr: portainer.EcrData{Region: "us-east-1"},
AccessToken: "no-colon-token",
AccessTokenExpiry: time.Now().Add(time.Hour).Unix(),
},
}
result := portainerRegistriesToAuthConfigs(registries)
require.Len(t, result, 1)
require.Equal(t, "valid.example.com", result[0].ServerAddress)
})
}

View File

@@ -1,93 +1,258 @@
package exec
import (
"bytes"
"context"
"fmt"
"errors"
"os"
"os/exec"
"path"
"runtime"
"strings"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/http/proxy"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/stacks/stackutils"
"github.com/portainer/portainer/pkg/libstack/swarm"
"github.com/rs/zerolog/log"
"github.com/segmentio/encoding/json"
)
// SwarmStackManager represents a service for managing stacks.
type SwarmStackManager struct {
deployer swarm.Deployer
proxyManager *proxy.Manager
binaryPath string
configPath string
signatureService portainer.DigitalSignatureService
fileService portainer.FileService
reverseTunnelService portainer.ReverseTunnelService
dataStore dataservices.DataStore
}
// NewSwarmStackManager creates a new SwarmStackManager.
// NewSwarmStackManager initializes a new SwarmStackManager service.
// It also updates the configuration of the Docker CLI binary.
func NewSwarmStackManager(
deployer swarm.Deployer,
proxyManager *proxy.Manager,
) *SwarmStackManager {
return &SwarmStackManager{
deployer: deployer,
proxyManager: proxyManager,
binaryPath, configPath string,
signatureService portainer.DigitalSignatureService,
fileService portainer.FileService,
reverseTunnelService portainer.ReverseTunnelService,
datastore dataservices.DataStore,
) (*SwarmStackManager, error) {
manager := &SwarmStackManager{
binaryPath: binaryPath,
configPath: configPath,
signatureService: signatureService,
fileService: fileService,
reverseTunnelService: reverseTunnelService,
dataStore: datastore,
}
if err := manager.updateDockerCLIConfiguration(manager.configPath); err != nil {
return nil, err
}
return manager, nil
}
// Deploy creates or updates a Docker Swarm stack.
func (manager *SwarmStackManager) Deploy(
ctx context.Context,
stack *portainer.Stack,
prune bool,
pullImage bool,
endpoint *portainer.Endpoint,
registries []portainer.Registry,
) error {
url, proxy, err := fetchEndpointProxy(manager.proxyManager, endpoint)
// Login executes the docker login command against a list of registries (including DockerHub).
func (manager *SwarmStackManager) Login(ctx context.Context, registries []portainer.Registry, endpoint *portainer.Endpoint) error {
command, args, err := manager.prepareDockerCommandAndArgs(manager.binaryPath, manager.configPath, endpoint)
if err != nil {
return fmt.Errorf("failed to fetch environment proxy: %w", err)
return err
}
if proxy != nil {
defer proxy.Close()
for _, registry := range registries {
if registry.Authentication {
username, password, err := getEffectiveRegUsernamePassword(manager.dataStore, &registry)
if err != nil {
continue
}
registryArgs := append(args, "login", "--username", username, "--password", password, registry.URL)
if err := runCommandAndCaptureStdErr(ctx, command, registryArgs, nil, ""); err != nil {
log.Warn().
Err(err).
Str("RegistryName", registry.Name).
Msg("Failed to login.")
}
}
}
return nil
}
// Logout executes the docker logout command.
func (manager *SwarmStackManager) Logout(ctx context.Context, endpoint *portainer.Endpoint) error {
command, args, err := manager.prepareDockerCommandAndArgs(manager.binaryPath, manager.configPath, endpoint)
if err != nil {
return err
}
args = append(args, "logout")
return runCommandAndCaptureStdErr(ctx, command, args, nil, "")
}
// Deploy executes the docker stack deploy command.
func (manager *SwarmStackManager) Deploy(ctx context.Context, stack *portainer.Stack, prune bool, pullImage bool, endpoint *portainer.Endpoint) error {
filePaths := stackutils.GetStackFilePaths(stack, true)
env := make([]string, 0, len(stack.Env))
for _, ev := range stack.Env {
env = append(env, ev.Name+"="+ev.Value)
}
return manager.deployer.Deploy(context.TODO(), filePaths, swarm.DeployOptions{
Options: swarm.Options{
ProjectName: stack.Name,
Host: url,
Env: env,
WorkingDir: stack.ProjectPath,
Registries: portainerRegistriesToAuthConfigs(registries),
},
RemoveOrphans: prune,
PullImage: pullImage,
})
}
// Remove deletes all resources belonging to a Swarm stack.
func (manager *SwarmStackManager) Remove(
ctx context.Context,
stack *portainer.Stack,
endpoint *portainer.Endpoint,
) error {
url, proxy, err := fetchEndpointProxy(manager.proxyManager, endpoint)
command, args, err := manager.prepareDockerCommandAndArgs(manager.binaryPath, manager.configPath, endpoint)
if err != nil {
return fmt.Errorf("failed to fetch environment proxy: %w", err)
return err
}
if proxy != nil {
defer proxy.Close()
if prune {
args = append(args, "stack", "deploy", "--prune", "--with-registry-auth")
} else {
args = append(args, "stack", "deploy", "--with-registry-auth")
}
return manager.deployer.Remove(context.TODO(), stack.Name, swarm.RemoveOptions{
Options: swarm.Options{
Host: url,
},
})
if !pullImage {
args = append(args, "--resolve-image=never")
}
args = configureFilePaths(args, filePaths)
args = append(args, stack.Name)
env := make([]string, 0)
for _, envvar := range stack.Env {
env = append(env, envvar.Name+"="+envvar.Value)
}
return runCommandAndCaptureStdErr(ctx, command, args, env, stack.ProjectPath)
}
// Remove executes the docker stack rm command.
func (manager *SwarmStackManager) Remove(ctx context.Context, stack *portainer.Stack, endpoint *portainer.Endpoint) error {
command, args, err := manager.prepareDockerCommandAndArgs(manager.binaryPath, manager.configPath, endpoint)
if err != nil {
return err
}
args = append(args, "stack", "rm", "--detach=false", stack.Name)
return runCommandAndCaptureStdErr(ctx, command, args, nil, "")
}
func runCommandAndCaptureStdErr(ctx context.Context, command string, args []string, env []string, workingDir string) error {
var stderr bytes.Buffer
var stdout bytes.Buffer
cmd := exec.CommandContext(ctx, command, args...)
cmd.Stderr = &stderr
cmd.Stdout = &stdout
if workingDir != "" {
cmd.Dir = workingDir
}
if env != nil {
cmd.Env = os.Environ()
cmd.Env = append(cmd.Env, env...)
}
if err := cmd.Run(); err != nil {
errMsg := strings.TrimSpace(stderr.String())
if errMsg == "" {
errMsg = strings.TrimSpace(stdout.String())
}
if errMsg == "" {
errMsg = err.Error()
}
return errors.New(errMsg)
}
return nil
}
func (manager *SwarmStackManager) prepareDockerCommandAndArgs(binaryPath, configPath string, endpoint *portainer.Endpoint) (string, []string, error) {
// Assume Linux as a default
command := path.Join(binaryPath, "docker")
if runtime.GOOS == "windows" {
command = path.Join(binaryPath, "docker.exe")
}
args := make([]string, 0)
args = append(args, "--config", configPath)
endpointURL := endpoint.URL
if endpoint.Type == portainer.EdgeAgentOnDockerEnvironment {
tunnelAddr, err := manager.reverseTunnelService.TunnelAddr(endpoint)
if err != nil {
return "", nil, err
}
endpointURL = "tcp://" + tunnelAddr
}
args = append(args, "-H", endpointURL)
if endpoint.TLSConfig.TLS {
args = append(args, "--tls")
if !endpoint.TLSConfig.TLSSkipVerify {
args = append(args, "--tlsverify", "--tlscacert", endpoint.TLSConfig.TLSCACertPath)
} else {
args = append(args, "--tlscacert", "")
}
if endpoint.TLSConfig.TLSCertPath != "" && endpoint.TLSConfig.TLSKeyPath != "" {
args = append(args, "--tlscert", endpoint.TLSConfig.TLSCertPath, "--tlskey", endpoint.TLSConfig.TLSKeyPath)
}
}
return command, args, nil
}
func (manager *SwarmStackManager) updateDockerCLIConfiguration(configPath string) error {
configFilePath := path.Join(configPath, "config.json")
config, err := manager.retrieveConfigurationFromDisk(configFilePath)
if err != nil {
log.Warn().Err(err).Msg("unable to retrieve the Swarm configuration from disk, proceeding without it")
}
signature, err := manager.signatureService.CreateSignature(portainer.PortainerAgentSignatureMessage)
if err != nil {
return err
}
if config["HttpHeaders"] == nil {
config["HttpHeaders"] = make(map[string]any)
}
headersObject := config["HttpHeaders"].(map[string]any)
headersObject["X-PortainerAgent-ManagerOperation"] = "1"
headersObject["X-PortainerAgent-Signature"] = signature
headersObject["X-PortainerAgent-PublicKey"] = manager.signatureService.EncodedPublicKey()
return manager.fileService.WriteJSONToFile(configFilePath, config)
}
func (manager *SwarmStackManager) retrieveConfigurationFromDisk(path string) (map[string]any, error) {
var config map[string]any
raw, err := manager.fileService.GetFileContent(path, "")
if err != nil {
return make(map[string]any), nil
}
if err := json.Unmarshal(raw, &config); err != nil {
return nil, err
}
return config, nil
}
// NormalizeStackName returns a new stack name with unsupported characters replaced.
func (manager *SwarmStackManager) NormalizeStackName(name string) string {
return normalizeStackName(name)
return stackNameNormalizeRegex.ReplaceAllString(strings.ToLower(name), "")
}
func configureFilePaths(args []string, filePaths []string) []string {
for _, path := range filePaths {
args = append(args, "--compose-file", path)
}
return args
}

View File

@@ -0,0 +1,86 @@
package exec
import (
"context"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestConfigFilePaths(t *testing.T) {
t.Parallel()
args := []string{"stack", "deploy", "--with-registry-auth"}
filePaths := []string{"dir/file", "dir/file-two", "dir/file-three"}
expected := []string{"stack", "deploy", "--with-registry-auth", "--compose-file", "dir/file", "--compose-file", "dir/file-two", "--compose-file", "dir/file-three"}
output := configureFilePaths(args, filePaths)
assert.ElementsMatch(t, expected, output, "wrong output file paths")
}
func TestPrepareDockerCommandAndArgs(t *testing.T) {
t.Parallel()
binaryPath := "/test/dist"
configPath := "/test/config"
manager := &SwarmStackManager{
binaryPath: binaryPath,
configPath: configPath,
}
endpoint := &portainer.Endpoint{
URL: "tcp://test:9000",
TLSConfig: portainer.TLSConfiguration{
TLS: true,
TLSSkipVerify: true,
},
}
command, args, err := manager.prepareDockerCommandAndArgs(binaryPath, configPath, endpoint)
require.NoError(t, err)
expectedCommand := "/test/dist/docker"
expectedArgs := []string{"--config", "/test/config", "-H", "tcp://test:9000", "--tls", "--tlscacert", ""}
require.Equal(t, expectedCommand, command)
require.Equal(t, expectedArgs, args)
}
func TestRunCommandAndCaptureStdErr(t *testing.T) {
t.Parallel()
t.Run("should return nil on successful command", func(t *testing.T) {
err := runCommandAndCaptureStdErr(context.Background(), "echo", []string{"hello"}, nil, "")
require.NoError(t, err)
})
t.Run("should capture stderr on failure", func(t *testing.T) {
err := runCommandAndCaptureStdErr(context.Background(), "sh", []string{"-c", "echo 'stderr error' >&2; exit 1"}, nil, "")
require.Error(t, err)
assert.Contains(t, err.Error(), "stderr error")
})
t.Run("should fall back to stdout when stderr is empty", func(t *testing.T) {
err := runCommandAndCaptureStdErr(context.Background(), "sh", []string{"-c", "echo 'stdout error'; exit 1"}, nil, "")
require.Error(t, err)
assert.Contains(t, err.Error(), "stdout error")
})
t.Run("should fall back to exec error when both are empty", func(t *testing.T) {
err := runCommandAndCaptureStdErr(context.Background(), "sh", []string{"-c", "exit 1"}, nil, "")
require.Error(t, err)
assert.NotEmpty(t, err.Error())
assert.Contains(t, err.Error(), "exit status 1")
})
t.Run("should prefer stderr over stdout", func(t *testing.T) {
err := runCommandAndCaptureStdErr(context.Background(), "sh", []string{"-c", "echo 'stdout msg'; echo 'stderr msg' >&2; exit 1"}, nil, "")
require.Error(t, err)
assert.Contains(t, err.Error(), "stderr msg")
assert.NotContains(t, err.Error(), "stdout msg")
})
t.Run("should return error for non-existent command", func(t *testing.T) {
err := runCommandAndCaptureStdErr(context.Background(), "nonexistent-cmd-12345", nil, nil, "")
require.Error(t, err)
})
}

View File

@@ -46,6 +46,8 @@ const (
BinaryStorePath = "bin"
// EdgeJobStorePath represents the subfolder where schedule files are stored.
EdgeJobStorePath = "edge_jobs"
// DockerConfigPath represents the subfolder where docker configuration is stored.
DockerConfigPath = "docker_config"
// ExtensionRegistryManagementStorePath represents the subfolder where files related to the
// registry management extension are stored.
ExtensionRegistryManagementStorePath = "extensions"
@@ -89,7 +91,7 @@ func JoinPaths(trustedRoot string, untrustedPaths ...string) string {
trustedRoot = "."
}
p := filepath.Join(trustedRoot, filepath.Join(append([]string{"/"}, untrustedPaths...)...))
p := filepath.Join(trustedRoot, filepath.Join(append([]string{"/"}, untrustedPaths...)...)) //nolint:forbidigo
// avoid setting a volume name from the untrusted paths
vnp := filepath.VolumeName(p)
@@ -133,6 +135,11 @@ func NewService(dataStorePath, fileStorePath string) (*Service, error) {
return nil, err
}
err = service.createDirectoryInStore(DockerConfigPath)
if err != nil {
return nil, err
}
return service, nil
}
@@ -141,6 +148,11 @@ func (service *Service) GetBinaryFolder() string {
return JoinPaths(service.fileStorePath, BinaryStorePath)
}
// GetDockerConfigPath returns the full path to the docker config store on the filesystem
func (service *Service) GetDockerConfigPath() string {
return JoinPaths(service.fileStorePath, DockerConfigPath)
}
// RemoveDirectory removes a directory on the filesystem.
func (service *Service) RemoveDirectory(directoryPath string) error {
return os.RemoveAll(directoryPath)

View File

@@ -15,7 +15,7 @@ type DirEntry struct {
Name string
Content string
IsFile bool
Permissions os.FileMode `swaggertype:"integer"`
Permissions os.FileMode
}
// FilterDirForEntryFile filers the given dirEntries, returns entries of the entryFile and .env file

View File

@@ -14,13 +14,12 @@ import (
"github.com/portainer/portainer/api/crypto"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/portainer/portainer/api/logs"
"github.com/portainer/portainer/pkg/libhttp/ssrf"
"github.com/rs/zerolog/log"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing/filemode"
githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
"github.com/segmentio/encoding/json"
)
@@ -65,10 +64,15 @@ func NewAzureClient() *azureClient {
}
func newHttpClientForAzure(insecureSkipVerify bool) *http.Client {
return &http.Client{
Transport: ssrf.NewTransport(crypto.CreateTLSConfiguration(insecureSkipVerify)),
Timeout: 300 * time.Second,
httpsCli := &http.Client{
Transport: &http.Transport{
TLSClientConfig: crypto.CreateTLSConfiguration(insecureSkipVerify),
Proxy: http.ProxyFromEnvironment,
},
Timeout: 300 * time.Second,
}
return httpsCli
}
func (a *azureClient) Download(ctx context.Context, destination string, opt *git.CloneOptions) error {

View File

@@ -3,7 +3,6 @@ package git
import (
"context"
"os"
"path/filepath"
"strings"
"github.com/portainer/portainer/api/filesystem"
@@ -48,19 +47,11 @@ func NewGitClient(preserveGitDir bool) *gitClient {
}
func (c *gitClient) Download(ctx context.Context, dst string, opt *git.CloneOptions) error {
resolved, err := filepath.EvalSymlinks(dst)
if err != nil && !errors.Is(err, os.ErrNotExist) {
return errors.Wrap(err, "failed to resolve destination path")
}
if err == nil {
dst = resolved
}
wt := NewNoSymlinkFS(osfs.New(dst))
dot := osfs.New(filesystem.JoinPaths(dst, ".git"))
storer := gogitfs.NewStorage(dot, cache.NewObjectLRU(0))
_, err = git.CloneContext(ctx, storer, wt, opt)
_, err := git.CloneContext(ctx, storer, wt, opt)
if err != nil {
if err.Error() == "authentication required" {
return gittypes.ErrAuthenticationFailure
@@ -86,7 +77,7 @@ func (c *gitClient) LatestCommitID(ctx context.Context, repositoryUrl, reference
URLs: []string{repositoryUrl},
})
refs, err := remote.ListContext(ctx, opt)
refs, err := remote.List(opt)
if err != nil {
if err.Error() == "authentication required" {
return "", gittypes.ErrAuthenticationFailure
@@ -118,7 +109,7 @@ func (c *gitClient) ListRefs(ctx context.Context, repositoryUrl string, opt *git
URLs: []string{repositoryUrl},
})
refs, err := rem.ListContext(ctx, opt)
refs, err := rem.List(opt)
if err != nil {
return nil, checkGitError(err)
}

View File

@@ -99,19 +99,6 @@ func Test_ClonePublicRepository_NoGitDirectory(t *testing.T) {
assert.NoDirExists(t, filesystem.JoinPaths(dir, ".git"))
}
func Test_ClonePublicRepository_NonExistentDst(t *testing.T) {
t.Parallel()
service := Service{git: NewGitClient(false)}
repositoryURL := setup(t)
referenceName := "refs/heads/main"
dir := filesystem.JoinPaths(t.TempDir(), "sub", "dir")
err := service.CloneRepository(t.Context(), dir, repositoryURL, referenceName, "", "", false)
require.NoError(t, err)
assert.DirExists(t, dir)
assert.NoDirExists(t, filesystem.JoinPaths(dir, ".git"))
}
func Test_latestCommitID(t *testing.T) {
t.Parallel()
service := Service{git: NewGitClient(true)} // no need for http client since the test access the repo via file system.
@@ -275,7 +262,6 @@ func createBareRepoWithSymlink(t *testing.T) string {
}
func Test_Download_RejectsSymlink(t *testing.T) {
t.Parallel()
client := NewGitClient(false)
repoURL := createBareRepoWithSymlink(t)

View File

@@ -1,53 +0,0 @@
package git
import (
"context"
"fmt"
"net"
"strconv"
"github.com/portainer/portainer/pkg/libhttp/ssrf"
gittransport "github.com/go-git/go-git/v5/plumbing/transport"
)
const gitDefaultPort = 9418
// ssrfGitTransport wraps a git:// transport and validates the resolved IP
// against the SSRF policy before establishing connections.
type ssrfGitTransport struct {
inner gittransport.Transport
}
// NewSSRFGitTransport wraps inner and blocks connections to private IP ranges
// according to the active SSRF policy.
func NewSSRFGitTransport(inner gittransport.Transport) gittransport.Transport {
return &ssrfGitTransport{inner: inner}
}
func (t *ssrfGitTransport) NewUploadPackSession(ep *gittransport.Endpoint, auth gittransport.AuthMethod) (gittransport.UploadPackSession, error) {
if err := checkEndpointSSRF(ep); err != nil {
return nil, err
}
return t.inner.NewUploadPackSession(ep, auth)
}
func (t *ssrfGitTransport) NewReceivePackSession(ep *gittransport.Endpoint, auth gittransport.AuthMethod) (gittransport.ReceivePackSession, error) {
if err := checkEndpointSSRF(ep); err != nil {
return nil, err
}
return t.inner.NewReceivePackSession(ep, auth)
}
func checkEndpointSSRF(ep *gittransport.Endpoint) error {
port := ep.Port
if port <= 0 {
port = gitDefaultPort
}
rawURL := fmt.Sprintf("git://%s/", net.JoinHostPort(ep.Host, strconv.Itoa(port)))
return ssrf.CheckURL(context.Background(), rawURL)
}

View File

@@ -1,11 +1,7 @@
package gittypes
import (
"cmp"
"errors"
"net/url"
"path"
"strings"
)
var (
@@ -14,10 +10,6 @@ var (
ErrSymlinkDetected = errors.New("repository contains a symlink, which is not allowed for security reasons")
)
type GitCredentialAuthType int
type GitProvider int
// RepoConfig represents a configuration for a repo
type RepoConfig struct {
// The repo url
@@ -28,72 +20,18 @@ type RepoConfig struct {
// NOTE: For stacks, this mirrors Stack.EntryPoint and the two are kept in sync by stackUpdateGit.
ConfigFilePath string `example:"docker-compose.yml"`
// Git credentials
Authentication *GitAuthentication `json:",omitempty"`
Authentication *GitAuthentication
// Repository hash
ConfigHash string `example:"bc4c183d756879ea4d173315338110b31004b8e0"`
// TLSSkipVerify skips SSL verification when cloning the Git repository
TLSSkipVerify bool `example:"false"`
}
// RepoName extracts the repository name from a git URL for use as a display name.
// e.g. "https://github.com/org/app-config.git" results in "app-config"
func RepoName(rawURL string) string {
return strings.TrimSuffix(path.Base(rawURL), ".git")
}
// NormalizeURL returns a canonical form of rawURL for deduplication purposes:
// scheme and host are lowercased, embedded credentials are removed, trailing
// slashes and the .git suffix are stripped from the path. If the scheme is
// absent it defaults to https.
func NormalizeURL(rawURL string) (string, error) {
u, err := url.Parse(rawURL)
if err != nil {
return "", err
}
u.Scheme = strings.ToLower(cmp.Or(u.Scheme, "https"))
u.Host = strings.ToLower(u.Host)
u.User = nil
u.Path = strings.TrimSuffix(strings.TrimRight(u.Path, "/"), ".git")
return u.String(), nil
}
// SanitizeURL strips any userinfo (username/password) embedded in rawURL,
// returning a URL safe to store or return to clients.
func SanitizeURL(rawURL string) string {
u, err := url.Parse(rawURL)
if err != nil || u.User == nil {
return rawURL
}
u.User = nil
return u.String()
}
// SanitizeRepoConfig returns a copy of gc with the URL sanitized and password cleared,
// safe to return to clients.
func SanitizeRepoConfig(gc *RepoConfig) *RepoConfig {
if gc == nil {
return nil
}
result := *gc
result.URL = SanitizeURL(result.URL)
if result.Authentication != nil && result.Authentication.Password != "" {
auth := *result.Authentication
auth.Password = ""
result.Authentication = &auth
}
return &result
}
type GitAuthentication struct {
Username string
Password string
Provider GitProvider `json:",omitempty"`
AuthorizationType GitCredentialAuthType `json:",omitempty"`
Username string
Password string
// Git credentials identifier when the value is not 0
// When the value is 0, Username and Password are set without using saved credential
// This is introduced since 2.15.0
GitCredentialID int `example:"0"`
}

View File

@@ -1,26 +0,0 @@
package gittypes
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestNormalizeURL(t *testing.T) {
t.Parallel()
f := func(input, expected string) {
t.Helper()
got, err := NormalizeURL(input)
require.NoError(t, err)
require.Equal(t, expected, got)
}
f("https://github.com/org/repo.git", "https://github.com/org/repo")
f("https://github.com/org/repo/", "https://github.com/org/repo")
f("https://github.com/org/repo.git/", "https://github.com/org/repo")
f("HTTPS://github.com/org/repo", "https://github.com/org/repo")
f("https://GitHub.COM/org/repo", "https://github.com/org/repo")
f("https://user:pass@github.com/org/repo.git", "https://github.com/org/repo")
f("https://github.com/org/repo", "https://github.com/org/repo")
}

View File

@@ -2,9 +2,7 @@ package update
import (
"context"
"os"
"strings"
"time"
"github.com/pkg/errors"
@@ -29,21 +27,15 @@ func UpdateGitObject(ctx context.Context, gitService portainer.GitService, objId
username, password := git.GetCredentials(gitConfig.Authentication)
fetchCtx, cancel := context.WithTimeout(ctx, time.Minute)
newHash, err := gitService.LatestCommitID(
fetchCtx,
ctx,
gitConfig.URL,
gitConfig.ReferenceName,
username,
password,
gitConfig.TLSSkipVerify,
)
cancel()
if err != nil {
if fetchCtx.Err() == context.DeadlineExceeded {
log.Error().Str("object", objId).Msg("git fetch timed out after 1 minute")
}
return false, "", errors.WithMessagef(err, "failed to fetch latest commit id of %v", objId)
}
@@ -79,11 +71,6 @@ func UpdateGitObject(ctx context.Context, gitService portainer.GitService, objId
}
if err := cloneGitRepository(ctx, gitService, cloneParams); err != nil {
if enableVersionFolder {
if removeErr := os.RemoveAll(toDir); removeErr != nil {
log.Warn().Err(removeErr).Str("dir", toDir).Msg("failed to remove partial clone directory")
}
}
return false, "", errors.WithMessagef(err, "failed to do a fresh clone of %v", objId)
}

View File

@@ -1,55 +0,0 @@
package sources
import (
portainer "github.com/portainer/portainer/api"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/portainer/portainer/pkg/fips"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
)
// RepoConfigInput holds the raw payload fields needed to resolve a git RepoConfig.
// Set SourceID to resolve URL/auth from a stored source; otherwise provide the inline fields.
type RepoConfigInput struct {
SourceID portainer.SourceID
ReferenceName string
ConfigFilePath string
RepositoryURL string
TLSSkipVerify bool
RepositoryAuthentication bool
Username string
Password string
Provider gittypes.GitProvider
AuthorizationType gittypes.GitCredentialAuthType
}
// ResolveRepoConfig builds a RepoConfig from either a SourceID or inline URL/auth fields.
func ResolveRepoConfig(tx gitSourceStore, input RepoConfigInput) (gittypes.RepoConfig, *httperror.HandlerError) {
cfg := gittypes.RepoConfig{
ReferenceName: input.ReferenceName,
ConfigFilePath: input.ConfigFilePath,
}
if input.SourceID != 0 {
src, httpErr := ValidateGitSourceAccess(tx, input.SourceID)
if httpErr != nil {
return gittypes.RepoConfig{}, httpErr
}
cfg.URL = src.Git.URL
cfg.Authentication = src.Git.Authentication
cfg.TLSSkipVerify = src.Git.TLSSkipVerify
} else {
cfg.URL = input.RepositoryURL
cfg.TLSSkipVerify = input.TLSSkipVerify
if input.RepositoryAuthentication {
cfg.Authentication = &gittypes.GitAuthentication{
Username: input.Username,
Password: input.Password,
Provider: input.Provider,
AuthorizationType: input.AuthorizationType,
}
}
}
cfg.TLSSkipVerify = cfg.TLSSkipVerify && fips.CanTLSSkipVerify()
return cfg, nil
}

View File

@@ -1,70 +0,0 @@
package sources
import (
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/datastore"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/portainer/portainer/pkg/fips"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func init() {
fips.InitFIPS(false)
}
func TestResolveRepoConfig_WithSourceID_ReturnsSourceConfig(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, false)
src := &portainer.Source{
Type: portainer.SourceTypeGit,
Git: &gittypes.RepoConfig{
URL: "https://github.com/org/repo",
TLSSkipVerify: true,
Authentication: &gittypes.GitAuthentication{
Username: "user",
Password: "token",
},
},
}
require.NoError(t, store.Source().Create(src))
cfg, httpErr := ResolveRepoConfig(store, RepoConfigInput{
SourceID: src.ID,
ReferenceName: "refs/heads/main",
ConfigFilePath: "docker-compose.yml",
RepositoryURL: "https://ignored.example.com",
})
require.Nil(t, httpErr)
assert.Equal(t, src.Git.URL, cfg.URL)
assert.Equal(t, src.Git.Authentication, cfg.Authentication)
assert.Equal(t, src.Git.TLSSkipVerify, cfg.TLSSkipVerify)
assert.Equal(t, "refs/heads/main", cfg.ReferenceName)
assert.Equal(t, "docker-compose.yml", cfg.ConfigFilePath)
}
func TestResolveRepoConfig_WithInlineURL_ReturnsInlineConfig(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, false)
cfg, httpErr := ResolveRepoConfig(store, RepoConfigInput{
ReferenceName: "refs/heads/main",
ConfigFilePath: "docker-compose.yml",
RepositoryURL: "https://github.com/org/repo",
TLSSkipVerify: true,
RepositoryAuthentication: true,
Username: "user",
Password: "pass",
})
require.Nil(t, httpErr)
assert.Equal(t, "https://github.com/org/repo", cfg.URL)
assert.True(t, cfg.TLSSkipVerify)
require.NotNil(t, cfg.Authentication)
assert.Equal(t, "user", cfg.Authentication.Username)
assert.Equal(t, "pass", cfg.Authentication.Password)
}

View File

@@ -1,38 +0,0 @@
package sources
import (
"fmt"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
)
// gitSourceStore is the minimal intersection of CE and EE DataStoreTx that these functions need.
// Both EE and CE DataStoreTx satisfy it, even though they are incompatible as full interface types.
type gitSourceStore interface {
Source() dataservices.SourceService
IsErrObjectNotFound(err error) bool
}
// ValidateGitSourceAccess checks that the given Source exists and is a git Source, and returns it.
// TODO(BE-12905): enforce per-user access policies once Source ownership is introduced.
func ValidateGitSourceAccess(tx gitSourceStore, sourceID portainer.SourceID) (*portainer.Source, *httperror.HandlerError) {
src, err := tx.Source().Read(sourceID)
if err != nil {
if tx.IsErrObjectNotFound(err) {
return nil, httperror.NotFound("Source not found", err)
}
return nil, httperror.InternalServerError("Unable to read source", err)
}
if src.Type != portainer.SourceTypeGit {
return nil, httperror.BadRequest(fmt.Sprintf("source %d is not a git source", sourceID), nil)
}
if src.Git == nil {
return nil, httperror.BadRequest("Source has no git configuration", nil)
}
return src, nil
}

View File

@@ -1,49 +0,0 @@
package sources
import (
"net/http"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/datastore"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestValidateSourceForStack_ValidGitSource_ReturnsNil(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, false)
src := &portainer.Source{
Type: portainer.SourceTypeGit,
Git: &gittypes.RepoConfig{URL: "https://github.com/org/repo"},
}
require.NoError(t, store.Source().Create(src))
_, httpErr := ValidateGitSourceAccess(store, src.ID)
assert.Nil(t, httpErr)
}
func TestValidateSourceForStack_SourceNotFound_Returns404(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, false)
_, httpErr := ValidateGitSourceAccess(store, portainer.SourceID(999))
require.NotNil(t, httpErr)
assert.Equal(t, http.StatusNotFound, httpErr.StatusCode)
}
func TestValidateSourceForStack_NonGitSource_Returns400(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, false)
src := &portainer.Source{
Type: portainer.SourceType(99), // not a git source
}
require.NoError(t, store.Source().Create(src))
_, httpErr := ValidateGitSourceAccess(store, src.ID)
require.NotNil(t, httpErr)
assert.Equal(t, http.StatusBadRequest, httpErr.StatusCode)
}

View File

@@ -1,203 +0,0 @@
package workflows
import (
"context"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/kubernetes/cli"
"github.com/portainer/portainer/api/set"
)
// FetchWorkflows returns all GitOps workflows visible to the given user.
func FetchWorkflows(
ctx context.Context,
tx dataservices.DataStoreTx,
gitService portainer.GitService,
k8sFactory *cli.ClientFactory,
sc *security.RestrictedRequestContext,
endpointIDSet set.Set[portainer.EndpointID],
) ([]Workflow, error) {
gitConfigs := map[portainer.StackID]*gittypes.RepoConfig{}
stacks, err := tx.Stack().ReadAll(func(s portainer.Stack) bool {
return s.WorkflowID != 0 && (len(endpointIDSet) == 0 || endpointIDSet.Contains(s.EndpointID))
})
if err != nil {
return nil, err
}
endpointMap, err := buildEndpointMap(tx, stacks)
if err != nil {
return nil, err
}
stacks, err = filterDockerStacksByAccess(tx, stacks, sc)
if err != nil {
return nil, err
}
// First pass: filter by endpoint/stack-type match and collect workflow IDs.
preFiltered := make([]portainer.Stack, 0, len(stacks))
workflowIDSet := make(set.Set[portainer.WorkflowID], len(stacks))
for _, stack := range stacks {
if ep, ok := endpointMap[stack.EndpointID]; ok && !EndpointMatchesStackType(ep, stack.Type) {
continue
}
preFiltered = append(preFiltered, stack)
workflowIDSet.Add(stack.WorkflowID)
}
workflowMap, sourceMap, err := LoadWorkflowAndSourceMaps(tx, workflowIDSet)
if err != nil {
return nil, err
}
// Second pass: build filtered list using in-memory lookups.
var filtered []portainer.Stack
for _, stack := range preFiltered {
wf := workflowMap[stack.WorkflowID]
outer:
for _, as := range wf.Artifacts {
if as.StackID != stack.ID {
continue
}
for _, f := range as.Files {
src := sourceMap[f.SourceID]
if src.Type == portainer.SourceTypeGit {
gitConfigs[stack.ID] = MergeSourceAndFile(&src, &f)
break outer
}
}
}
filtered = append(filtered, stack)
}
stacks = filtered
accessMap, err := buildEndpointAccessMap(k8sFactory, sc, endpointMap)
if err != nil {
return nil, err
}
stacks, err = filterK8SStacks(stacks, endpointMap, k8sFactory, accessMap)
if err != nil {
return nil, err
}
items := make([]Workflow, 0, len(stacks))
for _, stack := range stacks {
gitConfig := gitConfigs[stack.ID]
source, artifact := ComputeGitPhasesForConfig(ctx, gitService, gitConfig)
items = append(items, MapStackToWorkflow(stack, gitConfig, source, artifact))
}
return items, nil
}
// SourceStats holds aggregated statistics for a GitOps source.
type SourceStats struct {
WorkflowCount int
EndpointIDs set.Set[portainer.EndpointID]
LastSync int64
}
// FetchSourceStats returns all sources and per-source stats for sources accessible to the given user.
// It applies the same access control as FetchWorkflows but skips git phase checks.
func FetchSourceStats(
tx dataservices.DataStoreTx,
k8sFactory *cli.ClientFactory,
sc *security.RestrictedRequestContext,
) ([]portainer.Source, map[portainer.SourceID]SourceStats, error) {
sources, err := tx.Source().ReadAll()
if err != nil {
return nil, nil, err
}
allStacks, err := tx.Stack().ReadAll(func(s portainer.Stack) bool { return s.WorkflowID != 0 })
if err != nil {
return nil, nil, err
}
endpointMap, err := buildEndpointMap(tx, allStacks)
if err != nil {
return nil, nil, err
}
allStacks, err = filterDockerStacksByAccess(tx, allStacks, sc)
if err != nil {
return nil, nil, err
}
workflowIDSet := make(set.Set[portainer.WorkflowID], len(allStacks))
preFiltered := make([]portainer.Stack, 0, len(allStacks))
for _, stack := range allStacks {
if ep, ok := endpointMap[stack.EndpointID]; ok && !EndpointMatchesStackType(ep, stack.Type) {
continue
}
preFiltered = append(preFiltered, stack)
workflowIDSet.Add(stack.WorkflowID)
}
wfMap, err := LoadWorkflowMap(tx, workflowIDSet)
if err != nil {
return nil, nil, err
}
wfSources := make(map[portainer.WorkflowID][]portainer.SourceID, len(wfMap))
for id, wf := range wfMap {
for _, as := range wf.Artifacts {
for _, f := range as.Files {
wfSources[id] = append(wfSources[id], f.SourceID)
}
}
}
stackSourceIDs := make(map[portainer.StackID][]portainer.SourceID)
for _, stack := range preFiltered {
if srcIDs := wfSources[stack.WorkflowID]; len(srcIDs) > 0 {
stackSourceIDs[stack.ID] = srcIDs
}
}
accessMap, err := buildEndpointAccessMap(k8sFactory, sc, endpointMap)
if err != nil {
return nil, nil, err
}
stacks, err := filterK8SStacks(preFiltered, endpointMap, k8sFactory, accessMap)
if err != nil {
return nil, nil, err
}
stats := make(map[portainer.SourceID]SourceStats)
for _, stack := range stacks {
var epIDs []portainer.EndpointID
if stack.EndpointID != 0 {
epIDs = []portainer.EndpointID{stack.EndpointID}
}
addSourceStats(stats, stackSourceIDs[stack.ID], epIDs, StackLastSyncDate(stack))
}
return sources, stats, nil
}
func addSourceStats(result map[portainer.SourceID]SourceStats, srcIDs []portainer.SourceID, epIDs []portainer.EndpointID, lastSync int64) {
for _, srcID := range srcIDs {
st := result[srcID]
if st.EndpointIDs == nil {
st.EndpointIDs = make(set.Set[portainer.EndpointID])
}
st.WorkflowCount++
for _, epID := range epIDs {
st.EndpointIDs.Add(epID)
}
st.LastSync = max(lastSync, st.LastSync)
result[srcID] = st
}
}

View File

@@ -1,282 +0,0 @@
package workflows
import (
"strconv"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/datastore"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/set"
"github.com/stretchr/testify/require"
)
func adminContext() *security.RestrictedRequestContext {
return &security.RestrictedRequestContext{IsAdmin: true, UserID: 1}
}
func mustCreateGitWorkflow(t *testing.T, tx dataservices.DataStoreTx, stack *portainer.Stack) {
t.Helper()
cfg := stack.GitConfig
src := &portainer.Source{Type: portainer.SourceTypeGit, Git: cfg}
require.NoError(t, tx.Source().Create(src))
wf := &portainer.Workflow{Artifacts: []portainer.Artifact{{
StackID: stack.ID,
Files: []portainer.ArtifactFile{{SourceID: src.ID}},
}}}
require.NoError(t, tx.Workflow().Create(wf))
stack.WorkflowID = wf.ID
stack.GitConfig = nil
require.NoError(t, tx.Stack().Create(stack))
}
func TestAddSourceStats_NoOp(t *testing.T) {
t.Parallel()
result := make(map[portainer.SourceID]SourceStats)
addSourceStats(result, nil, nil, 0)
require.Empty(t, result)
}
func TestAddSourceStats_AccumulatesWorkflowCount(t *testing.T) {
t.Parallel()
result := make(map[portainer.SourceID]SourceStats)
addSourceStats(result, []portainer.SourceID{1}, nil, 0)
addSourceStats(result, []portainer.SourceID{1}, nil, 0)
require.Equal(t, 2, result[1].WorkflowCount)
}
func TestAddSourceStats_CollectsUniqueEndpointIDs(t *testing.T) {
t.Parallel()
result := make(map[portainer.SourceID]SourceStats)
addSourceStats(result, []portainer.SourceID{1}, []portainer.EndpointID{10, 20}, 0)
addSourceStats(result, []portainer.SourceID{1}, []portainer.EndpointID{20, 30}, 0)
require.Len(t, result[1].EndpointIDs, 3)
require.True(t, result[1].EndpointIDs[10])
require.True(t, result[1].EndpointIDs[20])
require.True(t, result[1].EndpointIDs[30])
}
func TestAddSourceStats_MaxLastSync(t *testing.T) {
t.Parallel()
result := make(map[portainer.SourceID]SourceStats)
addSourceStats(result, []portainer.SourceID{1}, nil, 100)
addSourceStats(result, []portainer.SourceID{1}, nil, 500)
addSourceStats(result, []portainer.SourceID{1}, nil, 200)
require.Equal(t, int64(500), result[1].LastSync)
}
func TestAddSourceStats_MultipleSourceIDs(t *testing.T) {
t.Parallel()
result := make(map[portainer.SourceID]SourceStats)
addSourceStats(result, []portainer.SourceID{1, 2}, []portainer.EndpointID{10}, 100)
require.Equal(t, 1, result[1].WorkflowCount)
require.Equal(t, 1, result[2].WorkflowCount)
require.True(t, result[1].EndpointIDs[10])
require.True(t, result[2].EndpointIDs[10])
}
func TestFetchWorkflows_ReturnsOnlyGitopsStacks(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
mustCreateGitWorkflow(t, tx, &portainer.Stack{
ID: 1,
Name: "gitops-stack",
GitConfig: &gittypes.RepoConfig{URL: "https://github.com/x/repo"},
})
require.NoError(t, tx.Stack().Create(&portainer.Stack{ID: 2, Name: "plain-stack"}))
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
var items []Workflow
require.NoError(t, store.ViewTx(func(tx dataservices.DataStoreTx) error {
var err error
items, err = FetchWorkflows(t.Context(), tx, nil, nil, adminContext(), nil)
return err
}))
require.Len(t, items, 1)
require.Equal(t, "gitops-stack", items[0].Name)
}
func TestFetchWorkflows_FiltersByEndpointID(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
for i := 1; i <= 3; i++ {
mustCreateGitWorkflow(t, tx, &portainer.Stack{
ID: portainer.StackID(i),
Name: "stack-" + strconv.Itoa(i),
EndpointID: portainer.EndpointID(i),
GitConfig: &gittypes.RepoConfig{URL: "https://github.com/x/" + strconv.Itoa(i)},
})
}
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
var items []Workflow
require.NoError(t, store.ViewTx(func(tx dataservices.DataStoreTx) error {
var err error
items, err = FetchWorkflows(t.Context(), tx, nil, nil, adminContext(), set.ToSet([]portainer.EndpointID{1, 2}))
return err
}))
require.Len(t, items, 2)
names := []string{items[0].Name, items[1].Name}
require.Contains(t, names, "stack-1")
require.Contains(t, names, "stack-2")
}
func TestFetchWorkflows_EmptyWhenNoGitopsStacks(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
require.NoError(t, tx.Stack().Create(&portainer.Stack{ID: 1, Name: "plain-1"}))
require.NoError(t, tx.Stack().Create(&portainer.Stack{ID: 2, Name: "plain-2"}))
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
var items []Workflow
require.NoError(t, store.ViewTx(func(tx dataservices.DataStoreTx) error {
var err error
items, err = FetchWorkflows(t.Context(), tx, nil, nil, adminContext(), nil)
return err
}))
require.Empty(t, items)
}
func TestFetchWorkflows_NilEndpointSetReturnsAll(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
for i := 1; i <= 3; i++ {
mustCreateGitWorkflow(t, tx, &portainer.Stack{
ID: portainer.StackID(i),
Name: "stack-" + strconv.Itoa(i),
EndpointID: portainer.EndpointID(i),
GitConfig: &gittypes.RepoConfig{URL: "https://github.com/x/" + strconv.Itoa(i)},
})
}
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
var items []Workflow
require.NoError(t, store.ViewTx(func(tx dataservices.DataStoreTx) error {
var err error
items, err = FetchWorkflows(t.Context(), tx, nil, nil, adminContext(), nil)
return err
}))
require.Len(t, items, 3)
}
func TestFetchSourceStats_ReturnsAllSources(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
require.NoError(t, tx.Source().Create(&portainer.Source{Name: "source-1", Type: portainer.SourceTypeGit}))
require.NoError(t, tx.Source().Create(&portainer.Source{Name: "source-2", Type: portainer.SourceTypeGit}))
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
var sources []portainer.Source
require.NoError(t, store.ViewTx(func(tx dataservices.DataStoreTx) error {
var err error
sources, _, err = FetchSourceStats(tx, nil, adminContext())
return err
}))
require.Len(t, sources, 2)
}
func TestFetchSourceStats_TracksWorkflowCountAndEndpoints(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
var srcID portainer.SourceID
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
src := &portainer.Source{Name: "shared", Type: portainer.SourceTypeGit}
require.NoError(t, tx.Source().Create(src))
srcID = src.ID
for i := 1; i <= 2; i++ {
wf := &portainer.Workflow{Artifacts: []portainer.Artifact{{Files: []portainer.ArtifactFile{{SourceID: srcID}}}}}
require.NoError(t, tx.Workflow().Create(wf))
require.NoError(t, tx.Stack().Create(&portainer.Stack{
ID: portainer.StackID(i),
Name: "stack-" + strconv.Itoa(i),
EndpointID: portainer.EndpointID(i),
WorkflowID: wf.ID,
}))
}
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
var stats map[portainer.SourceID]SourceStats
require.NoError(t, store.ViewTx(func(tx dataservices.DataStoreTx) error {
var err error
_, stats, err = FetchSourceStats(tx, nil, adminContext())
return err
}))
st := stats[srcID]
require.Equal(t, 2, st.WorkflowCount)
require.Len(t, st.EndpointIDs, 2)
}
func TestFetchSourceStats_UnusedSourceHasZeroStats(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
var unusedID portainer.SourceID
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
src := &portainer.Source{Name: "unused", Type: portainer.SourceTypeGit}
require.NoError(t, tx.Source().Create(src))
unusedID = src.ID
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
var stats map[portainer.SourceID]SourceStats
require.NoError(t, store.ViewTx(func(tx dataservices.DataStoreTx) error {
var err error
_, stats, err = FetchSourceStats(tx, nil, adminContext())
return err
}))
st := stats[unusedID]
require.Zero(t, st.WorkflowCount)
require.Empty(t, st.EndpointIDs)
}

View File

@@ -1,289 +0,0 @@
package workflows
import (
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/datastore"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/authorization"
"github.com/portainer/portainer/api/kubernetes/cli"
appsv1 "k8s.io/api/apps/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kfake "k8s.io/client-go/kubernetes/fake"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestFilterDockerStacksByAccess_KubeStacksPassThrough(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
user := &portainer.User{
ID: 1,
Username: "standard",
Role: portainer.StandardUserRole,
PortainerAuthorizations: authorization.DefaultPortainerAuthorizations(),
}
require.NoError(t, store.User().Create(user))
sc := &security.RestrictedRequestContext{
IsAdmin: false,
UserID: 1,
}
kubeStack := portainer.Stack{ID: 1, Name: "kube-stack", Type: portainer.KubernetesStack}
dockerStack := portainer.Stack{ID: 2, Name: "docker-stack", Type: portainer.DockerComposeStack}
stacks := []portainer.Stack{kubeStack, dockerStack}
var result []portainer.Stack
err := store.ViewTx(func(tx dataservices.DataStoreTx) error {
var txErr error
result, txErr = filterDockerStacksByAccess(tx, stacks, sc)
return txErr
})
require.NoError(t, err)
require.Len(t, result, 1)
require.Equal(t, "kube-stack", result[0].Name)
}
func TestFilterDockerStacksByAccess_AdminGetsAll(t *testing.T) {
t.Parallel()
sc := &security.RestrictedRequestContext{
IsAdmin: true,
UserID: 1,
}
stacks := []portainer.Stack{
{ID: 1, Name: "kube-stack", Type: portainer.KubernetesStack},
{ID: 2, Name: "docker-stack", Type: portainer.DockerComposeStack},
}
result, err := filterDockerStacksByAccess(nil, stacks, sc)
require.NoError(t, err)
require.Len(t, result, 2)
}
func TestBuildEndpointAccessMap_AdminIsKubeAdmin(t *testing.T) {
t.Parallel()
sc := &security.RestrictedRequestContext{
IsAdmin: true,
UserID: 1,
}
endpointMap := map[portainer.EndpointID]portainer.Endpoint{
1: {ID: 1, Type: portainer.KubernetesLocalEnvironment},
2: {ID: 2, Type: portainer.DockerEnvironment},
}
result, err := buildEndpointAccessMap(nil, sc, endpointMap)
require.NoError(t, err)
require.Len(t, result, 1)
require.True(t, result[1].isKubeAdmin)
require.Empty(t, result[1].nonAdminNamespaces)
}
func TestFilterK8SStacks_IncludesMatchingStack(t *testing.T) {
t.Parallel()
fakeKubeClient := kfake.NewSimpleClientset()
deployment := &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: "my-app",
Namespace: "default",
Labels: map[string]string{
"io.portainer.kubernetes.application.stackid": "1",
},
},
Spec: appsv1.DeploymentSpec{
Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "my-app"}},
},
}
_, err := fakeKubeClient.AppsV1().Deployments("default").Create(t.Context(), deployment, metav1.CreateOptions{})
require.NoError(t, err)
kcl := cli.NewTestKubeClient(fakeKubeClient)
factory := cli.NewTestClientFactory(1, kcl)
endpointMap := map[portainer.EndpointID]portainer.Endpoint{
1: {ID: 1, Type: portainer.KubernetesLocalEnvironment},
}
stacks := []portainer.Stack{
{ID: 1, Name: "stack-name", EndpointID: 1, Type: portainer.KubernetesStack},
}
accessMap := map[portainer.EndpointID]endpointAccess{
1: {isKubeAdmin: true},
}
result, err := filterK8SStacks(stacks, endpointMap, factory, accessMap)
require.NoError(t, err)
require.Len(t, result, 1)
assert.Equal(t, "my-app", result[0].Name)
assert.Equal(t, "default", result[0].Namespace)
}
func TestFilterK8SStacks_ExcludesStackWhenNoMatchingDeployment(t *testing.T) {
t.Parallel()
fakeKubeClient := kfake.NewSimpleClientset()
kcl := cli.NewTestKubeClient(fakeKubeClient)
factory := cli.NewTestClientFactory(1, kcl)
endpointMap := map[portainer.EndpointID]portainer.Endpoint{
1: {ID: 1, Type: portainer.KubernetesLocalEnvironment},
}
stacks := []portainer.Stack{
{ID: 1, Name: "stack-name", EndpointID: 1, Type: portainer.KubernetesStack},
}
accessMap := map[portainer.EndpointID]endpointAccess{
1: {isKubeAdmin: true},
}
result, err := filterK8SStacks(stacks, endpointMap, factory, accessMap)
require.NoError(t, err)
require.Empty(t, result)
}
func TestFilterK8SStacks_NonAdminWithNamespaceAccess(t *testing.T) {
t.Parallel()
fakeKubeClient := kfake.NewSimpleClientset()
deployment := &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: "my-app",
Namespace: "ns1",
Labels: map[string]string{
"io.portainer.kubernetes.application.stackid": "1",
},
},
Spec: appsv1.DeploymentSpec{
Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "my-app"}},
},
}
_, err := fakeKubeClient.AppsV1().Deployments("ns1").Create(t.Context(), deployment, metav1.CreateOptions{})
require.NoError(t, err)
kcl := cli.NewTestKubeClient(fakeKubeClient)
factory := cli.NewTestClientFactory(1, kcl)
endpointMap := map[portainer.EndpointID]portainer.Endpoint{
1: {ID: 1, Type: portainer.KubernetesLocalEnvironment},
}
stacks := []portainer.Stack{
{ID: 1, Name: "stack-name", EndpointID: 1, Type: portainer.KubernetesStack},
}
accessMap := map[portainer.EndpointID]endpointAccess{
1: {isKubeAdmin: false, nonAdminNamespaces: []string{"ns1"}},
}
result, err := filterK8SStacks(stacks, endpointMap, factory, accessMap)
require.NoError(t, err)
require.Len(t, result, 1)
assert.Equal(t, "my-app", result[0].Name)
}
func TestResolveKubeAccess_NonAdminWithTeamMemberships(t *testing.T) {
t.Parallel()
fakeKubeClient := kfake.NewSimpleClientset()
kcl := cli.NewTestKubeClient(fakeKubeClient)
factory := cli.NewTestClientFactory(1, kcl)
ep := &portainer.Endpoint{
ID: 1,
Type: portainer.KubernetesLocalEnvironment,
}
sc := &security.RestrictedRequestContext{
IsAdmin: false,
UserID: 1,
UserMemberships: []portainer.TeamMembership{
{TeamID: 5},
},
}
access, err := resolveKubeAccess(factory, sc, ep)
require.NoError(t, err)
require.False(t, access.isKubeAdmin)
require.Equal(t, []string{"default"}, access.nonAdminNamespaces)
}
func TestResolveKubeAccess_NonAdmin(t *testing.T) {
t.Parallel()
fakeKubeClient := kfake.NewSimpleClientset()
kcl := cli.NewTestKubeClient(fakeKubeClient)
factory := cli.NewTestClientFactory(1, kcl)
ep := &portainer.Endpoint{
ID: 1,
Type: portainer.KubernetesLocalEnvironment,
}
sc := &security.RestrictedRequestContext{
IsAdmin: false,
UserID: 1,
}
access, err := resolveKubeAccess(factory, sc, ep)
require.NoError(t, err)
require.False(t, access.isKubeAdmin)
require.Equal(t, []string{"default"}, access.nonAdminNamespaces)
}
func TestFilterK8SStacks_NonAdminWithoutNamespaceAccess(t *testing.T) {
t.Parallel()
fakeKubeClient := kfake.NewSimpleClientset()
deployment := &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: "my-app",
Namespace: "ns1",
Labels: map[string]string{
"io.portainer.kubernetes.application.stackid": "1",
},
},
Spec: appsv1.DeploymentSpec{
Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "my-app"}},
},
}
_, err := fakeKubeClient.AppsV1().Deployments("ns1").Create(t.Context(), deployment, metav1.CreateOptions{})
require.NoError(t, err)
kcl := cli.NewTestKubeClient(fakeKubeClient)
factory := cli.NewTestClientFactory(1, kcl)
endpointMap := map[portainer.EndpointID]portainer.Endpoint{
1: {ID: 1, Type: portainer.KubernetesLocalEnvironment},
}
stacks := []portainer.Stack{
{ID: 1, Name: "stack-name", EndpointID: 1, Type: portainer.KubernetesStack},
}
accessMap := map[portainer.EndpointID]endpointAccess{
1: {isKubeAdmin: false, nonAdminNamespaces: []string{}},
}
result, err := filterK8SStacks(stacks, endpointMap, factory, accessMap)
require.NoError(t, err)
require.Empty(t, result)
}

View File

@@ -5,50 +5,17 @@ import (
"fmt"
"path"
"slices"
portainer "github.com/portainer/portainer/api"
gittypes "github.com/portainer/portainer/api/git/types"
)
// ListRefsFunc lists all git refs for a repository.
type ListRefsFunc func(ctx context.Context) ([]string, error)
// ListFilesFunc lists files in a repository branch filtered by extension.
type ListFilesFunc func(ctx context.Context, exts []string, dirOnly bool) ([]string, error)
// GitEntries represents a git entry which can be either a file or a directory.
type GitEntries struct {
Name string
IsFile bool
}
// ComputeGitPhasesForConfig computes source and artifact phases from a RepoConfig and a GitService.
func ComputeGitPhasesForConfig(ctx context.Context, gitSvc portainer.GitService, cfg *gittypes.RepoConfig) (source, artifact WorkflowPhaseStatus) {
if gitSvc == nil || cfg == nil {
return WorkflowPhaseStatus{Status: StatusUnknown}, WorkflowPhaseStatus{Status: StatusUnknown}
}
username, password := gitCredentials(cfg)
return ComputeGitPhases(ctx, cfg.ReferenceName, []GitEntries{{Name: cfg.ConfigFilePath, IsFile: true}},
func(ctx context.Context) ([]string, error) {
return gitSvc.ListRefs(ctx, cfg.URL, username, password, false, cfg.TLSSkipVerify)
},
func(ctx context.Context, exts []string, dirOnly bool) ([]string, error) {
return gitSvc.ListFiles(ctx, cfg.URL, cfg.ReferenceName, username, password, dirOnly, false, exts, cfg.TLSSkipVerify)
},
)
}
func gitCredentials(cfg *gittypes.RepoConfig) (username, password string) {
if cfg.Authentication != nil {
return cfg.Authentication.Username, cfg.Authentication.Password
}
return "", ""
}
type ListFilesFunc func(ctx context.Context, exts []string) ([]string, error)
// ComputeGitPhases checks source (ref reachability) and artifact (config file presence).
// If source fails, artifact is returned as unknown without making a network call.
func ComputeGitPhases(ctx context.Context, referenceName string, configFilePath []GitEntries, listRefs ListRefsFunc, listFiles ListFilesFunc) (source, artifact WorkflowPhaseStatus) {
func ComputeGitPhases(ctx context.Context, referenceName, configFilePath string, listRefs ListRefsFunc, listFiles ListFilesFunc) (source, artifact WorkflowPhaseStatus) {
source = computeSourcePhase(ctx, referenceName, listRefs)
if source.Status == StatusError {
return source, WorkflowPhaseStatus{Status: StatusUnknown}
@@ -70,58 +37,23 @@ func computeSourcePhase(ctx context.Context, referenceName string, listRefs List
return WorkflowPhaseStatus{Status: StatusHealthy}
}
func computeArtifactPhase(ctx context.Context, gitEntries []GitEntries, listFiles ListFilesFunc) WorkflowPhaseStatus {
if len(gitEntries) == 0 {
func computeArtifactPhase(ctx context.Context, configFilePath string, listFiles ListFilesFunc) WorkflowPhaseStatus {
if configFilePath == "" {
return WorkflowPhaseStatus{Status: StatusError, Error: "no config file path specified"}
}
var (
exts []string
fileEntries []string
dirEntries []string
)
for _, gitEntry := range gitEntries {
if gitEntry.IsFile {
ext := path.Ext(gitEntry.Name)
if len(ext) > 0 {
ext = ext[1:]
exts = append(exts, ext)
}
fileEntries = append(fileEntries, gitEntry.Name)
continue
}
dirEntries = append(dirEntries, gitEntry.Name)
ext := path.Ext(configFilePath)
var exts []string
if len(ext) > 0 {
ext = ext[1:]
exts = []string{ext}
}
// Check file entries
if len(fileEntries) > 0 {
files, err := listFiles(ctx, exts, false)
if err != nil {
return WorkflowPhaseStatus{Status: StatusError, Error: err.Error()}
}
for _, fileEntry := range fileEntries {
if !slices.Contains(files, fileEntry) {
return WorkflowPhaseStatus{Status: StatusError, Error: fmt.Sprintf("file %q not found", fileEntry)}
}
}
files, err := listFiles(ctx, exts)
if err != nil {
return WorkflowPhaseStatus{Status: StatusError, Error: err.Error()}
}
// Check directory entries
if len(dirEntries) > 0 {
dirs, err := listFiles(ctx, nil, true)
if err != nil {
return WorkflowPhaseStatus{Status: StatusError, Error: err.Error()}
}
for _, dirEntry := range dirEntries {
if !slices.Contains(dirs, dirEntry) {
return WorkflowPhaseStatus{Status: StatusError, Error: fmt.Sprintf("directory %q not found", dirEntry)}
}
}
if !slices.Contains(files, configFilePath) {
return WorkflowPhaseStatus{Status: StatusError, Error: fmt.Sprintf("file %q not found", configFilePath)}
}
return WorkflowPhaseStatus{Status: StatusHealthy}
}

View File

@@ -14,38 +14,38 @@ func TestComputeGitPhases(t *testing.T) {
okRefs := func(_ context.Context) ([]string, error) {
return []string{"refs/heads/main"}, nil
}
okFiles := func(_ context.Context, _ []string, _ bool) ([]string, error) {
okFiles := func(_ context.Context, _ []string) ([]string, error) {
return []string{"docker-compose.yml"}, nil
}
errRefs := func(_ context.Context) ([]string, error) {
return nil, errors.New("connection refused")
}
errFiles := func(_ context.Context, _ []string, _ bool) ([]string, error) {
errFiles := func(_ context.Context, _ []string) ([]string, error) {
return nil, errors.New("connection refused")
}
cases := []struct {
name string
referenceName string
configFilePath []GitEntries
configFilePath string
listRefs ListRefsFunc
listFiles ListFilesFunc
expectedSource Status
expectedArtifact Status
}{
{
name: "listRefs errors: source error, artifact unknown",
name: "listRefs errors source error, artifact unknown",
referenceName: "refs/heads/main",
configFilePath: []GitEntries{{Name: "docker-compose.yml", IsFile: true}},
configFilePath: "docker-compose.yml",
listRefs: errRefs,
listFiles: okFiles,
expectedSource: StatusError,
expectedArtifact: StatusUnknown,
},
{
name: "ref not in list: source error, artifact unknown",
name: "ref not in list source error, artifact unknown",
referenceName: "refs/heads/missing",
configFilePath: []GitEntries{{Name: "docker-compose.yml", IsFile: true}},
configFilePath: "docker-compose.yml",
listRefs: func(_ context.Context) ([]string, error) {
return []string{"refs/heads/main"}, nil
},
@@ -54,29 +54,29 @@ func TestComputeGitPhases(t *testing.T) {
expectedArtifact: StatusUnknown,
},
{
name: "empty configFilePath: artifact error",
name: "empty configFilePath artifact error",
referenceName: "refs/heads/main",
configFilePath: []GitEntries{},
configFilePath: "",
listRefs: okRefs,
listFiles: okFiles,
expectedSource: StatusHealthy,
expectedArtifact: StatusError,
},
{
name: "listFiles errors: artifact error",
name: "listFiles errors artifact error",
referenceName: "refs/heads/main",
configFilePath: []GitEntries{{Name: "docker-compose.yml", IsFile: true}},
configFilePath: "docker-compose.yml",
listRefs: okRefs,
listFiles: errFiles,
expectedSource: StatusHealthy,
expectedArtifact: StatusError,
},
{
name: "file not in list: artifact error",
name: "file not in list artifact error",
referenceName: "refs/heads/main",
configFilePath: []GitEntries{{Name: "docker-compose.yml", IsFile: true}},
configFilePath: "docker-compose.yml",
listRefs: okRefs,
listFiles: func(_ context.Context, _ []string, _ bool) ([]string, error) {
listFiles: func(_ context.Context, _ []string) ([]string, error) {
return []string{"other.yml"}, nil
},
expectedSource: StatusHealthy,
@@ -85,16 +85,16 @@ func TestComputeGitPhases(t *testing.T) {
{
name: "both healthy",
referenceName: "refs/heads/main",
configFilePath: []GitEntries{{Name: "docker-compose.yml", IsFile: true}},
configFilePath: "docker-compose.yml",
listRefs: okRefs,
listFiles: okFiles,
expectedSource: StatusHealthy,
expectedArtifact: StatusHealthy,
},
{
name: "empty referenceName: source healthy (default HEAD)",
name: "empty referenceName source healthy (default HEAD)",
referenceName: "",
configFilePath: []GitEntries{{Name: "docker-compose.yml", IsFile: true}},
configFilePath: "docker-compose.yml",
listRefs: okRefs,
listFiles: okFiles,
expectedSource: StatusHealthy,
@@ -132,9 +132,9 @@ func TestComputeArtifactPhase_ExtensionFilter(t *testing.T) {
ComputeGitPhases(
t.Context(),
"",
[]GitEntries{{Name: tc.configPath, IsFile: true}},
tc.configPath,
func(_ context.Context) ([]string, error) { return nil, nil },
func(_ context.Context, exts []string, dirOnly bool) ([]string, error) {
func(_ context.Context, exts []string) ([]string, error) {
capturedExts = exts
return []string{tc.configPath}, nil
},
@@ -151,12 +151,12 @@ func TestComputeGitPhases_ArtifactNotCalledOnSourceError(t *testing.T) {
listRefs := func(_ context.Context) ([]string, error) {
return nil, errors.New("repo unreachable")
}
listFiles := func(_ context.Context, _ []string, _ bool) ([]string, error) {
listFiles := func(_ context.Context, _ []string) ([]string, error) {
listFilesCalled = true
return nil, nil
}
ComputeGitPhases(t.Context(), "refs/heads/main", []GitEntries{{Name: "docker-compose.yml", IsFile: true}}, listRefs, listFiles)
ComputeGitPhases(t.Context(), "refs/heads/main", "docker-compose.yml", listRefs, listFiles)
assert.False(t, listFilesCalled, "listFiles must not be called when source fails")
}

Some files were not shown because too many files have changed in this diff Show More