feat(gitops): introduce sources list view [BE-12902] (#2550)

This commit is contained in:
Chaim Lev-Ari
2026-05-12 15:32:46 +03:00
committed by GitHub
parent edff47fd41
commit d9a415f011
52 changed files with 2655 additions and 995 deletions
-122
View File
@@ -1,122 +0,0 @@
package workflows
import (
"fmt"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/authorization"
"github.com/portainer/portainer/api/internal/endpointutils"
"github.com/portainer/portainer/api/internal/snapshot"
"github.com/portainer/portainer/api/kubernetes/cli"
"github.com/portainer/portainer/api/set"
"github.com/portainer/portainer/api/slicesx"
"github.com/portainer/portainer/api/stacks/stackutils"
)
func endpointMatchesStackType(ep portainer.Endpoint, stackType portainer.StackType) bool {
switch stackType {
case portainer.DockerSwarmStack:
return len(ep.Snapshots) > 0 && ep.Snapshots[0].Swarm
case portainer.DockerComposeStack:
return len(ep.Snapshots) == 0 || !ep.Snapshots[0].Swarm
case portainer.KubernetesStack:
return endpointutils.IsKubernetesEndpoint(&ep)
default:
return true
}
}
func buildEndpointMap(tx dataservices.DataStoreTx, stacks []portainer.Stack) (map[portainer.EndpointID]portainer.Endpoint, error) {
ids := set.ToSet(slicesx.Map(stacks, func(s portainer.Stack) portainer.EndpointID { return s.EndpointID }))
endpoints, err := tx.Endpoint().ReadAll(func(ep portainer.Endpoint) bool { return ids[ep.ID] })
if err != nil {
return nil, err
}
m := make(map[portainer.EndpointID]portainer.Endpoint, len(endpoints))
for i := range endpoints {
if err := snapshot.FillSnapshotData(tx, &endpoints[i], false); err != nil {
return nil, fmt.Errorf("unable to fill snapshot data for endpoint %d: %w", endpoints[i].ID, err)
}
m[endpoints[i].ID] = endpoints[i]
}
return m, nil
}
type endpointAccess struct {
isKubeAdmin bool
nonAdminNamespaces []string
}
// filterDockerStacksByAccess filters stacks to only those the current user can access.
func filterDockerStacksByAccess(tx dataservices.DataStoreTx, stacks []portainer.Stack, sc *security.RestrictedRequestContext) ([]portainer.Stack, error) {
if sc.IsAdmin {
return stacks, nil
}
// do not try to check UAC on kube stacks
filtered, dockerStacks := slicesx.Partition(stacks, func(s portainer.Stack) bool { return s.Type == portainer.KubernetesStack })
stackResourceIDSet := set.ToSet(slicesx.Map(dockerStacks, func(s portainer.Stack) string {
return stackutils.ResourceControlID(s.EndpointID, s.Name)
}))
resourceControls, err := tx.ResourceControl().ReadAll(func(rc portainer.ResourceControl) bool {
return rc.Type == portainer.StackResourceControl && stackResourceIDSet[rc.ResourceID]
})
if err != nil {
return nil, err
}
dockerStacks = authorization.DecorateStacks(dockerStacks, resourceControls)
userTeamIDs := authorization.TeamIDs(sc.UserMemberships)
filtered = append(filtered, authorization.FilterAuthorizedStacks(dockerStacks, sc.UserID, userTeamIDs)...)
return filtered, nil
}
func resolveKubeAccess(k8sFactory *cli.ClientFactory, sc *security.RestrictedRequestContext, ep *portainer.Endpoint) (endpointAccess, error) {
if sc.IsAdmin {
return endpointAccess{isKubeAdmin: true}, nil
}
pcli, err := k8sFactory.GetPrivilegedKubeClient(ep)
if err != nil {
return endpointAccess{}, fmt.Errorf("unable to get privileged kube client for endpoint %d: %w", ep.ID, err)
}
teamIDs := make([]int, 0, len(sc.UserMemberships))
for _, m := range sc.UserMemberships {
teamIDs = append(teamIDs, int(m.TeamID))
}
nonAdminNamespaces, err := pcli.GetNonAdminNamespaces(int(sc.UserID), teamIDs, ep.Kubernetes.Configuration.RestrictDefaultNamespace)
if err != nil {
return endpointAccess{}, fmt.Errorf("unable to retrieve non-admin namespaces for endpoint %d: %w", ep.ID, err)
}
return endpointAccess{isKubeAdmin: false, nonAdminNamespaces: nonAdminNamespaces}, nil
}
func buildEndpointAccessMap(k8sFactory *cli.ClientFactory, sc *security.RestrictedRequestContext, endpointMap map[portainer.EndpointID]portainer.Endpoint) (map[portainer.EndpointID]endpointAccess, error) {
result := make(map[portainer.EndpointID]endpointAccess, len(endpointMap))
for epID, ep := range endpointMap {
if !endpointutils.IsKubernetesEndpoint(&ep) {
continue
}
access, err := resolveKubeAccess(k8sFactory, sc, &ep)
if err != nil {
return nil, err
}
result[epID] = access
}
return result, nil
}
@@ -5,17 +5,10 @@ import (
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/datastore"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/authorization"
"github.com/portainer/portainer/api/kubernetes/cli"
"github.com/portainer/portainer/api/stacks/stackutils"
appsv1 "k8s.io/api/apps/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kfake "k8s.io/client-go/kubernetes/fake"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -85,274 +78,3 @@ func TestWorkflowsList_RBAC_NonAdminWithAccess(t *testing.T) {
require.Len(t, items, 1)
assert.Equal(t, stackName, items[0].Name)
}
func TestFilterDockerStacksByAccess_KubeStacksPassThrough(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
user := &portainer.User{
ID: 1,
Username: "standard",
Role: portainer.StandardUserRole,
PortainerAuthorizations: authorization.DefaultPortainerAuthorizations(),
}
require.NoError(t, store.User().Create(user))
sc := &security.RestrictedRequestContext{
IsAdmin: false,
UserID: 1,
}
kubeStack := portainer.Stack{ID: 1, Name: "kube-stack", Type: portainer.KubernetesStack}
dockerStack := portainer.Stack{ID: 2, Name: "docker-stack", Type: portainer.DockerComposeStack}
stacks := []portainer.Stack{kubeStack, dockerStack}
var result []portainer.Stack
err := store.ViewTx(func(tx dataservices.DataStoreTx) error {
var txErr error
result, txErr = filterDockerStacksByAccess(tx, stacks, sc)
return txErr
})
require.NoError(t, err)
require.Len(t, result, 1)
require.Equal(t, "kube-stack", result[0].Name)
}
func TestFilterDockerStacksByAccess_AdminGetsAll(t *testing.T) {
t.Parallel()
sc := &security.RestrictedRequestContext{
IsAdmin: true,
UserID: 1,
}
stacks := []portainer.Stack{
{ID: 1, Name: "kube-stack", Type: portainer.KubernetesStack},
{ID: 2, Name: "docker-stack", Type: portainer.DockerComposeStack},
}
result, err := filterDockerStacksByAccess(nil, stacks, sc)
require.NoError(t, err)
require.Len(t, result, 2)
}
func TestBuildEndpointAccessMap_AdminIsKubeAdmin(t *testing.T) {
t.Parallel()
sc := &security.RestrictedRequestContext{
IsAdmin: true,
UserID: 1,
}
endpointMap := map[portainer.EndpointID]portainer.Endpoint{
1: {ID: 1, Type: portainer.KubernetesLocalEnvironment},
2: {ID: 2, Type: portainer.DockerEnvironment},
}
result, err := buildEndpointAccessMap(nil, sc, endpointMap)
require.NoError(t, err)
require.Len(t, result, 1)
require.True(t, result[1].isKubeAdmin)
require.Empty(t, result[1].nonAdminNamespaces)
}
func TestFilterK8SStacks_IncludesMatchingStack(t *testing.T) {
t.Parallel()
fakeKubeClient := kfake.NewSimpleClientset()
deployment := &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: "my-app",
Namespace: "default",
Labels: map[string]string{
"io.portainer.kubernetes.application.stackid": "1",
},
},
Spec: appsv1.DeploymentSpec{
Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "my-app"}},
},
}
_, err := fakeKubeClient.AppsV1().Deployments("default").Create(t.Context(), deployment, metav1.CreateOptions{})
require.NoError(t, err)
kcl := cli.NewTestKubeClient(fakeKubeClient)
factory := cli.NewTestClientFactory(1, kcl)
endpointMap := map[portainer.EndpointID]portainer.Endpoint{
1: {ID: 1, Type: portainer.KubernetesLocalEnvironment},
}
stacks := []portainer.Stack{
{ID: 1, Name: "stack-name", EndpointID: 1, Type: portainer.KubernetesStack},
}
accessMap := map[portainer.EndpointID]endpointAccess{
1: {isKubeAdmin: true},
}
result, err := filterK8SStacks(stacks, endpointMap, factory, accessMap)
require.NoError(t, err)
require.Len(t, result, 1)
assert.Equal(t, "my-app", result[0].Name)
assert.Equal(t, "default", result[0].Namespace)
}
func TestFilterK8SStacks_ExcludesStackWhenNoMatchingDeployment(t *testing.T) {
t.Parallel()
fakeKubeClient := kfake.NewSimpleClientset()
kcl := cli.NewTestKubeClient(fakeKubeClient)
factory := cli.NewTestClientFactory(1, kcl)
endpointMap := map[portainer.EndpointID]portainer.Endpoint{
1: {ID: 1, Type: portainer.KubernetesLocalEnvironment},
}
stacks := []portainer.Stack{
{ID: 1, Name: "stack-name", EndpointID: 1, Type: portainer.KubernetesStack},
}
accessMap := map[portainer.EndpointID]endpointAccess{
1: {isKubeAdmin: true},
}
result, err := filterK8SStacks(stacks, endpointMap, factory, accessMap)
require.NoError(t, err)
require.Empty(t, result)
}
func TestFilterK8SStacks_NonAdminWithNamespaceAccess(t *testing.T) {
t.Parallel()
fakeKubeClient := kfake.NewSimpleClientset()
deployment := &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: "my-app",
Namespace: "ns1",
Labels: map[string]string{
"io.portainer.kubernetes.application.stackid": "1",
},
},
Spec: appsv1.DeploymentSpec{
Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "my-app"}},
},
}
_, err := fakeKubeClient.AppsV1().Deployments("ns1").Create(t.Context(), deployment, metav1.CreateOptions{})
require.NoError(t, err)
kcl := cli.NewTestKubeClient(fakeKubeClient)
factory := cli.NewTestClientFactory(1, kcl)
endpointMap := map[portainer.EndpointID]portainer.Endpoint{
1: {ID: 1, Type: portainer.KubernetesLocalEnvironment},
}
stacks := []portainer.Stack{
{ID: 1, Name: "stack-name", EndpointID: 1, Type: portainer.KubernetesStack},
}
accessMap := map[portainer.EndpointID]endpointAccess{
1: {isKubeAdmin: false, nonAdminNamespaces: []string{"ns1"}},
}
result, err := filterK8SStacks(stacks, endpointMap, factory, accessMap)
require.NoError(t, err)
require.Len(t, result, 1)
assert.Equal(t, "my-app", result[0].Name)
}
func TestResolveKubeAccess_NonAdminWithTeamMemberships(t *testing.T) {
t.Parallel()
fakeKubeClient := kfake.NewSimpleClientset()
kcl := cli.NewTestKubeClient(fakeKubeClient)
factory := cli.NewTestClientFactory(1, kcl)
ep := &portainer.Endpoint{
ID: 1,
Type: portainer.KubernetesLocalEnvironment,
}
sc := &security.RestrictedRequestContext{
IsAdmin: false,
UserID: 1,
UserMemberships: []portainer.TeamMembership{
{TeamID: 5},
},
}
access, err := resolveKubeAccess(factory, sc, ep)
require.NoError(t, err)
require.False(t, access.isKubeAdmin)
require.Equal(t, []string{"default"}, access.nonAdminNamespaces)
}
func TestResolveKubeAccess_NonAdmin(t *testing.T) {
t.Parallel()
fakeKubeClient := kfake.NewSimpleClientset()
kcl := cli.NewTestKubeClient(fakeKubeClient)
factory := cli.NewTestClientFactory(1, kcl)
ep := &portainer.Endpoint{
ID: 1,
Type: portainer.KubernetesLocalEnvironment,
}
sc := &security.RestrictedRequestContext{
IsAdmin: false,
UserID: 1,
}
access, err := resolveKubeAccess(factory, sc, ep)
require.NoError(t, err)
require.False(t, access.isKubeAdmin)
require.Equal(t, []string{"default"}, access.nonAdminNamespaces)
}
func TestFilterK8SStacks_NonAdminWithoutNamespaceAccess(t *testing.T) {
t.Parallel()
fakeKubeClient := kfake.NewSimpleClientset()
deployment := &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: "my-app",
Namespace: "ns1",
Labels: map[string]string{
"io.portainer.kubernetes.application.stackid": "1",
},
},
Spec: appsv1.DeploymentSpec{
Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "my-app"}},
},
}
_, err := fakeKubeClient.AppsV1().Deployments("ns1").Create(t.Context(), deployment, metav1.CreateOptions{})
require.NoError(t, err)
kcl := cli.NewTestKubeClient(fakeKubeClient)
factory := cli.NewTestClientFactory(1, kcl)
endpointMap := map[portainer.EndpointID]portainer.Endpoint{
1: {ID: 1, Type: portainer.KubernetesLocalEnvironment},
}
stacks := []portainer.Stack{
{ID: 1, Name: "stack-name", EndpointID: 1, Type: portainer.KubernetesStack},
}
accessMap := map[portainer.EndpointID]endpointAccess{
1: {isKubeAdmin: false, nonAdminNamespaces: []string{}},
}
result, err := filterK8SStacks(stacks, endpointMap, factory, accessMap)
require.NoError(t, err)
require.Empty(t, result)
}
@@ -1,32 +0,0 @@
package workflows
import (
"context"
portainer "github.com/portainer/portainer/api"
gittypes "github.com/portainer/portainer/api/git/types"
wf "github.com/portainer/portainer/api/gitops/workflows"
)
func computeGitPhases(ctx context.Context, gitSvc portainer.GitService, cfg *gittypes.RepoConfig) (source, artifact wf.WorkflowPhaseStatus) {
if gitSvc == nil || cfg == nil {
return wf.WorkflowPhaseStatus{Status: wf.StatusUnknown}, wf.WorkflowPhaseStatus{Status: wf.StatusUnknown}
}
username, password := gitCredentials(cfg)
return wf.ComputeGitPhases(ctx, cfg.ReferenceName, cfg.ConfigFilePath,
func(ctx context.Context) ([]string, error) {
return gitSvc.ListRefs(ctx, cfg.URL, username, password, false, cfg.TLSSkipVerify)
},
func(ctx context.Context, exts []string) ([]string, error) {
return gitSvc.ListFiles(ctx, cfg.URL, cfg.ReferenceName, username, password, false, false, exts, cfg.TLSSkipVerify)
},
)
}
func gitCredentials(cfg *gittypes.RepoConfig) (username, password string) {
if cfg.Authentication != nil {
return cfg.Authentication.Username, cfg.Authentication.Password
}
return "", ""
}
+1 -113
View File
@@ -10,13 +10,9 @@ import (
gocache "github.com/patrickmn/go-cache"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
svc "github.com/portainer/portainer/api/gitops/workflows"
"github.com/portainer/portainer/api/http/models/kubernetes"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/http/utils/filters"
"github.com/portainer/portainer/api/internal/endpointutils"
"github.com/portainer/portainer/api/kubernetes/cli"
"github.com/portainer/portainer/api/set"
"github.com/portainer/portainer/api/slicesx"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
@@ -141,115 +137,7 @@ func (h *Handler) getWorkflows(ctx context.Context, key string, sc *security.Res
}
func (h *Handler) fetchWorkflows(ctx context.Context, sc *security.RestrictedRequestContext, endpointIDSet set.Set[portainer.EndpointID]) ([]svc.Workflow, error) {
var entries []portainer.Stack
var endpointMap map[portainer.EndpointID]portainer.Endpoint
err := h.dataStore.ViewTx(func(tx dataservices.DataStoreTx) error {
stacks, err := tx.Stack().ReadAll(func(s portainer.Stack) bool {
return s.GitConfig != nil && (len(endpointIDSet) == 0 || endpointIDSet.Contains(s.EndpointID))
})
if err != nil {
return err
}
endpointMap, err = buildEndpointMap(tx, stacks)
if err != nil {
return err
}
stacks, err = filterDockerStacksByAccess(tx, stacks, sc)
if err != nil {
return err
}
for i := range stacks {
s := stacks[i]
if ep, ok := endpointMap[s.EndpointID]; ok && !endpointMatchesStackType(ep, s.Type) {
continue
}
entries = append(entries, s)
}
return nil
})
if err != nil {
return nil, err
}
accessMap, err := buildEndpointAccessMap(h.k8sFactory, sc, endpointMap)
if err != nil {
return nil, err
}
entries, err = filterK8SStacks(entries, endpointMap, h.k8sFactory, accessMap)
if err != nil {
return nil, err
}
items := make([]svc.Workflow, 0, len(entries))
for _, s := range entries {
source, artifact := computeGitPhases(ctx, h.gitService, s.GitConfig)
items = append(items, svc.MapStackToWorkflow(s, s.GitConfig, source, artifact))
}
return items, nil
}
// lookup only if env is kube and either not edge or (edge + not async)
func shouldPerformEnvLookup(endpoint *portainer.Endpoint) bool {
return endpointutils.IsKubernetesEndpoint(endpoint) &&
(!endpointutils.IsEdgeEndpoint(endpoint) ||
(endpointutils.IsEdgeEndpoint(endpoint) && !endpoint.Edge.AsyncMode))
}
func filterK8SStacks(items []portainer.Stack, endpointMap map[portainer.EndpointID]portainer.Endpoint, k8sFactory *cli.ClientFactory, accessMap map[portainer.EndpointID]endpointAccess) ([]portainer.Stack, error) {
k8sStacks, result := slicesx.Partition(items, func(s portainer.Stack) bool {
return s.Type == portainer.KubernetesStack
})
groupedByEnvId := slicesx.GroupBy(k8sStacks, func(s portainer.Stack) portainer.EndpointID {
return s.EndpointID
})
for envID, stacks := range groupedByEnvId {
ep, ok := endpointMap[envID]
if !ok || !shouldPerformEnvLookup(&ep) {
continue
}
kcl, err := k8sFactory.GetPrivilegedKubeClient(&ep)
if err != nil {
return nil, err
}
access := accessMap[envID]
kcl.SetIsKubeAdmin(access.isKubeAdmin)
kcl.SetClientNonAdminNamespaces(access.nonAdminNamespaces)
apps, err := kcl.GetApplications("", "")
if err != nil {
return nil, err
}
for _, s := range stacks {
idx := slices.IndexFunc(apps, func(app kubernetes.K8sApplication) bool {
return app.StackKind != "edge" && app.StackID == strconv.Itoa(int(s.ID))
})
if idx == -1 {
// if we don't find a matching application (deployment/statefulset/daemonset) in the environment workloads
// this workflow (stack) wouldn't show in the Applications list, so we don't keep it
continue
}
app := apps[idx]
s.Name = app.Name
s.Namespace = app.ResourcePool
result = append(result, s)
}
}
return result, nil
return svc.FetchWorkflows(ctx, h.dataStore, h.gitService, h.k8sFactory, sc, endpointIDSet)
}
func cacheKey(sc *security.RestrictedRequestContext, endpointIDs []portainer.EndpointID) string {