Compare commits

...

9 Commits

Author SHA1 Message Date
Oscar Zhou bdb84617fe chore(version): bump version to 2.21.1 (#12203) 2024-09-09 09:39:18 -03:00
andres-portainer 2d5c834590 fix(users): fix data-race in userCreate() BE-11209 (#12194) 2024-09-05 22:28:11 -03:00
andres-portainer 280ca22aeb fix(teams): fix data-race in teamCreate() BE-11210 (#12196) 2024-09-05 21:36:26 -03:00
Oscar Zhou 753150e03c fix(stack): env placeholder as host path [BE-11187] (#12186) 2024-09-06 08:42:55 +12:00
Yajith Dayarathna 517abc662a update ci workflow (release/2.21) (#12184) 2024-09-05 09:19:20 +12:00
andres-portainer 04e9ee3b3e fix(docker): avoid specifying the MAC address of container for Docker API < v1.44 BE-10880 (#12178) 2024-09-03 10:31:19 -03:00
andres-portainer 273ea5df23 fix(jwt): generate JWT IDs BE-11179 (#12176) 2024-09-02 12:06:44 -03:00
andres-portainer 6cc95e11ae fix(bouncer): add support for JWT revocation BE-11179 (#12165) 2024-08-30 20:24:14 -03:00
andres-portainer 9133cbf544 fix(git): optimize listFiles() BE-11184 (#12161) 2024-08-29 19:07:17 -03:00
27 changed files with 858 additions and 88 deletions
+3 -13
View File
@@ -34,7 +34,6 @@ jobs:
- { platform: linux, arch: arm64, version: "" }
- { platform: linux, arch: arm, version: "" }
- { platform: linux, arch: ppc64le, version: "" }
- { platform: linux, arch: s390x, version: "" }
- { platform: windows, arch: amd64, version: 1809 }
- { platform: windows, arch: amd64, version: ltsc2022 }
runs-on: ubuntu-latest
@@ -146,31 +145,22 @@ jobs:
# for instance, feature/1.0.0 -> feature-1.0.0
CONTAINER_IMAGE_TAG=$(echo $GITHUB_REF_NAME | sed 's/\//-/g')
fi
docker buildx imagetools create -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}" \
"${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-amd64" \
"${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm64" \
"${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm" \
"${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-ppc64le" \
"${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-s390x" \
"${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-windows1809-amd64" \
"${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-windowsltsc2022-amd64"
docker buildx imagetools create -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-alpine" \
"${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-amd64-alpine" \
"${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm64-alpine" \
"${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm-alpine"
"${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm-alpine" \
"${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-ppc64le-alpine"
if [[ "${GITHUB_REF_NAME}" =~ ^release/.*$ ]]; then
docker buildx imagetools create -t "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}" \
"${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-amd64" \
"${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm64" \
"${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm" \
"${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-ppc64le" \
"${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-s390x"
docker buildx imagetools create -t "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-alpine" \
"${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-amd64-alpine" \
"${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm64-alpine" \
"${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm-alpine"
"${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm64"
fi
+11 -2
View File
@@ -19,8 +19,7 @@ type Service struct {
// NewService creates a new instance of a service.
func NewService(connection portainer.Connection) (*Service, error) {
err := connection.SetServiceName(BucketName)
if err != nil {
if err := connection.SetServiceName(BucketName); err != nil {
return nil, err
}
@@ -32,6 +31,16 @@ func NewService(connection portainer.Connection) (*Service, error) {
}, nil
}
func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
return ServiceTx{
BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.Team, portainer.TeamID]{
Bucket: BucketName,
Connection: service.Connection,
Tx: tx,
},
}
}
// TeamByName returns a team by name.
func (service *Service) TeamByName(name string) (*portainer.Team, error) {
var t portainer.Team
+48
View File
@@ -0,0 +1,48 @@
package team
import (
"errors"
"strings"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
dserrors "github.com/portainer/portainer/api/dataservices/errors"
)
type ServiceTx struct {
dataservices.BaseDataServiceTx[portainer.Team, portainer.TeamID]
}
// TeamByName returns a team by name.
func (service ServiceTx) TeamByName(name string) (*portainer.Team, error) {
var t portainer.Team
err := service.Tx.GetAll(
BucketName,
&portainer.Team{},
dataservices.FirstFn(&t, func(e portainer.Team) bool {
return strings.EqualFold(e.Name, name)
}),
)
if errors.Is(err, dataservices.ErrStop) {
return &t, nil
}
if err == nil {
return nil, dserrors.ErrObjectNotFound
}
return nil, err
}
// CreateTeam creates a new Team.
func (service ServiceTx) Create(team *portainer.Team) error {
return service.Tx.CreateObject(
BucketName,
func(id uint64) (int, any) {
team.ID = portainer.TeamID(id)
return int(team.ID), team
},
)
}
+1 -1
View File
@@ -402,7 +402,6 @@ type storeExport struct {
}
func (store *Store) Export(filename string) (err error) {
backup := storeExport{}
if c, err := store.CustomTemplate().ReadAll(); err != nil {
@@ -606,6 +605,7 @@ func (store *Store) Export(filename string) (err error) {
if err != nil {
return err
}
return os.WriteFile(filename, b, 0600)
}
+4 -1
View File
@@ -80,7 +80,10 @@ func (tx *StoreTx) TeamMembership() dataservices.TeamMembershipService {
return tx.store.TeamMembershipService.Tx(tx.tx)
}
func (tx *StoreTx) Team() dataservices.TeamService { return nil }
func (tx *StoreTx) Team() dataservices.TeamService {
return tx.store.TeamService.Tx(tx.tx)
}
func (tx *StoreTx) TunnelServer() dataservices.TunnelServerService { return nil }
func (tx *StoreTx) User() dataservices.UserService {
@@ -941,6 +941,6 @@
}
],
"version": {
"VERSION": "{\"SchemaVersion\":\"2.21.0\",\"MigratorCount\":0,\"Edition\":1,\"InstanceID\":\"463d5c47-0ea5-4aca-85b1-405ceefee254\"}"
"VERSION": "{\"SchemaVersion\":\"2.21.1\",\"MigratorCount\":0,\"Edition\":1,\"InstanceID\":\"463d5c47-0ea5-4aca-85b1-405ceefee254\"}"
}
}
+57 -10
View File
@@ -4,15 +4,17 @@ import (
"context"
"strings"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
dockerclient "github.com/portainer/portainer/api/docker/client"
"github.com/portainer/portainer/api/docker/images"
"github.com/Masterminds/semver"
"github.com/docker/docker/api/types"
dockercontainer "github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/network"
"github.com/docker/docker/client"
"github.com/pkg/errors"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
dockerclient "github.com/portainer/portainer/api/docker/client"
"github.com/portainer/portainer/api/docker/images"
"github.com/rs/zerolog/log"
)
@@ -30,6 +32,44 @@ func NewContainerService(factory *dockerclient.ClientFactory, dataStore dataserv
}
}
// applyVersionConstraint uses the version to apply a transformation function to
// the value when the constraint is satisfied
func applyVersionConstraint[T any](currentVersion, versionConstraint string, value T, transform func(T) T) (T, error) {
newValue := value
constraint, err := semver.NewConstraint(versionConstraint)
if err != nil {
return newValue, errors.New("invalid version constraint specified")
}
currentVer, err := semver.NewVersion(currentVersion)
if err != nil {
log.Warn().Err(err).Msg("Unable to parse the Docker client version")
return newValue, nil
}
if satisfiesConstraint, _ := constraint.Validate(currentVer); satisfiesConstraint {
newValue = transform(value)
}
return newValue, nil
}
func clearMacAddrs(n network.NetworkingConfig) network.NetworkingConfig {
netConfig := network.NetworkingConfig{
EndpointsConfig: make(map[string]*network.EndpointSettings),
}
for k := range n.EndpointsConfig {
endpointConfig := n.EndpointsConfig[k].Copy()
endpointConfig.MacAddress = ""
netConfig.EndpointsConfig[k] = endpointConfig
}
return netConfig
}
// Recreate a container
func (c *ContainerService) Recreate(ctx context.Context, endpoint *portainer.Endpoint, containerId string, forcePullImage bool, imageTag, nodeName string) (*types.ContainerJSON, error) {
cli, err := c.factory.CreateClient(endpoint, nodeName, nil)
@@ -90,7 +130,7 @@ func (c *ContainerService) Recreate(ctx context.Context, endpoint *portainer.End
return nil, errors.Wrap(err, "rename container error")
}
networkWithCreation := network.NetworkingConfig{
initialNetwork := network.NetworkingConfig{
EndpointsConfig: make(map[string]*network.EndpointSettings),
}
@@ -103,10 +143,10 @@ func (c *ContainerService) Recreate(ctx context.Context, endpoint *portainer.End
}
// 5. get the first network attached to the current container
if len(networkWithCreation.EndpointsConfig) == 0 {
if len(initialNetwork.EndpointsConfig) == 0 {
// Retrieve the first network that is linked to the present container, which
// will be utilized when creating the container.
networkWithCreation.EndpointsConfig[name] = network
initialNetwork.EndpointsConfig[name] = network
}
}
c.sr.enable()
@@ -130,7 +170,15 @@ func (c *ContainerService) Recreate(ctx context.Context, endpoint *portainer.End
// to retain the same network settings we have to connect on creation to one of the old
// container's networks, and connect to the other networks after creation.
// see: https://portainer.atlassian.net/browse/EE-5448
create, err := cli.ContainerCreate(ctx, container.Config, container.HostConfig, &networkWithCreation, nil, container.Name)
// Docker API < 1.44 does not support specifying MAC addresses
// https://github.com/moby/moby/blob/6aea26b431ea152a8b085e453da06ea403f89886/client/container_create.go#L44-L46
initialNetwork, err = applyVersionConstraint(cli.ClientVersion(), "< 1.44", initialNetwork, clearMacAddrs)
if err != nil {
return nil, err
}
create, err := cli.ContainerCreate(ctx, container.Config, container.HostConfig, &initialNetwork, nil, container.Name)
c.sr.push(func() {
log.Debug().Str("container_id", create.ID).Msg("removing the new container")
@@ -150,8 +198,7 @@ func (c *ContainerService) Recreate(ctx context.Context, endpoint *portainer.End
log.Debug().Str("container_id", newContainerId).Msg("connecting networks to container")
networks := container.NetworkSettings.Networks
for key, network := range networks {
_, ok := networkWithCreation.EndpointsConfig[key]
if ok {
if _, ok := initialNetwork.EndpointsConfig[key]; ok {
// skip the network that is used during container creation
continue
}
+52
View File
@@ -0,0 +1,52 @@
package docker
import (
"testing"
"github.com/docker/docker/api/types/network"
"github.com/stretchr/testify/require"
)
func TestApplyVersionConstraint(t *testing.T) {
initialNet := network.NetworkingConfig{
EndpointsConfig: map[string]*network.EndpointSettings{
"key1": {
MacAddress: "mac1",
EndpointID: "endpointID1",
},
"key2": {
MacAddress: "mac2",
EndpointID: "endpointID2",
},
},
}
f := func(currentVer string, constraint string, success, emptyMac bool) {
t.Helper()
transformedNet, err := applyVersionConstraint(currentVer, constraint, initialNet, clearMacAddrs)
if success {
require.NoError(t, err)
} else {
require.Error(t, err)
}
require.Len(t, transformedNet.EndpointsConfig, len(initialNet.EndpointsConfig))
for k := range initialNet.EndpointsConfig {
if emptyMac {
require.NotEqual(t, initialNet.EndpointsConfig[k], transformedNet.EndpointsConfig[k])
require.Empty(t, transformedNet.EndpointsConfig[k].MacAddress)
continue
}
require.Equal(t, initialNet.EndpointsConfig[k], transformedNet.EndpointsConfig[k])
}
}
f("1.45", "< 1.44", true, false) // No transformation
f("1.43", "< 1.44", true, true) // Transformation
f("a.b.", "< 1.44", true, false) // Invalid current version
f("1.45", "z 1.44", false, false) // Invalid version constraint
}
+4
View File
@@ -143,6 +143,7 @@ func (c *gitClient) listFiles(ctx context.Context, opt fetchOption) ([]string, e
ReferenceName: plumbing.ReferenceName(opt.referenceName),
Auth: getAuth(opt.username, opt.password),
InsecureSkipTLS: opt.tlsSkipVerify,
Tags: git.NoTags,
}
repo, err := git.Clone(memory.NewStorage(), nil, cloneOption)
@@ -166,7 +167,10 @@ func (c *gitClient) listFiles(ctx context.Context, opt fetchOption) ([]string, e
}
var allPaths []string
w := object.NewTreeWalker(tree, true, nil)
defer w.Close()
for {
name, entry, err := w.Next()
if err != nil {
+23
View File
@@ -91,6 +91,29 @@ func Test_latestCommitID(t *testing.T) {
assert.Equal(t, "68dcaa7bd452494043c64252ab90db0f98ecf8d2", id)
}
func Test_ListRefs(t *testing.T) {
service := Service{git: NewGitClient(true)}
repositoryURL := setup(t)
fs, err := service.ListRefs(repositoryURL, "", "", false, false)
assert.NoError(t, err)
assert.Equal(t, []string{"refs/heads/main"}, fs)
}
func Test_ListFiles(t *testing.T) {
service := Service{git: NewGitClient(true)}
repositoryURL := setup(t)
referenceName := "refs/heads/main"
fs, err := service.ListFiles(repositoryURL, referenceName, "", "", false, false, []string{".yml"}, false)
assert.NoError(t, err)
assert.Equal(t, []string{"docker-compose.yml"}, fs)
}
func getCommitHistoryLength(t *testing.T, err error, dir string) int {
repo, err := git.PlainOpen(dir)
if err != nil {
+19 -12
View File
@@ -9,6 +9,7 @@ import (
lru "github.com/hashicorp/golang-lru"
"github.com/rs/zerolog/log"
"golang.org/x/sync/singleflight"
)
const (
@@ -223,11 +224,23 @@ func (service *Service) ListRefs(repositoryURL, username, password string, hardR
return refs, nil
}
var singleflightGroup = &singleflight.Group{}
// ListFiles will list all the files of the target repository with specific extensions.
// If extension is not provided, it will list all the files under the target repository
func (service *Service) ListFiles(repositoryURL, referenceName, username, password string, dirOnly, hardRefresh bool, includedExts []string, tlsSkipVerify bool) ([]string, error) {
repoKey := generateCacheKey(repositoryURL, referenceName, username, password, strconv.FormatBool(tlsSkipVerify), strconv.FormatBool(dirOnly))
fs, err, _ := singleflightGroup.Do(repoKey, func() (any, error) {
return service.listFiles(repositoryURL, referenceName, username, password, dirOnly, hardRefresh, tlsSkipVerify)
})
return filterFiles(fs.([]string), includedExts), err
}
func (service *Service) listFiles(repositoryURL, referenceName, username, password string, dirOnly, hardRefresh bool, tlsSkipVerify bool) ([]string, error) {
repoKey := generateCacheKey(repositoryURL, referenceName, username, password, strconv.FormatBool(tlsSkipVerify), strconv.FormatBool(dirOnly))
if service.cacheEnabled && hardRefresh {
// Should remove the cache explicitly, so that the following normal list can show the correct result
service.repoFileCache.Remove(repoKey)
@@ -235,14 +248,9 @@ func (service *Service) ListFiles(repositoryURL, referenceName, username, passwo
if service.repoFileCache != nil {
// lookup the files cache first
cache, ok := service.repoFileCache.Get(repoKey)
if ok {
files, success := cache.([]string)
if success {
// For the case while searching files in a repository without include extensions for the first time,
// but with include extensions for the second time
includedFiles := filterFiles(files, includedExts)
return includedFiles, nil
if cache, ok := service.repoFileCache.Get(repoKey); ok {
if files, ok := cache.([]string); ok {
return files, nil
}
}
}
@@ -274,12 +282,11 @@ func (service *Service) ListFiles(repositoryURL, referenceName, username, passwo
}
}
includedFiles := filterFiles(files, includedExts)
if service.cacheEnabled && service.repoFileCache != nil {
service.repoFileCache.Add(repoKey, includedFiles)
return includedFiles, nil
service.repoFileCache.Add(repoKey, files)
}
return includedFiles, nil
return files, nil
}
func (service *Service) purgeCache() {
+2
View File
@@ -28,5 +28,7 @@ func (handler *Handler) logout(w http.ResponseWriter, r *http.Request) *httperro
security.RemoveAuthCookie(w)
handler.bouncer.RevokeJWT(tokenData.Token)
return response.Empty(w)
}
+1 -1
View File
@@ -85,7 +85,7 @@ type Handler struct {
}
// @title PortainerCE API
// @version 2.21.0
// @version 2.21.1
// @description.markdown api-description.md
// @termsOfService
+33 -16
View File
@@ -5,6 +5,7 @@ import (
"net/http"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
@@ -23,6 +24,7 @@ func (payload *teamCreatePayload) Validate(r *http.Request) error {
if govalidator.IsNull(payload.Name) {
return errors.New("Invalid team name")
}
return nil
}
@@ -43,26 +45,42 @@ func (payload *teamCreatePayload) Validate(r *http.Request) error {
// @router /teams [post]
func (handler *Handler) teamCreate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
var payload teamCreatePayload
err := request.DecodeAndValidateJSONPayload(r, &payload)
if err != nil {
if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil {
return httperror.BadRequest("Invalid request payload", err)
}
team, err := handler.DataStore.Team().TeamByName(payload.Name)
if err != nil && !handler.DataStore.IsErrObjectNotFound(err) {
return httperror.InternalServerError("Unable to retrieve teams from the database", err)
var team *portainer.Team
if err := handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
var err error
team, err = createTeam(tx, payload)
return err
}); err != nil {
var httpErr *httperror.HandlerError
if errors.As(err, &httpErr) {
return httpErr
}
return httperror.InternalServerError("Unexpected error", err)
}
return response.JSON(w, team)
}
func createTeam(tx dataservices.DataStoreTx, payload teamCreatePayload) (*portainer.Team, error) {
team, err := tx.Team().TeamByName(payload.Name)
if err != nil && !tx.IsErrObjectNotFound(err) {
return nil, httperror.InternalServerError("Unable to retrieve teams from the database", err)
}
if team != nil {
return httperror.Conflict("A team with the same name already exists", errors.New("Team already exists"))
return nil, httperror.Conflict("A team with the same name already exists", errors.New("Team already exists"))
}
team = &portainer.Team{
Name: payload.Name,
}
team = &portainer.Team{Name: payload.Name}
err = handler.DataStore.Team().Create(team)
if err != nil {
return httperror.InternalServerError("Unable to persist the team inside the database", err)
if err := tx.Team().Create(team); err != nil {
return nil, httperror.InternalServerError("Unable to persist the team inside the database", err)
}
for _, teamLeader := range payload.TeamLeaders {
@@ -72,11 +90,10 @@ func (handler *Handler) teamCreate(w http.ResponseWriter, r *http.Request) *http
Role: portainer.TeamLeader,
}
err = handler.DataStore.TeamMembership().Create(membership)
if err != nil {
return httperror.InternalServerError("Unable to persist team leadership inside the database", err)
if err := tx.TeamMembership().Create(membership); err != nil {
return nil, httperror.InternalServerError("Unable to persist team leadership inside the database", err)
}
}
return response.JSON(w, team)
return team, nil
}
@@ -0,0 +1,65 @@
package teams
import (
"bytes"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"github.com/portainer/portainer/api/datastore"
"github.com/stretchr/testify/require"
"golang.org/x/sync/errgroup"
)
func TestConcurrentTeamCreation(t *testing.T) {
_, store := datastore.MustNewTestStore(t, true, false)
h := &Handler{
DataStore: store,
}
tcp := teamCreatePayload{
Name: "portainer",
}
m, err := json.Marshal(tcp)
require.NoError(t, err)
errGroup := &errgroup.Group{}
n := 100
for i := 0; i < n; i++ {
errGroup.Go(func() error {
req, err := http.NewRequest(http.MethodPost, "/teams", bytes.NewReader(m))
if err != nil {
return err
}
if err := h.teamCreate(httptest.NewRecorder(), req); err != nil {
return err
}
return nil
})
}
err = errGroup.Wait()
require.Error(t, err)
teams, err := store.Team().ReadAll()
require.NotEmpty(t, teams)
require.NoError(t, err)
teamCreated := false
for _, team := range teams {
if team.Name == tcp.Name {
require.False(t, teamCreated)
teamCreated = true
}
}
require.True(t, teamCreated)
}
+37 -15
View File
@@ -5,6 +5,7 @@ import (
"net/http"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
@@ -54,12 +55,33 @@ func (handler *Handler) userCreate(w http.ResponseWriter, r *http.Request) *http
return httperror.BadRequest("Invalid request payload", err)
}
user, err := handler.DataStore.User().UserByUsername(payload.Username)
if err != nil && !handler.DataStore.IsErrObjectNotFound(err) {
return httperror.InternalServerError("Unable to retrieve users from the database", err)
var user *portainer.User
if err := handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
var err error
user, err = handler.createUser(tx, payload)
return err
}); err != nil {
var httpErr *httperror.HandlerError
if errors.As(err, &httpErr) {
return httpErr
}
return httperror.InternalServerError("Unexpected error", err)
}
return response.JSON(w, user)
}
func (handler *Handler) createUser(tx dataservices.DataStoreTx, payload userCreatePayload) (*portainer.User, error) {
user, err := tx.User().UserByUsername(payload.Username)
if err != nil && !tx.IsErrObjectNotFound(err) {
return nil, httperror.InternalServerError("Unable to retrieve users from the database", err)
}
if user != nil {
return httperror.Conflict("Another user with the same username already exists", errUserAlreadyExists)
return nil, httperror.Conflict("Another user with the same username already exists", errUserAlreadyExists)
}
user = &portainer.User{
@@ -67,33 +89,33 @@ func (handler *Handler) userCreate(w http.ResponseWriter, r *http.Request) *http
Role: portainer.UserRole(payload.Role),
}
settings, err := handler.DataStore.Settings().Settings()
settings, err := tx.Settings().Settings()
if err != nil {
return httperror.InternalServerError("Unable to retrieve settings from the database", err)
return nil, httperror.InternalServerError("Unable to retrieve settings from the database", err)
}
// when ldap/oauth is on, can only add users without password
// When LDAP/OAuth is on, can only add users without password
if (settings.AuthenticationMethod == portainer.AuthenticationLDAP || settings.AuthenticationMethod == portainer.AuthenticationOAuth) && payload.Password != "" {
errMsg := "A user with password can not be created when authentication method is Oauth or LDAP"
return httperror.BadRequest(errMsg, errors.New(errMsg))
errMsg := "a user with password can not be created when authentication method is Oauth or LDAP"
return nil, httperror.BadRequest(errMsg, errors.New(errMsg))
}
if settings.AuthenticationMethod == portainer.AuthenticationInternal {
if !handler.passwordStrengthChecker.Check(payload.Password) {
return httperror.BadRequest("Password does not meet the requirements", nil)
return nil, httperror.BadRequest("Password does not meet the requirements", nil)
}
user.Password, err = handler.CryptoService.Hash(payload.Password)
if err != nil {
return httperror.InternalServerError("Unable to hash user password", errCryptoHashFailure)
return nil, httperror.InternalServerError("Unable to hash user password", errCryptoHashFailure)
}
}
err = handler.DataStore.User().Create(user)
if err != nil {
return httperror.InternalServerError("Unable to persist user inside the database", err)
if err := tx.User().Create(user); err != nil {
return nil, httperror.InternalServerError("Unable to persist user inside the database", err)
}
hideFields(user)
return response.JSON(w, user)
return user, nil
}
@@ -0,0 +1,77 @@
package users
import (
"bytes"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/crypto"
"github.com/portainer/portainer/api/datastore"
"github.com/stretchr/testify/require"
"golang.org/x/sync/errgroup"
)
type mockPasswordStrengthChecker struct{}
func (m *mockPasswordStrengthChecker) Check(string) bool {
return true
}
func TestConcurrentUserCreation(t *testing.T) {
_, store := datastore.MustNewTestStore(t, true, false)
h := &Handler{
passwordStrengthChecker: &mockPasswordStrengthChecker{},
CryptoService: &crypto.Service{},
DataStore: store,
}
ucp := userCreatePayload{
Username: "portainer",
Password: "password",
Role: int(portainer.AdministratorRole),
}
m, err := json.Marshal(ucp)
require.NoError(t, err)
errGroup := &errgroup.Group{}
n := 100
for i := 0; i < n; i++ {
errGroup.Go(func() error {
req, err := http.NewRequest(http.MethodPost, "/users", bytes.NewReader(m))
if err != nil {
return err
}
if err := h.userCreate(httptest.NewRecorder(), req); err != nil {
return err
}
return nil
})
}
err = errGroup.Wait()
require.Error(t, err)
users, err := store.User().ReadAll()
require.NotEmpty(t, users)
require.NoError(t, err)
userCreated := false
for _, u := range users {
if u.Username == ucp.Username {
require.False(t, userCreated)
userCreated = true
}
}
require.True(t, userCreated)
}
+51 -6
View File
@@ -4,6 +4,7 @@ import (
"fmt"
"net/http"
"strings"
"sync"
"time"
portainer "github.com/portainer/portainer/api"
@@ -16,6 +17,9 @@ import (
"github.com/pkg/errors"
)
const apiKeyHeader = "X-API-KEY"
const jwtTokenHeader = "Authorization"
type (
BouncerService interface {
PublicAccess(http.Handler) http.Handler
@@ -30,6 +34,7 @@ type (
TrustedEdgeEnvironmentAccess(dataservices.DataStoreTx, *portainer.Endpoint) error
CookieAuthLookup(*http.Request) (*portainer.TokenData, error)
JWTAuthLookup(*http.Request) (*portainer.TokenData, error)
RevokeJWT(string)
}
// RequestBouncer represents an entity that manages API request accesses
@@ -37,6 +42,7 @@ type (
dataStore dataservices.DataStore
jwtService portainer.JWTService
apiKeyService apikey.APIKeyService
revokedJWT sync.Map
}
// RestrictedRequestContext is a data structure containing information
@@ -52,16 +58,22 @@ type (
tokenLookup func(*http.Request) (*portainer.TokenData, error)
)
const apiKeyHeader = "X-API-KEY"
const jwtTokenHeader = "Authorization"
var (
ErrInvalidKey = errors.New("Invalid API key")
ErrRevokedJWT = errors.New("the JWT has been revoked")
)
// NewRequestBouncer initializes a new RequestBouncer
func NewRequestBouncer(dataStore dataservices.DataStore, jwtService portainer.JWTService, apiKeyService apikey.APIKeyService) *RequestBouncer {
return &RequestBouncer{
b := &RequestBouncer{
dataStore: dataStore,
jwtService: jwtService,
apiKeyService: apiKeyService,
}
go b.cleanUpExpiredJWT()
return b
}
// PublicAccess defines a security check for public API endpoints.
@@ -317,11 +329,15 @@ func (bouncer *RequestBouncer) CookieAuthLookup(r *http.Request) (*portainer.Tok
return nil, nil
}
tokenData, err := bouncer.jwtService.ParseAndVerifyToken(token)
tokenData, jti, _, err := bouncer.jwtService.ParseAndVerifyToken(token)
if err != nil {
return nil, err
}
if _, ok := bouncer.revokedJWT.Load(jti); ok {
return nil, ErrRevokedJWT
}
return tokenData, nil
}
@@ -333,15 +349,44 @@ func (bouncer *RequestBouncer) JWTAuthLookup(r *http.Request) (*portainer.TokenD
return nil, nil
}
tokenData, err := bouncer.jwtService.ParseAndVerifyToken(token)
tokenData, jti, _, err := bouncer.jwtService.ParseAndVerifyToken(token)
if err != nil {
return nil, err
}
if _, ok := bouncer.revokedJWT.Load(jti); ok {
return nil, ErrRevokedJWT
}
return tokenData, nil
}
var ErrInvalidKey = errors.New("Invalid API key")
func (bouncer *RequestBouncer) RevokeJWT(token string) {
_, jti, exp, err := bouncer.jwtService.ParseAndVerifyToken(token)
if err != nil {
return
}
bouncer.revokedJWT.Store(jti, exp)
}
func (bouncer *RequestBouncer) cleanUpExpiredJWTPass() {
bouncer.revokedJWT.Range(func(key, value any) bool {
if time.Now().After(value.(time.Time)) {
bouncer.revokedJWT.Delete(key)
}
return true
})
}
func (bouncer *RequestBouncer) cleanUpExpiredJWT() {
ticker := time.NewTicker(time.Hour)
for range ticker.C {
bouncer.cleanUpExpiredJWTPass()
}
}
// apiKeyLookup looks up an verifies an api-key by:
// - computing the digest of the raw api-key
+59
View File
@@ -5,6 +5,7 @@ import (
"net/http"
"net/http/httptest"
"testing"
"time"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/apikey"
@@ -14,6 +15,7 @@ import (
"github.com/portainer/portainer/api/jwt"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// testHandler200 is a simple handler which returns HTTP status 200 OK
@@ -459,3 +461,60 @@ func Test_ShouldSkipCSRFCheck(t *testing.T) {
})
}
}
func TestJWTRevocation(t *testing.T) {
_, store := datastore.MustNewTestStore(t, true, true)
jwtService, err := jwt.NewService("1h", store)
require.NoError(t, err)
err = store.User().Create(&portainer.User{ID: 1})
require.NoError(t, err)
jwtService.SetUserSessionDuration(time.Second)
token, _, err := jwtService.GenerateToken(&portainer.TokenData{ID: 1})
require.NoError(t, err)
apiKeyService := apikey.NewAPIKeyService(nil, nil)
bouncer := NewRequestBouncer(store, jwtService, apiKeyService)
r, err := http.NewRequest(http.MethodGet, "url", nil)
require.NoError(t, err)
r.Header.Add(jwtTokenHeader, "Bearer "+token)
r.AddCookie(&http.Cookie{Name: portainer.AuthCookieKey, Value: token})
_, err = bouncer.JWTAuthLookup(r)
require.NoError(t, err)
_, err = bouncer.CookieAuthLookup(r)
require.NoError(t, err)
bouncer.RevokeJWT(token)
revokeLen := func() (l int) {
bouncer.revokedJWT.Range(func(key, value any) bool {
l++
return true
})
return l
}
require.Equal(t, 1, revokeLen())
_, err = bouncer.JWTAuthLookup(r)
require.Error(t, err)
_, err = bouncer.CookieAuthLookup(r)
require.Error(t, err)
time.Sleep(time.Second)
bouncer.cleanUpExpiredJWTPass()
require.Equal(t, 0, revokeLen())
}
@@ -58,6 +58,8 @@ func (testRequestBouncer) JWTAuthLookup(r *http.Request) (*portainer.TokenData,
return nil, nil
}
func (testRequestBouncer) RevokeJWT(jti string) {}
// AddTestSecurityCookie adds a security cookie to the request
func AddTestSecurityCookie(r *http.Request, jwt string) {
r.AddCookie(&http.Cookie{
+15 -7
View File
@@ -7,9 +7,10 @@ import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/golang-jwt/jwt/v4"
"github.com/portainer/portainer/api/internal/securecookie"
"github.com/gofrs/uuid"
"github.com/golang-jwt/jwt/v4"
"github.com/rs/zerolog/log"
)
@@ -103,7 +104,7 @@ func (service *Service) GenerateToken(data *portainer.TokenData) (string, time.T
}
// ParseAndVerifyToken parses a JWT token and verify its validity. It returns an error if token is invalid.
func (service *Service) ParseAndVerifyToken(token string) (*portainer.TokenData, error) {
func (service *Service) ParseAndVerifyToken(token string) (*portainer.TokenData, string, time.Time, error) {
scope := parseScope(token)
secret := service.secrets[scope]
parsedToken, err := jwt.ParseWithClaims(token, &claims{}, func(token *jwt.Token) (interface{}, error) {
@@ -119,10 +120,10 @@ func (service *Service) ParseAndVerifyToken(token string) (*portainer.TokenData,
user, err := service.dataStore.User().Read(portainer.UserID(cl.UserID))
if err != nil {
return nil, errInvalidJWTToken
return nil, "", time.Time{}, errInvalidJWTToken
}
if user.TokenIssueAt > cl.StandardClaims.IssuedAt {
return nil, errInvalidJWTToken
return nil, "", time.Time{}, errInvalidJWTToken
}
return &portainer.TokenData{
@@ -131,10 +132,11 @@ func (service *Service) ParseAndVerifyToken(token string) (*portainer.TokenData,
Role: portainer.UserRole(cl.Role),
Token: token,
ForceChangePassword: cl.ForceChangePassword,
}, nil
}, cl.Id, time.Unix(cl.ExpiresAt, 0), nil
}
}
return nil, errInvalidJWTToken
return nil, "", time.Time{}, errInvalidJWTToken
}
// parse a JWT token, fallback to defaultScope if no scope is present in the JWT
@@ -173,6 +175,11 @@ func (service *Service) generateSignedToken(data *portainer.TokenData, expiresAt
expiresAt = time.Now().Add(time.Hour * 8760 * 99).Unix()
}
uuid, err := uuid.NewV4()
if err != nil {
return "", fmt.Errorf("unable to generate the JWT ID: %w", err)
}
cl := claims{
UserID: int(data.ID),
Username: data.Username,
@@ -180,6 +187,7 @@ func (service *Service) generateSignedToken(data *portainer.TokenData, expiresAt
Scope: scope,
ForceChangePassword: data.ForceChangePassword,
StandardClaims: jwt.StandardClaims{
Id: uuid.String(),
ExpiresAt: expiresAt,
IssuedAt: time.Now().Unix(),
},
+55
View File
@@ -6,8 +6,10 @@ import (
"github.com/golang-jwt/jwt/v4"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/datastore"
i "github.com/portainer/portainer/api/internal/testhelpers"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestGenerateSignedToken(t *testing.T) {
@@ -55,3 +57,56 @@ func TestGenerateSignedToken_InvalidScope(t *testing.T) {
assert.Error(t, err)
assert.Equal(t, "invalid scope: testing", err.Error())
}
func TestGenerationAndParsing(t *testing.T) {
_, store := datastore.MustNewTestStore(t, true, false)
err := store.User().Create(&portainer.User{ID: 1})
require.NoError(t, err)
service, err := NewService("1h", store)
require.NoError(t, err)
expectedToken := &portainer.TokenData{
Username: "User",
ID: 1,
Role: 1,
}
tokenString, _, err := service.GenerateToken(expectedToken)
require.NoError(t, err)
expectedToken.Token = tokenString
token, _, _, err := service.ParseAndVerifyToken(tokenString)
require.NoError(t, err)
require.Equal(t, expectedToken, token)
}
func TestExpiration(t *testing.T) {
_, store := datastore.MustNewTestStore(t, true, false)
err := store.User().Create(&portainer.User{ID: 1})
require.NoError(t, err)
service, err := NewService("1h", store)
require.NoError(t, err)
expectedToken := &portainer.TokenData{
Username: "User",
ID: 1,
Role: 1,
}
service.SetUserSessionDuration(time.Second)
tokenString, _, err := service.GenerateToken(expectedToken)
require.NoError(t, err)
expectedToken.Token = tokenString
time.Sleep(2 * time.Second)
_, _, _, err = service.ParseAndVerifyToken(tokenString)
require.Error(t, err)
}
+2 -2
View File
@@ -1490,7 +1490,7 @@ type (
JWTService interface {
GenerateToken(data *TokenData) (string, time.Time, error)
GenerateTokenForKubeconfig(data *TokenData) (string, error)
ParseAndVerifyToken(token string) (*TokenData, error)
ParseAndVerifyToken(token string) (*TokenData, string, time.Time, error)
SetUserSessionDuration(userSessionDuration time.Duration)
}
@@ -1601,7 +1601,7 @@ type (
const (
// APIVersion is the version number of the Portainer API
APIVersion = "2.21.0"
APIVersion = "2.21.1"
// Edition is what this edition of Portainer is called
Edition = PortainerCE
// ComposeSyntaxMaxVersion is a maximum supported version of the docker compose syntax
+1 -1
View File
@@ -2,7 +2,7 @@
"author": "Portainer.io",
"name": "portainer",
"homepage": "http://portainer.io",
"version": "2.21.0",
"version": "2.21.1",
"repository": {
"type": "git",
"url": "git@github.com:portainer/portainer.git"
@@ -110,6 +110,11 @@ func (wrapper *PluginWrapper) Validate(ctx context.Context, filePaths []string,
return err
}
func (wrapper *PluginWrapper) Config(ctx context.Context, filePaths []string, options libstack.Options) ([]byte, error) {
configArgs := append([]string{"config"}, options.ConfigOptions...)
return wrapper.command(newCommand(configArgs, filePaths), options)
}
// Command execute a docker-compose command
func (wrapper *PluginWrapper) command(command composeCommand, options libstack.Options) ([]byte, error) {
program := utils.ProgramPath(wrapper.binaryPath, "docker-compose")
@@ -143,3 +143,230 @@ func containerExists(containerName string) bool {
return strings.Contains(string(out), containerName)
}
func Test_Config(t *testing.T) {
checkPrerequisites(t)
ctx := context.Background()
dir := t.TempDir()
projectName := "configtest"
defer os.RemoveAll(dir)
testCases := []struct {
name string
composeFileContent string
expectFileContent string
envFileContent string
}{
{
name: "compose file with relative path",
composeFileContent: `services:
app:
image: 'nginx:latest'
ports:
- '80:80'
volumes:
- ./nginx-data:/data`,
expectFileContent: `name: configtest
services:
app:
image: nginx:latest
networks:
default: null
ports:
- mode: ingress
target: 80
published: "80"
protocol: tcp
volumes:
- type: bind
source: ./nginx-data
target: /data
bind:
create_host_path: true
networks:
default:
name: configtest_default
`,
envFileContent: "",
},
{
name: "compose file with absolute path",
composeFileContent: `services:
app:
image: 'nginx:latest'
ports:
- '80:80'
volumes:
- /nginx-data:/data`,
expectFileContent: `name: configtest
services:
app:
image: nginx:latest
networks:
default: null
ports:
- mode: ingress
target: 80
published: "80"
protocol: tcp
volumes:
- type: bind
source: /nginx-data
target: /data
bind:
create_host_path: true
networks:
default:
name: configtest_default
`,
envFileContent: "",
},
{
name: "compose file with declared volume",
composeFileContent: `services:
app:
image: 'nginx:latest'
ports:
- '80:80'
volumes:
- nginx-data:/data
volumes:
nginx-data:
driver: local`,
expectFileContent: `name: configtest
services:
app:
image: nginx:latest
networks:
default: null
ports:
- mode: ingress
target: 80
published: "80"
protocol: tcp
volumes:
- type: volume
source: nginx-data
target: /data
volume: {}
networks:
default:
name: configtest_default
volumes:
nginx-data:
name: configtest_nginx-data
driver: local
`,
envFileContent: "",
},
{
name: "compose file with relative path environment variable placeholder",
composeFileContent: `services:
nginx:
image: nginx:latest
ports:
- 8019:80
volumes:
- ${WEB_HOME}:/usr/share/nginx/html/
env_file:
- stack.env
`,
expectFileContent: `name: configtest
services:
nginx:
environment:
WEB_HOME: ./html
image: nginx:latest
networks:
default: null
ports:
- mode: ingress
target: 80
published: "8019"
protocol: tcp
volumes:
- type: bind
source: ./html
target: /usr/share/nginx/html
bind:
create_host_path: true
networks:
default:
name: configtest_default
`,
envFileContent: `WEB_HOME=./html`,
},
{
name: "compose file with absolute path environment variable placeholder",
composeFileContent: `services:
nginx:
image: nginx:latest
ports:
- 8019:80
volumes:
- ${WEB_HOME}:/usr/share/nginx/html/
env_file:
- stack.env
`,
expectFileContent: `name: configtest
services:
nginx:
environment:
WEB_HOME: /usr/share/nginx/html
image: nginx:latest
networks:
default: null
ports:
- mode: ingress
target: 80
published: "8019"
protocol: tcp
volumes:
- type: bind
source: /usr/share/nginx/html
target: /usr/share/nginx/html
bind:
create_host_path: true
networks:
default:
name: configtest_default
`,
envFileContent: `WEB_HOME=/usr/share/nginx/html`,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
composeFilePath, err := createFile(dir, "docker-compose.yml", tc.composeFileContent)
if err != nil {
t.Fatal(err)
}
envFilePath := ""
if tc.envFileContent != "" {
envFilePath, err = createFile(dir, "stack.env", tc.envFileContent)
if err != nil {
t.Fatal(err)
}
}
w := setup(t)
actual, err := w.Config(ctx, []string{composeFilePath}, libstack.Options{
WorkingDir: dir,
ProjectName: projectName,
EnvFilePath: envFilePath,
ConfigOptions: []string{"--no-path-resolution"},
})
if err != nil {
t.Fatalf("failed to get config: %s. Error: %s", string(actual), err)
}
if string(actual) != tc.expectFileContent {
t.Fatalf("unexpected config output: %s(len=%d), expect: %s(len=%d)", actual, len(actual), tc.expectFileContent, len(tc.expectFileContent))
}
})
}
}
+3
View File
@@ -14,6 +14,7 @@ type Deployer interface {
Pull(ctx context.Context, filePaths []string, options Options) error
Validate(ctx context.Context, filePaths []string, options Options) error
WaitForStatus(ctx context.Context, name string, status Status) <-chan WaitResult
Config(ctx context.Context, filePaths []string, options Options) ([]byte, error)
}
type Status string
@@ -47,6 +48,8 @@ type Options struct {
// By default, it is an empty string, which means it corresponds to the path of the compose file itself.
// This is particularly helpful when mounting a relative path.
ProjectDir string
// ConfigOptions is a list of options to pass to the docker-compose config command
ConfigOptions []string
}
type DeployOptions struct {