From 8f1ac3896373053dc1446fe1855db94f1910abca Mon Sep 17 00:00:00 2001 From: matias-portainer <104775949+matias-portainer@users.noreply.github.com> Date: Thu, 13 Oct 2022 11:03:19 -0300 Subject: [PATCH 01/31] fix(tags): get tags when loading associated endpoints selector EE-4140 (#7857) --- .../associatedEndpointsSelectorController.js | 10 ++++++++++ app/portainer/helpers/tagHelper.js | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/app/portainer/components/associated-endpoints-selector/associatedEndpointsSelectorController.js b/app/portainer/components/associated-endpoints-selector/associatedEndpointsSelectorController.js index 940573b95..923485188 100644 --- a/app/portainer/components/associated-endpoints-selector/associatedEndpointsSelectorController.js +++ b/app/portainer/components/associated-endpoints-selector/associatedEndpointsSelectorController.js @@ -3,6 +3,7 @@ import _ from 'lodash-es'; import { EdgeTypes } from '@/portainer/environments/types'; import { getEnvironments } from '@/portainer/environments/environment.service'; +import { getTags } from '@/portainer/tags/tags.service'; class AssoicatedEndpointsSelectorController { /* @ngInject */ @@ -47,6 +48,7 @@ class AssoicatedEndpointsSelectorController { } loadData() { + this.getTags(); this.getAvailableEndpoints(); this.getAssociatedEndpoints(); } @@ -79,6 +81,14 @@ class AssoicatedEndpointsSelectorController { this.setTableData('associated', response.value, response.totalCount); }); } + + getTags() { + return this.$async(async () => { + let tags = { value: [], totalCount: 0 }; + tags = await getTags(); + this.tags = tags; + }); + } /* #endregion */ /* #region On endpoint click (either available or associated) */ diff --git a/app/portainer/helpers/tagHelper.js b/app/portainer/helpers/tagHelper.js index 133d56a61..555a3ffa2 100644 --- a/app/portainer/helpers/tagHelper.js +++ b/app/portainer/helpers/tagHelper.js @@ -1,7 +1,7 @@ import _ from 'lodash'; export function idsToTagNames(tags, ids) { - const filteredTags = _.filter(tags, (tag) => _.includes(ids, tag.Id)); + const filteredTags = _.filter(tags, (tag) => _.includes(ids, tag.ID)); const tagNames = _.map(filteredTags, 'Name'); return tagNames; } From 367f3dd6d4d0504705e0dc2c661861da6f20458c Mon Sep 17 00:00:00 2001 From: andres-portainer <91705312+andres-portainer@users.noreply.github.com> Date: Thu, 13 Oct 2022 11:12:12 -0300 Subject: [PATCH 02/31] fix(tags): remove a data race EE-4310 (#7862) --- api/connection.go | 1 + api/database/boltdb/db.go | 34 ++++++++++++++++--- api/dataservices/interface.go | 1 + api/dataservices/tag/tag.go | 14 +++++++- .../endpointgroups/endpointgroup_create.go | 14 ++++---- .../endpointgroups/endpointgroup_delete.go | 14 ++++---- .../endpointgroups/endpointgroup_update.go | 26 +++++++------- api/http/handler/endpoints/endpoint_create.go | 11 ++---- api/http/handler/endpoints/endpoint_delete.go | 14 ++++---- api/http/handler/endpoints/endpoint_update.go | 27 +++++++-------- 10 files changed, 90 insertions(+), 66 deletions(-) diff --git a/api/connection.go b/api/connection.go index 643767058..6c9e4f440 100644 --- a/api/connection.go +++ b/api/connection.go @@ -24,6 +24,7 @@ type Connection interface { SetServiceName(bucketName string) error GetObject(bucketName string, key []byte, object interface{}) error UpdateObject(bucketName string, key []byte, object interface{}) error + UpdateObjectFunc(bucketName string, key []byte, object any, updateFn func()) error DeleteObject(bucketName string, key []byte) error DeleteAllObjects(bucketName string, matching func(o interface{}) (id int, ok bool)) error GetNextIdentifier(bucketName string) int diff --git a/api/database/boltdb/db.go b/api/database/boltdb/db.go index 203f9255c..a829cedd8 100644 --- a/api/database/boltdb/db.go +++ b/api/database/boltdb/db.go @@ -179,7 +179,7 @@ func (connection *DbConnection) ConvertToKey(v int) []byte { return b } -// CreateBucket is a generic function used to create a bucket inside a database database. +// CreateBucket is a generic function used to create a bucket inside a database. func (connection *DbConnection) SetServiceName(bucketName string) error { return connection.Batch(func(tx *bolt.Tx) error { _, err := tx.CreateBucketIfNotExists([]byte(bucketName)) @@ -187,7 +187,7 @@ func (connection *DbConnection) SetServiceName(bucketName string) error { }) } -// GetObject is a generic function used to retrieve an unmarshalled object from a database database. +// GetObject is a generic function used to retrieve an unmarshalled object from a database. func (connection *DbConnection) GetObject(bucketName string, key []byte, object interface{}) error { var data []byte @@ -219,7 +219,7 @@ func (connection *DbConnection) getEncryptionKey() []byte { return connection.EncryptionKey } -// UpdateObject is a generic function used to update an object inside a database database. +// UpdateObject is a generic function used to update an object inside a database. func (connection *DbConnection) UpdateObject(bucketName string, key []byte, object interface{}) error { data, err := connection.MarshalObject(object) if err != nil { @@ -232,7 +232,33 @@ func (connection *DbConnection) UpdateObject(bucketName string, key []byte, obje }) } -// DeleteObject is a generic function used to delete an object inside a database database. +// UpdateObjectFunc is a generic function used to update an object safely without race conditions. +func (connection *DbConnection) UpdateObjectFunc(bucketName string, key []byte, object any, updateFn func()) error { + return connection.Batch(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(bucketName)) + + data := bucket.Get(key) + if data == nil { + return dserrors.ErrObjectNotFound + } + + err := connection.UnmarshalObjectWithJsoniter(data, object) + if err != nil { + return err + } + + updateFn() + + data, err = connection.MarshalObject(object) + if err != nil { + return err + } + + return bucket.Put(key, data) + }) +} + +// DeleteObject is a generic function used to delete an object inside a database. func (connection *DbConnection) DeleteObject(bucketName string, key []byte) error { return connection.Batch(func(tx *bolt.Tx) error { bucket := tx.Bucket([]byte(bucketName)) diff --git a/api/dataservices/interface.go b/api/dataservices/interface.go index 619cc6f4d..e936f5de1 100644 --- a/api/dataservices/interface.go +++ b/api/dataservices/interface.go @@ -251,6 +251,7 @@ type ( Tag(ID portainer.TagID) (*portainer.Tag, error) Create(tag *portainer.Tag) error UpdateTag(ID portainer.TagID, tag *portainer.Tag) error + UpdateTagFunc(ID portainer.TagID, updateFunc func(tag *portainer.Tag)) error DeleteTag(ID portainer.TagID) error BucketName() string } diff --git a/api/dataservices/tag/tag.go b/api/dataservices/tag/tag.go index 42c289598..7641afc37 100644 --- a/api/dataservices/tag/tag.go +++ b/api/dataservices/tag/tag.go @@ -80,12 +80,24 @@ func (service *Service) Create(tag *portainer.Tag) error { ) } -// UpdateTag updates a tag. +// Deprecated: Use UpdateTagFunc instead. func (service *Service) UpdateTag(ID portainer.TagID, tag *portainer.Tag) error { identifier := service.connection.ConvertToKey(int(ID)) return service.connection.UpdateObject(BucketName, identifier, tag) } +// UpdateTagFunc updates a tag inside a transaction avoiding data races. +func (service *Service) UpdateTagFunc(ID portainer.TagID, updateFunc func(tag *portainer.Tag)) error { + id := service.connection.ConvertToKey(int(ID)) + tag := &portainer.Tag{} + + service.connection.UpdateObjectFunc(BucketName, id, tag, func() { + updateFunc(tag) + }) + + return nil +} + // DeleteTag deletes a tag. func (service *Service) DeleteTag(ID portainer.TagID) error { identifier := service.connection.ConvertToKey(int(ID)) diff --git a/api/http/handler/endpointgroups/endpointgroup_create.go b/api/http/handler/endpointgroups/endpointgroup_create.go index cd41498f3..3a9096433 100644 --- a/api/http/handler/endpointgroups/endpointgroup_create.go +++ b/api/http/handler/endpointgroups/endpointgroup_create.go @@ -91,15 +91,13 @@ func (handler *Handler) endpointGroupCreate(w http.ResponseWriter, r *http.Reque } for _, tagID := range endpointGroup.TagIDs { - tag, err := handler.DataStore.Tag().Tag(tagID) - if err != nil { - return httperror.InternalServerError("Unable to retrieve tag from the database", err) - } + handler.DataStore.Tag().UpdateTagFunc(tagID, func(tag *portainer.Tag) { + tag.EndpointGroups[endpointGroup.ID] = true + }) - tag.EndpointGroups[endpointGroup.ID] = true - - err = handler.DataStore.Tag().UpdateTag(tagID, tag) - if err != nil { + if handler.DataStore.IsErrObjectNotFound(err) { + return httperror.InternalServerError("Unable to find a tag inside the database", err) + } else if err != nil { return httperror.InternalServerError("Unable to persist tag changes inside the database", err) } } diff --git a/api/http/handler/endpointgroups/endpointgroup_delete.go b/api/http/handler/endpointgroups/endpointgroup_delete.go index feea30c53..a96bd7278 100644 --- a/api/http/handler/endpointgroups/endpointgroup_delete.go +++ b/api/http/handler/endpointgroups/endpointgroup_delete.go @@ -66,15 +66,13 @@ func (handler *Handler) endpointGroupDelete(w http.ResponseWriter, r *http.Reque } for _, tagID := range endpointGroup.TagIDs { - tag, err := handler.DataStore.Tag().Tag(tagID) - if err != nil { - return httperror.InternalServerError("Unable to retrieve tag from the database", err) - } + handler.DataStore.Tag().UpdateTagFunc(tagID, func(tag *portainer.Tag) { + delete(tag.EndpointGroups, endpointGroup.ID) + }) - delete(tag.EndpointGroups, endpointGroup.ID) - - err = handler.DataStore.Tag().UpdateTag(tagID, tag) - if err != nil { + if handler.DataStore.IsErrObjectNotFound(err) { + return httperror.InternalServerError("Unable to find a tag inside the database", err) + } else if err != nil { return httperror.InternalServerError("Unable to persist tag changes inside the database", err) } } diff --git a/api/http/handler/endpointgroups/endpointgroup_update.go b/api/http/handler/endpointgroups/endpointgroup_update.go index 8f9578f3c..fbbd3384a 100644 --- a/api/http/handler/endpointgroups/endpointgroup_update.go +++ b/api/http/handler/endpointgroups/endpointgroup_update.go @@ -81,28 +81,26 @@ func (handler *Handler) endpointGroupUpdate(w http.ResponseWriter, r *http.Reque removeTags := tag.Difference(endpointGroupTagSet, payloadTagSet) for tagID := range removeTags { - tag, err := handler.DataStore.Tag().Tag(tagID) - if err != nil { + handler.DataStore.Tag().UpdateTagFunc(tagID, func(tag *portainer.Tag) { + delete(tag.EndpointGroups, endpointGroup.ID) + }) + + if handler.DataStore.IsErrObjectNotFound(err) { return httperror.InternalServerError("Unable to find a tag inside the database", err) - } - delete(tag.EndpointGroups, endpointGroup.ID) - err = handler.DataStore.Tag().UpdateTag(tag.ID, tag) - if err != nil { + } else if err != nil { return httperror.InternalServerError("Unable to persist tag changes inside the database", err) } } endpointGroup.TagIDs = payload.TagIDs for _, tagID := range payload.TagIDs { - tag, err := handler.DataStore.Tag().Tag(tagID) - if err != nil { + handler.DataStore.Tag().UpdateTagFunc(tagID, func(tag *portainer.Tag) { + tag.EndpointGroups[endpointGroup.ID] = true + }) + + if handler.DataStore.IsErrObjectNotFound(err) { return httperror.InternalServerError("Unable to find a tag inside the database", err) - } - - tag.EndpointGroups[endpointGroup.ID] = true - - err = handler.DataStore.Tag().UpdateTag(tag.ID, tag) - if err != nil { + } else if err != nil { return httperror.InternalServerError("Unable to persist tag changes inside the database", err) } } diff --git a/api/http/handler/endpoints/endpoint_create.go b/api/http/handler/endpoints/endpoint_create.go index a54dad51c..c2df56ab8 100644 --- a/api/http/handler/endpoints/endpoint_create.go +++ b/api/http/handler/endpoints/endpoint_create.go @@ -530,14 +530,9 @@ func (handler *Handler) saveEndpointAndUpdateAuthorizations(endpoint *portainer. } for _, tagID := range endpoint.TagIDs { - tag, err := handler.DataStore.Tag().Tag(tagID) - if err != nil { - return err - } - - tag.Endpoints[endpoint.ID] = true - - err = handler.DataStore.Tag().UpdateTag(tagID, tag) + err = handler.DataStore.Tag().UpdateTagFunc(tagID, func(tag *portainer.Tag) { + tag.Endpoints[endpoint.ID] = true + }) if err != nil { return err } diff --git a/api/http/handler/endpoints/endpoint_delete.go b/api/http/handler/endpoints/endpoint_delete.go index e271fbbe7..45a840970 100644 --- a/api/http/handler/endpoints/endpoint_delete.go +++ b/api/http/handler/endpoints/endpoint_delete.go @@ -62,15 +62,13 @@ func (handler *Handler) endpointDelete(w http.ResponseWriter, r *http.Request) * } for _, tagID := range endpoint.TagIDs { - tag, err := handler.DataStore.Tag().Tag(tagID) - if err != nil { + err = handler.DataStore.Tag().UpdateTagFunc(tagID, func(tag *portainer.Tag) { + delete(tag.Endpoints, endpoint.ID) + }) + + if handler.DataStore.IsErrObjectNotFound(err) { return httperror.NotFound("Unable to find tag inside the database", err) - } - - delete(tag.Endpoints, endpoint.ID) - - err = handler.DataStore.Tag().UpdateTag(tagID, tag) - if err != nil { + } else if err != nil { return httperror.InternalServerError("Unable to persist tag relation inside the database", err) } } diff --git a/api/http/handler/endpoints/endpoint_update.go b/api/http/handler/endpoints/endpoint_update.go index d34d33428..a783a1b01 100644 --- a/api/http/handler/endpoints/endpoint_update.go +++ b/api/http/handler/endpoints/endpoint_update.go @@ -139,29 +139,26 @@ func (handler *Handler) endpointUpdate(w http.ResponseWriter, r *http.Request) * removeTags := tag.Difference(endpointTagSet, payloadTagSet) for tagID := range removeTags { - tag, err := handler.DataStore.Tag().Tag(tagID) - if err != nil { - return httperror.InternalServerError("Unable to find a tag inside the database", err) - } + err = handler.DataStore.Tag().UpdateTagFunc(tagID, func(tag *portainer.Tag) { + delete(tag.Endpoints, endpoint.ID) + }) - delete(tag.Endpoints, endpoint.ID) - err = handler.DataStore.Tag().UpdateTag(tag.ID, tag) - if err != nil { + if handler.DataStore.IsErrObjectNotFound(err) { + return httperror.InternalServerError("Unable to find a tag inside the database", err) + } else if err != nil { return httperror.InternalServerError("Unable to persist tag changes inside the database", err) } } endpoint.TagIDs = payload.TagIDs for _, tagID := range payload.TagIDs { - tag, err := handler.DataStore.Tag().Tag(tagID) - if err != nil { + err = handler.DataStore.Tag().UpdateTagFunc(tagID, func(tag *portainer.Tag) { + tag.Endpoints[endpoint.ID] = true + }) + + if handler.DataStore.IsErrObjectNotFound(err) { return httperror.InternalServerError("Unable to find a tag inside the database", err) - } - - tag.Endpoints[endpoint.ID] = true - - err = handler.DataStore.Tag().UpdateTag(tag.ID, tag) - if err != nil { + } else if err != nil { return httperror.InternalServerError("Unable to persist tag changes inside the database", err) } } From ae2bec4bd9db3565acb65ceff363b467715b3afd Mon Sep 17 00:00:00 2001 From: andres-portainer <91705312+andres-portainer@users.noreply.github.com> Date: Fri, 14 Oct 2022 18:09:07 -0300 Subject: [PATCH 03/31] fix(code): clean up EE-4432 (#7865) --- api/cmd/portainer/main.go | 7 +------ api/crypto/aes.go | 2 +- api/exec/swarm_stack.go | 7 +------ api/filesystem/filesystem.go | 2 +- api/hostmanagement/openamt/deviceFeatures.go | 5 +---- api/hostmanagement/openamt/openamt.go | 12 ++--------- .../customtemplates/customtemplate_create.go | 21 +++---------------- .../customtemplates/customtemplate_update.go | 7 +------ api/http/handler/endpoints/endpoint_update.go | 6 +----- api/http/handler/hostmanagement/fdo/fdo.go | 7 +------ .../openamt/amtconfiguration.go | 6 +----- .../handler/hostmanagement/openamt/amtrpc.go | 10 ++------- api/http/handler/kubernetes/ingresses.go | 2 +- api/http/handler/settings/settings_update.go | 7 +------ api/http/handler/websocket/attach.go | 7 +------ api/http/handler/websocket/exec.go | 7 +------ .../proxy/factory/azure/containergroup.go | 5 +---- api/http/security/rate_limiter.go | 2 +- api/internal/ssl/ssl.go | 7 +------ api/kubernetes/cli/access.go | 5 +---- api/stacks/stackutils/validation.go | 2 +- 21 files changed, 25 insertions(+), 111 deletions(-) diff --git a/api/cmd/portainer/main.go b/api/cmd/portainer/main.go index ed503ad65..fd5fa1b72 100644 --- a/api/cmd/portainer/main.go +++ b/api/cmd/portainer/main.go @@ -316,12 +316,7 @@ func updateSettingsFromFlags(dataStore dataservices.DataStore, flags *portainer. sslSettings.HTTPEnabled = true } - err = dataStore.SSLSettings().UpdateSettings(sslSettings) - if err != nil { - return err - } - - return nil + return dataStore.SSLSettings().UpdateSettings(sslSettings) } // enableFeaturesFromFlags turns on or off feature flags diff --git a/api/crypto/aes.go b/api/crypto/aes.go index 5a6f58fe7..6dd273310 100644 --- a/api/crypto/aes.go +++ b/api/crypto/aes.go @@ -13,7 +13,7 @@ import ( // Person with better knowledge is welcomed to improve it. // sourced from https://golang.org/src/crypto/cipher/example_test.go -var emptySalt []byte = make([]byte, 0, 0) +var emptySalt []byte = make([]byte, 0) // AesEncrypt reads from input, encrypts with AES-256 and writes to the output. // passphrase is used to generate an encryption key. diff --git a/api/exec/swarm_stack.go b/api/exec/swarm_stack.go index 78686333c..114ac3fd4 100644 --- a/api/exec/swarm_stack.go +++ b/api/exec/swarm_stack.go @@ -203,12 +203,7 @@ func (manager *SwarmStackManager) updateDockerCLIConfiguration(configPath string headersObject["X-PortainerAgent-Signature"] = signature headersObject["X-PortainerAgent-PublicKey"] = manager.signatureService.EncodedPublicKey() - err = manager.fileService.WriteJSONToFile(configFilePath, config) - if err != nil { - return err - } - - return nil + return manager.fileService.WriteJSONToFile(configFilePath, config) } func (manager *SwarmStackManager) retrieveConfigurationFromDisk(path string) (map[string]interface{}, error) { diff --git a/api/filesystem/filesystem.go b/api/filesystem/filesystem.go index f0202feec..981f1ea97 100644 --- a/api/filesystem/filesystem.go +++ b/api/filesystem/filesystem.go @@ -163,7 +163,7 @@ func (service *Service) Copy(fromFilePath string, toFilePath string, deleteIfExi } if !exists { - return errors.New(fmt.Sprintf("File (%s) doesn't exist", fromFilePath)) + return fmt.Errorf("File (%s) doesn't exist", fromFilePath) } finput, err := os.Open(fromFilePath) diff --git a/api/hostmanagement/openamt/deviceFeatures.go b/api/hostmanagement/openamt/deviceFeatures.go index e71d96684..cce18a8db 100644 --- a/api/hostmanagement/openamt/deviceFeatures.go +++ b/api/hostmanagement/openamt/deviceFeatures.go @@ -21,9 +21,6 @@ func (service *Service) enableDeviceFeatures(configuration portainer.OpenAMTConf jsonValue, _ := json.Marshal(payload) _, err := service.executeSaveRequest(http.MethodPost, url, configuration.MPSToken, jsonValue) - if err != nil { - return err - } - return nil + return err } diff --git a/api/hostmanagement/openamt/openamt.go b/api/hostmanagement/openamt/openamt.go index ca03c999e..863410d14 100644 --- a/api/hostmanagement/openamt/openamt.go +++ b/api/hostmanagement/openamt/openamt.go @@ -83,11 +83,8 @@ func (service *Service) Configure(configuration portainer.OpenAMTConfiguration) } _, err = service.createOrUpdateDomain(configuration) - if err != nil { - return err - } - return nil + return err } func (service *Service) executeSaveRequest(method string, url string, token string, payload []byte) ([]byte, error) { @@ -229,12 +226,7 @@ func (service *Service) ExecuteDeviceAction(configuration portainer.OpenAMTConfi } configuration.MPSToken = token - err = service.executeDeviceAction(configuration, deviceGUID, int(parsedAction)) - if err != nil { - return err - } - - return nil + return service.executeDeviceAction(configuration, deviceGUID, int(parsedAction)) } func (service *Service) EnableDeviceFeatures(configuration portainer.OpenAMTConfiguration, deviceGUID string, features portainer.OpenAMTDeviceEnabledFeatures) (string, error) { diff --git a/api/http/handler/customtemplates/customtemplate_create.go b/api/http/handler/customtemplates/customtemplate_create.go index c026a540e..34a291fb5 100644 --- a/api/http/handler/customtemplates/customtemplate_create.go +++ b/api/http/handler/customtemplates/customtemplate_create.go @@ -142,12 +142,7 @@ func (payload *customTemplateFromFileContentPayload) Validate(r *http.Request) e return errors.New("Invalid note. tag is not supported") } - err := validateVariablesDefinitions(payload.Variables) - if err != nil { - return err - } - - return nil + return validateVariablesDefinitions(payload.Variables) } func isValidNote(note string) bool { @@ -251,12 +246,7 @@ func (payload *customTemplateFromGitRepositoryPayload) Validate(r *http.Request) return errors.New("Invalid note. tag is not supported") } - err := validateVariablesDefinitions(payload.Variables) - if err != nil { - return err - } - - return nil + return validateVariablesDefinitions(payload.Variables) } func (handler *Handler) createCustomTemplateFromGitRepository(r *http.Request) (*portainer.CustomTemplate, error) { @@ -395,12 +385,7 @@ func (payload *customTemplateFromFileUploadPayload) Validate(r *http.Request) er return errors.New("Invalid variables. Ensure that the variables are valid JSON") } - err = validateVariablesDefinitions(payload.Variables) - if err != nil { - return err - } - - return nil + return validateVariablesDefinitions(payload.Variables) } func (handler *Handler) createCustomTemplateFromFileUpload(r *http.Request) (*portainer.CustomTemplate, error) { diff --git a/api/http/handler/customtemplates/customtemplate_update.go b/api/http/handler/customtemplates/customtemplate_update.go index 7b0a5e750..38e10fe9a 100644 --- a/api/http/handler/customtemplates/customtemplate_update.go +++ b/api/http/handler/customtemplates/customtemplate_update.go @@ -55,12 +55,7 @@ func (payload *customTemplateUpdatePayload) Validate(r *http.Request) error { return errors.New("Invalid note. tag is not supported") } - err := validateVariablesDefinitions(payload.Variables) - if err != nil { - return err - } - - return nil + return validateVariablesDefinitions(payload.Variables) } // @id CustomTemplateUpdate diff --git a/api/http/handler/endpoints/endpoint_update.go b/api/http/handler/endpoints/endpoint_update.go index a783a1b01..a35f383ab 100644 --- a/api/http/handler/endpoints/endpoint_update.go +++ b/api/http/handler/endpoints/endpoint_update.go @@ -190,12 +190,8 @@ func (handler *Handler) endpointUpdate(w http.ResponseWriter, r *http.Request) * switch *payload.Status { case 1: endpoint.Status = portainer.EndpointStatusUp - break case 2: endpoint.Status = portainer.EndpointStatusDown - break - default: - break } } @@ -328,7 +324,7 @@ func (handler *Handler) endpointUpdate(w http.ResponseWriter, r *http.Request) * err = handler.SnapshotService.FillSnapshotData(endpoint) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to add snapshot data", err} + return httperror.InternalServerError("Unable to add snapshot data", err) } return response.JSON(w, endpoint) diff --git a/api/http/handler/hostmanagement/fdo/fdo.go b/api/http/handler/hostmanagement/fdo/fdo.go index ef846a648..69851f454 100644 --- a/api/http/handler/hostmanagement/fdo/fdo.go +++ b/api/http/handler/hostmanagement/fdo/fdo.go @@ -128,12 +128,7 @@ func (handler *Handler) addDefaultProfile() error { profile.FilePath = filePath profile.DateCreated = time.Now().Unix() - err = handler.DataStore.FDOProfile().Create(profile) - if err != nil { - return err - } - - return nil + return handler.DataStore.FDOProfile().Create(profile) } const defaultProfileFileContent = ` diff --git a/api/http/handler/hostmanagement/openamt/amtconfiguration.go b/api/http/handler/hostmanagement/openamt/amtconfiguration.go index 6ffe5667f..ff8db308b 100644 --- a/api/http/handler/hostmanagement/openamt/amtconfiguration.go +++ b/api/http/handler/hostmanagement/openamt/amtconfiguration.go @@ -169,12 +169,8 @@ func (handler *Handler) saveConfiguration(configuration portainer.OpenAMTConfigu configuration.MPSToken = "" settings.OpenAMTConfiguration = configuration - err = handler.DataStore.Settings().UpdateSettings(settings) - if err != nil { - return err - } - return nil + return handler.DataStore.Settings().UpdateSettings(settings) } func (handler *Handler) disableOpenAMT() error { diff --git a/api/http/handler/hostmanagement/openamt/amtrpc.go b/api/http/handler/hostmanagement/openamt/amtrpc.go index a4922d276..643e5dd65 100644 --- a/api/http/handler/hostmanagement/openamt/amtrpc.go +++ b/api/http/handler/hostmanagement/openamt/amtrpc.go @@ -295,11 +295,8 @@ func (handler *Handler) activateDevice(endpoint *portainer.Endpoint, settings po } _, err := handler.PullAndRunContainer(ctx, endpoint, rpcGoImageName, rpcGoContainerName, cmdLine) - if err != nil { - return err - } - return nil + return err } func (handler *Handler) deactivateDevice(endpoint *portainer.Endpoint, settings portainer.Settings) error { @@ -315,9 +312,6 @@ func (handler *Handler) deactivateDevice(endpoint *portainer.Endpoint, settings } _, err := handler.PullAndRunContainer(ctx, endpoint, rpcGoImageName, rpcGoContainerName, cmdLine) - if err != nil { - return err - } - return nil + return err } diff --git a/api/http/handler/kubernetes/ingresses.go b/api/http/handler/kubernetes/ingresses.go index 0dfb8fadb..741436f82 100644 --- a/api/http/handler/kubernetes/ingresses.go +++ b/api/http/handler/kubernetes/ingresses.go @@ -328,7 +328,7 @@ PayloadLoop: updatedClass.GloballyBlocked = existingClass.GloballyBlocked // Handle "allow" - if p.Availability == true { + if p.Availability { // remove the namespace from the list of blocked namespaces // in the existingClass. for _, blockedNS := range existingClass.BlockedNamespaces { diff --git a/api/http/handler/settings/settings_update.go b/api/http/handler/settings/settings_update.go index 49a005207..da5ee2d0c 100644 --- a/api/http/handler/settings/settings_update.go +++ b/api/http/handler/settings/settings_update.go @@ -251,12 +251,7 @@ func (handler *Handler) settingsUpdate(w http.ResponseWriter, r *http.Request) * func (handler *Handler) updateSnapshotInterval(settings *portainer.Settings, snapshotInterval string) error { settings.SnapshotInterval = snapshotInterval - err := handler.SnapshotService.SetSnapshotInterval(snapshotInterval) - if err != nil { - return err - } - - return nil + return handler.SnapshotService.SetSnapshotInterval(snapshotInterval) } func (handler *Handler) updateTLS(settings *portainer.Settings) *httperror.HandlerError { diff --git a/api/http/handler/websocket/attach.go b/api/http/handler/websocket/attach.go index b4a5afb48..fcd9dcc8f 100644 --- a/api/http/handler/websocket/attach.go +++ b/api/http/handler/websocket/attach.go @@ -116,12 +116,7 @@ func hijackAttachStartOperation(websocketConn *websocket.Conn, endpoint *portain return err } - err = hijackRequest(websocketConn, httpConn, attachStartRequest) - if err != nil { - return err - } - - return nil + return hijackRequest(websocketConn, httpConn, attachStartRequest) } func createAttachStartRequest(attachID string) (*http.Request, error) { diff --git a/api/http/handler/websocket/exec.go b/api/http/handler/websocket/exec.go index fb50403eb..ae95e99a0 100644 --- a/api/http/handler/websocket/exec.go +++ b/api/http/handler/websocket/exec.go @@ -121,12 +121,7 @@ func hijackExecStartOperation(websocketConn *websocket.Conn, endpoint *portainer return err } - err = hijackRequest(websocketConn, httpConn, execStartRequest) - if err != nil { - return err - } - - return nil + return hijackRequest(websocketConn, httpConn, execStartRequest) } func createExecStartRequest(execID string) (*http.Request, error) { diff --git a/api/http/proxy/factory/azure/containergroup.go b/api/http/proxy/factory/azure/containergroup.go index d5eeb7a10..1f8ca581c 100644 --- a/api/http/proxy/factory/azure/containergroup.go +++ b/api/http/proxy/factory/azure/containergroup.go @@ -85,11 +85,8 @@ func (transport *Transport) proxyContainerGroupPutRequest(request *http.Request) responseObject = decorateObject(responseObject, resourceControl) err = utils.RewriteResponse(response, responseObject, http.StatusOK) - if err != nil { - return response, err - } - return response, nil + return response, err } func (transport *Transport) proxyContainerGroupGetRequest(request *http.Request) (*http.Response, error) { diff --git a/api/http/security/rate_limiter.go b/api/http/security/rate_limiter.go index 65eea5d4a..58ad63315 100644 --- a/api/http/security/rate_limiter.go +++ b/api/http/security/rate_limiter.go @@ -29,7 +29,7 @@ func NewRateLimiter(maxRequests int, duration time.Duration, banDuration time.Du func (limiter *RateLimiter) LimitAccess(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ip := StripAddrPort(r.RemoteAddr) - if banned := limiter.Inc(ip); banned == true { + if banned := limiter.Inc(ip); banned { httperror.WriteError(w, http.StatusForbidden, "Access denied", errors.ErrResourceAccessDenied) return } diff --git a/api/internal/ssl/ssl.go b/api/internal/ssl/ssl.go index 419b54ad0..f52b6bec5 100644 --- a/api/internal/ssl/ssl.go +++ b/api/internal/ssl/ssl.go @@ -166,10 +166,5 @@ func (service *Service) cacheInfo(certPath string, keyPath string, selfSigned bo settings.KeyPath = keyPath settings.SelfSigned = selfSigned - err = service.dataStore.SSLSettings().UpdateSettings(settings) - if err != nil { - return err - } - - return nil + return service.dataStore.SSLSettings().UpdateSettings(settings) } diff --git a/api/kubernetes/cli/access.go b/api/kubernetes/cli/access.go index a510c73d8..729e3c801 100644 --- a/api/kubernetes/cli/access.go +++ b/api/kubernetes/cli/access.go @@ -117,9 +117,6 @@ func (kcl *KubeClient) UpdateNamespaceAccessPolicies(accessPolicies map[string]p configMap.Data[portainerConfigMapAccessPoliciesKey] = string(data) _, err = kcl.cli.CoreV1().ConfigMaps(portainerNamespace).Update(context.TODO(), configMap, metav1.UpdateOptions{}) - if err != nil { - return err - } - return nil + return err } diff --git a/api/stacks/stackutils/validation.go b/api/stacks/stackutils/validation.go index e5ffe564d..daace6e60 100644 --- a/api/stacks/stackutils/validation.go +++ b/api/stacks/stackutils/validation.go @@ -43,7 +43,7 @@ func IsValidStackFile(stackFileContent []byte, securitySettings *portainer.Endpo } } - if !securitySettings.AllowPrivilegedModeForRegularUsers && service.Privileged == true { + if !securitySettings.AllowPrivilegedModeForRegularUsers && service.Privileged { return errors.New("privileged mode disabled for non administrator users") } From 191f8e17eea9ef2d255a7b6eb505d24581a63245 Mon Sep 17 00:00:00 2001 From: andres-portainer <91705312+andres-portainer@users.noreply.github.com> Date: Fri, 14 Oct 2022 19:42:31 -0300 Subject: [PATCH 04/31] fix(code): remove unused code EE-4431 (#7866) --- api/cmd/portainer/import.go | 28 ------ .../migrator/migrate_dbversion34_test.go | 94 ------------------- api/http/handler/backup/handler.go | 9 -- api/http/handler/customtemplates/handler.go | 18 ---- .../handler/hostmanagement/openamt/amtrpc.go | 17 ---- .../registries/registry_create_test.go | 88 ----------------- .../registries/registry_update_test.go | 88 ----------------- .../resourcecontrol_create.go | 5 +- api/http/handler/webhooks/handler.go | 45 +-------- api/http/proxy/factory/agent/transport.go | 13 +-- 10 files changed, 8 insertions(+), 397 deletions(-) delete mode 100644 api/cmd/portainer/import.go delete mode 100644 api/datastore/migrator/migrate_dbversion34_test.go delete mode 100644 api/http/handler/registries/registry_update_test.go diff --git a/api/cmd/portainer/import.go b/api/cmd/portainer/import.go deleted file mode 100644 index 7ce53f4a8..000000000 --- a/api/cmd/portainer/import.go +++ /dev/null @@ -1,28 +0,0 @@ -package main - -import ( - portainer "github.com/portainer/portainer/api" - "github.com/portainer/portainer/api/datastore" - - "github.com/rs/zerolog/log" -) - -func importFromJson(fileService portainer.FileService, store *datastore.Store) { - // EXPERIMENTAL - if used with an incomplete json file, it will fail, as we don't have a way to default the model values - importFile := "/data/import.json" - if exists, _ := fileService.FileExists(importFile); exists { - if err := store.Import(importFile); err != nil { - log.Error().Str("filename", importFile).Err(err).Msg("import failed") - // TODO: should really rollback on failure, but then we have nothing. - } else { - log.Info().Str("filename", importFile).Msg("successfully imported the file to a new portainer database") - } - - // TODO: this is bad - its to ensure that any defaults that were broken in import, or migrations get set back to what we want - // I also suspect that everything from "Init to Init" is potentially a migration - err := store.Init() - if err != nil { - log.Fatal().Err(err).Msg("failed initializing data store") - } - } -} diff --git a/api/datastore/migrator/migrate_dbversion34_test.go b/api/datastore/migrator/migrate_dbversion34_test.go deleted file mode 100644 index 5938a4caa..000000000 --- a/api/datastore/migrator/migrate_dbversion34_test.go +++ /dev/null @@ -1,94 +0,0 @@ -package migrator - -const ( - db35TestFile = "portainer-mig-35.db" - username = "portainer" - password = "password" -) - -// TODO: this is exactly the kind of reaching into the internals of the store we should not do -// func setupDB35Test(t *testing.T) *Migrator { -// is := assert.New(t) -// dbConn, err := bolt.Open(path.Join(t.TempDir(), db35TestFile), 0600, &bolt.Options{Timeout: 1 * time.Second}) -// is.NoError(err, "failed to init testing DB connection") - -// // Create an old style dockerhub authenticated account -// dockerhubService, err := dockerhub.NewService(&database.DbConnection{DB: dbConn}) -// is.NoError(err, "failed to init testing registry service") -// err = dockerhubService.UpdateDockerHub(&portainer.DockerHub{true, username, password}) -// is.NoError(err, "failed to create dockerhub account") - -// registryService, err := registry.NewService(&database.DbConnection{DB: dbConn}) -// is.NoError(err, "failed to init testing registry service") - -// endpointService, err := endpoint.NewService(&database.DbConnection{DB: dbConn}) -// is.NoError(err, "failed to init endpoint service") - -// m := &Migrator{ -// db: dbConn, -// dockerhubService: dockerhubService, -// registryService: registryService, -// endpointService: endpointService, -// } - -// return m -// } - -// // TestUpdateDockerhubToDB32 tests a normal upgrade -// func TestUpdateDockerhubToDB32(t *testing.T) { -// is := assert.New(t) -// m := setupDB35Test(t) -// defer m.db.Close() -// defer os.Remove(db35TestFile) - -// if err := m.updateDockerhubToDB32(); err != nil { -// t.Errorf("failed to update settings: %v", err) -// } - -// // Verify we have a single registry were created -// registries, err := m.registryService.Registries() -// is.NoError(err, "failed to read registries from the RegistryService") -// is.Equal(len(registries), 1, "only one migrated registry expected") -// } - -// // TestUpdateDockerhubToDB32_with_duplicate_migrations tests an upgrade where in earlier versions a broken migration -// // created a large number of duplicate "dockerhub migrated" registry entries. -// func TestUpdateDockerhubToDB32_with_duplicate_migrations(t *testing.T) { -// is := assert.New(t) -// m := setupDB35Test(t) -// defer m.db.Close() -// defer os.Remove(db35TestFile) - -// // Create lots of duplicate entries... -// registry := &portainer.Registry{ -// Type: portainer.DockerHubRegistry, -// Name: "Dockerhub (authenticated - migrated)", -// URL: "docker.io", -// Authentication: true, -// Username: "portainer", -// Password: "password", -// RegistryAccesses: portainer.RegistryAccesses{}, -// } - -// for i := 1; i < 150; i++ { -// err := m.registryService.CreateRegistry(registry) -// assert.NoError(t, err, "create registry failed") -// } - -// // Verify they were created -// registries, err := m.registryService.Registries() -// is.NoError(err, "failed to read registries from the RegistryService") -// is.Condition(func() bool { -// return len(registries) > 1 -// }, "expected multiple duplicate registry entries") - -// // Now run the migrator -// if err := m.updateDockerhubToDB32(); err != nil { -// t.Errorf("failed to update settings: %v", err) -// } - -// // Verify we have a single registry were created -// registries, err = m.registryService.Registries() -// is.NoError(err, "failed to read registries from the RegistryService") -// is.Equal(len(registries), 1, "only one migrated registry expected") -// } diff --git a/api/http/handler/backup/handler.go b/api/http/handler/backup/handler.go index 500a3c374..cd95d71b8 100644 --- a/api/http/handler/backup/handler.go +++ b/api/http/handler/backup/handler.go @@ -6,7 +6,6 @@ import ( "github.com/gorilla/mux" httperror "github.com/portainer/libhttp/error" - portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/adminmonitor" "github.com/portainer/portainer/api/dataservices" "github.com/portainer/portainer/api/demo" @@ -71,11 +70,3 @@ func adminAccess(next http.Handler) http.Handler { next.ServeHTTP(w, r) }) } - -func systemWasInitialized(dataStore dataservices.DataStore) (bool, error) { - users, err := dataStore.User().UsersByRole(portainer.AdministratorRole) - if err != nil { - return false, err - } - return len(users) > 0, nil -} diff --git a/api/http/handler/customtemplates/handler.go b/api/http/handler/customtemplates/handler.go index 7bc9b4b3d..54fad7d67 100644 --- a/api/http/handler/customtemplates/handler.go +++ b/api/http/handler/customtemplates/handler.go @@ -8,7 +8,6 @@ import ( portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/dataservices" "github.com/portainer/portainer/api/http/security" - "github.com/portainer/portainer/api/internal/authorization" ) // Handler is the HTTP handler used to handle environment(endpoint) group operations. @@ -42,20 +41,3 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler { func userCanEditTemplate(customTemplate *portainer.CustomTemplate, securityContext *security.RestrictedRequestContext) bool { return securityContext.IsAdmin || customTemplate.CreatedByUserID == securityContext.UserID } - -func userCanAccessTemplate(customTemplate portainer.CustomTemplate, securityContext *security.RestrictedRequestContext, resourceControl *portainer.ResourceControl) bool { - if securityContext.IsAdmin || customTemplate.CreatedByUserID == securityContext.UserID { - return true - } - - userTeamIDs := make([]portainer.TeamID, 0) - for _, membership := range securityContext.UserMemberships { - userTeamIDs = append(userTeamIDs, membership.TeamID) - } - - if resourceControl != nil && authorization.UserCanAccessResource(securityContext.UserID, userTeamIDs, resourceControl) { - return true - } - - return false -} diff --git a/api/http/handler/hostmanagement/openamt/amtrpc.go b/api/http/handler/hostmanagement/openamt/amtrpc.go index 643e5dd65..fe0c9e162 100644 --- a/api/http/handler/hostmanagement/openamt/amtrpc.go +++ b/api/http/handler/hostmanagement/openamt/amtrpc.go @@ -298,20 +298,3 @@ func (handler *Handler) activateDevice(endpoint *portainer.Endpoint, settings po return err } - -func (handler *Handler) deactivateDevice(endpoint *portainer.Endpoint, settings portainer.Settings) error { - ctx := context.TODO() - - config := settings.OpenAMTConfiguration - cmdLine := []string{ - "deactivate", - "-n", - "-v", - "-u", fmt.Sprintf("wss://%s/activate", config.MPSServer), - "-password", config.MPSPassword, - } - - _, err := handler.PullAndRunContainer(ctx, endpoint, rpcGoImageName, rpcGoContainerName, cmdLine) - - return err -} diff --git a/api/http/handler/registries/registry_create_test.go b/api/http/handler/registries/registry_create_test.go index 711a6ef36..515dc6c5e 100644 --- a/api/http/handler/registries/registry_create_test.go +++ b/api/http/handler/registries/registry_create_test.go @@ -1,15 +1,9 @@ package registries import ( - "bytes" - "encoding/json" - "net/http" - "net/http/httptest" "testing" portainer "github.com/portainer/portainer/api" - "github.com/portainer/portainer/api/dataservices" - "github.com/portainer/portainer/api/http/security" "github.com/stretchr/testify/assert" ) @@ -49,85 +43,3 @@ func Test_registryCreatePayload_Validate(t *testing.T) { assert.NoError(t, err) }) } - -type testRegistryService struct { - dataservices.RegistryService - createRegistry func(r *portainer.Registry) error - updateRegistry func(ID portainer.RegistryID, r *portainer.Registry) error - getRegistry func(ID portainer.RegistryID) (*portainer.Registry, error) -} - -type testDataStore struct { - dataservices.DataStore - registry *testRegistryService -} - -func (t testDataStore) Registry() dataservices.RegistryService { - return t.registry -} - -func (t testRegistryService) CreateRegistry(r *portainer.Registry) error { - return t.createRegistry(r) -} - -func (t testRegistryService) UpdateRegistry(ID portainer.RegistryID, r *portainer.Registry) error { - return t.updateRegistry(ID, r) -} - -func (t testRegistryService) Registry(ID portainer.RegistryID) (*portainer.Registry, error) { - return t.getRegistry(ID) -} - -func (t testRegistryService) Registries() ([]portainer.Registry, error) { - return nil, nil -} - -func (t testRegistryService) Create(registry *portainer.Registry) error { - return nil -} - -// Not entirely sure what this is intended to test -func deleteTestHandler_registryCreate(t *testing.T) { - payload := registryCreatePayload{ - Name: "Test registry", - Type: portainer.ProGetRegistry, - URL: "http://example.com", - BaseURL: "http://example.com", - Authentication: false, - Username: "username", - Password: "password", - Gitlab: portainer.GitlabRegistryData{}, - } - payloadBytes, err := json.Marshal(payload) - assert.NoError(t, err) - r := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(payloadBytes)) - w := httptest.NewRecorder() - - restrictedContext := &security.RestrictedRequestContext{ - IsAdmin: true, - UserID: portainer.UserID(1), - } - - ctx := security.StoreRestrictedRequestContext(r, restrictedContext) - r = r.WithContext(ctx) - - registry := portainer.Registry{} - handler := Handler{} - handler.DataStore = testDataStore{ - registry: &testRegistryService{ - createRegistry: func(r *portainer.Registry) error { - registry = *r - return nil - }, - }, - } - handlerError := handler.registryCreate(w, r) - assert.Nil(t, handlerError) - assert.Equal(t, payload.Name, registry.Name) - assert.Equal(t, payload.Type, registry.Type) - assert.Equal(t, payload.URL, registry.URL) - assert.Equal(t, payload.BaseURL, registry.BaseURL) - assert.Equal(t, payload.Authentication, registry.Authentication) - assert.Equal(t, payload.Username, registry.Username) - assert.Equal(t, payload.Password, registry.Password) -} diff --git a/api/http/handler/registries/registry_update_test.go b/api/http/handler/registries/registry_update_test.go deleted file mode 100644 index 59c28d695..000000000 --- a/api/http/handler/registries/registry_update_test.go +++ /dev/null @@ -1,88 +0,0 @@ -package registries - -import ( - "bytes" - "encoding/json" - "net/http" - "net/http/httptest" - "testing" - - portainer "github.com/portainer/portainer/api" - "github.com/portainer/portainer/api/http/security" - "github.com/stretchr/testify/assert" -) - -func ps(s string) *string { - return &s -} - -func pb(b bool) *bool { - return &b -} - -type TestBouncer struct{} - -func (t TestBouncer) AdminAccess(h http.Handler) http.Handler { - return h -} - -func (t TestBouncer) AuthenticatedAccess(h http.Handler) http.Handler { - return h -} - -func (t TestBouncer) AuthorizedEndpointOperation(r *http.Request, endpoint *portainer.Endpoint) error { - return nil -} - -// TODO, no i don't know what this is actually intended to test either. -func delete_TestHandler_registryUpdate(t *testing.T) { - payload := registryUpdatePayload{ - Name: ps("Updated test registry"), - URL: ps("http://example.org/feed"), - BaseURL: ps("http://example.org"), - Authentication: pb(true), - Username: ps("username"), - Password: ps("password"), - } - payloadBytes, err := json.Marshal(payload) - assert.NoError(t, err) - registry := portainer.Registry{Type: portainer.ProGetRegistry, ID: 5} - r := httptest.NewRequest(http.MethodPut, "/registries/5", bytes.NewReader(payloadBytes)) - w := httptest.NewRecorder() - - restrictedContext := &security.RestrictedRequestContext{ - IsAdmin: true, - UserID: portainer.UserID(1), - } - - ctx := security.StoreRestrictedRequestContext(r, restrictedContext) - r = r.WithContext(ctx) - - updatedRegistry := portainer.Registry{} - handler := newHandler(nil) - handler.initRouter(TestBouncer{}) - handler.DataStore = testDataStore{ - registry: &testRegistryService{ - getRegistry: func(_ portainer.RegistryID) (*portainer.Registry, error) { - return ®istry, nil - }, - updateRegistry: func(ID portainer.RegistryID, r *portainer.Registry) error { - assert.Equal(t, ID, r.ID) - updatedRegistry = *r - return nil - }, - }, - } - - handler.ServeHTTP(w, r) - assert.Equal(t, http.StatusOK, w.Code) - // Registry type should remain intact - assert.Equal(t, registry.Type, updatedRegistry.Type) - - assert.Equal(t, *payload.Name, updatedRegistry.Name) - assert.Equal(t, *payload.URL, updatedRegistry.URL) - assert.Equal(t, *payload.BaseURL, updatedRegistry.BaseURL) - assert.Equal(t, *payload.Authentication, updatedRegistry.Authentication) - assert.Equal(t, *payload.Username, updatedRegistry.Username) - assert.Equal(t, *payload.Password, updatedRegistry.Password) -} diff --git a/api/http/handler/resourcecontrols/resourcecontrol_create.go b/api/http/handler/resourcecontrols/resourcecontrol_create.go index b3bd5741e..5eeb2e873 100644 --- a/api/http/handler/resourcecontrols/resourcecontrol_create.go +++ b/api/http/handler/resourcecontrols/resourcecontrol_create.go @@ -29,10 +29,7 @@ type resourceControlCreatePayload struct { SubResourceIDs []string `example:"617c5f22bb9b023d6daab7cba43a57576f83492867bc767d1c59416b065e5f08"` } -var ( - errResourceControlAlreadyExists = errors.New("A resource control is already applied on this resource") //http/resourceControl - errInvalidResourceControlType = errors.New("Unsupported resource control type") //http/resourceControl -) +var errResourceControlAlreadyExists = errors.New("A resource control is already applied on this resource") //http/resourceControl func (payload *resourceControlCreatePayload) Validate(r *http.Request) error { if govalidator.IsNull(payload.ResourceID) { diff --git a/api/http/handler/webhooks/handler.go b/api/http/handler/webhooks/handler.go index 45cacfc73..eb0d8c030 100644 --- a/api/http/handler/webhooks/handler.go +++ b/api/http/handler/webhooks/handler.go @@ -1,12 +1,11 @@ package webhooks import ( - "github.com/portainer/portainer/api/dataservices" - "github.com/portainer/portainer/api/internal/authorization" "net/http" + "github.com/portainer/portainer/api/dataservices" + httperror "github.com/portainer/libhttp/error" - portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/docker" "github.com/portainer/portainer/api/http/security" @@ -39,43 +38,3 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler { bouncer.PublicAccess(httperror.LoggerHandler(h.webhookExecute))).Methods(http.MethodPost) return h } - -func (handler *Handler) checkResourceAccess(r *http.Request, resourceID string, resourceControlType portainer.ResourceControlType) *httperror.HandlerError { - securityContext, err := security.RetrieveRestrictedRequestContext(r) - if err != nil { - return httperror.InternalServerError("Unable to retrieve user info from request context", err) - } - // non-admins - rc, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(resourceID, resourceControlType) - if rc == nil || err != nil { - return httperror.InternalServerError("Unable to retrieve a resource control associated to the resource", err) - } - userTeamIDs := make([]portainer.TeamID, 0) - for _, membership := range securityContext.UserMemberships { - userTeamIDs = append(userTeamIDs, membership.TeamID) - } - canAccess := authorization.UserCanAccessResource(securityContext.UserID, userTeamIDs, rc) - if !canAccess { - return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "This operation is disabled for non-admin users and unassigned access users"} - } - return nil -} - -func (handler *Handler) checkAuthorization(r *http.Request, endpoint *portainer.Endpoint, authorizations []portainer.Authorization) (bool, *httperror.HandlerError) { - err := handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint) - if err != nil { - return false, httperror.Forbidden("Permission denied to access environment", err) - } - - securityContext, err := security.RetrieveRestrictedRequestContext(r) - if err != nil { - return false, httperror.InternalServerError("Unable to retrieve user info from request context", err) - } - - authService := authorization.NewService(handler.DataStore) - isAdminOrAuthorized, err := authService.UserIsAdminOrAuthorized(securityContext.UserID, endpoint.ID, authorizations) - if err != nil { - return false, httperror.InternalServerError("Unable to get user authorizations", err) - } - return isAdminOrAuthorized, nil -} diff --git a/api/http/proxy/factory/agent/transport.go b/api/http/proxy/factory/agent/transport.go index 4250f861d..8abe497eb 100644 --- a/api/http/proxy/factory/agent/transport.go +++ b/api/http/proxy/factory/agent/transport.go @@ -6,14 +6,11 @@ import ( portainer "github.com/portainer/portainer/api" ) -type ( - // Transport is an http.Transport wrapper that adds custom http headers to communicate to an Agent - Transport struct { - httpTransport *http.Transport - signatureService portainer.DigitalSignatureService - endpointIdentifier portainer.EndpointID - } -) +// Transport is an http.Transport wrapper that adds custom http headers to communicate to an Agent +type Transport struct { + httpTransport *http.Transport + signatureService portainer.DigitalSignatureService +} // NewTransport returns a new transport that can be used to send signed requests to a Portainer agent func NewTransport(signatureService portainer.DigitalSignatureService, httpTransport *http.Transport) *Transport { From 669327da7c38114f841348c9e09f1ebfde86859a Mon Sep 17 00:00:00 2001 From: Prabhat Khera <91852476+prabhat-org@users.noreply.github.com> Date: Mon, 17 Oct 2022 10:44:17 +1300 Subject: [PATCH 05/31] fix reloading page when ing class disallowed (#7830) --- .../ingresses/CreateIngressView/CreateIngressView.tsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/kubernetes/react/views/networks/ingresses/CreateIngressView/CreateIngressView.tsx b/app/kubernetes/react/views/networks/ingresses/CreateIngressView/CreateIngressView.tsx index 7a1ad6ba0..a8d715936 100644 --- a/app/kubernetes/react/views/networks/ingresses/CreateIngressView/CreateIngressView.tsx +++ b/app/kubernetes/react/views/networks/ingresses/CreateIngressView/CreateIngressView.tsx @@ -175,7 +175,7 @@ export function CreateIngressView() { (!existingIngressClass || (existingIngressClass && !existingIngressClass.Availability)) && ingressRule.IngressClassName && - ingressControllersResults.data + !ingressControllersResults.isLoading ) { ingressClassOptions.push({ label: !ingressRule.IngressType @@ -206,7 +206,7 @@ export function CreateIngressView() { !!params.name && ingressesResults.data && !ingressRule.IngressName && - ingressControllersResults.data + !ingressControllersResults.isLoading ) { // if it is an edit screen, prepare the rule from the ingress const ing = ingressesResults.data?.find( @@ -221,6 +221,7 @@ export function CreateIngressView() { setIngressRule(r); } } + // eslint-disable-next-line react-hooks/exhaustive-deps }, [ params.name, ingressesResults.data, From 69f498c431ab09fbe6396a3740ed3d6f6595d3bc Mon Sep 17 00:00:00 2001 From: andres-portainer <91705312+andres-portainer@users.noreply.github.com> Date: Mon, 17 Oct 2022 13:57:41 -0300 Subject: [PATCH 06/31] fix(tests): add missing context cancel EE-4433 (#7879) --- api/git/git_integration_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/git/git_integration_test.go b/api/git/git_integration_test.go index 45504101e..886003c8d 100644 --- a/api/git/git_integration_test.go +++ b/api/git/git_integration_test.go @@ -274,7 +274,8 @@ func TestService_purgeCacheByTTL_Github(t *testing.T) { func TestService_canStopCacheCleanTimer_whenContextDone(t *testing.T) { timeout := 10 * time.Millisecond - deadlineCtx, _ := context.WithDeadline(context.TODO(), time.Now().Add(10*timeout)) + deadlineCtx, cancel := context.WithDeadline(context.TODO(), time.Now().Add(10*timeout)) + defer cancel() service := NewService(deadlineCtx) assert.False(t, service.timerHasStopped(), "timer should not be stopped") From 5488389278050461f3d29508c10da99c598ee6fb Mon Sep 17 00:00:00 2001 From: andres-portainer <91705312+andres-portainer@users.noreply.github.com> Date: Mon, 17 Oct 2022 15:29:12 -0300 Subject: [PATCH 07/31] fix(code): replace calls to ioutil EE-4425 (#7878) --- api/archive/targz_test.go | 17 ++++++++--------- api/archive/zip.go | 6 +++--- api/archive/zip_test.go | 3 ++- api/crypto/aes_test.go | 19 +++++++++---------- api/crypto/tls.go | 4 ++-- api/database/boltdb/db.go | 3 +-- api/datastore/services.go | 6 +++--- api/exec/compose_stack_test.go | 6 +++--- api/filesystem/copy_test.go | 9 ++++----- api/filesystem/write_test.go | 8 ++++---- api/git/azure.go | 3 +-- api/hostmanagement/openamt/authorization.go | 4 ++-- api/hostmanagement/openamt/openamt.go | 6 +++--- api/http/client/client.go | 4 ++-- api/http/handler/backup/backup_test.go | 5 ++--- .../edgegroups/associated_endpoints.go | 2 +- api/http/handler/helm/helm_install.go | 3 +-- .../handler/hostmanagement/openamt/amtrpc.go | 6 +++--- api/http/handler/stacks/stack_delete.go | 3 +-- .../handler/stacks/update_kubernetes_stack.go | 3 +-- api/http/handler/upload/handler.go | 2 +- api/http/handler/users/admin_check.go | 3 ++- api/http/handler/webhooks/webhook_delete.go | 3 ++- api/http/handler/webhooks/webhook_execute.go | 3 ++- api/http/handler/webhooks/webhook_list.go | 3 ++- api/http/handler/websocket/dial_windows.go | 3 ++- api/http/handler/websocket/initdial.go | 2 +- api/http/handler/websocket/proxy.go | 2 +- api/http/handler/websocket/types.go | 2 +- api/http/proxy/factory/docker/build.go | 8 ++++---- api/http/proxy/factory/docker/containers.go | 6 +++--- api/http/proxy/factory/docker/services.go | 6 +++--- api/http/proxy/factory/docker/transport.go | 4 ++-- api/http/proxy/factory/kubernetes/token.go | 4 ++-- .../proxy/factory/kubernetes/transport.go | 4 ++-- api/http/proxy/factory/utils/json.go | 3 +-- api/http/proxy/factory/utils/request.go | 4 ++-- api/http/proxy/factory/utils/response.go | 4 ++-- api/internal/edge/edgestack.go | 2 +- api/internal/edge/endpoint.go | 2 +- api/internal/tag/tag.go | 2 +- api/jwt/jwt_test.go | 3 ++- api/kubernetes/cli/nodes_limits_test.go | 7 ++++--- api/kubernetes/cli/service_account.go | 2 +- api/kubernetes/kubeclusteraccess_service.go | 4 ++-- .../kubeclusteraccess_service_test.go | 4 ++-- api/oauth/oauth.go | 4 ++-- .../deployment_kubernetes_config.go | 5 ++--- 48 files changed, 109 insertions(+), 112 deletions(-) diff --git a/api/archive/targz_test.go b/api/archive/targz_test.go index df17b8f4f..f3694f16f 100644 --- a/api/archive/targz_test.go +++ b/api/archive/targz_test.go @@ -2,7 +2,6 @@ package archive import ( "fmt" - "io/ioutil" "os" "os/exec" "path" @@ -28,10 +27,10 @@ func listFiles(dir string) []string { func Test_shouldCreateArhive(t *testing.T) { tmpdir := t.TempDir() content := []byte("content") - ioutil.WriteFile(path.Join(tmpdir, "outer"), content, 0600) + os.WriteFile(path.Join(tmpdir, "outer"), content, 0600) os.MkdirAll(path.Join(tmpdir, "dir"), 0700) - ioutil.WriteFile(path.Join(tmpdir, "dir", ".dotfile"), content, 0600) - ioutil.WriteFile(path.Join(tmpdir, "dir", "inner"), content, 0600) + os.WriteFile(path.Join(tmpdir, "dir", ".dotfile"), content, 0600) + os.WriteFile(path.Join(tmpdir, "dir", "inner"), content, 0600) gzPath, err := TarGzDir(tmpdir) assert.Nil(t, err) @@ -48,7 +47,7 @@ func Test_shouldCreateArhive(t *testing.T) { wasExtracted := func(p string) { fullpath := path.Join(extractionDir, p) assert.Contains(t, extractedFiles, fullpath) - copyContent, _ := ioutil.ReadFile(fullpath) + copyContent, _ := os.ReadFile(fullpath) assert.Equal(t, content, copyContent) } @@ -60,10 +59,10 @@ func Test_shouldCreateArhive(t *testing.T) { func Test_shouldCreateArhiveXXXXX(t *testing.T) { tmpdir := t.TempDir() content := []byte("content") - ioutil.WriteFile(path.Join(tmpdir, "outer"), content, 0600) + os.WriteFile(path.Join(tmpdir, "outer"), content, 0600) os.MkdirAll(path.Join(tmpdir, "dir"), 0700) - ioutil.WriteFile(path.Join(tmpdir, "dir", ".dotfile"), content, 0600) - ioutil.WriteFile(path.Join(tmpdir, "dir", "inner"), content, 0600) + os.WriteFile(path.Join(tmpdir, "dir", ".dotfile"), content, 0600) + os.WriteFile(path.Join(tmpdir, "dir", "inner"), content, 0600) gzPath, err := TarGzDir(tmpdir) assert.Nil(t, err) @@ -80,7 +79,7 @@ func Test_shouldCreateArhiveXXXXX(t *testing.T) { wasExtracted := func(p string) { fullpath := path.Join(extractionDir, p) assert.Contains(t, extractedFiles, fullpath) - copyContent, _ := ioutil.ReadFile(fullpath) + copyContent, _ := os.ReadFile(fullpath) assert.Equal(t, content, copyContent) } diff --git a/api/archive/zip.go b/api/archive/zip.go index 2aaee5a4f..50328ef09 100644 --- a/api/archive/zip.go +++ b/api/archive/zip.go @@ -4,12 +4,12 @@ import ( "archive/zip" "bytes" "fmt" - "github.com/pkg/errors" "io" - "io/ioutil" "os" "path/filepath" "strings" + + "github.com/pkg/errors" ) // UnzipArchive will unzip an archive from bytes into the dest destination folder on disk @@ -36,7 +36,7 @@ func extractFileFromArchive(file *zip.File, dest string) error { } defer f.Close() - data, err := ioutil.ReadAll(f) + data, err := io.ReadAll(f) if err != nil { return err } diff --git a/api/archive/zip_test.go b/api/archive/zip_test.go index 268109358..056bc1c0d 100644 --- a/api/archive/zip_test.go +++ b/api/archive/zip_test.go @@ -1,9 +1,10 @@ package archive import ( - "github.com/stretchr/testify/assert" "path/filepath" "testing" + + "github.com/stretchr/testify/assert" ) func TestUnzipFile(t *testing.T) { diff --git a/api/crypto/aes_test.go b/api/crypto/aes_test.go index b53d0e87d..09dea9c6d 100644 --- a/api/crypto/aes_test.go +++ b/api/crypto/aes_test.go @@ -2,7 +2,6 @@ package crypto import ( "io" - "io/ioutil" "os" "path/filepath" "testing" @@ -20,7 +19,7 @@ func Test_encryptAndDecrypt_withTheSamePassword(t *testing.T) { ) content := []byte("content") - ioutil.WriteFile(originFilePath, content, 0600) + os.WriteFile(originFilePath, content, 0600) originFile, _ := os.Open(originFilePath) defer originFile.Close() @@ -30,7 +29,7 @@ func Test_encryptAndDecrypt_withTheSamePassword(t *testing.T) { err := AesEncrypt(originFile, encryptedFileWriter, []byte("passphrase")) assert.Nil(t, err, "Failed to encrypt a file") - encryptedContent, err := ioutil.ReadFile(encryptedFilePath) + encryptedContent, err := os.ReadFile(encryptedFilePath) assert.Nil(t, err, "Couldn't read encrypted file") assert.NotEqual(t, encryptedContent, content, "Content wasn't encrypted") @@ -45,7 +44,7 @@ func Test_encryptAndDecrypt_withTheSamePassword(t *testing.T) { io.Copy(decryptedFileWriter, decryptedReader) - decryptedContent, _ := ioutil.ReadFile(decryptedFilePath) + decryptedContent, _ := os.ReadFile(decryptedFilePath) assert.Equal(t, content, decryptedContent, "Original and decrypted content should match") } @@ -59,7 +58,7 @@ func Test_encryptAndDecrypt_withEmptyPassword(t *testing.T) { ) content := []byte("content") - ioutil.WriteFile(originFilePath, content, 0600) + os.WriteFile(originFilePath, content, 0600) originFile, _ := os.Open(originFilePath) defer originFile.Close() @@ -69,7 +68,7 @@ func Test_encryptAndDecrypt_withEmptyPassword(t *testing.T) { err := AesEncrypt(originFile, encryptedFileWriter, []byte("")) assert.Nil(t, err, "Failed to encrypt a file") - encryptedContent, err := ioutil.ReadFile(encryptedFilePath) + encryptedContent, err := os.ReadFile(encryptedFilePath) assert.Nil(t, err, "Couldn't read encrypted file") assert.NotEqual(t, encryptedContent, content, "Content wasn't encrypted") @@ -84,7 +83,7 @@ func Test_encryptAndDecrypt_withEmptyPassword(t *testing.T) { io.Copy(decryptedFileWriter, decryptedReader) - decryptedContent, _ := ioutil.ReadFile(decryptedFilePath) + decryptedContent, _ := os.ReadFile(decryptedFilePath) assert.Equal(t, content, decryptedContent, "Original and decrypted content should match") } @@ -98,7 +97,7 @@ func Test_decryptWithDifferentPassphrase_shouldProduceWrongResult(t *testing.T) ) content := []byte("content") - ioutil.WriteFile(originFilePath, content, 0600) + os.WriteFile(originFilePath, content, 0600) originFile, _ := os.Open(originFilePath) defer originFile.Close() @@ -108,7 +107,7 @@ func Test_decryptWithDifferentPassphrase_shouldProduceWrongResult(t *testing.T) err := AesEncrypt(originFile, encryptedFileWriter, []byte("passphrase")) assert.Nil(t, err, "Failed to encrypt a file") - encryptedContent, err := ioutil.ReadFile(encryptedFilePath) + encryptedContent, err := os.ReadFile(encryptedFilePath) assert.Nil(t, err, "Couldn't read encrypted file") assert.NotEqual(t, encryptedContent, content, "Content wasn't encrypted") @@ -123,6 +122,6 @@ func Test_decryptWithDifferentPassphrase_shouldProduceWrongResult(t *testing.T) io.Copy(decryptedFileWriter, decryptedReader) - decryptedContent, _ := ioutil.ReadFile(decryptedFilePath) + decryptedContent, _ := os.ReadFile(decryptedFilePath) assert.NotEqual(t, content, decryptedContent, "Original and decrypted content should NOT match") } diff --git a/api/crypto/tls.go b/api/crypto/tls.go index e46998898..e3418f66e 100644 --- a/api/crypto/tls.go +++ b/api/crypto/tls.go @@ -3,7 +3,7 @@ package crypto import ( "crypto/tls" "crypto/x509" - "io/ioutil" + "os" ) // CreateServerTLSConfiguration creates a basic tls.Config to be used by servers with recommended TLS settings @@ -63,7 +63,7 @@ func CreateTLSConfigurationFromDisk(caCertPath, certPath, keyPath string, skipSe } if !skipServerVerification && caCertPath != "" { - caCert, err := ioutil.ReadFile(caCertPath) + caCert, err := os.ReadFile(caCertPath) if err != nil { return nil, err } diff --git a/api/database/boltdb/db.go b/api/database/boltdb/db.go index a829cedd8..c7fa80bd5 100644 --- a/api/database/boltdb/db.go +++ b/api/database/boltdb/db.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "os" "path" "time" @@ -167,7 +166,7 @@ func (connection *DbConnection) ExportRaw(filename string) error { if err != nil { return err } - return ioutil.WriteFile(filename, b, 0600) + return os.WriteFile(filename, b, 0600) } // ConvertToKey returns an 8-byte big endian representation of v. diff --git a/api/datastore/services.go b/api/datastore/services.go index c4bd90003..eb05b3329 100644 --- a/api/datastore/services.go +++ b/api/datastore/services.go @@ -3,7 +3,7 @@ package datastore import ( "encoding/json" "fmt" - "io/ioutil" + "os" "strconv" portainer "github.com/portainer/portainer/api" @@ -607,13 +607,13 @@ func (store *Store) Export(filename string) (err error) { if err != nil { return err } - return ioutil.WriteFile(filename, b, 0600) + return os.WriteFile(filename, b, 0600) } func (store *Store) Import(filename string) (err error) { backup := storeExport{} - s, err := ioutil.ReadFile(filename) + s, err := os.ReadFile(filename) if err != nil { return err } diff --git a/api/exec/compose_stack_test.go b/api/exec/compose_stack_test.go index 733eb7fb2..b1904438a 100644 --- a/api/exec/compose_stack_test.go +++ b/api/exec/compose_stack_test.go @@ -1,7 +1,7 @@ package exec import ( - "io/ioutil" + "io" "os" "path" "testing" @@ -55,7 +55,7 @@ func Test_createEnvFile(t *testing.T) { assert.Equal(t, "stack.env", result) f, _ := os.Open(path.Join(dir, "stack.env")) - content, _ := ioutil.ReadAll(f) + content, _ := io.ReadAll(f) assert.Equal(t, tt.expected, string(content)) } else { @@ -80,7 +80,7 @@ func Test_createEnvFile_mergesDefultAndInplaceEnvVars(t *testing.T) { assert.NoError(t, err) assert.FileExists(t, path.Join(dir, "stack.env")) f, _ := os.Open(path.Join(dir, "stack.env")) - content, _ := ioutil.ReadAll(f) + content, _ := io.ReadAll(f) assert.Equal(t, []byte("VAR1=VAL1\nVAR2=VAL2\n\nVAR1=NEW_VAL1\nVAR3=VAL3\n"), content) } diff --git a/api/filesystem/copy_test.go b/api/filesystem/copy_test.go index a451fdb8f..a91754975 100644 --- a/api/filesystem/copy_test.go +++ b/api/filesystem/copy_test.go @@ -1,7 +1,6 @@ package filesystem import ( - "io/ioutil" "os" "path" "path/filepath" @@ -19,12 +18,12 @@ func Test_copyFile_returnsError_whenSourceDoesNotExist(t *testing.T) { func Test_copyFile_shouldMakeAbackup(t *testing.T) { tmpdir := t.TempDir() content := []byte("content") - ioutil.WriteFile(path.Join(tmpdir, "origin"), content, 0600) + os.WriteFile(path.Join(tmpdir, "origin"), content, 0600) err := copyFile(path.Join(tmpdir, "origin"), path.Join(tmpdir, "copy")) assert.NoError(t, err) - copyContent, _ := ioutil.ReadFile(path.Join(tmpdir, "copy")) + copyContent, _ := os.ReadFile(path.Join(tmpdir, "copy")) assert.Equal(t, content, copyContent) } @@ -59,13 +58,13 @@ func Test_CopyPath_shouldSkipWhenNotExist(t *testing.T) { func Test_CopyPath_shouldCopyFile(t *testing.T) { tmpdir := t.TempDir() content := []byte("content") - ioutil.WriteFile(path.Join(tmpdir, "file"), content, 0600) + os.WriteFile(path.Join(tmpdir, "file"), content, 0600) os.MkdirAll(path.Join(tmpdir, "backup"), 0700) err := CopyPath(path.Join(tmpdir, "file"), path.Join(tmpdir, "backup")) assert.NoError(t, err) - copyContent, err := ioutil.ReadFile(path.Join(tmpdir, "backup", "file")) + copyContent, err := os.ReadFile(path.Join(tmpdir, "backup", "file")) assert.NoError(t, err) assert.Equal(t, content, copyContent) } diff --git a/api/filesystem/write_test.go b/api/filesystem/write_test.go index 89223a20e..17606ace0 100644 --- a/api/filesystem/write_test.go +++ b/api/filesystem/write_test.go @@ -1,7 +1,7 @@ package filesystem import ( - "io/ioutil" + "os" "path" "testing" @@ -16,7 +16,7 @@ func Test_WriteFile_CanStoreContentInANewFile(t *testing.T) { err := WriteToFile(tmpFilePath, content) assert.NoError(t, err) - fileContent, _ := ioutil.ReadFile(tmpFilePath) + fileContent, _ := os.ReadFile(tmpFilePath) assert.Equal(t, content, fileContent) } @@ -31,7 +31,7 @@ func Test_WriteFile_CanOverwriteExistingFile(t *testing.T) { err = WriteToFile(tmpFilePath, content) assert.NoError(t, err) - fileContent, _ := ioutil.ReadFile(tmpFilePath) + fileContent, _ := os.ReadFile(tmpFilePath) assert.Equal(t, content, fileContent) } @@ -43,6 +43,6 @@ func Test_WriteFile_CanWriteANestedPath(t *testing.T) { err := WriteToFile(tmpFilePath, content) assert.NoError(t, err) - fileContent, _ := ioutil.ReadFile(tmpFilePath) + fileContent, _ := os.ReadFile(tmpFilePath) assert.Equal(t, content, fileContent) } diff --git a/api/git/azure.go b/api/git/azure.go index 7aa7b5097..6036fe5ac 100644 --- a/api/git/azure.go +++ b/api/git/azure.go @@ -6,7 +6,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "net/http" "net/url" "os" @@ -100,7 +99,7 @@ func (a *azureClient) downloadZipFromAzureDevOps(ctx context.Context, opt cloneO if err != nil { return "", errors.WithMessage(err, "failed to build download url") } - zipFile, err := ioutil.TempFile("", "azure-git-repo-*.zip") + zipFile, err := os.CreateTemp("", "azure-git-repo-*.zip") if err != nil { return "", errors.WithMessage(err, "failed to create temp file") } diff --git a/api/hostmanagement/openamt/authorization.go b/api/hostmanagement/openamt/authorization.go index 5e9d0773f..9d5050d2e 100644 --- a/api/hostmanagement/openamt/authorization.go +++ b/api/hostmanagement/openamt/authorization.go @@ -4,7 +4,7 @@ import ( "bytes" "encoding/json" "fmt" - "io/ioutil" + "io" "net/http" portainer "github.com/portainer/portainer/api" @@ -33,7 +33,7 @@ func (service *Service) Authorization(configuration portainer.OpenAMTConfigurati if err != nil { return "", err } - responseBody, readErr := ioutil.ReadAll(response.Body) + responseBody, readErr := io.ReadAll(response.Body) if readErr != nil { return "", readErr } diff --git a/api/hostmanagement/openamt/openamt.go b/api/hostmanagement/openamt/openamt.go index 863410d14..8c9902866 100644 --- a/api/hostmanagement/openamt/openamt.go +++ b/api/hostmanagement/openamt/openamt.go @@ -6,7 +6,7 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" + "io" "net/http" "time" @@ -99,7 +99,7 @@ func (service *Service) executeSaveRequest(method string, url string, token stri if err != nil { return nil, err } - responseBody, readErr := ioutil.ReadAll(response.Body) + responseBody, readErr := io.ReadAll(response.Body) if readErr != nil { return nil, readErr } @@ -128,7 +128,7 @@ func (service *Service) executeGetRequest(url string, token string) ([]byte, err if err != nil { return nil, err } - responseBody, readErr := ioutil.ReadAll(response.Body) + responseBody, readErr := io.ReadAll(response.Body) if readErr != nil { return nil, readErr } diff --git a/api/http/client/client.go b/api/http/client/client.go index dcd070375..343e8fa2a 100644 --- a/api/http/client/client.go +++ b/api/http/client/client.go @@ -5,7 +5,7 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" + "io" "net/http" "net/url" "strings" @@ -96,7 +96,7 @@ func Get(url string, timeout int) ([]byte, error) { return nil, errInvalidResponseStatus } - body, err := ioutil.ReadAll(response.Body) + body, err := io.ReadAll(response.Body) if err != nil { return nil, err } diff --git a/api/http/handler/backup/backup_test.go b/api/http/handler/backup/backup_test.go index 8c2fb979f..a35754db3 100644 --- a/api/http/handler/backup/backup_test.go +++ b/api/http/handler/backup/backup_test.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "io" - "io/ioutil" "net/http" "net/http/httptest" "os" @@ -38,7 +37,7 @@ func listFiles(dir string) []string { func contains(t *testing.T, list []string, path string) { assert.Contains(t, list, path) - copyContent, _ := ioutil.ReadFile(path) + copyContent, _ := os.ReadFile(path) assert.Equal(t, "content\n", string(copyContent)) } @@ -58,7 +57,7 @@ func Test_backupHandlerWithoutPassword_shouldCreateATarballArchive(t *testing.T) tmpdir := t.TempDir() archivePath := filepath.Join(tmpdir, "archive.tar.gz") - err := ioutil.WriteFile(archivePath, body, 0600) + err := os.WriteFile(archivePath, body, 0600) if err != nil { t.Fatal("Failed to save downloaded .tar.gz archive: ", err) } diff --git a/api/http/handler/edgegroups/associated_endpoints.go b/api/http/handler/edgegroups/associated_endpoints.go index b34711eda..6c240083c 100644 --- a/api/http/handler/edgegroups/associated_endpoints.go +++ b/api/http/handler/edgegroups/associated_endpoints.go @@ -1,7 +1,7 @@ package edgegroups import ( - "github.com/portainer/portainer/api" + portainer "github.com/portainer/portainer/api" ) type endpointSetType map[portainer.EndpointID]bool diff --git a/api/http/handler/helm/helm_install.go b/api/http/handler/helm/helm_install.go index 248e0659c..33f8f45da 100644 --- a/api/http/handler/helm/helm_install.go +++ b/api/http/handler/helm/helm_install.go @@ -2,7 +2,6 @@ package helm import ( "fmt" - "io/ioutil" "net/http" "os" "strings" @@ -192,7 +191,7 @@ func (handler *Handler) updateHelmAppManifest(r *http.Request, manifest []byte, for _, resource := range yamlResources { resource := resource // https://golang.org/doc/faq#closures_and_goroutines g.Go(func() error { - tmpfile, err := ioutil.TempFile("", "helm-manifest-*") + tmpfile, err := os.CreateTemp("", "helm-manifest-*") if err != nil { return errors.Wrap(err, "failed to create a tmp helm manifest file") } diff --git a/api/http/handler/hostmanagement/openamt/amtrpc.go b/api/http/handler/hostmanagement/openamt/amtrpc.go index fe0c9e162..a0c2ccbf2 100644 --- a/api/http/handler/hostmanagement/openamt/amtrpc.go +++ b/api/http/handler/hostmanagement/openamt/amtrpc.go @@ -4,7 +4,7 @@ import ( "context" "encoding/json" "fmt" - "io/ioutil" + "io" "net/http" "time" @@ -139,7 +139,7 @@ func pullImage(ctx context.Context, docker *client.Client, imageName string) err } defer out.Close() - outputBytes, err := ioutil.ReadAll(out) + outputBytes, err := io.ReadAll(out) if err != nil { log.Error().Str("image_name", imageName).Err(err).Msg("could not read image pull output") @@ -261,7 +261,7 @@ func runContainer(ctx context.Context, docker *client.Client, imageName, contain return "", err } - outputBytes, err := ioutil.ReadAll(out) + outputBytes, err := io.ReadAll(out) if err != nil { log.Error(). Str("image_name", imageName). diff --git a/api/http/handler/stacks/stack_delete.go b/api/http/handler/stacks/stack_delete.go index ca447c903..d4d353978 100644 --- a/api/http/handler/stacks/stack_delete.go +++ b/api/http/handler/stacks/stack_delete.go @@ -3,7 +3,6 @@ package stacks import ( "context" "fmt" - "io/ioutil" "net/http" "os" "strconv" @@ -200,7 +199,7 @@ func (handler *Handler) deleteStack(userID portainer.UserID, stack *portainer.St //then process the remove operation if stack.IsComposeFormat { fileNames := stackutils.GetStackFilePaths(stack, false) - tmpDir, err := ioutil.TempDir("", "kube_delete") + tmpDir, err := os.MkdirTemp("", "kube_delete") if err != nil { return errors.Wrap(err, "failed to create temp directory for deleting kub stack") } diff --git a/api/http/handler/stacks/update_kubernetes_stack.go b/api/http/handler/stacks/update_kubernetes_stack.go index 0764c58d7..ed939addb 100644 --- a/api/http/handler/stacks/update_kubernetes_stack.go +++ b/api/http/handler/stacks/update_kubernetes_stack.go @@ -2,7 +2,6 @@ package stacks import ( "fmt" - "io/ioutil" "net/http" "os" "strconv" @@ -105,7 +104,7 @@ func (handler *Handler) updateKubernetesStack(r *http.Request, stack *portainer. return httperror.BadRequest("Failed to retrieve user token data", err) } - tempFileDir, _ := ioutil.TempDir("", "kub_file_content") + tempFileDir, _ := os.MkdirTemp("", "kub_file_content") defer os.RemoveAll(tempFileDir) if err := filesystem.WriteToFile(filesystem.JoinPaths(tempFileDir, stack.EntryPoint), []byte(payload.StackFileContent)); err != nil { diff --git a/api/http/handler/upload/handler.go b/api/http/handler/upload/handler.go index dd6a459a1..9c8cca2a4 100644 --- a/api/http/handler/upload/handler.go +++ b/api/http/handler/upload/handler.go @@ -2,7 +2,7 @@ package upload import ( httperror "github.com/portainer/libhttp/error" - "github.com/portainer/portainer/api" + portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/http/security" "net/http" diff --git a/api/http/handler/users/admin_check.go b/api/http/handler/users/admin_check.go index 1925981ee..5fbec62a5 100644 --- a/api/http/handler/users/admin_check.go +++ b/api/http/handler/users/admin_check.go @@ -1,9 +1,10 @@ package users import ( - "github.com/portainer/portainer/api/dataservices/errors" "net/http" + "github.com/portainer/portainer/api/dataservices/errors" + httperror "github.com/portainer/libhttp/error" "github.com/portainer/libhttp/response" portainer "github.com/portainer/portainer/api" diff --git a/api/http/handler/webhooks/webhook_delete.go b/api/http/handler/webhooks/webhook_delete.go index 0b7f64747..3596130f9 100644 --- a/api/http/handler/webhooks/webhook_delete.go +++ b/api/http/handler/webhooks/webhook_delete.go @@ -2,9 +2,10 @@ package webhooks import ( "errors" - "github.com/portainer/portainer/api/http/security" "net/http" + "github.com/portainer/portainer/api/http/security" + httperror "github.com/portainer/libhttp/error" "github.com/portainer/libhttp/request" "github.com/portainer/libhttp/response" diff --git a/api/http/handler/webhooks/webhook_execute.go b/api/http/handler/webhooks/webhook_execute.go index 0168e231d..511448270 100644 --- a/api/http/handler/webhooks/webhook_execute.go +++ b/api/http/handler/webhooks/webhook_execute.go @@ -3,11 +3,12 @@ package webhooks import ( "context" "errors" - "github.com/portainer/portainer/api/internal/registryutils" "io" "net/http" "strings" + "github.com/portainer/portainer/api/internal/registryutils" + dockertypes "github.com/docker/docker/api/types" httperror "github.com/portainer/libhttp/error" "github.com/portainer/libhttp/request" diff --git a/api/http/handler/webhooks/webhook_list.go b/api/http/handler/webhooks/webhook_list.go index 966611010..c2097a47d 100644 --- a/api/http/handler/webhooks/webhook_list.go +++ b/api/http/handler/webhooks/webhook_list.go @@ -1,9 +1,10 @@ package webhooks import ( - "github.com/portainer/portainer/api/http/security" "net/http" + "github.com/portainer/portainer/api/http/security" + httperror "github.com/portainer/libhttp/error" "github.com/portainer/libhttp/request" "github.com/portainer/libhttp/response" diff --git a/api/http/handler/websocket/dial_windows.go b/api/http/handler/websocket/dial_windows.go index e6d725a40..8f181e956 100644 --- a/api/http/handler/websocket/dial_windows.go +++ b/api/http/handler/websocket/dial_windows.go @@ -4,8 +4,9 @@ package websocket import ( - "github.com/Microsoft/go-winio" "net" + + "github.com/Microsoft/go-winio" ) func createDial(scheme, host string) (net.Conn, error) { diff --git a/api/http/handler/websocket/initdial.go b/api/http/handler/websocket/initdial.go index 327753d28..e9160ccf1 100644 --- a/api/http/handler/websocket/initdial.go +++ b/api/http/handler/websocket/initdial.go @@ -5,7 +5,7 @@ import ( "net" "net/url" - "github.com/portainer/portainer/api" + portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/crypto" ) diff --git a/api/http/handler/websocket/proxy.go b/api/http/handler/websocket/proxy.go index 647399760..d1325cc03 100644 --- a/api/http/handler/websocket/proxy.go +++ b/api/http/handler/websocket/proxy.go @@ -8,7 +8,7 @@ import ( "github.com/gorilla/websocket" "github.com/koding/websocketproxy" - "github.com/portainer/portainer/api" + portainer "github.com/portainer/portainer/api" ) func (handler *Handler) proxyEdgeAgentWebsocketRequest(w http.ResponseWriter, r *http.Request, params *webSocketRequestParams) error { diff --git a/api/http/handler/websocket/types.go b/api/http/handler/websocket/types.go index 614143aa0..97b3334f6 100644 --- a/api/http/handler/websocket/types.go +++ b/api/http/handler/websocket/types.go @@ -1,6 +1,6 @@ package websocket -import "github.com/portainer/portainer/api" +import portainer "github.com/portainer/portainer/api" type webSocketRequestParams struct { ID string diff --git a/api/http/proxy/factory/docker/build.go b/api/http/proxy/factory/docker/build.go index 6b5dfa91f..cbe2d0e99 100644 --- a/api/http/proxy/factory/docker/build.go +++ b/api/http/proxy/factory/docker/build.go @@ -4,7 +4,7 @@ import ( "bytes" "encoding/json" "errors" - "io/ioutil" + "io" "mime" "net/http" @@ -37,7 +37,7 @@ func buildOperation(request *http.Request) error { var buffer []byte switch mediaType { case "": - body, err := ioutil.ReadAll(request.Body) + body, err := io.ReadAll(request.Body) if err != nil { return err } @@ -81,7 +81,7 @@ func buildOperation(request *http.Request) error { log.Info().Str("filename", hdr.Filename).Int64("size", hdr.Size).Msg("upload the file to build image") - content, err := ioutil.ReadAll(f) + content, err := io.ReadAll(f) if err != nil { return err } @@ -105,7 +105,7 @@ func buildOperation(request *http.Request) error { return nil } - request.Body = ioutil.NopCloser(bytes.NewReader(buffer)) + request.Body = io.NopCloser(bytes.NewReader(buffer)) request.ContentLength = int64(len(buffer)) request.Header.Set("Content-Type", "application/x-tar") diff --git a/api/http/proxy/factory/docker/containers.go b/api/http/proxy/factory/docker/containers.go index 573bfc21d..c137e2606 100644 --- a/api/http/proxy/factory/docker/containers.go +++ b/api/http/proxy/factory/docker/containers.go @@ -5,7 +5,7 @@ import ( "context" "encoding/json" "errors" - "io/ioutil" + "io" "net/http" "strings" @@ -190,7 +190,7 @@ func (transport *Transport) decorateContainerCreationOperation(request *http.Req return nil, err } - body, err := ioutil.ReadAll(request.Body) + body, err := io.ReadAll(request.Body) if err != nil { return nil, err } @@ -229,7 +229,7 @@ func (transport *Transport) decorateContainerCreationOperation(request *http.Req } } - request.Body = ioutil.NopCloser(bytes.NewBuffer(body)) + request.Body = io.NopCloser(bytes.NewBuffer(body)) } response, err := transport.executeDockerRequest(request) diff --git a/api/http/proxy/factory/docker/services.go b/api/http/proxy/factory/docker/services.go index 205c48c60..4df1d316a 100644 --- a/api/http/proxy/factory/docker/services.go +++ b/api/http/proxy/factory/docker/services.go @@ -5,7 +5,7 @@ import ( "context" "encoding/json" "errors" - "io/ioutil" + "io" "net/http" "github.com/docker/docker/api/types" @@ -116,7 +116,7 @@ func (transport *Transport) decorateServiceCreationOperation(request *http.Reque return nil, err } - body, err := ioutil.ReadAll(request.Body) + body, err := io.ReadAll(request.Body) if err != nil { return nil, err } @@ -135,7 +135,7 @@ func (transport *Transport) decorateServiceCreationOperation(request *http.Reque } } - request.Body = ioutil.NopCloser(bytes.NewBuffer(body)) + request.Body = io.NopCloser(bytes.NewBuffer(body)) } return transport.replaceRegistryAuthenticationHeader(request) diff --git a/api/http/proxy/factory/docker/transport.go b/api/http/proxy/factory/docker/transport.go index 439988a96..86a252dae 100644 --- a/api/http/proxy/factory/docker/transport.go +++ b/api/http/proxy/factory/docker/transport.go @@ -6,7 +6,7 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" + "io" "net/http" "path" "regexp" @@ -197,7 +197,7 @@ func (transport *Transport) proxyAgentRequest(r *http.Request) (*http.Response, r.Method = http.MethodPost - r.Body = ioutil.NopCloser(bytes.NewReader(newBody)) + r.Body = io.NopCloser(bytes.NewReader(newBody)) r.ContentLength = int64(len(newBody)) } diff --git a/api/http/proxy/factory/kubernetes/token.go b/api/http/proxy/factory/kubernetes/token.go index 997898ea1..f2d620630 100644 --- a/api/http/proxy/factory/kubernetes/token.go +++ b/api/http/proxy/factory/kubernetes/token.go @@ -1,7 +1,7 @@ package kubernetes import ( - "io/ioutil" + "os" portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/dataservices" @@ -28,7 +28,7 @@ func NewTokenManager(kubecli portainer.KubeClient, dataStore dataservices.DataSt } if setLocalAdminToken { - token, err := ioutil.ReadFile(defaultServiceAccountTokenFile) + token, err := os.ReadFile(defaultServiceAccountTokenFile) if err != nil { return nil, err } diff --git a/api/http/proxy/factory/kubernetes/transport.go b/api/http/proxy/factory/kubernetes/transport.go index 86b793eed..1f05092f8 100644 --- a/api/http/proxy/factory/kubernetes/transport.go +++ b/api/http/proxy/factory/kubernetes/transport.go @@ -4,7 +4,7 @@ import ( "bytes" "encoding/json" "fmt" - "io/ioutil" + "io" "net/http" "path" "regexp" @@ -189,7 +189,7 @@ func decorateAgentDockerHubRequest(r *http.Request, dataStore dataservices.DataS } r.Method = http.MethodPost - r.Body = ioutil.NopCloser(bytes.NewReader(newBody)) + r.Body = io.NopCloser(bytes.NewReader(newBody)) r.ContentLength = int64(len(newBody)) return nil diff --git a/api/http/proxy/factory/utils/json.go b/api/http/proxy/factory/utils/json.go index cab676834..7d12567e0 100644 --- a/api/http/proxy/factory/utils/json.go +++ b/api/http/proxy/factory/utils/json.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "mime" "gopkg.in/yaml.v3" @@ -50,7 +49,7 @@ func getBody(body io.ReadCloser, contentType string, isGzip bool) (interface{}, defer reader.Close() - bodyBytes, err := ioutil.ReadAll(reader) + bodyBytes, err := io.ReadAll(reader) if err != nil { return nil, err } diff --git a/api/http/proxy/factory/utils/request.go b/api/http/proxy/factory/utils/request.go index 92724b7fa..9f809fc20 100644 --- a/api/http/proxy/factory/utils/request.go +++ b/api/http/proxy/factory/utils/request.go @@ -2,7 +2,7 @@ package utils import ( "bytes" - "io/ioutil" + "io" "net/http" "strconv" ) @@ -25,7 +25,7 @@ func RewriteRequest(request *http.Request, newData interface{}) error { return err } - body := ioutil.NopCloser(bytes.NewReader(data)) + body := io.NopCloser(bytes.NewReader(data)) request.Body = body request.ContentLength = int64(len(data)) diff --git a/api/http/proxy/factory/utils/response.go b/api/http/proxy/factory/utils/response.go index 674529883..868590295 100644 --- a/api/http/proxy/factory/utils/response.go +++ b/api/http/proxy/factory/utils/response.go @@ -3,7 +3,7 @@ package utils import ( "bytes" "fmt" - "io/ioutil" + "io" "net/http" "strconv" @@ -82,7 +82,7 @@ func RewriteResponse(response *http.Response, newResponseData interface{}, statu return err } - body := ioutil.NopCloser(bytes.NewReader(data)) + body := io.NopCloser(bytes.NewReader(data)) response.StatusCode = statusCode response.Body = body diff --git a/api/internal/edge/edgestack.go b/api/internal/edge/edgestack.go index 9a0837bc2..5e2aa520b 100644 --- a/api/internal/edge/edgestack.go +++ b/api/internal/edge/edgestack.go @@ -3,7 +3,7 @@ package edge import ( "errors" - "github.com/portainer/portainer/api" + portainer "github.com/portainer/portainer/api" ) // EdgeStackRelatedEndpoints returns a list of environments(endpoints) related to this Edge stack diff --git a/api/internal/edge/endpoint.go b/api/internal/edge/endpoint.go index 492c803a7..843561107 100644 --- a/api/internal/edge/endpoint.go +++ b/api/internal/edge/endpoint.go @@ -1,6 +1,6 @@ package edge -import "github.com/portainer/portainer/api" +import portainer "github.com/portainer/portainer/api" // EndpointRelatedEdgeStacks returns a list of Edge stacks related to this Environment(Endpoint) func EndpointRelatedEdgeStacks(endpoint *portainer.Endpoint, endpointGroup *portainer.EndpointGroup, edgeGroups []portainer.EdgeGroup, edgeStacks []portainer.EdgeStack) []portainer.EdgeStackID { diff --git a/api/internal/tag/tag.go b/api/internal/tag/tag.go index 8dd5ad58b..ada130c88 100644 --- a/api/internal/tag/tag.go +++ b/api/internal/tag/tag.go @@ -1,6 +1,6 @@ package tag -import "github.com/portainer/portainer/api" +import portainer "github.com/portainer/portainer/api" type tagSet map[portainer.TagID]bool diff --git a/api/jwt/jwt_test.go b/api/jwt/jwt_test.go index 5a6a5c988..82e559aed 100644 --- a/api/jwt/jwt_test.go +++ b/api/jwt/jwt_test.go @@ -1,10 +1,11 @@ package jwt import ( - i "github.com/portainer/portainer/api/internal/testhelpers" "testing" "time" + i "github.com/portainer/portainer/api/internal/testhelpers" + "github.com/golang-jwt/jwt" portainer "github.com/portainer/portainer/api" "github.com/stretchr/testify/assert" diff --git a/api/kubernetes/cli/nodes_limits_test.go b/api/kubernetes/cli/nodes_limits_test.go index bf880c2ff..25ebe56a4 100644 --- a/api/kubernetes/cli/nodes_limits_test.go +++ b/api/kubernetes/cli/nodes_limits_test.go @@ -1,14 +1,15 @@ package cli import ( + "reflect" + "testing" + portainer "github.com/portainer/portainer/api" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" kfake "k8s.io/client-go/kubernetes/fake" - "reflect" - "testing" ) func newNodes() *v1.NodeList { diff --git a/api/kubernetes/cli/service_account.go b/api/kubernetes/cli/service_account.go index f9c421ec6..a50620b4d 100644 --- a/api/kubernetes/cli/service_account.go +++ b/api/kubernetes/cli/service_account.go @@ -37,7 +37,7 @@ func (kcl *KubeClient) GetServiceAccountBearerToken(userID int) (string, error) // SetupUserServiceAccount will make sure that all the required resources are created inside the Kubernetes // cluster before creating a ServiceAccount and a ServiceAccountToken for the specified Portainer user. -//It will also create required default RoleBinding and ClusterRoleBinding rules. +// It will also create required default RoleBinding and ClusterRoleBinding rules. func (kcl *KubeClient) SetupUserServiceAccount(userID int, teamIDs []int, restrictDefaultNamespace bool) error { serviceAccountName := UserServiceAccountName(userID, kcl.instanceID) diff --git a/api/kubernetes/kubeclusteraccess_service.go b/api/kubernetes/kubeclusteraccess_service.go index 0b1e785d5..d7da7bd2e 100644 --- a/api/kubernetes/kubeclusteraccess_service.go +++ b/api/kubernetes/kubeclusteraccess_service.go @@ -5,7 +5,7 @@ import ( "encoding/base64" "encoding/pem" "fmt" - "io/ioutil" + "os" "strings" portainer "github.com/portainer/portainer/api" @@ -63,7 +63,7 @@ func getCertificateAuthorityData(tlsCertPath string) (string, error) { return "", errTLSCertNotProvided } - data, err := ioutil.ReadFile(tlsCertPath) + data, err := os.ReadFile(tlsCertPath) if err != nil { return "", errors.Wrap(errTLSCertFileMissing, err.Error()) } diff --git a/api/kubernetes/kubeclusteraccess_service_test.go b/api/kubernetes/kubeclusteraccess_service_test.go index 51f421668..71ce50f30 100644 --- a/api/kubernetes/kubeclusteraccess_service_test.go +++ b/api/kubernetes/kubeclusteraccess_service_test.go @@ -2,7 +2,7 @@ package kubernetes import ( "fmt" - "io/ioutil" + "os" "strings" "testing" @@ -38,7 +38,7 @@ const certDataString = "MIIC5TCCAc2gAwIBAgIJAJ+poiEBdsplMA0GCSqGSIb3DQEBCwUAMBQx func createTempFile(filename, content string, t *testing.T) string { tempPath := t.TempDir() filePath := fmt.Sprintf("%s/%s", tempPath, filename) - ioutil.WriteFile(filePath, []byte(content), 0644) + os.WriteFile(filePath, []byte(content), 0644) return filePath } diff --git a/api/oauth/oauth.go b/api/oauth/oauth.go index 3088ca821..59b960d45 100644 --- a/api/oauth/oauth.go +++ b/api/oauth/oauth.go @@ -3,7 +3,7 @@ package oauth import ( "context" "encoding/json" - "io/ioutil" + "io" "mime" "net/http" "net/url" @@ -127,7 +127,7 @@ func getResource(token string, configuration *portainer.OAuthSettings) (map[stri } defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { return nil, err } diff --git a/api/stacks/deployments/deployment_kubernetes_config.go b/api/stacks/deployments/deployment_kubernetes_config.go index faf974a8c..b94d6c4b4 100644 --- a/api/stacks/deployments/deployment_kubernetes_config.go +++ b/api/stacks/deployments/deployment_kubernetes_config.go @@ -2,7 +2,6 @@ package deployments import ( "fmt" - "io/ioutil" "os" "github.com/pkg/errors" @@ -41,7 +40,7 @@ func (config *KubernetesStackDeploymentConfig) Deploy() error { manifestFilePaths := make([]string, len(fileNames)) - tmpDir, err := ioutil.TempDir("", "kub_deployment") + tmpDir, err := os.MkdirTemp("", "kub_deployment") if err != nil { return errors.Wrap(err, "failed to create temp kub deployment directory") } @@ -50,7 +49,7 @@ func (config *KubernetesStackDeploymentConfig) Deploy() error { for _, fileName := range fileNames { manifestFilePath := filesystem.JoinPaths(tmpDir, fileName) - manifestContent, err := ioutil.ReadFile(filesystem.JoinPaths(config.stack.ProjectPath, fileName)) + manifestContent, err := os.ReadFile(filesystem.JoinPaths(config.stack.ProjectPath, fileName)) if err != nil { return errors.Wrap(err, "failed to read manifest file") } From f6d6be90e4232c924cf2668a64e59764d4fe6115 Mon Sep 17 00:00:00 2001 From: itsconquest Date: Tue, 18 Oct 2022 09:45:47 +1300 Subject: [PATCH 08/31] fix(UAC): provide required UI context [EE-4415] (#7854) --- app/portainer/react/components/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/portainer/react/components/index.ts b/app/portainer/react/components/index.ts index e2f2e2410..17e28fd3e 100644 --- a/app/portainer/react/components/index.ts +++ b/app/portainer/react/components/index.ts @@ -109,7 +109,7 @@ export const componentsModule = angular .component('boxSelectorBadgeIcon', r2a(BadgeIcon, ['featherIcon', 'icon'])) .component( 'accessControlPanel', - r2a(withReactQuery(withCurrentUser(AccessControlPanel)), [ + r2a(withUIRouter(withReactQuery(withCurrentUser(AccessControlPanel))), [ 'disableOwnershipChange', 'onUpdateSuccess', 'resourceControl', From 0c995ae1c845d0867da3e542b2f111f47bc86c98 Mon Sep 17 00:00:00 2001 From: Dakota Walsh <101994734+dakota-portainer@users.noreply.github.com> Date: Tue, 18 Oct 2022 10:46:27 +1300 Subject: [PATCH 09/31] fix(kubernetes): create proxied kubeclient EE-4326 (#7850) --- api/cmd/portainer/main.go | 6 +- api/go.mod | 1 + api/go.sum | 2 + .../kubernetes/configmaps_and_secrets.go | 21 ++- api/http/handler/kubernetes/handler.go | 102 +++++++++++++-- api/http/handler/kubernetes/ingresses.go | 121 ++++++++++++++++-- api/http/handler/kubernetes/namespaces.go | 85 ++++++++++-- api/http/handler/kubernetes/services.go | 83 ++++++++++-- api/kubernetes/cli/client.go | 64 ++++++++- api/kubernetes/cli/ingress.go | 14 +- api/kubernetes/cli/resource_test.go | 2 +- api/kubernetes/cli/role.go | 5 + api/kubernetes/kubeclusteraccess_service.go | 2 +- api/portainer.go | 2 +- .../IngressDatatable/IngressDataTable.tsx | 11 +- .../IngressDatatable/columns/name.tsx | 27 ++-- .../ingress-table/ingress-table.html | 4 +- app/portainer/hooks/useUser.tsx | 4 +- 18 files changed, 485 insertions(+), 71 deletions(-) diff --git a/api/cmd/portainer/main.go b/api/cmd/portainer/main.go index fd5fa1b72..360e3b663 100644 --- a/api/cmd/portainer/main.go +++ b/api/cmd/portainer/main.go @@ -239,8 +239,8 @@ func initDockerClientFactory(signatureService portainer.DigitalSignatureService, return docker.NewClientFactory(signatureService, reverseTunnelService) } -func initKubernetesClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService, instanceID string, dataStore dataservices.DataStore) *kubecli.ClientFactory { - return kubecli.NewClientFactory(signatureService, reverseTunnelService, instanceID, dataStore) +func initKubernetesClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService, dataStore dataservices.DataStore, instanceID, addrHTTPS, userSessionTimeout string) (*kubecli.ClientFactory, error) { + return kubecli.NewClientFactory(signatureService, reverseTunnelService, dataStore, instanceID, addrHTTPS, userSessionTimeout) } func initSnapshotService( @@ -612,7 +612,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server { reverseTunnelService := chisel.NewService(dataStore, shutdownCtx) dockerClientFactory := initDockerClientFactory(digitalSignatureService, reverseTunnelService) - kubernetesClientFactory := initKubernetesClientFactory(digitalSignatureService, reverseTunnelService, instanceID, dataStore) + kubernetesClientFactory, err := initKubernetesClientFactory(digitalSignatureService, reverseTunnelService, dataStore, instanceID, *flags.AddrHTTPS, settings.UserSessionTimeout) snapshotService, err := initSnapshotService(*flags.SnapshotInterval, dataStore, dockerClientFactory, kubernetesClientFactory, shutdownCtx) if err != nil { diff --git a/api/go.mod b/api/go.mod index 60d1da09d..e7f2ae2c5 100644 --- a/api/go.mod +++ b/api/go.mod @@ -31,6 +31,7 @@ require ( github.com/json-iterator/go v1.1.12 github.com/koding/websocketproxy v0.0.0-20181220232114-7ed82d81a28c github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6 + github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 github.com/portainer/docker-compose-wrapper v0.0.0-20220708023447-a69a4ebaa021 github.com/portainer/libcrypto v0.0.0-20220506221303-1f4fb3b30f9a diff --git a/api/go.sum b/api/go.sum index 650360013..1a60e9a02 100644 --- a/api/go.sum +++ b/api/go.sum @@ -352,6 +352,8 @@ github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrB github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6 h1:lNCW6THrCKBiJBpz8kbVGjC7MgdCGKwuvBgc7LoD6sw= github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI= +github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= +github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/api/http/handler/kubernetes/configmaps_and_secrets.go b/api/http/handler/kubernetes/configmaps_and_secrets.go index b38b16640..d1c7af22f 100644 --- a/api/http/handler/kubernetes/configmaps_and_secrets.go +++ b/api/http/handler/kubernetes/configmaps_and_secrets.go @@ -2,6 +2,7 @@ package kubernetes import ( "net/http" + "strconv" httperror "github.com/portainer/libhttp/error" "github.com/portainer/libhttp/request" @@ -9,7 +10,23 @@ import ( ) func (handler *Handler) getKubernetesConfigMaps(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { - cli := handler.KubernetesClient + endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id") + if err != nil { + return httperror.BadRequest( + "Invalid environment identifier route variable", + err, + ) + } + + cli, ok := handler.KubernetesClientFactory.GetProxyKubeClient( + strconv.Itoa(endpointID), r.Header.Get("Authorization"), + ) + if !ok { + return httperror.InternalServerError( + "Failed to lookup KubeClient", + nil, + ) + } namespace, err := request.RetrieveRouteVariableValue(r, "namespace") if err != nil { @@ -22,7 +39,7 @@ func (handler *Handler) getKubernetesConfigMaps(w http.ResponseWriter, r *http.R configmaps, err := cli.GetConfigMapsAndSecrets(namespace) if err != nil { return httperror.InternalServerError( - "Unable to retrieve nodes limits", + "Unable to retrieve configmaps and secrets", err, ) } diff --git a/api/http/handler/kubernetes/handler.go b/api/http/handler/kubernetes/handler.go index 680e4ea46..7d46c00d3 100644 --- a/api/http/handler/kubernetes/handler.go +++ b/api/http/handler/kubernetes/handler.go @@ -3,6 +3,8 @@ package kubernetes import ( "errors" "net/http" + "net/url" + "strconv" portainer "github.com/portainer/portainer/api" portainerDsErrors "github.com/portainer/portainer/api/dataservices/errors" @@ -24,7 +26,6 @@ type Handler struct { *mux.Router authorizationService *authorization.Service DataStore dataservices.DataStore - KubernetesClient portainer.KubeClient KubernetesClientFactory *cli.ClientFactory JwtService dataservices.JWTService kubeClusterAccessService kubernetes.KubeClusterAccessService @@ -39,7 +40,6 @@ func NewHandler(bouncer *security.RequestBouncer, authorizationService *authoriz JwtService: jwtService, kubeClusterAccessService: kubeClusterAccessService, KubernetesClientFactory: kubernetesClientFactory, - KubernetesClient: kubernetesClient, } kubeRouter := h.PathPrefix("/kubernetes").Subrouter() @@ -85,13 +85,19 @@ func kubeOnlyMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(rw http.ResponseWriter, request *http.Request) { endpoint, err := middlewares.FetchEndpoint(request) if err != nil { - httperror.WriteError(rw, http.StatusInternalServerError, "Unable to find an environment on request context", err) + httperror.InternalServerError( + "Unable to find an environment on request context", + err, + ) return } if !endpointutils.IsKubernetesEndpoint(endpoint) { errMessage := "environment is not a Kubernetes environment" - httperror.WriteError(rw, http.StatusBadRequest, errMessage, errors.New(errMessage)) + httperror.BadRequest( + errMessage, + errors.New(errMessage), + ) return } @@ -109,6 +115,7 @@ func (handler *Handler) kubeClient(next http.Handler) http.Handler { "Invalid environment identifier route variable", err, ) + return } endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID)) @@ -119,6 +126,7 @@ func (handler *Handler) kubeClient(next http.Handler) http.Handler { "Unable to find an environment with the specified identifier inside the database", err, ) + return } else if err != nil { httperror.WriteError( w, @@ -126,23 +134,101 @@ func (handler *Handler) kubeClient(next http.Handler) http.Handler { "Unable to find an environment with the specified identifier inside the database", err, ) + return } if handler.KubernetesClientFactory == nil { next.ServeHTTP(w, r) return } - kubeCli, err := handler.KubernetesClientFactory.GetKubeClient(endpoint) + // Generate a proxied kubeconfig, then create a kubeclient using it. + tokenData, err := security.RetrieveTokenData(r) + if err != nil { + httperror.WriteError( + w, + http.StatusForbidden, + "Permission denied to access environment", + err, + ) + return + } + bearerToken, err := handler.JwtService.GenerateTokenForKubeconfig(tokenData) if err != nil { httperror.WriteError( w, http.StatusInternalServerError, - "Unable to create Kubernetes client", + "Unable to create JWT token", err, ) - + return } - handler.KubernetesClient = kubeCli + singleEndpointList := []portainer.Endpoint{ + *endpoint, + } + config, handlerErr := handler.buildConfig( + r, + tokenData, + bearerToken, + singleEndpointList, + ) + if err != nil { + httperror.WriteError( + w, + http.StatusInternalServerError, + "Unable to build endpoint kubeconfig", + handlerErr.Err, + ) + return + } + + if len(config.Clusters) == 0 { + httperror.WriteError( + w, + http.StatusInternalServerError, + "Unable build cluster kubeconfig", + errors.New("Unable build cluster kubeconfig"), + ) + return + } + + // Manually setting the localhost to route + // the request to proxy server + serverURL, err := url.Parse(config.Clusters[0].Cluster.Server) + if err != nil { + httperror.WriteError( + w, + http.StatusInternalServerError, + "Unable parse cluster's kubeconfig server URL", + nil, + ) + return + } + serverURL.Scheme = "https" + serverURL.Host = "localhost" + handler.KubernetesClientFactory.AddrHTTPS + config.Clusters[0].Cluster.Server = serverURL.String() + + yaml, err := cli.GenerateYAML(config) + if err != nil { + httperror.WriteError( + w, + http.StatusInternalServerError, + "Unable to generate yaml from endpoint kubeconfig", + err, + ) + return + } + kubeCli, err := handler.KubernetesClientFactory.CreateKubeClientFromKubeConfig(endpoint.Name, []byte(yaml)) + if err != nil { + httperror.WriteError( + w, + http.StatusInternalServerError, + "Failed to create client from kubeconfig", + err, + ) + return + } + + handler.KubernetesClientFactory.SetProxyKubeClient(strconv.Itoa(int(endpoint.ID)), r.Header.Get("Authorization"), kubeCli) next.ServeHTTP(w, r) }) } diff --git a/api/http/handler/kubernetes/ingresses.go b/api/http/handler/kubernetes/ingresses.go index 741436f82..abc9842ed 100644 --- a/api/http/handler/kubernetes/ingresses.go +++ b/api/http/handler/kubernetes/ingresses.go @@ -2,6 +2,7 @@ package kubernetes import ( "net/http" + "strconv" httperror "github.com/portainer/libhttp/error" "github.com/portainer/libhttp/request" @@ -49,7 +50,13 @@ func (handler *Handler) getKubernetesIngressControllers(w http.ResponseWriter, r ) } - controllers := cli.GetIngressControllers() + controllers, err := cli.GetIngressControllers() + if err != nil { + return httperror.InternalServerError( + "Failed to fetch ingressclasses", + err, + ) + } existingClasses := endpoint.Kubernetes.Configuration.IngressClasses var updatedClasses []portainer.KubernetesIngressClassConfig for i := range controllers { @@ -129,8 +136,23 @@ func (handler *Handler) getKubernetesIngressControllersByNamespace(w http.Respon ) } - cli := handler.KubernetesClient - currentControllers := cli.GetIngressControllers() + cli, ok := handler.KubernetesClientFactory.GetProxyKubeClient( + strconv.Itoa(endpointID), r.Header.Get("Authorization"), + ) + if !ok { + return httperror.InternalServerError( + "Failed to lookup KubeClient", + nil, + ) + } + + currentControllers, err := cli.GetIngressControllers() + if err != nil { + return httperror.InternalServerError( + "Failed to fetch ingressclasses", + err, + ) + } kubernetesConfig := endpoint.Kubernetes.Configuration existingClasses := kubernetesConfig.IngressClasses ingressAvailabilityPerNamespace := kubernetesConfig.IngressAvailabilityPerNamespace @@ -229,7 +251,13 @@ func (handler *Handler) updateKubernetesIngressControllers(w http.ResponseWriter } existingClasses := endpoint.Kubernetes.Configuration.IngressClasses - controllers := cli.GetIngressControllers() + controllers, err := cli.GetIngressControllers() + if err != nil { + return httperror.InternalServerError( + "Unable to get ingress controllers", + err, + ) + } var updatedClasses []portainer.KubernetesIngressClassConfig for i := range controllers { controllers[i].Availability = true @@ -401,11 +429,28 @@ func (handler *Handler) getKubernetesIngresses(w http.ResponseWriter, r *http.Re ) } - cli := handler.KubernetesClient + endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id") + if err != nil { + return httperror.BadRequest( + "Invalid environment identifier route variable", + err, + ) + } + + cli, ok := handler.KubernetesClientFactory.GetProxyKubeClient( + strconv.Itoa(endpointID), r.Header.Get("Authorization"), + ) + if !ok { + return httperror.InternalServerError( + "Failed to lookup KubeClient", + nil, + ) + } + ingresses, err := cli.GetIngresses(namespace) if err != nil { return httperror.InternalServerError( - "Unable to retrieve nodes limits", + "Unable to retrieve ingresses", err, ) } @@ -431,11 +476,28 @@ func (handler *Handler) createKubernetesIngress(w http.ResponseWriter, r *http.R ) } - cli := handler.KubernetesClient + endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id") + if err != nil { + return httperror.BadRequest( + "Invalid environment identifier route variable", + err, + ) + } + + cli, ok := handler.KubernetesClientFactory.GetProxyKubeClient( + strconv.Itoa(endpointID), r.Header.Get("Authorization"), + ) + if !ok { + return httperror.InternalServerError( + "Failed to lookup KubeClient", + nil, + ) + } + err = cli.CreateIngress(namespace, payload) if err != nil { return httperror.InternalServerError( - "Unable to retrieve nodes limits", + "Unable to retrieve the ingress", err, ) } @@ -443,10 +505,26 @@ func (handler *Handler) createKubernetesIngress(w http.ResponseWriter, r *http.R } func (handler *Handler) deleteKubernetesIngresses(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { - cli := handler.KubernetesClient + endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id") + if err != nil { + return httperror.BadRequest( + "Invalid environment identifier route variable", + err, + ) + } + + cli, ok := handler.KubernetesClientFactory.GetProxyKubeClient( + strconv.Itoa(endpointID), r.Header.Get("Authorization"), + ) + if !ok { + return httperror.InternalServerError( + "Failed to lookup KubeClient", + nil, + ) + } var payload models.K8sIngressDeleteRequests - err := request.DecodeAndValidateJSONPayload(r, &payload) + err = request.DecodeAndValidateJSONPayload(r, &payload) if err != nil { return httperror.BadRequest("Invalid request payload", err) } @@ -454,7 +532,7 @@ func (handler *Handler) deleteKubernetesIngresses(w http.ResponseWriter, r *http err = cli.DeleteIngresses(payload) if err != nil { return httperror.InternalServerError( - "Unable to retrieve nodes limits", + "Unable to delete ingresses", err, ) } @@ -479,11 +557,28 @@ func (handler *Handler) updateKubernetesIngress(w http.ResponseWriter, r *http.R ) } - cli := handler.KubernetesClient + endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id") + if err != nil { + return httperror.BadRequest( + "Invalid environment identifier route variable", + err, + ) + } + + cli, ok := handler.KubernetesClientFactory.GetProxyKubeClient( + strconv.Itoa(endpointID), r.Header.Get("Authorization"), + ) + if !ok { + return httperror.InternalServerError( + "Failed to lookup KubeClient", + nil, + ) + } + err = cli.UpdateIngress(namespace, payload) if err != nil { return httperror.InternalServerError( - "Unable to retrieve nodes limits", + "Unable to update the ingress", err, ) } diff --git a/api/http/handler/kubernetes/namespaces.go b/api/http/handler/kubernetes/namespaces.go index 884ae7038..68ba70261 100644 --- a/api/http/handler/kubernetes/namespaces.go +++ b/api/http/handler/kubernetes/namespaces.go @@ -2,6 +2,7 @@ package kubernetes import ( "net/http" + "strconv" httperror "github.com/portainer/libhttp/error" "github.com/portainer/libhttp/request" @@ -10,12 +11,28 @@ import ( ) func (handler *Handler) getKubernetesNamespaces(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { - cli := handler.KubernetesClient + endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id") + if err != nil { + return httperror.BadRequest( + "Invalid environment identifier route variable", + err, + ) + } + + cli, ok := handler.KubernetesClientFactory.GetProxyKubeClient( + strconv.Itoa(endpointID), r.Header.Get("Authorization"), + ) + if !ok { + return httperror.InternalServerError( + "Failed to lookup KubeClient", + nil, + ) + } namespaces, err := cli.GetNamespaces() if err != nil { return httperror.InternalServerError( - "Unable to retrieve nodes limits", + "Unable to retrieve namespaces", err, ) } @@ -24,10 +41,26 @@ func (handler *Handler) getKubernetesNamespaces(w http.ResponseWriter, r *http.R } func (handler *Handler) createKubernetesNamespace(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { - cli := handler.KubernetesClient + endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id") + if err != nil { + return httperror.BadRequest( + "Invalid environment identifier route variable", + err, + ) + } + + cli, ok := handler.KubernetesClientFactory.GetProxyKubeClient( + strconv.Itoa(endpointID), r.Header.Get("Authorization"), + ) + if !ok { + return httperror.InternalServerError( + "Failed to lookup KubeClient", + nil, + ) + } var payload models.K8sNamespaceDetails - err := request.DecodeAndValidateJSONPayload(r, &payload) + err = request.DecodeAndValidateJSONPayload(r, &payload) if err != nil { return httperror.BadRequest( "Invalid request payload", @@ -38,7 +71,7 @@ func (handler *Handler) createKubernetesNamespace(w http.ResponseWriter, r *http err = cli.CreateNamespace(payload) if err != nil { return httperror.InternalServerError( - "Unable to retrieve nodes limits", + "Unable to create namespace", err, ) } @@ -46,7 +79,23 @@ func (handler *Handler) createKubernetesNamespace(w http.ResponseWriter, r *http } func (handler *Handler) deleteKubernetesNamespaces(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { - cli := handler.KubernetesClient + endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id") + if err != nil { + return httperror.BadRequest( + "Invalid environment identifier route variable", + err, + ) + } + + cli, ok := handler.KubernetesClientFactory.GetProxyKubeClient( + strconv.Itoa(endpointID), r.Header.Get("Authorization"), + ) + if !ok { + return httperror.InternalServerError( + "Failed to lookup KubeClient", + nil, + ) + } namespace, err := request.RetrieveRouteVariableValue(r, "namespace") if err != nil { @@ -59,7 +108,7 @@ func (handler *Handler) deleteKubernetesNamespaces(w http.ResponseWriter, r *htt err = cli.DeleteNamespace(namespace) if err != nil { return httperror.InternalServerError( - "Unable to retrieve nodes limits", + "Unable to delete namespace", err, ) } @@ -68,17 +117,33 @@ func (handler *Handler) deleteKubernetesNamespaces(w http.ResponseWriter, r *htt } func (handler *Handler) updateKubernetesNamespace(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { - cli := handler.KubernetesClient + endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id") + if err != nil { + return httperror.BadRequest( + "Invalid environment identifier route variable", + err, + ) + } + + cli, ok := handler.KubernetesClientFactory.GetProxyKubeClient( + strconv.Itoa(endpointID), r.Header.Get("Authorization"), + ) + if !ok { + return httperror.InternalServerError( + "Failed to lookup KubeClient", + nil, + ) + } var payload models.K8sNamespaceDetails - err := request.DecodeAndValidateJSONPayload(r, &payload) + err = request.DecodeAndValidateJSONPayload(r, &payload) if err != nil { return httperror.BadRequest("Invalid request payload", err) } err = cli.UpdateNamespace(payload) if err != nil { - return httperror.InternalServerError("Unable to retrieve nodes limits", err) + return httperror.InternalServerError("Unable to update namespace", err) } return nil } diff --git a/api/http/handler/kubernetes/services.go b/api/http/handler/kubernetes/services.go index 2e60513ce..0cc223eb9 100644 --- a/api/http/handler/kubernetes/services.go +++ b/api/http/handler/kubernetes/services.go @@ -2,6 +2,7 @@ package kubernetes import ( "net/http" + "strconv" httperror "github.com/portainer/libhttp/error" "github.com/portainer/libhttp/request" @@ -18,7 +19,24 @@ func (handler *Handler) getKubernetesServices(w http.ResponseWriter, r *http.Req ) } - cli := handler.KubernetesClient + endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id") + if err != nil { + return httperror.BadRequest( + "Invalid environment identifier route variable", + err, + ) + } + + cli, ok := handler.KubernetesClientFactory.GetProxyKubeClient( + strconv.Itoa(endpointID), r.Header.Get("Authorization"), + ) + if !ok { + return httperror.InternalServerError( + "Failed to lookup KubeClient", + nil, + ) + } + services, err := cli.GetServices(namespace) if err != nil { return httperror.InternalServerError( @@ -48,11 +66,28 @@ func (handler *Handler) createKubernetesService(w http.ResponseWriter, r *http.R ) } - cli := handler.KubernetesClient + endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id") + if err != nil { + return httperror.BadRequest( + "Invalid environment identifier route variable", + err, + ) + } + + cli, ok := handler.KubernetesClientFactory.GetProxyKubeClient( + strconv.Itoa(endpointID), r.Header.Get("Authorization"), + ) + if !ok { + return httperror.InternalServerError( + "Failed to lookup KubeClient", + nil, + ) + } + err = cli.CreateService(namespace, payload) if err != nil { return httperror.InternalServerError( - "Unable to retrieve nodes limits", + "Unable to create sercice", err, ) } @@ -60,10 +95,26 @@ func (handler *Handler) createKubernetesService(w http.ResponseWriter, r *http.R } func (handler *Handler) deleteKubernetesServices(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { - cli := handler.KubernetesClient + endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id") + if err != nil { + return httperror.BadRequest( + "Invalid environment identifier route variable", + err, + ) + } + + cli, ok := handler.KubernetesClientFactory.GetProxyKubeClient( + strconv.Itoa(endpointID), r.Header.Get("Authorization"), + ) + if !ok { + return httperror.InternalServerError( + "Failed to lookup KubeClient", + nil, + ) + } var payload models.K8sServiceDeleteRequests - err := request.DecodeAndValidateJSONPayload(r, &payload) + err = request.DecodeAndValidateJSONPayload(r, &payload) if err != nil { return httperror.BadRequest( "Invalid request payload", @@ -74,7 +125,7 @@ func (handler *Handler) deleteKubernetesServices(w http.ResponseWriter, r *http. err = cli.DeleteServices(payload) if err != nil { return httperror.InternalServerError( - "Unable to retrieve nodes limits", + "Unable to delete service", err, ) } @@ -99,11 +150,27 @@ func (handler *Handler) updateKubernetesService(w http.ResponseWriter, r *http.R ) } - cli := handler.KubernetesClient + endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id") + if err != nil { + return httperror.BadRequest( + "Invalid environment identifier route variable", + err, + ) + } + + cli, ok := handler.KubernetesClientFactory.GetProxyKubeClient( + strconv.Itoa(endpointID), r.Header.Get("Authorization"), + ) + if !ok { + return httperror.InternalServerError( + "Failed to lookup KubeClient", + nil, + ) + } err = cli.UpdateService(namespace, payload) if err != nil { return httperror.InternalServerError( - "Unable to retrieve nodes limits", + "Unable to update service", err, ) } diff --git a/api/kubernetes/cli/client.go b/api/kubernetes/cli/client.go index 23bd59993..821a17a0e 100644 --- a/api/kubernetes/cli/client.go +++ b/api/kubernetes/cli/client.go @@ -5,8 +5,10 @@ import ( "net/http" "strconv" "sync" + "time" cmap "github.com/orcaman/concurrent-map" + "github.com/patrickmn/go-cache" "github.com/pkg/errors" portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/dataservices" @@ -23,6 +25,8 @@ type ( signatureService portainer.DigitalSignatureService instanceID string endpointClients cmap.ConcurrentMap + endpointProxyClients *cache.Cache + AddrHTTPS string } // KubeClient represent a service used to execute Kubernetes operations @@ -34,14 +38,24 @@ type ( ) // NewClientFactory returns a new instance of a ClientFactory -func NewClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService, instanceID string, dataStore dataservices.DataStore) *ClientFactory { +func NewClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService, dataStore dataservices.DataStore, instanceID, addrHTTPS, userSessionTimeout string) (*ClientFactory, error) { + if userSessionTimeout == "" { + userSessionTimeout = portainer.DefaultUserSessionTimeout + } + timeout, err := time.ParseDuration(userSessionTimeout) + if err != nil { + return nil, err + } + return &ClientFactory{ dataStore: dataStore, signatureService: signatureService, reverseTunnelService: reverseTunnelService, instanceID: instanceID, endpointClients: cmap.New(), - } + endpointProxyClients: cache.New(timeout, timeout), + AddrHTTPS: addrHTTPS, + }, nil } func (factory *ClientFactory) GetInstanceID() (instanceID string) { @@ -59,7 +73,7 @@ func (factory *ClientFactory) GetKubeClient(endpoint *portainer.Endpoint) (porta key := strconv.Itoa(int(endpoint.ID)) client, ok := factory.endpointClients.Get(key) if !ok { - client, err := factory.createKubeClient(endpoint) + client, err := factory.createCachedAdminKubeClient(endpoint) if err != nil { return nil, err } @@ -71,7 +85,49 @@ func (factory *ClientFactory) GetKubeClient(endpoint *portainer.Endpoint) (porta return client.(portainer.KubeClient), nil } -func (factory *ClientFactory) createKubeClient(endpoint *portainer.Endpoint) (portainer.KubeClient, error) { +// GetProxyKubeClient retrieves a KubeClient from the cache. You should be +// calling SetProxyKubeClient before first. It is normally, called the +// kubernetes middleware. +func (factory *ClientFactory) GetProxyKubeClient(endpointID, token string) (portainer.KubeClient, bool) { + client, ok := factory.endpointProxyClients.Get(endpointID + "." + token) + if !ok { + return nil, false + } + return client.(portainer.KubeClient), true +} + +// SetProxyKubeClient stores a kubeclient in the cache. +func (factory *ClientFactory) SetProxyKubeClient(endpointID, token string, cli portainer.KubeClient) { + factory.endpointProxyClients.Set(endpointID+"."+token, cli, 0) +} + +// CreateKubeClientFromKubeConfig creates a KubeClient from a clusterID, and +// Kubernetes config. +func (factory *ClientFactory) CreateKubeClientFromKubeConfig(clusterID string, kubeConfig []byte) (portainer.KubeClient, error) { + config, err := clientcmd.NewClientConfigFromBytes([]byte(kubeConfig)) + if err != nil { + return nil, err + } + cliConfig, err := config.ClientConfig() + if err != nil { + return nil, err + } + + cli, err := kubernetes.NewForConfig(cliConfig) + if err != nil { + return nil, err + } + + kubecli := &KubeClient{ + cli: cli, + instanceID: factory.instanceID, + lock: &sync.Mutex{}, + } + + return kubecli, nil +} + +func (factory *ClientFactory) createCachedAdminKubeClient(endpoint *portainer.Endpoint) (portainer.KubeClient, error) { cli, err := factory.CreateClient(endpoint) if err != nil { return nil, err diff --git a/api/kubernetes/cli/ingress.go b/api/kubernetes/cli/ingress.go index ce7ffc442..7eaf3e593 100644 --- a/api/kubernetes/cli/ingress.go +++ b/api/kubernetes/cli/ingress.go @@ -5,11 +5,12 @@ import ( "strings" "github.com/portainer/portainer/api/database/models" + "github.com/rs/zerolog/log" netv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func (kcl *KubeClient) GetIngressControllers() models.K8sIngressControllers { +func (kcl *KubeClient) GetIngressControllers() (models.K8sIngressControllers, error) { var controllers []models.K8sIngressController // We know that each existing class points to a controller so we can start @@ -17,19 +18,22 @@ func (kcl *KubeClient) GetIngressControllers() models.K8sIngressControllers { classClient := kcl.cli.NetworkingV1().IngressClasses() classList, err := classClient.List(context.Background(), metav1.ListOptions{}) if err != nil { - return nil + return nil, err } // We want to know which of these controllers is in use. var ingresses []models.K8sIngressInfo namespaces, err := kcl.GetNamespaces() if err != nil { - return nil + return nil, err } for namespace := range namespaces { t, err := kcl.GetIngresses(namespace) if err != nil { - return nil + // User might not be able to list ingresses in system/not allowed + // namespaces. + log.Debug().Err(err).Msg("failed to list ingresses for the current user, skipped sending ingress") + continue } ingresses = append(ingresses, t...) } @@ -58,7 +62,7 @@ func (kcl *KubeClient) GetIngressControllers() models.K8sIngressControllers { } controllers = append(controllers, controller) } - return controllers + return controllers, nil } // GetIngresses gets all the ingresses for a given namespace in a k8s endpoint. diff --git a/api/kubernetes/cli/resource_test.go b/api/kubernetes/cli/resource_test.go index 902671103..d86044e98 100644 --- a/api/kubernetes/cli/resource_test.go +++ b/api/kubernetes/cli/resource_test.go @@ -85,7 +85,7 @@ func Test_GenerateYAML(t *testing.T) { t.Errorf("generateYamlConfig failed; err=%s", err) } - if compareYAMLStrings(yaml, ryt.wantYAML) != 0 { + if compareYAMLStrings(string(yaml), ryt.wantYAML) != 0 { t.Errorf("generateYamlConfig failed;\ngot=\n%s\nwant=\n%s", yaml, ryt.wantYAML) } }) diff --git a/api/kubernetes/cli/role.go b/api/kubernetes/cli/role.go index 4bf7f58fc..80fc825e2 100644 --- a/api/kubernetes/cli/role.go +++ b/api/kubernetes/cli/role.go @@ -25,6 +25,11 @@ func getPortainerUserDefaultPolicies() []rbacv1.PolicyRule { Resources: []string{"namespaces", "pods", "nodes"}, APIGroups: []string{"metrics.k8s.io"}, }, + { + Verbs: []string{"list"}, + Resources: []string{"ingressclasses"}, + APIGroups: []string{"networking.k8s.io"}, + }, } } diff --git a/api/kubernetes/kubeclusteraccess_service.go b/api/kubernetes/kubeclusteraccess_service.go index d7da7bd2e..4a97885d4 100644 --- a/api/kubernetes/kubeclusteraccess_service.go +++ b/api/kubernetes/kubeclusteraccess_service.go @@ -106,7 +106,7 @@ func (service *kubeClusterAccessService) GetData(hostURL string, endpointID port baseURL = fmt.Sprintf("/%s/", strings.Trim(baseURL, "/")) } - log.Info(). + log.Debug(). Str("host_URL", hostURL). Str("HTTPS_bind_address", service.httpsBindAddr). Str("base_URL", baseURL). diff --git a/api/portainer.go b/api/portainer.go index 11ae98781..0277de258 100644 --- a/api/portainer.go +++ b/api/portainer.go @@ -1357,7 +1357,7 @@ type ( GetNamespaces() (map[string]K8sNamespaceInfo, error) DeleteNamespace(namespace string) error GetConfigMapsAndSecrets(namespace string) ([]models.K8sConfigMapOrSecret, error) - GetIngressControllers() models.K8sIngressControllers + GetIngressControllers() (models.K8sIngressControllers, error) CreateIngress(namespace string, info models.K8sIngressInfo) error UpdateIngress(namespace string, info models.K8sIngressInfo) error GetIngresses(namespace string) ([]models.K8sIngressInfo, error) diff --git a/app/kubernetes/react/views/networks/ingresses/IngressDatatable/IngressDataTable.tsx b/app/kubernetes/react/views/networks/ingresses/IngressDatatable/IngressDataTable.tsx index 738c432ef..f4f201873 100644 --- a/app/kubernetes/react/views/networks/ingresses/IngressDatatable/IngressDataTable.tsx +++ b/app/kubernetes/react/views/networks/ingresses/IngressDatatable/IngressDataTable.tsx @@ -3,7 +3,7 @@ import { useRouter } from '@uirouter/react'; import { useEnvironmentId } from '@/portainer/hooks/useEnvironmentId'; import { useNamespaces } from '@/react/kubernetes/namespaces/queries'; -import { Authorized } from '@/portainer/hooks/useUser'; +import { useAuthorizations, Authorized } from '@/portainer/hooks/useUser'; import { confirmDeletionAsync } from '@/portainer/services/modal.service/confirm'; import { Datatable } from '@@/datatables'; @@ -55,6 +55,7 @@ export function IngressDataTable() { }} getRowId={(row) => row.Name + row.Type + row.Namespace} renderTableActions={tableActions} + disableSelect={useCheckboxes()} /> ); @@ -80,7 +81,7 @@ export function IngressDataTable() { - + {{ $ctrl.formValues.File.name }} - - - From ee5600b6affd2d5085dd5d9e91019b8b6d56aac0 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Thu, 20 Oct 2022 17:23:56 +0300 Subject: [PATCH 18/31] chore(build): incremental ts build [EE-4204] (#7888) --- tsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/tsconfig.json b/tsconfig.json index 476f76334..17b874743 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -5,6 +5,7 @@ "module": "es6", // "module": "commonjs", // "module": "esnext", + "incremental": true, "target": "es2017", "allowJs": true, "checkJs": false, From 535a26412f18f65898efe3ef92694c3895e3604c Mon Sep 17 00:00:00 2001 From: andres-portainer <91705312+andres-portainer@users.noreply.github.com> Date: Thu, 20 Oct 2022 11:33:54 -0300 Subject: [PATCH 19/31] fix(logging): default to pretty logging [EE-4371] (#7847) * fix(logging): default to pretty logging EE-4371 * feat(app/logs): prettify stack traces in JSON logs * feat(nomad/logs): prettify JSON logs in log viewer * feat(kubernetes/logs): prettigy JSON logs in log viewers * feat(app/logs): format and color zerolog prettified logs * fix(app/logs): pre-parse logs when they are double serialized Co-authored-by: andres-portainer Co-authored-by: LP B --- api/cli/cli.go | 1 + api/cmd/portainer/log.go | 22 ++ api/cmd/portainer/main.go | 2 + api/portainer.go | 1 + .../components/log-viewer/logViewer.html | 2 +- .../log-viewer/logViewerController.js | 8 +- app/docker/helpers/logHelper.js | 223 --------------- app/docker/helpers/logHelper/colors/colors.ts | 63 +++++ app/docker/helpers/logHelper/colors/index.ts | 7 + .../helpers/logHelper/colors/rawColors.json | 258 ++++++++++++++++++ .../helpers/logHelper/concatLogsToString.ts | 15 + .../helpers/logHelper/formatJSONLogs.ts | 55 ++++ app/docker/helpers/logHelper/formatLogs.ts | 141 ++++++++++ .../helpers/logHelper/formatZerologLogs.ts | 119 ++++++++ app/docker/helpers/logHelper/formatters.ts | 154 +++++++++++ app/docker/helpers/logHelper/index.ts | 2 + app/docker/helpers/logHelper/types.ts | 53 ++++ app/docker/services/containerService.js | 5 +- app/docker/services/serviceService.js | 9 +- app/docker/services/taskService.js | 6 +- app/kubernetes/pod/service.js | 2 +- .../views/applications/logs/logs.html | 7 +- .../views/applications/logs/logsController.js | 18 +- app/kubernetes/views/stacks/logs/logs.html | 8 +- .../views/stacks/logs/logsController.js | 27 +- package.json | 1 - yarn.lock | 5 - 27 files changed, 935 insertions(+), 279 deletions(-) delete mode 100644 app/docker/helpers/logHelper.js create mode 100644 app/docker/helpers/logHelper/colors/colors.ts create mode 100644 app/docker/helpers/logHelper/colors/index.ts create mode 100644 app/docker/helpers/logHelper/colors/rawColors.json create mode 100644 app/docker/helpers/logHelper/concatLogsToString.ts create mode 100644 app/docker/helpers/logHelper/formatJSONLogs.ts create mode 100644 app/docker/helpers/logHelper/formatLogs.ts create mode 100644 app/docker/helpers/logHelper/formatZerologLogs.ts create mode 100644 app/docker/helpers/logHelper/formatters.ts create mode 100644 app/docker/helpers/logHelper/index.ts create mode 100644 app/docker/helpers/logHelper/types.ts diff --git a/api/cli/cli.go b/api/cli/cli.go index 83ced70ea..69dd61b75 100644 --- a/api/cli/cli.go +++ b/api/cli/cli.go @@ -62,6 +62,7 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) { MaxBatchDelay: kingpin.Flag("max-batch-delay", "Maximum delay before a batch starts").Duration(), SecretKeyName: kingpin.Flag("secret-key-name", "Secret key name for encryption and will be used as /run/secrets/.").Default(defaultSecretKeyName).String(), LogLevel: kingpin.Flag("log-level", "Set the minimum logging level to show").Default("INFO").Enum("DEBUG", "INFO", "WARN", "ERROR"), + LogMode: kingpin.Flag("log-mode", "Set the logging output mode").Default("PRETTY").Enum("PRETTY", "JSON"), } kingpin.Parse() diff --git a/api/cmd/portainer/log.go b/api/cmd/portainer/log.go index 1623b4fd7..ef95b26b0 100644 --- a/api/cmd/portainer/log.go +++ b/api/cmd/portainer/log.go @@ -1,7 +1,9 @@ package main import ( + "fmt" stdlog "log" + "os" "github.com/rs/zerolog" "github.com/rs/zerolog/log" @@ -31,3 +33,23 @@ func setLoggingLevel(level string) { zerolog.SetGlobalLevel(zerolog.DebugLevel) } } + +func setLoggingMode(mode string) { + switch mode { + case "PRETTY": + log.Logger = log.Output(zerolog.ConsoleWriter{ + Out: os.Stderr, + NoColor: true, + TimeFormat: "2006/01/02 03:04PM", + FormatMessage: formatMessage}) + case "JSON": + log.Logger = log.Output(os.Stderr) + } +} + +func formatMessage(i interface{}) string { + if i == nil { + return "" + } + return fmt.Sprintf("%s |", i) +} diff --git a/api/cmd/portainer/main.go b/api/cmd/portainer/main.go index 360e3b663..031482b8d 100644 --- a/api/cmd/portainer/main.go +++ b/api/cmd/portainer/main.go @@ -758,10 +758,12 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server { func main() { configureLogger() + setLoggingMode("PRETTY") flags := initCLI() setLoggingLevel(*flags.LogLevel) + setLoggingMode(*flags.LogMode) for { server := buildServer(flags) diff --git a/api/portainer.go b/api/portainer.go index 0277de258..d1df3c2be 100644 --- a/api/portainer.go +++ b/api/portainer.go @@ -130,6 +130,7 @@ type ( MaxBatchDelay *time.Duration SecretKeyName *string LogLevel *string + LogMode *string } // CustomTemplateVariableDefinition diff --git a/app/docker/components/log-viewer/logViewer.html b/app/docker/components/log-viewer/logViewer.html index 61b0ccb24..2f3ac1f15 100644 --- a/app/docker/components/log-viewer/logViewer.html +++ b/app/docker/components/log-viewer/logViewer.html @@ -86,7 +86,7 @@ 
-      
{{ span.text }}

+      
{{ span.text }}

       
No log line matching the '{{ $ctrl.state.search }}' filter

       
No logs available
diff --git a/app/docker/components/log-viewer/logViewerController.js b/app/docker/components/log-viewer/logViewerController.js index 4cee6ec92..07b47267e 100644 --- a/app/docker/components/log-viewer/logViewerController.js +++ b/app/docker/components/log-viewer/logViewerController.js @@ -1,6 +1,7 @@ import moment from 'moment'; -import _ from 'lodash-es'; + import { NEW_LINE_BREAKER } from '@/constants'; +import { concatLogsToString } from '@/docker/helpers/logHelper'; angular.module('portainer.docker').controller('LogViewerController', [ '$scope', @@ -74,9 +75,8 @@ angular.module('portainer.docker').controller('LogViewerController', [ }; this.downloadLogs = function () { - // To make the feature of downloading container logs working both on Windows and Linux, - // we need to use correct new line breakers on corresponding OS. - const data = new Blob([_.reduce(this.state.filteredLogs, (acc, log) => acc + log.line + NEW_LINE_BREAKER, '')]); + const logsAsString = concatLogsToString(this.state.filteredLogs); + const data = new Blob([logsAsString]); FileSaver.saveAs(data, this.resourceName + '_logs.txt'); }; }, diff --git a/app/docker/helpers/logHelper.js b/app/docker/helpers/logHelper.js deleted file mode 100644 index d2214b3c2..000000000 --- a/app/docker/helpers/logHelper.js +++ /dev/null @@ -1,223 +0,0 @@ -import tokenize from '@nxmix/tokenize-ansi'; -import x256 from 'x256'; -import { takeRight, without } from 'lodash'; -import { format } from 'date-fns'; - -const FOREGROUND_COLORS_BY_ANSI = { - black: x256.colors[0], - red: x256.colors[1], - green: x256.colors[2], - yellow: x256.colors[3], - blue: x256.colors[4], - magenta: x256.colors[5], - cyan: x256.colors[6], - white: x256.colors[7], - brightBlack: x256.colors[8], - brightRed: x256.colors[9], - brightGreen: x256.colors[10], - brightYellow: x256.colors[11], - brightBlue: x256.colors[12], - brightMagenta: x256.colors[13], - brightCyan: x256.colors[14], - brightWhite: x256.colors[15], -}; - -const BACKGROUND_COLORS_BY_ANSI = { - bgBlack: x256.colors[0], - bgRed: x256.colors[1], - bgGreen: x256.colors[2], - bgYellow: x256.colors[3], - bgBlue: x256.colors[4], - bgMagenta: x256.colors[5], - bgCyan: x256.colors[6], - bgWhite: x256.colors[7], - bgBrightBlack: x256.colors[8], - bgBrightRed: x256.colors[9], - bgBrightGreen: x256.colors[10], - bgBrightYellow: x256.colors[11], - bgBrightBlue: x256.colors[12], - bgBrightMagenta: x256.colors[13], - bgBrightCyan: x256.colors[14], - bgBrightWhite: x256.colors[15], -}; - -const TIMESTAMP_LENGTH = 31; // 30 for timestamp + 1 for trailing space - -angular.module('portainer.docker').factory('LogHelper', [ - function LogHelperFactory() { - 'use strict'; - var helper = {}; - - function stripHeaders(logs) { - logs = logs.substring(8); - logs = logs.replace(/\r?\n(.{8})/g, '\n'); - - return logs; - } - - function stripEscapeCodes(logs) { - return logs.replace(/[\u001b\u009b][[()#;?]*(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-ORZcf-nqry=><]/g, ''); - } - - function cssColorFromRgb(rgb) { - const [r, g, b] = rgb; - - return `rgb(${r}, ${g}, ${b})`; - } - - function extendedColorForToken(token) { - const colorMode = token[1]; - - if (colorMode === 2) { - return cssColorFromRgb(token.slice(2)); - } - - if (colorMode === 5 && x256.colors[token[2]]) { - return cssColorFromRgb(x256.colors[token[2]]); - } - - return ''; - } - - // Return an array with each log including a line and styled spans for each entry. - // If the stripHeaders param is specified, it will strip the 8 first characters of each line. - // withTimestamps param is needed to find the start of JSON for Zerolog logs parsing - helper.formatLogs = function (logs, { stripHeaders: skipHeaders, withTimestamps }) { - if (skipHeaders) { - logs = stripHeaders(logs); - } - - const tokens = tokenize(logs); - const formattedLogs = []; - - let foregroundColor = null; - let backgroundColor = null; - let line = ''; - let spans = []; - - for (const token of tokens) { - const type = token[0]; - - if (FOREGROUND_COLORS_BY_ANSI[type]) { - foregroundColor = cssColorFromRgb(FOREGROUND_COLORS_BY_ANSI[type]); - } else if (type === 'moreColor') { - foregroundColor = extendedColorForToken(token); - } else if (type === 'fgDefault') { - foregroundColor = null; - } else if (BACKGROUND_COLORS_BY_ANSI[type]) { - backgroundColor = cssColorFromRgb(BACKGROUND_COLORS_BY_ANSI[type]); - } else if (type === 'bgMoreColor') { - backgroundColor = extendedColorForToken(token); - } else if (type === 'bgDefault') { - backgroundColor = null; - } else if (type === 'reset') { - foregroundColor = null; - backgroundColor = null; - } else if (type === 'text') { - const tokenLines = token[1].split('\n'); - - for (let i = 0; i < tokenLines.length; i++) { - if (i !== 0) { - formattedLogs.push({ line, spans }); - - line = ''; - spans = []; - } - - const text = stripEscapeCodes(tokenLines[i]); - if ((!withTimestamps && text.startsWith('{')) || (withTimestamps && text.substring(TIMESTAMP_LENGTH).startsWith('{'))) { - line += JSONToFormattedLine(text, spans, withTimestamps); - } else { - spans.push({ foregroundColor, backgroundColor, text }); - line += text; - } - } - } - } - - if (line) { - formattedLogs.push({ line, spans }); - } - - return formattedLogs; - }; - - return helper; - }, -]); - -const JSONColors = { - Grey: 'var(--text-log-viewer-color-json-grey)', - Magenta: 'var(--text-log-viewer-color-json-magenta)', - Yellow: 'var(--text-log-viewer-color-json-yellow)', - Green: 'var(--text-log-viewer-color-json-green)', - Red: 'var(--text-log-viewer-color-json-red)', - Blue: 'var(--text-log-viewer-color-json-blue)', -}; - -const spaceSpan = { text: ' ' }; - -function logLevelToSpan(level) { - switch (level) { - case 'debug': - return { foregroundColor: JSONColors.Grey, text: 'DBG', fontWeight: 'bold' }; - case 'info': - return { foregroundColor: JSONColors.Green, text: 'INF', fontWeight: 'bold' }; - case 'warn': - return { foregroundColor: JSONColors.Yellow, text: 'WRN', fontWeight: 'bold' }; - case 'error': - return { foregroundColor: JSONColors.Red, text: 'ERR', fontWeight: 'bold' }; - default: - return { text: level }; - } -} - -function JSONToFormattedLine(rawText, spans, withTimestamps) { - const text = withTimestamps ? rawText.substring(TIMESTAMP_LENGTH) : rawText; - const json = JSON.parse(text); - const { level, caller, message, time } = json; - let line = ''; - - if (withTimestamps) { - const timestamp = rawText.substring(0, TIMESTAMP_LENGTH); - spans.push({ text: timestamp }); - line += `${timestamp}`; - } - if (time) { - const date = format(new Date(time * 1000), 'Y/MM/dd hh:mmaa'); - spans.push({ foregroundColor: JSONColors.Grey, text: date }, spaceSpan); - line += `${date} `; - } - if (level) { - const levelSpan = logLevelToSpan(level); - spans.push(levelSpan, spaceSpan); - line += `${levelSpan.text} `; - } - if (caller) { - const trimmedCaller = takeRight(caller.split('/'), 2).join('/'); - spans.push({ foregroundColor: JSONColors.Magenta, text: trimmedCaller, fontWeight: 'bold' }, spaceSpan); - spans.push({ foregroundColor: JSONColors.Blue, text: '>' }, spaceSpan); - line += `${trimmedCaller} > `; - } - - const keys = without(Object.keys(json), 'time', 'level', 'caller', 'message'); - if (message) { - spans.push({ foregroundColor: JSONColors.Magenta, text: `${message}` }, spaceSpan); - line += `${message} `; - - if (keys.length) { - spans.push({ foregroundColor: JSONColors.Magenta, text: `|` }, spaceSpan); - line += '| '; - } - } - - keys.forEach((key) => { - const value = json[key]; - spans.push({ foregroundColor: JSONColors.Blue, text: `${key}=` }); - spans.push({ foregroundColor: key === 'error' ? JSONColors.Red : JSONColors.Magenta, text: value }); - spans.push(spaceSpan); - line += `${key}=${value} `; - }); - - return line; -} diff --git a/app/docker/helpers/logHelper/colors/colors.ts b/app/docker/helpers/logHelper/colors/colors.ts new file mode 100644 index 000000000..f9183c2b6 --- /dev/null +++ b/app/docker/helpers/logHelper/colors/colors.ts @@ -0,0 +1,63 @@ +// original code comes from https://www.npmjs.com/package/x256 +// only picking the used parts as there is no type definition +// package is unmaintained and repository doesn't exist anymore + +// colors scraped from +// http://www.calmar.ws/vim/256-xterm-24bit-rgb-color-chart.html +// %s/ *\d\+ \+#\([^ ]\+\)/\1\r/g + +import rawColors from './rawColors.json'; + +export type RGBColor = [number, number, number]; +export type TextColor = string | undefined; + +function hexToRGB(hex: string): RGBColor { + const r = parseInt(hex.slice(0, 2), 16); + const g = parseInt(hex.slice(2, 4), 16); + const b = parseInt(hex.slice(4, 6), 16); + return [r, g, b]; +} + +export const colors = rawColors.map(hexToRGB); + +export const FOREGROUND_COLORS_BY_ANSI: { + [k: string]: RGBColor; +} = { + black: colors[0], + red: colors[1], + green: colors[2], + yellow: colors[3], + blue: colors[4], + magenta: colors[5], + cyan: colors[6], + white: colors[7], + brightBlack: colors[8], + brightRed: colors[9], + brightGreen: colors[10], + brightYellow: colors[11], + brightBlue: colors[12], + brightMagenta: colors[13], + brightCyan: colors[14], + brightWhite: colors[15], +}; + +export const BACKGROUND_COLORS_BY_ANSI: { + [k: string]: RGBColor; +} = { + bgBlack: colors[0], + bgRed: colors[1], + bgGreen: colors[2], + bgYellow: colors[3], + bgBlue: colors[4], + bgMagenta: colors[5], + bgCyan: colors[6], + bgWhite: colors[7], + bgBrightBlack: colors[8], + bgBrightRed: colors[9], + bgBrightGreen: colors[10], + bgBrightYellow: colors[11], + bgBrightBlue: colors[12], + bgBrightMagenta: colors[13], + bgBrightCyan: colors[14], + bgBrightWhite: colors[15], +}; diff --git a/app/docker/helpers/logHelper/colors/index.ts b/app/docker/helpers/logHelper/colors/index.ts new file mode 100644 index 000000000..25eed2280 --- /dev/null +++ b/app/docker/helpers/logHelper/colors/index.ts @@ -0,0 +1,7 @@ +export { + type RGBColor, + type TextColor, + colors, + FOREGROUND_COLORS_BY_ANSI, + BACKGROUND_COLORS_BY_ANSI, +} from './colors'; diff --git a/app/docker/helpers/logHelper/colors/rawColors.json b/app/docker/helpers/logHelper/colors/rawColors.json new file mode 100644 index 000000000..b9bba82d4 --- /dev/null +++ b/app/docker/helpers/logHelper/colors/rawColors.json @@ -0,0 +1,258 @@ +[ + "000000", + "800000", + "008000", + "808000", + "000080", + "800080", + "008080", + "c0c0c0", + "808080", + "ff0000", + "00ff00", + "ffff00", + "0000ff", + "ff00ff", + "00ffff", + "ffffff", + "000000", + "00005f", + "000087", + "0000af", + "0000d7", + "0000ff", + "005f00", + "005f5f", + "005f87", + "005faf", + "005fd7", + "005fff", + "008700", + "00875f", + "008787", + "0087af", + "0087d7", + "0087ff", + "00af00", + "00af5f", + "00af87", + "00afaf", + "00afd7", + "00afff", + "00d700", + "00d75f", + "00d787", + "00d7af", + "00d7d7", + "00d7ff", + "00ff00", + "00ff5f", + "00ff87", + "00ffaf", + "00ffd7", + "00ffff", + "5f0000", + "5f005f", + "5f0087", + "5f00af", + "5f00d7", + "5f00ff", + "5f5f00", + "5f5f5f", + "5f5f87", + "5f5faf", + "5f5fd7", + "5f5fff", + "5f8700", + "5f875f", + "5f8787", + "5f87af", + "5f87d7", + "5f87ff", + "5faf00", + "5faf5f", + "5faf87", + "5fafaf", + "5fafd7", + "5fafff", + "5fd700", + "5fd75f", + "5fd787", + "5fd7af", + "5fd7d7", + "5fd7ff", + "5fff00", + "5fff5f", + "5fff87", + "5fffaf", + "5fffd7", + "5fffff", + "870000", + "87005f", + "870087", + "8700af", + "8700d7", + "8700ff", + "875f00", + "875f5f", + "875f87", + "875faf", + "875fd7", + "875fff", + "878700", + "87875f", + "878787", + "8787af", + "8787d7", + "8787ff", + "87af00", + "87af5f", + "87af87", + "87afaf", + "87afd7", + "87afff", + "87d700", + "87d75f", + "87d787", + "87d7af", + "87d7d7", + "87d7ff", + "87ff00", + "87ff5f", + "87ff87", + "87ffaf", + "87ffd7", + "87ffff", + "af0000", + "af005f", + "af0087", + "af00af", + "af00d7", + "af00ff", + "af5f00", + "af5f5f", + "af5f87", + "af5faf", + "af5fd7", + "af5fff", + "af8700", + "af875f", + "af8787", + "af87af", + "af87d7", + "af87ff", + "afaf00", + "afaf5f", + "afaf87", + "afafaf", + "afafd7", + "afafff", + "afd700", + "afd75f", + "afd787", + "afd7af", + "afd7d7", + "afd7ff", + "afff00", + "afff5f", + "afff87", + "afffaf", + "afffd7", + "afffff", + "d70000", + "d7005f", + "d70087", + "d700af", + "d700d7", + "d700ff", + "d75f00", + "d75f5f", + "d75f87", + "d75faf", + "d75fd7", + "d75fff", + "d78700", + "d7875f", + "d78787", + "d787af", + "d787d7", + "d787ff", + "d7af00", + "d7af5f", + "d7af87", + "d7afaf", + "d7afd7", + "d7afff", + "d7d700", + "d7d75f", + "d7d787", + "d7d7af", + "d7d7d7", + "d7d7ff", + "d7ff00", + "d7ff5f", + "d7ff87", + "d7ffaf", + "d7ffd7", + "d7ffff", + "ff0000", + "ff005f", + "ff0087", + "ff00af", + "ff00d7", + "ff00ff", + "ff5f00", + "ff5f5f", + "ff5f87", + "ff5faf", + "ff5fd7", + "ff5fff", + "ff8700", + "ff875f", + "ff8787", + "ff87af", + "ff87d7", + "ff87ff", + "ffaf00", + "ffaf5f", + "ffaf87", + "ffafaf", + "ffafd7", + "ffafff", + "ffd700", + "ffd75f", + "ffd787", + "ffd7af", + "ffd7d7", + "ffd7ff", + "ffff00", + "ffff5f", + "ffff87", + "ffffaf", + "ffffd7", + "ffffff", + "080808", + "121212", + "1c1c1c", + "262626", + "303030", + "3a3a3a", + "444444", + "4e4e4e", + "585858", + "606060", + "666666", + "767676", + "808080", + "8a8a8a", + "949494", + "9e9e9e", + "a8a8a8", + "b2b2b2", + "bcbcbc", + "c6c6c6", + "d0d0d0", + "dadada", + "e4e4e4", + "eeeeee" +] diff --git a/app/docker/helpers/logHelper/concatLogsToString.ts b/app/docker/helpers/logHelper/concatLogsToString.ts new file mode 100644 index 000000000..b67d4312e --- /dev/null +++ b/app/docker/helpers/logHelper/concatLogsToString.ts @@ -0,0 +1,15 @@ +import { NEW_LINE_BREAKER } from '@/constants'; + +import { FormattedLine } from './types'; + +type FormatFunc = (line: FormattedLine) => string; + +export function concatLogsToString( + logs: FormattedLine[], + formatFunc: FormatFunc = (line) => line.line +) { + return logs.reduce( + (acc, formattedLine) => acc + formatFunc(formattedLine) + NEW_LINE_BREAKER, + '' + ); +} diff --git a/app/docker/helpers/logHelper/formatJSONLogs.ts b/app/docker/helpers/logHelper/formatJSONLogs.ts new file mode 100644 index 000000000..5a08cc360 --- /dev/null +++ b/app/docker/helpers/logHelper/formatJSONLogs.ts @@ -0,0 +1,55 @@ +import { without } from 'lodash'; + +import { FormattedLine, Span, JSONLogs, TIMESTAMP_LENGTH } from './types'; +import { + formatCaller, + formatKeyValuePair, + formatLevel, + formatMessage, + formatStackTrace, + formatTime, +} from './formatters'; + +function removeKnownKeys(keys: string[]) { + return without(keys, 'time', 'level', 'caller', 'message', 'stack_trace'); +} + +export function formatJSONLine( + rawText: string, + withTimestamps?: boolean +): FormattedLine[] { + const spans: Span[] = []; + const lines: FormattedLine[] = []; + let line = ''; + + const text = withTimestamps ? rawText.substring(TIMESTAMP_LENGTH) : rawText; + + const json: JSONLogs = JSON.parse(text); + const { time, level, caller, message, stack_trace: stackTrace } = json; + const keys = removeKnownKeys(Object.keys(json)); + + if (withTimestamps) { + const timestamp = rawText.substring(0, TIMESTAMP_LENGTH); + spans.push({ text: timestamp }); + line += `${timestamp}`; + } + line += formatTime(time, spans, line); + line += formatLevel(level, spans, line); + line += formatCaller(caller, spans, line); + line += formatMessage(message, spans, line, !!keys.length); + + keys.forEach((key, idx) => { + line += formatKeyValuePair( + key, + json[key], + spans, + line, + idx === keys.length - 1 + ); + }); + + lines.push({ line, spans }); + formatStackTrace(stackTrace, lines); + + return lines; +} diff --git a/app/docker/helpers/logHelper/formatLogs.ts b/app/docker/helpers/logHelper/formatLogs.ts new file mode 100644 index 000000000..e0af64375 --- /dev/null +++ b/app/docker/helpers/logHelper/formatLogs.ts @@ -0,0 +1,141 @@ +import tokenize from '@nxmix/tokenize-ansi'; +import { FontWeight } from 'xterm'; + +import { + colors, + BACKGROUND_COLORS_BY_ANSI, + FOREGROUND_COLORS_BY_ANSI, + RGBColor, +} from './colors'; +import { formatJSONLine } from './formatJSONLogs'; +import { formatZerologLogs, ZerologRegex } from './formatZerologLogs'; +import { Token, Span, TIMESTAMP_LENGTH, FormattedLine } from './types'; + +type FormatOptions = { + stripHeaders?: boolean; + withTimestamps?: boolean; + splitter?: string; +}; + +const defaultOptions: FormatOptions = { + splitter: '\n', +}; + +export function formatLogs( + rawLogs: string, + { + stripHeaders, + withTimestamps, + splitter = '\n', + }: FormatOptions = defaultOptions +) { + let logs = rawLogs; + if (stripHeaders) { + logs = stripHeadersFunc(logs); + } + if (logs.includes('\\n')) { + logs = JSON.parse(logs); + } + + const tokens: Token[][] = tokenize(logs); + const formattedLogs: FormattedLine[] = []; + + let fgColor: string | undefined; + let bgColor: string | undefined; + let fontWeight: FontWeight | undefined; + let line = ''; + let spans: Span[] = []; + + tokens.forEach((token) => { + const [type] = token; + + const fgAnsi = FOREGROUND_COLORS_BY_ANSI[type]; + const bgAnsi = BACKGROUND_COLORS_BY_ANSI[type]; + + if (fgAnsi) { + fgColor = cssColorFromRgb(fgAnsi); + } else if (type === 'moreColor') { + fgColor = extendedColorForToken(token); + } else if (type === 'fgDefault') { + fgColor = undefined; + } else if (bgAnsi) { + bgColor = cssColorFromRgb(bgAnsi); + } else if (type === 'bgMoreColor') { + bgColor = extendedColorForToken(token); + } else if (type === 'bgDefault') { + bgColor = undefined; + } else if (type === 'reset') { + fgColor = undefined; + bgColor = undefined; + fontWeight = undefined; + } else if (type === 'bold') { + fontWeight = 'bold'; + } else if (type === 'normal') { + fontWeight = 'normal'; + } else if (type === 'text') { + const tokenLines = (token[1] as string).split(splitter); + + tokenLines.forEach((tokenLine, idx) => { + if (idx && line) { + formattedLogs.push({ line, spans }); + line = ''; + spans = []; + } + + const text = stripEscapeCodes(tokenLine); + if ( + (!withTimestamps && text.startsWith('{')) || + (withTimestamps && text.substring(TIMESTAMP_LENGTH).startsWith('{')) + ) { + const lines = formatJSONLine(text, withTimestamps); + formattedLogs.push(...lines); + } else if (ZerologRegex.test(text)) { + const lines = formatZerologLogs(text, withTimestamps); + formattedLogs.push(...lines); + } else { + spans.push({ fgColor, bgColor, text, fontWeight }); + line += text; + } + }); + } + }); + if (line) { + formattedLogs.push({ line, spans }); + } + + return formattedLogs; +} + +function stripHeadersFunc(logs: string) { + return logs.substring(8).replace(/\r?\n(.{8})/g, '\n'); +} + +function stripEscapeCodes(logs: string) { + return logs.replace( + // eslint-disable-next-line no-control-regex + /[\u001b\u009b][[()#;?]*(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-ORZcf-nqry=><]/g, + '' + ); +} + +function cssColorFromRgb(rgb: RGBColor) { + const [r, g, b] = rgb; + + return `rgb(${r}, ${g}, ${b})`; +} + +// assuming types based on original JS implementation +// there is not much type definitions for the tokenize library +function extendedColorForToken(token: Token[]) { + const [, colorMode, colorRef] = token as [undefined, number, number]; + + if (colorMode === 2) { + return cssColorFromRgb(token.slice(2) as RGBColor); + } + + if (colorMode === 5 && colors[colorRef]) { + return cssColorFromRgb(colors[colorRef]); + } + + return ''; +} diff --git a/app/docker/helpers/logHelper/formatZerologLogs.ts b/app/docker/helpers/logHelper/formatZerologLogs.ts new file mode 100644 index 000000000..e983e3141 --- /dev/null +++ b/app/docker/helpers/logHelper/formatZerologLogs.ts @@ -0,0 +1,119 @@ +import { + formatCaller, + formatKeyValuePair, + formatLevel, + formatMessage, + formatStackTrace, + formatTime, +} from './formatters'; +import { + FormattedLine, + JSONStackTrace, + Level, + Span, + TIMESTAMP_LENGTH, +} from './types'; + +const dateRegex = /(\d{4}\/\d{2}\/\d{2} \d{2}:\d{2}[AP]M) /; // "2022/02/01 04:30AM " +const levelRegex = /(\w{3}) /; // "INF " or "ERR " +const callerRegex = /(.+?.go:\d+) /; // "path/to/file.go:line " +const chevRegex = /> /; // "> " +const messageAndPairsRegex = /(.*)/; // include the rest of the string in a separate group + +const keyRegex = /(\S+=)/g; // "" + +export const ZerologRegex = concatRegex( + dateRegex, + levelRegex, + callerRegex, + chevRegex, + messageAndPairsRegex +); + +function concatRegex(...regs: RegExp[]) { + const flags = Array.from( + new Set( + regs + .map((r) => r.flags) + .join('') + .split('') + ) + ).join(''); + const source = regs.map((r) => r.source).join(''); + return new RegExp(source, flags); +} + +type Pair = { + key: string; + value: string; +}; + +export function formatZerologLogs(rawText: string, withTimestamps?: boolean) { + const spans: Span[] = []; + const lines: FormattedLine[] = []; + let line = ''; + + const text = withTimestamps ? rawText.substring(TIMESTAMP_LENGTH) : rawText; + + const [, date, level, caller, messageAndPairs] = + text.match(ZerologRegex) || []; + + const [message, pairs] = extractPairs(messageAndPairs); + + line += formatTime(date, spans, line); + line += formatLevel(level as Level, spans, line); + line += formatCaller(caller, spans, line); + line += formatMessage(message, spans, line, !!pairs.length); + + let stackTrace: JSONStackTrace | undefined; + const stackTraceIndex = pairs.findIndex((p) => p.key === 'stack_trace'); + + if (stackTraceIndex !== -1) { + stackTrace = JSON.parse(pairs[stackTraceIndex].value); + pairs.splice(stackTraceIndex); + } + + pairs.forEach(({ key, value }, idx) => { + line += formatKeyValuePair( + key, + value, + spans, + line, + idx === pairs.length - 1 + ); + }); + lines.push({ line, spans }); + + formatStackTrace(stackTrace, lines); + + return lines; +} + +function extractPairs(messageAndPairs: string): [string, Pair[]] { + const pairs: Pair[] = []; + let [message, rawPairs] = messageAndPairs.split('|'); + + if (!messageAndPairs.includes('|') && !rawPairs) { + rawPairs = message; + message = ''; + } + message = message.trim(); + rawPairs = rawPairs.trim(); + + const matches = [...rawPairs.matchAll(keyRegex)]; + + matches.forEach((m, idx) => { + const rawKey = m[0]; + const key = rawKey.slice(0, -1); + const start = m.index || 0; + const end = idx !== matches.length - 1 ? matches[idx + 1].index : undefined; + const value = ( + end + ? rawPairs.slice(start + rawKey.length, end) + : rawPairs.slice(start + rawKey.length) + ).trim(); + pairs.push({ key, value }); + }); + + return [message, pairs]; +} diff --git a/app/docker/helpers/logHelper/formatters.ts b/app/docker/helpers/logHelper/formatters.ts new file mode 100644 index 000000000..10c5b284f --- /dev/null +++ b/app/docker/helpers/logHelper/formatters.ts @@ -0,0 +1,154 @@ +import { format } from 'date-fns'; +import { takeRight } from 'lodash'; + +import { Span, Level, Colors, JSONStackTrace, FormattedLine } from './types'; + +const spaceSpan: Span = { text: ' ' }; + +function logLevelToSpan(level: Level): Span { + switch (level) { + case 'debug': + case 'DBG': + return { + fgColor: Colors.Grey, + text: 'DBG', + fontWeight: 'bold', + }; + case 'info': + case 'INF': + return { + fgColor: Colors.Green, + text: 'INF', + fontWeight: 'bold', + }; + case 'warn': + case 'WRN': + return { + fgColor: Colors.Yellow, + text: 'WRN', + fontWeight: 'bold', + }; + case 'error': + case 'ERR': + return { + fgColor: Colors.Red, + text: 'ERR', + fontWeight: 'bold', + }; + default: + return { text: level }; + } +} + +export function formatTime( + time: number | string | undefined, + spans: Span[], + line: string +) { + let nl = line; + if (time) { + let date = ''; + if (typeof time === 'number') { + date = format(new Date(time * 1000), 'Y/MM/dd hh:mmaa'); + } else { + date = time; + } + spans.push({ fgColor: Colors.Grey, text: date }, spaceSpan); + nl += `${date} `; + } + return nl; +} + +export function formatLevel( + level: Level | undefined, + spans: Span[], + line: string +) { + let nl = line; + if (level) { + const levelSpan = logLevelToSpan(level); + spans.push(levelSpan, spaceSpan); + nl += `${levelSpan.text} `; + } + return nl; +} + +export function formatCaller( + caller: string | undefined, + spans: Span[], + line: string +) { + let nl = line; + if (caller) { + const trim = takeRight(caller.split('/'), 2).join('/'); + spans.push( + { fgColor: Colors.Magenta, text: trim, fontWeight: 'bold' }, + spaceSpan + ); + spans.push({ fgColor: Colors.Blue, text: '>' }, spaceSpan); + nl += `${trim} > `; + } + return nl; +} + +export function formatMessage( + message: string, + spans: Span[], + line: string, + hasKeys: boolean +) { + let nl = line; + if (message) { + spans.push({ fgColor: Colors.Magenta, text: `${message}` }, spaceSpan); + nl += `${message} `; + + if (hasKeys) { + spans.push({ fgColor: Colors.Magenta, text: `|` }, spaceSpan); + nl += '| '; + } + } + return nl; +} + +export function formatKeyValuePair( + key: string, + value: unknown, + spans: Span[], + line: string, + isLastKey: boolean +) { + let nl = line; + + spans.push( + { fgColor: Colors.Blue, text: `${key}=` }, + { + fgColor: key === 'error' || key === 'ERR' ? Colors.Red : Colors.Magenta, + text: value as string, + } + ); + if (!isLastKey) spans.push(spaceSpan); + nl += `${key}=${value}${!isLastKey ? ' ' : ''}`; + + return nl; +} + +export function formatStackTrace( + stackTrace: JSONStackTrace | undefined, + lines: FormattedLine[] +) { + if (stackTrace) { + stackTrace.forEach(({ func, line: lineNumber, source }) => { + const line = ` at ${func} (${source}:${lineNumber})`; + const spans: Span[] = [ + spaceSpan, + spaceSpan, + spaceSpan, + spaceSpan, + { text: 'at ', fgColor: Colors.Grey }, + { text: func, fgColor: Colors.Red }, + { text: `(${source}:${lineNumber})`, fgColor: Colors.Grey }, + ]; + lines.push({ line, spans }); + }); + } +} diff --git a/app/docker/helpers/logHelper/index.ts b/app/docker/helpers/logHelper/index.ts new file mode 100644 index 000000000..91fe97ed7 --- /dev/null +++ b/app/docker/helpers/logHelper/index.ts @@ -0,0 +1,2 @@ +export { formatLogs } from './formatLogs'; +export { concatLogsToString } from './concatLogsToString'; diff --git a/app/docker/helpers/logHelper/types.ts b/app/docker/helpers/logHelper/types.ts new file mode 100644 index 000000000..7d467e296 --- /dev/null +++ b/app/docker/helpers/logHelper/types.ts @@ -0,0 +1,53 @@ +import { FontWeight } from 'xterm'; + +import { type TextColor } from './colors'; + +export type Token = string | number; + +export type Level = + | 'debug' + | 'info' + | 'warn' + | 'error' + | 'DBG' + | 'INF' + | 'WRN' + | 'ERR'; + +export type JSONStackTrace = { + func: string; + line: string; + source: string; +}[]; + +export type JSONLogs = { + [k: string]: unknown; + time: number; + level: Level; + caller: string; + message: string; + stack_trace?: JSONStackTrace; +}; + +export type Span = { + fgColor?: TextColor; + bgColor?: TextColor; + text: string; + fontWeight?: FontWeight; +}; + +export type FormattedLine = { + spans: Span[]; + line: string; +}; + +export const TIMESTAMP_LENGTH = 31; // 30 for timestamp + 1 for trailing space + +export const Colors = { + Grey: 'var(--text-log-viewer-color-json-grey)', + Magenta: 'var(--text-log-viewer-color-json-magenta)', + Yellow: 'var(--text-log-viewer-color-json-yellow)', + Green: 'var(--text-log-viewer-color-json-green)', + Red: 'var(--text-log-viewer-color-json-red)', + Blue: 'var(--text-log-viewer-color-json-blue)', +}; diff --git a/app/docker/services/containerService.js b/app/docker/services/containerService.js index fc1514b6f..175720b09 100644 --- a/app/docker/services/containerService.js +++ b/app/docker/services/containerService.js @@ -10,11 +10,12 @@ import { stopContainer, } from '@/react/docker/containers/containers.service'; import { ContainerDetailsViewModel, ContainerStatsViewModel, ContainerViewModel } from '../models/container'; +import { formatLogs } from '../helpers/logHelper'; angular.module('portainer.docker').factory('ContainerService', ContainerServiceFactory); /* @ngInject */ -function ContainerServiceFactory($q, Container, LogHelper, $timeout, EndpointProvider) { +function ContainerServiceFactory($q, Container, $timeout, EndpointProvider) { const service = { killContainer: withEndpointId(killContainer), pauseContainer: withEndpointId(pauseContainer), @@ -159,7 +160,7 @@ function ContainerServiceFactory($q, Container, LogHelper, $timeout, EndpointPro Container.logs(parameters) .$promise.then(function success(data) { - var logs = LogHelper.formatLogs(data.logs, { stripHeaders, withTimestamps: !!timestamps }); + var logs = formatLogs(data.logs, { stripHeaders, withTimestamps: !!timestamps }); deferred.resolve(logs); }) .catch(function error(err) { diff --git a/app/docker/services/serviceService.js b/app/docker/services/serviceService.js index cab03024b..749bc91c8 100644 --- a/app/docker/services/serviceService.js +++ b/app/docker/services/serviceService.js @@ -1,13 +1,10 @@ +import { formatLogs } from '../helpers/logHelper'; import { ServiceViewModel } from '../models/service'; angular.module('portainer.docker').factory('ServiceService', [ '$q', 'Service', - 'ServiceHelper', - 'TaskService', - 'ResourceControlService', - 'LogHelper', - function ServiceServiceFactory($q, Service, ServiceHelper, TaskService, ResourceControlService, LogHelper) { + function ServiceServiceFactory($q, Service) { 'use strict'; var service = {}; @@ -88,7 +85,7 @@ angular.module('portainer.docker').factory('ServiceService', [ Service.logs(parameters) .$promise.then(function success(data) { - var logs = LogHelper.formatLogs(data.logs, { stripHeaders: true, withTimestamps: !!timestamps }); + var logs = formatLogs(data.logs, { stripHeaders: true, withTimestamps: !!timestamps }); deferred.resolve(logs); }) .catch(function error(err) { diff --git a/app/docker/services/taskService.js b/app/docker/services/taskService.js index 5c38a316f..0f68183a0 100644 --- a/app/docker/services/taskService.js +++ b/app/docker/services/taskService.js @@ -1,10 +1,10 @@ +import { formatLogs } from '../helpers/logHelper'; import { TaskViewModel } from '../models/task'; angular.module('portainer.docker').factory('TaskService', [ '$q', 'Task', - 'LogHelper', - function TaskServiceFactory($q, Task, LogHelper) { + function TaskServiceFactory($q, Task) { 'use strict'; var service = {}; @@ -54,7 +54,7 @@ angular.module('portainer.docker').factory('TaskService', [ Task.logs(parameters) .$promise.then(function success(data) { - var logs = LogHelper.formatLogs(data.logs, { stripHeaders: true, withTimestamps: !!timestamps }); + var logs = formatLogs(data.logs, { stripHeaders: true, withTimestamps: !!timestamps }); deferred.resolve(logs); }) .catch(function error(err) { diff --git a/app/kubernetes/pod/service.js b/app/kubernetes/pod/service.js index 925b09797..45d735700 100644 --- a/app/kubernetes/pod/service.js +++ b/app/kubernetes/pod/service.js @@ -67,7 +67,7 @@ class KubernetesPodService { params.container = containerName; } const data = await this.KubernetesPods(namespace).logs(params).$promise; - return data.logs.length === 0 ? [] : data.logs.split('\n'); + return data.logs; } catch (err) { throw new PortainerError('Unable to retrieve pod logs', err); } diff --git a/app/kubernetes/views/applications/logs/logs.html b/app/kubernetes/views/applications/logs/logs.html index 833f1670c..42f35524c 100644 --- a/app/kubernetes/views/applications/logs/logs.html +++ b/app/kubernetes/views/applications/logs/logs.html @@ -77,9 +77,10 @@
- 
{{ line }}
No log line matching the '{{ ctrl.state.search }}' filter
No logs available
+ 
+        
{{ span.text }}

+        
No log line matching the '{{ ctrl.state.search }}' filter

+        
No logs available
diff --git a/app/kubernetes/views/applications/logs/logsController.js b/app/kubernetes/views/applications/logs/logsController.js index a72e6acb1..66601d98b 100644 --- a/app/kubernetes/views/applications/logs/logsController.js +++ b/app/kubernetes/views/applications/logs/logsController.js @@ -1,5 +1,6 @@ import angular from 'angular'; -import _ from 'lodash-es'; + +import { concatLogsToString, formatLogs } from '@/docker/helpers/logHelper'; class KubernetesApplicationLogsController { /* @ngInject */ @@ -39,13 +40,15 @@ class KubernetesApplicationLogsController { } downloadLogs() { - const data = new this.Blob([_.reduce(this.applicationLogs, (acc, log) => acc + '\n' + log, '')]); + const logsAsString = concatLogsToString(this.applicationLogs); + const data = new this.Blob([logsAsString]); this.FileSaver.saveAs(data, this.podName + '_logs.txt'); } async getApplicationLogsAsync() { try { - this.applicationLogs = await this.KubernetesPodService.logs(this.application.ResourcePool, this.podName, this.containerName); + const rawLogs = await this.KubernetesPodService.logs(this.application.ResourcePool, this.podName, this.containerName); + this.applicationLogs = formatLogs(rawLogs); } catch (err) { this.stopRepeater(); this.Notifications.error('Failure', err, 'Unable to retrieve application logs'); @@ -70,13 +73,8 @@ class KubernetesApplicationLogsController { this.containerName = containerName; try { - const [application, applicationLogs] = await Promise.all([ - this.KubernetesApplicationService.get(namespace, applicationName), - this.KubernetesPodService.logs(namespace, podName, containerName), - ]); - - this.application = application; - this.applicationLogs = applicationLogs; + this.application = await this.KubernetesApplicationService.get(namespace, applicationName); + await this.getApplicationLogsAsync(); } catch (err) { this.Notifications.error('Failure', err, 'Unable to retrieve application logs'); } finally { diff --git a/app/kubernetes/views/stacks/logs/logs.html b/app/kubernetes/views/stacks/logs/logs.html index 5ec715af5..3847c5a96 100644 --- a/app/kubernetes/views/stacks/logs/logs.html +++ b/app/kubernetes/views/stacks/logs/logs.html @@ -69,9 +69,11 @@ ctrl.state.transition.name,
- 
{{ line.AppName }} {{ line.Line }}
No log line matching the '{{ ctrl.state.search }}' filter
No logs available
+ 
+        
{{ log.appName }} {{ span.text }}

+        
No log line matching the '{{ ctrl.state.search }}' filter

+        
No logs available

+
diff --git a/app/kubernetes/views/stacks/logs/logsController.js b/app/kubernetes/views/stacks/logs/logsController.js index 547ba9e6c..536ea2ae4 100644 --- a/app/kubernetes/views/stacks/logs/logsController.js +++ b/app/kubernetes/views/stacks/logs/logsController.js @@ -1,6 +1,7 @@ -import _ from 'lodash-es'; +import { filter, flatMap, map } from 'lodash'; import angular from 'angular'; import $allSettled from 'Portainer/services/allSettled'; +import { concatLogsToString, formatLogs } from '@/docker/helpers/logHelper'; const colors = ['red', 'orange', 'lime', 'green', 'darkgreen', 'cyan', 'turquoise', 'teal', 'deepskyblue', 'blue', 'darkblue', 'slateblue', 'magenta', 'darkviolet']; @@ -58,7 +59,7 @@ class KubernetesStackLogsController { Pods: [], }; - const promises = _.flatMap(_.map(app.Pods, (pod) => _.map(pod.Containers, (container) => this.generateLogsPromise(pod, container)))); + const promises = flatMap(map(app.Pods, (pod) => map(pod.Containers, (container) => this.generateLogsPromise(pod, container)))); const result = await $allSettled(promises); res.Pods = result.fulfilled; return res; @@ -67,21 +68,12 @@ class KubernetesStackLogsController { async getStackLogsAsync() { try { const applications = await this.KubernetesApplicationService.get(this.state.transition.namespace); - const filteredApplications = _.filter(applications, (app) => app.StackName === this.state.transition.name); - const logsPromises = _.map(filteredApplications, this.generateAppPromise); + const filteredApplications = filter(applications, (app) => app.StackName === this.state.transition.name); + const logsPromises = map(filteredApplications, this.generateAppPromise); const data = await Promise.all(logsPromises); - const logs = _.flatMap(data, (app, index) => { - return _.flatMap(app.Pods, (pod) => { - return _.map(pod.Logs, (line) => { - const res = { - Color: colors[index % colors.length], - Line: line, - AppName: pod.Pod.Name, - }; - return res; - }); - }); - }); + const logs = flatMap(data, (app, index) => + flatMap(app.Pods, (pod) => formatLogs(pod.Logs).map((line) => ({ ...line, appColor: colors[index % colors.length], appName: pod.Pod.Name }))) + ); this.stackLogs = logs; } catch (err) { this.stopRepeater(); @@ -90,7 +82,8 @@ class KubernetesStackLogsController { } downloadLogs() { - const data = new this.Blob([(this.dataLogs = _.reduce(this.stackLogs, (acc, log) => acc + '\n' + log.AppName + ' ' + log.Line, ''))]); + const logsAsString = concatLogsToString(this.state.filteredLogs, (line) => `${line.appName} ${line.line}`); + const data = new this.Blob([logsAsString]); this.FileSaver.saveAs(data, this.state.transition.name + '_logs.txt'); } diff --git a/package.json b/package.json index 113234da6..244cdfa6d 100644 --- a/package.json +++ b/package.json @@ -140,7 +140,6 @@ "strip-ansi": "^6.0.0", "toastr": "^2.1.4", "uuid": "^3.3.2", - "x256": "^0.0.2", "xterm": "^3.8.0", "yaml": "^1.10.2", "yup": "^0.32.11", diff --git a/yarn.lock b/yarn.lock index 3854e115f..022e55f14 100644 --- a/yarn.lock +++ b/yarn.lock @@ -18981,11 +18981,6 @@ x-default-browser@^0.4.0: optionalDependencies: default-browser-id "^1.0.4" -x256@^0.0.2: - version "0.0.2" - resolved "https://registry.yarnpkg.com/x256/-/x256-0.0.2.tgz#c9af18876f7a175801d564fe70ad9e8317784934" - integrity sha1-ya8Yh296F1gB1WT+cK2egxd4STQ= - xml-name-validator@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a" From 7624ff10ee7f6728dcd1a26b42374252dbf050d8 Mon Sep 17 00:00:00 2001 From: Chaim Lev-Ari Date: Fri, 21 Oct 2022 08:22:49 +0300 Subject: [PATCH 20/31] chore(edge): add aria-label for edge-group selector [EE-4466] (#7896) * chore(edge): add aria-label for edge-group selector * style(edge): remove comment --- app/react/edge/components/EdgeGroupsSelector.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/app/react/edge/components/EdgeGroupsSelector.tsx b/app/react/edge/components/EdgeGroupsSelector.tsx index 8a8343d14..9e7a47820 100644 --- a/app/react/edge/components/EdgeGroupsSelector.tsx +++ b/app/react/edge/components/EdgeGroupsSelector.tsx @@ -19,6 +19,7 @@ export function EdgeGroupsSelector({ items, value, onChange }: Props) { return ( ); } diff --git a/app/react/portainer/users/teams/ListView/CreateTeamForm/CreateTeamForm.tsx b/app/react/portainer/users/teams/ListView/CreateTeamForm/CreateTeamForm.tsx index 662ad32d5..a13619ce5 100644 --- a/app/react/portainer/users/teams/ListView/CreateTeamForm/CreateTeamForm.tsx +++ b/app/react/portainer/users/teams/ListView/CreateTeamForm/CreateTeamForm.tsx @@ -5,12 +5,14 @@ import { useReducer } from 'react'; import { Icon } from '@/react/components/Icon'; import { User } from '@/portainer/users/types'; import { notifySuccess } from '@/portainer/services/notifications'; +import { usePublicSettings } from '@/react/portainer/settings/queries'; import { FormControl } from '@@/form-components/FormControl'; import { Widget } from '@@/Widget'; import { Input } from '@@/form-components/Input'; import { UsersSelector } from '@@/UsersSelector'; import { LoadingButton } from '@@/buttons/LoadingButton'; +import { TextTip } from '@@/Tip/TextTip'; import { createTeam } from '../../teams.service'; import { Team } from '../../types'; @@ -26,6 +28,9 @@ interface Props { export function CreateTeamForm({ users, teams }: Props) { const addTeamMutation = useAddTeamMutation(); const [formKey, incFormKey] = useReducer((state: number) => state + 1, 0); + const teamSyncQuery = usePublicSettings({ + select: (settings) => settings.TeamSync, + }); const initialValues = { name: '', @@ -95,10 +100,22 @@ export function CreateTeamForm({ users, teams }: Props) { dataCy="team-teamLeaderSelect" inputId="users-input" placeholder="Select one or more team leaders" + disabled={teamSyncQuery.data} /> )} + {teamSyncQuery.data && ( +
+
+ + The team leader feature is disabled as external + authentication is currently enabled with team sync. + +
+
+ )} +
Date: Tue, 25 Oct 2022 15:02:59 +0800 Subject: [PATCH 26/31] fix(image): hide button issues [EE-4166] (#7845) * fix(image): hide button issues [EE-4166] --- .../BEFeatureIndicator/BEFeatureIndicator.html | 2 +- .../registries-datatable/registriesDatatable.html | 2 +- .../DefaultRegistry/DefaultRegistryAction.tsx | 13 +++++-------- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/app/portainer/components/BEFeatureIndicator/BEFeatureIndicator.html b/app/portainer/components/BEFeatureIndicator/BEFeatureIndicator.html index ce3114a64..e68b4a4f6 100644 --- a/app/portainer/components/BEFeatureIndicator/BEFeatureIndicator.html +++ b/app/portainer/components/BEFeatureIndicator/BEFeatureIndicator.html @@ -1,4 +1,4 @@ - + Business Edition Feature diff --git a/app/portainer/components/datatables/registries-datatable/registriesDatatable.html b/app/portainer/components/datatables/registries-datatable/registriesDatatable.html index b76479f6a..8683d2780 100644 --- a/app/portainer/components/datatables/registries-datatable/registriesDatatable.html +++ b/app/portainer/components/datatables/registries-datatable/registriesDatatable.html @@ -110,7 +110,7 @@ Manage access - Browse + Browse - diff --git a/app/react/portainer/registries/ListView/DefaultRegistry/DefaultRegistryAction.tsx b/app/react/portainer/registries/ListView/DefaultRegistry/DefaultRegistryAction.tsx index 3ccc52d59..8f76b49ee 100644 --- a/app/react/portainer/registries/ListView/DefaultRegistry/DefaultRegistryAction.tsx +++ b/app/react/portainer/registries/ListView/DefaultRegistry/DefaultRegistryAction.tsx @@ -1,3 +1,5 @@ +import { Eye, EyeOff } from 'react-feather'; + import { notifySuccess } from '@/portainer/services/notifications'; import { FeatureId } from '@/portainer/feature-flags/enums'; import { isLimitedToBE } from '@/portainer/feature-flags/feature-flags.service'; @@ -8,7 +10,6 @@ import { import { Tooltip } from '@@/Tip/Tooltip'; import { Button } from '@@/buttons'; -import { Icon } from '@@/Icon'; import { BEFeatureIndicator } from '@@/BEFeatureIndicator'; export function DefaultRegistryAction() { @@ -29,11 +30,11 @@ export function DefaultRegistryAction() { {!hideDefaultRegistry ? (
@@ -46,11 +47,7 @@ export function DefaultRegistryAction() {
) : (
- Date: Thu, 27 Oct 2022 16:14:54 +1300 Subject: [PATCH 27/31] fix showing namespaces for standard user (#7917) --- api/http/handler/kubernetes/handler.go | 1 + api/http/handler/kubernetes/namespaces.go | 37 ++++++++++++++++++++++ api/kubernetes/cli/namespace.go | 15 +++++++++ api/portainer.go | 1 + app/react/kubernetes/namespaces/queries.ts | 18 ++++++++++- app/react/kubernetes/namespaces/service.ts | 12 +++---- 6 files changed, 77 insertions(+), 7 deletions(-) diff --git a/api/http/handler/kubernetes/handler.go b/api/http/handler/kubernetes/handler.go index 7d46c00d3..f3f1a688b 100644 --- a/api/http/handler/kubernetes/handler.go +++ b/api/http/handler/kubernetes/handler.go @@ -62,6 +62,7 @@ func NewHandler(bouncer *security.RequestBouncer, authorizationService *authoriz endpointRouter.Path("/namespaces").Handler(httperror.LoggerHandler(h.updateKubernetesNamespace)).Methods(http.MethodPut) endpointRouter.Path("/namespaces").Handler(httperror.LoggerHandler(h.getKubernetesNamespaces)).Methods(http.MethodGet) endpointRouter.Path("/namespace/{namespace}").Handler(httperror.LoggerHandler(h.deleteKubernetesNamespaces)).Methods(http.MethodDelete) + endpointRouter.Path("/namespaces/{namespace}").Handler(httperror.LoggerHandler(h.getKubernetesNamespace)).Methods(http.MethodGet) // namespaces // in the future this piece of code might be in another package (or a few different packages - namespaces/namespace?) diff --git a/api/http/handler/kubernetes/namespaces.go b/api/http/handler/kubernetes/namespaces.go index 68ba70261..d2e4d0c17 100644 --- a/api/http/handler/kubernetes/namespaces.go +++ b/api/http/handler/kubernetes/namespaces.go @@ -40,6 +40,43 @@ func (handler *Handler) getKubernetesNamespaces(w http.ResponseWriter, r *http.R return response.JSON(w, namespaces) } +func (handler *Handler) getKubernetesNamespace(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { + endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id") + if err != nil { + return httperror.BadRequest( + "Invalid environment identifier route variable", + err, + ) + } + + cli, ok := handler.KubernetesClientFactory.GetProxyKubeClient( + strconv.Itoa(endpointID), r.Header.Get("Authorization"), + ) + if !ok { + return httperror.InternalServerError( + "Failed to lookup KubeClient", + nil, + ) + } + + ns, err := request.RetrieveRouteVariableValue(r, "namespace") + if err != nil { + return httperror.BadRequest( + "Invalid namespace identifier route variable", + err, + ) + } + namespace, err := cli.GetNamespace(ns) + if err != nil { + return httperror.InternalServerError( + "Unable to retrieve namespace", + err, + ) + } + + return response.JSON(w, namespace) +} + func (handler *Handler) createKubernetesNamespace(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id") if err != nil { diff --git a/api/kubernetes/cli/namespace.go b/api/kubernetes/cli/namespace.go index 18701822d..0baacf09a 100644 --- a/api/kubernetes/cli/namespace.go +++ b/api/kubernetes/cli/namespace.go @@ -44,6 +44,21 @@ func (kcl *KubeClient) GetNamespaces() (map[string]portainer.K8sNamespaceInfo, e return results, nil } +// GetNamespace gets the namespace in the current k8s environment(endpoint). +func (kcl *KubeClient) GetNamespace(name string) (portainer.K8sNamespaceInfo, error) { + namespace, err := kcl.cli.CoreV1().Namespaces().Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + return portainer.K8sNamespaceInfo{}, err + } + + result := portainer.K8sNamespaceInfo{ + IsSystem: isSystemNamespace(*namespace), + IsDefault: namespace.Name == defaultNamespace, + } + + return result, nil +} + // CreateIngress creates a new ingress in a given namespace in a k8s endpoint. func (kcl *KubeClient) CreateNamespace(info models.K8sNamespaceDetails) error { client := kcl.cli.CoreV1().Namespaces() diff --git a/api/portainer.go b/api/portainer.go index 4fbd3d61c..3afa7322a 100644 --- a/api/portainer.go +++ b/api/portainer.go @@ -1357,6 +1357,7 @@ type ( CreateNamespace(info models.K8sNamespaceDetails) error UpdateNamespace(info models.K8sNamespaceDetails) error GetNamespaces() (map[string]K8sNamespaceInfo, error) + GetNamespace(string) (K8sNamespaceInfo, error) DeleteNamespace(namespace string) error GetConfigMapsAndSecrets(namespace string) ([]models.K8sConfigMapOrSecret, error) GetIngressControllers() (models.K8sIngressControllers, error) diff --git a/app/react/kubernetes/namespaces/queries.ts b/app/react/kubernetes/namespaces/queries.ts index 00d1b65cd..f79a6c555 100644 --- a/app/react/kubernetes/namespaces/queries.ts +++ b/app/react/kubernetes/namespaces/queries.ts @@ -2,13 +2,29 @@ import { useQuery } from 'react-query'; import { EnvironmentId } from '@/react/portainer/environments/types'; import { error as notifyError } from '@/portainer/services/notifications'; +import { getIngresses } from '@/kubernetes/react/views/networks/ingresses/service'; import { getNamespaces, getNamespace } from './service'; +import { Namespaces } from './types'; export function useNamespaces(environmentId: EnvironmentId) { return useQuery( ['environments', environmentId, 'kubernetes', 'namespaces'], - () => getNamespaces(environmentId), + async () => { + const namespaces = await getNamespaces(environmentId); + const settledNamespacesPromise = await Promise.allSettled( + Object.keys(namespaces).map((namespace) => + getIngresses(environmentId, namespace).then(() => namespace) + ) + ); + const ns: Namespaces = {}; + settledNamespacesPromise.forEach((namespace) => { + if (namespace.status === 'fulfilled') { + ns[namespace.value] = namespaces[namespace.value]; + } + }); + return ns; + }, { onError: (err) => { notifyError('Failure', err as Error, 'Unable to get namespaces.'); diff --git a/app/react/kubernetes/namespaces/service.ts b/app/react/kubernetes/namespaces/service.ts index 43612d62e..57e91689d 100644 --- a/app/react/kubernetes/namespaces/service.ts +++ b/app/react/kubernetes/namespaces/service.ts @@ -8,23 +8,23 @@ export async function getNamespace( namespace: string ) { try { - const { data: ingress } = await axios.get( + const { data: ns } = await axios.get( buildUrl(environmentId, namespace) ); - return ingress; + return ns; } catch (e) { - throw parseAxiosError(e as Error, 'Unable to retrieve network details'); + throw parseAxiosError(e as Error, 'Unable to retrieve namespace'); } } export async function getNamespaces(environmentId: EnvironmentId) { try { - const { data: ingresses } = await axios.get( + const { data: namespaces } = await axios.get( buildUrl(environmentId) ); - return ingresses; + return namespaces; } catch (e) { - throw parseAxiosError(e as Error, 'Unable to retrieve network details'); + throw parseAxiosError(e as Error, 'Unable to retrieve namespaces'); } } From 903cf284e7fa7d665dcf70034224ed3ac01b5502 Mon Sep 17 00:00:00 2001 From: Dmitry Salakhov Date: Thu, 27 Oct 2022 23:31:31 +1300 Subject: [PATCH 28/31] fix(image): build image from file (#7929) [EE-4501] --- api/http/proxy/factory/docker/build.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/api/http/proxy/factory/docker/build.go b/api/http/proxy/factory/docker/build.go index cbe2d0e99..27f3622d7 100644 --- a/api/http/proxy/factory/docker/build.go +++ b/api/http/proxy/factory/docker/build.go @@ -29,9 +29,13 @@ type postDockerfileRequest struct { func buildOperation(request *http.Request) error { contentTypeHeader := request.Header.Get("Content-Type") - mediaType, _, err := mime.ParseMediaType(contentTypeHeader) - if err != nil { - return err + mediaType := "" + if contentTypeHeader != "" { + var err error + mediaType, _, err = mime.ParseMediaType(contentTypeHeader) + if err != nil { + return err + } } var buffer []byte @@ -49,7 +53,8 @@ func buildOperation(request *http.Request) error { case "application/json": var req postDockerfileRequest - if err := json.NewDecoder(request.Body).Decode(&req); err != nil { + err := json.NewDecoder(request.Body).Decode(&req) + if err != nil { return err } From 4edf232e41f6134aeed88871512f85e2af005a16 Mon Sep 17 00:00:00 2001 From: Dmitry Salakhov Date: Fri, 28 Oct 2022 13:00:12 +1300 Subject: [PATCH 29/31] fix: document edge endoint url requirement (#7735) [EE-3425] --- api/http/handler/endpoints/endpoint_create.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/api/http/handler/endpoints/endpoint_create.go b/api/http/handler/endpoints/endpoint_create.go index c2df56ab8..97858aa71 100644 --- a/api/http/handler/endpoints/endpoint_create.go +++ b/api/http/handler/endpoints/endpoint_create.go @@ -131,6 +131,17 @@ func (payload *endpointCreatePayload) Validate(r *http.Request) error { return errors.New("Invalid Azure authentication key") } payload.AzureAuthenticationKey = azureAuthenticationKey + + case edgeAgentEnvironment: + endpointURL, err := request.RetrieveMultiPartFormValue(r, "URL", false) + if err != nil || strings.EqualFold("", strings.Trim(endpointURL, " ")) { + return errors.New("URL cannot be empty") + } + payload.URL = endpointURL + + publicURL, _ := request.RetrieveMultiPartFormValue(r, "PublicURL", true) + payload.PublicURL = publicURL + default: endpointURL, err := request.RetrieveMultiPartFormValue(r, "URL", true) if err != nil { @@ -169,7 +180,7 @@ func (payload *endpointCreatePayload) Validate(r *http.Request) error { // @produce json // @param Name formData string true "Name that will be used to identify this environment(endpoint) (example: my-environment)" // @param EndpointCreationType formData integer true "Environment(Endpoint) type. Value must be one of: 1 (Local Docker environment), 2 (Agent environment), 3 (Azure environment), 4 (Edge agent environment) or 5 (Local Kubernetes Environment" Enum(1,2,3,4,5) -// @param URL formData string false "URL or IP address of a Docker host (example: docker.mydomain.tld:2375). Defaults to local if not specified (Linux: /var/run/docker.sock, Windows: //./pipe/docker_engine)" +// @param URL formData string false "URL or IP address of a Docker host (example: docker.mydomain.tld:2375). Defaults to local if not specified (Linux: /var/run/docker.sock, Windows: //./pipe/docker_engine)". Cannot be empty if EndpointCreationType is set to 4 (Edge agent environment) // @param PublicURL formData string false "URL or IP address where exposed containers will be reachable. Defaults to URL if not specified (example: docker.mydomain.tld:2375)" // @param GroupID formData int false "Environment(Endpoint) group identifier. If not specified will default to 1 (unassigned)." // @param TLS formData bool false "Require TLS to connect against this environment(endpoint)" From 95a4f83466e4eac1f885ed5baea93e9f0ee9ed68 Mon Sep 17 00:00:00 2001 From: Rex Wang <109048808+RexWangPT@users.noreply.github.com> Date: Sun, 30 Oct 2022 14:56:23 +0800 Subject: [PATCH 30/31] fix(docker): docker template UI bug fix [EE-4034] (#7912) * EE-4034 fix(docker): docker template UI bug fix * EE-4034 fix(docker): fix ui --- .../stackFromTemplateForm.html | 33 +++++++----- .../template-list/templateList.html | 50 ++++++++++--------- app/portainer/views/templates/templates.html | 24 +++++++-- 3 files changed, 67 insertions(+), 40 deletions(-) diff --git a/app/portainer/components/forms/stack-from-template-form/stackFromTemplateForm.html b/app/portainer/components/forms/stack-from-template-form/stackFromTemplateForm.html index a45f42525..42bd81a06 100644 --- a/app/portainer/components/forms/stack-from-template-form/stackFromTemplateForm.html +++ b/app/portainer/components/forms/stack-from-template-form/stackFromTemplateForm.html @@ -17,19 +17,20 @@
-
-
-
-
-
-

- - This field must consist of lower case alphanumeric characters, '_' or '-' (e.g. 'my-name', or 'abc-123'). -

-

This field is required.

+
+
+
+

+ + This field must consist of lower case alphanumeric characters, '_' or '-' (e.g. 'my-name', or 'abc-123'). +

+

This field is required.

+
+
+
@@ -66,8 +67,16 @@ Deployment in progress... -
{{ $ctrl.state.formValidationError }}
-
This template type cannot be deployed on this environment.
+
+
+

{{ $ctrl.state.formValidationError }}

+
+
+
+
+

This template type cannot be deployed on this environment.

+
+
diff --git a/app/portainer/components/template-list/templateList.html b/app/portainer/components/template-list/templateList.html index fd65eae0e..c4d4b30de 100644 --- a/app/portainer/components/template-list/templateList.html +++ b/app/portainer/components/template-list/templateList.html @@ -1,15 +1,33 @@
- +
+
+
+ +
+ {{ $ctrl.titleText }} +
+
+ + +
+
+ +
+
-
-
- -
-
-
-
-
- - -
-
-
-
Add map additional port
+
+ + Add map additional port + +
@@ -171,7 +175,11 @@ -
Add map additional volume
+
+ + Add map additional volume + +
@@ -191,7 +199,11 @@ -
Add additional entry
+
+ + Add additional entry + +
@@ -216,7 +228,11 @@ -
Add label
+
+ + Add label + +
From e785d1572ebe915bc314e65accbe80049bd3b454 Mon Sep 17 00:00:00 2001 From: fhanportainer <79428273+fhanportainer@users.noreply.github.com> Date: Mon, 31 Oct 2022 11:03:50 +1300 Subject: [PATCH 31/31] fix(web-editor): fixed web editor scroll bar. (#7941) --- .../form-components/web-editor-form/web-editor-form.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/portainer/components/form-components/web-editor-form/web-editor-form.html b/app/portainer/components/form-components/web-editor-form/web-editor-form.html index f72bfbf41..b644c2964 100644 --- a/app/portainer/components/form-components/web-editor-form/web-editor-form.html +++ b/app/portainer/components/form-components/web-editor-form/web-editor-form.html @@ -1,5 +1,5 @@ -
+
Web editor