From faa04c188b315b48b2ee94ef648f1fb60e25d4e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Busso?= Date: Fri, 20 Nov 2020 12:40:01 +1300 Subject: [PATCH] feat(bolt/backup): backup and restore db for migration and edition upgrades (#87) * refactor backup Update upgrade texts * Restore Failed Upgrade to EE to initial CE version * Store version before upgrading * Check rollback command line * Fix version display * Update template url only for CE 1.xx * Fix comments * revert go modules * remove duplicate migration * remove unused files --- api/bolt/backup.go | 181 ++++++++++ api/bolt/backup_test.go | 118 +++++++ api/bolt/datastore.go | 245 +------------- api/bolt/datastore_test.go | 1 + api/bolt/errors/errors.go | 1 + api/bolt/helpers_test.go | 115 +++++++ api/bolt/init.go | 2 +- api/bolt/log/log.test.go | 1 + api/bolt/migrate_data.go | 206 ++++++++++++ api/bolt/migrate_data_test.go | 99 ++++++ api/bolt/migrator/downgrade_dbversion25.go | 15 + api/bolt/migrator/migrate_ce.go | 262 +++++++++++++++ api/bolt/migrator/migrator.go | 312 +++--------------- api/bolt/migrator/migrator_test.go | 4 + .../migrate_ce.go => migrator/upgrade_ee.go} | 56 ++-- api/bolt/migratoree/migrator.go | 71 ---- api/bolt/migratoree/rollback.go | 20 -- api/bolt/version/version.go | 26 +- api/cli/cli.go | 4 +- api/cmd/portainer/main.go | 4 +- api/go.sum | 1 + api/portainer.go | 20 +- 22 files changed, 1149 insertions(+), 615 deletions(-) create mode 100644 api/bolt/backup.go create mode 100644 api/bolt/backup_test.go create mode 100644 api/bolt/datastore_test.go create mode 100644 api/bolt/helpers_test.go create mode 100644 api/bolt/log/log.test.go create mode 100644 api/bolt/migrate_data.go create mode 100644 api/bolt/migrate_data_test.go create mode 100644 api/bolt/migrator/downgrade_dbversion25.go create mode 100644 api/bolt/migrator/migrate_ce.go create mode 100644 api/bolt/migrator/migrator_test.go rename api/bolt/{migratoree/migrate_ce.go => migrator/upgrade_ee.go} (77%) delete mode 100644 api/bolt/migratoree/migrator.go delete mode 100644 api/bolt/migratoree/rollback.go diff --git a/api/bolt/backup.go b/api/bolt/backup.go new file mode 100644 index 000000000..e674560b1 --- /dev/null +++ b/api/bolt/backup.go @@ -0,0 +1,181 @@ +package bolt + +import ( + "fmt" + "io/ioutil" + "os" + "path" + "time" + + portainer "github.com/portainer/portainer/api" + plog "github.com/portainer/portainer/api/bolt/log" +) + +var backupDefaults = struct { + backupDir string + editions []string + databaseFileName string +}{ + "backups", + []string{"CE", "BE", "EE"}, + databaseFileName, +} + +var backupLog = plog.NewScopedLog("bolt, backup") + +// +// Backup Helpers +// + +// createBackupFolders create initial folders for backups +func (store *Store) createBackupFolders() { + for _, e := range backupDefaults.editions { + + p := path.Join(store.path, backupDefaults.backupDir, e) + + if exists, _ := store.fileService.FileExists(p); !exists { + err := os.MkdirAll(p, 0700) + if err != nil { + backupLog.Error("Error while creating backup folders", err) + } + } + } +} + +func (store *Store) databasePath() string { + return path.Join(store.path, databaseFileName) +} + +func (store *Store) editionBackupDir(edition portainer.SoftwareEdition) string { + return path.Join(store.path, backupDefaults.backupDir, edition.GetEditionLabel()) +} + +func (store *Store) copyDBFile(from string, to string) error { + backupLog.Info(fmt.Sprintf("Copying db file from %s to %s", from, to)) + err := store.fileService.Copy(from, to, true) + if err != nil { + backupLog.Error("Failed", err) + } + return err +} + +// BackupOptions provide a helper to inject backup options +type BackupOptions struct { + Edition portainer.SoftwareEdition + Version int + BackupDir string + BackupFileName string + BackupPath string +} + +func (store *Store) setupOptions(options *BackupOptions) *BackupOptions { + if options == nil { + options = &BackupOptions{} + } + if options.Edition == 0 { + options.Edition = store.edition() + } + if options.Version == 0 { + options.Version, _ = store.version() + } + if options.BackupDir == "" { + options.BackupDir = store.editionBackupDir(options.Edition) + } + if options.BackupFileName == "" { + options.BackupFileName = fmt.Sprintf("%s.%s.%s", backupDefaults.databaseFileName, fmt.Sprintf("%03d", options.Version), time.Now().Format("20060102150405")) + } + if options.BackupPath == "" { + options.BackupPath = path.Join(options.BackupDir, options.BackupFileName) + } + return options +} + +func (store *Store) listEditionBackups(edition portainer.SoftwareEdition) ([]string, error) { + var fileNames = []string{} + + files, err := ioutil.ReadDir(store.editionBackupDir(edition)) + + if err != nil { + backupLog.Error("Error while retrieving backup files", err) + return fileNames, err + } + + for _, f := range files { + fileNames = append(fileNames, f.Name()) + } + + return fileNames, nil +} + +func (store *Store) lastestEditionBackup() (string, error) { + edition := store.edition() + + files, err := store.listEditionBackups(edition) + if err != nil { + backupLog.Error("Error while retrieving backup files", err) + return "", err + } + + if len(files) == 0 { + return "", nil + } + + return files[len(files)-1], nil +} + +// BackupWithOptions backup current database with options +func (store *Store) BackupWithOptions(options *BackupOptions) (string, error) { + backupLog.Info("creating db backup") + store.createBackupFolders() + + options = store.setupOptions(options) + + return options.BackupPath, store.copyDBFile(store.databasePath(), options.BackupPath) +} + +// Backup current database with default options +func (store *Store) Backup() (string, error) { + return store.BackupWithOptions(nil) +} + +// RestoreWithOptions previously saved backup for the current Edition with options +// Restore strategies: +// - default: restore latest from current edition +// - restore a specific +func (store *Store) RestoreWithOptions(options *BackupOptions) error { + + // Check if backup file exist before restoring + + options = store.setupOptions(options) + + _, err := os.Stat(options.BackupPath) + + if os.IsNotExist(err) { + backupLog.Error(fmt.Sprintf("Backup file to restore does not exist %s", options.BackupPath), err) + return err + } + + err = store.Close() + if err != nil { + backupLog.Error("Error while closing store before restore", err) + return err + } + backupLog.Info("Restoring db backup") + err = store.copyDBFile(options.BackupPath, store.databasePath()) + if err != nil { + return err + } + return store.Open() +} + +// Restore previously saved backup for the current Edition with default options +func (store *Store) Restore() error { + var options = &BackupOptions{} + var err error + options.BackupFileName, err = store.lastestEditionBackup() + if err != nil { + return err + } + return store.RestoreWithOptions(options) + +} diff --git a/api/bolt/backup_test.go b/api/bolt/backup_test.go new file mode 100644 index 000000000..50a57a2fa --- /dev/null +++ b/api/bolt/backup_test.go @@ -0,0 +1,118 @@ +package bolt + +import ( + "fmt" + "log" + "testing" + + portainer "github.com/portainer/portainer/api" +) + +func TestCreateBackupFolders(t *testing.T) { + store := NewTestStore(portainer.PortainerEE, portainer.DBVersionEE, false) + if exists, _ := store.fileService.FileExists("tmp/backups"); exists { + t.Error("Expect backups folder to not exist") + } + store.createBackupFolders() + if exists, _ := store.fileService.FileExists("tmp/backups"); !exists { + t.Error("Expect backups folder to exist") + } + store.createBackupFolders() + store.Close() + teardown() +} + +func TestStoreCreation(t *testing.T) { + store := NewTestStore(portainer.PortainerEE, portainer.DBVersionEE, false) + if store == nil { + t.Error("Expect to create a store") + } + + if store.edition() != portainer.PortainerEE { + t.Error("Expect to get EE Edition") + } + + version, err := store.version() + if err != nil { + log.Fatal(err) + } + + if version != portainer.DBVersionEE { + t.Error("Expect to get EE DBVersion") + } + + store.Close() + teardown() +} + +func TestBackup(t *testing.T) { + + tests := []struct { + edition portainer.SoftwareEdition + version int + }{ + {edition: portainer.PortainerCE, version: portainer.DBVersion}, + {edition: portainer.PortainerEE, version: portainer.DBVersionEE}, + } + + for _, tc := range tests { + backupFileName := fmt.Sprintf("tmp/backups/%s/portainer.db.%03d.*", tc.edition.GetEditionLabel(), tc.version) + t.Run(fmt.Sprintf("Backup should create %s", backupFileName), func(t *testing.T) { + store := NewTestStore(tc.edition, tc.version, false) + store.Backup() + + if !isFileExist(backupFileName) { + t.Errorf("Expect backup file to be created %s", backupFileName) + } + store.Close() + }) + } + t.Run("BackupWithOption should create a name specific backup", func(t *testing.T) { + edition := portainer.PortainerCE + version := portainer.DBVersion + store := NewTestStore(edition, version, false) + store.BackupWithOptions(&BackupOptions{ + BackupFileName: beforePortainerUpgradeToEEBackup, + Edition: portainer.PortainerCE, + }) + backupFileName := fmt.Sprintf("tmp/backups/%s/%s", edition.GetEditionLabel(), beforePortainerUpgradeToEEBackup) + if !isFileExist(backupFileName) { + t.Errorf("Expect backup file to be created %s", backupFileName) + } + store.Close() + }) + + teardown() +} + +// TODO restore / backup failed test cases +func TestRestore(t *testing.T) { + + editions := []portainer.SoftwareEdition{portainer.PortainerCE, portainer.PortainerEE} + var currentVersion = 0 + + for i, e := range editions { + editionLabel := e.GetEditionLabel() + currentVersion = 10 ^ i + 1 + store := NewTestStore(e, currentVersion, false) + t.Run(fmt.Sprintf("Basic Restore for %s", editionLabel), func(t *testing.T) { + store.Backup() + updateVersion(store, currentVersion+1) + testVersion(store, currentVersion+1, t) + store.Restore() + testVersion(store, currentVersion, t) + }) + t.Run(fmt.Sprintf("Basic Restore After Multiple Backup for %s", editionLabel), func(t *testing.T) { + currentVersion = currentVersion + 5 + updateVersion(store, currentVersion) + store.Backup() + updateVersion(store, currentVersion+2) + testVersion(store, currentVersion+2, t) + store.Restore() + testVersion(store, currentVersion, t) + }) + store.Close() + } + + teardown() +} diff --git a/api/bolt/datastore.go b/api/bolt/datastore.go index 5b874f175..160e0807f 100644 --- a/api/bolt/datastore.go +++ b/api/bolt/datastore.go @@ -1,11 +1,10 @@ package bolt import ( - "errors" - "log" "path" "time" + "github.com/portainer/portainer/api/bolt/errors" "github.com/portainer/portainer/api/bolt/license" "github.com/boltdb/bolt" @@ -18,10 +17,7 @@ import ( "github.com/portainer/portainer/api/bolt/endpoint" "github.com/portainer/portainer/api/bolt/endpointgroup" "github.com/portainer/portainer/api/bolt/endpointrelation" - bolterrors "github.com/portainer/portainer/api/bolt/errors" "github.com/portainer/portainer/api/bolt/extension" - "github.com/portainer/portainer/api/bolt/migrator" - "github.com/portainer/portainer/api/bolt/migratoree" "github.com/portainer/portainer/api/bolt/registry" "github.com/portainer/portainer/api/bolt/resourcecontrol" "github.com/portainer/portainer/api/bolt/role" @@ -35,11 +31,9 @@ import ( "github.com/portainer/portainer/api/bolt/user" "github.com/portainer/portainer/api/bolt/version" "github.com/portainer/portainer/api/bolt/webhook" - "github.com/portainer/portainer/api/cli" - "github.com/portainer/portainer/api/internal/authorization" ) -const ( +var ( databaseFileName = "portainer.db" ) @@ -75,6 +69,22 @@ type Store struct { WebhookService *webhook.Service } +func (store *Store) version() (int, error) { + version, err := store.VersionService.DBVersion() + if err == errors.ErrObjectNotFound { + version = 0 + } + return version, err +} + +func (store *Store) edition() portainer.SoftwareEdition { + edition, err := store.VersionService.Edition() + if err == errors.ErrObjectNotFound { + edition = portainer.PortainerCE + } + return edition +} + // NewStore initializes a new Store and the associated services func NewStore(storePath string, fileService portainer.FileService) (*Store, error) { store := &Store{ @@ -121,222 +131,3 @@ func (store *Store) Close() error { func (store *Store) IsNew() bool { return store.isNew } - -// MigrateData automatically migrate the data based on the DBVersion. -// This process is only triggered on an existing database, not if the database was just created. -func (store *Store) MigrateData() error { - if store.isNew { - err := store.VersionService.StoreDBVersion(portainer.DBVersionEE) - if err != nil { - return err - } - - err = store.VersionService.StoreEdition(portainer.PortainerEE) - if err != nil { - return err - } - - return nil - } - - version, err := store.VersionService.DBVersion() - if err == bolterrors.ErrObjectNotFound { - version = 0 - } else if err != nil { - return err - } - - edition, err := store.VersionService.Edition() - if err == bolterrors.ErrObjectNotFound { - edition = portainer.PortainerCE - } else if err != nil { - return err - } - - if edition == portainer.PortainerCE && version < portainer.DBVersion { - migratorParams := &migrator.Parameters{ - DB: store.db, - DatabaseVersion: version, - EndpointGroupService: store.EndpointGroupService, - EndpointService: store.EndpointService, - EndpointRelationService: store.EndpointRelationService, - ExtensionService: store.ExtensionService, - RegistryService: store.RegistryService, - ResourceControlService: store.ResourceControlService, - RoleService: store.RoleService, - ScheduleService: store.ScheduleService, - SettingsService: store.SettingsService, - StackService: store.StackService, - TagService: store.TagService, - TeamMembershipService: store.TeamMembershipService, - UserService: store.UserService, - VersionService: store.VersionService, - FileService: store.fileService, - AuthorizationService: authorization.NewService(store), - } - migrator := migrator.NewMigrator(migratorParams) - - log.Printf("[INFO] [bolt, migrate] [message: Migrating CE database from version %v to %v.]", version, portainer.DBVersion) - err = migrator.Migrate() - if err != nil { - log.Printf("[ERROR] [bolt, migrate] [message: An error occurred during database migration: %s]", err) - return err - } - - version = portainer.DBVersion - } - - if edition < portainer.PortainerEE { - migratorParams := &migratoree.Parameters{ - CurrentEdition: edition, - DB: store.db, - DatabaseVersion: version, - - AuthorizationService: authorization.NewService(store), - EndpointGroupService: store.EndpointGroupService, - EndpointService: store.EndpointService, - ExtensionService: store.ExtensionService, - SettingsService: store.SettingsService, - UserService: store.UserService, - VersionService: store.VersionService, - RoleService: store.RoleService, - } - migrator := migratoree.NewMigrator(migratorParams) - - log.Printf("[INFO] [bolt, migrate] [message: Migrating CE database version %d to EE database version %d.]", version, portainer.DBVersionEE) - err = migrator.MigrateFromCEdbv25() - if err != nil { - log.Printf("[ERROR] [bolt, migrate] [message: An error occurred during database migration: %s]", err) - return err - } - - version = portainer.DBVersionEE - edition = portainer.PortainerEE - } - - if edition == portainer.PortainerBE && version < portainer.DBVersionEE { - migratorParams := &migratoree.Parameters{ - CurrentEdition: edition, - DB: store.db, - DatabaseVersion: version, - - AuthorizationService: authorization.NewService(store), - EndpointGroupService: store.EndpointGroupService, - EndpointService: store.EndpointService, - ExtensionService: store.ExtensionService, - SettingsService: store.SettingsService, - UserService: store.UserService, - VersionService: store.VersionService, - RoleService: store.RoleService, - } - migrator := migratoree.NewMigrator(migratorParams) - - log.Printf("[INFO] [bolt, migrate] Migrating EE database from version %v to %v.", version, portainer.DBVersionEE) - err = migrator.Migrate() - if err != nil { - log.Printf("[ERROR] [bolt, migrate] [message: An error occurred during database migration: %s]", err) - return err - } - - version = portainer.DBVersionEE - } - - return nil -} - -func (store *Store) backupDBAndRestoreIfFailed(action func() error) error { - databasePath := path.Join(store.path, databaseFileName) - backupPath := databasePath + ".old" - log.Printf("[INFO] [bolt, backup] [message: creating db backup at %s]", backupPath) - - err := store.fileService.Copy(databasePath, backupPath, true) - if err != nil { - return err - } - - err = action() - if err != nil { - databasePath := path.Join(store.path, databaseFileName) - backupPath := databasePath + ".old" - log.Printf("[INFO] [bolt, backup] [message: restoring db backup from %s after failure] [error: %s]", backupPath, err) - - copyErr := store.fileService.Copy(backupPath, databasePath, true) - if copyErr != nil { - log.Printf("[ERROR] [bolt, backup] [message: failed restoring db backup from %s] [error: %s]", backupPath, copyErr) - return copyErr - } - - return err - } - - return nil -} - -// RollbackToCE rollbacks the store to the current ce version -func (store *Store) RollbackToCE() error { - version, err := store.Version().DBVersion() - if err == bolterrors.ErrObjectNotFound { - version = 0 - } else if err != nil { - return err - } - - edition, err := store.Version().Edition() - if err == bolterrors.ErrObjectNotFound { - edition = portainer.PortainerCE - } else if err != nil { - return err - } - - log.Printf("Current Software Edition: %s\n", getEditionLabel(edition)) - log.Printf("Current DB Version: %d\n", version) - - if edition == portainer.PortainerCE { - return errors.New("DB is already on CE edition") - } - - confirmed, err := cli.Confirm("Are you sure you want to rollback your database?") - if err != nil { - return err - } - - if !confirmed { - return nil - } - - err = store.backupDBAndRestoreIfFailed(func() error { - migratorParams := &migratoree.Parameters{ - CurrentEdition: edition, - DB: store.db, - DatabaseVersion: version, - - AuthorizationService: authorization.NewService(store), - EndpointGroupService: store.EndpointGroupService, - EndpointService: store.EndpointService, - ExtensionService: store.ExtensionService, - UserService: store.UserService, - VersionService: store.VersionService, - } - migrator := migratoree.NewMigrator(migratorParams) - - return migrator.Rollback() - }) - - if err != nil { - return err - } - - log.Printf("Rolled back to CE Edition, DB Version %d\n", portainer.DBVersion) - return nil -} - -func getEditionLabel(edition portainer.SoftwareEdition) string { - switch edition { - case portainer.PortainerCE: - return "CE" - case portainer.PortainerEE: - return "EE" - } - - return "" -} diff --git a/api/bolt/datastore_test.go b/api/bolt/datastore_test.go new file mode 100644 index 000000000..a3ce8fd08 --- /dev/null +++ b/api/bolt/datastore_test.go @@ -0,0 +1 @@ +package bolt diff --git a/api/bolt/errors/errors.go b/api/bolt/errors/errors.go index c9a142189..4abf0daef 100644 --- a/api/bolt/errors/errors.go +++ b/api/bolt/errors/errors.go @@ -4,4 +4,5 @@ import "errors" var ( ErrObjectNotFound = errors.New("Object not found inside the database") + ErrMigrationToCE = errors.New("DB is already on CE edition") ) diff --git a/api/bolt/helpers_test.go b/api/bolt/helpers_test.go new file mode 100644 index 000000000..c962648c9 --- /dev/null +++ b/api/bolt/helpers_test.go @@ -0,0 +1,115 @@ +package bolt + +import ( + "fmt" + "log" + "math/rand" + "os" + "path" + "path/filepath" + "testing" + "time" + + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/filesystem" +) + +var ( + dataStorePath string + testBackupPath string +) + +func init() { + rand.Seed(time.Now().UnixNano()) + databaseFileName = fmt.Sprintf("portainer-%08d.db", rand.Intn(100000000)) + + pwd, err := os.Getwd() + if err != nil { + log.Println(err) + } + + dataStorePath = path.Join(pwd, "tmp") + testBackupPath = path.Join(dataStorePath, "backups") + + teardown() +} + +func NewTestStore(edition portainer.SoftwareEdition, version int, init bool) *Store { + fileService, err := filesystem.NewService(dataStorePath, "") + if err != nil { + log.Fatal(err) + } + + store, err := NewStore(dataStorePath, fileService) + if err != nil { + log.Fatal(err) + } + + err = store.Open() + if err != nil { + log.Fatal(err) + } + + if init { + err = store.Init() + if err != nil { + log.Fatal(err) + } + } + + err = store.VersionService.StoreEdition(edition) + if err != nil { + log.Fatal(err) + } + + err = store.VersionService.StoreDBVersion(version) + if err != nil { + log.Fatal(err) + } + + return store +} + +func teardown() { + err := os.RemoveAll(testBackupPath) + if err != nil { + log.Fatalln(err) + } + + files, err := filepath.Glob(path.Join(dataStorePath, "portainer-*.*")) + if err != nil { + log.Fatalln(err) + } + for _, f := range files { + if err := os.Remove(f); err != nil { + log.Fatalln(err) + } + } +} + +func isFileExist(path string) bool { + matches, err := filepath.Glob(path) + if err != nil { + return false + } + return len(matches) > 0 +} + +func updateVersion(store *Store, v int) { + err := store.VersionService.StoreDBVersion(v) + if err != nil { + log.Fatal(err) + } +} + +func testVersion(store *Store, versionWant int, t *testing.T) { + if v, _ := store.version(); v != versionWant { + t.Errorf("Expect store version to be %d but was %d", versionWant, v) + } +} + +func testEdition(store *Store, editionWant portainer.SoftwareEdition, t *testing.T) { + if e := store.edition(); e != editionWant { + t.Errorf("Expect store edition to be %s but was %s", editionWant.GetEditionLabel(), e.GetEditionLabel()) + } +} diff --git a/api/bolt/init.go b/api/bolt/init.go index c40f0dfc9..206d27abf 100644 --- a/api/bolt/init.go +++ b/api/bolt/init.go @@ -2,7 +2,7 @@ package bolt import ( "github.com/gofrs/uuid" - "github.com/portainer/portainer/api" + portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/bolt/errors" "github.com/portainer/portainer/api/internal/authorization" ) diff --git a/api/bolt/log/log.test.go b/api/bolt/log/log.test.go new file mode 100644 index 000000000..7330d5405 --- /dev/null +++ b/api/bolt/log/log.test.go @@ -0,0 +1 @@ +package log diff --git a/api/bolt/migrate_data.go b/api/bolt/migrate_data.go new file mode 100644 index 000000000..2eca0b70b --- /dev/null +++ b/api/bolt/migrate_data.go @@ -0,0 +1,206 @@ +package bolt + +import ( + "fmt" + + portainer "github.com/portainer/portainer/api" + errors "github.com/portainer/portainer/api/bolt/errors" + plog "github.com/portainer/portainer/api/bolt/log" + "github.com/portainer/portainer/api/bolt/migrator" + "github.com/portainer/portainer/api/cli" + "github.com/portainer/portainer/api/internal/authorization" +) + +const beforePortainerUpgradeToEEBackup = "portainer.db.before-EE-upgrade" + +var migrateLog = plog.NewScopedLog("bolt, migrate") + +// FailSafeMigrate backup and restore DB if migration fail +func (store *Store) FailSafeMigrate(migrator *migrator.Migrator, version int) error { + defer func() { + if err := recover(); err != nil { + migrateLog.Info(fmt.Sprintf("Error during migration, recovering [%v]", err)) + store.Restore() + } + }() + return migrator.Migrate(version) +} + +// MigrateData automatically migrate the data based on the DBVersion. +// This process is only triggered on an existing database, not if the database was just created. +func (store *Store) MigrateData() error { + // 0 – if DB is new then we don't need to migrate any data and just set version and edition to latest EE + if store.isNew { + err := store.VersionService.StoreDBVersion(portainer.DBVersionEE) + if err != nil { + return err + } + + err = store.VersionService.StoreEdition(portainer.PortainerEE) + if err != nil { + return err + } + + return nil + } + + migrator, err := store.newMigrator() + if err != nil { + return err + } + + if migrator.Edition() == portainer.PortainerCE { + // backup before migrating + store.BackupWithOptions(&BackupOptions{ + BackupFileName: beforePortainerUpgradeToEEBackup, + Edition: portainer.PortainerCE, + }) + + store.VersionService.StorePreviousDBVersion(migrator.Version()) + + // 1 – We need to migrate DB to latest CE version + + if migrator.Version() < portainer.DBVersion { + store.Backup() + err = store.FailSafeMigrate(migrator, portainer.DBVersion) + if err != nil { + store.Restore() + migrateLog.Error("An error occurred while migrating CE database to latest version", err) + return err + } + } + + } + + if portainer.Edition == portainer.PortainerEE { + // 2 – if DB is CE Edition we need to upgrade settings to EE + if migrator.Edition() < portainer.PortainerEE { + err = migrator.UpgradeToEE() + if err != nil { + migrateLog.Error("An error occurred while upgrading database to EE", err) + store.RollbackFailedUpgradeToEE() + return err + } + + } + + // 3 – if DB is EE Edition we need to migrate to latest version of EE + if migrator.Edition() == portainer.PortainerEE && migrator.Version() < portainer.DBVersionEE { + store.Backup() + err = store.FailSafeMigrate(migrator, portainer.DBVersionEE) + if err != nil { + migrateLog.Error("An error occurred while migrating EE database to latest version", err) + store.Restore() + return err + } + } + } + + return nil +} + +// RollbackFailedUpgradeToEE down migrate to previous version +func (store *Store) RollbackFailedUpgradeToEE() error { + return store.RestoreWithOptions(&BackupOptions{ + BackupFileName: beforePortainerUpgradeToEEBackup, + Edition: portainer.PortainerCE, + }) +} + +// RollbackToCE rollbacks the store to the current ce version +func (store *Store) RollbackToCE() error { + + migrator, err := store.newMigrator() + if err != nil { + return err + } + + migrateLog.Info(fmt.Sprintf("Current Software Edition: %s", migrator.Edition().GetEditionLabel())) + migrateLog.Info(fmt.Sprintf("Current DB Version: %d", migrator.Version())) + + if migrator.Edition() == portainer.PortainerCE { + return errors.ErrMigrationToCE + } + + previousVersion, err := store.VersionService.PreviousDBVersion() + if err != nil { + migrateLog.Error("An Error occurred with retrieving previous DB version", err) + return err + } + + confirmed, err := cli.Confirm(fmt.Sprintf("Are you sure you want to rollback your database to %d?", previousVersion)) + if err != nil || !confirmed { + return err + } + + if previousVersion < 25 { + migrator.DowngradeSettingsFrom25() + } + + err = store.VersionService.StoreDBVersion(previousVersion) + if err != nil { + migrateLog.Error(fmt.Sprintf("An Error occurred with rolling back to CE Edition, DB Version %d", previousVersion), err) + return err + } + + err = store.VersionService.StoreEdition(portainer.PortainerCE) + if err != nil { + migrateLog.Error(fmt.Sprintf("An Error occurred with rolling back to CE Edition, DB Version %d", previousVersion), err) + return err + } + + migrateLog.Info(fmt.Sprintf("Rolled back to CE Edition, DB Version %d", previousVersion)) + return nil +} + +func (store *Store) newMigrator() (*migrator.Migrator, error) { + + version, err := store.version() + if err != nil { + return nil, err + } + + edition := store.edition() + + params := &migrator.Parameters{ + DB: store.db, + DatabaseVersion: version, + CurrentEdition: edition, + + EndpointGroupService: store.EndpointGroupService, + EndpointService: store.EndpointService, + EndpointRelationService: store.EndpointRelationService, + ExtensionService: store.ExtensionService, + RegistryService: store.RegistryService, + ResourceControlService: store.ResourceControlService, + RoleService: store.RoleService, + ScheduleService: store.ScheduleService, + SettingsService: store.SettingsService, + StackService: store.StackService, + TagService: store.TagService, + TeamMembershipService: store.TeamMembershipService, + UserService: store.UserService, + VersionService: store.VersionService, + FileService: store.fileService, + AuthorizationService: authorization.NewService(store), + } + + return migrator.NewMigrator(params), nil +} + +// RollbackVersion down migrate to previous version +func (store *Store) RollbackVersion(version int) error { + // TODO + backupLog.NotImplemented("RollbackVersion") + return nil +} + +// RollbackEdition downgrade to previous edition +func (store *Store) RollbackEdition(edition portainer.SoftwareEdition) error { + // TODO + backupLog.NotImplemented("RollbackEdition") + // Change Edition + // Migrate Services + // Restore Latest + return nil +} diff --git a/api/bolt/migrate_data_test.go b/api/bolt/migrate_data_test.go new file mode 100644 index 000000000..83150474a --- /dev/null +++ b/api/bolt/migrate_data_test.go @@ -0,0 +1,99 @@ +package bolt + +import ( + "fmt" + "log" + "testing" + + "github.com/boltdb/bolt" + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/filesystem" +) + +// New Database should be EE and DBVersion +// + +func TestMigrateData(t *testing.T) { + var store *Store + + t.Run("MigrateData for New Store", func(t *testing.T) { + fileService, err := filesystem.NewService(dataStorePath, "") + if err != nil { + log.Fatal(err) + } + + store, err := NewStore(dataStorePath, fileService) + if err != nil { + log.Fatal(err) + } + + err = store.Open() + if err != nil { + log.Fatal(err) + } + + err = store.Init() + if err != nil { + log.Fatal(err) + } + + store.MigrateData() + + testVersion(store, portainer.DBVersionEE, t) + testEdition(store, portainer.PortainerEE, t) + + store.Close() + }) + + tests := []struct { + edition portainer.SoftwareEdition + version int + expectedVersion int + }{ + {edition: portainer.PortainerCE, version: 5, expectedVersion: portainer.DBVersionEE}, + {edition: portainer.PortainerCE, version: 21, expectedVersion: portainer.DBVersionEE}, + } + + for _, tc := range tests { + store = NewTestStore(tc.edition, tc.version, true) + t.Run(fmt.Sprintf("MigrateData for %s version %d", tc.edition.GetEditionLabel(), tc.version), func(t *testing.T) { + store.MigrateData() + testVersion(store, tc.expectedVersion, t) + testEdition(store, portainer.PortainerEE, t) + }) + + t.Run(fmt.Sprintf("Restoring DB after migrateData for %s version %d", tc.edition.GetEditionLabel(), tc.version), func(t *testing.T) { + store.RollbackToCE() + testVersion(store, tc.version, t) + testEdition(store, tc.edition, t) + }) + + store.Close() + } + + t.Run("Error in MigrateData should restore backup before MigrateData", func(t *testing.T) { + version := 21 + store = NewTestStore(portainer.PortainerCE, version, true) + + deleteBucket(store.db, "settings") + store.MigrateData() + + testVersion(store, version, t) + testEdition(store, portainer.PortainerCE, t) + + store.Close() + }) + + teardown() +} + +func deleteBucket(db *bolt.DB, bucketName string) { + db.Update(func(tx *bolt.Tx) error { + log.Printf("Delete bucket %s\n", bucketName) + err := tx.DeleteBucket([]byte(bucketName)) + if err != nil { + log.Println(err) + } + return err + }) +} diff --git a/api/bolt/migrator/downgrade_dbversion25.go b/api/bolt/migrator/downgrade_dbversion25.go new file mode 100644 index 000000000..80637dd6a --- /dev/null +++ b/api/bolt/migrator/downgrade_dbversion25.go @@ -0,0 +1,15 @@ +package migrator + +// DowngradeSettingsFrom25 downgrade template settings for portainer v1.2 +func (migrator *Migrator) DowngradeSettingsFrom25() error { + legacySettings, err := migrator.settingsService.Settings() + if err != nil { + return err + } + + legacySettings.TemplatesURL = "https://raw.githubusercontent.com/portainer/templates/master/templates-1.20.0.json" + + err = migrator.settingsService.UpdateSettings(legacySettings) + + return err +} diff --git a/api/bolt/migrator/migrate_ce.go b/api/bolt/migrator/migrate_ce.go new file mode 100644 index 000000000..eb3d31d22 --- /dev/null +++ b/api/bolt/migrator/migrate_ce.go @@ -0,0 +1,262 @@ +package migrator + +import ( + "log" + + portainer "github.com/portainer/portainer/api" +) + +// MigrateCE checks the database version and migrate the existing data to the most recent data model. +func (m *Migrator) MigrateCE() error { + // Portainer < 1.12 + if m.currentDBVersion < 1 { + err := m.updateAdminUserToDBVersion1() + if err != nil { + return err + } + } + + // Portainer 1.12.x + if m.currentDBVersion < 2 { + err := m.updateResourceControlsToDBVersion2() + if err != nil { + return err + } + err = m.updateEndpointsToDBVersion2() + if err != nil { + return err + } + } + + // Portainer 1.13.x + if m.currentDBVersion < 3 { + err := m.updateSettingsToDBVersion3() + if err != nil { + return err + } + } + + // Portainer 1.14.0 + if m.currentDBVersion < 4 { + err := m.updateEndpointsToDBVersion4() + if err != nil { + return err + } + } + + // https://github.com/portainer/portainer/issues/1235 + if m.currentDBVersion < 5 { + err := m.updateSettingsToVersion5() + if err != nil { + return err + } + } + + // https://github.com/portainer/portainer/issues/1236 + if m.currentDBVersion < 6 { + err := m.updateSettingsToVersion6() + if err != nil { + return err + } + } + + // https://github.com/portainer/portainer/issues/1449 + if m.currentDBVersion < 7 { + err := m.updateSettingsToVersion7() + if err != nil { + return err + } + } + + if m.currentDBVersion < 8 { + err := m.updateEndpointsToVersion8() + if err != nil { + return err + } + } + + // https: //github.com/portainer/portainer/issues/1396 + if m.currentDBVersion < 9 { + err := m.updateEndpointsToVersion9() + if err != nil { + return err + } + } + + // https://github.com/portainer/portainer/issues/461 + if m.currentDBVersion < 10 { + err := m.updateEndpointsToVersion10() + if err != nil { + return err + } + } + + // https://github.com/portainer/portainer/issues/1906 + if m.currentDBVersion < 11 { + err := m.updateEndpointsToVersion11() + if err != nil { + return err + } + } + + // Portainer 1.18.0 + if m.currentDBVersion < 12 { + err := m.updateEndpointsToVersion12() + if err != nil { + return err + } + + err = m.updateEndpointGroupsToVersion12() + if err != nil { + return err + } + + err = m.updateStacksToVersion12() + if err != nil { + return err + } + } + + // Portainer 1.19.0 + if m.currentDBVersion < 13 { + err := m.updateSettingsToVersion13() + if err != nil { + return err + } + } + + // Portainer 1.19.2 + if m.currentDBVersion < 14 { + err := m.updateResourceControlsToDBVersion14() + if err != nil { + return err + } + } + + // Portainer 1.20.0 + if m.currentDBVersion < 15 { + err := m.updateSettingsToDBVersion15() + if err != nil { + return err + } + + err = m.updateTemplatesToVersion15() + if err != nil { + return err + } + } + + if m.currentDBVersion < 16 { + err := m.updateSettingsToDBVersion16() + if err != nil { + return err + } + } + + // Portainer 1.20.1 + if m.currentDBVersion < 17 { + err := m.updateExtensionsToDBVersion17() + if err != nil { + return err + } + } + + // Portainer 1.21.0 + if m.currentDBVersion < 18 { + err := m.updateUsersToDBVersion18() + if err != nil { + return err + } + + err = m.updateEndpointsToDBVersion18() + if err != nil { + return err + } + + err = m.updateEndpointGroupsToDBVersion18() + if err != nil { + return err + } + + err = m.updateRegistriesToDBVersion18() + if err != nil { + return err + } + } + + // Portainer 1.22.0 + if m.currentDBVersion < 19 { + err := m.updateSettingsToDBVersion19() + if err != nil { + return err + } + } + + // Portainer 1.22.1 + if m.currentDBVersion < 20 { + err := m.updateUsersToDBVersion20() + if err != nil { + return err + } + + err = m.updateSettingsToDBVersion20() + if err != nil { + return err + } + + err = m.updateSchedulesToDBVersion20() + if err != nil { + return err + } + } + + // Portainer 1.23.0 + // DBVersion 21 is missing as it was shipped as via hotfix 1.22.2 + if m.currentDBVersion < 22 { + err := m.updateResourceControlsToDBVersion22() + if err != nil { + return err + } + + err = m.updateUsersAndRolesToDBVersion22() + if err != nil { + return err + } + } + + // Portainer 1.24.0 + if m.currentDBVersion < 23 { + err := m.updateTagsToDBVersion23() + if err != nil { + return err + } + + err = m.updateEndpointsAndEndpointGroupsToDBVersion23() + if err != nil { + return err + } + } + + // Portainer 1.24.1 + if m.currentDBVersion < 24 { + err := m.updateSettingsToDB24() + if err != nil { + return err + } + } + + // Portainer 2.0.0 + if m.currentDBVersion < 25 { + err := m.updateSettingsToDB25() + if err != nil { + return err + } + + err = m.updateStacksToDB24() + if err != nil { + return err + } + } + log.Println("Update DB version to ", portainer.DBVersion) + return m.versionService.StoreDBVersion(portainer.DBVersion) +} diff --git a/api/bolt/migrator/migrator.go b/api/bolt/migrator/migrator.go index 8217dc302..84f03ae37 100644 --- a/api/bolt/migrator/migrator.go +++ b/api/bolt/migrator/migrator.go @@ -1,12 +1,15 @@ package migrator import ( + "fmt" + "github.com/boltdb/bolt" - "github.com/portainer/portainer/api" + portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/bolt/endpoint" "github.com/portainer/portainer/api/bolt/endpointgroup" "github.com/portainer/portainer/api/bolt/endpointrelation" "github.com/portainer/portainer/api/bolt/extension" + plog "github.com/portainer/portainer/api/bolt/log" "github.com/portainer/portainer/api/bolt/registry" "github.com/portainer/portainer/api/bolt/resourcecontrol" "github.com/portainer/portainer/api/bolt/role" @@ -20,11 +23,15 @@ import ( "github.com/portainer/portainer/api/internal/authorization" ) +var migrateLog = plog.NewScopedLog("bolt, migrate") + type ( // Migrator defines a service to migrate data after a Portainer version update. Migrator struct { - currentDBVersion int - db *bolt.DB + db *bolt.DB + currentDBVersion int + currentEdition portainer.SoftwareEdition + endpointGroupService *endpointgroup.Service endpointService *endpoint.Service endpointRelationService *endpointrelation.Service @@ -45,8 +52,10 @@ type ( // Parameters represents the required parameters to create a new Migrator instance. Parameters struct { - DB *bolt.DB - DatabaseVersion int + DB *bolt.DB + DatabaseVersion int + CurrentEdition portainer.SoftwareEdition + EndpointGroupService *endpointgroup.Service EndpointService *endpoint.Service EndpointRelationService *endpointrelation.Service @@ -71,6 +80,7 @@ func NewMigrator(parameters *Parameters) *Migrator { return &Migrator{ db: parameters.DB, currentDBVersion: parameters.DatabaseVersion, + currentEdition: parameters.CurrentEdition, endpointGroupService: parameters.EndpointGroupService, endpointService: parameters.EndpointService, endpointRelationService: parameters.EndpointRelationService, @@ -90,257 +100,43 @@ func NewMigrator(parameters *Parameters) *Migrator { } } -// Migrate checks the database version and migrate the existing data to the most recent data model. -func (m *Migrator) Migrate() error { - // Portainer < 1.12 - if m.currentDBVersion < 1 { - err := m.updateAdminUserToDBVersion1() - if err != nil { - return err - } - } - - // Portainer 1.12.x - if m.currentDBVersion < 2 { - err := m.updateResourceControlsToDBVersion2() - if err != nil { - return err - } - err = m.updateEndpointsToDBVersion2() - if err != nil { - return err - } - } - - // Portainer 1.13.x - if m.currentDBVersion < 3 { - err := m.updateSettingsToDBVersion3() - if err != nil { - return err - } - } - - // Portainer 1.14.0 - if m.currentDBVersion < 4 { - err := m.updateEndpointsToDBVersion4() - if err != nil { - return err - } - } - - // https://github.com/portainer/portainer/issues/1235 - if m.currentDBVersion < 5 { - err := m.updateSettingsToVersion5() - if err != nil { - return err - } - } - - // https://github.com/portainer/portainer/issues/1236 - if m.currentDBVersion < 6 { - err := m.updateSettingsToVersion6() - if err != nil { - return err - } - } - - // https://github.com/portainer/portainer/issues/1449 - if m.currentDBVersion < 7 { - err := m.updateSettingsToVersion7() - if err != nil { - return err - } - } - - if m.currentDBVersion < 8 { - err := m.updateEndpointsToVersion8() - if err != nil { - return err - } - } - - // https: //github.com/portainer/portainer/issues/1396 - if m.currentDBVersion < 9 { - err := m.updateEndpointsToVersion9() - if err != nil { - return err - } - } - - // https://github.com/portainer/portainer/issues/461 - if m.currentDBVersion < 10 { - err := m.updateEndpointsToVersion10() - if err != nil { - return err - } - } - - // https://github.com/portainer/portainer/issues/1906 - if m.currentDBVersion < 11 { - err := m.updateEndpointsToVersion11() - if err != nil { - return err - } - } - - // Portainer 1.18.0 - if m.currentDBVersion < 12 { - err := m.updateEndpointsToVersion12() - if err != nil { - return err - } - - err = m.updateEndpointGroupsToVersion12() - if err != nil { - return err - } - - err = m.updateStacksToVersion12() - if err != nil { - return err - } - } - - // Portainer 1.19.0 - if m.currentDBVersion < 13 { - err := m.updateSettingsToVersion13() - if err != nil { - return err - } - } - - // Portainer 1.19.2 - if m.currentDBVersion < 14 { - err := m.updateResourceControlsToDBVersion14() - if err != nil { - return err - } - } - - // Portainer 1.20.0 - if m.currentDBVersion < 15 { - err := m.updateSettingsToDBVersion15() - if err != nil { - return err - } - - err = m.updateTemplatesToVersion15() - if err != nil { - return err - } - } - - if m.currentDBVersion < 16 { - err := m.updateSettingsToDBVersion16() - if err != nil { - return err - } - } - - // Portainer 1.20.1 - if m.currentDBVersion < 17 { - err := m.updateExtensionsToDBVersion17() - if err != nil { - return err - } - } - - // Portainer 1.21.0 - if m.currentDBVersion < 18 { - err := m.updateUsersToDBVersion18() - if err != nil { - return err - } - - err = m.updateEndpointsToDBVersion18() - if err != nil { - return err - } - - err = m.updateEndpointGroupsToDBVersion18() - if err != nil { - return err - } - - err = m.updateRegistriesToDBVersion18() - if err != nil { - return err - } - } - - // Portainer 1.22.0 - if m.currentDBVersion < 19 { - err := m.updateSettingsToDBVersion19() - if err != nil { - return err - } - } - - // Portainer 1.22.1 - if m.currentDBVersion < 20 { - err := m.updateUsersToDBVersion20() - if err != nil { - return err - } - - err = m.updateSettingsToDBVersion20() - if err != nil { - return err - } - - err = m.updateSchedulesToDBVersion20() - if err != nil { - return err - } - } - - // Portainer 1.23.0 - // DBVersion 21 is missing as it was shipped as via hotfix 1.22.2 - if m.currentDBVersion < 22 { - err := m.updateResourceControlsToDBVersion22() - if err != nil { - return err - } - - err = m.updateUsersAndRolesToDBVersion22() - if err != nil { - return err - } - } - - // Portainer 1.24.0 - if m.currentDBVersion < 23 { - err := m.updateTagsToDBVersion23() - if err != nil { - return err - } - - err = m.updateEndpointsAndEndpointGroupsToDBVersion23() - if err != nil { - return err - } - } - - // Portainer 1.24.1 - if m.currentDBVersion < 24 { - err := m.updateSettingsToDB24() - if err != nil { - return err - } - } - - // Portainer 2.0.0 - if m.currentDBVersion < 25 { - err := m.updateSettingsToDB25() - if err != nil { - return err - } - - err = m.updateStacksToDB24() - if err != nil { - return err - } - } - - return m.versionService.StoreDBVersion(portainer.DBVersion) +// Version exposes version of database +func (migrator *Migrator) Version() int { + return migrator.currentDBVersion +} + +// Edition exposes edition of portainer +func (migrator *Migrator) Edition() portainer.SoftwareEdition { + return migrator.currentEdition +} + +// Migrate helper to upgrade DB +func (migrator *Migrator) Migrate(version int) error { + + migrateLog.Info(fmt.Sprintf("Migrating %s database from version %d to %d.", migrator.Edition().GetEditionLabel(), migrator.currentDBVersion, version)) + // TODO : run backup before migration and restore if failed + err := migrator.MigrateCE() //CE + if err != nil { + migrateLog.Error("An error occurred during database migration", err) + return err + } + + migrator.versionService.StoreDBVersion(version) + migrator.currentDBVersion = version + + return nil +} + +// RollbackVersion rolls back the db to version +func (migrator *Migrator) RollbackVersion(version int) error { + + err := migrator.versionService.StoreDBVersion(version) // portainer.DBVersion + return err +} + +// RollbackEdition rolls back the db to portainer CE +func (migrator *Migrator) RollbackEdition(edition portainer.SoftwareEdition) error { + + err := migrator.versionService.StoreEdition(portainer.PortainerCE) + return err } diff --git a/api/bolt/migrator/migrator_test.go b/api/bolt/migrator/migrator_test.go new file mode 100644 index 000000000..1a179f43e --- /dev/null +++ b/api/bolt/migrator/migrator_test.go @@ -0,0 +1,4 @@ +package migrator + +// test CE version is always upgraded to latest version of CE +// test EE version is always upgraded to latest version of EE diff --git a/api/bolt/migratoree/migrate_ce.go b/api/bolt/migrator/upgrade_ee.go similarity index 77% rename from api/bolt/migratoree/migrate_ce.go rename to api/bolt/migrator/upgrade_ee.go index 9ed462466..4d49d8963 100644 --- a/api/bolt/migratoree/migrate_ce.go +++ b/api/bolt/migrator/upgrade_ee.go @@ -1,54 +1,56 @@ -package migratoree +package migrator import ( - "log" + "fmt" portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/internal/authorization" ) -// MigrateFromCEdbv25 will migrate the db from latest ce version to latest ee version -func (m *Migrator) MigrateFromCEdbv25() error { - log.Printf("[INFO] [bolt, migrate] Updating LDAP settings to EE.") +// UpgradeToEE will migrate the db from latest ce version to latest ee version +// Latest version is v25 on 06/11/2020 +func (m *Migrator) UpgradeToEE() error { + + migrateLog.Info(fmt.Sprintf("Migrating CE database version %d to EE database version %d.", m.Version(), portainer.DBVersion)) + + migrateLog.Info("Updating LDAP settings to EE") err := m.updateSettingsToEE() if err != nil { return err } - log.Printf("[INFO] [bolt, migrate] Updating user roles to EE.") + migrateLog.Info("Updating user roles to EE") err = m.updateUserRolesToEE() if err != nil { return err } - log.Printf("[INFO] [bolt, migrate] Updating role authorizations to EE.") + migrateLog.Info("Updating role authorizations to EE") err = m.updateRoleAuthorizationsToEE() if err != nil { return err } - log.Printf("[INFO] [bolt, migrate] Updating user authorizations.") + migrateLog.Info("Updating user authorizations") err = m.authorizationService.UpdateUsersAuthorizations() if err != nil { return err } - log.Printf("[INFO] [bolt, migrate] Updating LDAP settings to EE.") - err = m.updateSettingsToEE() - if err != nil { - return err - } - - log.Printf("[INFO] [bolt, migrate] Setting db version to %v.", portainer.DBVersionEE) + migrateLog.Info(fmt.Sprintf("Setting db version to %d", portainer.DBVersionEE)) err = m.versionService.StoreDBVersion(portainer.DBVersionEE) if err != nil { return err } - log.Printf("[INFO] [bolt, migrate] Setting edition to %v.", portainer.PortainerEE) + migrateLog.Info(fmt.Sprintf("Setting edition to %s", portainer.PortainerEE.GetEditionLabel())) err = m.versionService.StoreEdition(portainer.PortainerEE) if err != nil { return err } + m.currentDBVersion = portainer.DBVersionEE + m.currentEdition = portainer.PortainerEE + return nil } @@ -71,13 +73,13 @@ func (m *Migrator) updateSettingsToEE() error { // Updating role authorizations because of the new policies in Kube RBAC func (m *Migrator) updateRoleAuthorizationsToEE() error { - log.Printf("[DEBUG] [bolt, migrate] Retriving settings") + migrateLog.Debug("Retriving settings") settings, err := m.settingsService.Settings() if err != nil { return err } - log.Printf("[DEBUG] [bolt, migrate] Updating Endpoint Admin Role") + migrateLog.Debug("Updating Endpoint Admin Role") endpointAdministratorRole, err := m.roleService.Role(portainer.RoleID(1)) if err != nil { return err @@ -87,7 +89,7 @@ func (m *Migrator) updateRoleAuthorizationsToEE() error { err = m.roleService.UpdateRole(endpointAdministratorRole.ID, endpointAdministratorRole) - log.Printf("[DEBUG] [bolt, migrate] Updating Help Desk Role") + migrateLog.Debug("Updating Help Desk Role") helpDeskRole, err := m.roleService.Role(portainer.RoleID(2)) if err != nil { return err @@ -97,7 +99,7 @@ func (m *Migrator) updateRoleAuthorizationsToEE() error { err = m.roleService.UpdateRole(helpDeskRole.ID, helpDeskRole) - log.Printf("[DEBUG] [bolt, migrate] Updating Standard User Role") + migrateLog.Debug("Updating Standard User Role") standardUserRole, err := m.roleService.Role(portainer.RoleID(3)) if err != nil { return err @@ -107,7 +109,7 @@ func (m *Migrator) updateRoleAuthorizationsToEE() error { err = m.roleService.UpdateRole(standardUserRole.ID, standardUserRole) - log.Printf("[DEBUG] [bolt, migrate] Updating Read Only User Role") + migrateLog.Debug("Updating Read Only User Role") readOnlyUserRole, err := m.roleService.Role(portainer.RoleID(4)) if err != nil { return err @@ -131,23 +133,23 @@ func (m *Migrator) updateUserRolesToEE() error { return err } - log.Printf("[DEBUG] [bolt, migrate] Retriving extension info") + migrateLog.Debug("Retriving extension info") extensions, err := m.extensionService.Extensions() for _, extension := range extensions { if extension.ID == 3 && extension.Enabled { - log.Printf("[INFO] [bolt, migrate] RBAC extensions were enabled before; Skip updating User Roles") + migrateLog.Info("RBAC extensions were enabled before; Skip updating User Roles") return nil } } - log.Printf("[DEBUG] [bolt, migrate] Retriving endpoint groups") + migrateLog.Debug("Retriving endpoint groups") endpointGroups, err := m.endpointGroupService.EndpointGroups() if err != nil { return err } for _, endpointGroup := range endpointGroups { - log.Printf("[DEBUG] [bolt, migrate] Updating user policies for endpoint group %v", endpointGroup.ID) + migrateLog.Debug(fmt.Sprintf("Updating user policies for endpoint group %v", endpointGroup.ID)) for key := range endpointGroup.UserAccessPolicies { updateUserAccessPolicyToReadOnlyRole(endpointGroup.UserAccessPolicies, key) } @@ -162,14 +164,14 @@ func (m *Migrator) updateUserRolesToEE() error { } } - log.Printf("[DEBUG] [bolt, migrate] Retriving endpoints") + migrateLog.Debug("Retriving endpoints") endpoints, err := m.endpointService.Endpoints() if err != nil { return err } for _, endpoint := range endpoints { - log.Printf("[DEBUG] [bolt, migrate] Updating user policies for endpoint %v", endpoint.ID) + migrateLog.Debug(fmt.Sprintf("Updating user policies for endpoint %v", endpoint.ID)) for key := range endpoint.UserAccessPolicies { updateUserAccessPolicyToReadOnlyRole(endpoint.UserAccessPolicies, key) } diff --git a/api/bolt/migratoree/migrator.go b/api/bolt/migratoree/migrator.go deleted file mode 100644 index 6e02a8d87..000000000 --- a/api/bolt/migratoree/migrator.go +++ /dev/null @@ -1,71 +0,0 @@ -package migratoree - -import ( - "github.com/boltdb/bolt" - portainer "github.com/portainer/portainer/api" - "github.com/portainer/portainer/api/bolt/endpoint" - "github.com/portainer/portainer/api/bolt/endpointgroup" - "github.com/portainer/portainer/api/bolt/extension" - "github.com/portainer/portainer/api/bolt/role" - "github.com/portainer/portainer/api/bolt/settings" - "github.com/portainer/portainer/api/bolt/user" - "github.com/portainer/portainer/api/internal/authorization" -) - -type ( - // Migrator defines a service to migrate data after a Portainer version update. - Migrator struct { - db *bolt.DB - currentDBVersion int - currentEdition portainer.SoftwareEdition - - authorizationService *authorization.Service - endpointGroupService *endpointgroup.Service - endpointService *endpoint.Service - extensionService *extension.Service - settingsService *settings.Service - userService *user.Service - versionService portainer.VersionService - roleService *role.Service - } - - // Parameters represents the required parameters to create a new Migrator instance. - Parameters struct { - DB *bolt.DB - DatabaseVersion int - CurrentEdition portainer.SoftwareEdition - - AuthorizationService *authorization.Service - EndpointGroupService *endpointgroup.Service - EndpointService *endpoint.Service - ExtensionService *extension.Service - SettingsService *settings.Service - UserService *user.Service - VersionService portainer.VersionService - RoleService *role.Service - } -) - -// NewMigrator creates a new Migrator. -func NewMigrator(parameters *Parameters) *Migrator { - return &Migrator{ - db: parameters.DB, - currentDBVersion: parameters.DatabaseVersion, - currentEdition: parameters.CurrentEdition, - - authorizationService: parameters.AuthorizationService, - endpointGroupService: parameters.EndpointGroupService, - endpointService: parameters.EndpointService, - extensionService: parameters.ExtensionService, - settingsService: parameters.SettingsService, - userService: parameters.UserService, - versionService: parameters.VersionService, - roleService: parameters.RoleService, - } -} - -// Migrate checks the database version and migrate the existing data to the most recent data model. -func (m *Migrator) Migrate() error { - - return nil -} diff --git a/api/bolt/migratoree/rollback.go b/api/bolt/migratoree/rollback.go deleted file mode 100644 index 933daf387..000000000 --- a/api/bolt/migratoree/rollback.go +++ /dev/null @@ -1,20 +0,0 @@ -package migratoree - -import ( - "github.com/portainer/portainer/api" -) - -// Rollback rolls back the db to portainer CE latest db version -func (m *Migrator) Rollback() error { - err := m.versionService.StoreDBVersion(portainer.DBVersion) - if err != nil { - return err - } - - err = m.versionService.StoreEdition(portainer.PortainerCE) - if err != nil { - return err - } - - return nil -} diff --git a/api/bolt/version/version.go b/api/bolt/version/version.go index bfc996b5f..c5ff7c440 100644 --- a/api/bolt/version/version.go +++ b/api/bolt/version/version.go @@ -4,17 +4,18 @@ import ( "strconv" "github.com/boltdb/bolt" - "github.com/portainer/portainer/api" + portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/bolt/errors" "github.com/portainer/portainer/api/bolt/internal" ) const ( // BucketName represents the name of the bucket where this service stores data. - BucketName = "version" - versionKey = "DB_VERSION" - instanceKey = "INSTANCE_ID" - editionKey = "EDITION" + BucketName = "version" + versionKey = "DB_VERSION" + previousVersionKey = "PREVIOUS_DB_VERSION" + instanceKey = "INSTANCE_ID" + editionKey = "EDITION" ) // Service represents a service to manage stored versions. @@ -54,6 +55,16 @@ func (service *Service) StoreEdition(edition portainer.SoftwareEdition) error { return service.setKey(editionKey, strconv.Itoa(int(edition))) } +// PreviousDBVersion retrieves the stored database version. +func (service *Service) PreviousDBVersion() (int, error) { + version, err := service.getKey(previousVersionKey) + if err != nil { + return 0, err + } + + return strconv.Atoi(string(version)) +} + // DBVersion retrieves the stored database version. func (service *Service) DBVersion() (int, error) { version, err := service.getKey(versionKey) @@ -64,6 +75,11 @@ func (service *Service) DBVersion() (int, error) { return strconv.Atoi(string(version)) } +// StorePreviousDBVersion store the database version. +func (service *Service) StorePreviousDBVersion(version int) error { + return service.setKey(previousVersionKey, strconv.Itoa(version)) +} + // StoreDBVersion store the database version. func (service *Service) StoreDBVersion(version int) error { return service.setKey(versionKey, strconv.Itoa(version)) diff --git a/api/cli/cli.go b/api/cli/cli.go index 0df0bd104..7ecfd3992 100644 --- a/api/cli/cli.go +++ b/api/cli/cli.go @@ -5,7 +5,7 @@ import ( "log" "time" - "github.com/portainer/portainer/api" + portainer "github.com/portainer/portainer/api" "os" "path/filepath" @@ -42,7 +42,7 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) { TLSCacert: kingpin.Flag("tlscacert", "Path to the CA").Default(defaultTLSCACertPath).String(), TLSCert: kingpin.Flag("tlscert", "Path to the TLS certificate file").Default(defaultTLSCertPath).String(), TLSKey: kingpin.Flag("tlskey", "Path to the TLS key").Default(defaultTLSKeyPath).String(), - RollbackDB: kingpin.Flag("rollback-db", "Rollback the database store").Bool(), + RollbackToCE: kingpin.Flag("rollback-to-ce", "Rollback the database store to CE").Bool(), SSL: kingpin.Flag("ssl", "Secure Portainer instance using SSL").Default(defaultSSL).Bool(), SSLCert: kingpin.Flag("sslcert", "Path to the SSL certificate used to secure the Portainer instance").Default(defaultSSLCertPath).String(), SSLKey: kingpin.Flag("sslkey", "Path to the SSL key used to secure the Portainer instance").Default(defaultSSLKeyPath).String(), diff --git a/api/cmd/portainer/main.go b/api/cmd/portainer/main.go index a5191c86e..d5e90e02a 100644 --- a/api/cmd/portainer/main.go +++ b/api/cmd/portainer/main.go @@ -6,7 +6,7 @@ import ( "strings" "time" - "github.com/portainer/portainer/api" + portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/bolt" "github.com/portainer/portainer/api/chisel" "github.com/portainer/portainer/api/cli" @@ -350,7 +350,7 @@ func main() { fileService := initFileService(*flags.Data) - dataStore := initDataStore(*flags.Data, *flags.RollbackDB, fileService) + dataStore := initDataStore(*flags.Data, *flags.RollbackToCE, fileService) defer dataStore.Close() jwtService, err := initJWTService(dataStore) diff --git a/api/go.sum b/api/go.sum index f6c668d26..2cc475a51 100644 --- a/api/go.sum +++ b/api/go.sum @@ -349,6 +349,7 @@ github.com/portainer/libcrypto v0.0.0-20190723020515-23ebe86ab2c2 h1:0PfgGLys9yH github.com/portainer/libcrypto v0.0.0-20190723020515-23ebe86ab2c2/go.mod h1:/wIeGwJOMYc1JplE/OvYMO5korce39HddIfI8VKGyAM= github.com/portainer/libhttp v0.0.0-20190806161843-ba068f58be33 h1:H8HR2dHdBf8HANSkUyVw4o8+4tegGcd+zyKZ3e599II= github.com/portainer/libhttp v0.0.0-20190806161843-ba068f58be33/go.mod h1:Y2TfgviWI4rT2qaOTHr+hq6MdKIE5YjgQAu7qwptTV0= +github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/portainer/liblicense v0.0.0-20200926131517-4c35b5b3a96f h1:YrLxuK4YNS46rMSooaYUI33i7PiobPQ0KBMlhum36cc= github.com/portainer/liblicense v0.0.0-20200926131517-4c35b5b3a96f/go.mod h1:6OSugUz07QF1WwrMmRNIMhgiq+drVMIZ98UTu5LVZP8= github.com/portainer/liblicense v0.0.0-20201008102752-a2860d99d01b h1:fQAHzTior2Kk15U6Bz79slXo+oJqUjq2c2vh4MFuXXM= diff --git a/api/portainer.go b/api/portainer.go index 021580031..d470d432f 100644 --- a/api/portainer.go +++ b/api/portainer.go @@ -60,7 +60,7 @@ type ( TLSCacert *string TLSCert *string TLSKey *string - RollbackDB *bool + RollbackToCE *bool SSL *bool SSLCert *string SSLKey *string @@ -1220,7 +1220,9 @@ const ( // DBVersion is the version number of the Portainer CE database DBVersion = 25 // DBVersionEE is the version number of the Portainer EE database - DBVersionEE = 1 + DBVersionEE = 25 + // Edition is the edition of the Portainer API + Edition = PortainerEE // AssetsServerURL represents the URL of the Portainer asset server AssetsServerURL = "https://portainer-io-assets.sfo2.digitaloceanspaces.com" // MessageOfTheDayURL represents the URL where Portainer EE MOTD message can be retrieved @@ -1733,3 +1735,17 @@ const ( OperationK8sClusterNodeW Authorization = "K8sClusterNodeW" OperationK8sClusterSetupRW Authorization = "K8sClusterSetupRW" ) + +// GetEditionLabel returns the portainer edition label +func (e SoftwareEdition) GetEditionLabel() string { + switch e { + case PortainerCE: + return "CE" + case PortainerBE: + return "BE" + case PortainerEE: + return "EE" + } + + return "CE" +}