Merge pull request onflow#6503 from onflow/bastian/master-port-intern…

…al-6971

Port internal 6971

cmd/util/cmd/execution-state-extract/cmd.go

@@ -1,8 +1,10 @@
 package extract
 
 import (
+	"compress/gzip"
 	"encoding/hex"
 	"fmt"
+	"io"
 	"os"
 	"path"
 	"runtime/pprof"
@@ -32,6 +34,8 @@ var (
 	flagChain                              string
 	flagNWorker                            int
 	flagNoMigration                        bool
+	flagMigration                          string
+	flagAuthorizationFixes                 string
 	flagNoReport                           bool
 	flagValidateMigration                  bool
 	flagAllowPartialStateFromPayloads      bool
@@ -87,6 +91,12 @@ func init() {
 	Cmd.Flags().BoolVar(&flagNoMigration, "no-migration", false,
 		"don't migrate the state")
 
+	Cmd.Flags().StringVar(&flagMigration, "migration", "cadence-1.0",
+		"migration name. 'cadence-1.0' (default) or 'fix-authorizations'")
+
+	Cmd.Flags().StringVar(&flagAuthorizationFixes, "authorization-fixes", "",
+		"authorization fixes to apply. requires '--migration=fix-authorizations'")
+
 	Cmd.Flags().BoolVar(&flagNoReport, "no-report", false,
 		"don't report the state")
 
@@ -195,7 +205,10 @@ func run(*cobra.Command, []string) {
 		defer pprof.StopCPUProfile()
 	}
 
-	var stateCommitment flow.StateCommitment
+	err := os.MkdirAll(flagOutputDir, 0755)
+	if err != nil {
+		log.Fatal().Err(err).Msgf("cannot create output directory %s", flagOutputDir)
+	}
 
 	if len(flagBlockHash) > 0 && len(flagStateCommitment) > 0 {
 		log.Fatal().Msg("cannot run the command with both block hash and state commitment as inputs, only one of them should be provided")
@@ -219,6 +232,21 @@ func run(*cobra.Command, []string) {
 		log.Fatal().Msg("Both --validate and --diff are enabled, please specify only one (or none) of these")
 	}
 
+	switch flagMigration {
+	case "cadence-1.0":
+		// valid, no-op
+
+	case "fix-authorizations":
+		if flagAuthorizationFixes == "" {
+			log.Fatal().Msg("--migration=fix-authorizations requires --authorization-fixes")
+		}
+
+	default:
+		log.Fatal().Msg("Invalid --migration: got %s, expected 'cadence-1.0' or 'fix-authorizations'")
+	}
+
+	var stateCommitment flow.StateCommitment
+
 	if len(flagBlockHash) > 0 {
 		blockID, err := flow.HexStringToIdentifier(flagBlockHash)
 		if err != nil {
@@ -429,9 +457,29 @@ func run(*cobra.Command, []string) {
 	// Migrate payloads.
 
 	if !flagNoMigration {
-		migrations := newMigrations(log.Logger, flagOutputDir, opts)
+		var migs []migrations.NamedMigration
+
+		switch flagMigration {
+		case "cadence-1.0":
+			migs = newCadence1Migrations(
+				log.Logger,
+				flagOutputDir,
+				opts,
+			)
+
+		case "fix-authorizations":
+			migs = newFixAuthorizationsMigrations(
+				log.Logger,
+				flagAuthorizationFixes,
+				flagOutputDir,
+				opts,
+			)
+
+		default:
+			log.Fatal().Msgf("unknown migration: %s", flagMigration)
+		}
 
-		migration := newMigration(log.Logger, migrations, flagNWorker)
+		migration := newMigration(log.Logger, migs, flagNWorker)
 
 		payloads, err = migration(payloads)
 		if err != nil {
@@ -509,3 +557,35 @@ func ensureCheckpointFileExist(dir string) error {
 
 	return fmt.Errorf("no checkpoint file was found, no root checkpoint file was found in %v, check the --execution-state-dir flag", dir)
 }
+
+func readAuthorizationFixes(path string) migrations.AuthorizationFixes {
+
+	file, err := os.Open(path)
+	if err != nil {
+		log.Fatal().Err(err).Msgf("can't open authorization fixes: %s", path)
+	}
+	defer file.Close()
+
+	var reader io.Reader = file
+	if isGzip(file) {
+		reader, err = gzip.NewReader(file)
+		if err != nil {
+			log.Fatal().Err(err).Msgf("failed to create gzip reader for %s", path)
+		}
+	}
+
+	log.Info().Msgf("Reading authorization fixes from %s ...", path)
+
+	fixes, err := migrations.ReadAuthorizationFixes(reader, nil)
+	if err != nil {
+		log.Fatal().Err(err).Msgf("failed to read authorization fixes %s", path)
+	}
+
+	log.Info().Msgf("Read %d authorization fixes", len(fixes))
+
+	return fixes
+}
+
+func isGzip(file *os.File) bool {
+	return strings.HasSuffix(file.Name(), ".gz")
+}

cmd/util/cmd/execution-state-extract/execution_state_extract.go

@@ -358,13 +358,13 @@ func createTrieFromPayloads(logger zerolog.Logger, payloads []*ledger.Payload) (
 	return newTrie, nil
 }
 
-func newMigrations(
+func newCadence1Migrations(
 	log zerolog.Logger,
 	outputDir string,
 	opts migrators.Options,
 ) []migrators.NamedMigration {
 
-	log.Info().Msg("initializing migrations")
+	log.Info().Msg("initializing Cadence 1.0 migrations ...")
 
 	rwf := reporters.NewReportFileWriterFactory(outputDir, log)
 
@@ -394,3 +394,43 @@ func newMigrations(
 
 	return namedMigrations
 }
+
+func newFixAuthorizationsMigrations(
+	log zerolog.Logger,
+	authorizationFixesPath string,
+	outputDir string,
+	opts migrators.Options,
+) []migrators.NamedMigration {
+
+	log.Info().Msg("initializing authorization fix migrations ...")
+
+	rwf := reporters.NewReportFileWriterFactory(outputDir, log)
+
+	authorizationFixes := readAuthorizationFixes(authorizationFixesPath)
+
+	namedMigrations := migrators.NewFixAuthorizationsMigrations(
+		log,
+		rwf,
+		authorizationFixes,
+		opts,
+	)
+
+	// At the end, fix up storage-used discrepancies
+	namedMigrations = append(
+		namedMigrations,
+		migrators.NamedMigration{
+			Name: "account-usage-migration",
+			Migrate: migrators.NewAccountBasedMigration(
+				log,
+				opts.NWorker,
+				[]migrators.AccountBasedMigration{
+					migrators.NewAccountUsageMigration(rwf),
+				},
+			),
+		},
+	)
+
+	log.Info().Msg("initialized migrations")
+
+	return namedMigrations
+}