From 6e12e088d423604ad0ec0d4377e80d319d59140a Mon Sep 17 00:00:00 2001 From: Benjamin Wang Date: Sun, 7 May 2023 14:07:14 +0800 Subject: [PATCH 1/3] cmd: migrate 'surgery copy-page' command to cobra style command Signed-off-by: Benjamin Wang --- ...nd_surgery_cobra.go => command_surgery.go} | 52 ++++++- ..._cobra_test.go => command_surgery_test.go} | 37 +++++ cmd/bbolt/main.go | 2 - cmd/bbolt/surgery_commands.go | 146 ------------------ cmd/bbolt/surgery_commands_test.go | 47 ------ 5 files changed, 86 insertions(+), 198 deletions(-) rename cmd/bbolt/{command_surgery_cobra.go => command_surgery.go} (86%) rename cmd/bbolt/{command_surgery_cobra_test.go => command_surgery_test.go} (93%) delete mode 100644 cmd/bbolt/surgery_commands.go delete mode 100644 cmd/bbolt/surgery_commands_test.go diff --git a/cmd/bbolt/command_surgery_cobra.go b/cmd/bbolt/command_surgery.go similarity index 86% rename from cmd/bbolt/command_surgery_cobra.go rename to cmd/bbolt/command_surgery.go index 50ea686cf..bb930d9e5 100644 --- a/cmd/bbolt/command_surgery_cobra.go +++ b/cmd/bbolt/command_surgery.go @@ -26,6 +26,7 @@ func newSurgeryCobraCommand() *cobra.Command { surgeryCmd.AddCommand(newSurgeryRevertMetaPageCommand()) surgeryCmd.AddCommand(newSurgeryCopyPageCommand()) + surgeryCmd.AddCommand(newSurgeryClearPageCommand()) surgeryCmd.AddCommand(newSurgeryClearPageElementsCommand()) surgeryCmd.AddCommand(newSurgeryFreelistCommand()) @@ -183,6 +184,54 @@ func (o *surgeryClearPageElementsOptions) Validate() error { return nil } +func newSurgeryClearPageCommand() *cobra.Command { + cfg := defaultSurgeryOptions() + clearPageCmd := &cobra.Command{ + Use: "clear-page [options]", + Short: "Clears all elements from the given page, which can be a branch or leaf page", + Args: func(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return errors.New("db file path not provided") + } + if len(args) > 1 { + return errors.New("too many arguments") + } + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + return surgeryClearPageFunc(args[0], cfg) + }, + } + + clearPageCmd.Flags().StringVar(&cfg.surgeryTargetDBFilePath, "output", "", "path to the target db file") + clearPageCmd.Flags().Uint64VarP(&cfg.surgeryPageId, "pageId", "", 0, "page id") + + return clearPageCmd +} + +func surgeryClearPageFunc(srcDBPath string, cfg surgeryOptions) error { + if err := common.CopyFile(srcDBPath, cfg.surgeryTargetDBFilePath); err != nil { + return fmt.Errorf("[clear-page] copy file failed: %w", err) + } + + if cfg.surgeryPageId < 2 { + return fmt.Errorf("the pageId must be at least 2, but got %d", cfg.surgeryPageId) + } + + needAbandonFreelist, err := surgeon.ClearPage(cfg.surgeryTargetDBFilePath, common.Pgid(cfg.surgeryPageId)) + if err != nil { + return fmt.Errorf("clear-page command failed: %w", err) + } + + if needAbandonFreelist { + fmt.Fprintf(os.Stdout, "WARNING: The clearing has abandoned some pages that are not yet referenced from free list.\n") + fmt.Fprintf(os.Stdout, "Please consider executing `./bbolt surgery abandon-freelist ...`\n") + } + + fmt.Fprintf(os.Stdout, "The page (%d) was cleared\n", cfg.surgeryPageId) + return nil +} + func newSurgeryClearPageElementsCommand() *cobra.Command { var o surgeryClearPageElementsOptions clearElementCmd := &cobra.Command{ @@ -227,9 +276,6 @@ func surgeryClearPageElementFunc(srcDBPath string, cfg surgeryClearPageElementsO return nil } -// TODO(ahrtr): add `bbolt surgery freelist rebuild/check ...` commands, -// and move all `surgery freelist` commands into a separate file, -// e.g command_surgery_freelist.go. func newSurgeryFreelistCommand() *cobra.Command { cmd := &cobra.Command{ Use: "freelist ", diff --git a/cmd/bbolt/command_surgery_cobra_test.go b/cmd/bbolt/command_surgery_test.go similarity index 93% rename from cmd/bbolt/command_surgery_cobra_test.go rename to cmd/bbolt/command_surgery_test.go index 5c506a91a..2c99e21cd 100644 --- a/cmd/bbolt/command_surgery_cobra_test.go +++ b/cmd/bbolt/command_surgery_test.go @@ -99,6 +99,43 @@ func TestSurgery_CopyPage(t *testing.T) { assert.Equal(t, pageDataWithoutPageId(srcPageId3Data), pageDataWithoutPageId(dstPageId2Data)) } +// TODO(ahrtr): add test case below for `surgery clear-page` command: +// 1. The page is a branch page. All its children should become free pages. +func TestSurgery_ClearPage(t *testing.T) { + pageSize := 4096 + db := btesting.MustCreateDBWithOption(t, &bolt.Options{PageSize: pageSize}) + srcPath := db.Path() + + // Insert some sample data + t.Log("Insert some sample data") + err := db.Fill([]byte("data"), 1, 20, + func(tx int, k int) []byte { return []byte(fmt.Sprintf("%04d", k)) }, + func(tx int, k int) []byte { return make([]byte, 10) }, + ) + require.NoError(t, err) + + defer requireDBNoChange(t, dbData(t, srcPath), srcPath) + + // clear page 3 + t.Log("clear page 3") + rootCmd := main.NewRootCommand() + output := filepath.Join(t.TempDir(), "dstdb") + rootCmd.SetArgs([]string{ + "surgery", "clear-page", srcPath, + "--output", output, + "--pageId", "3", + }) + err = rootCmd.Execute() + require.NoError(t, err) + + t.Log("Verify result") + dstPageId3Data := readPage(t, output, 3, pageSize) + + p := common.LoadPage(dstPageId3Data) + assert.Equal(t, uint16(0), p.Count()) + assert.Equal(t, uint32(0), p.Overflow()) +} + func TestSurgery_ClearPageElements_Without_Overflow(t *testing.T) { testCases := []struct { name string diff --git a/cmd/bbolt/main.go b/cmd/bbolt/main.go index 31d0d62b0..6e2e7310e 100644 --- a/cmd/bbolt/main.go +++ b/cmd/bbolt/main.go @@ -140,8 +140,6 @@ func (m *Main) Run(args ...string) error { return newPagesCommand(m).Run(args[1:]...) case "stats": return newStatsCommand(m).Run(args[1:]...) - case "surgery": - return newSurgeryCommand(m).Run(args[1:]...) default: return ErrUnknownCommand } diff --git a/cmd/bbolt/surgery_commands.go b/cmd/bbolt/surgery_commands.go deleted file mode 100644 index 64b970ae3..000000000 --- a/cmd/bbolt/surgery_commands.go +++ /dev/null @@ -1,146 +0,0 @@ -package main - -import ( - "errors" - "flag" - "fmt" - "os" - "strconv" - "strings" - - "go.etcd.io/bbolt/internal/common" - "go.etcd.io/bbolt/internal/surgeon" -) - -// surgeryCommand represents the "surgery" command execution. -type surgeryCommand struct { - baseCommand - - srcPath string - dstPath string -} - -// newSurgeryCommand returns a SurgeryCommand. -func newSurgeryCommand(m *Main) *surgeryCommand { - c := &surgeryCommand{} - c.baseCommand = m.baseCommand - return c -} - -// Run executes the `surgery` program. -func (cmd *surgeryCommand) Run(args ...string) error { - // Require a command at the beginning. - if len(args) == 0 || strings.HasPrefix(args[0], "-") { - fmt.Fprintln(cmd.Stderr, cmd.Usage()) - return ErrUsage - } - - // Execute command. - switch args[0] { - case "help": - fmt.Fprintln(cmd.Stderr, cmd.Usage()) - return ErrUsage - case "clear-page": - return newClearPageCommand(cmd).Run(args[1:]...) - default: - return ErrUnknownCommand - } -} - -func (cmd *surgeryCommand) parsePathsAndCopyFile(fs *flag.FlagSet) error { - // Require database paths. - cmd.srcPath = fs.Arg(0) - if cmd.srcPath == "" { - return ErrPathRequired - } - - cmd.dstPath = fs.Arg(1) - if cmd.dstPath == "" { - return errors.New("output file required") - } - - // Copy database from SrcPath to DstPath - if err := common.CopyFile(cmd.srcPath, cmd.dstPath); err != nil { - return fmt.Errorf("failed to copy file: %w", err) - } - - return nil -} - -// Usage returns the help message. -func (cmd *surgeryCommand) Usage() string { - return strings.TrimLeft(` -Surgery is a command for performing low level update on bbolt databases. - -Usage: - - bbolt surgery command [arguments] - -The commands are: - help print this screen - clear-page clear all elements at the given pageId - -Use "bbolt surgery [command] -h" for more information about a command. -`, "\n") -} - -// clearPageCommand represents the "surgery clear-page" command execution. -type clearPageCommand struct { - *surgeryCommand -} - -// newClearPageCommand returns a clearPageCommand. -func newClearPageCommand(m *surgeryCommand) *clearPageCommand { - c := &clearPageCommand{} - c.surgeryCommand = m - return c -} - -// Run executes the command. -func (cmd *clearPageCommand) Run(args ...string) error { - // Parse flags. - fs := flag.NewFlagSet("", flag.ContinueOnError) - help := fs.Bool("h", false, "") - if err := fs.Parse(args); err != nil { - return err - } else if *help { - fmt.Fprintln(cmd.Stderr, cmd.Usage()) - return ErrUsage - } - - if err := cmd.parsePathsAndCopyFile(fs); err != nil { - return fmt.Errorf("clearPageCommand failed to parse paths and copy file: %w", err) - } - - // Read page id. - pageId, err := strconv.ParseUint(fs.Arg(2), 10, 64) - if err != nil { - return err - } - - needAbandonFreelist, err := surgeon.ClearPage(cmd.dstPath, common.Pgid(pageId)) - if err != nil { - return fmt.Errorf("clearPageCommand failed: %w", err) - } - - if needAbandonFreelist { - fmt.Fprintf(os.Stdout, "WARNING: The clearing has abandoned some pages that are not yet referenced from free list.\n") - fmt.Fprintf(os.Stdout, "Please consider executing `./bbolt surgery abandon-freelist ...`\n") - } - - fmt.Fprintf(cmd.Stdout, "Page (%d) was cleared\n", pageId) - return nil -} - -// Usage returns the help message. -func (cmd *clearPageCommand) Usage() string { - return strings.TrimLeft(` -usage: bolt surgery clear-page SRC DST pageId - -ClearPage copies the database file at SRC to a newly created database -file at DST. Afterwards, it clears all elements in the page at pageId -in DST. - -The original database is left untouched. -`, "\n") -} diff --git a/cmd/bbolt/surgery_commands_test.go b/cmd/bbolt/surgery_commands_test.go deleted file mode 100644 index af3b1393e..000000000 --- a/cmd/bbolt/surgery_commands_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package main_test - -import ( - "fmt" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - bolt "go.etcd.io/bbolt" - "go.etcd.io/bbolt/internal/btesting" - "go.etcd.io/bbolt/internal/common" -) - -// TODO(ahrtr): add test case below for `surgery clear-page` command: -// 1. The page is a branch page. All its children should become free pages. -func TestSurgery_ClearPage(t *testing.T) { - pageSize := 4096 - db := btesting.MustCreateDBWithOption(t, &bolt.Options{PageSize: pageSize}) - srcPath := db.Path() - - // Insert some sample data - t.Log("Insert some sample data") - err := db.Fill([]byte("data"), 1, 20, - func(tx int, k int) []byte { return []byte(fmt.Sprintf("%04d", k)) }, - func(tx int, k int) []byte { return make([]byte, 10) }, - ) - require.NoError(t, err) - - defer requireDBNoChange(t, dbData(t, srcPath), srcPath) - - // clear page 3 - t.Log("clear page 3") - dstPath := filepath.Join(t.TempDir(), "dstdb") - m := NewMain() - err = m.Run("surgery", "clear-page", srcPath, dstPath, "3") - require.NoError(t, err) - - // The page 2 should have exactly the same data as page 3. - t.Log("Verify result") - dstPageId3Data := readPage(t, dstPath, 3, pageSize) - - p := common.LoadPage(dstPageId3Data) - assert.Equal(t, uint16(0), p.Count()) - assert.Equal(t, uint32(0), p.Overflow()) -} From 383d99079464699fb07cbe01d03ba675e5e87804 Mon Sep 17 00:00:00 2001 From: Benjamin Wang Date: Tue, 9 May 2023 08:38:50 +0800 Subject: [PATCH 2/3] cmd: wrap 'surgery clear-page' options into surgeryClearPageOptions Signed-off-by: Benjamin Wang --- cmd/bbolt/command_surgery.go | 62 +++++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 23 deletions(-) diff --git a/cmd/bbolt/command_surgery.go b/cmd/bbolt/command_surgery.go index bb930d9e5..59e525c1c 100644 --- a/cmd/bbolt/command_surgery.go +++ b/cmd/bbolt/command_surgery.go @@ -160,21 +160,17 @@ func surgeryCopyPageFunc(srcDBPath string, cfg surgeryCopyPageOptions) error { return nil } -type surgeryClearPageElementsOptions struct { +type surgeryClearPageOptions struct { surgeryBaseOptions - pageId uint64 - startElementIdx int - endElementIdx int + pageId uint64 } -func (o *surgeryClearPageElementsOptions) AddFlags(fs *pflag.FlagSet) { +func (o *surgeryClearPageOptions) AddFlags(fs *pflag.FlagSet) { o.surgeryBaseOptions.AddFlags(fs) - fs.Uint64VarP(&o.pageId, "pageId", "", o.pageId, "page id") - fs.IntVarP(&o.startElementIdx, "from-index", "", o.startElementIdx, "start element index (included) to clear, starting from 0") - fs.IntVarP(&o.endElementIdx, "to-index", "", o.endElementIdx, "end element index (excluded) to clear, starting from 0, -1 means to the end of page") + fs.Uint64VarP(&o.pageId, "pageId", "", o.pageId, "page Id") } -func (o *surgeryClearPageElementsOptions) Validate() error { +func (o *surgeryClearPageOptions) Validate() error { if err := o.surgeryBaseOptions.Validate(); err != nil { return err } @@ -185,7 +181,7 @@ func (o *surgeryClearPageElementsOptions) Validate() error { } func newSurgeryClearPageCommand() *cobra.Command { - cfg := defaultSurgeryOptions() + var o surgeryClearPageOptions clearPageCmd := &cobra.Command{ Use: "clear-page [options]", Short: "Clears all elements from the given page, which can be a branch or leaf page", @@ -199,26 +195,22 @@ func newSurgeryClearPageCommand() *cobra.Command { return nil }, RunE: func(cmd *cobra.Command, args []string) error { - return surgeryClearPageFunc(args[0], cfg) + if err := o.Validate(); err != nil { + return err + } + return surgeryClearPageFunc(args[0], o) }, } - - clearPageCmd.Flags().StringVar(&cfg.surgeryTargetDBFilePath, "output", "", "path to the target db file") - clearPageCmd.Flags().Uint64VarP(&cfg.surgeryPageId, "pageId", "", 0, "page id") - + o.AddFlags(clearPageCmd.Flags()) return clearPageCmd } -func surgeryClearPageFunc(srcDBPath string, cfg surgeryOptions) error { - if err := common.CopyFile(srcDBPath, cfg.surgeryTargetDBFilePath); err != nil { +func surgeryClearPageFunc(srcDBPath string, cfg surgeryClearPageOptions) error { + if err := common.CopyFile(srcDBPath, cfg.outputDBFilePath); err != nil { return fmt.Errorf("[clear-page] copy file failed: %w", err) } - if cfg.surgeryPageId < 2 { - return fmt.Errorf("the pageId must be at least 2, but got %d", cfg.surgeryPageId) - } - - needAbandonFreelist, err := surgeon.ClearPage(cfg.surgeryTargetDBFilePath, common.Pgid(cfg.surgeryPageId)) + needAbandonFreelist, err := surgeon.ClearPage(cfg.outputDBFilePath, common.Pgid(cfg.pageId)) if err != nil { return fmt.Errorf("clear-page command failed: %w", err) } @@ -228,7 +220,31 @@ func surgeryClearPageFunc(srcDBPath string, cfg surgeryOptions) error { fmt.Fprintf(os.Stdout, "Please consider executing `./bbolt surgery abandon-freelist ...`\n") } - fmt.Fprintf(os.Stdout, "The page (%d) was cleared\n", cfg.surgeryPageId) + fmt.Fprintf(os.Stdout, "The page (%d) was cleared\n", cfg.pageId) + return nil +} + +type surgeryClearPageElementsOptions struct { + surgeryBaseOptions + pageId uint64 + startElementIdx int + endElementIdx int +} + +func (o *surgeryClearPageElementsOptions) AddFlags(fs *pflag.FlagSet) { + o.surgeryBaseOptions.AddFlags(fs) + fs.Uint64VarP(&o.pageId, "pageId", "", o.pageId, "page id") + fs.IntVarP(&o.startElementIdx, "from-index", "", o.startElementIdx, "start element index (included) to clear, starting from 0") + fs.IntVarP(&o.endElementIdx, "to-index", "", o.endElementIdx, "end element index (excluded) to clear, starting from 0, -1 means to the end of page") +} + +func (o *surgeryClearPageElementsOptions) Validate() error { + if err := o.surgeryBaseOptions.Validate(); err != nil { + return err + } + if o.pageId < 2 { + return fmt.Errorf("the pageId must be at least 2, but got %d", o.pageId) + } return nil } From 8974e912fb58598c5bded04b69092cfb1877ff2b Mon Sep 17 00:00:00 2001 From: Benjamin Wang Date: Tue, 9 May 2023 08:50:17 +0800 Subject: [PATCH 3/3] cmd: check source db path for all surgery commands Signed-off-by: Benjamin Wang --- cmd/bbolt/command_surgery.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/cmd/bbolt/command_surgery.go b/cmd/bbolt/command_surgery.go index 59e525c1c..9f75603c3 100644 --- a/cmd/bbolt/command_surgery.go +++ b/cmd/bbolt/command_surgery.go @@ -139,6 +139,10 @@ func newSurgeryCopyPageCommand() *cobra.Command { } func surgeryCopyPageFunc(srcDBPath string, cfg surgeryCopyPageOptions) error { + if _, err := checkSourceDBPath(srcDBPath); err != nil { + return err + } + if err := common.CopyFile(srcDBPath, cfg.outputDBFilePath); err != nil { return fmt.Errorf("[copy-page] copy file failed: %w", err) } @@ -206,6 +210,10 @@ func newSurgeryClearPageCommand() *cobra.Command { } func surgeryClearPageFunc(srcDBPath string, cfg surgeryClearPageOptions) error { + if _, err := checkSourceDBPath(srcDBPath); err != nil { + return err + } + if err := common.CopyFile(srcDBPath, cfg.outputDBFilePath); err != nil { return fmt.Errorf("[clear-page] copy file failed: %w", err) } @@ -274,6 +282,10 @@ func newSurgeryClearPageElementsCommand() *cobra.Command { } func surgeryClearPageElementFunc(srcDBPath string, cfg surgeryClearPageElementsOptions) error { + if _, err := checkSourceDBPath(srcDBPath); err != nil { + return err + } + if err := common.CopyFile(srcDBPath, cfg.outputDBFilePath); err != nil { return fmt.Errorf("[clear-page-element] copy file failed: %w", err) } @@ -331,6 +343,10 @@ func newSurgeryFreelistAbandonCommand() *cobra.Command { } func surgeryFreelistAbandonFunc(srcDBPath string, cfg surgeryBaseOptions) error { + if _, err := checkSourceDBPath(srcDBPath); err != nil { + return err + } + if err := common.CopyFile(srcDBPath, cfg.outputDBFilePath); err != nil { return fmt.Errorf("[freelist abandon] copy file failed: %w", err) }