Merge pull request #816 from fuweid/13-failpoint-backport

[1.3] backport failpoints(resizeFileError, lackOfDiskSpace) and dmflakey on XFS

.github/workflows/robustness_test.yaml

@@ -12,6 +12,9 @@ jobs:
         with:
           go-version: ${{ steps.goversion.outputs.goversion }}
       - run: |
+          set -euo pipefail
+          sudo apt-get install -y dmsetup xfsprogs
+
           make gofail-enable
           # build bbolt with failpoint
           go install ./cmd/bbolt

db.go

@@ -1159,6 +1159,8 @@ func (db *DB) grow(sz int) error {
 	// https://github.com/boltdb/bolt/issues/284
 	if !db.NoGrowSync && !db.readOnly {
 		if runtime.GOOS != "windows" {
+			// gofail: var resizeFileError string
+			// return errors.New(resizeFileError)
 			if err := db.file.Truncate(int64(sz)); err != nil {
 				return fmt.Errorf("file resize error: %s", err)
 			}

tests/dmflakey/dmflakey.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+	"path"
 	"path/filepath"
 	"strings"
 	"time"
@@ -89,9 +90,9 @@ const (
 // The device-mapper device will be /dev/mapper/$flakeyDevice. And the filesystem
 // image will be created at $dataStorePath/$flakeyDevice.img. By default, the
 // device is available for 2 minutes and size is 10 GiB.
-func InitFlakey(flakeyDevice, dataStorePath string, fsType FSType) (_ Flakey, retErr error) {
+func InitFlakey(flakeyDevice, dataStorePath string, fsType FSType, mkfsOpt string) (_ Flakey, retErr error) {
 	imgPath := filepath.Join(dataStorePath, fmt.Sprintf("%s.img", flakeyDevice))
-	if err := createEmptyFSImage(imgPath, fsType); err != nil {
+	if err := createEmptyFSImage(imgPath, fsType, mkfsOpt); err != nil {
 		return nil, err
 	}
 	defer func() {
@@ -275,7 +276,7 @@ func (f *flakey) Teardown() error {
 
 // createEmptyFSImage creates empty filesystem on dataStorePath folder with
 // default size - 10 GiB.
-func createEmptyFSImage(imgPath string, fsType FSType) error {
+func createEmptyFSImage(imgPath string, fsType FSType, mkfsOpt string) error {
 	if err := validateFSType(fsType); err != nil {
 		return err
 	}
@@ -289,6 +290,10 @@ func createEmptyFSImage(imgPath string, fsType FSType) error {
 		return fmt.Errorf("failed to create image because %s already exists", imgPath)
 	}
 
+	if err := os.MkdirAll(path.Dir(imgPath), 0600); err != nil {
+		return fmt.Errorf("failed to ensure parent directory %s: %w", path.Dir(imgPath), err)
+	}
+
 	f, err := os.Create(imgPath)
 	if err != nil {
 		return fmt.Errorf("failed to create image %s: %w", imgPath, err)
@@ -303,10 +308,16 @@ func createEmptyFSImage(imgPath string, fsType FSType) error {
 			imgPath, defaultImgSize, err)
 	}
 
-	output, err := exec.Command(mkfs, imgPath).CombinedOutput()
+	args := []string{imgPath}
+	if mkfsOpt != "" {
+		splitArgs := strings.Split(mkfsOpt, " ")
+		args = append(splitArgs, imgPath)
+	}
+
+	output, err := exec.Command(mkfs, args...).CombinedOutput()
 	if err != nil {
-		return fmt.Errorf("failed to mkfs.%s on %s (out: %s): %w",
-			fsType, imgPath, string(output), err)
+		return fmt.Errorf("failed to mkfs on %s (%s %v) (out: %s): %w",
+			imgPath, mkfs, args, string(output), err)
 	}
 	return nil
 }

tests/dmflakey/dmflakey_test.go

@@ -26,31 +26,35 @@ func TestMain(m *testing.M) {
 }
 
 func TestBasic(t *testing.T) {
-	tmpDir := t.TempDir()
+	for _, fsType := range []FSType{FSTypeEXT4, FSTypeXFS} {
+		t.Run(string(fsType), func(t *testing.T) {
+			tmpDir := t.TempDir()
 
-	flakey, err := InitFlakey("go-dmflakey", tmpDir, FSTypeEXT4)
-	require.NoError(t, err, "init flakey")
-	defer func() {
-		assert.NoError(t, flakey.Teardown())
-	}()
+			flakey, err := InitFlakey("go-dmflakey", tmpDir, fsType, "")
+			require.NoError(t, err, "init flakey")
+			defer func() {
+				assert.NoError(t, flakey.Teardown())
+			}()
 
-	target := filepath.Join(tmpDir, "root")
-	require.NoError(t, os.MkdirAll(target, 0600))
+			target := filepath.Join(tmpDir, "root")
+			require.NoError(t, os.MkdirAll(target, 0600))
 
-	require.NoError(t, mount(target, flakey.DevicePath(), ""))
-	defer func() {
-		assert.NoError(t, unmount(target))
-	}()
+			require.NoError(t, mount(target, flakey.DevicePath(), ""))
+			defer func() {
+				assert.NoError(t, unmount(target))
+			}()
 
-	file := filepath.Join(target, "test")
-	assert.NoError(t, writeFile(file, []byte("hello, world"), 0600, true))
+			file := filepath.Join(target, "test")
+			assert.NoError(t, writeFile(file, []byte("hello, world"), 0600, true))
 
-	assert.NoError(t, unmount(target))
+			assert.NoError(t, unmount(target))
 
-	assert.NoError(t, flakey.Teardown())
+			assert.NoError(t, flakey.Teardown())
+		})
+	}
 }
 
-func TestDropWrites(t *testing.T) {
+func TestDropWritesExt4(t *testing.T) {
 	flakey, root := initFlakey(t, FSTypeEXT4)
 
 	// commit=1000 is to delay commit triggered by writeback thread
@@ -82,7 +86,7 @@ func TestDropWrites(t *testing.T) {
 	assert.True(t, errors.Is(err, os.ErrNotExist))
 }
 
-func TestErrorWrites(t *testing.T) {
+func TestErrorWritesExt4(t *testing.T) {
 	flakey, root := initFlakey(t, FSTypeEXT4)
 
 	// commit=1000 is to delay commit triggered by writeback thread
@@ -114,7 +118,7 @@ func initFlakey(t *testing.T, fsType FSType) (_ Flakey, root string) {
 	target := filepath.Join(tmpDir, "root")
 	require.NoError(t, os.MkdirAll(target, 0600))
 
-	flakey, err := InitFlakey("go-dmflakey", tmpDir, FSTypeEXT4)
+	flakey, err := InitFlakey("go-dmflakey", tmpDir, fsType, "")
 	require.NoError(t, err, "init flakey")
 
 	t.Cleanup(func() {

tests/failpoint/db_failpoint_test.go

@@ -209,3 +209,63 @@ func TestIssue72(t *testing.T) {
 func idToBytes(id int) []byte {
 	return []byte(fmt.Sprintf("%010d", id))
 }
+
+func TestFailpoint_ResizeFileFail(t *testing.T) {
+	db := btesting.MustCreateDB(t)
+
+	err := gofail.Enable("resizeFileError", `return("resizeFile somehow failed")`)
+	require.NoError(t, err)
+
+	err = db.Fill([]byte("data"), 1, 10000,
+		func(tx int, k int) []byte { return []byte(fmt.Sprintf("%04d", k)) },
+		func(tx int, k int) []byte { return make([]byte, 100) },
+	)
+
+	require.Error(t, err)
+	require.ErrorContains(t, err, "resizeFile somehow failed")
+
+	// It should work after disabling the failpoint.
+	err = gofail.Disable("resizeFileError")
+	require.NoError(t, err)
+	db.MustClose()
+	db.MustReopen()
+
+	err = db.Fill([]byte("data"), 1, 10000,
+		func(tx int, k int) []byte { return []byte(fmt.Sprintf("%04d", k)) },
+		func(tx int, k int) []byte { return make([]byte, 100) },
+	)
+
+	require.NoError(t, err)
+}
+
+func TestFailpoint_LackOfDiskSpace(t *testing.T) {
+	db := btesting.MustCreateDB(t)
+
+	err := gofail.Enable("lackOfDiskSpace", `return("grow somehow failed")`)
+	require.NoError(t, err)
+
+	tx, err := db.Begin(true)
+	require.NoError(t, err)
+
+	err = tx.Commit()
+	require.Error(t, err)
+	require.ErrorContains(t, err, "grow somehow failed")
+
+	err = tx.Rollback()
+	require.Error(t, err)
+	require.ErrorIs(t, err, bolt.ErrTxClosed)
+
+	// It should work after disabling the failpoint.
+	err = gofail.Disable("lackOfDiskSpace")
+	require.NoError(t, err)
+
+	tx, err = db.Begin(true)
+	require.NoError(t, err)
+
+	err = tx.Commit()
+	require.NoError(t, err)
+
+	err = tx.Rollback()
+	require.Error(t, err)
+	require.ErrorIs(t, err, bolt.ErrTxClosed)
+}