Init support for WASM plugins

go.mod

@@ -34,6 +34,7 @@ require (
  github.com/spf13/cobra v1.8.1
  github.com/spf13/pflag v1.0.5
  github.com/stretchr/testify v1.9.0
+ github.com/tetratelabs/wazero v1.8.0
  go.lsp.dev/jsonrpc2 v0.10.0
  go.lsp.dev/protocol v0.12.0
  go.opentelemetry.io/otel v1.30.0

go.sum

@@ -252,6 +252,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/tetratelabs/wazero v1.8.0 h1:iEKu0d4c2Pd+QSRieYbnQC9yiFlMS9D+Jr0LsRmcF4g=
+github.com/tetratelabs/wazero v1.8.0/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs=
 github.com/vbatts/tar-split v0.11.5 h1:3bHCTIheBm1qFTcgh9oPu+nNBtX+XJIupG/vacinCts=
 github.com/vbatts/tar-split v0.11.5/go.mod h1:yZbwRsSeGjusneWgA781EKej9HF8vme8okylkAeNKLk=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

make/buf/all.mk

@@ -25,6 +25,13 @@ GO_TEST_BINS := $(GO_TEST_BINS) \
  private/bufpkg/bufcheck/internal/cmd/buf-plugin-rpc-ext \
  private/bufpkg/bufcheck/internal/cmd/buf-plugin-duplicate-category \
  private/bufpkg/bufcheck/internal/cmd/buf-plugin-duplicate-rule
+GO_TEST_WASM_BINS := $(GO_TEST_WASM_BINS) \
+ private/bufpkg/bufcheck/internal/cmd/buf-plugin-panic \
+ private/bufpkg/bufcheck/internal/cmd/buf-plugin-suffix \
+ private/bufpkg/bufcheck/internal/cmd/buf-plugin-protovalidate-ext \
+ private/bufpkg/bufcheck/internal/cmd/buf-plugin-rpc-ext \
+ private/bufpkg/bufcheck/internal/cmd/buf-plugin-duplicate-category \
+ private/bufpkg/bufcheck/internal/cmd/buf-plugin-duplicate-rule
 GO_MOD_VERSION := 1.22
 DOCKER_BINS := $(DOCKER_BINS) buf
 FILE_IGNORES := $(FILE_IGNORES) \

make/go/go.mk

@@ -16,6 +16,8 @@ GO_BINS ?=
 # Settable
 GO_TEST_BINS ?=
 # Settable
+GO_TEST_WASM_BINS ?=
+# Settable
 GO_GET_PKGS ?=
 # Settable
 GO_MOD_VERSION ?= 1.21
@@ -142,7 +144,7 @@ build: prebuild ## Run go build.
 pretest::
 
 .PHONY: test
-test: pretest installtest ## Run all go tests.
+test: pretest installtest installtestwasm ## Run all go tests.
  go test $(GO_TEST_FLAGS) $(GOPKGS)
 
 .PHONY: testrace
@@ -203,3 +205,17 @@ endef
 
 $(foreach gobin,$(sort $(GO_TEST_BINS)),$(eval $(call gotestbinfunc,$(gobin))))
 $(foreach gobin,$(sort $(GO_TEST_BINS)),$(eval FILE_IGNORES := $(FILE_IGNORES) $(gobin)/$(notdir $(gobin))))
+
+.PHONY: installtestwasm
+installtestwasm::
+
+define gotestwasmfunc
+.PHONY: installtestwasm$(notdir $(1))
+installtestwasm$(notdir $(1)):
+ GOOS=wasip1 GOARCH=wasm go build -o $(GOBIN)/$(notdir $(1)).wasm ./$(1)
+
+installtestwasm:: installtestwasm$(notdir $(1))
+endef
+
+$(foreach gobin,$(sort $(GO_TEST_WASM_BINS)),$(eval $(call gotestwasmfunc,$(gobin))))
+$(foreach gobin,$(sort $(GO_TEST_WASM_BINS)),$(eval FILE_IGNORES := $(FILE_IGNORES) $(gobin)/$(notdir $(gobin))))

private/buf/bufcli/cache.go

@@ -103,6 +103,10 @@ var (
  //
  // Normalized.
  v3CacheModuleLockRelDirPath = normalpath.Join("v3", "modulelocks")
+ //v3CachePluginsRelDirPath is the relative path to the plugins cache directory in its newest iteration.
+ //
+ // Normalized.
+ v3CachePluginsRelDirPath = normalpath.Join("v3", "plugins")
 )
 
 // NewModuleDataProvider returns a new ModuleDataProvider while creating the
@@ -135,6 +139,15 @@ func NewCommitProvider(container appext.Container) (bufmodule.CommitProvider, er
  )
 }
 
+// CreatePluginCacheDir creates the cache directory for plugins.
+func CreatePluginCacheDir(container appext.Container) (string, error) {
+ if err := createCacheDir(container.CacheDirPath(), v3CachePluginsRelDirPath); err != nil {
+ return "", err
+ }
+ fullCacheDirPath := normalpath.Join(container.CacheDirPath(), v3CachePluginsRelDirPath)
+ return fullCacheDirPath, nil
+}
+
 // newWKTStore returns a new bufwktstore.Store while creating the required cache directories.
 func newWKTStore(container appext.Container) (bufwktstore.Store, error) {
  if err := createCacheDir(container.CacheDirPath(), v3CacheWKTRelDirPath); err != nil {

private/buf/buflsp/buflsp.go

@@ -58,7 +58,12 @@ func Serve(
  }
 
  tracer := tracing.NewTracer(container.Tracer())
- checkClient, err := bufcheck.NewClient(container.Logger(), tracer, bufcheck.NewRunnerProvider(command.NewRunner()), bufcheck.ClientWithStderr(container.Stderr()))
+ checkClient, err := bufcheck.NewClient(
+ container.Logger(),
+ tracer,
+ bufcheck.NewRunnerProvider(command.NewRunner()),
+ bufcheck.ClientWithStderr(container.Stderr()),
+ )
  if err != nil {
  return nil, err
  }

private/buf/bufmigrate/bufmigrate.go

@@ -21,6 +21,7 @@ import (
 
  "github.com/bufbuild/buf/private/bufpkg/bufconfig"
  "github.com/bufbuild/buf/private/bufpkg/bufmodule"
+ "github.com/bufbuild/buf/private/bufpkg/bufwasm"
  "github.com/bufbuild/buf/private/pkg/command"
  "github.com/bufbuild/buf/private/pkg/normalpath"
  "github.com/bufbuild/buf/private/pkg/storage"
@@ -80,13 +81,15 @@ func NewMigrator(
  logger *zap.Logger,
  tracer tracing.Tracer,
  runner command.Runner,
+ wasmRuntime bufwasm.Runtime,
  moduleKeyProvider bufmodule.ModuleKeyProvider,
  commitProvider bufmodule.CommitProvider,
 ) Migrator {
  return newMigrator(
  logger,
  tracer,
  runner,
+ wasmRuntime,
  moduleKeyProvider,
  commitProvider,
  )

private/buf/bufmigrate/migrate_builder.go

@@ -22,6 +22,7 @@ import (
 
  "github.com/bufbuild/buf/private/bufpkg/bufconfig"
  "github.com/bufbuild/buf/private/bufpkg/bufmodule"
+ "github.com/bufbuild/buf/private/bufpkg/bufwasm"
  "github.com/bufbuild/buf/private/pkg/command"
  "github.com/bufbuild/buf/private/pkg/normalpath"
  "github.com/bufbuild/buf/private/pkg/slicesext"
@@ -38,6 +39,7 @@ type migrateBuilder struct {
  logger *zap.Logger
  tracer tracing.Tracer
  runner command.Runner
+ wasmRuntime bufwasm.Runtime
  commitProvider bufmodule.CommitProvider
  bucket storage.ReadBucket
  destinationDirPath string
@@ -59,6 +61,7 @@ func newMigrateBuilder(
  logger *zap.Logger,
  tracer tracing.Tracer,
  runner command.Runner,
+ wasmRuntime bufwasm.Runtime,
  commitProvider bufmodule.CommitProvider,
  bucket storage.ReadBucket,
  destinationDirPath string,
@@ -67,6 +70,7 @@ func newMigrateBuilder(
  logger: logger,
  tracer: tracer,
  runner: runner,
+ wasmRuntime: wasmRuntime,
  commitProvider: commitProvider,
  bucket: bucket,
  destinationDirPath: destinationDirPath,
@@ -272,11 +276,11 @@ func (m *migrateBuilder) addModule(ctx context.Context, moduleDirPath string) (r
  if err != nil {
  return err
  }
- lintConfigForRoot, err := equivalentLintConfigInV2(ctx, m.logger, m.tracer, m.runner, moduleConfig.LintConfig())
+ lintConfigForRoot, err := equivalentLintConfigInV2(ctx, m.logger, m.tracer, m.runner, m.wasmRuntime, moduleConfig.LintConfig())
  if err != nil {
  return err
  }
- breakingConfigForRoot, err := equivalentBreakingConfigInV2(ctx, m.logger, m.tracer, m.runner, moduleConfig.BreakingConfig())
+ breakingConfigForRoot, err := equivalentBreakingConfigInV2(ctx, m.logger, m.tracer, m.runner, m.wasmRuntime, moduleConfig.BreakingConfig())
  if err != nil {
  return err
  }
@@ -308,11 +312,11 @@ func (m *migrateBuilder) addModule(ctx context.Context, moduleDirPath string) (r
  if err != nil {
  return err
  }
- lintConfig, err := equivalentLintConfigInV2(ctx, m.logger, m.tracer, m.runner, moduleConfig.LintConfig())
+ lintConfig, err := equivalentLintConfigInV2(ctx, m.logger, m.tracer, m.runner, m.wasmRuntime, moduleConfig.LintConfig())
  if err != nil {
  return err
  }
- breakingConfig, err := equivalentBreakingConfigInV2(ctx, m.logger, m.tracer, m.runner, moduleConfig.BreakingConfig())
+ breakingConfig, err := equivalentBreakingConfigInV2(ctx, m.logger, m.tracer, m.runner, m.wasmRuntime, moduleConfig.BreakingConfig())
  if err != nil {
  return err
  }

private/buf/bufmigrate/migrator.go

@@ -26,6 +26,7 @@ import (
  "github.com/bufbuild/buf/private/bufpkg/bufcheck"
  "github.com/bufbuild/buf/private/bufpkg/bufconfig"
  "github.com/bufbuild/buf/private/bufpkg/bufmodule"
+ "github.com/bufbuild/buf/private/bufpkg/bufwasm"
  "github.com/bufbuild/buf/private/pkg/command"
  "github.com/bufbuild/buf/private/pkg/normalpath"
  "github.com/bufbuild/buf/private/pkg/slicesext"
@@ -41,6 +42,7 @@ type migrator struct {
  logger *zap.Logger
  tracer tracing.Tracer
  runner command.Runner
+ wasmRuntime bufwasm.Runtime
  moduleKeyProvider bufmodule.ModuleKeyProvider
  commitProvider bufmodule.CommitProvider
 }
@@ -49,13 +51,15 @@ func newMigrator(
  logger *zap.Logger,
  tracer tracing.Tracer,
  runner command.Runner,
+ wasmRuntime bufwasm.Runtime,
  moduleKeyProvider bufmodule.ModuleKeyProvider,
  commitProvider bufmodule.CommitProvider,
 ) *migrator {
  return &migrator{
  logger: logger,
  tracer: tracer,
  runner: runner,
+ wasmRuntime: wasmRuntime,
  moduleKeyProvider: moduleKeyProvider,
  commitProvider: commitProvider,
  }
@@ -144,6 +148,7 @@ func (m *migrator) getMigrateBuilder(
  m.logger,
  m.tracer,
  m.runner,
+ m.wasmRuntime,
  m.commitProvider,
  bucket,
  destinationDirPath,
@@ -652,13 +657,15 @@ func equivalentLintConfigInV2(
  logger *zap.Logger,
  tracer tracing.Tracer,
  runner command.Runner,
+ wasmRuntime bufwasm.Runtime,
  lintConfig bufconfig.LintConfig,
 ) (bufconfig.LintConfig, error) {
  equivalentCheckConfigV2, err := equivalentCheckConfigInV2(
  ctx,
  logger,
  tracer,
  runner,
+ wasmRuntime,
  check.RuleTypeLint,
  lintConfig,
  )
@@ -681,13 +688,15 @@ func equivalentBreakingConfigInV2(
  logger *zap.Logger,
  tracer tracing.Tracer,
  runner command.Runner,
+ wasmRuntime bufwasm.Runtime,
  breakingConfig bufconfig.BreakingConfig,
 ) (bufconfig.BreakingConfig, error) {
  equivalentCheckConfigV2, err := equivalentCheckConfigInV2(
  ctx,
  logger,
  tracer,
  runner,
+ wasmRuntime,
  check.RuleTypeBreaking,
  breakingConfig,
  )
@@ -707,12 +716,17 @@ func equivalentCheckConfigInV2(
  logger *zap.Logger,
  tracer tracing.Tracer,
  runner command.Runner,
+ wasmRuntime bufwasm.Runtime,
  ruleType check.RuleType,
  checkConfig bufconfig.CheckConfig,
 ) (bufconfig.CheckConfig, error) {
  // No need for custom lint/breaking plugins since there's no plugins to migrate from <=v1.
  // TODO: If we ever need v3, then we will have to deal with this.
- client, err := bufcheck.NewClient(logger, tracer, bufcheck.NewRunnerProvider(runner))
+ client, err := bufcheck.NewClient(
+ logger,
+ tracer,
+ bufcheck.NewRunnerProvider(runner, bufcheck.RunnerProviderWithWASMRuntime(wasmRuntime)),
+ )
  if err != nil {
  return nil, err
  }

private/buf/cmd/buf/command/breaking/breaking.go

@@ -25,13 +25,15 @@ import (
  "github.com/bufbuild/buf/private/bufpkg/bufanalysis"
  "github.com/bufbuild/buf/private/bufpkg/bufcheck"
  "github.com/bufbuild/buf/private/bufpkg/bufimage"
+ "github.com/bufbuild/buf/private/bufpkg/bufwasm"
  "github.com/bufbuild/buf/private/pkg/app/appcmd"
  "github.com/bufbuild/buf/private/pkg/app/appext"
  "github.com/bufbuild/buf/private/pkg/command"
  "github.com/bufbuild/buf/private/pkg/slicesext"
  "github.com/bufbuild/buf/private/pkg/stringutil"
  "github.com/bufbuild/buf/private/pkg/tracing"
  "github.com/spf13/pflag"
+ "go.uber.org/multierr"
 )
 
 const (
@@ -145,7 +147,7 @@ func run(
  ctx context.Context,
  container appext.Container,
  flags *flags,
-) error {
+) (retErr error) {
  if err := bufcli.ValidateRequiredFlag(againstFlagName, flags.Against); err != nil {
  return err
  }
@@ -206,10 +208,30 @@ func run(
  len(againstImageWithConfigs),
  )
  }
+ pluginCacheDir, err := bufcli.CreatePluginCacheDir(container)
+ if err != nil {
+ return err
+ }
+ wasmRuntime, err := bufwasm.NewRuntime(
+ ctx,
+ bufwasm.WithLocalCacheDir(pluginCacheDir),
+ )
+ if err != nil {
+ return err
+ }
+ defer func() { retErr = multierr.Append(retErr, wasmRuntime.Release(ctx)) }()
  tracer := tracing.NewTracer(container.Tracer())
  var allFileAnnotations []bufanalysis.FileAnnotation
  for i, imageWithConfig := range imageWithConfigs {
- client, err := bufcheck.NewClient(container.Logger(), tracer, bufcheck.NewRunnerProvider(command.NewRunner()), bufcheck.ClientWithStderr(container.Stderr()))
+ client, err := bufcheck.NewClient(
+ container.Logger(),
+ tracer,
+ bufcheck.NewRunnerProvider(
+ command.NewRunner(),
+ bufcheck.RunnerProviderWithWASMRuntime(wasmRuntime),
+ ),
+ bufcheck.ClientWithStderr(container.Stderr()),
+ )
  if err != nil {
  return err
  }

private/buf/cmd/buf/command/config/configmigrate/configmigrate.go

@@ -19,6 +19,7 @@ import (
 
  "github.com/bufbuild/buf/private/buf/bufcli"
  "github.com/bufbuild/buf/private/buf/bufmigrate"
+ "github.com/bufbuild/buf/private/bufpkg/bufwasm"
  "github.com/bufbuild/buf/private/pkg/app/appcmd"
  "github.com/bufbuild/buf/private/pkg/app/appext"
  "github.com/bufbuild/buf/private/pkg/command"
@@ -111,6 +112,10 @@ func run(
  flags *flags,
 ) error {
  runner := command.NewRunner()
+ wasmRuntime, err := bufwasm.NewRuntime(ctx)
+ if err != nil {
+ return err
+ }
  moduleKeyProvider, err := bufcli.NewModuleKeyProvider(container)
  if err != nil {
  return err
@@ -130,6 +135,7 @@ func run(
  container.Logger(),
  tracing.NewTracer(container.Tracer()),
  runner,
+ wasmRuntime,
  moduleKeyProvider,
  commitProvider,
  )