wasi preview 2 support (#4027)

* all: wasip2 support

Co-authored-by: Randy Reddig <[email protected]>

.github/workflows/linux.yml

@@ -134,17 +134,19 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+        with:
+          submodules: true
       - name: Install Go
         uses: actions/setup-go@v5
         with:
           go-version: '1.22'
           cache: true
       - name: Install wasmtime
-        run: |
-          mkdir -p $HOME/.wasmtime $HOME/.wasmtime/bin
-          curl https://github.com/bytecodealliance/wasmtime/releases/download/v14.0.4/wasmtime-v14.0.4-x86_64-linux.tar.xz -o wasmtime-v14.0.4-x86_64-linux.tar.xz -SfL
-          tar -C $HOME/.wasmtime/bin --wildcards -xf wasmtime-v14.0.4-x86_64-linux.tar.xz --strip-components=1 wasmtime-v14.0.4-x86_64-linux/*
-          echo "$HOME/.wasmtime/bin" >> $GITHUB_PATH
+        uses: bytecodealliance/actions/wasmtime/setup@v1
+        with:
+          version: "19.0.1"
+      - name: Install wasm-tools
+        uses: bytecodealliance/actions/wasm-tools/setup@v1
       - name: Download release artifact
         uses: actions/download-artifact@v4
         with:
@@ -154,8 +156,8 @@ jobs:
           mkdir -p ~/lib
           tar -C ~/lib -xf tinygo.linux-amd64.tar.gz
           ln -s ~/lib/tinygo/bin/tinygo ~/go/bin/tinygo
-      - run: make tinygo-test-wasi-fast
       - run: make tinygo-test-wasip1-fast
+      - run: make tinygo-test-wasip2-fast
       - run: make smoketest
   assert-test-linux:
     # Run all tests that can run on Linux, with LLVM assertions enabled to catch
@@ -187,11 +189,11 @@ jobs:
         with:
           node-version: '18'
       - name: Install wasmtime
-        run: |
-          mkdir -p $HOME/.wasmtime $HOME/.wasmtime/bin
-          curl -L https://github.com/bytecodealliance/wasmtime/releases/download/v14.0.4/wasmtime-v14.0.4-x86_64-linux.tar.xz -o wasmtime-v14.0.4-x86_64-linux.tar.xz -SfL
-          tar -C $HOME/.wasmtime/bin --wildcards -xf wasmtime-v14.0.4-x86_64-linux.tar.xz --strip-components=1 wasmtime-v14.0.4-x86_64-linux/*
-          echo "$HOME/.wasmtime/bin" >> $GITHUB_PATH
+        uses: bytecodealliance/actions/wasmtime/setup@v1
+        with:
+          version: "19.0.1"
+      - name: Setup `wasm-tools`
+        uses: bytecodealliance/actions/wasm-tools/setup@v1
       - name: Restore LLVM source cache
         uses: actions/cache/restore@v4
         id: cache-llvm-source

.github/workflows/windows.yml

@@ -220,5 +220,5 @@ jobs:
         shell: bash
         working-directory: build
         run: 7z x release.zip -r
-      - name: Test stdlib packages on wasi
-        run: make tinygo-test-wasi-fast TINYGO=$(PWD)/build/tinygo/bin/tinygo
+      - name: Test stdlib packages on wasip1
+        run: make tinygo-test-wasip1-fast TINYGO=$(PWD)/build/tinygo/bin/tinygo

.gitignore

@@ -1,3 +1,6 @@
+go.work
+go.work.sum
+
 docs/_build
 src/device/avr/*.go
 src/device/avr/*.ld
@@ -17,7 +20,7 @@ src/device/kendryte/*.go
 src/device/kendryte/*.s
 src/device/rp/*.go
 src/device/rp/*.s
-vendor
+./vendor
 llvm-build
 llvm-project
 build/*

.gitmodules

@@ -39,3 +39,9 @@
 	path = src/net
 	url = https://github.com/tinygo-org/net.git
 	branch = dev
+[submodule "lib/wasi-cli"]
+	path = lib/wasi-cli
+	url = https://github.com/WebAssembly/wasi-cli
+[submodule "src/vendor/github.com/ydnar/wasm-tools-go"]
+	path = src/vendor/github.com/ydnar/wasm-tools-go
+	url = https://github.com/ydnar/wasm-tools-go.git

GNUmakefile

@@ -268,6 +268,11 @@ lib/wasi-libc/sysroot/lib/wasm32-wasi/libc.a:
 	@if [ ! -e lib/wasi-libc/Makefile ]; then echo "Submodules have not been downloaded. Please download them using:\n  git submodule update --init"; exit 1; fi
 	cd lib/wasi-libc && $(MAKE) -j4 EXTRA_CFLAGS="-O2 -g -DNDEBUG -mnontrapping-fptoint -msign-ext" MALLOC_IMPL=none CC="$(CLANG)" AR=$(LLVM_AR) NM=$(LLVM_NM)
 
+# Generate WASI syscall bindings
+.PHONY: wasi-syscall
+wasi-syscall:
+	wit-bindgen-go generate -o ./src/internal -p internal --versioned ./lib/wasi-cli/wit
+
 # Check for Node.js used during WASM tests.
 NODEJS_VERSION := $(word 1,$(subst ., ,$(shell node -v | cut -c 2-)))
 MIN_NODEJS_VERSION=18
@@ -430,15 +435,36 @@ tinygo-test-wasi:
 	$(TINYGO) test -target wasip1 $(TEST_PACKAGES_FAST) $(TEST_PACKAGES_SLOW) ./tests/runtime_wasi
 tinygo-test-wasip1:
 	GOOS=wasip1 GOARCH=wasm $(TINYGO) test $(TEST_PACKAGES_FAST) $(TEST_PACKAGES_SLOW) ./tests/runtime_wasi
-tinygo-test-wasi-fast:
-	$(TINYGO) test -target wasip1 $(TEST_PACKAGES_FAST) ./tests/runtime_wasi
 tinygo-test-wasip1-fast:
-	GOOS=wasip1 GOARCH=wasm $(TINYGO) test $(TEST_PACKAGES_FAST) ./tests/runtime_wasi
-tinygo-bench-wasi:
+	$(TINYGO) test -target=wasip1 $(TEST_PACKAGES_FAST) ./tests/runtime_wasi
+
+tinygo-test-wasip2-slow:
+	$(TINYGO) test -target=wasip2 $(TEST_PACKAGES_SLOW)
+tinygo-test-wasip2-fast:
+	$(TINYGO) test -target=wasip2 $(TEST_PACKAGES_FAST) ./tests/runtime_wasi
+
+tinygo-test-wasip2-sum-slow:
+	TINYGO=$(TINYGO) \
+	TARGET=wasip2 \
+	TESTOPTS="-x -work" \
+	PACKAGES="$(TEST_PACKAGES_SLOW)" \
+	gotestsum --raw-command -- ./tools/tgtestjson.sh
+tinygo-test-wasip2-sum-fast:
+	TINYGO=$(TINYGO) \
+	TARGET=wasip2 \
+	TESTOPTS="-x -work" \
+	PACKAGES="$(TEST_PACKAGES_FAST)" \
+	gotestsum --raw-command -- ./tools/tgtestjson.sh
+tinygo-bench-wasip1:
 	$(TINYGO) test -target wasip1 -bench . $(TEST_PACKAGES_FAST) $(TEST_PACKAGES_SLOW)
-tinygo-bench-wasi-fast:
+tinygo-bench-wasip1-fast:
 	$(TINYGO) test -target wasip1 -bench . $(TEST_PACKAGES_FAST)
 
+tinygo-bench-wasip2:
+	$(TINYGO) test -target wasip2 -bench . $(TEST_PACKAGES_FAST) $(TEST_PACKAGES_SLOW)
+tinygo-bench-wasip2-fast:
+	$(TINYGO) test -target wasip2 -bench . $(TEST_PACKAGES_FAST)
+
 # Test external packages in a large corpus.
 test-corpus:
 	CGO_CPPFLAGS="$(CGO_CPPFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GO) test $(GOTESTFLAGS) -timeout=1h -buildmode exe -tags byollvm -run TestCorpus . -corpus=testdata/corpus.yaml
@@ -832,6 +858,7 @@ build/release: tinygo gen-device wasi-libc $(if $(filter 1,$(USE_SYSTEM_BINARYEN
 	@mkdir -p build/release/tinygo/lib/wasi-libc/libc-bottom-half/headers
 	@mkdir -p build/release/tinygo/lib/wasi-libc/libc-top-half/musl/arch
 	@mkdir -p build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
+	@mkdir -p build/release/tinygo/lib/wasi-cli/
 	@mkdir -p build/release/tinygo/pkg/thumbv6m-unknown-unknown-eabi-cortex-m0
 	@mkdir -p build/release/tinygo/pkg/thumbv6m-unknown-unknown-eabi-cortex-m0plus
 	@mkdir -p build/release/tinygo/pkg/thumbv7em-unknown-unknown-eabi-cortex-m4
@@ -891,6 +918,7 @@ endif
 	@cp -rp lib/wasi-libc/libc-top-half/musl/src/string             build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
 	@cp -rp lib/wasi-libc/libc-top-half/musl/include                build/release/tinygo/lib/wasi-libc/libc-top-half/musl
 	@cp -rp lib/wasi-libc/sysroot                                   build/release/tinygo/lib/wasi-libc/sysroot
+	@cp -rp lib/wasi-cli/wit                                        build/release/tinygo/lib/wasi-cli/wit
 	@cp -rp llvm-project/compiler-rt/lib/builtins build/release/tinygo/lib/compiler-rt-builtins
 	@cp -rp llvm-project/compiler-rt/LICENSE.TXT  build/release/tinygo/lib/compiler-rt-builtins
 	@cp -rp src                          build/release/tinygo/src

builder/build.go

@@ -840,11 +840,11 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe
 					"--output", result.Executable,
 				)
 
+				wasmopt := goenv.Get("WASMOPT")
 				if config.Options.PrintCommands != nil {
-					config.Options.PrintCommands(goenv.Get("WASMOPT"), args...)
+					config.Options.PrintCommands(wasmopt, args...)
 				}
-
-				cmd := exec.Command(goenv.Get("WASMOPT"), args...)
+				cmd := exec.Command(wasmopt, args...)
 				cmd.Stdout = os.Stdout
 				cmd.Stderr = os.Stderr
 
@@ -854,6 +854,62 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe
 				}
 			}
 
+			// Run wasm-tools for component-model binaries
+			witPackage := strings.ReplaceAll(config.Target.WITPackage, "{root}", goenv.Get("TINYGOROOT"))
+			if config.Options.WITPackage != "" {
+				witPackage = config.Options.WITPackage
+			}
+			witWorld := config.Target.WITWorld
+			if config.Options.WITWorld != "" {
+				witWorld = config.Options.WITWorld
+			}
+			if witPackage != "" && witWorld != "" {
+
+				// wasm-tools component embed -w wasi:cli/command
+				// 		$$(tinygo env TINYGOROOT)/lib/wasi-cli/wit/ main.wasm -o embedded.wasm
+				args := []string{
+					"component",
+					"embed",
+					"-w", witWorld,
+					witPackage,
+					result.Executable,
+					"-o", result.Executable,
+				}
+
+				wasmtools := goenv.Get("WASMTOOLS")
+				if config.Options.PrintCommands != nil {
+					config.Options.PrintCommands(wasmtools, args...)
+				}
+				cmd := exec.Command(wasmtools, args...)
+				cmd.Stdout = os.Stdout
+				cmd.Stderr = os.Stderr
+
+				err := cmd.Run()
+				if err != nil {
+					return fmt.Errorf("wasm-tools failed: %w", err)
+				}
+
+				// wasm-tools component new embedded.wasm -o component.wasm
+				args = []string{
+					"component",
+					"new",
+					result.Executable,
+					"-o", result.Executable,
+				}
+
+				if config.Options.PrintCommands != nil {
+					config.Options.PrintCommands(wasmtools, args...)
+				}
+				cmd = exec.Command(wasmtools, args...)
+				cmd.Stdout = os.Stdout
+				cmd.Stderr = os.Stderr
+
+				err = cmd.Run()
+				if err != nil {
+					return fmt.Errorf("wasm-tools failed: %w", err)
+				}
+			}
+
 			// Print code size if requested.
 			if config.Options.PrintSizes == "short" || config.Options.PrintSizes == "full" {
 				packagePathMap := make(map[string]string, len(lprogram.Packages))

builder/builder_test.go

@@ -34,6 +34,7 @@ func TestClangAttributes(t *testing.T) {
 		"nintendoswitch",
 		"riscv-qemu",
 		"wasip1",
+		"wasip2",
 		"wasm",
 		"wasm-unknown",
 	}
@@ -61,6 +62,7 @@ func TestClangAttributes(t *testing.T) {
 		{GOOS: "windows", GOARCH: "amd64"},
 		{GOOS: "windows", GOARCH: "arm64"},
 		{GOOS: "wasip1", GOARCH: "wasm"},
+		{GOOS: "wasip2", GOARCH: "wasm"},
 	} {
 		name := "GOOS=" + options.GOOS + ",GOARCH=" + options.GOARCH
 		if options.GOARCH == "arm" {

compileopts/options.go

@@ -53,6 +53,8 @@ type Options struct {
 	Monitor         bool
 	BaudRate        int
 	Timeout         time.Duration
+	WITPackage      string // pass through to wasm-tools component embed invocation
+	WITWorld        string // pass through to wasm-tools component embed -w option
 }
 
 // Verify performs a validation on the given options, raising an error if options are not valid.

compileopts/target.go

@@ -62,6 +62,8 @@ type TargetSpec struct {
 	JLinkDevice      string   `json:"jlink-device,omitempty"`
 	CodeModel        string   `json:"code-model,omitempty"`
 	RelocationModel  string   `json:"relocation-model,omitempty"`
+	WITPackage       string   `json:"wit-package,omitempty"`
+	WITWorld         string   `json:"wit-world,omitempty"`
 }
 
 // overrideProperties overrides all properties that are set in child into itself using reflection.

compiler/symbol.go

@@ -346,7 +346,7 @@ func (c *compilerContext) parsePragmas(info *functionInfo, f *ssa.Function) {
 // The list of allowed types is based on this proposal:
 // https://github.com/golang/go/issues/59149
 func (c *compilerContext) checkWasmImport(f *ssa.Function, pragma string) {
-	if c.pkg.Path() == "runtime" || c.pkg.Path() == "syscall/js" {
+	if c.pkg.Path() == "runtime" || c.pkg.Path() == "syscall/js" || c.pkg.Path() == "syscall" {
 		// The runtime is a special case. Allow all kinds of parameters
 		// (importantly, including pointers).
 		return
@@ -360,38 +360,68 @@ func (c *compilerContext) checkWasmImport(f *ssa.Function, pragma string) {
 		c.addError(f.Signature.Results().At(1).Pos(), fmt.Sprintf("%s: too many return values", pragma))
 	} else if f.Signature.Results().Len() == 1 {
 		result := f.Signature.Results().At(0)
-		if !isValidWasmType(result.Type(), true) {
+		if !isValidWasmType(result.Type(), siteResult) {
 			c.addError(result.Pos(), fmt.Sprintf("%s: unsupported result type %s", pragma, result.Type().String()))
 		}
 	}
 	for _, param := range f.Params {
 		// Check whether the type is allowed.
 		// Only a very limited number of types can be mapped to WebAssembly.
-		if !isValidWasmType(param.Type(), false) {
+		if !isValidWasmType(param.Type(), siteParam) {
 			c.addError(param.Pos(), fmt.Sprintf("%s: unsupported parameter type %s", pragma, param.Type().String()))
 		}
 	}
 }
 
-// Check whether the type maps directly to a WebAssembly type, according to:
+// Check whether the type maps directly to a WebAssembly type.
+//
+// This reflects the relaxed type restrictions proposed here (except for structs.HostLayout):
+// https://github.com/golang/go/issues/66984
+//
+// This previously reflected the additional restrictions documented here:
 // https://github.com/golang/go/issues/59149
-func isValidWasmType(typ types.Type, isReturn bool) bool {
+func isValidWasmType(typ types.Type, site wasmSite) bool {
 	switch typ := typ.Underlying().(type) {
 	case *types.Basic:
 		switch typ.Kind() {
-		case types.Int32, types.Uint32, types.Int64, types.Uint64:
+		case types.Bool:
+			return true
+		case types.Int, types.Uint, types.Int8, types.Uint8, types.Int16, types.Uint16, types.Int32, types.Uint32, types.Int64, types.Uint64:
 			return true
 		case types.Float32, types.Float64:
 			return true
-		case types.UnsafePointer:
-			if !isReturn {
-				return true
+		case types.Uintptr, types.UnsafePointer:
+			return true
+		case types.String:
+			// string flattens to two values, so disallowed as a result
+			return site == siteParam || site == siteIndirect
+		}
+	case *types.Array:
+		return site == siteIndirect && isValidWasmType(typ.Elem(), siteIndirect)
+	case *types.Struct:
+		if site != siteIndirect {
+			return false
+		}
+		for i := 0; i < typ.NumFields(); i++ {
+			if !isValidWasmType(typ.Field(i).Type(), siteIndirect) {
+				return false
 			}
 		}
+		return true
+	case *types.Pointer:
+		return isValidWasmType(typ.Elem(), siteIndirect)
 	}
 	return false
 }
 
+type wasmSite int
+
+const (
+	siteParam wasmSite = iota
+	siteResult
+	siteIndirect // pointer or field
+)
+
 // getParams returns the function parameters, including the receiver at the
 // start. This is an alternative to the Params member of *ssa.Function, which is
 // not yet populated when the package has not yet been built.