From aba7e2265a6e7ad7759e47ea30218274e920eca3 Mon Sep 17 00:00:00 2001 From: Ayke van Laethem Date: Wed, 19 Mar 2025 08:14:52 +0100 Subject: [PATCH 1/3] builder: don't use precompiled libraries We used these libraries in the past, but stopped doing so a while ago. This is a small cleanup to remove support for these entirely. --- builder/library.go | 7 +------ compileopts/config.go | 23 +++++++---------------- 2 files changed, 8 insertions(+), 22 deletions(-) diff --git a/builder/library.go b/builder/library.go index 1b6afe2fcd..09bd53fc81 100644 --- a/builder/library.go +++ b/builder/library.go @@ -48,13 +48,8 @@ type Library struct { // target config. In other words, pass this libc if the library needs a libc to // compile. func (l *Library) load(config *compileopts.Config, tmpdir string, libc *compileJob) (job *compileJob, abortLock func(), err error) { - outdir, precompiled := config.LibcPath(l.name) + outdir := config.LibcPath(l.name) archiveFilePath := filepath.Join(outdir, "lib.a") - if precompiled { - // Found a precompiled library for this OS/architecture. Return the path - // directly. - return dummyCompileJob(archiveFilePath), func() {}, nil - } // Create a lock on the output (if supported). // This is a bit messy, but avoids a deadlock because it is ordered consistently with other library loads within a build. diff --git a/compileopts/config.go b/compileopts/config.go index 3d8a73627c..411cd7c84f 100644 --- a/compileopts/config.go +++ b/compileopts/config.go @@ -247,10 +247,9 @@ func MuslArchitecture(triple string) string { return CanonicalArchName(triple) } -// LibcPath returns the path to the libc directory. The libc path will be either -// a precompiled libc shipped with a TinyGo build, or a libc path in the cache -// directory (which might not yet be built). -func (c *Config) LibcPath(name string) (path string, precompiled bool) { +// LibcPath returns the path to the libc directory. The libc path will be a libc +// path in the cache directory (which might not yet be built). +func (c *Config) LibcPath(name string) string { archname := c.Triple() if c.CPU() != "" { archname += "-" + c.CPU() @@ -266,17 +265,9 @@ func (c *Config) LibcPath(name string) (path string, precompiled bool) { archname += "-" + c.Target.Libc } - // Try to load a precompiled library. - precompiledDir := filepath.Join(goenv.Get("TINYGOROOT"), "pkg", archname, name) - if _, err := os.Stat(precompiledDir); err == nil { - // Found a precompiled library for this OS/architecture. Return the path - // directly. - return precompiledDir, true - } - // No precompiled library found. Determine the path name that will be used // in the build cache. - return filepath.Join(goenv.Get("GOCACHE"), name+"-"+archname), false + return filepath.Join(goenv.Get("GOCACHE"), name+"-"+archname) } // DefaultBinaryExtension returns the default extension for binaries, such as @@ -360,7 +351,7 @@ func (c *Config) LibcCFlags() []string { case "picolibc": root := goenv.Get("TINYGOROOT") picolibcDir := filepath.Join(root, "lib", "picolibc", "newlib", "libc") - path, _ := c.LibcPath("picolibc") + path := c.LibcPath("picolibc") return []string{ "-nostdlibinc", "-isystem", filepath.Join(path, "include"), @@ -370,7 +361,7 @@ func (c *Config) LibcCFlags() []string { } case "musl": root := goenv.Get("TINYGOROOT") - path, _ := c.LibcPath("musl") + path := c.LibcPath("musl") arch := MuslArchitecture(c.Triple()) return []string{ "-nostdlibinc", @@ -390,7 +381,7 @@ func (c *Config) LibcCFlags() []string { return nil case "mingw-w64": root := goenv.Get("TINYGOROOT") - path, _ := c.LibcPath("mingw-w64") + path := c.LibcPath("mingw-w64") return []string{ "-nostdlibinc", "-isystem", filepath.Join(path, "include"), From 435edce3907032c580208c4feae5e113d856cb79 Mon Sep 17 00:00:00 2001 From: Ayke van Laethem Date: Wed, 19 Mar 2025 08:58:10 +0100 Subject: [PATCH 2/3] builder: simplify bdwgc libc dependency Header files are built immediately, not in a separate job, so no dependency is needed here. All we need is a flag whether to add flags for that given libc. --- builder/bdwgc.go | 1 + builder/build.go | 19 +++++++------------ builder/library.go | 17 +++++------------ 3 files changed, 13 insertions(+), 24 deletions(-) diff --git a/builder/bdwgc.go b/builder/bdwgc.go index 88e7a26732..8341005d2e 100644 --- a/builder/bdwgc.go +++ b/builder/bdwgc.go @@ -49,6 +49,7 @@ var BoehmGC = Library{ "-I" + libdir + "/include", } }, + needsLibc: true, sourceDir: func() string { return filepath.Join(goenv.Get("TINYGOROOT"), "lib/bdwgc") }, diff --git a/builder/build.go b/builder/build.go index 46f4dcbdfa..d921f95a0d 100644 --- a/builder/build.go +++ b/builder/build.go @@ -147,14 +147,13 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe // the libc needs them. root := goenv.Get("TINYGOROOT") var libcDependencies []*compileJob - var libcJob *compileJob switch config.Target.Libc { case "darwin-libSystem": - libcJob = makeDarwinLibSystemJob(config, tmpdir) + libcJob := makeDarwinLibSystemJob(config, tmpdir) libcDependencies = append(libcDependencies, libcJob) case "musl": var unlock func() - libcJob, unlock, err = libMusl.load(config, tmpdir, nil) + libcJob, unlock, err := libMusl.load(config, tmpdir) if err != nil { return BuildResult{}, err } @@ -162,7 +161,7 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe libcDependencies = append(libcDependencies, dummyCompileJob(filepath.Join(filepath.Dir(libcJob.result), "crt1.o"))) libcDependencies = append(libcDependencies, libcJob) case "picolibc": - libcJob, unlock, err := libPicolibc.load(config, tmpdir, nil) + libcJob, unlock, err := libPicolibc.load(config, tmpdir) if err != nil { return BuildResult{}, err } @@ -175,15 +174,14 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe } libcDependencies = append(libcDependencies, dummyCompileJob(path)) case "wasmbuiltins": - libcJob, unlock, err := libWasmBuiltins.load(config, tmpdir, nil) + libcJob, unlock, err := libWasmBuiltins.load(config, tmpdir) if err != nil { return BuildResult{}, err } defer unlock() libcDependencies = append(libcDependencies, libcJob) case "mingw-w64": - var unlock func() - libcJob, unlock, err = libMinGW.load(config, tmpdir, nil) + libcJob, unlock, err := libMinGW.load(config, tmpdir) if err != nil { return BuildResult{}, err } @@ -704,7 +702,7 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe // Add compiler-rt dependency if needed. Usually this is a simple load from // a cache. if config.Target.RTLib == "compiler-rt" { - job, unlock, err := libCompilerRT.load(config, tmpdir, nil) + job, unlock, err := libCompilerRT.load(config, tmpdir) if err != nil { return result, err } @@ -714,10 +712,7 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe // The Boehm collector is stored in a separate C library. if config.GC() == "boehm" { - if libcJob == nil { - return BuildResult{}, fmt.Errorf("boehm GC isn't supported with libc %s", config.Target.Libc) - } - job, unlock, err := BoehmGC.load(config, tmpdir, libcJob) + job, unlock, err := BoehmGC.load(config, tmpdir) if err != nil { return BuildResult{}, err } diff --git a/builder/library.go b/builder/library.go index 09bd53fc81..0c73a515ad 100644 --- a/builder/library.go +++ b/builder/library.go @@ -25,6 +25,9 @@ type Library struct { // cflags returns the C flags specific to this library cflags func(target, headerPath string) []string + // needsLibc is set to true if this library needs libc headers. + needsLibc bool + // The source directory. sourceDir func() string @@ -43,11 +46,7 @@ type Library struct { // output archive file, it is expected to be removed after use. // As a side effect, this call creates the library header files if they didn't // exist yet. -// The provided libc job (if not null) will cause this libc to be added as a -// dependency for all C compiler jobs, and adds libc headers for the given -// target config. In other words, pass this libc if the library needs a libc to -// compile. -func (l *Library) load(config *compileopts.Config, tmpdir string, libc *compileJob) (job *compileJob, abortLock func(), err error) { +func (l *Library) load(config *compileopts.Config, tmpdir string) (job *compileJob, abortLock func(), err error) { outdir := config.LibcPath(l.name) archiveFilePath := filepath.Join(outdir, "lib.a") @@ -180,7 +179,7 @@ func (l *Library) load(config *compileopts.Config, tmpdir string, libc *compileJ args = append(args, "-mfpu=vfpv2") } } - if libc != nil { + if l.needsLibc { args = append(args, config.LibcCFlags()...) } @@ -251,9 +250,6 @@ func (l *Library) load(config *compileopts.Config, tmpdir string, libc *compileJ return nil }, } - if libc != nil { - objfile.dependencies = append(objfile.dependencies, libc) - } job.dependencies = append(job.dependencies, objfile) } @@ -284,9 +280,6 @@ func (l *Library) load(config *compileopts.Config, tmpdir string, libc *compileJ return os.Rename(tmpfile.Name(), filepath.Join(outdir, "crt1.o")) }, } - if libc != nil { - crt1Job.dependencies = append(crt1Job.dependencies, libc) - } job.dependencies = append(job.dependencies, crt1Job) } From 820b30794ccc689082dab252f3640fecc52910ce Mon Sep 17 00:00:00 2001 From: Ayke van Laethem Date: Thu, 20 Mar 2025 14:41:52 +0100 Subject: [PATCH 3/3] builder: build wasi-libc inside TinyGo Instead of relying on a build when TinyGo is being built, do it like all other libraries when it is needed. This brings a few benefits: * No more running `make wasi-libc` on the command line as a special case for WebAssembly. The generic `git submodule update --init` step is now enough for wasi-libc. * It becomes much easier to customize the build per system. For example: include/exclude malloc as needed, disable/enable bulk memory operations per target, etc. --- .circleci/config.yml | 8 - .github/workflows/build-macos.yml | 9 - .github/workflows/linux.yml | 18 -- .github/workflows/windows.yml | 9 - GNUmakefile | 44 +++-- builder/build.go | 10 +- builder/library.go | 7 + builder/musl.go | 71 ++++---- builder/wasilibc.go | 278 ++++++++++++++++++++++++++++++ compileopts/config.go | 4 +- flake.nix | 9 - 11 files changed, 362 insertions(+), 105 deletions(-) create mode 100644 builder/wasilibc.go diff --git a/.circleci/config.yml b/.circleci/config.yml index a3a052c28c..257e5971ab 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -73,14 +73,6 @@ commands: - go-cache-v4-{{ checksum "go.mod" }} - llvm-source-linux - run: go install -tags=llvm<> . - - restore_cache: - keys: - - wasi-libc-sysroot-systemclang-v7 - - run: make wasi-libc - - save_cache: - key: wasi-libc-sysroot-systemclang-v7 - paths: - - lib/wasi-libc/sysroot - when: condition: <> steps: diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml index 25b5971783..46ec2cc7c9 100644 --- a/.github/workflows/build-macos.yml +++ b/.github/workflows/build-macos.yml @@ -89,15 +89,6 @@ jobs: with: key: ${{ steps.cache-llvm-build.outputs.cache-primary-key }} path: llvm-build - - name: Cache wasi-libc sysroot - uses: actions/cache@v4 - id: cache-wasi-libc - with: - key: wasi-libc-sysroot-${{ matrix.os }}-v1 - path: lib/wasi-libc/sysroot - - name: Build wasi-libc - if: steps.cache-wasi-libc.outputs.cache-hit != 'true' - run: make wasi-libc - name: make gen-device run: make -j3 gen-device - name: Test TinyGo diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 0a8fabf1f2..803759892f 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -104,15 +104,6 @@ jobs: run: | apk add cmake samurai python3 make binaryen STATIC=1 - - name: Cache wasi-libc - uses: actions/cache@v4 - id: cache-wasi-libc - with: - key: wasi-libc-sysroot-linux-alpine-v2 - path: lib/wasi-libc/sysroot - - name: Build wasi-libc - if: steps.cache-wasi-libc.outputs.cache-hit != 'true' - run: make wasi-libc - name: Install fpm run: | gem install --version 4.0.7 public_suffix @@ -258,15 +249,6 @@ jobs: - name: Build Binaryen if: steps.cache-binaryen.outputs.cache-hit != 'true' run: make binaryen - - name: Cache wasi-libc - uses: actions/cache@v4 - id: cache-wasi-libc - with: - key: wasi-libc-sysroot-linux-asserts-v6 - path: lib/wasi-libc/sysroot - - name: Build wasi-libc - if: steps.cache-wasi-libc.outputs.cache-hit != 'true' - run: make wasi-libc - run: make gen-device -j4 - name: Test TinyGo run: make ASSERT=1 test diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 42365f59b7..692790bafe 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -91,15 +91,6 @@ jobs: with: key: ${{ steps.cache-llvm-build.outputs.cache-primary-key }} path: llvm-build - - name: Cache wasi-libc sysroot - uses: actions/cache@v4 - id: cache-wasi-libc - with: - key: wasi-libc-sysroot-v5 - path: lib/wasi-libc/sysroot - - name: Build wasi-libc - if: steps.cache-wasi-libc.outputs.cache-hit != 'true' - run: make wasi-libc - name: Cache Go cache uses: actions/cache@v4 with: diff --git a/GNUmakefile b/GNUmakefile index fcae9a2057..13bf599f1c 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -259,13 +259,6 @@ build/wasm-opt$(EXE): cp lib/binaryen/bin/wasm-opt$(EXE) build/wasm-opt$(EXE) endif -# Build wasi-libc sysroot -.PHONY: wasi-libc -wasi-libc: lib/wasi-libc/sysroot/lib/wasm32-wasi/libc.a -lib/wasi-libc/sysroot/lib/wasm32-wasi/libc.a: - @if [ ! -e lib/wasi-libc/Makefile ]; then echo "Submodules have not been downloaded. Please download them using:\n git submodule update --init"; exit 1; fi - cd lib/wasi-libc && $(MAKE) -j4 EXTRA_CFLAGS="-O2 -g -DNDEBUG -mnontrapping-fptoint -msign-ext" MALLOC_IMPL=none CC="$(CLANG)" AR=$(LLVM_AR) NM=$(LLVM_NM) - # Generate WASI syscall bindings WASM_TOOLS_MODULE=go.bytecodealliance.org .PHONY: wasi-syscall @@ -293,7 +286,7 @@ endif tinygo: ## Build the TinyGo compiler @if [ ! -f "$(LLVM_BUILDDIR)/bin/llvm-config" ]; then echo "Fetch and build LLVM first by running:"; echo " $(MAKE) llvm-source"; echo " $(MAKE) $(LLVM_BUILDDIR)"; exit 1; fi CGO_CPPFLAGS="$(CGO_CPPFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GOENVFLAGS) $(GO) build -buildmode exe -o build/tinygo$(EXE) -tags "byollvm osusergo" . -test: wasi-libc check-nodejs-version +test: check-nodejs-version CGO_CPPFLAGS="$(CGO_CPPFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GO) test $(GOTESTFLAGS) -timeout=1h -buildmode exe -tags "byollvm osusergo" $(GOTESTPKGS) # Standard library packages that pass tests on darwin, linux, wasi, and windows, but take over a minute in wasi @@ -526,9 +519,9 @@ test-corpus: CGO_CPPFLAGS="$(CGO_CPPFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GO) test $(GOTESTFLAGS) -timeout=1h -buildmode exe -tags byollvm -run TestCorpus . -corpus=testdata/corpus.yaml test-corpus-fast: CGO_CPPFLAGS="$(CGO_CPPFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GO) test $(GOTESTFLAGS) -timeout=1h -buildmode exe -tags byollvm -run TestCorpus -short . -corpus=testdata/corpus.yaml -test-corpus-wasi: wasi-libc +test-corpus-wasi: CGO_CPPFLAGS="$(CGO_CPPFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GO) test $(GOTESTFLAGS) -timeout=1h -buildmode exe -tags byollvm -run TestCorpus . -corpus=testdata/corpus.yaml -target=wasip1 -test-corpus-wasip2: wasi-libc +test-corpus-wasip2: CGO_CPPFLAGS="$(CGO_CPPFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GO) test $(GOTESTFLAGS) -timeout=1h -buildmode exe -tags byollvm -run TestCorpus . -corpus=testdata/corpus.yaml -target=wasip2 .PHONY: testchdir @@ -939,7 +932,7 @@ endif wasmtest: $(GO) test ./tests/wasm -build/release: tinygo gen-device wasi-libc $(if $(filter 1,$(USE_SYSTEM_BINARYEN)),,binaryen) +build/release: tinygo gen-device $(if $(filter 1,$(USE_SYSTEM_BINARYEN)),,binaryen) @mkdir -p build/release/tinygo/bin @mkdir -p build/release/tinygo/lib/bdwgc @mkdir -p build/release/tinygo/lib/clang/include @@ -954,7 +947,7 @@ build/release: tinygo gen-device wasi-libc $(if $(filter 1,$(USE_SYSTEM_BINARYEN @mkdir -p build/release/tinygo/lib/nrfx @mkdir -p build/release/tinygo/lib/picolibc/newlib/libc @mkdir -p build/release/tinygo/lib/picolibc/newlib/libm - @mkdir -p build/release/tinygo/lib/wasi-libc/libc-bottom-half/headers + @mkdir -p build/release/tinygo/lib/wasi-libc/libc-bottom-half @mkdir -p build/release/tinygo/lib/wasi-libc/libc-top-half/musl/arch @mkdir -p build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src @mkdir -p build/release/tinygo/lib/wasi-cli/ @@ -1017,15 +1010,36 @@ endif @cp -rp lib/picolibc/newlib/libm/common build/release/tinygo/lib/picolibc/newlib/libm @cp -rp lib/picolibc/newlib/libm/math build/release/tinygo/lib/picolibc/newlib/libm @cp -rp lib/picolibc-stdio.c build/release/tinygo/lib - @cp -rp lib/wasi-libc/libc-bottom-half/headers/public build/release/tinygo/lib/wasi-libc/libc-bottom-half/headers + @cp -rp lib/wasi-libc/libc-bottom-half/cloudlibc build/release/tinygo/lib/wasi-libc/libc-bottom-half + @cp -rp lib/wasi-libc/libc-bottom-half/headers build/release/tinygo/lib/wasi-libc/libc-bottom-half + @cp -rp lib/wasi-libc/libc-bottom-half/sources build/release/tinygo/lib/wasi-libc/libc-bottom-half + @cp -rp lib/wasi-libc/libc-top-half/headers build/release/tinygo/lib/wasi-libc/libc-top-half @cp -rp lib/wasi-libc/libc-top-half/musl/arch/generic build/release/tinygo/lib/wasi-libc/libc-top-half/musl/arch @cp -rp lib/wasi-libc/libc-top-half/musl/arch/wasm32 build/release/tinygo/lib/wasi-libc/libc-top-half/musl/arch + @cp -rp lib/wasi-libc/libc-top-half/musl/include build/release/tinygo/lib/wasi-libc/libc-top-half/musl + @cp -rp lib/wasi-libc/libc-top-half/musl/src/conf build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src + @cp -rp lib/wasi-libc/libc-top-half/musl/src/dirent build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src + @cp -rp lib/wasi-libc/libc-top-half/musl/src/env build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src + @cp -rp lib/wasi-libc/libc-top-half/musl/src/errno build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src + @cp -rp lib/wasi-libc/libc-top-half/musl/src/exit build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src + @cp -rp lib/wasi-libc/libc-top-half/musl/src/fcntl build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src + @cp -rp lib/wasi-libc/libc-top-half/musl/src/fenv build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src @cp -rp lib/wasi-libc/libc-top-half/musl/src/include build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src @cp -rp lib/wasi-libc/libc-top-half/musl/src/internal build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src + @cp -rp lib/wasi-libc/libc-top-half/musl/src/legacy build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src + @cp -rp lib/wasi-libc/libc-top-half/musl/src/locale build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src @cp -rp lib/wasi-libc/libc-top-half/musl/src/math build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src + @cp -rp lib/wasi-libc/libc-top-half/musl/src/misc build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src + @cp -rp lib/wasi-libc/libc-top-half/musl/src/multibyte build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src + @cp -rp lib/wasi-libc/libc-top-half/musl/src/network build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src + @cp -rp lib/wasi-libc/libc-top-half/musl/src/stat build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src + @cp -rp lib/wasi-libc/libc-top-half/musl/src/stdio build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src + @cp -rp lib/wasi-libc/libc-top-half/musl/src/stdlib build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src @cp -rp lib/wasi-libc/libc-top-half/musl/src/string build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src - @cp -rp lib/wasi-libc/libc-top-half/musl/include build/release/tinygo/lib/wasi-libc/libc-top-half/musl - @cp -rp lib/wasi-libc/sysroot build/release/tinygo/lib/wasi-libc/sysroot + @cp -rp lib/wasi-libc/libc-top-half/musl/src/thread build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src + @cp -rp lib/wasi-libc/libc-top-half/musl/src/time build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src + @cp -rp lib/wasi-libc/libc-top-half/musl/src/unistd build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src + @cp -rp lib/wasi-libc/libc-top-half/sources build/release/tinygo/lib/wasi-libc/libc-top-half @cp -rp lib/wasi-cli/wit build/release/tinygo/lib/wasi-cli/wit @cp -rp llvm-project/compiler-rt/lib/builtins build/release/tinygo/lib/compiler-rt-builtins @cp -rp llvm-project/compiler-rt/LICENSE.TXT build/release/tinygo/lib/compiler-rt-builtins diff --git a/builder/build.go b/builder/build.go index d921f95a0d..ee5f97ec59 100644 --- a/builder/build.go +++ b/builder/build.go @@ -14,7 +14,6 @@ import ( "fmt" "go/types" "hash/crc32" - "io/fs" "math/bits" "os" "os/exec" @@ -168,11 +167,12 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe defer unlock() libcDependencies = append(libcDependencies, libcJob) case "wasi-libc": - path := filepath.Join(root, "lib/wasi-libc/sysroot/lib/wasm32-wasi/libc.a") - if _, err := os.Stat(path); errors.Is(err, fs.ErrNotExist) { - return BuildResult{}, errors.New("could not find wasi-libc, perhaps you need to run `make wasi-libc`?") + libcJob, unlock, err := libWasiLibc.load(config, tmpdir) + if err != nil { + return BuildResult{}, err } - libcDependencies = append(libcDependencies, dummyCompileJob(path)) + defer unlock() + libcDependencies = append(libcDependencies, libcJob) case "wasmbuiltins": libcJob, unlock, err := libWasmBuiltins.load(config, tmpdir) if err != nil { diff --git a/builder/library.go b/builder/library.go index 0c73a515ad..53ae5d9d60 100644 --- a/builder/library.go +++ b/builder/library.go @@ -25,6 +25,9 @@ type Library struct { // cflags returns the C flags specific to this library cflags func(target, headerPath string) []string + // cflagsForFile returns additional C flags for a particular source file. + cflagsForFile func(path string) []string + // needsLibc is set to true if this library needs libc headers. needsLibc bool @@ -226,6 +229,7 @@ func (l *Library) load(config *compileopts.Config, tmpdir string) (job *compileJ } for _, path := range paths { // Strip leading "../" parts off the path. + path := path cleanpath := path for strings.HasPrefix(cleanpath, "../") { cleanpath = cleanpath[3:] @@ -239,6 +243,9 @@ func (l *Library) load(config *compileopts.Config, tmpdir string) (job *compileJ run: func(*compileJob) error { var compileArgs []string compileArgs = append(compileArgs, args...) + if l.cflagsForFile != nil { + compileArgs = append(compileArgs, l.cflagsForFile(path)...) + } compileArgs = append(compileArgs, "-o", objpath, srcpath) if config.Options.PrintCommands != nil { config.Options.PrintCommands("clang", compileArgs...) diff --git a/builder/musl.go b/builder/musl.go index dc03be46f7..4195332550 100644 --- a/builder/musl.go +++ b/builder/musl.go @@ -12,6 +12,45 @@ import ( "github.com/tinygo-org/tinygo/goenv" ) +// Create the alltypes.h file from the appropriate alltypes.h.in files. +func buildMuslAllTypes(arch, muslDir, outputBitsDir string) error { + // Create the file alltypes.h. + f, err := os.Create(filepath.Join(outputBitsDir, "alltypes.h")) + if err != nil { + return err + } + infiles := []string{ + filepath.Join(muslDir, "arch", arch, "bits", "alltypes.h.in"), + filepath.Join(muslDir, "include", "alltypes.h.in"), + } + for _, infile := range infiles { + data, err := os.ReadFile(infile) + if err != nil { + return err + } + lines := strings.Split(string(data), "\n") + for _, line := range lines { + if strings.HasPrefix(line, "TYPEDEF ") { + matches := regexp.MustCompile(`TYPEDEF (.*) ([^ ]*);`).FindStringSubmatch(line) + value := matches[1] + name := matches[2] + line = fmt.Sprintf("#if defined(__NEED_%s) && !defined(__DEFINED_%s)\ntypedef %s %s;\n#define __DEFINED_%s\n#endif\n", name, name, value, name, name) + } + if strings.HasPrefix(line, "STRUCT ") { + matches := regexp.MustCompile(`STRUCT * ([^ ]*) (.*);`).FindStringSubmatch(line) + name := matches[1] + value := matches[2] + line = fmt.Sprintf("#if defined(__NEED_struct_%s) && !defined(__DEFINED_struct_%s)\nstruct %s %s;\n#define __DEFINED_struct_%s\n#endif\n", name, name, name, value, name) + } + _, err := f.WriteString(line + "\n") + if err != nil { + return err + } + } + } + return f.Close() +} + var libMusl = Library{ name: "musl", makeHeaders: func(target, includeDir string) error { @@ -24,41 +63,13 @@ var libMusl = Library{ arch := compileopts.MuslArchitecture(target) muslDir := filepath.Join(goenv.Get("TINYGOROOT"), "lib", "musl") - // Create the file alltypes.h. - f, err := os.Create(filepath.Join(bits, "alltypes.h")) + err = buildMuslAllTypes(arch, muslDir, bits) if err != nil { return err } - infiles := []string{ - filepath.Join(muslDir, "arch", arch, "bits", "alltypes.h.in"), - filepath.Join(muslDir, "include", "alltypes.h.in"), - } - for _, infile := range infiles { - data, err := os.ReadFile(infile) - if err != nil { - return err - } - lines := strings.Split(string(data), "\n") - for _, line := range lines { - if strings.HasPrefix(line, "TYPEDEF ") { - matches := regexp.MustCompile(`TYPEDEF (.*) ([^ ]*);`).FindStringSubmatch(line) - value := matches[1] - name := matches[2] - line = fmt.Sprintf("#if defined(__NEED_%s) && !defined(__DEFINED_%s)\ntypedef %s %s;\n#define __DEFINED_%s\n#endif\n", name, name, value, name, name) - } - if strings.HasPrefix(line, "STRUCT ") { - matches := regexp.MustCompile(`STRUCT * ([^ ]*) (.*);`).FindStringSubmatch(line) - name := matches[1] - value := matches[2] - line = fmt.Sprintf("#if defined(__NEED_struct_%s) && !defined(__DEFINED_struct_%s)\nstruct %s %s;\n#define __DEFINED_struct_%s\n#endif\n", name, name, name, value, name) - } - f.WriteString(line + "\n") - } - } - f.Close() // Create the file syscall.h. - f, err = os.Create(filepath.Join(bits, "syscall.h")) + f, err := os.Create(filepath.Join(bits, "syscall.h")) if err != nil { return err } diff --git a/builder/wasilibc.go b/builder/wasilibc.go new file mode 100644 index 0000000000..9b8ad7a5e2 --- /dev/null +++ b/builder/wasilibc.go @@ -0,0 +1,278 @@ +package builder + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/tinygo-org/tinygo/goenv" +) + +var libWasiLibc = Library{ + name: "wasi-libc", + makeHeaders: func(target, includeDir string) error { + bits := filepath.Join(includeDir, "bits") + err := os.Mkdir(bits, 0777) + if err != nil { + return err + } + + muslDir := filepath.Join(goenv.Get("TINYGOROOT"), "lib", "wasi-libc/libc-top-half/musl") + err = buildMuslAllTypes("wasm32", muslDir, bits) + if err != nil { + return err + } + + // See MUSL_OMIT_HEADERS in the Makefile. + omitHeaders := map[string]struct{}{ + "syslog.h": {}, + "wait.h": {}, + "ucontext.h": {}, + "paths.h": {}, + "utmp.h": {}, + "utmpx.h": {}, + "lastlog.h": {}, + "elf.h": {}, + "link.h": {}, + "pwd.h": {}, + "shadow.h": {}, + "grp.h": {}, + "mntent.h": {}, + "netdb.h": {}, + "resolv.h": {}, + "pty.h": {}, + "dlfcn.h": {}, + "setjmp.h": {}, + "ulimit.h": {}, + "wordexp.h": {}, + "spawn.h": {}, + "termios.h": {}, + "libintl.h": {}, + "aio.h": {}, + + "stdarg.h": {}, + "stddef.h": {}, + + "pthread.h": {}, + } + + for _, glob := range [][2]string{ + {"libc-bottom-half/headers/public/*.h", ""}, + {"libc-bottom-half/headers/public/wasi/*.h", "wasi"}, + {"libc-top-half/musl/arch/wasm32/bits/*.h", "bits"}, + {"libc-top-half/musl/include/*.h", ""}, + {"libc-top-half/musl/include/netinet/*.h", "netinet"}, + {"libc-top-half/musl/include/sys/*.h", "sys"}, + } { + matches, _ := filepath.Glob(filepath.Join(goenv.Get("TINYGOROOT"), "lib/wasi-libc", glob[0])) + outDir := filepath.Join(includeDir, glob[1]) + os.MkdirAll(outDir, 0o777) + for _, match := range matches { + name := filepath.Base(match) + if _, ok := omitHeaders[name]; ok { + continue + } + data, err := os.ReadFile(match) + if err != nil { + return err + } + err = os.WriteFile(filepath.Join(outDir, name), data, 0o666) + if err != nil { + return err + } + } + } + + return nil + }, + cflags: func(target, headerPath string) []string { + libcDir := filepath.Join(goenv.Get("TINYGOROOT"), "lib/wasi-libc") + return []string{ + "-Werror", + "-Wall", + "-std=gnu11", + "-nostdlibinc", + "-mnontrapping-fptoint", "-msign-ext", + "-Wno-null-pointer-arithmetic", "-Wno-unused-parameter", "-Wno-sign-compare", "-Wno-unused-variable", "-Wno-unused-function", "-Wno-ignored-attributes", "-Wno-missing-braces", "-Wno-ignored-pragmas", "-Wno-unused-but-set-variable", "-Wno-unknown-warning-option", + "-Wno-parentheses", "-Wno-shift-op-parentheses", "-Wno-bitwise-op-parentheses", "-Wno-logical-op-parentheses", "-Wno-string-plus-int", "-Wno-dangling-else", "-Wno-unknown-pragmas", + "-DNDEBUG", + "-D__wasilibc_printscan_no_long_double", + "-D__wasilibc_printscan_full_support_option=\"long double support is disabled\"", + "-isystem", headerPath, + "-I" + libcDir + "/libc-top-half/musl/src/include", + "-I" + libcDir + "/libc-top-half/musl/src/internal", + "-I" + libcDir + "/libc-top-half/musl/arch/wasm32", + "-I" + libcDir + "/libc-top-half/musl/arch/generic", + "-I" + libcDir + "/libc-top-half/headers/private", + } + }, + cflagsForFile: func(path string) []string { + if strings.HasPrefix(path, "libc-bottom-half"+string(os.PathSeparator)) { + libcDir := filepath.Join(goenv.Get("TINYGOROOT"), "lib/wasi-libc") + return []string{ + "-I" + libcDir + "/libc-bottom-half/headers/private", + "-I" + libcDir + "/libc-bottom-half/cloudlibc/src/include", + "-I" + libcDir + "/libc-bottom-half/cloudlibc/src", + } + } + return nil + }, + sourceDir: func() string { return filepath.Join(goenv.Get("TINYGOROOT"), "lib/wasi-libc") }, + librarySources: func(target string) ([]string, error) { + type filePattern struct { + glob string + exclude []string + } + + // See: LIBC_TOP_HALF_MUSL_SOURCES in the Makefile + globs := []filePattern{ + // Top half: mostly musl sources. + {glob: "libc-top-half/sources/*.c"}, + {glob: "libc-top-half/musl/src/conf/*.c"}, + {glob: "libc-top-half/musl/src/internal/*.c", exclude: []string{ + "procfdname.c", "syscall.c", "syscall_ret.c", "vdso.c", "version.c", + }}, + {glob: "libc-top-half/musl/src/locale/*.c", exclude: []string{ + "dcngettext.c", "textdomain.c", "bind_textdomain_codeset.c"}}, + {glob: "libc-top-half/musl/src/math/*.c", exclude: []string{ + "__signbit.c", "__signbitf.c", "__signbitl.c", + "__fpclassify.c", "__fpclassifyf.c", "__fpclassifyl.c", + "ceilf.c", "ceil.c", + "floorf.c", "floor.c", + "truncf.c", "trunc.c", + "rintf.c", "rint.c", + "nearbyintf.c", "nearbyint.c", + "sqrtf.c", "sqrt.c", + "fabsf.c", "fabs.c", + "copysignf.c", "copysign.c", + "fminf.c", "fmaxf.c", + "fmin.c", "fmax.c,", + }}, + {glob: "libc-top-half/musl/src/multibyte/*.c"}, + {glob: "libc-top-half/musl/src/stdio/*.c", exclude: []string{ + "vfwscanf.c", "vfwprintf.c", // long double is unsupported + "__lockfile.c", "flockfile.c", "funlockfile.c", "ftrylockfile.c", + "rename.c", + "tmpnam.c", "tmpfile.c", "tempnam.c", + "popen.c", "pclose.c", + "remove.c", + "gets.c"}}, + {glob: "libc-top-half/musl/src/stdlib/*.c"}, + {glob: "libc-top-half/musl/src/string/*.c", exclude: []string{ + "strsignal.c"}}, + + // Bottom half: connect top half to WASI equivalents. + {glob: "libc-bottom-half/cloudlibc/src/libc/*/*.c"}, + {glob: "libc-bottom-half/cloudlibc/src/libc/sys/*/*.c"}, + {glob: "libc-bottom-half/sources/*.c"}, + } + + // See: LIBC_TOP_HALF_MUSL_SOURCES in the Makefile + sources := []string{ + "libc-top-half/musl/src/misc/a64l.c", + "libc-top-half/musl/src/misc/basename.c", + "libc-top-half/musl/src/misc/dirname.c", + "libc-top-half/musl/src/misc/ffs.c", + "libc-top-half/musl/src/misc/ffsl.c", + "libc-top-half/musl/src/misc/ffsll.c", + "libc-top-half/musl/src/misc/fmtmsg.c", + "libc-top-half/musl/src/misc/getdomainname.c", + "libc-top-half/musl/src/misc/gethostid.c", + "libc-top-half/musl/src/misc/getopt.c", + "libc-top-half/musl/src/misc/getopt_long.c", + "libc-top-half/musl/src/misc/getsubopt.c", + "libc-top-half/musl/src/misc/uname.c", + "libc-top-half/musl/src/misc/nftw.c", + "libc-top-half/musl/src/errno/strerror.c", + "libc-top-half/musl/src/network/htonl.c", + "libc-top-half/musl/src/network/htons.c", + "libc-top-half/musl/src/network/ntohl.c", + "libc-top-half/musl/src/network/ntohs.c", + "libc-top-half/musl/src/network/inet_ntop.c", + "libc-top-half/musl/src/network/inet_pton.c", + "libc-top-half/musl/src/network/inet_aton.c", + "libc-top-half/musl/src/network/in6addr_any.c", + "libc-top-half/musl/src/network/in6addr_loopback.c", + "libc-top-half/musl/src/fenv/fenv.c", + "libc-top-half/musl/src/fenv/fesetround.c", + "libc-top-half/musl/src/fenv/feupdateenv.c", + "libc-top-half/musl/src/fenv/fesetexceptflag.c", + "libc-top-half/musl/src/fenv/fegetexceptflag.c", + "libc-top-half/musl/src/fenv/feholdexcept.c", + "libc-top-half/musl/src/exit/exit.c", + "libc-top-half/musl/src/exit/atexit.c", + "libc-top-half/musl/src/exit/assert.c", + "libc-top-half/musl/src/exit/quick_exit.c", + "libc-top-half/musl/src/exit/at_quick_exit.c", + "libc-top-half/musl/src/time/strftime.c", + "libc-top-half/musl/src/time/asctime.c", + "libc-top-half/musl/src/time/asctime_r.c", + "libc-top-half/musl/src/time/ctime.c", + "libc-top-half/musl/src/time/ctime_r.c", + "libc-top-half/musl/src/time/wcsftime.c", + "libc-top-half/musl/src/time/strptime.c", + "libc-top-half/musl/src/time/difftime.c", + "libc-top-half/musl/src/time/timegm.c", + "libc-top-half/musl/src/time/ftime.c", + "libc-top-half/musl/src/time/gmtime.c", + "libc-top-half/musl/src/time/gmtime_r.c", + "libc-top-half/musl/src/time/timespec_get.c", + "libc-top-half/musl/src/time/getdate.c", + "libc-top-half/musl/src/time/localtime.c", + "libc-top-half/musl/src/time/localtime_r.c", + "libc-top-half/musl/src/time/mktime.c", + "libc-top-half/musl/src/time/__tm_to_secs.c", + "libc-top-half/musl/src/time/__month_to_secs.c", + "libc-top-half/musl/src/time/__secs_to_tm.c", + "libc-top-half/musl/src/time/__year_to_secs.c", + "libc-top-half/musl/src/time/__tz.c", + "libc-top-half/musl/src/fcntl/creat.c", + "libc-top-half/musl/src/dirent/alphasort.c", + "libc-top-half/musl/src/dirent/versionsort.c", + "libc-top-half/musl/src/env/__stack_chk_fail.c", + "libc-top-half/musl/src/env/clearenv.c", + "libc-top-half/musl/src/env/getenv.c", + "libc-top-half/musl/src/env/putenv.c", + "libc-top-half/musl/src/env/setenv.c", + "libc-top-half/musl/src/env/unsetenv.c", + "libc-top-half/musl/src/unistd/posix_close.c", + "libc-top-half/musl/src/stat/futimesat.c", + "libc-top-half/musl/src/legacy/getpagesize.c", + "libc-top-half/musl/src/thread/thrd_sleep.c", + } + + basepath := goenv.Get("TINYGOROOT") + "/lib/wasi-libc/" + for _, pattern := range globs { + matches, err := filepath.Glob(basepath + pattern.glob) + if err != nil { + // From the documentation: + // > Glob ignores file system errors such as I/O errors reading + // > directories. The only possible returned error is + // > ErrBadPattern, when pattern is malformed. + // So the only possible error is when the (statically defined) + // pattern is wrong. In other words, a programming bug. + return nil, fmt.Errorf("wasi-libc: could not glob source dirs: %w", err) + } + if len(matches) == 0 { + return nil, fmt.Errorf("wasi-libc: did not find any files for pattern %#v", pattern) + } + excludeSet := map[string]struct{}{} + for _, exclude := range pattern.exclude { + excludeSet[exclude] = struct{}{} + } + for _, match := range matches { + if _, ok := excludeSet[filepath.Base(match)]; ok { + continue + } + relpath, err := filepath.Rel(basepath, match) + if err != nil { + // Not sure if this is even possible. + return nil, err + } + sources = append(sources, relpath) + } + } + return sources, nil + }, +} diff --git a/compileopts/config.go b/compileopts/config.go index 411cd7c84f..9fa482952f 100644 --- a/compileopts/config.go +++ b/compileopts/config.go @@ -371,10 +371,10 @@ func (c *Config) LibcCFlags() []string { "-isystem", filepath.Join(root, "lib", "musl", "include"), } case "wasi-libc": - root := goenv.Get("TINYGOROOT") + path := c.LibcPath("wasi-libc") return []string{ "-nostdlibinc", - "-isystem", root + "/lib/wasi-libc/sysroot/include", + "-isystem", filepath.Join(path, "include"), } case "wasmbuiltins": // nothing to add (library is purely for builtins) diff --git a/flake.nix b/flake.nix index 25ffc70205..136e1014fb 100644 --- a/flake.nix +++ b/flake.nix @@ -21,7 +21,6 @@ # make llvm-source # fetch compiler-rt # git submodule update --init # fetch lots of other libraries and SVD files # make gen-device -j4 # build src/device/*/*.go files -# make wasi-libc # build support for wasi/wasm # # With this, you should have an environment that can compile anything - except # for the Xtensa architecture (ESP8266/ESP32) because support for that lives in @@ -64,14 +63,6 @@ #openocd ]; shellHook= '' - # Configure CLANG, LLVM_AR, and LLVM_NM for `make wasi-libc`. - # Without setting these explicitly, Homebrew versions might be used - # or the default `ar` and `nm` tools might be used (which don't - # support wasi). - export CLANG="clang-18 -resource-dir ${llvmPackages_18.clang.cc.lib}/lib/clang/18" - export LLVM_AR=llvm-ar - export LLVM_NM=llvm-nm - # Make `make smoketest` work (the default is `md5`, while Nix only # has `md5sum`). export MD5SUM=md5sum