diff --git a/.github/workflows/tests_wasm.yaml b/.github/workflows/tests_wasm.yaml new file mode 100644 index 000000000..43fcc1af4 --- /dev/null +++ b/.github/workflows/tests_wasm.yaml @@ -0,0 +1,42 @@ +--- +name: Tests WASM +permissions: read-all +on: [push, pull_request] +jobs: + test-wasm-js: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - id: goversion + run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" + - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 + with: + go-version: ${{ steps.goversion.outputs.goversion }} + - name: Setup Node.js + uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2 + with: + node-version: '20' + - name: Run WASM JS tests + run: | + env -i \ + PATH="$PATH" \ + HOME="$HOME" \ + GOCACHE="$GOCACHE" \ + GOPATH="$GOPATH" \ + GOROOT="$(go env GOROOT)" \ + make test-wasm + + test-wasm-wasip1: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - id: goversion + run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" + - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 + with: + go-version: ${{ steps.goversion.outputs.goversion }} + - name: Install wazero + run: go install github.com/tetratelabs/wazero/cmd/wazero@latest + - name: Run WASM wasip1 tests + run: | + make test-wasip1 diff --git a/Makefile b/Makefile index b1062cabe..5734b5d2a 100644 --- a/Makefile +++ b/Makefile @@ -51,6 +51,32 @@ test: BBOLT_VERIFY=all TEST_FREELIST_TYPE=array go test -v ${TESTFLAGS} ./internal/... BBOLT_VERIFY=all TEST_FREELIST_TYPE=array go test -v ${TESTFLAGS} ./cmd/bbolt/... +.PHONY: test-wasm +test-wasm: + @echo "WASM js test" + export PATH="$$PATH:$$(go env GOROOT)/lib/wasm" && \ + GOOS=js GOARCH=wasm go test -v ${TESTFLAGS} -timeout ${TESTFLAGS_TIMEOUT} + +.PHONY: test-wasip1 +test-wasip1: + @echo "WASM wasip1 test" + if command -v wazero >/dev/null 2>&1; then \ + echo "Using wazero runtime"; \ + GOOS=wasip1 GOARCH=wasm go test -v ${TESTFLAGS} -timeout ${TESTFLAGS_TIMEOUT} \ + -exec="wazero run -mount=$(shell mktemp -d):/tmp -mount=.:/test"; \ + elif command -v wasmtime >/dev/null 2>&1; then \ + WASI_TEMP=$$(mktemp -d); \ + echo "Using wasmtime runtime with temp dir: $$WASI_TEMP"; \ + GOOS=wasip1 GOARCH=wasm go test -v ${TESTFLAGS} -timeout ${TESTFLAGS_TIMEOUT} \ + -exec="wasmtime --dir=. --dir=$$WASI_TEMP --env=TMPDIR=$$WASI_TEMP"; \ + else \ + echo "No WASI runtime found - install wazero (recommended) or wasmtime"; \ + echo " go install github.com/tetratelabs/wazero/cmd/wazero@latest"; \ + echo " brew install wasmtime"; \ + echo "Building wasip1 binary to verify compilation..."; \ + GOOS=wasip1 GOARCH=wasm go build .; \ + fi + .PHONY: coverage coverage: @echo "hashmap freelist test" diff --git a/bolt_unix.go b/bolt_unix.go index f68e721f5..9c4aa4044 100644 --- a/bolt_unix.go +++ b/bolt_unix.go @@ -1,4 +1,4 @@ -//go:build !windows && !plan9 && !solaris && !aix && !android +//go:build !windows && !plan9 && !solaris && !aix && !android && !js && !wasip1 package bbolt diff --git a/bolt_wasm.go b/bolt_wasm.go new file mode 100644 index 000000000..d0a2ce83e --- /dev/null +++ b/bolt_wasm.go @@ -0,0 +1,147 @@ +//go:build js || wasip1 + +package bbolt + +import ( + "fmt" + "io" + "time" + "unsafe" + + berrors "go.etcd.io/bbolt/errors" + "go.etcd.io/bbolt/internal/common" +) + +// mmap memory maps a DB's data file. +func mmap(db *DB, sz int) error { + // Check MaxSize constraint for WASM platforms + if !db.readOnly && db.MaxSize > 0 && sz > db.MaxSize { + // The max size only limits future writes; however, we don't block opening + // and mapping the database if it already exceeds the limit. + fileSize, err := db.fileSize() + if err != nil { + return fmt.Errorf("could not check existing db file size: %s", err) + } + + if sz > fileSize { + return berrors.ErrMaxSizeReached + } + } + + // Truncate and fsync to ensure file size metadata is flushed. + // https://github.com/boltdb/bolt/issues/284 + if !db.NoGrowSync && !db.readOnly { + if err := db.file.Truncate(int64(sz)); err != nil { + return fmt.Errorf("file resize error: %s", err) + } + if err := db.file.Sync(); err != nil { + return fmt.Errorf("file sync error: %s", err) + } + } + + // Map the data file to memory. + b := make([]byte, sz) + if sz > 0 { + // Read the data file. + if _, err := db.file.ReadAt(b, 0); err != nil && err != io.EOF { + return err + } + } + + // Save the original byte slice and convert to a byte array pointer. + db.dataref = b + db.datasz = sz + if sz > 0 { + db.data = (*[common.MaxMapSize]byte)(unsafe.Pointer(&b[0])) + } + + return nil +} + +// munmap unmaps a DB's data file from memory. +func munmap(db *DB) error { + // In WASM, we just clear the references + db.dataref = nil + db.data = nil + db.datasz = 0 + return nil +} + +// madvise is not supported in WASM. +func madvise(b []byte, advice int) error { + // Not implemented - no memory advice in WASM + return nil +} + +// mlock is not supported in WASM. +func mlock(db *DB, fileSize int) error { + // Not implemented - no memory locking in WASM + return nil +} + +// munlock is not supported in WASM. +func munlock(db *DB, fileSize int) error { + // Not implemented - no memory unlocking in WASM + return nil +} + +// flock acquires an advisory lock on a file descriptor. +func flock(db *DB, exclusive bool, timeout time.Duration) error { + // Not implemented - no file locking in WASM + return nil +} + +// funlock releases an advisory lock on a file descriptor. +func funlock(db *DB) error { + // Not implemented - no file unlocking in WASM + return nil +} + +// fdatasync flushes written data to a file descriptor. +func fdatasync(db *DB) error { + if db.file == nil { + return nil + } + return db.file.Sync() +} + +// txInit refreshes the memory buffer from the file for WASM platforms. +// This is needed because WASM doesn't have real mmap, so we need to manually +// sync the memory buffer with the file to see changes from previous transactions. +func (db *DB) txInit() error { + // For read-only databases or initial state, skip refresh + if db.file == nil { + return nil + } + + // Check if the file has grown + fileInfo, err := db.file.Stat() + if err != nil { + return err + } + fileSize := int(fileInfo.Size()) + + // If file has grown or we need to initialize, refresh memory + if fileSize > db.datasz || db.datasz == 0 { + // Re-mmap with the new size + if err := mmap(db, fileSize); err != nil { + return err + } + } else if db.datasz > 0 { + // Refresh the existing buffer + b := make([]byte, db.datasz) + if _, err := db.file.ReadAt(b, 0); err != nil && err != io.EOF { + return err + } + db.dataref = b + db.data = (*[common.MaxMapSize]byte)(unsafe.Pointer(&b[0])) + } + + // Update meta page pointers + if db.pageSize > 0 && db.datasz >= db.pageSize*2 { + db.meta0 = db.page(0).Meta() + db.meta1 = db.page(1).Meta() + } + + return nil +} diff --git a/boltsync_unix.go b/boltsync_unix.go index 27face752..77266173b 100644 --- a/boltsync_unix.go +++ b/boltsync_unix.go @@ -1,4 +1,4 @@ -//go:build !windows && !plan9 && !linux && !openbsd +//go:build !windows && !plan9 && !linux && !openbsd && !js && !wasip1 package bbolt diff --git a/bucket_test.go b/bucket_test.go index 493d133a7..533222c65 100644 --- a/bucket_test.go +++ b/bucket_test.go @@ -8,6 +8,7 @@ import ( "log" "math/rand" "os" + "runtime" "strconv" "strings" "testing" @@ -208,6 +209,9 @@ func TestDB_Put_VeryLarge(t *testing.T) { if testing.Short() { t.Skip("skipping test in short mode.") } + if runtime.GOOS == "js" || runtime.GOOS == "wasip1" { + t.Skip("skipping test on WASM due to memory constraints") + } n, batchN := 400000, 200000 ksize, vsize := 8, 500 @@ -377,6 +381,9 @@ func TestBucket_Delete_FreelistOverflow(t *testing.T) { if testing.Short() { t.Skip("skipping test in short mode.") } + if runtime.GOOS == "js" || runtime.GOOS == "wasip1" { + t.Skip("skipping test on WASM due to memory constraints") + } db := btesting.MustCreateDB(t) @@ -1207,6 +1214,9 @@ func TestBucket_Put_KeyTooLarge(t *testing.T) { // Ensure that an error is returned when inserting a value that's too large. func TestBucket_Put_ValueTooLarge(t *testing.T) { + if runtime.GOARCH == "wasm" { + t.Skip("skipping test on wasm") + } // Skip this test on DroneCI because the machine is resource constrained. if os.Getenv("DRONE") == "true" { t.Skip("not enough RAM for test") diff --git a/db.go b/db.go index 5d3e26496..acbbd6286 100644 --- a/db.go +++ b/db.go @@ -772,12 +772,28 @@ func (db *DB) Logger() Logger { return db.logger } +// txIniter is an interface that allows for platform-specific transaction +// initialization. +type txIniter interface { + txInit() error +} + func (db *DB) beginTx() (*Tx, error) { // Lock the meta pages while we initialize the transaction. We obtain // the meta lock before the mmap lock because that's the order that the // write transaction will obtain them. db.metalock.Lock() + // Allow WASM-specific transaction initialization + if runtime.GOARCH == "wasm" { + if initer, ok := any(db).(txIniter); ok { + if err := initer.txInit(); err != nil { + db.metalock.Unlock() + return nil, err + } + } + } + // Obtain a read-only lock on the mmap. When the mmap is remapped it will // obtain a write lock so all transactions must finish before it can be // remapped. @@ -834,6 +850,16 @@ func (db *DB) beginRWTx() (*Tx, error) { db.metalock.Lock() defer db.metalock.Unlock() + // Allow WASM-specific transaction initialization + if runtime.GOARCH == "wasm" { + if initer, ok := any(db).(txIniter); ok { + if err := initer.txInit(); err != nil { + db.rwlock.Unlock() + return nil, err + } + } + } + // Exit if the database is not open yet. if !db.opened { db.rwlock.Unlock() diff --git a/db_test.go b/db_test.go index feb3368ce..dc2c05a13 100644 --- a/db_test.go +++ b/db_test.go @@ -247,6 +247,9 @@ func TestOpen_ReadPageSize_FromMeta1_OS(t *testing.T) { // Ensure that it can read the page size from the second meta page if the first one is invalid. // The page size is expected to be the given page size in this case. func TestOpen_ReadPageSize_FromMeta1_Given(t *testing.T) { + if runtime.GOOS == "js" || runtime.GOOS == "wasip1" { + t.Skip("skipping test on WASM due to memory constraints") + } // test page size from 1KB (1024<<0) to 16MB(1024<<14) for i := 0; i <= 14; i++ { givenPageSize := 1024 << uint(i) @@ -332,6 +335,9 @@ func TestOpen_Size_Large(t *testing.T) { if testing.Short() { t.Skip("short mode") } + if runtime.GOOS == "js" || runtime.GOOS == "wasip1" { + t.Skip("skipping large file test in WASM") + } // Open a data file. db := btesting.MustCreateDB(t) @@ -456,6 +462,10 @@ func TestOpen_FileTooSmall(t *testing.T) { // read transaction blocks the write transaction and causes deadlock. // This is a very hacky test since the mmap size is not exposed. func TestDB_Open_InitialMmapSize(t *testing.T) { + t.Parallel() + if runtime.GOOS == "js" || runtime.GOOS == "wasip1" { + t.Skip("skipping test on WASM due to memory constraints") + } path := tempfile() defer os.Remove(path) diff --git a/internal/common/bolt_wasm.go b/internal/common/bolt_wasm.go new file mode 100644 index 000000000..d7ab938cb --- /dev/null +++ b/internal/common/bolt_wasm.go @@ -0,0 +1,9 @@ +//go:build js || wasip1 + +package common + +// MaxMapSize represents the largest mmap size supported by Bolt. +const MaxMapSize = 0x10000000 + +// MaxAllocSize is the size used when creating array pointers. +const MaxAllocSize = 0x10100000 diff --git a/mlock_unix.go b/mlock_unix.go index 9a0fd332c..80c36fa1d 100644 --- a/mlock_unix.go +++ b/mlock_unix.go @@ -1,4 +1,4 @@ -//go:build !windows +//go:build !windows && !js && !wasip1 package bbolt diff --git a/simulation_test.go b/simulation_test.go index 6f4d5b236..48ed99414 100644 --- a/simulation_test.go +++ b/simulation_test.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" "math/rand" + "runtime" "sync" "sync/atomic" "testing" @@ -34,6 +35,9 @@ func testSimulate(t *testing.T, openOption *bolt.Options, round, threadCount, pa if testing.Short() { t.Skip("skipping test in short mode.") } + if runtime.GOARCH == "wasm" && threadCount >= 1000 { + t.Skip("skipping test on wasm with 1000+ concurrency") + } // A list of operations that readers and writers can perform. var readerHandlers = []simulateHandler{simulateGetHandler} diff --git a/unix_test.go b/unix_test.go index ac53ad559..ccb18201d 100644 --- a/unix_test.go +++ b/unix_test.go @@ -1,4 +1,4 @@ -//go:build !windows +//go:build !windows && !js && !wasip1 package bbolt_test