diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 3a18acca4..d7380a8ad 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -164,7 +164,7 @@ jobs: with: issue-number: ${{ github.event.pull_request.number }} comment-author: 'github-actions[bot]' - body-includes: '- PGlite:' + body-includes: '- PGlite with node:' - name: Create or update build outputs comment uses: peter-evans/create-or-update-comment@v4 @@ -174,7 +174,7 @@ jobs: comment-id: ${{ steps.fc.outputs.comment-id }} issue-number: ${{ github.event.pull_request.number }} body: | - - PGlite: ${{ steps.upload-pglite-package.outputs.artifact-url }} + - PGlite with node v${{ matrix.node }}: ${{ steps.upload-pglite-package.outputs.artifact-url }} edit-mode: append build-and-test-pglite-dependents: diff --git a/packages/pglite-tools/src/pgDumpModFactory.ts b/packages/pglite-tools/src/pgDumpModFactory.ts index dadc2e7a3..67dd3f37e 100644 --- a/packages/pglite-tools/src/pgDumpModFactory.ts +++ b/packages/pglite-tools/src/pgDumpModFactory.ts @@ -22,16 +22,16 @@ export interface PgDumpMod FS: FS WASM_PREFIX: string INITIAL_MEMORY: number - _set_read_write_cbs: (read_cb: number, write_cb: number) => void + _pgl_set_rw_cbs: (read_cb: number, write_cb: number) => void addFunction: ( cb: (ptr: any, length: number) => void, signature: string, ) => number removeFunction: (f: number) => void - _main: (args: string[]) => number onExit: (status: number) => void print: (test: string) => void printErr: (text: string) => void + callMain: (args?: string[]) => number } type PgDumpFactory = ( diff --git a/packages/pglite-tools/src/pg_dump.ts b/packages/pglite-tools/src/pg_dump.ts index 5e07d5f4e..a948a61da 100644 --- a/packages/pglite-tools/src/pg_dump.ts +++ b/packages/pglite-tools/src/pg_dump.ts @@ -36,11 +36,10 @@ async function execPgDump({ args: string[] }): Promise { let pgdump_write, pgdump_read - let exitStatus = 0 + let exitCode = 0 let stderrOutput: string = '' let stdoutOutput: string = '' const emscriptenOpts: Partial = { - arguments: args, noExitRuntime: false, print: (text) => { stdoutOutput += text @@ -49,7 +48,7 @@ async function execPgDump({ stderrOutput += text }, onExit: (status: number) => { - exitStatus = status + exitCode = status }, preRun: [ (mod: PgDumpMod) => { @@ -83,7 +82,7 @@ async function execPgDump({ return length }, 'iii') - mod._set_read_write_cbs(pgdump_read, pgdump_write) + mod._pgl_set_rw_cbs(pgdump_read, pgdump_write) // default $HOME in emscripten is /home/web_user mod.FS.chmod('/home/web_user/.pgpass', 0o0600) // https://www.postgresql.org/docs/current/libpq-pgpass.html } @@ -92,13 +91,14 @@ async function execPgDump({ } const mod = await PgDumpModFactory(emscriptenOpts) + mod.callMain(args) let fileContents = '' - if (!exitStatus) { + if (!exitCode) { fileContents = mod.FS.readFile(dumpFilePath, { encoding: 'utf8' }) } return { - exitCode: exitStatus, + exitCode, fileContents, stderr: stderrOutput, stdout: stdoutOutput, diff --git a/packages/pglite/src/pglite.ts b/packages/pglite/src/pglite.ts index c0da6cba8..8c3416fa7 100644 --- a/packages/pglite/src/pglite.ts +++ b/packages/pglite/src/pglite.ts @@ -76,14 +76,14 @@ export class PGlite #globalNotifyListeners = new Set<(channel: string, payload: string) => void>() // receive data from wasm - #pglite_write: number = -1 + #pglite_socket_write: number = -1 #currentResults: BackendMessage[] = [] #currentThrowOnError: boolean = false #currentOnNotice: ((notice: NoticeMessage) => void) | undefined // send data to wasm - #pglite_read: number = -1 + #pglite_socket_read: number = -1 // buffer that holds the data to be sent to wasm #outputData: any = [] // read index in the buffer @@ -214,12 +214,12 @@ export class PGlite const extensionInitFns: Array<() => Promise> = [] const args = [ - `PGDATA=${PGDATA}`, - `PREFIX=${WASM_PREFIX}`, - `PGUSER=${options.username ?? 'postgres'}`, - `PGDATABASE=${options.database ?? 'template1'}`, - 'MODE=REACT', - 'REPL=N', + // `PGDATA=${PGDATA}`, + // `PREFIX=${WASM_PREFIX}`, + // `PGUSER=${options.username ?? 'postgres'}`, + // `PGDATABASE=${options.database ?? 'template1'}`, + // 'MODE=REACT', + // 'REPL=N', // "-F", // Disable fsync (TODO: Only for in-memory mode?) ...(this.debug ? ['-d', this.debug.toString()] : []), ] @@ -332,6 +332,20 @@ export class PGlite } mod.FS.registerDevice(devId, devOpt) mod.FS.mkdev('/dev/blob', devId) + // mod.FS.mkdir('/tmp') && mod.FS.chmod('/tmp', 0o700) + }, + (mod: any) => { + mod.ENV.MODE = 'REACT' + mod.ENV.PGDATA = PGDATA + mod.ENV.PREFIX = WASM_PREFIX + mod.ENV.PGUSER = options.username ?? 'postgres' + mod.ENV.PGDATABASE = options.database ?? 'template1' + mod.ENV.LC_CTYPE = 'en_US.UTF-8' + mod.ENV.TZ = 'UTC' + mod.ENV.PGTZ = 'UTC' + mod.ENV.PGCLIENTENCODING = 'UTF8' + // mod.ENV.PGDATABASE = 'template1' + // mod.ENV.PG_COLOR = 'always' }, ], } @@ -387,66 +401,73 @@ export class PGlite this.mod = await PostgresModFactory(emscriptenOpts) // set the write callback - this.#pglite_write = this.mod.addFunction((ptr: any, length: number) => { - let bytes - try { - bytes = this.mod!.HEAPU8.subarray(ptr, ptr + length) - } catch (e: any) { - console.error('error', e) - throw e - } - this.#protocolParser.parse(bytes, (msg) => { - this.#parse(msg) - }) - if (this.#keepRawResponse) { - const copied = bytes.slice() - - let requiredSize = this.#writeOffset + copied.length - - if (requiredSize > this.#inputData.length) { - const newSize = - this.#inputData.length + - (this.#inputData.length >> 1) + - requiredSize - if (requiredSize > PGlite.MAX_BUFFER_SIZE) { - requiredSize = PGlite.MAX_BUFFER_SIZE + this.#pglite_socket_write = this.mod.addFunction( + (ptr: any, length: number) => { + let bytes + try { + bytes = this.mod!.HEAPU8.subarray(ptr, ptr + length) + } catch (e: any) { + console.error('error', e) + throw e + } + this.#protocolParser.parse(bytes, (msg) => { + this.#parse(msg) + }) + if (this.#keepRawResponse) { + const copied = bytes.slice() + let requiredSize = this.#writeOffset + copied.length + if (requiredSize > this.#inputData.length) { + const newSize = + this.#inputData.length + + (this.#inputData.length >> 1) + + requiredSize + if (requiredSize > PGlite.MAX_BUFFER_SIZE) { + requiredSize = PGlite.MAX_BUFFER_SIZE + } + const newBuffer = new Uint8Array(newSize) + newBuffer.set(this.#inputData.subarray(0, this.#writeOffset)) + this.#inputData = newBuffer } - const newBuffer = new Uint8Array(newSize) - newBuffer.set(this.#inputData.subarray(0, this.#writeOffset)) - this.#inputData = newBuffer + this.#inputData.set(copied, this.#writeOffset) + this.#writeOffset += copied.length + return this.#inputData.length } - - this.#inputData.set(copied, this.#writeOffset) - this.#writeOffset += copied.length - - return this.#inputData.length - } - return length - }, 'iii') + return length + }, + 'iii', + ) // set the read callback - this.#pglite_read = this.mod.addFunction((ptr: any, max_length: number) => { - // copy current data to wasm buffer - let length = this.#outputData.length - this.#readOffset - if (length > max_length) { - length = max_length - } - try { - this.mod!.HEAP8.set( - (this.#outputData as Uint8Array).subarray( - this.#readOffset, - this.#readOffset + length, - ), - ptr, - ) - this.#readOffset += length - } catch (e) { - console.log(e) - } - return length - }, 'iii') + this.#pglite_socket_read = this.mod.addFunction( + (ptr: any, max_length: number) => { + // copy current data to wasm buffer + let length = this.#outputData.length - this.#readOffset + if (length > max_length) { + length = max_length + } + try { + this.mod!.HEAP8.set( + (this.#outputData as Uint8Array).subarray( + this.#readOffset, + this.#readOffset + length, + ), + ptr, + ) + this.#readOffset += length + } catch (e) { + console.error(e) + } + return length + }, + 'iii', + ) + + this.mod._pgl_set_rw_cbs( + this.#pglite_socket_read, + this.#pglite_socket_write, + ) - this.mod._set_read_write_cbs(this.#pglite_read, this.#pglite_write) + this.mod._pgl_startup(args) // Sync the filesystem from any previous store await this.fs!.initialSyncFs() @@ -576,8 +597,8 @@ export class PGlite try { await this.execProtocol(serialize.end()) this.mod!._pgl_shutdown() - this.mod!.removeFunction(this.#pglite_read) - this.mod!.removeFunction(this.#pglite_write) + this.mod!.removeFunction(this.#pglite_socket_read) + this.mod!.removeFunction(this.#pglite_socket_write) } catch (e) { const err = e as { name: string; status: number } if (err.name === 'ExitStatus' && err.status === 0) { @@ -671,7 +692,7 @@ export class PGlite } // execute the message - mod._interactive_one(message.length, message[0]) + mod._pgl_interactive_one(message.length, message[0]) this.#outputData = [] diff --git a/packages/pglite/src/postgresMod.ts b/packages/pglite/src/postgresMod.ts index 2a3e8ee1e..3acca50da 100644 --- a/packages/pglite/src/postgresMod.ts +++ b/packages/pglite/src/postgresMod.ts @@ -20,21 +20,21 @@ export interface PostgresMod preRun: Array<{ (mod: PostgresMod): void }> postRun: Array<{ (mod: PostgresMod): void }> FS: FS - FD_BUFFER_MAX: number WASM_PREFIX: string INITIAL_MEMORY: number pg_extensions: Record> _pgl_initdb: () => number _pgl_backend: () => void _pgl_shutdown: () => void - _interactive_write: (msgLength: number) => void - _interactive_one: (length: number, peek: number) => void - _set_read_write_cbs: (read_cb: number, write_cb: number) => void + _pgl_interactive_one: (length: number, peek: number) => void + _pgl_set_rw_cbs: (read_cb: number, write_cb: number) => void + _pgl_startup: (args?: string[]) => number addFunction: ( cb: (ptr: any, length: number) => void, signature: string, ) => number removeFunction: (f: number) => void + callMain: (args?: string[]) => number } type PostgresFactory = ( diff --git a/postgres-pglite b/postgres-pglite index 1195d5388..1d88faa52 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit 1195d5388bd5529e0013c45fa816cfcd953d84e0 +Subproject commit 1d88faa526bd2e3256a397241b0eef9b76651048