diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index eb5bab1a1..e49de0c7c 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -61,13 +61,6 @@ jobs: run: | pnpm build:all - - name: Typecheck pglite - working-directory: ${{ github.workspace }}/packages/pglite - run: pnpm typecheck - - name: Test pglite - working-directory: ${{ github.workspace }}/packages/pglite - run: pnpm test - - name: Upload PGlite Interim to Github artifacts id: upload-pglite-interim-build-files uses: actions/upload-artifact@v4 @@ -84,6 +77,26 @@ jobs: path: ./packages/pglite-tools/release/** retention-days: 60 + - name: Upload pglite-postgis build artifacts to Github artifacts + id: upload-pglite-postgis-release-files + uses: actions/upload-artifact@v4 + with: + name: pglite-postgis-release-files-node-v20.x + path: ./packages/pglite-postgis/release/** + retention-days: 60 + + - name: Typecheck pglite + working-directory: ${{ github.workspace }}/packages/pglite + run: pnpm typecheck + + - name: Test pglite + working-directory: ${{ github.workspace }}/packages/pglite + run: pnpm test + + - name: Test pglite-postgis + working-directory: ${{ github.workspace }}/packages/pglite-postgis + run: pnpm test + build-and-test-pglite: name: Build and Test packages/pglite runs-on: blacksmith-32vcpu-ubuntu-2204 @@ -119,6 +132,12 @@ jobs: name: pglite-tools-release-files-node-v20.x path: ./packages/pglite-tools/release + - name: Download pglite-postgis build artifacts + uses: actions/download-artifact@v4 + with: + name: pglite-postgis-release-files-node-v20.x + path: ./packages/pglite-postgis/release + - name: Install dependencies run: | pnpm install --frozen-lockfile @@ -204,6 +223,12 @@ jobs: with: name: pglite-tools-release-files-node-v20.x path: ./packages/pglite-tools/release/ + + - name: Download pglite-postgis build artifacts + uses: actions/download-artifact@v4 + with: + name: pglite-postgis-release-files-node-v20.x + path: ./packages/pglite-postgis/release/ - name: Install dependencies run: pnpm install --frozen-lockfile @@ -225,7 +250,14 @@ jobs: uses: actions/upload-artifact@v4 with: name: pglite-tools-dist-node-v${{ matrix.node }} - path: ./packages/pglite-tools/dist/* + path: ./packages/pglite-tools/dist/* + + - name: Upload pglite-postgis distribution artifact + id: upload-pglite-postgis-dist + uses: actions/upload-artifact@v4 + with: + name: pglite-postgis-dist-node-v${{ matrix.node }} + path: ./packages/pglite-postgis/dist/* publish-website-with-demos: name: Publish website with demos @@ -254,12 +286,24 @@ jobs: name: pglite-tools-release-files-node-v20.x path: ./packages/pglite-tools/release + - name: Download pglite-postgis build artifacts + uses: actions/download-artifact@v4 + with: + name: pglite-postgis-release-files-node-v20.x + path: ./packages/pglite-postgis/release/ + - name: Download PGlite build artifacts uses: actions/download-artifact@v4 with: name: pglite-dist-node-v20.x path: ./packages/pglite/dist/ + - name: Download pglite-postgis dist artifacts + uses: actions/download-artifact@v4 + with: + name: pglite-postgis-dist-node-v20.x + path: ./packages/pglite-postgis/dist/ + - name: Install dependencies run: pnpm install --frozen-lockfile @@ -368,6 +412,12 @@ jobs: name: pglite-tools-release-files-node-v20.x path: ./packages/pglite-tools/release/ + - name: Download pglite-postgis build artifacts + uses: actions/download-artifact@v4 + with: + name: pglite-postgis-release-files-node-v20.x + path: ./packages/pglite-postgis/release/ + - run: pnpm install --frozen-lockfile - run: pnpm --filter "./packages/**" build - name: Create Release Pull Request or Publish diff --git a/docs/extensions/extensions.data.ts b/docs/extensions/extensions.data.ts index d66992983..6f5caf808 100644 --- a/docs/extensions/extensions.data.ts +++ b/docs/extensions/extensions.data.ts @@ -292,6 +292,20 @@ const baseExtensions: Extension[] = [ importName: 'pgtap', size: 239428, }, + { + name: 'postgis', + description: ` + PostGIS extends the capabilities of the PostgreSQL relational database by adding + support for storing, indexing, and querying geospatial data. + *No GDAL support atm. + `, + shortDescription: 'Storing, indexing, and querying geospatial data.', + docs: 'postgis.net', + tags: ['postgres extension'], + importPath: '@electric-sql/pglite-postgis', + importName: 'postgis', + size: 7901736, + }, { name: 'pg_uuidv7', description: ` diff --git a/docs/package.json b/docs/package.json index 03864e194..95ad3ef61 100644 --- a/docs/package.json +++ b/docs/package.json @@ -20,6 +20,7 @@ "dependencies": { "@electric-sql/pglite": "workspace:*", "@electric-sql/pglite-repl": "workspace:*", + "@electric-sql/pglite-postgis": "workspace:*", "@uiw/codemirror-theme-github": "^4.23.0", "dedent": "^1.5.3" } diff --git a/docs/repl/allExtensions.ts b/docs/repl/allExtensions.ts index 9c8c75f9f..21ba9290a 100644 --- a/docs/repl/allExtensions.ts +++ b/docs/repl/allExtensions.ts @@ -25,6 +25,7 @@ export { pg_visibility } from '@electric-sql/pglite/contrib/pg_visibility' export { pg_walinspect } from '@electric-sql/pglite/contrib/pg_walinspect' export { pgcrypto } from '@electric-sql/pglite/contrib/pgcrypto' export { pgtap } from '@electric-sql/pglite/pgtap' +export { postgis } from '@electric-sql/pglite-postgis' export { seg } from '@electric-sql/pglite/contrib/seg' export { tablefunc } from '@electric-sql/pglite/contrib/tablefunc' export { tcn } from '@electric-sql/pglite/contrib/tcn' diff --git a/package.json b/package.json index 600dd8143..d983ff93b 100644 --- a/package.json +++ b/package.json @@ -12,9 +12,10 @@ "ci:publish": "pnpm changeset publish", "ts:build": "pnpm -r --filter \"./packages/**\" build", "ts:build:debug": "DEBUG=true pnpm ts:build", + "wasm:copy-postgis": "mkdir -p ./packages/pglite-postgis/release && cp ./postgres-pglite/dist/extensions/postgis/postgis.tar.gz ./packages/pglite-postgis/release", "wasm:copy-pgdump": "mkdir -p ./packages/pglite-tools/release && cp ./postgres-pglite/dist/bin/pg_dump.* ./packages/pglite-tools/release", "wasm:copy-pglite": "mkdir -p ./packages/pglite/release/ && cp ./postgres-pglite/dist/bin/pglite.* ./packages/pglite/release/ && cp ./postgres-pglite/dist/extensions/*.tar.gz ./packages/pglite/release/", - "wasm:build": "cd postgres-pglite && ./build-with-docker.sh && cd .. && pnpm wasm:copy-pglite && pnpm wasm:copy-pgdump", + "wasm:build": "cd postgres-pglite && ./build-with-docker.sh && cd .. && pnpm wasm:copy-pglite && pnpm wasm:copy-pgdump && pnpm wasm:copy-postgis", "wasm:build:debug": "DEBUG=true pnpm wasm:build", "build:all": "pnpm wasm:build && pnpm ts:build", "build:all:debug": "DEBUG=true pnpm build:all" diff --git a/packages/pglite-postgis/.gitignore b/packages/pglite-postgis/.gitignore new file mode 100644 index 000000000..ef8abb0e7 --- /dev/null +++ b/packages/pglite-postgis/.gitignore @@ -0,0 +1,2 @@ +release/* +dist \ No newline at end of file diff --git a/packages/pglite-postgis/CHANGELOG.md b/packages/pglite-postgis/CHANGELOG.md new file mode 100644 index 000000000..41b251d32 --- /dev/null +++ b/packages/pglite-postgis/CHANGELOG.md @@ -0,0 +1,7 @@ +# @electric-sql/pglite-postgis + +## 0.0.1 + +- Initial release +- PostGIS extension extracted from `@electric-sql/pglite` + diff --git a/packages/pglite-postgis/README.md b/packages/pglite-postgis/README.md new file mode 100644 index 000000000..1e671b7b3 --- /dev/null +++ b/packages/pglite-postgis/README.md @@ -0,0 +1,50 @@ +# @electric-sql/pglite-postgis + +PostGIS extension for [PGlite](https://pglite.dev). + +## Installation + +```bash +npm install @electric-sql/pglite-postgis +``` + +## Usage + +```typescript +import { PGlite } from '@electric-sql/pglite' +import { postgis } from '@electric-sql/pglite-postgis' + +const pg = new PGlite({ + extensions: { + postgis, + }, +}) + +await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') + +// Create a table with geometry columns +await pg.exec(` + CREATE TABLE cities ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + location GEOMETRY(Point, 4326) + ); +`) + +// Insert data +await pg.query(` + INSERT INTO cities (name, location) + VALUES ('New York', ST_GeomFromText('POINT(-74.0060 40.7128)', 4326)) +`) + +// Query with spatial functions +const result = await pg.query(` + SELECT name, ST_AsText(location) as location + FROM cities +`) +``` + +## License + +Apache-2.0 + diff --git a/packages/pglite-postgis/eslint.config.js b/packages/pglite-postgis/eslint.config.js new file mode 100644 index 000000000..e001f1b83 --- /dev/null +++ b/packages/pglite-postgis/eslint.config.js @@ -0,0 +1,22 @@ +import globals from 'globals' +import rootConfig from '../../eslint.config.js' + +export default [ + ...rootConfig, + { + ignores: ['release/**/*', 'dist/**/*'], + }, + { + languageOptions: { + globals: { + ...globals.browser, + ...globals.node, + }, + }, + rules: { + ...rootConfig.rules, + '@typescript-eslint/no-explicit-any': 'off', + }, + }, +] + diff --git a/packages/pglite-postgis/package.json b/packages/pglite-postgis/package.json new file mode 100644 index 000000000..d5a768298 --- /dev/null +++ b/packages/pglite-postgis/package.json @@ -0,0 +1,66 @@ +{ + "name": "@electric-sql/pglite-postgis", + "version": "0.0.1", + "description": "PostGIS extension for PGlite", + "author": "Electric DB Limited", + "homepage": "https://pglite.dev", + "license": "Apache-2.0", + "keywords": [ + "postgres", + "sql", + "database", + "wasm", + "pglite", + "postgis", + "gis", + "geospatial" + ], + "private": false, + "publishConfig": { + "access": "public" + }, + "type": "module", + "main": "./dist/index.cjs", + "module": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "import": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + }, + "require": { + "types": "./dist/index.d.cts", + "default": "./dist/index.cjs" + } + } + }, + "files": [ + "dist" + ], + "repository": { + "type": "git", + "url": "git+https://github.com/electric-sql/pglite.git", + "directory": "packages/pglite-postgis" + }, + "scripts": { + "build": "tsup", + "check:exports": "attw . --pack --profile node16", + "lint": "eslint ./src ./tests --report-unused-disable-directives --max-warnings 0", + "format": "prettier --write ./src ./tests", + "typecheck": "tsc", + "stylecheck": "pnpm lint && prettier --check ./src ./tests", + "test": "vitest", + "prepublishOnly": "pnpm check:exports" + }, + "devDependencies": { + "@arethetypeswrong/cli": "^0.18.1", + "@electric-sql/pglite": "workspace:*", + "@types/node": "^20.16.11", + "vitest": "^2.1.2" + }, + "peerDependencies": { + "@electric-sql/pglite": "workspace:0.3.14" + } +} + diff --git a/packages/pglite-postgis/src/index.ts b/packages/pglite-postgis/src/index.ts new file mode 100644 index 000000000..7638fbd62 --- /dev/null +++ b/packages/pglite-postgis/src/index.ts @@ -0,0 +1,17 @@ +import type { + Extension, + ExtensionSetupResult, + PGliteInterface, +} from '@electric-sql/pglite' + +const setup = async (_pg: PGliteInterface, emscriptenOpts: any) => { + return { + emscriptenOpts, + bundlePath: new URL('../release/postgis.tar.gz', import.meta.url), + } satisfies ExtensionSetupResult +} + +export const postgis = { + name: 'postgis', + setup, +} satisfies Extension diff --git a/packages/pglite-postgis/tests/postgis.test.ts b/packages/pglite-postgis/tests/postgis.test.ts new file mode 100644 index 000000000..c38ba765e --- /dev/null +++ b/packages/pglite-postgis/tests/postgis.test.ts @@ -0,0 +1,284 @@ +import { describe, it, expect } from 'vitest' +import { PGlite } from '@electric-sql/pglite' +import { postgis } from '../src/index.js' + +describe(`postgis`, () => { + it('basic', async () => { + const pg = new PGlite({ + extensions: { + postgis, + }, + }) + + await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') + await pg.exec(` + CREATE TABLE vehicle_location ( + time TIMESTAMPTZ NOT NULL, + vehicle_id INT NOT NULL, + location GEOGRAPHY(POINT, 4326) +); + `) + const inserted = await pg.query(`INSERT INTO vehicle_location VALUES + ('2023-05-29 20:00:00', 1, 'POINT(15.3672 -87.7231)'), + ('2023-05-30 20:00:00', 1, 'POINT(15.3652 -80.7331)'), + ('2023-05-31 20:00:00', 1, 'POINT(15.2672 -85.7431)');`) + + expect(inserted.affectedRows).toEqual(3) + }), + it('cities', async () => { + const pg = new PGlite({ + extensions: { + postgis, + }, + }) + + await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') + await pg.exec(` + CREATE TABLE cities ( + id SERIAL PRIMARY KEY, + name VARCHAR(100), + location GEOMETRY(Point, 4326) +); + `) + const inserted = await pg.query(`INSERT INTO cities (name, location) +VALUES + ('New York', ST_GeomFromText('POINT(-74.0060 40.7128)', 4326)), + ('Los Angeles', ST_GeomFromText('POINT(-118.2437 34.0522)', 4326)), + ('Chicago', ST_GeomFromText('POINT(-87.6298 41.8781)', 4326));`) + + expect(inserted.affectedRows).toEqual(3) + + const cities = await pg.query(`WITH state_boundary AS ( + SELECT ST_GeomFromText( + 'POLYGON((-91 36, -91 43, -87 43, -87 36, -91 36))', 4326 + ) AS geom +) +SELECT c.name +FROM cities c, state_boundary s +WHERE ST_Within(c.location, s.geom);`) + + expect(cities.affectedRows).toBe(0) + expect(cities.rows[0]).toEqual({ + name: 'Chicago', + }) + }) +}) + +it('areas', async () => { + const pg = new PGlite({ + extensions: { + postgis, + }, + }) + await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') + + const area1 = await pg.exec(` + select ST_Area(geom) sqft, + ST_Area(geom) * 0.3048 ^ 2 sqm + from ( + select 'SRID=2249;POLYGON((743238 2967416,743238 2967450, + 743265 2967450,743265.625 2967416,743238 2967416))' :: geometry geom + ) subquery;`) + + expect(area1).toEqual([ + { + rows: [ + { + sqft: 928.625, + sqm: 86.27208552, + }, + ], + fields: [ + { + name: 'sqft', + dataTypeID: 701, + }, + { + name: 'sqm', + dataTypeID: 701, + }, + ], + affectedRows: 0, + }, + ]) + + const area2 = await pg.exec(` + select ST_Area(geom) sqft, + ST_Area(ST_Transform(geom, 26986)) As sqm + from ( + select + 'SRID=2249;POLYGON((743238 2967416,743238 2967450, + 743265 2967450,743265.625 2967416,743238 2967416))' :: geometry geom + ) subquery; + + -- Cleanup test schema + -- DROP SCHEMA postgis_test CASCADE; + `) + + expect(area2).toEqual([ + { + rows: [ + { + sqft: 928.625, + sqm: 86.27243061926092, + }, + ], + fields: [ + { + name: 'sqft', + dataTypeID: 701, + }, + { + name: 'sqm', + dataTypeID: 701, + }, + ], + affectedRows: 0, + }, + ]) + + const area3 = await pg.exec(` + select ST_Area(geog) / 0.3048 ^ 2 sqft_spheroid, + ST_Area(geog, false) / 0.3048 ^ 2 sqft_sphere, + ST_Area(geog) sqm_spheroid + from ( + select ST_Transform( + 'SRID=2249;POLYGON((743238 2967416,743238 2967450,743265 2967450,743265.625 2967416,743238 2967416))'::geometry, + 4326 + ) :: geography geog + ) as subquery; + `) + + expect(area3).toEqual([ + { + rows: [ + { + sqft_spheroid: 928.6844047556697, + sqft_sphere: 926.609762750544, + sqm_spheroid: 86.27760440239217, + }, + ], + fields: [ + { + name: 'sqft_spheroid', + dataTypeID: 701, + }, + { + name: 'sqft_sphere', + dataTypeID: 701, + }, + { + name: 'sqm_spheroid', + dataTypeID: 701, + }, + ], + affectedRows: 0, + }, + ]) +}) + +it('ST_Polygonize', async () => { + const pg = new PGlite({ + extensions: { + postgis, + }, + }) + await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') + const res = await pg.exec(` + WITH data(geom) AS (VALUES + ('LINESTRING (180 40, 30 20, 20 90)'::geometry) + ,('LINESTRING (180 40, 160 160)'::geometry) + ,('LINESTRING (80 60, 120 130, 150 80)'::geometry) + ,('LINESTRING (80 60, 150 80)'::geometry) + ,('LINESTRING (20 90, 70 70, 80 130)'::geometry) + ,('LINESTRING (80 130, 160 160)'::geometry) + ,('LINESTRING (20 90, 20 160, 70 190)'::geometry) + ,('LINESTRING (70 190, 80 130)'::geometry) + ,('LINESTRING (70 190, 160 160)'::geometry) + ) + SELECT ST_AsText( ST_Polygonize( geom )) + FROM data; + `) + + expect(res).toEqual([ + { + rows: [ + { + st_astext: + 'GEOMETRYCOLLECTION(POLYGON((180 40,30 20,20 90,70 70,80 130,160 160,180 40),(150 80,120 130,80 60,150 80)),POLYGON((80 60,120 130,150 80,80 60)),POLYGON((80 130,70 70,20 90,20 160,70 190,80 130)),POLYGON((160 160,80 130,70 190,160 160)))', + }, + ], + fields: [ + { + name: 'st_astext', + dataTypeID: 25, + }, + ], + affectedRows: 0, + }, + ]) +}) + +it('complex1', async () => { + const pg = new PGlite({ + extensions: { + postgis, + }, + }) + await pg.exec('CREATE EXTENSION IF NOT EXISTS postgis;') + + await pg.exec(` + -- Create test schema + -- CREATE SCHEMA IF NOT EXISTS postgis_test; + -- SET search_path TO postgis_test; + + -- Create a table with geometry columns + CREATE TABLE cities ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + population INTEGER, + geom GEOMETRY(Point, 4326) + );`) + + await pg.exec(` + CREATE TABLE rivers ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + geom GEOMETRY(LineString, 4326) + ); + + -- Insert sample data + INSERT INTO cities (name, population, geom) VALUES + ('Paris', 2148000, ST_SetSRID(ST_MakePoint(2.3522, 48.8566), 4326)), + ('Berlin', 3769000, ST_SetSRID(ST_MakePoint(13.4050, 52.5200), 4326)), + ('London', 8982000, ST_SetSRID(ST_MakePoint(-0.1276, 51.5072), 4326)), + ('Amsterdam', 872757, ST_SetSRID(ST_MakePoint(4.9041, 52.3676), 4326)); + + INSERT INTO rivers (name, geom) VALUES + ('Seine', ST_SetSRID(ST_MakeLine(ARRAY[ + ST_MakePoint(2.1, 48.8), + ST_MakePoint(2.35, 48.85), + ST_MakePoint(2.45, 48.9) + ]), 4326)), + ('Spree', ST_SetSRID(ST_MakeLine(ARRAY[ + ST_MakePoint(13.1, 52.4), + ST_MakePoint(13.35, 52.5), + ST_MakePoint(13.45, 52.52) + ]), 4326)); + + -- Create spatial index + CREATE INDEX idx_cities_geom ON cities USING GIST (geom); + CREATE INDEX idx_rivers_geom ON rivers USING GIST (geom); + + -- Query: Find cities within 10 km of any river + SELECT + c.name AS city, + r.name AS river, + ST_Distance(c.geom::geography, r.geom::geography) AS distance_km + FROM cities c + JOIN rivers r + ON ST_DWithin(c.geom::geography, r.geom::geography, 10000) + ORDER BY distance_km; + + `) +}) diff --git a/packages/pglite-postgis/tsconfig.json b/packages/pglite-postgis/tsconfig.json new file mode 100644 index 000000000..6aab239b1 --- /dev/null +++ b/packages/pglite-postgis/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "types": ["node"] + }, + "include": ["src", "tsup.config.ts", "vitest.config.ts"] +} + diff --git a/packages/pglite-postgis/tsup.config.ts b/packages/pglite-postgis/tsup.config.ts new file mode 100644 index 000000000..0020b9fe5 --- /dev/null +++ b/packages/pglite-postgis/tsup.config.ts @@ -0,0 +1,26 @@ +import { cpSync } from 'fs' +import { resolve } from 'path' +import { defineConfig } from 'tsup' + +const entryPoints = ['src/index.ts'] + +const minify = process.env.DEBUG === 'true' ? false : true + +export default defineConfig([ + { + entry: entryPoints, + sourcemap: true, + dts: { + entry: entryPoints, + resolve: true, + }, + clean: true, + minify: minify, + shims: true, + format: ['esm', 'cjs'], + onSuccess: async () => { + cpSync(resolve('release/postgis.tar.gz'), resolve('dist/postgis.tar.gz')) + }, + }, +]) + diff --git a/packages/pglite-postgis/vitest.config.ts b/packages/pglite-postgis/vitest.config.ts new file mode 100644 index 000000000..3144cb036 --- /dev/null +++ b/packages/pglite-postgis/vitest.config.ts @@ -0,0 +1,10 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + globals: true, + environment: 'node', + testTimeout: 30000, + }, +}) + diff --git a/packages/pglite-socket/package.json b/packages/pglite-socket/package.json index 5cb84d022..27b4fee20 100644 --- a/packages/pglite-socket/package.json +++ b/packages/pglite-socket/package.json @@ -57,6 +57,7 @@ "@arethetypeswrong/cli": "^0.18.1", "@electric-sql/pg-protocol": "workspace:*", "@electric-sql/pglite": "workspace:*", + "@electric-sql/pglite-postgis": "workspace:*", "@types/emscripten": "^1.41.1", "@types/node": "^20.16.11", "pg": "^8.14.0", @@ -65,6 +66,7 @@ "vitest": "^1.3.1" }, "peerDependencies": { - "@electric-sql/pglite": "workspace:*" + "@electric-sql/pglite": "workspace:*", + "@electric-sql/pglite-postgis": "workspace:*" } } diff --git a/packages/pglite-socket/src/scripts/server.ts b/packages/pglite-socket/src/scripts/server.ts index 606a6bd37..b87e21ad0 100644 --- a/packages/pglite-socket/src/scripts/server.ts +++ b/packages/pglite-socket/src/scripts/server.ts @@ -43,7 +43,7 @@ const args = parseArgs({ type: 'string', short: 'e', default: undefined, - help: 'Comma-separated list of extensions to load (e.g., vector,pgcrypto)', + help: 'Comma-separated list of extensions to load (e.g., vector,pgcrypto,postgis)', }, run: { type: 'string', @@ -86,7 +86,7 @@ Options: -u, --path=UNIX Unix socket to bind to (default: undefined). Takes precedence over host:port -v, --debug=LEVEL Debug level 0-5 (default: 0) -e, --extensions=LIST Comma-separated list of extensions to load - Formats: vector, pgcrypto (built-in/contrib) + Formats: vector, pgcrypto,postgis (built-in/contrib) @org/package/path:exportedName (npm package) -r, --run=COMMAND Command to run after server starts --include-database-url Include DATABASE_URL in subprocess environment diff --git a/packages/pglite/src/extensionUtils.ts b/packages/pglite/src/extensionUtils.ts index ea225566d..3d2f55279 100644 --- a/packages/pglite/src/extensionUtils.ts +++ b/packages/pglite/src/extensionUtils.ts @@ -53,7 +53,8 @@ export async function loadExtensionBundle( export async function loadExtensions( mod: PostgresMod, log: (...args: any[]) => void, -) { +): Promise { + const promises = new Array>() for (const ext in mod.pg_extensions) { let blob try { @@ -64,11 +65,12 @@ export async function loadExtensions( } if (blob) { const bytes = new Uint8Array(await blob.arrayBuffer()) - loadExtension(mod, ext, bytes, log) + promises.push(...loadExtension(mod, ext, bytes, log)) } else { console.error('Could not get binary data for extension:', ext) } } + return Promise.all(promises) } function loadExtension( @@ -76,28 +78,43 @@ function loadExtension( _ext: string, bytes: Uint8Array, log: (...args: any[]) => void, -) { +): Promise[] { + const soPreloadPromises: Promise[] = [] const data = tinyTar.untar(bytes) data.forEach((file: any) => { if (!file.name.startsWith('.')) { const filePath = mod.WASM_PREFIX + '/' + file.name if (file.name.endsWith('.so')) { - const extOk = (...args: any[]) => { - log('pgfs:ext OK', filePath, args) - } - const extFail = (...args: any[]) => { - log('pgfs:ext FAIL', filePath, args) - } - mod.FS.createPreloadedFile( - dirname(filePath), - file.name.split('/').pop()!.slice(0, -3), - file.data as any, // There is a type error in Emscripten's FS.createPreloadedFile, this excepts a Uint8Array, but the type is defined as any - true, - true, - extOk, - extFail, - false, - ) + const soName = file.name.split('/').pop()! // e.g. 'postgis-3.so' + const dirPath = dirname(filePath) + // Wrap createPreloadedFile in a Promise so loadExtensions can await the + // async WASM compilation done by Emscripten's wasm preload plugin. + // The plugin calls extOk only after preloadedWasm[path] is set, so + // awaiting this ensures dlopen finds the pre-compiled module. + const soPreload = new Promise((resolve, reject) => { + const extOk = (...args: any[]) => { + log('pgfs:ext OK', filePath, args) + resolve() + } + const extFail = (...args: any[]) => { + log('pgfs:ext FAIL', filePath, args) + reject(new Error(`Failed to preload ${filePath}`)) + } + // Keep the .so suffix so Emscripten's wasm preload plugin canHandle() matches, + // triggering async WebAssembly.instantiate. The compiled module is stored in + // preloadedWasm under the path with .so. + mod.FS.createPreloadedFile( + dirPath, + soName, + file.data as any, // There is a type error in Emscripten's FS.createPreloadedFile, this excepts a Uint8Array, but the type is defined as any + true, + true, + extOk, + extFail, + false, + ) + }) + soPreloadPromises.push(soPreload) } else { try { const dirPath = filePath.substring(0, filePath.lastIndexOf('/')) @@ -111,6 +128,7 @@ function loadExtension( } } }) + return soPreloadPromises } function dirname(path: string) { diff --git a/packages/pglite/src/pglite.ts b/packages/pglite/src/pglite.ts index de5fedefa..6dfe5541b 100644 --- a/packages/pglite/src/pglite.ts +++ b/packages/pglite/src/pglite.ts @@ -669,6 +669,10 @@ export class PGlite // the previous call might have increased the size of the buffer so reset it to its default this.#inputData = new Uint8Array(PGlite.DEFAULT_RECV_BUF_SIZE) } + this.#readOffset = 0 + this.#outputData = message + + this.#writeOffset = 0 // execute the message mod._interactive_one(message.length, message[0]) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1ccfd45bf..162c10daa 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -44,6 +44,9 @@ importers: '@electric-sql/pglite': specifier: workspace:* version: link:../packages/pglite + '@electric-sql/pglite-postgis': + specifier: workspace:* + version: link:../packages/pglite-postgis '@electric-sql/pglite-repl': specifier: workspace:* version: link:../packages/pglite-repl @@ -202,6 +205,21 @@ importers: specifier: ^2.1.2 version: 2.1.2(@types/node@20.16.11)(jsdom@24.1.3)(terser@5.34.1) + packages/pglite-postgis: + devDependencies: + '@arethetypeswrong/cli': + specifier: ^0.18.1 + version: 0.18.1 + '@electric-sql/pglite': + specifier: workspace:* + version: link:../pglite + '@types/node': + specifier: ^20.16.11 + version: 20.16.11 + vitest: + specifier: ^2.1.2 + version: 2.1.2(@types/node@20.16.11)(jsdom@24.1.3)(terser@5.34.1) + packages/pglite-react: devDependencies: '@arethetypeswrong/cli': @@ -340,6 +358,9 @@ importers: '@electric-sql/pglite': specifier: workspace:* version: link:../pglite + '@electric-sql/pglite-postgis': + specifier: workspace:* + version: link:../pglite-postgis '@types/emscripten': specifier: ^1.41.1 version: 1.41.1 diff --git a/postgres-pglite b/postgres-pglite index bee4a36b7..50b5fc0c5 160000 --- a/postgres-pglite +++ b/postgres-pglite @@ -1 +1 @@ -Subproject commit bee4a36b76d2607f5c1d2ca61fd013958b17d0e9 +Subproject commit 50b5fc0c589b916d8ce4df10b2b33c64bd7c27bf