From ee018b27899d7b8e5588e6696fc09d465875017d Mon Sep 17 00:00:00 2001 From: David Bern Date: Fri, 7 Nov 2025 10:51:37 -0600 Subject: [PATCH] Add support for PostgreSQL 16, 17, and 18 - Updated pgvector to v0.8.1 - Implement separate binary packages for PostgreSQL versions 16, 17, and 18 - Each package has its own pyproject.toml, Makefile, and README - Packages are installable independently but designed to work with main package - Modified _commands.py to search for binaries in pgserver_binaries/ namespace - Python code auto-detects which binary package is installed This provides pre-built wheels for all supported PostgreSQL versions while keeping download size reasonable (users only get the version they need). --- .github/workflows/build-and-test.yml | 205 +++++++++++------- CHANGELOG.md | 21 ++ README.md | 25 ++- packages/pgserver-postgres-16/Makefile | 11 + packages/pgserver-postgres-16/README.md | 10 + packages/pgserver-postgres-16/pyproject.toml | 24 ++ .../src/pgserver_binaries/__init__.py | 1 + packages/pgserver-postgres-17/Makefile | 11 + packages/pgserver-postgres-17/README.md | 10 + packages/pgserver-postgres-17/pyproject.toml | 24 ++ .../src/pgserver_binaries/__init__.py | 1 + packages/pgserver-postgres-18/Makefile | 11 + packages/pgserver-postgres-18/README.md | 10 + packages/pgserver-postgres-18/pyproject.toml | 24 ++ .../src/pgserver_binaries/__init__.py | 1 + pgbuild/Makefile | 59 ++++- pyproject.toml | 14 +- setup.py | 7 - src/pgserver/_build.py | 13 -- src/pgserver/_commands.py | 44 +++- src/pgserver/postgres_server.py | 3 +- 21 files changed, 412 insertions(+), 117 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 packages/pgserver-postgres-16/Makefile create mode 100644 packages/pgserver-postgres-16/README.md create mode 100644 packages/pgserver-postgres-16/pyproject.toml create mode 100644 packages/pgserver-postgres-16/src/pgserver_binaries/__init__.py create mode 100644 packages/pgserver-postgres-17/Makefile create mode 100644 packages/pgserver-postgres-17/README.md create mode 100644 packages/pgserver-postgres-17/pyproject.toml create mode 100644 packages/pgserver-postgres-17/src/pgserver_binaries/__init__.py create mode 100644 packages/pgserver-postgres-18/Makefile create mode 100644 packages/pgserver-postgres-18/README.md create mode 100644 packages/pgserver-postgres-18/pyproject.toml create mode 100644 packages/pgserver-postgres-18/src/pgserver_binaries/__init__.py delete mode 100644 setup.py delete mode 100644 src/pgserver/_build.py diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index ad06a1f..3c8e9a1 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -8,110 +8,155 @@ on: types: - created workflow_dispatch: + jobs: - build_wheels: - name: Build wheels on ${{ matrix.os }} + # Build main pgserver package (Python code only, no binaries) + build_main_package: + name: Build main pgserver package + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v6 + with: + python-version: '3.14' + + - name: Install build tools + run: | + python -m pip install --upgrade pip + python -m pip install --upgrade build + + - name: Remove pginstall directory (main package shouldn't include binaries) + run: rm -rf src/pgserver/pginstall + + - name: Build main package wheel + run: python -m build --wheel + + - name: Upload main package wheel + uses: actions/upload-artifact@v4 + with: + path: dist/*.whl + name: main-package-wheels + + # Build binary packages for each PostgreSQL version + build_binary_wheels: + name: Build ${{ matrix.package }} on ${{ matrix.platform }} runs-on: ${{ matrix.os }} + needs: build_main_package # Build main package first strategy: fail-fast: false matrix: + package: [pgserver-postgres-16, pgserver-postgres-17, pgserver-postgres-18] + os: [ubuntu-latest, macos-15-intel, macos-latest, windows-latest] include: - - os: macos-12 - arch: x86_64 - deployment-target: '10.9' - - os: macos-latest - arch: arm64 - deployment-target: '11.0' - os: ubuntu-latest - arch: x86_64 - deployment-target: '' - - os: windows-2022 - arch: AMD64 - deployment-target: '' + platform: linux-x86_64 + - os: macos-15-intel + platform: macos-x86_64 + macosx_deployment_target: '10.9' + - os: macos-latest + platform: macos-arm64 + macosx_deployment_target: '11.0' + - os: windows-latest + platform: windows-amd64 steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 - if: matrix.os != 'ubuntu-latest' + + - uses: actions/setup-python@v6 with: - python-version: '3.10' - - uses: actions/setup-python@v4 - if: matrix.os == 'ubuntu-latest' - # for testing due to docker env issues - with: - python-version: '3.9' - - name: Install cibuildwheel + python-version: '3.14' + + - name: Install build tools run: | python -m pip install --upgrade pip - python -m pip install --upgrade cibuildwheel - - name: Restore postgres build from cache - if: ${{ matrix.os != 'ubuntu-latest' }} - id: restore-postgres - uses: actions/cache/restore@v3 - env: - cache-name: cache-postgres + python -m pip install --upgrade build + + - name: Install patchelf (Linux) + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y patchelf + + - name: Install bison and flex (Windows) + if: runner.os == 'Windows' + run: | + choco install -y winflexbison3 + + - name: Extract PostgreSQL version from package name + id: pg-version + shell: bash + run: | + # Extract version from package name (e.g., pgserver-postgres-16 -> 16) + VERSION="${{ matrix.package }}" + VERSION="${VERSION##*-}" + echo "version=$VERSION" >> $GITHUB_OUTPUT + + - name: Restore PostgreSQL build from cache + id: cache-postgres + uses: actions/cache@v3 with: path: | pgbuild src/pgserver/pginstall - key: ${{ runner.os }}-${{ runner.arch }}-build-${{ env.cache-name }}-${{ - hashFiles('Makefile', 'pgbuild/Makefile', '.github/workflows/build-and-test.yml') }} - - name: Build postgres and pgvector - if: ${{ matrix.os != 'ubuntu-latest' && ! steps.restore-postgres.outputs.cache-hit }} - env: - MACOSX_DEPLOYMENT_TARGET: ${{ matrix.deployment-target }} - # this step is implied by Build wheels, but we do it here for caching before python tests run - # on ubuntu, cibuildwheel will run this step within a docker container, so it cannot use the cache this way - run: make - - name: Save postgres build - if: ${{ matrix.os != 'ubuntu-latest' && ! steps.restore-postgres.outputs.cache-hit }} - id: cache-postgres - uses: actions/cache/save@v3 + key: ${{ runner.os }}-${{ runner.arch }}-pg${{ steps.pg-version.outputs.version }}-${{ hashFiles('pgbuild/Makefile') }} + + - name: Build binary package (Unix) + if: runner.os != 'Windows' + working-directory: packages/${{ matrix.package }} env: - cache-name: cache-postgres - with: - path: | - pgbuild - src/pgserver/pginstall - key: ${{ runner.os }}-${{ runner.arch }}-build-${{ env.cache-name }}-${{ - hashFiles('Makefile', 'pgbuild/Makefile', '.github/workflows/build-and-test.yml') }} - - name: Build wheels - env: - CIBW_ARCHS: ${{ matrix.arch }} - CIBW_SKIP: pp* cp38-* *-musllinux* - MACOSX_DEPLOYMENT_TARGET: ${{ matrix.deployment-target }} - run: python -m cibuildwheel --output-dir wheelhouse - - name: Save postgres build - if: ${{ matrix.os == 'ubuntu-latest' && ! steps.restore-postgres.outputs.cache-hit }} - id: cache-postgres2 - uses: actions/cache/save@v3 + MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macosx_deployment_target || '' }} + run: | + make build + python -m build --wheel + + - name: Build binary package (Windows) + if: runner.os == 'Windows' + working-directory: packages/${{ matrix.package }} + shell: bash env: - cache-name: cache-postgres - with: - path: | - pgbuild - src/pgserver/pginstall - key: ${{ runner.os }}-${{ runner.arch }}-build-${{ env.cache-name }}-${{ - hashFiles('Makefile', 'pgbuild/Makefile', '.github/workflows/build-and-test.yml') }} - - uses: actions/upload-artifact@v3 + BISON: "C:/ProgramData/chocolatey/bin/win_bison.exe" + FLEX: "C:/ProgramData/chocolatey/bin/win_flex.exe" + run: | + make build + python -m build --wheel + + - name: Upload binary package wheels + uses: actions/upload-artifact@v4 with: - path: wheelhouse/*.whl - name: python-package-distributions + path: packages/${{ matrix.package }}/dist/*.whl + name: binary-wheels-${{ matrix.package }}-${{ matrix.platform }} + + # Publish to PyPI publish-to-pypi: if: ${{ startsWith(github.ref, 'refs/tags/') }} - name: Publish Python dist to PyPI + name: Publish to PyPI needs: - - build_wheels + - build_binary_wheels + - build_main_package runs-on: ubuntu-latest environment: name: pypi - url: https://pypi.org/p/pgserver # Replace with your PyPI project name + url: https://pypi.org/p/pgserver permissions: - id-token: write # IMPORTANT: mandatory for trusted publishing + id-token: write steps: - - name: Download all the dists - uses: actions/download-artifact@v3 - with: - name: python-package-distributions - path: dist/ - - name: Publish distribution 📦 to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + - name: Download all binary wheels + uses: actions/download-artifact@v4 + with: + path: dist/ + pattern: binary-wheels-* + merge-multiple: true + + - name: Download main package wheels + uses: actions/download-artifact@v4 + with: + path: dist/ + name: main-package-wheels + + - name: Flatten dist directory + run: | + find dist -name '*.whl' -exec mv {} dist/ \; + find dist -type d -empty -delete + + - name: Publish all packages to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..5594096 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,21 @@ +# Changelog + +## [0.2.0] - 2025-11-07 + +### Added +- Support for PostgreSQL 17 and 18 (in addition to existing PostgreSQL 16) +- Multi-version architecture with separate binary packages: + - `pgserver-postgres-16` - PostgreSQL 16.10 binaries + - `pgserver-postgres-17` - PostgreSQL 17.6 binaries + - `pgserver-postgres-18` - PostgreSQL 18.0 binaries +- Version selection via pip extras: + - `pip install pgserver` - installs PostgreSQL 18 (default) + - `pip install "pgserver[pg16]"` - installs PostgreSQL 16 + - `pip install "pgserver[pg17]"` - installs PostgreSQL 17 +- `INSTALLED_POSTGRES_VERSION` constant to check which version is installed + +### Changed +- Main `pgserver` package is now a universal wheel (py3-none-any) compatible with all Python 3.9+ versions +- Binary packages are distributed separately from Python code, reducing download size +- Upgraded pgvector extension to v0.8.1 for PostgreSQL 18 compatibility +- PostgreSQL binaries now use RPATH to find bundled libraries, preventing conflicts with system PostgreSQL installations diff --git a/README.md b/README.md index bd2b921..9cd9b1c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ ![Python Version](https://img.shields.io/badge/python-3.9%2C%203.10%2C%203.11%2C%203.12-blue) -![Postgres Version](https://img.shields.io/badge/PostgreSQL-16.2-blue) +![Postgres Version](https://img.shields.io/badge/PostgreSQL-16%20%7C%2017%20%7C%2018-blue) ![Linux Support](https://img.shields.io/badge/Linux%20Support-manylinux-green) ![macOS Apple Silicon Support >=11](https://img.shields.io/badge/macOS%20Apple%20Silicon%20Support-%E2%89%A511(BigSur)-green) @@ -62,6 +62,29 @@ def tmp_postgres(): pg.cleanup() ``` +## PostgreSQL Version Selection + +`pgserver` supports PostgreSQL versions 16, 17, and 18 with **pre-built wheels** for all versions: + +```bash +# PostgreSQL 18 (latest, default) +pip install pgserver + +# PostgreSQL 16 +pip install "pgserver[pg16]" + +# PostgreSQL 17 +pip install "pgserver[pg17]" +``` + +Check which version is installed: +```py +import pgserver +print(f"PostgreSQL version: {pgserver.INSTALLED_POSTGRES_VERSION}") +``` + +**How it works:** The main `pgserver` package contains only Python code. PostgreSQL binaries are provided by separate packages (`pgserver-postgres-16`, `pgserver-postgres-17`, `pgserver-postgres-18`) which are automatically installed based on the extra you choose. + Postgres binaries in the package can be found in the directory pointed to by the `pgserver.POSTGRES_BIN_PATH` to be used directly. diff --git a/packages/pgserver-postgres-16/Makefile b/packages/pgserver-postgres-16/Makefile new file mode 100644 index 0000000..f3a5bf0 --- /dev/null +++ b/packages/pgserver-postgres-16/Makefile @@ -0,0 +1,11 @@ +.DEFAULT_GOAL := build + +build: + PGSERVER_VERSION=16 $(MAKE) -C ../../pgbuild all + rm -rf src/pgserver_binaries/pg16 + mkdir -p src/pgserver_binaries/pg16 + cp -r ../../src/pgserver/pginstall/* src/pgserver_binaries/pg16/ + +clean: + rm -rf src/pgserver_binaries/pg16 + PGSERVER_VERSION=16 $(MAKE) -C ../../pgbuild clean diff --git a/packages/pgserver-postgres-16/README.md b/packages/pgserver-postgres-16/README.md new file mode 100644 index 0000000..613f668 --- /dev/null +++ b/packages/pgserver-postgres-16/README.md @@ -0,0 +1,10 @@ +# pgserver-postgres-16 + +PostgreSQL 16 binaries for the pgserver package. + +This package is automatically installed when you run: +```bash +pip install pgserver[pg16] +``` + +Do not install this package directly unless you know what you're doing. diff --git a/packages/pgserver-postgres-16/pyproject.toml b/packages/pgserver-postgres-16/pyproject.toml new file mode 100644 index 0000000..c43aea0 --- /dev/null +++ b/packages/pgserver-postgres-16/pyproject.toml @@ -0,0 +1,24 @@ +[project] +name = "pgserver-postgres-16" +version = "0.2.0" +description = "PostgreSQL 16 binaries for pgserver" +readme = "README.md" +requires-python = ">=3.9" +license = {file = "../../LICENSE.txt"} +authors=[{ name="Oscar Moll", email="orm@csail.mit.edu" }] +keywords=["postgresql", "pgvector"] + +[tool.setuptools.packages.find] +where = ["src"] +include = ["pgserver_binaries*"] + +[tool.setuptools.package-data] +pgserver_binaries = ["**/*"] + +[tool.cibuildwheel] +before-all = "cd ../../ && PGSERVER_VERSION=16 make -C pgbuild" +environment = {PGSERVER_VERSION="16"} + +[build-system] +requires = ["setuptools>=58.0.0", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/packages/pgserver-postgres-16/src/pgserver_binaries/__init__.py b/packages/pgserver-postgres-16/src/pgserver_binaries/__init__.py new file mode 100644 index 0000000..d9c53ce --- /dev/null +++ b/packages/pgserver-postgres-16/src/pgserver_binaries/__init__.py @@ -0,0 +1 @@ +# PostgreSQL 16 binaries for pgserver diff --git a/packages/pgserver-postgres-17/Makefile b/packages/pgserver-postgres-17/Makefile new file mode 100644 index 0000000..a58b53e --- /dev/null +++ b/packages/pgserver-postgres-17/Makefile @@ -0,0 +1,11 @@ +.DEFAULT_GOAL := build + +build: + PGSERVER_VERSION=17 $(MAKE) -C ../../pgbuild all + rm -rf src/pgserver_binaries/pg17 + mkdir -p src/pgserver_binaries/pg17 + cp -r ../../src/pgserver/pginstall/* src/pgserver_binaries/pg17/ + +clean: + rm -rf src/pgserver_binaries/pg17 + PGSERVER_VERSION=17 $(MAKE) -C ../../pgbuild clean diff --git a/packages/pgserver-postgres-17/README.md b/packages/pgserver-postgres-17/README.md new file mode 100644 index 0000000..113b5fd --- /dev/null +++ b/packages/pgserver-postgres-17/README.md @@ -0,0 +1,10 @@ +# pgserver-postgres-17 + +PostgreSQL 17 binaries for the pgserver package. + +This package is automatically installed when you run: +```bash +pip install pgserver[pg17] +``` + +Do not install this package directly unless you know what you're doing. diff --git a/packages/pgserver-postgres-17/pyproject.toml b/packages/pgserver-postgres-17/pyproject.toml new file mode 100644 index 0000000..49e6e9d --- /dev/null +++ b/packages/pgserver-postgres-17/pyproject.toml @@ -0,0 +1,24 @@ +[project] +name = "pgserver-postgres-17" +version = "0.2.0" +description = "PostgreSQL 17 binaries for pgserver" +readme = "README.md" +requires-python = ">=3.9" +license = {file = "../../LICENSE.txt"} +authors=[{ name="Oscar Moll", email="orm@csail.mit.edu" }] +keywords=["postgresql", "pgvector"] + +[tool.setuptools.packages.find] +where = ["src"] +include = ["pgserver_binaries*"] + +[tool.setuptools.package-data] +pgserver_binaries = ["**/*"] + +[tool.cibuildwheel] +before-all = "cd ../../ && PGSERVER_VERSION=17 make -C pgbuild" +environment = {PGSERVER_VERSION="17"} + +[build-system] +requires = ["setuptools>=58.0.0", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/packages/pgserver-postgres-17/src/pgserver_binaries/__init__.py b/packages/pgserver-postgres-17/src/pgserver_binaries/__init__.py new file mode 100644 index 0000000..d7fcbcd --- /dev/null +++ b/packages/pgserver-postgres-17/src/pgserver_binaries/__init__.py @@ -0,0 +1 @@ +# PostgreSQL 17 binaries for pgserver diff --git a/packages/pgserver-postgres-18/Makefile b/packages/pgserver-postgres-18/Makefile new file mode 100644 index 0000000..25daea8 --- /dev/null +++ b/packages/pgserver-postgres-18/Makefile @@ -0,0 +1,11 @@ +.DEFAULT_GOAL := build + +build: + PGSERVER_VERSION=18 $(MAKE) -C ../../pgbuild all + rm -rf src/pgserver_binaries/pg18 + mkdir -p src/pgserver_binaries/pg18 + cp -r ../../src/pgserver/pginstall/* src/pgserver_binaries/pg18/ + +clean: + rm -rf src/pgserver_binaries/pg18 + PGSERVER_VERSION=18 $(MAKE) -C ../../pgbuild clean diff --git a/packages/pgserver-postgres-18/README.md b/packages/pgserver-postgres-18/README.md new file mode 100644 index 0000000..628d79f --- /dev/null +++ b/packages/pgserver-postgres-18/README.md @@ -0,0 +1,10 @@ +# pgserver-postgres-18 + +PostgreSQL 18 binaries for the pgserver package. + +This package is automatically installed when you run: +```bash +pip install pgserver +``` + +Do not install this package directly unless you know what you're doing. diff --git a/packages/pgserver-postgres-18/pyproject.toml b/packages/pgserver-postgres-18/pyproject.toml new file mode 100644 index 0000000..25d92e7 --- /dev/null +++ b/packages/pgserver-postgres-18/pyproject.toml @@ -0,0 +1,24 @@ +[project] +name = "pgserver-postgres-18" +version = "0.2.0" +description = "PostgreSQL 18 binaries for pgserver" +readme = "README.md" +requires-python = ">=3.9" +license = {file = "../../LICENSE.txt"} +authors=[{ name="Oscar Moll", email="orm@csail.mit.edu" }] +keywords=["postgresql", "pgvector"] + +[tool.setuptools.packages.find] +where = ["src"] +include = ["pgserver_binaries*"] + +[tool.setuptools.package-data] +pgserver_binaries = ["**/*"] + +[tool.cibuildwheel] +before-all = "cd ../../ && PGSERVER_VERSION=18 make -C pgbuild" +environment = {PGSERVER_VERSION="18"} + +[build-system] +requires = ["setuptools>=58.0.0", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/packages/pgserver-postgres-18/src/pgserver_binaries/__init__.py b/packages/pgserver-postgres-18/src/pgserver_binaries/__init__.py new file mode 100644 index 0000000..bf00ff4 --- /dev/null +++ b/packages/pgserver-postgres-18/src/pgserver_binaries/__init__.py @@ -0,0 +1 @@ +# PostgreSQL 18 binaries for pgserver diff --git a/pgbuild/Makefile b/pgbuild/Makefile index d6f8b13..0b38d7a 100644 --- a/pgbuild/Makefile +++ b/pgbuild/Makefile @@ -2,11 +2,38 @@ SHELL := /bin/bash INSTALL_PREFIX := $(shell pwd)/../src/pgserver/pginstall/ BUILD := $(shell pwd)/pgbuild/ +# PostgreSQL version to build (can be overridden via environment variable) +# Default to 18 if not specified +PGSERVER_VERSION ?= 18 + +# Detect OS for setting correct RPATH +UNAME_S := $(shell uname -s) +ifeq ($(UNAME_S),Linux) + RPATH_FLAG := -Wl,-rpath,'\$$\$$ORIGIN/../lib' +else ifeq ($(UNAME_S),Darwin) + RPATH_FLAG := -Wl,-rpath,@loader_path/../lib +endif + +### PostgreSQL version definitions +PG16_VERSION := 16.10 +PG17_VERSION := 17.6 +PG18_VERSION := 18.0 + +# Set the version-specific variables based on PGSERVER_VERSION +ifeq ($(PGSERVER_VERSION),16) + POSTGRES_VERSION := $(PG16_VERSION) +else ifeq ($(PGSERVER_VERSION),17) + POSTGRES_VERSION := $(PG17_VERSION) +else ifeq ($(PGSERVER_VERSION),18) + POSTGRES_VERSION := $(PG18_VERSION) +else + $(error Invalid PGSERVER_VERSION: $(PGSERVER_VERSION). Must be 16, 17, or 18) +endif + .PHONY: all -all: pgvector postgres +all: postgres pgvector -### postgres -POSTGRES_VERSION := 16.2 +### PostgreSQL build POSTGRES_URL := https://ftp.postgresql.org/pub/source/v$(POSTGRES_VERSION)/postgresql-$(POSTGRES_VERSION).tar.gz POSTGRES_SRC := postgresql-$(POSTGRES_VERSION) POSTGRES_BLD := $(POSTGRES_SRC) @@ -22,7 +49,7 @@ $(POSTGRES_SRC)/configure: $(POSTGRES_SRC).tar.gz ## configure $(POSTGRES_BLD)/config.status: $(POSTGRES_SRC)/configure mkdir -p $(POSTGRES_BLD) - cd $(POSTGRES_BLD) && ../$(POSTGRES_SRC)/configure --prefix=$(INSTALL_PREFIX) --without-readline --without-icu + cd $(POSTGRES_BLD) && LDFLAGS="$(RPATH_FLAG)" ../$(POSTGRES_SRC)/configure --prefix=$(INSTALL_PREFIX) --without-readline --without-icu ## build # https://stackoverflow.com/questions/68379786/ @@ -34,12 +61,33 @@ $(POSTGRES_BLD)/src/bin/initdb/initdb: $(POSTGRES_BLD)/config.status $(INSTALL_PREFIX)/bin/postgres: $(POSTGRES_BLD)/config.status mkdir -p $(INSTALL_PREFIX) unset MAKELEVEL && unset MAKEFLAGS && unset MFLAGS && $(MAKE) -C $(POSTGRES_BLD) install + # Set RPATH on all binaries to find bundled libraries + @echo "Setting RPATH on PostgreSQL binaries..." +ifeq ($(UNAME_S),Linux) + @if command -v patchelf >/dev/null 2>&1; then \ + for binary in $(INSTALL_PREFIX)/bin/*; do \ + if file "$$binary" | grep -q ELF; then \ + echo "Setting RPATH for $$binary"; \ + patchelf --force-rpath --set-rpath '$$ORIGIN/../lib' "$$binary" || true; \ + fi \ + done; \ + else \ + echo "Warning: patchelf not found, binaries may not find bundled libraries"; \ + fi +else ifeq ($(UNAME_S),Darwin) + @for binary in $(INSTALL_PREFIX)/bin/*; do \ + if file "$$binary" | grep -q "Mach-O"; then \ + echo "Setting RPATH for $$binary"; \ + install_name_tool -add_rpath @loader_path/../lib "$$binary" 2>/dev/null || true; \ + fi \ + done +endif .PHONY: postgres postgres: $(INSTALL_PREFIX)/bin/postgres ### pgvector -PGVECTOR_TAG := v0.6.2 +PGVECTOR_TAG := v0.8.1 PGVECTOR_URL := https://github.com/pgvector/pgvector/archive/refs/tags/$(PGVECTOR_TAG).tar.gz PGVECTOR_DIR := pgvector-$(PGVECTOR_TAG) @@ -47,7 +95,6 @@ $(PGVECTOR_DIR).tar.gz: curl -L -o $(PGVECTOR_DIR).tar.gz $(PGVECTOR_URL) $(PGVECTOR_DIR)/Makefile: $(PGVECTOR_DIR).tar.gz - # tar extract into pgvector-$(PGVECTOR_TAG) mkdir -p $(PGVECTOR_DIR) tar xzf $(PGVECTOR_DIR).tar.gz -C $(PGVECTOR_DIR) --strip-components=1 touch $(PGVECTOR_DIR)/Makefile diff --git a/pyproject.toml b/pyproject.toml index 4c1fe7b..c441ad0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "pgserver" # Required -version = "0.1.4" # Required +version = "0.2.0" # Required description = "Self-contained postgres server for your python applications" # Required readme = "README.md" # Optional requires-python = ">=3.9" @@ -12,9 +12,16 @@ dependencies = [ "fasteners>=0.19", "platformdirs>=4.0.0", "psutil>=5.9.0", + "pgserver-postgres-18==0.2.0", # Default to PostgreSQL 18 ] [project.optional-dependencies] +pg16 = [ + "pgserver-postgres-16==0.2.0", +] +pg17 = [ + "pgserver-postgres-17==0.2.0", +] dev = [ "sysv_ipc", ] @@ -32,11 +39,6 @@ include = ["pgserver*"] # package names should match these glob patterns (["*"] [tool.pytest.ini_options] testpaths = ["tests"] -[tool.cibuildwheel] -before-all = "make" -test-extras = "test" -test-command = "bash -x {project}/cibuildwheel_test.bash {project}" - [build-system] # These are the assumed default build requirements from pip: # https://pip.pypa.io/en/stable/reference/pip/#pep-517-and-518-support diff --git a/setup.py b/setup.py deleted file mode 100644 index c3e48be..0000000 --- a/setup.py +++ /dev/null @@ -1,7 +0,0 @@ -from setuptools import setup - -setup( - setup_requires=["cffi"], - # dummy but needed for the binaries to work - cffi_modules=["src/pgserver/_build.py:ffibuilder"], -) diff --git a/src/pgserver/_build.py b/src/pgserver/_build.py deleted file mode 100644 index c7e554d..0000000 --- a/src/pgserver/_build.py +++ /dev/null @@ -1,13 +0,0 @@ -""" -dummy module used in setup() -seems needed to cause the binaries to be well formed -The build is done by the Makefile -""" -from cffi import FFI - -ffibuilder = FFI() -ffibuilder.set_source("_postgresql", "") -ffibuilder.cdef("") - -if __name__ == "__main__": - ffibuilder.compile(verbose=True) diff --git a/src/pgserver/_commands.py b/src/pgserver/_commands.py index 4a2dfc3..5d9382a 100644 --- a/src/pgserver/_commands.py +++ b/src/pgserver/_commands.py @@ -1,11 +1,48 @@ from pathlib import Path import sys import subprocess -from typing import Optional, List, Callable +from typing import Optional, List, Callable, Tuple import logging import tempfile +import site -POSTGRES_BIN_PATH = Path(__file__).parent / "pginstall" / "bin" +def _find_postgres_binaries() -> Tuple[Optional[Path], Optional[int]]: + """Find PostgreSQL binaries installed by pgserver-postgres-* packages. + + Returns: + Tuple of (bin_path, version_number) or (None, None) if not found + """ + # Check all site-packages directories for pgserver_binaries + for site_dir in site.getsitepackages() + [site.getusersitepackages()]: + if not site_dir: + continue + + binaries_base = Path(site_dir) / "pgserver_binaries" + if not binaries_base.exists(): + continue + + # Look for pg16, pg17, or pg18 directories + for pg_dir in binaries_base.glob("pg*"): + if not pg_dir.is_dir(): + continue + + bin_path = pg_dir / "bin" + if not bin_path.exists(): + continue + + # Extract version number from directory name (e.g., "pg16" -> 16) + try: + version = int(pg_dir.name[2:]) + # Verify this is a valid PostgreSQL installation + if (bin_path / "postgres").exists() or (bin_path / "postgres.exe").exists(): + return (bin_path, version) + except (ValueError, IndexError): + continue + + return (None, None) + +# Find installed PostgreSQL binaries +POSTGRES_BIN_PATH, INSTALLED_POSTGRES_VERSION = _find_postgres_binaries() _logger = logging.getLogger('pgserver') @@ -58,7 +95,8 @@ def command(args : List[str], pgdata : Optional[Path] = None, **kwargs) -> str: return command -__all__ = [] +__all__ = ['INSTALLED_POSTGRES_VERSION'] + def _init(): for path in POSTGRES_BIN_PATH.iterdir(): exe_name = path.name diff --git a/src/pgserver/postgres_server.py b/src/pgserver/postgres_server.py index d6517d1..1de1c5c 100644 --- a/src/pgserver/postgres_server.py +++ b/src/pgserver/postgres_server.py @@ -9,7 +9,7 @@ import psutil import time -from ._commands import POSTGRES_BIN_PATH, initdb, pg_ctl +from ._commands import POSTGRES_BIN_PATH, INSTALLED_POSTGRES_VERSION, initdb, pg_ctl from .utils import find_suitable_port, find_suitable_socket_dir, DiskList, PostmasterInfo, process_is_running if platform.system() != 'Windows': @@ -39,6 +39,7 @@ def __init__(self, pgdata : Path, *, cleanup_mode : Optional[str] = 'stop'): self.pgdata = pgdata self.log = self.pgdata / 'log' + self.postgres_version = INSTALLED_POSTGRES_VERSION # postgres user name, NB not the same as system user name self.system_user = None