diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..4d116a9f --- /dev/null +++ b/.env.example @@ -0,0 +1,25 @@ +POSTGRES_USER=username +POSTGRES_PASSWORD=password +POSTGRES_DB=postgis + +PGUSER=username +PGPASSWORD=password +PGDATABASE=postgis +PGHOST=localhost +PGPORT=5439 + +DOCKER_BUILDKIT=1 +BUILDKIT_INLINE_CACHE=1 + +# Host-script defaults. +PGSTAC_BUILD_POLICY=always +PGSTAC_FAST=0 +PGSTAC_WATCH=0 +PGSTAC_STRICT=1 + +# Migration helper defaults. +# PGSTAC_VERSION=0.9.11 +# PGSTAC_FROM_VERSION=0.9.10 +# PGSTAC_TO_VERSION=0.9.11 +# PGSTAC_OVERWRITE=0 +# PGSTAC_DEBUG=0 \ No newline at end of file diff --git a/.github/dependabot.yml b/.github/dependabot.yml index b1b1e35d..f2090777 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,11 +5,50 @@ updates: schedule: interval: "weekly" day: "monday" + open-pull-requests-limit: 5 groups: - minor-and-patch: + actions-all: applies-to: version-updates patterns: - "*" - update-types: - - "minor" - - "patch" + + - package-ecosystem: "docker" + directory: "/" + schedule: + interval: "weekly" + day: "monday" + open-pull-requests-limit: 5 + groups: + docker-base-images: + applies-to: version-updates + patterns: + - "*" + + - package-ecosystem: "pip" + directory: "/src/pypgstac" + schedule: + interval: "weekly" + day: "monday" + open-pull-requests-limit: 5 + groups: + python-dev-tooling: + applies-to: version-updates + patterns: + - "ruff" + - "ty" + - "pre-commit" + - "types-*" + python-runtime: + applies-to: version-updates + patterns: + - "cachetools" + - "fire" + - "hydraters" + - "orjson" + - "plpygis" + - "pydantic" + - "python-dateutil" + - "smart-open" + - "tenacity" + - "version-parser" + - "psycopg*" diff --git a/.github/instructions/scripts.instructions.md b/.github/instructions/scripts.instructions.md index 1382f83d..6cec1b70 100644 --- a/.github/instructions/scripts.instructions.md +++ b/.github/instructions/scripts.instructions.md @@ -7,5 +7,6 @@ applyTo: "scripts/**" See CLAUDE.md "Development Workflow" for usage. All scripts require the Docker compose environment. - `runinpypgstac` is the foundation — most scripts delegate to it +- `scripts/container-scripts/` contains the in-container script payload copied into the pypgstac image; keep host wrappers in `scripts/` - `stageversion` modifies version files AND generates migrations — see CLAUDE.md "Migration Process" - DO NOT run `stageversion` without understanding its side effects diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 4ac4f3b7..d30090cf 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -5,6 +5,9 @@ on: branches: - main pull_request: + workflow_dispatch: + schedule: + - cron: '23 4 * * 0' env: REGISTRY: ghcr.io @@ -18,14 +21,14 @@ jobs: permissions: pull-requests: read outputs: - pgdocker: ${{ steps.check.outputs.pgtag }} + pgtagprefix: ${{ steps.check.outputs.pgtagprefix }} buildpgdocker: ${{ steps.check.outputs.buildpg }} pyrustdocker: ${{ steps.check.outputs.pytag }} buildpyrustdocker: ${{ steps.check.outputs.buildpy }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2 + - uses: dorny/paths-filter@6852f92c20ea7fd3b0c25de3b5112db3a98da050 # v3 id: filter with: filters: | @@ -37,12 +40,22 @@ jobs: run: | buildpg=false; ref=$(echo ${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}} | tr / _); - [[ "${{ steps.filter.outputs.pgstac }}" == "true" ]] && buildpg=true || ref=main; - echo "pgtag=${{ env.REGISTRY }}/${GITHUB_REPOSITORY_OWNER}/pgstac-postgres:$ref" >>$GITHUB_OUTPUT; + pgref=$ref; + [[ "${{ steps.filter.outputs.pgstac }}" == "true" ]] && buildpg=true || pgref=main; + if [[ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then + buildpg=true; + pgref=main; + fi + echo "pgtagprefix=${{ env.REGISTRY }}/${GITHUB_REPOSITORY_OWNER}/pgstac-postgres:$pgref" >>$GITHUB_OUTPUT; echo "buildpg=$buildpg" >>$GITHUB_OUTPUT; buildpy=false; - [[ "${{ steps.filter.outputs.pypgstac }}" == "true" ]] && buildpy=true || ref=main; - echo "pytag=${{ env.REGISTRY }}/${GITHUB_REPOSITORY_OWNER}/pgstac-pyrust:$ref" >>$GITHUB_OUTPUT; + pyref=$ref; + [[ "${{ steps.filter.outputs.pypgstac }}" == "true" ]] && buildpy=true || pyref=main; + if [[ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then + buildpy=true; + pyref=main; + fi + echo "pytag=${{ env.REGISTRY }}/${GITHUB_REPOSITORY_OWNER}/pgstac-pyrust:$pyref" >>$GITHUB_OUTPUT; echo "buildpy=$buildpy" >>$GITHUB_OUTPUT; # This builds a base postgres image that has everything installed to be able to run pgstac. This image does not have pgstac itself installed. @@ -50,6 +63,9 @@ jobs: name: Build and push base postgres image runs-on: ubuntu-latest needs: [changes] + strategy: + matrix: + pg_major: [16, 17, 18] steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 @@ -60,14 +76,16 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and Push Base Postgres - if: ${{ needs.changes.outputs.buildpgdocker == 'true' }} - uses: docker/build-push-action@0a97817b6ade9f46837855d676c4cca3a2471fc9 # v4 + if: ${{ needs.changes.outputs.buildpgdocker == 'true' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 with: platforms: linux/amd64,linux/arm64 context: . target: pgstacbase file: docker/pgstac/Dockerfile - tags: ${{ needs.changes.outputs.pgdocker }} + build-args: | + PG_MAJOR=${{ matrix.pg_major }} + tags: ${{ needs.changes.outputs.pgtagprefix }}-pg${{ matrix.pg_major }} push: true cache-from: type=gha cache-to: type=gha, mode=max @@ -86,8 +104,8 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and Push Base pyrust - if: ${{ needs.changes.outputs.buildpyrustdocker == 'true' }} - uses: docker/build-push-action@0a97817b6ade9f46837855d676c4cca3a2471fc9 # v4 + if: ${{ needs.changes.outputs.buildpyrustdocker == 'true' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 with: platforms: linux/amd64,linux/arm64 context: . @@ -104,6 +122,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: + pg_major: [16, 17, 18] flags: - "" - "--resolution lowest-direct" @@ -115,11 +134,12 @@ jobs: PGHOST: postgres PGDATABASE: postgres PGUSER: postgres + UV_CACHE_DIR: /tmp/.uv-cache services: postgres: env: POSTGRES_PASSWORD: postgres - image: ${{ needs.changes.outputs.pgdocker }} + image: ${{ needs.changes.outputs.pgtagprefix }}-pg${{ matrix.pg_major }} options: >- --health-cmd pg_isready --health-interval 10s @@ -127,18 +147,25 @@ jobs: --health-retries 5 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Ensure PostgreSQL 17 client tools + - name: Ensure PostgreSQL client tools run: | apt-get update - apt-get install -y --no-install-recommends postgresql-client-17 + pg_client_pkg="postgresql-client-${{ matrix.pg_major }}" + if apt-cache show "$pg_client_pkg" >/dev/null 2>&1; then + apt-get install -y --no-install-recommends "$pg_client_pkg" + else + apt-get install -y --no-install-recommends postgresql-client + fi - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 - name: Install pypgstac working-directory: /__w/pgstac/pgstac/src/pypgstac - run: uv pip install ${{ matrix.flags }} .[dev,test,psycopg] - env: - UV_SYSTEM_PYTHON: 1 - - name: Run tests - working-directory: /__w/pgstac/pgstac/docker/pypgstac/bin run: | - export PATH=/opt/docker/pypgstac/bin:$PATH - ./test + export UV_CACHE_DIR=/tmp/.uv-cache + export XDG_CACHE_HOME=/tmp/.cache + mkdir -p "$UV_CACHE_DIR" "$XDG_CACHE_HOME" + uv --cache-dir "$UV_CACHE_DIR" venv /tmp/ci-venv + uv --cache-dir "$UV_CACHE_DIR" pip install --python /tmp/ci-venv/bin/python ${{ matrix.flags }} .[dev,test,psycopg] + echo "/tmp/ci-venv/bin" >> "$GITHUB_PATH" + - name: Run tests + working-directory: /__w/pgstac/pgstac + run: scripts/container-scripts/test diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 73c4b902..6ba18921 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -42,11 +42,11 @@ jobs: - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # 98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-postgres - name: Build and Push Base Postgres - uses: docker/build-push-action@0a97817b6ade9f46837855d676c4cca3a2471fc9 # v4 + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 with: platforms: linux/amd64,linux/arm64 context: . @@ -79,11 +79,11 @@ jobs: - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # 98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - name: Build and Push Base Postgres - uses: docker/build-push-action@0a97817b6ade9f46837855d676c4cca3a2471fc9 # v4 + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 with: platforms: linux/amd64,linux/arm64 context: . @@ -115,11 +115,11 @@ jobs: - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # 98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-pyrust - name: Build and Push Base Postgres - uses: docker/build-push-action@0a97817b6ade9f46837855d676c4cca3a2471fc9 # v4 + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 with: platforms: linux/amd64,linux/arm64 context: . @@ -151,11 +151,11 @@ jobs: - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # 98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-pypgstac - name: Build and Push Base Postgres - uses: docker/build-push-action@0a97817b6ade9f46837855d676c4cca3a2471fc9 # v4 + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 with: platforms: linux/amd64,linux/arm64 context: . @@ -167,6 +167,40 @@ jobs: cache-from: type=gha cache-to: type=gha, mode=max + buildpypgstacruntime: + name: Build and push pypgstac runtime image + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3 + with: + image: tonistiigi/binfmt:qemu-v7.0.0-28 + - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + - name: Log in to the Container registry + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-pypgstac-runtime + - name: Build and Push pypgstac runtime + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + with: + platforms: linux/amd64,linux/arm64 + context: . + target: pypgstac-runtime + file: docker/pypgstac/Dockerfile + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + push: true + cache-from: type=gha + cache-to: type=gha, mode=max + releasetopypi: name: Release runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 32565592..c57bfb98 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,7 @@ src/pypgstac/python/pypgstac/*.so .vscode .ipynb_checkpoints .venv -.pytest_cache \ No newline at end of file +.pytest_cache +.plans/ +.env +src/pgstacrust/target/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 89ad3173..d6c1df84 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v5.0.0 hooks: - id: trailing-whitespace - id: check-yaml @@ -17,17 +17,6 @@ repos: - id: check-executables-have-shebangs - id: check-symlinks - -- repo: https://github.com/astral-sh/ruff-pre-commit - rev: 'v0.8.2' - hooks: - - id: ruff - files: src/pypgstac\/.*\.py$ - args: [--fix, --exit-non-zero-on-fix] - - id: ruff-format - files: src/pypgstac\/.*\.py$ - - - repo: local hooks: - id: dockerbuild @@ -55,7 +44,7 @@ repos: pass_filenames: false verbose: true fail_fast: true - always_run: true + files: ^(src/pypgstac/(src/pypgstac|tests)/.*\.py|src/pypgstac/pyproject\.toml)$ - id: pypgstac name: pypgstac entry: scripts/test diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c440b79..953a4d2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,82 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [Unreleased] + +### Added +- `scripts/makemigration` host wrapper for the in-container `makemigration` helper. +- `.env.example` documenting all supported environment variables for local development. +- All host-facing scripts (`test`, `format`, `migrate`, `server`, `stageversion`, + `runinpypgstac`, `console`) now accept `--help` / `-h` and honor environment-variable + counterparts for common flags (`PGSTAC_BUILD_POLICY`, `PGSTAC_FAST`, `PGSTAC_WATCH`, + `PGSTAC_STRICT`). +- `scripts/pgstacenv` gains `ensure_env_file` (auto-creates `.env` from `.env.example` + on first run) and `first_available_pgport` (avoids port collisions on shared machines). +- `scripts/test` expanded from a 3-line wrapper to a full-featured test runner supporting + `--fast`, `--watch`, `--build-policy`, `--no-strict`, and stale-image detection. +- PostgreSQL 16, 17, and 18-beta added to the CI and Docker build matrix. (Closes #334) +- Weekly scheduled CI run (`cron: '23 4 * * 0'`) to catch upstream base-image CVEs without + requiring a code change. (Closes #202) +- `workflow_dispatch` trigger for manual CI runs. +- `pg_tle` v1.5.2 built and pre-loaded in the `pgstacbase` image; database init runs + `CREATE EXTENSION IF NOT EXISTS pg_tle`. +- `pypgstac-runtime` Docker target: slim Python 3.13-trixie image without the Rust/build + toolchain, for production deployments where the Rust build environment is not needed. +- Dependabot coverage expanded to Docker base images and pip packages (two new + ecosystems with grouped update policies). + +### Changed +- In-container helper scripts moved from `docker/pypgstac/bin/` to + `scripts/container-scripts/`; container `PATH` updated accordingly. +- `docker/pgstac/Dockerfile` and `docker/pypgstac/Dockerfile` base images updated from + `bullseye` to `trixie`. (Closes #231) +- All Docker `RUN` layers now use BuildKit cache mounts for apt, uv, and git caches, + significantly reducing incremental rebuild times. +- `docker-compose.yml`: adds `env_file: .env`, explicit `PGHOST`/`PGPORT` defaults, + a pgstac healthcheck, and a `service_healthy` dependency on pypgstac. +- `runinpypgstac` gains `--build-policy {always,missing,never}` replacing the bare + `--build` flag; `PGSTAC_BUILD_POLICY` env var provides a persistent default. +- Dev tooling: `flake8`, `black`, and `mypy` removed in favour of `ruff==0.15.11` and + `ty==0.0.31`. `pre-commit` pinned to `3.5.0`. `pre-commit-hooks` updated to v5.0.0. +- `pypgstac` package floor raised to Python 3.11; metadata now advertises 3.11-3.14. +- `pypgstac` settings now use `pydantic-settings` (`BaseSettings` from + `pydantic_settings`) and require `pydantic>=2,<3`. +- `cachetools` upper bound removed (`cachetools>=5.3.0`) since `pypgstac` only uses + `cachetools.func.lru_cache`; no known incompatible API changes affect this usage. +- `pypgstac` developer tooling config now consistently targets Ruff + ty: + removes stale mypy config, pins Ruff to `0.15.11` to match pre-commit, + and adds minimal `[tool.ty]` project settings. +- Formatting/type-check pipeline now uses `scripts/test --formatting` as the + single pre-commit entry point (removing duplicate direct Ruff pre-commit hooks) + and aligns Ruff line-length handling with the formatter (`E501` ignored; + explicit `line-length = 88`). +- GitHub Actions updated: `dorny/paths-filter` v2→v3, `docker/build-push-action` + v4→v6, `astral-sh/setup-uv` v8.0.0→v8.1.0; all SHA pins refreshed. +- Dependabot groups reworked: `actions-all` (replaces `minor-and-patch`), new + `docker-base-images`, `python-dev-tooling`, and `python-runtime` groups. +- `docker-compose.yml` removes explicit `container_name` entries to avoid conflicts + between concurrent local instances. + +### Removed +- PL/Rust support: `pgstacbase-plrust` and `pgstac-plrust` Docker targets removed; the + pgstac image no longer builds or ships PL/Rust or the Rust toolchain. (Closes #339) +- `flake8`, `black`, and `mypy` removed from dev dependencies. + +### Fixed +- `load.py`: Use timezone-aware `MIN_DATETIME_UTC` / `MAX_DATETIME_UTC` sentinel + constants (instead of naive `datetime.min` / `datetime.max`) to avoid + `TypeError: can't compare offset-naive and offset-aware datetimes`. +- CI `lowest-direct` dependency install: avoid uv cache permission failures by using + a writable temp cache path in the test job install step. +- `pypgstac` dependency floor for `orjson` raised to `>=3.11.0` to avoid selecting + the broken `3.9.0` sdist under `--resolution lowest-direct`. +- `pydantic` minimum raised to `>=2.10` so `--resolution lowest-direct` on Python 3.13 + does not resolve to `pydantic-core==2.0.1`, which fails to build. + + ## [v0.9.11] + ### Fixed - Fix timestamp regex in partition constraint parsing to handle fractional seconds (microseconds), preventing incorrect `(-infinity, infinity)` constraint bounds. - Add explicit ANALYZE before `st_estimatedextent()` in `update_partition_stats` for deterministic spatial extent calculation. @@ -616,6 +690,7 @@ _TODO_ - Fixed issue with pypgstac loads which caused some writes to fail ([#18](https://github.com/stac-utils/pgstac/pull/18)) +[Unreleased]: https://github.com/stac-utils/pgstac/compare/v0.9.11...HEAD [v0.9.11]: https://github.com/stac-utils/pgstac/compare/v0.9.10...v0.9.11 [v0.9.10]: https://github.com/stac-utils/pgstac/compare/v0.9.9...v0.9.10 [v0.9.9]: https://github.com/stac-utils/pgstac/compare/v0.9.8...v0.9.9 diff --git a/CLAUDE.md b/CLAUDE.md index f4b6274b..4849c58a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -17,7 +17,8 @@ src/pgstac/migrations/ ← Base + incremental migration files src/pgstac/tests/ ← PGTap and basic SQL tests src/pypgstac/src/pypgstac/ ← Python package source src/pypgstac/tests/ ← pytest tests -docker/pypgstac/bin/ ← Build/test/utility scripts (pgstac_restore, test, etc.) +scripts/ ← Host-facing entrypoint scripts +scripts/container-scripts/ ← Scripts copied into the pypgstac container image ``` ### Documentation Files @@ -64,7 +65,7 @@ PgSTAC functions reference PostGIS functions (e.g. `st_makeenvelope`, `st_geomfr - **Do NOT schema-qualify PostGIS function calls** in PgSTAC SQL - **Avoid cross-function dependencies in SQL functions used by GENERATED columns** — pg_dump orders functions alphabetically, so `func_a` calling `func_b` may be created before `func_b` exists. Inline the logic instead. -- Use `pgstac_restore` (in `docker/pypgstac/bin/`) to restore dumps — it installs a temporary event trigger that sets the correct `search_path` before each DDL command +- Use `pgstac_restore` (via `scripts/container-scripts/pgstac_restore` in the image) to restore dumps — it installs a temporary event trigger that sets the correct `search_path` before each DDL command - Test with `scripts/test --pgdump` ## Development Workflow @@ -84,7 +85,7 @@ scripts/test --pypgstac # pytest only scripts/test --pgtap # PGTap SQL tests scripts/test --basicsql # SQL output comparison tests scripts/test --migrations # Full migration chain test -scripts/test --formatting # ruff + mypy +scripts/test --formatting # ruff + ty scripts/test --pgdump # pg_dump/pg_restore round-trip test ``` @@ -114,7 +115,7 @@ This runs inside Docker and: ### How makemigration Works -`makemigration` (in `docker/pypgstac/bin/makemigration`) generates incremental migrations by diffing schemas: +`makemigration` (copied from `scripts/container-scripts/makemigration` into the image) generates incremental migrations by diffing schemas: 1. Creates two temp databases: `migra_from`, `migra_to` 2. Loads old base migration into `migra_from` diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 327d6305..1aac96df 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,10 +1,10 @@ # Development - Contributing -PgSTAC uses a dockerized development environment. However, -it still needs a local install of pypgstac to allow an editable -install inside the docker container. This is installed automatically -if you have set up a virtual environment for the project. Otherwise -you'll need to install a local copy yourself by running `scripts/install`. +PgSTAC uses a dockerized development environment. + +Local-only planning notes now live under `.plans/` and are intentionally gitignored. +If you keep local execution settings, start from `.env.example` and write overrides to +`.env`. To build the docker images and set up the test database, use: @@ -22,11 +22,34 @@ To run tests, use: scripts/test ``` +To set up pre-commit using the project uv workflow: + +```bash +uv tool install pre-commit==3.5.0 +pre-commit install +``` + +Useful options: + +```bash +scripts/test --fast +scripts/test --pypgstac +scripts/test --build-policy always +``` + +`scripts/test` defaults to `PGSTAC_BUILD_POLICY=always` so the container image reflects +your current checkout. If you intentionally want to reuse an existing image, set +`PGSTAC_BUILD_POLICY=missing` or pass `--build-policy missing`. + To rebuild docker images: ```bash scripts/update ``` +Container-only helper scripts now live in `scripts/container-scripts/` and are copied +into the `pypgstac` image during build. Top-level `scripts/` remain the host-facing +entrypoint surface. + To drop into a console, use ```bash scripts/console @@ -47,6 +70,12 @@ To stage code and configurations and create template migrations for a version re scripts/stageversion [version] ``` +To generate only the incremental migration, use: + +```bash +scripts/makemigration --from 0.9.10 --to 0.9.11 +``` + Examples: ``` scripts/stageversion 0.2.8 @@ -71,7 +100,7 @@ PyPgSTAC tests are pytest tests, and they are located in `/src/pypgstac/tests` All tests can be found in tests/pgtap.sql and are run using `scripts/test`. -Individual tests can be run with any combination of the following flags `--formatting --basicsql --pgtap --migrations --pypgstac`. If pre-commit is installed, tests will be run on commit based on which files have changed. +Individual tests can be run with any combination of the following flags `--formatting --basicsql --pgtap --migrations --pypgstac`. The `--formatting` suite runs Ruff lint/format checks and Ty type checks. If pre-commit is installed, tests will be run on commit based on which files have changed. ### To make a PR @@ -108,4 +137,4 @@ Rehydration is the process of adding the stripped attributes back to the STAC it PgSTAC, a versatile tool, is designed to seamlessly integrate with PyPgSTAC or alternative backends. This flexibility allows for direct calls for both rehydration and dehydration, giving developers and technical users a sense of control over the process. -Hydration and dehydration are de-facto settings that users can not opt out of. In the future, we may provide a configuration for use cases where the size benefits do not justify the added complexity. \ No newline at end of file +Hydration and dehydration are de-facto settings that users can not opt out of. In the future, we may provide a configuration for use cases where the size benefits do not justify the added complexity. diff --git a/docker-compose.yml b/docker-compose.yml index 47ad9327..d7fe59eb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,8 @@ services: pgstac: - container_name: pgstac image: pgstac + env_file: + - .env build: context: . network: host @@ -9,20 +10,29 @@ services: target: pgstac platform: linux/amd64 environment: - - POSTGRES_USER=username - - POSTGRES_PASSWORD=password - - POSTGRES_DB=postgis - - PGUSER=username - - PGPASSWORD=password - - PGDATABASE=postgis + - POSTGRES_USER=${POSTGRES_USER:-username} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-password} + - POSTGRES_DB=${POSTGRES_DB:-postgis} + - PGHOST=/var/run/postgresql + - PGPORT=5432 + - PGUSER=${PGUSER:-username} + - PGPASSWORD=${PGPASSWORD:-password} + - PGDATABASE=${PGDATABASE:-postgis} ports: - - "5439:5432" + - "${PGPORT:-5439}:5432" volumes: - pgstac-pgdata:/var/lib/postgresql/data command: postgres + healthcheck: + test: ["CMD-SHELL", "pg_isready -h /var/run/postgresql -p 5432 -U ${PGUSER:-username} -d ${PGDATABASE:-postgis}"] + interval: 5s + timeout: 5s + retries: 12 + start_period: 5s pypgstac: - container_name: pypgstac image: pypgstac + env_file: + - .env build: context: . network: host @@ -31,10 +41,12 @@ services: platform: linux/amd64 environment: - PGHOST=pgstac - - PGUSER=username - - PGPASSWORD=password - - PGDATABASE=postgis + - PGPORT=5432 + - PGUSER=${PGUSER:-username} + - PGPASSWORD=${PGPASSWORD:-password} + - PGDATABASE=${PGDATABASE:-postgis} depends_on: - - pgstac + pgstac: + condition: service_healthy volumes: pgstac-pgdata: diff --git a/docker/pgstac/Dockerfile b/docker/pgstac/Dockerfile index 14114be1..671ac025 100644 --- a/docker/pgstac/Dockerfile +++ b/docker/pgstac/Dockerfile @@ -1,76 +1,51 @@ +# syntax=docker/dockerfile:1.7 ARG PG_MAJOR=17 ARG POSTGIS_MAJOR=3 +ARG PG_TLE_VERSION=1.5.2 +ARG DEBIAN_SUITE=trixie # Base postgres image that pgstac can be installed onto -FROM postgres:${PG_MAJOR}-bullseye AS pgstacbase +FROM postgres:${PG_MAJOR}-${DEBIAN_SUITE} AS pgstacbase ARG POSTGIS_MAJOR ARG PG_MAJOR -RUN \ +ARG PG_TLE_VERSION +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \ + --mount=type=cache,target=/root/.cache/git,sharing=locked \ apt-get update \ - && apt-get upgrade -y \ && apt-get install -y --no-install-recommends \ postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR \ postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR-scripts \ postgresql-$PG_MAJOR-pgtap \ postgresql-$PG_MAJOR-plpgsql-check \ postgresql-$PG_MAJOR-partman \ - && apt-get remove -y apt-transport-https \ - && apt-get clean && apt-get -y autoremove \ - && rm -rf /var/lib/apt/lists/* -COPY docker/pgstac/dbinit/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh - -# Base postgres image with plrust installed that can be used for future development using plrust -FROM pgstacbase AS pgstacbase-plrust -ENV PLRUSTVERSION=1.2.7 -ENV RUSTVERSION=1.72.0 -ENV PLRUSTDOWNLOADURL=https://github.com/tcdi/plrust/releases/download/ -ENV PLRUSTFILE=plrust-trusted-${PLRUSTVERSION}_${RUSTVERSION}-debian-pg${PG_MAJOR}-amd64.deb -ENV PLRUSTURL=${PLRUSTDOWNLOADURL}v${PLRUSTVERSION}/${PLRUSTFILE} -ADD $PLRUSTURL . -ENV PATH=/home/postgres/.cargo/bin:$PATH -RUN \ - apt-get update \ - && apt-get upgrade -y \ - && apt-get install -y --no-install-recommends \ postgresql-server-dev-$PG_MAJOR \ build-essential \ ca-certificates \ - clang \ - clang-11 \ - gcc \ + curl \ + git \ + flex \ + bison \ + libkrb5-dev \ + && GIT_TERMINAL_PROMPT=0 git clone --branch v${PG_TLE_VERSION} --depth 1 https://github.com/aws/pg_tle.git /tmp/pg_tle \ + && make -C /tmp/pg_tle \ + && make -C /tmp/pg_tle install \ + && rm -rf /tmp/pg_tle \ + && sed -i "s/^#shared_preload_libraries = .*/shared_preload_libraries = 'pg_tle'/" /usr/share/postgresql/$PG_MAJOR/postgresql.conf.sample \ + && sed -i "s/^#shared_preload_libraries = .*/shared_preload_libraries = 'pg_tle'/" /usr/share/postgresql/postgresql.conf.sample \ + && apt-get purge -y --auto-remove \ + postgresql-server-dev-$PG_MAJOR \ + build-essential \ + curl \ git \ - gnupg \ - libssl-dev \ - llvm-11 \ - lsb-release \ - make \ - pkg-config \ - wget \ - && apt-get remove -y apt-transport-https \ + flex \ + bison \ + libkrb5-dev \ && apt-get clean && apt-get -y autoremove \ && rm -rf /var/lib/apt/lists/* -USER postgres -RUN \ - wget -qO- https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain=${RUSTVERSION} \ - && $HOME/.cargo/bin/rustup toolchain install ${RUSTVERSION} \ - && $HOME/.cargo/bin/rustup default ${RUSTVERSION} \ - && $HOME/.cargo/bin/rustup component add rustc-dev -WORKDIR /docker-entrypoint-preinitdb.d -COPY docker/pgstac/dbinit/pgstac-rust-preinit.sh preloadplrust.sh -WORKDIR /docker-entrypoint-initdb.d -COPY docker/pgstac/dbinit/pgstac-rust.sh 991_plrust.sh - -USER root -RUN apt-get install -y /${PLRUSTFILE} # The pgstacbase image with latest version of pgstac installed FROM pgstacbase AS pgstac WORKDIR /docker-entrypoint-initdb.d COPY docker/pgstac/dbinit/pgstac.sh 990_pgstac.sh COPY src/pgstac/pgstac.sql 999_pgstac.sql - -# The pgstacbase-plrust image with the latest version of pgstac installed -FROM pgstacbase-plrust AS pgstac-plrust -WORKDIR /docker-entrypoint-initdb.d -COPY docker/pgstac/dbinit/pgstac.sh 990_pgstac.sh -COPY src/pgstac/pgstac.sql 999_pgstac.sql diff --git a/docker/pgstac/dbinit/docker-entrypoint.sh b/docker/pgstac/dbinit/docker-entrypoint.sh deleted file mode 100755 index b6c55481..00000000 --- a/docker/pgstac/dbinit/docker-entrypoint.sh +++ /dev/null @@ -1,354 +0,0 @@ -#!/usr/bin/env bash -set -Eeo pipefail -# TODO swap to -Eeuo pipefail above (after handling all potentially-unset variables) - -# usage: file_env VAR [DEFAULT] -# ie: file_env 'XYZ_DB_PASSWORD' 'example' -# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of -# "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature) -file_env() { - local var="$1" - local fileVar="${var}_FILE" - local def="${2:-}" - if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then - printf >&2 'error: both %s and %s are set (but are exclusive)\n' "$var" "$fileVar" - exit 1 - fi - local val="$def" - if [ "${!var:-}" ]; then - val="${!var}" - elif [ "${!fileVar:-}" ]; then - val="$(< "${!fileVar}")" - fi - export "$var"="$val" - unset "$fileVar" -} - -# check to see if this file is being run or sourced from another script -_is_sourced() { - # https://unix.stackexchange.com/a/215279 - [ "${#FUNCNAME[@]}" -ge 2 ] \ - && [ "${FUNCNAME[0]}" = '_is_sourced' ] \ - && [ "${FUNCNAME[1]}" = 'source' ] -} - -# used to create initial postgres directories and if run as root, ensure ownership to the "postgres" user -docker_create_db_directories() { - local user; user="$(id -u)" - - mkdir -p "$PGDATA" - # ignore failure since there are cases where we can't chmod (and PostgreSQL might fail later anyhow - it's picky about permissions of this directory) - chmod 00700 "$PGDATA" || : - - # ignore failure since it will be fine when using the image provided directory; see also https://github.com/docker-library/postgres/pull/289 - mkdir -p /var/run/postgresql || : - chmod 03775 /var/run/postgresql || : - - # Create the transaction log directory before initdb is run so the directory is owned by the correct user - if [ -n "${POSTGRES_INITDB_WALDIR:-}" ]; then - mkdir -p "$POSTGRES_INITDB_WALDIR" - if [ "$user" = '0' ]; then - find "$POSTGRES_INITDB_WALDIR" \! -user postgres -exec chown postgres '{}' + - fi - chmod 700 "$POSTGRES_INITDB_WALDIR" - fi - - # allow the container to be started with `--user` - if [ "$user" = '0' ]; then - find "$PGDATA" \! -user postgres -exec chown postgres '{}' + - find /var/run/postgresql \! -user postgres -exec chown postgres '{}' + - fi -} - -# initialize empty PGDATA directory with new database via 'initdb' -# arguments to `initdb` can be passed via POSTGRES_INITDB_ARGS or as arguments to this function -# `initdb` automatically creates the "postgres", "template0", and "template1" dbnames -# this is also where the database user is created, specified by `POSTGRES_USER` env -docker_init_database_dir() { - # "initdb" is particular about the current user existing in "/etc/passwd", so we use "nss_wrapper" to fake that if necessary - # see https://github.com/docker-library/postgres/pull/253, https://github.com/docker-library/postgres/issues/359, https://cwrap.org/nss_wrapper.html - local uid; uid="$(id -u)" - if ! getent passwd "$uid" &> /dev/null; then - # see if we can find a suitable "libnss_wrapper.so" (https://salsa.debian.org/sssd-team/nss-wrapper/-/commit/b9925a653a54e24d09d9b498a2d913729f7abb15) - local wrapper - for wrapper in {/usr,}/lib{/*,}/libnss_wrapper.so; do - if [ -s "$wrapper" ]; then - NSS_WRAPPER_PASSWD="$(mktemp)" - NSS_WRAPPER_GROUP="$(mktemp)" - export LD_PRELOAD="$wrapper" NSS_WRAPPER_PASSWD NSS_WRAPPER_GROUP - local gid; gid="$(id -g)" - printf 'postgres:x:%s:%s:PostgreSQL:%s:/bin/false\n' "$uid" "$gid" "$PGDATA" > "$NSS_WRAPPER_PASSWD" - printf 'postgres:x:%s:\n' "$gid" > "$NSS_WRAPPER_GROUP" - break - fi - done - fi - - if [ -n "${POSTGRES_INITDB_WALDIR:-}" ]; then - set -- --waldir "$POSTGRES_INITDB_WALDIR" "$@" - fi - - # --pwfile refuses to handle a properly-empty file (hence the "\n"): https://github.com/docker-library/postgres/issues/1025 - eval 'initdb --username="$POSTGRES_USER" --pwfile=<(printf "%s\n" "$POSTGRES_PASSWORD") '"$POSTGRES_INITDB_ARGS"' "$@"' - - # unset/cleanup "nss_wrapper" bits - if [[ "${LD_PRELOAD:-}" == */libnss_wrapper.so ]]; then - rm -f "$NSS_WRAPPER_PASSWD" "$NSS_WRAPPER_GROUP" - unset LD_PRELOAD NSS_WRAPPER_PASSWD NSS_WRAPPER_GROUP - fi -} - -# print large warning if POSTGRES_PASSWORD is long -# error if both POSTGRES_PASSWORD is empty and POSTGRES_HOST_AUTH_METHOD is not 'trust' -# print large warning if POSTGRES_HOST_AUTH_METHOD is set to 'trust' -# assumes database is not set up, ie: [ -z "$DATABASE_ALREADY_EXISTS" ] -docker_verify_minimum_env() { - # check password first so we can output the warning before postgres - # messes it up - if [ "${#POSTGRES_PASSWORD}" -ge 100 ]; then - cat >&2 <<-'EOWARN' - - WARNING: The supplied POSTGRES_PASSWORD is 100+ characters. - - This will not work if used via PGPASSWORD with "psql". - - https://www.postgresql.org/message-id/flat/E1Rqxp2-0004Qt-PL%40wrigleys.postgresql.org (BUG #6412) - https://github.com/docker-library/postgres/issues/507 - - EOWARN - fi - if [ -z "$POSTGRES_PASSWORD" ] && [ 'trust' != "$POSTGRES_HOST_AUTH_METHOD" ]; then - # The - option suppresses leading tabs but *not* spaces. :) - cat >&2 <<-'EOE' - Error: Database is uninitialized and superuser password is not specified. - You must specify POSTGRES_PASSWORD to a non-empty value for the - superuser. For example, "-e POSTGRES_PASSWORD=password" on "docker run". - - You may also use "POSTGRES_HOST_AUTH_METHOD=trust" to allow all - connections without a password. This is *not* recommended. - - See PostgreSQL documentation about "trust": - https://www.postgresql.org/docs/current/auth-trust.html - EOE - exit 1 - fi - if [ 'trust' = "$POSTGRES_HOST_AUTH_METHOD" ]; then - cat >&2 <<-'EOWARN' - ******************************************************************************** - WARNING: POSTGRES_HOST_AUTH_METHOD has been set to "trust". This will allow - anyone with access to the Postgres port to access your database without - a password, even if POSTGRES_PASSWORD is set. See PostgreSQL - documentation about "trust": - https://www.postgresql.org/docs/current/auth-trust.html - In Docker's default configuration, this is effectively any other - container on the same system. - - It is not recommended to use POSTGRES_HOST_AUTH_METHOD=trust. Replace - it with "-e POSTGRES_PASSWORD=password" instead to set a password in - "docker run". - ******************************************************************************** - EOWARN - fi -} - -# usage: docker_process_init_files [file [file [...]]] -# ie: docker_process_init_files /always-initdb.d/* -# process initializer files, based on file extensions and permissions -docker_process_init_files() { - # psql here for backwards compatibility "${psql[@]}" - psql=( docker_process_sql ) - - printf '\n' - local f - for f; do - case "$f" in - *.sh) - # https://github.com/docker-library/postgres/issues/450#issuecomment-393167936 - # https://github.com/docker-library/postgres/pull/452 - if [ -x "$f" ]; then - printf '%s: running %s\n' "$0" "$f" - "$f" - else - printf '%s: sourcing %s\n' "$0" "$f" - . "$f" - fi - ;; - *.sql) printf '%s: running %s\n' "$0" "$f"; docker_process_sql -f "$f"; printf '\n' ;; - *.sql.gz) printf '%s: running %s\n' "$0" "$f"; gunzip -c "$f" | docker_process_sql; printf '\n' ;; - *.sql.xz) printf '%s: running %s\n' "$0" "$f"; xzcat "$f" | docker_process_sql; printf '\n' ;; - *.sql.zst) printf '%s: running %s\n' "$0" "$f"; zstd -dc "$f" | docker_process_sql; printf '\n' ;; - *) printf '%s: ignoring %s\n' "$0" "$f" ;; - esac - printf '\n' - done -} - -# Execute sql script, passed via stdin (or -f flag of pqsl) -# usage: docker_process_sql [psql-cli-args] -# ie: docker_process_sql --dbname=mydb <<<'INSERT ...' -# ie: docker_process_sql -f my-file.sql -# ie: docker_process_sql > "$PGDATA/pg_hba.conf" -} - -# start socket-only postgresql server for setting up or running scripts -# all arguments will be passed along as arguments to `postgres` (via pg_ctl) -docker_temp_server_start() { - if [ "$1" = 'postgres' ]; then - shift - fi - - # internal start of server in order to allow setup using psql client - # does not listen on external TCP/IP and waits until start finishes - set -- "$@" -c listen_addresses='' -p "${PGPORT:-5432}" - - PGUSER="${PGUSER:-$POSTGRES_USER}" \ - pg_ctl -D "$PGDATA" \ - -o "$(printf '%q ' "$@")" \ - -w start -} - -# stop postgresql server after done setting up user and running scripts -docker_temp_server_stop() { - PGUSER="${PGUSER:-postgres}" \ - pg_ctl -D "$PGDATA" -m fast -w stop -} - -# check arguments for an option that would cause postgres to stop -# return true if there is one -_pg_want_help() { - local arg - for arg; do - case "$arg" in - # postgres --help | grep 'then exit' - # leaving out -C on purpose since it always fails and is unhelpful: - # postgres: could not access the server configuration file "/var/lib/postgresql/data/postgresql.conf": No such file or directory - -'?'|--help|--describe-config|-V|--version) - return 0 - ;; - esac - done - return 1 -} - -_main() { - # if first arg looks like a flag, assume we want to run postgres server - if [ "${1:0:1}" = '-' ]; then - set -- postgres "$@" - fi - - if [ "$1" = 'postgres' ] && ! _pg_want_help "$@"; then - docker_setup_env - # setup data directories and permissions (when run as root) - docker_create_db_directories - if [ "$(id -u)" = '0' ]; then - # then restart script as postgres user - exec gosu postgres "$BASH_SOURCE" "$@" - fi - - # only run initialization on an empty data directory - if [ -z "$DATABASE_ALREADY_EXISTS" ]; then - docker_verify_minimum_env - - # check dir permissions to reduce likelihood of half-initialized database - ls /docker-entrypoint-initdb.d/ > /dev/null - - docker_init_database_dir - pg_setup_hba_conf "$@" - - # PGPASSWORD is required for psql when authentication is required for 'local' connections via pg_hba.conf and is otherwise harmless - # e.g. when '--auth=md5' or '--auth-local=md5' is used in POSTGRES_INITDB_ARGS - export PGPASSWORD="${PGPASSWORD:-$POSTGRES_PASSWORD}" - docker_temp_server_start "$@" - - docker_setup_db - docker_process_init_files /docker-entrypoint-preinitdb.d/* - docker_temp_server_stop - docker_temp_server_start "$@" - docker_process_init_files /docker-entrypoint-initdb.d/* - - docker_temp_server_stop - unset PGPASSWORD - - cat <<-'EOM' - - PostgreSQL init process complete; ready for start up. - - EOM - else - cat <<-'EOM' - - PostgreSQL Database directory appears to contain a database; Skipping initialization - - EOM - fi - fi - - exec "$@" -} - -if ! _is_sourced; then - _main "$@" -fi diff --git a/docker/pgstac/dbinit/pgstac-rust-preinit.sh b/docker/pgstac/dbinit/pgstac-rust-preinit.sh deleted file mode 100755 index 91b734c2..00000000 --- a/docker/pgstac/dbinit/pgstac-rust-preinit.sh +++ /dev/null @@ -1,3 +0,0 @@ -psql -X -q -v ON_ERROR_STOP=1 < /etc/apt/sources.list.d/pgdg.list \ - && apt-get update \ && apt-get install -y --no-install-recommends \ + adduser \ + ca-certificates \ + curl \ postgresql-client-${PG_MAJOR} \ python3 python-is-python3 python3-pip python3-venv \ - build-essential clang clang-11 gcc git libssl-dev llvm-11 make pkg-config \ - && python3 -m pip install --upgrade pip uv wheel setuptools \ - && apt-get remove -y apt-transport-https \ + build-essential clang gcc git libssl-dev llvm make pkg-config \ + && curl -LsSf https://astral.sh/uv/install.sh | UV_INSTALL_DIR=/usr/local/bin sh \ && apt-get clean && apt-get -y autoremove \ && rm -rf /var/lib/apt/lists/* FROM pyrustbase AS pypgstac COPY ./src/pypgstac/pyproject.toml /tmp/pyproject.toml WORKDIR /tmp -RUN \ - uv pip compile --all-extras /tmp/pyproject.toml >/tmp/requirements.txt \ +RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked \ + uv pip compile /tmp/pyproject.toml \ + --extra dev \ + --extra test \ + --extra psycopg \ + --extra migrations \ + >/tmp/requirements.txt \ && uv pip install --system -r /tmp/requirements.txt -COPY docker/pypgstac/bin /opt/docker/pypgstac/bin +COPY scripts/container-scripts /opt/pgstac/container-scripts COPY src/pypgstac /opt/src/pypgstac COPY src/pgstac /opt/src/pgstac WORKDIR /opt/src/pypgstac -RUN uv pip install --system -e . && rm -rf /usr/local/cargo/registry +RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked \ + uv pip install --system -e . \ + && rm -rf /usr/local/cargo/registry RUN addgroup --gid 1000 user && \ adduser --uid 1000 --gid 1000 --disabled-password --gecos "" --home /home/user user && \ chown -R user:user /opt/src/pypgstac /opt/src/pgstac USER user + +# Optional runtime-optimized image: no build toolchain, only pypgstac package + runtime deps. +FROM python:3.13-slim-trixie AS pypgstac-runtime +ENV PYTHONWRITEBYTECODE=1 +ENV PYTHONBUFFERED=1 +ENV UV_BREAK_SYSTEM_PACKAGES=1 +ENV PATH="/opt/pgstac/container-scripts:$PATH" +ENV UV_CACHE_DIR=/root/.cache/uv +ARG PG_MAJOR=17 +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \ + --mount=type=cache,target=/root/.cache/uv,sharing=locked \ + apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + postgresql-client-${PG_MAJOR} \ + && curl -LsSf https://astral.sh/uv/install.sh | UV_INSTALL_DIR=/usr/local/bin sh \ + && apt-get clean && rm -rf /var/lib/apt/lists/* +COPY ./src/pypgstac/pyproject.toml /tmp/pyproject.toml +RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked \ + uv pip compile /tmp/pyproject.toml \ + --extra psycopg \ + --extra migrations \ + >/tmp/requirements.txt \ + && uv pip install --system -r /tmp/requirements.txt +COPY scripts/container-scripts /opt/pgstac/container-scripts +COPY src/pypgstac /opt/src/pypgstac +COPY src/pgstac /opt/src/pgstac +WORKDIR /opt/src/pypgstac +RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked \ + uv pip install --system . diff --git a/docker/pypgstac/bin/tmpdb b/docker/pypgstac/bin/tmpdb deleted file mode 100755 index b4788406..00000000 --- a/docker/pypgstac/bin/tmpdb +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -set -e - -if [[ "${CI}" ]]; then - set -x -fi - -if [ ! $PGSTACDOCKER == 1 ]; then - echo "This script should only be run within pgstac docker"; exit 1; -fi - -function tmppg(){ - -# Make temp directories for pgdata and logs -WORKSPACE=$(mktemp -d) - -# Find an unused port - -# Run tests using postgres user and trust authentication - -export PGDATA=$WORKSPACE/pgdata -export PGLOG=$WORKSPACE/pglog - -export PGDATABASE=postgres -export PGUSER=postgres - -# Leverage scripts from docker-entrypoint.sh -source /usr/local/bin/docker-entrypoint.sh - -gosu postgres initdb --username $PGUSER --auth=trust -D $PGDATA - -# Start postgres with minimal logging settings -gosu postgres pg_ctl -l "$PGLOG" -w -o "-F -c fsync=off -c full_page_writes=off -c synchronous_commit=off -c archive_mode=off" start -trap "gosu postgres pg_ctl stop && rm -fr $WORKSPACE " 0 2 3 15 - -# Get location of script directory -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) - -cd /opt/src/pgstac -# Create template database with pgstac installed -psql -X -q -v ON_ERROR_STOP=1 < /dev/null && pwd ) cd $SCRIPT_DIR/.. +source "$SCRIPT_DIR/pgstacenv" set -e if [[ "${CI}" ]]; then @@ -9,7 +10,7 @@ fi function usage() { echo -n \ - "Usage: $(basename "$0") [--db] + "Usage: $(basename "$0") [--db] [--help] Start a console in the dev container --db: Instead, start a psql console in the database container. @@ -21,14 +22,19 @@ while [[ "$#" > 0 ]]; do case $1 in DB_CONSOLE=1 shift ;; + -h|--help) + usage + exit 0 + ;; *) - usage "Unknown parameter passed: $1" - shift - shift + echo "Unknown parameter passed: $1" >&2 + usage + exit 1 ;; esac; done if [ "${BASH_SOURCE[0]}" = "${0}" ]; then + ensure_env_file docker compose up -d @@ -39,6 +45,6 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then exit 0 fi - docker compose exec pgstac /bin/bash + docker compose exec pypgstac /bin/bash fi diff --git a/docker/pypgstac/bin/format b/scripts/container-scripts/format similarity index 60% rename from docker/pypgstac/bin/format rename to scripts/container-scripts/format index 15053b66..6c22233e 100755 --- a/docker/pypgstac/bin/format +++ b/scripts/container-scripts/format @@ -5,20 +5,20 @@ if [[ "${CI}" ]]; then set -x fi SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -cd $SCRIPT_DIR/../../../src +cd ${PGSTAC_REPO_DIR:-/opt/src} function usage() { echo -n \ "Usage: $(basename "$0") Format code. -This scripts is meant to be run inside the dev container. +This script is meant to be run inside the dev container. " } if [ "${BASH_SOURCE[0]}" = "${0}" ]; then echo "Formatting pypgstac..." - ruff --fix pypgstac/pypgstac - ruff --fix pypgstac/tests + ruff check --fix pypgstac/src/pypgstac pypgstac/tests + ruff format pypgstac/src/pypgstac pypgstac/tests fi diff --git a/docker/pypgstac/bin/initpgstac b/scripts/container-scripts/initpgstac similarity index 79% rename from docker/pypgstac/bin/initpgstac rename to scripts/container-scripts/initpgstac index b205b7cf..880b350d 100755 --- a/docker/pypgstac/bin/initpgstac +++ b/scripts/container-scripts/initpgstac @@ -1,11 +1,11 @@ #!/bin/bash SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -cd $SCRIPT_DIR/../../../src/pgstac +cd ${PGSTAC_PGSTAC_DIR:-/opt/src/pgstac} psql -X -q -v ON_ERROR_STOP=1 < /dev/null && pwd ) -cd $SCRIPT_DIR/../../../src/pgstac +cd ${PGSTAC_PGSTAC_DIR:-/opt/src/pgstac} psql -f pgstac.sql psql -v ON_ERROR_STOP=1 <<-EOSQL \copy collections (content) FROM 'tests/testdata/collections.ndjson' diff --git a/docker/pypgstac/bin/makemigration b/scripts/container-scripts/makemigration similarity index 76% rename from docker/pypgstac/bin/makemigration rename to scripts/container-scripts/makemigration index e2a6f7a3..bb24831e 100755 --- a/docker/pypgstac/bin/makemigration +++ b/scripts/container-scripts/makemigration @@ -1,6 +1,6 @@ #!/bin/bash SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -SRCDIR=$SCRIPT_DIR/../../../src +SRCDIR=${PGSTAC_REPO_DIR:-/opt/src} cd $SRCDIR SHORT=f:,t:,o,d,h @@ -29,8 +29,23 @@ do shift 1 ;; -h | --help) - "Help" - exit 2 + cat <&2 + exit 1 +fi + BASEDIR=$SRCDIR @@ -72,13 +97,8 @@ fi MIGRATIONSQL=$MIGRATIONSDIR/pgstac.$FROM-$TO.sql if [[ -f "$MIGRATIONSQL" ]]; then if [[ "$OVERWRITE" != 1 ]]; then - echo "$MIGRATIONSQL Already exists." - select yn in "Yes" "No"; do - case $yn in - Yes ) break;; - No ) exit 1;; - esac - done + echo "ERROR: $MIGRATIONSQL already exists. Use --overwrite to replace." >&2 + exit 1 else echo "Removing existing $MIGRATIONSQL" rm $MIGRATIONSQL diff --git a/docker/pypgstac/bin/pgstac_restore b/scripts/container-scripts/pgstac_restore similarity index 100% rename from docker/pypgstac/bin/pgstac_restore rename to scripts/container-scripts/pgstac_restore diff --git a/docker/pypgstac/bin/resetpgstac b/scripts/container-scripts/resetpgstac similarity index 90% rename from docker/pypgstac/bin/resetpgstac rename to scripts/container-scripts/resetpgstac index 097c3d5b..6c428cee 100755 --- a/docker/pypgstac/bin/resetpgstac +++ b/scripts/container-scripts/resetpgstac @@ -1,6 +1,6 @@ #!/bin/bash SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -cd $SCRIPT_DIR/../../../src/pgstac +cd ${PGSTAC_PGSTAC_DIR:-/opt/src/pgstac} set -e psql -v ON_ERROR_STOP=1 <<-EOSQL DROP SCHEMA IF EXISTS pgstac CASCADE; diff --git a/docker/pypgstac/bin/stageversion b/scripts/container-scripts/stageversion similarity index 74% rename from docker/pypgstac/bin/stageversion rename to scripts/container-scripts/stageversion index e99a6121..fcac770c 100755 --- a/docker/pypgstac/bin/stageversion +++ b/scripts/container-scripts/stageversion @@ -1,19 +1,41 @@ #!/bin/bash SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -SRCDIR=$SCRIPT_DIR/../../../src +SRCDIR=${PGSTAC_REPO_DIR:-/opt/src} cd $SRCDIR BASEDIR=$SRCDIR SQLDIR=$BASEDIR/pgstac/sql PYPGSTACDIR=$BASEDIR/pypgstac MIGRATIONSDIR=$BASEDIR/pgstac/migrations +function usage() { + cat < /dev/null && pwd ) +REPO_DIR=${PGSTAC_REPO_DIR:-/opt/src} export PATH="$SCRIPT_DIR:$PATH" -export SRCDIR=$SCRIPT_DIR/../../../src +if [[ -d "$REPO_DIR/pgstac" && -d "$REPO_DIR/pypgstac" ]]; then + export SRCDIR="$REPO_DIR" +elif [[ -d "$REPO_DIR/src/pgstac" && -d "$REPO_DIR/src/pypgstac" ]]; then + export SRCDIR="$REPO_DIR/src" +elif [[ -d "$SCRIPT_DIR/../../src/pgstac" && -d "$SCRIPT_DIR/../../src/pypgstac" ]]; then + export SRCDIR="$(cd "$SCRIPT_DIR/../../src" && pwd)" +else + echo "Unable to find pgstac/pypgstac sources under '$REPO_DIR'." >&2 + echo "Set PGSTAC_REPO_DIR to either a directory containing pgstac/ and pypgstac/ or a repository root containing src/pgstac and src/pypgstac." >&2 + exit 1 +fi export PGSTACDIR=$SRCDIR/pgstac -echo $SCRIPT_DIR -echo $SRCDIR -echo $PGSTACDIR - if [[ "${CI}" ]]; then set -x fi function usage() { - echo -n \ - "Usage: $(basename "$0") -Run PgSTAC tests. -This scripts is meant to be run inside the dev container. + cat </dev/null 2>&1 || true + psql -X -q -d postgres -c "ALTER DATABASE postgis REFRESH COLLATION VERSION;" >/dev/null 2>&1 || true +} + function test_formatting(){ cd $SRCDIR/pypgstac echo "Running ruff" ruff check src/pypgstac tests + ruff format --check src/pypgstac tests - echo "Running mypy" - mypy src/pypgstac + echo "Running ty" + ty check echo "Checking if there are any staged migrations." find $SRCDIR/pgstac/migrations | grep 'staged' && { echo "There are staged migrations in pypgstac/migrations. Please check migrations and remove staged suffix."; exit 1; } @@ -378,7 +405,7 @@ then fi [ $FORMATTING -eq 1 ] && test_formatting -[ $SETUPDB -eq 1 ] && setuptestdb +[ $SETUPDB -eq 1 ] && refresh_collation_versions && setuptestdb [ $PGTAP -eq 1 ] && test_pgtap [ $BASICSQL -eq 1 ] && test_basicsql [ $PYPGSTAC -eq 1 ] && test_pypgstac diff --git a/scripts/format b/scripts/format index 0a47433a..84d8565f 100755 --- a/scripts/format +++ b/scripts/format @@ -1,4 +1,37 @@ #!/bin/bash SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) cd $SCRIPT_DIR/.. -$SCRIPT_DIR/runinpypgstac format "$@" + +function usage() { + cat <&2 + usage + exit 1 + ;; + esac +done + +$SCRIPT_DIR/runinpypgstac --build-policy "$BUILD_POLICY" --cpfiles format diff --git a/scripts/makemigration b/scripts/makemigration new file mode 100755 index 00000000..64572a5c --- /dev/null +++ b/scripts/makemigration @@ -0,0 +1,64 @@ +#!/bin/bash +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +cd $SCRIPT_DIR/.. + +function usage() { + cat < /dev/null && pwd ) cd $SCRIPT_DIR/.. -$SCRIPT_DIR/runinpypgstac pypgstac migrate "$@" + +function usage() { + cat </dev/null | tail -n +2 | grep -q .; then + echo "$port" + return 0 + fi + done + + echo 5439 +} + +function ensure_env_file() { + if [[ ! -f .env && -f .env.example ]]; then + cp .env.example .env + local selected_port + selected_port=$(first_available_pgport) + if [[ "$selected_port" != "5439" ]]; then + sed -i "s/^PGPORT=.*/PGPORT=${selected_port}/" .env + fi + echo "Created .env from .env.example" + fi +} diff --git a/scripts/runinpypgstac b/scripts/runinpypgstac index bbd0517e..8e978725 100755 --- a/scripts/runinpypgstac +++ b/scripts/runinpypgstac @@ -1,56 +1,141 @@ #!/bin/bash SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) cd $SCRIPT_DIR/.. +source "$SCRIPT_DIR/pgstacenv" set -e if [[ "${CI}" ]]; then set -x fi + function usage() { - echo -n \ - "Usage: $(basename "$0") <--build> <--no-cache>