From 013aa4eaf92861790f4d13e91e5fa170d04a3fc6 Mon Sep 17 00:00:00 2001 From: David W Bitner Date: Mon, 20 Apr 2026 14:03:41 -0500 Subject: [PATCH 01/22] chore: ignore planning artifacts, env files, and Rust build output Add .plans/, .env, and src/pgstacrust/target/ to .gitignore. Local planning files are working notes only and must not be committed. The .env file holds local secret/port overrides from .env.example. --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 32565592..c57bfb98 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,7 @@ src/pypgstac/python/pypgstac/*.so .vscode .ipynb_checkpoints .venv -.pytest_cache \ No newline at end of file +.pytest_cache +.plans/ +.env +src/pgstacrust/target/ From c617b9300db5dcc838df917760710afedff6ab02 Mon Sep 17 00:00:00 2001 From: David W Bitner Date: Mon, 20 Apr 2026 14:03:48 -0500 Subject: [PATCH 02/22] refactor(scripts): move in-container scripts to scripts/container-scripts/ docker/pypgstac/bin/ contained scripts that run inside the pypgstac container. Move them to scripts/container-scripts/ to make the layout explicit: scripts/ is the host-facing entrypoint surface, scripts/container-scripts/ is the in-container payload. The Dockerfiles are updated in the next commit to COPY from the new path. A new scripts/makemigration host wrapper is added for the in-container makemigration helper. --- docker/pypgstac/bin/tmpdb | 54 ---------------- .../bin => scripts/container-scripts}/format | 8 +-- .../container-scripts}/initpgstac | 4 +- .../container-scripts}/loadsampledata | 2 +- .../container-scripts}/makemigration | 40 +++++++++--- .../container-scripts}/pgstac_restore | 0 .../container-scripts}/resetpgstac | 2 +- .../container-scripts}/stageversion | 26 +++++++- .../bin => scripts/container-scripts}/test | 35 ++++++---- scripts/makemigration | 64 +++++++++++++++++++ 10 files changed, 149 insertions(+), 86 deletions(-) delete mode 100755 docker/pypgstac/bin/tmpdb rename {docker/pypgstac/bin => scripts/container-scripts}/format (60%) rename {docker/pypgstac/bin => scripts/container-scripts}/initpgstac (79%) rename {docker/pypgstac/bin => scripts/container-scripts}/loadsampledata (87%) rename {docker/pypgstac/bin => scripts/container-scripts}/makemigration (76%) rename {docker/pypgstac/bin => scripts/container-scripts}/pgstac_restore (100%) rename {docker/pypgstac/bin => scripts/container-scripts}/resetpgstac (90%) rename {docker/pypgstac/bin => scripts/container-scripts}/stageversion (74%) rename {docker/pypgstac/bin => scripts/container-scripts}/test (92%) create mode 100755 scripts/makemigration diff --git a/docker/pypgstac/bin/tmpdb b/docker/pypgstac/bin/tmpdb deleted file mode 100755 index b4788406..00000000 --- a/docker/pypgstac/bin/tmpdb +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -set -e - -if [[ "${CI}" ]]; then - set -x -fi - -if [ ! $PGSTACDOCKER == 1 ]; then - echo "This script should only be run within pgstac docker"; exit 1; -fi - -function tmppg(){ - -# Make temp directories for pgdata and logs -WORKSPACE=$(mktemp -d) - -# Find an unused port - -# Run tests using postgres user and trust authentication - -export PGDATA=$WORKSPACE/pgdata -export PGLOG=$WORKSPACE/pglog - -export PGDATABASE=postgres -export PGUSER=postgres - -# Leverage scripts from docker-entrypoint.sh -source /usr/local/bin/docker-entrypoint.sh - -gosu postgres initdb --username $PGUSER --auth=trust -D $PGDATA - -# Start postgres with minimal logging settings -gosu postgres pg_ctl -l "$PGLOG" -w -o "-F -c fsync=off -c full_page_writes=off -c synchronous_commit=off -c archive_mode=off" start -trap "gosu postgres pg_ctl stop && rm -fr $WORKSPACE " 0 2 3 15 - -# Get location of script directory -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) - -cd /opt/src/pgstac -# Create template database with pgstac installed -psql -X -q -v ON_ERROR_STOP=1 < /dev/null && pwd ) -cd $SCRIPT_DIR/../../../src +cd ${PGSTAC_REPO_DIR:-/opt/src} function usage() { echo -n \ "Usage: $(basename "$0") Format code. -This scripts is meant to be run inside the dev container. +This script is meant to be run inside the dev container. " } if [ "${BASH_SOURCE[0]}" = "${0}" ]; then echo "Formatting pypgstac..." - ruff --fix pypgstac/pypgstac - ruff --fix pypgstac/tests + ruff check --fix pypgstac/src/pypgstac pypgstac/tests + ruff format pypgstac/src/pypgstac pypgstac/tests fi diff --git a/docker/pypgstac/bin/initpgstac b/scripts/container-scripts/initpgstac similarity index 79% rename from docker/pypgstac/bin/initpgstac rename to scripts/container-scripts/initpgstac index b205b7cf..880b350d 100755 --- a/docker/pypgstac/bin/initpgstac +++ b/scripts/container-scripts/initpgstac @@ -1,11 +1,11 @@ #!/bin/bash SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -cd $SCRIPT_DIR/../../../src/pgstac +cd ${PGSTAC_PGSTAC_DIR:-/opt/src/pgstac} psql -X -q -v ON_ERROR_STOP=1 < /dev/null && pwd ) -cd $SCRIPT_DIR/../../../src/pgstac +cd ${PGSTAC_PGSTAC_DIR:-/opt/src/pgstac} psql -f pgstac.sql psql -v ON_ERROR_STOP=1 <<-EOSQL \copy collections (content) FROM 'tests/testdata/collections.ndjson' diff --git a/docker/pypgstac/bin/makemigration b/scripts/container-scripts/makemigration similarity index 76% rename from docker/pypgstac/bin/makemigration rename to scripts/container-scripts/makemigration index e2a6f7a3..bb24831e 100755 --- a/docker/pypgstac/bin/makemigration +++ b/scripts/container-scripts/makemigration @@ -1,6 +1,6 @@ #!/bin/bash SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -SRCDIR=$SCRIPT_DIR/../../../src +SRCDIR=${PGSTAC_REPO_DIR:-/opt/src} cd $SRCDIR SHORT=f:,t:,o,d,h @@ -29,8 +29,23 @@ do shift 1 ;; -h | --help) - "Help" - exit 2 + cat <&2 + exit 1 +fi + BASEDIR=$SRCDIR @@ -72,13 +97,8 @@ fi MIGRATIONSQL=$MIGRATIONSDIR/pgstac.$FROM-$TO.sql if [[ -f "$MIGRATIONSQL" ]]; then if [[ "$OVERWRITE" != 1 ]]; then - echo "$MIGRATIONSQL Already exists." - select yn in "Yes" "No"; do - case $yn in - Yes ) break;; - No ) exit 1;; - esac - done + echo "ERROR: $MIGRATIONSQL already exists. Use --overwrite to replace." >&2 + exit 1 else echo "Removing existing $MIGRATIONSQL" rm $MIGRATIONSQL diff --git a/docker/pypgstac/bin/pgstac_restore b/scripts/container-scripts/pgstac_restore similarity index 100% rename from docker/pypgstac/bin/pgstac_restore rename to scripts/container-scripts/pgstac_restore diff --git a/docker/pypgstac/bin/resetpgstac b/scripts/container-scripts/resetpgstac similarity index 90% rename from docker/pypgstac/bin/resetpgstac rename to scripts/container-scripts/resetpgstac index 097c3d5b..6c428cee 100755 --- a/docker/pypgstac/bin/resetpgstac +++ b/scripts/container-scripts/resetpgstac @@ -1,6 +1,6 @@ #!/bin/bash SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -cd $SCRIPT_DIR/../../../src/pgstac +cd ${PGSTAC_PGSTAC_DIR:-/opt/src/pgstac} set -e psql -v ON_ERROR_STOP=1 <<-EOSQL DROP SCHEMA IF EXISTS pgstac CASCADE; diff --git a/docker/pypgstac/bin/stageversion b/scripts/container-scripts/stageversion similarity index 74% rename from docker/pypgstac/bin/stageversion rename to scripts/container-scripts/stageversion index e99a6121..fcac770c 100755 --- a/docker/pypgstac/bin/stageversion +++ b/scripts/container-scripts/stageversion @@ -1,19 +1,41 @@ #!/bin/bash SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -SRCDIR=$SCRIPT_DIR/../../../src +SRCDIR=${PGSTAC_REPO_DIR:-/opt/src} cd $SRCDIR BASEDIR=$SRCDIR SQLDIR=$BASEDIR/pgstac/sql PYPGSTACDIR=$BASEDIR/pypgstac MIGRATIONSDIR=$BASEDIR/pgstac/migrations +function usage() { + cat < /dev/null && pwd ) +REPO_DIR=${PGSTAC_REPO_DIR:-/opt/src} export PATH="$SCRIPT_DIR:$PATH" -export SRCDIR=$SCRIPT_DIR/../../../src +export SRCDIR=$REPO_DIR export PGSTACDIR=$SRCDIR/pgstac -echo $SCRIPT_DIR -echo $SRCDIR -echo $PGSTACDIR - if [[ "${CI}" ]]; then set -x fi function usage() { - echo -n \ - "Usage: $(basename "$0") -Run PgSTAC tests. -This scripts is meant to be run inside the dev container. + cat < /dev/null && pwd ) +cd $SCRIPT_DIR/.. + +function usage() { + cat < Date: Mon, 20 Apr 2026 14:03:57 -0500 Subject: [PATCH 03/22] feat(scripts): expand script UX with --help, --build-policy, and env-var support All host-facing scripts (test, format, migrate, server, stageversion, runinpypgstac, console) now: - Accept --help / -h - Honor PGSTAC_BUILD_POLICY={always,missing,never} (replaces bare --build) - Honor PGSTAC_FAST, PGSTAC_WATCH, PGSTAC_STRICT where applicable scripts/test expanded from a 3-line wrapper to a full-featured runner with --fast, --watch, --build-policy, --no-strict, and stale-image detection (refuses to run with build-policy!=always when scripts/docker/src have uncommitted changes). scripts/pgstacenv gains: - ensure_env_file: auto-creates .env from .env.example on first run - first_available_pgport: scans 5439-5445 to avoid port conflicts .env.example documents all supported variables. --- .env.example | 25 ++++++++ scripts/console | 16 +++-- scripts/format | 35 +++++++++- scripts/migrate | 35 +++++++++- scripts/pgstacenv | 25 ++++++++ scripts/runinpypgstac | 145 +++++++++++++++++++++++++++++++++--------- scripts/server | 24 +++++-- scripts/setup | 16 ++++- scripts/stageversion | 53 ++++++++++++++- scripts/test | 108 ++++++++++++++++++++++++++++++- scripts/update | 6 +- 11 files changed, 440 insertions(+), 48 deletions(-) create mode 100644 .env.example diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..4d116a9f --- /dev/null +++ b/.env.example @@ -0,0 +1,25 @@ +POSTGRES_USER=username +POSTGRES_PASSWORD=password +POSTGRES_DB=postgis + +PGUSER=username +PGPASSWORD=password +PGDATABASE=postgis +PGHOST=localhost +PGPORT=5439 + +DOCKER_BUILDKIT=1 +BUILDKIT_INLINE_CACHE=1 + +# Host-script defaults. +PGSTAC_BUILD_POLICY=always +PGSTAC_FAST=0 +PGSTAC_WATCH=0 +PGSTAC_STRICT=1 + +# Migration helper defaults. +# PGSTAC_VERSION=0.9.11 +# PGSTAC_FROM_VERSION=0.9.10 +# PGSTAC_TO_VERSION=0.9.11 +# PGSTAC_OVERWRITE=0 +# PGSTAC_DEBUG=0 \ No newline at end of file diff --git a/scripts/console b/scripts/console index 60068e4b..7defafa4 100755 --- a/scripts/console +++ b/scripts/console @@ -1,6 +1,7 @@ #!/bin/bash SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) cd $SCRIPT_DIR/.. +source "$SCRIPT_DIR/pgstacenv" set -e if [[ "${CI}" ]]; then @@ -9,7 +10,7 @@ fi function usage() { echo -n \ - "Usage: $(basename "$0") [--db] + "Usage: $(basename "$0") [--db] [--help] Start a console in the dev container --db: Instead, start a psql console in the database container. @@ -21,14 +22,19 @@ while [[ "$#" > 0 ]]; do case $1 in DB_CONSOLE=1 shift ;; + -h|--help) + usage + exit 0 + ;; *) - usage "Unknown parameter passed: $1" - shift - shift + echo "Unknown parameter passed: $1" >&2 + usage + exit 1 ;; esac; done if [ "${BASH_SOURCE[0]}" = "${0}" ]; then + ensure_env_file docker compose up -d @@ -39,6 +45,6 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then exit 0 fi - docker compose exec pgstac /bin/bash + docker compose exec pypgstac /bin/bash fi diff --git a/scripts/format b/scripts/format index 0a47433a..227e37a5 100755 --- a/scripts/format +++ b/scripts/format @@ -1,4 +1,37 @@ #!/bin/bash SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) cd $SCRIPT_DIR/.. -$SCRIPT_DIR/runinpypgstac format "$@" + +function usage() { + cat <&2 + usage + exit 1 + ;; + esac +done + +$SCRIPT_DIR/runinpypgstac --build-policy "$BUILD_POLICY" format diff --git a/scripts/migrate b/scripts/migrate index 445a5179..dd97f82a 100755 --- a/scripts/migrate +++ b/scripts/migrate @@ -1,4 +1,37 @@ #!/bin/bash SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) cd $SCRIPT_DIR/.. -$SCRIPT_DIR/runinpypgstac pypgstac migrate "$@" + +function usage() { + cat </dev/null | tail -n +2 | grep -q .; then + echo "$port" + return 0 + fi + done + + echo 5439 +} + +function ensure_env_file() { + if [[ ! -f .env && -f .env.example ]]; then + cp .env.example .env + local selected_port + selected_port=$(first_available_pgport) + if [[ "$selected_port" != "5439" ]]; then + sed -i "s/^PGPORT=.*/PGPORT=${selected_port}/" .env + fi + echo "Created .env from .env.example" + fi +} diff --git a/scripts/runinpypgstac b/scripts/runinpypgstac index bbd0517e..afc70ae2 100755 --- a/scripts/runinpypgstac +++ b/scripts/runinpypgstac @@ -1,56 +1,141 @@ #!/bin/bash SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) cd $SCRIPT_DIR/.. +source "$SCRIPT_DIR/pgstacenv" set -e if [[ "${CI}" ]]; then set -x fi + function usage() { - echo -n \ - "Usage: $(basename "$0") <--build> <--no-cache>