support multiple worktrees at once

.github/copilot-instructions.md

@@ -27,7 +27,7 @@ scripts/setup-openapi-generator.sh
 cd api && pip3 install -r requirements.txt -r requirements_dev.txt
 
 # Start local database
-docker-compose --env-file ./config/.env.local up -d --force-recreate
+scripts/docker-localdb-rebuild-data.sh   # per-worktree project + ports; prefer over bare `docker compose up`
 
 # Generate API stubs (run after schema changes)
 scripts/api-gen.sh

CLAUDE.md

@@ -43,7 +43,9 @@ scripts/setup-openapi-generator.sh
 cd api && pip3 install -r requirements.txt -r requirements_dev.txt
 
 # Local Postgres + Liquibase migrations
-docker-compose --env-file ./config/.env.local up -d --force-recreate
+scripts/docker-localdb-rebuild-data.sh   # prefer this over a bare `docker compose up`:
+                                        # it loads config/.env.worktree, which pins this
+                                        # worktree's Compose project and Postgres ports
 
 # Re-init local env after checking out a branch: rebuilds main+test DBs, regenerates
 # SQLAlchemy models, FastAPI stubs (Feeds + User Service + Operations API), and
@@ -84,7 +86,10 @@ scripts/function-python-run.sh --function_name <name>     # installs deps into a
 scripts/function-python-build.sh --function_name <name>   # zips into .dist/ for deploy
 ```
 
-`api-tests.sh` copies `config/.env.local` to `.env` before running, so local env vars match the docker-compose Postgres instance.
+`api-tests.sh` copies `config/.env.local` to `.env` before running, then appends `config/.env.worktree` if present, so local env vars match this worktree's docker-compose Postgres instance.
+
+### Per-worktree local databases
+Each git worktree runs its own Compose project, with its own published Postgres host ports and its own named data volumes, so several worktrees can run their stacks and test suites concurrently. `scripts/worktree-env.sh` generates `config/.env.worktree` (gitignored) on first use and every other script sources it; CI never creates that file, so CI keeps the default `5432` / `54320`. Release the resources with `scripts/docker-localdb-cleanup.sh` (add `--all` to reclaim idle leftovers from deleted worktrees). Note `config/.env.local` is *tracked* - never put per-worktree values there.
 
 ## Architecture Notes
 

README.md

@@ -91,9 +91,30 @@ pip3 install -r requirements_dev.txt
 - Generates an instance of the database locally using docker-compose
 
 ```bash
-docker-compose --env-file ./config/.env.local  up -d --force-recreate
+scripts/docker-localdb-rebuild-data.sh
 ```
 
+  Prefer this over a bare `docker compose up`. Each git worktree gets its own
+  Compose project and its own published Postgres ports (generated once into
+  `config/.env.worktree` by `scripts/worktree-env.sh`), so several worktrees can
+  run their databases - and their test suites - at the same time. A bare
+  `docker compose up` does not load that file and will fall back to the shared
+  default ports, colliding with whichever worktree already holds them.
+
+- Releases the Docker resources again when you are done with a worktree
+
+```bash
+scripts/docker-localdb-cleanup.sh            # this worktree only
+scripts/docker-localdb-cleanup.sh --all      # also reclaim idle leftovers of deleted worktrees
+```
+
+  Worth knowing: every Compose project creates its own Docker network, and
+  Docker's default address pool only fits about 31 of them. Worktrees that are
+  deleted without being torn down leak a network each; once the pool is
+  exhausted, every `docker compose up` fails with "all predefined address pools
+  have been fully subnetted", which shows up as a flood of connection errors
+  from liquibase and the db-gen scripts.
+
 - Generates the api and database stubs on the first run and every time the schema changes
 
 ```bash

api/tests/integration/cascade_delete/conftest.py

@@ -8,6 +8,7 @@
 from shared.database.database import Database
 from main import app as application
 from tests.test_utils.database import populate_database
+from tests.test_utils.db_url import feeds_test_database_url
 
 
 @pytest.fixture(scope="package")
@@ -19,7 +20,7 @@ def app() -> FastAPI:
 @pytest.fixture(scope="package")
 def test_database():
     # Restrict the tests to the test database
-    os.environ["FEEDS_DATABASE_URL"] = "postgresql://postgres:postgres@localhost:54320/MobilityDatabaseTest"
+    os.environ["FEEDS_DATABASE_URL"] = feeds_test_database_url()
 
     data_dirs = []
     second_phase_data_dirs = []

api/tests/integration/conftest.py

@@ -7,6 +7,7 @@
 from shared.database.database import Database
 from main import app as application
 from tests.test_utils.database import populate_database
+from tests.test_utils.db_url import feeds_test_database_url
 
 
 @pytest.fixture(scope="package")
@@ -18,7 +19,7 @@ def app() -> FastAPI:
 @pytest.fixture(scope="package")
 def test_database():
     # Restrict the tests to the test database
-    os.environ["FEEDS_DATABASE_URL"] = "postgresql://postgres:postgres@localhost:54320/MobilityDatabaseTest"
+    os.environ["FEEDS_DATABASE_URL"] = feeds_test_database_url()
 
     current_path = os.path.dirname(os.path.abspath(__file__))
 

api/tests/integration/populate_tests/conftest.py

@@ -7,6 +7,7 @@
 from shared.database.database import Database
 from main import app as application
 from tests.test_utils.database import populate_database
+from tests.test_utils.db_url import feeds_test_database_url
 
 
 @pytest.fixture(scope="package")
@@ -18,7 +19,7 @@ def app() -> FastAPI:
 @pytest.fixture(scope="package")
 def test_database():
     # Restrict the tests to the test database
-    os.environ["FEEDS_DATABASE_URL"] = "postgresql://postgres:postgres@localhost:54320/MobilityDatabaseTest"
+    os.environ["FEEDS_DATABASE_URL"] = feeds_test_database_url()
 
     current_path = os.path.dirname(os.path.abspath(__file__))
 

api/tests/integration/populate_twice_gbfs/conftest.py

@@ -7,6 +7,7 @@
 from shared.database.database import Database
 from main import app as application
 from tests.test_utils.database import populate_database
+from tests.test_utils.db_url import feeds_test_database_url
 
 
 @pytest.fixture(scope="package")
@@ -18,7 +19,7 @@ def app() -> FastAPI:
 @pytest.fixture(scope="package")
 def test_database():
     # Restrict the tests to the test database
-    os.environ["FEEDS_DATABASE_URL"] = "postgresql://postgres:postgres@localhost:54320/MobilityDatabaseTest"
+    os.environ["FEEDS_DATABASE_URL"] = feeds_test_database_url()
 
     current_path = os.path.dirname(os.path.abspath(__file__))
 

api/tests/test_utils/db_url.py

@@ -0,0 +1,38 @@
+"""Resolution of the test database URLs.
+
+These URLs are per worktree. ``scripts/worktree-env.sh`` publishes Postgres on a
+different host port in each git worktree so that several stacks - and several
+test runs - can be active at once, and writes the resulting URLs into
+``config/.env.worktree``. ``scripts/api-tests.sh`` merges that file into the
+``.env`` that sits next to the tests.
+
+The literals below are the values for a checkout with no such override, and for
+CI, which never creates one. Behaviour is therefore unchanged when it is absent.
+"""
+
+import os
+from typing import Final
+
+from dotenv import load_dotenv
+
+# Populate os.environ from the .env that scripts/api-tests.sh writes next to the
+# tests. load_dotenv never overrides a variable already set in the environment.
+load_dotenv()
+
+DEFAULT_FEEDS_TEST_DATABASE_URL: Final[str] = "postgresql://postgres:postgres@localhost:54320/MobilityDatabaseTest"
+
+DEFAULT_USERS_TEST_DATABASE_URL: Final[str] = "postgresql://postgres:postgres@localhost:54320/MobilityDatabaseUsersTest"
+
+
+def feeds_test_database_url() -> str:
+    """URL of the feeds test database for this worktree."""
+    return os.getenv("FEEDS_DATABASE_URL_TEST", DEFAULT_FEEDS_TEST_DATABASE_URL)
+
+
+def users_test_database_url() -> str:
+    """URL of the users test database for this worktree.
+
+    ``TEST_USERS_DATABASE_URL`` is honoured first because it predates the
+    per-worktree override and is already used to redirect individual tests.
+    """
+    return os.getenv("TEST_USERS_DATABASE_URL") or os.getenv("USERS_DATABASE_URL_TEST", DEFAULT_USERS_TEST_DATABASE_URL)

api/tests/unittest/conftest.py

@@ -7,6 +7,7 @@
 from shared.database.database import Database
 from main import app as application
 from tests.test_utils.database import populate_database
+from tests.test_utils.db_url import feeds_test_database_url
 
 
 @pytest.fixture(scope="package")
@@ -19,7 +20,7 @@ def app() -> FastAPI:
 def test_database():
 
     # Restrict the tests to the test database
-    os.environ["FEEDS_DATABASE_URL"] = "postgresql://postgres:postgres@localhost:54320/MobilityDatabaseTest"
+    os.environ["FEEDS_DATABASE_URL"] = feeds_test_database_url()
 
     current_path = os.path.dirname(os.path.abspath(__file__))
     data_dirs = [current_path + "/../test_data"]

api/tests/unittest/user_service/conftest.py

@@ -6,9 +6,12 @@
 # Ensure src is on the path for these standalone unit tests.
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../../../src"))
 
-# Users test database, matching the Liquibase target in the CI "Run tests" job
+from tests.test_utils.db_url import users_test_database_url as resolve_users_test_database_url  # noqa: E402
+
+# Users test database. Per worktree - see tests/test_utils/db_url.py. Defaults to
+# the CI "Run tests" Liquibase target
 # (liquibase update on jdbc:postgresql://localhost:54320/MobilityDatabaseUsersTest).
-USERS_TEST_DATABASE_URL = "postgresql://postgres:postgres@localhost:54320/MobilityDatabaseUsersTest"
+USERS_TEST_DATABASE_URL = resolve_users_test_database_url()
 
 
 @pytest.fixture
@@ -20,7 +23,7 @@ def users_test_database_url():
     ``USERS_DATABASE_URL`` and on the CI test database.
     """
     previous = os.environ.get("USERS_DATABASE_URL")
-    os.environ["USERS_DATABASE_URL"] = os.getenv("TEST_USERS_DATABASE_URL", USERS_TEST_DATABASE_URL)
+    os.environ["USERS_DATABASE_URL"] = USERS_TEST_DATABASE_URL
     try:
         yield os.environ["USERS_DATABASE_URL"]
     finally:

config/.env.local

@@ -4,7 +4,11 @@ POSTGRES_DB=MobilityDatabase
 POSTGRES_TEST_DB=MobilityDatabaseTest
 POSTGRES_USER_DB=MobilityDatabaseUsers
 POSTGRES_USER_TEST_DB=MobilityDatabaseUsersTest
+# In-container Postgres port. Do not make this per-worktree: the liquibase and
+# schemaspy services use it for container-to-container URLs.
 POSTGRES_PORT=5432
+# Published host port for the main DB. Overridden per worktree in config/.env.worktree.
+POSTGRES_HOST_PORT=5432
 POSTGRES_TEST_PORT=54320
 PGUSER=postgres
 POSTGRES_HOST=localhost
@@ -13,6 +17,7 @@ SCHEMA_SPY_DOC=schemaspy-dev
 FEEDS_DATABASE_URL=postgresql://postgres:postgres@localhost:5432/MobilityDatabase
 FEEDS_DATABASE_URL_TEST=postgresql://postgres:postgres@localhost:54320/MobilityDatabaseTest
 USERS_DATABASE_URL=postgresql://postgres:postgres@localhost:5432/MobilityDatabaseUsers
+USERS_DATABASE_URL_TEST=postgresql://postgres:postgres@localhost:54320/MobilityDatabaseUsersTest
 FEEDS_AUTHORIZATION=Bearer
 LOCAL_ENV=True
 PROJECT_ID=mobility-feeds-dev

docker-compose.yaml

@@ -1,14 +1,24 @@
+# The Compose project name is what isolates one git worktree's local stack from
+# another's. scripts/worktree-env.sh generates a stable per-worktree value in
+# config/.env.worktree; the fallback keeps a plain `docker compose` invocation
+# (and CI, which never creates that file) on the historical behaviour.
+name: ${COMPOSE_PROJECT_NAME:-mobility-feed-api}
+
 services:
   postgres:
-    container_name: database
     image: postgis/postgis:15-3.5
     # Fix warning message on arm64 machines
     platform: linux/amd64
     healthcheck:
-      test: [ "CMD-SHELL", "pg_isready" ]
-      interval: 20s
+      # Must probe over TCP, not the Unix socket. During first-time init the
+      # entrypoint runs a temporary server with listen_addresses='' that answers
+      # on the socket, so a bare `pg_isready` reports healthy while TCP is still
+      # refused - releasing the liquibase services too early.
+      test: [ "CMD-SHELL", "pg_isready -h 127.0.0.1 -p ${POSTGRES_PORT} -U ${POSTGRES_USER}" ]
+      interval: 5s
       timeout: 5s
-      retries: 5
+      retries: 10
+      start_period: 30s
     environment:
       - POSTGRES_USER=${POSTGRES_USER}
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
@@ -17,24 +27,29 @@ services:
       - POSTGRES_USER_DB=${POSTGRES_USER_DB}
       - PGUSER=${PGUSER}
     volumes:
-      - ./data:/var/lib/postgresql/data
+      # A named volume, not a bind mount: Docker Desktop's file sharing ignores
+      # chown on bind-mounted directories, so PGDATA stays root-owned and
+      # postgres (uid 999) refuses to start with "data directory has wrong
+      # ownership". The volume is project-scoped, so each worktree gets its own.
+      - pgdata:/var/lib/postgresql/data
       - ./liquibase/init:/docker-entrypoint-initdb.d
     expose:
       - ${POSTGRES_PORT}
     ports:
-      - ${POSTGRES_PORT}:${POSTGRES_PORT}
+      # Host port is per worktree; container port is always ${POSTGRES_PORT}.
+      - ${POSTGRES_HOST_PORT:-5432}:${POSTGRES_PORT}
     networks:
       - local
   postgres-test:
-    container_name: database_test
     image: postgis/postgis:15-3.5
     # Fix warning message on arm64 machines
     platform: linux/amd64
     healthcheck:
-      test: [ "CMD-SHELL", "pg_isready" ]
-      interval: 20s
+      test: [ "CMD-SHELL", "pg_isready -h 127.0.0.1 -p ${POSTGRES_PORT} -U ${POSTGRES_USER}" ]
+      interval: 5s
       timeout: 5s
-      retries: 5
+      retries: 10
+      start_period: 30s
     environment:
       - POSTGRES_USER=${POSTGRES_USER}
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
@@ -43,17 +58,19 @@ services:
       - POSTGRES_USER_DB=${POSTGRES_USER_TEST_DB}
       - PGUSER=${PGUSER}
     volumes:
-      - ./data-test:/var/lib/postgresql/data
+      - pgdata_test:/var/lib/postgresql/data
       - ./liquibase/init:/docker-entrypoint-initdb.d
     expose:
-      - ${POSTGRES_TEST_PORT}
+      - ${POSTGRES_PORT}
     ports:
-      - ${POSTGRES_TEST_PORT}:${POSTGRES_PORT}
+      - ${POSTGRES_TEST_PORT:-54320}:${POSTGRES_PORT}
     networks:
-      - local      
+      - local
   liquibase:
     image: liquibase/liquibase
-    restart: on-failure
+    # Capped: an uncapped on-failure policy turns a permanently broken migration
+    # (e.g. a missing JDBC driver jar) into a container that restarts forever.
+    restart: on-failure:5
     volumes:
       - ./liquibase:/liquibase/changelog
     command:
@@ -72,7 +89,7 @@ services:
       - local
   liquibase-test:
     image: liquibase/liquibase
-    restart: on-failure
+    restart: on-failure:5
     volumes:
       - ./liquibase:/liquibase/changelog
     command:
@@ -88,10 +105,10 @@ services:
       postgres-test:
         condition: service_healthy
     networks:
-      - local      
+      - local
   liquibase-user:
     image: liquibase/liquibase
-    restart: on-failure
+    restart: on-failure:5
     volumes:
       - ./liquibase:/liquibase/changelog
     command:
@@ -110,7 +127,7 @@ services:
       - local
   liquibase-user-test:
     image: liquibase/liquibase
-    restart: on-failure
+    restart: on-failure:5
     volumes:
       - ./liquibase:/liquibase/changelog
     command:
@@ -129,7 +146,6 @@ services:
       - local
   schemaspy:
     image: andrewjones/schemaspy-postgres:latest
-    container_name: schema_generation
     command: [ "-db", "${POSTGRES_DB}", "-host", "postgres", "-port", "${POSTGRES_PORT}", "-s", "public", "-u", "${POSTGRES_USER}", "-p", "${POSTGRES_PASSWORD}" ]
     volumes:
       - ./docs/${SCHEMA_SPY_DOC}:/output
@@ -138,6 +154,9 @@ services:
        condition: service_completed_successfully
     networks:
       - local
+volumes:
+  pgdata:
+  pgdata_test:
 networks:
   local:
-    driver: bridge
+    driver: bridge

functions-python/export_csv/src/main.py

@@ -469,8 +469,10 @@ def upload_file_to_storage(source_file_path, target_path):
         default=csv_default_file_path,
         help="Path to the output csv file.",
     )
-    os.environ["FEEDS_DATABASE_URL"] = (
-        "postgresql://postgres:postgres@localhost:54320/MobilityDatabaseTest"
+    # Per-worktree test DB (config/.env.worktree); falls back to the default port.
+    os.environ["FEEDS_DATABASE_URL"] = os.getenv(
+        "FEEDS_DATABASE_URL_TEST",
+        "postgresql://postgres:postgres@localhost:54320/MobilityDatabaseTest",
     )
     args = parser.parse_args()
     export_csv(args.outpath)