diff --git a/Makefile b/Makefile index caa2ca0..a08b63d 100644 --- a/Makefile +++ b/Makefile @@ -5,12 +5,18 @@ MODULE_big = lolor EXTENSION = lolor DATA = lolor--1.0.sql \ lolor--1.0--1.2.1.sql lolor--1.2.1--1.2.2.sql \ - lolor--1.2.2--1.3.0.sql + lolor--1.2.2--1.2.3.sql lolor--1.2.3--1.3.0.sql PGFILEDESC = "lolor - drop in large objects replacement for logical replication" OBJS = src/lolor.o src/lolor_fsstubs.o src/lolor_inv_api.o src/lolor_largeobject.o REGRESS = lolor +# The drop guard is an object_access_hook, so lolor has to be preloaded and the +# regression suite can only run against a server that preloads it. Run it in a +# temporary instance configured that way instead of against whatever server +# pg_config points at; this also keeps the suite's cluster-global test roles out +# of any shared server. +REGRESS_OPTS = --temp-instance=./tmp_check --temp-config=regress.conf TAP_TESTS = 1 ifdef USE_PGXS diff --git a/README.md b/README.md index 07861a9..33e1845 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ make USE_PGXS=1 make USE_PGXS=1 install ``` -After installing the lolor extension, connect to your Postgres database and create the extension with the command: +After installing the lolor extension, add it to `shared_preload_libraries` and restart the server (see [Configuring lolor](#configuring-lolor)), then connect to your Postgres database and create the extension with the command: ``` CREATE EXTENSION lolor; @@ -46,7 +46,16 @@ CREATE EXTENSION lolor; ### Configuring lolor -You must set the `lolor.node` parameter before using the extension. The value can be from 1 to 2^28; the value is used to help in generation of new large object OID. +lolor must be loaded at server start. Add it to `shared_preload_libraries` in +`postgresql.conf` and restart; `CREATE EXTENSION lolor`, `LOAD 'lolor'` and any call that reaches one of lolor's functions fail with `lolor must be loaded via "shared_preload_libraries"` when the library was loaded on demand; the native `pg_catalog.lo_*` functions are not affected while lolor is not installed or is disabled. The guard that protects large +objects when the extension is dropped is an object access hook, which is only +in place in every backend when the library is preloaded. + +``` +shared_preload_libraries = 'lolor' +``` + +You must set the `lolor.node` parameter before using the extension. The value can be from 1 to 15 (0 means unset); it is encoded in the four low bits of every large object OID lolor generates, so each node must use a different value. ``` lolor.node = 1 @@ -70,8 +79,8 @@ SELECT spock.repset_add_table('spock_replication_set', 'lolor.pg_largeobject_met ### Migrating large objects -Migration from native to lolor is **manual**; migration back is **automatic** -on `DROP EXTENSION` so no objects are ever lost. +Migration is a manual step in both directions. Dropping the extension never +moves data: it is refused while any object remains in lolor storage. Migrate existing native large objects into lolor storage (requires superuser): @@ -80,16 +89,31 @@ CREATE EXTENSION lolor; SELECT lolor.migrate_from_native(); ``` -Reverse migration happens automatically when the extension is dropped, or can -be triggered manually: +To remove the extension, migrate the objects back first: ```sql -SELECT lolor.migrate_to_native(); -- manual -DROP EXTENSION lolor; -- automatic +SELECT lolor.migrate_to_native(); +DROP EXTENSION lolor; ``` +An object access hook refuses to remove the extension while it is enabled or +while any object remains in lolor storage, on every path that reaches it +(`DROP EXTENSION`, `DROP SCHEMA lolor CASCADE`, `DROP OWNED BY`). The drop +itself only puts the native `pg_catalog.lo_*` names back. To discard whatever +is in lolor storage instead, a superuser can set `lolor.allow_unsafe_drop = on` +for the session that runs the drop; it skips both checks. The drop renames the +native functions back, so it has the same requirements as `lolor.disable()`: +no other session connected to the database, and a client session. + Both directions preserve original OIDs, owners, ACLs, and data. +Two helpers find and repair objects whose roles have been dropped: + +```sql +SELECT * FROM lolor.check_orphans(); -- objects referring to a dropped role +SELECT lolor.fix_orphans('some_role'); -- reassign them and drop dead ACL entries +``` + When the spock extension is installed, both migration functions run under `spock.repair_mode()`, so the row-shuffling migration DML is **not** replicated to other nodes. This is essential for `migrate_to_native()`: its @@ -124,4 +148,7 @@ database user. Upgrading to 1.3.0 revokes the privilege; see the release notes. - Native large object functionality cannot be used while you are using the lolor extension. - lolor does not support the following statements: `ALTER LARGE OBJECT`, `GRANT ON LARGE OBJECT`, `COMMENT ON LARGE OBJECT`, and `REVOKE ON LARGE OBJECT`. +- `lolor.enable()`, `lolor.disable()`, and dropping the extension while lolor is enabled, refuse while any other session is connected to the database, the same rule as `ALTER DATABASE ... RENAME`: a renamed function keeps its OID, so a session that already resolved the `lo_*` functions keeps calling the previous implementation, and the row movement cannot be made atomic for other backends either. The calling session must reconnect after `enable()` or `disable()`. They also refuse to run from anything but a client session, so replicated DDL cannot drive them from an apply worker. +- Objects in lolor storage are rows in ordinary tables and so cannot participate in `pg_shdepend`. `DROP ROLE`, `DROP OWNED BY` and `REASSIGN OWNED BY` do not see them: a role that owns them or appears in their ACL can be dropped without a warning, and `REASSIGN OWNED` / `DROP OWNED` leave them untouched. Before dropping a role, run `REASSIGN OWNED BY` or `DROP OWNED BY` in each database that has lolor, then `DROP ROLE`. Afterwards run `lolor.check_orphans()` in each such database and repair anything it reports with `lolor.fix_orphans(new_owner)`; `lolor.migrate_to_native()` refuses while orphans exist. +- Role OIDs come from a cluster-wide counter that wraps around, so a new role can receive a dropped role's OID and silently become the owner or grantee of that role's orphaned objects. This cannot be detected after the fact, which is another reason to run `lolor.check_orphans()` promptly after dropping roles. - Large object migration is node-local. Native large objects live in `pg_catalog.pg_largeobject`, which is never replicated, so each node holds an independent set and `migrate_from_native()` migrates only the local node's objects; with spock installed, the migration DML runs in repair mode and is not replicated. Run the migration on every node that holds native large objects — for example with `spock.replicate_ddl('SELECT lolor.migrate_from_native()')`, which queues the command so that each node executes it locally. Migrated objects keep their original native OIDs, which are not node-encoded: if different nodes hold different objects under the same OID, the nodes' lolor contents will diverge and later replicated changes to those objects can conflict. Newly created large objects are collision-free, since new OIDs are node-encoded via `lolor.node` and checked against existing rows. diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 37e9b2f..5534e49 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -137,6 +137,13 @@ cd /tmp/lolor-build make USE_PGXS=1 with_llvm=no make USE_PGXS=1 with_llvm=no install +# lolor refuses to load on demand, and the library did not exist when the +# server above was started. Preload it from here on; the later setting wins. +cat >> "$PGDATA/postgresql.conf" <<_EOF_ +shared_preload_libraries = 'spock, lolor' +_EOF_ +pg_ctl -D "$PGDATA" -l /home/pgedge/logfile.log -o "-k /tmp" -m fast -w restart + psql -U admin -d demo -h /tmp -v ON_ERROR_STOP=1 <<_EOF_ create extension lolor; alter system set lolor.node to ${HOSTNAME: -1}; diff --git a/docs/index.md b/docs/index.md index 0232b6f..87479a6 100644 --- a/docs/index.md +++ b/docs/index.md @@ -18,9 +18,10 @@ Use of the `lolor` extension requires Postgres 16 or newer. ## Migrating large objects -Migration from native to lolor storage is **manual** — you decide when to move -existing large objects. Migration back to native is **automatic** — dropping -the extension moves all objects back so nothing is lost. +Migration is a manual step in both directions: you decide when to move +existing large objects into lolor storage, and you move them back before +dropping the extension. Dropping the extension never moves data: it is +refused while any object remains in lolor storage. ### Native to lolor (manual) @@ -39,25 +40,39 @@ when there are no native large objects. This step is intentionally not automatic: it requires superuser privileges and should be performed during a maintenance window. -### Lolor to native (automatic on DROP EXTENSION) +### Lolor to native (before DROP EXTENSION) -Large objects are automatically migrated back to native Postgres storage when the -extension is dropped: +Move the large objects back to native Postgres storage, then drop the +extension: ```sql +SELECT lolor.migrate_to_native(); DROP EXTENSION lolor; ``` -This ensures that no large objects are lost if the extension is removed. You -can also trigger the reverse migration manually while the extension is still -installed: +Both paths preserve OIDs, owners, ACLs, and data. + +An object access hook refuses to remove the extension while it is enabled or +while any object remains in lolor storage, on every path that reaches it +(`DROP EXTENSION`, `DROP SCHEMA lolor CASCADE`, `DROP OWNED BY`). The drop +itself only puts the native `pg_catalog.lo_*` names back. To discard whatever +is in lolor storage instead, a superuser can set `lolor.allow_unsafe_drop = on` +for the session that runs the drop; it skips both checks. The drop renames the +native functions back, so it has the same requirements as `lolor.disable()`: +no other session connected to the database, and a client session. + +`lolor.enable()` and `lolor.disable()` refuse while +any other session is connected to the database, and refuse to run from +anything but a client session. Reconnect client sessions after `enable()` or +`disable()`: libpq caches the large object function OIDs per connection. + +Two helpers find and repair objects whose roles have been dropped: ```sql -SELECT lolor.migrate_to_native(); +SELECT * FROM lolor.check_orphans(); -- objects referring to a dropped role +SELECT lolor.fix_orphans('some_role'); -- reassign them and drop dead ACL entries ``` -Both paths preserve OIDs, owners, ACLs, and data. - When the spock extension is installed, both migration functions run under `spock.repair_mode()`, so the row-shuffling migration DML is **not** replicated to other nodes. This is essential for `migrate_to_native()`: its deletes from @@ -83,4 +98,7 @@ retains the changes and delivers them when replication resumes. - Native Postgres large object functionality cannot be used while you are using the lolor extension. - lolor does not support the following statements: `ALTER LARGE OBJECT`, `GRANT ON LARGE OBJECT`, `COMMENT ON LARGE OBJECT`, and `REVOKE ON LARGE OBJECT`. +- `lolor.enable()`, `lolor.disable()`, and dropping the extension while lolor is enabled, refuse while any other session is connected to the database, the same rule as `ALTER DATABASE ... RENAME`: a renamed function keeps its OID, so a session that already resolved the `lo_*` functions keeps calling the previous implementation, and the row movement cannot be made atomic for other backends either. The calling session must reconnect after `enable()` or `disable()`. They also refuse to run from anything but a client session, so replicated DDL cannot drive them from an apply worker. +- Objects in lolor storage are rows in ordinary tables and so cannot participate in `pg_shdepend`. `DROP ROLE`, `DROP OWNED BY` and `REASSIGN OWNED BY` do not see them: a role that owns them or appears in their ACL can be dropped without a warning, and `REASSIGN OWNED` / `DROP OWNED` leave them untouched. Before dropping a role, run `REASSIGN OWNED BY` or `DROP OWNED BY` in each database that has lolor, then `DROP ROLE`. Afterwards run `lolor.check_orphans()` in each such database and repair anything it reports with `lolor.fix_orphans(new_owner)`; `lolor.migrate_to_native()` refuses while orphans exist. +- Role OIDs come from a cluster-wide counter that wraps around, so a new role can receive a dropped role's OID and silently become the owner or grantee of that role's orphaned objects. This cannot be detected after the fact, which is another reason to run `lolor.check_orphans()` promptly after dropping roles. - Large object migration is node-local. Native large objects live in `pg_catalog.pg_largeobject`, which is never replicated, so each node holds an independent set and `migrate_from_native()` migrates only the local node's objects; with spock installed, the migration DML runs in repair mode and is not replicated. Run the migration on every node that holds native large objects — for example with `spock.replicate_ddl('SELECT lolor.migrate_from_native()')`, which queues the command so that each node executes it locally. Migrated objects keep their original native OIDs, which are not node-encoded: if different nodes hold different objects under the same OID, the nodes' lolor contents will diverge and later replicated changes to those objects can conflict. Newly created large objects are collision-free, since new OIDs are node-encoded via `lolor.node` and checked against existing rows. diff --git a/docs/install_configure.md b/docs/install_configure.md index 6c50a0c..469aca4 100644 --- a/docs/install_configure.md +++ b/docs/install_configure.md @@ -14,7 +14,13 @@ make USE_PGXS=1 make USE_PGXS=1 install ``` -After the lolor extension is installed, connect to your Postgres database and create the extension with the command: +lolor must be loaded at server start. Add it to `shared_preload_libraries` in `postgresql.conf` and restart; `CREATE EXTENSION lolor`, `LOAD 'lolor'` and any call that reaches one of lolor's functions fail with `lolor must be loaded via "shared_preload_libraries"` when the library was loaded on demand; the native `pg_catalog.lo_*` functions are not affected while lolor is not installed or is disabled. The guard that protects large objects when the extension is dropped is an object access hook, which is only in place in every backend when the library is preloaded. + +``` +shared_preload_libraries = 'lolor' +``` + +Then connect to your Postgres database and create the extension with the command: ``` CREATE EXTENSION lolor; @@ -22,7 +28,7 @@ CREATE EXTENSION lolor; ## Configuring lolor -You must set the `lolor.node` parameter on each node in your replication cluster before using the extension. The value can be from 1 to 2^28; the value is used to help in generation of new large object OID. +You must set the `lolor.node` parameter on each node in your replication cluster before using the extension. The value can be from 1 to 15 (0 means unset); it is encoded in the four low bits of every large object OID lolor generates, so each node must use a different value. ``` lolor.node = 1 diff --git a/docs/lolor_release_notes.md b/docs/lolor_release_notes.md index 0e9d82b..c2421e9 100644 --- a/docs/lolor_release_notes.md +++ b/docs/lolor_release_notes.md @@ -5,15 +5,29 @@ * Add bidirectional large object migration between native PostgreSQL and lolor storage: * `lolor.migrate_from_native()` migrates existing native large objects into lolor storage. This is a manual step (run after `CREATE EXTENSION lolor`) and requires superuser privileges. * `lolor.migrate_to_native()` migrates lolor large objects back to native storage. - * Reverse migration runs automatically on `DROP EXTENSION lolor`, so large objects are never lost when the extension is removed. + * Dropping the extension no longer migrates anything: it is refused while any object remains in lolor storage (see the object access hook below). Run `lolor.migrate_to_native()` first. * With the spock extension installed, both migration functions run under `spock.repair_mode()` so migration is replication-safe. If a logical replication slot that spock cannot suppress is present — a non-spock slot, or any logical slot when spock is absent — the functions refuse to migrate rather than risk losing objects. * Migration is node-local: native large objects are never replicated, so each node holds an independent set and `migrate_from_native()` migrates only the local node's objects. Run it on every node that holds native large objects, for example via `spock.replicate_ddl('SELECT lolor.migrate_from_native()')`. Migrated objects keep their original native OIDs, which are not node-encoded and can collide across nodes if different nodes hold different objects under the same OID; newly created large objects are collision-free, since new OIDs are node-encoded via `lolor.node` and checked against existing rows. -* **Security fix: `lo_import()` and `lo_export()` were executable by any database user.** These read and write files on the server as the account PostgreSQL runs under, so core revokes `EXECUTE` on them from `PUBLIC`. lolor replaces them by renaming the originals to `*_orig`; an ACL belongs to a function rather than a name, so the restriction stayed on the parked original while each replacement got the default `EXECUTE TO PUBLIC`. Any user could read an arbitrary server file with `lo_import()` or overwrite one with `lo_export()`. The replacements are now locked down at install time, and upgrading revokes the privilege on existing installations in either state. Versions 1.0 through 1.2.2 are affected. +* New helpers `lolor.check_orphans()` and `lolor.fix_orphans(new_owner)` for objects whose owner, grantee or grantor has been dropped. Objects in lolor storage cannot participate in `pg_shdepend`, so `DROP ROLE` does not notice them; `check_orphans()` lists them and which reference is dangling, and `fix_orphans()` reassigns dead owners, replacing them in the ACL as `REASSIGN OWNED` would so that grants they made survive, and drops entries that still name a missing role. +* **Security fix: `lo_import()` and `lo_export()` were executable by any database user.** These read and write files on the server as the account PostgreSQL runs under, so core revokes `EXECUTE` on them from `PUBLIC`. lolor replaces them by renaming the originals to `*_orig`; an ACL belongs to a function rather than a name, so the restriction stayed on the parked original while each replacement got the default `EXECUTE TO PUBLIC`. Any user could read an arbitrary server file with `lo_import()` or overwrite one with `lo_export()`. The replacements are now locked down at install time, and upgrading revokes the privilege on existing installations in either state. Versions 1.0 through 1.2.2 are affected. A pre-release build installed from the development branch under the 1.3.0 number before this change does not pick the fix up through `ALTER EXTENSION lolor UPDATE`, which does nothing when the version is unchanged; reinstall the extension, or run the three `REVOKE` statements from `lolor--1.2.2--1.3.0.sql` by hand. * The extension is no longer marked `trusted`. Installing lolor renames functions in `pg_catalog` for the whole database, which a non-superuser should not be able to do. +* **Fixed `DROP ROLE` failing with "unrecognized object class".** Creating a large object recorded a `pg_shdepend` row whose `classId` was the OID of `lolor.pg_largeobject`, an ordinary table rather than a catalog. Any role that had created a large object became undroppable, and the rows were never cleaned up because `inv_drop()` deleted with `PERFORM_DELETION_SKIP_ORIGINAL`. lolor no longer records these rows, and the upgrade removes the ones already present. + * The cleanup is per database, so `ALTER EXTENSION lolor UPDATE` must be run in every database that has lolor. Until it is, `DROP ROLE` anywhere in the cluster keeps reporting "owner of objects in database X". + * A database that ran `DROP EXTENSION lolor` on an earlier version still has the rows, now pointing at a table that no longer exists, and no upgrade script will run there. Clean it by hand as superuser in that database: find the stale class OIDs with `SELECT DISTINCT classid FROM pg_shdepend WHERE dbid = (SELECT oid FROM pg_database WHERE datname = current_database()) AND classid NOT IN (SELECT oid FROM pg_class)`, then `DELETE FROM pg_shdepend WHERE dbid = AND classid IN ()`. +* **lolor must now be listed in `shared_preload_libraries`.** Loading it on demand is refused: `CREATE EXTENSION lolor`, `LOAD 'lolor'` and any call that reaches one of lolor's functions fail with `lolor must be loaded via "shared_preload_libraries"`; the native `pg_catalog.lo_*` functions are not affected while lolor is not installed or is disabled. The drop guard below is an object access hook, which is only present in every backend and worker when the library is preloaded, and a missing or broken library now fails at server start rather than at the first `lo_open()`. Add `lolor` to `shared_preload_libraries` and restart before installing the new library; a server that installs it first sees the error from every new session until it is restarted. +* Removing the extension is guarded by an object access hook. `DROP EXTENSION`, `DROP SCHEMA lolor CASCADE`, `DROP OWNED BY` and any other path that reaches the extension are refused while lolor is enabled or while any object remains in lolor storage, checked in the catalog at the moment of deletion from whichever backend performs it. **The drop no longer migrates the objects out itself**: run `lolor.migrate_to_native()` first. The event trigger only puts the native `pg_catalog.lo_*` names back. The new superuser setting `lolor.allow_unsafe_drop` skips both checks and discards the contents of lolor storage. Because the drop renames the native functions back, it has the same requirements as `lolor.disable()`: no other session connected to the database, and a client session. +* `lolor.enable()` and `lolor.disable()` refuse to run from anything but a client session, so replicated DDL cannot execute them inside an apply worker, where a refusal would leave that node's replication retrying forever. +* `lolor.enable()`, `lolor.disable()` and `lolor.is_enabled()` now probe exact function signatures in `pg_catalog`. They previously matched on `proname` across every schema, so any user with `CREATE` on any schema could define a function named `lolor_lo_open` and wedge lolor into a permanent "inconsistent state" that also blocked `DROP EXTENSION`. Both now emit a notice that client sessions must reconnect, since libpq caches the large object function OIDs per connection, and refuse while any other session is connected to the database, since a renamed function keeps its OID and other sessions cannot be made to see the switch. * Fixed the `lolor.node` upper bound. The GUC accepted 0..16 while a generated OID reserves four bits for the node id, so node 16 did not fit: the node field of every OID it generated read back as 0. The bound is now derived from the encoding (`LOLOR_MAX_NODE_ID`), giving 0..15. **If you have `lolor.node = 16` configured**, the server still starts but logs `16 is outside the valid range for parameter "lolor.node" (0 .. 15)` and falls back to 0, which is the node id it was effectively using already. Set it to a value in 0..15, and check for OID collisions against whichever node is genuinely 0. -* Expanded test coverage: TAP tests for dump/restore, streaming and logical replication, and standby promotion; regression tests for `lo_lseek`, `lo_tell`, and `lo_truncate`. +* Expanded test coverage: TAP tests for dump/restore, streaming and logical replication, standby promotion and the drop paths; regression tests for `lo_lseek`, `lo_tell`, `lo_truncate`, permission enforcement and the orphan helpers. The regression suite now runs in a temporary instance that preloads lolor (`make installcheck`), since the preload requirement means it cannot run against an arbitrary server. * Security hardening: addressed Codacy/Flawfinder warnings. +## lolor 1.2.3 + +* Add support for PostgreSQL 19 +* Add in-repo release workflow and RPM/DEB packaging +* Improve CI: build the test cluster from source, add PostgreSQL 19 to the test matrix, and bound test run time + ## lolor 1.2.2 * Fix lolor upgrades diff --git a/docs/pg_upgrade_with_lolor.md b/docs/pg_upgrade_with_lolor.md index b816190..a950b9e 100644 --- a/docs/pg_upgrade_with_lolor.md +++ b/docs/pg_upgrade_with_lolor.md @@ -4,6 +4,8 @@ The `pg_upgrade` utility is used for upgrading Postgres versions. You can use `pg_upgrade` with the `lolor` extension installed provided that you are using lolor version 1.2.2 or later. If you are using an older version of lolor, you will need to upgrade the extension first. +Both clusters must list `lolor` in `shared_preload_libraries`; without it the new cluster cannot load lolor's library, so restoring the extension fails. + Before running `pg_upgrade`, you must disable `lolor`. After executing pg_upgrade, you can enable lolor. Use `psql` or another client to disable lolor: ``` @@ -25,3 +27,9 @@ Then, use psql to enable `lolor`: ``` db1_18=# SELECT lolor.enable(); ``` + +Reconnect any client sessions afterwards. `lolor.enable()` and +`lolor.disable()` change which function OID owns each `pg_catalog.lo_*` name, +and libpq resolves those OIDs once per connection and caches them for the life +of the connection. A session that used a large object before the switch would +otherwise keep calling the previous implementation. diff --git a/expected/lolor.out b/expected/lolor.out index 674ec34..7cfc278 100644 --- a/expected/lolor.out +++ b/expected/lolor.out @@ -216,8 +216,15 @@ SELECT lo_close(:fd); (1 row) END; -DROP EXTENSION lolor; +SELECT lolor.migrate_to_native(); NOTICE: migrated 4 large object(s) from lolor to native storage + migrate_to_native +------------------- + 4 +(1 row) + +DROP EXTENSION lolor; +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs -- Check extension upgrade CREATE EXTENSION lolor VERSION '1.0'; SELECT lo_creat(-1) AS loid \gset @@ -241,7 +248,21 @@ SELECT convert_from(loread(:fd, 1024), 'UTF8'); (1 row) END; +-- 1.2.3 is the released version; 1.3.0 is reached from it +ALTER EXTENSION lolor UPDATE TO '1.2.3'; +SELECT extversion FROM pg_extension WHERE extname = 'lolor'; + extversion +------------ + 1.2.3 +(1 row) + ALTER EXTENSION lolor UPDATE TO '1.3.0'; +SELECT extversion FROM pg_extension WHERE extname = 'lolor'; + extversion +------------ + 1.3.0 +(1 row) + -- Verify migration functions are available after upgrade SELECT lolor.migrate_to_native(); -- One LO object has been created before LOLOR NOTICE: migrated 1 large object(s) from lolor to native storage @@ -276,7 +297,7 @@ NOTICE: migrated 5 large object(s) (5 data page(s)) from native to lolor storag -- Basic checks for enable/disable routines. -- SELECT lolor.enable(); -- ERROR -NOTICE: lolor still not disabled +NOTICE: lolor is already enabled enable -------- f @@ -289,6 +310,7 @@ SELECT lo_from_bytea(1, 'Example large object stored in lolor LO storage'); (1 row) SELECT lolor.disable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs disable --------- t @@ -304,6 +326,7 @@ SELECT lo_from_bytea(2, 'Example large object stored in built-in LO storage'); -- We should see the object SELECT lolor.enable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs enable -------- t @@ -322,12 +345,14 @@ SELECT convert_from(loread(:fd, 1024), 'UTF8'); -- OK, see the object END; -- To be sure that the behaviour is repeatable SELECT lolor.disable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs disable --------- t (1 row) SELECT lolor.enable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs enable -------- t @@ -335,46 +360,71 @@ SELECT lolor.enable(); -- Check that no tails existing after the extension drop in both enabled and -- disabled states. -DROP EXTENSION lolor; +SELECT lolor.migrate_to_native(); NOTICE: migrated 6 large object(s) from lolor to native storage + migrate_to_native +------------------- + 6 +(1 row) + +DROP EXTENSION lolor; +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs SELECT oid, proname FROM pg_proc WHERE proname IN ('lo_open_orig', 'lolor_lo_open'); oid | proname -----+--------- (0 rows) --- Check: we can't just delete LOLOR without LO migration in disabled mode. --- XXX: should we introduce a 'forced' flag to allow this? +-- DROP EXTENSION while lolor is disabled. The drop is refused while the +-- object is still in lolor storage, whichever state lolor is in. Move the +-- object out first -- the migration runs through the renamed _orig functions +-- and so needs lolor enabled -- and the drop goes through. CREATE EXTENSION lolor; +SELECT lo_from_bytea(0, 'stored before disabling') AS disabled_drop_oid \gset SELECT lolor.disable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs disable --------- t (1 row) DROP EXTENSION lolor; -ERROR: lolor must be enabled before migration to native -SELECT extname FROM pg_extension; -- lolor is here - extname ---------- - plpgsql - lolor -(2 rows) - +ERROR: cannot drop lolor storage while it holds 1 large object SELECT lolor.enable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs enable -------- t (1 row) +SELECT lolor.migrate_to_native(); +NOTICE: migrated 1 large object(s) from lolor to native storage + migrate_to_native +------------------- + 1 +(1 row) + DROP EXTENSION lolor; -NOTICE: no lolor large objects to migrate +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs SELECT extname FROM pg_extension; -- check lolor removal extname --------- plpgsql (1 row) +-- The object was migrated to native storage, not dropped with lolor's tables +SELECT convert_from(lo_get(:disabled_drop_oid), 'UTF8') AS survived_disabled_drop; + survived_disabled_drop +------------------------- + stored before disabling +(1 row) + +SELECT lo_unlink(:disabled_drop_oid); + lo_unlink +----------- + 1 +(1 row) + -- -- Migration tests: migrate_from_native / migrate_to_native / DROP EXTENSION -- @@ -444,9 +494,16 @@ SELECT lo_close(:fd); END; -- Create an additional LO directly in lolor storage SELECT lo_from_bytea(0, 'Created directly in lolor') AS lolor_direct_oid \gset --- Reverse migration via DROP EXTENSION -DROP EXTENSION lolor; +-- Reverse migration, then DROP EXTENSION +SELECT lolor.migrate_to_native(); NOTICE: migrated 10 large object(s) from lolor to native storage + migrate_to_native +------------------- + 10 +(1 row) + +DROP EXTENSION lolor; +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs SELECT count(*) AS native_after_drop FROM pg_catalog.pg_largeobject_metadata; native_after_drop ------------------- @@ -501,8 +558,15 @@ NOTICE: migrated 7 large object(s) (7 data page(s)) from native to lolor storag 7 (1 row) -DROP EXTENSION lolor; +SELECT lolor.migrate_to_native(); NOTICE: migrated 7 large object(s) from lolor to native storage + migrate_to_native +------------------- + 7 +(1 row) + +DROP EXTENSION lolor; +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs -- -- Manual migrate_to_native (not via DROP EXTENSION) -- @@ -531,6 +595,7 @@ SELECT count(*) AS lolor_after_manual FROM lolor.pg_largeobject_metadata; BEGIN; -- Disable lolor to read from native storage directly SELECT lolor.disable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs disable --------- t @@ -551,13 +616,21 @@ SELECT lo_unlink(:'manual_oid'::oid); (1 row) SELECT lolor.enable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs enable -------- t (1 row) -DROP EXTENSION lolor; +SELECT lolor.migrate_to_native(); NOTICE: no lolor large objects to migrate + migrate_to_native +------------------- + 0 +(1 row) + +DROP EXTENSION lolor; +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs -- -- OID conflict detection -- @@ -572,8 +645,15 @@ SELECT lolor.migrate_from_native(); ERROR: OID conflict: some native large objects already exist in lolor storage -- Cleanup: remove the conflicting row and drop cleanly DELETE FROM lolor.pg_largeobject_metadata WHERE oid = :'conflict_oid'; -DROP EXTENSION lolor; +SELECT lolor.migrate_to_native(); NOTICE: no lolor large objects to migrate + migrate_to_native +------------------- + 0 +(1 row) + +DROP EXTENSION lolor; +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs SELECT lo_unlink(:'conflict_oid'::oid); lo_unlink ----------- @@ -585,6 +665,7 @@ CREATE EXTENSION lolor; SELECT lo_from_bytea(0, 'Lolor side object') AS conflict_oid2 \gset -- Disable lolor to create a native LO with the same OID SELECT lolor.disable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs disable --------- t @@ -599,6 +680,7 @@ SELECT :'created_oid' = :'conflict_oid2' AS oid_matches; (1 row) SELECT lolor.enable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs enable -------- t @@ -609,6 +691,7 @@ SELECT lolor.migrate_to_native(); ERROR: OID conflict: some lolor large objects already exist in native storage -- Cleanup: remove the native duplicate, then drop cleanly SELECT lolor.disable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs disable --------- t @@ -621,18 +704,28 @@ SELECT lo_unlink(:'conflict_oid2'::oid); (1 row) SELECT lolor.enable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs enable -------- t (1 row) -DROP EXTENSION lolor; +SELECT lolor.migrate_to_native(); NOTICE: migrated 1 large object(s) from lolor to native storage --- DROP EXTENSION should be rejected when migrate_to_native has OID conflict + migrate_to_native +------------------- + 1 +(1 row) + +DROP EXTENSION lolor; +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs +-- A conflicting native object blocks the removal: the drop is refused while +-- objects remain in lolor storage, and they cannot be moved out while the +-- native duplicate exists CREATE EXTENSION lolor; SELECT lo_from_bytea(0, 'Drop conflict test') AS drop_conflict_oid \gset --- Create a native LO with the same OID to force conflict at DROP time SELECT lolor.disable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs disable --------- t @@ -646,13 +739,16 @@ SELECT lo_create(:'drop_conflict_oid') = :'drop_conflict_oid'::oid AS native_oid (1 row) SELECT lolor.enable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs enable -------- t (1 row) --- DROP EXTENSION should ERROR to prevent data loss DROP EXTENSION lolor; +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs +ERROR: cannot drop lolor storage while it holds 1 large object +SELECT lolor.migrate_to_native(); ERROR: OID conflict: some lolor large objects already exist in native storage -- Extension should still be installed SELECT extname FROM pg_extension WHERE extname = 'lolor'; @@ -670,6 +766,7 @@ SELECT count(*) FROM lolor.pg_largeobject; -- Resolve the conflict: remove the native duplicate, then retry SELECT lolor.disable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs disable --------- t @@ -682,14 +779,100 @@ SELECT lo_unlink(:'drop_conflict_oid'::oid); (1 row) SELECT lolor.enable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs enable -------- t (1 row) --- Now DROP should succeed +-- Now the migration and the drop go through +SELECT lolor.migrate_to_native(); +NOTICE: migrated 1 large object(s) from lolor to native storage + migrate_to_native +------------------- + 1 +(1 row) + +DROP EXTENSION lolor; +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs +-- +-- The drop guard is an object access hook, so it holds even when the event +-- trigger that restores the function names does not run. +-- +CREATE EXTENSION lolor; +ALTER EVENT TRIGGER lo_on_drop_extension DISABLE; +-- Enabled with nothing stored: refused because the native functions are +-- still parked under their _orig names +DROP EXTENSION lolor; +ERROR: cannot drop extension "lolor" while it is enabled +-- Disabled with an object stored: refused on every path that reaches the +-- extension +SELECT lo_from_bytea(0, 'guarded by the drop hook') AS guarded_oid \gset +SELECT lolor.disable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs + disable +--------- + t +(1 row) + DROP EXTENSION lolor; +ERROR: cannot drop lolor storage while it holds 1 large object +DROP SCHEMA lolor CASCADE; +NOTICE: drop cascades to extension lolor +ERROR: cannot drop lolor storage while it holds 1 large object +SELECT count(*) AS extension_still_installed FROM pg_extension WHERE extname = 'lolor'; + extension_still_installed +--------------------------- + 1 +(1 row) + +SELECT count(*) AS still_in_lolor_storage FROM lolor.pg_largeobject_metadata; + still_in_lolor_storage +------------------------ + 1 +(1 row) + +-- The escape hatch is a superuser setting and discards the objects +BEGIN; +SET LOCAL lolor.allow_unsafe_drop = on; +DROP EXTENSION lolor; +SELECT count(*) AS extension_gone FROM pg_extension WHERE extname = 'lolor'; + extension_gone +---------------- + 0 +(1 row) + +ROLLBACK; +-- Back on the supported path: migrate out by hand, then drop +ALTER EVENT TRIGGER lo_on_drop_extension ENABLE ALWAYS; +SELECT lolor.enable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs + enable +-------- + t +(1 row) + +SELECT lolor.migrate_to_native(); NOTICE: migrated 1 large object(s) from lolor to native storage + migrate_to_native +------------------- + 1 +(1 row) + +DROP EXTENSION lolor; +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs +SELECT convert_from(lo_get(:guarded_oid), 'UTF8') AS migrated_out; + migrated_out +-------------------------- + guarded by the drop hook +(1 row) + +SELECT lo_unlink(:guarded_oid); + lo_unlink +----------- + 1 +(1 row) + -- -- lo_import() and lo_export() read and write files on the server, so core -- revokes EXECUTE on them from PUBLIC. lolor replaces them by renaming the @@ -718,8 +901,15 @@ ORDER BY 1; lo_import(text, oid) | f | f (3 rows) -DROP EXTENSION lolor; +SELECT lolor.migrate_to_native(); NOTICE: no lolor large objects to migrate + migrate_to_native +------------------- + 0 +(1 row) + +DROP EXTENSION lolor; +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs -- -- lolor.node is bounded by the OID encoding, not by an independently written -- constant: the low LOLOR_NODEID_BITS of a generated OID carry the node id, so @@ -730,6 +920,283 @@ ERROR: 16 is outside the valid range for parameter "lolor.node" (0 .. 15) SET lolor.node = 15; SET lolor.node = 1; -- +-- Permission enforcement. +-- +-- Objects in lolor storage have no catalog entry, so lolor cannot use the +-- syscache-backed owner and ACL checks and reimplements them against its own +-- tables. Exercise that path rather than assuming it matches core. +-- +CREATE EXTENSION lolor; +CREATE ROLE lolor_alice; +CREATE ROLE lolor_bob; +-- Large object error messages quote the OID, which is generated and so +-- differs between runs. Report the message with digits masked instead. +CREATE FUNCTION lolor_expect_error(cmd text) RETURNS text AS $$ +BEGIN + EXECUTE cmd; + RETURN 'unexpectedly succeeded'; +EXCEPTION WHEN OTHERS THEN + RETURN regexp_replace(SQLERRM, '[0-9]+', 'NNN', 'g'); +END +$$ LANGUAGE plpgsql; +SET ROLE lolor_alice; +SELECT lo_from_bytea(0, 'alice private data') AS alice_oid \gset +SELECT convert_from(lo_get(:alice_oid), 'UTF8') AS owner_can_read; + owner_can_read +-------------------- + alice private data +(1 row) + +RESET ROLE; +-- A different role gets nothing without a grant. +SET ROLE lolor_bob; +SELECT lolor_expect_error(format('SELECT lo_get(%s)', :alice_oid)) AS read_denied; + read_denied +---------------------------------------- + permission denied for large object NNN +(1 row) + +SELECT lolor_expect_error(format('SELECT lo_open(%s, 262144)', :alice_oid)) AS open_denied; + open_denied +---------------------------------------- + permission denied for large object NNN +(1 row) + +SELECT lolor_expect_error(format('SELECT lo_put(%s, 0, ''x'')', :alice_oid)) AS write_denied; + write_denied +---------------------------------------- + permission denied for large object NNN +(1 row) + +SELECT lolor_expect_error(format('SELECT lo_unlink(%s)', :alice_oid)) AS unlink_denied; + unlink_denied +----------------------------------- + must be owner of large object NNN +(1 row) + +RESET ROLE; +-- The superuser bypasses the check, as in core. +SELECT convert_from(lo_get(:alice_oid), 'UTF8') AS superuser_can_read; + superuser_can_read +-------------------- + alice private data +(1 row) + +-- GRANT/ALTER on a large object act on pg_largeobject_metadata, where an +-- object in lolor storage has no row. This limitation is documented; assert +-- it so that a change in behaviour is noticed. +SELECT lolor_expect_error( + format('GRANT SELECT ON LARGE OBJECT %s TO lolor_bob', :alice_oid)) AS grant_unsupported; + grant_unsupported +--------------------------------- + large object NNN does not exist +(1 row) + +SELECT lolor_expect_error( + format('ALTER LARGE OBJECT %s OWNER TO lolor_bob', :alice_oid)) AS alter_unsupported; + alter_unsupported +--------------------------------- + large object NNN does not exist +(1 row) + +SELECT lo_unlink(:alice_oid); + lo_unlink +----------- + 1 +(1 row) + +-- +-- Objects in lolor storage are rows in ordinary tables and cannot participate +-- in pg_shdepend, so DROP ROLE does not notice that a role still owns one. +-- lolor.check_orphans() exists to make the consequence findable. +-- +SET ROLE lolor_alice; +SELECT lo_from_bytea(0, 'owned by a role about to vanish') AS orphan_oid \gset +RESET ROLE; +SELECT count(*) AS orphans_before FROM lolor.check_orphans(); + orphans_before +---------------- + 0 +(1 row) + +DROP ROLE lolor_alice; +SELECT count(*) AS orphans_after FROM lolor.check_orphans(); + orphans_after +--------------- + 1 +(1 row) + +SELECT lo_unlink(:orphan_oid); + lo_unlink +----------- + 1 +(1 row) + +SELECT count(*) AS orphans_cleared FROM lolor.check_orphans(); + orphans_cleared +----------------- + 0 +(1 row) + +-- +-- fix_orphans() must behave like REASSIGN OWNED: grants the dead owner made +-- survive with the new owner as grantor, an entry the new owner already held +-- merges into its owner entry, and only entries that still name a missing +-- role are dropped. ACLs can only be set in native storage, so build them +-- there and migrate in. +-- +CREATE ROLE lolor_dead_owner; +CREATE ROLE lolor_carol; +CREATE ROLE lolor_dave; +SELECT lolor.disable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs + disable +--------- + t +(1 row) + +SET ROLE lolor_dead_owner; +SELECT lo_from_bytea(0, 'granted to bob') AS granted_oid \gset +GRANT SELECT ON LARGE OBJECT :granted_oid TO lolor_bob; +SELECT lo_from_bytea(0, 'grant chain') AS chain_oid \gset +GRANT SELECT ON LARGE OBJECT :chain_oid TO lolor_dave WITH GRANT OPTION; +SELECT lo_from_bytea(0, 'new owner already a grantee') AS merge_oid \gset +GRANT SELECT ON LARGE OBJECT :merge_oid TO lolor_carol WITH GRANT OPTION; +SELECT lo_from_bytea(0, 'mixed grant options') AS mixed_oid \gset +GRANT SELECT ON LARGE OBJECT :mixed_oid TO lolor_bob WITH GRANT OPTION; +GRANT UPDATE ON LARGE OBJECT :mixed_oid TO lolor_bob; +RESET ROLE; +SET ROLE lolor_dave; +GRANT SELECT ON LARGE OBJECT :chain_oid TO lolor_bob; +RESET ROLE; +SELECT lolor.enable(); +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs + enable +-------- + t +(1 row) + +SELECT lolor.migrate_from_native(); +NOTICE: migrated 13 large object(s) (13 data page(s)) from native to lolor storage + migrate_from_native +--------------------- + 13 +(1 row) + +DROP ROLE lolor_dead_owner; +SELECT role_kind, count(*) FROM lolor.check_orphans() GROUP BY 1 ORDER BY 1; + role_kind | count +-----------+------- + grantee | 4 + grantor | 4 + owner | 4 +(3 rows) + +SELECT lolor.fix_orphans('lolor_carol') AS objects_repaired; + objects_repaired +------------------ + 4 +(1 row) + +SELECT count(*) AS orphans_left FROM lolor.check_orphans(); + orphans_left +-------------- + 0 +(1 row) + +-- Role OIDs vary per run, so compare the rebuilt ACLs by name +SELECT CASE m.oid WHEN :granted_oid THEN 'granted' WHEN :chain_oid THEN 'chain' + WHEN :merge_oid THEN 'merge' ELSE 'mixed' END AS obj, + pg_get_userbyid(m.lomowner) AS owner, + a.grantee::regrole::text AS grantee, a.grantor::regrole::text AS grantor, + a.privilege_type, a.is_grantable + FROM lolor.pg_largeobject_metadata m, aclexplode(m.lomacl) a + WHERE m.oid IN (:granted_oid, :chain_oid, :merge_oid, :mixed_oid) + ORDER BY 1, 3, 4, 5; + obj | owner | grantee | grantor | privilege_type | is_grantable +---------+-------------+-------------+-------------+----------------+-------------- + chain | lolor_carol | lolor_bob | lolor_dave | SELECT | f + chain | lolor_carol | lolor_carol | lolor_carol | SELECT | f + chain | lolor_carol | lolor_carol | lolor_carol | UPDATE | f + chain | lolor_carol | lolor_dave | lolor_carol | SELECT | t + granted | lolor_carol | lolor_bob | lolor_carol | SELECT | f + granted | lolor_carol | lolor_carol | lolor_carol | SELECT | f + granted | lolor_carol | lolor_carol | lolor_carol | UPDATE | f + merge | lolor_carol | lolor_carol | lolor_carol | SELECT | t + merge | lolor_carol | lolor_carol | lolor_carol | UPDATE | f + mixed | lolor_carol | lolor_bob | lolor_carol | SELECT | t + mixed | lolor_carol | lolor_bob | lolor_carol | UPDATE | f + mixed | lolor_carol | lolor_carol | lolor_carol | SELECT | f + mixed | lolor_carol | lolor_carol | lolor_carol | UPDATE | f +(13 rows) + +-- One entry per (grantee, grantor), as GRANT and REVOKE expect: a grant +-- option that differs between privileges lives inside the entry +SELECT count(*) AS objects_with_duplicate_entries + FROM lolor.pg_largeobject_metadata m + WHERE m.oid IN (:granted_oid, :chain_oid, :merge_oid, :mixed_oid) + AND cardinality(m.lomacl) <> (SELECT count(DISTINCT (a.grantee, a.grantor)) + FROM aclexplode(m.lomacl) a); + objects_with_duplicate_entries +-------------------------------- + 0 +(1 row) + +-- The grantee kept access and the new owner has it +SET ROLE lolor_bob; +SELECT convert_from(lo_get(:granted_oid), 'UTF8') AS bob_still_reads; + bob_still_reads +----------------- + granted to bob +(1 row) + +RESET ROLE; +SET ROLE lolor_carol; +SELECT convert_from(lo_get(:chain_oid), 'UTF8') AS new_owner_reads; + new_owner_reads +----------------- + grant chain +(1 row) + +RESET ROLE; +SELECT lo_unlink(:granted_oid); + lo_unlink +----------- + 1 +(1 row) + +SELECT lo_unlink(:chain_oid); + lo_unlink +----------- + 1 +(1 row) + +SELECT lo_unlink(:merge_oid); + lo_unlink +----------- + 1 +(1 row) + +SELECT lo_unlink(:mixed_oid); + lo_unlink +----------- + 1 +(1 row) + +DROP ROLE lolor_carol; +DROP ROLE lolor_dave; +DROP ROLE lolor_bob; +DROP FUNCTION lolor_expect_error(text); +SELECT lolor.migrate_to_native(); +NOTICE: migrated 9 large object(s) from lolor to native storage + migrate_to_native +------------------- + 9 +(1 row) + +DROP EXTENSION lolor; +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs +-- -- 64-bit interface and page-boundary I/O. lo_put(), lo_tell64() and -- lo_truncate64() had no coverage. -- @@ -1050,9 +1517,147 @@ SELECT lo_unlink(:multi_oid); -- SELECT lo_get(0); ERROR: large object 0 does not exist +SELECT lo_unlink(0); +ERROR: large object 0 does not exist BEGIN; SELECT lo_open(0, 262144); ERROR: large object 0 does not exist ROLLBACK; +SELECT lolor.migrate_to_native(); +NOTICE: no lolor large objects to migrate + migrate_to_native +------------------- + 0 +(1 row) + DROP EXTENSION lolor; +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs +-- +-- An unprivileged user must not be able to wedge lolor by squatting the names +-- the state probes look for. Matching on proname alone would make +-- is_enabled() raise "inconsistent state" and block DROP EXTENSION. +-- +CREATE EXTENSION lolor; +CREATE ROLE lolor_squatter; +GRANT CREATE ON SCHEMA public TO lolor_squatter; +SET ROLE lolor_squatter; +CREATE FUNCTION public.lolor_lo_open(oid, int4) RETURNS int4 + AS 'SELECT 1' LANGUAGE sql; +CREATE FUNCTION public.lo_close_orig(int4) RETURNS int4 + AS 'SELECT 1' LANGUAGE sql; +RESET ROLE; +SELECT lolor.is_enabled() AS unaffected_by_squatting; + unaffected_by_squatting +------------------------- + t +(1 row) + +DROP FUNCTION public.lolor_lo_open(oid, int4); +DROP FUNCTION public.lo_close_orig(int4); +REVOKE CREATE ON SCHEMA public FROM lolor_squatter; +DROP ROLE lolor_squatter; +SELECT lolor.migrate_to_native(); NOTICE: no lolor large objects to migrate + migrate_to_native +------------------- + 0 +(1 row) + +DROP EXTENSION lolor; +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs +-- +-- DROP SCHEMA lolor CASCADE reaches the extension by dependency cascade rather +-- than as DROP EXTENSION. It is refused while objects remain in lolor +-- storage, and once they are migrated out the cleanup trigger must still run, +-- or pg_catalog is left without a working lo_open(). +-- +CREATE EXTENSION lolor; +SELECT lo_from_bytea(0, 'rescued from drop schema') AS rescued_oid \gset +DROP SCHEMA lolor CASCADE; +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs +NOTICE: drop cascades to extension lolor +ERROR: cannot drop lolor storage while it holds 1 large object +SELECT lolor.migrate_to_native(); +NOTICE: migrated 1 large object(s) from lolor to native storage + migrate_to_native +------------------- + 1 +(1 row) + +DROP SCHEMA lolor CASCADE; +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs +NOTICE: drop cascades to extension lolor +SELECT count(*) AS ext_left FROM pg_extension WHERE extname = 'lolor'; + ext_left +---------- + 0 +(1 row) + +SELECT to_regprocedure('pg_catalog.lo_open(oid,int4)') IS NOT NULL AS lo_open_restored; + lo_open_restored +------------------ + t +(1 row) + +SELECT to_regprocedure('pg_catalog.lo_open_orig(oid,int4)') IS NULL AS no_orig_left; + no_orig_left +-------------- + t +(1 row) + +SELECT convert_from(lo_get(:rescued_oid), 'UTF8') AS rescued_content; + rescued_content +-------------------------- + rescued from drop schema +(1 row) + +SELECT lo_unlink(:rescued_oid); + lo_unlink +----------- + 1 +(1 row) + +-- +-- DROP SCHEMA without CASCADE is RESTRICT and cannot remove a schema that +-- still holds the extension's tables. The cleanup must not run for a command +-- that is going to be rejected. +-- +CREATE EXTENSION lolor; +SELECT lo_from_bytea(0, 'still here afterwards') AS kept_oid \gset +DROP SCHEMA lolor; +ERROR: cannot drop schema lolor because other objects depend on it +SELECT count(*) AS extension_still_installed + FROM pg_extension WHERE extname = 'lolor'; + extension_still_installed +--------------------------- + 1 +(1 row) + +SELECT convert_from(lo_get(:kept_oid), 'UTF8') AS object_untouched; + object_untouched +----------------------- + still here afterwards +(1 row) + +SELECT count(*) AS still_in_lolor_storage + FROM lolor.pg_largeobject_metadata WHERE oid = :kept_oid; + still_in_lolor_storage +------------------------ + 1 +(1 row) + +SELECT lo_unlink(:kept_oid); + lo_unlink +----------- + 1 +(1 row) + +SELECT lolor.migrate_to_native(); +NOTICE: no lolor large objects to migrate + migrate_to_native +------------------- + 0 +(1 row) + +DROP EXTENSION lolor; +NOTICE: lolor: reconnect existing client sessions; they cache large object function OIDs diff --git a/lolor--1.2.2--1.2.3.sql b/lolor--1.2.2--1.2.3.sql new file mode 100644 index 0000000..7d65822 --- /dev/null +++ b/lolor--1.2.2--1.2.3.sql @@ -0,0 +1,6 @@ +/* lolor--1.2.2--1.2.3.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION lolor" to load this file. \quit + +-- No SQL-level changes in 1.2.3; the version is bumped for the release. diff --git a/lolor--1.2.2--1.3.0.sql b/lolor--1.2.2--1.3.0.sql deleted file mode 100644 index 53fc468..0000000 --- a/lolor--1.2.2--1.3.0.sql +++ /dev/null @@ -1,396 +0,0 @@ -/* lolor--1.2.2--1.3.0.sql */ - --- complain if script is sourced in psql, rather than via CREATE EXTENSION -\echo Use "ALTER EXTENSION lolor UPDATE" to load this file. \quit - --- Warn if there are active streaming replicas — they need lolor installed too -DO $$ -BEGIN - IF EXISTS (SELECT 1 FROM pg_stat_replication WHERE state = 'streaming') THEN - RAISE NOTICE 'lolor: active streaming replica(s) detected. ' - 'Ensure the lolor extension is also installed on each replica, ' - 'otherwise large object operations will fail if a replica is promoted.'; - END IF; -END; -$$; - -/* - * Restore the ACLs on the server-side file access functions. - * - * pg_catalog.lo_import() and lo_export() read and write files as the account - * PostgreSQL runs under, so core revokes EXECUTE on them from PUBLIC. lolor - * replaces them by renaming the originals to *_orig and creating its own. An - * ACL belongs to a function rather than to a name, so the restriction stayed - * on the parked original while each replacement was created with the default - * of EXECUTE TO PUBLIC. Every version through 1.2.2 is affected. - * - * Revoke on both spellings, since the installation may be enabled or - * disabled. Revoking on an already restricted function is a no-op. - */ -DO $$ -DECLARE - target text; -BEGIN - FOREACH target IN ARRAY ARRAY[ - 'lo_import(text)', - 'lo_import(text,oid)', - 'lo_export(oid,text)', - 'lolor_lo_import(text)', - 'lolor_lo_import(text,oid)', - 'lolor_lo_export(oid,text)' - ] - LOOP - IF to_regprocedure('pg_catalog.' || target) IS NOT NULL THEN - EXECUTE format('REVOKE ALL ON FUNCTION pg_catalog.%s FROM PUBLIC', target); - END IF; - END LOOP; -END; -$$; - -/* - * lolor.migrate_from_native() - * - * Migrate all native PostgreSQL large objects from pg_catalog.pg_largeobject - * into lolor's storage, preserving original OIDs, owners, ACLs, and data. - * After migration, the native copies are removed. - * - * The entire operation is transactional: if anything fails, ROLLBACK undoes - * all changes and no data is lost. - * - * Returns the number of large objects migrated, or -1 if migration was - * refused because logical replication slots exist whose decoding of the - * migration DML cannot be suppressed: either spock is not available to - * silence its own slots, or a non-spock slot (e.g. pgoutput, wal2json) is - * present that spock.repair_mode() cannot exclude. - * - * Refusal is reported as a soft -1 return (alongside a WARNING), not an - * error, and this asymmetry with migrate_to_native() is deliberate. This - * function is a manual, non-destructive operation: on refusal the native - * large objects are left untouched, so the caller can drop the offending - * slots and simply retry. - * - * Callers acting on the result MUST check for a non-positive return: a -1 - * means nothing was migrated, and ignoring it treats a refused migration - * as success. - */ -CREATE FUNCTION lolor.migrate_from_native() -RETURNS bigint AS $$ -DECLARE - lo_count bigint; - inserted_count bigint; - page_count bigint; - native_page_count bigint; - repair_enabled boolean := false; - lr_slots_exists boolean := false; - foreign_lr_slots_exists boolean := false; -BEGIN - -- Only superusers can read pg_largeobject.data and unlink others' objects - IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = current_user AND rolsuper) THEN - RAISE EXCEPTION 'lolor.migrate_from_native() requires superuser privileges'; - END IF; - - -- Verify lolor is enabled (functions are replaced) - IF NOT lolor.is_enabled() THEN - RAISE EXCEPTION 'lolor must be enabled before migration'; - END IF; - - -- Check for OID conflicts: native LOs that already exist in lolor storage - IF EXISTS ( - SELECT 1 - FROM pg_catalog.pg_largeobject_metadata native - JOIN lolor.pg_largeobject_metadata lm ON lm.oid = native.oid - ) THEN - RAISE EXCEPTION 'OID conflict: some native large objects already exist in lolor storage'; - END IF; - - -- Count what we are about to migrate - SELECT count(*) INTO lo_count FROM pg_catalog.pg_largeobject_metadata; - - IF lo_count = 0 THEN - RAISE NOTICE 'no native large objects to migrate'; - RETURN 0; - END IF; - - -- Logical slots indicate subscribers (replica nodes) that will not receive - -- the node-local migration DML. - SELECT EXISTS ( - SELECT 1 FROM pg_catalog.pg_replication_slots - WHERE slot_type = 'logical' AND database = current_database() - ) INTO lr_slots_exists; - - -- XXX: there are no evidence that the 'spock' name has been used anywhere. - -- Fix this mess later. - SELECT EXISTS ( - SELECT 1 FROM pg_catalog.pg_replication_slots - WHERE slot_type = 'logical' AND database = current_database() - AND plugin NOT IN ('spock_output', 'spock') - ) INTO foreign_lr_slots_exists; - - -- Suppress spock replication of the bulk migration DML. The migration only - -- shuffles rows between native and lolor storage on this node. - -- - -- Use spock only when it is fully operational: the extension is installed - -- (pg_extension is superuser-gated, so the schema name cannot be squatted - -- by an unprivileged user), the function exists (older spock versions lack - -- it) and the GUC exists (the library is actually preloaded). Anything - -- less falls through to the refusal branch below. - -- - -- Without spock the migration DML cannot be excluded from logical decoding, - -- so if any logical replication slot exists the migrated rows would leak to - -- subscribers. - IF EXISTS (SELECT 1 FROM pg_catalog.pg_extension WHERE extname = 'spock') - AND to_regprocedure('spock.repair_mode(boolean)') IS NOT NULL - AND current_setting('spock.replication_repair_mode', true) IS NOT NULL - THEN - -- If a non-spock logical slot is present its consumer would still decode - -- the bulk migration DML and receive the node-local row shuffling. - -- Refuse rather than leak the migration to that subscriber. - IF foreign_lr_slots_exists THEN - RAISE WARNING 'not migrating: non-spock logical replication slot(s) exist' - USING DETAIL = 'This call is a no-op: no large objects were migrated. ' - 'spock repair mode cannot exclude the migration DML from ' - 'non-spock output plugins (e.g. pgoutput, wal2json), so the ' - 'migrated rows would leak to those subscribers', - HINT = 'Drop all non-spock logical replication slots in this database before executing this procedure'; - RETURN -1; - END IF; - - IF current_setting('spock.replication_repair_mode', true) = 'off' THEN - PERFORM spock.repair_mode(true); - repair_enabled := true; - END IF; - ELSIF lr_slots_exists THEN - RAISE WARNING 'not migrating: logical replication slot(s) exist' - USING DETAIL = 'This call is a no-op: no large objects were migrated', - HINT = 'Drop all logical replication slots in this database before executing this procedure'; - RETURN -1; - END IF; - - SELECT count(*) INTO native_page_count - FROM pg_catalog.pg_largeobject; - - -- Copy metadata (preserving OIDs, owners, and ACLs) - INSERT INTO lolor.pg_largeobject_metadata (oid, lomowner, lomacl) - SELECT oid, lomowner, lomacl - FROM pg_catalog.pg_largeobject_metadata; - - GET DIAGNOSTICS inserted_count = ROW_COUNT; - IF inserted_count <> lo_count THEN - RAISE EXCEPTION 'metadata row count mismatch: expected %, inserted %', - lo_count, inserted_count; - END IF; - - -- Copy data pages - INSERT INTO lolor.pg_largeobject (loid, pageno, data) - SELECT loid, pageno, data FROM pg_catalog.pg_largeobject; - - GET DIAGNOSTICS page_count = ROW_COUNT; - IF page_count <> native_page_count THEN - RAISE EXCEPTION 'data page count mismatch: expected %, inserted %', - native_page_count, page_count; - END IF; - - -- Remove native large objects using the original (renamed) function. - -- Materialize the OID list first to avoid scanning the catalog while - -- lo_unlink_orig modifies it. - PERFORM pg_catalog.lo_unlink_orig(oid) - FROM (SELECT oid FROM pg_catalog.pg_largeobject_metadata) AS native_oids; - - -- Re-enable replication for the remainder of the caller's transaction, so - -- repair mode covers exactly the migration DML and nothing after it. - -- Error paths need no cleanup: they abort the whole transaction. - IF repair_enabled THEN - PERFORM spock.repair_mode(false); - END IF; - - RAISE NOTICE 'migrated % large object(s) (% data page(s)) from native to lolor storage', - lo_count, page_count; - - -- The migration DML was excluded from replication, so connected subscribers - -- (replica nodes) did not receive it. - IF lr_slots_exists THEN - RAISE NOTICE 'this migration is local to this node: ' - 'execute lolor.migrate_from_native() on each replica as well before modifying LOs'; - END IF; - - RETURN lo_count; -END; -$$ LANGUAGE plpgsql VOLATILE; - -/* - * lolor.migrate_to_native() - * - * Migrate all large objects from lolor storage back into native PostgreSQL - * storage, preserving original OIDs, owners, ACLs, and data. After - * migration, the lolor copies are removed. - * - * Called automatically by the DROP EXTENSION event trigger, but can also - * be invoked manually to revert to native large object storage. - * - * The _orig functions (native LO API) must be available, which means lolor - * must be in the enabled state. - * - * Returns the number of large objects migrated. RAISEs an ERROR if something - * goes wrong. - * The hard failure is deliberate: accidental ignore of a failure may result in - * loosing LOs. So, user must solve the issue manually before moving forward. - */ -CREATE FUNCTION lolor.migrate_to_native() -RETURNS bigint AS $$ -DECLARE - lo_count bigint; - loblksize bigint; - r_meta record; - r_data record; - fd integer; - repair_enabled boolean := false; - lr_slots_exists boolean := false; - foreign_lr_slots_exists boolean := false; -BEGIN - -- Only superusers can UPDATE pg_catalog.pg_largeobject_metadata - IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = current_user AND rolsuper) THEN - RAISE EXCEPTION 'lolor.migrate_to_native() requires superuser privileges'; - END IF; - - -- Verify lolor is enabled so _orig functions point to native API - IF NOT lolor.is_enabled() THEN - RAISE EXCEPTION 'lolor must be enabled before migration to native'; - END IF; - - -- Derive LOBLKSIZE at runtime. PostgreSQL defines it as BLCKSZ / 4. - -- Hard-coding 2048 would break on non-default block size builds. - loblksize := current_setting('block_size')::bigint / 4; - - -- Count what we are about to migrate - SELECT count(*) INTO lo_count FROM lolor.pg_largeobject_metadata; - - IF lo_count = 0 THEN - RAISE NOTICE 'no lolor large objects to migrate'; - RETURN 0; - END IF; - - -- Check for OID conflicts - IF EXISTS ( - SELECT 1 - FROM lolor.pg_largeobject_metadata lm - JOIN pg_catalog.pg_largeobject_metadata native ON native.oid = lm.oid - ) THEN - RAISE EXCEPTION 'OID conflict: some lolor large objects already exist in native storage'; - END IF; - - -- Logical slots indicate subscribers (replica nodes) that will not receive - -- the node-local migration DML. - SELECT EXISTS ( - SELECT 1 FROM pg_catalog.pg_replication_slots - WHERE slot_type = 'logical' AND database = current_database() - ) INTO lr_slots_exists; - - -- A logical slot whose output plugin is not spock's own cannot be silenced - -- by spock.repair_mode(). Track those separately: their consumers would - -- still decode the migration DML even when spock is fully operational. - SELECT EXISTS ( - SELECT 1 FROM pg_catalog.pg_replication_slots - WHERE slot_type = 'logical' AND database = current_database() - AND plugin NOT IN ('spock_output', 'spock') - ) INTO foreign_lr_slots_exists; - - -- Suppress spock replication of the bulk migration DML. The migration only - -- shuffles rows between lolor and native storage on this node. - -- - -- Use spock only when it is fully operational: the extension is installed - -- (pg_extension is superuser-gated, so the schema name cannot be squatted - -- by an unprivileged user), the function exists (older spock versions lack - -- it) and the GUC exists (the library is actually preloaded). Anything - -- less falls through to the refusal branch below. - -- - -- Without spock the migration DML cannot be excluded from logical decoding, - -- so if any logical replication slot exists the deletes from the lolor - -- tables would replicate while the re-created native large objects would - -- not, losing large objects on the subscriber side. - IF EXISTS (SELECT 1 FROM pg_catalog.pg_extension WHERE extname = 'spock') - AND to_regprocedure('spock.repair_mode(boolean)') IS NOT NULL - AND current_setting('spock.replication_repair_mode', true) IS NOT NULL - THEN - -- spock.repair_mode() only suppresses spock's own output plugin - -- ('spock_output'/'spock'). If a non-spock logical slot is present - -- (pgoutput, wal2json, decoderbufs, ...) its consumer would still decode - -- the migration DML: the lolor deletes would replicate while the - -- re-created native large objects would not, losing LOs on the subscriber. - IF foreign_lr_slots_exists THEN - RAISE EXCEPTION 'cannot migrate LOs to pg_catalog: non-spock logical replication slot(s) exist' - USING DETAIL = 'spock repair mode only silences spock''s own output plugin; ' - 'a non-spock slot (e.g. pgoutput, wal2json) would replicate the ' - 'lolor deletes while the re-created native large objects would not, ' - 'losing large objects on the subscriber side', - HINT = 'Drop all non-spock logical replication slots in this database before executing this procedure'; - END IF; - - IF current_setting('spock.replication_repair_mode', true) = 'off' THEN - PERFORM spock.repair_mode(true); - repair_enabled := true; - END IF; - ELSIF lr_slots_exists THEN - RAISE EXCEPTION 'cannot migrate LOs to pg_catalog: logical replication slot(s) exist' - USING DETAIL = 'Without spock the migration DML cannot be excluded from logical decoding: ' - 'the deletes from the lolor tables would replicate to subscribers while ' - 'the re-created native large objects would not, losing large objects on ' - 'the subscriber side', - HINT = 'Drop all logical replication slots in this database before executing this procedure'; - END IF; - - -- Migrate each object using the native LO API (_orig functions). - -- We cannot INSERT directly into pg_catalog.pg_largeobject from SQL, - -- so we use lo_create_orig + lo_open_orig + lowrite_orig. - -- - -- Zero-data-page LOs (metadata only) are handled correctly: lo_create_orig - -- creates an empty LO and the inner FOR loop simply does not execute. - -- - -- Note on sparse LOs: any gap between non-consecutive page numbers will - -- be filled with zeroes by the native LO write API. This preserves read - -- semantics (holes already returned zeroes) but may increase storage. - FOR r_meta IN SELECT oid, lomowner, lomacl FROM lolor.pg_largeobject_metadata - LOOP - -- Create native LO with the exact same OID - PERFORM pg_catalog.lo_create_orig(r_meta.oid); - - -- Write data pages through the native LO write API. - -- Use lo_lseek64_orig (bigint offset) to handle LOs larger than 2 GB. - fd := pg_catalog.lo_open_orig(r_meta.oid, x'60000'::int); - FOR r_data IN - SELECT pageno, data FROM lolor.pg_largeobject - WHERE loid = r_meta.oid ORDER BY pageno - LOOP - PERFORM pg_catalog.lo_lseek64_orig(fd, r_data.pageno::bigint * loblksize, 0); - PERFORM pg_catalog.lowrite_orig(fd, r_data.data); - END LOOP; - PERFORM pg_catalog.lo_close_orig(fd); - - -- Restore ownership and ACL (lo_create sets current user as owner) - UPDATE pg_catalog.pg_largeobject_metadata - SET lomowner = r_meta.lomowner, lomacl = r_meta.lomacl - WHERE pg_catalog.pg_largeobject_metadata.oid = r_meta.oid; - END LOOP; - - -- Clean lolor storage - DELETE FROM lolor.pg_largeobject; - DELETE FROM lolor.pg_largeobject_metadata; - - -- Re-enable replication for the remainder of the caller's transaction, so - -- repair mode covers exactly the migration DML and nothing after it. - -- Error paths need no cleanup: they abort the whole transaction. - IF repair_enabled THEN - PERFORM spock.repair_mode(false); - END IF; - - RAISE NOTICE 'migrated % large object(s) from lolor to native storage', lo_count; - - -- The migration DML was excluded from replication, so connected subscribers - -- (replica nodes) did not receive it. - IF lr_slots_exists THEN - RAISE NOTICE 'this migration is local to this node: ' - 'execute lolor.migrate_to_native() on each replica as well before modifying LOs'; - END IF; - - RETURN lo_count; -END; -$$ LANGUAGE plpgsql VOLATILE; diff --git a/lolor--1.2.3--1.3.0.sql b/lolor--1.2.3--1.3.0.sql new file mode 100644 index 0000000..c843b82 --- /dev/null +++ b/lolor--1.2.3--1.3.0.sql @@ -0,0 +1,758 @@ +/* lolor--1.2.3--1.3.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION lolor UPDATE" to load this file. \quit + +-- Warn if there are active streaming replicas — they need lolor installed too +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_stat_replication WHERE state = 'streaming') THEN + RAISE NOTICE 'lolor: active streaming replica(s) detected. ' + 'Ensure the lolor extension is also installed on each replica, ' + 'otherwise large object operations will fail if a replica is promoted.'; + END IF; +END; +$$; + +/* + * Restore the ACLs on the server-side file access functions. + * + * pg_catalog.lo_import() and lo_export() read and write files as the account + * PostgreSQL runs under, so core revokes EXECUTE on them from PUBLIC. lolor + * replaces them by renaming the originals to *_orig and creating its own. An + * ACL belongs to a function rather than to a name, so the restriction stayed + * on the parked original while each replacement was created with the default + * of EXECUTE TO PUBLIC. Every version through 1.2.2 is affected. + * + * Revoke on both spellings, since the installation may be enabled or + * disabled. Revoking on an already restricted function is a no-op. + */ +DO $$ +DECLARE + target text; +BEGIN + FOREACH target IN ARRAY ARRAY[ + 'lo_import(text)', + 'lo_import(text,oid)', + 'lo_export(oid,text)', + 'lolor_lo_import(text)', + 'lolor_lo_import(text,oid)', + 'lolor_lo_export(oid,text)' + ] + LOOP + IF to_regprocedure('pg_catalog.' || target) IS NOT NULL THEN + EXECUTE format('REVOKE ALL ON FUNCTION pg_catalog.%s FROM PUBLIC', target); + END IF; + END LOOP; +END; +$$; + +/* + * Large objects in lolor storage that refer to a role which no longer exists, + * as owner, grantee or grantor. + * + * Objects in lolor storage are rows in ordinary tables, so they cannot + * participate in pg_shdepend: DROP ROLE will not notice them the way it + * notices native large objects. That is inherent to storing them outside the + * catalogs; this function makes the consequence findable, and fix_orphans() + * repairs it. Joins pg_roles rather than pg_authid so that a grantee of + * EXECUTE can actually run it. + */ +CREATE FUNCTION lolor.check_orphans() +RETURNS TABLE (loid oid, role_oid oid, role_kind text) AS $$ + SELECT m.oid, m.lomowner, 'owner' + FROM lolor.pg_largeobject_metadata m + WHERE NOT EXISTS (SELECT 1 FROM pg_catalog.pg_roles r WHERE r.oid = m.lomowner) + UNION + SELECT m.oid, a.grantee, 'grantee' + FROM lolor.pg_largeobject_metadata m, pg_catalog.aclexplode(m.lomacl) a + WHERE a.grantee <> 0 + AND NOT EXISTS (SELECT 1 FROM pg_catalog.pg_roles r WHERE r.oid = a.grantee) + UNION + SELECT m.oid, a.grantor, 'grantor' + FROM lolor.pg_largeobject_metadata m, pg_catalog.aclexplode(m.lomacl) a + WHERE NOT EXISTS (SELECT 1 FROM pg_catalog.pg_roles r WHERE r.oid = a.grantor) + ORDER BY 1, 3, 2 +$$ LANGUAGE sql STABLE; + +REVOKE ALL ON FUNCTION lolor.check_orphans() FROM PUBLIC; + +/* + * Repair what check_orphans() reports. Objects whose owner is gone go to + * new_owner, and the ACL is rebuilt the way core's aclnewowner() does it: the + * old owner is replaced by new_owner wherever it appears as grantee or + * grantor, entries that then coincide are merged, and only entries that still + * name a missing role are dropped. Grants the old owner made therefore + * survive, as they do under REASSIGN OWNED. Returns the number of objects + * changed. + * + * One UPDATE is essential: the ACL rewrite has to see the old owner, and a + * separate UPDATE of lomowner would already have lost it. + */ +CREATE FUNCTION lolor.fix_orphans(new_owner regrole) +RETURNS bigint AS $$ +DECLARE + fixed bigint; +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = current_user AND rolsuper) THEN + RAISE EXCEPTION 'must be superuser to repair orphaned large objects'; + END IF; + + WITH o AS ( + SELECT m.oid, m.lomowner AS old_owner, m.lomacl, + NOT EXISTS (SELECT 1 FROM pg_catalog.pg_roles r WHERE r.oid = m.lomowner) AS owner_dead + FROM lolor.pg_largeobject_metadata m) + UPDATE lolor.pg_largeobject_metadata m + SET lomowner = CASE WHEN o.owner_dead THEN new_owner::oid ELSE m.lomowner END, + -- aclexplode() yields one row per privilege. Substitute the owner, + -- OR the grant option per privilege where entries now coincide, and + -- rebuild exactly one aclitem per (grantee, grantor) through the + -- aclitem input syntax: an ACL must not hold two entries for the + -- same pair, since GRANT and REVOKE update only the first, and + -- makeaclitem() cannot mix grant options within one entry. Large + -- objects carry only SELECT (r) and UPDATE (w). An empty result is + -- NULL, meaning default privileges. + lomacl = ( + SELECT array_agg(s.item ORDER BY s.grantee, s.grantor) + FROM (SELECT p.grantee, p.grantor, + format('%s=%s/%s', + CASE WHEN p.grantee = 0 THEN '' ELSE quote_ident(ge.rolname) END, + string_agg(CASE p.privilege_type WHEN 'SELECT' THEN 'r' WHEN 'UPDATE' THEN 'w' END + || CASE WHEN p.grantable THEN '*' ELSE '' END, + '' ORDER BY p.privilege_type), + quote_ident(gr.rolname))::pg_catalog.aclitem AS item + FROM (SELECT CASE WHEN o.owner_dead AND a.grantee = o.old_owner + THEN new_owner::oid ELSE a.grantee END AS grantee, + CASE WHEN o.owner_dead AND a.grantor = o.old_owner + THEN new_owner::oid ELSE a.grantor END AS grantor, + a.privilege_type, + bool_or(a.is_grantable) AS grantable + FROM pg_catalog.aclexplode(o.lomacl) a + GROUP BY 1, 2, 3) p + LEFT JOIN pg_catalog.pg_roles ge ON ge.oid = p.grantee + JOIN pg_catalog.pg_roles gr ON gr.oid = p.grantor + WHERE p.grantee = 0 OR ge.oid IS NOT NULL + GROUP BY p.grantee, p.grantor, ge.rolname, gr.rolname) s) + FROM o + WHERE m.oid = o.oid + AND (o.owner_dead + OR EXISTS (SELECT 1 FROM pg_catalog.aclexplode(o.lomacl) a + WHERE (a.grantee <> 0 + AND NOT EXISTS (SELECT 1 FROM pg_catalog.pg_roles r WHERE r.oid = a.grantee)) + OR NOT EXISTS (SELECT 1 FROM pg_catalog.pg_roles r WHERE r.oid = a.grantor))); + GET DIAGNOSTICS fixed = ROW_COUNT; + + RETURN fixed; +END; +$$ LANGUAGE plpgsql VOLATILE; + +REVOKE ALL ON FUNCTION lolor.fix_orphans(regrole) FROM PUBLIC; + +/* + * Remove the bogus pg_shdepend rows left by earlier versions. + * + * Through 1.2.2, creating a large object recorded a pg_shdepend row whose + * classId was the OID of lolor.pg_largeobject -- an ordinary table, not a + * catalog the dependency machinery can describe. DROP ROLE on any role that + * had created one failed with "unrecognized object class", and the rows were + * never removed. + * + * pg_shdepend is shared across the cluster, so restrict the delete to this + * database: the same classId in another database is an unrelated relation. + */ +DELETE FROM pg_catalog.pg_shdepend +WHERE dbid = (SELECT oid FROM pg_catalog.pg_database + WHERE datname = current_database()) + AND classid IN ('lolor.pg_largeobject'::regclass, + 'lolor.pg_largeobject_metadata'::regclass); + +/* + * Hardened enable / disable / is_enabled. + */ + +/* + * Refuse unless called from a client session that is the only one connected + * to this database. + * + * Renaming the pg_catalog large object functions cannot be made atomic for + * other backends: a rename keeps the function OID, so a session that has + * already resolved lo_open() keeps calling the previous implementation, and + * neither cached plans nor libpq's fastpath OIDs are invalidated. Require + * sole access instead, as ALTER DATABASE ... RENAME does. This is a snapshot + * of pg_stat_activity, so a session connecting in the same instant is not + * excluded; that window is why the migration also takes its own locks. + */ +CREATE OR REPLACE FUNCTION lolor._require_no_other_sessions(what text) +RETURNS void AS $$ +DECLARE + n int; + self text; +BEGIN + -- A background worker, such as an apply worker executing replicated DDL, + -- is not a client session and must not do this at all: the change is + -- node-local and a refusal would leave that worker retrying forever. + SELECT backend_type INTO self + FROM pg_catalog.pg_stat_activity + WHERE pid = pg_backend_pid(); + + IF self IS DISTINCT FROM 'client backend' THEN + RAISE EXCEPTION 'cannot % from a %', what, coalesce(self, 'non-client process') + USING HINT = 'Run it from a client session connected directly to this node.'; + END IF; + + SELECT count(*) INTO n + FROM pg_catalog.pg_stat_activity + WHERE datname = current_database() + AND backend_type = 'client backend' + AND pid <> pg_backend_pid(); + + IF n > 0 THEN + RAISE EXCEPTION 'cannot % while other sessions are connected to the database', what + USING DETAIL = CASE WHEN n = 1 THEN '1 other session is connected.' + ELSE n || ' other sessions are connected.' END, + HINT = 'Disconnect them first. A session that already resolved the ' + 'large object functions keeps calling the previous implementation.'; + END IF; +END; +$$ LANGUAGE plpgsql VOLATILE; + +REVOKE ALL ON FUNCTION lolor._require_no_other_sessions(text) FROM PUBLIC; + +CREATE OR REPLACE FUNCTION lolor.is_enabled() +RETURNS boolean AS $$ +DECLARE + parked_present boolean; + orig_present boolean; +BEGIN + -- Exact signatures, qualified to pg_catalog: see the note in lolor.enable(). + parked_present := to_regprocedure('pg_catalog.lolor_lo_open(oid,int4)') IS NOT NULL; + orig_present := to_regprocedure('pg_catalog.lo_open_orig(oid,int4)') IS NOT NULL; + + IF parked_present = orig_present THEN + RAISE EXCEPTION 'lolor is in inconsistent state' + USING DETAIL = format('pg_catalog.lolor_lo_open present: %s; pg_catalog.lo_open_orig present: %s', + parked_present, orig_present); + END IF; + + -- Our functions parked under lolor_* means the native ones are in place. + RETURN NOT parked_present; +END; +$$ LANGUAGE plpgsql STRICT STABLE; + + +/* + * Disable lolor functionality. + * + * Parks the lolor implementations under pg_catalog.lolor_* and restores the + * native pg_catalog names from their *_orig parking spot. Creates and drops + * nothing. Returns true on success, false on a handled no-op. + */ +CREATE OR REPLACE FUNCTION lolor.disable() +RETURNS boolean AS $$ +BEGIN + PERFORM lolor._require_no_other_sessions('disable lolor'); + + -- The probes below are a check-then-act, but no lock is needed: the whole + -- body runs in one transaction, so a concurrent caller that loses the race + -- fails on a rename and rolls back, leaving the state consistent. + -- Probe exact signatures in pg_catalog. Earlier versions matched on + -- proname alone across every schema, so any user with CREATE on any schema + -- could squat 'lolor_lo_open' and wedge lolor into a permanent + -- 'inconsistent state', which also blocked DROP EXTENSION. + IF to_regprocedure('pg_catalog.lo_close_orig(int4)') IS NULL THEN + RAISE NOTICE 'lolor is already disabled'; + RETURN false; + END IF; + IF to_regprocedure('pg_catalog.lolor_lo_open(oid,int4)') IS NOT NULL THEN + RAISE NOTICE 'lolor.disable() has been called before'; + RETURN false; + END IF; + + ALTER FUNCTION pg_catalog.lo_open(oid, int4) RENAME TO lolor_lo_open; + ALTER FUNCTION pg_catalog.lo_open_orig(oid, int4) RENAME TO lo_open; + ALTER FUNCTION pg_catalog.lo_close(int4) RENAME TO lolor_lo_close; + ALTER FUNCTION pg_catalog.lo_close_orig(int4) RENAME TO lo_close; + ALTER FUNCTION pg_catalog.lo_creat(integer) RENAME TO lolor_lo_creat; + ALTER FUNCTION pg_catalog.lo_creat_orig(integer) RENAME TO lo_creat; + ALTER FUNCTION pg_catalog.lo_create(oid) RENAME TO lolor_lo_create; + ALTER FUNCTION pg_catalog.lo_create_orig(oid) RENAME TO lo_create; + ALTER FUNCTION pg_catalog.loread(integer, integer) RENAME TO lolor_loread; + ALTER FUNCTION pg_catalog.loread_orig(integer, integer) RENAME TO loread; + ALTER FUNCTION pg_catalog.lowrite(integer, bytea) RENAME TO lolor_lowrite; + ALTER FUNCTION pg_catalog.lowrite_orig(integer, bytea) RENAME TO lowrite; + ALTER FUNCTION pg_catalog.lo_export(oid, text) RENAME TO lolor_lo_export; + ALTER FUNCTION pg_catalog.lo_export_orig(oid, text) RENAME TO lo_export; + ALTER FUNCTION pg_catalog.lo_from_bytea(oid, bytea) RENAME TO lolor_lo_from_bytea; + ALTER FUNCTION pg_catalog.lo_from_bytea_orig(oid, bytea) RENAME TO lo_from_bytea; + ALTER FUNCTION pg_catalog.lo_get(oid) RENAME TO lolor_lo_get; + ALTER FUNCTION pg_catalog.lo_get_orig(oid) RENAME TO lo_get; + ALTER FUNCTION pg_catalog.lo_get(oid, bigint, integer) RENAME TO lolor_lo_get; + ALTER FUNCTION pg_catalog.lo_get_orig(oid, bigint, integer) RENAME TO lo_get; + ALTER FUNCTION pg_catalog.lo_import(text) RENAME TO lolor_lo_import; + ALTER FUNCTION pg_catalog.lo_import_orig(text) RENAME TO lo_import; + ALTER FUNCTION pg_catalog.lo_import(text, oid) RENAME TO lolor_lo_import; + ALTER FUNCTION pg_catalog.lo_import_orig(text, oid) RENAME TO lo_import; + ALTER FUNCTION pg_catalog.lo_lseek(integer, integer, integer) RENAME TO lolor_lo_lseek; + ALTER FUNCTION pg_catalog.lo_lseek_orig(integer, integer, integer) RENAME TO lo_lseek; + ALTER FUNCTION pg_catalog.lo_lseek64(integer, bigint, integer) RENAME TO lolor_lo_lseek64; + ALTER FUNCTION pg_catalog.lo_lseek64_orig(integer, bigint, integer) RENAME TO lo_lseek64; + ALTER FUNCTION pg_catalog.lo_put(oid, bigint, bytea) RENAME TO lolor_lo_put; + ALTER FUNCTION pg_catalog.lo_put_orig(oid, bigint, bytea) RENAME TO lo_put; + ALTER FUNCTION pg_catalog.lo_tell(integer) RENAME TO lolor_lo_tell; + ALTER FUNCTION pg_catalog.lo_tell_orig(integer) RENAME TO lo_tell; + ALTER FUNCTION pg_catalog.lo_tell64(integer) RENAME TO lolor_lo_tell64; + ALTER FUNCTION pg_catalog.lo_tell64_orig(integer) RENAME TO lo_tell64; + ALTER FUNCTION pg_catalog.lo_truncate(integer, integer) RENAME TO lolor_lo_truncate; + ALTER FUNCTION pg_catalog.lo_truncate_orig(integer, integer) RENAME TO lo_truncate; + ALTER FUNCTION pg_catalog.lo_truncate64(integer, bigint) RENAME TO lolor_lo_truncate64; + ALTER FUNCTION pg_catalog.lo_truncate64_orig(integer, bigint) RENAME TO lo_truncate64; + ALTER FUNCTION pg_catalog.lo_unlink(oid) RENAME TO lolor_lo_unlink; + ALTER FUNCTION pg_catalog.lo_unlink_orig(oid) RENAME TO lo_unlink; + + -- Renaming changes which OID owns the name lo_open, and libpq caches the + -- large object fastpath OIDs per connection: sessions that touched a large + -- object before this call keep using the previous implementation. + RAISE NOTICE 'lolor: reconnect existing client sessions; they cache large object function OIDs'; + + RETURN true; +END; +$$ LANGUAGE plpgsql STRICT VOLATILE; + + +/* + * Enable lolor functionality, undoing lolor.disable(). + */ +CREATE OR REPLACE FUNCTION lolor.enable() +RETURNS boolean AS $$ +BEGIN + PERFORM lolor._require_no_other_sessions('enable lolor'); + + -- The probes below are a check-then-act, but no lock is needed: the whole + -- body runs in one transaction, so a concurrent caller that loses the race + -- fails on a rename and rolls back, leaving the state consistent. + -- Probe exact signatures in pg_catalog. Earlier versions matched on + -- proname alone across every schema, so any user with CREATE on any schema + -- could squat 'lolor_lo_open' and wedge lolor into a permanent + -- 'inconsistent state', which also blocked DROP EXTENSION. + IF to_regprocedure('pg_catalog.lolor_lo_open(oid,int4)') IS NULL THEN + RAISE NOTICE 'lolor is already enabled'; + RETURN false; + END IF; + IF to_regprocedure('pg_catalog.lo_close_orig(int4)') IS NOT NULL THEN + RAISE NOTICE 'lolor.enable() has been called before'; + RETURN false; + END IF; + + ALTER FUNCTION pg_catalog.lo_open(oid, int4) RENAME TO lo_open_orig; + ALTER FUNCTION pg_catalog.lolor_lo_open(oid, int4) RENAME TO lo_open; + ALTER FUNCTION pg_catalog.lo_close(int4) RENAME TO lo_close_orig; + ALTER FUNCTION pg_catalog.lolor_lo_close(int4) RENAME TO lo_close; + ALTER FUNCTION pg_catalog.lo_creat(integer) RENAME TO lo_creat_orig; + ALTER FUNCTION pg_catalog.lolor_lo_creat(integer) RENAME TO lo_creat; + ALTER FUNCTION pg_catalog.lo_create(oid) RENAME TO lo_create_orig; + ALTER FUNCTION pg_catalog.lolor_lo_create(oid) RENAME TO lo_create; + ALTER FUNCTION pg_catalog.loread(integer, integer) RENAME TO loread_orig; + ALTER FUNCTION pg_catalog.lolor_loread(integer, integer) RENAME TO loread; + ALTER FUNCTION pg_catalog.lowrite(integer, bytea) RENAME TO lowrite_orig; + ALTER FUNCTION pg_catalog.lolor_lowrite(integer, bytea) RENAME TO lowrite; + ALTER FUNCTION pg_catalog.lo_export(oid, text) RENAME TO lo_export_orig; + ALTER FUNCTION pg_catalog.lolor_lo_export(oid, text) RENAME TO lo_export; + ALTER FUNCTION pg_catalog.lo_from_bytea(oid, bytea) RENAME TO lo_from_bytea_orig; + ALTER FUNCTION pg_catalog.lolor_lo_from_bytea(oid, bytea) RENAME TO lo_from_bytea; + ALTER FUNCTION pg_catalog.lo_get(oid) RENAME TO lo_get_orig; + ALTER FUNCTION pg_catalog.lolor_lo_get(oid) RENAME TO lo_get; + ALTER FUNCTION pg_catalog.lo_get(oid, bigint, integer) RENAME TO lo_get_orig; + ALTER FUNCTION pg_catalog.lolor_lo_get(oid, bigint, integer) RENAME TO lo_get; + ALTER FUNCTION pg_catalog.lo_import(text) RENAME TO lo_import_orig; + ALTER FUNCTION pg_catalog.lolor_lo_import(text) RENAME TO lo_import; + ALTER FUNCTION pg_catalog.lo_import(text, oid) RENAME TO lo_import_orig; + ALTER FUNCTION pg_catalog.lolor_lo_import(text, oid) RENAME TO lo_import; + ALTER FUNCTION pg_catalog.lo_lseek(integer, integer, integer) RENAME TO lo_lseek_orig; + ALTER FUNCTION pg_catalog.lolor_lo_lseek(integer, integer, integer) RENAME TO lo_lseek; + ALTER FUNCTION pg_catalog.lo_lseek64(integer, bigint, integer) RENAME TO lo_lseek64_orig; + ALTER FUNCTION pg_catalog.lolor_lo_lseek64(integer, bigint, integer) RENAME TO lo_lseek64; + ALTER FUNCTION pg_catalog.lo_put(oid, bigint, bytea) RENAME TO lo_put_orig; + ALTER FUNCTION pg_catalog.lolor_lo_put(oid, bigint, bytea) RENAME TO lo_put; + ALTER FUNCTION pg_catalog.lo_tell(integer) RENAME TO lo_tell_orig; + ALTER FUNCTION pg_catalog.lolor_lo_tell(integer) RENAME TO lo_tell; + ALTER FUNCTION pg_catalog.lo_tell64(integer) RENAME TO lo_tell64_orig; + ALTER FUNCTION pg_catalog.lolor_lo_tell64(integer) RENAME TO lo_tell64; + ALTER FUNCTION pg_catalog.lo_truncate(integer, integer) RENAME TO lo_truncate_orig; + ALTER FUNCTION pg_catalog.lolor_lo_truncate(integer, integer) RENAME TO lo_truncate; + ALTER FUNCTION pg_catalog.lo_truncate64(integer, bigint) RENAME TO lo_truncate64_orig; + ALTER FUNCTION pg_catalog.lolor_lo_truncate64(integer, bigint) RENAME TO lo_truncate64; + ALTER FUNCTION pg_catalog.lo_unlink(oid) RENAME TO lo_unlink_orig; + ALTER FUNCTION pg_catalog.lolor_lo_unlink(oid) RENAME TO lo_unlink; + + -- Renaming changes which OID owns the name lo_open, and libpq caches the + -- large object fastpath OIDs per connection: sessions that touched a large + -- object before this call keep using the previous implementation. + RAISE NOTICE 'lolor: reconnect existing client sessions; they cache large object function OIDs'; + + RETURN true; +END; +$$ LANGUAGE plpgsql STRICT VOLATILE; + +/* + * Re-register the drop cleanup for every spelling of the drop. + * + * DROP SCHEMA lolor CASCADE and DROP OWNED BY reach the extension by + * dependency cascade rather than as DROP EXTENSION, and fire under their own + * command tags. Without them the cleanup never runs, the large objects are + * destroyed with the lolor tables, and pg_catalog is left without a working + * lo_open(). Tags cannot be altered in place, so re-create the trigger. + */ +DROP EVENT TRIGGER lo_on_drop_extension; +CREATE EVENT TRIGGER lo_on_drop_extension + ON ddl_command_start + WHEN tag IN ('DROP EXTENSION', 'DROP SCHEMA', 'DROP OWNED') + EXECUTE FUNCTION pg_catalog.lo_on_drop_extension(); +ALTER EVENT TRIGGER lo_on_drop_extension ENABLE ALWAYS; + +/* + * lolor.migrate_from_native() + * + * Migrate all native PostgreSQL large objects from pg_catalog.pg_largeobject + * into lolor's storage, preserving original OIDs, owners, ACLs, and data. + * After migration, the native copies are removed. + * + * The entire operation is transactional: if anything fails, ROLLBACK undoes + * all changes and no data is lost. + * + * Returns the number of large objects migrated, or -1 if migration was + * refused because logical replication slots exist whose decoding of the + * migration DML cannot be suppressed: either spock is not available to + * silence its own slots, or a non-spock slot (e.g. pgoutput, wal2json) is + * present that spock.repair_mode() cannot exclude. + * + * Refusal is reported as a soft -1 return (alongside a WARNING), not an + * error, and this asymmetry with migrate_to_native() is deliberate. This + * function is a manual, non-destructive operation: on refusal the native + * large objects are left untouched, so the caller can drop the offending + * slots and simply retry. + * + * Callers acting on the result MUST check for a non-positive return: a -1 + * means nothing was migrated, and ignoring it treats a refused migration + * as success. + */ +CREATE FUNCTION lolor.migrate_from_native() +RETURNS bigint AS $$ +DECLARE + lo_count bigint; + inserted_count bigint; + page_count bigint; + native_page_count bigint; + repair_enabled boolean := false; + lr_slots_exists boolean := false; + foreign_lr_slots_exists boolean := false; +BEGIN + -- Only superusers can read pg_largeobject.data and unlink others' objects + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = current_user AND rolsuper) THEN + RAISE EXCEPTION 'lolor.migrate_from_native() requires superuser privileges'; + END IF; + + -- Verify lolor is enabled (functions are replaced) + IF NOT lolor.is_enabled() THEN + RAISE EXCEPTION 'lolor must be enabled before migration'; + END IF; + + -- Check for OID conflicts: native LOs that already exist in lolor storage + IF EXISTS ( + SELECT 1 + FROM pg_catalog.pg_largeobject_metadata native + JOIN lolor.pg_largeobject_metadata lm ON lm.oid = native.oid + ) THEN + RAISE EXCEPTION 'OID conflict: some native large objects already exist in lolor storage'; + END IF; + + -- Count what we are about to migrate + SELECT count(*) INTO lo_count FROM pg_catalog.pg_largeobject_metadata; + + IF lo_count = 0 THEN + RAISE NOTICE 'no native large objects to migrate'; + RETURN 0; + END IF; + + -- Logical slots indicate subscribers (replica nodes) that will not receive + -- the node-local migration DML. + SELECT EXISTS ( + SELECT 1 FROM pg_catalog.pg_replication_slots + WHERE slot_type = 'logical' AND database = current_database() + ) INTO lr_slots_exists; + + -- XXX: there are no evidence that the 'spock' name has been used anywhere. + -- Fix this mess later. + SELECT EXISTS ( + SELECT 1 FROM pg_catalog.pg_replication_slots + WHERE slot_type = 'logical' AND database = current_database() + AND plugin NOT IN ('spock_output', 'spock') + ) INTO foreign_lr_slots_exists; + + -- Suppress spock replication of the bulk migration DML. The migration only + -- shuffles rows between native and lolor storage on this node. + -- + -- Use spock only when it is fully operational: the extension is installed + -- (pg_extension is superuser-gated, so the schema name cannot be squatted + -- by an unprivileged user), the function exists (older spock versions lack + -- it) and the GUC exists (the library is actually preloaded). Anything + -- less falls through to the refusal branch below. + -- + -- Without spock the migration DML cannot be excluded from logical decoding, + -- so if any logical replication slot exists the migrated rows would leak to + -- subscribers. + IF EXISTS (SELECT 1 FROM pg_catalog.pg_extension WHERE extname = 'spock') + AND to_regprocedure('spock.repair_mode(boolean)') IS NOT NULL + AND current_setting('spock.replication_repair_mode', true) IS NOT NULL + THEN + -- If a non-spock logical slot is present its consumer would still decode + -- the bulk migration DML and receive the node-local row shuffling. + -- Refuse rather than leak the migration to that subscriber. + IF foreign_lr_slots_exists THEN + RAISE WARNING 'not migrating: non-spock logical replication slot(s) exist' + USING DETAIL = 'This call is a no-op: no large objects were migrated. ' + 'spock repair mode cannot exclude the migration DML from ' + 'non-spock output plugins (e.g. pgoutput, wal2json), so the ' + 'migrated rows would leak to those subscribers', + HINT = 'Drop all non-spock logical replication slots in this database before executing this procedure'; + RETURN -1; + END IF; + + IF current_setting('spock.replication_repair_mode', true) = 'off' THEN + PERFORM spock.repair_mode(true); + repair_enabled := true; + END IF; + ELSIF lr_slots_exists THEN + RAISE WARNING 'not migrating: logical replication slot(s) exist' + USING DETAIL = 'This call is a no-op: no large objects were migrated', + HINT = 'Drop all logical replication slots in this database before executing this procedure'; + RETURN -1; + END IF; + + SELECT count(*) INTO native_page_count + FROM pg_catalog.pg_largeobject; + + -- Copy metadata (preserving OIDs, owners, and ACLs) + INSERT INTO lolor.pg_largeobject_metadata (oid, lomowner, lomacl) + SELECT oid, lomowner, lomacl + FROM pg_catalog.pg_largeobject_metadata; + + GET DIAGNOSTICS inserted_count = ROW_COUNT; + IF inserted_count <> lo_count THEN + RAISE EXCEPTION 'metadata row count mismatch: expected %, inserted %', + lo_count, inserted_count; + END IF; + + -- Copy data pages + INSERT INTO lolor.pg_largeobject (loid, pageno, data) + SELECT loid, pageno, data FROM pg_catalog.pg_largeobject; + + GET DIAGNOSTICS page_count = ROW_COUNT; + IF page_count <> native_page_count THEN + RAISE EXCEPTION 'data page count mismatch: expected %, inserted %', + native_page_count, page_count; + END IF; + + -- Remove native large objects using the original (renamed) function. + -- Materialize the OID list first to avoid scanning the catalog while + -- lo_unlink_orig modifies it. + PERFORM pg_catalog.lo_unlink_orig(oid) + FROM (SELECT oid FROM pg_catalog.pg_largeobject_metadata) AS native_oids; + + -- Re-enable replication for the remainder of the caller's transaction, so + -- repair mode covers exactly the migration DML and nothing after it. + -- Error paths need no cleanup: they abort the whole transaction. + IF repair_enabled THEN + PERFORM spock.repair_mode(false); + END IF; + + RAISE NOTICE 'migrated % large object(s) (% data page(s)) from native to lolor storage', + lo_count, page_count; + + -- The migration DML was excluded from replication, so connected subscribers + -- (replica nodes) did not receive it. + IF lr_slots_exists THEN + RAISE NOTICE 'this migration is local to this node: ' + 'execute lolor.migrate_from_native() on each replica as well before modifying LOs'; + END IF; + + RETURN lo_count; +END; +$$ LANGUAGE plpgsql VOLATILE; + +/* + * lolor.migrate_to_native() + * + * Migrate all large objects from lolor storage back into native PostgreSQL + * storage, preserving original OIDs, owners, ACLs, and data. After + * migration, the lolor copies are removed. + * + * Called automatically by the DROP EXTENSION event trigger, but can also + * be invoked manually to revert to native large object storage. + * + * The _orig functions (native LO API) must be available, which means lolor + * must be in the enabled state. + * + * Returns the number of large objects migrated. RAISEs an ERROR if something + * goes wrong. + * The hard failure is deliberate: accidental ignore of a failure may result in + * loosing LOs. So, user must solve the issue manually before moving forward. + */ +CREATE FUNCTION lolor.migrate_to_native() +RETURNS bigint AS $$ +DECLARE + lo_count bigint; + loblksize bigint; + r_meta record; + r_data record; + fd integer; + repair_enabled boolean := false; + lr_slots_exists boolean := false; + foreign_lr_slots_exists boolean := false; +BEGIN + -- Only superusers can UPDATE pg_catalog.pg_largeobject_metadata + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = current_user AND rolsuper) THEN + RAISE EXCEPTION 'lolor.migrate_to_native() requires superuser privileges'; + END IF; + + -- Verify lolor is enabled so _orig functions point to native API + IF NOT lolor.is_enabled() THEN + RAISE EXCEPTION 'lolor must be enabled before migration to native'; + END IF; + + -- Derive LOBLKSIZE at runtime. PostgreSQL defines it as BLCKSZ / 4. + -- Hard-coding 2048 would break on non-default block size builds. + loblksize := current_setting('block_size')::bigint / 4; + + -- Count what we are about to migrate + SELECT count(*) INTO lo_count FROM lolor.pg_largeobject_metadata; + + IF lo_count = 0 THEN + RAISE NOTICE 'no lolor large objects to migrate'; + RETURN 0; + END IF; + + -- Check for OID conflicts + IF EXISTS ( + SELECT 1 + FROM lolor.pg_largeobject_metadata lm + JOIN pg_catalog.pg_largeobject_metadata native ON native.oid = lm.oid + ) THEN + RAISE EXCEPTION 'OID conflict: some lolor large objects already exist in native storage'; + END IF; + + -- Logical slots indicate subscribers (replica nodes) that will not receive + -- the node-local migration DML. + SELECT EXISTS ( + SELECT 1 FROM pg_catalog.pg_replication_slots + WHERE slot_type = 'logical' AND database = current_database() + ) INTO lr_slots_exists; + + -- A logical slot whose output plugin is not spock's own cannot be silenced + -- by spock.repair_mode(). Track those separately: their consumers would + -- still decode the migration DML even when spock is fully operational. + SELECT EXISTS ( + SELECT 1 FROM pg_catalog.pg_replication_slots + WHERE slot_type = 'logical' AND database = current_database() + AND plugin NOT IN ('spock_output', 'spock') + ) INTO foreign_lr_slots_exists; + + -- Suppress spock replication of the bulk migration DML. The migration only + -- shuffles rows between lolor and native storage on this node. + -- + -- Use spock only when it is fully operational: the extension is installed + -- (pg_extension is superuser-gated, so the schema name cannot be squatted + -- by an unprivileged user), the function exists (older spock versions lack + -- it) and the GUC exists (the library is actually preloaded). Anything + -- less falls through to the refusal branch below. + -- + -- Without spock the migration DML cannot be excluded from logical decoding, + -- so if any logical replication slot exists the deletes from the lolor + -- tables would replicate while the re-created native large objects would + -- not, losing large objects on the subscriber side. + IF EXISTS (SELECT 1 FROM pg_catalog.pg_extension WHERE extname = 'spock') + AND to_regprocedure('spock.repair_mode(boolean)') IS NOT NULL + AND current_setting('spock.replication_repair_mode', true) IS NOT NULL + THEN + -- spock.repair_mode() only suppresses spock's own output plugin + -- ('spock_output'/'spock'). If a non-spock logical slot is present + -- (pgoutput, wal2json, decoderbufs, ...) its consumer would still decode + -- the migration DML: the lolor deletes would replicate while the + -- re-created native large objects would not, losing LOs on the subscriber. + IF foreign_lr_slots_exists THEN + RAISE EXCEPTION 'cannot migrate LOs to pg_catalog: non-spock logical replication slot(s) exist' + USING DETAIL = 'spock repair mode only silences spock''s own output plugin; ' + 'a non-spock slot (e.g. pgoutput, wal2json) would replicate the ' + 'lolor deletes while the re-created native large objects would not, ' + 'losing large objects on the subscriber side', + HINT = 'Drop all non-spock logical replication slots in this database before executing this procedure'; + END IF; + + IF current_setting('spock.replication_repair_mode', true) = 'off' THEN + PERFORM spock.repair_mode(true); + repair_enabled := true; + END IF; + ELSIF lr_slots_exists THEN + RAISE EXCEPTION 'cannot migrate LOs to pg_catalog: logical replication slot(s) exist' + USING DETAIL = 'Without spock the migration DML cannot be excluded from logical decoding: ' + 'the deletes from the lolor tables would replicate to subscribers while ' + 'the re-created native large objects would not, losing large objects on ' + 'the subscriber side', + HINT = 'Drop all logical replication slots in this database before executing this procedure'; + END IF; + + -- Migrate each object using the native LO API (_orig functions). + -- We cannot INSERT directly into pg_catalog.pg_largeobject from SQL, + -- so we use lo_create_orig + lo_open_orig + lowrite_orig. + -- + -- Zero-data-page LOs (metadata only) are handled correctly: lo_create_orig + -- creates an empty LO and the inner FOR loop simply does not execute. + -- + -- Note on sparse LOs: any gap between non-consecutive page numbers will + -- be filled with zeroes by the native LO write API. This preserves read + -- semantics (holes already returned zeroes) but may increase storage. + FOR r_meta IN SELECT oid, lomowner, lomacl FROM lolor.pg_largeobject_metadata + LOOP + -- Create native LO with the exact same OID + PERFORM pg_catalog.lo_create_orig(r_meta.oid); + + -- Write data pages through the native LO write API. + -- Use lo_lseek64_orig (bigint offset) to handle LOs larger than 2 GB. + fd := pg_catalog.lo_open_orig(r_meta.oid, x'60000'::int); + FOR r_data IN + SELECT pageno, data FROM lolor.pg_largeobject + WHERE loid = r_meta.oid ORDER BY pageno + LOOP + PERFORM pg_catalog.lo_lseek64_orig(fd, r_data.pageno::bigint * loblksize, 0); + PERFORM pg_catalog.lowrite_orig(fd, r_data.data); + END LOOP; + PERFORM pg_catalog.lo_close_orig(fd); + + -- Restore ownership and ACL (lo_create sets current user as owner) + UPDATE pg_catalog.pg_largeobject_metadata + SET lomowner = r_meta.lomowner, lomacl = r_meta.lomacl + WHERE pg_catalog.pg_largeobject_metadata.oid = r_meta.oid; + END LOOP; + + -- Clean lolor storage + DELETE FROM lolor.pg_largeobject; + DELETE FROM lolor.pg_largeobject_metadata; + + -- Re-enable replication for the remainder of the caller's transaction, so + -- repair mode covers exactly the migration DML and nothing after it. + -- Error paths need no cleanup: they abort the whole transaction. + IF repair_enabled THEN + PERFORM spock.repair_mode(false); + END IF; + + RAISE NOTICE 'migrated % large object(s) from lolor to native storage', lo_count; + + -- The migration DML was excluded from replication, so connected subscribers + -- (replica nodes) did not receive it. + IF lr_slots_exists THEN + RAISE NOTICE 'this migration is local to this node: ' + 'execute lolor.migrate_to_native() on each replica as well before modifying LOs'; + END IF; + + RETURN lo_count; +END; +$$ LANGUAGE plpgsql VOLATILE; diff --git a/regress.conf b/regress.conf new file mode 100644 index 0000000..d67dbb4 --- /dev/null +++ b/regress.conf @@ -0,0 +1,2 @@ +shared_preload_libraries = 'lolor' +lolor.node = 1 diff --git a/sql/lolor.sql b/sql/lolor.sql index cf25c11..edf3532 100644 --- a/sql/lolor.sql +++ b/sql/lolor.sql @@ -90,6 +90,7 @@ SELECT convert_from(loread(:fd, 1024), 'UTF8'); SELECT lo_close(:fd); END; +SELECT lolor.migrate_to_native(); DROP EXTENSION lolor; -- Check extension upgrade @@ -105,7 +106,11 @@ BEGIN; SELECT lo_open(:loid, 262144) AS fd \gset SELECT convert_from(loread(:fd, 1024), 'UTF8'); END; +-- 1.2.3 is the released version; 1.3.0 is reached from it +ALTER EXTENSION lolor UPDATE TO '1.2.3'; +SELECT extversion FROM pg_extension WHERE extname = 'lolor'; ALTER EXTENSION lolor UPDATE TO '1.3.0'; +SELECT extversion FROM pg_extension WHERE extname = 'lolor'; -- Verify migration functions are available after upgrade SELECT lolor.migrate_to_native(); -- One LO object has been created before LOLOR SELECT lolor.migrate_from_native(); -- two objects @@ -138,19 +143,26 @@ SELECT lolor.enable(); -- Check that no tails existing after the extension drop in both enabled and -- disabled states. +SELECT lolor.migrate_to_native(); DROP EXTENSION lolor; SELECT oid, proname FROM pg_proc WHERE proname IN ('lo_open_orig', 'lolor_lo_open'); --- Check: we can't just delete LOLOR without LO migration in disabled mode. --- XXX: should we introduce a 'forced' flag to allow this? +-- DROP EXTENSION while lolor is disabled. The drop is refused while the +-- object is still in lolor storage, whichever state lolor is in. Move the +-- object out first -- the migration runs through the renamed _orig functions +-- and so needs lolor enabled -- and the drop goes through. CREATE EXTENSION lolor; +SELECT lo_from_bytea(0, 'stored before disabling') AS disabled_drop_oid \gset SELECT lolor.disable(); DROP EXTENSION lolor; -SELECT extname FROM pg_extension; -- lolor is here SELECT lolor.enable(); +SELECT lolor.migrate_to_native(); DROP EXTENSION lolor; SELECT extname FROM pg_extension; -- check lolor removal +-- The object was migrated to native storage, not dropped with lolor's tables +SELECT convert_from(lo_get(:disabled_drop_oid), 'UTF8') AS survived_disabled_drop; +SELECT lo_unlink(:disabled_drop_oid); -- -- Migration tests: migrate_from_native / migrate_to_native / DROP EXTENSION @@ -188,7 +200,8 @@ END; -- Create an additional LO directly in lolor storage SELECT lo_from_bytea(0, 'Created directly in lolor') AS lolor_direct_oid \gset --- Reverse migration via DROP EXTENSION +-- Reverse migration, then DROP EXTENSION +SELECT lolor.migrate_to_native(); DROP EXTENSION lolor; SELECT count(*) AS native_after_drop FROM pg_catalog.pg_largeobject_metadata; @@ -207,6 +220,7 @@ SELECT lo_unlink(:'lolor_direct_oid'::oid); CREATE EXTENSION lolor; SELECT lolor.migrate_from_native(); +SELECT lolor.migrate_to_native(); DROP EXTENSION lolor; -- @@ -228,6 +242,7 @@ END; -- Cleanup SELECT lo_unlink(:'manual_oid'::oid); SELECT lolor.enable(); +SELECT lolor.migrate_to_native(); DROP EXTENSION lolor; -- @@ -244,6 +259,7 @@ INSERT INTO lolor.pg_largeobject_metadata (oid, lomowner, lomacl) SELECT lolor.migrate_from_native(); -- Cleanup: remove the conflicting row and drop cleanly DELETE FROM lolor.pg_largeobject_metadata WHERE oid = :'conflict_oid'; +SELECT lolor.migrate_to_native(); DROP EXTENSION lolor; SELECT lo_unlink(:'conflict_oid'::oid); @@ -262,18 +278,20 @@ SELECT lolor.migrate_to_native(); SELECT lolor.disable(); SELECT lo_unlink(:'conflict_oid2'::oid); SELECT lolor.enable(); +SELECT lolor.migrate_to_native(); DROP EXTENSION lolor; --- DROP EXTENSION should be rejected when migrate_to_native has OID conflict +-- A conflicting native object blocks the removal: the drop is refused while +-- objects remain in lolor storage, and they cannot be moved out while the +-- native duplicate exists CREATE EXTENSION lolor; SELECT lo_from_bytea(0, 'Drop conflict test') AS drop_conflict_oid \gset --- Create a native LO with the same OID to force conflict at DROP time SELECT lolor.disable(); -- Print a stable boolean rather than the generated OID, which varies per run SELECT lo_create(:'drop_conflict_oid') = :'drop_conflict_oid'::oid AS native_oid_honored; SELECT lolor.enable(); --- DROP EXTENSION should ERROR to prevent data loss DROP EXTENSION lolor; +SELECT lolor.migrate_to_native(); -- Extension should still be installed SELECT extname FROM pg_extension WHERE extname = 'lolor'; -- Objects should be in place @@ -282,9 +300,41 @@ SELECT count(*) FROM lolor.pg_largeobject; SELECT lolor.disable(); SELECT lo_unlink(:'drop_conflict_oid'::oid); SELECT lolor.enable(); --- Now DROP should succeed +-- Now the migration and the drop go through +SELECT lolor.migrate_to_native(); DROP EXTENSION lolor; +-- +-- The drop guard is an object access hook, so it holds even when the event +-- trigger that restores the function names does not run. +-- +CREATE EXTENSION lolor; +ALTER EVENT TRIGGER lo_on_drop_extension DISABLE; +-- Enabled with nothing stored: refused because the native functions are +-- still parked under their _orig names +DROP EXTENSION lolor; +-- Disabled with an object stored: refused on every path that reaches the +-- extension +SELECT lo_from_bytea(0, 'guarded by the drop hook') AS guarded_oid \gset +SELECT lolor.disable(); +DROP EXTENSION lolor; +DROP SCHEMA lolor CASCADE; +SELECT count(*) AS extension_still_installed FROM pg_extension WHERE extname = 'lolor'; +SELECT count(*) AS still_in_lolor_storage FROM lolor.pg_largeobject_metadata; +-- The escape hatch is a superuser setting and discards the objects +BEGIN; +SET LOCAL lolor.allow_unsafe_drop = on; +DROP EXTENSION lolor; +SELECT count(*) AS extension_gone FROM pg_extension WHERE extname = 'lolor'; +ROLLBACK; +-- Back on the supported path: migrate out by hand, then drop +ALTER EVENT TRIGGER lo_on_drop_extension ENABLE ALWAYS; +SELECT lolor.enable(); +SELECT lolor.migrate_to_native(); +DROP EXTENSION lolor; +SELECT convert_from(lo_get(:guarded_oid), 'UTF8') AS migrated_out; +SELECT lo_unlink(:guarded_oid); + -- -- lo_import() and lo_export() read and write files on the server, so core -- revokes EXECUTE on them from PUBLIC. lolor replaces them by renaming the @@ -306,6 +356,7 @@ JOIN pg_proc o ON o.pronamespace = r.pronamespace AND o.proargtypes = r.proargtypes WHERE r.proname IN ('lo_import', 'lo_export') ORDER BY 1; +SELECT lolor.migrate_to_native(); DROP EXTENSION lolor; -- @@ -317,6 +368,133 @@ SET lolor.node = 16; SET lolor.node = 15; SET lolor.node = 1; +-- +-- Permission enforcement. +-- +-- Objects in lolor storage have no catalog entry, so lolor cannot use the +-- syscache-backed owner and ACL checks and reimplements them against its own +-- tables. Exercise that path rather than assuming it matches core. +-- +CREATE EXTENSION lolor; +CREATE ROLE lolor_alice; +CREATE ROLE lolor_bob; + +-- Large object error messages quote the OID, which is generated and so +-- differs between runs. Report the message with digits masked instead. +CREATE FUNCTION lolor_expect_error(cmd text) RETURNS text AS $$ +BEGIN + EXECUTE cmd; + RETURN 'unexpectedly succeeded'; +EXCEPTION WHEN OTHERS THEN + RETURN regexp_replace(SQLERRM, '[0-9]+', 'NNN', 'g'); +END +$$ LANGUAGE plpgsql; + +SET ROLE lolor_alice; +SELECT lo_from_bytea(0, 'alice private data') AS alice_oid \gset +SELECT convert_from(lo_get(:alice_oid), 'UTF8') AS owner_can_read; +RESET ROLE; + +-- A different role gets nothing without a grant. +SET ROLE lolor_bob; +SELECT lolor_expect_error(format('SELECT lo_get(%s)', :alice_oid)) AS read_denied; +SELECT lolor_expect_error(format('SELECT lo_open(%s, 262144)', :alice_oid)) AS open_denied; +SELECT lolor_expect_error(format('SELECT lo_put(%s, 0, ''x'')', :alice_oid)) AS write_denied; +SELECT lolor_expect_error(format('SELECT lo_unlink(%s)', :alice_oid)) AS unlink_denied; +RESET ROLE; + +-- The superuser bypasses the check, as in core. +SELECT convert_from(lo_get(:alice_oid), 'UTF8') AS superuser_can_read; + +-- GRANT/ALTER on a large object act on pg_largeobject_metadata, where an +-- object in lolor storage has no row. This limitation is documented; assert +-- it so that a change in behaviour is noticed. +SELECT lolor_expect_error( + format('GRANT SELECT ON LARGE OBJECT %s TO lolor_bob', :alice_oid)) AS grant_unsupported; +SELECT lolor_expect_error( + format('ALTER LARGE OBJECT %s OWNER TO lolor_bob', :alice_oid)) AS alter_unsupported; + +SELECT lo_unlink(:alice_oid); + +-- +-- Objects in lolor storage are rows in ordinary tables and cannot participate +-- in pg_shdepend, so DROP ROLE does not notice that a role still owns one. +-- lolor.check_orphans() exists to make the consequence findable. +-- +SET ROLE lolor_alice; +SELECT lo_from_bytea(0, 'owned by a role about to vanish') AS orphan_oid \gset +RESET ROLE; +SELECT count(*) AS orphans_before FROM lolor.check_orphans(); +DROP ROLE lolor_alice; +SELECT count(*) AS orphans_after FROM lolor.check_orphans(); +SELECT lo_unlink(:orphan_oid); +SELECT count(*) AS orphans_cleared FROM lolor.check_orphans(); + +-- +-- fix_orphans() must behave like REASSIGN OWNED: grants the dead owner made +-- survive with the new owner as grantor, an entry the new owner already held +-- merges into its owner entry, and only entries that still name a missing +-- role are dropped. ACLs can only be set in native storage, so build them +-- there and migrate in. +-- +CREATE ROLE lolor_dead_owner; +CREATE ROLE lolor_carol; +CREATE ROLE lolor_dave; +SELECT lolor.disable(); +SET ROLE lolor_dead_owner; +SELECT lo_from_bytea(0, 'granted to bob') AS granted_oid \gset +GRANT SELECT ON LARGE OBJECT :granted_oid TO lolor_bob; +SELECT lo_from_bytea(0, 'grant chain') AS chain_oid \gset +GRANT SELECT ON LARGE OBJECT :chain_oid TO lolor_dave WITH GRANT OPTION; +SELECT lo_from_bytea(0, 'new owner already a grantee') AS merge_oid \gset +GRANT SELECT ON LARGE OBJECT :merge_oid TO lolor_carol WITH GRANT OPTION; +SELECT lo_from_bytea(0, 'mixed grant options') AS mixed_oid \gset +GRANT SELECT ON LARGE OBJECT :mixed_oid TO lolor_bob WITH GRANT OPTION; +GRANT UPDATE ON LARGE OBJECT :mixed_oid TO lolor_bob; +RESET ROLE; +SET ROLE lolor_dave; +GRANT SELECT ON LARGE OBJECT :chain_oid TO lolor_bob; +RESET ROLE; +SELECT lolor.enable(); +SELECT lolor.migrate_from_native(); +DROP ROLE lolor_dead_owner; +SELECT role_kind, count(*) FROM lolor.check_orphans() GROUP BY 1 ORDER BY 1; +SELECT lolor.fix_orphans('lolor_carol') AS objects_repaired; +SELECT count(*) AS orphans_left FROM lolor.check_orphans(); +-- Role OIDs vary per run, so compare the rebuilt ACLs by name +SELECT CASE m.oid WHEN :granted_oid THEN 'granted' WHEN :chain_oid THEN 'chain' + WHEN :merge_oid THEN 'merge' ELSE 'mixed' END AS obj, + pg_get_userbyid(m.lomowner) AS owner, + a.grantee::regrole::text AS grantee, a.grantor::regrole::text AS grantor, + a.privilege_type, a.is_grantable + FROM lolor.pg_largeobject_metadata m, aclexplode(m.lomacl) a + WHERE m.oid IN (:granted_oid, :chain_oid, :merge_oid, :mixed_oid) + ORDER BY 1, 3, 4, 5; +-- One entry per (grantee, grantor), as GRANT and REVOKE expect: a grant +-- option that differs between privileges lives inside the entry +SELECT count(*) AS objects_with_duplicate_entries + FROM lolor.pg_largeobject_metadata m + WHERE m.oid IN (:granted_oid, :chain_oid, :merge_oid, :mixed_oid) + AND cardinality(m.lomacl) <> (SELECT count(DISTINCT (a.grantee, a.grantor)) + FROM aclexplode(m.lomacl) a); +-- The grantee kept access and the new owner has it +SET ROLE lolor_bob; +SELECT convert_from(lo_get(:granted_oid), 'UTF8') AS bob_still_reads; +RESET ROLE; +SET ROLE lolor_carol; +SELECT convert_from(lo_get(:chain_oid), 'UTF8') AS new_owner_reads; +RESET ROLE; +SELECT lo_unlink(:granted_oid); +SELECT lo_unlink(:chain_oid); +SELECT lo_unlink(:merge_oid); +SELECT lo_unlink(:mixed_oid); +DROP ROLE lolor_carol; +DROP ROLE lolor_dave; +DROP ROLE lolor_bob; +DROP FUNCTION lolor_expect_error(text); +SELECT lolor.migrate_to_native(); +DROP EXTENSION lolor; + -- -- 64-bit interface and page-boundary I/O. lo_put(), lo_tell64() and -- lo_truncate64() had no coverage. @@ -429,7 +607,65 @@ SELECT lo_unlink(:multi_oid); -- Error paths. -- SELECT lo_get(0); +SELECT lo_unlink(0); BEGIN; SELECT lo_open(0, 262144); ROLLBACK; +SELECT lolor.migrate_to_native(); +DROP EXTENSION lolor; + +-- +-- An unprivileged user must not be able to wedge lolor by squatting the names +-- the state probes look for. Matching on proname alone would make +-- is_enabled() raise "inconsistent state" and block DROP EXTENSION. +-- +CREATE EXTENSION lolor; +CREATE ROLE lolor_squatter; +GRANT CREATE ON SCHEMA public TO lolor_squatter; +SET ROLE lolor_squatter; +CREATE FUNCTION public.lolor_lo_open(oid, int4) RETURNS int4 + AS 'SELECT 1' LANGUAGE sql; +CREATE FUNCTION public.lo_close_orig(int4) RETURNS int4 + AS 'SELECT 1' LANGUAGE sql; +RESET ROLE; +SELECT lolor.is_enabled() AS unaffected_by_squatting; +DROP FUNCTION public.lolor_lo_open(oid, int4); +DROP FUNCTION public.lo_close_orig(int4); +REVOKE CREATE ON SCHEMA public FROM lolor_squatter; +DROP ROLE lolor_squatter; +SELECT lolor.migrate_to_native(); +DROP EXTENSION lolor; + +-- +-- DROP SCHEMA lolor CASCADE reaches the extension by dependency cascade rather +-- than as DROP EXTENSION. It is refused while objects remain in lolor +-- storage, and once they are migrated out the cleanup trigger must still run, +-- or pg_catalog is left without a working lo_open(). +-- +CREATE EXTENSION lolor; +SELECT lo_from_bytea(0, 'rescued from drop schema') AS rescued_oid \gset +DROP SCHEMA lolor CASCADE; +SELECT lolor.migrate_to_native(); +DROP SCHEMA lolor CASCADE; +SELECT count(*) AS ext_left FROM pg_extension WHERE extname = 'lolor'; +SELECT to_regprocedure('pg_catalog.lo_open(oid,int4)') IS NOT NULL AS lo_open_restored; +SELECT to_regprocedure('pg_catalog.lo_open_orig(oid,int4)') IS NULL AS no_orig_left; +SELECT convert_from(lo_get(:rescued_oid), 'UTF8') AS rescued_content; +SELECT lo_unlink(:rescued_oid); + +-- +-- DROP SCHEMA without CASCADE is RESTRICT and cannot remove a schema that +-- still holds the extension's tables. The cleanup must not run for a command +-- that is going to be rejected. +-- +CREATE EXTENSION lolor; +SELECT lo_from_bytea(0, 'still here afterwards') AS kept_oid \gset +DROP SCHEMA lolor; +SELECT count(*) AS extension_still_installed + FROM pg_extension WHERE extname = 'lolor'; +SELECT convert_from(lo_get(:kept_oid), 'UTF8') AS object_untouched; +SELECT count(*) AS still_in_lolor_storage + FROM lolor.pg_largeobject_metadata WHERE oid = :kept_oid; +SELECT lo_unlink(:kept_oid); +SELECT lolor.migrate_to_native(); DROP EXTENSION lolor; diff --git a/src/lolor.c b/src/lolor.c index cdaf70d..0d7f4a0 100644 --- a/src/lolor.c +++ b/src/lolor.c @@ -17,25 +17,42 @@ #include "miscadmin.h" #include "fmgr.h" #include "access/xact.h" +#include "access/genam.h" +#include "access/htup_details.h" +#include "access/table.h" +#include "catalog/dependency.h" #include "catalog/namespace.h" +#include "catalog/objectaccess.h" +#include "catalog/pg_extension.h" +#include "catalog/pg_namespace.h" +#include "catalog/pg_type.h" #include "commands/event_trigger.h" +#include "commands/extension.h" #include "executor/spi.h" #include "nodes/parsenodes.h" #include "nodes/value.h" #include "nodes/print.h" +#include "utils/acl.h" #include "utils/builtins.h" +#include "utils/fmgroids.h" #include "utils/inval.h" #include "utils/guc.h" #include "utils/rel.h" #include "utils/lsyscache.h" +#include "utils/syscache.h" #include "lolor.h" PG_MODULE_MAGIC; int32 lolor_node_id = 0; +static bool lolor_allow_unsafe_drop = false; + +static object_access_hook_type prev_object_access_hook = NULL; void _PG_init(void); +static void lolor_object_access(ObjectAccessType access, Oid classId, + Oid objectId, int subId, void *arg); /* keep Oids of the large object catalog. */ static Oid LOLOR_LargeObjectRelationId = InvalidOid; @@ -150,6 +167,18 @@ relcache_invalidate_callback(Datum arg, Oid reloid) void _PG_init(void) { + /* + * The drop guard below is an object_access_hook. A hook installed by a + * library loaded on demand exists only in the backend that loaded it, and + * the guard has to hold in every backend and worker, so insist on being + * preloaded. This also makes a missing or broken library fail at startup + * rather than at the first lo_open(). + */ + if (!process_shared_preload_libraries_in_progress) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("lolor must be loaded via \"shared_preload_libraries\""))); + DefineCustomIntVariable("lolor.node", "Unique id of current node.", NULL, @@ -161,6 +190,15 @@ _PG_init(void) 0, NULL, NULL, NULL); + DefineCustomBoolVariable("lolor.allow_unsafe_drop", + "Allows dropping lolor while it is enabled or still holds large objects.", + NULL, + &lolor_allow_unsafe_drop, + false, + PGC_SUSET, + 0, + NULL, NULL, NULL); + /* register transaction callbacks for cleanup. */ RegisterXactCallback(lolor_xact_callback, NULL); RegisterSubXactCallback(lolor_subxact_callback, NULL); @@ -170,116 +208,283 @@ _PG_init(void) * So, it is necessary to invalidate cache of Oids. */ CacheRegisterRelcacheCallback(relcache_invalidate_callback, (Datum) 0); + + prev_object_access_hook = object_access_hook; + object_access_hook = lolor_object_access; +} + +/* + * lolor_extension_owner + * + * Owner of the installed lolor extension, or InvalidOid when it is not + * installed. + */ +static Oid +lolor_extension_owner(void) +{ + Relation rel; + ScanKeyData skey[1]; + SysScanDesc scan; + HeapTuple tup; + Oid owner = InvalidOid; + + rel = table_open(ExtensionRelationId, AccessShareLock); + + ScanKeyInit(&skey[0], + Anum_pg_extension_extname, + BTEqualStrategyNumber, F_NAMEEQ, + CStringGetDatum(EXTENSION_NAME)); + + scan = systable_beginscan(rel, ExtensionNameIndexId, true, NULL, 1, skey); + + tup = systable_getnext(scan); + if (HeapTupleIsValid(tup)) + owner = ((Form_pg_extension) GETSTRUCT(tup))->extowner; + + systable_endscan(scan); + table_close(rel, AccessShareLock); + + return owner; +} + +/* + * lolor_storage_drop_check + * + * Refuse to delete lolor.pg_largeobject_metadata while it holds rows. + * + * The lolor tables are members of the extension, and members are deleted + * before the extension itself, so this is the last moment at which the + * contents can still be counted. + */ +static void +lolor_storage_drop_check(Oid relid) +{ + Oid nspoid; + Oid extoid; + Relation rel; + SysScanDesc scan; + int64 count = 0; + + /* Fast exit for the usual case: some table that is not ours. */ + nspoid = get_namespace_oid(EXTENSION_NAME, true); + if (!OidIsValid(nspoid) || + get_relname_relid(LOLOR_LARGEOBJECT_METADATA, nspoid) != relid) + return; + + /* A same-named table in a squatted schema is not ours either. */ + extoid = get_extension_oid(EXTENSION_NAME, true); + if (!OidIsValid(extoid) || + getExtensionOfObject(RelationRelationId, relid) != extoid) + return; + + /* The dropper already holds AccessExclusiveLock. */ + rel = table_open(relid, NoLock); + scan = systable_beginscan(rel, InvalidOid, false, NULL, 0, NULL); + while (HeapTupleIsValid(systable_getnext(scan))) + count++; + systable_endscan(scan); + table_close(rel, NoLock); + + if (count > 0) + ereport(ERROR, + (errcode(ERRCODE_DEPENDENT_OBJECTS_STILL_EXIST), + errmsg_plural("cannot drop lolor storage while it holds %lld large object", + "cannot drop lolor storage while it holds %lld large objects", + count, (long long) count), + errhint("Run lolor.migrate_to_native() first, or set lolor.allow_unsafe_drop to discard them."))); +} + +/* + * lolor_extension_drop_check + * + * Refuse to delete the extension while the native large object functions + * are still parked under their *_orig names: the drop would take lolor's + * replacements with it and leave pg_catalog without any lo_open() at all. + * The probe is the one lolor.is_enabled() uses. + */ +static void +lolor_extension_drop_check(Oid extoid) +{ + Oid argtypes[2] = {OIDOID, INT4OID}; + + if (extoid != get_extension_oid(EXTENSION_NAME, true)) + return; + + if (SearchSysCacheExists3(PROCNAMEARGSNSP, + CStringGetDatum("lo_open_orig"), + PointerGetDatum(buildoidvector(argtypes, 2)), + ObjectIdGetDatum(PG_CATALOG_NAMESPACE))) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cannot drop extension \"%s\" while it is enabled", + EXTENSION_NAME), + errdetail("The native large object functions are still parked under their *_orig names."), + errhint("Run lolor.disable() first."))); +} + +/* + * lolor_object_access + * + * Guard every path that removes lolor: DROP EXTENSION, DROP SCHEMA CASCADE, + * DROP OWNED BY and anything else that reaches the extension by dependency. + * + * The event trigger migrates the large objects out and restores the native + * functions before the drop, but an event trigger can be disabled, and the + * drop usually runs in a different backend from the migration, so a flag + * set by the migration would not be visible to it. Check the catalog state + * at the moment of deletion instead, from whichever process performs it. + * lolor.allow_unsafe_drop skips both checks. + */ +static void +lolor_object_access(ObjectAccessType access, Oid classId, Oid objectId, + int subId, void *arg) +{ + if (access == OAT_DROP && !lolor_allow_unsafe_drop) + { + /* subId != 0 is a column being dropped, not the table. */ + if (classId == RelationRelationId && subId == 0) + lolor_storage_drop_check(objectId); + else if (classId == ExtensionRelationId) + lolor_extension_drop_check(objectId); + } + + if (prev_object_access_hook) + prev_object_access_hook(access, classId, objectId, subId, arg); +} + +/* + * lolor_is_being_dropped + * + * Decide whether the DDL about to run will remove the lolor extension. + * + * DROP EXTENSION is the obvious spelling, but the extension also goes away + * through DROP SCHEMA lolor CASCADE and DROP OWNED BY , + * which reach it by dependency cascade rather than by naming it. Missing one + * is not cosmetic: the large objects would be destroyed along with the lolor + * tables and the renamed pg_catalog functions never restored, leaving a + * database with no working lo_open(). + */ +static bool +lolor_is_being_dropped(Node *parsetree) +{ + ListCell *lc; + + if (IsA(parsetree, DropStmt)) + { + DropStmt *stmt = (DropStmt *) parsetree; + + /* + * DROP EXTENSION lolor and DROP SCHEMA lolor both name the object with + * a bare String, so one loop covers both; the extension and its schema + * share a name, which lolor.control enforces. + * + * Only CASCADE reaches the extension through the schema: a plain DROP + * SCHEMA is RESTRICT and cannot remove a schema that still holds the + * extension's tables. Acting on it would run the whole migration and + * take the storage locks before PostgreSQL rejected the command. + */ + if (stmt->removeType != OBJECT_EXTENSION && + (stmt->removeType != OBJECT_SCHEMA || + stmt->behavior != DROP_CASCADE)) + return false; + + foreach(lc, stmt->objects) + { + Node *objname = (Node *) lfirst(lc); + + if (IsA(objname, String) && + strcmp(strVal(objname), EXTENSION_NAME) == 0) + return true; + } + + return false; + } + + if (IsA(parsetree, DropOwnedStmt)) + { + DropOwnedStmt *stmt = (DropOwnedStmt *) parsetree; + Oid extowner = lolor_extension_owner(); + + if (!OidIsValid(extowner)) + return false; + + foreach(lc, stmt->roles) + { + RoleSpec *rolespec = lfirst_node(RoleSpec, lc); + + if (get_rolespec_oid(rolespec, true) == extowner) + return true; + } + + return false; + } + + return false; } /* * lolor_on_drop_extension * - * In order to be a drop-in replacement for the PostgreSQL built - * in large object access functions, we must replace them with - * our own ones. We do that in the extension's install script - * by renaming the build-in ones to _orig and then - * creating our versions of them. The PostgreSQL system has no - * mechanism to invoke a cleanup or uninstall script on DROP - * EXTENSION. We therefore must do the cleanup in an event trigger. - * However only C-Language event triggers that fire on - * ddl_command_start have access to the list of object that get - * dropped. + * The install script replaces the pg_catalog large object functions by + * renaming the built-in ones to _orig and creating ours under + * the original names. PostgreSQL has no uninstall script, so the renames + * are undone here, from an event trigger on ddl_command_start, which is + * the only kind that still sees the command before anything is dropped. * - * We cannot drop our own functions here as the dependencies of - * the extension itself won't allow that. Likewise we cannot - * drop the original PostgreSQL functions because the PostgreSQL - * system depends on them. But we can get around that with - * renaming (which makes no sense). + * Only the function names are restored. Objects still in lolor storage + * are not migrated here: moving data from inside a DROP is fragile, and + * the object access hook refuses the drop while any remain, so the user + * runs lolor.migrate_to_native() first. */ Datum lolor_on_drop_extension(PG_FUNCTION_ARGS) { - EventTriggerData *trigdata; - DropStmt *dropstmt; - ListCell *lc; - bool has_lolor_objs = false; + EventTriggerData *trigdata; /* Make sure we are called as an event trigger */ if (!CALLED_AS_EVENT_TRIGGER(fcinfo)) elog(ERROR, "not fired by event trigger manager"); - /* Make sure we have a parsetree and that this is for a DROP EXTENSION */ trigdata = (EventTriggerData *) fcinfo->context; if (trigdata->parsetree == NULL) { - elog(LOG, "lo_on_drop_extension(): parsetree = NULL"); + elog(LOG, "lolor_on_drop_extension(): parsetree = NULL"); PG_RETURN_NULL(); } /* - * Check that this is DROP EXTENSION lolor + * The trigger is registered for several command tags, so most invocations + * are for drops that have nothing to do with lolor. Let them proceed. */ - if (!IsA(trigdata->parsetree, DropStmt)) - { - elog(WARNING, "lo_on_drop_extension(): not a DropStmt"); - PG_RETURN_NULL(); - } - dropstmt = (DropStmt *)trigdata->parsetree; - if (dropstmt->removeType != OBJECT_EXTENSION) - { - elog(WARNING, "lo_on_drop_extension(): not a DropStmt for extension"); - PG_RETURN_NULL(); - } - foreach(lc, dropstmt->objects) - { - Node *objname = (Node *) lfirst(lc); - - if (strcmp(strVal(objname), "lolor") == 0) - { - has_lolor_objs = true; - break; - } - } - if (!has_lolor_objs) + if (!lolor_is_being_dropped(trigdata->parsetree)) PG_RETURN_NULL(); + SPI_connect(); /* - * OK, this is DROP EXTENSION lolor. - * - * First, migrate any large objects stored in lolor tables back to - * native PostgreSQL storage. This must happen while lolor is still - * enabled so the _orig functions (native LO API) are available. - * The event trigger fires on ddl_command_start, so lolor tables - * still exist and are readable at this point. - * - * Then rename our replacement functions out of the way and restore - * the original PostgreSQL function names. The DROP EXTENSION itself - * will then drop the lolor schema and its objects. - * - * Guard the migrate_to_native() call with a pg_proc check so that - * upgrades from versions < 1.3.0 (where the function does not exist) - * do not fail. + * disable() renames the native functions back. That has the same + * requirements whether the user calls it or the drop does: no other + * session may be connected, because a renamed function keeps its OID and + * other sessions keep calling the previous implementation, and it has to + * run from a client session. Check them here first, under the drop's own + * name, so that the refusal says what was refused. A disabled lolor + * renames nothing, so the check only applies while it is enabled. The + * helper exists from 1.3.0; dropping an older installed version runs that + * version's disable(), which has no such requirement. */ - SPI_connect(); - if (SPI_execute("SELECT 1 FROM pg_proc p " - "JOIN pg_namespace n ON n.oid = p.pronamespace " - "WHERE n.nspname = 'lolor' " - "AND p.proname = 'migrate_to_native'", - true, 1) == SPI_OK_SELECT && + "JOIN pg_namespace n ON n.oid = p.pronamespace " + "WHERE n.nspname = 'lolor' " + "AND p.proname = '_require_no_other_sessions'", + true, 1) == SPI_OK_SELECT && SPI_processed > 0) - { - /* - * If migrate_to_native() fails (e.g. OID conflict), the ERROR - * propagates and aborts the DROP EXTENSION. This is intentional: - * losing large objects silently is worse than a failed DROP. The - * user must resolve the conflict and retry. - */ - if (SPI_execute("SELECT lolor.migrate_to_native()", false, 0) != SPI_OK_SELECT) - ereport(ERROR, - (errmsg("lolor: failed to migrate large objects back to native storage"))); - } + SPI_execute("SELECT CASE WHEN lolor.is_enabled() THEN " + "lolor._require_no_other_sessions(" + "'drop the lolor extension') END", + false, 0); SPI_execute("SELECT CASE WHEN lolor.is_enabled() " "THEN lolor.disable() ELSE true END", false, 0); - SPI_finish(); PG_RETURN_NULL(); diff --git a/src/lolor_inv_api.c b/src/lolor_inv_api.c index 6307186..659bd99 100644 --- a/src/lolor_inv_api.c +++ b/src/lolor_inv_api.c @@ -39,9 +39,7 @@ #include "access/sysattr.h" #include "access/table.h" #include "access/xact.h" -#include "catalog/dependency.h" #include "catalog/indexing.h" -#include "catalog/objectaccess.h" #include "catalog/pg_largeobject.h" #include "catalog/pg_largeobject_metadata.h" #include "libpq/libpq-fs.h" @@ -217,18 +215,23 @@ lolor_inv_create(Oid lobjId) lobjId_new = LOLOR_LargeObjectCreate(lobjId); /* - * dependency on the owner of largeobject + * No shared dependency is recorded for the owner, and no object access + * hook is invoked. Both take a classId, and core passes + * LargeObjectRelationId: a genuine catalog that the dependency machinery + * and security modules know how to describe. An object in lolor storage + * is a row in an ordinary table, and there is no classId that describes + * it. Passing the OID of lolor.pg_largeobject produced pg_shdepend rows + * that DROP ROLE could not interpret, failing with "unrecognized object + * class" and leaving the role undroppable; they were never removed + * either, because inv_drop() deleted with PERFORM_DELETION_SKIP_ORIGINAL. + * pg_shdepend is a shared catalog, so the stored classId was a + * per-database relation OID meaningless elsewhere. Handing that same OID + * to an object access hook would mislead a security module the same way. * - * Note that LO dependencies are recorded using classId - * LOLOR_LargeObjectRelationId for backwards-compatibility reasons. Using - * LOLOR_LargeObjectMetadataRelationId instead would simplify matters for the - * backend, but it'd complicate pg_dump and possibly break other clients. + * Not tracking ownership is a real limitation, and inherent in storing + * large objects outside the catalogs. lolor.check_orphans() reports + * objects whose owner no longer exists. */ - recordDependencyOnOwner(get_LOLOR_LargeObjectRelationId(), - lobjId_new, GetUserId()); - - /* Post creation hook for new large object */ - InvokeObjectPostCreateHook(get_LOLOR_LargeObjectRelationId(), lobjId_new, 0); /* * Advance command counter to make new tuple visible to later operations. @@ -347,16 +350,14 @@ lolor_inv_close(LargeObjectDesc *obj_desc) int lolor_inv_drop(Oid lobjId) { - ObjectAddress object; - /* - * Delete any comments and dependencies on the large object + * There are no comments, security labels or dependencies to remove: an + * object in lolor storage is not a catalog object, so nothing can be + * attached to it. See the note in lolor_inv_create(). The performDeletion() + * call that stood here searched pg_depend under a classId that is an + * ordinary relation OID and could never match. */ - object.classId = get_LOLOR_LargeObjectRelationId(); - object.objectId = lobjId; - object.objectSubId = 0; - performDeletion(&object, DROP_CASCADE, PERFORM_DELETION_SKIP_ORIGINAL); - LOLOR_LargeObjectDrop(object.objectId); + LOLOR_LargeObjectDrop(lobjId); /* * Advance command counter so that tuple removal will be seen by later diff --git a/t/001_lolor_basic.pl b/t/001_lolor_basic.pl index 8502b47..a130ba7 100644 --- a/t/001_lolor_basic.pl +++ b/t/001_lolor_basic.pl @@ -19,32 +19,20 @@ # ############################################################################## # -# Lolor is loaded dynamically, on demand +# lolor must be preloaded: its drop guard is an object_access_hook, which has +# to be present in every backend. # # ############################################################################## $node->start; -$node->safe_psql('postgres', "CREATE EXTENSION lolor"); -# Check -($result, $stdout, $stderr) = $node->psql('postgres', qq( - SET lolor.node = 0; - SELECT lo_creat(-1) -)); -like($stderr, qr/value for lolor.node is not set/, "Zero value of lolor node is treated as an unset"); - -$result = $node->safe_psql('postgres', qq( - SET lolor.node = 1; - SELECT lo_creat(-1); -)); -ok($result > 0, "Lolor works and produces LO IDs"); +($result, $stdout, $stderr) = + $node->psql('postgres', "CREATE EXTENSION lolor"); +like( + $stderr, + qr/lolor must be loaded via "shared_preload_libraries"/, + "CREATE EXTENSION is refused when lolor was not preloaded"); -$node->safe_psql('postgres', "DROP EXTENSION lolor"); -$result = $node->safe_psql('postgres', qq( - SET lolor.node = 0; - SELECT lo_creat(-1); -)); -ok($result > 0, "Lolor has been removed and standard lo_creat routine is used"); $node->stop(); # ############################################################################## @@ -60,7 +48,34 @@ is($result, '', 'Basic check on create extension script'); -$result = $node->safe_psql('postgres', "DROP EXTENSION lolor"); +# Check +($result, $stdout, $stderr) = $node->psql( + 'postgres', qq( + SET lolor.node = 0; + SELECT lo_creat(-1) +)); +like( + $stderr, + qr/value for lolor.node is not set/, + "Zero value of lolor node is treated as an unset"); + +$result = $node->safe_psql( + 'postgres', qq( + SET lolor.node = 1; + SELECT lo_creat(-1); +)); +ok($result > 0, "Lolor works and produces LO IDs"); + +# The drop is refused while lolor storage holds objects +$node->safe_psql('postgres', "SELECT lolor.migrate_to_native()"); +$node->safe_psql('postgres', "DROP EXTENSION lolor"); +$result = $node->safe_psql( + 'postgres', qq( + SET lolor.node = 0; + SELECT lo_creat(-1); +)); +ok($result > 0, + "Lolor has been removed and standard lo_creat routine is used"); $node->stop(); diff --git a/t/002_pg_upgrade.pl b/t/002_pg_upgrade.pl index b2cda08..c16936b 100644 --- a/t/002_pg_upgrade.pl +++ b/t/002_pg_upgrade.pl @@ -20,7 +20,11 @@ # Prepare old node to be upgraded $old->init; -$old->append_conf('postgresql.conf', qq{lolor.node = 1}); +$old->append_conf( + 'postgresql.conf', qq{ +shared_preload_libraries = 'lolor' +lolor.node = 1 +}); $old->start; $old->safe_psql('postgres', "CREATE EXTENSION lolor"); $old->safe_psql('postgres', @@ -35,7 +39,11 @@ $old->stop(); $new->init; -$new->append_conf('postgresql.conf', qq{lolor.node = 1}); +$new->append_conf( + 'postgresql.conf', qq{ +shared_preload_libraries = 'lolor' +lolor.node = 1 +}); command_ok( [ diff --git a/t/003_dump_restore.pl b/t/003_dump_restore.pl index 9dcdd22..e834669 100644 --- a/t/003_dump_restore.pl +++ b/t/003_dump_restore.pl @@ -16,7 +16,11 @@ # Setup source node with lolor extension $src->init; -$src->append_conf('postgresql.conf', qq{lolor.node = 1}); +$src->append_conf( + 'postgresql.conf', qq{ +shared_preload_libraries = 'lolor' +lolor.node = 1 +}); $src->start; $src->safe_psql('postgres', "CREATE EXTENSION lolor"); @@ -53,7 +57,11 @@ # Setup destination node and restore $dst->init; -$dst->append_conf('postgresql.conf', qq{lolor.node = 1}); +$dst->append_conf( + 'postgresql.conf', qq{ +shared_preload_libraries = 'lolor' +lolor.node = 1 +}); $dst->start; command_ok( diff --git a/t/004_streaming_replication.pl b/t/004_streaming_replication.pl index 2d04ec6..5a8a7c2 100644 --- a/t/004_streaming_replication.pl +++ b/t/004_streaming_replication.pl @@ -15,7 +15,11 @@ # Setup primary node with lolor extension $primary->init(allows_streaming => 1); -$primary->append_conf('postgresql.conf', qq{lolor.node = 1}); +$primary->append_conf( + 'postgresql.conf', qq{ +shared_preload_libraries = 'lolor' +lolor.node = 1 +}); $primary->start; $primary->safe_psql('postgres', "CREATE EXTENSION lolor"); diff --git a/t/005_logical_replication.pl b/t/005_logical_replication.pl index e60d019..28c088e 100644 --- a/t/005_logical_replication.pl +++ b/t/005_logical_replication.pl @@ -16,7 +16,11 @@ # Setup publisher with logical replication support $publisher->init(allows_streaming => 'logical'); -$publisher->append_conf('postgresql.conf', qq{lolor.node = 1}); +$publisher->append_conf( + 'postgresql.conf', qq{ +shared_preload_libraries = 'lolor' +lolor.node = 1 +}); $publisher->start; $publisher->safe_psql('postgres', "CREATE EXTENSION lolor"); @@ -30,7 +34,11 @@ # Setup subscriber with lolor extension (tables must exist before subscription) $subscriber->init; -$subscriber->append_conf('postgresql.conf', qq{lolor.node = 2}); +$subscriber->append_conf( + 'postgresql.conf', qq{ +shared_preload_libraries = 'lolor' +lolor.node = 2 +}); $subscriber->start; $subscriber->safe_psql('postgres', "CREATE EXTENSION lolor"); diff --git a/t/006_promote_standby.pl b/t/006_promote_standby.pl index 147c0c3..bf71524 100644 --- a/t/006_promote_standby.pl +++ b/t/006_promote_standby.pl @@ -15,7 +15,11 @@ # Setup primary node with lolor extension $primary->init(allows_streaming => 1); -$primary->append_conf('postgresql.conf', qq{lolor.node = 1}); +$primary->append_conf( + 'postgresql.conf', qq{ +shared_preload_libraries = 'lolor' +lolor.node = 1 +}); $primary->start; $primary->safe_psql('postgres', "CREATE EXTENSION lolor"); diff --git a/t/007_file_privileges.pl b/t/007_file_privileges.pl index 1870e40..2a539fd 100644 --- a/t/007_file_privileges.pl +++ b/t/007_file_privileges.pl @@ -16,7 +16,11 @@ my ($result, $stdout, $stderr); $node->init; -$node->append_conf('postgresql.conf', qq{lolor.node = 1}); +$node->append_conf( + 'postgresql.conf', qq{ +shared_preload_libraries = 'lolor' +lolor.node = 1 +}); $node->start; $node->safe_psql('postgres', "CREATE EXTENSION lolor"); $node->safe_psql('postgres', "CREATE ROLE lo_plain LOGIN"); diff --git a/t/008_drop_paths.pl b/t/008_drop_paths.pl new file mode 100644 index 0000000..4811960 --- /dev/null +++ b/t/008_drop_paths.pl @@ -0,0 +1,161 @@ +# Check the drop guard and the cleanup for every spelling of the drop +# +# Copyright (c) 2022-2026, pgEdge, Inc. +# + +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('drop_paths'); + +$node->init; +$node->append_conf( + 'postgresql.conf', qq{ +shared_preload_libraries = 'lolor' +lolor.node = 1 +}); +$node->start; + +# DROP SCHEMA reaches the extension by dependency cascade rather than as DROP +# EXTENSION. It must be refused while objects remain in lolor storage, and +# once they are migrated out the cleanup has to run anyway, or pg_catalog is +# left without a working lo_open(). + +$node->safe_psql('postgres', "CREATE EXTENSION lolor"); +my $rescued = $node->safe_psql('postgres', + "SELECT lo_from_bytea(0, 'rescued from drop schema')"); + +my ($result, $stdout, $stderr) = + $node->psql('postgres', "DROP SCHEMA lolor CASCADE"); +isnt($result, 0, "DROP SCHEMA CASCADE is refused while objects remain"); +like( + $stderr, + qr/cannot drop lolor storage while it holds 1 large object/, + "the refusal names the storage"); +is( $node->safe_psql( + 'postgres', + "SELECT count(*) FROM lolor.pg_largeobject_metadata WHERE oid = $rescued" + ), + '1', + "the object is untouched"); + +$node->safe_psql('postgres', "SELECT lolor.migrate_to_native()"); +$node->safe_psql('postgres', "DROP SCHEMA lolor CASCADE"); + +is( $node->safe_psql( + 'postgres', + "SELECT count(*) FROM pg_extension WHERE extname = 'lolor'"), + '0', + "DROP SCHEMA CASCADE removed the extension"); + +is( $node->safe_psql( + 'postgres', + "SELECT to_regprocedure('pg_catalog.lo_open(oid,int4)') IS NOT NULL"), + 't', + "the native lo_open() was put back"); + +is( $node->safe_psql( + 'postgres', + "SELECT to_regprocedure('pg_catalog.lo_open_orig(oid,int4)') IS NULL" + ), + 't', + "no *_orig functions were left behind"); + +is( $node->safe_psql( + 'postgres', "SELECT convert_from(lo_get($rescued), 'UTF8')"), + 'rescued from drop schema', + "the large object survived in native storage"); + +$node->safe_psql('postgres', "SELECT lo_unlink($rescued)"); + +# DROP OWNED BY the extension owner reaches it the same way. + +# There is no ALTER EXTENSION ... OWNER TO, so install it as the role whose +# objects are about to be dropped. lolor is not trusted, hence SUPERUSER. +$node->safe_psql('postgres', "CREATE ROLE lolor_ext_owner SUPERUSER LOGIN"); +$node->safe_psql( + 'postgres', + "CREATE EXTENSION lolor", + extra_params => [ '-U', 'lolor_ext_owner' ]); + +my $owned = $node->safe_psql('postgres', + "SELECT lo_from_bytea(0, 'rescued from drop owned')"); + +($result, $stdout, $stderr) = + $node->psql('postgres', "DROP OWNED BY lolor_ext_owner"); +isnt($result, 0, "DROP OWNED BY is refused while objects remain"); +like( + $stderr, + qr/cannot drop lolor storage while it holds 1 large object/, + "the refusal names the storage"); + +$node->safe_psql('postgres', "SELECT lolor.migrate_to_native()"); +$node->safe_psql('postgres', "DROP OWNED BY lolor_ext_owner"); + +is( $node->safe_psql( + 'postgres', + "SELECT count(*) FROM pg_extension WHERE extname = 'lolor'"), + '0', + "DROP OWNED BY the extension owner removed the extension"); + +is( $node->safe_psql( + 'postgres', + "SELECT to_regprocedure('pg_catalog.lo_open(oid,int4)') IS NOT NULL"), + 't', + "the native lo_open() was put back"); + +is( $node->safe_psql( + 'postgres', "SELECT convert_from(lo_get($owned), 'UTF8')"), + 'rescued from drop owned', + "the large object survived in native storage"); + +# enable() and disable() rename functions in pg_catalog. A rename keeps the +# OID, so a session that has already resolved lo_open() keeps calling the +# previous implementation. Both therefore refuse while another session is +# connected to the database. + +$node->safe_psql('postgres', "CREATE EXTENSION lolor"); +my $other = $node->background_psql('postgres'); +$other->query_safe("SELECT 1"); + +($result, $stdout, $stderr) = + $node->psql('postgres', "SELECT lolor.disable()"); +isnt($result, 0, "disable() refuses while another session is connected"); +like( + $stderr, + qr/while other sessions are connected to the database/, + "the refusal says why"); + +# Dropping the extension renames the functions back too, so it is refused +# under the same condition, and says so under its own name. +($result, $stdout, $stderr) = + $node->psql('postgres', "DROP EXTENSION lolor"); +isnt($result, 0, "the drop refuses while another session is connected"); +like( + $stderr, + qr/cannot drop the lolor extension while other sessions are connected/, + "and the refusal names the drop"); + +$other->quit; +# The backend exits after the client has gone; a disable() issued at once +# could still count it. +$node->poll_query_until( + 'postgres', qq( + SELECT count(*) = 0 FROM pg_stat_activity + WHERE datname = current_database() + AND backend_type = 'client backend' + AND pid <> pg_backend_pid())) + or die "the other session's backend did not exit"; +is($node->safe_psql('postgres', "SELECT lolor.disable()"), + 't', "disable() works once the other session is gone"); +is($node->safe_psql('postgres', "SELECT lolor.enable()"), + 't', "and so does enable()"); +$node->safe_psql('postgres', "DROP EXTENSION lolor"); + +$node->stop; + +done_testing();