Protect the shutdown reap from release failures too

Extract the rescued release into attempt_to_release_claimed_jobs_by and
use it from reap_terminated_forks as well: a database that is down when
the supervisor shuts down should not abort the reap loop or skip exit
hooks, for the same reason it should not stop a replacement.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: rosa

lib/solid_queue/fork_supervisor.rb

@@ -57,8 +57,7 @@ def reap_terminated_forks
           terminated_fork.mark_as_reaped
 
           if !status.exited? || status.exitstatus.to_i > 0
-            error = Processes::ProcessExitError.new(status)
-            release_claimed_jobs_by(terminated_fork, with_error: error)
+            attempt_to_release_claimed_jobs_by(terminated_fork, status)
           end
         end
 
@@ -74,21 +73,23 @@ def replace_fork(pid, status)
           terminated_fork.mark_as_reaped
           payload[:fork] = terminated_fork
 
-          begin
-            release_claimed_jobs_by(terminated_fork, with_error: Processes::ProcessExitError.new(status))
-          rescue StandardError => error
-            # The database may be unreachable — likely the same reason the fork
-            # terminated. Starting the replacement can't depend on it: the jobs
-            # claimed by the terminated fork will be failed when its stale
-            # registration is pruned once the database is back.
-            handle_thread_error(error)
-          end
+          attempt_to_release_claimed_jobs_by(terminated_fork, status)
 
           start_process(configured_processes.delete(pid))
         end
       end
     end
 
+    # The database may be unreachable — likely the same reason the fork
+    # terminated. Neither starting a replacement nor shutting down can depend
+    # on it: the jobs claimed by the terminated fork will be failed when its
+    # stale registration is pruned once the database is back.
+    def attempt_to_release_claimed_jobs_by(terminated_fork, status)
+      release_claimed_jobs_by(terminated_fork, with_error: Processes::ProcessExitError.new(status))
+    rescue StandardError => error
+      handle_thread_error(error)
+    end
+
     def all_processes_terminated?
       process_instances.empty?
     end

test/unit/fork_supervisor_test.rb

@@ -152,6 +152,28 @@ def register
     end
   end
 
+  test "reap a terminated fork on shutdown even if releasing its claimed jobs fails" do
+    configuration = SolidQueue::Configuration.new(workers: [ { queues: "*", processes: 1 } ], dispatchers: [], skip_recurring: true)
+    supervisor = SolidQueue::ForkSupervisor.new(configuration)
+
+    terminated_fork = stub(kind: "Worker", name: "worker-42", hostname: "localhost", mark_as_reaped: true)
+    supervisor.send(:process_instances)[42] = terminated_fork
+    supervisor.send(:configured_processes)[42] = configuration.configured_processes.first
+
+    # The fork was killed and the database is unreachable when the supervisor
+    # reaps it during shutdown
+    status = stub(exited?: false, exitstatus: nil, pid: 42, signaled?: true, stopsig: nil, termsig: 9)
+    ::Process.stubs(:waitpid2).returns([ 42, status ]).then.returns(nil)
+    supervisor.expects(:release_claimed_jobs_by).raises(ActiveRecord::ConnectionNotEstablished.new("connection is closed"))
+
+    assert_nothing_raised do
+      supervisor.send(:reap_terminated_forks)
+    end
+
+    assert_empty supervisor.send(:process_instances)
+    assert_empty supervisor.send(:configured_processes)
+  end
+
   test "fail orphaned executions" do
     3.times { |i| StoreResultJob.set(queue: :new_queue).perform_later(i) }
     process = SolidQueue::Process.register(kind: "Worker", pid: 42, name: "worker-123")