Fix 'around thread' callback ordering

PR Betterment#42 inadvertently flipped the order of `:thread` and `:perform`, and
also pushed `:thread` far enough in that cleanup steps would happen
after the `with_connection`'s `end` block in the connection plugin.

This introduces the possibility of thread safety issues, or connections
being held longer than intended (exhausting the connection pool /
connection limits).

Author: smudge

lib/delayed.rb

@@ -42,8 +42,8 @@ module Delayed
   mattr_accessor(:default_log_level) { 'info'.freeze }
   mattr_accessor(:plugins) do
     [
-      Delayed::Plugins::Instrumentation,
       Delayed::Plugins::Connection,
+      Delayed::Plugins::Instrumentation,
     ]
   end
 

lib/delayed/worker.rb

@@ -101,7 +101,12 @@ def work_off(num = 100)
         pool = Concurrent::FixedThreadPool.new(jobs.length)
         jobs.each do |job|
           pool.post do
-            success.increment if run_job(job)
+            self.class.lifecycle.run_callbacks(:thread, self, job) do
+              success.increment if perform(job)
+            end
+          rescue Exception => e # rubocop:disable Lint/RescueException
+            job_say job, "Job thread crashed with #{e.class.name}: #{e.message}", 'error'
+            job.error = e
           end
         end
 
@@ -117,12 +122,8 @@ def work_off(num = 100)
       [success.value, total - success.value]
     end
 
-    def run_thread_callbacks(job, &block)
-      self.class.lifecycle.run_callbacks(:thread, self, job, &block)
-    end
-
-    def run(job)
-      run_thread_callbacks(job) do
+    def perform(job)
+      self.class.lifecycle.run_callbacks(:perform, self, job) do
         metadata = {
           status: 'RUNNING',
           name: job.name,
@@ -209,10 +210,6 @@ def handle_failed_job(job, error)
       reschedule(job)
     end
 
-    def run_job(job)
-      self.class.lifecycle.run_callbacks(:perform, self, job) { run(job) }
-    end
-
     # The backend adapter may return either a list or a single job
     # In some backends, this can be controlled with the `max_claims` config
     # Either way, we map this to an array of job instances