(#920) Download envelope and graph dumps using S3 client

Author: excelsior

app/services/envelope_dumps/base.rb

@@ -3,14 +3,20 @@ module EnvelopeDumps
   class Base # rubocop:todo Metrics/ClassLength
     attr_reader :envelope_download, :last_dumped_at
 
-    delegate :envelope_community, to: :envelope_download
+    delegate :envelope_community, :url, to: :envelope_download
 
     def initialize(envelope_download, last_dumped_at)
       @envelope_download = envelope_download
       @last_dumped_at = last_dumped_at
     end
 
     def bucket
+      @bucket ||= Aws::S3::Resource
+                  .new(region: ENV.fetch('AWS_REGION'))
+                  .bucket(bucket_name)
+    end
+
+    def bucket_name
       raise NotImplementedError
     end
 
@@ -40,15 +46,12 @@ def dirname
     end
 
     def download_file # rubocop:todo Metrics/AbcSize
-      return unless envelope_download.url?
-
-      log("Downloading the existing dump from #{envelope_download.url}")
+      return unless url.present?
 
-      File.open(filename, 'wb') do |file|
-        URI.parse(envelope_download.url).open do |data|
-          file.write(data.read)
-        end
-      end
+      log("Downloading the existing dump from #{url}")
+      previous_filename = url.split('/').last
+      object = bucket.object(previous_filename)
+      object.get(response_target: filename)
 
       log("Unarchiving the downloaded dump into #{dirname}")
       system("unzip -qq #{filename} -d #{dirname}", exception: true)
@@ -83,10 +86,6 @@ def published_envelopes
       end
     end
 
-    def region
-      ENV.fetch('AWS_REGION')
-    end
-
     def remove_entries
       log('Removing recently deleted envelopes from the dump')
 
@@ -116,7 +115,7 @@ def run
     end
 
     def up_to_date?
-      envelope_download.url? && published_envelopes.none? && destroy_envelope_events.none?
+      url.present? && published_envelopes.none? && destroy_envelope_events.none?
     end
 
     def upload_file
@@ -129,7 +128,7 @@ def upload_file
 
       log('Uploading the updated dump to S3.')
 
-      object = Aws::S3::Resource.new(region:).bucket(bucket).object(filename)
+      object = bucket.object(filename)
       object.upload_file(filename)
       object.public_url
     end

app/services/envelope_dumps/envelope_builder.rb

@@ -2,7 +2,7 @@
 
 module EnvelopeDumps
   class EnvelopeBuilder < Base # rubocop:todo Style/Documentation
-    def bucket
+    def bucket_name
       ENV.fetch('ENVELOPE_DOWNLOADS_BUCKET')
     end
 

app/services/envelope_dumps/graph_builder.rb

@@ -2,7 +2,7 @@
 
 module EnvelopeDumps
   class GraphBuilder < Base # rubocop:todo Style/Documentation
-    def bucket
+    def bucket_name
       ENV.fetch('ENVELOPE_GRAPHS_BUCKET')
     end
 

spec/services/envelope_dumps/envelope_builder_spec.rb

@@ -115,6 +115,8 @@
       # rubocop:todo RSpec/NestedGroups
       context 'with previous download' do # rubocop:todo RSpec/MultipleMemoizedHelpers, RSpec/NestedGroups
         # rubocop:enable RSpec/NestedGroups
+        let(:previous_key) { Faker::Lorem.characters.first(32) }
+
         let(:dump) do
           buffer = StringIO.new
 
@@ -133,7 +135,7 @@
             :envelope_download,
             envelope_community:,
             started_at: now + 1.second,
-            url: Faker::Internet.url
+            url: "#{Faker::Internet.url}/#{previous_key}"
           )
         end
 
@@ -144,13 +146,17 @@
         before do
           PaperTrail.enabled = true
 
+          allow(bucket).to receive(:object).with(previous_key).and_return(s3_object)
+
+          allow(s3_object).to receive(:get).with(response_target: key) do
+            File.write(key, dump)
+          end
+
           envelope2.update_column(:updated_at, envelope_download.started_at)
 
           travel_to envelope_download.started_at do
             envelope3.destroy
           end
-
-          stub_request(:get, envelope_download.url).to_return(body: dump)
         end
 
         after do

spec/services/envelope_dumps/graph_builder_spec.rb

@@ -88,6 +88,8 @@
 
       # rubocop:todo RSpec/NestedGroups
       context 'with previous download' do # rubocop:todo RSpec/MultipleMemoizedHelpers, RSpec/NestedGroups
+        let(:previous_key) { Faker::Lorem.characters.first(32) }
+
         # rubocop:enable RSpec/NestedGroups
         let(:dump) do
           buffer = StringIO.new
@@ -107,7 +109,7 @@
             :envelope_download,
             envelope_community:,
             started_at: now + 1.second,
-            url: Faker::Internet.url
+            url: "#{Faker::Internet.url}/#{previous_key}"
           )
         end
 
@@ -118,6 +120,12 @@
         before do
           PaperTrail.enabled = true
 
+          allow(bucket).to receive(:object).with(previous_key).and_return(s3_object)
+
+          allow(s3_object).to receive(:get).with(response_target: key) do
+            File.write(key, dump)
+          end
+
           envelope2.update_column(:updated_at, envelope_download.started_at)
 
           travel_to envelope_download.started_at do