Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@ def self.materialized?
true
end

def self.partitioned?
true
end

NON_COUNT_FIELDS = %i[
block_region_id
district_region_id
Expand Down
6 changes: 0 additions & 6 deletions app/models/reports/patient_state.rb
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,6 @@ def self.partitioned?
true
end

def self.partitioned_refresh(refresh_month)
ActiveRecord::Base.connection.exec_query(
"CALL simple_reporting.add_shard_to_table('#{refresh_month}', 'reporting_patient_states')"
)
end

def self.by_assigned_region(region_or_source)
region = region_or_source.region
where("assigned_#{region.region_type}_region_id" => region.id)
Expand Down
6 changes: 6 additions & 0 deletions app/models/reports/view.rb
Original file line number Diff line number Diff line change
Expand Up @@ -53,5 +53,11 @@ def self.fetch_refresh_month_for_date(date_of_refresh)
month_offset = (day_of_month / 2) + 1
day_of_month.odd? ? month_of_refresh.prev_month : (month_of_refresh - month_offset.month)
end

def self.partitioned_refresh(refresh_month)
ActiveRecord::Base.connection.exec_query(
"CALL simple_reporting.add_shard_to_table('#{refresh_month}', '#{table_name}')"
)
end
end
end

Large diffs are not rendered by default.

246 changes: 245 additions & 1 deletion db/structure.sql

Large diffs are not rendered by default.

21 changes: 9 additions & 12 deletions lib/tasks/reporting.rake
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,16 @@

namespace :reporting do
desc "Do a full refresh of the partitioned reporting table"
task full_partitioned_refresh: :environment do
reporting_tables = %w[
Reports::PatientState
].freeze
task :full_partitioned_refresh, [:table_name] => :environment do |t, args|
unless args[:table_name].present?
puts "ERROR: table_name is required."
puts "Usage: rake reporting:full_partitioned_refresh[reporting_patient_states]"
exit 1
end

reporting_tables.each do |reporting_table|
klass = reporting_table.constantize
ActiveRecord::Base.connection.exec_query(
"TRUNCATE TABLE simple_reporting.#{klass.table_name}"
)
Reports::Month.order(:month_date).select(:month_date).each_with_index do |reporting_month, index|
RefreshReportingPartitionedTableJob.set(wait_until: Time.now + (index * 45).minutes).perform_async(reporting_month.month_date.to_s, klass.table_name)
end
reporting_table = args[:table_name]
Reports::Month.order(:month_date).select(:month_date).each_with_index do |reporting_month, index|
RefreshReportingPartitionedTableJob.set(wait_until: Time.now + (index * 45).minutes).perform_async(reporting_month.month_date.to_s, reporting_table)
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How do we know 45 minutes works for a table? or How do we know if it's too much?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Honestly, 45 mins is a little too much for a partition refresh currently.
This just schedules every partition refresh at a marginal interval to prevent every refresh happening at once and not block any other jobs that would be come as part of regular functionality.
Original thought behind this is to use this job just once for each table initially when they are moved from materialized view to a partitioned table

end
end

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -225,4 +225,10 @@
expect(total.monthly_follow_ups_htn_only + total.monthly_follow_ups_htn_and_dm).to eq(dashboard_htn_follow_ups)
expect(total.monthly_follow_ups_dm_only + total.monthly_follow_ups_htn_and_dm).to eq(dashboard_dm_follow_ups)
end

describe "#partitioned?" do
it "returns true" do
expect(described_class.partitioned?).to be(true)
end
end
end
12 changes: 0 additions & 12 deletions spec/models/reports/patient_state_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -863,18 +863,6 @@ def patient_states(patient, from: nil, to: nil)
end
end

describe "#partitioned_refresh" do
let(:refresh_month) { Date.new(2024, 6, 1) }

it "executes the correct SQL to refresh the partition" do
expect(ActiveRecord::Base.connection).to receive(:exec_query).with(
"CALL simple_reporting.add_shard_to_table('#{refresh_month}', 'reporting_patient_states')"
)

described_class.partitioned_refresh(refresh_month)
end
end

context "screening" do
it "doesn't consider the patient if it's only screened" do
diagnosed_patient = create(:patient, recorded_at: Date.new(2024, 5, 1), diagnosed_confirmed_at: Date.new(2024, 6, 1))
Expand Down
14 changes: 14 additions & 0 deletions spec/models/reports/view_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -80,4 +80,18 @@ def table_descriptions_sql(table_name)
end
end
end

describe "#partitioned_refresh" do
let(:refresh_month) { Date.new(2024, 6, 1) }

it "executes the correct SQL to refresh the partition" do
[Reports::PatientState, Reports::FacilityMonthlyFollowUpAndRegistration].each do |klass|
expect(ActiveRecord::Base.connection).to receive(:exec_query).with(
"CALL simple_reporting.add_shard_to_table('#{refresh_month}', '#{klass.table_name}')"
)

klass.partitioned_refresh(refresh_month)
end
end
end
end