Emissions GQL, load and cache from etsource

Gemfile

@@ -75,7 +75,7 @@ gem 'ruby-progressbar'
 
 # own gems
 gem 'quintel_merit', ref: '54d2be1', github: 'quintel/merit'
-gem 'atlas',         ref: '33f32a4', github: 'quintel/atlas'
+gem 'atlas',         ref: '90fb02b', github: 'quintel/atlas'
 gem 'fever',         ref: '2a91194', github: 'quintel/fever'
 gem 'refinery',      ref: 'c39c9b1', github: 'quintel/refinery'
 gem 'rubel',         ref: 'e36554a', github: 'quintel/rubel'

Gemfile.lock

@@ -1,7 +1,7 @@
 GIT
   remote: https://github.com/quintel/atlas.git
-  revision: 33f32a48c868e92f055a68c517c6b9fd5ae01bfa
-  ref: 33f32a4
+  revision: 90fb02bf9a66189502b35351d20fe4dc1a27fe49
+  ref: 90fb02b
   specs:
     atlas (1.0.0)
       activemodel (>= 7)

app/models/etsource/dataset.rb

@@ -39,6 +39,15 @@ def self.weather_properties(region_code, variant_name)
       end
     end
 
+    # Gets the emission keys from the default dataset
+    def self.emissions_keys
+      NastyCache.instance.fetch('emission_keys') do
+        Atlas::Dataset.find(
+          Etsource::Config.default_dataset_key
+        ).emissions.to_hash.keys
+      end
+    end
+
     def self.region_codes(refresh: false)
       NastyCache.instance.delete('region_codes') if refresh
 

app/models/etsource/dataset/import.rb

@@ -79,6 +79,7 @@ def load_dataset_hash
 
       {
         area: load_region_data,
+        emissions: load_emission_data,
         carriers: load_carrier_data(precalculated_objects),
         energy_graph: load_energy_graph_dataset(precalculated_objects),
         molecules_graph: load_molecules_graph_dataset(precalculated_objects)
@@ -114,6 +115,13 @@ def load_region_data
       { area_data: @atlas_ds.to_hash }
     end
 
+    # Internal: Loads the regions emission data.
+    #
+    # Returns a hash.
+    def load_emission_data
+      { emissions_data: @atlas_ds.emissions.to_hash }
+    end
+
     # Internal: Loads the carrier data.
     #
     # Returns a hash, each key-pair being a carrier.

app/models/etsource/loader.rb

@@ -45,6 +45,13 @@ def area_attributes(area_code)
       end
     end
 
+    def emissions(area_code)
+      cache("emissions/#{area_code}") do
+        area_emissions = Atlas::Dataset.find(area_code).emissions
+        IceNine.deep_freeze!(area_emissions.to_hash.with_indifferent_access)
+      end
+    end
+
     # @return [Qernel::Dataset] Dataset to be used for a country. Is in a uncalculated state.
     def dataset(country)
       instrument("etsource.loader: dataset(#{country.inspect})") do

app/models/gql/runtime/functions/lookup.rb

@@ -179,6 +179,57 @@ def AREA(*keys)
         keys.empty? ? scope.graph.area : scope.area(keys.first)
       end
 
+      # Returns an attribute {Qernel::Emissions} or {Qernel::Emissions::ScopedSector}
+      #
+      # Emissions data is loaded from CSV files in ETSource with the following structure:
+      #   etm_sector, etm_subsector, type, ghg, unit, value
+      #
+      # Parameters:
+      #   - sector: ETM sector name (e.g., 'households', 'energy.electricity_and_heat_production')
+      #             Dots in sector names are converted to underscores for key generation
+      #   - type: Emission type (energetic, non_energetic) - REQUIRED when accessing values
+      #   - ghg: GHG type (co2, other_ghg) - optional
+      #   - year: Year of emission (e.g., 1990) - optional, reads from emissions_YEAR.csv files
+      #
+      # Key generation combines: sector_[subsector_]type_ghg[_year]
+      # Note: Unit column from CSV is not included in keys, blank values return nil
+      #
+      #
+      # EMISSIONS() without any keys returns {Qernel::Emissions}
+      #
+      #   EMISSIONS() # => <Qernel::Emissions>
+      #
+      #
+      # EMISSIONS(sector, type) returns {Qernel::Emissions::ScopedSector}
+      #
+      # Which can be used to update emission factors:
+      #   UPDATE(EMISSIONS(households, energetic), co2, VALUE )
+      #   UPDATE(EMISSIONS('energy.electricity_and_heat_production', energetic), co2, VALUE )
+      #
+      # Examples
+      #
+      #   EMISSIONS('energy.electricity_and_heat_production', energetic)
+      #   # => <Qernel::Emissions::ScopedSector energy_electricity_and_heat_production_energetic>
+      #
+      #
+      # EMISSIONS(sector, type, ghg) or EMISSIONS(sector, type, ghg, year) returns an emission value
+      #
+      # Examples
+      #   EMISSIONS(households, energetic, other_ghg) # => 12.0 (from emissions_default.csv)
+      #   EMISSIONS(households, energetic, co2, 1990) # => value (from emissions_1990.csv)
+      #   EMISSIONS(energy.electricity_and_heat_production, energetic, other_ghg) # => 18.0
+      #
+      def EMISSIONS(*keys)
+        return scope.graph.emissions if keys.empty?
+
+        keys[0] = keys.first.to_s.tr('.', '_').to_sym
+
+        # EMISSIONS(sector, type) -> return ScopedSector
+        return scope.graph.emissions.scope(keys.join('_').to_sym) if keys.size == 2
+
+        scope.graph.emissions[keys.join('_').to_sym]
+      end
+
       # Public: Retrieves a single value from the weather_properties.csv file
       # associated with the currently-selected weather curve set.
       def WEATHER_PROPERTY(key)

app/models/qernel/area.rb

@@ -18,12 +18,14 @@ class Area
     dataset_accessors :weather_curve_set
     dataset_accessors :disabled_sectors
 
-    attr_accessor :graph
+    attr_accessor :graph, :emissions
     attr_reader :dataset_key, :key
 
     def initialize(graph = nil)
       self.graph = graph unless graph.nil?
       @dataset_key = @key = :area_data
+
+      @emissions   = Qernel::Emissions.new(graph)
     end
 
     # Remove when we replace :area with :area_code
@@ -55,6 +57,11 @@ def weather_properties
       Etsource::Dataset.weather_properties(area_code, weather_curve_set)
     end
 
+    # # TODO: will change after fixing ETScource::Loader
+    # def emissions
+    #   fetch(:emissions) { Qernel::Emissions.new(**Etsource::Dataset.emissions(area_code)) }
+    # end
+
     # ----- attributes/methods still used in gqueries. should be properly added to etsource or change gqueries.
 
     def co2_emission_1990_billions

app/models/qernel/dataset.rb

@@ -26,6 +26,7 @@ def initialize(id = nil)
 
       @data = {
         area: { area_data: {} },
+        emissions: { emissions_data: {} },
         energy_graph: { graph: {} },
         molecules_graph: { graph: {} }
       }

app/models/qernel/emissions.rb

@@ -0,0 +1,69 @@
+module Qernel
+  # Class for getting and setting sector emmissions
+  # Behaves much like Qernel::Area, can been seen as an extension of
+  # area attributes, scoped for emissions
+  class Emissions
+    include DatasetAttributes
+
+    dataset_accessors ::Etsource::Dataset.emissions_keys
+    attr_accessor :graph
+
+    # TODO: write a spec for scope
+
+    # Queryable object that defers sector methods to main Emissions
+    #
+    # GQL uses this to scope the sector for easier queries and input/update
+    # statements in etsource
+    class ScopedSector
+      def initialize(emissions, scope)
+        @emissions = emissions
+        @scope = scope
+      end
+
+      def [](attr_name)
+        @emissions[scoped_method(attr_name)]
+      end
+
+      def []=(attr_name, value)
+        @emissions[scoped_method(attr_name)] = value
+      end
+
+      def inspect
+        "<Qernel::Emissions::ScopedSector #{@scope}>"
+      end
+
+      def scoped_method(method_name)
+        "#{@scope}_#{method_name}"
+      end
+
+      def respond_to_missing?(method_name, include_private = false)
+        data_key = scoped_method(method_name).split('=').first
+
+        @emissions.respond_to?(data_key) || super
+      end
+
+      def method_missing(method_name, *args)
+        data_key = scoped_method(method_name).split('=').first
+
+        if data_key == scoped_method(method_name)
+          @emissions[data_key]
+        else
+          @emissions[data_key] = args.first
+        end
+      end
+    end
+
+    def initialize(graph = nil)
+      self.graph = graph unless graph.nil?
+
+      @dataset_key = @key = :emissions_data
+    end
+
+    # Public: define the sector scope for access to the hashed emission keys
+    #
+    # Returns a scoped version of the emissions data
+    def scope(sector)
+      ScopedSector.new(self, sector)
+    end
+  end
+end

app/models/qernel/graph.rb

@@ -62,6 +62,7 @@ def graph
                 :area
 
   delegate :weather_properties, to: :area
+  delegate :emissions, to: :area
 
   def self.dataset_group_with_name(name)
     :"#{name}_graph"
@@ -162,6 +163,7 @@ def retaining_lifecycle
   def call_on_each_qernel_object(method_name)
     self.send(method_name)
     area.send(method_name)
+    emissions.send(method_name)
     carriers.each(&method_name)
 
     nodes.each do |n|

db/schema.rb

@@ -161,7 +161,6 @@
     t.string "user_email"
   end
 
-  add_foreign_key "active_storage_attachments", "active_storage_blobs", column: "blob_id"
   add_foreign_key "active_storage_variant_records", "active_storage_blobs", column: "blob_id"
   add_foreign_key "forecast_storage_orders", "scenarios"
   add_foreign_key "heat_network_orders", "scenarios"

spec/fixtures/etsource/datasets/nl/emissions/emissions_1990.csv

@@ -0,0 +1,40 @@
+etm_sector,etm_subsector,type,ghg,unit,value
+Energy,Electricity and heat production,energetic,other_ghg,kg,15.0
+Energy,Electricity and heat production,energetic,co2,kg,18.0
+Industry,Refineries,energetic,other_ghg,kg,
+Industry,Refineries,energetic,co2,kg,
+Energy,Other fuels production,energetic,other_ghg,kg,
+Energy,Other fuels production,energetic,co2,kg,
+Industry,,energetic,other_ghg,kg,
+Industry,,energetic,co2,kg,
+National transport,,energetic,other_ghg,kg,
+National transport,,energetic,co2,kg,
+Other,Other transportation,energetic,co2,kg,
+Other,Other transportation,energetic,other_ghg,kg,
+Buildings,,energetic,other_ghg,kg,
+Buildings,,energetic,co2,kg,
+Households,,energetic,other_ghg,kg,7.0
+Households,,energetic,co2,kg,12.0
+Agriculture,,energetic,other_ghg,kg,
+Agriculture,,energetic,co2,kg,
+Other,,energetic,other_ghg,kg,
+Other,,energetic,co2,kg,
+Energy,Electricity and heat production,non_energetic,co2,kg,18.0
+Energy,Fugitive emissions,non_energetic,other_ghg,kg,
+Energy,Fugitive emissions,non_energetic,co2,kg,
+Energy,CCUS,non_energetic,co2,kg,
+Industry,,non_energetic,other_ghg,kg,
+Industry,,non_energetic,co2,kg,
+Energy,Methanol production,non_energetic,other_ghg,kg,
+Energy,Methanol production,non_energetic,co2,kg,
+Energy,Hydrogen production,non_energetic,other_ghg,kg,
+Energy,Hydrogen production,non_energetic,co2,kg,
+Agriculture,,non_energetic,co2,kg,
+Agriculture,,non_energetic,other_ghg,kg,
+LULUCF,,non_energetic,co2,kg,
+LULUCF,,non_energetic,other_ghg,kg,
+Waste,,non_energetic,co2,kg,
+Waste,,non_energetic,other_ghg,kg,
+Other,Indirect emissions,non_energetic,co2,kg,
+International transport,,energetic,co2,kg,
+International transport,,energetic,other_ghg,kg,

spec/fixtures/etsource/datasets/nl/emissions/emissions_default.csv

@@ -0,0 +1,38 @@
+etm_sector,etm_subsector,type,ghg,unit,value
+Energy,Electricity and heat production,energetic,other_ghg,kg,18.0
+Industry,Refineries,energetic,other_ghg,kg,
+Energy,Other fuels production,energetic,other_ghg,kg,
+Industry,,energetic,other_ghg,kg,
+National transport,,energetic,other_ghg,kg,
+Other,Other transportation,energetic,other_ghg,kg,
+Buildings,,energetic,other_ghg,kg,
+Households,,energetic,other_ghg,kg,7.0
+Households,,energetic,co2,kg,12.0
+Agriculture,,energetic,other_ghg,kg,
+Other,,energetic,other_ghg,kg,
+Energy,Electricity and heat production,non_energetic,co2,kg,18.0
+Energy,Fugitive emissions,non_energetic,other_ghg,kg,
+Energy,Fugitive emissions,non_energetic,co2,kg,
+Energy,CCUS,non_energetic,co2,kg,
+Industry,Other,non_energetic,other_ghg,kg,
+Industry,Other,non_energetic,co2,kg,
+Industry,Fertilizers,non_energetic,other_ghg,kg,
+Industry,Chemicals,non_energetic,other_ghg,kg,
+Industry,Chemicals,non_energetic,co2,kg,
+Energy,Methanol production,non_energetic,other_ghg,kg,
+Energy,Hydrogen production,non_energetic,other_ghg,kg,
+Industry,Steel,non_energetic,co2,kg,
+Industry,Steel,non_energetic,other_ghg,kg,
+Industry,Aluminium,non_energetic,co2,kg,
+Industry,Aluminium,non_energetic,other_ghg,kg,
+Industry,Other metals,non_energetic,co2,kg,
+Industry,Other metals,non_energetic,other_ghg,kg,
+Agriculture,,non_energetic,co2,kg,
+Agriculture,,non_energetic,other_ghg,kg,
+LULUCF,,non_energetic,co2,kg,
+LULUCF,,non_energetic,other_ghg,kg,
+Waste,,non_energetic,co2,kg,
+Waste,,non_energetic,other_ghg,kg,
+Other,Indirect emissions,non_energetic,co2,kg,
+International transport,International aviation,energetic,other_ghg,kg,
+International transport,International navigation,energetic,other_ghg,kg,