feat(NET-92) Handle missing parquet columns as nulls

crates/query/Cargo.toml

@@ -24,6 +24,7 @@ sqd-bloom-filter = { path = "../bloom-filter" }
 sqd-polars = { path = "../polars" }
 sqd-primitives = { path = "../primitives", features = ["range"] }
 sqd-storage = { path = "../storage", optional = true }
+tracing = "0.1.40"
 
 [dev-dependencies]
 glob = "0.3.1"
@@ -37,6 +38,9 @@ sqd-dataset = { path = "../dataset" }
 tikv-jemallocator = "0.6"
 
 [features]
+max_level_trace = ["tracing/max_level_trace"]
+max_level_debug = ["tracing/max_level_debug"]
+max_level_info = ["tracing/max_level_info"]
 parquet = [
     "dep:parquet",
     "dep:memmap2"

crates/query/fixtures/fuel/queries/missing_block_fields/README.md

@@ -0,0 +1,8 @@
+This test validates that when nullable columns are missing intirely from the parquet file, such as:
+
+- eventInboxRoot
+- consensusParametersVersion
+- stateTransitionBytecodeVersion
+- messageOutboxRoot
+
+The output will include these columnds with null values

crates/query/fixtures/fuel/queries/missing_block_fields/query.json

@@ -0,0 +1,16 @@
+{
+  "type": "fuel",
+  "fromBlock": 9000000,
+  "toBlock": 9000003,
+  "fields": {
+    "block": {
+      "number": true,
+      "hash": true,
+      "eventInboxRoot": true,
+      "consensusParametersVersion": true,
+      "stateTransitionBytecodeVersion": true,
+      "messageOutboxRoot": true
+    }
+  },
+  "includeAllBlocks": false
+}

crates/query/fixtures/fuel/queries/missing_field_with_join/README.md

@@ -0,0 +1,3 @@
+This test validates that when nullable columns are missing intirely from the parquet file, such as:
+- upgrade_purpose
+The output will include these columnds with null values

crates/query/fixtures/fuel/queries/missing_field_with_join/query.json

@@ -0,0 +1,28 @@
+{
+  "type": "fuel",
+  "fromBlock": 9025510,
+  "toBlock": 9025549,
+  "fields": {
+    "block": {
+      "number": true,
+      "hash": true
+    },
+    "receipt": {
+      "receiptType": true,
+      "contract": true,
+      "index": true,
+      "transactionIndex": true
+    },
+    "transaction": {
+      "index": true,
+      "hash": true,
+      "upgradePurpose": true
+    }
+  },
+  "receipts": [
+    {
+      "type": ["LOG_DATA"],
+      "transaction": true
+    }
+  ]
+}

crates/query/src/json/exp.rs

@@ -218,6 +218,16 @@ fn eval_object(array: &dyn Array, props: &Vec<(Name, Exp)>) -> Result<EncoderObj
 }
 
 
+struct AllNullEncoder;
+
+
+impl Encoder for AllNullEncoder {
+    fn encode(&mut self, _idx: usize, out: &mut Vec<u8>) {
+        out.extend_from_slice(b"null")
+    }
+}
+
+
 fn eval_prop(array: &dyn Array, name: Name, exp: &Exp) -> Result<EncoderObject, SchemaError> {
     let array: &StructArray = array.as_any().downcast_ref().ok_or_else(|| {
         schema_error!("expected a StructArray, but got {}", array.data_type())
@@ -227,6 +237,15 @@ fn eval_prop(array: &dyn Array, name: Name, exp: &Exp) -> Result<EncoderObject,
         schema_error!("column `{}` not found", name)
     })?;
 
+    if column_array.data_type() == &DataType::Null {
+        let encoder: EncoderObject = Box::new(AllNullEncoder);
+        if let Some(nulls) = array.nulls() {
+            return Ok(Box::new(NullableEncoder::new(encoder, nulls.clone())))
+        } else {
+            return Ok(encoder)
+        }
+    }
+
     let encoder = exp.eval(column_array).map_err(|err| err.at(name))?;
 
     if let Some(nulls) = array.nulls() {

crates/query/src/lib.rs

@@ -13,5 +13,5 @@ pub use primitives::BlockNumber;
 pub use query::*;
 #[cfg(feature = "parquet")]
 pub use scan::parquet::ParquetChunk;
-pub use scan::{Chunk, TableDoesNotExist};
+pub use scan::{Chunk, TableDoesNotExist, ColumnDoesNotExist};
 pub use sqd_polars::set_polars_thread_pool_size;