Added sparse and HUffman encoding to the CMVContainer, updatet the drawCMV.C macro and TPCDistributeCMVSpec.h accordingly

Author: tubagundem

Detectors/TPC/calibration/include/TPCCalibration/CMVContainer.h

@@ -28,7 +28,9 @@
 namespace o2::tpc
 {
 
-struct CMVPerTF; // forward declaration
+struct CMVPerTF;        // forward declaration
+struct CMVPerTFSparse;  // forward declaration
+struct CMVPerTFHuffman; // forward declaration
 
 /// Delta+zigzag+varint compressed CMV data for one TF across all CRUs
 /// Produced by CMVPerTF::compress(), restored with decompress()
@@ -53,6 +55,56 @@ struct CMVPerTFCompressed {
   ClassDefNV(CMVPerTFCompressed, 1)
 };
 
+/// Sparse-encoded CMV data for one TF across all CRUs
+/// Produced by CMVPerTF::compressSparse(), restored with decompress()
+/// Each TTree entry corresponds to one CMVPerTFSparse object (one TF)
+///
+/// Encoding format (stored in mSparseData):
+///   For each CRU 0..MaxCRU-1:
+///     varint(N)  — number of non-zero timebins in this CRU
+///     For each of the N entries (in timebin order):
+///       varint(delta)  — absolute timeBin for the first entry; tb − prev_tb for subsequent ones
+///       uint16_t(raw)  — raw sign-magnitude CMV value (little-endian, 2 bytes)
+struct CMVPerTFSparse {
+  uint32_t firstOrbit{0}; ///< First orbit of this TF (copied from CMVPerTF)
+  uint16_t firstBC{0};    ///< First bunch crossing of this TF (copied from CMVPerTF)
+
+  /// Sparse-encoded CMV values
+  std::vector<uint8_t> mSparseData;
+
+  /// Restore a CMVPerTF from this sparse object into *cmv (must not be null)
+  void decompress(CMVPerTF* cmv) const;
+
+ private:
+  static uint32_t decodeVarint(const uint8_t*& data, const uint8_t* end); ///< Varint decode
+
+ public:
+  ClassDefNV(CMVPerTFSparse, 1)
+};
+
+/// Delta+zigzag+canonical Huffman compressed CMV data for one TF across all CRUs
+/// Produced by CMVPerTF::compressHuffman(), restored with decompress()
+///
+/// Serialisation layout (mHuffmanData):
+///   4 bytes LE uint32_t : numSymbols — number of distinct zigzag-encoded delta symbols
+///   numSymbols * 5 bytes : symbol table (canonical order, sorted by codeLen ASC then symbol ASC):
+///       4 bytes LE uint32_t : zigzag-encoded symbol value
+///       1 byte              : canonical code length (1..32)
+///   8 bytes LE uint64_t : totalBits — number of valid bits in the bitstream
+///   ceil(totalBits/8) bytes : bitstream, MSB-first packing within each byte
+struct CMVPerTFHuffman {
+  uint32_t firstOrbit{0}; ///< First orbit of this TF (copied from CMVPerTF)
+  uint16_t firstBC{0};    ///< First bunch crossing of this TF (copied from CMVPerTF)
+
+  /// Huffman-coded payload
+  std::vector<uint8_t> mHuffmanData;
+
+  /// Restore a CMVPerTF from this Huffman object into *cmv (must not be null)
+  void decompress(CMVPerTF* cmv) const;
+
+  ClassDefNV(CMVPerTFHuffman, 1)
+};
+
 /// CMV data for one TF across all CRUs
 /// Raw 16-bit CMV values are stored in a flat C array indexed as [cru * NTimeBinsPerTF + timeBin]
 /// CRU::MaxCRU and cmv::NTimeBinsPerTF are compile-time constants, so no dynamic allocation is needed
@@ -69,13 +121,30 @@ struct CMVPerTF {
   /// Return the float CMV value for a given CRU and timebin within this TF
   float getCMVFloat(const int cru, const int timeBin) const;
 
-  /// Zero out raw CMV values whose float magnitude is below threshold (default 1.0 ADC)
+  /// Zero out raw CMV values whose float magnitude is below threshold
   /// This converts the sign-magnitude raw value to 0x0000 for all entries with |float value| < threshold
   void zeroSmallValues(float threshold = 1.0f);
 
+  /// Apply dynamic precision reduction: round values to the nearest integer ADC for all values whose rounded magnitude is <= steps
+  /// Example: steps=3
+  ///   |v| <  0.5 ADC  -> 0
+  ///   |v| in [0.5, 1.5) ADC  -> 1.0 ADC
+  ///   |v| in [1.5, 2.5) ADC  -> 2.0 ADC
+  ///   |v| in [2.5, 3.5) ADC  -> 3.0 ADC
+  ///   |v| >= 3.5 ADC         -> unchanged (full precision)
+  ///
+  /// steps=0 dynamic precision is not applied
+  void applyDynamicPrecision(uint16_t steps);
+
   /// Compress this object into a CMVPerTFCompressed using delta+zigzag+varint encoding
   CMVPerTFCompressed compress() const;
 
+  /// Compress this object into a CMVPerTFSparse storing only non-zero timebins
+  CMVPerTFSparse compressSparse() const;
+
+  /// Compress this object using delta+zigzag+canonical-Huffman encoding
+  CMVPerTFHuffman compressHuffman() const;
+
   /// Serialise into a TTree; each Fill() call appends one entry (one TF)
   std::unique_ptr<TTree> toTTree() const;
 

Detectors/TPC/calibration/macro/drawCMV.C

@@ -80,22 +80,31 @@ TObjArray* drawCMV(std::string_view filename, std::string_view outDir)
                        110, -100.5, 9.5);
   h2d->SetStats(0);
 
-  // auto-detect branch format: compressed (delta+zigzag+varint) or raw CMVPerTF
+  // auto-detect branch format: sparse, huffman, compressed (delta+zigzag+varint), or raw CMVPerTF
+  const bool isSparse = (tree->GetBranch("CMVPerTFSparse") != nullptr);
+  const bool isHuffman = (tree->GetBranch("CMVPerTFHuffman") != nullptr);
   const bool isCompressed = (tree->GetBranch("CMVPerTFCompressed") != nullptr);
   const bool isRaw = (tree->GetBranch("CMVPerTF") != nullptr);
-  if (!isCompressed && !isRaw) {
-    fmt::print("ERROR: neither 'CMVPerTFCompressed' nor 'CMVPerTF' branch found\n");
+  if (!isSparse && !isHuffman && !isCompressed && !isRaw) {
+    fmt::print("ERROR: no recognised branch found (expected CMVPerTFSparse, CMVPerTFHuffman, CMVPerTFCompressed, or CMVPerTF)\n");
     return arrCanvases;
   }
-  fmt::print("Branch format: {}\n", isCompressed ? "CMVPerTFCompressed (delta+zigzag+varint)" : "CMVPerTF (raw)");
+  const std::string branchFormat = isSparse ? "CMVPerTFSparse (sparse)" : (isHuffman ? "CMVPerTFHuffman (delta+zigzag+Huffman)" : (isCompressed ? "CMVPerTFCompressed (delta+zigzag+varint)" : "CMVPerTF (raw)"));
+  fmt::print("Branch format: {}\n", branchFormat);
 
-  // branch setup
+  // branch setup — only one pointer is active depending on the detected format
+  o2::tpc::CMVPerTFSparse* tfSparse = nullptr;
+  o2::tpc::CMVPerTFHuffman* tfHuffman = nullptr;
   o2::tpc::CMVPerTFCompressed* tfCompressed = nullptr;
   o2::tpc::CMVPerTF* tfRaw = nullptr;
-  // staging object for decompression (only used in compressed path)
-  CMVPerTF* tfDecoded = isCompressed ? new CMVPerTF() : nullptr;
-
-  if (isCompressed) {
+  // staging object for decompression
+  CMVPerTF* tfDecoded = (isSparse || isHuffman || isCompressed) ? new CMVPerTF() : nullptr;
+
+  if (isSparse) {
+    tree->SetBranchAddress("CMVPerTFSparse", &tfSparse);
+  } else if (isHuffman) {
+    tree->SetBranchAddress("CMVPerTFHuffman", &tfHuffman);
+  } else if (isCompressed) {
     tree->SetBranchAddress("CMVPerTFCompressed", &tfCompressed);
   } else {
     tree->SetBranchAddress("CMVPerTF", &tfRaw);
@@ -108,7 +117,13 @@ TObjArray* drawCMV(std::string_view filename, std::string_view outDir)
 
     // resolve to a unified pointer regardless of storage format
     const CMVPerTF* tf = nullptr;
-    if (isCompressed) {
+    if (isSparse) {
+      tfSparse->decompress(tfDecoded);
+      tf = tfDecoded;
+    } else if (isHuffman) {
+      tfHuffman->decompress(tfDecoded);
+      tf = tfDecoded;
+    } else if (isCompressed) {
       tfCompressed->decompress(tfDecoded);
       tf = tfDecoded;
     } else {
@@ -128,6 +143,8 @@ TObjArray* drawCMV(std::string_view filename, std::string_view outDir)
 
   delete tfDecoded;
   tree->ResetBranchAddresses();
+  delete tfSparse;
+  delete tfHuffman;
   delete tfCompressed;
 
   fmt::print("firstOrbit: {}\n", firstOrbit);