Skip to content

Commit 27b73e3

Browse files
tubagundemtubagundem
committed
Refactored CMVContainer, unified CMV compression in a flag based container
1 parent 8e5cf37 commit 27b73e3

File tree

5 files changed

+317
-778
lines changed

5 files changed

+317
-778
lines changed

‎Detectors/TPC/calibration/include/TPCCalibration/CMVContainer.h‎

Lines changed: 49 additions & 149 deletions
Original file line numberDiff line numberDiff line change
@@ -28,142 +28,71 @@
2828
namespace o2::tpc
2929
{
3030

31-
struct CMVPerTF; // forward declaration
32-
struct CMVPerTFSparse; // forward declaration
33-
struct CMVPerTFHuffman; // forward declaration
34-
struct CMVPerTFCombined; // forward declaration
35-
struct CMVPerTFQuantized; // forward declaration
36-
37-
/// Delta+zigzag+varint compressed CMV data for one TF across all CRUs
38-
/// Produced by CMVPerTF::compressVarint(), restored with decompress()
39-
/// Each TTree entry corresponds to one CMVPerTFVarint object (one TF)
40-
struct CMVPerTFVarint {
41-
uint32_t firstOrbit{0}; ///< First orbit of this TF (copied from CMVPerTF)
42-
uint16_t firstBC{0}; ///< First bunch crossing of this TF (copied from CMVPerTF)
43-
44-
/// Delta+zigzag+varint encoded CMV values
45-
/// Layout: CRU-major, time-minor; delta is reset to zero at each CRU boundary
46-
std::vector<uint8_t> mCompressedData;
47-
48-
/// Restore a CMVPerTF from this compressed object into *cmv (must not be null)
49-
void decompress(CMVPerTF* cmv) const;
50-
51-
private:
52-
static uint16_t signedToCmv(int32_t val); ///< Signed integer -> sign-magnitude uint16_t
53-
static int32_t zigzagDecode(uint32_t value); ///< Zigzag decode
54-
static uint32_t decodeVarint(const uint8_t*& data, const uint8_t* end); ///< Varint decode
55-
56-
public:
57-
ClassDefNV(CMVPerTFVarint, 1)
58-
};
59-
60-
/// Sparse-encoded CMV data for one TF across all CRUs
61-
/// Produced by CMVPerTF::compressSparse(), restored with decompress()
62-
/// Each TTree entry corresponds to one CMVPerTFSparse object (one TF)
63-
///
64-
/// Encoding format (stored in mSparseData):
65-
/// For each CRU 0..MaxCRU-1:
66-
/// varint(N) — number of non-zero timebins in this CRU
67-
/// For each of the N entries (in timebin order):
68-
/// varint(delta) — absolute timeBin for the first entry; tb − prev_tb for subsequent ones
69-
/// uint16_t(raw) — raw sign-magnitude CMV value (little-endian, 2 bytes)
70-
struct CMVPerTFSparse {
71-
uint32_t firstOrbit{0}; ///< First orbit of this TF (copied from CMVPerTF)
72-
uint16_t firstBC{0}; ///< First bunch crossing of this TF (copied from CMVPerTF)
73-
74-
/// Sparse-encoded CMV values
75-
std::vector<uint8_t> mSparseData;
76-
77-
/// Restore a CMVPerTF from this sparse object into *cmv (must not be null)
78-
void decompress(CMVPerTF* cmv) const;
79-
80-
private:
81-
static uint32_t decodeVarint(const uint8_t*& data, const uint8_t* end); ///< Varint decode
82-
83-
public:
84-
ClassDefNV(CMVPerTFSparse, 1)
85-
};
86-
87-
/// Delta+zigzag+canonical Huffman compressed CMV data for one TF across all CRUs
88-
/// Produced by CMVPerTF::compressHuffman(), restored with decompress()
89-
///
90-
/// Serialisation layout (mHuffmanData):
91-
/// 4 bytes LE uint32_t : numSymbols — number of distinct zigzag-encoded delta symbols
92-
/// numSymbols * 5 bytes : symbol table (canonical order, sorted by codeLen ASC then symbol ASC):
93-
/// 4 bytes LE uint32_t : zigzag-encoded symbol value
94-
/// 1 byte : canonical code length (1..32)
95-
/// 8 bytes LE uint64_t : totalBits — number of valid bits in the bitstream
96-
/// ceil(totalBits/8) bytes : bitstream, MSB-first packing within each byte
97-
struct CMVPerTFHuffman {
98-
uint32_t firstOrbit{0}; ///< First orbit of this TF (copied from CMVPerTF)
99-
uint16_t firstBC{0}; ///< First bunch crossing of this TF (copied from CMVPerTF)
100-
101-
/// Huffman-coded payload
102-
std::vector<uint8_t> mHuffmanData;
103-
104-
/// Restore a CMVPerTF from this Huffman object into *cmv (must not be null)
105-
void decompress(CMVPerTF* cmv) const;
106-
107-
ClassDefNV(CMVPerTFHuffman, 1)
31+
struct CMVPerTF; // forward declaration
32+
struct CMVPerTFCompressed; // forward declaration
33+
34+
/// Bitmask flags describing which encoding stages are applied in CMVPerTFCompressed
35+
struct CMVEncoding {
36+
static constexpr uint8_t kNone = 0x00; ///< No compression — raw uint16 values stored flat
37+
static constexpr uint8_t kSparse = 0x01; ///< Non-zero positions stored sparsely (varint-encoded deltas)
38+
static constexpr uint8_t kDelta = 0x02; ///< Delta coding between consecutive values (dense only)
39+
static constexpr uint8_t kZigzag = 0x04; ///< Zigzag encoding of deltas or signed values
40+
static constexpr uint8_t kVarint = 0x08; ///< Varint compression of the value stream
41+
static constexpr uint8_t kHuffman = 0x10; ///< Canonical Huffman compression of the value stream
10842
};
10943

110-
/// Hybrid sparse+compressed-value encoding for one TF across all CRUs
111-
///
112-
/// Non-zero positions are stored as sparse varint deltas (same as CMVPerTFSparse)
113-
/// The remaining non-zero values are encoded according to mValueMode:
44+
/// Single compressed representation for one TF across all CRUs, stored in a TTree
45+
/// mFlags is a bitmask of CMVEncoding values that fully describes the encoding pipeline
46+
/// mData holds the encoded payload whose binary layout depends on mFlags:
11447
///
115-
/// Mode 0 raw uint16_t — identical value encoding to CMVPerTFSparse
116-
/// Mode 1 varint signed — zigzag+varint of the exact signed CMV value
117-
/// Mode 2 Huffman signed — canonical Huffman over the same zigzag-encoded exact values
48+
/// Dense path (!kSparse):
49+
/// kZigzag absent → N × uint16_t LE (raw values, CRU-major order)
50+
/// kZigzag + kVarint → N × varint(zigzag(delta(signed(raw))))
51+
/// kZigzag + kHuffman → [Huffman table] + [bitstream] of zigzag(delta(signed(raw)))
11852
///
119-
/// Binary layout of mData:
120-
/// 4 bytes LE posStreamSize
121-
/// [posStream] for each CRU: varint(N), N×varint(tb_delta)
122-
/// [valStream]
123-
/// mode 0: N_total × uint16_t LE
124-
/// mode 1: N_total × varint(zigzag(cmvToSigned(raw)))
125-
/// mode 2: [canonical Huffman table] + [8-byte totalBits] + [bitstream]
126-
struct CMVPerTFCombined {
53+
/// Sparse path (kSparse):
54+
/// 4 bytes LE uint32_t : posStreamSize
55+
/// posStream: for each CRU: varint(N), N × varint(tb_delta)
56+
/// valStream (one entry per non-zero):
57+
/// default → uint16_t LE raw value
58+
/// kZigzag + kVarint → varint(zigzag(signed(raw)))
59+
/// kZigzag + kHuffman → [Huffman table] + [bitstream] of zigzag(signed(raw))
60+
struct CMVPerTFCompressed {
12761
uint32_t firstOrbit{0}; ///< First orbit of this TF
12862
uint16_t firstBC{0}; ///< First bunch crossing of this TF
129-
uint8_t mValueMode{0}; ///< 0 = raw uint16, 1 = varint signed CMV, 2 = Huffman signed CMV
63+
uint8_t mFlags{0}; ///< Bitmask of CMVEncoding values
13064

13165
std::vector<uint8_t> mData; ///< Encoded payload
13266

133-
/// Restore a CMVPerTF from this object into *cmv (must not be null)
67+
/// Restore a CMVPerTF from this compressed object into *cmv (must not be null)
13468
void decompress(CMVPerTF* cmv) const;
13569

136-
ClassDefNV(CMVPerTFCombined, 1)
137-
};
70+
/// Serialise into a TTree; each Fill() call appends one entry (one TF)
71+
std::unique_ptr<TTree> toTTree() const;
13872

139-
/// Sparse positions + quantized-value symbols for one TF across all CRUs
140-
///
141-
/// Non-zero positions are stored as sparse varint deltas.
142-
/// The corresponding values are encoded as quantized signed symbols:
143-
/// Mode 0 raw symbols — 4-byte LE uint32 per symbol
144-
/// Mode 1 varint symbols — varint(symbol)
145-
/// Mode 2 Huffman symbols — canonical Huffman over symbol stream
146-
/// Quantized symbols represent decimal-style values below the internally derived full-precision cutoff
147-
/// and exact raw signed I8F7 values above it.
148-
struct CMVPerTFQuantized {
149-
uint32_t firstOrbit{0}; ///< First orbit of this TF
150-
uint16_t firstBC{0}; ///< First bunch crossing of this TF
151-
uint8_t mValueMode{0}; ///< 0 = raw symbols, 1 = varint symbols, 2 = Huffman symbols
73+
private:
74+
/// Decode the sparse position stream; advances ptr past the position block
75+
/// Returns (cru, timeBin) pairs for every non-zero entry, in CRU-major order
76+
static std::vector<std::pair<int, uint32_t>> decodeSparsePositions(const uint8_t*& ptr, const uint8_t* end);
15277

153-
std::vector<uint8_t> mData; ///< Encoded payload
78+
/// Decode the value stream into raw uint32_t symbols
79+
/// Dispatches to Huffman, varint, or raw uint16 based on flags
80+
static std::vector<uint32_t> decodeValueStream(const uint8_t*& ptr, const uint8_t* end, uint32_t N, uint8_t flags);
15481

155-
/// Restore nearest raw-I8F7 values into *cmv (must not be null)
156-
void decompress(CMVPerTF* cmv) const;
82+
/// Apply inverse zigzag and scatter decoded values into the sparse positions of *cmv
83+
static void decodeSparseValues(const std::vector<uint32_t>& symbols,
84+
const std::vector<std::pair<int, uint32_t>>& positions,
85+
uint8_t flags, CMVPerTF* cmv);
15786

158-
/// Restore exact quantized float values into `values`, sized to CRU::MaxCRU * cmv::NTimeBinsPerTF
159-
void decompressToFloatBuffer(std::vector<float>& values) const;
87+
/// Apply inverse zigzag and inverse delta, then fill the full dense CMV array in *cmv
88+
static void decodeDenseValues(const std::vector<uint32_t>& symbols, uint8_t flags, CMVPerTF* cmv);
16089

161-
ClassDefNV(CMVPerTFQuantized, 1)
90+
public:
91+
ClassDefNV(CMVPerTFCompressed, 1)
16292
};
16393

16494
/// CMV data for one TF across all CRUs
16595
/// Raw 16-bit CMV values are stored in a flat C array indexed as [cru * NTimeBinsPerTF + timeBin]
166-
/// CRU::MaxCRU and cmv::NTimeBinsPerTF are compile-time constants, so no dynamic allocation is needed
16796
struct CMVPerTF {
16897
uint32_t firstOrbit{0}; ///< First orbit of this TF, from heartbeatOrbit of the first CMV packet
16998
uint16_t firstBC{0}; ///< First bunch crossing of this TF, from heartbeatBC of the first CMV packet
@@ -178,47 +107,18 @@ struct CMVPerTF {
178107
float getCMVFloat(const int cru, const int timeBin) const;
179108

180109
/// Zero out raw CMV values whose float magnitude is below threshold
181-
/// This converts the sign-magnitude raw value to 0x0000 for all entries with |float value| < threshold
182110
void zeroSmallValues(float threshold = 1.0f);
183111

184112
/// Round values to the nearest integer ADC for all values whose rounded magnitude is <= threshold
185-
/// Example: threshold=3
186-
/// |v| < 0.5 ADC -> 0
187-
/// |v| in [0.5, 1.5) ADC -> 1.0 ADC
188-
/// |v| in [1.5, 2.5) ADC -> 2.0 ADC
189-
/// |v| in [2.5, 3.5) ADC -> 3.0 ADC
190-
/// |v| >= 3.5 ADC -> unchanged (full precision)
191-
///
192-
/// threshold=0 rounding is not applied
193113
void roundToIntegers(uint16_t threshold);
194114

195115
/// Quantise |v| with a Gaussian-CDF recovery profile:
196-
/// coarse decimal-style precision below and around mean, then a smooth return to the
197-
/// full native I8F7 precision as the magnitude increases with width sigma
116+
/// Coarse decimal-style precision below and around mean, then a smooth return to the full native I8F7 precision as the magnitude increases with width sigma
198117
void trimGaussianPrecision(float mean, float sigma);
199118

200-
/// Compress this object into a CMVPerTFVarint using delta+zigzag+varint encoding
201-
CMVPerTFVarint compressVarint() const;
202-
203-
/// Compress this object into a CMVPerTFSparse storing only non-zero timebins
204-
CMVPerTFSparse compressSparse() const;
205-
206-
/// Dedicated sparse + quantized-value compression
207-
/// valueMode:
208-
/// 0 = raw symbol stream
209-
/// 1 = varint symbol stream
210-
/// 2 = Huffman symbol stream
211-
CMVPerTFQuantized compressQuantized(uint8_t valueMode = 0, float quantizationMean = 1.f, float quantizationSigma = 0.f) const;
212-
213-
/// Hybrid sparse+compressed-value compression
214-
/// Positions encoded as sparse varint deltas; values encoded according to valueMode:
215-
/// 0 = raw uint16_t (same as compressSparse, no additional gain)
216-
/// 1 = varint signed CMV value
217-
/// 2 = Huffman signed CMV value
218-
CMVPerTFCombined compressCombined(uint8_t valueMode = 0) const;
219-
220-
/// Compress this object using delta+zigzag+canonical-Huffman encoding
221-
CMVPerTFHuffman compressHuffman() const;
119+
/// Compress this object into a CMVPerTFCompressed using the encoding pipeline described by flags
120+
/// Quantisation (trimGaussianPrecision / roundToIntegers / zeroSmallValues) should be applied to this object before calling compress(); it is not part of the flags pipeline
121+
CMVPerTFCompressed compress(uint8_t flags) const;
222122

223123
/// Serialise into a TTree; each Fill() call appends one entry (one TF)
224124
std::unique_ptr<TTree> toTTree() const;

‎Detectors/TPC/calibration/macro/drawCMV.C‎

Lines changed: 17 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -85,47 +85,21 @@ TObjArray* drawCMV(std::string_view filename, std::string_view outDir)
8585
1100, -100.5, 9.5);
8686
h1d->SetStats(1);
8787

88-
// auto-detect branch format
89-
const bool isQuantized = (tree->GetBranch("CMVPerTFQuantized") != nullptr);
90-
const bool isCombined = (tree->GetBranch("CMVPerTFCombined") != nullptr);
91-
const bool isSparse = (tree->GetBranch("CMVPerTFSparse") != nullptr);
92-
const bool isHuffman = (tree->GetBranch("CMVPerTFHuffman") != nullptr);
93-
const bool isVarint = (tree->GetBranch("CMVPerTFVarint") != nullptr);
88+
// auto-detect branch format: compressed or raw
89+
const bool isCompressed = (tree->GetBranch("CMVPerTFCompressed") != nullptr);
9490
const bool isRaw = (tree->GetBranch("CMVPerTF") != nullptr);
95-
if (!isQuantized && !isCombined && !isSparse && !isHuffman && !isVarint && !isRaw) {
96-
fmt::print("ERROR: no recognised branch found (expected CMVPerTFQuantized, CMVPerTFCombined, CMVPerTFSparse, CMVPerTFHuffman, CMVPerTFVarint, or CMVPerTF)\n");
91+
if (!isCompressed && !isRaw) {
92+
fmt::print("ERROR: no recognised branch found (expected 'CMVPerTFCompressed' or 'CMVPerTF')\n");
9793
return arrCanvases;
9894
}
99-
const std::string branchFormat =
100-
isQuantized ? "CMVPerTFQuantized (sparse + quantized float symbols)"
101-
: isCombined ? "CMVPerTFCombined (sparse + compressed exact values)"
102-
: isSparse ? "CMVPerTFSparse (sparse)"
103-
: isHuffman ? "CMVPerTFHuffman (delta+zigzag+Huffman)"
104-
: isVarint ? "CMVPerTFVarint (delta+zigzag+varint)"
105-
: "CMVPerTF (raw)";
106-
fmt::print("Branch format: {}\n", branchFormat);
107-
108-
// branch setup — only one pointer is active depending on the detected format
109-
o2::tpc::CMVPerTFQuantized* tfQuantized = nullptr;
110-
o2::tpc::CMVPerTFCombined* tfCombined = nullptr;
111-
o2::tpc::CMVPerTFSparse* tfSparse = nullptr;
112-
o2::tpc::CMVPerTFHuffman* tfHuffman = nullptr;
113-
o2::tpc::CMVPerTFVarint* tfVarint = nullptr;
95+
fmt::print("Branch format: {}\n", isCompressed ? "CMVPerTFCompressed" : "CMVPerTF (raw)");
96+
97+
o2::tpc::CMVPerTFCompressed* tfCompressed = nullptr;
11498
o2::tpc::CMVPerTF* tfRaw = nullptr;
115-
// staging object for decompression
116-
CMVPerTF* tfDecoded = (isCombined || isSparse || isHuffman || isVarint || isQuantized) ? new CMVPerTF() : nullptr;
117-
std::vector<float> tfDecodedFloats;
118-
119-
if (isQuantized) {
120-
tree->SetBranchAddress("CMVPerTFQuantized", &tfQuantized);
121-
} else if (isCombined) {
122-
tree->SetBranchAddress("CMVPerTFCombined", &tfCombined);
123-
} else if (isSparse) {
124-
tree->SetBranchAddress("CMVPerTFSparse", &tfSparse);
125-
} else if (isHuffman) {
126-
tree->SetBranchAddress("CMVPerTFHuffman", &tfHuffman);
127-
} else if (isVarint) {
128-
tree->SetBranchAddress("CMVPerTFVarint", &tfVarint);
99+
CMVPerTF* tfDecoded = isCompressed ? new CMVPerTF() : nullptr;
100+
101+
if (isCompressed) {
102+
tree->SetBranchAddress("CMVPerTFCompressed", &tfCompressed);
129103
} else {
130104
tree->SetBranchAddress("CMVPerTF", &tfRaw);
131105
}
@@ -135,23 +109,10 @@ TObjArray* drawCMV(std::string_view filename, std::string_view outDir)
135109
for (int i = 0; i < nEntries; ++i) {
136110
tree->GetEntry(i);
137111

138-
// resolve to a unified pointer regardless of storage format
112+
// Decompress if needed; resolve to a unified CMVPerTF pointer
139113
const CMVPerTF* tf = nullptr;
140-
if (isQuantized) {
141-
tfQuantized->decompress(tfDecoded);
142-
tfQuantized->decompressToFloatBuffer(tfDecodedFloats);
143-
tf = tfDecoded;
144-
} else if (isCombined) {
145-
tfCombined->decompress(tfDecoded);
146-
tf = tfDecoded;
147-
} else if (isSparse) {
148-
tfSparse->decompress(tfDecoded);
149-
tf = tfDecoded;
150-
} else if (isHuffman) {
151-
tfHuffman->decompress(tfDecoded);
152-
tf = tfDecoded;
153-
} else if (isVarint) {
154-
tfVarint->decompress(tfDecoded);
114+
if (isCompressed) {
115+
tfCompressed->decompress(tfDecoded);
155116
tf = tfDecoded;
156117
} else {
157118
tf = tfRaw;
@@ -163,21 +124,17 @@ TObjArray* drawCMV(std::string_view filename, std::string_view outDir)
163124

164125
for (int cru = 0; cru < nCRUs; ++cru) {
165126
for (int tb = 0; tb < nTimeBins; ++tb) {
166-
const float cmvValue = isQuantized ? tfDecodedFloats[cru * nTimeBins + tb] : tf->getCMVFloat(cru, tb);
127+
const float cmvValue = tf->getCMVFloat(cru, tb);
167128
h2d->Fill(tb, cmvValue);
168129
h1d->Fill(cmvValue);
169-
fmt::print("cru: {}, tb: {}, cmv: {}\n", cru, tb, cmvValue);
130+
// fmt::print("cru: {}, tb: {}, cmv: {}\n", cru, tb, cmvValue);
170131
}
171132
}
172133
}
173134

174135
delete tfDecoded;
175136
tree->ResetBranchAddresses();
176-
delete tfQuantized;
177-
delete tfCombined;
178-
delete tfSparse;
179-
delete tfHuffman;
180-
delete tfVarint;
137+
delete tfCompressed;
181138

182139
fmt::print("firstOrbit: {}\n", firstOrbit);
183140

0 commit comments

Comments
 (0)