bwt_compress_v8.hpp

// BWT Compression v8 - Fixed Model Range Coding with Context Models
//
// Key insight: BWT+MTF output distribution depends on content type:
// - Prose: high zero runs (~55%), predictable letter patterns
// - Code: higher symbol diversity, more brackets/operators
// - Config: key-value patterns, structured whitespace
// - Binary: near-uniform distribution
//
// By selecting the right model, we get 3-8% better compression.

#pragma once
#include <vector>
#include <algorithm>
#include <numeric>
#include <cstdint>
#include <cstring>
#include <queue>
#include <functional>

namespace bwt8 {

constexpr int MAX_ALPHA = 258;

// =============================================================================
// Content Type Detection
// =============================================================================
enum class ContentModel : uint8_t {
    PROSE = 0,   // English text, documentation
    CODE = 1,    // Source code (high bracket/operator density)
    CONFIG = 2,  // Config files (key=value patterns)
    BINARY = 3,  // Binary/random data (uniform distribution)
};

// Detect content type by sampling first 1024 bytes
inline ContentModel detect_content_model(const uint8_t* data, size_t n) {
    size_t sample_size = std::min(n, (size_t)1024);

    int bracket_count = 0;      // {}[]()<>
    int operator_count = 0;     // +-*/%&|^~!
    int equals_colon = 0;       // = :
    int non_printable = 0;      // bytes < 32 (except tab/newline) or > 126
    int alpha_count = 0;        // a-zA-Z
    int semicolon_count = 0;    // ;

    for (size_t i = 0; i < sample_size; i++) {
        uint8_t c = data[i];
        if (c == '{' || c == '}' || c == '[' || c == ']' ||
            c == '(' || c == ')' || c == '<' || c == '>') bracket_count++;
        if (c == '+' || c == '-' || c == '*' || c == '/' || c == '%' ||
            c == '&' || c == '|' || c == '^' || c == '~' || c == '!') operator_count++;
        if (c == '=' || c == ':') equals_colon++;
        if (c == ';') semicolon_count++;
        if ((c < 32 && c != '\t' && c != '\n' && c != '\r') || c > 126) non_printable++;
        if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) alpha_count++;
    }

    double pct = 100.0 / sample_size;
    double bracket_pct = bracket_count * pct;
    double operator_pct = operator_count * pct;
    double equals_pct = equals_colon * pct;
    double nonprint_pct = non_printable * pct;
    double alpha_pct = alpha_count * pct;
    double semi_pct = semicolon_count * pct;

    // Binary: >15% non-printable bytes
    if (nonprint_pct > 15.0) return ContentModel::BINARY;

    // Code: high brackets AND (operators OR semicolons)
    // Typical code has 2-5% brackets, 1-3% operators
    if (bracket_pct > 1.5 && (operator_pct > 0.5 || semi_pct > 0.5)) return ContentModel::CODE;

    // Config: high = or : with moderate structure
    // YAML/TOML/INI have 2-5% equals/colons
    if (equals_pct > 2.0 && bracket_pct < 2.0) return ContentModel::CONFIG;

    // Default: prose (high alpha, low punctuation)
    return ContentModel::PROSE;
}

// =============================================================================
// Context-Specific Probability Models
// =============================================================================

// PROSE model: Trained on English text (high zero runs, letter patterns)
constexpr uint16_t FREQ_PROSE[MAX_ALPHA] = {
    // Symbols 0-7: RUNA, RUNB, and high-frequency MTF values
    12407, 5678, 13921, 7292, 4963, 3494, 2609, 2228,  // 0-7
     1991, 1771,  1568, 1394, 1292, 1074,  926,  434,  // 8-15
       88,   77,    29,   40,   37,   37,   52,   59,  // 16-23
       82,   68,    96,   55,   59,   53,   19,    1,  // 24-31
        1,    1,     1,    1,    1,    1,    1,    1,  // 32-39
        1,    1,     1,    1,    1,    1,    1,   55,  // 40-47
       10,   45,     1,    1,    1,    1,    1,    1,  // 48-55
        1,    1,     1,    1,    1,    1,    1,    1,  // 56-63
        1,    1,     1,    1,    1,    1,    1,    1,  // 64-71
        1,    2,     1,    2,    4,    4,    5,    8,  // 72-79
       31,    4,     7,    8,   11,   15,   24,   58,  // 80-87
        6,    5,     7,    8,   43,    7,   39,    3,  // 88-95
        6,   45,    37,    9,   45,   22,   65,   28,  // 96-103
       55,   21,    17,   56,   23,   29,   39,   61,  // 104-111
       65,   47,    25,   37,  104,  100,   69,   59,  // 112-119
       26,   84,    55,    1,    1,    1,    1,    1,  // 120-127
        1,    1,     1,    1,    1,    1,    1,    1,  // 128-135
        1,    1,     1,    1,    1,    1,    1,    1,  // 136-143
        1,    1,     1,    1,    1,    1,    1,    1,  // 144-151
        1,    1,     1,    1,    1,    1,    1,    1,  // 152-159
        1,    1,     1,    1,    1,    1,    1,    1,  // 160-167
        1,    1,     1,    1,    1,    1,    1,    1,  // 168-175
        1,    1,     1,    1,    1,    1,    1,    1,  // 176-183
        1,    1,     1,    1,    1,    1,    1,    1,  // 184-191
        1,    1,     1,    1,    1,    1,    1,    1,  // 192-199
        1,    1,     1,    1,    1,    1,    1,    1,  // 200-207
        1,    1,     1,    1,    1,    1,    1,    1,  // 208-215
        1,    1,     1,    1,    1,    1,    1,    1,  // 216-223
        1,    1,     1,    1,    1,    1,    1,    1,  // 224-231
        1,    1,     1,    1,    1,    1,    1,    1,  // 232-239
        1,    1,     1,    1,    1,    1,    1,    1,  // 240-247
        1,    1,     1,    1,    1,    1,    1,    1,  // 248-255
        1,    1,                                       // 256-257
};

// CODE model: Higher symbol diversity, more MTF movement
// Code has more unique characters -> flatter distribution after MTF
constexpr uint16_t FREQ_CODE[MAX_ALPHA] = {
    // Less concentrated at low ranks due to more symbol variety
    10000, 6000, 11000, 8000, 6500, 5000, 4000, 3500,  // 0-7
     3000, 2600,  2300, 2000, 1800, 1600, 1400, 1200,  // 8-15
     1000,  900,   800,  700,  600,  550,  500,  450,  // 16-23
      400,  360,   320,  290,  260,  230,  200,  180,  // 24-31
      160,  145,   130,  115,  100,   90,   80,   72,  // 32-39
       65,   58,    52,   47,   42,   38,   34,   55,  // 40-47
       30,   45,    25,   22,   20,   18,   16,   14,  // 48-55
       13,   12,    11,   10,    9,    8,    7,    6,  // 56-63
        5,    5,     5,    5,    5,    4,    4,    4,  // 64-71
        4,    4,     4,    4,    4,    4,    5,    8,  // 72-79
       31,   15,    15,   15,   15,   15,   24,   58,  // 80-87
       15,   15,    15,   15,   43,   15,   39,   15,  // 88-95
       15,   45,    37,   15,   45,   22,   65,   28,  // 96-103
       55,   21,    17,   56,   23,   29,   39,   61,  // 104-111
       65,   47,    25,   37,  104,  100,   69,   59,  // 112-119
       26,   84,    55,    3,    3,    3,    3,    3,  // 120-127
        3,    3,     3,    3,    3,    3,    3,    3,  // 128-135
        3,    3,     3,    3,    3,    3,    3,    3,  // 136-143
        3,    3,     3,    3,    3,    3,    3,    3,  // 144-151
        3,    3,     3,    3,    3,    3,    3,    3,  // 152-159
        3,    3,     3,    3,    3,    3,    3,    3,  // 160-167
        3,    3,     3,    3,    3,    3,    3,    3,  // 168-175
        3,    3,     3,    3,    3,    3,    3,    3,  // 176-183
        3,    3,     3,    3,    3,    3,    3,    3,  // 184-191
        3,    3,     3,    3,    3,    3,    3,    3,  // 192-199
        3,    3,     3,    3,    3,    3,    3,    3,  // 200-207
        3,    3,     3,    3,    3,    3,    3,    3,  // 208-215
        3,    3,     3,    3,    3,    3,    3,    3,  // 216-223
        3,    3,     3,    3,    3,    3,    3,    3,  // 224-231
        3,    3,     3,    3,    3,    3,    3,    3,  // 232-239
        3,    3,     3,    3,    3,    3,    3,    3,  // 240-247
        3,    3,     3,    3,    3,    3,    3,    3,  // 248-255
        3,    3,                                       // 256-257
};

// CONFIG model: Structured whitespace, key-value patterns
// Similar to prose but with more repeated structures
constexpr uint16_t FREQ_CONFIG[MAX_ALPHA] = {
    // High repetition -> more zero runs
    14000, 6500, 12000, 7000, 5000, 3800, 2800, 2400,  // 0-7
     2100, 1900,  1700, 1500, 1350, 1200, 1050,  500,  // 8-15
      120,  100,    80,   65,   55,   48,   42,   38,  // 16-23
       35,   32,    29,   26,   23,   20,   18,   16,  // 24-31
       14,   12,    10,    9,    8,    7,    6,    5,  // 32-39
        5,    4,     4,    4,    3,    3,    3,   55,  // 40-47
       10,   45,     3,    3,    3,    3,    3,    3,  // 48-55
        3,    3,     3,    3,    3,    3,    3,    3,  // 56-63
        3,    3,     3,    3,    3,    3,    3,    3,  // 64-71
        3,    3,     3,    3,    4,    4,    5,    8,  // 72-79
       31,    4,     7,    8,   11,   15,   24,   58,  // 80-87
        6,    5,     7,    8,   43,    7,   39,    3,  // 88-95
        6,   45,    37,    9,   45,   22,   65,   28,  // 96-103
       55,   21,    17,   56,   23,   29,   39,   61,  // 104-111
       65,   47,    25,   37,  104,  100,   69,   59,  // 112-119
       26,   84,    55,    2,    2,    2,    2,    2,  // 120-127
        2,    2,     2,    2,    2,    2,    2,    2,  // 128-135
        2,    2,     2,    2,    2,    2,    2,    2,  // 136-143
        2,    2,     2,    2,    2,    2,    2,    2,  // 144-151
        2,    2,     2,    2,    2,    2,    2,    2,  // 152-159
        2,    2,     2,    2,    2,    2,    2,    2,  // 160-167
        2,    2,     2,    2,    2,    2,    2,    2,  // 168-175
        2,    2,     2,    2,    2,    2,    2,    2,  // 176-183
        2,    2,     2,    2,    2,    2,    2,    2,  // 184-191
        2,    2,     2,    2,    2,    2,    2,    2,  // 192-199
        2,    2,     2,    2,    2,    2,    2,    2,  // 200-207
        2,    2,     2,    2,    2,    2,    2,    2,  // 208-215
        2,    2,     2,    2,    2,    2,    2,    2,  // 216-223
        2,    2,     2,    2,    2,    2,    2,    2,  // 224-231
        2,    2,     2,    2,    2,    2,    2,    2,  // 232-239
        2,    2,     2,    2,    2,    2,    2,    2,  // 240-247
        2,    2,     2,    2,    2,    2,    2,    2,  // 248-255
        2,    2,                                       // 256-257
};

// BINARY model: Near-uniform distribution
// Random data has flat MTF distribution
constexpr uint16_t FREQ_BINARY[MAX_ALPHA] = {
    // Much flatter - random data doesn't cluster at low MTF ranks
     5000, 4800, 4600, 4400, 4200, 4000, 3800, 3600,  // 0-7
     3400, 3200, 3000, 2850, 2700, 2550, 2400, 2250,  // 8-15
     2100, 2000, 1900, 1800, 1700, 1600, 1500, 1400,  // 16-23
     1300, 1220, 1140, 1060,  980,  900,  840,  780,  // 24-31
      720,  670,  620,  575,  530,  490,  450,  415,  // 32-39
      380,  350,  320,  295,  270,  248,  228,  210,  // 40-47
      193,  178,  164,  151,  139,  128,  118,  109,  // 48-55
      100,   92,   85,   78,   72,   66,   61,   56,  // 56-63
       52,   48,   44,   41,   38,   35,   32,   30,  // 64-71
       28,   26,   24,   22,   20,   19,   18,   17,  // 72-79
       16,   15,   14,   13,   12,   11,   10,   10,  // 80-87
        9,    9,    8,    8,    7,    7,    7,    6,  // 88-95
        6,    6,    5,    5,    5,    5,    5,    4,  // 96-103
        4,    4,    4,    4,    4,    4,    3,    3,  // 104-111
        3,    3,    3,    3,    3,    3,    3,    3,  // 112-119
        3,    3,    3,    3,    3,    3,    3,    3,  // 120-127
        3,    3,    3,    3,    3,    3,    3,    3,  // 128-135
        3,    3,    3,    3,    3,    3,    3,    3,  // 136-143
        3,    3,    3,    3,    3,    3,    3,    3,  // 144-151
        3,    3,    3,    3,    3,    3,    3,    3,  // 152-159
        3,    3,    3,    3,    3,    3,    3,    3,  // 160-167
        3,    3,    3,    3,    3,    3,    3,    3,  // 168-175
        3,    3,    3,    3,    3,    3,    3,    3,  // 176-183
        3,    3,    3,    3,    3,    3,    3,    3,  // 184-191
        3,    3,    3,    3,    3,    3,    3,    3,  // 192-199
        3,    3,    3,    3,    3,    3,    3,    3,  // 200-207
        3,    3,    3,    3,    3,    3,    3,    3,  // 208-215
        3,    3,    3,    3,    3,    3,    3,    3,  // 216-223
        3,    3,    3,    3,    3,    3,    3,    3,  // 224-231
        3,    3,    3,    3,    3,    3,    3,    3,  // 232-239
        3,    3,    3,    3,    3,    3,    3,    3,  // 240-247
        3,    3,    3,    3,    3,    3,    3,    3,  // 248-255
        3,    3,                                       // 256-257
};

// Legacy alias for backwards compatibility
constexpr const uint16_t* FIXED_FREQ = FREQ_PROSE;

// Get frequency table for a content model
inline const uint16_t* get_freq_table(ContentModel model) {
    switch (model) {
        case ContentModel::CODE:   return FREQ_CODE;
        case ContentModel::CONFIG: return FREQ_CONFIG;
        case ContentModel::BINARY: return FREQ_BINARY;
        default:                   return FREQ_PROSE;
    }
}

// Precompute cumulative frequencies for a specific model
inline void build_cumul(uint32_t* cumul, int alpha_size, ContentModel model = ContentModel::PROSE) {
    const uint16_t* freq = get_freq_table(model);
    cumul[0] = 0;
    for (int i = 0; i < alpha_size; i++) {
        cumul[i + 1] = cumul[i] + freq[i];
    }
}

// =============================================================================
// BWT Transform
// =============================================================================
inline std::vector<uint8_t> bwt_encode(const uint8_t* data, size_t n, uint32_t& primary_idx) {
    if (n == 0) return {};
    std::vector<uint32_t> sa(n);
    std::iota(sa.begin(), sa.end(), 0);
    std::sort(sa.begin(), sa.end(), [&](uint32_t a, uint32_t b) {
        for (size_t i = 0; i < n; i++) {
            uint8_t ca = data[(a + i) % n];
            uint8_t cb = data[(b + i) % n];
            if (ca != cb) return ca < cb;
        }
        return false;
    });
    primary_idx = 0;
    for (size_t i = 0; i < n; i++) {
        if (sa[i] == 0) { primary_idx = i; break; }
    }
    std::vector<uint8_t> bwt(n);
    for (size_t i = 0; i < n; i++) {
        bwt[i] = data[(sa[i] + n - 1) % n];
    }
    return bwt;
}

inline std::vector<uint8_t> bwt_decode(const uint8_t* bwt, size_t n, uint32_t primary_idx) {
    if (n == 0) return {};
    std::vector<uint32_t> count(256, 0);
    for (size_t i = 0; i < n; i++) count[bwt[i]]++;
    std::vector<uint32_t> cumul(256, 0);
    uint32_t sum = 0;
    for (int i = 0; i < 256; i++) { cumul[i] = sum; sum += count[i]; }
    std::vector<uint32_t> transform(n);
    std::vector<uint32_t> occ(256, 0);
    for (size_t i = 0; i < n; i++) {
        transform[i] = cumul[bwt[i]] + occ[bwt[i]];
        occ[bwt[i]]++;
    }
    std::vector<uint8_t> output(n);
    uint32_t idx = primary_idx;
    for (size_t i = n; i > 0; i--) {
        output[i - 1] = bwt[idx];
        idx = transform[idx];
    }
    return output;
}

// =============================================================================
// MTF Transform
// =============================================================================
inline std::vector<uint8_t> mtf_encode(const uint8_t* data, size_t n) {
    std::vector<uint8_t> alphabet(256);
    std::iota(alphabet.begin(), alphabet.end(), 0);
    std::vector<uint8_t> output(n);
    for (size_t i = 0; i < n; i++) {
        uint8_t c = data[i];
        uint8_t pos = 0;
        while (alphabet[pos] != c) pos++;
        output[i] = pos;
        for (int j = pos; j > 0; j--) alphabet[j] = alphabet[j-1];
        alphabet[0] = c;
    }
    return output;
}

inline std::vector<uint8_t> mtf_decode(const uint8_t* data, size_t n) {
    std::vector<uint8_t> alphabet(256);
    std::iota(alphabet.begin(), alphabet.end(), 0);
    std::vector<uint8_t> output(n);
    for (size_t i = 0; i < n; i++) {
        uint8_t pos = data[i];
        uint8_t c = alphabet[pos];
        output[i] = c;
        for (int j = pos; j > 0; j--) alphabet[j] = alphabet[j-1];
        alphabet[0] = c;
    }
    return output;
}

// =============================================================================
// Zero-RLE
// =============================================================================
inline std::vector<uint16_t> zrle_encode(const uint8_t* data, size_t n) {
    std::vector<uint16_t> output;
    size_t i = 0;
    while (i < n) {
        if (data[i] == 0) {
            size_t run = 0;
            while (i < n && data[i] == 0) { run++; i++; }
            while (run > 0) {
                if (run & 1) {
                    output.push_back(0);
                    run = (run - 1) / 2;
                } else {
                    output.push_back(1);
                    run = (run - 2) / 2;
                }
            }
        } else {
            output.push_back(data[i] + 1);
            i++;
        }
    }
    return output;
}

inline std::vector<uint8_t> zrle_decode(const uint16_t* data, size_t n) {
    std::vector<uint8_t> output;
    size_t i = 0;
    while (i < n) {
        if (data[i] <= 1) {
            size_t run = 0;
            size_t power = 1;
            while (i < n && data[i] <= 1) {
                run += (data[i] + 1) * power;
                power *= 2;
                i++;
            }
            for (size_t j = 0; j < run; j++) output.push_back(0);
        } else {
            output.push_back(data[i] - 1);
            i++;
        }
    }
    return output;
}

// =============================================================================
// Range Encoder/Decoder with Fixed Model
// =============================================================================
class RangeEncoder {
    std::vector<uint8_t>& output;
    uint32_t low = 0;
    uint32_t range = 0xFFFFFFFF;

    void normalize() {
        while (true) {
            if ((low ^ (low + range)) >= 0x1000000) {
                if (range >= 0x10000) break;
                range = -low & 0xFFFF;
            }
            output.push_back(low >> 24);
            low <<= 8;
            range <<= 8;
        }
    }

public:
    RangeEncoder(std::vector<uint8_t>& out) : output(out) {}

    void encode(int symbol, const uint32_t* cumul, uint32_t total) {
        // Ensure range >= total to avoid division yielding 0
        while (range < total) {
            output.push_back(low >> 24);
            low <<= 8;
            range <<= 8;
        }
        uint32_t r = range / total;
        low += cumul[symbol] * r;
        range = (cumul[symbol + 1] - cumul[symbol]) * r;
        normalize();
    }

    void finish() {
        for (int i = 0; i < 4; i++) {
            output.push_back(low >> 24);
            low <<= 8;
        }
    }
};

class RangeDecoder {
    const uint8_t* data;
    size_t size;
    size_t pos = 0;
    uint32_t low = 0;
    uint32_t code = 0;
    uint32_t range = 0xFFFFFFFF;

    uint8_t get_byte() {
        return pos < size ? data[pos++] : 0;
    }

    void normalize() {
        while (true) {
            if ((low ^ (low + range)) >= 0x1000000) {
                if (range >= 0x10000) break;
                range = -low & 0xFFFF;
            }
            code = (code << 8) | get_byte();
            low <<= 8;
            range <<= 8;
        }
    }

public:
    RangeDecoder(const uint8_t* d, size_t n) : data(d), size(n) {
        for (int i = 0; i < 4; i++) {
            code = (code << 8) | get_byte();
        }
    }

    int decode(const uint32_t* cumul, uint32_t total, int alpha_size) {
        // Ensure range >= total to avoid division yielding 0
        while (range < total) {
            code = (code << 8) | get_byte();
            low <<= 8;
            range <<= 8;
        }
        uint32_t r = range / total;
        uint32_t target = (code - low) / r;

        // Binary search for symbol
        int lo = 0, hi = alpha_size;
        while (lo + 1 < hi) {
            int mid = (lo + hi) / 2;
            if (cumul[mid] <= target) lo = mid;
            else hi = mid;
        }
        int symbol = lo;

        low += cumul[symbol] * r;
        range = (cumul[symbol + 1] - cumul[symbol]) * r;
        normalize();

        return symbol;
    }
};

// =============================================================================
// Main Compress/Decompress
// =============================================================================
inline std::vector<uint8_t> compress(const uint8_t* data, size_t n) {
    if (n == 0) return {};

    // Detect content type for optimal model selection
    ContentModel model = detect_content_model(data, n);

    uint32_t primary_idx;
    auto bwt = bwt_encode(data, n, primary_idx);
    auto mtf = mtf_encode(bwt.data(), bwt.size());
    auto rle = zrle_encode(mtf.data(), mtf.size());

    if (rle.empty()) return {};

    // Find max symbol to determine alpha size needed
    int max_sym = 0;
    for (auto s : rle) if (s > max_sym) max_sym = s;
    int alpha_size = max_sym + 1;

    // Build cumulative table from context-specific model
    std::vector<uint32_t> cumul(alpha_size + 1);
    build_cumul(cumul.data(), alpha_size, model);
    uint32_t total = cumul[alpha_size];

    // Output
    std::vector<uint8_t> output;

    // Header - 'B8' for version 8
    output.push_back('B');
    output.push_back('8');

    // Content model (1 byte) - NEW in v8.1
    output.push_back(static_cast<uint8_t>(model));

    // Original size (4 bytes)
    output.push_back(n & 0xFF);
    output.push_back((n >> 8) & 0xFF);
    output.push_back((n >> 16) & 0xFF);
    output.push_back((n >> 24) & 0xFF);

    // Primary index (4 bytes)
    output.push_back(primary_idx & 0xFF);
    output.push_back((primary_idx >> 8) & 0xFF);
    output.push_back((primary_idx >> 16) & 0xFF);
    output.push_back((primary_idx >> 24) & 0xFF);

    // RLE count (4 bytes)
    uint32_t rle_count = rle.size();
    output.push_back(rle_count & 0xFF);
    output.push_back((rle_count >> 8) & 0xFF);
    output.push_back((rle_count >> 16) & 0xFF);
    output.push_back((rle_count >> 24) & 0xFF);

    // Alpha size (2 bytes) - needed for decoding
    output.push_back(alpha_size & 0xFF);
    output.push_back((alpha_size >> 8) & 0xFF);

    // NO frequency table needed - using fixed model!

    // Range encode the data
    RangeEncoder encoder(output);
    for (auto sym : rle) {
        encoder.encode(sym, cumul.data(), total);
    }
    encoder.finish();

    return output;
}

inline std::vector<uint8_t> decompress(const uint8_t* data, size_t n) {
    if (n < 17) return {};  // Updated: 17 bytes header with model byte
    if (data[0] != 'B' || data[1] != '8') return {};

    // Read content model (1 byte) - NEW in v8.1
    ContentModel model = static_cast<ContentModel>(data[2]);

    uint32_t orig_size = data[3] | (data[4] << 8) | (data[5] << 16) | (data[6] << 24);
    uint32_t primary_idx = data[7] | (data[8] << 8) | (data[9] << 16) | (data[10] << 24);
    uint32_t rle_count = data[11] | (data[12] << 8) | (data[13] << 16) | (data[14] << 24);
    int alpha_size = data[15] | (data[16] << 8);

    // Build cumulative table from context-specific model
    std::vector<uint32_t> cumul(alpha_size + 1);
    build_cumul(cumul.data(), alpha_size, model);
    uint32_t total = cumul[alpha_size];

    // Range decode
    size_t encoded_start = 17;  // Updated: 17 bytes header with model byte
    RangeDecoder decoder(data + encoded_start, n - encoded_start);

    std::vector<uint16_t> rle(rle_count);
    for (size_t i = 0; i < rle_count; i++) {
        rle[i] = decoder.decode(cumul.data(), total, alpha_size);
    }

    // Reverse transforms
    auto mtf_data = zrle_decode(rle.data(), rle.size());
    auto bwt_data = mtf_decode(mtf_data.data(), mtf_data.size());
    auto output = bwt_decode(bwt_data.data(), bwt_data.size(), primary_idx);

    return output;
}

// =============================================================================
// Utility
// =============================================================================
inline const char* model_name(ContentModel m) {
    switch (m) {
        case ContentModel::PROSE:  return "PROSE";
        case ContentModel::CODE:   return "CODE";
        case ContentModel::CONFIG: return "CONFIG";
        case ContentModel::BINARY: return "BINARY";
        default:                   return "UNKNOWN";
    }
}

} // namespace bwt8