diff --git a/lib/core/settings.py b/lib/core/settings.py index 3cde0d5cad0..73ce6310099 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -20,7 +20,7 @@ from thirdparty import six # sqlmap version (...) -VERSION = "1.10.7.186" +VERSION = "1.10.7.187" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/lib/request/basic.py b/lib/request/basic.py index 8ed3c06ded4..43dc19f8102 100644 --- a/lib/request/basic.py +++ b/lib/request/basic.py @@ -325,9 +325,14 @@ def decodePage(page, contentEncoding, contentType, percentDecode=True): elif contentEncoding == "zstd": if _zstd is None: raise Exception("no Zstandard decoder available") - page = _zstd.decompress(page) + # bounded streaming decode: cap output at MAX_CONNECTION_TOTAL_SIZE without allocating the + # full result first, and enforce the 8 MB window the HTTP 'zstd' coding mandates + decompressor = _zstd.ZstdDecompressor(options={_zstd.DecompressionParameter.window_log_max: 23}) + page = decompressor.decompress(page, max_length=MAX_CONNECTION_TOTAL_SIZE + 1) if len(page) > MAX_CONNECTION_TOTAL_SIZE: raise Exception("size too large") + if not decompressor.eof: + raise Exception("incomplete Zstandard stream") else: data = gzip.GzipFile("", "rb", 9, io.BytesIO(page)) page = data.read(MAX_CONNECTION_TOTAL_SIZE + 1) diff --git a/lib/utils/brotli.py b/lib/utils/brotli.py index 631a97ae1fc..c0ee7a8f22f 100644 --- a/lib/utils/brotli.py +++ b/lib/utils/brotli.py @@ -12,11 +12,24 @@ # reference encoder across every quality/window/size. The 122 KB static dictionary + context-lookup # table live ZIP-packed in data/txt/brotli-dictionary.tx_ (same convention as wordlist.tx_). Py 2.7 / 3.x. +import hashlib import os +import threading import zipfile -_DICTIONARY = None # 122784-byte static dictionary (lazy-loaded) -_CONTEXT = None # 2048-byte context-lookup table (4 modes x 2 halves x 256) +_TABLES = None # (dictionary, context) published atomically on first use +_TABLES_LOCK = threading.Lock() + +# provenance: the RFC 7932 Appendix A static dictionary (122784 bytes) + the 2048-byte context-lookup +# table, extracted byte-for-byte from libbrotlicommon; verified on load so a swapped/corrupt resource +# fails loudly instead of silently mis-decoding +_TABLES_SHA256 = "20e42eb1b511c21806d4d227d07e5dd06877d8ce7b3a817f378f313653f35c70" # sha256 of the 122784-byte dictionary +_DICTIONARY_SIZE = 122784 +_CONTEXT_SIZE = 2048 + +# per-stream ceiling on total Huffman lookup-table entries: bounds decoder memory independently of the +# output cap (a hostile stream can declare many maximal 2^15-entry trees). ~10x the worst legitimate need. +_MAX_HUFFMAN_TABLE_ENTRIES = 1 << 20 # RFC 7932 Appendix A: words are bucketed by length (4..24); size_bits gives the index width per bucket, # offsets the cumulative byte offset of each bucket (derived from size_bits; last bucket end == 122784). @@ -180,28 +193,45 @@ class BrotliError(Exception): def _loadTables(): - global _DICTIONARY, _CONTEXT - if _DICTIONARY is not None: - return - - path = None - try: - from lib.core.data import paths - path = getattr(paths, "BROTLI_DICTIONARY", None) - except ImportError: - pass - if not path or not os.path.isfile(path): - path = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, "data", "txt", "brotli-dictionary.tx_") - - archive = zipfile.ZipFile(path) # ZIP-packed like wordlist.tx_ / catalog-identifiers.tx_ - try: - raw = archive.read(archive.namelist()[0]) - finally: - archive.close() - if len(raw) != 122784 + 2048: - raise BrotliError("invalid Brotli dictionary table") - _DICTIONARY = raw[:122784] - _CONTEXT = bytearray(raw[122784:]) + global _TABLES + tables = _TABLES + if tables is not None: # fast path: already published (dict, context) tuple + return tables + + with _TABLES_LOCK: + if _TABLES is not None: # another thread won the race + return _TABLES + try: + path = None + try: + from lib.core.data import paths + path = getattr(paths, "BROTLI_DICTIONARY", None) + except ImportError: + pass + if not path or not os.path.isfile(path): + path = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, "data", "txt", "brotli-dictionary.tx_") + + archive = zipfile.ZipFile(path) # ZIP-packed like wordlist.tx_ / catalog-identifiers.tx_ + try: + names = archive.namelist() + if len(names) != 1: + raise BrotliError("unexpected Brotli dictionary archive layout") + raw = archive.read(names[0]) + finally: + archive.close() + except BrotliError: + raise + except Exception as ex: + raise BrotliError("could not load the Brotli dictionary (%s)" % ex) + + if len(raw) != _DICTIONARY_SIZE + _CONTEXT_SIZE: + raise BrotliError("invalid Brotli dictionary length") + if hashlib.sha256(raw[:_DICTIONARY_SIZE]).hexdigest() != _TABLES_SHA256: + raise BrotliError("Brotli dictionary integrity check failed") + + # build both, then publish the pair atomically so a concurrent reader never sees a half-set state + _TABLES = (raw[:_DICTIONARY_SIZE], bytearray(raw[_DICTIONARY_SIZE:])) + return _TABLES class _BitReader(object): @@ -225,17 +255,23 @@ def readBits(self, count): return 0 if self.bits < count: self._fill() + if self.bits < count: # ran off the end of the stream -> truncated, not zero-padded + raise BrotliError("truncated Brotli stream") value = self.acc & ((1 << count) - 1) self.acc >>= count self.bits -= count return value def peek(self, count): + # lenient lookahead (a prefix-code peek may legitimately reach past the final byte); only the + # matching drop() actually consumes, and drop() rejects consuming more than really remains if self.bits < count: self._fill() return self.acc & ((1 << count) - 1) def drop(self, count): + if self.bits < count: # the matched code needs bits the stream does not have + raise BrotliError("truncated Brotli stream") self.acc >>= count self.bits -= count @@ -253,10 +289,17 @@ def readBytes(self, count): self.bits -= 8 count -= 1 if count > 0: + if self.pos + count > self.size: + raise BrotliError("truncated Brotli stream") out += self.data[self.pos:self.pos + count] self.pos += count return bytes(out) + def exhausted(self): + # true once no whole real bytes remain beyond the current (partial) byte - used to reject + # trailing garbage after the final meta-block + return self.pos >= self.size and self.bits < 8 + def _reverseBits(value, count): result = 0 @@ -269,54 +312,66 @@ def _reverseBits(value, count): class _Huffman(object): __slots__ = ("maxLength", "table", "single") - def __init__(self, lengths): + def __init__(self, lengths, budget=None): self.single = None self.table = None self.maxLength = max(lengths) if lengths else 0 - if self.maxLength == 0: - self.single = 0 - for symbol, length in enumerate(lengths): - if length > 0: - self.single = symbol + used = [(symbol, length) for symbol, length in enumerate(lengths) if length] + if not used: + raise BrotliError("empty Brotli prefix code") + if self.maxLength == 0 or len(used) == 1: # a one-symbol code is always that symbol (0 bits) + self.single = used[0][0] + self.maxLength = 0 return + if budget is not None: + budget[0] -= (1 << self.maxLength) + if budget[0] < 0: + raise BrotliError("Brotli decoder table budget exceeded") + counts = [0] * (self.maxLength + 1) - for length in lengths: - if length: - counts[length] += 1 + for _, length in used: + counts[length] += 1 nextCode = [0] * (self.maxLength + 2) code = 0 + space = 0 for bits in range(1, self.maxLength + 1): code = (code + counts[bits - 1]) << 1 nextCode[bits] = code - - self.table = [(0, 0)] * (1 << self.maxLength) - for symbol, length in enumerate(lengths): - if length: - reversed_ = _reverseBits(nextCode[length], length) - nextCode[length] += 1 - step = 1 << length - for index in range(reversed_, 1 << self.maxLength, step): - self.table[index] = (symbol, length) + space += counts[bits] << (self.maxLength - bits) + if space != (1 << self.maxLength): # over- or under-subscribed prefix code (must be complete) + raise BrotliError("invalid Brotli prefix code") + + self.table = [None] * (1 << self.maxLength) # None = unreachable slot (rejected on decode) + for symbol, length in used: + reversed_ = _reverseBits(nextCode[length], length) + nextCode[length] += 1 + step = 1 << length + for index in range(reversed_, 1 << self.maxLength, step): + self.table[index] = (symbol, length) def decode(self, reader): if self.table is None: return self.single - symbol, length = self.table[reader.peek(self.maxLength)] - reader.drop(length) - return symbol + entry = self.table[reader.peek(self.maxLength)] + if entry is None: # bits matched no code -> malformed stream + raise BrotliError("invalid Brotli prefix code") + reader.drop(entry[1]) + return entry[0] -def _readSimplePrefix(reader, alphabetSize): +def _readSimplePrefix(reader, alphabetSize, budget): count = reader.readBits(2) + 1 symbolBits = (alphabetSize - 1).bit_length() or 1 symbols = [reader.readBits(symbolBits) for _ in range(count)] - lengths = [0] * alphabetSize + for symbol in symbols: + if symbol >= alphabetSize: + raise BrotliError("out-of-range symbol in Brotli simple prefix code") + if len(set(symbols)) != count: + raise BrotliError("duplicate symbol in Brotli simple prefix code") if count == 1: - huffman = _Huffman([]) - huffman.single = symbols[0] - return huffman - if count == 2: + pairs = [(symbols[0], 1)] # one symbol -> _Huffman makes it a 0-bit code + elif count == 2: pairs = [(symbols[0], 1), (symbols[1], 1)] elif count == 3: pairs = [(symbols[0], 1), (symbols[1], 2), (symbols[2], 2)] @@ -324,12 +379,13 @@ def _readSimplePrefix(reader, alphabetSize): pairs = [(symbols[0], 1), (symbols[1], 2), (symbols[2], 3), (symbols[3], 3)] else: pairs = [(symbols[0], 2), (symbols[1], 2), (symbols[2], 2), (symbols[3], 2)] + lengths = [0] * alphabetSize for symbol, length in pairs: lengths[symbol] = length - return _Huffman(lengths) + return _Huffman(lengths, budget) -def _readComplexPrefix(reader, alphabetSize, skip): +def _readComplexPrefix(reader, alphabetSize, skip, budget): codeLengths = [0] * 18 space = 32 for symbol in _CL_ORDER[skip:]: @@ -340,7 +396,7 @@ def _readComplexPrefix(reader, alphabetSize, skip): space -= 32 >> codeLengths[symbol] if space <= 0: break - codeLengthHuffman = _Huffman(codeLengths) + codeLengthHuffman = _Huffman(codeLengths, budget) lengths = [0] * alphabetSize symbol = 0 @@ -370,20 +426,20 @@ def _readComplexPrefix(reader, alphabetSize, skip): repeat += delta + 3 emit = repeat - old for _ in range(emit): - if symbol >= alphabetSize: - break + if symbol >= alphabetSize: # a run past the alphabet is a malformed stream + raise BrotliError("Brotli code-length run exceeds alphabet") lengths[symbol] = repeatLength symbol += 1 if repeatLength: space -= emit << (15 - repeatLength) - return _Huffman(lengths) + return _Huffman(lengths, budget) -def _readPrefix(reader, alphabetSize): +def _readPrefix(reader, alphabetSize, budget): header = reader.readBits(2) if header == 1: - return _readSimplePrefix(reader, alphabetSize) - return _readComplexPrefix(reader, alphabetSize, header) + return _readSimplePrefix(reader, alphabetSize, budget) + return _readComplexPrefix(reader, alphabetSize, header, budget) def _readBlockTypeCount(reader): @@ -393,19 +449,24 @@ def _readBlockTypeCount(reader): return (1 << bits) + 1 + reader.readBits(bits) -def _readContextMap(reader, treeCount, size): +def _readContextMap(reader, treeCount, size, budget): maxRun = reader.readBits(4) + 1 if reader.readBits(1) else 0 - huffman = _readPrefix(reader, treeCount + maxRun) + huffman = _readPrefix(reader, treeCount + maxRun, budget) contextMap = [] while len(contextMap) < size: code = huffman.decode(reader) if code == 0: contextMap.append(0) elif code <= maxRun: - contextMap.extend([0] * ((1 << code) + reader.readBits(code))) + run = (1 << code) + reader.readBits(code) + if len(contextMap) + run > size: # a run past the declared map size is malformed + raise BrotliError("Brotli context map run overruns the map") + contextMap.extend([0] * run) else: - contextMap.append(code - maxRun) - del contextMap[size:] + value = code - maxRun + if value >= treeCount: # references a tree that was not declared + raise BrotliError("Brotli context map references an undefined tree") + contextMap.append(value) if reader.readBits(1): # inverse move-to-front moveToFront = list(range(256)) for i in range(len(contextMap)): @@ -456,11 +517,8 @@ def decompress(data, maxOutput=100 * 1024 * 1024): """Decompress a Brotli (RFC 7932) stream, returning the original bytes. Raises BrotliError on a malformed stream or if the output would exceed 'maxOutput' (an anti-decompression-bomb cap).""" - _loadTables() - dictionary = _DICTIONARY - context = _CONTEXT - try: + dictionary, context = _loadTables() reader = _BitReader(data) header = reader.readBits(1) if header == 0: @@ -506,12 +564,14 @@ def decompress(data, maxOutput=100 * 1024 * 1024): raise BrotliError("output too large") continue + budget = [_MAX_HUFFMAN_TABLE_ENTRIES] # per-meta-block Huffman memory ceiling + typesL = _readBlockTypeCount(reader) blockL, typeHuffmanL, lengthHuffmanL, prevTypeL = 1 << 28, None, None, 1 typeL = 0 if typesL >= 2: - typeHuffmanL = _readPrefix(reader, typesL + 2) - lengthHuffmanL = _readPrefix(reader, 26) + typeHuffmanL = _readPrefix(reader, typesL + 2, budget) + lengthHuffmanL = _readPrefix(reader, 26, budget) code = lengthHuffmanL.decode(reader) blockL = _BLEN_BASE[code] + reader.readBits(_BLEN_EXTRA[code]) @@ -519,8 +579,8 @@ def decompress(data, maxOutput=100 * 1024 * 1024): blockI, typeHuffmanI, lengthHuffmanI, prevTypeI = 1 << 28, None, None, 1 typeI = 0 if typesI >= 2: - typeHuffmanI = _readPrefix(reader, typesI + 2) - lengthHuffmanI = _readPrefix(reader, 26) + typeHuffmanI = _readPrefix(reader, typesI + 2, budget) + lengthHuffmanI = _readPrefix(reader, 26, budget) code = lengthHuffmanI.decode(reader) blockI = _BLEN_BASE[code] + reader.readBits(_BLEN_EXTRA[code]) @@ -528,8 +588,8 @@ def decompress(data, maxOutput=100 * 1024 * 1024): blockD, typeHuffmanD, lengthHuffmanD, prevTypeD = 1 << 28, None, None, 1 typeD = 0 if typesD >= 2: - typeHuffmanD = _readPrefix(reader, typesD + 2) - lengthHuffmanD = _readPrefix(reader, 26) + typeHuffmanD = _readPrefix(reader, typesD + 2, budget) + lengthHuffmanD = _readPrefix(reader, 26, budget) code = lengthHuffmanD.decode(reader) blockD = _BLEN_BASE[code] + reader.readBits(_BLEN_EXTRA[code]) @@ -538,14 +598,14 @@ def decompress(data, maxOutput=100 * 1024 * 1024): contextModes = [reader.readBits(2) for _ in range(typesL)] treesL = _readBlockTypeCount(reader) - contextMapL = _readContextMap(reader, treesL, typesL * 64) if treesL >= 2 else [0] * (typesL * 64) + contextMapL = _readContextMap(reader, treesL, typesL * 64, budget) if treesL >= 2 else [0] * (typesL * 64) treesD = _readBlockTypeCount(reader) - contextMapD = _readContextMap(reader, treesD, typesD * 4) if treesD >= 2 else [0] * (typesD * 4) + contextMapD = _readContextMap(reader, treesD, typesD * 4, budget) if treesD >= 2 else [0] * (typesD * 4) - huffmanL = [_readPrefix(reader, 256) for _ in range(treesL)] - huffmanI = [_readPrefix(reader, 704) for _ in range(typesI)] + huffmanL = [_readPrefix(reader, 256, budget) for _ in range(treesL)] + huffmanI = [_readPrefix(reader, 704, budget) for _ in range(typesI)] distanceAlphabet = 16 + direct + (48 << postfix) - huffmanD = [_readPrefix(reader, distanceAlphabet) for _ in range(treesD)] + huffmanD = [_readPrefix(reader, distanceAlphabet, budget) for _ in range(treesD)] produced = 0 while produced < metaLength: @@ -614,6 +674,9 @@ def decompress(data, maxOutput=100 * 1024 * 1024): low = value & ((1 << postfix) - 1) distance = ((((2 + (high & 1)) << extraBits) - 4 + extra) << postfix) + low + direct + 1 + if distance <= 0: # a ring/short-code computation must yield >= 1 + raise BrotliError("invalid Brotli distance") + maxDistance = min(len(out), maxBackward) if distanceCode != 0 and distance <= maxDistance: distRing[distIndex & 3] = distance @@ -637,6 +700,8 @@ def decompress(data, maxOutput=100 * 1024 * 1024): raise BrotliError("invalid dictionary transform") start = _OFFSETS[copyLength] + index * copyLength word = _applyTransform(transformId, dictionary[start:start + copyLength]) + if produced + len(word) > metaLength: # a transformed word must still fit the block + raise BrotliError("dictionary word exceeds meta-block length") out += word produced += len(word) @@ -645,6 +710,13 @@ def decompress(data, maxOutput=100 * 1024 * 1024): if isLast: break + + # after the final meta-block only zero byte-alignment padding may remain: no whole leftover bytes + # (trailing garbage) and the padding bits themselves must be zero (RFC 7932) + if reader.bits + (reader.size - reader.pos) * 8 >= 8: + raise BrotliError("trailing data after Brotli stream") + if reader.acc != 0: + raise BrotliError("non-zero Brotli padding bits") return bytes(out) except BrotliError: raise diff --git a/tests/test_brotli.py b/tests/test_brotli.py index 42525036661..4024a744c6b 100644 --- a/tests/test_brotli.py +++ b/tests/test_brotli.py @@ -4,12 +4,14 @@ Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org) See the file 'LICENSE' for copying permission -Tests for the dependency-free Brotli (RFC 7932) decompressor under lib/utils/brotli.py. The compressed -fixtures were produced by the reference encoder at various quality levels; the expected plaintext is -reconstructed here by construction, so the suite validates the decoder fully offline (no third-party -'brotli' module at test time) on Python 2.7 / 3.x. Cases deliberately exercise the static dictionary + -word transforms, long overlapping copies, UTF-8, the low-quality (near-uniform tree) path and the -repetitive content that relies on the higher insert-and-copy command ranges. +Tests for the dependency-free Brotli (RFC 7932) decompressor under lib/utils/brotli.py, and its wiring +into lib/request/basic.py::decodePage. The compressed fixtures were produced by the reference encoder at +various quality levels; the expected plaintext is reconstructed here by construction, so the suite +validates the decoder fully offline (no third-party 'brotli' module at test time) on Python 2.7 / 3.x. +Positive cases exercise the static dictionary + word transforms, long overlapping copies, UTF-8, the +low-quality (near-uniform tree) path and the repetitive content that relies on the higher insert-and-copy +command ranges. Negative cases assert that hostile input (truncation, garbage, output bombs) is rejected +with a BrotliError rather than silently producing corrupted output. """ import binascii @@ -35,12 +37,15 @@ (b"the quick brown fox " * 30, "1b570200047463a92ee78362f22082d628041695d90acef9a3f135e9c701"), (b"AB" * 400, "1b1f0300a48284a2b230b009"), - ((u"caf\xe9 na\xefve \u4f60\u597d ").encode("utf-8") * 12, + (u"caf\xe9 na\xefve \u4f60\u597d ".encode("utf-8") * 12, "1bef00004427477ad6d60ac38c93200a288ab462c2a06461d22d186dbbe0263e0707"), (b"hello hello hello world world foo bar baz " * 6, "8b7d000080aaaaaaeaff74e5f355048415f8c0000c201701d0ffbbeadf736f75cfa82e6f63b82b5e2c2c2c6c6cacea654675f0e1c38fc160308e33595583c16030180ce65067442a4aa370586827d97b828968074727f5b21e97eebd045d8baeefef94c3fca4fb1e"), ] +# a valid stream (~1.2 KB of text) whose truncations feed the negative corpus +_TRUNCATION_SAMPLE = binascii.unhexlify("1b570200047463a92ee78362f22082d628041695d90acef9a3f135e9c701") + class TestBrotli(unittest.TestCase): def test_known_fixtures(self): @@ -50,20 +55,66 @@ def test_known_fixtures(self): def test_empty_stream(self): self.assertEqual(decompress(binascii.unhexlify("3b")), b"") - def test_malformed_raises(self): - # a hostile/truncated stream must surface as BrotliError, never a raw exception - for blob in (b"\xff", b"\x00\x00\x00", b"\x1b\xff\xff\xff\xff", os.urandom(32)): + def test_truncation_is_rejected(self): + # every proper prefix of a valid stream is truncated -> must raise, never silently return + # corrupted or zero-padded output + for cut in range(1, len(_TRUNCATION_SAMPLE)): + self.assertRaises(BrotliError, decompress, _TRUNCATION_SAMPLE[:cut]) + + def test_malformed_is_rejected(self): + for blob in (b"\xff", b"\x00\x00\x00", b"\x1b\xff\xff\xff\xff", + _TRUNCATION_SAMPLE + b"\x00\x00\x00\x00", # trailing garbage + binascii.unhexlify("3b") + b"\xde\xad"): # data after a complete empty stream + self.assertRaises(BrotliError, decompress, blob) + + def test_non_brotlierror_never_escapes(self): + # arbitrary bytes must terminate quickly and only ever raise BrotliError (never a raw exception) + for i in range(400): + blob = bytes(bytearray((i * 37 + j * 13) & 0xff for j in range(i % 60))) try: decompress(blob) except BrotliError: pass - except Exception as ex: - self.fail("non-BrotliError on malformed input: %s" % ex) def test_bomb_cap(self): # a small stream must not be allowed to expand past the output cap self.assertRaises(BrotliError, decompress, binascii.unhexlify("1b1f0300a48284a2b230b009"), 16) +class TestBrotliDecodePage(unittest.TestCase): + _KB = ("pageCompress", "pageEncoding", "disableHtmlDecoding", "singleLogFlags") + _CONF = ("encoding", "nullConnection") + + def setUp(self): + from lib.core.data import conf, kb + self._kb = dict((name, kb.get(name)) for name in self._KB) + self._conf = dict((name, conf.get(name)) for name in self._CONF) + + def tearDown(self): + from lib.core.data import conf, kb + for name, value in self._kb.items(): + kb[name] = value + for name, value in self._conf.items(): + conf[name] = value + + def test_decodepage_br(self): + from lib.core.data import conf, kb + from lib.request.basic import decodePage + from lib.core.convert import getBytes + + conf.encoding = None + conf.nullConnection = False + kb.pageCompress = True + kb.pageEncoding = None + kb.singleLogFlags = set() + kb.disableHtmlDecoding = True + + body = b"secret uid=admin" * 25 + # brotli-compressed 'body' (reference encoder, quality 11) + compressed = binascii.unhexlify( + "1b190488c56d6c1ff52d8742bd820d3870892cd08016f661030e310d82f520b7a3513810bf66edf05d3500") + self.assertEqual(getBytes(decodePage(compressed, "br", "text/html; charset=utf-8")), body) + + if __name__ == "__main__": unittest.main()