diff --git a/data/txt/common-params.txt b/data/txt/common-params.txt new file mode 100644 index 00000000000..81b7bc46329 --- /dev/null +++ b/data/txt/common-params.txt @@ -0,0 +1,243 @@ +# Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org) +# See the file 'LICENSE' for copying permission + +id +page +q +query +search +s +keyword +keywords +name +username +user +uid +userid +email +mail +pass +password +pwd +token +key +apikey +api_key +access_token +auth +session +sid +sessionid +lang +language +locale +country +region +city +zip +sort +order +orderby +dir +direction +filter +category +cat +type +kind +class +group +tag +tags +status +state +action +act +cmd +command +op +operation +mode +method +func +function +callback +jsonp +format +output +view +tab +step +stage +debug +test +dev +admin +role +level +priv +file +filename +path +dir +folder +doc +document +url +uri +link +href +redirect +redirect_uri +return +returnurl +return_url +next +target +dest +destination +goto +continue +ref +referer +referrer +source +src +from +to +date +year +month +day +time +timestamp +start +end +begin +finish +limit +offset +count +num +number +size +length +width +height +amount +price +qty +quantity +value +val +data +input +content +text +msg +message +body +title +subject +description +desc +comment +note +code +hash +sig +signature +csrf +csrf_token +csrftoken +nonce +salt +enc +encrypt +decrypt +base64 +json +xml +raw +echo +reflect +show +hide +display +render +template +tpl +theme +skin +style +color +font +image +img +photo +avatar +icon +banner +video +audio +media +attachment +upload +download +export +import +backup +restore +sync +refresh +reload +reset +clear +flush +purge +enable +disable +active +enabled +visible +public +private +locked +verified +confirmed +approved +product +item +sku +model +brand +vendor +seller +store +shop +cart +basket +checkout +payment +invoice +receipt +transaction +account +profile +member +customer +client +company +organization +org +department +team +project +task +job +event +booking +reservation +appointment +schedule +calendar diff --git a/extra/vulnserver/vulnserver.py b/extra/vulnserver/vulnserver.py index e82f3850647..95c1a9da45c 100644 --- a/extra/vulnserver/vulnserver.py +++ b/extra/vulnserver/vulnserver.py @@ -407,6 +407,10 @@ def _xpath_element_to_dict(el): _server = None _alive = False _csrf_token = None +_ratelimit_hits = 0 + +# number of initial hits to '/ratelimit' answered with 429 before it behaves normally +RATELIMIT_INITIAL_429 = 1 def init(quiet=False): global _conn @@ -963,6 +967,22 @@ def do_REQUEST(self): self.wfile.write(b"Request blocked: security policy violation (WAF)") return + # rate-limit emulator ('/ratelimit'): the first hit(s) answer 429 with a 'Retry-After', then + # it behaves like the default SQLi endpoint - so a client that honors the backoff and retries + # eventually gets through (drives the adaptive rate-limit handling) + if self.url == "/ratelimit": + global _ratelimit_hits + _ratelimit_hits += 1 + if _ratelimit_hits <= RATELIMIT_INITIAL_429: + self.send_response(429) + self.send_header("Retry-After", "0") + self.send_header("Content-type", "text/html; charset=%s" % UNICODE_ENCODING) + self.send_header("Connection", "close") + self.end_headers() + self.wfile.write(b"Too Many Requests") + return + self.url = "/" + if self.url == "/xxe": self.send_response(OK) self.send_header("Content-type", "application/xml; charset=%s" % UNICODE_ENCODING) diff --git a/lib/controller/controller.py b/lib/controller/controller.py index 48f137e9a2b..3ec482316a4 100644 --- a/lib/controller/controller.py +++ b/lib/controller/controller.py @@ -529,6 +529,10 @@ def start(): checkWaf() + if conf.mineParams and not any((conf.graphql, conf.nosql, conf.ldap, conf.xpath, conf.ssti, conf.xxe, conf.hql)): + from lib.utils.paraminer import mineParameters + mineParameters() + if any((conf.graphql, conf.nosql, conf.ldap, conf.xpath, conf.ssti, conf.xxe, conf.hql)) and (conf.reportJson or conf.resultsFile): singleTimeWarnMessage("'--report-json'/'--results-file' do not (yet) capture non-SQL technique (--graphql/--nosql/--ldap/--xpath/--ssti/--xxe/--hql) findings; these are reported on the console only") diff --git a/lib/core/common.py b/lib/core/common.py index 6b70d688150..9220dcb9d81 100644 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -1589,6 +1589,7 @@ def setPaths(rootPath): paths.COMMON_COLUMNS = os.path.join(paths.SQLMAP_TXT_PATH, "common-columns.txt") paths.COMMON_FILES = os.path.join(paths.SQLMAP_TXT_PATH, "common-files.txt") paths.COMMON_TABLES = os.path.join(paths.SQLMAP_TXT_PATH, "common-tables.txt") + paths.COMMON_PARAMETERS = os.path.join(paths.SQLMAP_TXT_PATH, "common-params.txt") paths.SQL_KEYWORDS = os.path.join(paths.SQLMAP_TXT_PATH, "keywords.txt") paths.SMALL_DICT = os.path.join(paths.SQLMAP_TXT_PATH, "smalldict.txt") paths.USER_AGENTS = os.path.join(paths.SQLMAP_TXT_PATH, "user-agents.txt") diff --git a/lib/core/enums.py b/lib/core/enums.py index ced9700a854..7973d48b3f8 100644 --- a/lib/core/enums.py +++ b/lib/core/enums.py @@ -282,6 +282,7 @@ class HTTP_HEADER(object): RANGE = "Range" REFERER = "Referer" REFRESH = "Refresh" # Reference: http://stackoverflow.com/a/283794 + RETRY_AFTER = "Retry-After" SERVER = "Server" SET_COOKIE = "Set-Cookie" TRANSFER_ENCODING = "Transfer-Encoding" diff --git a/lib/core/optiondict.py b/lib/core/optiondict.py index 504182e22d7..8bf6951d2c5 100644 --- a/lib/core/optiondict.py +++ b/lib/core/optiondict.py @@ -241,6 +241,7 @@ "eta": "boolean", "flushSession": "boolean", "forms": "boolean", + "mineParams": "boolean", "freshQueries": "boolean", "googlePage": "integer", "harFile": "string", diff --git a/lib/core/settings.py b/lib/core/settings.py index e7b8a3cf4b3..f4abc3823ff 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -20,7 +20,7 @@ from thirdparty import six # sqlmap version (...) -VERSION = "1.10.7.180" +VERSION = "1.10.7.182" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) @@ -58,6 +58,17 @@ # false positive) rather than the back-end actually answering. WAF_BLOCK_HTTP_CODES = (403, 406, 429, 451, 501, 503) +# HTTP status signalling that the client is being rate-limited (kept as a literal because Python 2's +# httplib has no such constant) +TOO_MANY_REQUESTS_HTTP_CODE = 429 + +# Adaptive rate-limit handling: one-time backoff used when a rate-limited response carries no usable +# 'Retry-After', the additive step by which the inter-request delay is raised on each hit, and the +# ceiling for both the honored backoff and the auto-throttle (seconds) +RATE_LIMIT_DEFAULT_DELAY = 1.0 +RATE_LIMIT_DELAY_STEP = 0.5 +RATE_LIMIT_MAX_DELAY = 60.0 + # Candidate tamper scripts for automatic WAF-bypass, ordered by empirical WAF-bypass value # (structural token-substitution first, camouflage last; per identYwaf data). The back-end DBMS # is not pre-filtered here: semantics-preservation is verified at runtime by re-running detection @@ -92,6 +103,9 @@ LOWER_RATIO_BOUND = 0.02 UPPER_RATIO_BOUND = 0.98 +# Number of candidate names probed per request while mining for hidden parameters ('--mine-params') +PARAMETER_MINING_BUCKET_SIZE = 25 + # For filling in case of dumb push updates DUMMY_JUNK = "Phah5jue" diff --git a/lib/core/testing.py b/lib/core/testing.py index c8289a29ebb..e6d62cfa54e 100644 --- a/lib/core/testing.py +++ b/lib/core/testing.py @@ -66,6 +66,8 @@ def vulnTest(tests=None, label="vuln"): ("-u --data=\"security_level=4\" -p id --flush-session --technique=B --banner", ("random (non-scanner) User-Agent and browser-like headers to bypass the WAF/IPS", "Type: boolean-based blind", "banner: '3.")), # automatic WAF-bypass against a libinjection-class WAF: tampers cannot help, only the non-scanner User-Agent does ("-u --data=\"security_level=5\" -p id --flush-session --technique=B", ("unable to automatically bypass the WAF/IPS", "does not seem to be injectable")), # automatic WAF-bypass honest bail: a libinjection-class WAF that no User-Agent or tamper can defeat ("-u -p id --flush-session --technique=B --proof", ("sqlmap proved exploitation of the following injection point", "Parameter: id (GET)", "Technique: boolean-based blind", "TRUE (5/5)", "repeatably", "Retrieved: back-end DBMS banner '3.")), # --proof: report-grade proof in the injection-point style - forces the boolean technique (so a multi-technique point still proves), and actively reads a value out as the strongest proof + ("-u --mine-params --flush-session --technique=B", ("mining for hidden GET parameters", "found hidden parameter 'id'", "held back parameter(s) that break the base request", "Parameter: id (GET)", "Type: boolean-based blind")), # --mine-params: discover an injectable parameter absent from a bare URL, hold back the raw-SQL sink that would shadow it, then confirm the injection on the mined 'id' + ("-u \"ratelimit?id=1\" --flush-session --technique=B", ("target appears to be rate-limiting", "Parameter: id (GET)", "Type: boolean-based blind")), # adaptive rate-limit handling: the endpoint answers 429 with 'Retry-After' first, so detection only succeeds if sqlmap honors the backoff, throttles, and retries rather than treating 429 as a hard block ("-r --flush-session -v 5 --test-skip=\"heavy\" --save=", ("CloudFlare", "web application technology: Express", "possible DBMS: 'SQLite'", "User-Agent: foobar", "~Type: time-based blind", "saved command line options to the configuration file")), ("-c ", ("CloudFlare", "possible DBMS: 'SQLite'", "User-Agent: foobar", "~Type: time-based blind")), ("-l --flush-session --skip-waf -vvvvv --technique=U --union-from=users --banner --parse-errors", ("banner: '3.", "ORDER BY term out of range", "~xp_cmdshell", "Connection: keep-alive")), diff --git a/lib/parse/cmdline.py b/lib/parse/cmdline.py index bc838a88dc5..5bbc8c51cc9 100644 --- a/lib/parse/cmdline.py +++ b/lib/parse/cmdline.py @@ -721,6 +721,9 @@ def cmdLineParser(argv=None): general.add_argument("--forms", dest="forms", action="store_true", help="Parse and test forms on target URL") + general.add_argument("--mine-params", dest="mineParams", action="store_true", + help="Mine for hidden (unlinked) GET parameters to test") + general.add_argument("--fresh-queries", dest="freshQueries", action="store_true", help="Ignore query results stored in session file") diff --git a/lib/request/connect.py b/lib/request/connect.py index fc23cf99a7b..e3cd05b470a 100644 --- a/lib/request/connect.py +++ b/lib/request/connect.py @@ -6,6 +6,8 @@ """ import binascii +import calendar +import email.utils import inspect import io import logging @@ -119,9 +121,13 @@ from lib.core.settings import PLAIN_TEXT_CONTENT_TYPE from lib.core.settings import RANDOM_INTEGER_MARKER from lib.core.settings import RANDOM_STRING_MARKER +from lib.core.settings import RATE_LIMIT_DEFAULT_DELAY +from lib.core.settings import RATE_LIMIT_DELAY_STEP +from lib.core.settings import RATE_LIMIT_MAX_DELAY from lib.core.settings import REPLACEMENT_MARKER from lib.core.settings import SAFE_HEX_MARKER from lib.core.settings import TEXT_CONTENT_TYPE_REGEX +from lib.core.settings import TOO_MANY_REQUESTS_HTTP_CODE from lib.core.settings import UNENCODED_ORIGINAL_VALUE from lib.core.settings import UNICODE_ENCODING from lib.core.settings import URI_HTTP_HEADER @@ -223,6 +229,49 @@ def _retryProxy(**kwargs): kwargs['retrying'] = True return Connect._getPageProxy(**kwargs) + @staticmethod + def _parseRetryAfter(responseHeaders): + """ + Parses a 'Retry-After' response header (RFC 7231 delta-seconds or an HTTP-date) into a number + of seconds to wait, or None when it is absent or unparseable. + """ + + value = (responseHeaders.get(HTTP_HEADER.RETRY_AFTER) if responseHeaders else None) or "" + value = value.strip() + + if value.isdigit(): + return float(value) + + parsed = email.utils.parsedate(value) + return max(0.0, calendar.timegm(parsed) - time.time()) if parsed else None + + @staticmethod + def _rateLimitRetry(responseHeaders, code, **kwargs): + """ + Handles a rate-limited response by honoring its 'Retry-After' (capped), adaptively raising the + inter-request delay so subsequent requests self-throttle under the limit, then re-issuing the + request. Returns the retried (page, headers, code) or None when the retry budget is exhausted, + so the caller can surface the rate-limited response as-is. + """ + + threadData = getCurrentThreadData() + if threadData.retriesCount >= conf.retries or kb.threadException: + return None + + retryAfter = Connect._parseRetryAfter(responseHeaders) + backoff = min(retryAfter if retryAfter is not None else RATE_LIMIT_DEFAULT_DELAY, RATE_LIMIT_MAX_DELAY) + + # additive-increase throttle: nudge the inter-request delay up toward a sustainable pace. The + # auto-throttle is capped, but a larger user-set '--delay' is never lowered. It is monotonic, + # so a lost concurrent update across threads self-heals on the next hit. + conf.delay = max(conf.delay or 0, min(RATE_LIMIT_MAX_DELAY, (conf.delay or 0) + RATE_LIMIT_DELAY_STEP)) + + singleTimeWarnMessage("target appears to be rate-limiting requests; sqlmap is backing off and throttling accordingly (consider raising '--delay' or lowering '--threads')") + logger.debug("rate-limited (HTTP %d)%s; sleeping %.1f second(s), inter-request delay now %.1f second(s)" % (code, " honoring 'Retry-After'" if retryAfter is not None else "", backoff, conf.delay)) + + time.sleep(backoff) + return Connect._retryProxy(**kwargs) + @staticmethod def _connReadProxy(conn): parts = [] @@ -844,7 +893,13 @@ class _(dict): raise SystemExit if ex.code not in (conf.ignoreCode or []): - if ex.code == _http_client.UNAUTHORIZED: + if ex.code == TOO_MANY_REQUESTS_HTTP_CODE or (ex.code == _http_client.SERVICE_UNAVAILABLE and Connect._parseRetryAfter(responseHeaders) is not None): + retried = Connect._rateLimitRetry(responseHeaders, ex.code, **kwargs) + if retried is not None: + return retried + debugMsg = "target kept rate-limiting after %d retries (%d)" % (conf.retries, code) + logger.debug(debugMsg) + elif ex.code == _http_client.UNAUTHORIZED: errMsg = "not authorized, try to provide right HTTP " errMsg += "authentication type and valid credentials (%d). " % code errMsg += "If this is intended, try to rerun by providing " diff --git a/lib/utils/paraminer.py b/lib/utils/paraminer.py new file mode 100644 index 00000000000..a6a67f974b6 --- /dev/null +++ b/lib/utils/paraminer.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python + +""" +Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org) +See the file 'LICENSE' for copying permission +""" + +import difflib + +from lib.core.compat import xrange +from lib.core.common import getFileItems +from lib.core.common import paramToDict +from lib.core.common import randomStr +from lib.core.common import singleTimeWarnMessage +from lib.core.data import conf +from lib.core.data import logger +from lib.core.data import paths +from lib.core.enums import HTTPMETHOD +from lib.core.enums import PLACE +from lib.core.settings import DIFF_TOLERANCE +from lib.core.settings import MAX_DIFFLIB_SEQUENCE_LENGTH +from lib.core.settings import PARAMETER_MINING_BUCKET_SIZE +from lib.request.connect import Connect as Request + +# Benign, broadly-stable value used both to check whether a discovered parameter is safe to add and +# as its seed during testing (a random value would error out an integer/id context and defeat inference) +PROBE_VALUE = "1" + +def _canary(): + return randomStr(10, lowercase=True) + +def _fetch(get): + """ + Requests the target with the given GET query string, returning a (page, HTTP code) pair. + """ + + try: + page, _, code = Request.getPage(get=get or None, silent=True, raise404=False) + except Exception: + page, code = None, None + + return (page or ""), code + +def _ratio(first, second): + """ + Similarity of two response bodies, mirroring the core page comparison (see comparison.py): an + exact match, a length ratio for oversized bodies, otherwise difflib's quick_ratio(). + """ + + if not first or not second: + return 0.0 + + if first == second: + return 1.0 + + if any(len(_) > MAX_DIFFLIB_SEQUENCE_LENGTH for _ in (first, second)): + ratio = 1.0 * len(first) / len(second) + return ratio if ratio <= 1 else 1.0 / ratio + + return difflib.SequenceMatcher(None, first, second).quick_ratio() + +def _differs(page, base, floor): + """ + True when 'page' departs from the baseline by more than the target's own dynamic jitter ('floor'). + """ + + return bool(page) and _ratio(page, base) < floor - DIFF_TOLERANCE + +def _confirm(baseGet, name, base, floor): + """ + Confirms a single candidate in isolation with two differently-valued probes. Returns 'reflected' + when a value is echoed back (caught here even if a shared bucket hid it behind a preceding + parameter), 'behavioral' when both probes resemble each other yet depart from the baseline (its + presence, not its value, matters), otherwise None. + """ + + def _get(value): + pair = "%s=%s" % (name, value) + return "%s&%s" % (baseGet, pair) if baseGet else pair + + canaries = (_canary(), _canary()) + first, second = _fetch(_get(canaries[0]))[0], _fetch(_get(canaries[1]))[0] + + if not (first and second): + return None + + if (canaries[0] in first or canaries[1] in second) and not any(_ in base for _ in canaries): + return "reflected" + + if _ratio(first, second) < floor - DIFF_TOLERANCE: # value-dependent yet not reflected -> unreliable + return None + + if _differs(first, base, floor) and _differs(second, base, floor): + return "behavioral" + + return None + +def _chunks(sequence, size): + for i in xrange(0, len(sequence), size): + yield sequence[i:i + size] + +def _discover(candidates, baseGet, base, floor): + """ + Probes candidate names in buckets (one shared request per bucket, each name carrying its own + random canary) and returns the confirmed ones as (name, reason) pairs. Reflection is resolved + straight from the bucket response; the rest are confirmed individually only when the bucket + actually moved the response, so a target that ignores every candidate stays cheap. + """ + + found = [] + + for chunk in _chunks(candidates, PARAMETER_MINING_BUCKET_SIZE): + canaries = dict((name, _canary()) for name in chunk) + query = "&".join("%s=%s" % (name, canaries[name]) for name in chunk) + page = _fetch("%s&%s" % (baseGet, query) if baseGet else query)[0] + + if not page: + continue + + pending = [] + for name in chunk: + if canaries[name] in page and canaries[name] not in base: + found.append((name, "reflected")) + else: + pending.append(name) + + if pending and _differs(page, base, floor): + for name in pending: + reason = _confirm(baseGet, name, base, floor) + if reason: + found.append((name, reason)) + + return found + +def _commit(found, baseGet, baseCode): + """ + Adds the discovered parameters (seeded with PROBE_VALUE) to the GET test scope. One that turns + the request into a server error the baseline did not have would shadow and corrupt the testing + of every sibling parameter, so it is reported and held back rather than degrading detection. + """ + + safe, disruptive = [], [] + + for name, _ in found: + pair = "%s=%s" % (name, PROBE_VALUE) + code = _fetch("%s&%s" % (baseGet, pair) if baseGet else pair)[1] + if code is not None and code >= 500 and not (baseCode is not None and baseCode >= 500): + disruptive.append(name) + else: + safe.append(name) + + if disruptive: + logger.warning("held back parameter(s) that break the base request with a test value (test them explicitly with '-p'): %s" % ", ".join("'%s'" % _ for _ in disruptive)) + + if not safe: + return + + logger.info("adding %d discovered parameter(s) to the test scope: %s" % (len(safe), ", ".join("'%s'" % _ for _ in safe))) + + additions = "&".join("%s=%s" % (name, PROBE_VALUE) for name in safe) + conf.parameters[PLACE.GET] = "%s&%s" % (baseGet, additions) if baseGet else additions + conf.paramDict[PLACE.GET] = paramToDict(PLACE.GET, conf.parameters[PLACE.GET]) + +def mineParameters(): + """ + Discovers hidden (unlinked) GET parameters the target still processes and queues the confirmed + ones for the regular injection tests, using two independent oracles (value reflection and a + behavioral side effect on the response). + """ + + if conf.data or (conf.method and conf.method != HTTPMETHOD.GET): + singleTimeWarnMessage("'--mine-params' currently supports GET parameters only") + return + + baseGet = conf.parameters.get(PLACE.GET) or "" + existing = set(conf.paramDict.get(PLACE.GET) or {}) + candidates = [_ for _ in getFileItems(paths.COMMON_PARAMETERS, unique=True) if _ and _ not in existing] + + if not candidates: + return + + logger.info("mining for hidden GET parameters (%d candidate name(s))" % len(candidates)) + + base, baseCode = _fetch(baseGet) + if not base: + singleTimeWarnMessage("could not obtain a baseline response, skipping parameter mining") + return + + floor = _ratio(base, _fetch(baseGet)[0]) # the target's own between-request jitter + + found = _discover(candidates, baseGet, base, floor) + + for name, reason in found: + logger.info("found hidden parameter '%s' (%s)" % (name, reason)) + + if not found: + logger.info("no hidden parameters found") + return + + _commit(found, baseGet, baseCode) diff --git a/tests/test_brute.py b/tests/test_brute.py index c13395978d5..f85fe4c6c2a 100644 --- a/tests/test_brute.py +++ b/tests/test_brute.py @@ -161,8 +161,10 @@ def test_column_exists_collects_and_types(self): def _cbe(expression, expectingNone=True): calls["n"] += 1 - # initial sanity probe uses two random strings (no real column name) - if "id" not in expression and "name" not in expression: + # initial sanity probe queries a random table, not the real 'users' one - so keying on the + # table name is collision-proof (unlike a column-name substring, which a random probe value + # can incidentally contain, e.g. 'id') + if "users" not in expression: return False # MySQL numeric-type follow-up: `not checkBooleanExpression(... REGEXP '[^0-9]')`. # 'id' is numeric (no non-digit chars => probe False => numeric);