From b2f98b61ec4575c2da1946f4ac374ae67fab5c1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0tampar?= Date: Sat, 25 Jul 2026 19:10:58 +0200 Subject: [PATCH] Improved crawling --- lib/core/option.py | 1 + lib/core/settings.py | 21 ++- lib/parse/sitemap.py | 22 ++- lib/request/redirecthandler.py | 8 +- lib/utils/crawler.py | 320 +++++++++++++++++++++++++++++---- 5 files changed, 333 insertions(+), 39 deletions(-) diff --git a/lib/core/option.py b/lib/core/option.py index 5d9163e0c62..aadde6d5b57 100644 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -2360,6 +2360,7 @@ def _setKnowledgeBaseAttributes(flushAll=True): if flushAll: kb.checkSitemap = None + kb.crawledHosts = set() # hosts whose robots.txt / well-known paths were already probed kb.headerPaths = {} kb.keywords = set(getFileItems(paths.SQL_KEYWORDS)) kb.lastCtrlCTime = None diff --git a/lib/core/settings.py b/lib/core/settings.py index 73ce6310099..04f62b2c16c 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -20,7 +20,7 @@ from thirdparty import six # sqlmap version (...) -VERSION = "1.10.7.187" +VERSION = "1.10.7.188" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) @@ -790,6 +790,21 @@ # Extensions skipped by crawler CRAWL_EXCLUDE_EXTENSIONS = frozenset(("3ds", "3g2", "3gp", "7z", "DS_Store", "a", "aac", "accdb", "access", "adp", "ai", "aif", "aiff", "apk", "ar", "asf", "au", "avi", "bak", "bin", "bk", "bkp", "bmp", "btif", "bz2", "c", "cab", "caf", "cfg", "cgm", "cmx", "com", "conf", "config", "cpio", "cpp", "cr2", "cue", "dat", "db", "dbf", "deb", "debug", "djvu", "dll", "dmg", "dmp", "dng", "doc", "docx", "dot", "dotx", "dra", "dsk", "dts", "dtshd", "dvb", "dwg", "dxf", "dylib", "ear", "ecelp4800", "ecelp7470", "ecelp9600", "egg", "elf", "env", "eol", "eot", "epub", "error", "exe", "f4v", "fbs", "fh", "fla", "flac", "fli", "flv", "fpx", "fst", "fvt", "g3", "gif", "go", "gz", "h", "h261", "h263", "h264", "ico", "ief", "img", "ini", "ipa", "iso", "jar", "java", "jpeg", "jpg", "jpgv", "jpm", "js", "jxr", "ktx", "lock", "log", "lvp", "lz", "lzma", "lzo", "m3u", "m4a", "m4v", "mar", "mdb", "mdi", "mid", "mj2", "mka", "mkv", "mmr", "mng", "mov", "movie", "mp3", "mp4", "mp4a", "mpeg", "mpg", "mpga", "msi", "mxu", "nef", "npx", "nrg", "o", "oga", "ogg", "ogv", "old", "otf", "ova", "ovf", "pbm", "pcx", "pdf", "pea", "pgm", "pic", "pid", "pkg", "png", "pnm", "ppm", "pps", "ppt", "pptx", "ps", "psd", "py", "pya", "pyc", "pyo", "pyv", "qt", "rar", "ras", "raw", "rb", "rgb", "rip", "rlc", "rs", "run", "rz", "s3m", "s7z", "scm", "scpt", "service", "sgi", "shar", "sil", "smv", "so", "sock", "socket", "sqlite", "sqlitedb", "sub", "svc", "swf", "swo", "swp", "sys", "tar", "tbz2", "temp", "tga", "tgz", "tif", "tiff", "tlz", "tmp", "toast", "torrent", "ts", "ttf", "uvh", "uvi", "uvm", "uvp", "uvs", "uvu", "vbox", "vdi", "vhd", "vhdx", "viv", "vmdk", "vmx", "vob", "vxd", "war", "wav", "wax", "wbmp", "wdp", "weba", "webm", "webp", "whl", "wm", "wma", "wmv", "wmx", "woff", "woff2", "wvx", "xbm", "xif", "xls", "xlsx", "xlt", "xm", "xpi", "xpm", "xwd", "xz", "yaml", "yml", "z", "zip", "zipx")) +# Endpoint mining inside JavaScript bundles during crawling (SPA API routes live in .js, invisible to +# href/src scraping): quoted absolute-path or same-origin-URL string literals (fetch/axios/XHR targets). +# MAX_* caps per-bundle results so a noisy minified file cannot flood the target list. +JAVASCRIPT_ENDPOINT_REGEX = r'''["'`](?P(?:https?:)?//[\w.:@-]+/[^"'`\s]*|/[A-Za-z0-9_][^"'`\s]*)["'`]''' +MAX_JAVASCRIPT_ENDPOINTS = 200 +# only the leading slice of a bundle/source-map is scanned for endpoints - bounds CPU/RAM on a hostile response +MAX_JAVASCRIPT_MINE_SIZE = 1 * 1024 * 1024 +# max source distance (chars) between a string-constant assignment and a `name + "/suffix"` use it may fold into +MAX_JAVASCRIPT_FOLD_DISTANCE = 2048 +# cap on Disallow/Allow/Sitemap lines consumed from a (potentially hostile) robots.txt +MAX_ROBOTS_ENTRIES = 1000 +# bounds on sitemap parsing: number of sitemap documents fetched (recursion fan-out) and total URLs kept +MAX_SITEMAP_FETCHES = 100 +MAX_SITEMAP_URLS = 100000 + # Patterns often seen in HTTP headers containing custom injection marking character '*' # Note: the ';q=' quality-value class excludes '*' so a user-placed injection mark right after a # quality value (e.g. 'Accept: ...;q=0.9*') is not swallowed (ref: #5357 - header injection was then @@ -983,6 +998,10 @@ # GraphQL endpoint paths to probe when the user supplies a base URL with --graphql (no explicit /graphql) GRAPHQL_ENDPOINT_PATHS = ("/graphql", "/api/graphql", "/v1/graphql", "/api/v1/graphql", "/graphql/api", "/graphql/console", "/graphql.php", "/graphiql", "/graph", "/gql", "/query") +# Self-describing JSON endpoint directories probed once per host during crawling: OIDC discovery lists the +# auth/token/userinfo URLs, OpenAPI/Swagger specs enumerate the whole API (their paths are mined as endpoints) +WELL_KNOWN_ENDPOINT_PATHS = ("/.well-known/openid-configuration", "/swagger.json", "/openapi.json", "/swagger/v1/swagger.json", "/api-docs", "/v2/api-docs", "/v3/api-docs", "/api/swagger.json", "/api/openapi.json") + # Seed field/argument names used to recover a GraphQL schema from "Did you mean" suggestion error # messages when introspection is disabled (the field-suggestion / "Clairvoyance" technique) GRAPHQL_FIELD_WORDLIST = ("user", "users", "me", "search", "login", "node", "post", "posts", diff --git a/lib/parse/sitemap.py b/lib/parse/sitemap.py index 882c86b2013..1192dcb6798 100644 --- a/lib/parse/sitemap.py +++ b/lib/parse/sitemap.py @@ -13,12 +13,18 @@ from lib.core.data import logger from lib.core.datatype import OrderedSet from lib.core.exception import SqlmapSyntaxException +from lib.core.settings import MAX_SITEMAP_FETCHES +from lib.core.settings import MAX_SITEMAP_URLS from lib.request.connect import Connect as Request from thirdparty.six.moves import http_client as _http_client abortedFlag = None -def parseSitemap(url, retVal=None, visited=None): +def parseSitemap(url, retVal=None, visited=None, urlFilter=None): + """Parse a sitemap (recursively following nested sitemap indexes). 'urlFilter' - when given - must + return True for a URL to be fetched or kept; it is enforced on the initial URL AND every nested + sitemap fetched recursively, so a hostile sitemap index cannot pull the crawler out of scope.""" + global abortedFlag if retVal is not None: @@ -30,11 +36,17 @@ def parseSitemap(url, retVal=None, visited=None): retVal = OrderedSet() visited = set() - if url in visited: + if url in visited or (urlFilter is not None and not urlFilter(url)): return retVal visited.add(url) + if len(visited) > MAX_SITEMAP_FETCHES or len(retVal) >= MAX_SITEMAP_URLS: # bound a hostile sitemap graph + if not abortedFlag: # warn once on the False->True transition + logger.warning("sitemap parsing stopped after reaching the fetch/URL limit. sqlmap will use partial list") + abortedFlag = True + return retVal + try: content = Request.getPage(url=url, raise404=True)[0] if not abortedFlag else "" except _http_client.InvalidURL: @@ -45,7 +57,7 @@ def parseSitemap(url, retVal=None, visited=None): content = re.sub(r"", "", content, flags=re.DOTALL) # Note: strip (possibly multi-line) XML comments so commented-out entries aren't harvested for match in re.finditer(r"<\w*?loc[^>]*>\s*([^<]+)", content, re.I): - if abortedFlag: + if abortedFlag or len(retVal) >= MAX_SITEMAP_URLS: break foundUrl = htmlUnescape(match.group(1).strip()) @@ -60,8 +72,8 @@ def parseSitemap(url, retVal=None, visited=None): kb.followSitemapRecursion = readInput(message, default='N', boolean=True) if kb.followSitemapRecursion: - parseSitemap(foundUrl, retVal, visited) - else: + parseSitemap(foundUrl, retVal, visited, urlFilter) + elif urlFilter is None or urlFilter(foundUrl): retVal.add(foundUrl) except KeyboardInterrupt: diff --git a/lib/request/redirecthandler.py b/lib/request/redirecthandler.py index 64ee596e091..2f147221249 100644 --- a/lib/request/redirecthandler.py +++ b/lib/request/redirecthandler.py @@ -122,7 +122,13 @@ def http_error_302(self, req, fp, code, msg, headers): redurl = _urllib.parse.urljoin(req.get_full_url(), redurl) self._infinite_loop_check(req) - if conf.scope: + crawlRedirectFilter = getattr(threadData, "crawlRedirectFilter", None) + if crawlRedirectFilter is not None and not crawlRedirectFilter(redurl): + # a crawler recon/source-map fetch carries the session cookie; it must NOT follow an + # (attacker-controlled) redirect out of scope - reject the hop before it is made, so the + # cookie never reaches the off-scope destination (works even when no explicit --scope is set) + redurl = None + elif conf.scope: if not re.search(conf.scope, redurl, re.I): redurl = None else: diff --git a/lib/utils/crawler.py b/lib/utils/crawler.py index 787f0a15e54..05cc4a6b4e7 100644 --- a/lib/utils/crawler.py +++ b/lib/utils/crawler.py @@ -7,11 +7,15 @@ from __future__ import division +import bisect +import json import os import re import tempfile import time +from itertools import islice + from lib.core.common import checkSameHost from lib.core.common import clearConsoleLine from lib.core.common import dataToStdout @@ -32,6 +36,12 @@ from lib.core.exception import SqlmapConnectionException from lib.core.exception import SqlmapSyntaxException from lib.core.settings import CRAWL_EXCLUDE_EXTENSIONS +from lib.core.settings import JAVASCRIPT_ENDPOINT_REGEX +from lib.core.settings import MAX_JAVASCRIPT_ENDPOINTS +from lib.core.settings import MAX_JAVASCRIPT_FOLD_DISTANCE +from lib.core.settings import MAX_JAVASCRIPT_MINE_SIZE +from lib.core.settings import MAX_ROBOTS_ENTRIES +from lib.core.settings import WELL_KNOWN_ENDPOINT_PATHS from lib.core.threads import getCurrentThreadData from lib.core.threads import runThreads from lib.parse.sitemap import parseSitemap @@ -41,6 +51,109 @@ from thirdparty.six.moves import http_client as _http_client from thirdparty.six.moves import urllib as _urllib +def _inScope(url, target): + """Single predicate governing every crawler request/result: honor --scope if set, else same-host.""" + + return (re.search(conf.scope, url, re.I) is not None) if conf.scope else checkSameHost(url, target) + +def _mineJavaScript(content, base): + """Extract candidate API endpoints referenced inside a JavaScript bundle - the fetch/axios/XHR + targets and absolute-path string literals that power single-page apps and are invisible to + href/src scraping. Returns [(absoluteURL, parametrized), ...] (capped, static assets dropped); + 'parametrized' marks a path templated with a dynamic segment (e.g. `/user/${id}` -> `/user/1`) + or carrying a query string, i.e. a directly testable target rather than a page merely to crawl. + + Light constant folding resolves the common `var base="/api"; fetch(base+"/users")` idiom that + plain literal scraping would split into two useless halves. + + >>> r = _mineJavaScript('fetch("/api/users?id=1");x="/img/logo.png";t=`/user/${i}/x`', "http://h/a.js") + >>> ("http://h/api/users?id=1", True) in r and ("http://h/user/1/x", False) in r # query -> target, template -> crawl only + True + >>> any(_.endswith("logo.png") for _, __ in r) + False + >>> r = _mineJavaScript('var base="/api/v1";fetch(base+"/users")', "http://h/a.js") + >>> ("http://h/api/v1/users", False) in r # folded ... + True + >>> any(_ == "http://h/users" for _, __ in r) # ... and only THIS folded suffix occurrence is dropped + False + """ + + content = content[:MAX_JAVASCRIPT_MINE_SIZE] # endpoints live near the top; bound the work on a hostile bundle + _template = r"\$\{[^}]*\}|:[A-Za-z_]\w*|\{[A-Za-z_.]+\}|<[A-Za-z_]\w*>" + + # index path/URL-valued string constants by name (positions ascending) so a ` + "/suffix"` + # concatenation resolves against the NEAREST PRECEDING assignment via binary search - minified bundles + # reuse identifiers (a global last-wins map would fold the wrong value) and can hold tens of thousands + # of assignments (a linear scan per concatenation would be quadratic) + byName = {} + for match in re.finditer(r"""(?:\b(?:var|let|const)\s+|[,{(]\s*)(?P\w+)\s*[:=]\s*["'`](?P(?:https?:)?/[^"'`\s]{0,256})["'`]""", content): + byName.setdefault(match.group("name"), ([], [])) + byName[match.group("name")][0].append(match.start()) + byName[match.group("name")][1].append(match.group("value")) + + candidates = [] # (spanStart, text): spanStart correlates a folded suffix to the literal it consumes + for match in re.finditer(r"""(?P\w+)\s*\+\s*["'`](?P/[^"'`\s]{0,256})["'`]""", content): + entry = byName.get(match.group("name")) + if entry: + index = bisect.bisect_left(entry[0], match.start()) - 1 + # best-effort, deliberately not a JS parser: fold only against a NEARBY preceding assignment so a + # far-away or differently-scoped `var base=...` (e.g. inside another function) is not mis-applied + if index >= 0 and match.start() - entry[0][index] <= MAX_JAVASCRIPT_FOLD_DISTANCE: + candidates.append((match.start("suffix"), entry[1][index].rstrip("/") + match.group("suffix"))) + folded = set(_[0] for _ in candidates) # exact source spans consumed by folding + for match in re.finditer(JAVASCRIPT_ENDPOINT_REGEX, content): + if match.start("result") not in folded: # suppress only THIS occurrence, not every same-text literal + candidates.append((match.start("result"), match.group("result"))) + + candidates.sort(key=lambda _: _[0]) # process in source order so the endpoint cap keeps the earliest, not every folded one first + + results = [] + seen = set() + for _, candidate in candidates: + if any(_ in candidate for _ in ("\\", "^", "*", " ")): # regex/glob fragments, not endpoints + continue + candidate = re.sub(_template, "1", candidate) # concrete, crawlable path segment + url = _urllib.parse.urljoin(base, candidate) + if not re.search(r"(?i)\Ahttps?://[^/]+/", url): # must resolve to an absolute http(s) URL + continue + if (extractRegexResult(r"\A[^?#]+\.(?P\w+)([?#]|\Z)", url) or "").lower() in CRAWL_EXCLUDE_EXTENSIONS: + continue + # only a real query parameter makes a URL a directly testable target; a path with a (substituted) + # dynamic segment is crawled, not marked, because sqlmap's URI injection needs the '*' marker at the + # right segment (a middle template like /user/1/x would otherwise be mis-tested at its end) + isTarget = re.search(r"\?.*\b\w+=", url) is not None + if url not in seen: + seen.add(url) + results.append((url, isTarget)) + if len(results) >= MAX_JAVASCRIPT_ENDPOINTS: + break + + return results + +def _sourceMapEndpoints(mapContent, base): + """A '//# sourceMappingURL=' map ships the original, un-minified sources in 'sourcesContent'; + mining those recovers endpoints (and pre-minification structure) that the bundle alone hides - + a trick commercial crawlers (Burp, Acunetix) lean on. Returns the same shape as _mineJavaScript.""" + + if len(mapContent) > 8 * MAX_JAVASCRIPT_MINE_SIZE: # do not json-parse an oversized (hostile) map into RAM + return [] + try: + data = json.loads(mapContent) + except ValueError: + return [] + sources = data.get("sourcesContent") if isinstance(data, dict) else None + if not isinstance(sources, (list, tuple)): # a valid JSON map may carry a non-array here + return [] + parts = [] + size = 0 + for source in sources: # join a bounded slice (avoid repeated string realloc) + if isinstance(source, six.text_type): + parts.append(source[:MAX_JAVASCRIPT_MINE_SIZE - size]) + size += len(parts[-1]) + if size >= MAX_JAVASCRIPT_MINE_SIZE: + break + return _mineJavaScript("\n".join(parts), base) if parts else [] + def crawl(target, post=None, cookie=None): if not target: return @@ -50,9 +163,31 @@ def crawl(target, post=None, cookie=None): threadData = getCurrentThreadData() threadData.shared.value = OrderedSet() threadData.shared.formsFound = False + # host-level recon (robots/well-known/sitemap) runs on this thread and carries the session cookie; + # confine its redirects to scope (reset in 'finally' so later requests on this thread are unaffected) + threadData.crawlRedirectFilter = lambda url: _inScope(url, target) def crawlThread(): threadData = getCurrentThreadData() + # confine this worker's redirects (e.g. a source-map fetch) to scope; reset in 'finally' so a pooled + # thread later reused for injection is not left restricted to the crawl scope + threadData.crawlRedirectFilter = lambda url: _inScope(url, target) + + try: + _crawlThreadLoop(threadData) + finally: + threadData.crawlRedirectFilter = None + + def _crawlThreadLoop(threadData): + def consume(endpoints): + # feed mined endpoints through the same scope-check + deeper/value flow as scraped links + for url, isTarget in endpoints: + if not _inScope(url, target): + continue + with kb.locks.value: + threadData.shared.deeper.add(url) + if isTarget: + threadData.shared.value.add(url) while kb.threadContinue: with kb.locks.limit: @@ -88,7 +223,38 @@ def crawlThread(): if not kb.threadContinue: break - if isinstance(content, six.text_type): + if isinstance(content, six.text_type) and (current or "").split("?", 1)[0].lower().endswith((".js", ".mjs")): + try: + consume(_mineJavaScript(content, current)) + # follow a source map ('//# sourceMappingURL=') to the original un-minified sources; + # the URL is attacker-controlled, so it must stay same-host/in-scope (no SSRF) and be + # fetched at most once + smatch = re.search(r"(?m)[#@]\s*sourceMappingURL\s*=\s*(?P[^\s'\"]+)", content) + if smatch and not smatch.group("url").startswith("data:"): + mapURL = _urllib.parse.urljoin(current, smatch.group("url")) + with kb.locks.value: + fetch = mapURL.startswith(("http://", "https://")) and _inScope(mapURL, target) and mapURL not in visited + if fetch: + visited.add(mapURL) + if fetch: # GET the static map (not a re-POST) and keep the auth cookie + mapContent = Request.getPage(url=mapURL, post=None, cookie=cookie, crawling=True, raise404=False)[0] + # a same-host map that redirected off-scope must not have its (authenticated) body used + redirected = threadData.lastRedirectURL[1] if (threadData.lastRedirectURL and threadData.lastRedirectURL[0] == threadData.lastRequestUID) else None + if isinstance(mapContent, six.text_type) and (redirected is None or _inScope(redirected, target)): + consume(_sourceMapEndpoints(mapContent, current)) + except (ValueError, SqlmapConnectionException): + pass + elif isinstance(content, six.text_type) and content[:64].lstrip()[:1] in ("{", "["): + try: # a JSON API response: mine embedded resource links (REST/HATEOAS, pagination) + consume(_mineJavaScript(content, current)) + except ValueError: + pass + elif isinstance(content, six.text_type): + # base for resolving links AND forms: the redirect target (if any), refined by + # below; defined before the try so the 'finally' can rely on it even if parsing fails + linkBase = current + if threadData.lastRedirectURL and threadData.lastRedirectURL[0] == threadData.lastRequestUID: + linkBase = threadData.lastRedirectURL[1] try: match = re.search(r"(?si)]*>(.+)", content) if match: @@ -99,29 +265,55 @@ def crawlThread(): tags += re.finditer(r'(?i)\s(href|src)=["\'](?P[^>"\']+)', content) tags += re.finditer(r'(?i)window\.open\(["\'](?P[^)"\']+)["\']', content) + # URL-bearing data-* attributes and navigational sinks that mature crawlers also follow. + # Note:
/formaction are intentionally NOT scraped here - they need method + + # field semantics (handled by --forms/findPageForms), and data-action usually holds a + # command name, not a URL + tags += re.finditer(r'(?i)\s(?:data-(?:url|href|src|link|api|endpoint))=["\'](?P[^>"\']+)', content) + tags += re.finditer(r'(?i)]+?http-equiv=["\']?refresh\b[^>]+?url=(?P[^"\'>\s;]+)', content) + tags += re.finditer(r'(?i)]*?\sdata=["\'](?P[^>"\']+)', content) + + # honor for correct relative-URL resolution, but only if it stays in scope - + # a cross-origin must not become the resolution base for links or (via linkBase) forms + baseTag = re.search(r'(?i)]+?href=["\'](?P[^"\'>]+)', content) + if baseTag: + rebased = _urllib.parse.urljoin(linkBase, htmlUnescape(baseTag.group("href"))) + if (re.search(conf.scope, rebased, re.I) if conf.scope else checkSameHost(rebased, target)): + linkBase = rebased for tag in tags: href = tag.get("href") if hasattr(tag, "get") else tag.group("href") if href: - if threadData.lastRedirectURL and threadData.lastRedirectURL[0] == threadData.lastRequestUID: - current = threadData.lastRedirectURL[1] - url = _urllib.parse.urljoin(current, htmlUnescape(href)) + url = _urllib.parse.urldefrag(_urllib.parse.urljoin(linkBase, htmlUnescape(href)))[0] - # flag to know if we are dealing with the same target host - _ = checkSameHost(url, target) - - if conf.scope: - if not re.search(conf.scope, url, re.I): - continue - elif not _: + if not _inScope(url, target): continue - if (extractRegexResult(r"\A[^?]+\.(?P\w+)(\?|\Z)", url) or "").lower() not in CRAWL_EXCLUDE_EXTENSIONS: + extension = (extractRegexResult(r"\A[^?#]+\.(?P\w+)([?#]|\Z)", url) or "").lower() + if extension in ("js", "mjs"): + with kb.locks.value: # enqueue the bundle so its endpoints get mined + threadData.shared.deeper.add(url) + elif extension not in CRAWL_EXCLUDE_EXTENSIONS: with kb.locks.value: threadData.shared.deeper.add(url) - if re.search(r"(.*?)\?(.+)", url) and not re.search(r"\?(v=)?\d+\Z", url) and not re.search(r"(?i)\.(js|css)(\?|\Z)", url): + if re.search(r"(.*?)\?(.+)", url) and not re.search(r"\?(v=)?\d+\Z", url) and not re.search(r"(?i)\.(m?js|css)(\?|\Z)", url): threadData.shared.value.add(url) + + # inline ", content): + body = script.group(1)[:MAX_JAVASCRIPT_MINE_SIZE - inlineSize] + inline.append(body) + inlineSize += len(body) + if inlineSize >= MAX_JAVASCRIPT_MINE_SIZE: + break + if inline: + consume(_mineJavaScript("\n".join(inline), linkBase)) except UnicodeEncodeError: # for non-HTML files pass except ValueError: # for non-valid links @@ -130,7 +322,7 @@ def crawlThread(): pass finally: if conf.forms: - threadData.shared.formsFound |= len(findPageForms(content, current, False, True)) > 0 + threadData.shared.formsFound |= len(findPageForms(content, linkBase, False, True)) > 0 if conf.verbose in (1, 2): threadData.shared.count += 1 @@ -148,34 +340,97 @@ def crawlThread(): if re.search(r"\?.*\b\w+=", target): threadData.shared.value.add(target) + # host-level recon (robots.txt + well-known endpoint directories) is done at most once per host so a + # multi-target run does not re-probe (and re-404) the same host over and over + _split = _urllib.parse.urlsplit(target) + crawlHost = ("%s://%s" % (_split.scheme, _split.netloc)).lower() # scheme+netloc dedup key (finer than the host-level scope predicate on purpose - never re-probe the same origin) + with kb.locks.value: # atomic check-and-claim so concurrent target crawls do not double-probe a host + reconHost = bool(_split.netloc) and crawlHost not in kb.crawledHosts + if reconHost: + kb.crawledHosts.add(crawlHost) + + # every sitemap source (robots.txt 'Sitemap:' lines AND the /sitemap.xml guess below) shares ONE fetch/URL + # budget and ONE visited set, so a hostile robots.txt advertising many roots cannot multiply the per-root + # limits into ~100k fetches, and a sitemap listed twice is fetched once + sitemapItems = OrderedSet() + sitemapVisited = set() + + # robots.txt Disallow/Allow entries expose unlinked paths (admin panels, API roots) that crawlers + # (Burp, Acunetix) routinely harvest as seeds; the file itself must be in scope before it is fetched + robotsUrl = _urllib.parse.urljoin(target, "/robots.txt") + try: + robots = Request.getPage(url=robotsUrl, post=None, cookie=cookie, crawling=True, raise404=False)[0] if (reconHost and _inScope(robotsUrl, target)) else None + except Exception: + robots = None + if isinstance(robots, six.text_type): + # ONE combined budget across Disallow/Allow and Sitemap lines, consumed lazily (islice over finditer) + # so a huge/repetitive robots.txt is neither fully materialized nor over-processed + remaining = MAX_ROBOTS_ENTRIES + for match in islice(re.finditer(r"(?im)^\s*(?:dis)?allow\s*:\s*(/\S*)", robots), remaining): + remaining -= 1 + path = match.group(1) + if any(_ in path for _ in "*$"): # a pattern, not a concrete path + continue + url = _urllib.parse.urljoin(target, path) + if _inScope(url, target) and (extractRegexResult(r"\A[^?#]+\.(?P\w+)([?#]|\Z)", url) or "").lower() not in CRAWL_EXCLUDE_EXTENSIONS: + threadData.shared.unprocessed.add(url) + if re.search(r"\?.*\b\w+=", url): + threadData.shared.value.add(url) + + # follow the sitemaps robots.txt advertises into the SHARED budget/visited; the advertised URL AND every + # nested sitemap parseSitemap fetches recursively must pass the same scope predicate (enforced inside) + for match in islice(re.finditer(r"(?im)^\s*sitemap\s*:\s*(https?://\S+)", robots), max(remaining, 0)): + sitemapUrl = match.group(1) + if not _inScope(sitemapUrl, target): + continue + try: + parseSitemap(sitemapUrl, retVal=sitemapItems, visited=sitemapVisited, urlFilter=lambda _: _inScope(_, target)) + except Exception: + pass + + # heuristic path discovery from self-describing JSON documents (OIDC discovery, OpenAPI/Swagger). + # this mines endpoint-looking strings, NOT a full OpenAPI model - basePath/servers are not merged and + # $ref/examples are not resolved; '--openapi' does exact API enumeration + for path in (WELL_KNOWN_ENDPOINT_PATHS if reconHost else ()): + probe = _urllib.parse.urljoin(target, path) + if probe in visited or not _inScope(probe, target): + continue + visited.add(probe) + try: + blob = Request.getPage(url=probe, post=None, cookie=cookie, crawling=True, raise404=False)[0] + except Exception: + blob = None + if isinstance(blob, six.text_type) and blob[:64].lstrip()[:1] in ("{", "["): + for url, isTarget in _mineJavaScript(blob, probe): + if _inScope(url, target): + threadData.shared.unprocessed.add(url) + if isTarget: + threadData.shared.value.add(url) + if kb.checkSitemap is None: message = "do you want to check for the existence of " message += "site's sitemap(.xml) [y/N] " kb.checkSitemap = readInput(message, default='N', boolean=True) - if kb.checkSitemap: - found = True - items = None - url = _urllib.parse.urljoin(target, "/sitemap.xml") - try: - items = parseSitemap(url) + url = _urllib.parse.urljoin(target, "/sitemap.xml") + if kb.checkSitemap and _inScope(url, target): + try: # into the same shared budget/visited as the robots sitemaps + parseSitemap(url, retVal=sitemapItems, visited=sitemapVisited, urlFilter=lambda _: _inScope(_, target)) except SqlmapConnectionException as ex: if "page not found" in getSafeExString(ex): - found = False logger.warning("'sitemap.xml' not found") except: pass - finally: - if found: - if items: - # keep sitemap-derived URLs on-scope, exactly like the HTML-crawl path above - items = OrderedSet(_ for _ in items if (re.search(conf.scope, _, re.I) if conf.scope else checkSameHost(_, target))) - for item in items: - if re.search(r"(.*?)\?(.+)", item): - threadData.shared.value.add(item) - if conf.crawlDepth > 1: - threadData.shared.unprocessed.update(items) - logger.info("%s links found" % ("no" if not items else len(items))) + + # single consumption of every sitemap-derived URL (already scope-filtered inside parseSitemap): a URL with + # GET parameters is a target, and - at depth > 1 - all are queued for further crawling + if sitemapItems: + for item in sitemapItems: + if re.search(r"\?.*\b\w+=", item): + threadData.shared.value.add(item) + if conf.crawlDepth > 1: + threadData.shared.unprocessed.add(item) + logger.info("%d link(s) found via sitemap(s)" % len(sitemapItems)) if not conf.bulkFile: infoMsg = "starting crawler for target URL '%s'" % target @@ -204,6 +459,7 @@ def crawlThread(): finally: clearConsoleLine(True) + threadData.crawlRedirectFilter = None if not threadData.shared.value: if not (conf.forms and threadData.shared.formsFound):