diff --git a/doc/CHANGELOG.md b/doc/CHANGELOG.md index e46292bf92d..668543673e0 100644 --- a/doc/CHANGELOG.md +++ b/doc/CHANGELOG.md @@ -84,6 +84,10 @@ * Removed the time outliers from the time statistics. * Improved the detection of the SQL dialect of the target. * The switch `--xpath` no longer reports an injection when the page changes only because it shows the payload again. Reflection is not proof that an expression was evaluated. +* sqlmap now makes sure that the random markers around the retrieved data are all different. Two equal markers made the error-based technique give back the wrong character. +* Made the heuristic hints of the non-SQL switches exclusive. A signature no longer matches the errors of a different engine, an ordinary SQL error, or a page that only contains the name of a template engine. +* Corrected the GraphQL validation signatures. They now match the quotes in the way that the JSON body escapes them. +* Added the error signatures of Mako and of DynamoDB. sqlmap did not recognise the errors of these two back-ends. ## Quality diff --git a/lib/core/option.py b/lib/core/option.py index cb1c707bcfa..6b145ef5432 100644 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -2193,9 +2193,16 @@ def _setKnowledgeBaseAttributes(flushAll=True): kb.chars = AttribDict() kb.chars.delimiter = randomStr(length=6, lowercase=True) - kb.chars.start = "%s%s%s" % (KB_CHARS_BOUNDARY_CHAR, randomStr(length=3, alphabet=KB_CHARS_LOW_FREQUENCY_ALPHABET), KB_CHARS_BOUNDARY_CHAR) - kb.chars.stop = "%s%s%s" % (KB_CHARS_BOUNDARY_CHAR, randomStr(length=3, alphabet=KB_CHARS_LOW_FREQUENCY_ALPHABET), KB_CHARS_BOUNDARY_CHAR) - kb.chars.at, kb.chars.space, kb.chars.dollar, kb.chars.hash_ = ("%s%s%s" % (KB_CHARS_BOUNDARY_CHAR, _, KB_CHARS_BOUNDARY_CHAR) for _ in randomStr(length=4, lowercase=True)) + # NOTE: markers have to be mutually distinct (e.g. equal start/stop makes the delimited output ambiguous, while equal replacement markers make _errorReplaceChars() restore the wrong character) + _ = set() + while len(_) < 2: + _.add(randomStr(length=3, alphabet=KB_CHARS_LOW_FREQUENCY_ALPHABET)) + kb.chars.start, kb.chars.stop = ("%s%s%s" % (KB_CHARS_BOUNDARY_CHAR, __, KB_CHARS_BOUNDARY_CHAR) for __ in _) + + _ = set() + while len(_) < 4: + _.add(randomStr(length=1, lowercase=True)) + kb.chars.at, kb.chars.space, kb.chars.dollar, kb.chars.hash_ = ("%s%s%s" % (KB_CHARS_BOUNDARY_CHAR, __, KB_CHARS_BOUNDARY_CHAR) for __ in _) kb.checkWafMode = False kb.choices = AttribDict(keycheck=False) diff --git a/lib/core/settings.py b/lib/core/settings.py index 5e7293fb3f7..72b5885ea70 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -20,7 +20,7 @@ from thirdparty import six # sqlmap version (...) -VERSION = "1.10.8.16" +VERSION = "1.10.8.18" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) @@ -1008,13 +1008,19 @@ ("CouchDB", r'"error"\s*:\s*"(?:bad_request|query_parse_error|missing_named_query)"|invalid operator: ?\$'), ("Elasticsearch", r'"type"\s*:\s*"[a-z_]*?(?:query_shard|x_content_parse|parsing|search_phase_execution|illegal_argument|too_many_clauses|number_format|script)_exception"|Failed to parse query \['), ("Solr", r"org\.apache\.solr\.[\w.]*(?:SyntaxError|SolrException)"), - ("Neo4j", r"Neo\.(?:ClientError|DatabaseError|TransientError|ClientNotification)\.|\bNeo4jError\b|even number of non-escaped quotes|Failed to parse string literal|expected an expression|'(?:UNWIND|OPTIONAL|DETACH|FOREACH|MERGE|LOAD CSV)'"), + # NOTE: 'MERGE' is not Cypher-only. It is standard SQL, so it matched "Incorrect syntax near 'MERGE'" + ("Neo4j", r"Neo\.(?:ClientError|DatabaseError|TransientError|ClientNotification)\.|\bNeo4jError\b|even number of non-escaped quotes|Failed to parse string literal|expected an expression|'(?:UNWIND|OPTIONAL|DETACH|FOREACH|LOAD CSV)'"), ("ArangoDB", r"\bArangoError\b|AQL: (?:syntax|parse) error"), - ("Cassandra", r"line \d+:\d+ (?:no viable alternative at input|(?:mismatched|extraneous) input '.*?' expecting)|org\.apache\.cassandra|com\.datastax|\bInvalid(?:Request|Query)Exception\b"), + # NOTE: the ANTLR "line N:M no viable alternative" line is not evidence of Cassandra on its own - + # Hibernate 6 and Trino emit it word for word. What IS particular to the CQL grammar: the driver + # exception in front of it, the ANTLR3 lexer wording ('mismatched character', which the ANTLR4 + # parsers do not have), and the "(...[TOKEN]...)" excerpt that CQL appends to the offending token + ("Cassandra", r"\b(?:ResponseError|SyntaxException|InvalidRequestException|InvalidQueryException)\b[^\n]{0,60}?line \d+:\d+ (?:no viable alternative at input|(?:mismatched|extraneous) input)|line \d+:\d+ (?:mismatched character|no viable alternative at input '[^']*' \(\.\.\.)|org\.apache\.cassandra|com\.datastax|\bInvalid(?:Request|Query)Exception\b"), ("Redis", r"\bWRONGTYPE\b|ERR Error (?:compiling|running) script|@user_script|\bReplyError\b"), ("Memcached", r"CLIENT_ERROR bad|SERVER_ERROR object too large"), ("InfluxDB", r"error parsing query|unable to parse '[^']*': found"), ("HBase/Phoenix", r"org\.apache\.phoenix|PhoenixParserException|org\.apache\.hadoop\.hbase"), + ("DynamoDB", r"Statement wasn't well formed, can't be processed|software\.amazon\.awssdk\.services\.dynamodb|com\.amazonaws\.services\.dynamodbv2|\bDynamoDb(?:Exception|Error)\b"), ) NOSQL_ERROR_REGEX = "(?:%s)" % '|'.join(regex for _, regex in NOSQL_ERRORS) @@ -1087,14 +1093,17 @@ r"\bExpected Name,\s*found\b", r"\bUnexpected\s+\b", ) +# NOTE: graphql-js quotes the offending name, and the response carries those quotes backslash-escaped +# inside the JSON body ('Cannot query field \"x\" on type \"Query\"'). Without the optional backslash +# none of these ever matched a real answer - only a pretty-printed one GRAPHQL_VALIDATION_ERRORS = ( r'"code"\s*:\s*"GRAPHQL_VALIDATION_FAILED"', - r"\bCannot query field\s+\"[^\"]+\"\s+on type\s+\"[^\"]+\"", - r"\bUnknown argument\s+\"[^\"]+\"\s+on field\s+\"[^\"]+\"", - r"\bField\s+\"[^\"]+\"\s+argument\s+\"[^\"]+\"\s+of type\s+\"[^\"]+\"\s+is required\b", - r"\bVariable\s+\"\$[^\"]+\"\s+got invalid value\b", + r"\bCannot query field\s+\\?\"[^\"\\]+\\?\"\s+on type\s+\\?\"[^\"\\]+\\?\"", + r"\bUnknown argument\s+\\?\"[^\"\\]+\\?\"\s+on field\s+\\?\"[^\"\\]+\\?\"", + r"\bField\s+\\?\"[^\"\\]+\\?\"\s+argument\s+\\?\"[^\"\\]+\\?\"\s+of type\s+\\?\"[^\"\\]+\\?\"\s+is required\b", + r"\bVariable\s+\\?\"\$[^\"\\]+\\?\"\s+got invalid value\b", r"\bExpected type\s+[^,]+,\s*found\b", - r"\bDid you mean\s+\"[^\"]+\"\b", + r"\bDid you mean\s+\\?\"[^\"\\]+\\?\"", ) GRAPHQL_APQ_ERRORS = ( r"\bPersistedQueryNotFound\b", @@ -1150,17 +1159,25 @@ # fingerprinting (matched against HTTP response bodies). Each tuple is # (backend_name, regex_fragment). XPATH_ERROR_SIGNATURES = ( - ("Java JAXP / Xalan", r"(?:javax\.xml\.(?:xpath\.XPathExpressionException|transform\.Transformer(?:Configuration)?Exception)|com\.sun\.org\.apache\.xpath\.(?:XPathException|XPathProcessorException)|org\.apache\.xpath|org\.xml\.sax\.SAX(?:Parse)?Exception)"), + # NOTE: neither javax.xml.transform.Transformer*Exception nor org.xml.sax.SAX*Exception belongs + # here. The first is the XSLT transformer and the second is the XML parser, so claiming them made + # every stylesheet failure and every malformed-XML response suggest '--xpath' as well. A real + # Xalan XPath failure always carries javax.xml.xpath.XPathExpressionException + ("Java JAXP / Xalan", r"(?:javax\.xml\.xpath\.XPathExpressionException|com\.sun\.org\.apache\.xpath\.(?:XPathException|XPathProcessorException)|org\.apache\.xpath)"), ("Java JAXP / Xalan", r"XPath (?:expression|syntax) error"), ("Java JAXP / Saxon", r"net\.sf\.saxon\.(?:trans\.XPathException|s9api\.SaxonApiException)"), - ("Java JAXP / Saxon", r"(?:XPST|XPTY|XPDY|XQST|XTDE)\d{4}:"), - (".NET XPathNavigator", r"System\.Xml\.(?:XPath\.XPathException|XmlException)"), + # NOTE: XTDE is an XSLT Transformation Dynamic Error, so it stays with '--xslt' + ("Java JAXP / Saxon", r"(?:XPST|XPTY|XPDY|XQST)\d{4}:"), + (".NET XPathNavigator", r"System\.Xml\.XPath\.XPathException"), (".NET XPathNavigator", r"Expression must evaluate to a node-set"), (".NET XPathNavigator", r"has an invalid (?:token|qualified name)"), ("lxml / libxml2", r"(?:lxml\.etree\.(?:XPath(?:Eval|Document|Syntax)?Error)|libxml2|xmlXPath(?:CompOp|Eval|Err))"), - ("lxml / libxml2", r"(?:XPath error|Invalid (?:expression|predicate))"), + # NOTE: 'Invalid expression' on its own is not an XPath error. libxml2 always prefixes it with + # "XPath error : ", and PHP always names the failing method, so both are covered without the + # bare form - which otherwise matched any calculator or formula field + ("lxml / libxml2", r"XPath error"), ("PHP SimpleXML / DOMXPath", r"(?:SimpleXMLElement::xpath\(\)|DOMXPath::(?:query|evaluate)\(\))"), - ("PHP SimpleXML / DOMXPath", r"Invalid expression|xmlXPathEval"), + ("PHP SimpleXML / DOMXPath", r"xmlXPathEval"), ("Saxon (standalone)", r"(?:net\.sf\.saxon\.(?:s9api\.SaxonApiException|trans\.XPathException)|Saxon error)"), ("Saxon (standalone)", r"Static error\(s\) in query"), ("BaseX", r"org\.basex\.(?:query\.QueryException|core\.BaseXException)"), @@ -1226,14 +1243,22 @@ # the binding that actually tells the tester what they are talking to. XSLT_ERROR_SIGNATURES = ( ("PHP XSLTProcessor", r"XSLTProcessor::(?:importStylesheet|transformTo\w+)\(\)"), - ("libxslt / lxml", r"lxml\.etree\.(?:XSLT(?:Parse|Apply|)Error|XPathEvalError)"), - ("Saxon", r"(?:net\.sf\.saxon\.|SaxonApiException|Static error(?:s)? (?:in|at)|XTDE\d{4}|XTSE\d{4})"), - ("Xalan / Java JAXP", r"(?:javax\.xml\.transform\.Transformer(?:Configuration)?Exception|org\.apache\.xalan|XSLT Error)"), + # NOTE: XPathEvalError is raised by a plain tree.xpath() call, so it belongs to '--xpath'. lxml + # raises XSLTParseError / XSLTApplyError for a stylesheet + ("libxslt / lxml", r"lxml\.etree\.XSLT(?:Parse|Apply)?Error"), + # NOTE: Saxon is one product for XPath, XQuery and XSLT, so the package name alone proves nothing. + # Only the XSLT-exclusive evidence is kept (XTSE static / XTDE dynamic codes, the style package) + ("Saxon", r"(?:net\.sf\.saxon\.style\.|XTDE\d{4}|XTSE\d{4}|Failed to compile stylesheet)"), + # NOTE: only the 'Configuration' form is exclusive to a stylesheet. javax.xml.xpath wraps a plain + # XPath failure in a bare TransformerException, and a Xalan run-time failure names org.apache.xalan + ("Xalan / Java JAXP", r"(?:javax\.xml\.transform\.TransformerConfigurationException|org\.apache\.xalan|XSLT Error)"), (".NET XslCompiledTransform", r"System\.Xml\.Xsl\.(?:XslLoadException|XsltException)"), # Anchored to XSLT vocabulary on purpose: this regex also drives the GLOBAL heuristic hint in # checks.py, and bare "compilation error" / "Invalid expression" match gcc, javac and regex failures, # which would suggest '--xslt' on targets that have nothing to do with XSLT. - ("libxslt", r"(?:xsltParseStylesheet|xsltApplyStylesheet|xsltCompilePattern|xsl:\w+ : |xmlXPathEval|XPath error : )"), + # NOTE: xmlXPathEval is the libxml2 XPath entry point, which '--xpath' owns. libxslt names the + # stylesheet file instead, and that is what tells the two apart + ("libxslt", r"(?:xsltParseStylesheet|xsltApplyStylesheet|xsltCompilePattern|xsltLoadStylesheet|xsl:\w+ : |(?:runtime|compilation) error: file [^\n]{0,120}\.xsl)"), ("Generic XSLT", r"(?:XSLT|xsl:stylesheet).{0,40}?(?:error|exception|fail)"), ) @@ -1274,15 +1299,25 @@ # Each tuple is (engine_name, regex_fragment). SSTI_ERROR_SIGNATURES = ( ("Jinja2", r"jinja2\.exceptions\.\w+|TemplateSyntaxError|UndefinedError|TemplateNotFound|TemplateAssertionError"), - ("Twig", r"Twig[\\_]Error|Twig[\\_]Environment|Unknown (?:filter|function|test|tag)"), - ("Freemarker", r"freemarker\.(?:core|template|extract|cache)\.\w+|ParseException|InvalidReferenceException|TemplateException"), + # NOTE: 'at line: N char: N' (with those colons) is how Mako, and only Mako, points at the fault + ("Mako", r"mako\.exceptions\.\w+|at line: \d+ char: \d+"), + # NOTE: Twig quotes the offending name ('Unknown "upper" filter'). Without the quotes this also + # matched the 'Unknown function' of Neo4j and Cassandra, which suggested '--ssti' on a NoSQL error + ("Twig", "Twig[\\\\_]Error|Twig[\\\\_]Environment|Unknown \"[^\"]+\" (?:filter|function|test|tag)"), + # NOTE: a bare 'ParseException' is not Freemarker. It also matched java.text.ParseException, + # org.xml.sax.SAXParseException and REXML::ParseException. The package prefix is always present + ("Freemarker", r"freemarker\.(?:core|template|extract|cache)\.\w+|InvalidReferenceException|TemplateException"), ("Velocity", r"org\.apache\.velocity\.(?:runtime|exception)\.\w+|ParseErrorException|MethodInvocationException|ResourceNotFoundException"), ("Spring EL / Thymeleaf", r"org\.springframework\.expression\.\w+|org\.thymeleaf\.\w+|SpelEvaluationException|TemplateProcessingException|ExpressionParsingException"), ("Struts2 (OGNL)", r"ognl\.(?:OgnlException|NoSuchPropertyException|MethodFailedException|InappropriateExpressionException|ExpressionSyntaxException)|com\.opensymphony\.xwork2|org\.apache\.struts2|There is no Action mapped for|Struts (?:Problem Report|has detected an unhandled exception)"), ("ERB", r"\(erb\):\d+|NameError.*undefined local variable"), - ("Pug/Jade", r"pug|jade|ParseError"), - ("Handlebars", r"handlebars|Handlebars|Parse error on line"), - ("Generic SSTI", r"template.*?(?:error|syntax|exception)"), + # NOTE: these must stay anchored to a diagnostic. The bare product names matched any page that + # carries the word 'pug'/'jade'/'handlebars' (a surname, a colour, a "), + (NOBODY, "asset (handlebars)", ""), + (NOBODY, "colour name", ""), + (NOBODY, "error page", "

500 Internal Server Error

The server encountered an internal error.

"), + (NOBODY, "CSS comment", "/* template error state: .alert{color:red} */"), + (NOBODY, "documentation", "This page explains how to use a query exception handler in your application."), + (NOBODY, "calculator", "Invalid expression: 2++2"), +) + +# The only overlaps that are allowed, because the error text really does belong to both. libxslt and +# .NET report the failure of an XPath expression that sits inside a stylesheet, so both switches are +# worth trying - which is what the response says. Anything else is a bug in the signatures. +ACCEPTED_SHARED = { + ("xslt", ".NET"): ("xpath",), + ("xslt", "libxslt runtime"): ("xpath",), + # breaking a stylesheet also breaks the XML that carries it, so libxml2 reports a malformed + # document as well. Harmless: the XXE hint needs an XML/SOAP request body to fire at all + ("xslt", "libxslt (live)"): ("xxe",), +} + + +def _matches(text): + return tuple(name for name, regex in ENGINES if re.search(regex, text)) + + +class HeuristicSignatureTest(unittest.TestCase): + def test_each_engine_recognises_its_own_errors(self): + for owner, backend, text in CORPUS: + if owner == NOBODY: + continue + self.assertIn(owner, _matches(text), + msg="'--%s' no longer recognises its own %s error: %r" % (owner, backend, text)) + + def test_no_engine_claims_another_engines_error(self): + for owner, backend, text in CORPUS: + if owner == NOBODY: + continue + allowed = ACCEPTED_SHARED.get((owner, backend), ()) + stolen = [_ for _ in _matches(text) if _ != owner and _ not in allowed] + self.assertEqual(stolen, [], + msg="a %s %s error also suggests %s" % (owner, backend, '/'.join("'--%s'" % _ for _ in stolen))) + + def test_nothing_fires_on_sql_errors_or_ordinary_pages(self): + for owner, backend, text in CORPUS: + if owner != NOBODY: + continue + fired = _matches(text) + self.assertEqual(fired, (), + msg="%s output suggests %s: %r" % (backend, '/'.join("'--%s'" % _ for _ in fired), text)) + + def test_every_engine_is_covered(self): + # a new switch must arrive here with its own errors, or the matrix above proves nothing about it + owners = set(owner for owner, _, _ in CORPUS) + missing = set(name for name, _ in ENGINES) - owners + self.assertEqual(missing, set(), msg="engines with no corpus entry: %s" % missing) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tests/test_kbchars.py b/tests/test_kbchars.py new file mode 100644 index 00000000000..22a3112ab7c --- /dev/null +++ b/tests/test_kbchars.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python + +""" +Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org) +See the file 'LICENSE' for copying permission + +Invariants of the random markers in kb.chars (lib/core/option.py). + +These markers are drawn at random once per run, so a collision between two of them is a +heisenbug: it corrupts extraction only in the fraction of runs that happen to draw the same +value twice. Both collisions below were observed and are pinned here. + + start/stop wrap the delimited output. Equal values make the output ambiguous + (a two-row result then carries four identical markers instead of + two pairs), so the parsing returns garbage or nothing at all. + at/space/dollar/hash_ stand in for '@', ' ', '$' and '#' inside a single chained REPLACE() + (see the Oracle XMLType vectors in error_based.xml). Two equal + markers make _errorReplaceChars() restore whichever character it + tries first, so e.g. an '@' in an e-mail column comes back as ' '. + +The loop count is what matters: the smaller alphabet collides once every ~512 draws and the +single-letter one every ~5, so a single sample proves nothing. + +stdlib unittest only (no pytest / no pip); works on Python 2.7 and 3.x. +""" + +import os +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from _testutils import bootstrap +bootstrap() + +from lib.core.data import kb +from lib.core.option import _setKnowledgeBaseAttributes + +ROUNDS = 3000 + +MARKERS = ("start", "stop", "at", "space", "dollar", "hash_") + + +class KbCharsDistinctTest(unittest.TestCase): + """Re-draws the markers many times. kb is global, so it is snapshotted and put back - + leaking a fresh kb into the rest of the suite is how this bug got noticed in the first place.""" + + def setUp(self): + self._saved = dict(kb) + + def tearDown(self): + kb.clear() + kb.update(self._saved) + + def test_markers_never_collide(self): + for i in range(ROUNDS): + _setKnowledgeBaseAttributes() + drawn = [getattr(kb.chars, _) for _ in MARKERS] + self.assertEqual(len(set(drawn)), len(MARKERS), + msg="colliding kb.chars markers on round %d: %s" % (i, dict(zip(MARKERS, drawn)))) + + def test_markers_keep_their_shape(self): + # the fix must not change the on-the-wire length of a payload + for _ in range(ROUNDS // 100): + _setKnowledgeBaseAttributes() + self.assertEqual(len(kb.chars.start), 5) + self.assertEqual(len(kb.chars.stop), 5) + for name in ("at", "space", "dollar", "hash_"): + self.assertEqual(len(getattr(kb.chars, name)), 3, msg=name) + self.assertEqual(len(kb.chars.delimiter), 6) + + def test_kb_is_restored(self): + # guards the snapshot above, so a later failure elsewhere is never blamed on this file + _setKnowledgeBaseAttributes() + self.assertNotEqual(dict(kb).get("chars"), self._saved.get("chars")) + + +if __name__ == "__main__": + unittest.main(verbosity=2)