diff --git a/src/main/java/org/perlonjava/frontend/parser/StringParser.java b/src/main/java/org/perlonjava/frontend/parser/StringParser.java index 550e474b1d..9706ddebc2 100644 --- a/src/main/java/org/perlonjava/frontend/parser/StringParser.java +++ b/src/main/java/org/perlonjava/frontend/parser/StringParser.java @@ -25,6 +25,7 @@ import static org.perlonjava.runtime.perlmodule.Strict.HINT_RE_TAINT; import static org.perlonjava.runtime.perlmodule.Strict.HINT_RE_DEBUG; import static org.perlonjava.runtime.perlmodule.Strict.HINT_RE_DEBUGCOLOR; +import static org.perlonjava.runtime.perlmodule.Strict.HINT_RE_STRICT; import static org.perlonjava.runtime.perlmodule.Strict.HINT_LOCALE; import static org.perlonjava.runtime.runtimetypes.NameNormalizer.normalizeVariableName; import static org.perlonjava.runtime.runtimetypes.ScalarUtils.printable; @@ -710,6 +711,9 @@ static String addLexicalRegexContext(EmitterContext ctx, String modifiers) { if (ctx.symbolTable.isStrictOptionEnabled(HINT_RE_TAINT) && !result.contains("T")) { result = "T" + result; } + if (ctx.symbolTable.isStrictOptionEnabled(HINT_RE_STRICT)) { + result += RuntimeRegex.INTERNAL_RE_STRICT_MARKER; + } return addLexicalRegexDebugMarker(ctx, result); } diff --git a/src/main/java/org/perlonjava/runtime/perlmodule/Re.java b/src/main/java/org/perlonjava/runtime/perlmodule/Re.java index ebf26ff785..3ea47f5096 100644 --- a/src/main/java/org/perlonjava/runtime/perlmodule/Re.java +++ b/src/main/java/org/perlonjava/runtime/perlmodule/Re.java @@ -181,6 +181,7 @@ public static RuntimeList importRe(RuntimeArray args, int ctx) { RuntimeScalar targetCode = getGlobalCodeRef(caller + "::regexp_pattern"); targetCode.set(sourceCode); } else if (opt.equalsIgnoreCase("strict")) { + symbolTable.enableStrictOption(Strict.HINT_RE_STRICT); // Enable categories used by our preprocessor warnings Warnings.warningManager.enableWarning("experimental::re_strict"); Warnings.warningManager.enableWarning("experimental::uniprop_wildcards"); @@ -227,6 +228,7 @@ public static RuntimeList unimportRe(RuntimeArray args, int ctx) { opt = opt.replace("\"", "").replace("'", "").trim(); if (opt.equalsIgnoreCase("strict")) { + symbolTable.disableStrictOption(Strict.HINT_RE_STRICT); Warnings.warningManager.disableWarning("experimental::re_strict"); Warnings.warningManager.disableWarning("experimental::uniprop_wildcards"); Warnings.warningManager.disableWarning("experimental::vlb"); diff --git a/src/main/java/org/perlonjava/runtime/perlmodule/Strict.java b/src/main/java/org/perlonjava/runtime/perlmodule/Strict.java index 8122edc0db..427e759b50 100644 --- a/src/main/java/org/perlonjava/runtime/perlmodule/Strict.java +++ b/src/main/java/org/perlonjava/runtime/perlmodule/Strict.java @@ -51,6 +51,7 @@ private static void propagatePragmaFlags(ScopedSymbolTable source) { public static final int HINT_RE_TAINT = 0x00002000; // use re 'taint' public static final int HINT_RE_DEBUG = 0x00004000; // use re 'debug' public static final int HINT_RE_DEBUGCOLOR = 0x00008000; // use re 'debugcolor' + public static final int HINT_RE_STRICT = 0x00010000; // use re 'strict' /** * Constructor for Strict. diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index 27fdd3c19f..51ed7f54bb 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -525,9 +525,26 @@ static boolean requiresJoniBackend(String pattern, RegexFlags flags) { || pattern.contains("(*COMMIT") || pattern.contains("(*MARK") || pattern.contains("(*:") + || containsPerlEmptyCharacterClass(pattern) || hasSubroutineCall; } + private static boolean containsPerlEmptyCharacterClass(String pattern) { + boolean quoted = false; + for (int i = 0; i + 1 < pattern.length(); i++) { + char ch = pattern.charAt(i); + if (ch == '\\') { + char next = pattern.charAt(i + 1); + if (quoted && next == 'E') quoted = false; + else if (!quoted && next == 'Q') quoted = true; + i++; + continue; + } + if (!quoted && ch == '[' && pattern.charAt(i + 1) == ']') return true; + } + return false; + } + static boolean containsNamedCharacterEscape(String pattern) { if (pattern == null) return false; for (int i = 0; i + 2 < pattern.length(); i++) { diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java index d3c965cba1..7336379242 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java @@ -20,6 +20,7 @@ import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -58,6 +59,7 @@ public static RuntimeScalar stabilizeLiteralTarget(RuntimeScalar literal, int ca /** Private AST/runtime modifier markers; removed before Perl modifier parsing. */ public static final char INTERNAL_DEBUG_MARKER = '\u0001'; public static final char INTERNAL_DEBUGCOLOR_MARKER = '\u0002'; + public static final char INTERNAL_RE_STRICT_MARKER = '\u0003'; // Debug flag for regex compilation (set at class load time) private static final boolean DEBUG_REGEX = System.getenv("DEBUG_REGEX") != null; @@ -220,6 +222,7 @@ static void updateControlVerbVariables(String mark, String error) { private List inlineModifierWarnings = new ArrayList<>(); // 0 = off, 1 = debug, 2 = debugcolor. Captured at the regex call site. private int lexicalDebugMode; + private boolean lexicalReStrict; private static final String DYNAMIC_PATTERN_ERROR = "\u0000(??{...}) recursive regex patterns not implemented (dynamic pattern)"; @@ -262,6 +265,7 @@ public RuntimeRegex cloneTracked() { copy.warningsOnUse = new ArrayList<>(this.warningsOnUse); copy.inlineModifierWarnings = new ArrayList<>(this.inlineModifierWarnings); copy.lexicalDebugMode = this.lexicalDebugMode; + copy.lexicalReStrict = this.lexicalReStrict; // replacement and callerArgs are not copied — they are set per-substitution // matched is not copied — each qr// object tracks its own m?PAT? state copy.refCount = 0; // Enable refCount tracking @@ -516,9 +520,16 @@ private static RuntimeRegex compile(String patternString, String modifiers, int private static RuntimeRegex compile(String patternString, String modifiers, int lexicalDebugMode, int trustedCalloutCount, boolean patternByteBacked) { + return compile(patternString, modifiers, lexicalDebugMode, trustedCalloutCount, + patternByteBacked, reStrictMode(modifiers)); + } + + private static RuntimeRegex compile(String patternString, String modifiers, int lexicalDebugMode, + int trustedCalloutCount, boolean patternByteBacked, + boolean lexicalReStrict) { RuntimeScalar namedCharacterTranslator = org.perlonjava.runtime.HintHashRegistry.getCompileTimeHint("charnames"); - modifiers = stripDebugMarkers(modifiers); + modifiers = stripInternalMarkers(modifiers); // Dynamic/interpolated qr// compilation can begin during ordinary // execution, outside the Perl compiler lock. User-defined Unicode // properties execute arbitrary Perl and may block, so resolve them @@ -528,7 +539,8 @@ private static RuntimeRegex compile(String patternString, String modifiers, int UnicodeResolver.preloadUserDefinedProperties( patternString, preloadFlags.isCaseInsensitive()); return compileSynchronized(patternString, modifiers, lexicalDebugMode, - trustedCalloutCount, false, patternByteBacked, namedCharacterTranslator); + trustedCalloutCount, false, patternByteBacked, lexicalReStrict, + namedCharacterTranslator); } /** User properties execute Perl code and therefore cannot be validated while compiling a CV. */ @@ -549,8 +561,8 @@ public static boolean requiresRuntimeUnicodePropertyResolution(String patternStr */ public static void validateLiteralSyntax(String patternString, String modifiers) { try { - compileSynchronized(patternString, stripDebugMarkers(modifiers), - debugMode(modifiers), 0, true, false, + compileSynchronized(patternString, stripInternalMarkers(modifiers), + debugMode(modifiers), 0, true, false, reStrictMode(modifiers), org.perlonjava.runtime.HintHashRegistry.getCompileTimeHint("charnames")); } catch (PerlJavaUnimplementedException unsupported) { String message = unsupported.getMessage(); @@ -571,7 +583,7 @@ public static void validateLiteralSyntax(String patternString, String modifiers) private static synchronized RuntimeRegex compileSynchronized( String patternString, String modifiers, int lexicalDebugMode, int trustedCalloutCount, boolean literalSyntaxValidation, - boolean patternByteBacked, + boolean patternByteBacked, boolean lexicalReStrict, RuntimeScalar namedCharacterTranslator) { // Debug logging if (DEBUG_REGEX) { @@ -624,6 +636,7 @@ private static synchronized RuntimeRegex compileSynchronized( + "#callouts=" + trustedCalloutCount + "#backend=" + RegexBackendPolicy.cacheTag() + "#bytepattern=" + effectivePatternByteBacked + + "#strict=" + lexicalReStrict + (namedCharacterTranslator == null ? "" : "#charnames=" + namedCharacterTranslator.toString()) + (hasDynamicPattern ? (warnOnUnimplemented ? "\0warn" : "\0defer") : ""); @@ -640,6 +653,7 @@ private static synchronized RuntimeRegex compileSynchronized( regex.namedCharacterCache = new JoniRegexPattern.NamedCharacterCache(namedCharacterTranslator); regex.lexicalDebugMode = lexicalDebugMode; + regex.lexicalReStrict = lexicalReStrict; // Note: flags /e /ee are processed at parse time, in parseRegexReplace() @@ -648,6 +662,39 @@ private static synchronized RuntimeRegex compileSynchronized( regex.regexFlags = fromModifiers(modifiers, compilePatternString); regex.useGAssertion = regex.regexFlags.useGAssertion(); regex.patternFlags = regex.regexFlags.toPatternFlags(); + + LeftBraceIssue leftBraceIssue = unescapedLeftBraceIssue( + originalPatternString); + String sourcePolicyWarning = null; + String constructionPolicyWarning = null; + if (leftBraceIssue != null) { + String message = leftBraceIssue.alwaysFatal || lexicalReStrict + ? "Unescaped left brace in regex is illegal here" + : "Unescaped left brace in regex is passed through"; + String diagnostic = RegexDiagnosticFormatter.markedPerl( + originalPatternString, leftBraceIssue.offset + 1, message); + if (leftBraceIssue.alwaysFatal || lexicalReStrict) { + throw new PerlCompilerException(diagnostic); + } + sourcePolicyWarning = diagnostic; + } + NonHexIssue nonHexIssue = nonHexEscapeIssue(originalPatternString); + if (nonHexIssue != null) { + if (lexicalReStrict) { + throw new PerlCompilerException(RegexDiagnosticFormatter.markedPerl( + originalPatternString, nonHexIssue.offset + 1, + "Non-hex character")); + } + if (!nonHexIssue.braced) { + char invalid = originalPatternString.charAt(nonHexIssue.offset); + char digit = originalPatternString.charAt(nonHexIssue.offset - 1); + String message = "Non-hex character '" + invalid + + "' terminates \\x early. Resolved as \"\\x0" + + digit + invalid + "\""; + constructionPolicyWarning = RegexDiagnosticFormatter.markedPerl( + originalPatternString, nonHexIssue.offset, message); + } + } // Always compute Unicode flags - we need the Unicode variant for when // the input string contains non-ASCII characters (auto-Unicode detection) @@ -688,6 +735,13 @@ private static synchronized RuntimeRegex compileSynchronized( // Track if preprocessing deferred user-defined Unicode properties. // These need to be resolved later, once the corresponding Perl subs are defined. regex.warningsOnUse = new ArrayList<>(quoteMetaWarningsOnUse); + if (sourcePolicyWarning != null) { + regex.inlineModifierWarnings.add(sourcePolicyWarning); + regex.warningsOnUse.add(sourcePolicyWarning); + } + if (constructionPolicyWarning != null) { + regex.inlineModifierWarnings.add(constructionPolicyWarning); + } if (hasDeferredDynamicPattern) { regex.warningsOnUse.add(DYNAMIC_PATTERN_ERROR); } else if (hasWarnDynamicFallback) { @@ -714,8 +768,9 @@ private static synchronized RuntimeRegex compileSynchronized( regex.recursivePattern.hasDeferredUserDefinedUnicodeProperty() || regex.recursivePatternUnicode .hasDeferredUserDefinedUnicodeProperty(); - regex.inlineModifierWarnings.addAll( - regex.recursivePattern.compileWarnings()); + regex.inlineModifierWarnings.addAll(normalizeNonHexWarningCase( + regex.recursivePattern.compileWarnings(), + originalPatternString, nonHexIssue)); regex.warningsOnUse.addAll(regex.inlineModifierWarnings); regex.hasPreservesMatch = regex.regexFlags.preservesMatch() || RegexFlags.hasInlinePreserveModifier(compilePatternString); @@ -790,6 +845,10 @@ private static synchronized RuntimeRegex compileSynchronized( } } } catch (Exception e) { + if (e instanceof PatternSyntaxException syntaxError + && "Illegal character range".equals(syntaxError.getDescription())) { + throw new PerlCompilerException("Invalid [] range"); + } if ("invalid backref number/name".equals(e.getMessage()) || "invalid backref number".equals(e.getMessage())) { throw new PerlCompilerException("Reference to nonexistent group"); @@ -970,16 +1029,212 @@ private static int ordinaryUnmatchedOpeningParenthesis(String pattern) { return -1; } + private static LeftBraceIssue unescapedLeftBraceIssue(String pattern) { + if (pattern == null || pattern.isEmpty()) return null; + boolean escaped = false; + boolean inClass = false; + for (int i = 0; i < pattern.length(); i++) { + char ch = pattern.charAt(i); + if (escaped) { + escaped = false; + continue; + } + if (ch == '\\') { + escaped = true; + continue; + } + if (ch == '[') { + inClass = true; + continue; + } + if (ch == ']' && inClass) { + inClass = false; + continue; + } + if (inClass || ch != '{') continue; + if (isEscapeArgumentBrace(pattern, i) + || isValidQuantifier(pattern, i) + || isAllowedLiteralLeftBrace(pattern, i)) { + continue; + } + boolean followsAlphanumericEscape = i >= 2 + && Character.isLetterOrDigit(pattern.charAt(i - 1)) + && pattern.charAt(i - 2) == '\\' + && (i < 3 || pattern.charAt(i - 3) != '\\'); + return new LeftBraceIssue(i, followsAlphanumericEscape); + } + return null; + } + + private static boolean isEscapeArgumentBrace(String pattern, int offset) { + if (offset < 2 || pattern.charAt(offset - 2) != '\\' + || (offset >= 3 && pattern.charAt(offset - 3) == '\\')) { + return false; + } + return "pPxXoONkgbB".indexOf(pattern.charAt(offset - 1)) >= 0; + } + + private static boolean isValidQuantifier(String pattern, int offset) { + int cursor = offset + 1; + while (cursor < pattern.length() + && isPerlIntervalWhitespace(pattern.charAt(cursor))) cursor++; + int digitStart = cursor; + while (cursor < pattern.length() && Character.isDigit(pattern.charAt(cursor))) { + cursor++; + } + boolean hasLow = cursor > digitStart; + while (cursor < pattern.length() + && isPerlIntervalWhitespace(pattern.charAt(cursor))) cursor++; + if (cursor < pattern.length() && pattern.charAt(cursor) == ',') { + cursor++; + while (cursor < pattern.length() + && isPerlIntervalWhitespace(pattern.charAt(cursor))) cursor++; + int highStart = cursor; + while (cursor < pattern.length() && Character.isDigit(pattern.charAt(cursor))) { + cursor++; + } + boolean hasHigh = cursor > highStart; + while (cursor < pattern.length() + && isPerlIntervalWhitespace(pattern.charAt(cursor))) cursor++; + return (hasLow || hasHigh) && cursor < pattern.length() + && pattern.charAt(cursor) == '}'; + } + return hasLow && cursor < pattern.length() && pattern.charAt(cursor) == '}'; + } + + private static boolean isPerlIntervalWhitespace(char ch) { + return ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' || ch == '\f'; + } + + private static boolean isAllowedLiteralLeftBrace(String pattern, int offset) { + if (offset == 0) return true; + char previous = pattern.charAt(offset - 1); + if (previous == '^' || previous == '|' || previous == '(' || previous == '*' + || previous == '+' || previous == '?') { + return true; + } + if (offset >= 3 && (pattern.regionMatches(offset - 3, "(?:", 0, 3) + || pattern.regionMatches(offset - 3, "(:?", 0, 3))) { + return true; + } + if (previous != '}') return false; + int opening = pattern.lastIndexOf('{', offset - 2); + return opening >= 0 && isValidQuantifier(pattern, opening) + && pattern.indexOf('}', opening + 1) == offset - 1; + } + + private static final class LeftBraceIssue { + final int offset; + final boolean alwaysFatal; + + LeftBraceIssue(int offset, boolean alwaysFatal) { + this.offset = offset; + this.alwaysFatal = alwaysFatal; + } + } + + private static NonHexIssue nonHexEscapeIssue(String pattern) { + if (pattern == null || pattern.contains("(?[")) return null; + for (int i = 0; i + 2 < pattern.length(); i++) { + if (pattern.charAt(i) != '\\' || pattern.charAt(i + 1) != 'x' + || (i > 0 && pattern.charAt(i - 1) == '\\')) { + continue; + } + int cursor = i + 2; + if (pattern.charAt(cursor) != '{') { + if (isHexDigit(pattern.charAt(cursor)) + && cursor + 1 < pattern.length() + && Character.isLetterOrDigit(pattern.charAt(cursor + 1)) + && !isHexDigit(pattern.charAt(cursor + 1))) { + return new NonHexIssue(cursor + 1, false); + } + continue; + } + int close = pattern.indexOf('}', cursor + 1); + if (close < 0) continue; + cursor++; + while (cursor < close && pattern.charAt(cursor) == ' ') cursor++; + int digits = cursor; + while (cursor < close) { + if (isHexDigit(pattern.charAt(cursor))) { + cursor++; + continue; + } + if (pattern.charAt(cursor) == '_' && cursor > digits + && cursor + 1 < close + && isHexDigit(pattern.charAt(cursor + 1))) { + cursor++; + continue; + } + break; + } + if (cursor == digits || cursor == close) { + i = close; + continue; + } + if (pattern.charAt(cursor) != ' ') return new NonHexIssue(cursor, true); + int whitespace = cursor; + while (cursor < close && pattern.charAt(cursor) == ' ') cursor++; + if (cursor < close) return new NonHexIssue(whitespace, true); + i = close; + } + return null; + } + + private static final class NonHexIssue { + final int offset; + final boolean braced; + + NonHexIssue(int offset, boolean braced) { + this.offset = offset; + this.braced = braced; + } + } + + private static List normalizeNonHexWarningCase( + List warnings, String pattern, NonHexIssue issue) { + if (issue == null || !issue.braced || warnings.isEmpty()) return warnings; + int opening = pattern.lastIndexOf('{', issue.offset); + if (opening < 0) return warnings; + String sourceDigits = pattern.substring(opening + 1, issue.offset).trim(); + if (sourceDigits.isEmpty()) return warnings; + List normalized = new ArrayList<>(warnings.size()); + String prefix = "Resolved as \"\\x{"; + for (String warning : warnings) { + int valueStart = warning.indexOf(prefix); + int valueEnd = valueStart < 0 ? -1 : warning.indexOf('}', valueStart + prefix.length()); + if (valueEnd >= 0) { + String resolved = warning.substring(valueStart + prefix.length(), valueEnd); + if (resolved.equalsIgnoreCase(sourceDigits)) { + warning = warning.substring(0, valueStart + prefix.length()) + + sourceDigits + warning.substring(valueEnd); + } + } + normalized.add(warning); + } + return normalized; + } + + private static boolean isHexDigit(char ch) { + return ch >= '0' && ch <= '9' || ch >= 'a' && ch <= 'f' + || ch >= 'A' && ch <= 'F'; + } + private static int debugMode(String modifiers) { if (modifiers == null) return 0; if (modifiers.indexOf(INTERNAL_DEBUGCOLOR_MARKER) >= 0) return 2; return modifiers.indexOf(INTERNAL_DEBUG_MARKER) >= 0 ? 1 : 0; } - private static String stripDebugMarkers(String modifiers) { + private static boolean reStrictMode(String modifiers) { + return modifiers != null && modifiers.indexOf(INTERNAL_RE_STRICT_MARKER) >= 0; + } + + private static String stripInternalMarkers(String modifiers) { if (modifiers == null || modifiers.isEmpty()) return modifiers == null ? "" : modifiers; return modifiers.replace(String.valueOf(INTERNAL_DEBUG_MARKER), "") - .replace(String.valueOf(INTERNAL_DEBUGCOLOR_MARKER), ""); + .replace(String.valueOf(INTERNAL_DEBUGCOLOR_MARKER), "") + .replace(String.valueOf(INTERNAL_RE_STRICT_MARKER), ""); } private void emitCompileDebugTrace() { @@ -1055,7 +1310,8 @@ private static RuntimeRegex ensureCompiledForRuntime(RuntimeRegex regex) { // makes a later qr/\\p{Property}/ reuse its match-any stand-in. state().compiledRegexCache.remove(cacheKey + "#debug=" + regex.lexicalDebugMode + "#callouts=0#backend=" + RegexBackendPolicy.cacheTag() - + "#bytepattern=" + regex.patternByteBacked); + + "#bytepattern=" + regex.patternByteBacked + + "#strict=" + regex.lexicalReStrict); // User property subs can execute arbitrary Perl and block. Resolve them // before compile() takes its process-wide monitor; only simultaneous @@ -1064,7 +1320,8 @@ private static RuntimeRegex ensureCompiledForRuntime(RuntimeRegex regex) { regex.regexFlags != null && regex.regexFlags.isCaseInsensitive()); RuntimeRegex recompiled = compile(regex.patternString, regex.regexFlags == null ? "" : regex.regexFlags.toFlagString(), - regex.lexicalDebugMode, 0, regex.patternByteBacked); + regex.lexicalDebugMode, 0, regex.patternByteBacked, + regex.lexicalReStrict); regex.pattern = recompiled.pattern; regex.patternUnicode = recompiled.patternUnicode; regex.recursivePattern = recompiled.recursivePattern; @@ -1080,6 +1337,7 @@ private static RuntimeRegex ensureCompiledForRuntime(RuntimeRegex regex) { regex.warningsOnUse = new ArrayList<>(recompiled.warningsOnUse); regex.inlineModifierWarnings = new ArrayList<>(recompiled.inlineModifierWarnings); regex.lexicalDebugMode = recompiled.lexicalDebugMode; + regex.lexicalReStrict = recompiled.lexicalReStrict; return regex; } @@ -1457,7 +1715,7 @@ public static RuntimeScalar applyUnicodeStringsFeatureToModifiers(RuntimeScalar public static RuntimeScalar getQuotedRegex(RuntimeScalar patternString, RuntimeScalar modifiers) { String rawModifierStr = modifiers.toString(); int callSiteDebugMode = debugMode(rawModifierStr); - String modifierStr = stripDebugMarkers(rawModifierStr); + String modifierStr = stripInternalMarkers(rawModifierStr); // Unwrap readonly scalar if (patternString.type == RuntimeScalarType.READONLY_SCALAR) patternString = (RuntimeScalar) patternString.value; @@ -1485,7 +1743,7 @@ && containsExecutableSource(patternString.toString(), modifierStr.indexOf('x') > } if (patternString.value instanceof RuntimeRegexTemplate template) { - RuntimeRegex regex = compile(template.pattern(), modifierStr, callSiteDebugMode, + RuntimeRegex regex = compile(template.pattern(), rawModifierStr, callSiteDebugMode, template.callbacks().size(), template.byteBackedPattern()).cloneTracked(); regex.setExecutableCallbacks(template.callbacks()); return new RuntimeScalar(regex).propagateTaint(patternString); @@ -1520,6 +1778,7 @@ && containsExecutableSource(patternString.toString(), modifierStr.indexOf('x') > regex.inlineModifierWarnings = new ArrayList<>(originalRegex.inlineModifierWarnings); regex.lexicalDebugMode = callSiteDebugMode != 0 ? callSiteDebugMode : originalRegex.lexicalDebugMode; + regex.lexicalReStrict = originalRegex.lexicalReStrict; regex.regexFlags = mergeRegexFlags(originalRegex.regexFlags, modifierStr, originalRegex.patternString); regex.hasPreservesMatch = regex.hasPreservesMatch || regex.regexFlags.preservesMatch(); regex.useGAssertion = regex.regexFlags.useGAssertion(); @@ -1564,6 +1823,7 @@ && containsExecutableSource(patternString.toString(), modifierStr.indexOf('x') > regex.inlineModifierWarnings = new ArrayList<>(originalRegex.inlineModifierWarnings); regex.lexicalDebugMode = callSiteDebugMode != 0 ? callSiteDebugMode : originalRegex.lexicalDebugMode; + regex.lexicalReStrict = originalRegex.lexicalReStrict; regex.regexFlags = mergeRegexFlags(originalRegex.regexFlags, modifierStr, originalRegex.patternString); regex.hasPreservesMatch = regex.hasPreservesMatch || regex.regexFlags.preservesMatch(); regex.useGAssertion = regex.regexFlags.useGAssertion(); @@ -1579,7 +1839,8 @@ && containsExecutableSource(patternString.toString(), modifierStr.indexOf('x') > // Try fallback to string conversion RuntimeScalar fallbackResult = overloadCtx.tryOverloadFallback(patternString, "(\"\""); if (fallbackResult != null) { - return new RuntimeScalar(compile(fallbackResult.toString(), modifierStr, callSiteDebugMode).cloneTracked()) + return new RuntimeScalar(compile(fallbackResult.toString(), rawModifierStr, + callSiteDebugMode).cloneTracked()) .propagateTaint(patternString, fallbackResult); } } @@ -1587,7 +1848,7 @@ && containsExecutableSource(patternString.toString(), modifierStr.indexOf('x') > // Default: compile as string (cloneTracked() creates a tracked copy // so the cached RuntimeRegex is not corrupted by refCount changes) - RuntimeRegex compiled = compile(patternString.toString(), modifierStr, + RuntimeRegex compiled = compile(patternString.toString(), rawModifierStr, callSiteDebugMode, 0, patternString.type == RuntimeScalarType.BYTE_STRING).cloneTracked(); return new RuntimeScalar(compiled).propagateTaint(patternString); @@ -1660,7 +1921,7 @@ static RuntimeScalar compileExecutableTemplate( List callbacks, RuntimeScalar original, boolean patternByteBacked) { int lexicalDebugMode = debugMode(modifiers); - RuntimeRegex regex = compile(executablePattern, stripDebugMarkers(modifiers), + RuntimeRegex regex = compile(executablePattern, modifiers, lexicalDebugMode, callbacks.size(), patternByteBacked).cloneTracked(); regex.setExecutableCallbacks(callbacks); return new RuntimeScalar(regex).propagateTaint(original); @@ -1723,7 +1984,7 @@ private static void validateTaintedPatternSecurity(RuntimeScalar patternString) */ public static RuntimeScalar getQuotedRegex(RuntimeScalar patternString, RuntimeScalar modifiers, int callsiteId) { String rawModifierStr = modifiers.toString(); - String modifierStr = stripDebugMarkers(rawModifierStr); + String modifierStr = stripInternalMarkers(rawModifierStr); // Check if /o or m?PAT? modifier is present (both need per-callsite caching // to preserve state: /o caches the compiled pattern, m?PAT? preserves the @@ -1760,7 +2021,7 @@ public static RuntimeScalar getReplacementRegex(RuntimeScalar patternString, Run RuntimeRegex resolvedRegex = resolved.regex(); String rawModifierStr = modifiers.toString(); int callSiteDebugMode = debugMode(rawModifierStr); - String modifierStr = stripDebugMarkers(rawModifierStr); + String modifierStr = stripInternalMarkers(rawModifierStr); // Create a new regex instance with the replacement RuntimeRegex regex = new RuntimeRegex(); @@ -1788,6 +2049,7 @@ public static RuntimeScalar getReplacementRegex(RuntimeScalar patternString, Run regex.inlineModifierWarnings = new ArrayList<>(resolvedRegex.inlineModifierWarnings); regex.lexicalDebugMode = callSiteDebugMode != 0 ? callSiteDebugMode : resolvedRegex.lexicalDebugMode; + regex.lexicalReStrict = resolvedRegex.lexicalReStrict; // Only recompile if we have new modifiers that actually change the flags if (!modifierStr.isEmpty()) { @@ -1808,7 +2070,7 @@ public static RuntimeScalar getReplacementRegex(RuntimeScalar patternString, Run RuntimeRegex recompiledRegex = compile(resolvedRegex.patternString, newFlags.toFlagString(), regex.lexicalDebugMode, resolvedRegex.executableCallbacks.size(), - resolvedRegex.patternByteBacked); + resolvedRegex.patternByteBacked, regex.lexicalReStrict); regex.pattern = recompiledRegex.pattern; regex.patternUnicode = recompiledRegex.patternUnicode; regex.recursivePattern = recompiledRegex.recursivePattern; @@ -2117,6 +2379,8 @@ private static RuntimeBase matchRegexDirect(RuntimeScalar quotedRegex, RuntimeSc tempRegex.lexicalDebugMode = regex.lexicalDebugMode != 0 ? regex.lexicalDebugMode : regexState.lastSuccessfulPattern.lexicalDebugMode; + tempRegex.lexicalReStrict = regex.lexicalReStrict + || regexState.lastSuccessfulPattern.lexicalReStrict; tempRegex.regexFlags = originalFlags; tempRegex.useGAssertion = originalFlags != null && originalFlags.useGAssertion(); regex = tempRegex; diff --git a/src/test/java/org/perlonjava/runtime/regex/NativeEmptyClassRoutingTest.java b/src/test/java/org/perlonjava/runtime/regex/NativeEmptyClassRoutingTest.java new file mode 100644 index 0000000000..f0e6b403d7 --- /dev/null +++ b/src/test/java/org/perlonjava/runtime/regex/NativeEmptyClassRoutingTest.java @@ -0,0 +1,22 @@ +package org.perlonjava.runtime.regex; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +@Tag("unit") +class NativeEmptyClassRoutingTest { + @Test + void routesPerlEmptyClassToJoni() { + assertTrue(JoniRegexPattern.requiresJoniBackend("a[]b")); + assertTrue(JoniRegexPattern.requiresJoniBackend("(?i:a[]b)")); + } + + @Test + void ignoresEscapedAndQuotedBracketPairs() { + assertFalse(JoniRegexPattern.requiresJoniBackend("a\\[]b")); + assertFalse(JoniRegexPattern.requiresJoniBackend("\\Q[]\\E")); + } +} diff --git a/src/test/resources/unit/regex/boundary_empty_whitespace.t b/src/test/resources/unit/regex/boundary_empty_whitespace.t new file mode 100644 index 0000000000..4f3eead298 --- /dev/null +++ b/src/test/resources/unit/regex/boundary_empty_whitespace.t @@ -0,0 +1,17 @@ +use strict; +use warnings; +use Test::More; + +for my $name (qw(gcb sb wb lb)) { + my $positive = qr/\b{$name}/; + my $negative = qr/\B{$name}/; + my $spaced_positive = qr/\b{ $name }/; + my $spaced_negative = qr/\B{ $name }/; + + unlike('', $positive, "empty text has no $name boundary"); + like('', $negative, "empty text satisfies negated $name boundary"); + unlike('', $spaced_positive, "whitespace is accepted around $name"); + like('', $spaced_negative, "spaced negated $name retains empty semantics"); +} + +done_testing; diff --git a/src/test/resources/unit/regex/character_class_closing_bracket.t b/src/test/resources/unit/regex/character_class_closing_bracket.t new file mode 100644 index 0000000000..318b39ffd8 --- /dev/null +++ b/src/test/resources/unit/regex/character_class_closing_bracket.t @@ -0,0 +1,26 @@ +use strict; +use warnings; +use Test::More; + +my @cases = ( + [ 'a]', 'a]', 1, 'closing bracket outside a class is literal' ], + [ 'a[]]b', 'a]b', 1, 'leading closing bracket in a class is literal' ], + [ 'a[^]b]c', 'a]c', 0, 'negated class excludes its leading bracket' ], + [ 'a[^]b]c', 'adc', 1, 'negated class retains its ordinary members' ], + [ '2(]*)?$\\1', '2', 1, 'closing bracket class composes with backreference' ], +); + +for my $case (@cases) { + my ($pattern, $subject, $expected, $name) = @$case; + my @warnings; + my $regex; + { + local $SIG{__WARN__} = sub { push @warnings, @_ }; + $regex = eval { qr/$pattern/ }; + } + is($@, '', "$name compiles"); + is(scalar @warnings, 0, "$name has no warning"); + is(($subject =~ $regex) ? 1 : 0, $expected, $name); +} + +done_testing; diff --git a/src/test/resources/unit/regex/character_class_range_diagnostic.t b/src/test/resources/unit/regex/character_class_range_diagnostic.t new file mode 100644 index 0000000000..4bc611af7d --- /dev/null +++ b/src/test/resources/unit/regex/character_class_range_diagnostic.t @@ -0,0 +1,11 @@ +use strict; +use warnings; +use Test::More; + +for my $pattern ('[b-a]', '(?i:a[b-a])', '[\x{100}-\x{ff}]') { + my $regex = eval { qr/$pattern/ }; + ok(!defined($regex), "$pattern is rejected"); + like($@, qr/^Invalid \[\] range/, "$pattern uses Perl range diagnostic"); +} + +done_testing; diff --git a/src/test/resources/unit/regex/empty_character_class_diagnostic.t b/src/test/resources/unit/regex/empty_character_class_diagnostic.t new file mode 100644 index 0000000000..5cc70d35fb --- /dev/null +++ b/src/test/resources/unit/regex/empty_character_class_diagnostic.t @@ -0,0 +1,11 @@ +use strict; +use warnings; +use Test::More; + +for my $pattern ('a[]b', '(?i:a[]b)') { + my $regex = eval { qr/$pattern/ }; + ok(!defined($regex), "$pattern is rejected"); + like($@, qr/^Unmatched \[/, "$pattern uses Perl unmatched-class diagnostic"); +} + +done_testing; diff --git a/src/test/resources/unit/regex/posix_class_unknown_diagnostic.t b/src/test/resources/unit/regex/posix_class_unknown_diagnostic.t new file mode 100644 index 0000000000..0d0d3e8fba --- /dev/null +++ b/src/test/resources/unit/regex/posix_class_unknown_diagnostic.t @@ -0,0 +1,13 @@ +use strict; +use warnings; +use Test::More; + +for my $name ('foo', '^foo', 'xyz') { + my $pattern = "[[:$name:]]"; + my $regex = eval { qr/$pattern/ }; + ok(!defined($regex), "$pattern is rejected"); + like($@, qr/^POSIX class \[:\Q$name\E:\] unknown/, + "$pattern uses Perl POSIX-class diagnostic"); +} + +done_testing; diff --git a/src/test/resources/unit/regex/quantifier_follows_nothing_diagnostic.t b/src/test/resources/unit/regex/quantifier_follows_nothing_diagnostic.t new file mode 100644 index 0000000000..3230e1f482 --- /dev/null +++ b/src/test/resources/unit/regex/quantifier_follows_nothing_diagnostic.t @@ -0,0 +1,12 @@ +use strict; +use warnings; +use Test::More; + +for my $pattern ('*a', '(|*)b', '(?i:*a)', '(?i:(|*)b)') { + my $regex = eval { qr/$pattern/ }; + ok(!defined($regex), "$pattern is rejected"); + like($@, qr/^Quantifier follows nothing/, + "$pattern uses Perl leading-quantifier diagnostic"); +} + +done_testing; diff --git a/src/test/resources/unit/regex/quantifier_omitted_lower_bound.t b/src/test/resources/unit/regex/quantifier_omitted_lower_bound.t new file mode 100644 index 0000000000..4872d56663 --- /dev/null +++ b/src/test/resources/unit/regex/quantifier_omitted_lower_bound.t @@ -0,0 +1,38 @@ +use strict; +use warnings; +use Test::More; + +sub compile_pattern { + my ($pattern, $strict) = @_; + my (@warnings, $regex, $error); + { + local $SIG{__WARN__} = sub { push @warnings, @_ }; + if ($strict) { + no warnings 'experimental::re_strict'; + use re 'strict'; + $regex = eval { qr/$pattern/ }; + } + else { + $regex = eval { qr/$pattern/ }; + } + $error = $@; + } + return ($regex, $error, \@warnings); +} + +for my $case ( + [ 'a{,2}', 'aa' ], + [ 'a{, 2 }', 'aa' ], + [ 'a{ , 2 }', 'aa' ], + [ '[x]{, 2}', 'xx' ], + [ '\p{Latin}{ , 2 }', 'a' ], +) { + my ($pattern, $subject) = @$case; + for my $strict (0, 1) { + my ($regex, $error, $warnings) = compile_pattern($pattern, $strict); + ok(defined($regex) && $error eq '' && !@$warnings && $subject =~ /\A$regex\z/, + "$pattern is a quiet quantifier" . ($strict ? ' under re strict' : '')); + } +} + +done_testing; diff --git a/src/test/resources/unit/regex/regex_re_strict_left_brace_policy.t b/src/test/resources/unit/regex/regex_re_strict_left_brace_policy.t new file mode 100644 index 0000000000..97ec7c967a --- /dev/null +++ b/src/test/resources/unit/regex/regex_re_strict_left_brace_policy.t @@ -0,0 +1,80 @@ +use strict; +use warnings; +use Test::More; + +sub capture_eval_string { + my ($source) = @_; + my @warnings; + local $SIG{__WARN__} = sub { push @warnings, @_ }; + my $value = eval $source; + return ($value, $@, \@warnings); +} + +sub compile_default { + my ($pattern) = @_; + my @warnings; + local $SIG{__WARN__} = sub { push @warnings, @_ }; + my $value = eval { qr/$pattern/ }; + return ($value, $@, \@warnings); +} + +{ + no warnings 'experimental::re_strict'; + use re 'strict'; + + sub compile_strict { + my ($pattern) = @_; + my @warnings; + local $SIG{__WARN__} = sub { push @warnings, @_ }; + my $value = eval { qr/$pattern/ }; + return ($value, $@, \@warnings); + } +} + +my ($value, $error, $warnings) = compile_default('\\w{'); +ok(!defined($value), 'ambiguous brace after escape is fatal by default'); +like($error, qr/^Unescaped left brace in regex is illegal here in regex; marked by <-- HERE in m\/\\w\{ <-- HERE \/ at /, + 'default fatal marker follows brace'); +is(scalar(@$warnings), 0, 'default fatal emits no warning'); + +($value, $error, $warnings) = compile_default(':{4,a}'); +ok(defined($value) && $error eq '', 'malformed quantifier-like brace passes by default'); +like($warnings->[0] // '', qr/^Unescaped left brace in regex is passed through in regex; marked by <-- HERE in m\/:\{ <-- HERE 4,a\}\/ at /, + 'default warning marker follows brace'); + +($value, $error, $warnings) = compile_strict(':{4,a}'); +ok(!defined($value), 'malformed quantifier-like brace is fatal under lexical strict'); +like($error, qr/^Unescaped left brace in regex is illegal here in regex; marked by <-- HERE in m\/:\{ <-- HERE 4,a\}\/ at /, + 'strict fatal marker follows brace'); +is(scalar(@$warnings), 0, 'strict fatal emits no warning'); + +($value, $error, $warnings) = capture_eval_string(q{qr/:{4,a}/}); +ok(defined($value) && $error eq '' && @$warnings == 1, + 'literal malformed brace warns outside strict'); + +($value, $error, $warnings) = capture_eval_string( + q{no warnings 'experimental::re_strict'; use re 'strict'; qr/:{4,a}/}); +ok(!defined($value) && $error =~ /^Unescaped left brace in regex is illegal here/, + 'literal malformed brace is fatal inside strict'); + +for my $pattern ('^{', 'foo|{', '\\s*{', 'a{3,4}{', 'foo(:?{bar)') { + ($value, $error, $warnings) = compile_strict($pattern); + ok(defined($value) && $error eq '' && @$warnings == 0, + "allowed brace context remains quiet: $pattern"); +} + +my @boundary_cases = ( + ['\\B{gc}', qr/^'gc' is an unknown bound type in regex/], + ['\\B{}', qr/^Empty \\B\{\} in regex/], + ['a\\B{cde', qr/^Missing right brace on \\B\{\} in regex/], +); +for my $case (@boundary_cases) { + my ($pattern, $expected) = @$case; + for my $compiler (\&compile_default, \&compile_strict) { + ($value, $error, $warnings) = $compiler->($pattern); + ok(!defined($value) && $error =~ $expected && @$warnings == 0, + "boundary brace keeps its dedicated diagnostic: $pattern"); + } +} + +done_testing; diff --git a/src/test/resources/unit/regex/regex_re_strict_nonhex_policy.t b/src/test/resources/unit/regex/regex_re_strict_nonhex_policy.t new file mode 100644 index 0000000000..07470fe080 --- /dev/null +++ b/src/test/resources/unit/regex/regex_re_strict_nonhex_policy.t @@ -0,0 +1,63 @@ +use strict; +use warnings; +use Test::More; + +sub capture_eval_string { + my ($source) = @_; + my @warnings; + local $SIG{__WARN__} = sub { push @warnings, @_ }; + my $value = eval $source; + return ($value, $@, \@warnings); +} + +sub compile_default { + my ($pattern) = @_; + my @warnings; + local $SIG{__WARN__} = sub { push @warnings, @_ }; + my $value = eval { qr/$pattern/ }; + return ($value, $@, \@warnings); +} + +{ + no warnings 'experimental::re_strict'; + use re 'strict'; + + sub compile_strict { + my ($pattern) = @_; + my @warnings; + local $SIG{__WARN__} = sub { push @warnings, @_ }; + my $value = eval { qr/$pattern/ }; + return ($value, $@, \@warnings); + } +} + +my @cases = ( + ['\\xAG', 'm/\\xAG <-- HERE /'], + ['[\\xAG]', 'm/[\\xAG <-- HERE ]/'], + ['\\x{ABCDEFG}', 'm/\\x{ABCDEFG <-- HERE }/'], + ['[\\x{ABCDEFG}]', 'm/[\\x{ABCDEFG <-- HERE }]/'], + ['\\x{ 5 0 }', 'm/\\x{ 5 <-- HERE 0 }/'], +); + +for my $case (@cases) { + my ($pattern, $marked_pattern) = @$case; + my ($value, $error, $warnings) = compile_default($pattern); + ok(defined($value) && $error eq '', "non-hex escape passes by default: $pattern"); + is(scalar(@$warnings), 1, "default non-hex escape warns exactly once: $pattern"); + like($warnings->[0] // '', qr/^Non-hex character '.+' terminates \\x early\. Resolved as /, + "default non-hex warning retained: $pattern"); + + ($value, $error, $warnings) = compile_strict($pattern); + ok(!defined($value), "non-hex escape is fatal under lexical strict: $pattern"); + like($error, qr/^Non-hex character in regex; marked by <-- HERE in \Q$marked_pattern\E at /, + "strict non-hex marker: $pattern"); + is(scalar(@$warnings), 0, "strict non-hex fatal emits no warning: $pattern"); +} + +my ($literal, $literal_error, $literal_warnings) = capture_eval_string(q!qr/\xAG/!); +ok(defined($literal) && $literal_error eq '', 'literal unbraced non-hex escape compiles'); +is(scalar(@$literal_warnings), 1, 'literal unbraced non-hex escape warns exactly once'); +like($literal_warnings->[0], qr/^Non-hex character 'G' terminates \\x early/, + 'literal unbraced non-hex warning keeps Perl text'); + +done_testing; diff --git a/third_party/joni/src/org/joni/ByteCodeMachine.java b/third_party/joni/src/org/joni/ByteCodeMachine.java index 33020e4d96..f38475ab52 100644 --- a/third_party/joni/src/org/joni/ByteCodeMachine.java +++ b/third_party/joni/src/org/joni/ByteCodeMachine.java @@ -1399,6 +1399,7 @@ private void opWordBreakBoundary(boolean negated) { } private boolean isWordBreakBoundary() { + if (str == end) return false; if (s <= str || s >= end) return true; // WB1, WB2 int leftPosition = enc.prevCharHead(bytes, str, s, end); @@ -1858,6 +1859,7 @@ private int precedingLineRun(int position, short value) { } private boolean isSentenceBoundary() { + if (str == end) return false; if (s <= str || s >= end) return true; // SB1, SB2 int leftPosition = enc.prevCharHead(bytes, str, s, end); @@ -1981,6 +1983,7 @@ private boolean isSentenceTerminal(byte property) { } private boolean isGraphemeBoundary() { + if (str == end) return false; if (s <= str || s >= end) return true; // GB1, GB2 int leftPosition = enc.prevCharHead(bytes, str, s, end); diff --git a/third_party/joni/src/org/joni/CodeRangeBuffer.java b/third_party/joni/src/org/joni/CodeRangeBuffer.java index c5d0e6ca41..47af86360b 100644 --- a/third_party/joni/src/org/joni/CodeRangeBuffer.java +++ b/third_party/joni/src/org/joni/CodeRangeBuffer.java @@ -186,7 +186,7 @@ public static CodeRangeBuffer addCodeRange(CodeRangeBuffer pbuf, ScanEnvironment if (env.syntax.allowEmptyRangeInCC()) { return pbuf; } else { - throw new ValueException(ErrorMessages.EMPTY_RANGE_IN_CHAR_CLASS); + throw new ValueException(env.emptyRangeError()); } } return addCodeRangeToBuff(pbuf, env, from, to, checkDup); diff --git a/third_party/joni/src/org/joni/Lexer.java b/third_party/joni/src/org/joni/Lexer.java index 76b65e415b..0596502fe3 100644 --- a/third_party/joni/src/org/joni/Lexer.java +++ b/third_party/joni/src/org/joni/Lexer.java @@ -1991,7 +1991,7 @@ private boolean fetchTokenForPerlBoundary(boolean negated) { int characterStart = p; fetch(); if (c == '}') { - String boundaryName = name.toString(); + String boundaryName = name.toString().trim(); if (boundaryName.equals("gcb")) { fetchTokenFor_anchor(negated ? AnchorType.NOT_GRAPHEME_BOUNDARY diff --git a/third_party/joni/src/org/joni/Parser.java b/third_party/joni/src/org/joni/Parser.java index b7a37e523a..070c8f4d87 100644 --- a/third_party/joni/src/org/joni/Parser.java +++ b/third_party/joni/src/org/joni/Parser.java @@ -195,6 +195,7 @@ private boolean parsePosixBracket(CClassNode cc, CClassNode ascCc, } else { not = false; } + int nameStart = p; if (enc.strLength(bytes, p, stop) >= POSIX_BRACKET_NAME_MIN_LEN + 3) { // else goto not_posix_bracket boolean asciiRange = isAsciiRange(env.option) && !isPosixBracketAllRange(env.option); @@ -233,10 +234,19 @@ private boolean parsePosixBracket(CClassNode cc, CClassNode ascCc, } if (c == ':' && left()) { + int nameEnd = p; inc(); if (left()) { fetch(); - if (c == ']') newSyntaxException(INVALID_POSIX_BRACKET_TYPE); + if (c == ']') { + if (env.usesPerlDiagnostics()) { + String name = new String(bytes, nameStart, + nameEnd - nameStart, StandardCharsets.US_ASCII); + newSyntaxException("POSIX class [:" + (not ? "^" : "") + + name + ":] unknown"); + } + newSyntaxException(INVALID_POSIX_BRACKET_TYPE); + } } } restore(); @@ -341,6 +351,7 @@ int apply(int option, int modifier, boolean neg, boolean rejectLocale) { private ParsedCharClass parseCharClass(ObjPtr ascNode, ObjPtr foldNode) { + int classContentStart = p - getBegin(); final boolean neg; CClassNode cc, prevCc = null, ascCc = null, ascPrevCc = null, workCc = null, ascWorkCc = null, foldCc = null, @@ -357,7 +368,12 @@ private ParsedCharClass parseCharClass(ObjPtr ascNode, } if (token.type == TokenType.CC_CLOSE && !syntax.op3OptionECMAScript()) { - if (!codeExistCheck(']', true)) newSyntaxException(EMPTY_CHAR_CLASS); + if (!codeExistCheck(']', true)) { + if (env.usesPerlDiagnostics()) { + newSyntaxException(PERL_UNMATCHED_OPEN_BRACKET, classContentStart); + } + newSyntaxException(EMPTY_CHAR_CLASS); + } env.ccEscWarn("]"); token.type = TokenType.CHAR; /* allow []...] */ } @@ -1529,7 +1545,9 @@ private Node parseExp(TokenType term) { case INTERVAL: if (syntax.contextIndepRepeatOps()) { if (syntax.contextInvalidRepeatOps()) { - newSyntaxException(TARGET_OF_REPEAT_OPERATOR_NOT_SPECIFIED); + newSyntaxException(env.usesPerlDiagnostics() + ? PERL_QUANTIFIER_FOLLOWS_NOTHING + : TARGET_OF_REPEAT_OPERATOR_NOT_SPECIFIED); } else { node = StringNode.EMPTY; // node_new_empty } diff --git a/third_party/joni/src/org/joni/ScanEnvironment.java b/third_party/joni/src/org/joni/ScanEnvironment.java index 4405782f64..886eb2cab6 100644 --- a/third_party/joni/src/org/joni/ScanEnvironment.java +++ b/third_party/joni/src/org/joni/ScanEnvironment.java @@ -162,6 +162,10 @@ int convertBackslashValue(int c) { void ccEscWarn(String s) { if (warnings != WarnCallback.NONE) { + // Perl accepts a leading ']' as a literal character in a class + // without warning. Ruby/Oniguruma warns for this spelling, but + // PerlNG must preserve Perl's warning policy as well as its parse. + if (syntax.op2OptionPerl() && "]".equals(s)) return; if (syntax.warnCCOpNotEscaped() && syntax.backSlashEscapeInCC()) { warnings.warn("character class has '" + s + "' without escape"); } @@ -174,8 +178,22 @@ void unknownEscWarn(String s) { } } + public String emptyRangeError() { + return usesPerlDiagnostics() + ? ErrorMessages.PERL_INVALID_RANGE_IN_CHAR_CLASS + : ErrorMessages.EMPTY_RANGE_IN_CHAR_CLASS; + } + + public boolean usesPerlDiagnostics() { + return "PerlNG".equals(syntax.name) || "PERLONJAVA".equals(syntax.name); + } + void closeBracketWithoutEscapeWarn(String s) { if (warnings != WarnCallback.NONE) { + // A closing bracket outside a character class is an ordinary + // literal in Perl (except at the start, which the lexer already + // permits separately) and does not produce a regexp warning. + if (syntax.op2OptionPerl() && "]".equals(s)) return; if (syntax.warnCCOpNotEscaped()) { warnings.warn("regular expression has '" + s + "' without escape"); } diff --git a/third_party/joni/src/org/joni/ast/CClassNode.java b/third_party/joni/src/org/joni/ast/CClassNode.java index 30ec08a6aa..9cdc69ada5 100644 --- a/third_party/joni/src/org/joni/ast/CClassNode.java +++ b/third_party/joni/src/org/joni/ast/CClassNode.java @@ -684,7 +684,7 @@ public void nextStateValue(CCStateArg arg, CClassNode ascCc, arg.state = CCSTATE.COMPLETE; break; } else { - throw new ValueException(ErrorMessages.EMPTY_RANGE_IN_CHAR_CLASS); + throw new ValueException(env.emptyRangeError()); } } bs.setRange(env, (int)arg.from, (int)arg.to); @@ -696,7 +696,7 @@ public void nextStateValue(CCStateArg arg, CClassNode ascCc, arg.state = CCSTATE.COMPLETE; break; } - throw new ValueException(ErrorMessages.EMPTY_RANGE_IN_CHAR_CLASS); + throw new ValueException(env.emptyRangeError()); } addWideScalarRange(arg.from, arg.to); } else { @@ -711,7 +711,7 @@ public void nextStateValue(CCStateArg arg, CClassNode ascCc, arg.state = CCSTATE.COMPLETE; break; } else { - throw new ValueException(ErrorMessages.EMPTY_RANGE_IN_CHAR_CLASS); + throw new ValueException(env.emptyRangeError()); } } long normalTo = Math.min(arg.to, 0x10ffffL); diff --git a/third_party/joni/src/org/joni/exception/ErrorMessages.java b/third_party/joni/src/org/joni/exception/ErrorMessages.java index 25109cc58f..80d5f29abf 100644 --- a/third_party/joni/src/org/joni/exception/ErrorMessages.java +++ b/third_party/joni/src/org/joni/exception/ErrorMessages.java @@ -36,6 +36,7 @@ public interface ErrorMessages extends org.jcodings.exception.ErrorMessages { String END_PATTERN_AT_LEFT_BRACE = "end pattern at left brace"; String END_PATTERN_AT_LEFT_BRACKET = "end pattern at left bracket"; String EMPTY_CHAR_CLASS = "empty char-class"; + String PERL_UNMATCHED_OPEN_BRACKET = "Unmatched ["; String PREMATURE_END_OF_CHAR_CLASS = "premature end of char-class"; String END_PATTERN_AT_ESCAPE = "end pattern at escape"; String END_PATTERN_AT_META = "end pattern at meta"; @@ -46,6 +47,7 @@ public interface ErrorMessages extends org.jcodings.exception.ErrorMessages { String CHAR_CLASS_VALUE_AT_START_OF_RANGE = "char-class value at start of range"; String UNMATCHED_RANGE_SPECIFIER_IN_CHAR_CLASS = "unmatched range specifier in char-class"; String TARGET_OF_REPEAT_OPERATOR_NOT_SPECIFIED = "target of repeat operator is not specified"; + String PERL_QUANTIFIER_FOLLOWS_NOTHING = "Quantifier follows nothing"; String TARGET_OF_REPEAT_OPERATOR_INVALID = "target of repeat operator is invalid"; String NESTED_REPEAT_NOT_ALLOWED = "nested repeat is not allowed"; String NESTED_REPEAT_OPERATOR = "nested repeat operator"; @@ -147,6 +149,7 @@ public interface ErrorMessages extends org.jcodings.exception.ErrorMessages { String TOO_BIG_NUMBER_FOR_REPEAT_RANGE = "too big number for repeat range"; String UPPER_SMALLER_THAN_LOWER_IN_REPEAT_RANGE = "upper is smaller than lower in repeat range"; String EMPTY_RANGE_IN_CHAR_CLASS = "empty range in char class"; + String PERL_INVALID_RANGE_IN_CHAR_CLASS = "Invalid [] range"; String MISMATCH_CODE_LENGTH_IN_CLASS_RANGE = "mismatch multibyte code length in char-class range"; String TOO_MANY_MULTI_BYTE_RANGES = "too many multibyte code ranges are specified"; String TOO_SHORT_MULTI_BYTE_STRING = "too short multibyte code string"; diff --git a/third_party/joni/test/org/joni/test/TestPerlBoundaryDiagnostics.java b/third_party/joni/test/org/joni/test/TestPerlBoundaryDiagnostics.java index fc7ff78e90..dc559a9f32 100644 --- a/third_party/joni/test/org/joni/test/TestPerlBoundaryDiagnostics.java +++ b/third_party/joni/test/org/joni/test/TestPerlBoundaryDiagnostics.java @@ -66,6 +66,8 @@ public void retainsSupportedPerlBoundaryNames() { for (String name : new String[] {"gcb", "sb", "wb", "lb"}) { compile("\\b{" + name + "}", Syntax.PerlNG); compile("\\B{" + name + "}", Syntax.PerlNG); + compile("\\b{ " + name + " }", Syntax.PerlNG); + compile("\\B{ " + name + " }", Syntax.PerlNG); } } diff --git a/third_party/joni/test/org/joni/test/TestPerlCharacterClassDiagnostics.java b/third_party/joni/test/org/joni/test/TestPerlCharacterClassDiagnostics.java new file mode 100644 index 0000000000..cc1b5621bb --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestPerlCharacterClassDiagnostics.java @@ -0,0 +1,78 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +import java.nio.charset.StandardCharsets; + +import org.jcodings.specific.UTF8Encoding; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.joni.WarnCallback; +import org.joni.exception.SyntaxException; +import org.junit.Test; + +public class TestPerlCharacterClassDiagnostics { + @Test + public void descendingRangeUsesPerlDiagnostic() { + byte[] bytes = "[b-a]".getBytes(StandardCharsets.UTF_8); + SyntaxException error = assertThrows(SyntaxException.class, + () -> new Regex(bytes, 0, bytes.length, Option.NONE, + UTF8Encoding.INSTANCE, Syntax.PerlNG, WarnCallback.NONE)); + assertEquals("Invalid [] range", error.getMessage()); + } + + @Test + public void unknownPosixClassNamesUsePerlDiagnostic() { + assertPosixError("[[:foo:]]", "POSIX class [:foo:] unknown"); + assertPosixError("[[:^foo:]]", "POSIX class [:^foo:] unknown"); + } + + private static void assertPosixError(String pattern, String message) { + byte[] bytes = pattern.getBytes(StandardCharsets.UTF_8); + SyntaxException error = assertThrows(SyntaxException.class, + () -> new Regex(bytes, 0, bytes.length, Option.NONE, + UTF8Encoding.INSTANCE, Syntax.PerlNG, WarnCallback.NONE)); + assertEquals(message, error.getMessage()); + } + + @Test + public void emptyClassUsesPerlUnmatchedBracketDiagnostic() { + byte[] bytes = "a[]b".getBytes(StandardCharsets.UTF_8); + SyntaxException error = assertThrows(SyntaxException.class, + () -> new Regex(bytes, 0, bytes.length, Option.NONE, + UTF8Encoding.INSTANCE, Syntax.PerlNG, WarnCallback.NONE)); + assertEquals("Unmatched [", error.getMessage()); + assertEquals(2, error.getPatternPosition()); + } + + @Test + public void leadingQuantifierUsesPerlDiagnostic() { + byte[] bytes = "*a".getBytes(StandardCharsets.UTF_8); + SyntaxException error = assertThrows(SyntaxException.class, + () -> new Regex(bytes, 0, bytes.length, Option.NONE, + UTF8Encoding.INSTANCE, Syntax.PerlNG, WarnCallback.NONE)); + assertEquals("Quantifier follows nothing", error.getMessage()); + assertEquals(1, error.getPatternPosition()); + } +}