diff --git a/dev/implementation/regex.md b/dev/implementation/regex.md index bd8f4f6ad..9f0317c91 100644 --- a/dev/implementation/regex.md +++ b/dev/implementation/regex.md @@ -8,11 +8,11 @@ backtracking, captures, conditions, recursion, control verbs, Unicode matching, case folding, and matcher-visible callbacks. PerlOnJava owns Perl source policy, runtime integration, lexical warnings, diagnostics, and callback closures. -Migration is not complete. Automatic routing still uses Java for ordinary -patterns unless a Joni-only construct is present. Setting -`JPERL_REGEX_BACKEND=joni` or the `jperl.regex.backend=joni` system property -forces Joni and is the compatibility gate used by the unit corpus. The Java -matcher, selector, and Java-only rewrites are temporary migration scaffolding. +Migration is not complete. Automatic routing uses Joni for ordinary patterns. +Setting `JPERL_REGEX_BACKEND=java` or the `jperl.regex.backend=java` system +property retains the legacy matcher solely as a differential baseline; explicit +`joni` selects the production route. The Java matcher, selector, and Java-only +rewrites are temporary migration scaffolding. ## Compilation and routing diff --git a/dev/import-perl5/config.yaml b/dev/import-perl5/config.yaml index 60ecd866e..f45961dd2 100644 --- a/dev/import-perl5/config.yaml +++ b/dev/import-perl5/config.yaml @@ -144,9 +144,6 @@ imports: target: perl5_t/t/test.pl patch: test.pl.patch - - source: perl5/t/re/pat.t - target: perl5_t/t/re/pat.t - - source: perl5/t/porting/manifest.t target: perl5_t/t/porting/manifest.t patch: manifest.t.patch diff --git a/src/main/java/org/perlonjava/backend/bytecode/CompileOperator.java b/src/main/java/org/perlonjava/backend/bytecode/CompileOperator.java index 9db294d41..a902dffb3 100644 --- a/src/main/java/org/perlonjava/backend/bytecode/CompileOperator.java +++ b/src/main/java/org/perlonjava/backend/bytecode/CompileOperator.java @@ -1128,7 +1128,13 @@ public static void visitOperator(BytecodeCompiler bytecodeCompiler, OperatorNode && !modifiers.contains("u")) { modifiers += "u"; } - RuntimeRegex.validateLiteralSyntax(literalPattern, modifiers); + try { + RuntimeRegex.validateLiteralSyntax(literalPattern, modifiers); + } catch (PerlCompilerException exception) { + throw PerlCompilerException.withSourceLocation( + node.tokenIndex, exception.getMessage(), + bytecodeCompiler.errorUtil); + } } boolean needsCallsiteCache = false; Node flagsNode = operand.elements.get(1); diff --git a/src/main/java/org/perlonjava/frontend/parser/StringSegmentParser.java b/src/main/java/org/perlonjava/frontend/parser/StringSegmentParser.java index 0b2ecef3f..7686c5479 100644 --- a/src/main/java/org/perlonjava/frontend/parser/StringSegmentParser.java +++ b/src/main/java/org/perlonjava/frontend/parser/StringSegmentParser.java @@ -1531,6 +1531,10 @@ void handleUnicodeNameEscape() { appendToCurrentSegment("\\N{" + name + "}"); return; } + if (name.matches("(?i)U\\+[0-9A-F]+(?:\\.[0-9A-F]+)+")) { + throwNamedCharacterDiagnostic( + "Invalid hexadecimal number in \\N{U+...}"); + } NamedCharacterExpansion expansion = NamedCharacterExpansion.resolve(name, sourceMode); if (expansion.resolved()) { diff --git a/src/main/java/org/perlonjava/runtime/NamedCharacterExpansion.java b/src/main/java/org/perlonjava/runtime/NamedCharacterExpansion.java index 84d6eabae..341313b9a 100644 --- a/src/main/java/org/perlonjava/runtime/NamedCharacterExpansion.java +++ b/src/main/java/org/perlonjava/runtime/NamedCharacterExpansion.java @@ -66,6 +66,19 @@ public static NamedCharacterExpansion resolve( if (name.matches("(?i)U\\+[0-9A-F]+")) { return resolveStandard(name); } + if (name.matches("(?i)U\\+[0-9A-F]+(?:\\.[0-9A-F]+)+")) { + try { + StringBuilder sequence = new StringBuilder(); + for (String scalar : name.substring(2).split("\\.")) { + sequence.appendCodePoint(Integer.parseInt(scalar, 16)); + } + return new NamedCharacterExpansion( + sequence.toString(), SourceMode.UNICODE, + true, Status.RESOLVED, null); + } catch (IllegalArgumentException failure) { + // Fall through to Perl's common malformed-U+ diagnostic. + } + } return new NamedCharacterExpansion( "", SourceMode.UNICODE, true, Status.INVALID, "Invalid hexadecimal number in \\N{U+...}"); diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index 4ea903c22..27fdd3c19 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -17,6 +17,7 @@ import org.joni.Syntax; import org.joni.WarnCallback; import org.joni.WideScalarCodec; +import org.joni.exception.SyntaxException; import static org.joni.constants.SyntaxProperties.ALLOW_MULTIPLEX_DEFINITION_NAME_CALL; import static org.joni.constants.SyntaxProperties.OP2_ESC_H_HORIZONTAL_WHITESPACE; @@ -249,7 +250,9 @@ public boolean supportsPositions() { Syntax syntax = syntaxForNamedCharacters( namedCharacterCache, namedCharacterSourceMode, flags.isCaseInsensitive()); - regex = new Regex(bytes, 0, bytes.length, toJoniOptions(flags, forceAsciiClasses), + int options = toJoniOptions(flags, forceAsciiClasses); + if (byteMode && byteBackedPattern) options |= Option.PERL_BYTE_PATTERN; + regex = new Regex(bytes, 0, bytes.length, options, byteMode ? ISO8859_1Encoding.INSTANCE : UTF8Encoding.INSTANCE, syntax, warningCollector); NamedGroupMaps groupMaps = collectNamedGroups(regex); @@ -1063,7 +1066,7 @@ private boolean find(int option, boolean anchored) { matcher = regex.matcher(bytes); if (!callbacks.isEmpty()) { calloutHandler = new PerlCalloutHandler( - input, byteToChar, callbacks, flags, hasControlVerbState, subject); + input, byteToChar, callbacks, flags, hasControlVerbState, byteMode, subject); matcher.setCalloutHandler(calloutHandler); } int result; @@ -1281,6 +1284,7 @@ static CaptureSnapshot of(MatchView match) { private final List callbacks; private final RegexFlags outerFlags; private final boolean publishesControlVerbState; + private final boolean byteMode; private final RuntimeScalar subject; private final int initialLocalLevel; private final RegexState initialRegexState; @@ -1295,20 +1299,21 @@ static CaptureSnapshot of(MatchView match) { PerlCalloutHandler(String input, int[] byteToChar, List callbacks, RegexFlags outerFlags, boolean publishesControlVerbState, - RuntimeScalar subject) { + boolean byteMode, RuntimeScalar subject) { this(input, byteToChar, callbacks, outerFlags, publishesControlVerbState, - subject, null); + byteMode, subject, null); } private PerlCalloutHandler( String input, int[] byteToChar, List callbacks, RegexFlags outerFlags, boolean publishesControlVerbState, - RuntimeScalar subject, PerlCalloutHandler parent) { + boolean byteMode, RuntimeScalar subject, PerlCalloutHandler parent) { this.input = input; this.byteToChar = byteToChar; this.callbacks = callbacks; this.outerFlags = outerFlags; this.publishesControlVerbState = publishesControlVerbState; + this.byteMode = byteMode; this.subject = subject; this.parent = parent; this.nestedDepth = parent == null ? 0 : parent.nestedDepth + 1; @@ -1352,7 +1357,8 @@ public DynamicPatternResult executeDynamic(int id, MatchView match) { nestedCallbacks = runtimeRegex.executableCallbacks; } else if (value.value instanceof RuntimeRegexTemplate template) { nestedPattern = new JoniRegexPattern(template.pattern(), outerFlags, - template.callbacks().size()); + template.callbacks().size(), false, + byteMode && template.byteBackedPattern(), template.byteBackedPattern()); nestedCallbacks = template.callbacks(); } else { String dynamicSource = value.toString(); @@ -1371,7 +1377,18 @@ public DynamicPatternResult executeDynamic(int id, MatchView match) { runtimeRegex.executableCallbacks.size()); nestedCallbacks = runtimeRegex.executableCallbacks; } else { - nestedPattern = new JoniRegexPattern(dynamicSource, outerFlags); + try { + nestedPattern = new JoniRegexPattern(dynamicSource, outerFlags); + } catch (SyntaxException exception) { + String message = exception.getMessage(); + if (message != null && (message.contains("premature end of char-class") + || message.contains("Unclosed character class"))) { + int open = dynamicSource.indexOf('['); + throw new PerlCompilerException(RegexDiagnosticFormatter.markedPerl( + dynamicSource, open < 0 ? 0 : open + 1, "Unmatched [")); + } + throw exception; + } } } CalloutHandler nestedHandler = nestedCallbacks.isEmpty() ? null @@ -1380,6 +1397,7 @@ public DynamicPatternResult executeDynamic(int id, MatchView match) { && runtimeRegex.getRegexFlags() != null ? runtimeRegex.getRegexFlags() : outerFlags, nestedPattern.hasControlVerbState, + byteMode, subject, this); if (nestedHandler != null) executedNestedCallbackPattern = true; diff --git a/src/main/java/org/perlonjava/runtime/regex/RegexBackendPolicy.java b/src/main/java/org/perlonjava/runtime/regex/RegexBackendPolicy.java index fd4121b5d..d341c3c42 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RegexBackendPolicy.java +++ b/src/main/java/org/perlonjava/runtime/regex/RegexBackendPolicy.java @@ -1,12 +1,11 @@ package org.perlonjava.runtime.regex; /** - * Temporary migration policy for comparing the legacy Java-first routing with - * the canonical Joni matcher. Ordinary lookbehind also remains on Java until - * Joni's nested-lookahead admission is complete. Branch-reset subroutine calls - * also temporarily use Java pending Joni's native named-call patch. Other - * Joni-only constructs in the same pattern still force Joni. This class and its - * controls are removed when the Java matching backend is retired. + * Temporary migration policy for comparing the canonical Joni matcher with + * the legacy Java matcher. Default and auto modes use Joni; explicit Java mode + * remains only for differential diagnosis. Constructs unavailable in Java may + * still force Joni even in explicit Java mode. This class and its controls are + * removed when the Java matching backend is retired. */ final class RegexBackendPolicy { static final String PROPERTY = "jperl.regex.backend"; @@ -27,14 +26,14 @@ static Mode current() { } if (configured == null || configured.isBlank() || configured.equalsIgnoreCase("auto") - || configured.equalsIgnoreCase("java")) { - return Mode.JAVA; - } - if (configured.equalsIgnoreCase("joni")) { + || configured.equalsIgnoreCase("joni")) { return Mode.JONI; } + if (configured.equalsIgnoreCase("java")) { + return Mode.JAVA; + } throw new IllegalArgumentException("Invalid " + ENVIRONMENT + " value '" - + configured + "' (expected java or joni)"); + + configured + "' (expected auto, java, or joni)"); } static boolean useJoni(String pattern) { diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java index 62308c93e..d3c965cba 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java @@ -273,6 +273,10 @@ public RegexFlags getRegexFlags() { return regexFlags; } + boolean isPatternByteBacked() { + return patternByteBacked; + } + private void setExecutableCallbacks(List callbacks) { List retained = List.copyOf(callbacks); for (RuntimeRegexCallback callback : retained) callback.retainOwner(); @@ -315,8 +319,11 @@ public RegexMatcher matcher(RuntimeScalar string, String input) { } private JoniRegexPattern selectRecursivePattern(RuntimeScalar string) { - if (bytesSubstitution && recursivePatternBytes != null - && string.type == RuntimeScalarType.BYTE_STRING) { + boolean byteDefaultSemantics = patternByteBacked && regexFlags != null + && regexFlags.isCaseInsensitive() && !regexFlags.isUnicode() + && !regexFlags.isAscii(); + if ((bytesSubstitution || byteDefaultSemantics) && recursivePatternBytes != null + && !Utf8.isUtf8(string)) { return recursivePatternBytes; } if (recursivePatternUnicode != null && recursivePatternUnicode != recursivePattern @@ -499,11 +506,16 @@ public static RuntimeRegex compile(String patternString, String modifiers) { } private static RuntimeRegex compile(String patternString, String modifiers, int lexicalDebugMode) { - return compile(patternString, modifiers, lexicalDebugMode, 0); + return compile(patternString, modifiers, lexicalDebugMode, 0, false); } private static RuntimeRegex compile(String patternString, String modifiers, int lexicalDebugMode, int trustedCalloutCount) { + return compile(patternString, modifiers, lexicalDebugMode, trustedCalloutCount, false); + } + + private static RuntimeRegex compile(String patternString, String modifiers, int lexicalDebugMode, + int trustedCalloutCount, boolean patternByteBacked) { RuntimeScalar namedCharacterTranslator = org.perlonjava.runtime.HintHashRegistry.getCompileTimeHint("charnames"); modifiers = stripDebugMarkers(modifiers); @@ -516,7 +528,7 @@ private static RuntimeRegex compile(String patternString, String modifiers, int UnicodeResolver.preloadUserDefinedProperties( patternString, preloadFlags.isCaseInsensitive()); return compileSynchronized(patternString, modifiers, lexicalDebugMode, - trustedCalloutCount, false, namedCharacterTranslator); + trustedCalloutCount, false, patternByteBacked, namedCharacterTranslator); } /** User properties execute Perl code and therefore cannot be validated while compiling a CV. */ @@ -538,14 +550,14 @@ public static boolean requiresRuntimeUnicodePropertyResolution(String patternStr public static void validateLiteralSyntax(String patternString, String modifiers) { try { compileSynchronized(patternString, stripDebugMarkers(modifiers), - debugMode(modifiers), 0, true, + debugMode(modifiers), 0, true, false, org.perlonjava.runtime.HintHashRegistry.getCompileTimeHint("charnames")); } catch (PerlJavaUnimplementedException unsupported) { String message = unsupported.getMessage(); if (message != null && (message.contains("premature end of char-class") || message.contains("Unclosed character class"))) { - throw new PerlCompilerException("Unmatched [ in regex m/" - + patternString + "/"); + throw new PerlCompilerException( + unmatchedCharacterClassDiagnostic(patternString) + "\n"); } if (message != null && (message.contains("Unclosed group") || message.contains("Dangling meta character") @@ -559,6 +571,7 @@ public static void validateLiteralSyntax(String patternString, String modifiers) private static synchronized RuntimeRegex compileSynchronized( String patternString, String modifiers, int lexicalDebugMode, int trustedCalloutCount, boolean literalSyntaxValidation, + boolean patternByteBacked, RuntimeScalar namedCharacterTranslator) { // Debug logging if (DEBUG_REGEX) { @@ -604,10 +617,13 @@ private static synchronized RuntimeRegex compileSynchronized( .usesCustomTranslator(namedCharacterTranslator) && patternString != null && patternString.contains("\\N{"); + boolean effectivePatternByteBacked = patternByteBacked + && !hasUnicodePromotingPatternSyntax(patternString); String cacheKey = patternString + "/" + modifiers + "#debug=" + lexicalDebugMode + "#callouts=" + trustedCalloutCount + "#backend=" + RegexBackendPolicy.cacheTag() + + "#bytepattern=" + effectivePatternByteBacked + (namedCharacterTranslator == null ? "" : "#charnames=" + namedCharacterTranslator.toString()) + (hasDynamicPattern ? (warnOnUnimplemented ? "\0warn" : "\0defer") : ""); @@ -620,6 +636,7 @@ private static synchronized RuntimeRegex compileSynchronized( System.err.println(" cache miss, compiling new regex"); } regex = new RuntimeRegex(); + regex.patternByteBacked = effectivePatternByteBacked; regex.namedCharacterCache = new JoniRegexPattern.NamedCharacterCache(namedCharacterTranslator); regex.lexicalDebugMode = lexicalDebugMode; @@ -687,6 +704,12 @@ private static synchronized RuntimeRegex compileSynchronized( : new JoniRegexPattern(compilePatternString, regex.regexFlags, trustedCalloutCount, false, false, false, regex.namedCharacterCache); + if (effectivePatternByteBacked && regex.regexFlags.isCaseInsensitive() + && !regex.regexFlags.isUnicode() && !regex.regexFlags.isAscii()) { + regex.recursivePatternBytes = new JoniRegexPattern( + compilePatternString, regex.regexFlags, trustedCalloutCount, + true, true, true, regex.namedCharacterCache); + } regex.deferredUserDefinedUnicodeProperties = regex.recursivePattern.hasDeferredUserDefinedUnicodeProperty() || regex.recursivePatternUnicode @@ -791,16 +814,35 @@ && containsExecutableSource(originalPatternString, if ("Empty \\N{}".equals(message)) { throw new PerlCompilerException("Unknown charname ''"); } - if (literalSyntaxValidation && message != null + if (message != null && (message.contains("premature end of char-class") || message.contains("Unclosed character class"))) { - throw new PerlCompilerException("Unmatched [ in regex m/" - + originalPatternString + "/"); + String diagnostic = unmatchedCharacterClassDiagnostic(originalPatternString); + if (literalSyntaxValidation) { + diagnostic += "\n"; + } + throw new PerlCompilerException(diagnostic); } int bytePosition = ((SyntaxException) e).getPatternPosition(); if (bytePosition != SyntaxException.UNKNOWN_PATTERN_POSITION) { int characterPosition = utf8ByteOffsetToCharacterOffset( compilePatternString, bytePosition); + if ("undefined group option".equals(message) + && originalPatternString != null + && characterPosition >= 3 + && originalPatternString.regionMatches( + characterPosition - 3, "(?\\", 0, 3)) { + message = "Sequence (?\\...) not recognized"; + } else if ("too big number for repeat range".equals(message)) { + message = "Quantifier in {,} bigger than 2147483646"; + } else if ("end pattern with unmatched parenthesis".equals(message)) { + int unmatched = ordinaryUnmatchedOpeningParenthesis( + originalPatternString); + if (unmatched >= 0) { + message = "Unmatched ("; + characterPosition = unmatched + 1; + } + } throw new PerlCompilerException(RegexDiagnosticFormatter.markedPerl( originalPatternString, characterPosition, message)); } @@ -865,6 +907,12 @@ && containsExecutableSource(originalPatternString, return regex; } + private static String unmatchedCharacterClassDiagnostic(String pattern) { + int open = pattern == null ? -1 : pattern.indexOf('['); + return RegexDiagnosticFormatter.markedPerl( + pattern, open < 0 ? 0 : open + 1, "Unmatched ["); + } + private static String invalidUnicodePropertyName(String message) { if (message == null) return null; String prefix = "invalid character property name <"; @@ -882,6 +930,46 @@ private static int utf8ByteOffsetToCharacterOffset(String pattern, int byteOffse return new String(bytes, 0, boundedOffset, StandardCharsets.UTF_8).length(); } + /** Locate an unmatched ordinary group opener, excluding Perl's specialized {@code (?...} forms. */ + private static int ordinaryUnmatchedOpeningParenthesis(String pattern) { + if (pattern == null) return -1; + Deque openings = new ArrayDeque<>(); + boolean escaped = false; + boolean inClass = false; + for (int i = 0; i < pattern.length(); i++) { + char ch = pattern.charAt(i); + if (escaped) { + escaped = false; + continue; + } + if (ch == '\\') { + escaped = true; + continue; + } + if (ch == '[') { + inClass = true; + continue; + } + if (ch == ']' && inClass) { + inClass = false; + continue; + } + if (inClass) continue; + if (ch == '(') { + openings.push(i); + } else if (ch == ')' && !openings.isEmpty()) { + openings.pop(); + } + } + while (!openings.isEmpty()) { + int opening = openings.removeLast(); + if (opening + 1 >= pattern.length() || pattern.charAt(opening + 1) != '?') { + return opening; + } + } + return -1; + } + private static int debugMode(String modifiers) { if (modifiers == null) return 0; if (modifiers.indexOf(INTERNAL_DEBUGCOLOR_MARKER) >= 0) return 2; @@ -966,7 +1054,8 @@ private static RuntimeRegex ensureCompiledForRuntime(RuntimeRegex regex) { // in sync with compileSynchronized(): leaving the placeholder cached // makes a later qr/\\p{Property}/ reuse its match-any stand-in. state().compiledRegexCache.remove(cacheKey + "#debug=" + regex.lexicalDebugMode - + "#callouts=0#backend=" + RegexBackendPolicy.cacheTag()); + + "#callouts=0#backend=" + RegexBackendPolicy.cacheTag() + + "#bytepattern=" + regex.patternByteBacked); // User property subs can execute arbitrary Perl and block. Resolve them // before compile() takes its process-wide monitor; only simultaneous @@ -975,11 +1064,12 @@ private static RuntimeRegex ensureCompiledForRuntime(RuntimeRegex regex) { regex.regexFlags != null && regex.regexFlags.isCaseInsensitive()); RuntimeRegex recompiled = compile(regex.patternString, regex.regexFlags == null ? "" : regex.regexFlags.toFlagString(), - regex.lexicalDebugMode); + regex.lexicalDebugMode, 0, regex.patternByteBacked); regex.pattern = recompiled.pattern; regex.patternUnicode = recompiled.patternUnicode; regex.recursivePattern = recompiled.recursivePattern; regex.recursivePatternUnicode = recompiled.recursivePatternUnicode; + regex.recursivePatternBytes = recompiled.recursivePatternBytes; regex.patternNoInternalMarkers = recompiled.patternNoInternalMarkers; regex.patternUnicodeNoInternalMarkers = recompiled.patternUnicodeNoInternalMarkers; regex.patternFlags = recompiled.patternFlags; @@ -1396,7 +1486,7 @@ && containsExecutableSource(patternString.toString(), modifierStr.indexOf('x') > if (patternString.value instanceof RuntimeRegexTemplate template) { RuntimeRegex regex = compile(template.pattern(), modifierStr, callSiteDebugMode, - template.callbacks().size()).cloneTracked(); + template.callbacks().size(), template.byteBackedPattern()).cloneTracked(); regex.setExecutableCallbacks(template.callbacks()); return new RuntimeScalar(regex).propagateTaint(patternString); } @@ -1416,6 +1506,7 @@ && containsExecutableSource(patternString.toString(), modifierStr.indexOf('x') > regex.patternUnicode = originalRegex.patternUnicode; regex.recursivePattern = originalRegex.recursivePattern; regex.recursivePatternUnicode = originalRegex.recursivePatternUnicode; + regex.recursivePatternBytes = originalRegex.recursivePatternBytes; regex.setExecutableCallbacks(originalRegex.executableCallbacks); regex.patternNoInternalMarkers = originalRegex.patternNoInternalMarkers; regex.patternUnicodeNoInternalMarkers = originalRegex.patternUnicodeNoInternalMarkers; @@ -1459,6 +1550,7 @@ && containsExecutableSource(patternString.toString(), modifierStr.indexOf('x') > regex.patternUnicode = originalRegex.patternUnicode; regex.recursivePattern = originalRegex.recursivePattern; regex.recursivePatternUnicode = originalRegex.recursivePatternUnicode; + regex.recursivePatternBytes = originalRegex.recursivePatternBytes; regex.setExecutableCallbacks(originalRegex.executableCallbacks); regex.patternNoInternalMarkers = originalRegex.patternNoInternalMarkers; regex.patternUnicodeNoInternalMarkers = originalRegex.patternUnicodeNoInternalMarkers; @@ -1496,8 +1588,8 @@ && containsExecutableSource(patternString.toString(), modifierStr.indexOf('x') > // Default: compile as string (cloneTracked() creates a tracked copy // so the cached RuntimeRegex is not corrupted by refCount changes) RuntimeRegex compiled = compile(patternString.toString(), modifierStr, - callSiteDebugMode).cloneTracked(); - compiled.patternByteBacked = patternString.type == RuntimeScalarType.BYTE_STRING; + callSiteDebugMode, 0, + patternString.type == RuntimeScalarType.BYTE_STRING).cloneTracked(); return new RuntimeScalar(compiled).propagateTaint(patternString); } @@ -1565,10 +1657,11 @@ static boolean containsExecutableSource(String pattern, boolean extended) { static RuntimeScalar compileExecutableTemplate( String executablePattern, String modifiers, - List callbacks, RuntimeScalar original) { + List callbacks, RuntimeScalar original, + boolean patternByteBacked) { int lexicalDebugMode = debugMode(modifiers); RuntimeRegex regex = compile(executablePattern, stripDebugMarkers(modifiers), - lexicalDebugMode, callbacks.size()).cloneTracked(); + lexicalDebugMode, callbacks.size(), patternByteBacked).cloneTracked(); regex.setExecutableCallbacks(callbacks); return new RuntimeScalar(regex).propagateTaint(original); } @@ -1677,6 +1770,7 @@ public static RuntimeScalar getReplacementRegex(RuntimeScalar patternString, Run regex.patternUnicode = resolvedRegex.patternUnicode; regex.recursivePattern = resolvedRegex.recursivePattern; regex.recursivePatternUnicode = resolvedRegex.recursivePatternUnicode; + regex.recursivePatternBytes = resolvedRegex.recursivePatternBytes; regex.executableCallbacks = resolvedRegex.executableCallbacks; regex.patternNoInternalMarkers = resolvedRegex.patternNoInternalMarkers; regex.patternUnicodeNoInternalMarkers = resolvedRegex.patternUnicodeNoInternalMarkers; @@ -1713,11 +1807,13 @@ public static RuntimeScalar getReplacementRegex(RuntimeScalar patternString, Run if (flagsChanged && !resolved.fromCompiledRegex()) { RuntimeRegex recompiledRegex = compile(resolvedRegex.patternString, newFlags.toFlagString(), regex.lexicalDebugMode, - resolvedRegex.executableCallbacks.size()); + resolvedRegex.executableCallbacks.size(), + resolvedRegex.patternByteBacked); regex.pattern = recompiledRegex.pattern; regex.patternUnicode = recompiledRegex.patternUnicode; regex.recursivePattern = recompiledRegex.recursivePattern; regex.recursivePatternUnicode = recompiledRegex.recursivePatternUnicode; + regex.recursivePatternBytes = recompiledRegex.recursivePatternBytes; regex.executableCallbacks = resolvedRegex.executableCallbacks; regex.patternNoInternalMarkers = recompiledRegex.patternNoInternalMarkers; regex.patternUnicodeNoInternalMarkers = recompiledRegex.patternUnicodeNoInternalMarkers; @@ -1790,6 +1886,54 @@ private static boolean containsNonAscii(String value) { return false; } + /** Perl named characters and wide numeric escapes make an ASCII source Unicode. */ + private static boolean hasUnicodePromotingPatternSyntax(String pattern) { + if (pattern == null) return false; + boolean quoted = false; + for (int i = 0; i < pattern.length(); i++) { + char ch = pattern.charAt(i); + if (ch > 0xff) return true; + if (ch != '\\' || i + 1 >= pattern.length()) continue; + + char escape = pattern.charAt(++i); + if (escape == 'Q' && !quoted) { + quoted = true; + continue; + } + if (escape == 'E' && quoted) { + quoted = false; + continue; + } + if (quoted || escape == '\\') continue; + if (escape == 'N' && i + 1 < pattern.length() + && pattern.charAt(i + 1) == '{') { + return true; + } + if ((escape != 'x' && escape != 'o') || i + 1 >= pattern.length() + || pattern.charAt(i + 1) != '{') { + continue; + } + + int radix = escape == 'x' ? 16 : 8; + boolean sawDigit = false; + long value = 0; + int cursor = i + 2; + for (; cursor < pattern.length() && pattern.charAt(cursor) != '}'; cursor++) { + char digitCharacter = pattern.charAt(cursor); + if (digitCharacter == '_') continue; + int digit = Character.digit(digitCharacter, radix); + if (digit < 0) break; + sawDigit = true; + value = value * radix + digit; + if (value > 0xff) return true; + } + if (sawDigit && cursor < pattern.length() && pattern.charAt(cursor) == '}') { + i = cursor; + } + } + return false; + } + /** * Applies a Perl "qr" object on a string; returns true/false or a list, * and produces side-effects. diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegexSourceCompiler.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegexSourceCompiler.java index 15ccb8e9a..50b3b0851 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegexSourceCompiler.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegexSourceCompiler.java @@ -143,7 +143,8 @@ static RuntimeScalar compileTemplate(RuntimeScalar original, RuntimeRegexTemplate template, String modifiers) { RuntimeRegexTemplate.MaskedCallouts masked = template.maskCallouts(); - RuntimeScalar compiled = compile(new RuntimeScalar(masked.pattern()), modifiers); + RuntimeScalar compiled = compile(RuntimeRegexTemplate.patternScalar( + masked.pattern(), template.byteBackedPattern()), modifiers); if (!(compiled.value instanceof RuntimeRegex sourceRegex)) { throw new IllegalStateException("runtime regex source did not compile to qr//"); } @@ -156,7 +157,7 @@ static RuntimeScalar compileTemplate(RuntimeScalar original, List callbacks = new ArrayList<>(template.callbacks()); callbacks.addAll(sourceRegex.executableCallbacks); RuntimeScalar result = RuntimeRegex.compileExecutableTemplate( - executablePattern, modifiers, callbacks, original); + executablePattern, modifiers, callbacks, original, template.byteBackedPattern()); if (result.value != sourceRegex) { sourceRegex.releaseExecutableCallbacks(); } diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegexTemplate.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegexTemplate.java index 8d0e60884..3b5c6978a 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegexTemplate.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegexTemplate.java @@ -9,7 +9,9 @@ import org.perlonjava.runtime.runtimetypes.RuntimeList; import org.perlonjava.runtime.runtimetypes.RuntimeScalar; import org.perlonjava.runtime.runtimetypes.RuntimeScalarType; +import org.perlonjava.runtime.perlmodule.Utf8; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; @@ -23,6 +25,7 @@ public final class RuntimeRegexTemplate { "\\(\\?\\{=(CALL|DYNAMIC):(\\d+)\\}\\)"); private final String pattern; private final List callbacks; + private final boolean byteBackedPattern; /** * Deferred array interpolation. Keeping the joined operands separate until @@ -52,9 +55,11 @@ String restore(String compiledPattern) { } } - private RuntimeRegexTemplate(String pattern, List callbacks) { + private RuntimeRegexTemplate(String pattern, List callbacks, + boolean byteBackedPattern) { this.pattern = pattern; this.callbacks = List.copyOf(callbacks); + this.byteBackedPattern = byteBackedPattern; } public static RuntimeScalar build(RuntimeList parts) { @@ -85,9 +90,11 @@ public static RuntimeScalar build(RuntimeList parts) { StringBuilder pattern = new StringBuilder(); List callbacks = new ArrayList<>(); boolean tainted = false; + boolean byteBackedPattern = true; for (RuntimeBase part : parts.elements) { RuntimeScalar scalar = part.scalar(); tainted |= scalar.isTainted(); + byteBackedPattern &= isByteCompatiblePatternPart(scalar); if (scalar.value instanceof RuntimeRegexCallback callback) { int id = callbacks.size(); callbacks.add(callback); @@ -108,12 +115,26 @@ public static RuntimeScalar build(RuntimeList parts) { } } RuntimeScalar result = callbacks.isEmpty() - ? new RuntimeScalar(pattern.toString()) - : new RuntimeScalar(new RuntimeRegexTemplate(pattern.toString(), callbacks)); + ? patternScalar(pattern.toString(), byteBackedPattern) + : new RuntimeScalar(new RuntimeRegexTemplate( + pattern.toString(), callbacks, byteBackedPattern)); result.tainted = tainted; return result; } + private static boolean isByteCompatiblePatternPart(RuntimeScalar scalar) { + if (scalar.value instanceof RuntimeRegexCallback) return true; + if (scalar.value instanceof RuntimeRegex regex) return regex.isPatternByteBacked(); + if (scalar.value instanceof RuntimeRegexTemplate template) return template.byteBackedPattern; + return !Utf8.isUtf8(scalar); + } + + static RuntimeScalar patternScalar(String pattern, boolean byteBackedPattern) { + return byteBackedPattern + ? new RuntimeScalar(pattern.getBytes(StandardCharsets.ISO_8859_1)) + : new RuntimeScalar(pattern); + } + private static RuntimeScalar resolveLoneRegexOverload(RuntimeScalar scalar) { int blessId = RuntimeScalarType.blessedId(scalar); if (blessId >= 0) return null; @@ -285,6 +306,10 @@ List callbacks() { return callbacks; } + boolean byteBackedPattern() { + return byteBackedPattern; + } + /** * Hide parser-created callout markers while runtime-interpolated Perl * source is compiled. The source parser must see raw {@code (?{...})} and diff --git a/src/test/java/org/perlonjava/runtime/regex/RegexBackendPolicyTest.java b/src/test/java/org/perlonjava/runtime/regex/RegexBackendPolicyTest.java index f85636638..f8a03c6ed 100644 --- a/src/test/java/org/perlonjava/runtime/regex/RegexBackendPolicyTest.java +++ b/src/test/java/org/perlonjava/runtime/regex/RegexBackendPolicyTest.java @@ -29,20 +29,20 @@ void restoreBackendProperty() { } @Test - void defaultModeRoutesOrdinaryLookbehindToJoni() { - assertFalse(RegexBackendPolicy.useJoni("ordinary")); + void defaultModeUsesJoniForOrdinaryPatterns() { + assertTrue(RegexBackendPolicy.useJoni("ordinary")); assertTrue(RegexBackendPolicy.useJoni("(?<=x)y")); assertTrue(RegexBackendPolicy.useJoni("(?1)|(?2))(?&digit)")); diff --git a/src/test/java/org/perlonjava/runtime/regex/RuntimeRegexTemplateProvenanceTest.java b/src/test/java/org/perlonjava/runtime/regex/RuntimeRegexTemplateProvenanceTest.java new file mode 100644 index 000000000..fce01c6e6 --- /dev/null +++ b/src/test/java/org/perlonjava/runtime/regex/RuntimeRegexTemplateProvenanceTest.java @@ -0,0 +1,34 @@ +package org.perlonjava.runtime.regex; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.perlonjava.runtime.runtimetypes.RuntimeList; +import org.perlonjava.runtime.runtimetypes.RuntimeScalar; +import org.perlonjava.runtime.runtimetypes.RuntimeScalarType; + +@Tag("unit") +class RuntimeRegexTemplateProvenanceTest { + @Test + public void retainsByteProvenanceAcrossByteCompatibleParts() { + RuntimeScalar result = RuntimeRegexTemplate.build(new RuntimeList( + new RuntimeScalar(new byte[] {'^'}), + new RuntimeScalar(new byte[] {(byte) 0xdf}), + new RuntimeScalar(new byte[] {'$'}))); + + assertEquals(RuntimeScalarType.BYTE_STRING, result.type); + assertEquals("^\u00df$", result.toString()); + } + + @Test + public void upgradesWhenAnyPartIsACharacterString() { + RuntimeScalar result = RuntimeRegexTemplate.build(new RuntimeList( + new RuntimeScalar(new byte[] {'^'}), + new RuntimeScalar("\u00df"), + new RuntimeScalar(new byte[] {'$'}))); + + assertEquals(RuntimeScalarType.STRING, result.type); + assertEquals("^\u00df$", result.toString()); + } +} diff --git a/src/test/resources/unit/regex/casefold_generated_matrix.t b/src/test/resources/unit/regex/casefold_generated_matrix.t new file mode 100644 index 000000000..bf7038b63 --- /dev/null +++ b/src/test/resources/unit/regex/casefold_generated_matrix.t @@ -0,0 +1,54 @@ +use strict; +use warnings; +use utf8; +use Test::More; + +my $byte_sharp = chr 0xDF; +utf8::downgrade($byte_sharp, 1); +my $byte_pair = $byte_sharp . $byte_sharp; +utf8::downgrade($byte_pair, 1); +my $byte_a_umlaut = chr 0xE4; +my $byte_A_umlaut = chr 0xC4; +utf8::downgrade($byte_a_umlaut, 1); +utf8::downgrade($byte_A_umlaut, 1); +my $unicode_sharp = chr 0xDF; +utf8::upgrade($unicode_sharp); + +unlike('ss', qr/^$byte_sharp$/di, + 'byte /d literal rejects sharp-s full fold'); +unlike($byte_sharp, qr/^ss$/di, + 'byte /d reverse literal rejects sharp-s full fold'); +unlike($byte_A_umlaut, qr/^$byte_a_umlaut$/di, + 'byte /d rejects Latin-1 character folding'); +like($byte_pair, qr/^($byte_sharp)\1$/di, + 'byte /d numbered backreference retains byte capture'); +like($byte_pair, qr/^(?$byte_sharp)\k$/di, + 'byte /d named backreference retains byte capture'); + +like('ss', qr/^$unicode_sharp$/di, + 'character /d literal retains sharp-s full fold'); +like($unicode_sharp, qr/^ss$/di, + 'character /d reverse literal retains sharp-s full fold'); +like('ss', qr/^\x{DF}$/ui, + '/u retains sharp-s full fold'); +like('ss', qr/^\x{DF}$/ai, + '/a retains sharp-s full fold'); +unlike('ss', qr/^\x{DF}$/aai, + '/aa rejects sharp-s ASCII crossing'); + +like('ss', qr/^(?i:$unicode_sharp)$/, + 'scoped /i enables full fold'); +unlike('ss', qr/^(?aa-i:\x{DF})$/i, + 'scoped minus i disables outer folding'); +unlike('ss', qr/^(?aa:\x{DF})$/i, + 'scoped /aa blocks outer full fold'); + +my $kelvin = "\N{KELVIN SIGN}"; +like($kelvin, qr/^\p{Lowercase}$/i, + 'ignore-case property closure includes Kelvin sign'); +like($kelvin, qr/^\p{Lowercase}$/aai, + 'ascii-strict property closure retains Unicode property membership'); +like('A', qr/^[\p{Lowercase}]$/i, + 'ignore-case class property closure includes uppercase sibling'); + +done_testing; diff --git a/src/test/resources/unit/regex/casefold_literal_backreference.t b/src/test/resources/unit/regex/casefold_literal_backreference.t new file mode 100644 index 000000000..5a09e205b --- /dev/null +++ b/src/test/resources/unit/regex/casefold_literal_backreference.t @@ -0,0 +1,33 @@ +use strict; +use warnings; +use utf8; +use Test::More; + +like('ss', qr/^\x{00DF}$/iu, 'sharp-s forward full fold'); +like("\x{00DF}", qr/^ss$/iu, 'sharp-s reverse full fold'); +like("\x{FB03}", qr/^ffi$/i, 'ligature reverse full fold'); +like("\x{017F}\x{017F}", qr/^\x{00DF}$/i, 'long-s components fold to sharp s'); +like("\x{212A}", qr/^k$/i, 'Kelvin folds with ASCII k'); +unlike('I', qr/^\x{0131}$/i, 'Turkic dotless I remains excluded'); + +like("\x{1E9E}\x{1E9E}", qr/^(\x{00DF})\1$/i, + 'backreference consumes folded non-ASCII siblings'); +unlike('ssss', qr/^(\x{00DF})\1$/iaa, + 'aa prevents ASCII-crossing backreference folds'); +like("\x{1E9E}\x{1E9E}", qr/^(\x{00DF})\1$/iaa, + 'aa retains non-ASCII sibling backreference folds'); + +like('xssy', qr/^x(?i:\x{00DF})y$/u, 'scoped i enables fold locally'); +unlike('xssy', qr/^x(?-i:\x{00DF})y$/i, 'scoped minus i restores outer policy'); + +my $byte_sharp = chr 0xDF; +utf8::downgrade($byte_sharp, 1); +unlike('ss', qr/^$byte_sharp$/di, 'byte d literal blocks full fold'); +my $unicode_sharp = $byte_sharp; +utf8::upgrade($unicode_sharp); +like('ss', qr/^$unicode_sharp$/di, 'upgraded d literal enables full fold'); + +like('affi', qr/^aff\x{0069}$/i, 'positive adjacent reverse-fold boundary'); +unlike('affx', qr/^aff\x{0069}$/i, 'negative adjacent reverse-fold boundary'); + +done_testing; diff --git a/src/test/resources/unit/regex/casefold_provenance_modes.t b/src/test/resources/unit/regex/casefold_provenance_modes.t new file mode 100644 index 000000000..14a9c862a --- /dev/null +++ b/src/test/resources/unit/regex/casefold_provenance_modes.t @@ -0,0 +1,32 @@ +use strict; +use warnings; +use utf8; +use Test::More; + +my $sharp = chr 0xDF; +utf8::downgrade($sharp, 1); +my $ss = 'ss'; +utf8::downgrade($ss, 1); + +unlike($ss, qr/$sharp/di, 'd byte pattern and subject do not full-fold sharp s'); +like($ss, qr/$sharp/ui, 'u permits sharp-s full folding'); +like($ss, qr/$sharp/ai, 'a permits sharp-s full folding'); +unlike($ss, qr/$sharp/aai, 'aa rejects ASCII-crossing sharp-s folding'); + +my $upgraded_sharp = $sharp; +utf8::upgrade($upgraded_sharp); +like($ss, qr/$upgraded_sharp/di, 'd upgraded pattern permits sharp-s full folding'); +my $upgraded_ss = $ss; +utf8::upgrade($upgraded_ss); +like($upgraded_ss, qr/$sharp/di, 'd upgraded subject permits sharp-s full folding'); + +like('I', qr/i/i, 'ordinary I folds with ASCII i'); +unlike('I', qr/\x{0131}/i, 'ordinary folding excludes Turkic dotless i'); +unlike("\x{0130}", qr/i/i, 'ordinary folding excludes Turkic dotted I'); + +like("\x{1E9E}\x{1E9E}", qr/(\x{00DF})\1/iaa, + 'aa backreference preserves non-ASCII sharp-s siblings'); +unlike('ssss', qr/(\x{00DF})\1/iaa, + 'aa backreference rejects captured ASCII-crossing folds'); + +done_testing; diff --git a/src/test/resources/unit/regex/dynamic_contract_edges.t b/src/test/resources/unit/regex/dynamic_contract_edges.t new file mode 100644 index 000000000..59b01c57f --- /dev/null +++ b/src/test/resources/unit/regex/dynamic_contract_edges.t @@ -0,0 +1,46 @@ +use strict; +use warnings; +use Test::More; + +my @warnings; +my $undef_matches; +{ + local $SIG{__WARN__} = sub { push @warnings, @_ }; + $undef_matches = 'a' =~ /^a(??{ undef })$/; +} +ok($undef_matches, 'undef dynamic result is an empty pattern'); +like(join('', @warnings), qr/^Use of uninitialized value/, 'undef warns at match time'); + +ok('a' =~ /^a(??{ '' })$/, 'empty dynamic result matches empty'); + +my $position; +ok('ab' =~ /^a(??{ $position = pos; 'b' })$/, 'dynamic result matches at current position'); +is($position, 1, 'pos is the current dynamic entry offset'); + +my $repeat_count = 0; +ok('aa' =~ /^(?:(??{ ++$repeat_count; 'a' })){2}$/, + 'dynamic result executes inside a quantifier'); +is($repeat_count, 2, 'dynamic expression executes for each quantifier entry'); + +my $choice_count = 0; +ok('ab' =~ /^(??{ ++$choice_count; 'ab|a' })b$/, + 'nested alternatives can backtrack before the outer suffix'); +is($choice_count, 1, 'nested alternative backtracking does not reevaluate expression'); + +ok('a' =~ /^(??{ 'A' })$/i, 'returned string inherits outer modifiers'); +my $strict_qr = qr/A/; +ok(!('a' =~ /^(??{ $strict_qr })$/i), 'returned qr retains its own modifiers'); + +our $seed; +ok('seed' =~ /^(?{ $seed = 'seed' })(??{ $^R })$/, + 'dynamic expression sees prior callback result'); + +my $error = eval { 'x' =~ /^(??{ die "dynamic boom\n" })$/; 1 }; +ok(!$error, 'exception aborts matching'); +is($@, "dynamic boom\n", 'dynamic exception propagates unchanged'); + +my $invalid = eval { 'x' =~ /^(??{ '[' })$/; 1 }; +ok(!$invalid, 'invalid returned pattern fails at match time'); +like($@, qr/Unmatched \[/, 'invalid returned pattern reports compile error'); + +done_testing; diff --git a/src/test/resources/unit/regex/ordinary_constant_joni_matrix.t b/src/test/resources/unit/regex/ordinary_constant_joni_matrix.t new file mode 100644 index 000000000..d23df1203 --- /dev/null +++ b/src/test/resources/unit/regex/ordinary_constant_joni_matrix.t @@ -0,0 +1,65 @@ +use strict; +use warnings; +use Test::More; + +ok('alphabet' =~ /pha/, 'literal'); +ok('green' =~ /red|green|blue/, 'alternation'); + +my $captured = 'name=perl'; +ok($captured =~ /name=(?\w+)/, 'numbered and named capture match'); +is($1, 'perl', 'numbered capture value'); +is($+{language}, 'perl', 'named capture value'); + +ok("A\x{3b1}9" =~ /[A-Z]\p{Greek}\d/, 'class and Unicode property'); +ok("first\nsecond" =~ /^second$/m, 'multiline anchors'); +ok("a\nlevel\nz" =~ /a.*z/s, 'dot-all modifier'); +ok('item42' =~ /item\d{2}/, 'bounded quantifier'); +ok('colour' =~ /colou?r/, 'optional quantifier'); + +ok('foobar' =~ /foo(?=bar)/, 'positive lookahead'); +ok('foobar' =~ /(?<=foo)bar/, 'fixed positive lookbehind'); +ok('foobaz' =~ /foo(?!bar)/, 'negative lookahead'); + +my $substitution = 'red green red'; +my $replacements = ($substitution =~ s/red/blue/g); +is($replacements, 2, 'global substitution count'); +is($substitution, 'blue green blue', 'global substitution result'); + +my $word = qr/[a-z]+/i; +ok('Perl' =~ $word, 'qr reuse'); +my $suffix = qr/\d+/; +ok('item42' =~ /item$suffix/, 'qr interpolation'); + +my @global = ('a1b22c333' =~ /(\d+)/g); +is_deeply(\@global, [qw(1 22 333)], 'global match list'); + +my $continued = '12ab'; +pos($continued) = 0; +ok($continued =~ /\d+/gc, '/gc first match'); +is(pos($continued), 2, '/gc advances pos'); +ok(!($continued =~ /\d+/gc), '/gc failed continuation'); +is(pos($continued), 2, '/c preserves pos after failure'); + +my $once_text = 'first'; +my $once_pattern = 'first'; +sub once_matches { + my ($candidate) = @_; + return $candidate =~ /$once_pattern/o; +} +ok(once_matches($once_text), '/o initial compilation'); +$once_pattern = 'second'; +ok(once_matches('first'), '/o reuses the initial pattern'); + +my $byte = pack('C', 0xe9); +utf8::downgrade($byte, 1); +my $unicode = "\x{e9}"; +utf8::upgrade($unicode); +ok($byte =~ /\x{e9}/, 'byte scalar match'); +ok($unicode =~ /\x{e9}/u, 'Unicode scalar match'); + +ok('MiXeD' =~ /mixed/i, 'case-insensitive modifier'); +ok('a b' =~ /a \s+ b/x, 'extended modifier'); +ok('ab' =~ /(?:a)(b)/n, 'non-capturing-default modifier'); +is($1, undef, '/n suppresses unnamed captures'); + +done_testing; diff --git a/src/test/resources/unit/regex/regex_escaped_group_option_diagnostics.t b/src/test/resources/unit/regex/regex_escaped_group_option_diagnostics.t new file mode 100644 index 000000000..a0baf703f --- /dev/null +++ b/src/test/resources/unit/regex/regex_escaped_group_option_diagnostics.t @@ -0,0 +1,18 @@ +use strict; +use warnings; +use Test::More; + +my @cases = ( + [q{(?\ix}, q{Sequence (?\...) not recognized in regex; marked by <-- HERE in m/(?\ <-- HERE ix/}], + [q{(?\:x}, q{Sequence (?\...) not recognized in regex; marked by <-- HERE in m/(?\ <-- HERE :x/}], + [q{(?\<=x}, q{Sequence (?\...) not recognized in regex; marked by <-- HERE in m/(?\ <-- HERE <=x/}], +); + +for my $case (@cases) { + my ($pattern, $expected) = @$case; + eval "#line 1 regex_escaped_group_option_diagnostics.t\nqr/$pattern/"; + my ($error) = split /\n/, $@; + like($error, qr/^\Q$expected\E at /, "escaped group option $pattern"); +} + +done_testing; diff --git a/src/test/resources/unit/regex/regex_oversized_quantifier_diagnostics.t b/src/test/resources/unit/regex/regex_oversized_quantifier_diagnostics.t new file mode 100644 index 000000000..7fbe17af8 --- /dev/null +++ b/src/test/resources/unit/regex/regex_oversized_quantifier_diagnostics.t @@ -0,0 +1,20 @@ +use strict; +use warnings; +use Test::More; + +my @cases = ( + [q{x{2147483648}}, q{x{2147483648 <-- HERE }}], + [q{x{2147483648,}}, q{x{2147483648 <-- HERE ,}}], + [q{x{2147483648,2147483649}}, q{x{2147483648 <-- HERE ,2147483649}}], +); + +for my $case (@cases) { + my ($pattern, $marked) = @$case; + eval "#line 1 regex_oversized_quantifier_diagnostics.t\nqr/$pattern/"; + my ($error) = split /\n/, $@; + like($error, + qr/^Quantifier in \{,\} bigger than \d+ in regex; marked by <-- HERE in m\/\Q$marked\E\/ at /, + "oversized quantifier $pattern"); +} + +done_testing; diff --git a/src/test/resources/unit/regex/regex_unmatched_open_paren_diagnostics.t b/src/test/resources/unit/regex/regex_unmatched_open_paren_diagnostics.t new file mode 100644 index 000000000..38e6314fc --- /dev/null +++ b/src/test/resources/unit/regex/regex_unmatched_open_paren_diagnostics.t @@ -0,0 +1,21 @@ +use strict; +use warnings; +use utf8; +use Test::More; + +my @cases = ( + [q{((x)}, q{Unmatched ( in regex; marked by <-- HERE in m/( <-- HERE (x)/}], + [q|{(}|, q|Unmatched ( in regex; marked by <-- HERE in m/{( <-- HERE }/|], + [q{ネ((ネ)}, q{Unmatched ( in regex; marked by <-- HERE in m/ネ( <-- HERE (ネ)/}], +); + +my $case_number = 0; +for my $case (@cases) { + $case_number++; + my ($pattern, $expected) = @$case; + eval "#line 1 regex_unmatched_open_paren_diagnostics.t\nqr/$pattern/"; + my ($error) = split /\n/, $@; + like($error, qr/^\Q$expected\E at /, "unmatched opening parenthesis case $case_number"); +} + +done_testing; diff --git a/src/test/resources/unit/regex/unicode_named_uplus_sequence_priority.t b/src/test/resources/unit/regex/unicode_named_uplus_sequence_priority.t new file mode 100644 index 000000000..14390f475 --- /dev/null +++ b/src/test/resources/unit/regex/unicode_named_uplus_sequence_priority.t @@ -0,0 +1,36 @@ +use strict; +use warnings; +use Test::More; + +my @warnings; +my $value; +{ + local $SIG{__WARN__} = sub { push @warnings, join '', @_ }; + $value = eval "#line 1 unicode_named_uplus_sequence_priority.t\n" + . q!"\N{U+41.42}"!; +} +my ($scalar_error) = split /\n/, $@; +is($scalar_error, + 'Invalid hexadecimal number in \N{U+...} at unicode_named_uplus_sequence_priority.t line 1, within string', + 'dotted U+ form remains invalid in a string'); +is(scalar @warnings, 0, 'invalid string form has no preceding warning'); + +my $regex = eval "#line 1 unicode_named_uplus_sequence_priority.t\n" + . q!qr/\N{U+41.42}/!; +is($@, '', 'dotted U+ sequence is legal in a regex'); +is("$regex", q!(?^:\N{U+41.42})!, 'qr stringification preserves dotted U+ source'); +ok('AB' =~ $regex, 'dotted U+ sequence matches its code points'); + +$regex = eval "#line 1 unicode_named_uplus_sequence_priority.t\n" + . q!qr/[\N{U+0.00}]/!; +is($@, '', 'dotted U+ sequence is legal in a closed class'); +is("$regex", q!(?^:[\N{U+0.00}])!, 'closed-class stringification preserves source'); + +eval "#line 1 unicode_named_uplus_sequence_priority.t\n" + . q!qr/0000000000000000[\N{U+0.00}0000/!; +my ($priority_error) = split /\n/, $@; +is($priority_error, + 'Unmatched [ in regex; marked by <-- HERE in m/0000000000000000[ <-- HERE \N{U+0.00}0000/ at unicode_named_uplus_sequence_priority.t line 1.', + 'unmatched class diagnostic precedes valid dotted U+ resolution'); + +done_testing; diff --git a/third_party/joni/src/org/joni/Analyser.java b/third_party/joni/src/org/joni/Analyser.java index f7768b88f..924e835c3 100644 --- a/third_party/joni/src/org/joni/Analyser.java +++ b/third_party/joni/src/org/joni/Analyser.java @@ -2145,6 +2145,51 @@ private Node expandCaseFoldString(Node node) { return xnode; } + private Node expandPerlByteAsciiFoldString(StringNode source, int state) { + ListNode root = null; + ListNode sequenceTail = null; + boolean expanded = false; + + for (int p = source.p; p < source.end;) { + int next = p + enc.length(source.bytes, p, source.end); + int codePoint = enc.mbcToCode(source.bytes, p, source.end); + StringNode exact = new StringNode(source.bytes, p, next); + exact.setRaw(); + ListNode alternatives = newAlt(exact, null); + ListNode tail = alternatives; + + if (Encoding.isAscii(codePoint)) { + int foldLength = PerlCaseFold.simpleFoldClassLength(codePoint); + for (int index = 0; index < foldLength; index++) { + int folded = PerlCaseFold.simpleFoldClassCodePoint(codePoint, index); + if (folded == codePoint || !Encoding.isAscii(folded)) continue; + StringNode sibling = new StringNode(); + sibling.catCode(folded, enc); + sibling.setRaw(); + ListNode alternative = newAlt(sibling, null); + tail.setTail(alternative); + tail = alternative; + expanded = true; + } + } + + Node part = alternatives.tail == null ? exact : alternatives; + ListNode segment = ListNode.newList(part, null); + if (root == null) root = segment; + else sequenceTail.setTail(segment); + sequenceTail = segment; + p = next; + } + + if (!expanded) { + source.setRaw(); + return source; + } + source.replaceWith(root); + setupTree(root, state); + return root; + } + private boolean isUnsafePerlMultiFoldOptimizationBoundary(StringNode node) { if (!syntax.op2OptionPerl() || node.length() == 0) return false; @@ -2344,7 +2389,9 @@ protected final Node setupTree(Node node, int state) { case NodeType.STR: if (isIgnoreCase(regex.options) && !((StringNode)node).isRaw()) { - if (Option.isPerlAsciiStrict(regex.options)) { + if (Option.isPerlBytePattern(regex.options)) { + node = expandPerlByteAsciiFoldString((StringNode)node, state); + } else if (Option.isPerlAsciiStrict(regex.options)) { Node protectedNode = protectPerlAsciiStrictCrossings( (StringNode)node, state); node = protectedNode == node diff --git a/third_party/joni/src/org/joni/ApplyCaseFold.java b/third_party/joni/src/org/joni/ApplyCaseFold.java index 557cc018b..e5a250664 100644 --- a/third_party/joni/src/org/joni/ApplyCaseFold.java +++ b/third_party/joni/src/org/joni/ApplyCaseFold.java @@ -40,7 +40,7 @@ public void apply(int from, int[]to, int length, Object o) { BitSet bs = cc.bs; boolean addFlag; - if (Option.isPerlAsciiStrict(env.option) + if (!arg.preservePropertyAsciiCrossings && Option.isPerlAsciiStrict(env.option) && perlAsciiStrictRelationCrossesAscii(from, to, length)) { return; } diff --git a/third_party/joni/src/org/joni/ApplyCaseFoldArg.java b/third_party/joni/src/org/joni/ApplyCaseFoldArg.java index c3de87823..54ad870ef 100644 --- a/third_party/joni/src/org/joni/ApplyCaseFoldArg.java +++ b/third_party/joni/src/org/joni/ApplyCaseFoldArg.java @@ -25,14 +25,21 @@ final class ApplyCaseFoldArg { final ScanEnvironment env; final CClassNode cc, ascCc, foldCc; + final boolean preservePropertyAsciiCrossings; ListNode altRoot; ListNode tail; ApplyCaseFoldArg(ScanEnvironment env, CClassNode cc, CClassNode ascCc, CClassNode foldCc) { + this(env, cc, ascCc, foldCc, false); + } + + ApplyCaseFoldArg(ScanEnvironment env, CClassNode cc, CClassNode ascCc, + CClassNode foldCc, boolean preservePropertyAsciiCrossings) { this.env = env; this.cc = cc; this.ascCc = ascCc; this.foldCc = foldCc; + this.preservePropertyAsciiCrossings = preservePropertyAsciiCrossings; } } diff --git a/third_party/joni/src/org/joni/Option.java b/third_party/joni/src/org/joni/Option.java index 6e5a9d774..a4392cd1f 100644 --- a/third_party/joni/src/org/joni/Option.java +++ b/third_party/joni/src/org/joni/Option.java @@ -48,8 +48,10 @@ public final class Option { public static final int CR_7_BIT = (1 << 18); /** Perl /aa: forbid case-fold crossings between ASCII and non-ASCII. */ public static final int PERL_ASCII_STRICT = (1 << 19); + /** Perl /d byte strings use single-character Latin-1 folding only. */ + public static final int PERL_BYTE_PATTERN = (1 << 20); - public static final int MAXBIT = (1 << 20); /* limit */ + public static final int MAXBIT = (1 << 21); /* limit */ public static final int DEFAULT = NONE; @@ -69,6 +71,7 @@ public static String toString(int option) { if (isPosixRegion(option)) options += "POSIX_REGION"; if (isCR7Bit(option)) options += "CR_7_BIT"; if (isPerlAsciiStrict(option)) options += "PERL_ASCII_STRICT"; + if (isPerlBytePattern(option)) options += "PERL_BYTE_PATTERN"; return options; } @@ -148,6 +151,10 @@ public static boolean isPerlAsciiStrict(int option) { return (option & PERL_ASCII_STRICT) != 0; } + public static boolean isPerlBytePattern(int option) { + return (option & PERL_BYTE_PATTERN) != 0; + } + public static boolean isDynamic(int option) { // Ignore-case and multibyte status are included in compiled code. // Perl ASCII-strict folding also needs a match-time option because it diff --git a/third_party/joni/src/org/joni/Parser.java b/third_party/joni/src/org/joni/Parser.java index 6d03aaf61..b7a37e523 100644 --- a/third_party/joni/src/org/joni/Parser.java +++ b/third_party/joni/src/org/joni/Parser.java @@ -2230,7 +2230,13 @@ private Node parseCharType(Node node) { private Node cClassCaseFold(Node node, CClassNode cc, CClassNode ascCc, CClassNode foldCc) { - ApplyCaseFoldArg arg = new ApplyCaseFoldArg(env, cc, ascCc, foldCc); + return cClassCaseFold(node, cc, ascCc, foldCc, false); + } + + private Node cClassCaseFold(Node node, CClassNode cc, CClassNode ascCc, + CClassNode foldCc, boolean preservePropertyAsciiCrossings) { + ApplyCaseFoldArg arg = new ApplyCaseFoldArg( + env, cc, ascCc, foldCc, preservePropertyAsciiCrossings); enc.applyAllCaseFold(env.caseFoldFlagFor(env.option), ApplyCaseFold.INSTANCE, arg); if (syntax.op2OptionPerl()) { ApplyCaseFold.applyPerlSimpleClassClosure(arg); @@ -2258,7 +2264,7 @@ private Node parseCharProperty() { if (isIgnoreCase(env.option) && property.caseFold) { if (property.ranges != null || property.wideRanges != null || property.ctype != CharacterType.ASCII) { - node = cClassCaseFold(node, cc, cc, cc); + node = cClassCaseFold(node, cc, cc, cc, true); } } return node; diff --git a/third_party/joni/test/org/joni/test/TestPerlBytePatternCaseFold.java b/third_party/joni/test/org/joni/test/TestPerlBytePatternCaseFold.java new file mode 100644 index 000000000..b3886985f --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestPerlBytePatternCaseFold.java @@ -0,0 +1,66 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + */ +package org.joni.test; + +import static org.junit.Assert.assertEquals; + +import java.nio.charset.StandardCharsets; + +import org.jcodings.specific.ISO8859_1Encoding; +import org.jcodings.specific.UTF8Encoding; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.junit.Test; + +public class TestPerlBytePatternCaseFold { + private static int search(String pattern, String input) { + byte[] patternBytes = pattern.getBytes(StandardCharsets.ISO_8859_1); + byte[] inputBytes = input.getBytes(StandardCharsets.ISO_8859_1); + Regex regex = new Regex(patternBytes, 0, patternBytes.length, + Option.IGNORECASE | Option.PERL_BYTE_PATTERN, + ISO8859_1Encoding.INSTANCE, Syntax.PerlNG); + return regex.matcher(inputBytes).search(0, inputBytes.length, Option.NONE); + } + + @Test + public void bytePatternsRejectUnicodeMultiCharacterFolds() { + assertEquals(-1, search("^\u00df$", "ss")); + assertEquals(-1, search("^ss$", "\u00df")); + assertEquals(-1, search("^(\u00df)\\1$", "ssss")); + assertEquals(-1, search("^(ss)\\1$", "\u00df\u00df")); + } + + @Test + public void bytePatternsKeepAsciiButRejectLatinOneFolds() { + assertEquals(0, search("^a$", "A")); + assertEquals(-1, search("^\u00e4$", "\u00c4")); + assertEquals(-1, search("^(\u00e4)\\1$", "\u00c4\u00e4")); + } + + @Test + public void bytePatternsRetainEverySegmentOfAsciiStrings() { + assertEquals(0, search("^dbi:$", "dbi:")); + assertEquals(0, search("^dbi:$", "DBI:")); + assertEquals(-1, search("^dbi:$", "dbx:")); + } + + @Test + public void asciiStrictPropertyClosureRetainsUnicodePropertyMembership() { + byte[] pattern = "^\\p{Lowercase}$".getBytes(StandardCharsets.UTF_8); + byte[] kelvin = "\u212a".getBytes(StandardCharsets.UTF_8); + Regex regex = new Regex(pattern, 0, pattern.length, + Option.IGNORECASE | Option.PERL_ASCII_STRICT, + UTF8Encoding.INSTANCE, Syntax.PerlNG); + assertEquals(0, regex.matcher(kelvin).search(0, kelvin.length, Option.NONE)); + } +}