diff --git a/docs/reference/feature-matrix.md b/docs/reference/feature-matrix.md index 1691ad9344..c7bf5550e0 100644 --- a/docs/reference/feature-matrix.md +++ b/docs/reference/feature-matrix.md @@ -378,11 +378,11 @@ my @copy = @{$z}; # ERROR - ✅ **Named Capture Groups**: Defining named capture groups using `(?...)` or `(?'name'...)` is supported. - ✅ **Backreferences to Named Groups**: Using `\k` or `\g{name}` for backreferences to named groups is supported. - ✅ **Relative Backreferences**: Using `\g{-n}` for relative backreferences. -- ✅ **Basic Unicode Properties**: Common `\p{...}` and `\P{...}` forms such as `\p{L}` execute through Joni. -- 🟡 **Perl Unicode Property Syntax**: Perl-specific properties execute through Joni. `Age`, cumulative `In`/`Present_In`, General_Category, Canonical_Combining_Class, Bidi_Class, Decomposition_Type, East_Asian_Width, Numeric_Value, Joining_Group, Block, Script, and Script_Extensions assignments are generated from pinned Perl 5.44 Unicode 17.0 data with loose and wildcard aliases, ordered missing defaults, official compact aliases and shortcuts, Script-versus-Script_Extensions policy, reserved or composite values, exact rationals, and Perl's generated decimal keyword aliases; `ASCII_Hex_Digit`/`AHex` accepts Perl's eight boolean value aliases. Other generated property/value aliases still have pinned acceptance/rejection gaps. +- ✅ **Basic Unicode Properties**: Common `\p{...}` and `\P{...}` forms such as `\p{L}` execute through Joni. General_Category assignments now enter the forked Joni parser unchanged and resolve to pinned Perl ranges there. +- 🟡 **Perl Unicode Property Syntax**: Perl-specific properties execute through Joni. `Age`, cumulative `In`/`Present_In`, General_Category, Canonical_Combining_Class, Bidi_Class, Decomposition_Type, East_Asian_Width, Numeric_Value, Joining_Group, Block, Script, and Script_Extensions assignments are generated from pinned Perl 5.44 Unicode 17.0 data with loose and wildcard aliases, ordered missing defaults, official compact aliases and shortcuts, Script-versus-Script_Extensions policy, reserved or composite values, exact rationals, and Perl's generated decimal keyword aliases; `ASCII_Hex_Digit`/`AHex` accepts Perl's eight boolean value aliases. General_Category uses Joni's range-resolver API; the other families remain adapter-translated until their family-specific case-fold and wildcard semantics are represented natively. Other generated property/value aliases still have pinned acceptance/rejection gaps. - ✅ **Possessive Quantifiers**: Quantifiers like `*+`, `++`, `?+`, and `{n,m}+`, which disable backtracking, are supported. - ✅ **Atomic Grouping**: Use of `(?>...)` for atomic groups is supported. -- ✅ **`\K` assertion**: Keep left — in `s///`, text before `\K` is preserved; match variables reflect only the portion after `\K`. +- ✅ **`\K` assertion**: Keep left — in `s///`, text before `\K` is preserved; match variables reflect only the portion after `\K`. Ordinary KEEP assertions route through native Joni and no longer use the Java marker rewrite; the adapter still rejects KEEP inside lookaround until the Joni analyser emits Perl's diagnostic directly. - ✅ **Preprocessor**: `\Q`, `\L`, `\U`, `\l`, `\u`, `\E` are preprocessed in regex. - ✅ **Overloading**: `qr` overloading is implemented. See also [overload pragma](#pragmas). - ✅ **Python-style named groups**: `(?P...)` and `(?P=name)` are parsed natively by Joni with Perl capture numbering, duplicate-name behavior, and malformed/unknown-name diagnostics. @@ -394,7 +394,7 @@ my @copy = @{$z}; # ERROR - ✅ **Backtracking Control Verbs**: `(*ACCEPT)`, `(*FAIL)`/`(*F)`, `(*PRUNE)`, `(*SKIP)`, `(*THEN)`, and `(*COMMIT)` execute through Joni with matcher-owned cut boundaries. Atomic groups `(?>...)` are supported. - ✅ **Marks and named skip targets**: `(*MARK:NAME)` and its `(*:NAME)` shorthand, named `(*SKIP:NAME)`, `$REGMARK`, and `$REGERROR` execute through Joni and follow the selected backtracking path. - ✅ **Regex Definitions**: `(?(DEFINE)...)` containers and numbered or named calls to their subpatterns execute through Joni. -- ✅ **Lookbehind Assertions**: Fixed and bounded variable-length positive and negative lookbehind assertions execute through Joni. +- ✅ **Lookbehind Assertions**: Fixed and bounded variable-length positive and negative lookbehind assertions execute through Joni, including nested lookahead. Removing the remaining compatibility routing requires the complete lookbehind corpus and native diagnostics to pass. - ✅ **Branch Reset Groups**: `(?|...)` resets capture numbering across alternatives and preserves mapped match variables. - ✅ **Advanced Subroutine Calls**: Sub-pattern calls with numbered or named references like `(?1)` and `(?&name)` execute through Joni. - ✅ **Conditional Expressions**: Numbered and named capture conditions, positive and negative assertion conditions, recursion conditions `(?(R))`, `(?(R1))`, and `(?(R&name))`, executable callback conditions, and optimistic predicates execute through Joni. diff --git a/src/main/java/org/perlonjava/runtime/regex/CaptureNameEncoder.java b/src/main/java/org/perlonjava/runtime/regex/CaptureNameEncoder.java index d796d22976..736a92c589 100644 --- a/src/main/java/org/perlonjava/runtime/regex/CaptureNameEncoder.java +++ b/src/main/java/org/perlonjava/runtime/regex/CaptureNameEncoder.java @@ -269,9 +269,9 @@ public static boolean isDuplicateMarkerName(String javaName) { * from user-visible variables like %+ and %-. * * @param captureName The capture group name to check - * @return true if this is an internal capture (code block or \K marker) + * @return true if this is an internal code-block capture */ public static boolean isInternalCapture(String captureName) { - return isCodeBlockCapture(captureName) || "perlK".equals(captureName); + return isCodeBlockCapture(captureName); } } diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index a1fac2e2de..a8b8cf0c9d 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -1,5 +1,6 @@ package org.perlonjava.runtime.regex; +import org.jcodings.Encoding; import org.jcodings.specific.ISO8859_1Encoding; import org.jcodings.specific.UTF8Encoding; import org.joni.Matcher; @@ -13,6 +14,7 @@ import org.joni.Region; import org.joni.Syntax; +import static org.joni.constants.SyntaxProperties.ALLOW_MULTIPLEX_DEFINITION_NAME_CALL; import static org.joni.constants.SyntaxProperties.OP2_OPTION_PERL; import static org.joni.constants.SyntaxProperties.OP2_OPTION_RUBY; import static org.joni.constants.SyntaxProperties.OP2_PLUS_POSSESSIVE_INTERVAL; @@ -44,14 +46,32 @@ final class JoniRegexPattern { "PERLONJAVA", Syntax.RUBY.op, (Syntax.RUBY.op2 & ~OP2_OPTION_RUBY) | OP2_OPTION_PERL | OP2_PLUS_POSSESSIVE_INTERVAL, Syntax.RUBY.op3, - Syntax.RUBY.behavior, + Syntax.RUBY.behavior | ALLOW_MULTIPLEX_DEFINITION_NAME_CALL, Syntax.RUBY.options & ~(Option.ASCII_RANGE | Option.POSIX_BRACKET_ALL_RANGE | Option.WORD_BOUND_ALL_RANGE), - Syntax.RUBY.metaCharTable); + Syntax.RUBY.metaCharTable, + JoniRegexPattern::resolveNamedCharacter, + JoniRegexPattern::resolveCharacterProperty); + + private static int resolveNamedCharacter(byte[] bytes, int p, int end, + Encoding encoding) { + return UnicodeResolver.getCodePointFromName(new String(bytes, p, end - p, + encoding == ISO8859_1Encoding.INSTANCE + ? StandardCharsets.ISO_8859_1 : StandardCharsets.UTF_8)); + } + + private static int[] resolveCharacterProperty(byte[] bytes, int p, int end, + Encoding encoding) { + String property = new String(bytes, p, end - p, + encoding == ISO8859_1Encoding.INSTANCE + ? StandardCharsets.ISO_8859_1 : StandardCharsets.UTF_8); + return UnicodeResolver.resolveJoniPropertyRanges(property); + } private final Regex regex; private final String sourcePattern; private final Map namedGroups; + private final Map physicalNamedGroups; private final RegexFlags flags; private final boolean hasControlVerbState; private final boolean hasDeferredUserDefinedUnicodeProperty; @@ -73,6 +93,10 @@ final class JoniRegexPattern { JoniRegexPattern(String perlPattern, RegexFlags flags, int trustedCalloutCount, boolean forceAsciiClasses, boolean byteMode, boolean byteBackedPattern) { + KeepSyntax keepSyntax = analyzeKeepSyntax(perlPattern, flags.isExtended()); + if (keepSyntax.inLookaround()) { + throw new PerlCompilerException("\\K not permitted in lookahead/lookbehind in regex"); + } this.flags = flags; this.byteMode = byteMode; hasControlVerbState = hasControlVerbState(perlPattern); @@ -84,7 +108,9 @@ final class JoniRegexPattern { regex = new Regex(bytes, 0, bytes.length, toJoniOptions(flags, forceAsciiClasses), byteMode ? ISO8859_1Encoding.INSTANCE : UTF8Encoding.INSTANCE, PERLONJAVA_SYNTAX); - namedGroups = collectNamedGroups(regex); + NamedGroupMaps groupMaps = collectNamedGroups(regex); + namedGroups = groupMaps.logical(); + physicalNamedGroups = groupMaps.physical(); } RegexMatcher matcher(String input, List callbacks) { @@ -93,7 +119,7 @@ RegexMatcher matcher(String input, List callbacks) { RegexMatcher matcher(String input, List callbacks, RuntimeScalar subject) { - return new JoniRegexMatcher(regex, sourcePattern, namedGroups, flags, + return new JoniRegexMatcher(regex, sourcePattern, namedGroups, physicalNamedGroups, flags, hasControlVerbState, byteMode, input, callbacks, subject); } @@ -205,6 +231,14 @@ private static UserPropertyTranslation translateUserDefinedProperties( boolean frontendProperty = unnegated.matches( "(?i)^(?:script|sc|block|blk|age|in|present[_ ]?in)\\s*(?:=|:(?!:)).*"); boolean perlBuiltInAlias = UnicodeResolver.isPerlBuiltInPropertyAlias(unnegated); + boolean joniResolvedProperty = UnicodeResolver.resolveJoniPropertyRanges( + unnegated) != null; + if (!userDefined && joniResolvedProperty + && (frontendProperty || scriptExtensions || perlBuiltInAlias)) { + translated.append(pattern, i, end + 1); + i = end; + continue; + } if ((frontendProperty || scriptExtensions || perlBuiltInAlias) && extendedClassBracketDepth > 0) { translated.append(pattern, i, end + 1); @@ -292,14 +326,21 @@ private static int toJoniOptions(RegexFlags flags, boolean forceAsciiClasses) { } static boolean requiresJoniBackend(String pattern) { + return requiresJoniBackend(pattern, + pattern == null ? null : RegexFlags.fromModifiers("", pattern)); + } + + static boolean requiresJoniBackend(String pattern, RegexFlags flags) { if (pattern == null) return false; boolean hasSubroutineCall = pattern.matches("(?s).*\\(\\?[+-]?\\d+\\).*") || pattern.contains("(?&") || pattern.contains("(?P>"); - // Temporary parity fallback: Java handles the branch-reset named-call - // corpus until Joni's native named-call admission patch is complete. + // Keep automatic routing on Java until the native branch-reset call + // implementation passes its combined imported-corpus gate. Explicit + // Joni mode still exercises the native implementation directly. boolean branchResetCallUsesJava = pattern.contains("(?|") && hasSubroutineCall; - return pattern.contains("(?{=CALL:") + return analyzeKeepSyntax(pattern, flags != null && flags.isExtended()).present() + || pattern.contains("(?{=CALL:") || pattern.contains("(?{=DYNAMIC:") || pattern.contains("(*ACCEPT)") || pattern.contains("(*PRUNE") @@ -316,6 +357,102 @@ static boolean requiresJoniBackend(String pattern) { || (hasSubroutineCall && !branchResetCallUsesJava); } + private record KeepSyntax(boolean present, boolean inLookaround) {} + + private static KeepSyntax analyzeKeepSyntax(String pattern, boolean extended) { + boolean quoted = false; + boolean inClass = false; + boolean classStart = false; + int extendedClassDepth = 0; + int lookaroundDepth = 0; + java.util.ArrayDeque groups = new java.util.ArrayDeque<>(); + boolean present = false; + + for (int i = 0; i < pattern.length(); i++) { + char ch = pattern.charAt(i); + if (quoted) { + if (ch == '\\' && i + 1 < pattern.length() + && pattern.charAt(i + 1) == 'E') { + quoted = false; + i++; + } + continue; + } + if (extendedClassDepth > 0) { + if (ch == '\\' && i + 1 < pattern.length()) { + i++; + } else if (ch == '[') { + extendedClassDepth++; + } else if (ch == ']' && --extendedClassDepth == 0) { + // The following ')' closes the extended-class construct. + } + continue; + } + if (inClass) { + if (ch == '\\' && i + 1 < pattern.length()) { + i++; + classStart = false; + continue; + } + if (ch == '[' && i + 1 < pattern.length() + && (pattern.charAt(i + 1) == ':' + || pattern.charAt(i + 1) == '.' + || pattern.charAt(i + 1) == '=')) { + char delimiter = pattern.charAt(i + 1); + int close = pattern.indexOf("" + delimiter + ']', i + 2); + if (close >= 0) i = close + 1; + classStart = false; + continue; + } + if (ch == ']' && !classStart) inClass = false; + else if (!(classStart && ch == '^')) classStart = false; + continue; + } + if (extended && ch == '#') { + while (i + 1 < pattern.length() + && pattern.charAt(i + 1) != '\n') i++; + continue; + } + if (pattern.startsWith("(?[", i)) { + extendedClassDepth = 1; + i += 2; + continue; + } + if (ch == '[') { + inClass = true; + classStart = true; + continue; + } + if (ch == '\\' && i + 1 < pattern.length()) { + char escaped = pattern.charAt(++i); + if (escaped == 'Q') { + quoted = true; + } else if (escaped == 'K') { + present = true; + if (lookaroundDepth > 0) return new KeepSyntax(true, true); + } + continue; + } + if (ch == '(') { + if (pattern.startsWith("(?#", i)) { + int close = pattern.indexOf(')', i + 3); + if (close < 0) break; + i = close; + continue; + } + boolean lookaround = pattern.startsWith("(?=", i) + || pattern.startsWith("(?!", i) + || pattern.startsWith("(?<=", i) + || pattern.startsWith("(? i + 3) { int codePoint = UnicodeResolver.getCodePointFromName( @@ -792,27 +935,34 @@ private static int findGroupEnd(String pattern, int start) { return -1; } - private static Map collectNamedGroups(Regex regex) { + private record NamedGroupMaps(Map logical, + Map physical) {} + + private static NamedGroupMaps collectNamedGroups(Regex regex) { Map names = new LinkedHashMap<>(); + Map physicalNames = new LinkedHashMap<>(); Iterator iterator = regex.namedBackrefIterator(); while (iterator.hasNext()) { NameEntry entry = iterator.next(); String name = new String(entry.name, entry.nameP, entry.nameEnd - entry.nameP, StandardCharsets.UTF_8); int[] refs = entry.getBackRefs(); + int[] physicalRefs = entry.getPhysicalBackRefs(); for (int i = 0; i < refs.length; i++) { String key = i == 0 ? name : name + CaptureNameEncoder.DUPLICATE_MARKER + (i - 1); names.put(key, refs[i]); + physicalNames.put(key, physicalRefs[i]); } } - return names; + return new NamedGroupMaps(names, physicalNames); } private static final class JoniRegexMatcher implements RegexMatcher { private final Regex regex; private final String sourcePattern; private final Map namedGroups; + private final Map physicalNamedGroups; private final RegexFlags flags; private final String input; private final byte[] bytes; @@ -833,12 +983,14 @@ private static final class JoniRegexMatcher implements RegexMatcher { private PerlCalloutHandler calloutHandler; JoniRegexMatcher(Regex regex, String sourcePattern, Map namedGroups, + Map physicalNamedGroups, RegexFlags flags, boolean hasControlVerbState, boolean byteMode, String input, List callbacks, RuntimeScalar subject) { this.regex = regex; this.sourcePattern = sourcePattern; this.namedGroups = namedGroups; + this.physicalNamedGroups = physicalNamedGroups; this.flags = flags; this.hasControlVerbState = hasControlVerbState; this.byteMode = byteMode; @@ -952,8 +1104,13 @@ public String group(int index) { @Override public String group(String name) { - int group = namedGroupNumber(name); - return group(group); + requireMatch(); + Integer physical = physicalNamedGroups.get(name); + if (physical == null) return group(namedGroupNumber(name)); + int begin = matcher.physicalNamedCaptureBegin(physical); + int end = matcher.physicalNamedCaptureEnd(physical); + if (begin < 0 || end < 0) return null; + return input.substring(toCharOffset(begin), toCharOffset(end)); } @Override public int groupCount() { return regex.numberOfCaptures(); } @@ -978,6 +1135,12 @@ private static int deriveCommittedLastClosedCapture(Region region) { private int groupOffset(String name, boolean begin) { requireMatch(); + Integer physical = physicalNamedGroups.get(name); + if (physical != null) { + int offset = begin ? matcher.physicalNamedCaptureBegin(physical) + : matcher.physicalNamedCaptureEnd(physical); + return offset < 0 ? -1 : toCharOffset(offset); + } int group = namedGroupNumber(name); return groupOffset(group, begin); } diff --git a/src/main/java/org/perlonjava/runtime/regex/RegexBackendPolicy.java b/src/main/java/org/perlonjava/runtime/regex/RegexBackendPolicy.java index ccadb5146f..fd4121b5d8 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RegexBackendPolicy.java +++ b/src/main/java/org/perlonjava/runtime/regex/RegexBackendPolicy.java @@ -38,7 +38,11 @@ static Mode current() { } static boolean useJoni(String pattern) { - return current() == Mode.JONI || JoniRegexPattern.requiresJoniBackend(pattern); + return useJoni(pattern, pattern == null ? null : RegexFlags.fromModifiers("", pattern)); + } + + static boolean useJoni(String pattern, RegexFlags flags) { + return current() == Mode.JONI || JoniRegexPattern.requiresJoniBackend(pattern, flags); } static String cacheTag() { diff --git a/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessor.java b/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessor.java index a901ac9963..d078cf6f36 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessor.java +++ b/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessor.java @@ -61,7 +61,6 @@ public class RegexPreprocessor { static boolean deferredUnicodePropertyEncountered; static boolean inlinePFlagEncountered; static boolean branchResetEncountered; - static boolean backslashKEncountered; /** * Tracks named capture groups already emitted in the current pattern. * Used to detect duplicate names like `(?a)|(?b)` (legal in Perl, @@ -104,14 +103,6 @@ static boolean hadBranchReset() { return branchResetEncountered; } - static void markBackslashK() { - backslashKEncountered = true; - } - - static boolean hadBackslashK() { - return backslashKEncountered; - } - /** * Preprocesses a given regex string to make it compatible with Java's regex engine. * This involves handling various constructs and escape sequences that Java does not @@ -144,7 +135,6 @@ private static String preProcessRegexInternal(String s, RegexFlags regexFlags) { deferredUnicodePropertyEncountered = false; inlinePFlagEncountered = false; branchResetEncountered = false; - backslashKEncountered = false; seenNamedCaptures.clear(); emittedNamedCaptures.clear(); duplicateNameCounter = 0; diff --git a/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessorHelper.java b/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessorHelper.java index e289d53653..74fe5b1358 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessorHelper.java +++ b/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessorHelper.java @@ -221,13 +221,11 @@ static int handleEscapeSequences(String s, StringBuilder sb, int c, int offset, sb.append("[^\\n\\x0B\\f\\r\\x85\\x{2028}\\x{2029}]"); return offset; } else if (nextChar == 'K') { - // \K - keep assertion (reset start of match) - // Insert a zero-width named capture group to mark the \K position. - // During substitution, text before this position is preserved ("kept"). - sb.setLength(sb.length() - 1); // Remove the backslash - RegexPreprocessor.markBackslashK(); - RegexPreprocessor.captureGroupCount++; - sb.append("(?)"); + // Syntax-aware admission routes real KEEP assertions to Joni. A + // \K reaching the Java preprocessor is literal, for example in a + // character class or a \Q...\E region. + sb.setLength(sb.length() - 1); + sb.append('K'); return offset; } else if ((nextChar == 'b' || nextChar == 'B') && offset + 1 < length && s.charAt(offset + 1) == '{') { // Handle \b{...} and \B{...} boundary assertions @@ -804,6 +802,12 @@ static int handleRegexCharacterClassEscape(int offset, String s, StringBuilder s sb.append(Character.toChars(s.charAt(offset))); } lastChar = -1; // Unicode properties can't be range endpoints + } else if (offset < length && s.charAt(offset) == 'K') { + // KEEP is not active inside a class. Preserve Perl's + // literal K while emitting Java-compatible syntax. + sb.setLength(sb.length() - 1); + sb.append('K'); + lastChar = 'K'; } else if (offset < length && s.charAt(offset) == 'N') { if (offset + 1 < length && s.charAt(offset + 1) == '{') { // Handle \N{...} constructs diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java index bec8aab452..3ae0ca72c8 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java @@ -213,7 +213,6 @@ static void updateControlVerbVariables(String mark, String error) { private boolean hasCodeBlockCaptures = false; // True if regex has (?{...}) code blocks private boolean deferredUserDefinedUnicodeProperties = false; private boolean hasBranchReset = false; // True if pattern uses (?|...) branch reset - private boolean hasBackslashK = false; // True if pattern uses \K (keep assertion) // An empty qr// object keeps its own empty pattern when interpolated; // only empty match/substitution string syntax reuses the previous match. private boolean quoteConstruction = false; @@ -259,7 +258,6 @@ public RuntimeRegex cloneTracked() { copy.hasCodeBlockCaptures = this.hasCodeBlockCaptures; copy.deferredUserDefinedUnicodeProperties = this.deferredUserDefinedUnicodeProperties; copy.hasBranchReset = this.hasBranchReset; - copy.hasBackslashK = this.hasBackslashK; copy.quoteConstruction = this.quoteConstruction; copy.warningsOnUse = new ArrayList<>(this.warningsOnUse); copy.lexicalDebugMode = this.lexicalDebugMode; @@ -623,7 +621,8 @@ private static synchronized RuntimeRegex compileSynchronized( String javaPattern = null; try { - boolean usesRecursiveBackend = RegexBackendPolicy.useJoni(compilePatternString); + boolean usesRecursiveBackend = RegexBackendPolicy.useJoni( + compilePatternString, regex.regexFlags); if (usesRecursiveBackend && compilePatternString.contains("(?&") && (compilePatternString.contains("(?<=") @@ -668,14 +667,12 @@ private static synchronized RuntimeRegex compileSynchronized( regex.hasPreservesMatch = regex.regexFlags.preservesMatch() || RegexFlags.hasInlinePreserveModifier(compilePatternString); regex.hasBranchReset = false; - regex.hasBackslashK = false; } else { regex.deferredUserDefinedUnicodeProperties = RegexPreprocessor.hadDeferredUnicodePropertyEncountered(); regex.hasPreservesMatch = regex.regexFlags.preservesMatch() || RegexFlags.hasInlinePreserveModifier(compilePatternString) || RegexPreprocessor.hadInlinePFlag(); regex.hasBranchReset = RegexPreprocessor.hadBranchReset(); - regex.hasBackslashK = RegexPreprocessor.hadBackslashK(); regex.warningsOnUse.addAll(RegexPreprocessor.getWarningsOnUse()); } @@ -766,6 +763,9 @@ && containsExecutableSource(originalPatternString, regex.regexFlags.isExtended()); if (e instanceof SyntaxException && !validatesExecutableSource) { String message = e.getMessage(); + if ("Empty \\N{}".equals(message)) { + throw new PerlCompilerException("Unknown charname ''"); + } if (literalSyntaxValidation && message != null && (message.contains("premature end of char-class") || message.contains("Unclosed character class"))) { @@ -1638,7 +1638,6 @@ public static RuntimeScalar getReplacementRegex(RuntimeScalar patternString, Run regex.useGAssertion = resolvedRegex.useGAssertion; regex.patternFlags = resolvedRegex.patternFlags; regex.hasBranchReset = resolvedRegex.hasBranchReset; - regex.hasBackslashK = resolvedRegex.hasBackslashK; regex.hasCodeBlockCaptures = resolvedRegex.hasCodeBlockCaptures; regex.warningsOnUse = new ArrayList<>(resolvedRegex.warningsOnUse); regex.lexicalDebugMode = callSiteDebugMode != 0 @@ -1677,7 +1676,6 @@ public static RuntimeScalar getReplacementRegex(RuntimeScalar patternString, Run regex.useGAssertion = recompiledRegex.useGAssertion; regex.patternFlags = recompiledRegex.patternFlags; regex.hasBranchReset = recompiledRegex.hasBranchReset; - regex.hasBackslashK = recompiledRegex.hasBackslashK; regex.hasCodeBlockCaptures = recompiledRegex.hasCodeBlockCaptures; regex.warningsOnUse = new ArrayList<>(recompiledRegex.warningsOnUse); } else { @@ -1856,7 +1854,7 @@ private static void updateLastNamedCaptureGroups(RegexMatcher matcher) { } /** Populate Perl's numbered capture state from the backend-neutral view. */ - private static void updateNumberedCaptureGroups(RuntimeRegex regex, RegexMatcher matcher) { + private static void updateNumberedCaptureGroups(RegexMatcher matcher) { RuntimeRegexState regexState = state(); regexState.lastParenMatchOverrideActive = false; regexState.lastParenMatchOverride = null; @@ -1871,17 +1869,9 @@ private static void updateNumberedCaptureGroups(RuntimeRegex regex, RegexMatcher return; } - int perlKGroup = regex.hasBackslashK ? getPerlKGroup(matcher) : -1; - int userGroupCount = captureCount - (perlKGroup >= 0 ? 1 : 0); - if (userGroupCount == 0) { - regexState.lastCaptureGroups = null; - return; - } - regexState.lastCaptureGroups = new String[userGroupCount]; - int destination = 0; + regexState.lastCaptureGroups = new String[captureCount]; for (int group = 1; group <= captureCount; group++) { - if (group == perlKGroup) continue; - regexState.lastCaptureGroups[destination++] = matcher.group(group); + regexState.lastCaptureGroups[group - 1] = matcher.group(group); } } @@ -2124,31 +2114,21 @@ private static RuntimeBase matchRegexDirect(RuntimeScalar quotedRegex, RuntimeSc // Always initialize $1, $2, @+, @-, $`, $&, $' for every successful match regexState.globalMatcher = matcher; regexState.globalMatchString = inputStr; - regexState.lastMatchUsedBackslashK = regex.hasBackslashK; + regexState.lastMatchUsedBackslashK = false; updateLastNamedCaptureGroups(matcher); - updateNumberedCaptureGroups(regex, matcher); + updateNumberedCaptureGroups(matcher); - // For \K, adjust match start/string so $& is only the post-\K portion - if (regex.hasBackslashK) { - int keepEnd = matcher.end("perlK"); - regexState.lastMatchedString = inputStr.substring(keepEnd, matcher.end()); - regexState.lastMatchStart = keepEnd; - } else { - regexState.lastMatchedString = matcher.group(0); - regexState.lastMatchStart = matcher.start(); - } + regexState.lastMatchedString = matcher.group(0); + regexState.lastMatchStart = matcher.start(); regexState.lastMatchEnd = matcher.end(); if (regex.regexFlags.isGlobalMatch() && captureCount < 1 && ctx == RuntimeContextType.LIST) { // Global match and no captures, in list context return the matched string - String matchedStr = regex.hasBackslashK ? regexState.lastMatchedString : matcher.group(0); - matchedGroups.add(makeMatchResultScalar(matchedStr)); + matchedGroups.add(makeMatchResultScalar(matcher.group(0))); } else { // save captures in return list if needed if (ctx == RuntimeContextType.LIST) { - int perlKGroup = regex.hasBackslashK ? getPerlKGroup(matcher) : -1; for (int i = 1; i <= captureCount; i++) { - if (i == perlKGroup) continue; // skip internal \K marker group String matchedStr = matcher.group(i); // Include undef for groups that didn't participate in the match. // The matcher adapter exposes native Perl numbering for (?|...). @@ -2774,19 +2754,12 @@ private static void updateReplacementMatchState(RuntimeRegex regex, RegexMatcher // Initialize $1, $2, @+, @- only when we have a match state().globalMatcher = matcher; state().globalMatchString = inputStr; - state().lastMatchUsedBackslashK = regex.hasBackslashK; + state().lastMatchUsedBackslashK = false; updateLastNamedCaptureGroups(matcher); - updateNumberedCaptureGroups(regex, matcher); + updateNumberedCaptureGroups(matcher); - // For \K, adjust match start so $& is only the post-\K portion - if (regex.hasBackslashK) { - int keepEnd = matcher.end("perlK"); - state().lastMatchStart = keepEnd; - state().lastMatchedString = inputStr.substring(keepEnd, matcher.end()); - } else { - state().lastMatchStart = matcher.start(); - state().lastMatchedString = matcher.group(0); - } + state().lastMatchStart = matcher.start(); + state().lastMatchedString = matcher.group(0); state().lastMatchEnd = matcher.end(); } @@ -2928,7 +2901,6 @@ public static RuntimeBase replaceRegex(RuntimeScalar quotedRegex, RuntimeScalar // Don't reset state().globalMatcher here - only reset it if we actually find a match // This preserves capture variables from previous matches when substitution doesn't match - // Track position for manual replacement when \K is used int lastAppendEnd = 0; // Perform the substitution. Java's Matcher.find() skips ahead after a @@ -2983,18 +2955,9 @@ public static RuntimeBase replaceRegex(RuntimeScalar quotedRegex, RuntimeScalar } if (replacementStr != null) { - if (regex.hasBackslashK) { - // \K: preserve text before \K position, only replace after it - int keepEnd = matcher.end("perlK"); - resultBuffer.append(inputStr, lastAppendEnd, keepEnd); - resultBuffer.append(replacementStr); - lastAppendEnd = matcher.end(); - } else { - // Normal replacement: replace the entire match - resultBuffer.append(inputStr, lastAppendEnd, matcher.start()); - resultBuffer.append(replacementStr); - lastAppendEnd = matcher.end(); - } + resultBuffer.append(inputStr, lastAppendEnd, matcher.start()); + resultBuffer.append(replacementStr); + lastAppendEnd = matcher.end(); } // If not a global match, break after the first replacement @@ -3058,16 +3021,9 @@ public static RuntimeBase replaceRegex(RuntimeScalar quotedRegex, RuntimeScalar } if (retryReplacementStr != null) { - if (regex.hasBackslashK) { - int keepEnd = retryMatcher.end("perlK"); - resultBuffer.append(inputStr, lastAppendEnd, keepEnd); - resultBuffer.append(retryReplacementStr); - lastAppendEnd = retryMatcher.end(); - } else { - resultBuffer.append(inputStr, lastAppendEnd, retryMatcher.start()); - resultBuffer.append(retryReplacementStr); - lastAppendEnd = retryMatcher.end(); - } + resultBuffer.append(inputStr, lastAppendEnd, retryMatcher.start()); + resultBuffer.append(retryReplacementStr); + lastAppendEnd = retryMatcher.end(); } searchStart = retryMatcher.end(); globalPosition = searchStart; @@ -3256,12 +3212,10 @@ public static RuntimeScalar matcherStart(int group) { return scalarUndef; } try { - // Adjust group number to skip the internal perlK group - int javaGroup = adjustGroupForBackslashK(group); - if (javaGroup < 0 || javaGroup > state().globalMatcher.groupCount()) { + if (group < 0 || group > state().globalMatcher.groupCount()) { return scalarUndef; } - int start = state().globalMatcher.start(javaGroup); + int start = state().globalMatcher.start(group); if (start == -1) { return scalarUndef; } @@ -3282,12 +3236,10 @@ public static RuntimeScalar matcherEnd(int group) { return scalarUndef; } try { - // Adjust group number to skip the internal perlK group - int javaGroup = adjustGroupForBackslashK(group); - if (javaGroup < 0 || javaGroup > state().globalMatcher.groupCount()) { + if (group < 0 || group > state().globalMatcher.groupCount()) { return scalarUndef; } - int end = state().globalMatcher.end(javaGroup); + int end = state().globalMatcher.end(group); if (end == -1) { return scalarUndef; } @@ -3317,10 +3269,6 @@ public static int matcherSize() { return 0; } int size = state().globalMatcher.groupCount(); - // Subtract the internal perlK group if \K was used - if (state().lastMatchUsedBackslashK) { - size--; - } // +1 because groupCount is zero-based, and we include the entire match return size + 1; } @@ -3334,20 +3282,6 @@ public static int matcherStartSize() { return size; } - /** - * Adjust a Perl capture group number to a Java matcher group number, - * skipping the internal perlK named group when \K is active. - */ - private static int adjustGroupForBackslashK(int perlGroup) { - if (!state().lastMatchUsedBackslashK || state().globalMatcher == null) { - return perlGroup; - } - int perlKGroup = getPerlKGroup(state().globalMatcher); - if (perlKGroup < 0) return perlGroup; - // Perl groups before perlK: same number. At or after: add 1. - return perlGroup >= perlKGroup ? perlGroup + 1 : perlGroup; - } - /** * Check if a string contains any non-ASCII characters (code point > 127). * Used to determine if Unicode matching should be used. @@ -3817,13 +3751,4 @@ private static boolean containsWideChars(String s) { return false; } - /** - * Get the group number of the internal perlK named capture group. - * This group is inserted by the preprocessor at the \K position. - */ - private static int getPerlKGroup(RegexMatcher matcher) { - Map namedGroups = matcher.namedGroups(); - Integer group = namedGroups.get("perlK"); - return group != null ? group : -1; - } } diff --git a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java index 4810b02fc4..d1505bd9d6 100644 --- a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java +++ b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java @@ -978,6 +978,26 @@ static boolean isPerlBuiltInPropertyAlias(String property) { || resolvePerlBuiltInPropertyAlias(property) != null; } + /** Returns pinned Perl-property ranges in Joni's native range-array format. */ + static int[] resolveJoniPropertyRanges(String property) { + if (property == null) return null; + int assignment = propertyValueDelimiter(property); + if (assignment <= 0 || assignment == property.length() - 1 + || !isGeneralCategoryProperty(property.substring(0, assignment))) { + return null; + } + UnicodeSet set = resolvePerlBuiltInPropertyAlias(property); + if (set == null) return null; + + int[] ranges = new int[set.getRangeCount() * 2 + 1]; + ranges[0] = set.getRangeCount(); + for (int i = 0; i < set.getRangeCount(); i++) { + ranges[i * 2 + 1] = set.getRangeStart(i); + ranges[i * 2 + 2] = set.getRangeEnd(i); + } + return ranges; + } + private static boolean isPerlSpecialPropertyAlias(String property) { return switch (property) { case "lb=cr", "lb=CR", diff --git a/src/test/java/org/perlonjava/runtime/regex/JoniRegexPatternTest.java b/src/test/java/org/perlonjava/runtime/regex/JoniRegexPatternTest.java index d6d5c65fe8..8652867941 100644 --- a/src/test/java/org/perlonjava/runtime/regex/JoniRegexPatternTest.java +++ b/src/test/java/org/perlonjava/runtime/regex/JoniRegexPatternTest.java @@ -100,6 +100,21 @@ void translatesPerlOnlyUnicodePropertyAliasesBeforeJoniCompilation() { .matcher("_", java.util.List.of()).find()); } + @Test + void passesPinnedGeneralCategoriesToTheJoniParserWithoutTextExpansion() { + JoniRegexPattern category = new JoniRegexPattern("\\p{gc=Uppercase_Letter}", FLAGS); + + assertEquals("\\p{gc=Uppercase_Letter}", category.patternDescription()); + assertTrue(category.matcher("A", java.util.List.of()).find()); + assertFalse(category.matcher("a", java.util.List.of()).find()); + JoniRegexPattern folded = new JoniRegexPattern("\\p{gc=Uppercase_Letter}", + RegexFlags.fromModifiers("i", "\\p{gc=Uppercase_Letter}")); + JoniRegexPattern foldedNegated = new JoniRegexPattern("\\P{gc=Uppercase_Letter}", + RegexFlags.fromModifiers("i", "\\P{gc=Uppercase_Letter}")); + assertTrue(folded.matcher("a", java.util.List.of()).find()); + assertFalse(foldedNegated.matcher("A", java.util.List.of()).find()); + } + @Test void flattensTranslatedPropertiesInsideOrdinaryCharacterClasses() { JoniRegexPattern pattern = new JoniRegexPattern( diff --git a/src/test/java/org/perlonjava/runtime/regex/NativeKeepRoutingTest.java b/src/test/java/org/perlonjava/runtime/regex/NativeKeepRoutingTest.java new file mode 100644 index 0000000000..2202ddd875 --- /dev/null +++ b/src/test/java/org/perlonjava/runtime/regex/NativeKeepRoutingTest.java @@ -0,0 +1,43 @@ +package org.perlonjava.runtime.regex; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; +import org.perlonjava.runtime.runtimetypes.PerlCompilerException; + +class NativeKeepRoutingTest { + @Test + void admitsOnlyRealKeepAssertions() { + assertTrue(JoniRegexPattern.requiresJoniBackend("ab\\Kcd")); + assertFalse(JoniRegexPattern.requiresJoniBackend("ab\\\\Kcd")); + assertFalse(JoniRegexPattern.requiresJoniBackend("[\\K]")); + assertFalse(JoniRegexPattern.requiresJoniBackend("[[:alpha:]\\K]")); + assertFalse(JoniRegexPattern.requiresJoniBackend("\\Q\\K\\E")); + assertFalse(JoniRegexPattern.requiresJoniBackend("(?# \\K ignored)abc")); + assertFalse(JoniRegexPattern.requiresJoniBackend("(?[ [\\K] ])")); + assertFalse(JoniRegexPattern.requiresJoniBackend("abc\\")); + assertFalse(JoniRegexPattern.requiresJoniBackend("\\k")); + } + + @Test + void ignoresKeepTextInExtendedComments() { + String pattern = "abc # \\K ignored\ndef"; + RegexFlags flags = RegexFlags.fromModifiers("x", pattern); + assertFalse(JoniRegexPattern.requiresJoniBackend(pattern, flags)); + } + + @Test + void rejectsKeepInsideLookaroundBeforeJoniCompilation() { + for (String pattern : new String[] { + "ab(?=c\\Kd)", "ab(?!c\\Kd)", "(?<=a\\Kb)c", "(? new JoniRegexPattern(pattern, flags)); + assertTrue(error.getMessage().contains( + "\\K not permitted in lookahead/lookbehind")); + } + } +} diff --git a/src/test/resources/unit/regex/branch_reset_named_call.t b/src/test/resources/unit/regex/branch_reset_named_call.t new file mode 100644 index 0000000000..7b7c5aaf19 --- /dev/null +++ b/src/test/resources/unit/regex/branch_reset_named_call.t @@ -0,0 +1,18 @@ +use strict; +use warnings; +use Test::More tests => 10; + +my $regex = eval { qr/(?|(?1)|(?2))(?&digit)/ }; +ok(defined $regex, 'duplicate named branch-reset call compiles'); +is($@, '', 'compilation reports no error'); + +ok('11' =~ $regex, 'first alternative can call the named group'); +is($1, '1', 'first alternative publishes its numbered capture'); +is($+{digit}, '1', 'first alternative publishes its named capture'); + +ok('21' =~ $regex, 'second alternative calls the leftmost named definition'); +is($1, '2', 'second alternative preserves its numbered capture'); +is($+{digit}, '2', 'second alternative preserves its named capture'); + +ok('12' !~ $regex, 'named call does not target the second definition'); +ok('22' !~ $regex, 'second alternative still calls the leftmost definition'); diff --git a/src/test/resources/unit/regex/branch_reset_physical_named_capture.t b/src/test/resources/unit/regex/branch_reset_physical_named_capture.t new file mode 100644 index 0000000000..38ef2738c8 --- /dev/null +++ b/src/test/resources/unit/regex/branch_reset_physical_named_capture.t @@ -0,0 +1,21 @@ +use strict; +use warnings; +use Test::More tests => 3; + +for my $case ( + ['1', '1 ', 'first branch publishes first physical named slot'], + ['2', ' 2', 'second branch publishes second physical named slot'], +) { + my ($subject, $expected, $name) = @$case; + my $published = ''; + if ($subject =~ /(?|(?1)|(?2))/) { + $published = join ' ', map { defined $_ ? $_ : '' } @{$-{digit}}; + } + is($published, $expected, $name); +} + +my $called = ''; +if ('11' =~ /(?|(?1)|(?2))(?&digit)/) { + $called = join ' ', map { defined $_ ? $_ : '' } @{$-{digit}}; +} +is($called, '1 ', 'named call retains leftmost physical capture publication'); diff --git a/src/test/resources/unit/regex/nested_lookahead_lookbehind.t b/src/test/resources/unit/regex/nested_lookahead_lookbehind.t new file mode 100644 index 0000000000..ff31ccd1be --- /dev/null +++ b/src/test/resources/unit/regex/nested_lookahead_lookbehind.t @@ -0,0 +1,54 @@ +use strict; +use warnings; +no warnings 'experimental::vlb'; +use Test::More; + +ok('ab' =~ /(?<=a(?=b))b/, + 'positive lookahead is admitted inside positive lookbehind'); +ok('ac' =~ /(?<=a(?!b))c/, + 'negative lookahead is admitted inside positive lookbehind'); +ok('ab' !~ /(?<=a(?!b))b/, + 'nested negative lookahead still rejects its forbidden suffix'); + +ok('ab' !~ /(?/; +is($capture_substitution, 'ab', + 'KEEP substitution retains captures before the reset'); + +my $backtracking = 'abcx'; +ok($backtracking =~ /ab\Kc(?:z|x)/, 'KEEP survives successful backtracking'); +is($&, 'cx', 'backtracked match retains the final KEEP start'); + +my $failed_keep_branch = 'ax'; +ok($failed_keep_branch =~ /(?:a\Kb|a)x/, + 'failed KEEP branch restores the prior match start'); +is($&, 'ax', 'failed KEEP branch does not leak its reset'); +is($-[0], 0, 'failed KEEP branch restores offset zero'); + +ok('abc' =~ /a\Kb\Kc/, 'multiple KEEP assertions match'); +is($&, 'c', 'last successful KEEP assertion wins'); +ok('abc' =~ /abc\K/, 'terminal KEEP permits an empty match'); +is($&, '', 'terminal KEEP resets to an empty complete match'); +is($-[0], 3, 'terminal KEEP start is the input end'); + +my $unicode = "préfixe"; +ok($unicode =~ /pré\Kfixe/, 'Unicode KEEP pattern matches'); +is($&, 'fixe', 'Unicode KEEP reports character-aligned match text'); +is($-[0], 3, 'Unicode KEEP reports Perl character offset'); + +{ + use bytes; + my $bytes = pack('C*', 0x61, 0x62, 0x63, 0x64); + ok($bytes =~ /ab\Kcd/, 'byte input KEEP pattern matches'); + is($&, 'cd', 'byte input KEEP reports post-reset bytes'); + is($-[0], 2, 'byte input KEEP reports byte offset'); +} + +{ + no warnings 'regexp'; + ok('\\K' =~ /\A\\K\z/, 'escaped backslash K remains literal'); + ok('K' =~ /\A[\K]\z/, 'K inside a class remains a class member'); + ok('\\K' =~ /\A\Q\K\E\z/, 'K inside quote-meta remains literal'); +} + +my $lookahead = 'ab(?=c\\Kd)'; +eval { qr/$lookahead/ }; +like($@, qr/\\K not permitted in lookahead\/lookbehind/, + 'KEEP in lookahead remains a compile error'); + +my $lookbehind = '(?<=a\\Kb)c'; +eval { qr/$lookbehind/ }; +like($@, qr/\\K not permitted in lookahead\/lookbehind/, + 'KEEP in lookbehind remains a compile error'); + +done_testing; diff --git a/src/test/resources/unit/regex_joni_native_named_character.t b/src/test/resources/unit/regex_joni_native_named_character.t new file mode 100644 index 0000000000..5035affdb8 --- /dev/null +++ b/src/test/resources/unit/regex_joni_native_named_character.t @@ -0,0 +1,39 @@ +use strict; +use warnings; +use utf8; +use Test::More tests => 11; + +ok("A" =~ /\N{LATIN CAPITAL LETTER A}/, + 'official named character resolves outside a class'); +ok("A" =~ /[\N{LATIN CAPITAL LETTER A}]/, + 'official named character resolves inside a class'); +ok("\x{1F642}" =~ /\N{U+1F642}/, + 'supplementary named character resolves outside a class'); +ok("\x{1F642}" =~ /[\N{U+1F642}]/, + 'supplementary named character resolves inside a class'); +ok("a" =~ /\N{LATIN CAPITAL LETTER A}/i, + 'case folding applies after named-character resolution'); +ok('#' =~ /(?x:\N{NUMBER SIGN})/, + 'named punctuation remains significant under extended syntax'); +ok(' ' =~ /(?x:[\N{SPACE}])/, + 'named whitespace remains significant in an extended class'); + +{ + use bytes; + my $latin1 = pack('C', 0xE9); + ok($latin1 =~ /\N{U+00E9}/, + 'named character uses the active single-byte regex mode'); +} + +eval q{qr/\N{}/}; +like($@, qr/(?:Unknown charname|zero length)/, + 'empty named-character escape remains fatal'); + +eval q{qr/\N{NOT A REAL UNICODE CHARACTER}/}; +like($@, qr/(?:Unknown charname|Invalid Unicode character name)/, + 'resolver failure remains fatal'); + +my $unterminated = '\\N{LATIN CAPITAL LETTER A'; +eval { qr/$unterminated/ }; +like($@, qr/(?:terminator|right brace|terminated)/i, + 'unterminated named-character escape remains fatal'); diff --git a/third_party/joni/src/org/joni/Analyser.java b/third_party/joni/src/org/joni/Analyser.java index c588d495d3..1e9a8ef1fb 100644 --- a/third_party/joni/src/org/joni/Analyser.java +++ b/third_party/joni/src/org/joni/Analyser.java @@ -1308,6 +1308,9 @@ protected final int subexpRecursiveCheckTrav(Node node) { private void setCallAttr(CallNode cn) { EncloseNode en = env.memNodes[cn.groupNum]; if (en == null) newValueException(UNDEFINED_NAME_REFERENCE, cn.nameP, cn.nameEnd); + // Perl subroutine calls do not replace captures already visible in the + // caller. Reused branch-reset numbers need the existing snapshot path. + if (env.isMultiplexMemNode(cn.groupNum)) cn.setRecursion(); en.setCalled(); cn.setTarget(en); env.btMemStart = BitStatus.bsOnAt(env.btMemStart, cn.groupNum); @@ -1339,7 +1342,7 @@ protected final void setupSubExpCall(Node node) { en.recursionConditionNameP, en.recursionConditionNameEnd); } int[] refs = ne.getBackRefs(); - if (refs.length != 1) { + if (refs.length != 1 && !syntax.allowMultiplexDefinitionNameCall()) { newValueException(MULTIPLEX_DEFINITION_NAME_CALL, en.recursionConditionNameP, en.recursionConditionNameEnd); } @@ -1370,10 +1373,11 @@ protected final void setupSubExpCall(Node node) { if (ne == null) { newValueException(UNDEFINED_NAME_REFERENCE, cn.nameP, cn.nameEnd); - } else if (ne.backNum > 1) { + } else if (ne.backNum > 1 && !syntax.allowMultiplexDefinitionNameCall()) { newValueException(MULTIPLEX_DEFINITION_NAME_CALL, cn.nameP, cn.nameEnd); } else { cn.groupNum = ne.backRef1; // ne.backNum == 1 ? ne.backRef1 : ne.backRefs[0]; // ??? need to check ? + if (ne.backNum > 1) cn.setRecursion(); setCallAttr(cn); } } @@ -1983,6 +1987,7 @@ protected final int setupCombExpCheck(Node node, int state) { private static final int IN_VAR_REPEAT = (1<<3); private static final int IN_CALL = (1<<4); private static final int IN_RECCALL = (1<<5); + private static final int IN_LOOKAROUND = (1<<6); private static final int EXPAND_STRING_MAX_LENGTH = 100; /* setup_tree does the following work. @@ -2185,29 +2190,41 @@ protected final Node setupTree(Node node, int state) { AnchorNode an = (AnchorNode)node; switch (an.type) { case AnchorType.PREC_READ: - setupTree(an.target, state); + setupTree(an.target, (state | IN_LOOKAROUND)); break; case AnchorType.PREC_READ_NOT: - setupTree(an.target, (state | IN_NOT)); + setupTree(an.target, (state | IN_NOT | IN_LOOKAROUND)); break; case AnchorType.LOOK_BEHIND: - if (checkTypeTree(an.target, NodeType.ALLOWED_IN_LB, EncloseType.ALLOWED_IN_LB, AnchorType.ALLOWED_IN_LB)) newSyntaxException(INVALID_LOOK_BEHIND_PATTERN); + int allowedInLookBehind = syntax.op2OptionPerl() + ? AnchorType.ALLOWED_IN_PERL_LB + : AnchorType.ALLOWED_IN_LB; + if (checkTypeTree(an.target, NodeType.ALLOWED_IN_LB, EncloseType.ALLOWED_IN_LB, allowedInLookBehind)) newSyntaxException(INVALID_LOOK_BEHIND_PATTERN); node = setupLookBehind(an); if (node.getType() != NodeType.ANCHOR) continue restart; - setupTree(((AnchorNode)node).target, state); + setupTree(((AnchorNode)node).target, (state | IN_LOOKAROUND)); node = setupLookBehind(an); break; case AnchorType.LOOK_BEHIND_NOT: - if (checkTypeTree(an.target, NodeType.ALLOWED_IN_LB, EncloseType.ALLOWED_IN_LB_NOT, AnchorType.ALLOWED_IN_LB_NOT)) newSyntaxException(INVALID_LOOK_BEHIND_PATTERN); + int allowedInNegativeLookBehind = syntax.op2OptionPerl() + ? AnchorType.ALLOWED_IN_PERL_LB_NOT + : AnchorType.ALLOWED_IN_LB_NOT; + if (checkTypeTree(an.target, NodeType.ALLOWED_IN_LB, EncloseType.ALLOWED_IN_LB_NOT, allowedInNegativeLookBehind)) newSyntaxException(INVALID_LOOK_BEHIND_PATTERN); node = setupLookBehind(an); if (node.getType() != NodeType.ANCHOR) continue restart; - setupTree(((AnchorNode)node).target, (state | IN_NOT)); + setupTree(((AnchorNode)node).target, (state | IN_NOT | IN_LOOKAROUND)); node = setupLookBehind(an); break; + case AnchorType.KEEP: + if (syntax.op2OptionPerl() && (state & IN_LOOKAROUND) != 0) { + newSyntaxException(PERL_KEEP_NOT_PERMITTED_IN_LOOKAROUND); + } + break; + } // inner switch break; } // switch diff --git a/third_party/joni/src/org/joni/ArrayCompiler.java b/third_party/joni/src/org/joni/ArrayCompiler.java index ef595fd1a6..93b8a0511c 100644 --- a/third_party/joni/src/org/joni/ArrayCompiler.java +++ b/third_party/joni/src/org/joni/ArrayCompiler.java @@ -918,6 +918,10 @@ private int compileLengthEncloseNode(EncloseNode node) { } len += tlen + (bsAt(regex.btMemEnd, node.regNum) ? OPSize.MEMORY_END_PUSH : OPSize.MEMORY_END); } + if (node.physicalNamedCaptureId >= 0) { + len += OPSize.PHYSICAL_NAMED_CAPTURE_START + + OPSize.PHYSICAL_NAMED_CAPTURE_END; + } break; case EncloseType.STOP_BACKTRACK: @@ -977,6 +981,10 @@ protected void compileEncloseNode(EncloseNode node) { addMemNum(-1); len = compileLengthTree(node.target); len += OPSize.MEMORY_START_PUSH + OPSize.RETURN; + if (node.physicalNamedCaptureId >= 0) { + len += OPSize.PHYSICAL_NAMED_CAPTURE_START + + OPSize.PHYSICAL_NAMED_CAPTURE_END; + } if (bsAt(regex.btMemEnd, node.regNum)) { len += node.isRecursion() ? OPSize.MEMORY_END_PUSH_REC : OPSize.MEMORY_END_PUSH; } else { @@ -985,6 +993,12 @@ protected void compileEncloseNode(EncloseNode node) { addOpcodeRelAddr(OPCode.JUMP, len); } // USE_SUBEXP_CALL + if (node.physicalNamedCaptureId >= 0) { + regex.requireStack = true; + addOpcode(OPCode.PHYSICAL_NAMED_CAPTURE_START); + addMemNum(node.physicalNamedCaptureId); + } + if (bsAt(regex.btMemStart, node.regNum)) { regex.requireStack = true; addOpcode(OPCode.MEMORY_START_PUSH); @@ -1002,6 +1016,10 @@ protected void compileEncloseNode(EncloseNode node) { addOpcode(node.isRecursion() ? OPCode.MEMORY_END_REC : OPCode.MEMORY_END); } addMemNum(node.regNum); + if (node.physicalNamedCaptureId >= 0) { + addOpcode(OPCode.PHYSICAL_NAMED_CAPTURE_END); + addMemNum(node.physicalNamedCaptureId); + } addOpcode(OPCode.RETURN); } else if (Config.USE_SUBEXP_CALL && node.isRecursion()) { // USE_SUBEXP_CALL if (bsAt(regex.btMemEnd, node.regNum)) { @@ -1010,6 +1028,10 @@ protected void compileEncloseNode(EncloseNode node) { addOpcode(OPCode.MEMORY_END_REC); } addMemNum(node.regNum); + if (node.physicalNamedCaptureId >= 0) { + addOpcode(OPCode.PHYSICAL_NAMED_CAPTURE_END); + addMemNum(node.physicalNamedCaptureId); + } } else { if (bsAt(regex.btMemEnd, node.regNum)) { addOpcode(OPCode.MEMORY_END_PUSH); @@ -1017,6 +1039,10 @@ protected void compileEncloseNode(EncloseNode node) { addOpcode(OPCode.MEMORY_END); } addMemNum(node.regNum); + if (node.physicalNamedCaptureId >= 0) { + addOpcode(OPCode.PHYSICAL_NAMED_CAPTURE_END); + addMemNum(node.physicalNamedCaptureId); + } } break; diff --git a/third_party/joni/src/org/joni/ByteCodeMachine.java b/third_party/joni/src/org/joni/ByteCodeMachine.java index 2c7ffde711..65fd958029 100644 --- a/third_party/joni/src/org/joni/ByteCodeMachine.java +++ b/third_party/joni/src/org/joni/ByteCodeMachine.java @@ -258,6 +258,8 @@ private final int execute(final boolean checkThreadInterrupt) throws Interrupted case OPCode.NOT_WORD_BREAK_BOUNDARY: opWordBreakBoundary(true); continue; case OPCode.LINE_BOUNDARY: opLineBoundary(false); continue; case OPCode.NOT_LINE_BOUNDARY: opLineBoundary(true); continue; + case OPCode.PHYSICAL_NAMED_CAPTURE_START: opPhysicalNamedCaptureStart(); continue; + case OPCode.PHYSICAL_NAMED_CAPTURE_END: opPhysicalNamedCaptureEnd(); continue; case OPCode.WORD_BEGIN: opWordBegin(); continue; case OPCode.WORD_END: opWordEnd(); continue; @@ -423,6 +425,8 @@ private final int executeSb(final boolean checkThreadInterrupt) throws Interrupt case OPCode.NOT_WORD_BREAK_BOUNDARY: opWordBreakBoundary(true); continue; case OPCode.LINE_BOUNDARY: opLineBoundary(false); continue; case OPCode.NOT_LINE_BOUNDARY: opLineBoundary(true); continue; + case OPCode.PHYSICAL_NAMED_CAPTURE_START: opPhysicalNamedCaptureStart(); continue; + case OPCode.PHYSICAL_NAMED_CAPTURE_END: opPhysicalNamedCaptureEnd(); continue; case OPCode.WORD_BEGIN: opWordBeginSb(); continue; case OPCode.WORD_END: opWordEndSb(); continue; @@ -560,14 +564,31 @@ private boolean opEnd() { } // USE_FIND_LONGEST_SEARCH_ALL_OF_RANGE bestLen = n; + if (physicalNamedCaptureBeg != null) { + System.arraycopy(physicalNamedCaptureBeg, 0, + committedPhysicalNamedCaptureBeg, 0, physicalNamedCaptureBeg.length); + System.arraycopy(physicalNamedCaptureEnd, 0, + committedPhysicalNamedCaptureEnd, 0, physicalNamedCaptureEnd.length); + } final Region region = msaRegion; if (region != null) { // USE_POSIX_REGION_OPTION ... else ... region.setBeg(0, msaBegin = ((pkeep > s) ? s : pkeep) - str); region.setEnd(0, msaEnd = s - str); + CompletedRecursiveCall completed = completedRecursiveCall(); + int[] callerCaptures = completed == null + ? null : completed.frame.getCallFrameCaptureSnapshot(); + int captureCount = regex.numMem + 1; for (int i = 1; i <= regex.numMem; i++) { + boolean preserveCallerCapture = callerCaptures != null + && i == completed.frame.getCallFrameNum() + && callerCaptures[i] != INVALID_INDEX + && callerCaptures[captureCount + i] != INVALID_INDEX; int me = repeatStk[memEndStk + i]; - if (me != INVALID_INDEX) { + if (preserveCallerCapture) { + region.setBeg(i, captureBegin(i)); + region.setEnd(i, captureEnd(i)); + } else if (me != INVALID_INDEX) { int ms = repeatStk[memStartStk + i]; region.setBeg(i, (bsAt(regex.btMemStart, i) ? stack[ms].getMemPStr() : ms) - str); region.setEnd(i, (bsAt(regex.btMemEnd, i) ? stack[me].getMemPStr() : me) - str); @@ -2129,6 +2150,16 @@ private void opMemoryStartPush() { pushMemStart(mem, s); } + private void opPhysicalNamedCaptureStart() { + int capture = code[ip++]; + if (!isInsideSubexpCall(0)) pushPhysicalNamedCapture(capture, s); + } + + private void opPhysicalNamedCaptureEnd() { + int capture = code[ip++]; + if (!isInsideSubexpCall(0)) physicalNamedCaptureEnd[capture] = s; + } + private void opMemoryStart() { int mem = code[ip++]; repeatStk[memStartStk + mem] = s; @@ -2945,6 +2976,26 @@ public int captureEnd(int capture) { return (bsAt(regex.btMemEnd, capture) ? stack[value].getMemPStr() : value) - str; } + @Override + public int physicalNamedCaptureBegin(int capture) { + if (committedPhysicalNamedCaptureBeg == null + || capture <= 0 || capture >= committedPhysicalNamedCaptureBeg.length) { + return Region.REGION_NOTPOS; + } + int begin = committedPhysicalNamedCaptureBeg[capture]; + return begin == INVALID_INDEX ? Region.REGION_NOTPOS : begin - str; + } + + @Override + public int physicalNamedCaptureEnd(int capture) { + if (committedPhysicalNamedCaptureEnd == null + || capture <= 0 || capture >= committedPhysicalNamedCaptureEnd.length) { + return Region.REGION_NOTPOS; + } + int end = committedPhysicalNamedCaptureEnd[capture]; + return end == INVALID_INDEX ? Region.REGION_NOTPOS : end - str; + } + @Override public int lastClosedCapture() { CompletedRecursiveCall completed = completedRecursiveCall(); diff --git a/third_party/joni/src/org/joni/CharacterPropertyResolver.java b/third_party/joni/src/org/joni/CharacterPropertyResolver.java new file mode 100644 index 0000000000..dd31598dd1 --- /dev/null +++ b/third_party/joni/src/org/joni/CharacterPropertyResolver.java @@ -0,0 +1,32 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import org.jcodings.Encoding; + +/** Resolves syntax-specific character properties to inclusive code-point ranges. */ +@FunctionalInterface +public interface CharacterPropertyResolver { + /** + * Returns {@code [count, from1, to1, ...]} for a resolved property, or + * {@code null} to use the encoding's built-in property lookup. + */ + int[] resolve(byte[] bytes, int p, int end, Encoding encoding); +} diff --git a/third_party/joni/src/org/joni/Lexer.java b/third_party/joni/src/org/joni/Lexer.java index 99f180b79f..0e26280a87 100644 --- a/third_party/joni/src/org/joni/Lexer.java +++ b/third_party/joni/src/org/joni/Lexer.java @@ -683,6 +683,34 @@ private void fetchTokenFor_oBrace() { token.setCode(value); } + private boolean fetchTokenFor_namedCharacter() { + NamedCharacterResolver resolver = syntax.namedCharacterResolver; + if (resolver == null || !syntax.op2OptionPerl() || !left() || !peekIs('{')) { + return false; + } + + inc(); + int nameStart = p; + while (left()) { + int nameEnd = p; + fetch(); + if (c != '}') continue; + if (nameStart == nameEnd) { + newSyntaxException(PERL_EMPTY_NAMED_CHARACTER_ESCAPE); + } + int codePoint = resolver.resolve(bytes, nameStart, nameEnd, enc); + if (codePoint < 0 || codePoint > 0x10ffff) { + newValueException(ERR_INVALID_CODE_POINT_VALUE); + } + token.type = TokenType.CODE_POINT; + token.setCode(codePoint); + return true; + } + + newSyntaxException(PERL_MISSING_RIGHT_BRACE_ON_NAMED_CHARACTER_ESCAPE); + return true; // not reached + } + private void scanOriginalBracedHexCodePoint(int last) { inc(); int num = scanUnsignedHexadecimalNumber(0, 8); @@ -950,6 +978,16 @@ protected final TokenType fetchTokenInCC() { case 'u': fetchTokenInCCFor_u(); break; + case 'N': + if (!fetchTokenFor_namedCharacter()) { + unfetch(); + fetchEscapedValue(); + if (token.getC() != c) { + token.setCode(c); + token.type = TokenType.CODE_POINT; + } + } + break; case '0': case '1': case '2': @@ -1412,6 +1450,22 @@ protected final void fetchToken() { case 'u': fetchTokenFor_uHex(); break; + case 'N': + if (!fetchTokenFor_namedCharacter()) { + unfetch(); + fetchEscapedValue(); + if (token.getC() != c) { + token.type = TokenType.CODE_POINT; + token.setCode(c); + } else { + int encLength = enc.length(bytes, token.backP, stop); + if (encLength == Encoding.CHAR_INVALID) { + throw new IllegalArgumentException("Invalid character found."); + } + p = token.backP + encLength; + } + } + break; case '1': case '2': case '3': @@ -1641,20 +1695,55 @@ private void possessiveCheck() { } } - protected final int fetchCharPropertyToCType() { + protected static final class CharProperty { + final int ctype; + final int[] ranges; + + CharProperty(int ctype, int[] ranges) { + this.ctype = ctype; + this.ranges = ranges; + } + } + + protected final CharProperty fetchCharProperty() { mark(); while (left()) { int last = p; fetch(); if (c == '}') { - return enc.propertyNameToCType(bytes, _p, last); + if (syntax.characterPropertyResolver != null) { + int[] ranges = syntax.characterPropertyResolver.resolve( + bytes, _p, last, enc); + if (ranges != null) { + validateCharacterPropertyRanges(ranges); + return new CharProperty(0, ranges); + } + } + return new CharProperty( + enc.propertyNameToCType(bytes, _p, last), null); } else if (c == '(' || c == ')' || c == '{' || c == '|') { throw new CharacterPropertyException(EncodingError.ERR_INVALID_CHAR_PROPERTY_NAME, bytes, _p, last); } } newValueException(PROPERTY_NAME_NEVER_TERMINATED, _p, stop); - return 0; // not reached + return null; // not reached + } + + private static void validateCharacterPropertyRanges(int[] ranges) { + if (ranges.length == 0 || ranges.length != ranges[0] * 2 + 1) { + throw new IllegalArgumentException("invalid character property ranges"); + } + int previousEnd = -1; + for (int i = 0; i < ranges[0]; i++) { + int from = ranges[i * 2 + 1]; + int to = ranges[i * 2 + 2]; + if (from < 0 || from > to || to > CodeRangeBuffer.LAST_CODE_POINT + || from <= previousEnd) { + throw new IllegalArgumentException("invalid character property ranges"); + } + previousEnd = to; + } } protected final void syntaxWarn(String message, char c) { diff --git a/third_party/joni/src/org/joni/Matcher.java b/third_party/joni/src/org/joni/Matcher.java index a42e8473c7..d3678119d6 100644 --- a/third_party/joni/src/org/joni/Matcher.java +++ b/third_party/joni/src/org/joni/Matcher.java @@ -130,6 +130,15 @@ public int captureEnd(int capture) { return capture < region.getNumRegs() ? region.getEnd(capture) : Region.REGION_NOTPOS; } + /** Physical named-group definition offsets, used when definitions share a number. */ + public int physicalNamedCaptureBegin(int capture) { + return Region.REGION_NOTPOS; + } + + public int physicalNamedCaptureEnd(int capture) { + return Region.REGION_NOTPOS; + } + /** Most recently closed active capture; engines without this view return -1. */ public int lastClosedCapture() { return -1; diff --git a/third_party/joni/src/org/joni/NameEntry.java b/third_party/joni/src/org/joni/NameEntry.java index 2b62513056..373d89c043 100644 --- a/third_party/joni/src/org/joni/NameEntry.java +++ b/third_party/joni/src/org/joni/NameEntry.java @@ -29,6 +29,8 @@ public final class NameEntry { int backNum; int backRef1; int[] backRefs; + int physicalRef1; + int[] physicalRefs; public NameEntry(byte[]bytes, int p, int end) { name = bytes; @@ -49,8 +51,22 @@ public int[] getBackRefs() { } } + public int[] getPhysicalBackRefs() { + switch (backNum) { + case 0: + return new int[]{}; + case 1: + return new int[]{physicalRef1}; + default: + int[] result = new int[backNum]; + System.arraycopy(physicalRefs, 0, result, 0, backNum); + return result; + } + } + private void alloc() { backRefs = new int[INIT_NAME_BACKREFS_ALLOC_NUM]; + physicalRefs = new int[INIT_NAME_BACKREFS_ALLOC_NUM]; } private void ensureSize() { @@ -58,24 +74,31 @@ private void ensureSize() { int[]tmp = new int[backRefs.length << 1]; System.arraycopy(backRefs, 0, tmp, 0, backRefs.length); backRefs = tmp; + int[] physicalTmp = new int[physicalRefs.length << 1]; + System.arraycopy(physicalRefs, 0, physicalTmp, 0, physicalRefs.length); + physicalRefs = physicalTmp; } } - public void addBackref(int backRef) { + public void addBackref(int backRef, int physicalRef) { backNum++; switch (backNum) { case 1: backRef1 = backRef; + physicalRef1 = physicalRef; break; case 2: alloc(); backRefs[0] = backRef1; backRefs[1] = backRef; + physicalRefs[0] = physicalRef1; + physicalRefs[1] = physicalRef; break; default: ensureSize(); backRefs[backNum - 1] = backRef; + physicalRefs[backNum - 1] = physicalRef; } } diff --git a/third_party/joni/src/org/joni/NamedCharacterResolver.java b/third_party/joni/src/org/joni/NamedCharacterResolver.java new file mode 100644 index 0000000000..e815ab840a --- /dev/null +++ b/third_party/joni/src/org/joni/NamedCharacterResolver.java @@ -0,0 +1,27 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import org.jcodings.Encoding; + +@FunctionalInterface +public interface NamedCharacterResolver { + int resolve(byte[] bytes, int p, int end, Encoding encoding); +} diff --git a/third_party/joni/src/org/joni/Parser.java b/third_party/joni/src/org/joni/Parser.java index 247be8edce..af9cbfc07d 100644 --- a/third_party/joni/src/org/joni/Parser.java +++ b/third_party/joni/src/org/joni/Parser.java @@ -256,13 +256,8 @@ private CClassNode parseCharClass(ObjPtr ascNode) { break; case CHAR_PROPERTY: - int ctype = fetchCharPropertyToCType(); - cc.addCType(ctype, token.getPropNot(), false, env, this); - if (ascCc != null) { - if (ctype != CharacterType.ASCII) { - ascCc.addCType(ctype, token.getPropNot(), false, env, this); - } - } + CharProperty property = fetchCharProperty(); + addCharProperty(cc, ascCc, property, token.getPropNot()); cc.nextStateClass(arg, ascCc, env); // goto next_class break; @@ -1069,8 +1064,8 @@ private Node parseEncloseNamedGroup2(boolean listCapture) { int num = env.addMemEntry(); if (listCapture && num >= BitStatus.BIT_STATUS_BITS_NUM) newValueException(GROUP_NUMBER_OVER_FOR_CAPTURE_HISTORY); - regex.nameAdd(bytes, nm, nameEnd, num, syntax); EncloseNode en = EncloseNode.newMemory(env.option, true); + en.physicalNamedCaptureId = regex.nameAdd(bytes, nm, nameEnd, num, syntax); en.regNum = num; if (listCapture) env.captureHistory = bsOnAtSimple(env.captureHistory, num); @@ -1634,20 +1629,33 @@ private Node cClassCaseFold(Node node, CClassNode cc, CClassNode ascCc) { } private Node parseCharProperty() { - int ctype = fetchCharPropertyToCType(); + CharProperty property = fetchCharProperty(); CClassNode cc = new CClassNode(); Node node = cc; - cc.addCType(ctype, false, false, env, this); + addCharProperty(cc, null, property, false); if (token.getPropNot()) cc.setNot(); if (isIgnoreCase(env.option)) { - if (ctype != CharacterType.ASCII) { + if (property.ranges != null || property.ctype != CharacterType.ASCII) { node = cClassCaseFold(node, cc, cc); } } return node; } + private void addCharProperty(CClassNode cc, CClassNode ascCc, + CharProperty property, boolean not) { + if (property.ranges == null) { + cc.addCType(property.ctype, not, false, env, this); + if (ascCc != null && property.ctype != CharacterType.ASCII) { + ascCc.addCType(property.ctype, not, false, env, this); + } + return; + } + cc.addCodeRanges(property.ranges, not, env); + if (ascCc != null) ascCc.addCodeRanges(property.ranges, not, env); + } + private Node parseAnycharAnytime() { Node node = new AnyCharNode(); QuantifierNode qn = new QuantifierNode(0, QuantifierNode.REPEAT_INFINITE, false); diff --git a/third_party/joni/src/org/joni/Regex.java b/third_party/joni/src/org/joni/Regex.java index 9de66c6364..7aaf9b7859 100644 --- a/third_party/joni/src/org/joni/Regex.java +++ b/third_party/joni/src/org/joni/Regex.java @@ -46,6 +46,7 @@ public final class Regex { boolean hasDynamicOptions; int numMem; /* used memory(...) num counted from 1 */ + int numPhysicalNamedCaptures; int numRepeat; /* OP_REPEAT/OP_REPEAT_NG id-counter */ int numNullCheck; /* OP_NULL_CHECK_START/END id counter */ int numCombExpCheck; /* combination explosion check */ @@ -239,7 +240,7 @@ void renumberNameTable(int[]map) { } } - void nameAdd(byte[]name, int nameP, int nameEnd, int backRef, Syntax syntax) { + int nameAdd(byte[]name, int nameP, int nameEnd, int backRef, Syntax syntax) { if (nameEnd - nameP <= 0) throw new ValueException(ErrorMessages.EMPTY_GROUP_NAME); NameEntry e = null; @@ -257,7 +258,13 @@ void nameAdd(byte[]name, int nameP, int nameEnd, int backRef, Syntax syntax) { throw new ValueException(ErrorMessages.MULTIPLEX_DEFINED_NAME, new String(name, nameP, nameEnd - nameP)); } - e.addBackref(backRef); + int physicalRef = ++numPhysicalNamedCaptures; + e.addBackref(backRef, physicalRef); + return physicalRef; + } + + public int numberOfPhysicalNamedCaptures() { + return numPhysicalNamedCaptures; } NameEntry nameToGroupNumbers(byte[]name, int nameP, int nameEnd) { diff --git a/third_party/joni/src/org/joni/ScanEnvironment.java b/third_party/joni/src/org/joni/ScanEnvironment.java index 01090a03de..cc88fc0717 100644 --- a/third_party/joni/src/org/joni/ScanEnvironment.java +++ b/third_party/joni/src/org/joni/ScanEnvironment.java @@ -41,6 +41,7 @@ public final class ScanEnvironment { int numCall; UnsetAddrList unsetAddrList; // USE_SUBEXP_CALL public int numMem; + boolean[] multiplexMemNodes; int numNamed; // USE_NAMED_GROUP @@ -75,11 +76,19 @@ int caseFoldFlagFor(int option) { int addMemEntry() { if (numMem >= Config.MAX_CAPTURE_GROUP_NUM) throw new InternalException(ErrorMessages.TOO_MANY_CAPTURE_GROUPS); if (numMem++ == 0) { - memNodes = new EncloseNode[Config.SCANENV_MEMNODES_SIZE]; + // Branch-reset parsing can rewind numMem without starting a new + // parse. Keep nodes recorded by earlier alternatives. + if (memNodes == null) { + memNodes = new EncloseNode[Config.SCANENV_MEMNODES_SIZE]; + multiplexMemNodes = new boolean[Config.SCANENV_MEMNODES_SIZE]; + } } else if (numMem >= memNodes.length) { EncloseNode[]tmp = new EncloseNode[memNodes.length << 1]; System.arraycopy(memNodes, 0, tmp, 0, memNodes.length); memNodes = tmp; + boolean[] multiplexTmp = new boolean[multiplexMemNodes.length << 1]; + System.arraycopy(multiplexMemNodes, 0, multiplexTmp, 0, multiplexMemNodes.length); + multiplexMemNodes = multiplexTmp; } return numMem; @@ -87,12 +96,22 @@ int addMemEntry() { void setMemNode(int num, EncloseNode node) { if (numMem >= num) { - memNodes[num] = node; + // Branch-reset alternatives reuse capture numbers. Subexpression + // calls target the leftmost physical group with that number. + if (memNodes[num] == null) { + memNodes[num] = node; + } else if (memNodes[num] != node) { + multiplexMemNodes[num] = true; + } } else { throw new InternalException(ErrorMessages.PARSER_BUG); } } + boolean isMultiplexMemNode(int num) { + return multiplexMemNodes != null && multiplexMemNodes[num]; + } + void pushPrecReadNotNode(Node node) { numPrecReadNotNodes++; diff --git a/third_party/joni/src/org/joni/StackEntry.java b/third_party/joni/src/org/joni/StackEntry.java index 4258bc1f50..a9600ee438 100644 --- a/third_party/joni/src/org/joni/StackEntry.java +++ b/third_party/joni/src/org/joni/StackEntry.java @@ -128,6 +128,24 @@ int getMemEnd() { return E4; } + void setPhysicalNamedCapture(int capture, int begin, int end) { + E1 = capture; + E2 = begin; + E3 = end; + } + + int getPhysicalNamedCapture() { + return E1; + } + + int getPhysicalNamedCaptureBegin() { + return E2; + } + + int getPhysicalNamedCaptureEnd() { + return E3; + } + // fifth union member /* null check id */ void setNullCheckNum(int num) { diff --git a/third_party/joni/src/org/joni/StackMachine.java b/third_party/joni/src/org/joni/StackMachine.java index c0e13eb50f..61fc610fd2 100644 --- a/third_party/joni/src/org/joni/StackMachine.java +++ b/third_party/joni/src/org/joni/StackMachine.java @@ -34,6 +34,10 @@ abstract class StackMachine extends Matcher implements StackType { protected StackEntry[]stack; protected int stk; // stkEnd protected final int[]repeatStk; + protected final int[] physicalNamedCaptureBeg; + protected final int[] physicalNamedCaptureEnd; + protected final int[] committedPhysicalNamedCaptureBeg; + protected final int[] committedPhysicalNamedCaptureEnd; protected final int memStartStk, memEndStk; protected byte[] stateCheckBuff; // CEC, move to int[] ? protected int stateCheckBuffSize; @@ -54,6 +58,11 @@ protected StackMachine(Regex regex, Region region, byte[]bytes, int p , int end) /* for index start from 1, mem_end_stk[1]..mem_end_stk[num_mem] */ } repeatStk = n > 0 ? new int[n] : null; + int physicalCount = regex.numPhysicalNamedCaptures; + physicalNamedCaptureBeg = physicalCount == 0 ? null : new int[physicalCount + 1]; + physicalNamedCaptureEnd = physicalCount == 0 ? null : new int[physicalCount + 1]; + committedPhysicalNamedCaptureBeg = physicalCount == 0 ? null : new int[physicalCount + 1]; + committedPhysicalNamedCaptureEnd = physicalCount == 0 ? null : new int[physicalCount + 1]; } protected final void stackInit() { @@ -63,6 +72,28 @@ protected final void stackInit() { repeatStk[i + memStartStk] = repeatStk[i + memEndStk] = INVALID_INDEX; } } + if (physicalNamedCaptureBeg != null) { + Arrays.fill(physicalNamedCaptureBeg, INVALID_INDEX); + Arrays.fill(physicalNamedCaptureEnd, INVALID_INDEX); + Arrays.fill(committedPhysicalNamedCaptureBeg, INVALID_INDEX); + Arrays.fill(committedPhysicalNamedCaptureEnd, INVALID_INDEX); + } + } + + protected final void pushPhysicalNamedCapture(int capture, int position) { + StackEntry e = ensure1(); + e.type = PHYSICAL_NAMED_CAPTURE; + e.setPhysicalNamedCapture(capture, + physicalNamedCaptureBeg[capture], physicalNamedCaptureEnd[capture]); + physicalNamedCaptureBeg[capture] = position; + physicalNamedCaptureEnd[capture] = INVALID_INDEX; + stk++; + } + + private void restorePhysicalNamedCapture(StackEntry e) { + int capture = e.getPhysicalNamedCapture(); + physicalNamedCaptureBeg[capture] = e.getPhysicalNamedCaptureBegin(); + physicalNamedCaptureEnd[capture] = e.getPhysicalNamedCaptureEnd(); } private static StackEntry[] allocateStack() { @@ -408,6 +439,8 @@ private StackEntry popFree() { unwindCallout(e); } else if (e.type == CONTROL_MARK) { restoreControlMark(e.getPreviousControlMarkName()); + } else if (e.type == PHYSICAL_NAMED_CAPTURE) { + restorePhysicalNamedCapture(e); } else if (USE_CEC) { if (e.type == STATE_CHECK_MARK) stateCheckMark(); } @@ -423,6 +456,8 @@ private StackEntry popMemStart() { unwindCallout(e); } else if (e.type == CONTROL_MARK) { restoreControlMark(e.getPreviousControlMarkName()); + } else if (e.type == PHYSICAL_NAMED_CAPTURE) { + restorePhysicalNamedCapture(e); } else if (e.type == MEM_START) { repeatStk[memStartStk + e.getMemNum()] = e.getMemStart(); repeatStk[memEndStk + e.getMemNum()] = e.getMemEnd(); @@ -437,6 +472,8 @@ private void popRewrite(StackEntry e) { unwindCallout(e); } else if (e.type == CONTROL_MARK) { restoreControlMark(e.getPreviousControlMarkName()); + } else if (e.type == PHYSICAL_NAMED_CAPTURE) { + restorePhysicalNamedCapture(e); } else if (e.type == MEM_START) { repeatStk[memStartStk + e.getMemNum()] = e.getMemStart(); repeatStk[memEndStk + e.getMemNum()] = e.getMemEnd(); diff --git a/third_party/joni/src/org/joni/Syntax.java b/third_party/joni/src/org/joni/Syntax.java index c24db21a65..c7a9477f40 100644 --- a/third_party/joni/src/org/joni/Syntax.java +++ b/third_party/joni/src/org/joni/Syntax.java @@ -31,8 +31,24 @@ public final class Syntax implements SyntaxProperties { public final int behavior; public final int options; public final MetaCharTable metaCharTable; + public final NamedCharacterResolver namedCharacterResolver; + public final CharacterPropertyResolver characterPropertyResolver; public Syntax(String name, int op, int op2, int op3, int behavior, int options, MetaCharTable metaCharTable) { + this(name, op, op2, op3, behavior, options, metaCharTable, null); + } + + public Syntax(String name, int op, int op2, int op3, int behavior, int options, + MetaCharTable metaCharTable, + NamedCharacterResolver namedCharacterResolver) { + this(name, op, op2, op3, behavior, options, metaCharTable, + namedCharacterResolver, null); + } + + public Syntax(String name, int op, int op2, int op3, int behavior, int options, + MetaCharTable metaCharTable, + NamedCharacterResolver namedCharacterResolver, + CharacterPropertyResolver characterPropertyResolver) { this.name = name; this.op = op; this.op2 = op2; @@ -40,6 +56,8 @@ public Syntax(String name, int op, int op2, int op3, int behavior, int options, this.behavior = behavior; this.options = options; this.metaCharTable = metaCharTable; + this.namedCharacterResolver = namedCharacterResolver; + this.characterPropertyResolver = characterPropertyResolver; } public static class MetaCharTable { @@ -386,6 +404,10 @@ public boolean allowMultiplexDefinitionName() { return isBehavior(ALLOW_MULTIPLEX_DEFINITION_NAME); } + public boolean allowMultiplexDefinitionNameCall() { + return isBehavior(ALLOW_MULTIPLEX_DEFINITION_NAME_CALL); + } + public boolean fixedIntervalIsGreedyOnly() { return isBehavior(FIXED_INTERVAL_IS_GREEDY_ONLY); } diff --git a/third_party/joni/src/org/joni/ast/CClassNode.java b/third_party/joni/src/org/joni/ast/CClassNode.java index ecc724646f..1502271163 100644 --- a/third_party/joni/src/org/joni/ast/CClassNode.java +++ b/third_party/joni/src/org/joni/ast/CClassNode.java @@ -288,6 +288,16 @@ public void addCTypeByRange(int ctype, boolean not, ScanEnvironment env, int sbO } } + /** Adds resolver-provided {@code [count, from, to, ...]} code-point ranges. */ + public void addCodeRanges(int[] ranges, boolean not, ScanEnvironment env) { + int singleByteLimit = 0; + while (singleByteLimit < BitSet.SINGLE_BYTE_SIZE + && env.enc.codeToMbcLength(singleByteLimit) == 1) { + singleByteLimit++; + } + addCTypeByRange(0, not, env, singleByteLimit, ranges); + } + private static int CR_FROM(int[] range, int i) { return range[(i * 2) + 1]; } diff --git a/third_party/joni/src/org/joni/ast/EncloseNode.java b/third_party/joni/src/org/joni/ast/EncloseNode.java index 13ad0c5bc0..4a42a5b5de 100644 --- a/third_party/joni/src/org/joni/ast/EncloseNode.java +++ b/third_party/joni/src/org/joni/ast/EncloseNode.java @@ -26,6 +26,7 @@ public final class EncloseNode extends StateNode implements EncloseType { public final int type; // enclose type public int regNum; + public int physicalNamedCaptureId = -1; public int option; public Node target; /* EncloseNode : ENCLOSE_MEMORY */ public int callAddr; // AbsAddrType diff --git a/third_party/joni/src/org/joni/constants/internal/AnchorType.java b/third_party/joni/src/org/joni/constants/internal/AnchorType.java index df6138588c..3b2f912ae8 100644 --- a/third_party/joni/src/org/joni/constants/internal/AnchorType.java +++ b/third_party/joni/src/org/joni/constants/internal/AnchorType.java @@ -92,4 +92,7 @@ public interface AnchorType { NOT_LINE_BOUNDARY | WORD_BEGIN | WORD_END ); + + int ALLOWED_IN_PERL_LB = ALLOWED_IN_LB | PREC_READ | PREC_READ_NOT; + int ALLOWED_IN_PERL_LB_NOT = ALLOWED_IN_LB_NOT | PREC_READ | PREC_READ_NOT; } diff --git a/third_party/joni/src/org/joni/constants/internal/OPCode.java b/third_party/joni/src/org/joni/constants/internal/OPCode.java index f9a85e21dd..389ad5f9f4 100644 --- a/third_party/joni/src/org/joni/constants/internal/OPCode.java +++ b/third_party/joni/src/org/joni/constants/internal/OPCode.java @@ -162,6 +162,8 @@ public interface OPCode { int NOT_WORD_BREAK_BOUNDARY = 118; /* Perl \B{wb} */ int LINE_BOUNDARY = 119; /* Perl \b{lb} */ int NOT_LINE_BOUNDARY = 120; /* Perl \B{lb} */ + int PHYSICAL_NAMED_CAPTURE_START = 121; + int PHYSICAL_NAMED_CAPTURE_END = 122; String[] OpCodeNames = Config.DEBUG_COMPILE ? new String[] { "finish", /*OP_FINISH*/ @@ -286,6 +288,8 @@ public interface OPCode { "not-word-break-boundary", /*OP_NOT_WORD_BREAK_BOUNDARY*/ "line-boundary", /*OP_LINE_BOUNDARY*/ "not-line-boundary", /*OP_NOT_LINE_BOUNDARY*/ + "physical-named-capture-start", + "physical-named-capture-end", } : null; int[] OpCodeArgTypes = Config.DEBUG_COMPILE ? new int[] { @@ -411,5 +415,7 @@ public interface OPCode { Arguments.NON, /*OP_NOT_WORD_BREAK_BOUNDARY*/ Arguments.NON, /*OP_LINE_BOUNDARY*/ Arguments.NON, /*OP_NOT_LINE_BOUNDARY*/ + Arguments.MEMNUM, /*OP_PHYSICAL_NAMED_CAPTURE_START*/ + Arguments.MEMNUM, /*OP_PHYSICAL_NAMED_CAPTURE_END*/ } : null; } diff --git a/third_party/joni/src/org/joni/constants/internal/OPSize.java b/third_party/joni/src/org/joni/constants/internal/OPSize.java index 265f3cc662..241519343d 100644 --- a/third_party/joni/src/org/joni/constants/internal/OPSize.java +++ b/third_party/joni/src/org/joni/constants/internal/OPSize.java @@ -84,6 +84,8 @@ public interface OPSize { int RECURSION_CONDITION = (OPCODE + MEMNUM + RELADDR); int CHECK_POS_END = OPCODE; int CHECK_LOOK_BEHIND_END = OPCODE; + int PHYSICAL_NAMED_CAPTURE_START = (OPCODE + MEMNUM); + int PHYSICAL_NAMED_CAPTURE_END = (OPCODE + MEMNUM); // #ifdef USE_COMBINATION_EXPLOSION_CHECK int STATE_CHECK = (OPCODE + STATE_CHECK_NUM); diff --git a/third_party/joni/src/org/joni/constants/internal/StackType.java b/third_party/joni/src/org/joni/constants/internal/StackType.java index cc5ac9b65a..e13b640fc1 100644 --- a/third_party/joni/src/org/joni/constants/internal/StackType.java +++ b/third_party/joni/src/org/joni/constants/internal/StackType.java @@ -47,6 +47,7 @@ public interface StackType { int ABSENT = 0x0c00; /* absent inner loop marker */ int CALLOUT = 0x0d00; /* match-time callback unwind token */ int CONTROL_MARK = 0x0e00; /* Perl (*MARK:name) backtrack state */ + int PHYSICAL_NAMED_CAPTURE = 0x0f00; /* physical named-group span */ int DYNAMIC_ALT = 0x0004; /* resumable nested-program alternative */ /* stack type check mask */ diff --git a/third_party/joni/src/org/joni/exception/ErrorMessages.java b/third_party/joni/src/org/joni/exception/ErrorMessages.java index e7b4323cd4..5d80f246ae 100644 --- a/third_party/joni/src/org/joni/exception/ErrorMessages.java +++ b/third_party/joni/src/org/joni/exception/ErrorMessages.java @@ -55,6 +55,8 @@ public interface ErrorMessages extends org.jcodings.exception.ErrorMessages { String UNDEFINED_GROUP_OPTION = "undefined group option"; String INVALID_POSIX_BRACKET_TYPE = "invalid POSIX bracket type"; String INVALID_LOOK_BEHIND_PATTERN = "invalid pattern in look-behind"; + String PERL_KEEP_NOT_PERMITTED_IN_LOOKAROUND = + "\\K not permitted in lookahead/lookbehind in regex"; String INVALID_REPEAT_RANGE_PATTERN = "invalid repeat range {lower,upper}"; String INVALID_CONDITION_PATTERN = "invalid conditional pattern"; String PERL_GROUP_NAME_MUST_START_WITH_WORD = @@ -91,6 +93,9 @@ public interface ErrorMessages extends org.jcodings.exception.ErrorMessages { String PERL_EMPTY_OCTAL_ESCAPE = "Empty \\o{}"; String PERL_MISSING_RIGHT_BRACE_ON_HEX_ESCAPE = "Missing right brace on \\x{}"; + String PERL_MISSING_RIGHT_BRACE_ON_NAMED_CHARACTER_ESCAPE = + "Missing right brace on \\N{}"; + String PERL_EMPTY_NAMED_CHARACTER_ESCAPE = "Empty \\N{}"; /* values error (syntax error) */ String TOO_BIG_NUMBER = "too big number"; diff --git a/third_party/joni/test/org/joni/test/TestCharacterPropertyResolver.java b/third_party/joni/test/org/joni/test/TestCharacterPropertyResolver.java new file mode 100644 index 0000000000..974df8eeae --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestCharacterPropertyResolver.java @@ -0,0 +1,100 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.fail; + +import java.nio.charset.StandardCharsets; + +import org.jcodings.specific.UTF8Encoding; +import org.joni.CharacterPropertyResolver; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.junit.Test; + +public class TestCharacterPropertyResolver { + private static final CharacterPropertyResolver RESOLVER = + (bytes, p, end, encoding) -> { + String name = new String(bytes, p, end - p, StandardCharsets.UTF_8); + return switch (name) { + case "Fake" -> new int[] {2, 'A', 'A', 0x1f642, 0x1f642}; + default -> null; + }; + }; + + private static Syntax syntax(CharacterPropertyResolver resolver) { + return new Syntax("CharacterPropertyResolver", Syntax.PerlNG.op, + Syntax.PerlNG.op2, Syntax.PerlNG.op3, Syntax.PerlNG.behavior, + Syntax.PerlNG.options, Syntax.PerlNG.metaCharTable, null, resolver); + } + + private static Regex compile(String pattern, CharacterPropertyResolver resolver) { + byte[] bytes = pattern.getBytes(StandardCharsets.UTF_8); + return new Regex(bytes, 0, bytes.length, Option.NONE, + UTF8Encoding.INSTANCE, syntax(resolver)); + } + + private static int search(String pattern, String input) { + byte[] bytes = input.getBytes(StandardCharsets.UTF_8); + return compile(pattern, RESOLVER).matcher(bytes) + .search(0, bytes.length, Option.NONE); + } + + @Test + public void resolvesRangesInsideAndOutsideCharacterClasses() { + assertEquals(0, search("\\p{Fake}", "A")); + assertEquals(0, search("[\\p{Fake}]", "\ud83d\ude42")); + assertEquals(-1, search("\\p{Fake}", "B")); + assertEquals(0, search("\\P{Fake}", "B")); + assertEquals(-1, search("[\\P{Fake}]", "A")); + assertEquals(0, search("(?i)\\p{Fake}", "a")); + assertEquals(-1, search("(?i)\\P{Fake}", "A")); + } + + @Test + public void fallsBackToEncodingProperties() { + assertEquals(0, search("\\p{Digit}", "7")); + assertEquals(-1, search("\\p{Digit}", "A")); + } + + @Test + public void preservesResolverExceptions() { + IllegalArgumentException expected = new IllegalArgumentException("failure"); + try { + compile("\\p{Fake}", (bytes, p, end, encoding) -> { throw expected; }); + fail("expected resolver exception"); + } catch (IllegalArgumentException error) { + assertSame(expected, error); + } + } + + @Test + public void rejectsMalformedRangeResults() { + try { + compile("\\p{Fake}", (bytes, p, end, encoding) -> new int[] {1, 2}); + fail("expected invalid range result"); + } catch (IllegalArgumentException error) { + assertEquals("invalid character property ranges", error.getMessage()); + } + } +} diff --git a/third_party/joni/test/org/joni/test/TestPerlBranchResetNamedCall.java b/third_party/joni/test/org/joni/test/TestPerlBranchResetNamedCall.java new file mode 100644 index 0000000000..102c490a87 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestPerlBranchResetNamedCall.java @@ -0,0 +1,115 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import static org.joni.constants.SyntaxProperties.ALLOW_MULTIPLEX_DEFINITION_NAME_CALL; +import static org.junit.Assert.assertEquals; + +import java.nio.charset.StandardCharsets; + +import org.jcodings.specific.UTF8Encoding; +import org.joni.Matcher; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.junit.Test; + +public class TestPerlBranchResetNamedCall { + private static final Syntax PERL_SYNTAX = new Syntax( + "PERL_TEST", Syntax.RUBY.op, Syntax.RUBY.op2, Syntax.RUBY.op3, + Syntax.RUBY.behavior | ALLOW_MULTIPLEX_DEFINITION_NAME_CALL, + Syntax.RUBY.options, Syntax.RUBY.metaCharTable); + + private static Matcher matcher(String pattern, String input) { + byte[] patternBytes = pattern.getBytes(StandardCharsets.UTF_8); + byte[] inputBytes = input.getBytes(StandardCharsets.UTF_8); + Regex regex = new Regex(patternBytes, 0, patternBytes.length, + Option.CAPTURE_GROUP, UTF8Encoding.INSTANCE, PERL_SYNTAX); + return regex.matcher(inputBytes); + } + + private static Matcher assertMatches(String pattern, String input) { + Matcher matcher = matcher(pattern, input); + assertEquals(0, matcher.search(0, input.length(), Option.NONE)); + return matcher; + } + + private static void assertDoesNotMatch(String pattern, String input) { + assertEquals(-1, matcher(pattern, input).search(0, input.length(), Option.NONE)); + } + + @Test + public void namedBranchResetCallTargetsLeftmostPhysicalGroup() { + String pattern = "(?|(?1)|(?2))\\g"; + Matcher first = assertMatches(pattern, "11"); + assertEquals(0, first.getRegion().getBeg(1)); + assertEquals(1, first.getRegion().getEnd(1)); + + Matcher second = assertMatches(pattern, "21"); + assertEquals(0, second.getRegion().getBeg(1)); + assertEquals(1, second.getRegion().getEnd(1)); + + assertDoesNotMatch(pattern, "12"); + assertDoesNotMatch(pattern, "22"); + } + + @Test + public void numericBranchResetCallTargetsLeftmostPhysicalGroup() { + String pattern = "(?|(1)|(2))\\g<1>"; + assertMatches(pattern, "11"); + assertMatches(pattern, "21"); + assertDoesNotMatch(pattern, "12"); + assertDoesNotMatch(pattern, "22"); + } + + @Test + public void ordinaryMultiplexNameCallTargetsFirstDefinition() { + String pattern = "(?1)(?2)\\g"; + Matcher matcher = assertMatches(pattern, "121"); + assertEquals(0, matcher.getRegion().getBeg(1)); + assertEquals(1, matcher.getRegion().getEnd(1)); + assertDoesNotMatch(pattern, "122"); + } + + @Test + public void ordinaryCapturePublicationIsUnchanged() { + Matcher matcher = assertMatches("(a)(b)", "ab"); + assertEquals(0, matcher.getRegion().getBeg(1)); + assertEquals(1, matcher.getRegion().getEnd(1)); + assertEquals(1, matcher.getRegion().getBeg(2)); + assertEquals(2, matcher.getRegion().getEnd(2)); + } + + @Test + public void duplicateBranchResetNamesRetainPhysicalDefinitionSpans() { + String pattern = "(?|(?1)|(?2))"; + Matcher first = assertMatches(pattern, "1"); + assertEquals(0, first.physicalNamedCaptureBegin(1)); + assertEquals(1, first.physicalNamedCaptureEnd(1)); + assertEquals(-1, first.physicalNamedCaptureBegin(2)); + assertEquals(-1, first.physicalNamedCaptureEnd(2)); + + Matcher second = assertMatches(pattern, "2"); + assertEquals(-1, second.physicalNamedCaptureBegin(1)); + assertEquals(-1, second.physicalNamedCaptureEnd(1)); + assertEquals(0, second.physicalNamedCaptureBegin(2)); + assertEquals(1, second.physicalNamedCaptureEnd(2)); + } +} diff --git a/third_party/joni/test/org/joni/test/TestPerlKeepLookaround.java b/third_party/joni/test/org/joni/test/TestPerlKeepLookaround.java new file mode 100644 index 0000000000..da5052f665 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestPerlKeepLookaround.java @@ -0,0 +1,82 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import static org.joni.constants.SyntaxProperties.OP2_OPTION_PERL; +import static org.joni.constants.SyntaxProperties.OP2_OPTION_RUBY; +import static org.joni.exception.ErrorMessages.PERL_KEEP_NOT_PERMITTED_IN_LOOKAROUND; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.nio.charset.StandardCharsets; + +import org.jcodings.specific.UTF8Encoding; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.joni.exception.SyntaxException; +import org.junit.Test; + +public class TestPerlKeepLookaround { + private static final Syntax PERL_SYNTAX = new Syntax( + "PerlKeepLookaround", Syntax.RUBY.op, + (Syntax.RUBY.op2 & ~OP2_OPTION_RUBY) | OP2_OPTION_PERL, + Syntax.RUBY.op3, Syntax.RUBY.behavior, Syntax.RUBY.options, + Syntax.RUBY.metaCharTable); + + private static Regex compile(String pattern, Syntax syntax) { + byte[] bytes = pattern.getBytes(StandardCharsets.UTF_8); + return new Regex(bytes, 0, bytes.length, Option.NONE, + UTF8Encoding.INSTANCE, syntax); + } + + private static void assertPerlRejected(String pattern) { + try { + compile(pattern, PERL_SYNTAX); + fail("expected syntax error for " + pattern); + } catch (SyntaxException error) { + assertEquals(PERL_KEEP_NOT_PERMITTED_IN_LOOKAROUND, error.getMessage()); + } + } + + @Test + public void rejectsKeepInAllFourLookaroundForms() { + assertPerlRejected("(?=a\\K)"); + assertPerlRejected("(?!a\\K)"); + assertPerlRejected("(?<=a\\K)"); + assertPerlRejected("(?a\\K)(?=\\g)", PERL_SYNTAX); + } + + @Test + public void preservesNonPerlSyntaxBehavior() { + compile("(?=a\\K)", Syntax.RUBY); + } +} diff --git a/third_party/joni/test/org/joni/test/TestPerlNamedCharacter.java b/third_party/joni/test/org/joni/test/TestPerlNamedCharacter.java new file mode 100644 index 0000000000..affbed2b9a --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestPerlNamedCharacter.java @@ -0,0 +1,146 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import static org.joni.exception.ErrorMessages.PERL_EMPTY_NAMED_CHARACTER_ESCAPE; +import static org.joni.exception.ErrorMessages.PERL_MISSING_RIGHT_BRACE_ON_NAMED_CHARACTER_ESCAPE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.fail; + +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; + +import org.jcodings.Encoding; +import org.jcodings.specific.ISO8859_1Encoding; +import org.jcodings.specific.UTF8Encoding; +import org.joni.NamedCharacterResolver; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.joni.WarnCallback; +import org.joni.exception.JOniException; +import org.junit.Test; + +public class TestPerlNamedCharacter { + private static Syntax syntax(NamedCharacterResolver resolver) { + return new Syntax("PerlNamedCharacter", Syntax.PerlNG.op, Syntax.PerlNG.op2, + Syntax.PerlNG.op3, Syntax.PerlNG.behavior, Syntax.PerlNG.options, + Syntax.PerlNG.metaCharTable, resolver); + } + + private static String decode(byte[] bytes, int p, int end, Encoding encoding) { + Charset charset = encoding == ISO8859_1Encoding.INSTANCE + ? StandardCharsets.ISO_8859_1 : StandardCharsets.UTF_8; + return new String(bytes, p, end - p, charset); + } + + private static final NamedCharacterResolver RESOLVER = (bytes, p, end, encoding) -> + switch (decode(bytes, p, end, encoding)) { + case "CAPITAL" -> 'A'; + case "LOWER" -> 'a'; + case "HASH" -> '#'; + case "SPACE" -> ' '; + case "BYTE" -> 0xe9; + case "SUPPLEMENTARY" -> 0x1f642; + default -> throw new IllegalArgumentException("unknown fake name"); + }; + + private static Regex compile(String pattern, Encoding encoding, + NamedCharacterResolver resolver) { + Charset charset = encoding == ISO8859_1Encoding.INSTANCE + ? StandardCharsets.ISO_8859_1 : StandardCharsets.UTF_8; + byte[] bytes = pattern.getBytes(charset); + return new Regex(bytes, 0, bytes.length, Option.NONE, encoding, + syntax(resolver), WarnCallback.NONE); + } + + private static int search(String pattern, String input) { + byte[] bytes = input.getBytes(StandardCharsets.UTF_8); + return compile(pattern, UTF8Encoding.INSTANCE, RESOLVER) + .matcher(bytes).search(0, bytes.length, Option.NONE); + } + + private static void assertSyntaxError(String pattern, String expected) { + try { + compile(pattern, UTF8Encoding.INSTANCE, RESOLVER); + fail("expected syntax error for " + pattern); + } catch (JOniException error) { + assertEquals(expected, error.getMessage()); + } + } + + @Test + public void resolvesNamesInsideAndOutsideCharacterClasses() { + assertEquals(0, search("\\N{CAPITAL}", "A")); + assertEquals(0, search("[\\N{CAPITAL}]", "A")); + assertEquals(-1, search("\\N{CAPITAL}", "B")); + assertEquals(-1, search("[\\N{CAPITAL}]", "B")); + } + + @Test + public void preservesOptionsAndSupplementaryCodePoints() { + assertEquals(0, search("(?i)\\N{LOWER}", "A")); + assertEquals(0, search("(?x)\\N{HASH}", "#")); + assertEquals(0, search("(?x)[\\N{SPACE}]", " ")); + assertEquals(0, search("\\N{SUPPLEMENTARY}", "\ud83d\ude42")); + assertEquals(0, search("[\\N{SUPPLEMENTARY}]", "\ud83d\ude42")); + } + + @Test + public void reportsMalformedEscapesBeforeCallingResolver() { + assertSyntaxError("\\N{}", PERL_EMPTY_NAMED_CHARACTER_ESCAPE); + assertSyntaxError("\\N{CAPITAL", PERL_MISSING_RIGHT_BRACE_ON_NAMED_CHARACTER_ESCAPE); + } + + @Test + public void preservesResolverExceptions() { + IllegalArgumentException expected = new IllegalArgumentException("resolver failure"); + try { + compile("\\N{FAIL}", UTF8Encoding.INSTANCE, + (bytes, p, end, encoding) -> { throw expected; }); + fail("expected resolver exception"); + } catch (IllegalArgumentException error) { + assertSame(expected, error); + } + } + + @Test + public void suppliesTheActiveSingleByteEncoding() { + NamedCharacterResolver resolver = (bytes, p, end, encoding) -> { + assertSame(ISO8859_1Encoding.INSTANCE, encoding); + assertEquals("BYTE", decode(bytes, p, end, encoding)); + return 0xe9; + }; + byte[] input = {(byte)0xe9}; + Regex regex = compile("\\N{BYTE}", ISO8859_1Encoding.INSTANCE, resolver); + assertEquals(0, regex.matcher(input).search(0, input.length, Option.NONE)); + } + + @Test + public void leavesUnbracedEscapesAndSyntaxesWithoutAResolverUnchanged() { + assertEquals(0, search("\\N", "N")); + byte[] pattern = "\\N".getBytes(StandardCharsets.UTF_8); + byte[] input = "N".getBytes(StandardCharsets.UTF_8); + Regex regex = new Regex(pattern, 0, pattern.length, Option.NONE, + UTF8Encoding.INSTANCE, Syntax.PerlNG, WarnCallback.NONE); + assertEquals(0, regex.matcher(input).search(0, input.length, Option.NONE)); + } +} diff --git a/third_party/joni/test/org/joni/test/TestPerlVariableLookBehind.java b/third_party/joni/test/org/joni/test/TestPerlVariableLookBehind.java index 0a25187bab..b83ac623ee 100644 --- a/third_party/joni/test/org/joni/test/TestPerlVariableLookBehind.java +++ b/third_party/joni/test/org/joni/test/TestPerlVariableLookBehind.java @@ -20,6 +20,7 @@ package org.joni.test; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; import java.nio.charset.StandardCharsets; @@ -28,17 +29,26 @@ import org.joni.Option; import org.joni.Regex; import org.joni.Syntax; +import org.joni.exception.SyntaxException; import org.junit.Test; public class TestPerlVariableLookBehind { private static Matcher matcher(String pattern, String input) { + return matcher(pattern, input, Syntax.Perl); + } + + private static Matcher matcher(String pattern, String input, Syntax syntax) { byte[] patternBytes = pattern.getBytes(StandardCharsets.UTF_8); byte[] inputBytes = input.getBytes(StandardCharsets.UTF_8); Regex regex = new Regex(patternBytes, 0, patternBytes.length, Option.NONE, - UTF8Encoding.INSTANCE, Syntax.Perl); + UTF8Encoding.INSTANCE, syntax); return regex.matcher(inputBytes); } + private static void assertRejected(String pattern, Syntax syntax) { + assertThrows(SyntaxException.class, () -> matcher(pattern, "", syntax)); + } + @Test public void boundedPositiveLookBehindTriesEveryLength() { assertEquals(3, matcher("(?<=a{1,3})b", "aaab").search(0, 4, Option.NONE)); @@ -68,4 +78,46 @@ public void nestedFiniteAlternationSupportsCompoundLengths() { assertEquals(5, matcher("(?<=x(?:a|bc){1,2})z", "xbcbcz") .search(0, 6, Option.NONE)); } + + @Test + public void nestedLookAheadAssertionsFollowPerlLookBehindSemantics() { + assertEquals(1, matcher("(?<=a(?=b))b", "ab").search(0, 2, Option.NONE)); + assertEquals(1, matcher("(?<=a(?!b))c", "ac").search(0, 2, Option.NONE)); + assertEquals(-1, matcher("(?<=a(?!b))b", "ab").search(0, 2, Option.NONE)); + assertEquals(-1, matcher("(?