From 229448c183b8b2e87cb323cf08543e7945583abb Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Tue, 18 Aug 2026 21:32:32 +0200 Subject: [PATCH] feat(regex): route alpha assertions through native Joni Teach the forked parser the long Perl alpha-assertion spellings and allow alpha assertions as conditional predicates. Route these constructs directly to Joni and remove their Java preprocessor rewrite. The focused oracle passes 19/19 in all matcher and execution combinations. The imported alpha corpus has no numbered assertion losses against its exact parent and gains passing assertions in every cell. Generated with Codex (https://openai.com/codex) Co-Authored-By: Codex --- docs/reference/feature-matrix.md | 2 +- .../runtime/regex/JoniRegexPattern.java | 29 +++++++++-- .../runtime/regex/RegexPreprocessor.java | 52 +++++++------------ .../NativeAlphaAssertionRoutingTest.java | 34 ++++++++++++ .../regex/alpha_assertion_native_routing.t | 39 ++++++++++++++ third_party/joni/src/org/joni/Parser.java | 23 +++++--- .../joni/test/org/joni/test/TestPerl.java | 7 +++ .../org/joni/test/TestPerlConditions.java | 8 +++ 8 files changed, 149 insertions(+), 45 deletions(-) create mode 100644 src/test/java/org/perlonjava/runtime/regex/NativeAlphaAssertionRoutingTest.java create mode 100644 src/test/resources/unit/regex/alpha_assertion_native_routing.t diff --git a/docs/reference/feature-matrix.md b/docs/reference/feature-matrix.md index 3a7bb069c9..ecb0b961c7 100644 --- a/docs/reference/feature-matrix.md +++ b/docs/reference/feature-matrix.md @@ -386,7 +386,7 @@ my @copy = @{$z}; # ERROR - ✅ **Preprocessor**: `\Q`, `\L`, `\U`, `\l`, `\u`, `\E` are preprocessed in regex. - ✅ **Overloading**: `qr` overloading is implemented. See also [overload pragma](#pragmas). - ✅ **Python-style named groups**: `(?P...)` and `(?P=name)` are parsed natively by Joni with Perl capture numbering, duplicate-name behavior, and malformed/unknown-name diagnostics. -- ✅ **Alpha assertion aliases**: `(*pla:...)`, `(*plb:...)`, `(*nla:...)`, `(*nlb:...)`, and `(*atomic:...)` are parsed natively by Joni with Perl nesting, capture numbering, backtracking, and malformed-form diagnostics. +- ✅ **Alpha assertion aliases**: `(*pla:...)`, `(*plb:...)`, `(*nla:...)`, `(*nlb:...)`, `(*atomic:...)`, and the corresponding long spellings are parsed natively by Joni with Perl nesting, capture numbering, backtracking, assertion-condition predicates, and malformed-form diagnostics. - 🟡 **Underscored numeric regex escapes**: Joni natively parses Perl spellings such as `\x{0_0_4_1}` and `\o{0_0_1_0_1}` through U+10FFFF, including literal/class forms, bare high-octal UTF-8 code points, truncation behavior, and structural diagnostics. The frontend normalization remains for forced-Java compatibility; exact `use re 'strict'` diagnostics and Perl code points above U+10FFFF through signed IV max remain source-policy/representation debt. - ✅ **Dynamically-scoped regex variables**: Provisional captures, `$^R`, `$^N`, match positions, and callback locals follow matcher paths and unwind on backtracking. diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index a925034c19..8d95fb2921 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -346,6 +346,7 @@ static boolean requiresJoniBackend(String pattern, RegexFlags flags) { pattern, flags != null && flags.isExtended()); return syntaxFeatures.keepPresent() || syntaxFeatures.conditionalPresent() + || syntaxFeatures.alphaAssertionPresent() || pattern.contains("(?{=CALL:") || pattern.contains("(?{=DYNAMIC:") || pattern.contains("(*ACCEPT)") @@ -360,7 +361,8 @@ static boolean requiresJoniBackend(String pattern, RegexFlags flags) { private record PerlSyntaxFeatures(boolean keepPresent, boolean keepInLookaround, - boolean conditionalPresent) {} + boolean conditionalPresent, + boolean alphaAssertionPresent) {} private static PerlSyntaxFeatures analyzePerlSyntax(String pattern, boolean extended) { boolean quoted = false; @@ -371,6 +373,7 @@ private static PerlSyntaxFeatures analyzePerlSyntax(String pattern, boolean exte java.util.ArrayDeque groups = new java.util.ArrayDeque<>(); boolean keepPresent = false; boolean conditionalPresent = false; + boolean alphaAssertionPresent = false; for (int i = 0; i < pattern.length(); i++) { char ch = pattern.charAt(i); @@ -434,7 +437,8 @@ private static PerlSyntaxFeatures analyzePerlSyntax(String pattern, boolean exte } else if (escaped == 'K') { keepPresent = true; if (lookaroundDepth > 0) { - return new PerlSyntaxFeatures(true, true, conditionalPresent); + return new PerlSyntaxFeatures(true, true, conditionalPresent, + alphaAssertionPresent); } } continue; @@ -447,6 +451,24 @@ private static PerlSyntaxFeatures analyzePerlSyntax(String pattern, boolean exte continue; } if (pattern.startsWith("(?(", i)) conditionalPresent = true; + if (pattern.startsWith("(*", i)) { + int nameEnd = i + 2; + while (nameEnd < pattern.length()) { + char nameChar = pattern.charAt(nameEnd); + if (!Character.isLetter(nameChar) && nameChar != '_') break; + nameEnd++; + } + String name = pattern.substring(i + 2, nameEnd); + alphaAssertionPresent |= name.equals("pla") + || name.equals("positive_lookahead") + || name.equals("plb") + || name.equals("positive_lookbehind") + || name.equals("nla") + || name.equals("negative_lookahead") + || name.equals("nlb") + || name.equals("negative_lookbehind") + || name.equals("atomic"); + } boolean lookaround = pattern.startsWith("(?=", i) || pattern.startsWith("(?!", i) || pattern.startsWith("(?<=", i) @@ -457,7 +479,8 @@ private static PerlSyntaxFeatures analyzePerlSyntax(String pattern, boolean exte if (groups.pop()) lookaroundDepth--; } } - return new PerlSyntaxFeatures(keepPresent, false, conditionalPresent); + return new PerlSyntaxFeatures(keepPresent, false, conditionalPresent, + alphaAssertionPresent); } private static boolean hasControlVerbState(String pattern) { diff --git a/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessor.java b/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessor.java index 4897b45d7c..d1dfac45c2 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessor.java +++ b/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessor.java @@ -991,8 +991,8 @@ private static int handleParentheses(String s, int offset, int length, StringBui // Check for (*...) verb patterns FIRST, before checking (? if (c2 == '*') { - // (*...) control verbs like (*ACCEPT), (*FAIL), (*COMMIT), etc. - // Also handles alpha assertion aliases: (*pla:...), (*plb:...), etc. + // Java-backend compatibility for (*...) control verbs such as + // (*FAIL). Alpha assertions are routed to Joni before preprocessing. // Find the verb name (up to ':' or ')') int verbNameEnd = offset + 2; @@ -1012,42 +1012,26 @@ private static int handleParentheses(String s, int offset, int length, StringBui return verbNameEnd; } - // Check for alpha assertion aliases (Perl 5.28+) - String replacement = switch (verbName) { - case "pla", "positive_lookahead" -> "(?="; - case "plb", "positive_lookbehind" -> "(?<="; - case "nla", "negative_lookahead" -> "(?!"; - case "nlb", "negative_lookbehind" -> "(? "(?>"; - default -> null; - }; - - if (replacement != null && verbNameEnd < length && s.codePointAt(verbNameEnd) == ':') { - // Alpha assertion with content: (*pla:...) -> (?=...) - sb.append(replacement); - offset = handleRegex(s, verbNameEnd + 1, sb, regexFlags, true); - // Fall through to common ')' handling at end of handleParentheses - } else { - // Find the end of the verb for error reporting - int verbEnd = offset + 2; - while (verbEnd < length && s.codePointAt(verbEnd) != ')') { - verbEnd++; - } - if (verbEnd < length) { - verbEnd++; // Include the closing paren - } + // Find the end of the verb for error reporting + int verbEnd = offset + 2; + while (verbEnd < length && s.codePointAt(verbEnd) != ')') { + verbEnd++; + } + if (verbEnd < length) { + verbEnd++; // Include the closing paren + } - // Extract the verb name for error reporting - String verb = s.substring(offset, Math.min(verbEnd, length)); + // Extract the verb name for error reporting + String verb = s.substring(offset, Math.min(verbEnd, length)); - // Replace with empty non-capturing group as placeholder - sb.append("(?:)"); + // Replace with empty non-capturing group as placeholder + sb.append("(?:)"); - // Throw error that can be caught by JPERL_UNIMPLEMENTED=warn - regexUnimplemented(s, offset + 2, "Regex control verb " + verb + " not implemented"); + // Throw error that can be caught by JPERL_UNIMPLEMENTED=warn + regexUnimplemented(s, offset + 2, + "Regex control verb " + verb + " not implemented"); - return verbEnd; // Skip past the entire verb construct - } + return verbEnd; // Skip past the entire verb construct } else if (c2 == '?') { if (offset + 2 >= length) { // Marker should be after the ? diff --git a/src/test/java/org/perlonjava/runtime/regex/NativeAlphaAssertionRoutingTest.java b/src/test/java/org/perlonjava/runtime/regex/NativeAlphaAssertionRoutingTest.java new file mode 100644 index 0000000000..2a7f754223 --- /dev/null +++ b/src/test/java/org/perlonjava/runtime/regex/NativeAlphaAssertionRoutingTest.java @@ -0,0 +1,34 @@ +package org.perlonjava.runtime.regex; + +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@Tag("unit") +class NativeAlphaAssertionRoutingTest { + @Test + void routesShortAndLongAlphaAssertionsToJoni() { + assertTrue(JoniRegexPattern.requiresJoniBackend("a(*pla:b)b")); + assertTrue(JoniRegexPattern.requiresJoniBackend("a(*positive_lookahead:b)b")); + assertTrue(JoniRegexPattern.requiresJoniBackend("a(*plb:a)b")); + assertTrue(JoniRegexPattern.requiresJoniBackend("a(*positive_lookbehind:a)b")); + assertTrue(JoniRegexPattern.requiresJoniBackend("a(*nla:c)b")); + assertTrue(JoniRegexPattern.requiresJoniBackend("a(*negative_lookahead:c)b")); + assertTrue(JoniRegexPattern.requiresJoniBackend("a(*nlb:c)b")); + assertTrue(JoniRegexPattern.requiresJoniBackend("a(*negative_lookbehind:c)b")); + assertTrue(JoniRegexPattern.requiresJoniBackend("(*atomic:a|ab)c")); + assertTrue(JoniRegexPattern.requiresJoniBackend("(*pla)")); + assertTrue(JoniRegexPattern.requiresJoniBackend("(*positive_lookahead")); + } + + @Test + void ignoresAlphaAssertionLookalikes() { + assertFalse(JoniRegexPattern.requiresJoniBackend("\\(\\*pla:a\\)")); + assertFalse(JoniRegexPattern.requiresJoniBackend("[(?*pla:)]")); + assertFalse(JoniRegexPattern.requiresJoniBackend("\\Q(*pla:a)\\E")); + assertFalse(JoniRegexPattern.requiresJoniBackend("(?# (*pla:a))ordinary")); + assertFalse(JoniRegexPattern.requiresJoniBackend("(*planet:a)")); + } +} diff --git a/src/test/resources/unit/regex/alpha_assertion_native_routing.t b/src/test/resources/unit/regex/alpha_assertion_native_routing.t new file mode 100644 index 0000000000..4785365d10 --- /dev/null +++ b/src/test/resources/unit/regex/alpha_assertion_native_routing.t @@ -0,0 +1,39 @@ +use strict; +use warnings; +use Test::More; + +ok('ab' =~ /a(*pla:b)b/, 'short positive lookahead alias'); +ok('ab' =~ /a(*positive_lookahead:b)b/, 'long positive lookahead alias'); +ok('ab' =~ /a(*plb:a)b/, 'short positive lookbehind alias'); +ok('ab' =~ /a(*positive_lookbehind:a)b/, 'long positive lookbehind alias'); +ok('ab' =~ /a(*nla:c)b/, 'short negative lookahead alias'); +ok('ab' =~ /a(*negative_lookahead:c)b/, 'long negative lookahead alias'); +ok('ab' =~ /a(*nlb:c)b/, 'short negative lookbehind alias'); +ok('ab' =~ /a(*negative_lookbehind:c)b/, 'long negative lookbehind alias'); + +ok('abc' !~ /(*atomic:a|ab)c/, 'atomic alias prevents alternative retry'); +ok('ab' =~ /a(*pla:(*nla:c)b)b/, 'nested alpha assertions'); + +my $captured = 'ab'; +ok($captured =~ /a(*pla:(b))b/, 'capture inside alpha assertion participates'); +is($1, 'b', 'alpha assertion publishes its capture'); + +ok('a' =~ /(?(*pla:a)a|b)/, + 'positive alpha assertion works as a conditional predicate'); +ok('b' =~ /(?(*pla:a)a|b)/, + 'positive alpha assertion conditional takes its alternate'); +ok('a' =~ /(?(*nla:a)b|a)/, + 'negative alpha assertion conditional takes its alternate'); +ok('b' =~ /(?(*nla:a)b|a)/, + 'negative alpha assertion works as a conditional predicate'); + +for my $invalid ( + '(*positive_lookahead)', + '(*positive_lookahead:a', + '(*positive_lookaround:a)', +) { + my $compiled = eval "qr/$invalid/"; + ok(!defined($compiled) && length($@), "malformed long alias is rejected: $invalid"); +} + +done_testing; diff --git a/third_party/joni/src/org/joni/Parser.java b/third_party/joni/src/org/joni/Parser.java index 0c2c79b1f8..dba27461a4 100644 --- a/third_party/joni/src/org/joni/Parser.java +++ b/third_party/joni/src/org/joni/Parser.java @@ -624,6 +624,12 @@ private Node parseEnclose(TokenType term) { ? AnchorType.PREC_READ : AnchorType.PREC_READ_NOT); fetchToken(); assertionCondition.setTarget(parseSubExp(term)); + } else if (c == '*') { + Node alphaCondition = parseAlphaAssertion(); + if (!(alphaCondition instanceof AnchorNode)) { + newSyntaxException(INVALID_CONDITION_PATTERN); + } + assertionCondition = (AnchorNode)alphaCondition; } else if (c == 'R') { recursionConditionGroup = 0; if (!left()) newSyntaxException(INVALID_CONDITION_PATTERN); @@ -981,12 +987,15 @@ private Node parseAlphaAssertion() { int cursor = p; while (cursor < stop) { int code = enc.mbcToCode(bytes, cursor, stop); - if (!Character.isLetter(code)) break; + if (!Character.isLetter(code) && code != '_') break; cursor += enc.length(bytes, cursor, stop); } String name = new String(bytes, nameStart, cursor - nameStart, StandardCharsets.UTF_8); - if (!name.equals("pla") && !name.equals("plb") && !name.equals("nla") - && !name.equals("nlb") && !name.equals("atomic")) { + if (!name.equals("pla") && !name.equals("positive_lookahead") + && !name.equals("plb") && !name.equals("positive_lookbehind") + && !name.equals("nla") && !name.equals("negative_lookahead") + && !name.equals("nlb") && !name.equals("negative_lookbehind") + && !name.equals("atomic")) { return null; } if (cursor >= stop || enc.mbcToCode(bytes, cursor, stop) != ':') { @@ -996,16 +1005,16 @@ private Node parseAlphaAssertion() { Node node; switch (name) { - case "pla": + case "pla", "positive_lookahead": node = new AnchorNode(AnchorType.PREC_READ); break; - case "plb": + case "plb", "positive_lookbehind": node = new AnchorNode(AnchorType.LOOK_BEHIND); break; - case "nla": + case "nla", "negative_lookahead": node = new AnchorNode(AnchorType.PREC_READ_NOT); break; - case "nlb": + case "nlb", "negative_lookbehind": node = new AnchorNode(AnchorType.LOOK_BEHIND_NOT); break; default: diff --git a/third_party/joni/test/org/joni/test/TestPerl.java b/third_party/joni/test/org/joni/test/TestPerl.java index 708ab071d9..4ebd3448ca 100755 --- a/third_party/joni/test/org/joni/test/TestPerl.java +++ b/third_party/joni/test/org/joni/test/TestPerl.java @@ -77,20 +77,27 @@ public void test() throws Exception { ns("[(?P)]", "z"); x2s("\\(\\?P", "(?P", 0, 6); x2s("a(*pla:b)b", "ab", 0, 2); + x2s("a(*positive_lookahead:b)b", "ab", 0, 2); ns("a(*pla:c)b", "ab"); x2s("a(*plb:a)b", "ab", 0, 2); + x2s("a(*positive_lookbehind:a)b", "ab", 0, 2); ns("a(*plb:c)b", "ab"); x2s("a(*nla:c)b", "ab", 0, 2); + x2s("a(*negative_lookahead:c)b", "ab", 0, 2); ns("a(*nla:b)b", "ab"); x2s("a(*nlb:c)b", "ab", 0, 2); + x2s("a(*negative_lookbehind:c)b", "ab", 0, 2); ns("a(*nlb:a)b", "ab"); ns("(*atomic:a|ab)c", "abc"); x2s("a(*pla:(*nla:c)b)b", "ab", 0, 2); x2s("(*pla:)", "", 0, 0); xerrs("(*PLA:a)", "Unknown verb pattern 'PLA'"); xerrs("(*pla)", "'(*pla' requires a terminating ':'"); + xerrs("(*positive_lookahead)", + "'(*positive_lookahead' requires a terminating ':'"); xerrs("(*plx:a)", "Unknown '(*...)' construct 'plx'"); xerrs("(*pla:a", ErrorMessages.PERL_UNTERMINATED_CONTROL_ARGUMENT); + xerrs("(*positive_lookahead:a", ErrorMessages.PERL_UNTERMINATED_CONTROL_ARGUMENT); ns("[(?*pla:)]", "z"); x2s("\\(\\*pla:a\\)", "(*pla:a)", 0, 8); } diff --git a/third_party/joni/test/org/joni/test/TestPerlConditions.java b/third_party/joni/test/org/joni/test/TestPerlConditions.java index 82ced22466..aadc68f94c 100644 --- a/third_party/joni/test/org/joni/test/TestPerlConditions.java +++ b/third_party/joni/test/org/joni/test/TestPerlConditions.java @@ -53,6 +53,14 @@ public void negativeAssertionSelectsTheMatchingBranch() { assertEquals(0, matcher("(?(?!a)b|a)", "b").search(0, 1, Option.NONE)); } + @Test + public void alphaAssertionConditionsSelectTheMatchingBranch() { + assertEquals(0, matcher("(?(*pla:a)a|b)", "a").search(0, 1, Option.NONE)); + assertEquals(0, matcher("(?(*pla:a)a|b)", "b").search(0, 1, Option.NONE)); + assertEquals(0, matcher("(?(*nla:a)b|a)", "a").search(0, 1, Option.NONE)); + assertEquals(0, matcher("(?(*nla:a)b|a)", "b").search(0, 1, Option.NONE)); + } + @Test public void assertionCapturesRemainVisibleToTheSelectedBranch() { Matcher matcher = matcher("(?(?=(a))\\1|b)", "a");