Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/reference/feature-matrix.md
Original file line number Diff line number Diff line change
Expand Up @@ -386,7 +386,7 @@ my @copy = @{$z}; # ERROR
- ✅ **Preprocessor**: `\Q`, `\L`, `\U`, `\l`, `\u`, `\E` are preprocessed in regex.
- ✅ **Overloading**: `qr` overloading is implemented. See also [overload pragma](#pragmas).
- ✅ **Python-style named groups**: `(?P<name>...)` and `(?P=name)` are parsed natively by Joni with Perl capture numbering, duplicate-name behavior, and malformed/unknown-name diagnostics.
- ✅ **Alpha assertion aliases**: `(*pla:...)`, `(*plb:...)`, `(*nla:...)`, `(*nlb:...)`, and `(*atomic:...)` are parsed natively by Joni with Perl nesting, capture numbering, backtracking, and malformed-form diagnostics.
- ✅ **Alpha assertion aliases**: `(*pla:...)`, `(*plb:...)`, `(*nla:...)`, `(*nlb:...)`, `(*atomic:...)`, and the corresponding long spellings are parsed natively by Joni with Perl nesting, capture numbering, backtracking, assertion-condition predicates, and malformed-form diagnostics.
- 🟡 **Underscored numeric regex escapes**: Joni natively parses Perl spellings such as `\x{0_0_4_1}` and `\o{0_0_1_0_1}` through U+10FFFF, including literal/class forms, bare high-octal UTF-8 code points, truncation behavior, and structural diagnostics. The frontend normalization remains for forced-Java compatibility; exact `use re 'strict'` diagnostics and Perl code points above U+10FFFF through signed IV max remain source-policy/representation debt.

- ✅ **Dynamically-scoped regex variables**: Provisional captures, `$^R`, `$^N`, match positions, and callback locals follow matcher paths and unwind on backtracking.
Expand Down
29 changes: 26 additions & 3 deletions src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java
Original file line number Diff line number Diff line change
Expand Up @@ -346,6 +346,7 @@ static boolean requiresJoniBackend(String pattern, RegexFlags flags) {
pattern, flags != null && flags.isExtended());
return syntaxFeatures.keepPresent()
|| syntaxFeatures.conditionalPresent()
|| syntaxFeatures.alphaAssertionPresent()
|| pattern.contains("(?{=CALL:")
|| pattern.contains("(?{=DYNAMIC:")
|| pattern.contains("(*ACCEPT)")
Expand All @@ -360,7 +361,8 @@ static boolean requiresJoniBackend(String pattern, RegexFlags flags) {

private record PerlSyntaxFeatures(boolean keepPresent,
boolean keepInLookaround,
boolean conditionalPresent) {}
boolean conditionalPresent,
boolean alphaAssertionPresent) {}

private static PerlSyntaxFeatures analyzePerlSyntax(String pattern, boolean extended) {
boolean quoted = false;
Expand All @@ -371,6 +373,7 @@ private static PerlSyntaxFeatures analyzePerlSyntax(String pattern, boolean exte
java.util.ArrayDeque<Boolean> groups = new java.util.ArrayDeque<>();
boolean keepPresent = false;
boolean conditionalPresent = false;
boolean alphaAssertionPresent = false;

for (int i = 0; i < pattern.length(); i++) {
char ch = pattern.charAt(i);
Expand Down Expand Up @@ -434,7 +437,8 @@ private static PerlSyntaxFeatures analyzePerlSyntax(String pattern, boolean exte
} else if (escaped == 'K') {
keepPresent = true;
if (lookaroundDepth > 0) {
return new PerlSyntaxFeatures(true, true, conditionalPresent);
return new PerlSyntaxFeatures(true, true, conditionalPresent,
alphaAssertionPresent);
}
}
continue;
Expand All @@ -447,6 +451,24 @@ private static PerlSyntaxFeatures analyzePerlSyntax(String pattern, boolean exte
continue;
}
if (pattern.startsWith("(?(", i)) conditionalPresent = true;
if (pattern.startsWith("(*", i)) {
int nameEnd = i + 2;
while (nameEnd < pattern.length()) {
char nameChar = pattern.charAt(nameEnd);
if (!Character.isLetter(nameChar) && nameChar != '_') break;
nameEnd++;
}
String name = pattern.substring(i + 2, nameEnd);
alphaAssertionPresent |= name.equals("pla")
|| name.equals("positive_lookahead")
|| name.equals("plb")
|| name.equals("positive_lookbehind")
|| name.equals("nla")
|| name.equals("negative_lookahead")
|| name.equals("nlb")
|| name.equals("negative_lookbehind")
|| name.equals("atomic");
}
boolean lookaround = pattern.startsWith("(?=", i)
|| pattern.startsWith("(?!", i)
|| pattern.startsWith("(?<=", i)
Expand All @@ -457,7 +479,8 @@ private static PerlSyntaxFeatures analyzePerlSyntax(String pattern, boolean exte
if (groups.pop()) lookaroundDepth--;
}
}
return new PerlSyntaxFeatures(keepPresent, false, conditionalPresent);
return new PerlSyntaxFeatures(keepPresent, false, conditionalPresent,
alphaAssertionPresent);
}

private static boolean hasControlVerbState(String pattern) {
Expand Down
52 changes: 18 additions & 34 deletions src/main/java/org/perlonjava/runtime/regex/RegexPreprocessor.java
Original file line number Diff line number Diff line change
Expand Up @@ -991,8 +991,8 @@ private static int handleParentheses(String s, int offset, int length, StringBui

// Check for (*...) verb patterns FIRST, before checking (?
if (c2 == '*') {
// (*...) control verbs like (*ACCEPT), (*FAIL), (*COMMIT), etc.
// Also handles alpha assertion aliases: (*pla:...), (*plb:...), etc.
// Java-backend compatibility for (*...) control verbs such as
// (*FAIL). Alpha assertions are routed to Joni before preprocessing.

// Find the verb name (up to ':' or ')')
int verbNameEnd = offset + 2;
Expand All @@ -1012,42 +1012,26 @@ private static int handleParentheses(String s, int offset, int length, StringBui
return verbNameEnd;
}

// Check for alpha assertion aliases (Perl 5.28+)
String replacement = switch (verbName) {
case "pla", "positive_lookahead" -> "(?=";
case "plb", "positive_lookbehind" -> "(?<=";
case "nla", "negative_lookahead" -> "(?!";
case "nlb", "negative_lookbehind" -> "(?<!";
case "atomic" -> "(?>";
default -> null;
};

if (replacement != null && verbNameEnd < length && s.codePointAt(verbNameEnd) == ':') {
// Alpha assertion with content: (*pla:...) -> (?=...)
sb.append(replacement);
offset = handleRegex(s, verbNameEnd + 1, sb, regexFlags, true);
// Fall through to common ')' handling at end of handleParentheses
} else {
// Find the end of the verb for error reporting
int verbEnd = offset + 2;
while (verbEnd < length && s.codePointAt(verbEnd) != ')') {
verbEnd++;
}
if (verbEnd < length) {
verbEnd++; // Include the closing paren
}
// Find the end of the verb for error reporting
int verbEnd = offset + 2;
while (verbEnd < length && s.codePointAt(verbEnd) != ')') {
verbEnd++;
}
if (verbEnd < length) {
verbEnd++; // Include the closing paren
}

// Extract the verb name for error reporting
String verb = s.substring(offset, Math.min(verbEnd, length));
// Extract the verb name for error reporting
String verb = s.substring(offset, Math.min(verbEnd, length));

// Replace with empty non-capturing group as placeholder
sb.append("(?:)");
// Replace with empty non-capturing group as placeholder
sb.append("(?:)");

// Throw error that can be caught by JPERL_UNIMPLEMENTED=warn
regexUnimplemented(s, offset + 2, "Regex control verb " + verb + " not implemented");
// Throw error that can be caught by JPERL_UNIMPLEMENTED=warn
regexUnimplemented(s, offset + 2,
"Regex control verb " + verb + " not implemented");

return verbEnd; // Skip past the entire verb construct
}
return verbEnd; // Skip past the entire verb construct
} else if (c2 == '?') {
if (offset + 2 >= length) {
// Marker should be after the ?
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
package org.perlonjava.runtime.regex;

import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;

import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;

@Tag("unit")
class NativeAlphaAssertionRoutingTest {
@Test
void routesShortAndLongAlphaAssertionsToJoni() {
assertTrue(JoniRegexPattern.requiresJoniBackend("a(*pla:b)b"));
assertTrue(JoniRegexPattern.requiresJoniBackend("a(*positive_lookahead:b)b"));
assertTrue(JoniRegexPattern.requiresJoniBackend("a(*plb:a)b"));
assertTrue(JoniRegexPattern.requiresJoniBackend("a(*positive_lookbehind:a)b"));
assertTrue(JoniRegexPattern.requiresJoniBackend("a(*nla:c)b"));
assertTrue(JoniRegexPattern.requiresJoniBackend("a(*negative_lookahead:c)b"));
assertTrue(JoniRegexPattern.requiresJoniBackend("a(*nlb:c)b"));
assertTrue(JoniRegexPattern.requiresJoniBackend("a(*negative_lookbehind:c)b"));
assertTrue(JoniRegexPattern.requiresJoniBackend("(*atomic:a|ab)c"));
assertTrue(JoniRegexPattern.requiresJoniBackend("(*pla)"));
assertTrue(JoniRegexPattern.requiresJoniBackend("(*positive_lookahead"));
}

@Test
void ignoresAlphaAssertionLookalikes() {
assertFalse(JoniRegexPattern.requiresJoniBackend("\\(\\*pla:a\\)"));
assertFalse(JoniRegexPattern.requiresJoniBackend("[(?*pla:)]"));
assertFalse(JoniRegexPattern.requiresJoniBackend("\\Q(*pla:a)\\E"));
assertFalse(JoniRegexPattern.requiresJoniBackend("(?# (*pla:a))ordinary"));
assertFalse(JoniRegexPattern.requiresJoniBackend("(*planet:a)"));
}
}
39 changes: 39 additions & 0 deletions src/test/resources/unit/regex/alpha_assertion_native_routing.t
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
use strict;
use warnings;
use Test::More;

ok('ab' =~ /a(*pla:b)b/, 'short positive lookahead alias');
ok('ab' =~ /a(*positive_lookahead:b)b/, 'long positive lookahead alias');
ok('ab' =~ /a(*plb:a)b/, 'short positive lookbehind alias');
ok('ab' =~ /a(*positive_lookbehind:a)b/, 'long positive lookbehind alias');
ok('ab' =~ /a(*nla:c)b/, 'short negative lookahead alias');
ok('ab' =~ /a(*negative_lookahead:c)b/, 'long negative lookahead alias');
ok('ab' =~ /a(*nlb:c)b/, 'short negative lookbehind alias');
ok('ab' =~ /a(*negative_lookbehind:c)b/, 'long negative lookbehind alias');

ok('abc' !~ /(*atomic:a|ab)c/, 'atomic alias prevents alternative retry');
ok('ab' =~ /a(*pla:(*nla:c)b)b/, 'nested alpha assertions');

my $captured = 'ab';
ok($captured =~ /a(*pla:(b))b/, 'capture inside alpha assertion participates');
is($1, 'b', 'alpha assertion publishes its capture');

ok('a' =~ /(?(*pla:a)a|b)/,
'positive alpha assertion works as a conditional predicate');
ok('b' =~ /(?(*pla:a)a|b)/,
'positive alpha assertion conditional takes its alternate');
ok('a' =~ /(?(*nla:a)b|a)/,
'negative alpha assertion conditional takes its alternate');
ok('b' =~ /(?(*nla:a)b|a)/,
'negative alpha assertion works as a conditional predicate');

for my $invalid (
'(*positive_lookahead)',
'(*positive_lookahead:a',
'(*positive_lookaround:a)',
) {
my $compiled = eval "qr/$invalid/";
ok(!defined($compiled) && length($@), "malformed long alias is rejected: $invalid");
}

done_testing;
23 changes: 16 additions & 7 deletions third_party/joni/src/org/joni/Parser.java
Original file line number Diff line number Diff line change
Expand Up @@ -624,6 +624,12 @@ private Node parseEnclose(TokenType term) {
? AnchorType.PREC_READ : AnchorType.PREC_READ_NOT);
fetchToken();
assertionCondition.setTarget(parseSubExp(term));
} else if (c == '*') {
Node alphaCondition = parseAlphaAssertion();
if (!(alphaCondition instanceof AnchorNode)) {
newSyntaxException(INVALID_CONDITION_PATTERN);
}
assertionCondition = (AnchorNode)alphaCondition;
} else if (c == 'R') {
recursionConditionGroup = 0;
if (!left()) newSyntaxException(INVALID_CONDITION_PATTERN);
Expand Down Expand Up @@ -981,12 +987,15 @@ private Node parseAlphaAssertion() {
int cursor = p;
while (cursor < stop) {
int code = enc.mbcToCode(bytes, cursor, stop);
if (!Character.isLetter(code)) break;
if (!Character.isLetter(code) && code != '_') break;
cursor += enc.length(bytes, cursor, stop);
}
String name = new String(bytes, nameStart, cursor - nameStart, StandardCharsets.UTF_8);
if (!name.equals("pla") && !name.equals("plb") && !name.equals("nla")
&& !name.equals("nlb") && !name.equals("atomic")) {
if (!name.equals("pla") && !name.equals("positive_lookahead")
&& !name.equals("plb") && !name.equals("positive_lookbehind")
&& !name.equals("nla") && !name.equals("negative_lookahead")
&& !name.equals("nlb") && !name.equals("negative_lookbehind")
&& !name.equals("atomic")) {
return null;
}
if (cursor >= stop || enc.mbcToCode(bytes, cursor, stop) != ':') {
Expand All @@ -996,16 +1005,16 @@ private Node parseAlphaAssertion() {

Node node;
switch (name) {
case "pla":
case "pla", "positive_lookahead":
node = new AnchorNode(AnchorType.PREC_READ);
break;
case "plb":
case "plb", "positive_lookbehind":
node = new AnchorNode(AnchorType.LOOK_BEHIND);
break;
case "nla":
case "nla", "negative_lookahead":
node = new AnchorNode(AnchorType.PREC_READ_NOT);
break;
case "nlb":
case "nlb", "negative_lookbehind":
node = new AnchorNode(AnchorType.LOOK_BEHIND_NOT);
break;
default:
Expand Down
7 changes: 7 additions & 0 deletions third_party/joni/test/org/joni/test/TestPerl.java
Original file line number Diff line number Diff line change
Expand Up @@ -77,20 +77,27 @@ public void test() throws Exception {
ns("[(?P<x>)]", "z");
x2s("\\(\\?P<x>", "(?P<x>", 0, 6);
x2s("a(*pla:b)b", "ab", 0, 2);
x2s("a(*positive_lookahead:b)b", "ab", 0, 2);
ns("a(*pla:c)b", "ab");
x2s("a(*plb:a)b", "ab", 0, 2);
x2s("a(*positive_lookbehind:a)b", "ab", 0, 2);
ns("a(*plb:c)b", "ab");
x2s("a(*nla:c)b", "ab", 0, 2);
x2s("a(*negative_lookahead:c)b", "ab", 0, 2);
ns("a(*nla:b)b", "ab");
x2s("a(*nlb:c)b", "ab", 0, 2);
x2s("a(*negative_lookbehind:c)b", "ab", 0, 2);
ns("a(*nlb:a)b", "ab");
ns("(*atomic:a|ab)c", "abc");
x2s("a(*pla:(*nla:c)b)b", "ab", 0, 2);
x2s("(*pla:)", "", 0, 0);
xerrs("(*PLA:a)", "Unknown verb pattern 'PLA'");
xerrs("(*pla)", "'(*pla' requires a terminating ':'");
xerrs("(*positive_lookahead)",
"'(*positive_lookahead' requires a terminating ':'");
xerrs("(*plx:a)", "Unknown '(*...)' construct 'plx'");
xerrs("(*pla:a", ErrorMessages.PERL_UNTERMINATED_CONTROL_ARGUMENT);
xerrs("(*positive_lookahead:a", ErrorMessages.PERL_UNTERMINATED_CONTROL_ARGUMENT);
ns("[(?*pla:)]", "z");
x2s("\\(\\*pla:a\\)", "(*pla:a)", 0, 8);
}
Expand Down
8 changes: 8 additions & 0 deletions third_party/joni/test/org/joni/test/TestPerlConditions.java
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,14 @@ public void negativeAssertionSelectsTheMatchingBranch() {
assertEquals(0, matcher("(?(?!a)b|a)", "b").search(0, 1, Option.NONE));
}

@Test
public void alphaAssertionConditionsSelectTheMatchingBranch() {
assertEquals(0, matcher("(?(*pla:a)a|b)", "a").search(0, 1, Option.NONE));
assertEquals(0, matcher("(?(*pla:a)a|b)", "b").search(0, 1, Option.NONE));
assertEquals(0, matcher("(?(*nla:a)b|a)", "a").search(0, 1, Option.NONE));
assertEquals(0, matcher("(?(*nla:a)b|a)", "b").search(0, 1, Option.NONE));
}

@Test
public void assertionCapturesRemainVisibleToTheSelectedBranch() {
Matcher matcher = matcher("(?(?=(a))\\1|b)", "a");
Expand Down