From 5a71e022ef27966fdbae91c99475d49a41b1ffe9 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 15:44:50 +0200 Subject: [PATCH 1/8] fix: diagnose malformed named Unicode scalars Match Perl's fatal diagnostic and source context for malformed U+ forms in strings, regexes, ordinary character classes, and extended classes. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../frontend/parser/StringSegmentParser.java | 13 ++++++++- .../runtime/NamedCharacterExpansion.java | 9 ++++-- ...nicode_named_malformed_uplus_diagnostics.t | 28 +++++++++++++++++++ 3 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 src/test/resources/unit/regex/unicode_named_malformed_uplus_diagnostics.t diff --git a/src/main/java/org/perlonjava/frontend/parser/StringSegmentParser.java b/src/main/java/org/perlonjava/frontend/parser/StringSegmentParser.java index b76f1e307..d95aac1ce 100644 --- a/src/main/java/org/perlonjava/frontend/parser/StringSegmentParser.java +++ b/src/main/java/org/perlonjava/frontend/parser/StringSegmentParser.java @@ -1518,6 +1518,9 @@ void handleUnicodeNameEscape() { throwNamedSequenceExtendedClassDiagnostic(expansion.sequence()); } if (!expansion.resolved()) { + if (expansion.status() == NamedCharacterExpansion.Status.INVALID) { + throwNamedCharacterDiagnostic(expansion.diagnostic()); + } parser.throwError(expansion.diagnostic()); } } @@ -1529,7 +1532,7 @@ void handleUnicodeNameEscape() { if (expansion.resolved()) { appendToCurrentSegment(expansion.sequence()); } else if (expansion.status() == NamedCharacterExpansion.Status.INVALID) { - parser.throwError(expansion.diagnostic()); + throwNamedCharacterDiagnostic(expansion.diagnostic()); } else { // Preserve the historical literal fallback when no standard // name or lexical translator resolves this escape. @@ -1540,6 +1543,14 @@ void handleUnicodeNameEscape() { } } + private void throwNamedCharacterDiagnostic(String diagnostic) { + int errorIndex = this.tokenIndex; + var location = ctx.errorUtil.getSourceLocationAccurate(errorIndex); + throw new PerlParserException(diagnostic + + " at " + location.fileName() + " line " + location.lineNumber() + + ", within " + (isRegex ? "pattern" : "string") + "\n"); + } + private void throwMissingNamedCharacterBraceDiagnostic() { var location = ctx.errorUtil.getSourceLocationAccurate(parser.tokenIndex); String message = isRegex diff --git a/src/main/java/org/perlonjava/runtime/NamedCharacterExpansion.java b/src/main/java/org/perlonjava/runtime/NamedCharacterExpansion.java index 9feaf8351..84d6eabae 100644 --- a/src/main/java/org/perlonjava/runtime/NamedCharacterExpansion.java +++ b/src/main/java/org/perlonjava/runtime/NamedCharacterExpansion.java @@ -62,8 +62,13 @@ public static NamedCharacterExpansion resolve(String name, SourceMode inputMode) */ public static NamedCharacterExpansion resolve( String name, RuntimeScalar translator, SourceMode inputMode) { - if (name != null && name.matches("(?i)U\\+[0-9A-F]+")) { - return resolveStandard(name); + if (name != null && name.regionMatches(true, 0, "U+", 0, 2)) { + if (name.matches("(?i)U\\+[0-9A-F]+")) { + return resolveStandard(name); + } + return new NamedCharacterExpansion( + "", SourceMode.UNICODE, true, Status.INVALID, + "Invalid hexadecimal number in \\N{U+...}"); } RuntimeScalar callable = unwrapCallable(translator); if (callable != null) { diff --git a/src/test/resources/unit/regex/unicode_named_malformed_uplus_diagnostics.t b/src/test/resources/unit/regex/unicode_named_malformed_uplus_diagnostics.t new file mode 100644 index 000000000..3b22efc8f --- /dev/null +++ b/src/test/resources/unit/regex/unicode_named_malformed_uplus_diagnostics.t @@ -0,0 +1,28 @@ +use strict; +use warnings; +use Test::More; + +my @cases = ( + [string => q!"\N{U+XYZ}"!, 'string'], + [regex => q!qr/\N{U+XYZ}/!, 'pattern'], + [class => q!qr/[\N{U+XYZ}]/!, 'pattern'], + [extended => q!no warnings 'experimental::regex_sets'; qr/(?[\N{U+XYZ}])/!, + 'pattern'], +); + +for my $case (@cases) { + my ($label, $source, $context) = @$case; + my @warnings; + { + local $SIG{__WARN__} = sub { push @warnings, join '', @_ }; + eval "#line 1 unicode_named_malformed_uplus_diagnostics.t\n$source"; + } + my ($first_line) = split /\n/, $@; + is($first_line, + "Invalid hexadecimal number in \\N{U+...} at unicode_named_malformed_uplus_diagnostics.t line 1, within $context", + "$label malformed U+ escape has the Perl diagnostic"); + is(scalar @warnings, 0, + "$label malformed U+ escape emits no warning before the fatal"); +} + +done_testing; From 67c9787945c84c9262f77331dfe2dba2b8c8f99d Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 15:47:59 +0200 Subject: [PATCH 2/8] fix: report unknown named character escapes Match Perl's fatal diagnostics for unknown and empty character names in strings, regexes, ordinary character classes, and extended classes. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../frontend/parser/StringSegmentParser.java | 11 +----- .../regex/unicode_named_unknown_diagnostics.t | 38 +++++++++++++++++++ 2 files changed, 40 insertions(+), 9 deletions(-) create mode 100644 src/test/resources/unit/regex/unicode_named_unknown_diagnostics.t diff --git a/src/main/java/org/perlonjava/frontend/parser/StringSegmentParser.java b/src/main/java/org/perlonjava/frontend/parser/StringSegmentParser.java index d95aac1ce..eb485a89e 100644 --- a/src/main/java/org/perlonjava/frontend/parser/StringSegmentParser.java +++ b/src/main/java/org/perlonjava/frontend/parser/StringSegmentParser.java @@ -1518,10 +1518,7 @@ void handleUnicodeNameEscape() { throwNamedSequenceExtendedClassDiagnostic(expansion.sequence()); } if (!expansion.resolved()) { - if (expansion.status() == NamedCharacterExpansion.Status.INVALID) { - throwNamedCharacterDiagnostic(expansion.diagnostic()); - } - parser.throwError(expansion.diagnostic()); + throwNamedCharacterDiagnostic(expansion.diagnostic()); } } appendToCurrentSegment("\\N{" + name + "}"); @@ -1531,12 +1528,8 @@ void handleUnicodeNameEscape() { NamedCharacterExpansion.resolve(name, sourceMode); if (expansion.resolved()) { appendToCurrentSegment(expansion.sequence()); - } else if (expansion.status() == NamedCharacterExpansion.Status.INVALID) { - throwNamedCharacterDiagnostic(expansion.diagnostic()); } else { - // Preserve the historical literal fallback when no standard - // name or lexical translator resolves this escape. - appendToCurrentSegment("N{" + name + "}"); + throwNamedCharacterDiagnostic(expansion.diagnostic()); } } else { throwMissingNamedCharacterBraceDiagnostic(); diff --git a/src/test/resources/unit/regex/unicode_named_unknown_diagnostics.t b/src/test/resources/unit/regex/unicode_named_unknown_diagnostics.t new file mode 100644 index 000000000..388ff5d61 --- /dev/null +++ b/src/test/resources/unit/regex/unicode_named_unknown_diagnostics.t @@ -0,0 +1,38 @@ +use strict; +use warnings; +use Test::More; + +my @contexts = ( + [string => sub { qq!"$_[0]"! }, 'string'], + [regex => sub { qq!qr/$_[0]/! }, 'pattern'], + [class => sub { qq!qr/[$_[0]]/! }, 'pattern'], + [extended => sub { + qq!no warnings 'experimental::regex_sets'; qr/(?[$_[0]])/! + }, 'pattern'], +); + +my @names = ( + ['PHASE36 UNKNOWN NAME', q!\N{PHASE36 UNKNOWN NAME}!], + ['', q!\N{}!], +); + +for my $named (@names) { + my ($name, $escape) = @$named; + for my $context (@contexts) { + my ($label, $wrap, $suffix) = @$context; + my @warnings; + { + local $SIG{__WARN__} = sub { push @warnings, join '', @_ }; + my $source = $wrap->($escape); + eval "#line 1 unicode_named_unknown_diagnostics.t\n$source"; + } + my ($first_line) = split /\n/, $@; + is($first_line, + "Unknown charname '$name' at unicode_named_unknown_diagnostics.t line 1, within $suffix", + "$label unknown charname '$name' has the Perl diagnostic"); + is(scalar @warnings, 0, + "$label unknown charname '$name' emits no warning before the fatal"); + } +} + +done_testing; From ee246dbbfe1fa0cff6d934d235f67f013f9fde74 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 15:24:23 +0200 Subject: [PATCH 3/8] feat(joni): compile Perl DEFINE containers natively Parse and compile Perl (?(DEFINE)...) containers directly in Joni, preserving lexical group numbering, callable definitions, recursion, and Perl diagnostics. Remove the runtime adapter lowering and preserve enclosing captures across completed DEFINE calls. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../runtime/regex/JoniRegexPattern.java | 79 ---------------- .../resources/unit/regex_joni_native_define.t | 37 ++++++++ third_party/joni/src/org/joni/Analyser.java | 46 +++++++++- .../joni/src/org/joni/ArrayCompiler.java | 8 ++ .../joni/src/org/joni/ByteCodeMachine.java | 37 +++++++- third_party/joni/src/org/joni/Parser.java | 9 +- .../joni/src/org/joni/ast/EncloseNode.java | 5 + .../joni/constants/internal/EncloseType.java | 1 + .../src/org/joni/exception/ErrorMessages.java | 2 + .../joni/test/TestPerlDefineContainer.java | 92 +++++++++++++++++++ 10 files changed, 230 insertions(+), 86 deletions(-) create mode 100644 src/test/resources/unit/regex_joni_native_define.t create mode 100644 third_party/joni/test/org/joni/test/TestPerlDefineContainer.java diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index 1cf9267ed..aa8d14bb3 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -711,7 +711,6 @@ private static String translatePattern(String pattern, RegexFlags flags, private static String translatePattern(String pattern, RegexFlags flags, int trustedCalloutCount, boolean resolveNamedCharacters) { - pattern = translateDefineBlocks(pattern); StringBuilder out = new StringBuilder(pattern.length() + 16); boolean escaped = false; boolean inClass = false; @@ -968,84 +967,6 @@ private static void appendAsciiClassForJoni(StringBuilder out, String javaClass) out.append(']'); } - /** - * Ruby/Oniguruma syntax supports named subexpression calls but not PCRE's - * {@code (?(DEFINE) ...)} container. Keep the definitions in the compiled - * graph inside a negative lookahead whose body is forced to fail; the - * lookahead therefore always succeeds without consuming input, while the - * named groups remain available to later {@code (?&name)} calls. - */ - private static String translateDefineBlocks(String pattern) { - StringBuilder out = new StringBuilder(pattern.length() + 16); - boolean escaped = false; - boolean inClass = false; - for (int i = 0; i < pattern.length(); i++) { - char ch = pattern.charAt(i); - if (escaped) { - out.append(ch); - escaped = false; - continue; - } - if (ch == '\\') { - out.append(ch); - escaped = true; - continue; - } - if (ch == '[') { - inClass = true; - out.append(ch); - continue; - } - if (ch == ']' && inClass) { - inClass = false; - out.append(ch); - continue; - } - if (!inClass && pattern.startsWith("(?(DEFINE)", i)) { - int end = findGroupEnd(pattern, i); - if (end > i) { - String definitions = pattern.substring(i + 10, end); - out.append("(?!(?:") - .append(translateDefineBlocks(definitions)) - .append(")(?!))"); - i = end; - continue; - } - } - out.append(ch); - } - return out.toString(); - } - - private static int findGroupEnd(String pattern, int start) { - int depth = 0; - boolean escaped = false; - boolean inClass = false; - for (int i = start; i < pattern.length(); i++) { - char ch = pattern.charAt(i); - if (escaped) { - escaped = false; - continue; - } - if (ch == '\\') { - escaped = true; - continue; - } - if (ch == '[') { - inClass = true; - continue; - } - if (ch == ']' && inClass) { - inClass = false; - continue; - } - if (inClass) continue; - if (ch == '(') depth++; - else if (ch == ')' && --depth == 0) return i; - } - return -1; - } - private record NamedGroupMaps(Map logical, Map physical) {} diff --git a/src/test/resources/unit/regex_joni_native_define.t b/src/test/resources/unit/regex_joni_native_define.t new file mode 100644 index 000000000..4ac650bdf --- /dev/null +++ b/src/test/resources/unit/regex_joni_native_define.t @@ -0,0 +1,37 @@ +use strict; +use warnings; +use Test::More tests => 14; + +ok('A' =~ /^(?(DEFINE)(?FAIL))A$/, + 'DEFINE container does not execute on the main path'); +ok('FAIL' !~ /^(?(DEFINE)(?FAIL))$/, + 'definition body cannot match without a call'); +ok('word' =~ /^(?&word)(?(DEFINE)(?[a-z]+))$/, + 'named call binds to a following definition'); +ok('a' =~ /^(?(DEFINE)(a))(?1)$/, + 'absolute numbered call binds to a definition'); + +ok('xw' =~ /^(x)(?(DEFINE)(y)(?z))(w)$/, + 'captures after DEFINE retain lexical numbering'); +is($1, 'x', 'capture before DEFINE keeps number 1'); +ok(!defined($2) && !defined($3) && $4 eq 'w', + 'uncalled definitions stay undefined and following capture is number 4'); + +ok('b' =~ /^(?(DEFINE)(a)(b))(?-1)$/, + 'relative call binds from its lexical position'); +ok('((x))' =~ /^(?&par)(?(DEFINE)(?\((?:x|(?&par))*\)))$/, + 'named definition recurses'); +ok('abc' =~ /^(?&piece)c(?(DEFINE)(?a|ab))$/, + 'called definition backtracks into a later alternative'); + +ok('ab' =~ /^(?&capturing)(?(DEFINE)(?(a)b))$/, + 'capturing definition matches when called'); +ok(!defined($1) && !defined($2) && !defined($+{capturing}), + 'subroutine-local captures are restored after return'); + +my $missing = eval q{ qr/(?&missing)(?(DEFINE)(?x))/; 1 } ? '' : $@; +like($missing, qr/Reference to nonexistent named group/, + 'missing named definition is diagnosed'); +my $branch = eval q{ qr/(?(DEFINE)(?x)|y)/; 1 } ? '' : $@; +like($branch, qr/does not allow branches/, + 'DEFINE container rejects top-level alternatives'); diff --git a/third_party/joni/src/org/joni/Analyser.java b/third_party/joni/src/org/joni/Analyser.java index a3e5b8293..f7768b88f 100644 --- a/third_party/joni/src/org/joni/Analyser.java +++ b/third_party/joni/src/org/joni/Analyser.java @@ -419,6 +419,9 @@ private int quantifiersMemoryInfo(Node node) { info = quantifiersMemoryInfo(en.target); break; + case EncloseType.DEFINE: + break; + default: break; } // inner switch @@ -540,6 +543,7 @@ private int getMinMatchLength(Node node) { break; case EncloseType.ABSENT: + case EncloseType.DEFINE: break; } // inner switch break; @@ -653,6 +657,7 @@ private int getMaxMatchLength(Node node) { break; case EncloseType.ABSENT: + case EncloseType.DEFINE: break; } // inner switch break; @@ -767,6 +772,7 @@ private int getCharLengthTree(Node node, int level) { break; case EncloseType.ABSENT: + case EncloseType.DEFINE: break; } // inner switch break; @@ -1018,6 +1024,7 @@ private Node getHeadValueNode(Node node, boolean exact) { break; case EncloseType.ABSENT: + case EncloseType.DEFINE: break; } // inner switch break; @@ -1315,10 +1322,17 @@ private void setCallAttr(CallNode cn) { EncloseNode en = cn.lexicalTarget != null ? cn.lexicalTarget : env.memNodes[cn.groupNum]; - if (en == null) newValueException(UNDEFINED_NAME_REFERENCE, cn.nameP, cn.nameEnd); + if (en == null) { + if (syntax.op2OptionPerl()) { + newValueException(PERL_REFERENCE_TO_NONEXISTENT_NAMED_GROUP); + } + newValueException(UNDEFINED_NAME_REFERENCE, cn.nameP, cn.nameEnd); + } // Perl subroutine calls do not replace captures already visible in the - // caller. Reused branch-reset numbers need the existing snapshot path. - if (cn.lexicalTarget == null && env.isMultiplexMemNode(cn.groupNum)) { + // caller. Reused branch-reset numbers and DEFINE-only groups need the + // existing call-frame snapshot path. + if ((cn.lexicalTarget == null && env.isMultiplexMemNode(cn.groupNum)) + || (isInsideDefine(en) && !isInsideDefine(cn))) { cn.setRecursion(); } en.setCalled(); @@ -1327,6 +1341,15 @@ private void setCallAttr(CallNode cn) { cn.unsetAddrList = env.unsetAddrList; } + private static boolean isInsideDefine(Node node) { + for (Node current = node.parent; current != null; current = current.parent) { + if (current instanceof EncloseNode en && en.type == EncloseType.DEFINE) { + return true; + } + } + return false; + } + protected final void setupSubExpCall(Node node) { switch(node.getType()) { @@ -1382,6 +1405,9 @@ protected final void setupSubExpCall(Node node) { NameEntry ne = regex.nameToGroupNumbers(cn.name, cn.nameP, cn.nameEnd); if (ne == null) { + if (syntax.op2OptionPerl()) { + newValueException(PERL_REFERENCE_TO_NONEXISTENT_NAMED_GROUP); + } newValueException(UNDEFINED_NAME_REFERENCE, cn.nameP, cn.nameEnd); } else if (ne.backNum > 1 && !syntax.allowMultiplexDefinitionNameCall()) { newValueException(MULTIPLEX_DEFINITION_NAME_CALL, cn.nameP, cn.nameEnd); @@ -1647,6 +1673,11 @@ private AcceptLengthInfo getAcceptLengthInfo(Node node) { case NodeType.ENCLOSE: { EncloseNode enclose = (EncloseNode)node; if (enclose.type == EncloseType.ABSENT) return null; + if (enclose.type == EncloseType.DEFINE) { + AcceptLengthInfo info = new AcceptLengthInfo(); + info.addNormal(0, 0); + return info; + } return getAcceptLengthInfo(enclose.target); } case NodeType.ANCHOR: { @@ -1710,6 +1741,7 @@ private CharLengthRange getCharLengthRange(Node node) { case NodeType.ENCLOSE: { EncloseNode enclose = (EncloseNode)node; if (enclose.type == EncloseType.ABSENT) return null; + if (enclose.type == EncloseType.DEFINE) return new CharLengthRange(0, 0); return getCharLengthRange(enclose.target); } case NodeType.ANCHOR: @@ -2481,6 +2513,10 @@ protected final Node setupTree(Node node, int state) { case EncloseType.ABSENT: setupTree(en.target, state); break; + + case EncloseType.DEFINE: + setupTree(en.target, state); + break; } // inner switch break; @@ -2828,6 +2864,10 @@ private void optimizeNodeLeft(Node node, NodeOptInfo opt, OptEnvironment oenv) { case EncloseType.ABSENT: opt.length.set(0, MinMaxLen.INFINITE_DISTANCE); break; + + case EncloseType.DEFINE: + opt.length.set(0, 0); + break; } // inner switch break; } diff --git a/third_party/joni/src/org/joni/ArrayCompiler.java b/third_party/joni/src/org/joni/ArrayCompiler.java index 13c267bac..4a1baaafa 100644 --- a/third_party/joni/src/org/joni/ArrayCompiler.java +++ b/third_party/joni/src/org/joni/ArrayCompiler.java @@ -977,6 +977,9 @@ private int compileLengthEncloseNode(EncloseNode node) { case EncloseType.ABSENT: len = OPSize.PUSH_ABSENT_POS + OPSize.ABSENT + tlen + OPSize.ABSENT_END; break; + case EncloseType.DEFINE: + len = OPSize.JUMP + tlen; + break; default: newInternalException(PARSER_BUG); return 0; // not reached @@ -1139,6 +1142,11 @@ protected void compileEncloseNode(EncloseNode node) { addOpcode(OPCode.ABSENT_END); break; + case EncloseType.DEFINE: + addOpcodeRelAddr(OPCode.JUMP, compileLengthTree(node.target)); + compileTree(node.target); + break; + default: newInternalException(PARSER_BUG); break; diff --git a/third_party/joni/src/org/joni/ByteCodeMachine.java b/third_party/joni/src/org/joni/ByteCodeMachine.java index a41cfe207..33020e4d9 100644 --- a/third_party/joni/src/org/joni/ByteCodeMachine.java +++ b/third_party/joni/src/org/joni/ByteCodeMachine.java @@ -637,8 +637,12 @@ private boolean opEnd() { && i == completed.frame.getCallFrameNum() && callerCaptures[i] != INVALID_INDEX && callerCaptures[captureCount + i] != INVALID_INDEX; + boolean enclosingCallerCapture = callerCaptures != null + && callerCaptures[i] != INVALID_INDEX + && callerCaptures[captureCount + i] == INVALID_INDEX + && captureClosedAfterReturn(i, completed.returnIndex); int me = repeatStk[memEndStk + i]; - if (preserveCallerCapture) { + if (preserveCallerCapture || enclosingCallerCapture) { region.setBeg(i, captureBegin(i)); region.setEnd(i, captureEnd(i)); } else if (me != INVALID_INDEX) { @@ -3076,7 +3080,10 @@ public int captureBegin(int capture) { checkCapture(capture); if (capture == 0) return sstart - str; int value = visibleCapturePointer(capture, true); - if (value == INVALID_INDEX) return Region.REGION_NOTPOS; + if (value == INVALID_INDEX) { + int committed = committedCaptureOffset(capture, true); + return committed; + } return (bsAt(regex.btMemStart, capture) ? stack[value].getMemPStr() : value) - str; } @@ -3085,10 +3092,28 @@ public int captureEnd(int capture) { checkCapture(capture); if (capture == 0) return s - str; int value = visibleCapturePointer(capture, false); - if (value == INVALID_INDEX) return Region.REGION_NOTPOS; + if (value == INVALID_INDEX) { + int committed = committedCaptureOffset(capture, false); + return committed; + } return (bsAt(regex.btMemEnd, capture) ? stack[value].getMemPStr() : value) - str; } + private int committedCaptureOffset(int capture, boolean begin) { + CompletedRecursiveCall completed = completedRecursiveCall(); + if (completed == null || msaRegion == null || isFindLongest(regex.options | msaOptions) + || capture >= msaRegion.getNumRegs()) { + return Region.REGION_NOTPOS; + } + int[] snapshot = completed.frame.getCallFrameCaptureSnapshot(); + int count = regex.numMem + 1; + if (snapshot[capture] == INVALID_INDEX + || snapshot[count + capture] != INVALID_INDEX) { + return Region.REGION_NOTPOS; + } + return begin ? msaRegion.getBeg(capture) : msaRegion.getEnd(capture); + } + @Override public int physicalNamedCaptureBegin(int capture) { if (committedPhysicalNamedCaptureBeg == null @@ -3178,6 +3203,12 @@ private int visibleCapturePointer(int capture, boolean begin) { } return current; } + if (snapshot[capture] != INVALID_INDEX + && snapshot[count + capture] == INVALID_INDEX + && captureClosedAfterReturn(capture, completed.returnIndex)) { + int closed = previousClosedCapturePointer(capture, begin); + if (closed != INVALID_INDEX) return closed; + } return snapshot[(begin ? 0 : count) + capture]; } diff --git a/third_party/joni/src/org/joni/Parser.java b/third_party/joni/src/org/joni/Parser.java index 4c71dc3bf..6d03aaf61 100644 --- a/third_party/joni/src/org/joni/Parser.java +++ b/third_party/joni/src/org/joni/Parser.java @@ -882,7 +882,11 @@ && left() && enc.isDigit(peek())) { int recursionConditionNameP = -1; int recursionConditionNameEnd = -1; fetch(); - if (c == '?' && left() && peekIs('{')) { + if (c == 'D' && startsWith("EFINE)")) { + p += "EFINE)".length(); + node = new EncloseNode(EncloseType.DEFINE); + break; + } else if (c == '?' && left() && peekIs('{')) { fetch(); calloutConditionId = parseInternalCalloutId(); } else if (c == '?' && left() && (peekIs('=') || peekIs('!'))) { @@ -1150,6 +1154,9 @@ && left() && enc.isDigit(peek())) { } } else { EncloseNode en = (EncloseNode)node; + if (en.type == EncloseType.DEFINE && target.getType() == NodeType.ALT) { + newSyntaxException(PERL_DEFINE_DOES_NOT_ALLOW_BRANCHES); + } en.setTarget(target); if (en.type == EncloseType.MEMORY) { if (syntax.op3OptionECMAScript()) { diff --git a/third_party/joni/src/org/joni/ast/EncloseNode.java b/third_party/joni/src/org/joni/ast/EncloseNode.java index 4a42a5b5d..3271d0ccd 100644 --- a/third_party/joni/src/org/joni/ast/EncloseNode.java +++ b/third_party/joni/src/org/joni/ast/EncloseNode.java @@ -106,6 +106,7 @@ public String typeToString() { if (isOption()) types.append("OPTION "); if (isCondition()) types.append("CONDITION "); if (isAbsent()) types.append("ABSENT "); + if (isDefine()) types.append("DEFINE "); return types.toString(); } @@ -136,4 +137,8 @@ public boolean isStopBacktrack() { public boolean isAbsent() { return (type & ABSENT) != 0; } + + public boolean isDefine() { + return (type & DEFINE) != 0; + } } diff --git a/third_party/joni/src/org/joni/constants/internal/EncloseType.java b/third_party/joni/src/org/joni/constants/internal/EncloseType.java index 67e0b82df..5916fd315 100644 --- a/third_party/joni/src/org/joni/constants/internal/EncloseType.java +++ b/third_party/joni/src/org/joni/constants/internal/EncloseType.java @@ -25,6 +25,7 @@ public interface EncloseType { int STOP_BACKTRACK = 1<<2; int CONDITION = 1<<3; int ABSENT = 1<<4; + int DEFINE = 1<<5; int ALLOWED_IN_LB = MEMORY | OPTION; int ALLOWED_IN_LB_NOT = OPTION; diff --git a/third_party/joni/src/org/joni/exception/ErrorMessages.java b/third_party/joni/src/org/joni/exception/ErrorMessages.java index 42990bb69..a2ce92ac4 100644 --- a/third_party/joni/src/org/joni/exception/ErrorMessages.java +++ b/third_party/joni/src/org/joni/exception/ErrorMessages.java @@ -62,6 +62,8 @@ public interface ErrorMessages extends org.jcodings.exception.ErrorMessages { String INVALID_REPEAT_RANGE_PATTERN = "invalid repeat range {lower,upper}"; String PERL_INVALID_QUANTIFIER = "Invalid quantifier in {,}"; String INVALID_CONDITION_PATTERN = "invalid conditional pattern"; + String PERL_DEFINE_DOES_NOT_ALLOW_BRANCHES = + "(?(DEFINE)....) does not allow branches"; String PERL_GROUP_NAME_MUST_START_WITH_WORD = "Group name must start with a non-digit word character"; String PERL_REFERENCE_TO_NONEXISTENT_NAMED_GROUP = diff --git a/third_party/joni/test/org/joni/test/TestPerlDefineContainer.java b/third_party/joni/test/org/joni/test/TestPerlDefineContainer.java new file mode 100644 index 000000000..9caa7165c --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestPerlDefineContainer.java @@ -0,0 +1,92 @@ +package org.joni.test; + +import static org.joni.constants.SyntaxProperties.ALLOW_MULTIPLEX_DEFINITION_NAME_CALL; +import static org.joni.constants.SyntaxProperties.OP2_OPTION_PERL; +import static org.joni.constants.SyntaxProperties.OP2_OPTION_RUBY; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.nio.charset.StandardCharsets; + +import org.jcodings.specific.UTF8Encoding; +import org.joni.Matcher; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.joni.exception.JOniException; +import org.junit.Test; + +public class TestPerlDefineContainer { + private static final Syntax SYNTAX = new Syntax( + "PerlDefineContainer", Syntax.RUBY.op, + (Syntax.RUBY.op2 & ~OP2_OPTION_RUBY) | OP2_OPTION_PERL, + Syntax.RUBY.op3, + Syntax.RUBY.behavior | ALLOW_MULTIPLEX_DEFINITION_NAME_CALL, + Syntax.RUBY.options, Syntax.RUBY.metaCharTable); + + private static Matcher matcher(String pattern, String input) { + byte[] patternBytes = pattern.getBytes(StandardCharsets.UTF_8); + byte[] inputBytes = input.getBytes(StandardCharsets.UTF_8); + Regex regex = new Regex(patternBytes, 0, patternBytes.length, + Option.CAPTURE_GROUP, UTF8Encoding.INSTANCE, SYNTAX); + return regex.matcher(inputBytes); + } + + private static Matcher assertMatches(String pattern, String input) { + Matcher matcher = matcher(pattern, input); + assertEquals(pattern, 0, matcher.search(0, input.length(), Option.NONE)); + return matcher; + } + + private static void assertDoesNotMatch(String pattern, String input) { + assertEquals(pattern, -1, + matcher(pattern, input).search(0, input.length(), Option.NONE)); + } + + @Test + public void skipsDefinitionsAndRetainsForwardCalls() { + assertMatches("^(?(DEFINE)(?FAIL))A$", "A"); + assertDoesNotMatch("^(?(DEFINE)(?FAIL))$", "FAIL"); + assertMatches("^(?&word)(?(DEFINE)(?[a-z]+))$", "word"); + assertMatches("^(?(DEFINE)(a))(?1)$", "a"); + assertMatches("^(?(DEFINE)(a)(b))(?-1)$", "b"); + } + + @Test + public void preservesRecursionAndBacktracking() { + assertMatches("^(?&par)(?(DEFINE)(?\\((?:x|(?&par))*\\)))$", "((x))"); + assertMatches("^(?&piece)c(?(DEFINE)(?a|ab))$", "abc"); + } + + @Test + public void leavesUncalledCapturesUndefined() { + Matcher matcher = assertMatches("^(x)(?(DEFINE)(y)(?z))(w)$", "xw"); + assertEquals(0, matcher.getRegion().getBeg(1)); + assertEquals(-1, matcher.getRegion().getBeg(2)); + assertEquals(-1, matcher.getRegion().getBeg(3)); + assertEquals(1, matcher.getRegion().getBeg(4)); + } + + @Test + public void publishesAnEnclosingCaptureAroundADefineCall() { + Matcher matcher = assertMatches( + "((?&solution)|%)\\z(?(DEFINE)(?7% solution))", + "7% solution"); + assertEquals(0, matcher.captureBegin(1)); + assertEquals(11, matcher.captureEnd(1)); + assertEquals(0, matcher.getRegion().getBeg(1)); + assertEquals(11, matcher.getRegion().getEnd(1)); + } + + @Test + public void rejectsTopLevelBranches() { + try { + matcher("(?(DEFINE)(?x)|y)", ""); + fail("expected DEFINE branch diagnostic"); + } catch (JOniException error) { + if (!error.getMessage().contains("does not allow branches")) { + fail(error.getMessage()); + } + } + } +} From 3875ddd418a4d11a26ef31ce7adf00279971eb41 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 15:58:10 +0200 Subject: [PATCH 4/8] fix(regex): route ordinary lookbehind through Joni Use the existing syntax-aware route scanner for real positive and negative lookbehind assertions, and retire the unreachable Java lookbehind validator. Keep escape, class, quoted, and comment lookalikes on the normal path. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../runtime/regex/JoniRegexPattern.java | 9 +- .../runtime/regex/RegexPreprocessor.java | 152 ------------------ .../regex/NativeLookbehindRoutingTest.java | 27 ++++ 3 files changed, 34 insertions(+), 154 deletions(-) create mode 100644 src/test/java/org/perlonjava/runtime/regex/NativeLookbehindRoutingTest.java diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index aa8d14bb3..11eb693f3 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -512,6 +512,7 @@ static boolean requiresJoniBackend(String pattern, RegexFlags flags) { return flags != null && flags.isAsciiStrict() || syntaxFeatures.asciiStrictPresent() || syntaxFeatures.keepPresent() + || syntaxFeatures.lookbehindPresent() || syntaxFeatures.conditionalPresent() || syntaxFeatures.alphaAssertionPresent() || pattern.contains("(?{=CALL:") @@ -548,6 +549,7 @@ static boolean containsNamedCharacterEscape(String pattern) { private record PerlSyntaxFeatures(boolean keepPresent, boolean keepInLookaround, + boolean lookbehindPresent, boolean conditionalPresent, boolean alphaAssertionPresent, boolean asciiStrictPresent) {} @@ -560,6 +562,7 @@ private static PerlSyntaxFeatures analyzePerlSyntax(String pattern, boolean exte int lookaroundDepth = 0; java.util.ArrayDeque groups = new java.util.ArrayDeque<>(); boolean keepPresent = false; + boolean lookbehindPresent = false; boolean conditionalPresent = false; boolean alphaAssertionPresent = false; boolean asciiStrictPresent = false; @@ -626,7 +629,7 @@ private static PerlSyntaxFeatures analyzePerlSyntax(String pattern, boolean exte } else if (escaped == 'K') { keepPresent = true; if (lookaroundDepth > 0) { - return new PerlSyntaxFeatures(true, true, conditionalPresent, + return new PerlSyntaxFeatures(true, true, lookbehindPresent, conditionalPresent, alphaAssertionPresent, asciiStrictPresent); } } @@ -682,13 +685,15 @@ private static PerlSyntaxFeatures analyzePerlSyntax(String pattern, boolean exte || pattern.startsWith("(?!", i) || pattern.startsWith("(?<=", i) || pattern.startsWith("(? ... ) - name can start with letter or underscore offset = handleNamedCapture(c3, s, offset, length, sb, regexFlags); @@ -1568,29 +1558,6 @@ static void regexErrorNoPosition(String errMsg) { throw new PerlCompilerException(errMsg); } - /** - * Validates that a lookbehind assertion doesn't potentially match more than 255 characters. - */ - private static void validateLookbehindLength(String s, int offset) { - // System.err.println("DEBUG: validateLookbehindLength called with string length " + s.length()); - // System.err.println("DEBUG: String codepoints: "); - // s.codePoints().forEach(cp -> System.err.printf("U+%04X ", cp)); - // System.err.println(); - - int start = offset + 4; // Skip past (?<= or (?= 0) { - throw new PerlJavaUnimplementedException( - "Lookbehind longer than 255 not implemented in regex m/" + s + "/"); - } - int maxLength = calculateMaxLength(s, start); - - if (maxLength >= 255 || maxLength == -1) { // >= 255 means 255 or more - throw new PerlJavaUnimplementedException("Lookbehind longer than 255 not implemented in regex m/" + s + "/"); - } - } - static void regexErrorSimple(String s, String errMsg) { throw new PerlCompilerException(errMsg + " in regex m/" + s + "/"); } @@ -1654,125 +1621,6 @@ private static boolean isUnimplementedWarnMode() { .get("JPERL_UNIMPLEMENTED").toString()); } - /** - * Calculates the maximum length a pattern can match. - * Returns -1 if the pattern can match unlimited length. - */ - private static int calculateMaxLength(String pattern, int start) { - int pos = start; - int totalLength = 0; - int depth = 1; // We're inside the lookbehind parentheses - - while (pos < pattern.length() && depth > 0) { - char ch = pattern.charAt(pos); - - if (ch == '(') { - depth++; - pos++; - } else if (ch == ')') { - depth--; - if (depth == 0) break; - pos++; - } else if (ch == '\\' && pos + 1 < pattern.length()) { - // Handle escape sequences - pos += 2; - totalLength++; - } else if (ch == '[') { - // A character class inside lookbehind has fixed width 1. - // Skip over its contents so literal braces like [${FOO}] - // are not misread as {n,m} quantifiers. - pos++; - boolean inEscape = false; - boolean first = true; - while (pos < pattern.length()) { - char cc = pattern.charAt(pos); - if (inEscape) { - inEscape = false; - pos++; - first = false; - continue; - } - if (cc == '\\') { - inEscape = true; - pos++; - first = false; - continue; - } - if (cc == ']' && !first) { - pos++; - break; - } - if (cc == '^' && first) { - pos++; - first = false; - continue; - } - pos++; - first = false; - } - totalLength++; - } else if (ch == '.') { - // Check if followed by * or + - if (pos + 1 < pattern.length()) { - char next = pattern.charAt(pos + 1); - if (next == '*' || next == '+') { - return -1; // Unlimited length - } - } - totalLength++; - pos++; - } else if (ch == '*' || ch == '+') { - return -1; // Previous element can repeat unlimited times - } else if (ch == '?') { - // Handle special case of (? which might be a group - if (pos + 1 < pattern.length() && pattern.charAt(pos + 1) == '&') { - // This is (?&...) which is a subroutine call - return -1; // Can match unlimited - } - pos++; - } else if (ch == '{') { - // Handle {n,m} quantifiers - int endBrace = pattern.indexOf('}', pos); - if (endBrace > pos && isValidQuantifierAt(pattern, pos)) { - String quantifier = pattern.substring(pos + 1, endBrace); - int multiplier = parseQuantifierMax(quantifier); - if (multiplier == -1) { - return -1; // Unlimited - } - // The quantifier applies to the immediately preceding atom - totalLength = totalLength - 1 + multiplier; - pos = endBrace + 1; - } else { - totalLength++; - pos++; - } - } else { - // Regular character - totalLength++; - pos++; - } - } - - return totalLength; - } - - /** - * Parses a quantifier like "200", "0,255", "1000" and returns the maximum count. - * Returns -1 if unbounded (e.g., "5,"). - */ - private static int parseQuantifierMax(String quantifier) { - quantifier = quantifier.trim(); - if (quantifier.contains(",")) { - String[] parts = quantifier.split(","); - if (parts.length == 1 || parts[1].trim().isEmpty()) { - return -1; // {n,} is unbounded - } - return Integer.parseInt(parts[1].trim()); - } else { - return Integer.parseInt(quantifier); - } - } - private static int findClosingBrace(String s, int start, int length) { int depth = 1; int pos = start; diff --git a/src/test/java/org/perlonjava/runtime/regex/NativeLookbehindRoutingTest.java b/src/test/java/org/perlonjava/runtime/regex/NativeLookbehindRoutingTest.java new file mode 100644 index 000000000..57c6e4856 --- /dev/null +++ b/src/test/java/org/perlonjava/runtime/regex/NativeLookbehindRoutingTest.java @@ -0,0 +1,27 @@ +package org.perlonjava.runtime.regex; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Tag; + +@Tag("unit") +class NativeLookbehindRoutingTest { + @Test + void routesOnlySyntacticallyRealLookbehindsToJoni() { + assertTrue(JoniRegexPattern.requiresJoniBackend("(?<=ab)c")); + assertTrue(JoniRegexPattern.requiresJoniBackend("(? Date: Wed, 19 Aug 2026 16:05:19 +0200 Subject: [PATCH 5/8] fix(regex): route branch reset through Joni Route syntactically real branch-reset groups through native Joni and remove the unreachable Java branch-reset rewrite and capture-map state. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../runtime/regex/BranchResetCaptureMap.java | 118 -------------- .../runtime/regex/JavaRegexMatcher.java | 42 +---- .../runtime/regex/JoniRegexPattern.java | 14 +- .../runtime/regex/RegexPreprocessor.java | 153 ------------------ .../runtime/regex/RuntimeRegex.java | 32 +--- .../regex/NativeBranchResetRoutingTest.java | 19 +++ 6 files changed, 35 insertions(+), 343 deletions(-) delete mode 100644 src/main/java/org/perlonjava/runtime/regex/BranchResetCaptureMap.java create mode 100644 src/test/java/org/perlonjava/runtime/regex/NativeBranchResetRoutingTest.java diff --git a/src/main/java/org/perlonjava/runtime/regex/BranchResetCaptureMap.java b/src/main/java/org/perlonjava/runtime/regex/BranchResetCaptureMap.java deleted file mode 100644 index 5cc4a27ff..000000000 --- a/src/main/java/org/perlonjava/runtime/regex/BranchResetCaptureMap.java +++ /dev/null @@ -1,118 +0,0 @@ -package org.perlonjava.runtime.regex; - -import java.util.ArrayList; -import java.util.List; - -/** Builds Java-group to Perl-group mappings for {@code (?|...)} patterns. */ -final class BranchResetCaptureMap { - private BranchResetCaptureMap() { - } - - static int[] build(String pattern) { - if (pattern == null || !pattern.contains("(?|")) return null; - Parser parser = new Parser(pattern); - parser.parseAlternatives(false, '\0'); - return parser.mapping.stream().mapToInt(Integer::intValue).toArray(); - } - - private static final class Parser { - private final String pattern; - private final List mapping = new ArrayList<>(); - private int offset; - private int nextGroup; - - private Parser(String pattern) { - this.pattern = pattern; - } - - private void parseAlternatives(boolean resetNumbers, char terminator) { - int branchBase = nextGroup; - int branchMaximum = nextGroup; - while (offset < pattern.length()) { - char ch = pattern.charAt(offset); - if (ch == terminator) { - offset++; - break; - } - if (ch == '|' && terminator != '\0') { - branchMaximum = Math.max(branchMaximum, nextGroup); - if (resetNumbers) nextGroup = branchBase; - offset++; - continue; - } - if (ch == '\\') { - offset += Math.min(2, pattern.length() - offset); - continue; - } - if (ch == '[') { - skipCharacterClass(); - continue; - } - if (ch != '(') { - offset++; - continue; - } - parseGroup(); - } - if (resetNumbers) nextGroup = Math.max(branchMaximum, nextGroup); - } - - private void parseGroup() { - if (pattern.startsWith("(?#", offset)) { - offset += 3; - while (offset < pattern.length() && pattern.charAt(offset) != ')') offset++; - if (offset < pattern.length()) offset++; - return; - } - - boolean branchReset = pattern.startsWith("(?|", offset); - boolean capturing = isCapturingGroup(offset); - offset += branchReset ? 3 : groupPrefixLength(offset); - if (capturing) mapping.add(++nextGroup); - parseAlternatives(branchReset, ')'); - } - - private boolean isCapturingGroup(int start) { - if (start + 1 >= pattern.length() || pattern.charAt(start + 1) != '?') return true; - if (pattern.startsWith("(?<", start)) { - return start + 3 < pattern.length() - && pattern.charAt(start + 3) != '=' - && pattern.charAt(start + 3) != '!'; - } - return pattern.startsWith("(?P<", start) || pattern.startsWith("(?'", start); - } - - private int groupPrefixLength(int start) { - if (start + 1 >= pattern.length() || pattern.charAt(start + 1) != '?') return 1; - if (pattern.startsWith("(?P<", start)) return namedPrefixEnd(start, start + 4, '>'); - if (pattern.startsWith("(?<", start) && isCapturingGroup(start)) { - return namedPrefixEnd(start, start + 3, '>'); - } - if (pattern.startsWith("(?'", start)) return namedPrefixEnd(start, start + 3, '\''); - int colon = pattern.indexOf(':', start + 2); - int close = pattern.indexOf(')', start + 2); - if (colon >= 0 && (close < 0 || colon < close)) return colon - start + 1; - return 2; - } - - private int namedPrefixEnd(int groupStart, int nameStart, char delimiter) { - int end = pattern.indexOf(delimiter, nameStart); - return end < 0 ? 2 : end - groupStart + 1; - } - - private void skipCharacterClass() { - offset++; - boolean escaped = false; - while (offset < pattern.length()) { - char ch = pattern.charAt(offset++); - if (escaped) { - escaped = false; - } else if (ch == '\\') { - escaped = true; - } else if (ch == ']') { - return; - } - } - } - } -} diff --git a/src/main/java/org/perlonjava/runtime/regex/JavaRegexMatcher.java b/src/main/java/org/perlonjava/runtime/regex/JavaRegexMatcher.java index b14bee5b4..915825eb3 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JavaRegexMatcher.java +++ b/src/main/java/org/perlonjava/runtime/regex/JavaRegexMatcher.java @@ -5,24 +5,8 @@ final class JavaRegexMatcher implements RegexMatcher { private final Matcher matcher; - private final int[] javaToPerlGroup; - private final int perlGroupCount; - JavaRegexMatcher(Matcher matcher) { - this(matcher, null); - } - - JavaRegexMatcher(Matcher matcher, int[] javaToPerlGroup) { this.matcher = matcher; - this.javaToPerlGroup = javaToPerlGroup != null - && javaToPerlGroup.length == matcher.groupCount() - ? javaToPerlGroup - : null; - int maximum = 0; - if (this.javaToPerlGroup != null) { - for (int group : this.javaToPerlGroup) maximum = Math.max(maximum, group); - } - this.perlGroupCount = this.javaToPerlGroup == null ? matcher.groupCount() : maximum; } Matcher unwrap() { @@ -35,33 +19,15 @@ Matcher unwrap() { @Override public void useTransparentBounds(boolean enabled) { matcher.useTransparentBounds(enabled); } @Override public int start() { return matcher.start(); } @Override public int end() { return matcher.end(); } - @Override public int start(int index) { return mappedOffset(index, true); } - @Override public int end(int index) { return mappedOffset(index, false); } + @Override public int start(int index) { return matcher.start(index); } + @Override public int end(int index) { return matcher.end(index); } @Override public int start(String name) { return matcher.start(name); } @Override public int end(String name) { return matcher.end(name); } @Override public String group(int index) { - if (javaToPerlGroup == null || index == 0) return matcher.group(index); - for (int javaGroup = 1; javaGroup <= javaToPerlGroup.length; javaGroup++) { - if (javaToPerlGroup[javaGroup - 1] == index && matcher.start(javaGroup) >= 0) { - return matcher.group(javaGroup); - } - } - return null; + return matcher.group(index); } @Override public String group(String name) { return matcher.group(name); } - @Override public int groupCount() { return perlGroupCount; } + @Override public int groupCount() { return matcher.groupCount(); } @Override public Map namedGroups() { return matcher.pattern().namedGroups(); } @Override public String patternDescription() { return matcher.pattern().pattern(); } - - private int mappedOffset(int index, boolean start) { - if (javaToPerlGroup == null || index == 0) { - return start ? matcher.start(index) : matcher.end(index); - } - for (int javaGroup = 1; javaGroup <= javaToPerlGroup.length; javaGroup++) { - if (javaToPerlGroup[javaGroup - 1] == index && matcher.start(javaGroup) >= 0) { - return start ? matcher.start(javaGroup) : matcher.end(javaGroup); - } - } - return -1; - } } diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index 11eb693f3..4ea903c22 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -503,16 +503,13 @@ static boolean requiresJoniBackend(String pattern, RegexFlags flags) { boolean hasSubroutineCall = pattern.matches("(?s).*\\(\\?[+-]?\\d+\\).*") || pattern.contains("(?&") || pattern.contains("(?P>"); - // Keep automatic routing on Java until the native branch-reset call - // implementation passes its combined imported-corpus gate. Explicit - // Joni mode still exercises the native implementation directly. - boolean branchResetCallUsesJava = pattern.contains("(?|") && hasSubroutineCall; PerlSyntaxFeatures syntaxFeatures = analyzePerlSyntax( pattern, flags != null && flags.isExtended()); return flags != null && flags.isAsciiStrict() || syntaxFeatures.asciiStrictPresent() || syntaxFeatures.keepPresent() || syntaxFeatures.lookbehindPresent() + || syntaxFeatures.branchResetPresent() || syntaxFeatures.conditionalPresent() || syntaxFeatures.alphaAssertionPresent() || pattern.contains("(?{=CALL:") @@ -525,7 +522,7 @@ static boolean requiresJoniBackend(String pattern, RegexFlags flags) { || pattern.contains("(*COMMIT") || pattern.contains("(*MARK") || pattern.contains("(*:") - || (hasSubroutineCall && !branchResetCallUsesJava); + || hasSubroutineCall; } static boolean containsNamedCharacterEscape(String pattern) { @@ -550,6 +547,7 @@ static boolean containsNamedCharacterEscape(String pattern) { private record PerlSyntaxFeatures(boolean keepPresent, boolean keepInLookaround, boolean lookbehindPresent, + boolean branchResetPresent, boolean conditionalPresent, boolean alphaAssertionPresent, boolean asciiStrictPresent) {} @@ -563,6 +561,7 @@ private static PerlSyntaxFeatures analyzePerlSyntax(String pattern, boolean exte java.util.ArrayDeque groups = new java.util.ArrayDeque<>(); boolean keepPresent = false; boolean lookbehindPresent = false; + boolean branchResetPresent = false; boolean conditionalPresent = false; boolean alphaAssertionPresent = false; boolean asciiStrictPresent = false; @@ -629,7 +628,7 @@ private static PerlSyntaxFeatures analyzePerlSyntax(String pattern, boolean exte } else if (escaped == 'K') { keepPresent = true; if (lookaroundDepth > 0) { - return new PerlSyntaxFeatures(true, true, lookbehindPresent, conditionalPresent, + return new PerlSyntaxFeatures(true, true, lookbehindPresent, branchResetPresent, conditionalPresent, alphaAssertionPresent, asciiStrictPresent); } } @@ -687,13 +686,14 @@ private static PerlSyntaxFeatures analyzePerlSyntax(String pattern, boolean exte || pattern.startsWith("(?a)|(?b)` (legal in Perl, @@ -99,10 +98,6 @@ static boolean hadInlinePFlag() { return inlinePFlagEncountered; } - static boolean hadBranchReset() { - return branchResetEncountered; - } - /** * Preprocesses a given regex string to make it compatible with Java's regex engine. * This involves handling various constructs and escape sequences that Java does not @@ -134,7 +129,6 @@ private static String preProcessRegexInternal(String s, RegexFlags regexFlags) { captureGroupCount = 0; deferredUnicodePropertyEncountered = false; inlinePFlagEncountered = false; - branchResetEncountered = false; seenNamedCaptures.clear(); emittedNamedCaptures.clear(); duplicateNameCounter = 0; @@ -1144,9 +1138,6 @@ private static int handleParentheses(String s, int offset, int length, StringBui // Atomic group (?>...) - non-backtracking group sb.append("(?>"); offset = handleRegex(s, offset + 3, sb, regexFlags, true); - } else if (c3 == '|') { - // Handle (?|...) branch reset groups - offset = handleBranchReset(s, offset, length, sb, regexFlags); } else if (Character.isDigit(c3)) { // Recursive subpattern reference (?1), (?2), etc. // These refer to the subpattern with that number and are recursive @@ -1235,150 +1226,6 @@ static java.util.List namedBackreferenceTargets(String perlName) { return emitted; } - /** - * Handles branch reset groups (?|alt1|alt2|alt3) - *

- * Branch reset groups reset capture group numbering for each alternative. - * In Perl, (?|(a)|(b)) means both alternatives capture to $1. - *

- * Phase 1 Implementation: Converts to non-capturing group with alternatives. - * This allows compilation and works for same-structure alternatives. - * Full runtime remapping would be needed for perfect Perl emulation. - * - * @param s The regex string - * @param offset Current position (at '(' of '(?|') - * @param length Length of regex string - * @param sb StringBuilder for output - * @param regexFlags Regex flags - * @return New offset after processing the branch reset group - */ - private static int handleBranchReset(String s, int offset, int length, StringBuilder sb, RegexFlags regexFlags) { - // Mark that this pattern uses branch reset - branchResetEncountered = true; - - // Save the starting group count - int startGroupCount = captureGroupCount; - - // Skip past '(?|' - offset += 3; - - // First pass: collect raw alternative strings - java.util.List rawAlternatives = new java.util.ArrayList<>(); - StringBuilder altSb = new StringBuilder(); - int parenDepth = 1; // We're inside the (?| already - boolean inEscape = false; - boolean inCharClass = false; - - while (offset < length && parenDepth > 0) { - char c = s.charAt(offset); - - if (inEscape) { - altSb.append(c); - inEscape = false; - offset++; - continue; - } - - if (c == '\\') { - altSb.append(c); - inEscape = true; - offset++; - continue; - } - - if (inCharClass) { - altSb.append(c); - if (c == ']') { - inCharClass = false; - } - offset++; - continue; - } - - if (c == '[') { - altSb.append(c); - inCharClass = true; - offset++; - continue; - } - - if (c == '(') { - parenDepth++; - altSb.append(c); - offset++; - continue; - } - - if (c == ')') { - parenDepth--; - if (parenDepth == 0) { - // End of branch reset group - save last alternative - rawAlternatives.add(altSb.toString()); - break; - } - altSb.append(c); - offset++; - continue; - } - - if (c == '|' && parenDepth == 1) { - // End of this alternative, start of next - rawAlternatives.add(altSb.toString()); - altSb = new StringBuilder(); - offset++; - continue; - } - - // Regular character - altSb.append(c); - offset++; - } - - if (parenDepth != 0) { - regexError(s, offset, "Unmatched ( in branch reset group"); - } - - // Second pass: process each alternative and track capture counts - java.util.List processedAlternatives = new java.util.ArrayList<>(); - java.util.List captureCounts = new java.util.ArrayList<>(); - - for (String rawAlt : rawAlternatives) { - // Reset capture count for this alternative - captureGroupCount = startGroupCount; - - // Process this alternative through handleRegex - StringBuilder processedAlt = new StringBuilder(); - handleRegex(rawAlt, 0, processedAlt, regexFlags, false); - - processedAlternatives.add(processedAlt.toString()); - captureCounts.add(captureGroupCount - startGroupCount); - } - - // Find the maximum capture count across all alternatives - int maxCaptures = 0; - for (int count : captureCounts) { - if (count > maxCaptures) { - maxCaptures = count; - } - } - - // Set the final capture group count to start + max captures - captureGroupCount = startGroupCount + maxCaptures; - - // Build the output: (?:alt1|alt2|alt3) - // This is a non-capturing wrapper with all alternatives - // Note: We don't append the closing ')' here - the caller (handleParentheses) will do that - sb.append("(?:"); - for (int i = 0; i < processedAlternatives.size(); i++) { - if (i > 0) { - sb.append('|'); - } - sb.append(processedAlternatives.get(i)); - } - - return offset; - } - private static int handleCharacterClass(String s, boolean flag_xx, StringBuilder sb, int c, int offset) { final int length = s.length(); int classStart = sb.length(); diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java index 8f0088181..62308c93e 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java @@ -190,7 +190,6 @@ static void updateControlVerbVariables(String mark, String error) { private JoniRegexPattern.NamedCharacterCache namedCharacterCache; List executableCallbacks = List.of(); private boolean executableCallbacksReleased; - int[] branchResetCaptureMap; int patternFlags; int patternFlagsUnicode; public String patternString; @@ -214,7 +213,6 @@ static void updateControlVerbVariables(String mark, String error) { private boolean matched = false; private boolean hasCodeBlockCaptures = false; // True if regex has (?{...}) code blocks private boolean deferredUserDefinedUnicodeProperties = false; - private boolean hasBranchReset = false; // True if pattern uses (?|...) branch reset // An empty qr// object keeps its own empty pattern when interpolated; // only empty match/substitution string syntax reuses the previous match. private boolean quoteConstruction = false; @@ -249,7 +247,6 @@ public RuntimeRegex cloneTracked() { copy.recursivePatternBytes = this.recursivePatternBytes; copy.namedCharacterCache = this.namedCharacterCache; copy.setExecutableCallbacks(this.executableCallbacks); - copy.branchResetCaptureMap = this.branchResetCaptureMap; copy.patternFlags = this.patternFlags; copy.patternFlagsUnicode = this.patternFlagsUnicode; copy.patternString = this.patternString; @@ -261,7 +258,6 @@ public RuntimeRegex cloneTracked() { copy.regexFlags = this.regexFlags; copy.hasCodeBlockCaptures = this.hasCodeBlockCaptures; copy.deferredUserDefinedUnicodeProperties = this.deferredUserDefinedUnicodeProperties; - copy.hasBranchReset = this.hasBranchReset; copy.quoteConstruction = this.quoteConstruction; copy.warningsOnUse = new ArrayList<>(this.warningsOnUse); copy.inlineModifierWarnings = new ArrayList<>(this.inlineModifierWarnings); @@ -315,9 +311,7 @@ public RegexMatcher matcher(RuntimeScalar string, String input) { return selectRecursivePattern(string).matcher(input, executableCallbacks, string); } Pattern selected = selectPattern(string, input); - return new JavaRegexMatcher( - selected.matcher(new RegexTimeoutCharSequence(input)), - branchResetCaptureMap); + return new JavaRegexMatcher(selected.matcher(new RegexTimeoutCharSequence(input))); } private JoniRegexPattern selectRecursivePattern(RuntimeScalar string) { @@ -702,21 +696,16 @@ private static synchronized RuntimeRegex compileSynchronized( regex.warningsOnUse.addAll(regex.inlineModifierWarnings); regex.hasPreservesMatch = regex.regexFlags.preservesMatch() || RegexFlags.hasInlinePreserveModifier(compilePatternString); - regex.hasBranchReset = false; } else { regex.deferredUserDefinedUnicodeProperties = RegexPreprocessor.hadDeferredUnicodePropertyEncountered(); regex.hasPreservesMatch = regex.regexFlags.preservesMatch() || RegexFlags.hasInlinePreserveModifier(compilePatternString) || RegexPreprocessor.hadInlinePFlag(); - regex.hasBranchReset = RegexPreprocessor.hadBranchReset(); regex.warningsOnUse.addAll(RegexPreprocessor.getWarningsOnUse()); } regex.patternString = originalPatternString; regex.javaPatternString = javaPattern; - regex.branchResetCaptureMap = usesRecursiveBackend - ? null - : BranchResetCaptureMap.build(compilePatternString); regex.requiredLiteral = usesRecursiveBackend ? null : findTopLevelRequiredLiteral(compilePatternString, regex.regexFlags); @@ -991,7 +980,6 @@ private static RuntimeRegex ensureCompiledForRuntime(RuntimeRegex regex) { regex.patternUnicode = recompiled.patternUnicode; regex.recursivePattern = recompiled.recursivePattern; regex.recursivePatternUnicode = recompiled.recursivePatternUnicode; - regex.branchResetCaptureMap = recompiled.branchResetCaptureMap; regex.patternNoInternalMarkers = recompiled.patternNoInternalMarkers; regex.patternUnicodeNoInternalMarkers = recompiled.patternUnicodeNoInternalMarkers; regex.patternFlags = recompiled.patternFlags; @@ -1429,7 +1417,6 @@ && containsExecutableSource(patternString.toString(), modifierStr.indexOf('x') > regex.recursivePattern = originalRegex.recursivePattern; regex.recursivePatternUnicode = originalRegex.recursivePatternUnicode; regex.setExecutableCallbacks(originalRegex.executableCallbacks); - regex.branchResetCaptureMap = originalRegex.branchResetCaptureMap; regex.patternNoInternalMarkers = originalRegex.patternNoInternalMarkers; regex.patternUnicodeNoInternalMarkers = originalRegex.patternUnicodeNoInternalMarkers; regex.patternString = originalRegex.patternString; @@ -1473,7 +1460,6 @@ && containsExecutableSource(patternString.toString(), modifierStr.indexOf('x') > regex.recursivePattern = originalRegex.recursivePattern; regex.recursivePatternUnicode = originalRegex.recursivePatternUnicode; regex.setExecutableCallbacks(originalRegex.executableCallbacks); - regex.branchResetCaptureMap = originalRegex.branchResetCaptureMap; regex.patternNoInternalMarkers = originalRegex.patternNoInternalMarkers; regex.patternUnicodeNoInternalMarkers = originalRegex.patternUnicodeNoInternalMarkers; regex.patternString = originalRegex.patternString; @@ -1692,7 +1678,6 @@ public static RuntimeScalar getReplacementRegex(RuntimeScalar patternString, Run regex.recursivePattern = resolvedRegex.recursivePattern; regex.recursivePatternUnicode = resolvedRegex.recursivePatternUnicode; regex.executableCallbacks = resolvedRegex.executableCallbacks; - regex.branchResetCaptureMap = resolvedRegex.branchResetCaptureMap; regex.patternNoInternalMarkers = resolvedRegex.patternNoInternalMarkers; regex.patternUnicodeNoInternalMarkers = resolvedRegex.patternUnicodeNoInternalMarkers; regex.patternString = resolvedRegex.patternString; @@ -1704,7 +1689,6 @@ public static RuntimeScalar getReplacementRegex(RuntimeScalar patternString, Run regex.quoteConstruction = resolvedRegex.quoteConstruction; regex.useGAssertion = resolvedRegex.useGAssertion; regex.patternFlags = resolvedRegex.patternFlags; - regex.hasBranchReset = resolvedRegex.hasBranchReset; regex.hasCodeBlockCaptures = resolvedRegex.hasCodeBlockCaptures; regex.warningsOnUse = new ArrayList<>(resolvedRegex.warningsOnUse); regex.inlineModifierWarnings = new ArrayList<>(resolvedRegex.inlineModifierWarnings); @@ -1735,7 +1719,6 @@ public static RuntimeScalar getReplacementRegex(RuntimeScalar patternString, Run regex.recursivePattern = recompiledRegex.recursivePattern; regex.recursivePatternUnicode = recompiledRegex.recursivePatternUnicode; regex.executableCallbacks = resolvedRegex.executableCallbacks; - regex.branchResetCaptureMap = recompiledRegex.branchResetCaptureMap; regex.patternNoInternalMarkers = recompiledRegex.patternNoInternalMarkers; regex.patternUnicodeNoInternalMarkers = recompiledRegex.patternUnicodeNoInternalMarkers; regex.patternString = recompiledRegex.patternString; @@ -1743,7 +1726,6 @@ public static RuntimeScalar getReplacementRegex(RuntimeScalar patternString, Run regex.hasPreservesMatch = recompiledRegex.hasPreservesMatch; regex.useGAssertion = recompiledRegex.useGAssertion; regex.patternFlags = recompiledRegex.patternFlags; - regex.hasBranchReset = recompiledRegex.hasBranchReset; regex.hasCodeBlockCaptures = recompiledRegex.hasCodeBlockCaptures; regex.warningsOnUse = new ArrayList<>(recompiledRegex.warningsOnUse); regex.inlineModifierWarnings = new ArrayList<>( @@ -1979,7 +1961,6 @@ private static RuntimeBase matchRegexDirect(RuntimeScalar quotedRegex, RuntimeSc tempRegex.recursivePatternUnicode = regexState.lastSuccessfulPattern.recursivePatternUnicode; tempRegex.executableCallbacks = regexState.lastSuccessfulPattern.executableCallbacks; - tempRegex.branchResetCaptureMap = regexState.lastSuccessfulPattern.branchResetCaptureMap; tempRegex.patternNoInternalMarkers = regexState.lastSuccessfulPattern.patternNoInternalMarkers; tempRegex.patternUnicodeNoInternalMarkers = regexState.lastSuccessfulPattern.patternUnicodeNoInternalMarkers; tempRegex.patternString = regexState.lastSuccessfulPattern.patternString; @@ -2039,7 +2020,7 @@ private static RuntimeBase matchRegexDirect(RuntimeScalar quotedRegex, RuntimeSc if (inputStr.isEmpty() && (pattern.flags() & Pattern.MULTILINE) != 0) { pattern = Pattern.compile(pattern.pattern(), pattern.flags() & ~Pattern.MULTILINE); } - matcher = new JavaRegexMatcher(pattern.matcher(matchInput), regex.branchResetCaptureMap); + matcher = new JavaRegexMatcher(pattern.matcher(matchInput)); } // hexPrinter(inputStr); @@ -2769,8 +2750,7 @@ private static RegexMatcher findNonEmptyGlobalRetry(RuntimeRegex regex, return null; } } - retryMatcher = new JavaRegexMatcher( - notemptyPattern.matcher(matchInput), regex.branchResetCaptureMap); + retryMatcher = new JavaRegexMatcher(notemptyPattern.matcher(matchInput)); nativeNotEmpty = false; } @@ -2887,7 +2867,6 @@ public static RuntimeBase replaceRegex(RuntimeScalar quotedRegex, RuntimeScalar tempRegex.recursivePatternUnicode = state().lastSuccessfulPattern.recursivePatternUnicode; tempRegex.executableCallbacks = state().lastSuccessfulPattern.executableCallbacks; - tempRegex.branchResetCaptureMap = state().lastSuccessfulPattern.branchResetCaptureMap; tempRegex.patternNoInternalMarkers = state().lastSuccessfulPattern.patternNoInternalMarkers; tempRegex.patternUnicodeNoInternalMarkers = state().lastSuccessfulPattern.patternUnicodeNoInternalMarkers; tempRegex.patternString = state().lastSuccessfulPattern.patternString; @@ -2928,7 +2907,7 @@ public static RuntimeBase replaceRegex(RuntimeScalar quotedRegex, RuntimeScalar if (inputStr.isEmpty() && (pattern.flags() & Pattern.MULTILINE) != 0) { pattern = Pattern.compile(pattern.pattern(), pattern.flags() & ~Pattern.MULTILINE); } - matcher = new JavaRegexMatcher(pattern.matcher(matchInput), regex.branchResetCaptureMap); + matcher = new JavaRegexMatcher(pattern.matcher(matchInput)); } Pattern nonEmptySubstitutionPattern = pattern != null && regex.regexFlags != null && regex.regexFlags.isGlobalMatch() @@ -3051,8 +3030,7 @@ public static RuntimeBase replaceRegex(RuntimeScalar quotedRegex, RuntimeScalar if ((nonEmptySubstitutionPattern != null || regex.recursivePattern != null) && zeroLengthOffset <= inputStr.length()) { RegexMatcher retryMatcher = nonEmptySubstitutionPattern != null - ? new JavaRegexMatcher(nonEmptySubstitutionPattern.matcher(matchInput), - regex.branchResetCaptureMap) + ? new JavaRegexMatcher(nonEmptySubstitutionPattern.matcher(matchInput)) : regex.selectRecursivePattern(inputValue) .matcher(inputStr, regex.executableCallbacks, inputValue); // The synthetic (?<=[\s\S]) suffix relies on opaque bounds diff --git a/src/test/java/org/perlonjava/runtime/regex/NativeBranchResetRoutingTest.java b/src/test/java/org/perlonjava/runtime/regex/NativeBranchResetRoutingTest.java new file mode 100644 index 000000000..fcd51aa50 --- /dev/null +++ b/src/test/java/org/perlonjava/runtime/regex/NativeBranchResetRoutingTest.java @@ -0,0 +1,19 @@ +package org.perlonjava.runtime.regex; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +@Tag("unit") +class NativeBranchResetRoutingTest { + @Test + void routesOnlyRealBranchResetGroupsToJoni() { + assertTrue(JoniRegexPattern.requiresJoniBackend("(?|(a)|(b))")); + assertFalse(JoniRegexPattern.requiresJoniBackend("\\(?|a\\)")); + assertFalse(JoniRegexPattern.requiresJoniBackend("[(?|a)]")); + assertFalse(JoniRegexPattern.requiresJoniBackend("\\Q(?|a)\\E")); + assertFalse(JoniRegexPattern.requiresJoniBackend("(?# (?|a))x")); + } +} From 63e5d67f55abc506f3d0ad76b93e7d078ff8204a Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 16:36:30 +0200 Subject: [PATCH 6/8] feat(joni): implement Perl plain non-newline escape Preserve regex-only plain \\N and its interval forms through frontend parsing, compile it natively as a negated line-feed class, and retain named-character handling and Perl's character-class diagnostic. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../frontend/parser/StringSegmentParser.java | 15 ++- .../unit/regex/plain_non_newline_escape.t | 33 +++++++ third_party/joni/src/org/joni/Lexer.java | 98 ++++++++++++++++--- .../src/org/joni/exception/ErrorMessages.java | 2 + .../org/joni/test/TestPerlNonNewline.java | 96 ++++++++++++++++++ 5 files changed, 228 insertions(+), 16 deletions(-) create mode 100644 src/test/resources/unit/regex/plain_non_newline_escape.t create mode 100644 third_party/joni/test/org/joni/test/TestPerlNonNewline.java diff --git a/src/main/java/org/perlonjava/frontend/parser/StringSegmentParser.java b/src/main/java/org/perlonjava/frontend/parser/StringSegmentParser.java index eb485a89e..0b2ecef3f 100644 --- a/src/main/java/org/perlonjava/frontend/parser/StringSegmentParser.java +++ b/src/main/java/org/perlonjava/frontend/parser/StringSegmentParser.java @@ -1480,8 +1480,9 @@ private boolean isHexDigit(String chr) { */ void handleUnicodeNameEscape() { if (!"{".equals(TokenUtils.peekChar(parser))) { - // Not a Unicode name escape, treat as literal - appendToCurrentSegment("N"); + // In a regex, plain \N is Perl's non-newline atom. Keep the escape + // intact for the regex backend; quoted strings still treat it as N. + appendToCurrentSegment(isRegex ? "\\N" : "N"); return; } @@ -1498,6 +1499,12 @@ void handleUnicodeNameEscape() { if ("}".equals(chr)) { TokenUtils.consumeChar(parser); // consume '}' var name = nameBuilder.toString(); + if (isRegex && isPlainNonNewlineInterval(name)) { + // A brace immediately following plain \N can be its quantifier, + // not a named character. Leave both pieces for the regex lexer. + appendToCurrentSegment("\\N{" + name + "}"); + return; + } NamedCharacterExpansion.SourceMode sourceMode = ctx.compilerOptions.isByteStringSource || (!ctx.symbolTable.isStrictOptionEnabled(HINT_UTF8) @@ -1536,6 +1543,10 @@ void handleUnicodeNameEscape() { } } + private boolean isPlainNonNewlineInterval(String contents) { + return contents.matches("(?:[0-9]+(?:,[0-9]*)?|,[0-9]+)"); + } + private void throwNamedCharacterDiagnostic(String diagnostic) { int errorIndex = this.tokenIndex; var location = ctx.errorUtil.getSourceLocationAccurate(errorIndex); diff --git a/src/test/resources/unit/regex/plain_non_newline_escape.t b/src/test/resources/unit/regex/plain_non_newline_escape.t new file mode 100644 index 000000000..ce0a515f5 --- /dev/null +++ b/src/test/resources/unit/regex/plain_non_newline_escape.t @@ -0,0 +1,33 @@ +use strict; +use warnings; +use utf8; +use Test::More; + +like('a', qr/^\N$/, 'plain N matches an ordinary character'); +unlike("\n", qr/^\N$/, 'plain N excludes line feed'); +like("\r", qr/^\N$/, 'plain N includes carriage return'); +like("\x{2028}", qr/^\N$/, 'plain N includes Unicode line separator'); +unlike("\n", qr/^\N$/s, 'dotall does not broaden plain N'); + +like('ab', qr/^\N{2}$/, 'numeric braces quantify plain N'); +unlike('a', qr/^\N{2}$/, 'exact plain N quantifier enforces its lower bound'); +like('abc', qr/^\N{2,3}$/, 'bounded plain N quantifier accepts its upper bound'); +unlike('abcd', qr/^\N{2,3}$/, 'bounded plain N quantifier rejects longer input'); +like('abcd', qr/^\N{2,}$/, 'open plain N quantifier remains unbounded'); +like('ab', qr/^\N { 2 }$/x, 'extended mode permits spacing before the quantifier'); + +like('\\N', qr/^\\N$/, 'escaped backslash leaves literal N syntax inert'); +like('A', qr/^\N{LATIN CAPITAL LETTER A}$/, + 'named character syntax remains distinct from plain N'); + +for my $source ('qr/[\\N]/', 'qr/[\\N{2}]/') { + my @warnings; + local $SIG{__WARN__} = sub { push @warnings, @_ }; + my $value = eval $source; + ok(!defined $value, "$source is rejected inside a character class"); + like($@, qr/^\\N in a character class must be a named character: \\N\{\.\.\.\}/, + "$source reports Perl's class diagnostic"); + is(scalar @warnings, 0, "$source emits no warning before the fatal"); +} + +done_testing; diff --git a/third_party/joni/src/org/joni/Lexer.java b/third_party/joni/src/org/joni/Lexer.java index a28dd2b38..76b65e415 100644 --- a/third_party/joni/src/org/joni/Lexer.java +++ b/third_party/joni/src/org/joni/Lexer.java @@ -49,6 +49,7 @@ class Lexer extends ScannerSupport { private boolean perlHorizontalWhitespaceSingleByte; private int perlVerticalWhitespaceTokenIndex = -1; private boolean perlVerticalWhitespaceNegated; + private int perlNonNewlineTokenIndex = -1; private int perlCharacterPropertyEscape; protected Lexer(Regex regex, Syntax syntax, byte[]bytes, int p, int end, WarnCallback warnings) { @@ -715,6 +716,61 @@ private TokenType fetchPerlVerticalWhitespaceToken() { return token.type; } + private boolean usesPerlNonNewlineEscape() { + return syntax.op2OptionPerl() && "PERLONJAVA".equals(syntax.name); + } + + private boolean isPerlNonNewlineIntervalAhead() { + if (!left() || !peekIs('{')) return false; + + int cursor = nextChar(p, stop); + boolean sawLow = false; + boolean sawComma = false; + boolean sawHigh = false; + while (cursor < stop) { + int code = codeAt(cursor, stop); + if (code == '}') { + return sawComma ? sawLow || sawHigh : sawLow; + } + if (code >= '0' && code <= '9') { + if (sawComma) sawHigh = true; + else sawLow = true; + } else if (code == ',' && !sawComma) { + sawComma = true; + } else { + return false; + } + cursor = nextChar(cursor, stop); + } + return false; + } + + private void startPerlNonNewline(TokenType openType) { + perlNonNewlineTokenIndex = 0; + token.type = openType; + } + + private TokenType fetchPerlNonNewlineToken() { + token.base = 0; + token.escaped = false; + switch (perlNonNewlineTokenIndex++) { + case 0: + token.type = TokenType.CHAR; + token.setC('^'); + break; + case 1: + token.type = TokenType.CODE_POINT; + token.setCode('\n'); + break; + default: + token.type = TokenType.CC_CLOSE; + token.setC(']'); + perlNonNewlineTokenIndex = -1; + break; + } + return token.type; + } + private void fetchTokenInCCFor_p() { int c2 = peek(); // !!! migrate to peekIs if (c2 == '{' && syntax.op2EscPBraceCharProperty()) { @@ -789,6 +845,9 @@ private boolean fetchTokenFor_namedCharacter(boolean inCharacterClass) { if (resolver == null || !syntax.op2OptionPerl() || !left() || !peekIs('{')) { return false; } + if (usesPerlNonNewlineEscape() && isPerlNonNewlineIntervalAhead()) { + return false; + } inc(); int nameStart = p; @@ -1059,6 +1118,9 @@ private void fetchTokenInCCFor_and() { } protected final TokenType fetchTokenInCC() { + if (perlNonNewlineTokenIndex >= 0) { + return fetchPerlNonNewlineToken(); + } if (perlHorizontalWhitespaceTokenIndex >= 0) { return fetchPerlHorizontalWhitespaceToken(); } @@ -1144,11 +1206,15 @@ protected final TokenType fetchTokenInCC() { break; case 'N': if (!fetchTokenFor_namedCharacter(true)) { - unfetch(); - fetchEscapedValue(); - if (token.getC() != c) { - token.setCode(c); - token.type = TokenType.CODE_POINT; + if (usesPerlNonNewlineEscape()) { + newSyntaxException(PERL_NON_NEWLINE_IN_CHARACTER_CLASS); + } else { + unfetch(); + fetchEscapedValue(); + if (token.getC() != c) { + token.setCode(c); + token.type = TokenType.CODE_POINT; + } } } break; @@ -1736,17 +1802,21 @@ protected final void fetchToken() { break; case 'N': if (!fetchTokenFor_namedCharacter(false)) { - unfetch(); - fetchEscapedValue(); - if (token.getC() != c) { - token.type = TokenType.CODE_POINT; - token.setCode(c); + if (usesPerlNonNewlineEscape()) { + startPerlNonNewline(TokenType.CC_OPEN); } else { - int encLength = enc.length(bytes, token.backP, stop); - if (encLength == Encoding.CHAR_INVALID) { - throw new IllegalArgumentException("Invalid character found."); + unfetch(); + fetchEscapedValue(); + if (token.getC() != c) { + token.type = TokenType.CODE_POINT; + token.setCode(c); + } else { + int encLength = enc.length(bytes, token.backP, stop); + if (encLength == Encoding.CHAR_INVALID) { + throw new IllegalArgumentException("Invalid character found."); + } + p = token.backP + encLength; } - p = token.backP + encLength; } } break; diff --git a/third_party/joni/src/org/joni/exception/ErrorMessages.java b/third_party/joni/src/org/joni/exception/ErrorMessages.java index a2ce92ac4..25109cc58 100644 --- a/third_party/joni/src/org/joni/exception/ErrorMessages.java +++ b/third_party/joni/src/org/joni/exception/ErrorMessages.java @@ -105,6 +105,8 @@ public interface ErrorMessages extends org.jcodings.exception.ErrorMessages { String PERL_MISSING_RIGHT_BRACE_ON_NAMED_CHARACTER_ESCAPE = "Missing right brace on \\N{}"; String PERL_EMPTY_NAMED_CHARACTER_ESCAPE = "Empty \\N{}"; + String PERL_NON_NEWLINE_IN_CHARACTER_CLASS = + "\\N in a character class must be a named character: \\N{...}"; String PERL_MISSING_RIGHT_BRACE_ON_BOUNDARY = "Missing right brace on \\%n{}"; String PERL_EMPTY_BOUNDARY = "Empty \\%n{}"; diff --git a/third_party/joni/test/org/joni/test/TestPerlNonNewline.java b/third_party/joni/test/org/joni/test/TestPerlNonNewline.java new file mode 100644 index 000000000..f075a9cf2 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestPerlNonNewline.java @@ -0,0 +1,96 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import static org.joni.exception.ErrorMessages.PERL_NON_NEWLINE_IN_CHARACTER_CLASS; +import static org.joni.constants.SyntaxProperties.ALLOW_INTERVAL_LOW_ABBREV; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.nio.charset.StandardCharsets; + +import org.jcodings.specific.UTF8Encoding; +import org.joni.NamedCharacterResolver; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.joni.WarnCallback; +import org.joni.exception.JOniException; +import org.junit.Test; + +public class TestPerlNonNewline { + private static final NamedCharacterResolver RESOLVER = + (bytes, p, end, encoding) -> 'A'; + private static final Syntax PERLONJAVA = new Syntax( + "PERLONJAVA", Syntax.PerlNG.op, Syntax.PerlNG.op2, + Syntax.PerlNG.op3, + Syntax.PerlNG.behavior | ALLOW_INTERVAL_LOW_ABBREV, + Syntax.PerlNG.options, + Syntax.PerlNG.metaCharTable, RESOLVER); + + private static int search(String pattern, String input) { + byte[] patternBytes = pattern.getBytes(StandardCharsets.UTF_8); + byte[] inputBytes = input.getBytes(StandardCharsets.UTF_8); + Regex regex = new Regex(patternBytes, 0, patternBytes.length, Option.NONE, + UTF8Encoding.INSTANCE, PERLONJAVA, WarnCallback.NONE); + return regex.matcher(inputBytes).search(0, inputBytes.length, Option.NONE); + } + + private static void assertMatch(String pattern, String input) { + assertEquals(0, search("\\A(?:" + pattern + ")\\z", input)); + } + + private static void assertNoMatch(String pattern, String input) { + assertEquals(-1, search("\\A(?:" + pattern + ")\\z", input)); + } + + private static void assertSyntaxError(String pattern) { + try { + search(pattern, ""); + fail("expected syntax error for " + pattern); + } catch (JOniException error) { + assertEquals(PERL_NON_NEWLINE_IN_CHARACTER_CLASS, error.getMessage()); + } + } + + @Test + public void matchesEverythingExceptLineFeed() { + assertMatch("\\N", "a"); + assertMatch("\\N", "\r"); + assertMatch("\\N", "\u2028"); + assertNoMatch("\\N", "\n"); + assertNoMatch("(?s)\\N", "\n"); + } + + @Test + public void acceptsPerlIntervalsAfterTheAtom() { + assertMatch("\\N{2}", "ab"); + assertMatch("\\N{2,3}", "abc"); + assertMatch("\\N{2,}", "abcd"); + assertMatch("\\N{,2}", "a"); + assertNoMatch("\\N{2}", "a"); + } + + @Test + public void rejectsPlainNonNewlineInsideCharacterClasses() { + assertSyntaxError("[\\N]"); + assertSyntaxError("[\\N{2}]"); + } +} From aff2c868e319f8d46eea461df3549a322f5e80b9 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 16:56:35 +0200 Subject: [PATCH 7/8] test(regex): retire temporary Java route expectations Expect default and auto routing to select Joni for ordinary lookbehind and branch-reset subroutine calls now that their native implementations are in the combined Phase 36 integration branch. Generated with Codex (https://openai.com/codex) Co-Authored-By: OpenAI Codex --- .../runtime/regex/RegexBackendPolicyTest.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/test/java/org/perlonjava/runtime/regex/RegexBackendPolicyTest.java b/src/test/java/org/perlonjava/runtime/regex/RegexBackendPolicyTest.java index d9dde15a8..f85636638 100644 --- a/src/test/java/org/perlonjava/runtime/regex/RegexBackendPolicyTest.java +++ b/src/test/java/org/perlonjava/runtime/regex/RegexBackendPolicyTest.java @@ -29,22 +29,22 @@ void restoreBackendProperty() { } @Test - void defaultModeTemporarilyUsesJavaForOrdinaryLookbehind() { + void defaultModeRoutesOrdinaryLookbehindToJoni() { assertFalse(RegexBackendPolicy.useJoni("ordinary")); - assertFalse(RegexBackendPolicy.useJoni("(?<=x)y")); - assertFalse(RegexBackendPolicy.useJoni("(?1)|(?2))(?&digit)")); - assertFalse(RegexBackendPolicy.useJoni("(?|(1)|(2))(?1)")); + assertTrue(RegexBackendPolicy.useJoni("(?|(?1)|(?2))(?&digit)")); + assertTrue(RegexBackendPolicy.useJoni("(?|(1)|(2))(?1)")); assertTrue(RegexBackendPolicy.useJoni("(?1)(?&digit)")); assertTrue(RegexBackendPolicy.useJoni( "(?|(?1)|(?2))(?&digit)(?{=CALL:0})")); From 8d95ea9d2891f0e2ff3b88a50adf9394bd031c32 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 17:16:17 +0200 Subject: [PATCH 8/8] ci: trigger checks after retargeting PR The validated tree was force-pushed while the pull request still targeted its merged prerequisite branch. Add an empty synchronization commit after changing the base to master so the master-targeted pull_request workflow runs. Generated with Codex (https://openai.com/codex) Co-Authored-By: OpenAI Codex