From 82239ba6d93aee8abba48907045f04ccae931165 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 22:20:01 +0200 Subject: [PATCH 01/10] fix(regex): distinguish non-ASCII unfinished class ranges Map Perl non-ASCII unfinished ranges such as /[\xdf-/i to the native Invalid [] range diagnostic while preserving the ordinary unmatched class behavior for ASCII ranges. Add focused regression coverage. Generated with Codex (https://openai.com/codex) Co-Authored-By: Codex --- .../unit/regex/joni_extended_class_range_diagnostic.t | 9 +++++++++ third_party/joni/src/org/joni/Parser.java | 8 ++++++++ 2 files changed, 17 insertions(+) create mode 100644 src/test/resources/unit/regex/joni_extended_class_range_diagnostic.t diff --git a/src/test/resources/unit/regex/joni_extended_class_range_diagnostic.t b/src/test/resources/unit/regex/joni_extended_class_range_diagnostic.t new file mode 100644 index 000000000..36334cab2 --- /dev/null +++ b/src/test/resources/unit/regex/joni_extended_class_range_diagnostic.t @@ -0,0 +1,9 @@ +use strict; +use warnings; +use Test::More; + +my $invalid_range = eval q{ qr/[\xdf-/i; 1 } ? '' : $@; +like($invalid_range, qr/^Invalid \[\] range/, + 'unterminated character-class range uses the Perl diagnostic'); + +done_testing; diff --git a/third_party/joni/src/org/joni/Parser.java b/third_party/joni/src/org/joni/Parser.java index 5bc165739..73b529b1d 100644 --- a/third_party/joni/src/org/joni/Parser.java +++ b/third_party/joni/src/org/joni/Parser.java @@ -614,6 +614,14 @@ private ParsedCharClass parseCharClass(ObjPtr ascNode, break; case EOT: + // Perl distinguishes an unfinished range after a non-ASCII + // class value (for example /[\xdf-/i) from a bare unmatched + // opening bracket. Preserve the ordinary EOT diagnostic for + // ASCII ranges such as /[a-/. + if (arg.state == CCSTATE.RANGE && arg.to >= 0x80 + && env.usesPerlDiagnostics()) { + newSyntaxException(PERL_INVALID_RANGE_IN_CHAR_CLASS); + } newSyntaxException(PREMATURE_END_OF_CHAR_CLASS); default: From bd2cdd76929435ac106d9d9aaa6a2a5e4cf5b387 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 22:16:18 +0200 Subject: [PATCH 02/10] fix(regex): bound numeric backreference parsing Detect decimal overflow without wrapped integers, parse Perl numeric braced backreferences directly, reject nonexistent decimal references, and guard analyser capture-node lookups. Add focused Joni and Perl boundary coverage. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../native_numeric_backreference_boundaries.t | 32 +++++++ third_party/joni/src/org/joni/Analyser.java | 13 ++- third_party/joni/src/org/joni/Lexer.java | 48 ++++++++++- .../joni/src/org/joni/ScannerSupport.java | 16 ++-- ...estPerlNumericBackreferenceBoundaries.java | 85 +++++++++++++++++++ 5 files changed, 181 insertions(+), 13 deletions(-) create mode 100644 src/test/resources/unit/regex/native_numeric_backreference_boundaries.t create mode 100644 third_party/joni/test/org/joni/test/TestPerlNumericBackreferenceBoundaries.java diff --git a/src/test/resources/unit/regex/native_numeric_backreference_boundaries.t b/src/test/resources/unit/regex/native_numeric_backreference_boundaries.t new file mode 100644 index 000000000..22d6b44f2 --- /dev/null +++ b/src/test/resources/unit/regex/native_numeric_backreference_boundaries.t @@ -0,0 +1,32 @@ +use strict; +use warnings; +use Test::More; + +for my $pattern (q{\87}, q{a\87}, q{a\97}) { + my $compiled = eval "qr/$pattern/"; + like($@, qr/Reference to nonexistent group in regex/, + "$pattern is a nonexistent decimal backreference"); +} + +for my $digits (qw(2147483648 2147483649 2147483650 + 4294967296 4294967297 4294967298)) { + for my $prefix ('', 'a') { + for my $form ("\\g$digits}", "\\g{$digits}", "\\g{ $digits }") { + my $compiled = eval "qr/${prefix}(.)$form/"; + like($@, qr/Reference to nonexistent group in regex/, + "$form rejects overflow safely after ${prefix}capture"); + } + } + + my ($octal, $tail) = $digits =~ /^([0-7]{1,3})(.*)$/; + for my $prefix ('', 'a') { + my $pattern = "${prefix}(.)\\$digits"; + my $compiled = eval "qr/$pattern/"; + ok(defined($compiled), "$pattern compiles as octal plus literal tail"); + my $subject = $prefix . 'b' . chr(oct($octal)) . $tail; + ok($subject =~ $compiled && $1 eq 'b', + "$pattern matches its octal boundary without integer overflow"); + } +} + +done_testing; diff --git a/third_party/joni/src/org/joni/Analyser.java b/third_party/joni/src/org/joni/Analyser.java index d5ec4bfa5..19fa12d1a 100644 --- a/third_party/joni/src/org/joni/Analyser.java +++ b/third_party/joni/src/org/joni/Analyser.java @@ -449,14 +449,14 @@ private int getMinMatchLength(Node node) { BackRefNode br = (BackRefNode)node; if (br.isRecursion()) break; - if (br.back[0] > env.numMem) { + if (invalidBackrefNode(br.back[0])) { if (!syntax.op3OptionECMAScript()) newValueException(INVALID_BACKREF); } else { min = getMinMatchLength(env.memNodes[br.back[0]]); } for (int i=1; i env.numMem) { + if (invalidBackrefNode(br.back[i])) { if (!syntax.op3OptionECMAScript()) newValueException(INVALID_BACKREF); } else { int tmin = getMinMatchLength(env.memNodes[br.back[i]]); @@ -596,7 +596,7 @@ private int getMaxMatchLength(Node node) { } for (int i=0; i env.numMem) { + if (invalidBackrefNode(br.back[i])) { if(!syntax.op3OptionECMAScript()) newValueException(INVALID_BACKREF); } else { int tmax = getMaxMatchLength(env.memNodes[br.back[i]]); @@ -2500,7 +2500,7 @@ protected final Node setupTree(Node node, int state) { case NodeType.BREF: BackRefNode br = (BackRefNode)node; for (int i=0; i env.numMem) { + if (invalidBackrefNode(br.back[i])) { if (!syntax.op3OptionECMAScript()) newValueException(INVALID_BACKREF); } else { env.backrefedMem = bsOnAt(env.backrefedMem, br.back[i]); @@ -3066,4 +3066,9 @@ protected final void setOptimizedInfoFromTree(Node node) { Config.log.println(regex.optimizeInfoToString()); } } + + private boolean invalidBackrefNode(int number) { + return number <= 0 || number > env.numMem || env.memNodes == null + || number >= env.memNodes.length || env.memNodes[number] == null; + } } diff --git a/third_party/joni/src/org/joni/Lexer.java b/third_party/joni/src/org/joni/Lexer.java index 434470c3d..abe714c27 100644 --- a/third_party/joni/src/org/joni/Lexer.java +++ b/third_party/joni/src/org/joni/Lexer.java @@ -1356,6 +1356,7 @@ private void fetchTokenFor_digit() { } if (c == '8' || c == '9') { /* normal char */ // skip_backref: + if (syntax.op2OptionPerl()) newValueException(INVALID_BACKREF); p = last; inc(); return; @@ -1440,10 +1441,10 @@ private void fetchTokenFor_subexpCall() { return; } if (Config.USE_NAMED_GROUP) { - if (syntax.op2EscGBraceBackref() && left()) { + if ((syntax.op2EscGBraceBackref() || syntax.op2OptionPerl()) && left()) { fetch(); if (c == '{') { - fetchNamedBackrefToken(); + if (!fetchPerlNumericBackrefToken()) fetchNamedBackrefToken(); } else { unfetch(); } @@ -1497,6 +1498,46 @@ private void fetchTokenFor_subexpCall() { } } + private boolean fetchPerlNumericBackrefToken() { + if (!syntax.op2OptionPerl()) return false; + + int cursor = p; + while (cursor < stop && Character.isWhitespace(codeAt(cursor, stop))) { + cursor = nextChar(cursor, stop); + } + boolean negative = cursor < stop && codeAt(cursor, stop) == '-'; + if (negative) cursor = nextChar(cursor, stop); + int digitsStart = cursor; + long number = 0; + boolean overflow = false; + while (cursor < stop && enc.isDigit(codeAt(cursor, stop))) { + int digit = Encoding.digitVal(codeAt(cursor, stop)); + if (number > (Integer.MAX_VALUE - digit) / 10L) overflow = true; + else if (!overflow) number = number * 10 + digit; + cursor = nextChar(cursor, stop); + } + if (cursor == digitsStart) return false; + while (cursor < stop && Character.isWhitespace(codeAt(cursor, stop))) { + cursor = nextChar(cursor, stop); + } + if (cursor >= stop || codeAt(cursor, stop) != '}') return false; + p = nextChar(cursor, stop); + + if (overflow || number == 0) newValueException(INVALID_BACKREF); + long absolute = negative ? (long)env.numMem + 1 - number : number; + if (absolute <= 0 || absolute > env.numMem || env.memNodes == null + || absolute >= env.memNodes.length + || env.memNodes[(int)absolute] == null) { + newValueException(INVALID_BACKREF); + } + token.type = TokenType.BACKREF; + token.setBackrefByName(false); + token.setBackrefNum(1); + token.setBackrefRef1((int)absolute); + if (Config.USE_BACKREF_WITH_LEVEL) token.setBackrefExistLevel(false); + return true; + } + private void rejectMalformedPerlBackref() { if (!syntax.op2OptionPerl()) return; if (!left()) { @@ -1580,7 +1621,8 @@ protected void fetchNamedBackrefToken() { if (backNum <= 0) newValueException(INVALID_BACKREF); } - if (syntax.strictCheckBackref() && (backNum > env.numMem || env.memNodes == null)) { + if (syntax.strictCheckBackref() && (backNum > env.numMem || env.memNodes == null + || backNum >= env.memNodes.length || env.memNodes[backNum] == null)) { newValueException(INVALID_BACKREF); } token.type = TokenType.BACKREF; diff --git a/third_party/joni/src/org/joni/ScannerSupport.java b/third_party/joni/src/org/joni/ScannerSupport.java index 43d384957..caf446371 100644 --- a/third_party/joni/src/org/joni/ScannerSupport.java +++ b/third_party/joni/src/org/joni/ScannerSupport.java @@ -57,16 +57,18 @@ protected final int getPatternPosition() { return p - begin; } - private static final int INT_SIGN_BIT = 1 << 31; protected final int scanUnsignedNumber() { int last = c; int num = 0; // long ??? while(left()) { fetch(); if (enc.isDigit(c)) { - int onum = num; - num = num * 10 + Encoding.digitVal(c); - if (((onum ^ num) & INT_SIGN_BIT) != 0) return -1; + int digit = Encoding.digitVal(c); + if (num > (Integer.MAX_VALUE - digit) / 10) { + c = last; + return -1; + } + num = num * 10 + digit; } else { unfetch(); break; @@ -103,10 +105,12 @@ protected final int scanUnsignedOctalNumber(int maxLength) { while(left() && maxLength-- != 0) { fetch(); if (enc.isDigit(c) && c < '8') { - int onum = num; int val = Encoding.odigitVal(c); + if (num > (Integer.MAX_VALUE - val) / 8) { + c = last; + return -1; + } num = (num << 3) + val; - if (((onum ^ num) & INT_SIGN_BIT) != 0) return -1; } else { unfetch(); break; diff --git a/third_party/joni/test/org/joni/test/TestPerlNumericBackreferenceBoundaries.java b/third_party/joni/test/org/joni/test/TestPerlNumericBackreferenceBoundaries.java new file mode 100644 index 000000000..62b057018 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestPerlNumericBackreferenceBoundaries.java @@ -0,0 +1,85 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import static org.joni.exception.ErrorMessages.INVALID_BACKREF; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +import java.nio.charset.StandardCharsets; + +import org.jcodings.specific.ISO8859_1Encoding; +import org.jcodings.specific.UTF8Encoding; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.joni.WarnCallback; +import org.joni.exception.SyntaxException; +import org.junit.Test; + +public class TestPerlNumericBackreferenceBoundaries { + private static Regex compile(String pattern) { + byte[] bytes = pattern.getBytes(StandardCharsets.UTF_8); + return new Regex(bytes, 0, bytes.length, Option.CAPTURE_GROUP, + UTF8Encoding.INSTANCE, Syntax.PerlNG, WarnCallback.NONE); + } + + private static void assertInvalid(String pattern) { + SyntaxException error = assertThrows(SyntaxException.class, + () -> compile(pattern)); + assertEquals(INVALID_BACKREF, error.getMessage()); + } + + @Test + public void rejectsNonexistentAndOverflowingDecimalBackreferences() { + assertInvalid("\\87"); + assertInvalid("a\\87"); + assertInvalid("a\\97"); + for (String digits : new String[] {"2147483648", "2147483649", + "2147483650", "4294967296", "4294967297", "4294967298"}) { + assertInvalid("(.)\\g" + digits + "}"); + assertInvalid("(.)\\g{" + digits + "}"); + assertInvalid("(.)\\g{ " + digits + " }"); + assertInvalid("a(.)\\g" + digits + "}"); + assertInvalid("a(.)\\g{" + digits + "}"); + assertInvalid("a(.)\\g{ " + digits + " }"); + } + } + + @Test + public void longDecimalEscapesUseOnlyTheMaximalOctalPrefix() { + for (String digits : new String[] {"2147483648", "2147483649", + "2147483650", "4294967296", "4294967297", "4294967298"}) { + int octalLength = 0; + while (octalLength < 3 && octalLength < digits.length() + && digits.charAt(octalLength) <= '7') octalLength++; + int octal = Integer.parseInt(digits.substring(0, octalLength), 8); + String tail = digits.substring(octalLength); + String pattern = "(.)\\" + digits; + byte[] patternBytes = pattern.getBytes(StandardCharsets.ISO_8859_1); + Regex regex = new Regex(patternBytes, 0, patternBytes.length, + Option.CAPTURE_GROUP, ISO8859_1Encoding.INSTANCE, + Syntax.PerlNG, WarnCallback.NONE); + byte[] input = ("b" + (char)octal + tail) + .getBytes(StandardCharsets.ISO_8859_1); + assertEquals(0, regex.matcher(input).search(0, input.length, Option.NONE)); + } + } +} From 1152a188289d30167939929abaadcfc22f2deb16 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 22:36:44 +0200 Subject: [PATCH 03/10] refactor(regex): retire Joni backreference rewriting Pass Perl brace backreferences directly to the bounded Joni lexer instead of rewriting them as named backreferences, and retain Perl diagnostic wording for invalid numeric references. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../runtime/regex/JoniRegexPattern.java | 32 ------------------- .../runtime/regex/RuntimeRegex.java | 3 +- 2 files changed, 2 insertions(+), 33 deletions(-) diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index af6af11f5..9a60c4678 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -807,22 +807,6 @@ private static String translatePattern(String pattern, RegexFlags flags, continue; } } - // In Perl, \g{name} is a backreference. Ruby/Oniguruma uses - // \g for a subexpression call and \k for the - // backreference, so passing the brace form through makes Joni - // diagnose it as an invalid subexpression call. Numeric and - // relative brace forms follow the same translation. - if (!inClass && pattern.startsWith("\\g{", i)) { - int end = pattern.indexOf('}', i + 3); - if (end > i + 3) { - String backreference = pattern.substring(i + 3, end); - if (isValidPerlBraceBackreference(backreference)) { - out.append("\\k<").append(backreference).append('>'); - i = end; - continue; - } - } - } out.append(ch); escaped = true; continue; @@ -949,22 +933,6 @@ private static String translatePattern(String pattern, RegexFlags flags, return out.toString(); } - private static boolean isValidPerlBraceBackreference(String content) { - if (content.isEmpty()) return false; - int start = content.charAt(0) == '-' ? 1 : 0; - if (start == content.length()) return false; - if (start == 1 || Character.isDigit(content.charAt(0))) { - for (int i = start; i < content.length(); i++) { - if (!Character.isDigit(content.charAt(i))) return false; - } - return true; - } - for (int i = 0; i < content.length(); i++) { - if (Character.isWhitespace(content.charAt(i))) return false; - } - return true; - } - private static void appendResolvedNamedCharacter(StringBuilder out, int codePoint, RegexFlags flags) { boolean extendedSyntax = flags.isExtended() diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java index 38c281d9b..32faac8e3 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java @@ -824,7 +824,8 @@ private static synchronized RuntimeRegex compileSynchronized( } if ("invalid backref number/name".equals(e.getMessage()) || "invalid backref number".equals(e.getMessage())) { - throw new PerlCompilerException("Reference to nonexistent group"); + throw new PerlCompilerException( + "Reference to nonexistent group in regex"); } String invalidProperty = invalidUnicodePropertyName(e.getMessage()); if (invalidProperty != null) { From 64e7398dddcad9ae6f3f9ef994cc06967322634a Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 22:39:52 +0200 Subject: [PATCH 04/10] fix(regex): accept Perl false character class ranges Treat class endpoints around a hyphen as Perl's literal-hyphen union instead of a Joni range error, while retaining existing behavior for other syntaxes. Generated with Codex (https://openai.com/codex) Co-Authored-By: Codex --- .../unit/regex/joni_false_class_ranges.t | 20 +++++++++++++++++++ third_party/joni/src/org/joni/Parser.java | 10 ++++++++++ .../joni/src/org/joni/ast/CClassNode.java | 16 ++++++++++++++- 3 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 src/test/resources/unit/regex/joni_false_class_ranges.t diff --git a/src/test/resources/unit/regex/joni_false_class_ranges.t b/src/test/resources/unit/regex/joni_false_class_ranges.t new file mode 100644 index 000000000..3355ad416 --- /dev/null +++ b/src/test/resources/unit/regex/joni_false_class_ranges.t @@ -0,0 +1,20 @@ +use strict; +use warnings; +use Test::More; + +my @cases = ( + [ qr/([a-\d]+)/, 'za-9z', 'a-9', 'backslash d after a-dash' ], + [ qr/([\d-z]+)/, 'a0-za', '0-z', 'backslash d before dash-z' ], + [ qr/([\d-\s]+)/, 'a0- z', '0- ', 'backslash d through dash-space' ], + [ qr/([a-[:digit:]]+)/, 'za-9z', 'a-9', 'POSIX digit after a-dash' ], + [ qr/([[:digit:]-z]+)/, '=0-z=', '0-z', 'POSIX digit before dash-z' ], + [ qr/([[:digit:]-[:alpha:]]+)/, '=0-z=', '0-z', 'POSIX digit and alpha around dash' ], +); + +for my $case (@cases) { + my ($regex, $input, $want, $name) = @$case; + my ($got) = $input =~ $regex; + is($got, $want, $name); +} + +done_testing; diff --git a/third_party/joni/src/org/joni/Parser.java b/third_party/joni/src/org/joni/Parser.java index 73b529b1d..c035908c5 100644 --- a/third_party/joni/src/org/joni/Parser.java +++ b/third_party/joni/src/org/joni/Parser.java @@ -511,6 +511,16 @@ private ParsedCharClass parseCharClass(ObjPtr ascNode, case CC_RANGE: if (arg.state == CCSTATE.VALUE) { + if (arg.type == CCVALTYPE.CLASS && env.usesPerlDiagnostics()) { + // Perl accepts [\d-z] as a false range: retain the + // class and make the hyphen a pending literal value. + arg.state = CCSTATE.COMPLETE; + arg.to = '-'; + arg.toIsRaw = false; + arg.inType = CCVALTYPE.SB; + cc.nextStateValue(arg, ascCc, foldCc, env); + break; + } fetchTokenInCC(); fetched = true; if (token.type == TokenType.CC_CLOSE) { /* allow [x-] */ diff --git a/third_party/joni/src/org/joni/ast/CClassNode.java b/third_party/joni/src/org/joni/ast/CClassNode.java index 9cdc69ada..4db1028bf 100644 --- a/third_party/joni/src/org/joni/ast/CClassNode.java +++ b/third_party/joni/src/org/joni/ast/CClassNode.java @@ -637,7 +637,21 @@ public static final class CCStateArg { public void nextStateClass(CCStateArg arg, CClassNode ascCc, CClassNode foldCc, ScanEnvironment env) { - if (arg.state == CCSTATE.RANGE) throw new SyntaxException(ErrorMessages.CHAR_CLASS_VALUE_AT_END_OF_RANGE); + if (arg.state == CCSTATE.RANGE) { + if (!env.usesPerlDiagnostics()) { + throw new SyntaxException(ErrorMessages.CHAR_CLASS_VALUE_AT_END_OF_RANGE); + } + + // Perl accepts a character class as a false range endpoint, such + // as [a-\d]. The hyphen is literal and both operands remain + // members of the surrounding class. + arg.state = CCSTATE.VALUE; + nextStateValue(arg, ascCc, foldCc, env); + arg.to = '-'; + arg.toIsRaw = false; + arg.inType = CCVALTYPE.SB; + nextStateValue(arg, ascCc, foldCc, env); + } if (arg.state == CCSTATE.VALUE && arg.type != CCVALTYPE.CLASS) { if (arg.type == CCVALTYPE.SB) { From 72edb3c4898339302c2fe7c667e58530cf77d4d6 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 22:50:54 +0200 Subject: [PATCH 05/10] fix(regex): finish braced backreference tokenization Return after parsing a braced numeric or named backreference so the lexer does not continue into unrelated token handling. This incorporates the final delta from the validated POJ5 numeric-backreference delivery. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- third_party/joni/src/org/joni/Lexer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/third_party/joni/src/org/joni/Lexer.java b/third_party/joni/src/org/joni/Lexer.java index abe714c27..13da46611 100644 --- a/third_party/joni/src/org/joni/Lexer.java +++ b/third_party/joni/src/org/joni/Lexer.java @@ -1445,6 +1445,7 @@ private void fetchTokenFor_subexpCall() { fetch(); if (c == '{') { if (!fetchPerlNumericBackrefToken()) fetchNamedBackrefToken(); + return; } else { unfetch(); } From 1ad77505c4f361065b08b9acb7f061122f424ab5 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 22:51:20 +0200 Subject: [PATCH 06/10] docs(regex): refresh Phase 36 execution path Replace stale completed next steps with the current forward-only integration, semantic completion, scaffold removal, and final acceptance sequence. Record only the numeric-backreference and class-range slices supported by completed focused and differential gates. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- dev/design/phase36-regex-parity.md | 76 ++++++++++++++++++------------ 1 file changed, 45 insertions(+), 31 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index b515113cd..d2551dca1 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -145,6 +145,12 @@ affected corpus before taking another slice. when the complete match fails, remain visible when another alternative succeeds, and commit when destructive control verbs cut the path, including across a dynamic continuation. +- Native Joni parses bounded decimal, braced, relative, and overflow numeric + backreferences without Java-side rewriting. The focused four-backend fixture + passes 63/63, and the exact `regexp.t` numeric tranche has zero introductions. +- Native Joni distinguishes non-ASCII unfinished ranges and accepts Perl false + class ranges around `\d`, `\s`, and POSIX classes. The focused system-Perl, + direct-Joni, four-backend, and imported-row gates agree. ## Execution Phases @@ -256,38 +262,38 @@ behavior. ## Ordered Next Steps -1. Integrate the native `/xx`, adjacent full-fold partition, and `L_` alias - batch on the exact merged predecessor. Run one warning-free full build plus - zero-introduction `regexp.t` and fold/property gates, then publish a review - PR and require exact-head Ubuntu/Windows CI. -2. Complete recursive and runtime `(??{...})` as native nested Joni execution: - preserve captures, `$^R`, `pos`, modes, byte/Unicode provenance, callback - unwind, backtracking re-evaluation, and recursion safety. Route every embedded - closure to Joni and delete constant inlining, progressive errors, and the - dynamic Java adapter as their gates pass. -3. Complete byte/Unicode pattern provenance through runtime interpolation and - template composition, then finish `/d`/`u`/`a`/`aa` forward/reverse literal, - class, property, and backreference folding from generated data. Require - direct Joni plus ordinary/forced JVM/interpreter zero-introduction gates. -4. Finish the remaining lexical `use re 'strict'`, unescaped-brace, and non-hex - diagnostic families. Refresh complete - `reg_mesg.t`, `pat.t`, and `pat_advanced.t` maps after each combined batch. -5. Repeat the four-leg 80-file Java/Joni × JVM/interpreter matrix on the exact - successor artifact and compare every file with the PR 958 log. Resolve every - regression, zero-TAP record, timeout, truncation, or incomplete file before - user acceptance. -6. Remove each proven-obsolete regex transformation from `dev/import-perl5` - sync sources, regenerate a private unpatched corpus twice, prove byte-for-byte - idempotence, and run the affected upstream tests without editing them. -7. Use the refreshed impact report to move all remaining ordinary constants to - native Joni, deleting their Java routes and matcher-semantic preprocessing in - the same validated slices. Keep `pat_re_eval.t` at 555/555 throughout. -8. Delete Java matching, selector, fallback state, and unreachable preprocessors; - then run direct/thread regex, CPAN, performance, packaging, notice/license, - warning-free build, Ubuntu, Windows, and full CI gates. +1. Complete and integrate the current non-overlapping native-Joni batch: + ordinary repeated-capture clearing, physical branch-reset names/conditions, + named-character grammar/diagnostics, numeric backreferences, and class-range + semantics. Run one warning-free full `make`, four-backend focused fixtures, + and an exact zero-introduction `regexp.t` comparison on the combined SHA. +2. Close the remaining capture/region identities, including inactive named and + numeric branch-reset slots, final successful quantified iterations, failed + alternatives, recursive-frame publication, and nested match-state restore. +3. Move remaining Perl named-character, escape, strict-mode, brace, control- + character, and warning semantics into Joni lexer/compiler internals. Remove + each corresponding `JoniRegexPattern`/preprocessor rewrite in the same gated + slice; keep only source-policy and final diagnostic rendering outside Joni. +4. Finish whole-pattern recursion `(?R)`, recursive numbered/named calls, + recursion conditions, capture publication, and recursion safety. Keep + runtime `(??{...})`, callback unwind, and `pat_re_eval.t` 555/555 green. +5. Finish `/d`/`u`/`a`/`aa` forward/reverse literal and backreference folding + from generated data, then rerun complete Unicode, `pat.t`, `pat_advanced.t`, + `reg_mesg.t`, and bounded speed/psycho gates on one immutable artifact. +6. Use the refreshed impact ledger to close every remaining semantic regex + identity and move all ordinary constants to Joni. Reject zero-TAP, timeout, + truncated, incomplete, JVM/interpreter, or direct/thread mismatches. +7. Remove proven-obsolete `dev/import-perl5` regex patches, rerun targeted sync + twice, prove byte-for-byte idempotence, and validate the restored unchanged + upstream tests. +8. Delete Java matching, the backend selector, fallback state, matcher-semantic + preprocessors, and unreachable adapter code. Prove performance, CPAN, + packaging, notice/license, and warning-free build gates before removal is + accepted. 9. Update the feature matrix and final as-implemented/fork documents, remove or - summarize redundant design documents, rebase the final stack on `master`, and - run the complete PR 958 parity audit before declaring Phase 36 complete. + summarize redundant design documents, rebase each final PR on `master`, pass + Ubuntu/Windows CI, and compare the complete runner output file-by-file with + the immutable PR 958 baseline. ## Parallel Work @@ -375,6 +381,14 @@ gates may reopen it if a semantic regression appears. - [x] Native branch reset and removal of its capture-map adapter - [x] Native plain `\N` non-newline atom and interval forms - [x] Native recursive/runtime `(??{...})` and removal of dynamic adapters +- [x] Native bounded numeric backreference parsing and removal of Joni-side + brace-backreference rewriting +- [x] Native Perl false-class ranges and unfinished non-ASCII range diagnostics +- [ ] Final-iteration, optional, alternation, and failed-path capture clearing +- [ ] Physical branch-reset named calls, conditions, and inactive-slot publication +- [ ] Native named-character whitespace/missing-brace/comment diagnostics and + removal of the duplicate Java translation path +- [ ] Whole-pattern `(?R)` recursion and recursive capture publication - [ ] Retire proven-obsolete `dev/import-perl5` regex patches - [ ] Refresh the complete Unicode, `pat.t`, `pat_advanced.t`, `reg_mesg.t`, and 80-file forced-Joni gates on one integrated artifact From 8ccb9c0df286d1f6c6e21e9ca3c90a272bfc337a Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 22:34:24 +0200 Subject: [PATCH 07/10] fix(joni): bind unique branch-reset names to physical calls Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- third_party/joni/src/org/joni/Analyser.java | 4 ++++ .../joni/src/org/joni/ScanEnvironment.java | 22 +++++++++++++++++++ .../test/TestPerlBranchResetNamedCall.java | 14 ++++++++++++ 3 files changed, 40 insertions(+) diff --git a/third_party/joni/src/org/joni/Analyser.java b/third_party/joni/src/org/joni/Analyser.java index 19fa12d1a..e5fd9c3b6 100644 --- a/third_party/joni/src/org/joni/Analyser.java +++ b/third_party/joni/src/org/joni/Analyser.java @@ -1414,6 +1414,10 @@ protected final void setupSubExpCall(Node node) { newValueException(MULTIPLEX_DEFINITION_NAME_CALL, cn.nameP, cn.nameEnd); } else { cn.groupNum = ne.backRef1; // ne.backNum == 1 ? ne.backRef1 : ne.backRefs[0]; // ??? need to check ? + if (ne.backNum == 1) { + cn.lexicalTarget = env.physicalNamedMemNode( + ne.getPhysicalBackRefs()[0]); + } if (ne.backNum > 1) cn.setRecursion(); setCallAttr(cn); } diff --git a/third_party/joni/src/org/joni/ScanEnvironment.java b/third_party/joni/src/org/joni/ScanEnvironment.java index 886eb2cab..2f4c0b6fe 100644 --- a/third_party/joni/src/org/joni/ScanEnvironment.java +++ b/third_party/joni/src/org/joni/ScanEnvironment.java @@ -46,6 +46,7 @@ public final class ScanEnvironment { int numNamed; // USE_NAMED_GROUP public EncloseNode[] memNodes; + private EncloseNode[] physicalNamedMemNodes; // USE_COMBINATION_EXPLOSION_CHECK int numCombExpCheck; @@ -104,11 +105,32 @@ void setMemNode(int num, EncloseNode node) { } else if (memNodes[num] != node) { multiplexMemNodes[num] = true; } + if (node.physicalNamedCaptureId > 0) { + setPhysicalNamedMemNode(node.physicalNamedCaptureId, node); + } } else { throw new InternalException(ErrorMessages.PARSER_BUG); } } + private void setPhysicalNamedMemNode(int physicalId, EncloseNode node) { + if (physicalNamedMemNodes == null) { + physicalNamedMemNodes = new EncloseNode[Config.SCANENV_MEMNODES_SIZE]; + } else if (physicalId >= physicalNamedMemNodes.length) { + EncloseNode[] expanded = new EncloseNode[physicalNamedMemNodes.length << 1]; + System.arraycopy(physicalNamedMemNodes, 0, expanded, 0, + physicalNamedMemNodes.length); + physicalNamedMemNodes = expanded; + } + physicalNamedMemNodes[physicalId] = node; + } + + EncloseNode physicalNamedMemNode(int physicalId) { + return physicalNamedMemNodes == null || physicalId <= 0 + || physicalId >= physicalNamedMemNodes.length + ? null : physicalNamedMemNodes[physicalId]; + } + boolean isMultiplexMemNode(int num) { return multiplexMemNodes != null && multiplexMemNodes[num]; } diff --git a/third_party/joni/test/org/joni/test/TestPerlBranchResetNamedCall.java b/third_party/joni/test/org/joni/test/TestPerlBranchResetNamedCall.java index 102c490a8..a36a2a2f9 100644 --- a/third_party/joni/test/org/joni/test/TestPerlBranchResetNamedCall.java +++ b/third_party/joni/test/org/joni/test/TestPerlBranchResetNamedCall.java @@ -70,6 +70,20 @@ public void namedBranchResetCallTargetsLeftmostPhysicalGroup() { assertDoesNotMatch(pattern, "22"); } + @Test + public void distinctNamedBranchResetCallsUseTheirOwnPhysicalDefinitions() { + String pattern = "(?|(?a)|(?b))\\1\\g\\g"; + assertMatches(pattern, "bbab"); + } + + @Test + public void distinctNamedBranchResetConditionsUsePhysicalDefinitions() { + String pattern = "(?|(?a)|(?b))(?()x|y)\\1"; + assertMatches(pattern, "byb"); + assertDoesNotMatch(pattern, "bxb"); + assertMatches(pattern, "axa"); + } + @Test public void numericBranchResetCallTargetsLeftmostPhysicalGroup() { String pattern = "(?|(1)|(2))\\g<1>"; From 648d145880db88001a41d184bfe0aec2205add67 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 22:50:46 +0200 Subject: [PATCH 08/10] fix(joni): distinguish physical named branch conditions Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../regex/branch_reset_capture_semantics.t | 41 +++++++++++++++++++ .../joni/src/org/joni/ArrayCompiler.java | 1 + .../joni/src/org/joni/ByteCodeMachine.java | 10 +++++ third_party/joni/src/org/joni/Parser.java | 6 +++ .../joni/src/org/joni/ast/EncloseNode.java | 2 + 5 files changed, 60 insertions(+) create mode 100644 src/test/resources/unit/regex/branch_reset_capture_semantics.t diff --git a/src/test/resources/unit/regex/branch_reset_capture_semantics.t b/src/test/resources/unit/regex/branch_reset_capture_semantics.t new file mode 100644 index 000000000..beb44e096 --- /dev/null +++ b/src/test/resources/unit/regex/branch_reset_capture_semantics.t @@ -0,0 +1,41 @@ +use strict; +use warnings; +use Test::More; + +plan skip_all => 'GH 20653 semantics require Perl 5.44+' if $] < 5.044; + +ok('bbab' =~ /(?|(?a)|(?b))\1(?&a)(?&b)/, + 'branch-reset numbered and named subroutine targets share captures'); +ok('byb' =~ /(?|(?a)|(?b))(?()x|y)\1/, + 'named condition sees unset sibling capture'); +ok(!('bxb' =~ /(?|(?a)|(?b))(?()x|y)\1/), + 'named condition rejects wrong unset-capture branch'); +ok('axa' =~ /(?|(?a)|(?b))(?()x|y)\1/, + 'named condition sees set capture'); + +'a' =~ /(?|(?a)|(?b))/; +is("$1-$+{a}-" . (defined $+{b} ? $+{b} : ''), 'a-a-', + 'first branch publishes only its named capture'); +'b' =~ /(?|(?a)|(?b))/; +is("$1-" . (defined $+{a} ? $+{a} : '') . "-$+{b}", 'b--b', + 'second branch publishes only its named capture'); + +for my $case ( + ['preabcpost', 'a-b-c'], + ['predepost', 'd-e-'], + ['prefpost', 'f--'], +) { + my ($subject, $expected) = @$case; + $subject =~ /(?
pre)(?|(?a)(?b)(?c)|(?d)(?e)|(?f))(?post)/;
+    is("$2-" . (defined $3 ? $3 : '') . '-' . (defined $4 ? $4 : ''),
+        $expected, 'branch-reset physical slots preserve post-group numbering');
+}
+
+for my $letter (qw(a b c)) {
+    my $subject = $letter x 2;
+    ok($subject =~ /((?|(?a)(?-1)|(?b)(?-1)|(?c)(?-1)))/,
+        'relative subroutine target resolves inside each branch-reset alternative');
+    is($1, $subject, 'relative subroutine target consumes the matching pair');
+}
+
+done_testing;
diff --git a/third_party/joni/src/org/joni/ArrayCompiler.java b/third_party/joni/src/org/joni/ArrayCompiler.java
index 5c5c6f541..bfae66036 100644
--- a/third_party/joni/src/org/joni/ArrayCompiler.java
+++ b/third_party/joni/src/org/joni/ArrayCompiler.java
@@ -1123,6 +1123,7 @@ protected void compileEncloseNode(EncloseNode node) {
                             : OPCode.CONDITION);
                     addMemNum(node.calloutConditionId >= 0 ? node.calloutConditionId
                             : node.recursionConditionGroup >= 0 ? node.recursionConditionGroup
+                            : node.physicalNamedCondition > 0 ? -node.physicalNamedCondition
                             : node.regNum);
                     addRelAddr(len + OPSize.JUMP);
                 }
diff --git a/third_party/joni/src/org/joni/ByteCodeMachine.java b/third_party/joni/src/org/joni/ByteCodeMachine.java
index 257901d3e..b3a96b3f1 100644
--- a/third_party/joni/src/org/joni/ByteCodeMachine.java
+++ b/third_party/joni/src/org/joni/ByteCodeMachine.java
@@ -946,6 +946,16 @@ private void opExactNICSb() {
     private void opCondition() {
         int mem = code[ip++];
         int addr = code[ip++];
+        if (mem < 0) {
+            int physical = -mem;
+            if (physicalNamedCaptureBeg == null
+                    || physical >= physicalNamedCaptureBeg.length
+                    || physicalNamedCaptureBeg[physical] == INVALID_INDEX
+                    || physicalNamedCaptureEnd[physical] == INVALID_INDEX) {
+                ip += addr;
+            }
+            return;
+        }
         if (mem > regex.numMem || repeatStk[memEndStk + mem] == INVALID_INDEX || repeatStk[memStartStk + mem] == INVALID_INDEX) {
             ip += addr;
         }
diff --git a/third_party/joni/src/org/joni/Parser.java b/third_party/joni/src/org/joni/Parser.java
index c035908c5..28b7c2e84 100644
--- a/third_party/joni/src/org/joni/Parser.java
+++ b/third_party/joni/src/org/joni/Parser.java
@@ -914,6 +914,7 @@ && left() && enc.isDigit(peek())) {
                 if (left() && syntax.op2QMarkLParenCondition()) {
                     int num = -1;
                     int name = -1;
+                    int physicalNamedCondition = -1;
                     int calloutConditionId = -1;
                     AnchorNode assertionCondition = null;
                     int recursionConditionGroup = -1;
@@ -977,6 +978,10 @@ && left() && enc.isDigit(peek())) {
                                 fetchNamedBackrefToken();
                                 inc();
                                 num = token.getBackrefNum() > 1 ? token.getBackrefRefs()[0] : token.getBackrefRef1();
+                                NameEntry named = regex.nameToGroupNumbers(bytes, name, value);
+                                if (named != null && named.backNum == 1) {
+                                    physicalNamedCondition = named.getPhysicalBackRefs()[0];
+                                }
                             }
                         } else { // USE_NAMED_GROUP
                             newSyntaxException(INVALID_CONDITION_PATTERN);
@@ -984,6 +989,7 @@ && left() && enc.isDigit(peek())) {
                     }
                     EncloseNode en = new EncloseNode(EncloseType.CONDITION);
                     en.regNum = num;
+                    en.physicalNamedCondition = physicalNamedCondition;
                     en.calloutConditionId = calloutConditionId;
                     en.assertionCondition = assertionCondition;
                     en.recursionConditionGroup = recursionConditionGroup;
diff --git a/third_party/joni/src/org/joni/ast/EncloseNode.java b/third_party/joni/src/org/joni/ast/EncloseNode.java
index 3271d0ccd..8111803e4 100644
--- a/third_party/joni/src/org/joni/ast/EncloseNode.java
+++ b/third_party/joni/src/org/joni/ast/EncloseNode.java
@@ -35,6 +35,8 @@ public final class EncloseNode extends StateNode implements EncloseType {
     public int charLength;
     public int optCount;            // referenced count in optimize_node_left()
     public int calloutConditionId = -1;
+    /** Distinguishes unique named captures that reuse a branch-reset slot. */
+    public int physicalNamedCondition = -1;
     public AnchorNode assertionCondition;
     public int recursionConditionGroup = -1;
     public int recursionConditionNameP = -1;

From a2716be4572589980eca7897a322b6c93fa7a0df Mon Sep 17 00:00:00 2001
From: "Flavio S. Glock" 
Date: Wed, 19 Aug 2026 23:16:53 +0200
Subject: [PATCH 09/10] docs(regex): advance Phase 36 integration boundary

Record the validated 2074/2210 zero-introduction boundary and replace the
completed worker-batch step with the active successor lanes.

Generated with [Codex](https://openai.com/codex/)

Co-Authored-By: Codex 
---
 dev/design/phase36-regex-parity.md | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md
index d2551dca1..c38171b33 100644
--- a/dev/design/phase36-regex-parity.md
+++ b/dev/design/phase36-regex-parity.md
@@ -151,6 +151,11 @@ affected corpus before taking another slice.
 - Native Joni distinguishes non-ASCII unfinished ranges and accepts Perl false
   class ranges around `\d`, `\s`, and POSIX classes. The focused system-Perl,
   direct-Joni, four-backend, and imported-row gates agree.
+- Physical branch-reset names now bind named calls and named conditions to the
+  correct definition without changing numeric condition operands. The combined
+  numeric-backreference, class-range, and branch-reset boundary passes full
+  `make`; exact `regexp.t` is 2,074/2,210, fixing 57 identities from the prior
+  integrated boundary with no newly failing identity.
 
 ## Execution Phases
 
@@ -262,11 +267,12 @@ behavior.
 
 ## Ordered Next Steps
 
-1. Complete and integrate the current non-overlapping native-Joni batch:
-   ordinary repeated-capture clearing, physical branch-reset names/conditions,
-   named-character grammar/diagnostics, numeric backreferences, and class-range
-   semantics. Run one warning-free full `make`, four-backend focused fixtures,
-   and an exact zero-introduction `regexp.t` comparison on the combined SHA.
+1. Complete and integrate the active non-overlapping successor lanes: dynamic
+   source boundaries, ordinary repeated-capture clearing, native named-character
+   and escape diagnostics, and whole-pattern recursion. Preserve each lane as
+   reviewable semantic commits, then run one warning-free full `make`, four-leg
+   focused fixtures, and an exact zero-introduction `regexp.t` comparison on the
+   combined SHA.
 2. Close the remaining capture/region identities, including inactive named and
    numeric branch-reset slots, final successful quantified iterations, failed
    alternatives, recursive-frame publication, and nested match-state restore.
@@ -385,7 +391,8 @@ gates may reopen it if a semantic regression appears.
       brace-backreference rewriting
 - [x] Native Perl false-class ranges and unfinished non-ASCII range diagnostics
 - [ ] Final-iteration, optional, alternation, and failed-path capture clearing
-- [ ] Physical branch-reset named calls, conditions, and inactive-slot publication
+- [x] Physical branch-reset named calls and conditions
+- [ ] Inactive branch-reset slot publication
 - [ ] Native named-character whitespace/missing-brace/comment diagnostics and
       removal of the duplicate Java translation path
 - [ ] Whole-pattern `(?R)` recursion and recursive capture publication

From 0bd9d5b431c1a5219a72892c72ae0afb2da558bd Mon Sep 17 00:00:00 2001
From: "Flavio S. Glock" 
Date: Wed, 19 Aug 2026 23:19:36 +0200
Subject: [PATCH 10/10] docs(regex): keep Phase 36 tracker conservative

Leave the backtracking-visible state phase open until the active capture
lifetime and publication work is integrated and validated.

Generated with [Codex](https://openai.com/codex/)

Co-Authored-By: Codex 
---
 dev/design/phase36-regex-parity.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md
index c38171b33..8740efd58 100644
--- a/dev/design/phase36-regex-parity.md
+++ b/dev/design/phase36-regex-parity.md
@@ -353,7 +353,7 @@ failure blocks backend removal, not semantic fixes.
 
 - [x] Phase 0 — reproducible differential baseline
 - [ ] Phase 1 — ordinary-pattern Joni parity
-- [x] Phase 2 — conditions and backtracking-visible state
+- [ ] Phase 2 — conditions and backtracking-visible state
 - [ ] Phase 3 — Unicode and native pattern syntax
 - [ ] Phase 4 — runtime source and diagnostics
 - [ ] Phase 5 — remove migration scaffolding