From 37667aa764d777b6920191d54da8ca785f07b9bb Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Tue, 18 Aug 2026 16:02:24 +0200 Subject: [PATCH 01/11] fix(regex): support branch-reset named subroutine calls Preserve the leftmost physical capture target when branch-reset alternatives reuse a logical group number, admit duplicate-name calls only for the PerlOnJava syntax, and retain caller captures after multiplex subroutine calls. Add direct Joni and Perl regression coverage for named, numeric, duplicate-name, capture-publication, and ordinary-capture behavior. Generated with [Codex](https://openai.com/codex/) Co-Authored-By: Codex --- .../runtime/regex/JoniRegexPattern.java | 3 +- .../unit/regex/branch_reset_named_call.t | 18 ++++ third_party/joni/src/org/joni/Analyser.java | 8 +- .../joni/src/org/joni/ByteCodeMachine.java | 13 ++- .../joni/src/org/joni/ScanEnvironment.java | 23 ++++- third_party/joni/src/org/joni/Syntax.java | 4 + .../test/TestPerlBranchResetNamedCall.java | 99 +++++++++++++++++++ 7 files changed, 162 insertions(+), 6 deletions(-) create mode 100644 src/test/resources/unit/regex/branch_reset_named_call.t create mode 100644 third_party/joni/test/org/joni/test/TestPerlBranchResetNamedCall.java diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index a1fac2e2de..f09ad85ee6 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -13,6 +13,7 @@ import org.joni.Region; import org.joni.Syntax; +import static org.joni.constants.SyntaxProperties.ALLOW_MULTIPLEX_DEFINITION_NAME_CALL; import static org.joni.constants.SyntaxProperties.OP2_OPTION_PERL; import static org.joni.constants.SyntaxProperties.OP2_OPTION_RUBY; import static org.joni.constants.SyntaxProperties.OP2_PLUS_POSSESSIVE_INTERVAL; @@ -44,7 +45,7 @@ final class JoniRegexPattern { "PERLONJAVA", Syntax.RUBY.op, (Syntax.RUBY.op2 & ~OP2_OPTION_RUBY) | OP2_OPTION_PERL | OP2_PLUS_POSSESSIVE_INTERVAL, Syntax.RUBY.op3, - Syntax.RUBY.behavior, + Syntax.RUBY.behavior | ALLOW_MULTIPLEX_DEFINITION_NAME_CALL, Syntax.RUBY.options & ~(Option.ASCII_RANGE | Option.POSIX_BRACKET_ALL_RANGE | Option.WORD_BOUND_ALL_RANGE), Syntax.RUBY.metaCharTable); diff --git a/src/test/resources/unit/regex/branch_reset_named_call.t b/src/test/resources/unit/regex/branch_reset_named_call.t new file mode 100644 index 0000000000..7b7c5aaf19 --- /dev/null +++ b/src/test/resources/unit/regex/branch_reset_named_call.t @@ -0,0 +1,18 @@ +use strict; +use warnings; +use Test::More tests => 10; + +my $regex = eval { qr/(?|(?1)|(?2))(?&digit)/ }; +ok(defined $regex, 'duplicate named branch-reset call compiles'); +is($@, '', 'compilation reports no error'); + +ok('11' =~ $regex, 'first alternative can call the named group'); +is($1, '1', 'first alternative publishes its numbered capture'); +is($+{digit}, '1', 'first alternative publishes its named capture'); + +ok('21' =~ $regex, 'second alternative calls the leftmost named definition'); +is($1, '2', 'second alternative preserves its numbered capture'); +is($+{digit}, '2', 'second alternative preserves its named capture'); + +ok('12' !~ $regex, 'named call does not target the second definition'); +ok('22' !~ $regex, 'second alternative still calls the leftmost definition'); diff --git a/third_party/joni/src/org/joni/Analyser.java b/third_party/joni/src/org/joni/Analyser.java index c588d495d3..d21ff876e8 100644 --- a/third_party/joni/src/org/joni/Analyser.java +++ b/third_party/joni/src/org/joni/Analyser.java @@ -1308,6 +1308,9 @@ protected final int subexpRecursiveCheckTrav(Node node) { private void setCallAttr(CallNode cn) { EncloseNode en = env.memNodes[cn.groupNum]; if (en == null) newValueException(UNDEFINED_NAME_REFERENCE, cn.nameP, cn.nameEnd); + // Perl subroutine calls do not replace captures already visible in the + // caller. Reused branch-reset numbers need the existing snapshot path. + if (env.isMultiplexMemNode(cn.groupNum)) cn.setRecursion(); en.setCalled(); cn.setTarget(en); env.btMemStart = BitStatus.bsOnAt(env.btMemStart, cn.groupNum); @@ -1339,7 +1342,7 @@ protected final void setupSubExpCall(Node node) { en.recursionConditionNameP, en.recursionConditionNameEnd); } int[] refs = ne.getBackRefs(); - if (refs.length != 1) { + if (refs.length != 1 && !syntax.allowMultiplexDefinitionNameCall()) { newValueException(MULTIPLEX_DEFINITION_NAME_CALL, en.recursionConditionNameP, en.recursionConditionNameEnd); } @@ -1370,10 +1373,11 @@ protected final void setupSubExpCall(Node node) { if (ne == null) { newValueException(UNDEFINED_NAME_REFERENCE, cn.nameP, cn.nameEnd); - } else if (ne.backNum > 1) { + } else if (ne.backNum > 1 && !syntax.allowMultiplexDefinitionNameCall()) { newValueException(MULTIPLEX_DEFINITION_NAME_CALL, cn.nameP, cn.nameEnd); } else { cn.groupNum = ne.backRef1; // ne.backNum == 1 ? ne.backRef1 : ne.backRefs[0]; // ??? need to check ? + if (ne.backNum > 1) cn.setRecursion(); setCallAttr(cn); } } diff --git a/third_party/joni/src/org/joni/ByteCodeMachine.java b/third_party/joni/src/org/joni/ByteCodeMachine.java index 2c7ffde711..50fa4ceb7e 100644 --- a/third_party/joni/src/org/joni/ByteCodeMachine.java +++ b/third_party/joni/src/org/joni/ByteCodeMachine.java @@ -565,9 +565,20 @@ private boolean opEnd() { // USE_POSIX_REGION_OPTION ... else ... region.setBeg(0, msaBegin = ((pkeep > s) ? s : pkeep) - str); region.setEnd(0, msaEnd = s - str); + CompletedRecursiveCall completed = completedRecursiveCall(); + int[] callerCaptures = completed == null + ? null : completed.frame.getCallFrameCaptureSnapshot(); + int captureCount = regex.numMem + 1; for (int i = 1; i <= regex.numMem; i++) { + boolean preserveCallerCapture = callerCaptures != null + && i == completed.frame.getCallFrameNum() + && callerCaptures[i] != INVALID_INDEX + && callerCaptures[captureCount + i] != INVALID_INDEX; int me = repeatStk[memEndStk + i]; - if (me != INVALID_INDEX) { + if (preserveCallerCapture) { + region.setBeg(i, captureBegin(i)); + region.setEnd(i, captureEnd(i)); + } else if (me != INVALID_INDEX) { int ms = repeatStk[memStartStk + i]; region.setBeg(i, (bsAt(regex.btMemStart, i) ? stack[ms].getMemPStr() : ms) - str); region.setEnd(i, (bsAt(regex.btMemEnd, i) ? stack[me].getMemPStr() : me) - str); diff --git a/third_party/joni/src/org/joni/ScanEnvironment.java b/third_party/joni/src/org/joni/ScanEnvironment.java index 01090a03de..cc88fc0717 100644 --- a/third_party/joni/src/org/joni/ScanEnvironment.java +++ b/third_party/joni/src/org/joni/ScanEnvironment.java @@ -41,6 +41,7 @@ public final class ScanEnvironment { int numCall; UnsetAddrList unsetAddrList; // USE_SUBEXP_CALL public int numMem; + boolean[] multiplexMemNodes; int numNamed; // USE_NAMED_GROUP @@ -75,11 +76,19 @@ int caseFoldFlagFor(int option) { int addMemEntry() { if (numMem >= Config.MAX_CAPTURE_GROUP_NUM) throw new InternalException(ErrorMessages.TOO_MANY_CAPTURE_GROUPS); if (numMem++ == 0) { - memNodes = new EncloseNode[Config.SCANENV_MEMNODES_SIZE]; + // Branch-reset parsing can rewind numMem without starting a new + // parse. Keep nodes recorded by earlier alternatives. + if (memNodes == null) { + memNodes = new EncloseNode[Config.SCANENV_MEMNODES_SIZE]; + multiplexMemNodes = new boolean[Config.SCANENV_MEMNODES_SIZE]; + } } else if (numMem >= memNodes.length) { EncloseNode[]tmp = new EncloseNode[memNodes.length << 1]; System.arraycopy(memNodes, 0, tmp, 0, memNodes.length); memNodes = tmp; + boolean[] multiplexTmp = new boolean[multiplexMemNodes.length << 1]; + System.arraycopy(multiplexMemNodes, 0, multiplexTmp, 0, multiplexMemNodes.length); + multiplexMemNodes = multiplexTmp; } return numMem; @@ -87,12 +96,22 @@ int addMemEntry() { void setMemNode(int num, EncloseNode node) { if (numMem >= num) { - memNodes[num] = node; + // Branch-reset alternatives reuse capture numbers. Subexpression + // calls target the leftmost physical group with that number. + if (memNodes[num] == null) { + memNodes[num] = node; + } else if (memNodes[num] != node) { + multiplexMemNodes[num] = true; + } } else { throw new InternalException(ErrorMessages.PARSER_BUG); } } + boolean isMultiplexMemNode(int num) { + return multiplexMemNodes != null && multiplexMemNodes[num]; + } + void pushPrecReadNotNode(Node node) { numPrecReadNotNodes++; diff --git a/third_party/joni/src/org/joni/Syntax.java b/third_party/joni/src/org/joni/Syntax.java index c24db21a65..30b249bb2a 100644 --- a/third_party/joni/src/org/joni/Syntax.java +++ b/third_party/joni/src/org/joni/Syntax.java @@ -386,6 +386,10 @@ public boolean allowMultiplexDefinitionName() { return isBehavior(ALLOW_MULTIPLEX_DEFINITION_NAME); } + public boolean allowMultiplexDefinitionNameCall() { + return isBehavior(ALLOW_MULTIPLEX_DEFINITION_NAME_CALL); + } + public boolean fixedIntervalIsGreedyOnly() { return isBehavior(FIXED_INTERVAL_IS_GREEDY_ONLY); } diff --git a/third_party/joni/test/org/joni/test/TestPerlBranchResetNamedCall.java b/third_party/joni/test/org/joni/test/TestPerlBranchResetNamedCall.java new file mode 100644 index 0000000000..1e52dfc971 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestPerlBranchResetNamedCall.java @@ -0,0 +1,99 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import static org.joni.constants.SyntaxProperties.ALLOW_MULTIPLEX_DEFINITION_NAME_CALL; +import static org.junit.Assert.assertEquals; + +import java.nio.charset.StandardCharsets; + +import org.jcodings.specific.UTF8Encoding; +import org.joni.Matcher; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.junit.Test; + +public class TestPerlBranchResetNamedCall { + private static final Syntax PERL_SYNTAX = new Syntax( + "PERL_TEST", Syntax.RUBY.op, Syntax.RUBY.op2, Syntax.RUBY.op3, + Syntax.RUBY.behavior | ALLOW_MULTIPLEX_DEFINITION_NAME_CALL, + Syntax.RUBY.options, Syntax.RUBY.metaCharTable); + + private static Matcher matcher(String pattern, String input) { + byte[] patternBytes = pattern.getBytes(StandardCharsets.UTF_8); + byte[] inputBytes = input.getBytes(StandardCharsets.UTF_8); + Regex regex = new Regex(patternBytes, 0, patternBytes.length, + Option.CAPTURE_GROUP, UTF8Encoding.INSTANCE, PERL_SYNTAX); + return regex.matcher(inputBytes); + } + + private static Matcher assertMatches(String pattern, String input) { + Matcher matcher = matcher(pattern, input); + assertEquals(0, matcher.search(0, input.length(), Option.NONE)); + return matcher; + } + + private static void assertDoesNotMatch(String pattern, String input) { + assertEquals(-1, matcher(pattern, input).search(0, input.length(), Option.NONE)); + } + + @Test + public void namedBranchResetCallTargetsLeftmostPhysicalGroup() { + String pattern = "(?|(?1)|(?2))\\g"; + Matcher first = assertMatches(pattern, "11"); + assertEquals(0, first.getRegion().getBeg(1)); + assertEquals(1, first.getRegion().getEnd(1)); + + Matcher second = assertMatches(pattern, "21"); + assertEquals(0, second.getRegion().getBeg(1)); + assertEquals(1, second.getRegion().getEnd(1)); + + assertDoesNotMatch(pattern, "12"); + assertDoesNotMatch(pattern, "22"); + } + + @Test + public void numericBranchResetCallTargetsLeftmostPhysicalGroup() { + String pattern = "(?|(1)|(2))\\g<1>"; + assertMatches(pattern, "11"); + assertMatches(pattern, "21"); + assertDoesNotMatch(pattern, "12"); + assertDoesNotMatch(pattern, "22"); + } + + @Test + public void ordinaryMultiplexNameCallTargetsFirstDefinition() { + String pattern = "(?1)(?2)\\g"; + Matcher matcher = assertMatches(pattern, "121"); + assertEquals(0, matcher.getRegion().getBeg(1)); + assertEquals(1, matcher.getRegion().getEnd(1)); + assertDoesNotMatch(pattern, "122"); + } + + @Test + public void ordinaryCapturePublicationIsUnchanged() { + Matcher matcher = assertMatches("(a)(b)", "ab"); + assertEquals(0, matcher.getRegion().getBeg(1)); + assertEquals(1, matcher.getRegion().getEnd(1)); + assertEquals(1, matcher.getRegion().getBeg(2)); + assertEquals(2, matcher.getRegion().getEnd(2)); + } +} From 4efdfef75bc6b630f113641a7e84b4dca1700da2 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Tue, 18 Aug 2026 17:41:26 +0200 Subject: [PATCH 02/11] fix(regex): preserve physical branch-reset named captures Track each named-group definition independently inside the forked Joni bytecode matcher while retaining Perl's shared logical branch-reset numbering. Publish duplicate-name arrays from committed physical spans, restore them on backtracking, and keep subpattern calls from publishing capture slots. Generated with Codex (https://openai.com/codex) Co-Authored-By: Codex --- .../runtime/regex/JoniRegexPattern.java | 35 +++++++++++++--- .../branch_reset_physical_named_capture.t | 21 ++++++++++ .../joni/src/org/joni/ArrayCompiler.java | 26 ++++++++++++ .../joni/src/org/joni/ByteCodeMachine.java | 40 +++++++++++++++++++ third_party/joni/src/org/joni/Matcher.java | 9 +++++ third_party/joni/src/org/joni/NameEntry.java | 25 +++++++++++- third_party/joni/src/org/joni/Parser.java | 2 +- third_party/joni/src/org/joni/Regex.java | 11 ++++- third_party/joni/src/org/joni/StackEntry.java | 18 +++++++++ .../joni/src/org/joni/StackMachine.java | 37 +++++++++++++++++ .../joni/src/org/joni/ast/EncloseNode.java | 1 + .../org/joni/constants/internal/OPCode.java | 6 +++ .../org/joni/constants/internal/OPSize.java | 2 + .../joni/constants/internal/StackType.java | 1 + .../test/TestPerlBranchResetNamedCall.java | 16 ++++++++ 15 files changed, 240 insertions(+), 10 deletions(-) create mode 100644 src/test/resources/unit/regex/branch_reset_physical_named_capture.t diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index f09ad85ee6..e7403ccf17 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -53,6 +53,7 @@ final class JoniRegexPattern { private final Regex regex; private final String sourcePattern; private final Map namedGroups; + private final Map physicalNamedGroups; private final RegexFlags flags; private final boolean hasControlVerbState; private final boolean hasDeferredUserDefinedUnicodeProperty; @@ -85,7 +86,9 @@ final class JoniRegexPattern { regex = new Regex(bytes, 0, bytes.length, toJoniOptions(flags, forceAsciiClasses), byteMode ? ISO8859_1Encoding.INSTANCE : UTF8Encoding.INSTANCE, PERLONJAVA_SYNTAX); - namedGroups = collectNamedGroups(regex); + NamedGroupMaps groupMaps = collectNamedGroups(regex); + namedGroups = groupMaps.logical(); + physicalNamedGroups = groupMaps.physical(); } RegexMatcher matcher(String input, List callbacks) { @@ -94,7 +97,7 @@ RegexMatcher matcher(String input, List callbacks) { RegexMatcher matcher(String input, List callbacks, RuntimeScalar subject) { - return new JoniRegexMatcher(regex, sourcePattern, namedGroups, flags, + return new JoniRegexMatcher(regex, sourcePattern, namedGroups, physicalNamedGroups, flags, hasControlVerbState, byteMode, input, callbacks, subject); } @@ -793,27 +796,34 @@ private static int findGroupEnd(String pattern, int start) { return -1; } - private static Map collectNamedGroups(Regex regex) { + private record NamedGroupMaps(Map logical, + Map physical) {} + + private static NamedGroupMaps collectNamedGroups(Regex regex) { Map names = new LinkedHashMap<>(); + Map physicalNames = new LinkedHashMap<>(); Iterator iterator = regex.namedBackrefIterator(); while (iterator.hasNext()) { NameEntry entry = iterator.next(); String name = new String(entry.name, entry.nameP, entry.nameEnd - entry.nameP, StandardCharsets.UTF_8); int[] refs = entry.getBackRefs(); + int[] physicalRefs = entry.getPhysicalBackRefs(); for (int i = 0; i < refs.length; i++) { String key = i == 0 ? name : name + CaptureNameEncoder.DUPLICATE_MARKER + (i - 1); names.put(key, refs[i]); + physicalNames.put(key, physicalRefs[i]); } } - return names; + return new NamedGroupMaps(names, physicalNames); } private static final class JoniRegexMatcher implements RegexMatcher { private final Regex regex; private final String sourcePattern; private final Map namedGroups; + private final Map physicalNamedGroups; private final RegexFlags flags; private final String input; private final byte[] bytes; @@ -834,12 +844,14 @@ private static final class JoniRegexMatcher implements RegexMatcher { private PerlCalloutHandler calloutHandler; JoniRegexMatcher(Regex regex, String sourcePattern, Map namedGroups, + Map physicalNamedGroups, RegexFlags flags, boolean hasControlVerbState, boolean byteMode, String input, List callbacks, RuntimeScalar subject) { this.regex = regex; this.sourcePattern = sourcePattern; this.namedGroups = namedGroups; + this.physicalNamedGroups = physicalNamedGroups; this.flags = flags; this.hasControlVerbState = hasControlVerbState; this.byteMode = byteMode; @@ -953,8 +965,13 @@ public String group(int index) { @Override public String group(String name) { - int group = namedGroupNumber(name); - return group(group); + requireMatch(); + Integer physical = physicalNamedGroups.get(name); + if (physical == null) return group(namedGroupNumber(name)); + int begin = matcher.physicalNamedCaptureBegin(physical); + int end = matcher.physicalNamedCaptureEnd(physical); + if (begin < 0 || end < 0) return null; + return input.substring(toCharOffset(begin), toCharOffset(end)); } @Override public int groupCount() { return regex.numberOfCaptures(); } @@ -979,6 +996,12 @@ private static int deriveCommittedLastClosedCapture(Region region) { private int groupOffset(String name, boolean begin) { requireMatch(); + Integer physical = physicalNamedGroups.get(name); + if (physical != null) { + int offset = begin ? matcher.physicalNamedCaptureBegin(physical) + : matcher.physicalNamedCaptureEnd(physical); + return offset < 0 ? -1 : toCharOffset(offset); + } int group = namedGroupNumber(name); return groupOffset(group, begin); } diff --git a/src/test/resources/unit/regex/branch_reset_physical_named_capture.t b/src/test/resources/unit/regex/branch_reset_physical_named_capture.t new file mode 100644 index 0000000000..38ef2738c8 --- /dev/null +++ b/src/test/resources/unit/regex/branch_reset_physical_named_capture.t @@ -0,0 +1,21 @@ +use strict; +use warnings; +use Test::More tests => 3; + +for my $case ( + ['1', '1 ', 'first branch publishes first physical named slot'], + ['2', ' 2', 'second branch publishes second physical named slot'], +) { + my ($subject, $expected, $name) = @$case; + my $published = ''; + if ($subject =~ /(?|(?1)|(?2))/) { + $published = join ' ', map { defined $_ ? $_ : '' } @{$-{digit}}; + } + is($published, $expected, $name); +} + +my $called = ''; +if ('11' =~ /(?|(?1)|(?2))(?&digit)/) { + $called = join ' ', map { defined $_ ? $_ : '' } @{$-{digit}}; +} +is($called, '1 ', 'named call retains leftmost physical capture publication'); diff --git a/third_party/joni/src/org/joni/ArrayCompiler.java b/third_party/joni/src/org/joni/ArrayCompiler.java index ef595fd1a6..93b8a0511c 100644 --- a/third_party/joni/src/org/joni/ArrayCompiler.java +++ b/third_party/joni/src/org/joni/ArrayCompiler.java @@ -918,6 +918,10 @@ private int compileLengthEncloseNode(EncloseNode node) { } len += tlen + (bsAt(regex.btMemEnd, node.regNum) ? OPSize.MEMORY_END_PUSH : OPSize.MEMORY_END); } + if (node.physicalNamedCaptureId >= 0) { + len += OPSize.PHYSICAL_NAMED_CAPTURE_START + + OPSize.PHYSICAL_NAMED_CAPTURE_END; + } break; case EncloseType.STOP_BACKTRACK: @@ -977,6 +981,10 @@ protected void compileEncloseNode(EncloseNode node) { addMemNum(-1); len = compileLengthTree(node.target); len += OPSize.MEMORY_START_PUSH + OPSize.RETURN; + if (node.physicalNamedCaptureId >= 0) { + len += OPSize.PHYSICAL_NAMED_CAPTURE_START + + OPSize.PHYSICAL_NAMED_CAPTURE_END; + } if (bsAt(regex.btMemEnd, node.regNum)) { len += node.isRecursion() ? OPSize.MEMORY_END_PUSH_REC : OPSize.MEMORY_END_PUSH; } else { @@ -985,6 +993,12 @@ protected void compileEncloseNode(EncloseNode node) { addOpcodeRelAddr(OPCode.JUMP, len); } // USE_SUBEXP_CALL + if (node.physicalNamedCaptureId >= 0) { + regex.requireStack = true; + addOpcode(OPCode.PHYSICAL_NAMED_CAPTURE_START); + addMemNum(node.physicalNamedCaptureId); + } + if (bsAt(regex.btMemStart, node.regNum)) { regex.requireStack = true; addOpcode(OPCode.MEMORY_START_PUSH); @@ -1002,6 +1016,10 @@ protected void compileEncloseNode(EncloseNode node) { addOpcode(node.isRecursion() ? OPCode.MEMORY_END_REC : OPCode.MEMORY_END); } addMemNum(node.regNum); + if (node.physicalNamedCaptureId >= 0) { + addOpcode(OPCode.PHYSICAL_NAMED_CAPTURE_END); + addMemNum(node.physicalNamedCaptureId); + } addOpcode(OPCode.RETURN); } else if (Config.USE_SUBEXP_CALL && node.isRecursion()) { // USE_SUBEXP_CALL if (bsAt(regex.btMemEnd, node.regNum)) { @@ -1010,6 +1028,10 @@ protected void compileEncloseNode(EncloseNode node) { addOpcode(OPCode.MEMORY_END_REC); } addMemNum(node.regNum); + if (node.physicalNamedCaptureId >= 0) { + addOpcode(OPCode.PHYSICAL_NAMED_CAPTURE_END); + addMemNum(node.physicalNamedCaptureId); + } } else { if (bsAt(regex.btMemEnd, node.regNum)) { addOpcode(OPCode.MEMORY_END_PUSH); @@ -1017,6 +1039,10 @@ protected void compileEncloseNode(EncloseNode node) { addOpcode(OPCode.MEMORY_END); } addMemNum(node.regNum); + if (node.physicalNamedCaptureId >= 0) { + addOpcode(OPCode.PHYSICAL_NAMED_CAPTURE_END); + addMemNum(node.physicalNamedCaptureId); + } } break; diff --git a/third_party/joni/src/org/joni/ByteCodeMachine.java b/third_party/joni/src/org/joni/ByteCodeMachine.java index 50fa4ceb7e..65fd958029 100644 --- a/third_party/joni/src/org/joni/ByteCodeMachine.java +++ b/third_party/joni/src/org/joni/ByteCodeMachine.java @@ -258,6 +258,8 @@ private final int execute(final boolean checkThreadInterrupt) throws Interrupted case OPCode.NOT_WORD_BREAK_BOUNDARY: opWordBreakBoundary(true); continue; case OPCode.LINE_BOUNDARY: opLineBoundary(false); continue; case OPCode.NOT_LINE_BOUNDARY: opLineBoundary(true); continue; + case OPCode.PHYSICAL_NAMED_CAPTURE_START: opPhysicalNamedCaptureStart(); continue; + case OPCode.PHYSICAL_NAMED_CAPTURE_END: opPhysicalNamedCaptureEnd(); continue; case OPCode.WORD_BEGIN: opWordBegin(); continue; case OPCode.WORD_END: opWordEnd(); continue; @@ -423,6 +425,8 @@ private final int executeSb(final boolean checkThreadInterrupt) throws Interrupt case OPCode.NOT_WORD_BREAK_BOUNDARY: opWordBreakBoundary(true); continue; case OPCode.LINE_BOUNDARY: opLineBoundary(false); continue; case OPCode.NOT_LINE_BOUNDARY: opLineBoundary(true); continue; + case OPCode.PHYSICAL_NAMED_CAPTURE_START: opPhysicalNamedCaptureStart(); continue; + case OPCode.PHYSICAL_NAMED_CAPTURE_END: opPhysicalNamedCaptureEnd(); continue; case OPCode.WORD_BEGIN: opWordBeginSb(); continue; case OPCode.WORD_END: opWordEndSb(); continue; @@ -560,6 +564,12 @@ private boolean opEnd() { } // USE_FIND_LONGEST_SEARCH_ALL_OF_RANGE bestLen = n; + if (physicalNamedCaptureBeg != null) { + System.arraycopy(physicalNamedCaptureBeg, 0, + committedPhysicalNamedCaptureBeg, 0, physicalNamedCaptureBeg.length); + System.arraycopy(physicalNamedCaptureEnd, 0, + committedPhysicalNamedCaptureEnd, 0, physicalNamedCaptureEnd.length); + } final Region region = msaRegion; if (region != null) { // USE_POSIX_REGION_OPTION ... else ... @@ -2140,6 +2150,16 @@ private void opMemoryStartPush() { pushMemStart(mem, s); } + private void opPhysicalNamedCaptureStart() { + int capture = code[ip++]; + if (!isInsideSubexpCall(0)) pushPhysicalNamedCapture(capture, s); + } + + private void opPhysicalNamedCaptureEnd() { + int capture = code[ip++]; + if (!isInsideSubexpCall(0)) physicalNamedCaptureEnd[capture] = s; + } + private void opMemoryStart() { int mem = code[ip++]; repeatStk[memStartStk + mem] = s; @@ -2956,6 +2976,26 @@ public int captureEnd(int capture) { return (bsAt(regex.btMemEnd, capture) ? stack[value].getMemPStr() : value) - str; } + @Override + public int physicalNamedCaptureBegin(int capture) { + if (committedPhysicalNamedCaptureBeg == null + || capture <= 0 || capture >= committedPhysicalNamedCaptureBeg.length) { + return Region.REGION_NOTPOS; + } + int begin = committedPhysicalNamedCaptureBeg[capture]; + return begin == INVALID_INDEX ? Region.REGION_NOTPOS : begin - str; + } + + @Override + public int physicalNamedCaptureEnd(int capture) { + if (committedPhysicalNamedCaptureEnd == null + || capture <= 0 || capture >= committedPhysicalNamedCaptureEnd.length) { + return Region.REGION_NOTPOS; + } + int end = committedPhysicalNamedCaptureEnd[capture]; + return end == INVALID_INDEX ? Region.REGION_NOTPOS : end - str; + } + @Override public int lastClosedCapture() { CompletedRecursiveCall completed = completedRecursiveCall(); diff --git a/third_party/joni/src/org/joni/Matcher.java b/third_party/joni/src/org/joni/Matcher.java index a42e8473c7..d3678119d6 100644 --- a/third_party/joni/src/org/joni/Matcher.java +++ b/third_party/joni/src/org/joni/Matcher.java @@ -130,6 +130,15 @@ public int captureEnd(int capture) { return capture < region.getNumRegs() ? region.getEnd(capture) : Region.REGION_NOTPOS; } + /** Physical named-group definition offsets, used when definitions share a number. */ + public int physicalNamedCaptureBegin(int capture) { + return Region.REGION_NOTPOS; + } + + public int physicalNamedCaptureEnd(int capture) { + return Region.REGION_NOTPOS; + } + /** Most recently closed active capture; engines without this view return -1. */ public int lastClosedCapture() { return -1; diff --git a/third_party/joni/src/org/joni/NameEntry.java b/third_party/joni/src/org/joni/NameEntry.java index 2b62513056..373d89c043 100644 --- a/third_party/joni/src/org/joni/NameEntry.java +++ b/third_party/joni/src/org/joni/NameEntry.java @@ -29,6 +29,8 @@ public final class NameEntry { int backNum; int backRef1; int[] backRefs; + int physicalRef1; + int[] physicalRefs; public NameEntry(byte[]bytes, int p, int end) { name = bytes; @@ -49,8 +51,22 @@ public int[] getBackRefs() { } } + public int[] getPhysicalBackRefs() { + switch (backNum) { + case 0: + return new int[]{}; + case 1: + return new int[]{physicalRef1}; + default: + int[] result = new int[backNum]; + System.arraycopy(physicalRefs, 0, result, 0, backNum); + return result; + } + } + private void alloc() { backRefs = new int[INIT_NAME_BACKREFS_ALLOC_NUM]; + physicalRefs = new int[INIT_NAME_BACKREFS_ALLOC_NUM]; } private void ensureSize() { @@ -58,24 +74,31 @@ private void ensureSize() { int[]tmp = new int[backRefs.length << 1]; System.arraycopy(backRefs, 0, tmp, 0, backRefs.length); backRefs = tmp; + int[] physicalTmp = new int[physicalRefs.length << 1]; + System.arraycopy(physicalRefs, 0, physicalTmp, 0, physicalRefs.length); + physicalRefs = physicalTmp; } } - public void addBackref(int backRef) { + public void addBackref(int backRef, int physicalRef) { backNum++; switch (backNum) { case 1: backRef1 = backRef; + physicalRef1 = physicalRef; break; case 2: alloc(); backRefs[0] = backRef1; backRefs[1] = backRef; + physicalRefs[0] = physicalRef1; + physicalRefs[1] = physicalRef; break; default: ensureSize(); backRefs[backNum - 1] = backRef; + physicalRefs[backNum - 1] = physicalRef; } } diff --git a/third_party/joni/src/org/joni/Parser.java b/third_party/joni/src/org/joni/Parser.java index 247be8edce..83f12dcc84 100644 --- a/third_party/joni/src/org/joni/Parser.java +++ b/third_party/joni/src/org/joni/Parser.java @@ -1069,8 +1069,8 @@ private Node parseEncloseNamedGroup2(boolean listCapture) { int num = env.addMemEntry(); if (listCapture && num >= BitStatus.BIT_STATUS_BITS_NUM) newValueException(GROUP_NUMBER_OVER_FOR_CAPTURE_HISTORY); - regex.nameAdd(bytes, nm, nameEnd, num, syntax); EncloseNode en = EncloseNode.newMemory(env.option, true); + en.physicalNamedCaptureId = regex.nameAdd(bytes, nm, nameEnd, num, syntax); en.regNum = num; if (listCapture) env.captureHistory = bsOnAtSimple(env.captureHistory, num); diff --git a/third_party/joni/src/org/joni/Regex.java b/third_party/joni/src/org/joni/Regex.java index 9de66c6364..7aaf9b7859 100644 --- a/third_party/joni/src/org/joni/Regex.java +++ b/third_party/joni/src/org/joni/Regex.java @@ -46,6 +46,7 @@ public final class Regex { boolean hasDynamicOptions; int numMem; /* used memory(...) num counted from 1 */ + int numPhysicalNamedCaptures; int numRepeat; /* OP_REPEAT/OP_REPEAT_NG id-counter */ int numNullCheck; /* OP_NULL_CHECK_START/END id counter */ int numCombExpCheck; /* combination explosion check */ @@ -239,7 +240,7 @@ void renumberNameTable(int[]map) { } } - void nameAdd(byte[]name, int nameP, int nameEnd, int backRef, Syntax syntax) { + int nameAdd(byte[]name, int nameP, int nameEnd, int backRef, Syntax syntax) { if (nameEnd - nameP <= 0) throw new ValueException(ErrorMessages.EMPTY_GROUP_NAME); NameEntry e = null; @@ -257,7 +258,13 @@ void nameAdd(byte[]name, int nameP, int nameEnd, int backRef, Syntax syntax) { throw new ValueException(ErrorMessages.MULTIPLEX_DEFINED_NAME, new String(name, nameP, nameEnd - nameP)); } - e.addBackref(backRef); + int physicalRef = ++numPhysicalNamedCaptures; + e.addBackref(backRef, physicalRef); + return physicalRef; + } + + public int numberOfPhysicalNamedCaptures() { + return numPhysicalNamedCaptures; } NameEntry nameToGroupNumbers(byte[]name, int nameP, int nameEnd) { diff --git a/third_party/joni/src/org/joni/StackEntry.java b/third_party/joni/src/org/joni/StackEntry.java index 4258bc1f50..a9600ee438 100644 --- a/third_party/joni/src/org/joni/StackEntry.java +++ b/third_party/joni/src/org/joni/StackEntry.java @@ -128,6 +128,24 @@ int getMemEnd() { return E4; } + void setPhysicalNamedCapture(int capture, int begin, int end) { + E1 = capture; + E2 = begin; + E3 = end; + } + + int getPhysicalNamedCapture() { + return E1; + } + + int getPhysicalNamedCaptureBegin() { + return E2; + } + + int getPhysicalNamedCaptureEnd() { + return E3; + } + // fifth union member /* null check id */ void setNullCheckNum(int num) { diff --git a/third_party/joni/src/org/joni/StackMachine.java b/third_party/joni/src/org/joni/StackMachine.java index c0e13eb50f..61fc610fd2 100644 --- a/third_party/joni/src/org/joni/StackMachine.java +++ b/third_party/joni/src/org/joni/StackMachine.java @@ -34,6 +34,10 @@ abstract class StackMachine extends Matcher implements StackType { protected StackEntry[]stack; protected int stk; // stkEnd protected final int[]repeatStk; + protected final int[] physicalNamedCaptureBeg; + protected final int[] physicalNamedCaptureEnd; + protected final int[] committedPhysicalNamedCaptureBeg; + protected final int[] committedPhysicalNamedCaptureEnd; protected final int memStartStk, memEndStk; protected byte[] stateCheckBuff; // CEC, move to int[] ? protected int stateCheckBuffSize; @@ -54,6 +58,11 @@ protected StackMachine(Regex regex, Region region, byte[]bytes, int p , int end) /* for index start from 1, mem_end_stk[1]..mem_end_stk[num_mem] */ } repeatStk = n > 0 ? new int[n] : null; + int physicalCount = regex.numPhysicalNamedCaptures; + physicalNamedCaptureBeg = physicalCount == 0 ? null : new int[physicalCount + 1]; + physicalNamedCaptureEnd = physicalCount == 0 ? null : new int[physicalCount + 1]; + committedPhysicalNamedCaptureBeg = physicalCount == 0 ? null : new int[physicalCount + 1]; + committedPhysicalNamedCaptureEnd = physicalCount == 0 ? null : new int[physicalCount + 1]; } protected final void stackInit() { @@ -63,6 +72,28 @@ protected final void stackInit() { repeatStk[i + memStartStk] = repeatStk[i + memEndStk] = INVALID_INDEX; } } + if (physicalNamedCaptureBeg != null) { + Arrays.fill(physicalNamedCaptureBeg, INVALID_INDEX); + Arrays.fill(physicalNamedCaptureEnd, INVALID_INDEX); + Arrays.fill(committedPhysicalNamedCaptureBeg, INVALID_INDEX); + Arrays.fill(committedPhysicalNamedCaptureEnd, INVALID_INDEX); + } + } + + protected final void pushPhysicalNamedCapture(int capture, int position) { + StackEntry e = ensure1(); + e.type = PHYSICAL_NAMED_CAPTURE; + e.setPhysicalNamedCapture(capture, + physicalNamedCaptureBeg[capture], physicalNamedCaptureEnd[capture]); + physicalNamedCaptureBeg[capture] = position; + physicalNamedCaptureEnd[capture] = INVALID_INDEX; + stk++; + } + + private void restorePhysicalNamedCapture(StackEntry e) { + int capture = e.getPhysicalNamedCapture(); + physicalNamedCaptureBeg[capture] = e.getPhysicalNamedCaptureBegin(); + physicalNamedCaptureEnd[capture] = e.getPhysicalNamedCaptureEnd(); } private static StackEntry[] allocateStack() { @@ -408,6 +439,8 @@ private StackEntry popFree() { unwindCallout(e); } else if (e.type == CONTROL_MARK) { restoreControlMark(e.getPreviousControlMarkName()); + } else if (e.type == PHYSICAL_NAMED_CAPTURE) { + restorePhysicalNamedCapture(e); } else if (USE_CEC) { if (e.type == STATE_CHECK_MARK) stateCheckMark(); } @@ -423,6 +456,8 @@ private StackEntry popMemStart() { unwindCallout(e); } else if (e.type == CONTROL_MARK) { restoreControlMark(e.getPreviousControlMarkName()); + } else if (e.type == PHYSICAL_NAMED_CAPTURE) { + restorePhysicalNamedCapture(e); } else if (e.type == MEM_START) { repeatStk[memStartStk + e.getMemNum()] = e.getMemStart(); repeatStk[memEndStk + e.getMemNum()] = e.getMemEnd(); @@ -437,6 +472,8 @@ private void popRewrite(StackEntry e) { unwindCallout(e); } else if (e.type == CONTROL_MARK) { restoreControlMark(e.getPreviousControlMarkName()); + } else if (e.type == PHYSICAL_NAMED_CAPTURE) { + restorePhysicalNamedCapture(e); } else if (e.type == MEM_START) { repeatStk[memStartStk + e.getMemNum()] = e.getMemStart(); repeatStk[memEndStk + e.getMemNum()] = e.getMemEnd(); diff --git a/third_party/joni/src/org/joni/ast/EncloseNode.java b/third_party/joni/src/org/joni/ast/EncloseNode.java index 13ad0c5bc0..4a42a5b5de 100644 --- a/third_party/joni/src/org/joni/ast/EncloseNode.java +++ b/third_party/joni/src/org/joni/ast/EncloseNode.java @@ -26,6 +26,7 @@ public final class EncloseNode extends StateNode implements EncloseType { public final int type; // enclose type public int regNum; + public int physicalNamedCaptureId = -1; public int option; public Node target; /* EncloseNode : ENCLOSE_MEMORY */ public int callAddr; // AbsAddrType diff --git a/third_party/joni/src/org/joni/constants/internal/OPCode.java b/third_party/joni/src/org/joni/constants/internal/OPCode.java index f9a85e21dd..389ad5f9f4 100644 --- a/third_party/joni/src/org/joni/constants/internal/OPCode.java +++ b/third_party/joni/src/org/joni/constants/internal/OPCode.java @@ -162,6 +162,8 @@ public interface OPCode { int NOT_WORD_BREAK_BOUNDARY = 118; /* Perl \B{wb} */ int LINE_BOUNDARY = 119; /* Perl \b{lb} */ int NOT_LINE_BOUNDARY = 120; /* Perl \B{lb} */ + int PHYSICAL_NAMED_CAPTURE_START = 121; + int PHYSICAL_NAMED_CAPTURE_END = 122; String[] OpCodeNames = Config.DEBUG_COMPILE ? new String[] { "finish", /*OP_FINISH*/ @@ -286,6 +288,8 @@ public interface OPCode { "not-word-break-boundary", /*OP_NOT_WORD_BREAK_BOUNDARY*/ "line-boundary", /*OP_LINE_BOUNDARY*/ "not-line-boundary", /*OP_NOT_LINE_BOUNDARY*/ + "physical-named-capture-start", + "physical-named-capture-end", } : null; int[] OpCodeArgTypes = Config.DEBUG_COMPILE ? new int[] { @@ -411,5 +415,7 @@ public interface OPCode { Arguments.NON, /*OP_NOT_WORD_BREAK_BOUNDARY*/ Arguments.NON, /*OP_LINE_BOUNDARY*/ Arguments.NON, /*OP_NOT_LINE_BOUNDARY*/ + Arguments.MEMNUM, /*OP_PHYSICAL_NAMED_CAPTURE_START*/ + Arguments.MEMNUM, /*OP_PHYSICAL_NAMED_CAPTURE_END*/ } : null; } diff --git a/third_party/joni/src/org/joni/constants/internal/OPSize.java b/third_party/joni/src/org/joni/constants/internal/OPSize.java index 265f3cc662..241519343d 100644 --- a/third_party/joni/src/org/joni/constants/internal/OPSize.java +++ b/third_party/joni/src/org/joni/constants/internal/OPSize.java @@ -84,6 +84,8 @@ public interface OPSize { int RECURSION_CONDITION = (OPCODE + MEMNUM + RELADDR); int CHECK_POS_END = OPCODE; int CHECK_LOOK_BEHIND_END = OPCODE; + int PHYSICAL_NAMED_CAPTURE_START = (OPCODE + MEMNUM); + int PHYSICAL_NAMED_CAPTURE_END = (OPCODE + MEMNUM); // #ifdef USE_COMBINATION_EXPLOSION_CHECK int STATE_CHECK = (OPCODE + STATE_CHECK_NUM); diff --git a/third_party/joni/src/org/joni/constants/internal/StackType.java b/third_party/joni/src/org/joni/constants/internal/StackType.java index cc5ac9b65a..e13b640fc1 100644 --- a/third_party/joni/src/org/joni/constants/internal/StackType.java +++ b/third_party/joni/src/org/joni/constants/internal/StackType.java @@ -47,6 +47,7 @@ public interface StackType { int ABSENT = 0x0c00; /* absent inner loop marker */ int CALLOUT = 0x0d00; /* match-time callback unwind token */ int CONTROL_MARK = 0x0e00; /* Perl (*MARK:name) backtrack state */ + int PHYSICAL_NAMED_CAPTURE = 0x0f00; /* physical named-group span */ int DYNAMIC_ALT = 0x0004; /* resumable nested-program alternative */ /* stack type check mask */ diff --git a/third_party/joni/test/org/joni/test/TestPerlBranchResetNamedCall.java b/third_party/joni/test/org/joni/test/TestPerlBranchResetNamedCall.java index 1e52dfc971..102c490a87 100644 --- a/third_party/joni/test/org/joni/test/TestPerlBranchResetNamedCall.java +++ b/third_party/joni/test/org/joni/test/TestPerlBranchResetNamedCall.java @@ -96,4 +96,20 @@ public void ordinaryCapturePublicationIsUnchanged() { assertEquals(1, matcher.getRegion().getBeg(2)); assertEquals(2, matcher.getRegion().getEnd(2)); } + + @Test + public void duplicateBranchResetNamesRetainPhysicalDefinitionSpans() { + String pattern = "(?|(?1)|(?2))"; + Matcher first = assertMatches(pattern, "1"); + assertEquals(0, first.physicalNamedCaptureBegin(1)); + assertEquals(1, first.physicalNamedCaptureEnd(1)); + assertEquals(-1, first.physicalNamedCaptureBegin(2)); + assertEquals(-1, first.physicalNamedCaptureEnd(2)); + + Matcher second = assertMatches(pattern, "2"); + assertEquals(-1, second.physicalNamedCaptureBegin(1)); + assertEquals(-1, second.physicalNamedCaptureEnd(1)); + assertEquals(0, second.physicalNamedCaptureBegin(2)); + assertEquals(1, second.physicalNamedCaptureEnd(2)); + } } From c22cf1f51ec95579f65c9d3bc803142498c81de0 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Tue, 18 Aug 2026 17:40:07 +0200 Subject: [PATCH 03/11] feat(regex): parse named characters natively in Joni Add an optional syntax-level named-character resolver and let the Perl lexer emit code-point tokens for braced named escapes inside and outside classes. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- third_party/joni/src/org/joni/Lexer.java | 54 +++++++ .../src/org/joni/NamedCharacterResolver.java | 27 ++++ third_party/joni/src/org/joni/Syntax.java | 8 + .../src/org/joni/exception/ErrorMessages.java | 3 + .../org/joni/test/TestPerlNamedCharacter.java | 146 ++++++++++++++++++ 5 files changed, 238 insertions(+) create mode 100644 third_party/joni/src/org/joni/NamedCharacterResolver.java create mode 100644 third_party/joni/test/org/joni/test/TestPerlNamedCharacter.java diff --git a/third_party/joni/src/org/joni/Lexer.java b/third_party/joni/src/org/joni/Lexer.java index 99f180b79f..5fa871a67b 100644 --- a/third_party/joni/src/org/joni/Lexer.java +++ b/third_party/joni/src/org/joni/Lexer.java @@ -683,6 +683,34 @@ private void fetchTokenFor_oBrace() { token.setCode(value); } + private boolean fetchTokenFor_namedCharacter() { + NamedCharacterResolver resolver = syntax.namedCharacterResolver; + if (resolver == null || !syntax.op2OptionPerl() || !left() || !peekIs('{')) { + return false; + } + + inc(); + int nameStart = p; + while (left()) { + int nameEnd = p; + fetch(); + if (c != '}') continue; + if (nameStart == nameEnd) { + newSyntaxException(PERL_EMPTY_NAMED_CHARACTER_ESCAPE); + } + int codePoint = resolver.resolve(bytes, nameStart, nameEnd, enc); + if (codePoint < 0 || codePoint > 0x10ffff) { + newValueException(ERR_INVALID_CODE_POINT_VALUE); + } + token.type = TokenType.CODE_POINT; + token.setCode(codePoint); + return true; + } + + newSyntaxException(PERL_MISSING_RIGHT_BRACE_ON_NAMED_CHARACTER_ESCAPE); + return true; // not reached + } + private void scanOriginalBracedHexCodePoint(int last) { inc(); int num = scanUnsignedHexadecimalNumber(0, 8); @@ -950,6 +978,16 @@ protected final TokenType fetchTokenInCC() { case 'u': fetchTokenInCCFor_u(); break; + case 'N': + if (!fetchTokenFor_namedCharacter()) { + unfetch(); + fetchEscapedValue(); + if (token.getC() != c) { + token.setCode(c); + token.type = TokenType.CODE_POINT; + } + } + break; case '0': case '1': case '2': @@ -1412,6 +1450,22 @@ protected final void fetchToken() { case 'u': fetchTokenFor_uHex(); break; + case 'N': + if (!fetchTokenFor_namedCharacter()) { + unfetch(); + fetchEscapedValue(); + if (token.getC() != c) { + token.type = TokenType.CODE_POINT; + token.setCode(c); + } else { + int encLength = enc.length(bytes, token.backP, stop); + if (encLength == Encoding.CHAR_INVALID) { + throw new IllegalArgumentException("Invalid character found."); + } + p = token.backP + encLength; + } + } + break; case '1': case '2': case '3': diff --git a/third_party/joni/src/org/joni/NamedCharacterResolver.java b/third_party/joni/src/org/joni/NamedCharacterResolver.java new file mode 100644 index 0000000000..e815ab840a --- /dev/null +++ b/third_party/joni/src/org/joni/NamedCharacterResolver.java @@ -0,0 +1,27 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import org.jcodings.Encoding; + +@FunctionalInterface +public interface NamedCharacterResolver { + int resolve(byte[] bytes, int p, int end, Encoding encoding); +} diff --git a/third_party/joni/src/org/joni/Syntax.java b/third_party/joni/src/org/joni/Syntax.java index 30b249bb2a..55507751bc 100644 --- a/third_party/joni/src/org/joni/Syntax.java +++ b/third_party/joni/src/org/joni/Syntax.java @@ -31,8 +31,15 @@ public final class Syntax implements SyntaxProperties { public final int behavior; public final int options; public final MetaCharTable metaCharTable; + public final NamedCharacterResolver namedCharacterResolver; public Syntax(String name, int op, int op2, int op3, int behavior, int options, MetaCharTable metaCharTable) { + this(name, op, op2, op3, behavior, options, metaCharTable, null); + } + + public Syntax(String name, int op, int op2, int op3, int behavior, int options, + MetaCharTable metaCharTable, + NamedCharacterResolver namedCharacterResolver) { this.name = name; this.op = op; this.op2 = op2; @@ -40,6 +47,7 @@ public Syntax(String name, int op, int op2, int op3, int behavior, int options, this.behavior = behavior; this.options = options; this.metaCharTable = metaCharTable; + this.namedCharacterResolver = namedCharacterResolver; } public static class MetaCharTable { diff --git a/third_party/joni/src/org/joni/exception/ErrorMessages.java b/third_party/joni/src/org/joni/exception/ErrorMessages.java index e7b4323cd4..4f913ede4a 100644 --- a/third_party/joni/src/org/joni/exception/ErrorMessages.java +++ b/third_party/joni/src/org/joni/exception/ErrorMessages.java @@ -91,6 +91,9 @@ public interface ErrorMessages extends org.jcodings.exception.ErrorMessages { String PERL_EMPTY_OCTAL_ESCAPE = "Empty \\o{}"; String PERL_MISSING_RIGHT_BRACE_ON_HEX_ESCAPE = "Missing right brace on \\x{}"; + String PERL_MISSING_RIGHT_BRACE_ON_NAMED_CHARACTER_ESCAPE = + "Missing right brace on \\N{}"; + String PERL_EMPTY_NAMED_CHARACTER_ESCAPE = "Empty \\N{}"; /* values error (syntax error) */ String TOO_BIG_NUMBER = "too big number"; diff --git a/third_party/joni/test/org/joni/test/TestPerlNamedCharacter.java b/third_party/joni/test/org/joni/test/TestPerlNamedCharacter.java new file mode 100644 index 0000000000..affbed2b9a --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestPerlNamedCharacter.java @@ -0,0 +1,146 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import static org.joni.exception.ErrorMessages.PERL_EMPTY_NAMED_CHARACTER_ESCAPE; +import static org.joni.exception.ErrorMessages.PERL_MISSING_RIGHT_BRACE_ON_NAMED_CHARACTER_ESCAPE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.fail; + +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; + +import org.jcodings.Encoding; +import org.jcodings.specific.ISO8859_1Encoding; +import org.jcodings.specific.UTF8Encoding; +import org.joni.NamedCharacterResolver; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.joni.WarnCallback; +import org.joni.exception.JOniException; +import org.junit.Test; + +public class TestPerlNamedCharacter { + private static Syntax syntax(NamedCharacterResolver resolver) { + return new Syntax("PerlNamedCharacter", Syntax.PerlNG.op, Syntax.PerlNG.op2, + Syntax.PerlNG.op3, Syntax.PerlNG.behavior, Syntax.PerlNG.options, + Syntax.PerlNG.metaCharTable, resolver); + } + + private static String decode(byte[] bytes, int p, int end, Encoding encoding) { + Charset charset = encoding == ISO8859_1Encoding.INSTANCE + ? StandardCharsets.ISO_8859_1 : StandardCharsets.UTF_8; + return new String(bytes, p, end - p, charset); + } + + private static final NamedCharacterResolver RESOLVER = (bytes, p, end, encoding) -> + switch (decode(bytes, p, end, encoding)) { + case "CAPITAL" -> 'A'; + case "LOWER" -> 'a'; + case "HASH" -> '#'; + case "SPACE" -> ' '; + case "BYTE" -> 0xe9; + case "SUPPLEMENTARY" -> 0x1f642; + default -> throw new IllegalArgumentException("unknown fake name"); + }; + + private static Regex compile(String pattern, Encoding encoding, + NamedCharacterResolver resolver) { + Charset charset = encoding == ISO8859_1Encoding.INSTANCE + ? StandardCharsets.ISO_8859_1 : StandardCharsets.UTF_8; + byte[] bytes = pattern.getBytes(charset); + return new Regex(bytes, 0, bytes.length, Option.NONE, encoding, + syntax(resolver), WarnCallback.NONE); + } + + private static int search(String pattern, String input) { + byte[] bytes = input.getBytes(StandardCharsets.UTF_8); + return compile(pattern, UTF8Encoding.INSTANCE, RESOLVER) + .matcher(bytes).search(0, bytes.length, Option.NONE); + } + + private static void assertSyntaxError(String pattern, String expected) { + try { + compile(pattern, UTF8Encoding.INSTANCE, RESOLVER); + fail("expected syntax error for " + pattern); + } catch (JOniException error) { + assertEquals(expected, error.getMessage()); + } + } + + @Test + public void resolvesNamesInsideAndOutsideCharacterClasses() { + assertEquals(0, search("\\N{CAPITAL}", "A")); + assertEquals(0, search("[\\N{CAPITAL}]", "A")); + assertEquals(-1, search("\\N{CAPITAL}", "B")); + assertEquals(-1, search("[\\N{CAPITAL}]", "B")); + } + + @Test + public void preservesOptionsAndSupplementaryCodePoints() { + assertEquals(0, search("(?i)\\N{LOWER}", "A")); + assertEquals(0, search("(?x)\\N{HASH}", "#")); + assertEquals(0, search("(?x)[\\N{SPACE}]", " ")); + assertEquals(0, search("\\N{SUPPLEMENTARY}", "\ud83d\ude42")); + assertEquals(0, search("[\\N{SUPPLEMENTARY}]", "\ud83d\ude42")); + } + + @Test + public void reportsMalformedEscapesBeforeCallingResolver() { + assertSyntaxError("\\N{}", PERL_EMPTY_NAMED_CHARACTER_ESCAPE); + assertSyntaxError("\\N{CAPITAL", PERL_MISSING_RIGHT_BRACE_ON_NAMED_CHARACTER_ESCAPE); + } + + @Test + public void preservesResolverExceptions() { + IllegalArgumentException expected = new IllegalArgumentException("resolver failure"); + try { + compile("\\N{FAIL}", UTF8Encoding.INSTANCE, + (bytes, p, end, encoding) -> { throw expected; }); + fail("expected resolver exception"); + } catch (IllegalArgumentException error) { + assertSame(expected, error); + } + } + + @Test + public void suppliesTheActiveSingleByteEncoding() { + NamedCharacterResolver resolver = (bytes, p, end, encoding) -> { + assertSame(ISO8859_1Encoding.INSTANCE, encoding); + assertEquals("BYTE", decode(bytes, p, end, encoding)); + return 0xe9; + }; + byte[] input = {(byte)0xe9}; + Regex regex = compile("\\N{BYTE}", ISO8859_1Encoding.INSTANCE, resolver); + assertEquals(0, regex.matcher(input).search(0, input.length, Option.NONE)); + } + + @Test + public void leavesUnbracedEscapesAndSyntaxesWithoutAResolverUnchanged() { + assertEquals(0, search("\\N", "N")); + byte[] pattern = "\\N".getBytes(StandardCharsets.UTF_8); + byte[] input = "N".getBytes(StandardCharsets.UTF_8); + Regex regex = new Regex(pattern, 0, pattern.length, Option.NONE, + UTF8Encoding.INSTANCE, Syntax.PerlNG, WarnCallback.NONE); + assertEquals(0, regex.matcher(input).search(0, input.length, Option.NONE)); + } +} From 3f576e4602caaae0c0c415e078521643c9d082de Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Tue, 18 Aug 2026 17:40:30 +0200 Subject: [PATCH 04/11] refactor(regex): route named characters through Joni Inject the Perl named-character resolver through the Joni syntax and retire the frontend textual rewrite while preserving Unicode and byte-mode behavior. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../runtime/regex/JoniRegexPattern.java | 21 ++++++++-- .../runtime/regex/RuntimeRegex.java | 3 ++ .../unit/regex_joni_native_named_character.t | 39 +++++++++++++++++++ 3 files changed, 60 insertions(+), 3 deletions(-) create mode 100644 src/test/resources/unit/regex_joni_native_named_character.t diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index e7403ccf17..6fb220b9df 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -1,5 +1,6 @@ package org.perlonjava.runtime.regex; +import org.jcodings.Encoding; import org.jcodings.specific.ISO8859_1Encoding; import org.jcodings.specific.UTF8Encoding; import org.joni.Matcher; @@ -48,7 +49,15 @@ final class JoniRegexPattern { Syntax.RUBY.behavior | ALLOW_MULTIPLEX_DEFINITION_NAME_CALL, Syntax.RUBY.options & ~(Option.ASCII_RANGE | Option.POSIX_BRACKET_ALL_RANGE | Option.WORD_BOUND_ALL_RANGE), - Syntax.RUBY.metaCharTable); + Syntax.RUBY.metaCharTable, + JoniRegexPattern::resolveNamedCharacter); + + private static int resolveNamedCharacter(byte[] bytes, int p, int end, + Encoding encoding) { + return UnicodeResolver.getCodePointFromName(new String(bytes, p, end - p, + encoding == ISO8859_1Encoding.INSTANCE + ? StandardCharsets.ISO_8859_1 : StandardCharsets.UTF_8)); + } private final Regex regex; private final String sourcePattern; @@ -328,11 +337,17 @@ private static boolean hasControlVerbState(String pattern) { } static String translatePattern(String pattern) { - return translatePattern(pattern, RegexFlags.fromModifiers("", pattern), 0); + return translatePattern(pattern, RegexFlags.fromModifiers("", pattern), 0, true); } private static String translatePattern(String pattern, RegexFlags flags, int trustedCalloutCount) { + return translatePattern(pattern, flags, trustedCalloutCount, false); + } + + private static String translatePattern(String pattern, RegexFlags flags, + int trustedCalloutCount, + boolean resolveNamedCharacters) { pattern = translateDefineBlocks(pattern); pattern = translateBeyondUnicodeClassMembers(pattern); StringBuilder out = new StringBuilder(pattern.length() + 16); @@ -354,7 +369,7 @@ private static String translatePattern(String pattern, RegexFlags flags, atClassStart = false; classAllowsLeadingClose = false; } - if (pattern.startsWith("\\N{", i)) { + if (resolveNamedCharacters && pattern.startsWith("\\N{", i)) { int end = pattern.indexOf('}', i + 3); if (end > i + 3) { int codePoint = UnicodeResolver.getCodePointFromName( diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java index bec8aab452..2750686120 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java @@ -766,6 +766,9 @@ && containsExecutableSource(originalPatternString, regex.regexFlags.isExtended()); if (e instanceof SyntaxException && !validatesExecutableSource) { String message = e.getMessage(); + if ("Empty \\N{}".equals(message)) { + throw new PerlCompilerException("Unknown charname ''"); + } if (literalSyntaxValidation && message != null && (message.contains("premature end of char-class") || message.contains("Unclosed character class"))) { diff --git a/src/test/resources/unit/regex_joni_native_named_character.t b/src/test/resources/unit/regex_joni_native_named_character.t new file mode 100644 index 0000000000..5035affdb8 --- /dev/null +++ b/src/test/resources/unit/regex_joni_native_named_character.t @@ -0,0 +1,39 @@ +use strict; +use warnings; +use utf8; +use Test::More tests => 11; + +ok("A" =~ /\N{LATIN CAPITAL LETTER A}/, + 'official named character resolves outside a class'); +ok("A" =~ /[\N{LATIN CAPITAL LETTER A}]/, + 'official named character resolves inside a class'); +ok("\x{1F642}" =~ /\N{U+1F642}/, + 'supplementary named character resolves outside a class'); +ok("\x{1F642}" =~ /[\N{U+1F642}]/, + 'supplementary named character resolves inside a class'); +ok("a" =~ /\N{LATIN CAPITAL LETTER A}/i, + 'case folding applies after named-character resolution'); +ok('#' =~ /(?x:\N{NUMBER SIGN})/, + 'named punctuation remains significant under extended syntax'); +ok(' ' =~ /(?x:[\N{SPACE}])/, + 'named whitespace remains significant in an extended class'); + +{ + use bytes; + my $latin1 = pack('C', 0xE9); + ok($latin1 =~ /\N{U+00E9}/, + 'named character uses the active single-byte regex mode'); +} + +eval q{qr/\N{}/}; +like($@, qr/(?:Unknown charname|zero length)/, + 'empty named-character escape remains fatal'); + +eval q{qr/\N{NOT A REAL UNICODE CHARACTER}/}; +like($@, qr/(?:Unknown charname|Invalid Unicode character name)/, + 'resolver failure remains fatal'); + +my $unterminated = '\\N{LATIN CAPITAL LETTER A'; +eval { qr/$unterminated/ }; +like($@, qr/(?:terminator|right brace|terminated)/i, + 'unterminated named-character escape remains fatal'); From 74e5804efc8ae642ae9d4207dde2c6794d16728d Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Tue, 18 Aug 2026 18:07:29 +0200 Subject: [PATCH 05/11] fix(regex): admit nested lookahead in Perl lookbehind Allow positive and negative lookahead anchors within Perl-syntax lookbehind while preserving the existing Ruby-syntax restrictions. Add direct Joni and standard-Perl regression coverage for semantics, captures, bounded lengths, malformed patterns, and syntax isolation. Generated with [OpenAI Codex](https://openai.com/codex) Co-Authored-By: OpenAI Codex --- .../unit/regex/nested_lookahead_lookbehind.t | 54 +++++++++++++++++++ third_party/joni/src/org/joni/Analyser.java | 10 +++- .../joni/constants/internal/AnchorType.java | 3 ++ .../joni/test/TestPerlVariableLookBehind.java | 54 ++++++++++++++++++- 4 files changed, 118 insertions(+), 3 deletions(-) create mode 100644 src/test/resources/unit/regex/nested_lookahead_lookbehind.t diff --git a/src/test/resources/unit/regex/nested_lookahead_lookbehind.t b/src/test/resources/unit/regex/nested_lookahead_lookbehind.t new file mode 100644 index 0000000000..ff31ccd1be --- /dev/null +++ b/src/test/resources/unit/regex/nested_lookahead_lookbehind.t @@ -0,0 +1,54 @@ +use strict; +use warnings; +no warnings 'experimental::vlb'; +use Test::More; + +ok('ab' =~ /(?<=a(?=b))b/, + 'positive lookahead is admitted inside positive lookbehind'); +ok('ac' =~ /(?<=a(?!b))c/, + 'negative lookahead is admitted inside positive lookbehind'); +ok('ab' !~ /(?<=a(?!b))b/, + 'nested negative lookahead still rejects its forbidden suffix'); + +ok('ab' !~ /(? matcher(pattern, "", syntax)); + } + @Test public void boundedPositiveLookBehindTriesEveryLength() { assertEquals(3, matcher("(?<=a{1,3})b", "aaab").search(0, 4, Option.NONE)); @@ -68,4 +78,46 @@ public void nestedFiniteAlternationSupportsCompoundLengths() { assertEquals(5, matcher("(?<=x(?:a|bc){1,2})z", "xbcbcz") .search(0, 6, Option.NONE)); } + + @Test + public void nestedLookAheadAssertionsFollowPerlLookBehindSemantics() { + assertEquals(1, matcher("(?<=a(?=b))b", "ab").search(0, 2, Option.NONE)); + assertEquals(1, matcher("(?<=a(?!b))c", "ac").search(0, 2, Option.NONE)); + assertEquals(-1, matcher("(?<=a(?!b))b", "ab").search(0, 2, Option.NONE)); + assertEquals(-1, matcher("(? Date: Tue, 18 Aug 2026 18:24:49 +0200 Subject: [PATCH 06/11] refactor(regex): route Perl KEEP assertions through Joni Admit real Perl KEEP assertions to the native Joni opcode, reject KEEP inside lookaround, and retire the Java marker capture and replacement bookkeeping. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../runtime/regex/CaptureNameEncoder.java | 4 +- .../runtime/regex/JoniRegexPattern.java | 118 ++++++++++++++-- .../runtime/regex/RegexBackendPolicy.java | 6 +- .../runtime/regex/RegexPreprocessor.java | 10 -- .../regex/RegexPreprocessorHelper.java | 18 ++- .../runtime/regex/RuntimeRegex.java | 126 ++++-------------- .../runtime/regex/NativeKeepRoutingTest.java | 43 ++++++ .../resources/unit/regex_joni_native_keep.t | 93 +++++++++++++ 8 files changed, 288 insertions(+), 130 deletions(-) create mode 100644 src/test/java/org/perlonjava/runtime/regex/NativeKeepRoutingTest.java create mode 100644 src/test/resources/unit/regex_joni_native_keep.t diff --git a/src/main/java/org/perlonjava/runtime/regex/CaptureNameEncoder.java b/src/main/java/org/perlonjava/runtime/regex/CaptureNameEncoder.java index d796d22976..736a92c589 100644 --- a/src/main/java/org/perlonjava/runtime/regex/CaptureNameEncoder.java +++ b/src/main/java/org/perlonjava/runtime/regex/CaptureNameEncoder.java @@ -269,9 +269,9 @@ public static boolean isDuplicateMarkerName(String javaName) { * from user-visible variables like %+ and %-. * * @param captureName The capture group name to check - * @return true if this is an internal capture (code block or \K marker) + * @return true if this is an internal code-block capture */ public static boolean isInternalCapture(String captureName) { - return isCodeBlockCapture(captureName) || "perlK".equals(captureName); + return isCodeBlockCapture(captureName); } } diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index 6fb220b9df..8fbc14573e 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -84,6 +84,10 @@ private static int resolveNamedCharacter(byte[] bytes, int p, int end, JoniRegexPattern(String perlPattern, RegexFlags flags, int trustedCalloutCount, boolean forceAsciiClasses, boolean byteMode, boolean byteBackedPattern) { + KeepSyntax keepSyntax = analyzeKeepSyntax(perlPattern, flags.isExtended()); + if (keepSyntax.inLookaround()) { + throw new PerlCompilerException("\\K not permitted in lookahead/lookbehind in regex"); + } this.flags = flags; this.byteMode = byteMode; hasControlVerbState = hasControlVerbState(perlPattern); @@ -305,14 +309,14 @@ private static int toJoniOptions(RegexFlags flags, boolean forceAsciiClasses) { } static boolean requiresJoniBackend(String pattern) { + return requiresJoniBackend(pattern, + pattern == null ? null : RegexFlags.fromModifiers("", pattern)); + } + + static boolean requiresJoniBackend(String pattern, RegexFlags flags) { if (pattern == null) return false; - boolean hasSubroutineCall = pattern.matches("(?s).*\\(\\?[+-]?\\d+\\).*") - || pattern.contains("(?&") - || pattern.contains("(?P>"); - // Temporary parity fallback: Java handles the branch-reset named-call - // corpus until Joni's native named-call admission patch is complete. - boolean branchResetCallUsesJava = pattern.contains("(?|") && hasSubroutineCall; - return pattern.contains("(?{=CALL:") + return analyzeKeepSyntax(pattern, flags != null && flags.isExtended()).present() + || pattern.contains("(?{=CALL:") || pattern.contains("(?{=DYNAMIC:") || pattern.contains("(*ACCEPT)") || pattern.contains("(*PRUNE") @@ -326,7 +330,105 @@ static boolean requiresJoniBackend(String pattern) { || pattern.contains("(?(R") || pattern.contains("(?(<") || pattern.contains("(?('") - || (hasSubroutineCall && !branchResetCallUsesJava); + || pattern.matches("(?s).*\\(\\?[+-]?\\d+\\).*") + || pattern.contains("(?&") + || pattern.contains("(?P>"); + } + + private record KeepSyntax(boolean present, boolean inLookaround) {} + + private static KeepSyntax analyzeKeepSyntax(String pattern, boolean extended) { + boolean quoted = false; + boolean inClass = false; + boolean classStart = false; + int extendedClassDepth = 0; + int lookaroundDepth = 0; + java.util.ArrayDeque groups = new java.util.ArrayDeque<>(); + boolean present = false; + + for (int i = 0; i < pattern.length(); i++) { + char ch = pattern.charAt(i); + if (quoted) { + if (ch == '\\' && i + 1 < pattern.length() + && pattern.charAt(i + 1) == 'E') { + quoted = false; + i++; + } + continue; + } + if (extendedClassDepth > 0) { + if (ch == '\\' && i + 1 < pattern.length()) { + i++; + } else if (ch == '[') { + extendedClassDepth++; + } else if (ch == ']' && --extendedClassDepth == 0) { + // The following ')' closes the extended-class construct. + } + continue; + } + if (inClass) { + if (ch == '\\' && i + 1 < pattern.length()) { + i++; + classStart = false; + continue; + } + if (ch == '[' && i + 1 < pattern.length() + && (pattern.charAt(i + 1) == ':' + || pattern.charAt(i + 1) == '.' + || pattern.charAt(i + 1) == '=')) { + char delimiter = pattern.charAt(i + 1); + int close = pattern.indexOf("" + delimiter + ']', i + 2); + if (close >= 0) i = close + 1; + classStart = false; + continue; + } + if (ch == ']' && !classStart) inClass = false; + else if (!(classStart && ch == '^')) classStart = false; + continue; + } + if (extended && ch == '#') { + while (i + 1 < pattern.length() + && pattern.charAt(i + 1) != '\n') i++; + continue; + } + if (pattern.startsWith("(?[", i)) { + extendedClassDepth = 1; + i += 2; + continue; + } + if (ch == '[') { + inClass = true; + classStart = true; + continue; + } + if (ch == '\\' && i + 1 < pattern.length()) { + char escaped = pattern.charAt(++i); + if (escaped == 'Q') { + quoted = true; + } else if (escaped == 'K') { + present = true; + if (lookaroundDepth > 0) return new KeepSyntax(true, true); + } + continue; + } + if (ch == '(') { + if (pattern.startsWith("(?#", i)) { + int close = pattern.indexOf(')', i + 3); + if (close < 0) break; + i = close; + continue; + } + boolean lookaround = pattern.startsWith("(?=", i) + || pattern.startsWith("(?!", i) + || pattern.startsWith("(?<=", i) + || pattern.startsWith("(?a)|(?b)` (legal in Perl, @@ -104,14 +103,6 @@ static boolean hadBranchReset() { return branchResetEncountered; } - static void markBackslashK() { - backslashKEncountered = true; - } - - static boolean hadBackslashK() { - return backslashKEncountered; - } - /** * Preprocesses a given regex string to make it compatible with Java's regex engine. * This involves handling various constructs and escape sequences that Java does not @@ -144,7 +135,6 @@ private static String preProcessRegexInternal(String s, RegexFlags regexFlags) { deferredUnicodePropertyEncountered = false; inlinePFlagEncountered = false; branchResetEncountered = false; - backslashKEncountered = false; seenNamedCaptures.clear(); emittedNamedCaptures.clear(); duplicateNameCounter = 0; diff --git a/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessorHelper.java b/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessorHelper.java index e289d53653..74fe5b1358 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessorHelper.java +++ b/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessorHelper.java @@ -221,13 +221,11 @@ static int handleEscapeSequences(String s, StringBuilder sb, int c, int offset, sb.append("[^\\n\\x0B\\f\\r\\x85\\x{2028}\\x{2029}]"); return offset; } else if (nextChar == 'K') { - // \K - keep assertion (reset start of match) - // Insert a zero-width named capture group to mark the \K position. - // During substitution, text before this position is preserved ("kept"). - sb.setLength(sb.length() - 1); // Remove the backslash - RegexPreprocessor.markBackslashK(); - RegexPreprocessor.captureGroupCount++; - sb.append("(?)"); + // Syntax-aware admission routes real KEEP assertions to Joni. A + // \K reaching the Java preprocessor is literal, for example in a + // character class or a \Q...\E region. + sb.setLength(sb.length() - 1); + sb.append('K'); return offset; } else if ((nextChar == 'b' || nextChar == 'B') && offset + 1 < length && s.charAt(offset + 1) == '{') { // Handle \b{...} and \B{...} boundary assertions @@ -804,6 +802,12 @@ static int handleRegexCharacterClassEscape(int offset, String s, StringBuilder s sb.append(Character.toChars(s.charAt(offset))); } lastChar = -1; // Unicode properties can't be range endpoints + } else if (offset < length && s.charAt(offset) == 'K') { + // KEEP is not active inside a class. Preserve Perl's + // literal K while emitting Java-compatible syntax. + sb.setLength(sb.length() - 1); + sb.append('K'); + lastChar = 'K'; } else if (offset < length && s.charAt(offset) == 'N') { if (offset + 1 < length && s.charAt(offset + 1) == '{') { // Handle \N{...} constructs diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java index 2750686120..3ae0ca72c8 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java @@ -213,7 +213,6 @@ static void updateControlVerbVariables(String mark, String error) { private boolean hasCodeBlockCaptures = false; // True if regex has (?{...}) code blocks private boolean deferredUserDefinedUnicodeProperties = false; private boolean hasBranchReset = false; // True if pattern uses (?|...) branch reset - private boolean hasBackslashK = false; // True if pattern uses \K (keep assertion) // An empty qr// object keeps its own empty pattern when interpolated; // only empty match/substitution string syntax reuses the previous match. private boolean quoteConstruction = false; @@ -259,7 +258,6 @@ public RuntimeRegex cloneTracked() { copy.hasCodeBlockCaptures = this.hasCodeBlockCaptures; copy.deferredUserDefinedUnicodeProperties = this.deferredUserDefinedUnicodeProperties; copy.hasBranchReset = this.hasBranchReset; - copy.hasBackslashK = this.hasBackslashK; copy.quoteConstruction = this.quoteConstruction; copy.warningsOnUse = new ArrayList<>(this.warningsOnUse); copy.lexicalDebugMode = this.lexicalDebugMode; @@ -623,7 +621,8 @@ private static synchronized RuntimeRegex compileSynchronized( String javaPattern = null; try { - boolean usesRecursiveBackend = RegexBackendPolicy.useJoni(compilePatternString); + boolean usesRecursiveBackend = RegexBackendPolicy.useJoni( + compilePatternString, regex.regexFlags); if (usesRecursiveBackend && compilePatternString.contains("(?&") && (compilePatternString.contains("(?<=") @@ -668,14 +667,12 @@ private static synchronized RuntimeRegex compileSynchronized( regex.hasPreservesMatch = regex.regexFlags.preservesMatch() || RegexFlags.hasInlinePreserveModifier(compilePatternString); regex.hasBranchReset = false; - regex.hasBackslashK = false; } else { regex.deferredUserDefinedUnicodeProperties = RegexPreprocessor.hadDeferredUnicodePropertyEncountered(); regex.hasPreservesMatch = regex.regexFlags.preservesMatch() || RegexFlags.hasInlinePreserveModifier(compilePatternString) || RegexPreprocessor.hadInlinePFlag(); regex.hasBranchReset = RegexPreprocessor.hadBranchReset(); - regex.hasBackslashK = RegexPreprocessor.hadBackslashK(); regex.warningsOnUse.addAll(RegexPreprocessor.getWarningsOnUse()); } @@ -1641,7 +1638,6 @@ public static RuntimeScalar getReplacementRegex(RuntimeScalar patternString, Run regex.useGAssertion = resolvedRegex.useGAssertion; regex.patternFlags = resolvedRegex.patternFlags; regex.hasBranchReset = resolvedRegex.hasBranchReset; - regex.hasBackslashK = resolvedRegex.hasBackslashK; regex.hasCodeBlockCaptures = resolvedRegex.hasCodeBlockCaptures; regex.warningsOnUse = new ArrayList<>(resolvedRegex.warningsOnUse); regex.lexicalDebugMode = callSiteDebugMode != 0 @@ -1680,7 +1676,6 @@ public static RuntimeScalar getReplacementRegex(RuntimeScalar patternString, Run regex.useGAssertion = recompiledRegex.useGAssertion; regex.patternFlags = recompiledRegex.patternFlags; regex.hasBranchReset = recompiledRegex.hasBranchReset; - regex.hasBackslashK = recompiledRegex.hasBackslashK; regex.hasCodeBlockCaptures = recompiledRegex.hasCodeBlockCaptures; regex.warningsOnUse = new ArrayList<>(recompiledRegex.warningsOnUse); } else { @@ -1859,7 +1854,7 @@ private static void updateLastNamedCaptureGroups(RegexMatcher matcher) { } /** Populate Perl's numbered capture state from the backend-neutral view. */ - private static void updateNumberedCaptureGroups(RuntimeRegex regex, RegexMatcher matcher) { + private static void updateNumberedCaptureGroups(RegexMatcher matcher) { RuntimeRegexState regexState = state(); regexState.lastParenMatchOverrideActive = false; regexState.lastParenMatchOverride = null; @@ -1874,17 +1869,9 @@ private static void updateNumberedCaptureGroups(RuntimeRegex regex, RegexMatcher return; } - int perlKGroup = regex.hasBackslashK ? getPerlKGroup(matcher) : -1; - int userGroupCount = captureCount - (perlKGroup >= 0 ? 1 : 0); - if (userGroupCount == 0) { - regexState.lastCaptureGroups = null; - return; - } - regexState.lastCaptureGroups = new String[userGroupCount]; - int destination = 0; + regexState.lastCaptureGroups = new String[captureCount]; for (int group = 1; group <= captureCount; group++) { - if (group == perlKGroup) continue; - regexState.lastCaptureGroups[destination++] = matcher.group(group); + regexState.lastCaptureGroups[group - 1] = matcher.group(group); } } @@ -2127,31 +2114,21 @@ private static RuntimeBase matchRegexDirect(RuntimeScalar quotedRegex, RuntimeSc // Always initialize $1, $2, @+, @-, $`, $&, $' for every successful match regexState.globalMatcher = matcher; regexState.globalMatchString = inputStr; - regexState.lastMatchUsedBackslashK = regex.hasBackslashK; + regexState.lastMatchUsedBackslashK = false; updateLastNamedCaptureGroups(matcher); - updateNumberedCaptureGroups(regex, matcher); + updateNumberedCaptureGroups(matcher); - // For \K, adjust match start/string so $& is only the post-\K portion - if (regex.hasBackslashK) { - int keepEnd = matcher.end("perlK"); - regexState.lastMatchedString = inputStr.substring(keepEnd, matcher.end()); - regexState.lastMatchStart = keepEnd; - } else { - regexState.lastMatchedString = matcher.group(0); - regexState.lastMatchStart = matcher.start(); - } + regexState.lastMatchedString = matcher.group(0); + regexState.lastMatchStart = matcher.start(); regexState.lastMatchEnd = matcher.end(); if (regex.regexFlags.isGlobalMatch() && captureCount < 1 && ctx == RuntimeContextType.LIST) { // Global match and no captures, in list context return the matched string - String matchedStr = regex.hasBackslashK ? regexState.lastMatchedString : matcher.group(0); - matchedGroups.add(makeMatchResultScalar(matchedStr)); + matchedGroups.add(makeMatchResultScalar(matcher.group(0))); } else { // save captures in return list if needed if (ctx == RuntimeContextType.LIST) { - int perlKGroup = regex.hasBackslashK ? getPerlKGroup(matcher) : -1; for (int i = 1; i <= captureCount; i++) { - if (i == perlKGroup) continue; // skip internal \K marker group String matchedStr = matcher.group(i); // Include undef for groups that didn't participate in the match. // The matcher adapter exposes native Perl numbering for (?|...). @@ -2777,19 +2754,12 @@ private static void updateReplacementMatchState(RuntimeRegex regex, RegexMatcher // Initialize $1, $2, @+, @- only when we have a match state().globalMatcher = matcher; state().globalMatchString = inputStr; - state().lastMatchUsedBackslashK = regex.hasBackslashK; + state().lastMatchUsedBackslashK = false; updateLastNamedCaptureGroups(matcher); - updateNumberedCaptureGroups(regex, matcher); + updateNumberedCaptureGroups(matcher); - // For \K, adjust match start so $& is only the post-\K portion - if (regex.hasBackslashK) { - int keepEnd = matcher.end("perlK"); - state().lastMatchStart = keepEnd; - state().lastMatchedString = inputStr.substring(keepEnd, matcher.end()); - } else { - state().lastMatchStart = matcher.start(); - state().lastMatchedString = matcher.group(0); - } + state().lastMatchStart = matcher.start(); + state().lastMatchedString = matcher.group(0); state().lastMatchEnd = matcher.end(); } @@ -2931,7 +2901,6 @@ public static RuntimeBase replaceRegex(RuntimeScalar quotedRegex, RuntimeScalar // Don't reset state().globalMatcher here - only reset it if we actually find a match // This preserves capture variables from previous matches when substitution doesn't match - // Track position for manual replacement when \K is used int lastAppendEnd = 0; // Perform the substitution. Java's Matcher.find() skips ahead after a @@ -2986,18 +2955,9 @@ public static RuntimeBase replaceRegex(RuntimeScalar quotedRegex, RuntimeScalar } if (replacementStr != null) { - if (regex.hasBackslashK) { - // \K: preserve text before \K position, only replace after it - int keepEnd = matcher.end("perlK"); - resultBuffer.append(inputStr, lastAppendEnd, keepEnd); - resultBuffer.append(replacementStr); - lastAppendEnd = matcher.end(); - } else { - // Normal replacement: replace the entire match - resultBuffer.append(inputStr, lastAppendEnd, matcher.start()); - resultBuffer.append(replacementStr); - lastAppendEnd = matcher.end(); - } + resultBuffer.append(inputStr, lastAppendEnd, matcher.start()); + resultBuffer.append(replacementStr); + lastAppendEnd = matcher.end(); } // If not a global match, break after the first replacement @@ -3061,16 +3021,9 @@ public static RuntimeBase replaceRegex(RuntimeScalar quotedRegex, RuntimeScalar } if (retryReplacementStr != null) { - if (regex.hasBackslashK) { - int keepEnd = retryMatcher.end("perlK"); - resultBuffer.append(inputStr, lastAppendEnd, keepEnd); - resultBuffer.append(retryReplacementStr); - lastAppendEnd = retryMatcher.end(); - } else { - resultBuffer.append(inputStr, lastAppendEnd, retryMatcher.start()); - resultBuffer.append(retryReplacementStr); - lastAppendEnd = retryMatcher.end(); - } + resultBuffer.append(inputStr, lastAppendEnd, retryMatcher.start()); + resultBuffer.append(retryReplacementStr); + lastAppendEnd = retryMatcher.end(); } searchStart = retryMatcher.end(); globalPosition = searchStart; @@ -3259,12 +3212,10 @@ public static RuntimeScalar matcherStart(int group) { return scalarUndef; } try { - // Adjust group number to skip the internal perlK group - int javaGroup = adjustGroupForBackslashK(group); - if (javaGroup < 0 || javaGroup > state().globalMatcher.groupCount()) { + if (group < 0 || group > state().globalMatcher.groupCount()) { return scalarUndef; } - int start = state().globalMatcher.start(javaGroup); + int start = state().globalMatcher.start(group); if (start == -1) { return scalarUndef; } @@ -3285,12 +3236,10 @@ public static RuntimeScalar matcherEnd(int group) { return scalarUndef; } try { - // Adjust group number to skip the internal perlK group - int javaGroup = adjustGroupForBackslashK(group); - if (javaGroup < 0 || javaGroup > state().globalMatcher.groupCount()) { + if (group < 0 || group > state().globalMatcher.groupCount()) { return scalarUndef; } - int end = state().globalMatcher.end(javaGroup); + int end = state().globalMatcher.end(group); if (end == -1) { return scalarUndef; } @@ -3320,10 +3269,6 @@ public static int matcherSize() { return 0; } int size = state().globalMatcher.groupCount(); - // Subtract the internal perlK group if \K was used - if (state().lastMatchUsedBackslashK) { - size--; - } // +1 because groupCount is zero-based, and we include the entire match return size + 1; } @@ -3337,20 +3282,6 @@ public static int matcherStartSize() { return size; } - /** - * Adjust a Perl capture group number to a Java matcher group number, - * skipping the internal perlK named group when \K is active. - */ - private static int adjustGroupForBackslashK(int perlGroup) { - if (!state().lastMatchUsedBackslashK || state().globalMatcher == null) { - return perlGroup; - } - int perlKGroup = getPerlKGroup(state().globalMatcher); - if (perlKGroup < 0) return perlGroup; - // Perl groups before perlK: same number. At or after: add 1. - return perlGroup >= perlKGroup ? perlGroup + 1 : perlGroup; - } - /** * Check if a string contains any non-ASCII characters (code point > 127). * Used to determine if Unicode matching should be used. @@ -3820,13 +3751,4 @@ private static boolean containsWideChars(String s) { return false; } - /** - * Get the group number of the internal perlK named capture group. - * This group is inserted by the preprocessor at the \K position. - */ - private static int getPerlKGroup(RegexMatcher matcher) { - Map namedGroups = matcher.namedGroups(); - Integer group = namedGroups.get("perlK"); - return group != null ? group : -1; - } } diff --git a/src/test/java/org/perlonjava/runtime/regex/NativeKeepRoutingTest.java b/src/test/java/org/perlonjava/runtime/regex/NativeKeepRoutingTest.java new file mode 100644 index 0000000000..2202ddd875 --- /dev/null +++ b/src/test/java/org/perlonjava/runtime/regex/NativeKeepRoutingTest.java @@ -0,0 +1,43 @@ +package org.perlonjava.runtime.regex; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; +import org.perlonjava.runtime.runtimetypes.PerlCompilerException; + +class NativeKeepRoutingTest { + @Test + void admitsOnlyRealKeepAssertions() { + assertTrue(JoniRegexPattern.requiresJoniBackend("ab\\Kcd")); + assertFalse(JoniRegexPattern.requiresJoniBackend("ab\\\\Kcd")); + assertFalse(JoniRegexPattern.requiresJoniBackend("[\\K]")); + assertFalse(JoniRegexPattern.requiresJoniBackend("[[:alpha:]\\K]")); + assertFalse(JoniRegexPattern.requiresJoniBackend("\\Q\\K\\E")); + assertFalse(JoniRegexPattern.requiresJoniBackend("(?# \\K ignored)abc")); + assertFalse(JoniRegexPattern.requiresJoniBackend("(?[ [\\K] ])")); + assertFalse(JoniRegexPattern.requiresJoniBackend("abc\\")); + assertFalse(JoniRegexPattern.requiresJoniBackend("\\k")); + } + + @Test + void ignoresKeepTextInExtendedComments() { + String pattern = "abc # \\K ignored\ndef"; + RegexFlags flags = RegexFlags.fromModifiers("x", pattern); + assertFalse(JoniRegexPattern.requiresJoniBackend(pattern, flags)); + } + + @Test + void rejectsKeepInsideLookaroundBeforeJoniCompilation() { + for (String pattern : new String[] { + "ab(?=c\\Kd)", "ab(?!c\\Kd)", "(?<=a\\Kb)c", "(? new JoniRegexPattern(pattern, flags)); + assertTrue(error.getMessage().contains( + "\\K not permitted in lookahead/lookbehind")); + } + } +} diff --git a/src/test/resources/unit/regex_joni_native_keep.t b/src/test/resources/unit/regex_joni_native_keep.t new file mode 100644 index 0000000000..f69cb90d5d --- /dev/null +++ b/src/test/resources/unit/regex_joni_native_keep.t @@ -0,0 +1,93 @@ +use strict; +use warnings; +use utf8; +use Test::More; + +my $basic = 'abcdef'; +ok($basic =~ /abc\Kdef/, 'KEEP pattern matches'); +is($&, 'def', 'KEEP resets the complete match'); +is($-[0], 3, 'KEEP resets match start'); +is($+[0], 6, 'KEEP retains match end'); +is($`, 'abc', 'prematch ends at the KEEP position'); +is($', '', 'postmatch starts at the native match end'); + +my $captured = 'abc'; +ok($captured =~ /(a)(b)\K(c)/, 'captures span KEEP'); +is($&, 'c', 'complete match begins after KEEP with captures'); +is_deeply([$1, $2, $3], [qw(a b c)], 'capture numbering excludes KEEP'); +is_deeply([$-[1], $+[1], $-[2], $+[2], $-[3], $+[3]], + [0, 1, 1, 2, 2, 3], 'capture offsets remain physical subject offsets'); + +my @global = 'a1b2' =~ /[a-z]\K\d/g; +is_deeply(\@global, [qw(1 2)], 'global list match returns post-KEEP matches'); + +my $global_scalar = 'a1b2'; +ok($global_scalar =~ /[a-z]\K\d/g, 'first scalar global KEEP match succeeds'); +is($&, '1', 'first scalar global match value'); +is(pos($global_scalar), 2, 'first scalar global position uses match end'); +ok($global_scalar =~ /[a-z]\K\d/g, 'second scalar global KEEP match succeeds'); +is($&, '2', 'second scalar global match value'); +is(pos($global_scalar), 4, 'second scalar global position uses match end'); + +my $substitution = 'foobar'; +is($substitution =~ s/foo\Kbar/X/, 1, 'single KEEP substitution count'); +is($substitution, 'fooX', 'single KEEP substitution preserves prefix'); + +my $global_substitution = 'a1b2'; +is($global_substitution =~ s/[a-z]\K\d/X/g, 2, + 'global KEEP substitution count'); +is($global_substitution, 'aXbX', + 'global KEEP substitution preserves every prefix'); + +my $capture_substitution = 'abc'; +$capture_substitution =~ s/(a)(b)\K(c)/<$1$2$3>/; +is($capture_substitution, 'ab', + 'KEEP substitution retains captures before the reset'); + +my $backtracking = 'abcx'; +ok($backtracking =~ /ab\Kc(?:z|x)/, 'KEEP survives successful backtracking'); +is($&, 'cx', 'backtracked match retains the final KEEP start'); + +my $failed_keep_branch = 'ax'; +ok($failed_keep_branch =~ /(?:a\Kb|a)x/, + 'failed KEEP branch restores the prior match start'); +is($&, 'ax', 'failed KEEP branch does not leak its reset'); +is($-[0], 0, 'failed KEEP branch restores offset zero'); + +ok('abc' =~ /a\Kb\Kc/, 'multiple KEEP assertions match'); +is($&, 'c', 'last successful KEEP assertion wins'); +ok('abc' =~ /abc\K/, 'terminal KEEP permits an empty match'); +is($&, '', 'terminal KEEP resets to an empty complete match'); +is($-[0], 3, 'terminal KEEP start is the input end'); + +my $unicode = "préfixe"; +ok($unicode =~ /pré\Kfixe/, 'Unicode KEEP pattern matches'); +is($&, 'fixe', 'Unicode KEEP reports character-aligned match text'); +is($-[0], 3, 'Unicode KEEP reports Perl character offset'); + +{ + use bytes; + my $bytes = pack('C*', 0x61, 0x62, 0x63, 0x64); + ok($bytes =~ /ab\Kcd/, 'byte input KEEP pattern matches'); + is($&, 'cd', 'byte input KEEP reports post-reset bytes'); + is($-[0], 2, 'byte input KEEP reports byte offset'); +} + +{ + no warnings 'regexp'; + ok('\\K' =~ /\A\\K\z/, 'escaped backslash K remains literal'); + ok('K' =~ /\A[\K]\z/, 'K inside a class remains a class member'); + ok('\\K' =~ /\A\Q\K\E\z/, 'K inside quote-meta remains literal'); +} + +my $lookahead = 'ab(?=c\\Kd)'; +eval { qr/$lookahead/ }; +like($@, qr/\\K not permitted in lookahead\/lookbehind/, + 'KEEP in lookahead remains a compile error'); + +my $lookbehind = '(?<=a\\Kb)c'; +eval { qr/$lookbehind/ }; +like($@, qr/\\K not permitted in lookahead\/lookbehind/, + 'KEEP in lookbehind remains a compile error'); + +done_testing; From 05b5b2f14e5709019abc7f7a62e6c479e259723b Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Tue, 18 Aug 2026 18:46:34 +0200 Subject: [PATCH 07/11] feat(joni): accept character-property range resolvers Let syntax owners resolve property names to validated native code-point ranges while preserving JCodings fallback behavior and Joni's parser ownership. Generated with [OpenAI Codex](https://openai.com/codex) Co-Authored-By: OpenAI Codex --- .../org/joni/CharacterPropertyResolver.java | 32 ++++++ third_party/joni/src/org/joni/Lexer.java | 41 ++++++- third_party/joni/src/org/joni/Parser.java | 28 +++-- third_party/joni/src/org/joni/Syntax.java | 10 ++ .../joni/src/org/joni/ast/CClassNode.java | 10 ++ .../test/TestCharacterPropertyResolver.java | 100 ++++++++++++++++++ 6 files changed, 208 insertions(+), 13 deletions(-) create mode 100644 third_party/joni/src/org/joni/CharacterPropertyResolver.java create mode 100644 third_party/joni/test/org/joni/test/TestCharacterPropertyResolver.java diff --git a/third_party/joni/src/org/joni/CharacterPropertyResolver.java b/third_party/joni/src/org/joni/CharacterPropertyResolver.java new file mode 100644 index 0000000000..dd31598dd1 --- /dev/null +++ b/third_party/joni/src/org/joni/CharacterPropertyResolver.java @@ -0,0 +1,32 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import org.jcodings.Encoding; + +/** Resolves syntax-specific character properties to inclusive code-point ranges. */ +@FunctionalInterface +public interface CharacterPropertyResolver { + /** + * Returns {@code [count, from1, to1, ...]} for a resolved property, or + * {@code null} to use the encoding's built-in property lookup. + */ + int[] resolve(byte[] bytes, int p, int end, Encoding encoding); +} diff --git a/third_party/joni/src/org/joni/Lexer.java b/third_party/joni/src/org/joni/Lexer.java index 5fa871a67b..0e26280a87 100644 --- a/third_party/joni/src/org/joni/Lexer.java +++ b/third_party/joni/src/org/joni/Lexer.java @@ -1695,20 +1695,55 @@ private void possessiveCheck() { } } - protected final int fetchCharPropertyToCType() { + protected static final class CharProperty { + final int ctype; + final int[] ranges; + + CharProperty(int ctype, int[] ranges) { + this.ctype = ctype; + this.ranges = ranges; + } + } + + protected final CharProperty fetchCharProperty() { mark(); while (left()) { int last = p; fetch(); if (c == '}') { - return enc.propertyNameToCType(bytes, _p, last); + if (syntax.characterPropertyResolver != null) { + int[] ranges = syntax.characterPropertyResolver.resolve( + bytes, _p, last, enc); + if (ranges != null) { + validateCharacterPropertyRanges(ranges); + return new CharProperty(0, ranges); + } + } + return new CharProperty( + enc.propertyNameToCType(bytes, _p, last), null); } else if (c == '(' || c == ')' || c == '{' || c == '|') { throw new CharacterPropertyException(EncodingError.ERR_INVALID_CHAR_PROPERTY_NAME, bytes, _p, last); } } newValueException(PROPERTY_NAME_NEVER_TERMINATED, _p, stop); - return 0; // not reached + return null; // not reached + } + + private static void validateCharacterPropertyRanges(int[] ranges) { + if (ranges.length == 0 || ranges.length != ranges[0] * 2 + 1) { + throw new IllegalArgumentException("invalid character property ranges"); + } + int previousEnd = -1; + for (int i = 0; i < ranges[0]; i++) { + int from = ranges[i * 2 + 1]; + int to = ranges[i * 2 + 2]; + if (from < 0 || from > to || to > CodeRangeBuffer.LAST_CODE_POINT + || from <= previousEnd) { + throw new IllegalArgumentException("invalid character property ranges"); + } + previousEnd = to; + } } protected final void syntaxWarn(String message, char c) { diff --git a/third_party/joni/src/org/joni/Parser.java b/third_party/joni/src/org/joni/Parser.java index 83f12dcc84..af9cbfc07d 100644 --- a/third_party/joni/src/org/joni/Parser.java +++ b/third_party/joni/src/org/joni/Parser.java @@ -256,13 +256,8 @@ private CClassNode parseCharClass(ObjPtr ascNode) { break; case CHAR_PROPERTY: - int ctype = fetchCharPropertyToCType(); - cc.addCType(ctype, token.getPropNot(), false, env, this); - if (ascCc != null) { - if (ctype != CharacterType.ASCII) { - ascCc.addCType(ctype, token.getPropNot(), false, env, this); - } - } + CharProperty property = fetchCharProperty(); + addCharProperty(cc, ascCc, property, token.getPropNot()); cc.nextStateClass(arg, ascCc, env); // goto next_class break; @@ -1634,20 +1629,33 @@ private Node cClassCaseFold(Node node, CClassNode cc, CClassNode ascCc) { } private Node parseCharProperty() { - int ctype = fetchCharPropertyToCType(); + CharProperty property = fetchCharProperty(); CClassNode cc = new CClassNode(); Node node = cc; - cc.addCType(ctype, false, false, env, this); + addCharProperty(cc, null, property, false); if (token.getPropNot()) cc.setNot(); if (isIgnoreCase(env.option)) { - if (ctype != CharacterType.ASCII) { + if (property.ranges != null || property.ctype != CharacterType.ASCII) { node = cClassCaseFold(node, cc, cc); } } return node; } + private void addCharProperty(CClassNode cc, CClassNode ascCc, + CharProperty property, boolean not) { + if (property.ranges == null) { + cc.addCType(property.ctype, not, false, env, this); + if (ascCc != null && property.ctype != CharacterType.ASCII) { + ascCc.addCType(property.ctype, not, false, env, this); + } + return; + } + cc.addCodeRanges(property.ranges, not, env); + if (ascCc != null) ascCc.addCodeRanges(property.ranges, not, env); + } + private Node parseAnycharAnytime() { Node node = new AnyCharNode(); QuantifierNode qn = new QuantifierNode(0, QuantifierNode.REPEAT_INFINITE, false); diff --git a/third_party/joni/src/org/joni/Syntax.java b/third_party/joni/src/org/joni/Syntax.java index 55507751bc..c7a9477f40 100644 --- a/third_party/joni/src/org/joni/Syntax.java +++ b/third_party/joni/src/org/joni/Syntax.java @@ -32,6 +32,7 @@ public final class Syntax implements SyntaxProperties { public final int options; public final MetaCharTable metaCharTable; public final NamedCharacterResolver namedCharacterResolver; + public final CharacterPropertyResolver characterPropertyResolver; public Syntax(String name, int op, int op2, int op3, int behavior, int options, MetaCharTable metaCharTable) { this(name, op, op2, op3, behavior, options, metaCharTable, null); @@ -40,6 +41,14 @@ public Syntax(String name, int op, int op2, int op3, int behavior, int options, public Syntax(String name, int op, int op2, int op3, int behavior, int options, MetaCharTable metaCharTable, NamedCharacterResolver namedCharacterResolver) { + this(name, op, op2, op3, behavior, options, metaCharTable, + namedCharacterResolver, null); + } + + public Syntax(String name, int op, int op2, int op3, int behavior, int options, + MetaCharTable metaCharTable, + NamedCharacterResolver namedCharacterResolver, + CharacterPropertyResolver characterPropertyResolver) { this.name = name; this.op = op; this.op2 = op2; @@ -48,6 +57,7 @@ public Syntax(String name, int op, int op2, int op3, int behavior, int options, this.options = options; this.metaCharTable = metaCharTable; this.namedCharacterResolver = namedCharacterResolver; + this.characterPropertyResolver = characterPropertyResolver; } public static class MetaCharTable { diff --git a/third_party/joni/src/org/joni/ast/CClassNode.java b/third_party/joni/src/org/joni/ast/CClassNode.java index ecc724646f..1502271163 100644 --- a/third_party/joni/src/org/joni/ast/CClassNode.java +++ b/third_party/joni/src/org/joni/ast/CClassNode.java @@ -288,6 +288,16 @@ public void addCTypeByRange(int ctype, boolean not, ScanEnvironment env, int sbO } } + /** Adds resolver-provided {@code [count, from, to, ...]} code-point ranges. */ + public void addCodeRanges(int[] ranges, boolean not, ScanEnvironment env) { + int singleByteLimit = 0; + while (singleByteLimit < BitSet.SINGLE_BYTE_SIZE + && env.enc.codeToMbcLength(singleByteLimit) == 1) { + singleByteLimit++; + } + addCTypeByRange(0, not, env, singleByteLimit, ranges); + } + private static int CR_FROM(int[] range, int i) { return range[(i * 2) + 1]; } diff --git a/third_party/joni/test/org/joni/test/TestCharacterPropertyResolver.java b/third_party/joni/test/org/joni/test/TestCharacterPropertyResolver.java new file mode 100644 index 0000000000..974df8eeae --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestCharacterPropertyResolver.java @@ -0,0 +1,100 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.fail; + +import java.nio.charset.StandardCharsets; + +import org.jcodings.specific.UTF8Encoding; +import org.joni.CharacterPropertyResolver; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.junit.Test; + +public class TestCharacterPropertyResolver { + private static final CharacterPropertyResolver RESOLVER = + (bytes, p, end, encoding) -> { + String name = new String(bytes, p, end - p, StandardCharsets.UTF_8); + return switch (name) { + case "Fake" -> new int[] {2, 'A', 'A', 0x1f642, 0x1f642}; + default -> null; + }; + }; + + private static Syntax syntax(CharacterPropertyResolver resolver) { + return new Syntax("CharacterPropertyResolver", Syntax.PerlNG.op, + Syntax.PerlNG.op2, Syntax.PerlNG.op3, Syntax.PerlNG.behavior, + Syntax.PerlNG.options, Syntax.PerlNG.metaCharTable, null, resolver); + } + + private static Regex compile(String pattern, CharacterPropertyResolver resolver) { + byte[] bytes = pattern.getBytes(StandardCharsets.UTF_8); + return new Regex(bytes, 0, bytes.length, Option.NONE, + UTF8Encoding.INSTANCE, syntax(resolver)); + } + + private static int search(String pattern, String input) { + byte[] bytes = input.getBytes(StandardCharsets.UTF_8); + return compile(pattern, RESOLVER).matcher(bytes) + .search(0, bytes.length, Option.NONE); + } + + @Test + public void resolvesRangesInsideAndOutsideCharacterClasses() { + assertEquals(0, search("\\p{Fake}", "A")); + assertEquals(0, search("[\\p{Fake}]", "\ud83d\ude42")); + assertEquals(-1, search("\\p{Fake}", "B")); + assertEquals(0, search("\\P{Fake}", "B")); + assertEquals(-1, search("[\\P{Fake}]", "A")); + assertEquals(0, search("(?i)\\p{Fake}", "a")); + assertEquals(-1, search("(?i)\\P{Fake}", "A")); + } + + @Test + public void fallsBackToEncodingProperties() { + assertEquals(0, search("\\p{Digit}", "7")); + assertEquals(-1, search("\\p{Digit}", "A")); + } + + @Test + public void preservesResolverExceptions() { + IllegalArgumentException expected = new IllegalArgumentException("failure"); + try { + compile("\\p{Fake}", (bytes, p, end, encoding) -> { throw expected; }); + fail("expected resolver exception"); + } catch (IllegalArgumentException error) { + assertSame(expected, error); + } + } + + @Test + public void rejectsMalformedRangeResults() { + try { + compile("\\p{Fake}", (bytes, p, end, encoding) -> new int[] {1, 2}); + fail("expected invalid range result"); + } catch (IllegalArgumentException error) { + assertEquals("invalid character property ranges", error.getMessage()); + } + } +} From 7f4bf390f6417739b6fedc81d0eed18ab9081b11 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Tue, 18 Aug 2026 18:46:34 +0200 Subject: [PATCH 08/11] refactor(regex): parse pinned general categories in Joni Pass Perl 5.44 General_Category assignments to Joni unchanged and resolve them through pinned native ranges instead of expanding property text in the adapter. Generated with [OpenAI Codex](https://openai.com/codex) Co-Authored-By: OpenAI Codex --- .../runtime/regex/JoniRegexPattern.java | 19 +++++++++++++++++- .../runtime/regex/UnicodeResolver.java | 20 +++++++++++++++++++ .../runtime/regex/JoniRegexPatternTest.java | 15 ++++++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index 8fbc14573e..ca8113e6d4 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -50,7 +50,8 @@ final class JoniRegexPattern { Syntax.RUBY.options & ~(Option.ASCII_RANGE | Option.POSIX_BRACKET_ALL_RANGE | Option.WORD_BOUND_ALL_RANGE), Syntax.RUBY.metaCharTable, - JoniRegexPattern::resolveNamedCharacter); + JoniRegexPattern::resolveNamedCharacter, + JoniRegexPattern::resolveCharacterProperty); private static int resolveNamedCharacter(byte[] bytes, int p, int end, Encoding encoding) { @@ -59,6 +60,14 @@ private static int resolveNamedCharacter(byte[] bytes, int p, int end, ? StandardCharsets.ISO_8859_1 : StandardCharsets.UTF_8)); } + private static int[] resolveCharacterProperty(byte[] bytes, int p, int end, + Encoding encoding) { + String property = new String(bytes, p, end - p, + encoding == ISO8859_1Encoding.INSTANCE + ? StandardCharsets.ISO_8859_1 : StandardCharsets.UTF_8); + return UnicodeResolver.resolveJoniPropertyRanges(property); + } + private final Regex regex; private final String sourcePattern; private final Map namedGroups; @@ -222,6 +231,14 @@ private static UserPropertyTranslation translateUserDefinedProperties( boolean frontendProperty = unnegated.matches( "(?i)^(?:script|sc|block|blk|age|in|present[_ ]?in)\\s*(?:=|:(?!:)).*"); boolean perlBuiltInAlias = UnicodeResolver.isPerlBuiltInPropertyAlias(unnegated); + boolean joniResolvedProperty = UnicodeResolver.resolveJoniPropertyRanges( + unnegated) != null; + if (!userDefined && joniResolvedProperty + && (frontendProperty || scriptExtensions || perlBuiltInAlias)) { + translated.append(pattern, i, end + 1); + i = end; + continue; + } if ((frontendProperty || scriptExtensions || perlBuiltInAlias) && extendedClassBracketDepth > 0) { translated.append(pattern, i, end + 1); diff --git a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java index 4810b02fc4..d1505bd9d6 100644 --- a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java +++ b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java @@ -978,6 +978,26 @@ static boolean isPerlBuiltInPropertyAlias(String property) { || resolvePerlBuiltInPropertyAlias(property) != null; } + /** Returns pinned Perl-property ranges in Joni's native range-array format. */ + static int[] resolveJoniPropertyRanges(String property) { + if (property == null) return null; + int assignment = propertyValueDelimiter(property); + if (assignment <= 0 || assignment == property.length() - 1 + || !isGeneralCategoryProperty(property.substring(0, assignment))) { + return null; + } + UnicodeSet set = resolvePerlBuiltInPropertyAlias(property); + if (set == null) return null; + + int[] ranges = new int[set.getRangeCount() * 2 + 1]; + ranges[0] = set.getRangeCount(); + for (int i = 0; i < set.getRangeCount(); i++) { + ranges[i * 2 + 1] = set.getRangeStart(i); + ranges[i * 2 + 2] = set.getRangeEnd(i); + } + return ranges; + } + private static boolean isPerlSpecialPropertyAlias(String property) { return switch (property) { case "lb=cr", "lb=CR", diff --git a/src/test/java/org/perlonjava/runtime/regex/JoniRegexPatternTest.java b/src/test/java/org/perlonjava/runtime/regex/JoniRegexPatternTest.java index d6d5c65fe8..8652867941 100644 --- a/src/test/java/org/perlonjava/runtime/regex/JoniRegexPatternTest.java +++ b/src/test/java/org/perlonjava/runtime/regex/JoniRegexPatternTest.java @@ -100,6 +100,21 @@ void translatesPerlOnlyUnicodePropertyAliasesBeforeJoniCompilation() { .matcher("_", java.util.List.of()).find()); } + @Test + void passesPinnedGeneralCategoriesToTheJoniParserWithoutTextExpansion() { + JoniRegexPattern category = new JoniRegexPattern("\\p{gc=Uppercase_Letter}", FLAGS); + + assertEquals("\\p{gc=Uppercase_Letter}", category.patternDescription()); + assertTrue(category.matcher("A", java.util.List.of()).find()); + assertFalse(category.matcher("a", java.util.List.of()).find()); + JoniRegexPattern folded = new JoniRegexPattern("\\p{gc=Uppercase_Letter}", + RegexFlags.fromModifiers("i", "\\p{gc=Uppercase_Letter}")); + JoniRegexPattern foldedNegated = new JoniRegexPattern("\\P{gc=Uppercase_Letter}", + RegexFlags.fromModifiers("i", "\\P{gc=Uppercase_Letter}")); + assertTrue(folded.matcher("a", java.util.List.of()).find()); + assertFalse(foldedNegated.matcher("A", java.util.List.of()).find()); + } + @Test void flattensTranslatedPropertiesInsideOrdinaryCharacterClasses() { JoniRegexPattern pattern = new JoniRegexPattern( From 560261becad1862278752adebbf6bf58ff7ce9d9 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Tue, 18 Aug 2026 18:55:30 +0200 Subject: [PATCH 09/11] docs(regex): describe native Joni migration boundaries Record native nested lookahead in lookbehind, KEEP routing, and the General_Category range-resolver boundary in the feature matrix. Generated with Codex (https://openai.com/codex) Co-Authored-By: Codex --- docs/reference/feature-matrix.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/reference/feature-matrix.md b/docs/reference/feature-matrix.md index 1691ad9344..c7bf5550e0 100644 --- a/docs/reference/feature-matrix.md +++ b/docs/reference/feature-matrix.md @@ -378,11 +378,11 @@ my @copy = @{$z}; # ERROR - ✅ **Named Capture Groups**: Defining named capture groups using `(?...)` or `(?'name'...)` is supported. - ✅ **Backreferences to Named Groups**: Using `\k` or `\g{name}` for backreferences to named groups is supported. - ✅ **Relative Backreferences**: Using `\g{-n}` for relative backreferences. -- ✅ **Basic Unicode Properties**: Common `\p{...}` and `\P{...}` forms such as `\p{L}` execute through Joni. -- 🟡 **Perl Unicode Property Syntax**: Perl-specific properties execute through Joni. `Age`, cumulative `In`/`Present_In`, General_Category, Canonical_Combining_Class, Bidi_Class, Decomposition_Type, East_Asian_Width, Numeric_Value, Joining_Group, Block, Script, and Script_Extensions assignments are generated from pinned Perl 5.44 Unicode 17.0 data with loose and wildcard aliases, ordered missing defaults, official compact aliases and shortcuts, Script-versus-Script_Extensions policy, reserved or composite values, exact rationals, and Perl's generated decimal keyword aliases; `ASCII_Hex_Digit`/`AHex` accepts Perl's eight boolean value aliases. Other generated property/value aliases still have pinned acceptance/rejection gaps. +- ✅ **Basic Unicode Properties**: Common `\p{...}` and `\P{...}` forms such as `\p{L}` execute through Joni. General_Category assignments now enter the forked Joni parser unchanged and resolve to pinned Perl ranges there. +- 🟡 **Perl Unicode Property Syntax**: Perl-specific properties execute through Joni. `Age`, cumulative `In`/`Present_In`, General_Category, Canonical_Combining_Class, Bidi_Class, Decomposition_Type, East_Asian_Width, Numeric_Value, Joining_Group, Block, Script, and Script_Extensions assignments are generated from pinned Perl 5.44 Unicode 17.0 data with loose and wildcard aliases, ordered missing defaults, official compact aliases and shortcuts, Script-versus-Script_Extensions policy, reserved or composite values, exact rationals, and Perl's generated decimal keyword aliases; `ASCII_Hex_Digit`/`AHex` accepts Perl's eight boolean value aliases. General_Category uses Joni's range-resolver API; the other families remain adapter-translated until their family-specific case-fold and wildcard semantics are represented natively. Other generated property/value aliases still have pinned acceptance/rejection gaps. - ✅ **Possessive Quantifiers**: Quantifiers like `*+`, `++`, `?+`, and `{n,m}+`, which disable backtracking, are supported. - ✅ **Atomic Grouping**: Use of `(?>...)` for atomic groups is supported. -- ✅ **`\K` assertion**: Keep left — in `s///`, text before `\K` is preserved; match variables reflect only the portion after `\K`. +- ✅ **`\K` assertion**: Keep left — in `s///`, text before `\K` is preserved; match variables reflect only the portion after `\K`. Ordinary KEEP assertions route through native Joni and no longer use the Java marker rewrite; the adapter still rejects KEEP inside lookaround until the Joni analyser emits Perl's diagnostic directly. - ✅ **Preprocessor**: `\Q`, `\L`, `\U`, `\l`, `\u`, `\E` are preprocessed in regex. - ✅ **Overloading**: `qr` overloading is implemented. See also [overload pragma](#pragmas). - ✅ **Python-style named groups**: `(?P...)` and `(?P=name)` are parsed natively by Joni with Perl capture numbering, duplicate-name behavior, and malformed/unknown-name diagnostics. @@ -394,7 +394,7 @@ my @copy = @{$z}; # ERROR - ✅ **Backtracking Control Verbs**: `(*ACCEPT)`, `(*FAIL)`/`(*F)`, `(*PRUNE)`, `(*SKIP)`, `(*THEN)`, and `(*COMMIT)` execute through Joni with matcher-owned cut boundaries. Atomic groups `(?>...)` are supported. - ✅ **Marks and named skip targets**: `(*MARK:NAME)` and its `(*:NAME)` shorthand, named `(*SKIP:NAME)`, `$REGMARK`, and `$REGERROR` execute through Joni and follow the selected backtracking path. - ✅ **Regex Definitions**: `(?(DEFINE)...)` containers and numbered or named calls to their subpatterns execute through Joni. -- ✅ **Lookbehind Assertions**: Fixed and bounded variable-length positive and negative lookbehind assertions execute through Joni. +- ✅ **Lookbehind Assertions**: Fixed and bounded variable-length positive and negative lookbehind assertions execute through Joni, including nested lookahead. Removing the remaining compatibility routing requires the complete lookbehind corpus and native diagnostics to pass. - ✅ **Branch Reset Groups**: `(?|...)` resets capture numbering across alternatives and preserves mapped match variables. - ✅ **Advanced Subroutine Calls**: Sub-pattern calls with numbered or named references like `(?1)` and `(?&name)` execute through Joni. - ✅ **Conditional Expressions**: Numbered and named capture conditions, positive and negative assertion conditions, recursion conditions `(?(R))`, `(?(R1))`, and `(?(R&name))`, executable callback conditions, and optimistic predicates execute through Joni. From aae6d0a23c354cd43e7a2d9e4dd4c6d321f898db Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Tue, 18 Aug 2026 19:12:13 +0200 Subject: [PATCH 10/11] fix(regex): reject KEEP inside native Joni lookaround Track Perl lookaround analysis state so the native KEEP anchor reports Perl's exact compile-time diagnostic in all four assertion forms while preserving non-Perl Joni behavior and subexpression-call boundaries. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- third_party/joni/src/org/joni/Analyser.java | 15 +++- .../src/org/joni/exception/ErrorMessages.java | 2 + .../org/joni/test/TestPerlKeepLookaround.java | 82 +++++++++++++++++++ 3 files changed, 95 insertions(+), 4 deletions(-) create mode 100644 third_party/joni/test/org/joni/test/TestPerlKeepLookaround.java diff --git a/third_party/joni/src/org/joni/Analyser.java b/third_party/joni/src/org/joni/Analyser.java index 483c82c673..1e9a8ef1fb 100644 --- a/third_party/joni/src/org/joni/Analyser.java +++ b/third_party/joni/src/org/joni/Analyser.java @@ -1987,6 +1987,7 @@ protected final int setupCombExpCheck(Node node, int state) { private static final int IN_VAR_REPEAT = (1<<3); private static final int IN_CALL = (1<<4); private static final int IN_RECCALL = (1<<5); + private static final int IN_LOOKAROUND = (1<<6); private static final int EXPAND_STRING_MAX_LENGTH = 100; /* setup_tree does the following work. @@ -2189,11 +2190,11 @@ protected final Node setupTree(Node node, int state) { AnchorNode an = (AnchorNode)node; switch (an.type) { case AnchorType.PREC_READ: - setupTree(an.target, state); + setupTree(an.target, (state | IN_LOOKAROUND)); break; case AnchorType.PREC_READ_NOT: - setupTree(an.target, (state | IN_NOT)); + setupTree(an.target, (state | IN_NOT | IN_LOOKAROUND)); break; case AnchorType.LOOK_BEHIND: @@ -2203,7 +2204,7 @@ protected final Node setupTree(Node node, int state) { if (checkTypeTree(an.target, NodeType.ALLOWED_IN_LB, EncloseType.ALLOWED_IN_LB, allowedInLookBehind)) newSyntaxException(INVALID_LOOK_BEHIND_PATTERN); node = setupLookBehind(an); if (node.getType() != NodeType.ANCHOR) continue restart; - setupTree(((AnchorNode)node).target, state); + setupTree(((AnchorNode)node).target, (state | IN_LOOKAROUND)); node = setupLookBehind(an); break; @@ -2214,10 +2215,16 @@ protected final Node setupTree(Node node, int state) { if (checkTypeTree(an.target, NodeType.ALLOWED_IN_LB, EncloseType.ALLOWED_IN_LB_NOT, allowedInNegativeLookBehind)) newSyntaxException(INVALID_LOOK_BEHIND_PATTERN); node = setupLookBehind(an); if (node.getType() != NodeType.ANCHOR) continue restart; - setupTree(((AnchorNode)node).target, (state | IN_NOT)); + setupTree(((AnchorNode)node).target, (state | IN_NOT | IN_LOOKAROUND)); node = setupLookBehind(an); break; + case AnchorType.KEEP: + if (syntax.op2OptionPerl() && (state & IN_LOOKAROUND) != 0) { + newSyntaxException(PERL_KEEP_NOT_PERMITTED_IN_LOOKAROUND); + } + break; + } // inner switch break; } // switch diff --git a/third_party/joni/src/org/joni/exception/ErrorMessages.java b/third_party/joni/src/org/joni/exception/ErrorMessages.java index 4f913ede4a..5d80f246ae 100644 --- a/third_party/joni/src/org/joni/exception/ErrorMessages.java +++ b/third_party/joni/src/org/joni/exception/ErrorMessages.java @@ -55,6 +55,8 @@ public interface ErrorMessages extends org.jcodings.exception.ErrorMessages { String UNDEFINED_GROUP_OPTION = "undefined group option"; String INVALID_POSIX_BRACKET_TYPE = "invalid POSIX bracket type"; String INVALID_LOOK_BEHIND_PATTERN = "invalid pattern in look-behind"; + String PERL_KEEP_NOT_PERMITTED_IN_LOOKAROUND = + "\\K not permitted in lookahead/lookbehind in regex"; String INVALID_REPEAT_RANGE_PATTERN = "invalid repeat range {lower,upper}"; String INVALID_CONDITION_PATTERN = "invalid conditional pattern"; String PERL_GROUP_NAME_MUST_START_WITH_WORD = diff --git a/third_party/joni/test/org/joni/test/TestPerlKeepLookaround.java b/third_party/joni/test/org/joni/test/TestPerlKeepLookaround.java new file mode 100644 index 0000000000..da5052f665 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestPerlKeepLookaround.java @@ -0,0 +1,82 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import static org.joni.constants.SyntaxProperties.OP2_OPTION_PERL; +import static org.joni.constants.SyntaxProperties.OP2_OPTION_RUBY; +import static org.joni.exception.ErrorMessages.PERL_KEEP_NOT_PERMITTED_IN_LOOKAROUND; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.nio.charset.StandardCharsets; + +import org.jcodings.specific.UTF8Encoding; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.joni.exception.SyntaxException; +import org.junit.Test; + +public class TestPerlKeepLookaround { + private static final Syntax PERL_SYNTAX = new Syntax( + "PerlKeepLookaround", Syntax.RUBY.op, + (Syntax.RUBY.op2 & ~OP2_OPTION_RUBY) | OP2_OPTION_PERL, + Syntax.RUBY.op3, Syntax.RUBY.behavior, Syntax.RUBY.options, + Syntax.RUBY.metaCharTable); + + private static Regex compile(String pattern, Syntax syntax) { + byte[] bytes = pattern.getBytes(StandardCharsets.UTF_8); + return new Regex(bytes, 0, bytes.length, Option.NONE, + UTF8Encoding.INSTANCE, syntax); + } + + private static void assertPerlRejected(String pattern) { + try { + compile(pattern, PERL_SYNTAX); + fail("expected syntax error for " + pattern); + } catch (SyntaxException error) { + assertEquals(PERL_KEEP_NOT_PERMITTED_IN_LOOKAROUND, error.getMessage()); + } + } + + @Test + public void rejectsKeepInAllFourLookaroundForms() { + assertPerlRejected("(?=a\\K)"); + assertPerlRejected("(?!a\\K)"); + assertPerlRejected("(?<=a\\K)"); + assertPerlRejected("(?a\\K)(?=\\g)", PERL_SYNTAX); + } + + @Test + public void preservesNonPerlSyntaxBehavior() { + compile("(?=a\\K)", Syntax.RUBY); + } +} From 86e474835bfa8d8a61719ef0ceeafac9e7f73edb Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Tue, 18 Aug 2026 20:05:08 +0200 Subject: [PATCH 11/11] fix(regex): retain branch-reset routing gate Keep automatic branch-reset subroutine-call routing on Java until the native implementation passes its combined imported-corpus gate. Forced Joni mode continues to exercise the new engine support directly. Generated with Codex (https://openai.com/codex) Co-Authored-By: Codex --- .../perlonjava/runtime/regex/JoniRegexPattern.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index ca8113e6d4..a8b8cf0c9d 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -332,6 +332,13 @@ static boolean requiresJoniBackend(String pattern) { static boolean requiresJoniBackend(String pattern, RegexFlags flags) { if (pattern == null) return false; + boolean hasSubroutineCall = pattern.matches("(?s).*\\(\\?[+-]?\\d+\\).*") + || pattern.contains("(?&") + || pattern.contains("(?P>"); + // Keep automatic routing on Java until the native branch-reset call + // implementation passes its combined imported-corpus gate. Explicit + // Joni mode still exercises the native implementation directly. + boolean branchResetCallUsesJava = pattern.contains("(?|") && hasSubroutineCall; return analyzeKeepSyntax(pattern, flags != null && flags.isExtended()).present() || pattern.contains("(?{=CALL:") || pattern.contains("(?{=DYNAMIC:") @@ -347,9 +354,7 @@ static boolean requiresJoniBackend(String pattern, RegexFlags flags) { || pattern.contains("(?(R") || pattern.contains("(?(<") || pattern.contains("(?('") - || pattern.matches("(?s).*\\(\\?[+-]?\\d+\\).*") - || pattern.contains("(?&") - || pattern.contains("(?P>"); + || (hasSubroutineCall && !branchResetCallUsesJava); } private record KeepSyntax(boolean present, boolean inLookaround) {}