From e2ba8237571ed392e64c09e65ffb30ae86f36083 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 20:11:33 +0200 Subject: [PATCH 1/6] fix(regex): preserve Perl nested quantifier semantics Reject ungrouped redundant quantifiers with Perl diagnostics while retaining grouped repeat boundaries and extended-mode quantifier modifiers in the forked Joni lexer and AST. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../JoniNestedQuantifierPatternTest.java | 18 ++++++ .../unit/regex/nested_quantifier_policy.t | 34 +++++++++++ third_party/joni/src/org/joni/Lexer.java | 33 +++++++++- .../joni/src/org/joni/ast/QuantifierNode.java | 12 ++++ .../src/org/joni/exception/ErrorMessages.java | 1 + .../joni/test/TestPerlNestedQuantifier.java | 61 +++++++++++++++++++ 6 files changed, 157 insertions(+), 2 deletions(-) create mode 100644 src/test/java/org/perlonjava/runtime/regex/JoniNestedQuantifierPatternTest.java create mode 100644 src/test/resources/unit/regex/nested_quantifier_policy.t create mode 100644 third_party/joni/test/org/joni/test/TestPerlNestedQuantifier.java diff --git a/src/test/java/org/perlonjava/runtime/regex/JoniNestedQuantifierPatternTest.java b/src/test/java/org/perlonjava/runtime/regex/JoniNestedQuantifierPatternTest.java new file mode 100644 index 0000000000..37c46dbfcf --- /dev/null +++ b/src/test/java/org/perlonjava/runtime/regex/JoniNestedQuantifierPatternTest.java @@ -0,0 +1,18 @@ +package org.perlonjava.runtime.regex; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.joni.exception.SyntaxException; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +@Tag("unit") +class JoniNestedQuantifierPatternTest { + @Test + void rejectsNestedIntervalModifiersBeforeRuntimeWrapping() { + assertThrows(SyntaxException.class, () -> new JoniRegexPattern( + ".{1}??", RegexFlags.fromModifiers("", ".{1}??"))); + assertThrows(SyntaxException.class, () -> new JoniRegexPattern( + ".{1}?+", RegexFlags.fromModifiers("", ".{1}?+"))); + } +} diff --git a/src/test/resources/unit/regex/nested_quantifier_policy.t b/src/test/resources/unit/regex/nested_quantifier_policy.t new file mode 100644 index 0000000000..505fa807ca --- /dev/null +++ b/src/test/resources/unit/regex/nested_quantifier_policy.t @@ -0,0 +1,34 @@ +use strict; +use warnings; +use Test::More; + +for my $pattern ('a**', '.{1}??', '.{1}?+', '(?i:a**)') { + my ($regex, @warnings); + { + local $SIG{__WARN__} = sub { push @warnings, @_ }; + $regex = eval { qr/$pattern/ }; + } + ok(!defined($regex), "$pattern is rejected"); + like($@, qr/^Nested quantifiers/, "$pattern uses Perl nested-quantifier diagnostic"); +} + +for my $case ( + [ 'x(~~)*(?:(?:F)?)?', 'x~~', undef, 1 ], + [ '(?:r?)*?r|(.{2,4})', 'abcde', 'abcd', 1 ], + [ '^(.)(?:(.)+)*[BX]', 'ABCDE', undef, 1 ], + [ '(?x:( a | ( bc ) ) {0,0} ? xyz)', 'xyz', undef, 0 ], + [ '(?x:( a | ( bc ) ) {0,0} + xyz)', 'xyz', undef, 0 ], +) { + my ($pattern, $subject, $capture, $requires_quiet) = @$case; + my (@warnings, $regex); + { + local $SIG{__WARN__} = sub { push @warnings, @_ }; + $regex = eval { qr/$pattern/ }; + } + ok(defined($regex) && $@ eq '' && (!$requires_quiet || !@warnings), + "$pattern has Perl's legal grouped-quantifier compile policy"); + ok($subject =~ $regex, "$pattern retains match semantics"); + is($1, $capture, "$pattern retains capture semantics") if defined($capture); +} + +done_testing; diff --git a/third_party/joni/src/org/joni/Lexer.java b/third_party/joni/src/org/joni/Lexer.java index 0596502fe3..f0599750eb 100644 --- a/third_party/joni/src/org/joni/Lexer.java +++ b/third_party/joni/src/org/joni/Lexer.java @@ -20,6 +20,7 @@ package org.joni; import static org.joni.Option.isAsciiRange; +import static org.joni.Option.isExtend; import static org.joni.Option.isSingleline; import static org.joni.Option.isWordBoundAllRange; import static org.joni.ast.QuantifierNode.isRepeatInfinite; @@ -1272,7 +1273,7 @@ private void fetchTokenFor_openBrace() { greedyCheck(); break; case 2: - if (syntax.fixedIntervalIsGreedyOnly()) { + if (syntax.fixedIntervalIsGreedyOnly() && !syntax.op2OptionPerl()) { possessiveCheck(); } else { greedyCheck(); @@ -2032,18 +2033,20 @@ private boolean fetchTokenForPerlBoundary(boolean negated) { } private void greedyCheck() { + skipPerlExtendedQuantifierSpace(); if (left() && peekIs('?') && syntax.opQMarkNonGreedy()) { - fetch(); token.setRepeatGreedy(false); token.setRepeatPossessive(false); + rejectPerlNestedQuantifierModifier(); } else { possessiveCheck(); } } private void possessiveCheck() { + skipPerlExtendedQuantifierSpace(); if (left() && peekIs('+') && (syntax.op2PlusPossessiveRepeat() && token.type != TokenType.INTERVAL || syntax.op2PlusPossessiveInterval() && token.type == TokenType.INTERVAL)) { @@ -2052,12 +2055,38 @@ private void possessiveCheck() { token.setRepeatGreedy(true); token.setRepeatPossessive(true); + rejectPerlNestedQuantifierModifier(); } else { token.setRepeatGreedy(true); token.setRepeatPossessive(false); } } + private void rejectPerlNestedQuantifierModifier() { + if (!env.usesPerlDiagnostics() || !left()) return; + int next = peek(); + if (next == '?' || next == '*' || next == '+') { + newSyntaxException(PERL_NESTED_QUANTIFIERS); + } + } + + private void skipPerlExtendedQuantifierSpace() { + if (!syntax.op2OptionPerl() || !isExtend(env.option)) return; + while (left()) { + int next = peek(); + if (next == ' ' || next == '\t' || next == '\n' + || next == '\r' || next == '\f') { + inc(); + continue; + } + if (next != '#') return; + while (left()) { + fetch(); + if (enc.isNewLine(c)) break; + } + } + } + protected static final class CharProperty { final int ctype; final int[] ranges; diff --git a/third_party/joni/src/org/joni/ast/QuantifierNode.java b/third_party/joni/src/org/joni/ast/QuantifierNode.java index ed9e80ed82..8b1a655f1f 100644 --- a/third_party/joni/src/org/joni/ast/QuantifierNode.java +++ b/third_party/joni/src/org/joni/ast/QuantifierNode.java @@ -29,6 +29,8 @@ import org.joni.Config; import org.joni.ScanEnvironment; import org.joni.constants.internal.TargetInfo; +import org.joni.exception.ErrorMessages; +import org.joni.exception.ValueException; public final class QuantifierNode extends StateNode { public static final int REPEAT_INFINITE = -1; @@ -229,6 +231,16 @@ public int setQuantifier(Node tgt, boolean group, ScanEnvironment env, byte[]byt /* check redundant double repeat. */ /* verbose warn (?:.?)? etc... but not warn (.?)? etc... */ QuantifierNode qnt = (QuantifierNode)tgt; + if (env.usesPerlDiagnostics()) { + if (!group) { + throw new ValueException(ErrorMessages.PERL_NESTED_QUANTIFIERS); + } + // Parentheses are a semantic boundary in Perl. Keep both + // quantifiers instead of applying Oniguruma's reduction table + // or warning about a legal grouped repeat. + setTarget(qnt); + return 0; + } int nestQNum = popularNum(); int targetQNum = qnt.popularNum(); diff --git a/third_party/joni/src/org/joni/exception/ErrorMessages.java b/third_party/joni/src/org/joni/exception/ErrorMessages.java index 80d5f29abf..9956d6c9bf 100644 --- a/third_party/joni/src/org/joni/exception/ErrorMessages.java +++ b/third_party/joni/src/org/joni/exception/ErrorMessages.java @@ -48,6 +48,7 @@ public interface ErrorMessages extends org.jcodings.exception.ErrorMessages { String UNMATCHED_RANGE_SPECIFIER_IN_CHAR_CLASS = "unmatched range specifier in char-class"; String TARGET_OF_REPEAT_OPERATOR_NOT_SPECIFIED = "target of repeat operator is not specified"; String PERL_QUANTIFIER_FOLLOWS_NOTHING = "Quantifier follows nothing"; + String PERL_NESTED_QUANTIFIERS = "Nested quantifiers"; String TARGET_OF_REPEAT_OPERATOR_INVALID = "target of repeat operator is invalid"; String NESTED_REPEAT_NOT_ALLOWED = "nested repeat is not allowed"; String NESTED_REPEAT_OPERATOR = "nested repeat operator"; diff --git a/third_party/joni/test/org/joni/test/TestPerlNestedQuantifier.java b/third_party/joni/test/org/joni/test/TestPerlNestedQuantifier.java new file mode 100644 index 0000000000..b2cda5d6f0 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestPerlNestedQuantifier.java @@ -0,0 +1,61 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +import java.nio.charset.StandardCharsets; + +import org.jcodings.specific.UTF8Encoding; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.joni.WarnCallback; +import org.joni.exception.SyntaxException; +import org.junit.Test; + +public class TestPerlNestedQuantifier { + private static Regex compile(String pattern) { + byte[] bytes = pattern.getBytes(StandardCharsets.UTF_8); + return new Regex(bytes, 0, bytes.length, Option.CAPTURE_GROUP, + UTF8Encoding.INSTANCE, Syntax.PerlNG, WarnCallback.NONE); + } + + @Test + public void rejectsUngroupedNestedQuantifier() { + assertNestedError("a**"); + assertNestedError(".{1}??"); + assertNestedError(".{1}?+"); + } + + private static void assertNestedError(String pattern) { + SyntaxException error = assertThrows(SyntaxException.class, + () -> compile(pattern)); + assertEquals("Nested quantifiers", error.getMessage()); + } + + @Test + public void preservesGroupedNestedQuantifier() { + byte[] subject = "x~~".getBytes(StandardCharsets.UTF_8); + assertEquals(0, compile("x(~~)*(?:(?:F)?)?").matcher(subject) + .search(0, subject.length, Option.NONE)); + } +} From 15695fda10f2d4314602635c2e30f4320e2c1b32 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 19:58:17 +0200 Subject: [PATCH 2/6] fix(regex): support named accept and fail verbs Carry ACCEPT and explicit FAIL names through the forked Joni parser and bytecode matcher, then publish their Perl control state through REGMARK and REGERROR without conflating explicit FAIL with internal assertion failure. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../runtime/regex/JoniRegexPattern.java | 14 +++- .../runtime/regex/RuntimeRegex.java | 10 ++- .../regex/named_accept_fail_control_verbs.t | 36 ++++++++++ .../joni/src/org/joni/ArrayCompiler.java | 6 +- .../joni/src/org/joni/ByteCodeMachine.java | 12 ++++ third_party/joni/src/org/joni/Parser.java | 6 +- .../org/joni/constants/internal/OPCode.java | 3 + .../org/joni/constants/internal/OPSize.java | 3 +- .../joni/test/TestPerlNamedAcceptFail.java | 68 +++++++++++++++++++ 9 files changed, 146 insertions(+), 12 deletions(-) create mode 100644 src/test/resources/unit/regex/named_accept_fail_control_verbs.t create mode 100644 third_party/joni/test/org/joni/test/TestPerlNamedAcceptFail.java diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index 51ed7f54bb..66065f5979 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -519,6 +519,10 @@ static boolean requiresJoniBackend(String pattern, RegexFlags flags) { || pattern.contains("(?{=DYNAMIC:") || containsNamedCharacterEscape(pattern) || pattern.contains("(*ACCEPT)") + || pattern.contains("(*ACCEPT:") + || pattern.contains("(*FAIL") + || pattern.contains("(*F)") + || pattern.contains("(*F:") || pattern.contains("(*PRUNE") || pattern.contains("(*SKIP") || pattern.contains("(*THEN") @@ -719,6 +723,8 @@ private static PerlSyntaxFeatures analyzePerlSyntax(String pattern, boolean exte private static boolean hasControlVerbState(String pattern) { return pattern.contains("(*MARK") || pattern.contains("(*:") + || pattern.contains("(*ACCEPT") || pattern.contains("(*FAIL") + || pattern.contains("(*F)") || pattern.contains("(*F:") || pattern.contains("(*PRUNE") || pattern.contains("(*SKIP") || pattern.contains("(*THEN") || pattern.contains("(*COMMIT"); @@ -1077,7 +1083,6 @@ private boolean find(int option, boolean anchored) { if (nextStart > regionEnd) { matched = false; committedLastClosedCapture = -1; - if (hasControlVerbState) RuntimeRegex.updateControlVerbVariables(null, null); return false; } matcher = regex.matcher(bytes); @@ -1103,9 +1108,12 @@ private boolean find(int option, boolean anchored) { throw failure; } matched = result >= 0; - if (hasControlVerbState || matcher.hasEncounteredControlVerb()) { + boolean encounteredControlVerb = matcher.hasEncounteredControlVerb(); + if ((matched && hasControlVerbState) || encounteredControlVerb) { + String mark = matcher.getControlMark(); + if (matched && mark == null) mark = "1"; RuntimeRegex.updateControlVerbVariables( - matcher.getControlMark(), matcher.getControlError()); + mark, matcher.getControlError()); } if (calloutHandler != null) calloutHandler.finish(matched); if (!matched) { diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java index 7336379242..fbeeb788a8 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java @@ -158,10 +158,14 @@ static void updateControlVerbVariables(String mark, String error) { ? RuntimeScalarCache.scalarEmptyString : new RuntimeScalar(mark); RuntimeScalar errorValue = error == null ? RuntimeScalarCache.scalarEmptyString : new RuntimeScalar(error); + String currentPackage = InterpreterState.currentPackage.get().toString(); + if (currentPackage == null || currentPackage.isEmpty()) currentPackage = "main"; + GlobalVariable.getGlobalVariable(currentPackage + "::REGMARK").set(markValue); + GlobalVariable.getGlobalVariable(currentPackage + "::REGERROR").set(errorValue); // Perl activates these otherwise ordinary package variables through - // local(). The interpreter does not keep its runtime current-package - // facade synchronized with every lexical package statement, so use the - // localized scalar identities rather than guessing one package name. + // local(). Also update localized scalar identities directly because the + // interpreter does not keep its runtime current-package facade + // synchronized with every lexical package statement. for (Map.Entry entry : DynamicVariableManager.activeLocalizedGlobalScalars().entrySet()) { if (entry.getKey().endsWith("::REGMARK")) { diff --git a/src/test/resources/unit/regex/named_accept_fail_control_verbs.t b/src/test/resources/unit/regex/named_accept_fail_control_verbs.t new file mode 100644 index 0000000000..d7334d677d --- /dev/null +++ b/src/test/resources/unit/regex/named_accept_fail_control_verbs.t @@ -0,0 +1,36 @@ +use strict; +use warnings; +use Test::More; + +our ($REGMARK, $REGERROR); +$REGMARK = undef; +$REGERROR = undef; + +ok('ab' =~ /a(*ACCEPT:accepted)z/, 'named ACCEPT ends the current match'); +is($&, 'a', 'named ACCEPT preserves its match boundary'); +is($REGMARK, 'accepted', 'named ACCEPT publishes its argument'); +is($REGERROR, '', 'named ACCEPT clears REGERROR'); + +ok('ac' !~ /a(*FAIL:blocked)c/, 'named FAIL rejects the match'); +is($REGMARK, '', 'named FAIL clears REGMARK after failure'); +is($REGERROR, 'blocked', 'named FAIL publishes its argument'); + +ok('ac' !~ /a(*F:short)c/, 'named F shorthand rejects the match'); +is($REGERROR, 'short', 'named F publishes its argument'); + +ok('ab' =~ /a(*FAIL:first)b|ab/, 'named FAIL can backtrack to a successful branch'); +is($REGMARK, '1', 'successful controlled match without a mark publishes true'); +is($REGERROR, '', 'success clears a backtracked named FAIL argument'); + +ok('ab' =~ /(?=(a(*ACCEPT:inner)z))ab/, + 'named ACCEPT respects a nested assertion boundary'); +is($1, 'a', 'nested named ACCEPT closes its active capture'); +is($REGMARK, 'inner', 'nested named ACCEPT publishes its argument'); + +$REGMARK = 'sentinel mark'; +$REGERROR = 'sentinel error'; +ok('x' !~ /z|a(*FAIL:unreached)/, 'pattern can fail before reaching named FAIL'); +is($REGMARK, 'sentinel mark', 'unreached control verb leaves REGMARK untouched on failure'); +is($REGERROR, 'sentinel error', 'unreached control verb leaves REGERROR untouched on failure'); + +done_testing; diff --git a/third_party/joni/src/org/joni/ArrayCompiler.java b/third_party/joni/src/org/joni/ArrayCompiler.java index 4a1baaafac..5c5c6f541d 100644 --- a/third_party/joni/src/org/joni/ArrayCompiler.java +++ b/third_party/joni/src/org/joni/ArrayCompiler.java @@ -103,10 +103,12 @@ protected void compileControlVerbNode(ControlVerbNode node) { case ACCEPT: regex.requireStack = true; addOpcode(OPCode.ACCEPT); + addInt(controlVerbLabelId(node.name)); break; case FAIL: regex.requireStack = true; - addOpcode(OPCode.FAIL); + addOpcode(OPCode.CONTROL_FAIL); + addInt(controlVerbLabelId(node.name)); break; case PRUNE: regex.requireStack = true; @@ -1372,7 +1374,7 @@ private int compileLengthTree(Node node) { if (node instanceof ControlVerbNode control) { return switch (control.kind) { case ACCEPT -> OPSize.ACCEPT; - case FAIL -> OPSize.FAIL; + case FAIL -> OPSize.CONTROL_FAIL; case PRUNE -> OPSize.PRUNE; case SKIP -> OPSize.SKIP; case THEN -> OPSize.THEN; diff --git a/third_party/joni/src/org/joni/ByteCodeMachine.java b/third_party/joni/src/org/joni/ByteCodeMachine.java index f38475ab52..463d2dfdd8 100644 --- a/third_party/joni/src/org/joni/ByteCodeMachine.java +++ b/third_party/joni/src/org/joni/ByteCodeMachine.java @@ -389,6 +389,7 @@ private final int execute(final boolean checkThreadInterrupt) throws Interrupted case OPCode.CHECK_LOOK_BEHIND_END: opCheckLookBehindEnd(); continue; case OPCode.FINISH: return finish(); case OPCode.FAIL: opFail(); continue; + case OPCode.CONTROL_FAIL: opControlFail(); continue; case OPCode.CALLOUT: opCallout(); continue; case OPCode.CALLOUT_CONDITION: opCalloutCondition(); continue; case OPCode.DYNAMIC_CALLOUT: opDynamicCallout(); continue; @@ -559,6 +560,7 @@ private final int executeSb(final boolean checkThreadInterrupt) throws Interrupt case OPCode.CHECK_LOOK_BEHIND_END: opCheckLookBehindEnd(); continue; case OPCode.FINISH: return finish(); case OPCode.FAIL: opFail(); continue; + case OPCode.CONTROL_FAIL: opControlFail(); continue; case OPCode.CALLOUT: opCallout(); continue; case OPCode.CALLOUT_CONDITION: opCalloutCondition(); continue; case OPCode.DYNAMIC_CALLOUT: opDynamicCallout(); continue; @@ -2949,6 +2951,9 @@ private void opDynamicCallout() { * are not boundaries; calls and assertions are, matching Perl's behavior. */ private boolean opAccept() { + controlVerbEncountered = true; + String name = controlVerbName(code[ip++]); + if (name != null) controlMark = name; int callDepth = 0; for (int i = stk - 1; i >= 0; i--) { StackEntry entry = stack[i]; @@ -2989,6 +2994,13 @@ private boolean opAccept() { return opEnd(); } + private void opControlFail() { + controlVerbEncountered = true; + String name = controlVerbName(code[ip++]); + controlError = name == null ? "1" : name; + opFail(); + } + private void opPrune() { controlVerbEncountered = true; String name = controlVerbName(code[ip++]); diff --git a/third_party/joni/src/org/joni/Parser.java b/third_party/joni/src/org/joni/Parser.java index 070c8f4d87..6044be6b24 100644 --- a/third_party/joni/src/org/joni/Parser.java +++ b/third_party/joni/src/org/joni/Parser.java @@ -1202,13 +1202,13 @@ private Node parseControlVerb() { final ControlVerbNode.Kind kind; final String verb; - if (startsWith("ACCEPT)")) { + if (startsWith("ACCEPT)") || startsWith("ACCEPT:")) { kind = ControlVerbNode.Kind.ACCEPT; verb = "ACCEPT"; - } else if (startsWith("FAIL)")) { + } else if (startsWith("FAIL)") || startsWith("FAIL:")) { kind = ControlVerbNode.Kind.FAIL; verb = "FAIL"; - } else if (startsWith("F)")) { + } else if (startsWith("F)") || startsWith("F:")) { kind = ControlVerbNode.Kind.FAIL; verb = "F"; } else if (startsWith("PRUNE)") || startsWith("PRUNE:")) { diff --git a/third_party/joni/src/org/joni/constants/internal/OPCode.java b/third_party/joni/src/org/joni/constants/internal/OPCode.java index 3e0b47f503..15d182e88a 100644 --- a/third_party/joni/src/org/joni/constants/internal/OPCode.java +++ b/third_party/joni/src/org/joni/constants/internal/OPCode.java @@ -167,6 +167,7 @@ public interface OPCode { int WIDE_SCALAR = 123; int WIDE_SCALAR_CLASS = 124; int PUSH_BRANCH = 125; /* push a syntactic alternation branch */ + int CONTROL_FAIL = 126; /* explicit Perl (*FAIL[:name]) verb */ String[] OpCodeNames = Config.DEBUG_COMPILE ? new String[] { "finish", /*OP_FINISH*/ @@ -296,6 +297,7 @@ public interface OPCode { "wide-scalar", "wide-scalar-class", "push-branch", + "control-fail", } : null; int[] OpCodeArgTypes = Config.DEBUG_COMPILE ? new int[] { @@ -426,5 +428,6 @@ public interface OPCode { Arguments.SPECIAL, /*OP_WIDE_SCALAR*/ Arguments.MEMNUM, /*OP_WIDE_SCALAR_CLASS*/ Arguments.RELADDR, /*OP_PUSH_BRANCH*/ + Arguments.MEMNUM, /*OP_CONTROL_FAIL*/ } : null; } diff --git a/third_party/joni/src/org/joni/constants/internal/OPSize.java b/third_party/joni/src/org/joni/constants/internal/OPSize.java index ba1de3dc77..3b17b0e9a7 100644 --- a/third_party/joni/src/org/joni/constants/internal/OPSize.java +++ b/third_party/joni/src/org/joni/constants/internal/OPSize.java @@ -74,7 +74,8 @@ public interface OPSize { int CALLOUT = (OPCODE + MEMNUM); int CALLOUT_CONDITION = (OPCODE + MEMNUM + RELADDR); int DYNAMIC_CALLOUT = (OPCODE + MEMNUM); - int ACCEPT = OPCODE; + int ACCEPT = (OPCODE + INDEX); + int CONTROL_FAIL = (OPCODE + INDEX); int PRUNE = (OPCODE + INDEX); int SKIP = (OPCODE + INDEX); int THEN = (OPCODE + INDEX); diff --git a/third_party/joni/test/org/joni/test/TestPerlNamedAcceptFail.java b/third_party/joni/test/org/joni/test/TestPerlNamedAcceptFail.java new file mode 100644 index 0000000000..d7f1bcf754 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestPerlNamedAcceptFail.java @@ -0,0 +1,68 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import static org.junit.Assert.assertEquals; + +import java.nio.charset.StandardCharsets; + +import org.jcodings.specific.ASCIIEncoding; +import org.joni.Matcher; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.junit.Test; + +public class TestPerlNamedAcceptFail { + private static Matcher matcher(String pattern, String input) { + byte[] patternBytes = pattern.getBytes(StandardCharsets.US_ASCII); + byte[] inputBytes = input.getBytes(StandardCharsets.US_ASCII); + Regex regex = new Regex(patternBytes, 0, patternBytes.length, Option.NONE, + ASCIIEncoding.INSTANCE, Syntax.RUBY); + return regex.matcher(inputBytes); + } + + @Test + public void namedAcceptPublishesItsName() { + Matcher matcher = matcher("a(*ACCEPT:accepted)z", "ab"); + + assertEquals(0, matcher.search(0, 2, Option.NONE)); + assertEquals(1, matcher.getEnd()); + assertEquals("accepted", matcher.getControlMark()); + assertEquals(null, matcher.getControlError()); + } + + @Test + public void namedFailPublishesItsName() { + Matcher matcher = matcher("a(*FAIL:blocked)c", "ac"); + + assertEquals(-1, matcher.search(0, 2, Option.NONE)); + assertEquals(null, matcher.getControlMark()); + assertEquals("blocked", matcher.getControlError()); + } + + @Test + public void namedFailShorthandPublishesItsName() { + Matcher matcher = matcher("a(*F:short)c", "ac"); + + assertEquals(-1, matcher.search(0, 2, Option.NONE)); + assertEquals("short", matcher.getControlError()); + } +} From 2a581001cfbeca0421f466fc3d49f80d666e845f Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 20:13:31 +0200 Subject: [PATCH 3/6] docs(regex): refresh phase 36 execution position Record the integrated native nested-quantifier and named control-verb state and make the next ordered gate describe only forward work. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- dev/design/phase36-regex-parity.md | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index 9beb329587..bbfb4ffb7a 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -92,13 +92,10 @@ affected corpus before taking another slice. - `master` contains the validated named-character diagnostics plus native `(?(DEFINE)...)`, ordinary lookbehind, branch reset, and plain `\N`; its exact head passed warning-free local, Ubuntu, and Windows gates. -- The successor integration batch carries byte/Unicode provenance and fold - policy, dotted-U+ diagnostics, the ordinary-pattern Joni default, a dynamic- - pattern edge contract, and the first obsolete import retirement. Its fold, - property, and resolver-cache residuals are closed; the exact semantic head - passes a warning-free 17-task `make`. The prospective PR head also includes - three independent runtime-diagnostic corrections and passes the same full - combined gate. +- Byte/Unicode provenance and fold policy, dotted-U+ diagnostics, the ordinary- + pattern Joni default, a dynamic-pattern edge contract, and the first obsolete + import retirement are integrated. Fold, property, resolver-cache, and the + classified runtime-diagnostic residuals are closed. - Native DEFINE, ordinary lookbehind, and branch reset now route through Joni; their feature-specific Java rewrites and branch-reset capture-map adapter are deleted. Plain Perl `\N` is a native Joni non-line-feed atom, including @@ -118,6 +115,13 @@ affected corpus before taking another slice. evidence. - Exact `/aa` routing/folding gates pass on native Joni, and the Java `/aa` workaround is removed. +- Perl grouped nested-quantifier semantics and extended-mode quantifier + modifiers are native Joni behavior; the exact `regexp.t` differential removes + seven failures with no introductions. +- Named `(*ACCEPT:NAME)`, `(*FAIL:NAME)`, and `(*F:NAME)` carry control state + through native Joni bytecode and publish Perl-compatible `$REGMARK` and + `$REGERROR`; the exact `regexp.t` differential removes three failures with no + introductions. ## Execution Phases @@ -229,8 +233,9 @@ behavior. ## Ordered Next Steps -1. Open the validated successor review PR against `master` and require - exact-head Ubuntu/Windows CI. +1. Run one warning-free full build and affected-corpus differential on the + integrated nested-quantifier and named-control-verb batch, then open its + review PR and require exact-head Ubuntu/Windows CI. 2. Complete byte/Unicode pattern provenance through runtime interpolation and template composition, then finish `/d`/`u`/`a`/`aa` forward/reverse literal and backreference folding from generated data. Require direct Joni plus @@ -240,8 +245,8 @@ behavior. unwind, backtracking re-evaluation, and recursion safety. Route every embedded closure to Joni and delete constant inlining, progressive errors, and the dynamic Java adapter as their gates pass. -4. Carry lexical `use re 'strict'` policy through regex compilation and close - the unescaped-brace/non-hex diagnostic families. Refresh complete +4. Finish the remaining lexical `use re 'strict'`, unescaped-brace, and non-hex + diagnostic families. Refresh complete `reg_mesg.t`, `pat.t`, and `pat_advanced.t` maps after each combined batch. 5. Repeat the four-leg 80-file Java/Joni × JVM/interpreter matrix on the exact successor artifact and compare every file with the PR 958 log. Resolve every From ef92b5eba87cefb657be7e6ea6acc3a6a2f558be Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 20:28:03 +0200 Subject: [PATCH 4/6] fix(regex): preserve named control errors before fail Keep an earlier named cut-verb error authoritative when a path terminates with an unnamed FAIL, while retaining named and standalone FAIL publication. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../joni/src/org/joni/ByteCodeMachine.java | 7 ++- .../test/TestPerlControlFailPrecedence.java | 55 +++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 third_party/joni/test/org/joni/test/TestPerlControlFailPrecedence.java diff --git a/third_party/joni/src/org/joni/ByteCodeMachine.java b/third_party/joni/src/org/joni/ByteCodeMachine.java index 463d2dfdd8..8219c88f54 100644 --- a/third_party/joni/src/org/joni/ByteCodeMachine.java +++ b/third_party/joni/src/org/joni/ByteCodeMachine.java @@ -2997,7 +2997,12 @@ private boolean opAccept() { private void opControlFail() { controlVerbEncountered = true; String name = controlVerbName(code[ip++]); - controlError = name == null ? "1" : name; + // An unnamed FAIL terminates the current path without replacing a + // more specific PRUNE/SKIP/THEN/COMMIT error already encountered on + // that path. A named FAIL remains authoritative. + if (name != null || controlError == null) { + controlError = name == null ? "1" : name; + } opFail(); } diff --git a/third_party/joni/test/org/joni/test/TestPerlControlFailPrecedence.java b/third_party/joni/test/org/joni/test/TestPerlControlFailPrecedence.java new file mode 100644 index 0000000000..6dc8491904 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestPerlControlFailPrecedence.java @@ -0,0 +1,55 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import static org.junit.Assert.assertEquals; + +import java.nio.charset.StandardCharsets; + +import org.jcodings.specific.ASCIIEncoding; +import org.joni.Matcher; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.junit.Test; + +public class TestPerlControlFailPrecedence { + private static Matcher match(String pattern, String input) { + byte[] patternBytes = pattern.getBytes(StandardCharsets.US_ASCII); + byte[] inputBytes = input.getBytes(StandardCharsets.US_ASCII); + Regex regex = new Regex(patternBytes, 0, patternBytes.length, Option.NONE, + ASCIIEncoding.INSTANCE, Syntax.PerlNG); + Matcher matcher = regex.matcher(inputBytes); + assertEquals(-1, matcher.search(0, inputBytes.length, Option.NONE)); + return matcher; + } + + @Test + public void unnamedFailPreservesEarlierNamedCutError() { + assertEquals("blocked", + match("a(*PRUNE:blocked)(*FAIL)", "a").getControlError()); + } + + @Test + public void failStillPublishesItsOwnError() { + assertEquals("1", match("a(*FAIL)", "a").getControlError()); + assertEquals("named", match("a(*FAIL:named)", "a").getControlError()); + } +} From fb3aab77643e5a7a00f1498fb3eaba405984fb13 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 20:34:08 +0200 Subject: [PATCH 5/6] fix(regex): support captures in negative lookbehind Permit Perl capture groups in negative lookbehind and apply ACCEPT-aware width analysis to both lookbehind polarities so native variable-width execution keeps the correct assertion boundary. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../negative_lookbehind_capture_accept.t | 18 ++++++ third_party/joni/src/org/joni/Analyser.java | 8 ++- .../TestPerlNegativeLookBehindCapture.java | 57 +++++++++++++++++++ 3 files changed, 80 insertions(+), 3 deletions(-) create mode 100644 src/test/resources/unit/regex/negative_lookbehind_capture_accept.t create mode 100644 third_party/joni/test/org/joni/test/TestPerlNegativeLookBehindCapture.java diff --git a/src/test/resources/unit/regex/negative_lookbehind_capture_accept.t b/src/test/resources/unit/regex/negative_lookbehind_capture_accept.t new file mode 100644 index 0000000000..c4bdb2dd62 --- /dev/null +++ b/src/test/resources/unit/regex/negative_lookbehind_capture_accept.t @@ -0,0 +1,18 @@ +use strict; +use warnings; +use Test::More; + +my $accept = qr/(? Date: Wed, 19 Aug 2026 20:35:35 +0200 Subject: [PATCH 6/6] docs(regex): include negative lookbehind gate Keep the Phase 36 execution position and immediate combined acceptance gate aligned with the integrated native lookbehind work. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- dev/design/phase36-regex-parity.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index bbfb4ffb7a..752169ff24 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -122,6 +122,9 @@ affected corpus before taking another slice. through native Joni bytecode and publish Perl-compatible `$REGMARK` and `$REGERROR`; the exact `regexp.t` differential removes three failures with no introductions. +- Negative lookbehind accepts capture enclosures and uses ACCEPT-aware width + analysis in native Joni; named cut errors remain authoritative before an + unnamed FAIL. The combined exact `regexp.t` differential has no introductions. ## Execution Phases @@ -234,8 +237,8 @@ behavior. ## Ordered Next Steps 1. Run one warning-free full build and affected-corpus differential on the - integrated nested-quantifier and named-control-verb batch, then open its - review PR and require exact-head Ubuntu/Windows CI. + integrated nested-quantifier, named-control-verb, and negative-lookbehind + batch, then open its review PR and require exact-head Ubuntu/Windows CI. 2. Complete byte/Unicode pattern provenance through runtime interpolation and template composition, then finish `/d`/`u`/`a`/`aa` forward/reverse literal and backreference folding from generated data. Require direct Joni plus