From c281827f7a479c5e597233cebf095c4447b80b52 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 21:23:50 +0200 Subject: [PATCH 01/11] fix(regex): report unterminated Perl option groups Emit Perl's native sequence diagnostic when inline option modifiers end before a closing parenthesis or scoped-group colon. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../unterminated_option_group_diagnostics.t | 10 ++++ third_party/joni/src/org/joni/Parser.java | 6 ++- .../src/org/joni/exception/ErrorMessages.java | 1 + .../test/TestPerlOptionGroupDiagnostics.java | 53 +++++++++++++++++++ 4 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 src/test/resources/unit/regex/unterminated_option_group_diagnostics.t create mode 100644 third_party/joni/test/org/joni/test/TestPerlOptionGroupDiagnostics.java diff --git a/src/test/resources/unit/regex/unterminated_option_group_diagnostics.t b/src/test/resources/unit/regex/unterminated_option_group_diagnostics.t new file mode 100644 index 000000000..72ea7917b --- /dev/null +++ b/src/test/resources/unit/regex/unterminated_option_group_diagnostics.t @@ -0,0 +1,10 @@ +use strict; +use warnings; +use Test::More tests => 6; + +for my $pattern ('(?i', '(?a-x', '(?-') { + my $ok = eval "qr/$pattern/; 1"; + ok(!$ok, "$pattern is rejected"); + ok(index($@, 'Sequence (?... not terminated') >= 0, + "$pattern reports an unterminated option sequence"); +} diff --git a/third_party/joni/src/org/joni/Parser.java b/third_party/joni/src/org/joni/Parser.java index 0ee1339eb..69e0621cc 100644 --- a/third_party/joni/src/org/joni/Parser.java +++ b/third_party/joni/src/org/joni/Parser.java @@ -1145,7 +1145,11 @@ && left() && enc.isDigit(peek())) { returnCode = 0; return node; } - if (!left()) newSyntaxException(END_PATTERN_IN_GROUP); + if (!left()) { + newSyntaxException(syntax.op2OptionPerl() + ? PERL_OPTION_GROUP_NOT_TERMINATED + : END_PATTERN_IN_GROUP); + } fetch(); } // while diff --git a/third_party/joni/src/org/joni/exception/ErrorMessages.java b/third_party/joni/src/org/joni/exception/ErrorMessages.java index a9833c1f9..df27e242f 100644 --- a/third_party/joni/src/org/joni/exception/ErrorMessages.java +++ b/third_party/joni/src/org/joni/exception/ErrorMessages.java @@ -55,6 +55,7 @@ public interface ErrorMessages extends org.jcodings.exception.ErrorMessages { String UNMATCHED_CLOSE_PARENTHESIS = "unmatched close parenthesis"; String END_PATTERN_WITH_UNMATCHED_PARENTHESIS = "end pattern with unmatched parenthesis"; String END_PATTERN_IN_GROUP = "end pattern in group"; + String PERL_OPTION_GROUP_NOT_TERMINATED = "Sequence (?... not terminated"; String UNDEFINED_GROUP_OPTION = "undefined group option"; String INVALID_POSIX_BRACKET_TYPE = "invalid POSIX bracket type"; String INVALID_LOOK_BEHIND_PATTERN = "invalid pattern in look-behind"; diff --git a/third_party/joni/test/org/joni/test/TestPerlOptionGroupDiagnostics.java b/third_party/joni/test/org/joni/test/TestPerlOptionGroupDiagnostics.java new file mode 100644 index 000000000..1c9093cdb --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestPerlOptionGroupDiagnostics.java @@ -0,0 +1,53 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.nio.charset.StandardCharsets; + +import org.jcodings.specific.UTF8Encoding; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.joni.WarnCallback; +import org.joni.exception.SyntaxException; +import org.junit.Test; + +public class TestPerlOptionGroupDiagnostics { + private static void assertUnterminated(String pattern) { + byte[] bytes = pattern.getBytes(StandardCharsets.UTF_8); + try { + new Regex(bytes, 0, bytes.length, Option.NONE, UTF8Encoding.INSTANCE, + Syntax.PerlNG, WarnCallback.NONE); + fail("expected unterminated Perl option group for " + pattern); + } catch (SyntaxException error) { + assertEquals("Sequence (?... not terminated", error.getMessage()); + } + } + + @Test + public void reportsUnterminatedOptionGroups() { + assertUnterminated("(?i"); + assertUnterminated("(?a-x"); + assertUnterminated("(?-"); + } +} From 7689a47697af1736eef390b7acb1dad0a709547c Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 21:27:43 +0200 Subject: [PATCH 02/11] fix(regex): report unterminated Perl comment groups Emit Perl's native sequence diagnostic when a regex comment group reaches the end of the pattern without its closing parenthesis. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../unterminated_comment_group_diagnostics.t | 10 ++++ third_party/joni/src/org/joni/Lexer.java | 6 ++- .../src/org/joni/exception/ErrorMessages.java | 1 + .../test/TestPerlCommentGroupDiagnostics.java | 53 +++++++++++++++++++ 4 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 src/test/resources/unit/regex/unterminated_comment_group_diagnostics.t create mode 100644 third_party/joni/test/org/joni/test/TestPerlCommentGroupDiagnostics.java diff --git a/src/test/resources/unit/regex/unterminated_comment_group_diagnostics.t b/src/test/resources/unit/regex/unterminated_comment_group_diagnostics.t new file mode 100644 index 000000000..b0b365d6a --- /dev/null +++ b/src/test/resources/unit/regex/unterminated_comment_group_diagnostics.t @@ -0,0 +1,10 @@ +use strict; +use warnings; +use Test::More tests => 6; + +for my $pattern ('x(?#', 'x(?#:', '(?#abc') { + my $ok = eval "qr/$pattern/; 1"; + ok(!$ok, "$pattern is rejected"); + ok(index($@, 'Sequence (?#... not terminated') >= 0, + "$pattern reports an unterminated comment sequence"); +} diff --git a/third_party/joni/src/org/joni/Lexer.java b/third_party/joni/src/org/joni/Lexer.java index 033a38a62..434470c3d 100644 --- a/third_party/joni/src/org/joni/Lexer.java +++ b/third_party/joni/src/org/joni/Lexer.java @@ -1928,7 +1928,11 @@ protected final void fetchToken() { if (peekIs('#')) { fetch(); while (true) { - if (!left()) newSyntaxException(END_PATTERN_IN_GROUP); + if (!left()) { + newSyntaxException(syntax.op2OptionPerl() + ? PERL_COMMENT_GROUP_NOT_TERMINATED + : END_PATTERN_IN_GROUP); + } fetch(); if (c == syntax.metaCharTable.esc) { if (left()) fetch(); diff --git a/third_party/joni/src/org/joni/exception/ErrorMessages.java b/third_party/joni/src/org/joni/exception/ErrorMessages.java index df27e242f..6eb10154b 100644 --- a/third_party/joni/src/org/joni/exception/ErrorMessages.java +++ b/third_party/joni/src/org/joni/exception/ErrorMessages.java @@ -56,6 +56,7 @@ public interface ErrorMessages extends org.jcodings.exception.ErrorMessages { String END_PATTERN_WITH_UNMATCHED_PARENTHESIS = "end pattern with unmatched parenthesis"; String END_PATTERN_IN_GROUP = "end pattern in group"; String PERL_OPTION_GROUP_NOT_TERMINATED = "Sequence (?... not terminated"; + String PERL_COMMENT_GROUP_NOT_TERMINATED = "Sequence (?#... not terminated"; String UNDEFINED_GROUP_OPTION = "undefined group option"; String INVALID_POSIX_BRACKET_TYPE = "invalid POSIX bracket type"; String INVALID_LOOK_BEHIND_PATTERN = "invalid pattern in look-behind"; diff --git a/third_party/joni/test/org/joni/test/TestPerlCommentGroupDiagnostics.java b/third_party/joni/test/org/joni/test/TestPerlCommentGroupDiagnostics.java new file mode 100644 index 000000000..62c2dc317 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestPerlCommentGroupDiagnostics.java @@ -0,0 +1,53 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.nio.charset.StandardCharsets; + +import org.jcodings.specific.UTF8Encoding; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.joni.WarnCallback; +import org.joni.exception.SyntaxException; +import org.junit.Test; + +public class TestPerlCommentGroupDiagnostics { + private static void assertUnterminated(String pattern) { + byte[] bytes = pattern.getBytes(StandardCharsets.UTF_8); + try { + new Regex(bytes, 0, bytes.length, Option.NONE, UTF8Encoding.INSTANCE, + Syntax.PerlNG, WarnCallback.NONE); + fail("expected unterminated Perl comment group for " + pattern); + } catch (SyntaxException error) { + assertEquals("Sequence (?#... not terminated", error.getMessage()); + } + } + + @Test + public void reportsUnterminatedCommentGroups() { + assertUnterminated("x(?#"); + assertUnterminated("x(?#:"); + assertUnterminated("(?#abc"); + } +} From c2830d09cb50dfdad3f46b081c6ea8a69505066e Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 21:32:53 +0200 Subject: [PATCH 03/11] fix(regex): report incomplete Perl group effects Emit Perl's native incomplete-sequence diagnostic when a pattern ends immediately after the group-effect prefix. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../incomplete_group_effect_diagnostics.t | 10 ++++ third_party/joni/src/org/joni/Parser.java | 6 ++- .../src/org/joni/exception/ErrorMessages.java | 1 + .../test/TestPerlGroupEffectDiagnostics.java | 52 +++++++++++++++++++ 4 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 src/test/resources/unit/regex/incomplete_group_effect_diagnostics.t create mode 100644 third_party/joni/test/org/joni/test/TestPerlGroupEffectDiagnostics.java diff --git a/src/test/resources/unit/regex/incomplete_group_effect_diagnostics.t b/src/test/resources/unit/regex/incomplete_group_effect_diagnostics.t new file mode 100644 index 000000000..146ff5848 --- /dev/null +++ b/src/test/resources/unit/regex/incomplete_group_effect_diagnostics.t @@ -0,0 +1,10 @@ +use strict; +use warnings; +use Test::More tests => 4; + +for my $pattern ('(?', 'a(?') { + my $ok = eval "qr/$pattern/; 1"; + ok(!$ok, "$pattern is rejected"); + ok(index($@, 'Sequence (? incomplete') >= 0, + "$pattern reports an incomplete group effect"); +} diff --git a/third_party/joni/src/org/joni/Parser.java b/third_party/joni/src/org/joni/Parser.java index 69e0621cc..c6a9514e1 100644 --- a/third_party/joni/src/org/joni/Parser.java +++ b/third_party/joni/src/org/joni/Parser.java @@ -738,7 +738,11 @@ private Node parseEnclose(TokenType term) { if (peekIs('?') && syntax.op2QMarkGroupEffect()) { inc(); - if (!left()) newSyntaxException(END_PATTERN_IN_GROUP); + if (!left()) { + newSyntaxException(syntax.op2OptionPerl() + ? PERL_GROUP_EFFECT_INCOMPLETE + : END_PATTERN_IN_GROUP); + } boolean listCapture = false; diff --git a/third_party/joni/src/org/joni/exception/ErrorMessages.java b/third_party/joni/src/org/joni/exception/ErrorMessages.java index 6eb10154b..6a0cfab3a 100644 --- a/third_party/joni/src/org/joni/exception/ErrorMessages.java +++ b/third_party/joni/src/org/joni/exception/ErrorMessages.java @@ -57,6 +57,7 @@ public interface ErrorMessages extends org.jcodings.exception.ErrorMessages { String END_PATTERN_IN_GROUP = "end pattern in group"; String PERL_OPTION_GROUP_NOT_TERMINATED = "Sequence (?... not terminated"; String PERL_COMMENT_GROUP_NOT_TERMINATED = "Sequence (?#... not terminated"; + String PERL_GROUP_EFFECT_INCOMPLETE = "Sequence (? incomplete"; String UNDEFINED_GROUP_OPTION = "undefined group option"; String INVALID_POSIX_BRACKET_TYPE = "invalid POSIX bracket type"; String INVALID_LOOK_BEHIND_PATTERN = "invalid pattern in look-behind"; diff --git a/third_party/joni/test/org/joni/test/TestPerlGroupEffectDiagnostics.java b/third_party/joni/test/org/joni/test/TestPerlGroupEffectDiagnostics.java new file mode 100644 index 000000000..bcd680a30 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestPerlGroupEffectDiagnostics.java @@ -0,0 +1,52 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.nio.charset.StandardCharsets; + +import org.jcodings.specific.UTF8Encoding; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.joni.WarnCallback; +import org.joni.exception.SyntaxException; +import org.junit.Test; + +public class TestPerlGroupEffectDiagnostics { + private static void assertIncomplete(String pattern) { + byte[] bytes = pattern.getBytes(StandardCharsets.UTF_8); + try { + new Regex(bytes, 0, bytes.length, Option.NONE, UTF8Encoding.INSTANCE, + Syntax.PerlNG, WarnCallback.NONE); + fail("expected incomplete Perl group effect for " + pattern); + } catch (SyntaxException error) { + assertEquals("Sequence (? incomplete", error.getMessage()); + } + } + + @Test + public void reportsIncompleteGroupEffects() { + assertIncomplete("(?"); + assertIncomplete("a(?"); + } +} From c3ffa5148c0bbf0183dacd72dd7072d6dc65e2af Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 21:40:49 +0200 Subject: [PATCH 04/11] fix(regex): classify empty Perl control verbs Treat an empty star group as an unknown verb pattern, matching Perl's native diagnostic instead of reporting an unknown control construct. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../regex/empty_control_verb_diagnostics.t | 10 ++++ third_party/joni/src/org/joni/Parser.java | 3 +- .../test/TestPerlEmptyVerbDiagnostics.java | 52 +++++++++++++++++++ 3 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 src/test/resources/unit/regex/empty_control_verb_diagnostics.t create mode 100644 third_party/joni/test/org/joni/test/TestPerlEmptyVerbDiagnostics.java diff --git a/src/test/resources/unit/regex/empty_control_verb_diagnostics.t b/src/test/resources/unit/regex/empty_control_verb_diagnostics.t new file mode 100644 index 000000000..975563409 --- /dev/null +++ b/src/test/resources/unit/regex/empty_control_verb_diagnostics.t @@ -0,0 +1,10 @@ +use strict; +use warnings; +use Test::More tests => 4; + +for my $pattern ('(*)', '(*)b') { + my $ok = eval "qr/$pattern/; 1"; + ok(!$ok, "$pattern is rejected"); + ok(index($@, "Unknown verb pattern ''") >= 0, + "$pattern reports an empty verb pattern"); +} diff --git a/third_party/joni/src/org/joni/Parser.java b/third_party/joni/src/org/joni/Parser.java index c6a9514e1..5bc165739 100644 --- a/third_party/joni/src/org/joni/Parser.java +++ b/third_party/joni/src/org/joni/Parser.java @@ -1246,7 +1246,8 @@ private Node parseControlVerb() { verb = "MARK"; } else { String construct = controlConstructName(); - if (!construct.isEmpty() && Character.isUpperCase(construct.codePointAt(0))) { + if (construct.isEmpty() + || Character.isUpperCase(construct.codePointAt(0))) { newValueException(PERL_UNKNOWN_VERB_PATTERN, construct); } newValueException(PERL_UNKNOWN_CONTROL_CONSTRUCT, construct); diff --git a/third_party/joni/test/org/joni/test/TestPerlEmptyVerbDiagnostics.java b/third_party/joni/test/org/joni/test/TestPerlEmptyVerbDiagnostics.java new file mode 100644 index 000000000..f213d0a1b --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestPerlEmptyVerbDiagnostics.java @@ -0,0 +1,52 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.nio.charset.StandardCharsets; + +import org.jcodings.specific.UTF8Encoding; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.joni.WarnCallback; +import org.joni.exception.ValueException; +import org.junit.Test; + +public class TestPerlEmptyVerbDiagnostics { + private static void assertEmptyVerb(String pattern) { + byte[] bytes = pattern.getBytes(StandardCharsets.UTF_8); + try { + new Regex(bytes, 0, bytes.length, Option.NONE, UTF8Encoding.INSTANCE, + Syntax.PerlNG, WarnCallback.NONE); + fail("expected empty Perl verb failure for " + pattern); + } catch (ValueException error) { + assertEquals("Unknown verb pattern ''", error.getMessage()); + } + } + + @Test + public void reportsEmptyVerbPatterns() { + assertEmptyVerb("(*)"); + assertEmptyVerb("(*)b"); + } +} From 63bca117d8fbaa8d21c03259d850baee46a61818 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 21:44:01 +0200 Subject: [PATCH 05/11] docs(regex): record native diagnostic tranche Update the Phase 36 current-position summary after the focused Perl, direct-Joni, runtime, and imported-regexp gates passed. Generated with [Codex](https://openai.com/codex/) Co-Authored-By: Codex --- dev/design/phase36-regex-parity.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index 0e93bf785..0684909bc 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -136,6 +136,10 @@ affected corpus before taking another slice. before loose alias normalization, so uncased letters no longer enter that class. The focused system-Perl oracle, four runtime legs, and imported `regexp.t` identity agree. +- Native Joni diagnostics distinguish invalid non-braced `\p`/`\P` followers, + unterminated inline-option and comment groups, incomplete `(?` group effects, + and empty control verbs. The focused system-Perl/direct-Joni/four-leg gates + remove the corresponding `regexp.t` identities with no introductions. ## Execution Phases From 6df06bcd3c227ddf17aed3bace499085c44da7b4 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 21:09:58 +0200 Subject: [PATCH 06/11] fix(regex): retire dynamic pattern fallback adapter Route runtime (??{...}) execution exclusively through the structured native Joni callout path. Remove the obsolete marker, deferred warning/error cache split, and constant-inlining fallback while retaining syntax-only validation for raw executable source. Add a system-Perl contract and direct Joni coverage for nested alternatives, captures, modifiers, callback state, modes, recursion, and commit ordering. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../runtime/regex/RegexMarkers.java | 16 +-- .../runtime/regex/RegexPreprocessor.java | 48 +------- .../runtime/regex/RuntimeRegex.java | 35 +----- .../regex/native_dynamic_pattern_contract.t | 72 +++++++++++ .../joni/test/TestPerlDynamicExecution.java | 112 ++++++++++++++++++ 5 files changed, 192 insertions(+), 91 deletions(-) create mode 100644 src/test/resources/unit/regex/native_dynamic_pattern_contract.t create mode 100644 third_party/joni/test/org/joni/test/TestPerlDynamicExecution.java diff --git a/src/main/java/org/perlonjava/runtime/regex/RegexMarkers.java b/src/main/java/org/perlonjava/runtime/regex/RegexMarkers.java index 6f61e8793..a279578c4 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RegexMarkers.java +++ b/src/main/java/org/perlonjava/runtime/regex/RegexMarkers.java @@ -4,8 +4,7 @@ * Shared placeholder markers used by the string-interpolation parser to * stand in for regex constructs that PerlOnJava cannot compile literally * (because they require features unsupported by the underlying Java regex - * engine — e.g. arbitrary {@code (?{ CODE })} code blocks and - * {@code (??{ CODE })} recursive/dynamic patterns). + * engine — e.g. arbitrary {@code (?{ CODE })} code blocks). * *

The markers are emitted by {@code StringSegmentParser} when a code * block can't be constant-folded. {@link RegexPreprocessor} detects them @@ -15,12 +14,6 @@ * or a no-op fallback only when {@link #CODE_BLOCK_NOOP_ENV} is set. * Plain {@code JPERL_UNIMPLEMENTED=warn} still reports the unsupported * feature without pretending the callback ran. - *

  • {@link #RECURSIVE_PATTERN} — a hard error under default die mode, - * or a warning under {@code JPERL_UNIMPLEMENTED=warn} followed by - * the soft {@code (?:} fallback so the surrounding pattern still - * compiles (many CPAN modules build dynamic patterns that happen - * to work with the empty-group fallback; under warn mode we want - * tests to continue but the user must see a diagnostic).
    • * * *

    Why these specific spellings? The preprocessor performs some @@ -50,12 +43,5 @@ public final class RegexMarkers { */ public static final String CODE_BLOCK = "(?{UNIMPLEMENTED_CODE_BLOC})"; - /** - * Marker for a {@code (??{ CODE })} recursive/dynamic pattern that - * could not be constant-folded at parse time. Contains no - * fold-affected letters. - */ - public static final String RECURSIVE_PATTERN = "(??{UNIMPLEMENTED_RECURSIVE_PATTERN})"; - private RegexMarkers() {} } diff --git a/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessor.java b/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessor.java index 2d7cfd83c..8dabd0f75 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessor.java +++ b/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessor.java @@ -1041,28 +1041,10 @@ private static int handleParentheses(String s, int offset, int length, StringBui offset = handleCodeBlock(s, offset, length, sb, regexFlags); } } else if (c3 == '?' && c4 == '{') { - // Check if this is the unimplemented marker for (??{...}). - // Under JPERL_UNIMPLEMENTED=warn, warn and fall through to the - // existing non-constant handling (which appends "(?:"); under - // die mode, abort with a clean diagnostic. Either way the user - // sees the issue — silent substitution would be a lie. - if (s.startsWith(RegexMarkers.RECURSIVE_PATTERN, offset)) { - regexUnimplementedSoft(s, offset + 3, - "(??{...}) recursive/dynamic regex patterns not implemented"); - if (isUnimplementedWarnMode()) { - // The marker includes the source construct's closing - // parenthesis. Emit a complete empty-group fallback - // and return its closing position so the dynamic - // construct is not parsed a second time below. - sb.append("(?:)"); - offset += RegexMarkers.RECURSIVE_PATTERN.length() - 1; - return offset; - } - } - // Handle (??{ ... }) recursive/dynamic regex patterns - // These insert a regex pattern at runtime based on code execution - - // Skip the (??{ part to find the code content + // Runtime executable-source compilation replaces this construct + // with a structured DYNAMIC_CALLOUT before matching. The ordinary + // preprocessor sees it only during literal syntax validation, so + // validate its extent and use an inert group for that validation. int codeStart = offset + 4; int codeOffset = findRegexCodeBlockClosingBrace(s, codeStart); if (codeOffset < 0) { @@ -1070,28 +1052,8 @@ private static int handleParentheses(String s, int offset, int length, StringBui "Unmatched '{' in (??{...}) dynamic pattern"); } // codeOffset points at the closing '}' - String codeBlock = s.substring(codeStart, codeOffset).trim(); offset = codeOffset + 1; // Skip past '}' - - // For simple constant expressions, inline the value as a regex pattern. - // (??{1}) means "evaluate 1 and use result as pattern" → matches literal "1" - // (??{"[x]"}) → matches character class [x] - if (isSimpleConstant(codeBlock)) { - String value = evaluateSimpleConstant(codeBlock); - if (value != null) { - // Insert the constant value as a non-capturing group pattern. - // Run through handleRegex to process any regex constructs - // (e.g. (?[...]) from regex_sets transformation). - sb.append("(?:"); - handleRegex(value, 0, sb, regexFlags, false); - } else { - // Fallback: empty non-capturing group - sb.append("(?:"); - } - } else { - // Non-constant: replace with empty non-capturing group - sb.append("(?:"); - } + sb.append("(?:"); // offset now points at ')' closing the (??{...}) construct // Fall through to common ')' handling at end of handleParentheses diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java index fbeeb788a..38c281d9b 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java @@ -227,9 +227,6 @@ static void updateControlVerbVariables(String mark, String error) { // 0 = off, 1 = debug, 2 = debugcolor. Captured at the regex call site. private int lexicalDebugMode; private boolean lexicalReStrict; - private static final String DYNAMIC_PATTERN_ERROR = - "\u0000(??{...}) recursive regex patterns not implemented (dynamic pattern)"; - public RuntimeRegex() { this.regexFlags = null; } @@ -450,9 +447,6 @@ private void emitWarningsOnUse() { && !activeCodeEnablesRegexp) { return; } - if (warningsOnUse.contains(DYNAMIC_PATTERN_ERROR)) { - throw new PerlJavaUnimplementedException(DYNAMIC_PATTERN_ERROR.substring(1)); - } for (String warning : warningsOnUse) { WarnDie.warnWithCategory(new RuntimeScalar(warning), RuntimeScalarCache.scalarEmptyString, "regexp"); } @@ -597,22 +591,6 @@ private static synchronized RuntimeRegex compileSynchronized( String originalPatternString = patternString; String compilePatternString = patternString; - boolean hasDynamicPattern = compilePatternString != null - && compilePatternString.contains(RegexMarkers.RECURSIVE_PATTERN); - boolean warnOnUnimplemented = "warn".equals( - GlobalVariable.getGlobalHash("main::ENV") - .get("JPERL_UNIMPLEMENTED").toString()); - boolean hasDeferredDynamicPattern = hasDynamicPattern && !warnOnUnimplemented; - boolean hasWarnDynamicFallback = hasDynamicPattern && warnOnUnimplemented; - if (hasDeferredDynamicPattern || hasWarnDynamicFallback) { - // Perl permits qr// construction before the dynamic callback is needed. - // Default mode keeps a never-matching placeholder and reports the hard - // error on use. Warn mode retains the historical compatibility fallback - // that ignores the unsupported dynamic component after warning. - compilePatternString = compilePatternString.replace( - RegexMarkers.RECURSIVE_PATTERN, - hasWarnDynamicFallback ? "(?:)" : "(?!)"); - } List quoteMetaWarningsOnUse = new ArrayList<>(); if (compilePatternString != null && compilePatternString.contains("\\Q")) { // Interpolated-pattern warnings are lexical diagnostics for each @@ -621,9 +599,7 @@ private static synchronized RuntimeRegex compileSynchronized( quoteMetaWarningsOnUse = RegexQuoteMeta.getWarningsOnUse(); } - // Dynamic patterns compile differently in normal and warn modes. Do not - // let a placeholder cached in one mode leak into the other. Lexical - // regex debugging also changes the compiled representation. + // Lexical regex debugging changes the compiled representation. // A lexical charname translator may return a different expansion for // each compilation of the same spelling. Literal syntax validation is // the first leg of one logical compilation: refresh the raw-source @@ -642,8 +618,7 @@ private static synchronized RuntimeRegex compileSynchronized( + "#bytepattern=" + effectivePatternByteBacked + "#strict=" + lexicalReStrict + (namedCharacterTranslator == null ? "" : "#charnames=" - + namedCharacterTranslator.toString()) - + (hasDynamicPattern ? (warnOnUnimplemented ? "\0warn" : "\0defer") : ""); + + namedCharacterTranslator.toString()); // Check if the regex is already cached RuntimeRegex regex = refreshLexicalNamedCharacter @@ -746,12 +721,6 @@ private static synchronized RuntimeRegex compileSynchronized( if (constructionPolicyWarning != null) { regex.inlineModifierWarnings.add(constructionPolicyWarning); } - if (hasDeferredDynamicPattern) { - regex.warningsOnUse.add(DYNAMIC_PATTERN_ERROR); - } else if (hasWarnDynamicFallback) { - regex.warningsOnUse.add( - "(??{...}) recursive/dynamic regex patterns not implemented\n"); - } if (usesRecursiveBackend) { regex.recursivePattern = new JoniRegexPattern(compilePatternString, regex.regexFlags, trustedCalloutCount, diff --git a/src/test/resources/unit/regex/native_dynamic_pattern_contract.t b/src/test/resources/unit/regex/native_dynamic_pattern_contract.t new file mode 100644 index 000000000..9e19421fa --- /dev/null +++ b/src/test/resources/unit/regex/native_dynamic_pattern_contract.t @@ -0,0 +1,72 @@ +use strict; +use warnings; +use Test::More; + +my $evaluations = 0; +ok('ab' =~ /^(??{ ++$evaluations; 'ab|a' })b$/, + 'nested alternatives backtrack into the outer suffix'); +is($evaluations, 1, 'one dynamic entry retains its nested alternatives'); + +my $repeat_count = 0; +ok('aa' =~ /^(?:(??{ ++$repeat_count; 'a' })){2}$/, + 'a quantified dynamic program executes at each entry'); +is($repeat_count, 2, 'quantified dynamic expression is reevaluated'); + +my ($seen_capture, $entry_pos); +ok('abc' =~ /^(a)(??{ $seen_capture = $1; $entry_pos = pos; '(b)' })(c)$/, + 'dynamic program sees provisional state and may contain captures'); +is($seen_capture, 'a', 'dynamic expression sees the preceding outer capture'); +is($entry_pos, 1, 'pos reports the dynamic entry offset'); +is($1, 'a', 'outer capture before the dynamic program is preserved'); +is($2, 'c', 'nested captures do not consume outer capture numbers'); + +my $returned_qr = qr/b/i; +ok('aB' =~ /^a(??{ $returned_qr })$/, + 'dynamic expression accepts a compiled qr value'); +ok(!('aB' =~ /^a(??{ qr{b} })$/i), + 'returned qr retains its own modifiers'); +ok('aB' =~ /^a(??{ 'b' })$/i, + 'returned string inherits outer modifiers'); + +my $seed; +ok('seed' =~ /^(?{ $seed = 'seed' })(??{ $^R })$/, + 'dynamic expression sees the prior callback result'); +is($^R, 'seed', 'successful dynamic execution preserves callback result'); + +my $failed_effects = 0; +ok(!('x' =~ /^(??{ ++$failed_effects; 'y' })$/), + 'a returned program may fail normally'); +is($failed_effects, 1, 'ordinary dynamic side effects survive failure'); + +my $unreached = 0; +ok('x' =~ /^x|y(??{ ++$unreached; 'z' })$/, + 'an earlier branch can bypass a dynamic program'); +is($unreached, 0, 'an unreached dynamic expression is not executed'); + +my $exception_ok = eval { 'x' =~ /^(??{ die "dynamic boom\n" })$/; 1 }; +ok(!$exception_ok, 'dynamic exception aborts matching'); +is($@, "dynamic boom\n", 'dynamic exception propagates unchanged'); + +{ + no warnings 'uninitialized'; + ok('a' =~ /^a(??{ undef })$/, 'undef dynamic result is an empty pattern'); +} +ok('a' =~ /^a(??{ '' })$/, 'empty dynamic result is an empty pattern'); + +my $wide = "\x{100}"; +ok($wide =~ /^(??{ $wide })$/u, 'Unicode dynamic source preserves its scalar'); +{ + use bytes; + my $octet = "\xE9"; + ok($octet =~ /^(??{ $octet })$/, + 'byte-mode dynamic source preserves its octet provenance'); +} + +my $recursive; +$recursive = qr{ \( (?: [^()]+ | (??{ $recursive }) )* \) }x; +ok('(a(b)c)' =~ /^$recursive$/, + 'self-referential dynamic qr matches nested input'); +ok(!('(a(b)' =~ /^$recursive$/), + 'self-referential dynamic qr still rejects unbalanced input'); + +done_testing; diff --git a/third_party/joni/test/org/joni/test/TestPerlDynamicExecution.java b/third_party/joni/test/org/joni/test/TestPerlDynamicExecution.java new file mode 100644 index 000000000..1a21b5197 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestPerlDynamicExecution.java @@ -0,0 +1,112 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import static org.junit.Assert.assertEquals; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.jcodings.specific.ASCIIEncoding; +import org.joni.CalloutHandler; +import org.joni.CalloutResult; +import org.joni.DynamicPatternResult; +import org.joni.MatchView; +import org.joni.Matcher; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.junit.Test; + +public class TestPerlDynamicExecution { + private static Regex regex(String pattern) { + byte[] bytes = pattern.getBytes(StandardCharsets.US_ASCII); + return new Regex(bytes, 0, bytes.length, Option.NONE, + ASCIIEncoding.INSTANCE, Syntax.RUBY); + } + + @Test + public void nestedAlternativesResolveTokensOnlyWhenTheOuterPathCommits() { + List events = new ArrayList<>(); + Regex outer = regex("\\A(?{=DYNAMIC:1})b\\z"); + Regex nested = regex("ab(?{=CALL:2})|a(?{=CALL:3})"); + CalloutHandler nestedHandler = new CalloutHandler() { + @Override + public CalloutResult execute(int id, MatchView match) { + events.add("nested-execute:" + id); + return CalloutResult.continueWith("nested-" + id); + } + + @Override + public void unwind(Object token) { + events.add("nested-unwind:" + token); + } + + @Override + public void complete(Object token) { + events.add("nested-complete:" + token); + } + + @Override + public void finish(boolean matched) { + events.add("nested-finish:" + matched); + } + }; + CalloutHandler outerHandler = new CalloutHandler() { + @Override + public CalloutResult execute(int id, MatchView match) { + throw new AssertionError("plain outer callout not expected"); + } + + @Override + public DynamicPatternResult executeDynamic(int id, MatchView match) { + events.add("dynamic:" + id); + return new DynamicPatternResult(nested, nestedHandler, "outer-dynamic"); + } + + @Override + public void unwind(Object token) { + events.add("outer-unwind:" + token); + } + + @Override + public void complete(Object token) { + events.add("outer-complete:" + token); + } + + }; + + byte[] input = "ab".getBytes(StandardCharsets.US_ASCII); + Matcher matcher = outer.matcher(input); + matcher.setCalloutHandler(outerHandler); + + assertEquals(0, matcher.search(0, input.length, Option.NONE)); + assertEquals(Arrays.asList( + "dynamic:1", + "nested-execute:2", + "nested-unwind:nested-2", + "nested-execute:3", + "nested-complete:nested-3", + "nested-finish:true", + "outer-complete:outer-dynamic"), events); + } +} From e5de81d84d2286db23a2aa7bc6eec0f571c48634 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 21:47:47 +0200 Subject: [PATCH 07/11] fix(regex): preserve callback mutation unwind boundaries Track aggregate mutations made by native regex code blocks so ordinary backtracking restores them while scalar assignments retain Perl semantics. Commit callback state when destructive control verbs cut the active path, including across dynamic regex continuations, while MARK still unwinds. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../runtime/regex/JoniRegexPattern.java | 15 ++-- .../RegexCallbackMutationSnapshot.java | 83 +++++++++++++++++++ .../native_callback_backtrack_transaction.t | 29 +++++++ .../joni/src/org/joni/ByteCodeMachine.java | 23 +++++ .../joni/src/org/joni/StackMachine.java | 9 +- .../joni/test/TestPerlDynamicExecution.java | 44 ++++++++++ 6 files changed, 197 insertions(+), 6 deletions(-) create mode 100644 src/main/java/org/perlonjava/runtime/runtimetypes/RegexCallbackMutationSnapshot.java create mode 100644 src/test/resources/unit/regex/native_callback_backtrack_transaction.t diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index 6c27d524d..67d946f33 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -1318,7 +1318,8 @@ private static int[] buildByteToChar(String input, int byteLength, int[] charToB private static final class PerlCalloutHandler implements CalloutHandler { private record Token(int localLevel, RegexState regexState, RuntimeScalar previousR, RuntimeScalar result, boolean block, boolean dynamic, - CaptureSnapshot previousDynamicView) {} + CaptureSnapshot previousDynamicView, + RegexCallbackMutationSnapshot mutations) {} private record CaptureSnapshot(int position, int[] begins, int[] ends, int lastClosed, String controlMark) implements MatchView { @@ -1491,6 +1492,9 @@ private Evaluation evaluate(RuntimeRegexCallback callback, MatchView match) { } } CaptureSnapshot priorDynamicView = previousDynamicView; + RegexCallbackMutationSnapshot mutations = + callback.kind == RuntimeRegexCallback.Kind.BLOCK && parent == null + ? RegexCallbackMutationSnapshot.capture(callback.code) : null; MatchView provisional = callback.kind == RuntimeRegexCallback.Kind.DYNAMIC ? dynamicCaptureView(match, priorDynamicView) : match; publishProvisional(provisional); @@ -1533,7 +1537,7 @@ private Evaluation evaluate(RuntimeRegexCallback callback, MatchView match) { Token token = new Token(localLevel, savedRegex, previousR, result.clone(), block, callback.kind == RuntimeRegexCallback.Kind.DYNAMIC, - priorDynamicView); + priorDynamicView, mutations); return new Evaluation(result, token); } catch (RuntimeException | Error failure) { // The matcher cannot register an unwind token when the callout @@ -1558,12 +1562,12 @@ private Evaluation evaluate(RuntimeRegexCallback callback, MatchView match) { @Override public void unwind(Object value) { - restore((Token) value, false); + restore((Token) value, false, true); } @Override public void complete(Object value) { - restore((Token) value, true); + restore((Token) value, true, false); } @Override @@ -1604,10 +1608,11 @@ void abort() { DynamicVariableManager.popToLocalLevel(initialLocalLevel); } - private void restore(Token token, boolean completed) { + private void restore(Token token, boolean completed, boolean backtracked) { if (!completed) { DynamicVariableManager.popToLocalLevel(token.localLevel()); previousDynamicView = token.previousDynamicView(); + if (backtracked && token.mutations() != null) token.mutations().restore(); } token.regexState().restore(); if (!completed || !token.dynamic()) { diff --git a/src/main/java/org/perlonjava/runtime/runtimetypes/RegexCallbackMutationSnapshot.java b/src/main/java/org/perlonjava/runtime/runtimetypes/RegexCallbackMutationSnapshot.java new file mode 100644 index 000000000..06dc45b9d --- /dev/null +++ b/src/main/java/org/perlonjava/runtime/runtimetypes/RegexCallbackMutationSnapshot.java @@ -0,0 +1,83 @@ +package org.perlonjava.runtime.runtimetypes; + +import java.util.ArrayDeque; +import java.util.IdentityHashMap; +import java.util.Map; + +/** Backtracking savepoint for mutations made by a plain {@code (?{...})} block. */ +public final class RegexCallbackMutationSnapshot { + private final IdentityHashMap arrays = new IdentityHashMap<>(); + private final IdentityHashMap hashes = new IdentityHashMap<>(); + private final IdentityHashMap seen = new IdentityHashMap<>(); + + private RegexCallbackMutationSnapshot(RuntimeCode callback) { + ArrayDeque work = new ArrayDeque<>(); + if (callback.closedOverVariables != null) { + addAll(work, callback.closedOverVariables.values()); + } + addAll(work, callback.capturedScalars); + addAll(work, callback.capturedAggregates); + if (callback.ourVariableRegistry != null) { + for (Map.Entry entry : callback.ourVariableRegistry.entrySet()) { + String name = entry.getKey(); + String packageName = entry.getValue(); + if (name == null || name.length() < 2 || packageName == null) continue; + String fullName = packageName + "::" + name.substring(1); + RuntimeBase cell = switch (name.charAt(0)) { + case '@' -> GlobalVariable.getGlobalArray(fullName); + case '%' -> GlobalVariable.getGlobalHash(fullName); + default -> GlobalVariable.getGlobalVariable(fullName); + }; + work.add(cell); + } + } + capture(work); + } + + public static RegexCallbackMutationSnapshot capture(RuntimeCode callback) { + return new RegexCallbackMutationSnapshot(callback); + } + + private void capture(ArrayDeque work) { + while (!work.isEmpty()) { + RuntimeBase value = work.removeLast(); + if (value == null || seen.put(value, Boolean.TRUE) != null) continue; + if (value instanceof RuntimeScalar scalar) { + if (scalar.value instanceof RuntimeArray array) work.add(array); + else if (scalar.value instanceof RuntimeHash hash) work.add(hash); + else if (scalar.value instanceof RuntimeScalar nested) work.add(nested); + } else if (value instanceof RuntimeArray array) { + Object state = array.snapshotRegexMutationState(); + if (state == null) continue; + arrays.put(array, state); + addAll(work, array.elements); + } else if (value instanceof RuntimeHash hash) { + Object state = hash.snapshotRegexMutationState(); + if (state == null) continue; + hashes.put(hash, state); + addAll(work, hash.elements.values()); + } + } + } + + public void restore() { + for (Map.Entry entry : arrays.entrySet()) { + entry.getKey().restoreRegexMutationState(entry.getValue()); + } + for (Map.Entry entry : hashes.entrySet()) { + entry.getKey().restoreRegexMutationState(entry.getValue()); + } + MortalList.flush(); + } + + private static void addAll(ArrayDeque work, + Iterable values) { + if (values == null) return; + for (RuntimeBase value : values) if (value != null) work.add(value); + } + + private static void addAll(ArrayDeque work, RuntimeBase[] values) { + if (values == null) return; + for (RuntimeBase value : values) if (value != null) work.add(value); + } +} diff --git a/src/test/resources/unit/regex/native_callback_backtrack_transaction.t b/src/test/resources/unit/regex/native_callback_backtrack_transaction.t new file mode 100644 index 000000000..8de3a52e6 --- /dev/null +++ b/src/test/resources/unit/regex/native_callback_backtrack_transaction.t @@ -0,0 +1,29 @@ +use strict; +use warnings; +use Test::More tests => 10; + +my @lexical; +ok('abc' !~ /^a(?{ push @lexical, 1 })b(?{ push @lexical, 2 })$/, + 'path with two plain callbacks fails'); +is_deeply(\@lexical, [], 'both lexical mutations unwind in reverse order'); + +our @package; +ok('abc' !~ /^a(?{ push @package, 1 })b(?{ push @package, 2 })$/, + 'package mutation path with two callbacks fails'); +is_deeply(\@package, [], 'both package mutations unwind in reverse order'); + +my @dynamic; +ok('abc' !~ /^a(??{ push @dynamic, 1; 'b' })$/, + 'dynamic callback nested program fails'); +is_deeply(\@dynamic, [1], 'dynamic expression mutation survives failure'); + +my @nested; +ok('abc' !~ /^(??{ qr{a(?{ push @nested, 1 })b(?{ push @nested, 2 })} })$/, + 'returned program containing plain callbacks fails'); +is_deeply(\@nested, [1, 2], + 'plain callback mutations inside a dynamic program survive failure'); + +my @condition; +ok('ac' !~ /^a(?(?{ push @condition, 1; 1 })b|c)$/, + 'conditional callback chooses a failing branch'); +is_deeply(\@condition, [1], 'conditional expression mutation survives failure'); diff --git a/third_party/joni/src/org/joni/ByteCodeMachine.java b/third_party/joni/src/org/joni/ByteCodeMachine.java index 8219c88f5..257901d3e 100644 --- a/third_party/joni/src/org/joni/ByteCodeMachine.java +++ b/third_party/joni/src/org/joni/ByteCodeMachine.java @@ -60,6 +60,8 @@ class ByteCodeMachine extends StackMachine implements MatchView { private int pkeep; private int currentRegexOptions; private int pendingControlAction; + private boolean preserveCalloutMutations; + private boolean exportedDestructiveControl; private final int[]code; // byte code private int ip; // instruction pointer @@ -222,6 +224,8 @@ protected final int matchAt(int _range, int _sstart, int _sprev, boolean interru ip = 0; currentRegexOptions = regex.options; controlMark = null; + preserveCalloutMutations = false; + exportedDestructiveControl = false; if (Config.DEBUG_MATCH) debugMatchBegin(); stackInit(); @@ -2996,6 +3000,7 @@ private boolean opAccept() { private void opControlFail() { controlVerbEncountered = true; + markDestructiveControl(); String name = controlVerbName(code[ip++]); // An unnamed FAIL terminates the current path without replacing a // more specific PRUNE/SKIP/THEN/COMMIT error already encountered on @@ -3008,6 +3013,7 @@ private void opControlFail() { private void opPrune() { controlVerbEncountered = true; + markDestructiveControl(); String name = controlVerbName(code[ip++]); controlError = name == null ? "1" : name; cutAlternatives(false); @@ -3016,6 +3022,7 @@ private void opPrune() { private void opSkip() { controlVerbEncountered = true; + markDestructiveControl(); String name = controlVerbName(code[ip++]); controlError = name == null ? "1" : name; if (name != null) { @@ -3033,6 +3040,7 @@ private void opSkip() { private void opThen() { controlVerbEncountered = true; + markDestructiveControl(); String name = controlVerbName(code[ip++]); controlError = name == null ? "1" : name; cutAlternatives(true, s, sprev, pkeep); @@ -3041,6 +3049,7 @@ private void opThen() { private void opCommit() { controlVerbEncountered = true; + markDestructiveControl(); String name = controlVerbName(code[ip++]); controlError = name == null ? "1" : name; cutAlternatives(false); @@ -3055,6 +3064,16 @@ private void opMark() { controlMark = next; } + private void markDestructiveControl() { + preserveCalloutMutations = true; + exportedDestructiveControl = true; + } + + @Override + protected boolean completeCalloutsOnUnwind() { + return preserveCalloutMutations; + } + @Override protected void restoreControlMark(String name) { if (controlMark != null) controlError = controlMark; @@ -3406,6 +3425,10 @@ void propagateControlTo(ByteCodeMachine outer) { int action = machine.pendingControlAction; machine.pendingControlAction = CONTROL_NONE; if (machine.controlVerbEncountered) outer.controlVerbEncountered = true; + if (machine.exportedDestructiveControl) { + outer.markDestructiveControl(); + machine.exportedDestructiveControl = false; + } if (machine.controlError != null) outer.controlError = machine.controlError; switch (action) { case CONTROL_PRUNE: diff --git a/third_party/joni/src/org/joni/StackMachine.java b/third_party/joni/src/org/joni/StackMachine.java index a5711be34..c91cbaf71 100644 --- a/third_party/joni/src/org/joni/StackMachine.java +++ b/third_party/joni/src/org/joni/StackMachine.java @@ -578,7 +578,14 @@ private void completeCallout(StackEntry entry) { private void unwindCallout(StackEntry entry) { Object token = entry.takeCalloutToken(); - if (token != null) getCalloutHandler().unwind(token); + if (token == null) return; + if (completeCalloutsOnUnwind()) getCalloutHandler().complete(token); + else getCalloutHandler().unwind(token); + } + + /** Whether the current failure path commits callback side effects. */ + protected boolean completeCalloutsOnUnwind() { + return false; } private void completeDynamic(StackEntry entry) { diff --git a/third_party/joni/test/org/joni/test/TestPerlDynamicExecution.java b/third_party/joni/test/org/joni/test/TestPerlDynamicExecution.java index 1a21b5197..75c3d1c30 100644 --- a/third_party/joni/test/org/joni/test/TestPerlDynamicExecution.java +++ b/third_party/joni/test/org/joni/test/TestPerlDynamicExecution.java @@ -109,4 +109,48 @@ public void complete(Object token) { "nested-finish:true", "outer-complete:outer-dynamic"), events); } + + @Test + public void destructiveControlsCommitCalloutsButMarkDoesNot() { + String[] committing = {"FAIL", "PRUNE)(*FAIL", "SKIP)(*FAIL", + "THEN)(*FAIL", "COMMIT)(*FAIL"}; + for (String control : committing) { + List events = new ArrayList<>(); + Regex pattern = regex("a(?{=CALL:1})(*" + control + ")"); + assertEquals(-1, search(pattern, "a", recordingHandler(events))); + assertEquals(control, Arrays.asList("execute:1", "complete:1"), events); + } + + List markEvents = new ArrayList<>(); + Regex mark = regex("a(?{=CALL:1})(*MARK:seen)b"); + assertEquals(-1, search(mark, "a", recordingHandler(markEvents))); + assertEquals(Arrays.asList("execute:1", "unwind:1"), markEvents); + } + + private static int search(Regex regex, String input, CalloutHandler handler) { + byte[] bytes = input.getBytes(StandardCharsets.US_ASCII); + Matcher matcher = regex.matcher(bytes); + matcher.setCalloutHandler(handler); + return matcher.search(0, bytes.length, Option.NONE); + } + + private static CalloutHandler recordingHandler(List events) { + return new CalloutHandler() { + @Override + public CalloutResult execute(int id, MatchView match) { + events.add("execute:" + id); + return CalloutResult.continueWith(id); + } + + @Override + public void unwind(Object token) { + events.add("unwind:" + token); + } + + @Override + public void complete(Object token) { + events.add("complete:" + token); + } + }; + } } From c81bf4a407b1127bd860764c36ebefd2919a825e Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 21:53:38 +0200 Subject: [PATCH 08/11] docs(regex): record native dynamic completion Mark runtime dynamic patterns and adapter retirement complete after the native continuation and callback unwind gates passed. Generated with [Codex](https://openai.com/codex/) Co-Authored-By: Codex --- dev/design/phase36-regex-parity.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index 0684909bc..c56aae5de 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -140,6 +140,10 @@ affected corpus before taking another slice. unterminated inline-option and comment groups, incomplete `(?` group effects, and empty control verbs. The focused system-Perl/direct-Joni/four-leg gates remove the corresponding `regexp.t` identities with no introductions. +- Runtime `(??{...})` sources execute through native Joni continuations; the + dynamic Java fallback adapter is gone. Callback aggregate mutations unwind + on ordinary backtracking while destructive control verbs commit the cut + path, including across a dynamic continuation. ## Execution Phases @@ -369,7 +373,7 @@ gates may reopen it if a semantic regression appears. - [x] Native ordinary lookbehind and removal of its Java translation - [x] Native branch reset and removal of its capture-map adapter - [x] Native plain `\N` non-newline atom and interval forms -- [ ] Native recursive/runtime `(??{...})` and removal of dynamic adapters +- [x] Native recursive/runtime `(??{...})` and removal of dynamic adapters - [ ] Retire proven-obsolete `dev/import-perl5` regex patches - [ ] Refresh the complete Unicode, `pat.t`, `pat_advanced.t`, `reg_mesg.t`, and 80-file forced-Joni gates on one integrated artifact From 34406d1446be5000b2c4befb0c2bdc5c8d2db506 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 22:02:09 +0200 Subject: [PATCH 09/11] fix(regex): restore callback mutations at match boundary Keep ordinary callback aggregate side effects from abandoned alternatives when another branch succeeds, while restoring all plain-block aggregate mutations when the complete match fails. Preserve per-path local-scope unwind and commit mutations cut by destructive control verbs. Generated with [Codex](https://openai.com/codex/) Co-Authored-By: Codex --- .../runtime/regex/JoniRegexPattern.java | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index 67d946f33..af6af11f5 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -26,6 +26,7 @@ import static org.joni.constants.SyntaxProperties.OP2_PLUS_POSSESSIVE_INTERVAL; import java.nio.charset.StandardCharsets; +import java.util.ArrayDeque; import java.util.Iterator; import java.util.ArrayList; import java.util.LinkedHashMap; @@ -1318,8 +1319,7 @@ private static int[] buildByteToChar(String input, int byteLength, int[] charToB private static final class PerlCalloutHandler implements CalloutHandler { private record Token(int localLevel, RegexState regexState, RuntimeScalar previousR, RuntimeScalar result, boolean block, boolean dynamic, - CaptureSnapshot previousDynamicView, - RegexCallbackMutationSnapshot mutations) {} + CaptureSnapshot previousDynamicView) {} private record CaptureSnapshot(int position, int[] begins, int[] ends, int lastClosed, String controlMark) implements MatchView { @@ -1359,6 +1359,9 @@ static CaptureSnapshot of(MatchView match) { private boolean executedNestedCallbackPattern; private String failedNestedLastClosedCapture; private String failedNestedLastParenMatch; + private final ArrayDeque callbackMutations = + new ArrayDeque<>(); + private boolean preserveCallbackMutations; PerlCalloutHandler(String input, int[] byteToChar, List callbacks, RegexFlags outerFlags, boolean publishesControlVerbState, @@ -1492,9 +1495,9 @@ private Evaluation evaluate(RuntimeRegexCallback callback, MatchView match) { } } CaptureSnapshot priorDynamicView = previousDynamicView; - RegexCallbackMutationSnapshot mutations = - callback.kind == RuntimeRegexCallback.Kind.BLOCK && parent == null - ? RegexCallbackMutationSnapshot.capture(callback.code) : null; + if (callback.kind == RuntimeRegexCallback.Kind.BLOCK && parent == null) { + callbackMutations.addLast(RegexCallbackMutationSnapshot.capture(callback.code)); + } MatchView provisional = callback.kind == RuntimeRegexCallback.Kind.DYNAMIC ? dynamicCaptureView(match, priorDynamicView) : match; publishProvisional(provisional); @@ -1537,7 +1540,7 @@ private Evaluation evaluate(RuntimeRegexCallback callback, MatchView match) { Token token = new Token(localLevel, savedRegex, previousR, result.clone(), block, callback.kind == RuntimeRegexCallback.Kind.DYNAMIC, - priorDynamicView, mutations); + priorDynamicView); return new Evaluation(result, token); } catch (RuntimeException | Error failure) { // The matcher cannot register an unwind token when the callout @@ -1562,12 +1565,14 @@ private Evaluation evaluate(RuntimeRegexCallback callback, MatchView match) { @Override public void unwind(Object value) { - restore((Token) value, false, true); + restore((Token) value, false); } @Override public void complete(Object value) { - restore((Token) value, true, false); + Token token = (Token) value; + if (token.block() && parent == null) preserveCallbackMutations = true; + restore(token, true); } @Override @@ -1586,6 +1591,7 @@ public void finish(boolean matched) { GlobalVariable.getGlobalVariable(GlobalContext.encodeSpecialVar("R")) .set(completedResult); } else if (!matched) { + if (!preserveCallbackMutations) restoreCallbackMutations(); initialRegexState.restore(); if (hasFailedNestedCaptureState) { state.lastClosedCapture = failedNestedLastClosedCapture; @@ -1594,6 +1600,7 @@ public void finish(boolean matched) { } } } finally { + callbackMutations.clear(); DynamicVariableManager.popToLocalLevel(initialLocalLevel); } } @@ -1605,14 +1612,15 @@ private void recordFailedNestedCaptureState(String lastClosed, String lastParen) } void abort() { + restoreCallbackMutations(); + callbackMutations.clear(); DynamicVariableManager.popToLocalLevel(initialLocalLevel); } - private void restore(Token token, boolean completed, boolean backtracked) { + private void restore(Token token, boolean completed) { if (!completed) { DynamicVariableManager.popToLocalLevel(token.localLevel()); previousDynamicView = token.previousDynamicView(); - if (backtracked && token.mutations() != null) token.mutations().restore(); } token.regexState().restore(); if (!completed || !token.dynamic()) { @@ -1624,6 +1632,12 @@ private void restore(Token token, boolean completed, boolean backtracked) { } } + private void restoreCallbackMutations() { + while (!callbackMutations.isEmpty()) { + callbackMutations.removeLast().restore(); + } + } + private static MatchView dynamicCaptureView( MatchView current, CaptureSnapshot previous) { CaptureSnapshot adjusted = CaptureSnapshot.of(current); From c962a472cd3ea935415637474e0b361382c4f124 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 22:09:08 +0200 Subject: [PATCH 10/11] docs(regex): clarify callback transaction boundary Describe aggregate rollback at the complete-match boundary and preserve the distinct successful-alternative and destructive-control outcomes. Generated with [Codex](https://openai.com/codex/) Co-Authored-By: Codex --- dev/design/phase36-regex-parity.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index c56aae5de..b515113cd 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -142,8 +142,9 @@ affected corpus before taking another slice. remove the corresponding `regexp.t` identities with no introductions. - Runtime `(??{...})` sources execute through native Joni continuations; the dynamic Java fallback adapter is gone. Callback aggregate mutations unwind - on ordinary backtracking while destructive control verbs commit the cut - path, including across a dynamic continuation. + when the complete match fails, remain visible when another alternative + succeeds, and commit when destructive control verbs cut the path, including + across a dynamic continuation. ## Execution Phases From 186c964ddd441bd2e8e319bae8e9acec487b0be4 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 22:11:46 +0200 Subject: [PATCH 11/11] test(regex): cover destructive callback commits Lock in Perl's distinction between callback mutation commits for destructive control verbs and ordinary rollback after MARK followed by failure. Generated with [Codex](https://openai.com/codex/) Co-Authored-By: Codex --- .../regex/native_callback_destructive_commit.t | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 src/test/resources/unit/regex/native_callback_destructive_commit.t diff --git a/src/test/resources/unit/regex/native_callback_destructive_commit.t b/src/test/resources/unit/regex/native_callback_destructive_commit.t new file mode 100644 index 000000000..6cd62160c --- /dev/null +++ b/src/test/resources/unit/regex/native_callback_destructive_commit.t @@ -0,0 +1,17 @@ +use strict; +use warnings; +use Test::More; + +for my $verb (qw(FAIL PRUNE SKIP THEN COMMIT)) { + my @events; + my $pattern = qr/\A(?{ push @events, $verb })(*$verb)(*FAIL)\z/; + ok('x' !~ $pattern, "$verb path fails"); + is_deeply(\@events, [$verb], "$verb commits callback mutation"); +} + +my @marked; +ok('x' !~ /\A(?{ push @marked, 'MARK' })(*MARK:plain)z\z/, + 'MARK path fails normally'); +is_deeply(\@marked, [], 'MARK does not commit callback mutation'); + +done_testing;