WIP: complete native Joni dynamic regex execution

dev/design/phase36-regex-parity.md

@@ -136,6 +136,15 @@ affected corpus before taking another slice.
   before loose alias normalization, so uncased letters no longer enter that
   class. The focused system-Perl oracle, four runtime legs, and imported
   `regexp.t` identity agree.
+- Native Joni diagnostics distinguish invalid non-braced `\p`/`\P` followers,
+  unterminated inline-option and comment groups, incomplete `(?` group effects,
+  and empty control verbs. The focused system-Perl/direct-Joni/four-leg gates
+  remove the corresponding `regexp.t` identities with no introductions.
+- Runtime `(??{...})` sources execute through native Joni continuations; the
+  dynamic Java fallback adapter is gone. Callback aggregate mutations unwind
+  when the complete match fails, remain visible when another alternative
+  succeeds, and commit when destructive control verbs cut the path, including
+  across a dynamic continuation.
 
 ## Execution Phases
 
@@ -365,7 +374,7 @@ gates may reopen it if a semantic regression appears.
 - [x] Native ordinary lookbehind and removal of its Java translation
 - [x] Native branch reset and removal of its capture-map adapter
 - [x] Native plain `\N` non-newline atom and interval forms
-- [ ] Native recursive/runtime `(??{...})` and removal of dynamic adapters
+- [x] Native recursive/runtime `(??{...})` and removal of dynamic adapters
 - [ ] Retire proven-obsolete `dev/import-perl5` regex patches
 - [ ] Refresh the complete Unicode, `pat.t`, `pat_advanced.t`, `reg_mesg.t`, and
       80-file forced-Joni gates on one integrated artifact

src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java

@@ -26,6 +26,7 @@
 import static org.joni.constants.SyntaxProperties.OP2_PLUS_POSSESSIVE_INTERVAL;
 
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayDeque;
 import java.util.Iterator;
 import java.util.ArrayList;
 import java.util.LinkedHashMap;
@@ -1358,6 +1359,9 @@ static CaptureSnapshot of(MatchView match) {
         private boolean executedNestedCallbackPattern;
         private String failedNestedLastClosedCapture;
         private String failedNestedLastParenMatch;
+        private final ArrayDeque<RegexCallbackMutationSnapshot> callbackMutations =
+                new ArrayDeque<>();
+        private boolean preserveCallbackMutations;
 
         PerlCalloutHandler(String input, int[] byteToChar, List<RuntimeRegexCallback> callbacks,
                            RegexFlags outerFlags, boolean publishesControlVerbState,
@@ -1491,6 +1495,9 @@ private Evaluation evaluate(RuntimeRegexCallback callback, MatchView match) {
                 }
             }
             CaptureSnapshot priorDynamicView = previousDynamicView;
+            if (callback.kind == RuntimeRegexCallback.Kind.BLOCK && parent == null) {
+                callbackMutations.addLast(RegexCallbackMutationSnapshot.capture(callback.code));
+            }
             MatchView provisional = callback.kind == RuntimeRegexCallback.Kind.DYNAMIC
                     ? dynamicCaptureView(match, priorDynamicView) : match;
             publishProvisional(provisional);
@@ -1563,7 +1570,9 @@ public void unwind(Object value) {
 
         @Override
         public void complete(Object value) {
-            restore((Token) value, true);
+            Token token = (Token) value;
+            if (token.block() && parent == null) preserveCallbackMutations = true;
+            restore(token, true);
         }
 
         @Override
@@ -1582,6 +1591,7 @@ public void finish(boolean matched) {
                     GlobalVariable.getGlobalVariable(GlobalContext.encodeSpecialVar("R"))
                             .set(completedResult);
                 } else if (!matched) {
+                    if (!preserveCallbackMutations) restoreCallbackMutations();
                     initialRegexState.restore();
                     if (hasFailedNestedCaptureState) {
                         state.lastClosedCapture = failedNestedLastClosedCapture;
@@ -1590,6 +1600,7 @@ public void finish(boolean matched) {
                     }
                 }
             } finally {
+                callbackMutations.clear();
                 DynamicVariableManager.popToLocalLevel(initialLocalLevel);
             }
         }
@@ -1601,6 +1612,8 @@ private void recordFailedNestedCaptureState(String lastClosed, String lastParen)
         }
 
         void abort() {
+            restoreCallbackMutations();
+            callbackMutations.clear();
             DynamicVariableManager.popToLocalLevel(initialLocalLevel);
         }
 
@@ -1619,6 +1632,12 @@ private void restore(Token token, boolean completed) {
             }
         }
 
+        private void restoreCallbackMutations() {
+            while (!callbackMutations.isEmpty()) {
+                callbackMutations.removeLast().restore();
+            }
+        }
+
         private static MatchView dynamicCaptureView(
                 MatchView current, CaptureSnapshot previous) {
             CaptureSnapshot adjusted = CaptureSnapshot.of(current);

src/main/java/org/perlonjava/runtime/regex/RegexMarkers.java

@@ -4,8 +4,7 @@
  * Shared placeholder markers used by the string-interpolation parser to
  * stand in for regex constructs that PerlOnJava cannot compile literally
  * (because they require features unsupported by the underlying Java regex
- * engine — e.g. arbitrary {@code (?{ CODE })} code blocks and
- * {@code (??{ CODE })} recursive/dynamic patterns).
+ * engine — e.g. arbitrary {@code (?{ CODE })} code blocks).
  *
  * <p>The markers are emitted by {@code StringSegmentParser} when a code
  * block can't be constant-folded. {@link RegexPreprocessor} detects them
@@ -15,12 +14,6 @@
  *       or a no-op fallback only when {@link #CODE_BLOCK_NOOP_ENV} is set.
  *       Plain {@code JPERL_UNIMPLEMENTED=warn} still reports the unsupported
  *       feature without pretending the callback ran.</li>
- *   <li>{@link #RECURSIVE_PATTERN} — a hard error under default die mode,
- *       or a warning under {@code JPERL_UNIMPLEMENTED=warn} followed by
- *       the soft {@code (?:} fallback so the surrounding pattern still
- *       compiles (many CPAN modules build dynamic patterns that happen
- *       to work with the empty-group fallback; under warn mode we want
- *       tests to continue but the user must see a diagnostic).</li>
  * </ul>
  *
  * <p><b>Why these specific spellings?</b> The preprocessor performs some
@@ -50,12 +43,5 @@ public final class RegexMarkers {
      */
     public static final String CODE_BLOCK = "(?{UNIMPLEMENTED_CODE_BLOC})";
 
-    /**
-     * Marker for a {@code (??{ CODE })} recursive/dynamic pattern that
-     * could not be constant-folded at parse time. Contains no
-     * fold-affected letters.
-     */
-    public static final String RECURSIVE_PATTERN = "(??{UNIMPLEMENTED_RECURSIVE_PATTERN})";
-
     private RegexMarkers() {}
 }

src/main/java/org/perlonjava/runtime/regex/RegexPreprocessor.java

@@ -1041,57 +1041,19 @@ private static int handleParentheses(String s, int offset, int length, StringBui
                     offset = handleCodeBlock(s, offset, length, sb, regexFlags);
                 }
             } else if (c3 == '?' && c4 == '{') {
-                // Check if this is the unimplemented marker for (??{...}).
-                // Under JPERL_UNIMPLEMENTED=warn, warn and fall through to the
-                // existing non-constant handling (which appends "(?:"); under
-                // die mode, abort with a clean diagnostic. Either way the user
-                // sees the issue — silent substitution would be a lie.
-                if (s.startsWith(RegexMarkers.RECURSIVE_PATTERN, offset)) {
-                    regexUnimplementedSoft(s, offset + 3,
-                            "(??{...}) recursive/dynamic regex patterns not implemented");
-                    if (isUnimplementedWarnMode()) {
-                        // The marker includes the source construct's closing
-                        // parenthesis. Emit a complete empty-group fallback
-                        // and return its closing position so the dynamic
-                        // construct is not parsed a second time below.
-                        sb.append("(?:)");
-                        offset += RegexMarkers.RECURSIVE_PATTERN.length() - 1;
-                        return offset;
-                    }
-                }
-                // Handle (??{ ... }) recursive/dynamic regex patterns
-                // These insert a regex pattern at runtime based on code execution
-
-                // Skip the (??{ part to find the code content
+                // Runtime executable-source compilation replaces this construct
+                // with a structured DYNAMIC_CALLOUT before matching. The ordinary
+                // preprocessor sees it only during literal syntax validation, so
+                // validate its extent and use an inert group for that validation.
                 int codeStart = offset + 4;
                 int codeOffset = findRegexCodeBlockClosingBrace(s, codeStart);
                 if (codeOffset < 0) {
                     regexError(s, offset + 3,
                             "Unmatched '{' in (??{...}) dynamic pattern");
                 }
                 // codeOffset points at the closing '}'
-                String codeBlock = s.substring(codeStart, codeOffset).trim();
                 offset = codeOffset + 1; // Skip past '}'
-
-                // For simple constant expressions, inline the value as a regex pattern.
-                // (??{1}) means "evaluate 1 and use result as pattern" → matches literal "1"
-                // (??{"[x]"}) → matches character class [x]
-                if (isSimpleConstant(codeBlock)) {
-                    String value = evaluateSimpleConstant(codeBlock);
-                    if (value != null) {
-                        // Insert the constant value as a non-capturing group pattern.
-                        // Run through handleRegex to process any regex constructs
-                        // (e.g. (?[...]) from regex_sets transformation).
-                        sb.append("(?:");
-                        handleRegex(value, 0, sb, regexFlags, false);
-                    } else {
-                        // Fallback: empty non-capturing group
-                        sb.append("(?:");
-                    }
-                } else {
-                    // Non-constant: replace with empty non-capturing group
-                    sb.append("(?:");
-                }
+                sb.append("(?:");
 
                 // offset now points at ')' closing the (??{...}) construct
                 // Fall through to common ')' handling at end of handleParentheses

src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java

@@ -227,9 +227,6 @@ static void updateControlVerbVariables(String mark, String error) {
     // 0 = off, 1 = debug, 2 = debugcolor. Captured at the regex call site.
     private int lexicalDebugMode;
     private boolean lexicalReStrict;
-    private static final String DYNAMIC_PATTERN_ERROR =
-            "\u0000(??{...}) recursive regex patterns not implemented (dynamic pattern)";
-
     public RuntimeRegex() {
         this.regexFlags = null;
     }
@@ -450,9 +447,6 @@ private void emitWarningsOnUse() {
                 && !activeCodeEnablesRegexp) {
             return;
         }
-        if (warningsOnUse.contains(DYNAMIC_PATTERN_ERROR)) {
-            throw new PerlJavaUnimplementedException(DYNAMIC_PATTERN_ERROR.substring(1));
-        }
         for (String warning : warningsOnUse) {
             WarnDie.warnWithCategory(new RuntimeScalar(warning), RuntimeScalarCache.scalarEmptyString, "regexp");
         }
@@ -597,22 +591,6 @@ private static synchronized RuntimeRegex compileSynchronized(
 
         String originalPatternString = patternString;
         String compilePatternString = patternString;
-        boolean hasDynamicPattern = compilePatternString != null
-                && compilePatternString.contains(RegexMarkers.RECURSIVE_PATTERN);
-        boolean warnOnUnimplemented = "warn".equals(
-                GlobalVariable.getGlobalHash("main::ENV")
-                        .get("JPERL_UNIMPLEMENTED").toString());
-        boolean hasDeferredDynamicPattern = hasDynamicPattern && !warnOnUnimplemented;
-        boolean hasWarnDynamicFallback = hasDynamicPattern && warnOnUnimplemented;
-        if (hasDeferredDynamicPattern || hasWarnDynamicFallback) {
-            // Perl permits qr// construction before the dynamic callback is needed.
-            // Default mode keeps a never-matching placeholder and reports the hard
-            // error on use. Warn mode retains the historical compatibility fallback
-            // that ignores the unsupported dynamic component after warning.
-            compilePatternString = compilePatternString.replace(
-                    RegexMarkers.RECURSIVE_PATTERN,
-                    hasWarnDynamicFallback ? "(?:)" : "(?!)");
-        }
         List<String> quoteMetaWarningsOnUse = new ArrayList<>();
         if (compilePatternString != null && compilePatternString.contains("\\Q")) {
             // Interpolated-pattern warnings are lexical diagnostics for each
@@ -621,9 +599,7 @@ private static synchronized RuntimeRegex compileSynchronized(
             quoteMetaWarningsOnUse = RegexQuoteMeta.getWarningsOnUse();
         }
 
-        // Dynamic patterns compile differently in normal and warn modes. Do not
-        // let a placeholder cached in one mode leak into the other. Lexical
-        // regex debugging also changes the compiled representation.
+        // Lexical regex debugging changes the compiled representation.
         // A lexical charname translator may return a different expansion for
         // each compilation of the same spelling. Literal syntax validation is
         // the first leg of one logical compilation: refresh the raw-source
@@ -642,8 +618,7 @@ private static synchronized RuntimeRegex compileSynchronized(
                 + "#bytepattern=" + effectivePatternByteBacked
                 + "#strict=" + lexicalReStrict
                 + (namedCharacterTranslator == null ? "" : "#charnames="
-                        + namedCharacterTranslator.toString())
-                + (hasDynamicPattern ? (warnOnUnimplemented ? "\0warn" : "\0defer") : "");
+                        + namedCharacterTranslator.toString());
 
         // Check if the regex is already cached
         RuntimeRegex regex = refreshLexicalNamedCharacter
@@ -746,12 +721,6 @@ private static synchronized RuntimeRegex compileSynchronized(
                 if (constructionPolicyWarning != null) {
                     regex.inlineModifierWarnings.add(constructionPolicyWarning);
                 }
-                if (hasDeferredDynamicPattern) {
-                    regex.warningsOnUse.add(DYNAMIC_PATTERN_ERROR);
-                } else if (hasWarnDynamicFallback) {
-                    regex.warningsOnUse.add(
-                            "(??{...}) recursive/dynamic regex patterns not implemented\n");
-                }
                 if (usesRecursiveBackend) {
                     regex.recursivePattern = new JoniRegexPattern(compilePatternString,
                             regex.regexFlags, trustedCalloutCount,

src/main/java/org/perlonjava/runtime/runtimetypes/RegexCallbackMutationSnapshot.java

@@ -0,0 +1,83 @@
+package org.perlonjava.runtime.runtimetypes;
+
+import java.util.ArrayDeque;
+import java.util.IdentityHashMap;
+import java.util.Map;
+
+/** Backtracking savepoint for mutations made by a plain {@code (?{...})} block. */
+public final class RegexCallbackMutationSnapshot {
+    private final IdentityHashMap<RuntimeArray, Object> arrays = new IdentityHashMap<>();
+    private final IdentityHashMap<RuntimeHash, Object> hashes = new IdentityHashMap<>();
+    private final IdentityHashMap<RuntimeBase, Boolean> seen = new IdentityHashMap<>();
+
+    private RegexCallbackMutationSnapshot(RuntimeCode callback) {
+        ArrayDeque<RuntimeBase> work = new ArrayDeque<>();
+        if (callback.closedOverVariables != null) {
+            addAll(work, callback.closedOverVariables.values());
+        }
+        addAll(work, callback.capturedScalars);
+        addAll(work, callback.capturedAggregates);
+        if (callback.ourVariableRegistry != null) {
+            for (Map.Entry<String, String> entry : callback.ourVariableRegistry.entrySet()) {
+                String name = entry.getKey();
+                String packageName = entry.getValue();
+                if (name == null || name.length() < 2 || packageName == null) continue;
+                String fullName = packageName + "::" + name.substring(1);
+                RuntimeBase cell = switch (name.charAt(0)) {
+                    case '@' -> GlobalVariable.getGlobalArray(fullName);
+                    case '%' -> GlobalVariable.getGlobalHash(fullName);
+                    default -> GlobalVariable.getGlobalVariable(fullName);
+                };
+                work.add(cell);
+            }
+        }
+        capture(work);
+    }
+
+    public static RegexCallbackMutationSnapshot capture(RuntimeCode callback) {
+        return new RegexCallbackMutationSnapshot(callback);
+    }
+
+    private void capture(ArrayDeque<RuntimeBase> work) {
+        while (!work.isEmpty()) {
+            RuntimeBase value = work.removeLast();
+            if (value == null || seen.put(value, Boolean.TRUE) != null) continue;
+            if (value instanceof RuntimeScalar scalar) {
+                if (scalar.value instanceof RuntimeArray array) work.add(array);
+                else if (scalar.value instanceof RuntimeHash hash) work.add(hash);
+                else if (scalar.value instanceof RuntimeScalar nested) work.add(nested);
+            } else if (value instanceof RuntimeArray array) {
+                Object state = array.snapshotRegexMutationState();
+                if (state == null) continue;
+                arrays.put(array, state);
+                addAll(work, array.elements);
+            } else if (value instanceof RuntimeHash hash) {
+                Object state = hash.snapshotRegexMutationState();
+                if (state == null) continue;
+                hashes.put(hash, state);
+                addAll(work, hash.elements.values());
+            }
+        }
+    }
+
+    public void restore() {
+        for (Map.Entry<RuntimeArray, Object> entry : arrays.entrySet()) {
+            entry.getKey().restoreRegexMutationState(entry.getValue());
+        }
+        for (Map.Entry<RuntimeHash, Object> entry : hashes.entrySet()) {
+            entry.getKey().restoreRegexMutationState(entry.getValue());
+        }
+        MortalList.flush();
+    }
+
+    private static void addAll(ArrayDeque<RuntimeBase> work,
+                               Iterable<? extends RuntimeBase> values) {
+        if (values == null) return;
+        for (RuntimeBase value : values) if (value != null) work.add(value);
+    }
+
+    private static void addAll(ArrayDeque<RuntimeBase> work, RuntimeBase[] values) {
+        if (values == null) return;
+        for (RuntimeBase value : values) if (value != null) work.add(value);
+    }
+}

src/test/resources/unit/regex/empty_control_verb_diagnostics.t

@@ -0,0 +1,10 @@
+use strict;
+use warnings;
+use Test::More tests => 4;
+
+for my $pattern ('(*)', '(*)b') {
+    my $ok = eval "qr/$pattern/; 1";
+    ok(!$ok, "$pattern is rejected");
+    ok(index($@, "Unknown verb pattern ''") >= 0,
+        "$pattern reports an empty verb pattern");
+}

src/test/resources/unit/regex/incomplete_group_effect_diagnostics.t

@@ -0,0 +1,10 @@
+use strict;
+use warnings;
+use Test::More tests => 4;
+
+for my $pattern ('(?', 'a(?') {
+    my $ok = eval "qr/$pattern/; 1";
+    ok(!$ok, "$pattern is rejected");
+    ok(index($@, 'Sequence (? incomplete') >= 0,
+        "$pattern reports an incomplete group effect");
+}