Phase 36: make Joni the ordinary regex default

dev/implementation/regex.md

@@ -8,11 +8,11 @@ backtracking, captures, conditions, recursion, control verbs, Unicode matching,
 case folding, and matcher-visible callbacks. PerlOnJava owns Perl source policy,
 runtime integration, lexical warnings, diagnostics, and callback closures.
 
-Migration is not complete. Automatic routing still uses Java for ordinary
-patterns unless a Joni-only construct is present. Setting
-`JPERL_REGEX_BACKEND=joni` or the `jperl.regex.backend=joni` system property
-forces Joni and is the compatibility gate used by the unit corpus. The Java
-matcher, selector, and Java-only rewrites are temporary migration scaffolding.
+Migration is not complete. Automatic routing uses Joni for ordinary patterns.
+Setting `JPERL_REGEX_BACKEND=java` or the `jperl.regex.backend=java` system
+property retains the legacy matcher solely as a differential baseline; explicit
+`joni` selects the production route. The Java matcher, selector, and Java-only
+rewrites are temporary migration scaffolding.
 
 ## Compilation and routing
 

dev/import-perl5/config.yaml

@@ -144,9 +144,6 @@ imports:
     target: perl5_t/t/test.pl
     patch: test.pl.patch
 
-  - source: perl5/t/re/pat.t
-    target: perl5_t/t/re/pat.t
-
   - source: perl5/t/porting/manifest.t
     target: perl5_t/t/porting/manifest.t
     patch: manifest.t.patch

src/main/java/org/perlonjava/backend/bytecode/CompileOperator.java

@@ -1128,7 +1128,13 @@ public static void visitOperator(BytecodeCompiler bytecodeCompiler, OperatorNode
                             && !modifiers.contains("u")) {
                         modifiers += "u";
                     }
-                    RuntimeRegex.validateLiteralSyntax(literalPattern, modifiers);
+                    try {
+                        RuntimeRegex.validateLiteralSyntax(literalPattern, modifiers);
+                    } catch (PerlCompilerException exception) {
+                        throw PerlCompilerException.withSourceLocation(
+                                node.tokenIndex, exception.getMessage(),
+                                bytecodeCompiler.errorUtil);
+                    }
                 }
                 boolean needsCallsiteCache = false;
                 Node flagsNode = operand.elements.get(1);

src/main/java/org/perlonjava/frontend/parser/StringSegmentParser.java

@@ -1531,6 +1531,10 @@ void handleUnicodeNameEscape() {
                 appendToCurrentSegment("\\N{" + name + "}");
                 return;
             }
+            if (name.matches("(?i)U\\+[0-9A-F]+(?:\\.[0-9A-F]+)+")) {
+                throwNamedCharacterDiagnostic(
+                        "Invalid hexadecimal number in \\N{U+...}");
+            }
             NamedCharacterExpansion expansion =
                     NamedCharacterExpansion.resolve(name, sourceMode);
             if (expansion.resolved()) {

src/main/java/org/perlonjava/runtime/NamedCharacterExpansion.java

@@ -66,6 +66,19 @@ public static NamedCharacterExpansion resolve(
             if (name.matches("(?i)U\\+[0-9A-F]+")) {
                 return resolveStandard(name);
             }
+            if (name.matches("(?i)U\\+[0-9A-F]+(?:\\.[0-9A-F]+)+")) {
+                try {
+                    StringBuilder sequence = new StringBuilder();
+                    for (String scalar : name.substring(2).split("\\.")) {
+                        sequence.appendCodePoint(Integer.parseInt(scalar, 16));
+                    }
+                    return new NamedCharacterExpansion(
+                            sequence.toString(), SourceMode.UNICODE,
+                            true, Status.RESOLVED, null);
+                } catch (IllegalArgumentException failure) {
+                    // Fall through to Perl's common malformed-U+ diagnostic.
+                }
+            }
             return new NamedCharacterExpansion(
                     "", SourceMode.UNICODE, true, Status.INVALID,
                     "Invalid hexadecimal number in \\N{U+...}");

src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java

@@ -17,6 +17,7 @@
 import org.joni.Syntax;
 import org.joni.WarnCallback;
 import org.joni.WideScalarCodec;
+import org.joni.exception.SyntaxException;
 
 import static org.joni.constants.SyntaxProperties.ALLOW_MULTIPLEX_DEFINITION_NAME_CALL;
 import static org.joni.constants.SyntaxProperties.OP2_ESC_H_HORIZONTAL_WHITESPACE;
@@ -249,7 +250,9 @@ public boolean supportsPositions() {
         Syntax syntax = syntaxForNamedCharacters(
                 namedCharacterCache, namedCharacterSourceMode,
                 flags.isCaseInsensitive());
-        regex = new Regex(bytes, 0, bytes.length, toJoniOptions(flags, forceAsciiClasses),
+        int options = toJoniOptions(flags, forceAsciiClasses);
+        if (byteMode && byteBackedPattern) options |= Option.PERL_BYTE_PATTERN;
+        regex = new Regex(bytes, 0, bytes.length, options,
                 byteMode ? ISO8859_1Encoding.INSTANCE : UTF8Encoding.INSTANCE,
                 syntax, warningCollector);
         NamedGroupMaps groupMaps = collectNamedGroups(regex);
@@ -1063,7 +1066,7 @@ private boolean find(int option, boolean anchored) {
             matcher = regex.matcher(bytes);
             if (!callbacks.isEmpty()) {
                 calloutHandler = new PerlCalloutHandler(
-                        input, byteToChar, callbacks, flags, hasControlVerbState, subject);
+                        input, byteToChar, callbacks, flags, hasControlVerbState, byteMode, subject);
                 matcher.setCalloutHandler(calloutHandler);
             }
             int result;
@@ -1281,6 +1284,7 @@ static CaptureSnapshot of(MatchView match) {
         private final List<RuntimeRegexCallback> callbacks;
         private final RegexFlags outerFlags;
         private final boolean publishesControlVerbState;
+        private final boolean byteMode;
         private final RuntimeScalar subject;
         private final int initialLocalLevel;
         private final RegexState initialRegexState;
@@ -1295,20 +1299,21 @@ static CaptureSnapshot of(MatchView match) {
 
         PerlCalloutHandler(String input, int[] byteToChar, List<RuntimeRegexCallback> callbacks,
                            RegexFlags outerFlags, boolean publishesControlVerbState,
-                           RuntimeScalar subject) {
+                           boolean byteMode, RuntimeScalar subject) {
             this(input, byteToChar, callbacks, outerFlags, publishesControlVerbState,
-                    subject, null);
+                    byteMode, subject, null);
         }
 
         private PerlCalloutHandler(
                 String input, int[] byteToChar, List<RuntimeRegexCallback> callbacks,
                 RegexFlags outerFlags, boolean publishesControlVerbState,
-                RuntimeScalar subject, PerlCalloutHandler parent) {
+                boolean byteMode, RuntimeScalar subject, PerlCalloutHandler parent) {
             this.input = input;
             this.byteToChar = byteToChar;
             this.callbacks = callbacks;
             this.outerFlags = outerFlags;
             this.publishesControlVerbState = publishesControlVerbState;
+            this.byteMode = byteMode;
             this.subject = subject;
             this.parent = parent;
             this.nestedDepth = parent == null ? 0 : parent.nestedDepth + 1;
@@ -1352,7 +1357,8 @@ public DynamicPatternResult executeDynamic(int id, MatchView match) {
                 nestedCallbacks = runtimeRegex.executableCallbacks;
             } else if (value.value instanceof RuntimeRegexTemplate template) {
                 nestedPattern = new JoniRegexPattern(template.pattern(), outerFlags,
-                        template.callbacks().size());
+                        template.callbacks().size(), false,
+                        byteMode && template.byteBackedPattern(), template.byteBackedPattern());
                 nestedCallbacks = template.callbacks();
             } else {
                 String dynamicSource = value.toString();
@@ -1371,7 +1377,18 @@ public DynamicPatternResult executeDynamic(int id, MatchView match) {
                             runtimeRegex.executableCallbacks.size());
                     nestedCallbacks = runtimeRegex.executableCallbacks;
                 } else {
-                    nestedPattern = new JoniRegexPattern(dynamicSource, outerFlags);
+                    try {
+                        nestedPattern = new JoniRegexPattern(dynamicSource, outerFlags);
+                    } catch (SyntaxException exception) {
+                        String message = exception.getMessage();
+                        if (message != null && (message.contains("premature end of char-class")
+                                || message.contains("Unclosed character class"))) {
+                            int open = dynamicSource.indexOf('[');
+                            throw new PerlCompilerException(RegexDiagnosticFormatter.markedPerl(
+                                    dynamicSource, open < 0 ? 0 : open + 1, "Unmatched ["));
+                        }
+                        throw exception;
+                    }
                 }
             }
             CalloutHandler nestedHandler = nestedCallbacks.isEmpty() ? null
@@ -1380,6 +1397,7 @@ public DynamicPatternResult executeDynamic(int id, MatchView match) {
                                     && runtimeRegex.getRegexFlags() != null
                                     ? runtimeRegex.getRegexFlags() : outerFlags,
                             nestedPattern.hasControlVerbState,
+                            byteMode,
                             subject,
                             this);
             if (nestedHandler != null) executedNestedCallbackPattern = true;

src/main/java/org/perlonjava/runtime/regex/RegexBackendPolicy.java

@@ -1,12 +1,11 @@
 package org.perlonjava.runtime.regex;
 
 /**
- * Temporary migration policy for comparing the legacy Java-first routing with
- * the canonical Joni matcher. Ordinary lookbehind also remains on Java until
- * Joni's nested-lookahead admission is complete. Branch-reset subroutine calls
- * also temporarily use Java pending Joni's native named-call patch. Other
- * Joni-only constructs in the same pattern still force Joni. This class and its
- * controls are removed when the Java matching backend is retired.
+ * Temporary migration policy for comparing the canonical Joni matcher with
+ * the legacy Java matcher. Default and auto modes use Joni; explicit Java mode
+ * remains only for differential diagnosis. Constructs unavailable in Java may
+ * still force Joni even in explicit Java mode. This class and its controls are
+ * removed when the Java matching backend is retired.
  */
 final class RegexBackendPolicy {
     static final String PROPERTY = "jperl.regex.backend";
@@ -27,14 +26,14 @@ static Mode current() {
         }
         if (configured == null || configured.isBlank()
                 || configured.equalsIgnoreCase("auto")
-                || configured.equalsIgnoreCase("java")) {
-            return Mode.JAVA;
-        }
-        if (configured.equalsIgnoreCase("joni")) {
+                || configured.equalsIgnoreCase("joni")) {
             return Mode.JONI;
         }
+        if (configured.equalsIgnoreCase("java")) {
+            return Mode.JAVA;
+        }
         throw new IllegalArgumentException("Invalid " + ENVIRONMENT + " value '"
-                + configured + "' (expected java or joni)");
+                + configured + "' (expected auto, java, or joni)");
     }
 
     static boolean useJoni(String pattern) {