fix(regex): advance native Joni syntax and fold parity

dev/design/phase36-regex-parity.md

@@ -125,6 +125,17 @@ affected corpus before taking another slice.
 - Negative lookbehind accepts capture enclosures and uses ACCEPT-aware width
   analysis in native Joni; named cut errors remain authoritative before an
   unnamed FAIL. The combined exact `regexp.t` differential has no introductions.
+- Perl `/xx` character-class whitespace and nested inline `x`/`xx` mode changes
+  are native Joni lexer/parser behavior. The corresponding `regexp.t` identities
+  pass without introductions on both execution backends.
+- Reverse full-fold alternatives can repartition across adjacent source
+  literals without changing single-literal lookbehind width. The targeted
+  `regexp.t` identity and the existing literal/backreference fold contract pass
+  on default and forced Joni for JVM and interpreter.
+- Perl's exact `L_` General_Category compatibility spelling resolves as `LC`
+  before loose alias normalization, so uncased letters no longer enter that
+  class. The focused system-Perl oracle, four runtime legs, and imported
+  `regexp.t` identity agree.
 
 ## Execution Phases
 
@@ -236,18 +247,19 @@ behavior.
 
 ## Ordered Next Steps
 
-1. Run one warning-free full build and affected-corpus differential on the
-   integrated nested-quantifier, named-control-verb, and negative-lookbehind
-   batch, then open its review PR and require exact-head Ubuntu/Windows CI.
-2. Complete byte/Unicode pattern provenance through runtime interpolation and
-   template composition, then finish `/d`/`u`/`a`/`aa` forward/reverse literal
-   and backreference folding from generated data. Require direct Joni plus
-   ordinary/forced JVM/interpreter zero-introduction gates.
-3. Implement recursive and runtime `(??{...})` as native nested Joni execution:
+1. Integrate the native `/xx`, adjacent full-fold partition, and `L_` alias
+   batch on the exact merged predecessor. Run one warning-free full build plus
+   zero-introduction `regexp.t` and fold/property gates, then publish a review
+   PR and require exact-head Ubuntu/Windows CI.
+2. Complete recursive and runtime `(??{...})` as native nested Joni execution:
    preserve captures, `$^R`, `pos`, modes, byte/Unicode provenance, callback
    unwind, backtracking re-evaluation, and recursion safety. Route every embedded
    closure to Joni and delete constant inlining, progressive errors, and the
    dynamic Java adapter as their gates pass.
+3. Complete byte/Unicode pattern provenance through runtime interpolation and
+   template composition, then finish `/d`/`u`/`a`/`aa` forward/reverse literal,
+   class, property, and backreference folding from generated data. Require
+   direct Joni plus ordinary/forced JVM/interpreter zero-introduction gates.
 4. Finish the remaining lexical `use re 'strict'`, unescaped-brace, and non-hex
    diagnostic families. Refresh complete
    `reg_mesg.t`, `pat.t`, and `pat_advanced.t` maps after each combined batch.

src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java

@@ -482,6 +482,7 @@ private static int toJoniOptions(RegexFlags flags, boolean forceAsciiClasses) {
         int options = Option.NONE;
         if (flags.isCaseInsensitive()) options |= Option.IGNORECASE;
         if (flags.isExtended()) options |= Option.EXTEND;
+        if (flags.isExtendedWhitespace()) options |= Option.EXTEND | Option.PERL_EXTEND_MORE;
         // Oniguruma's MULTILINE option controls whether dot matches newline.
         if (flags.isDotAll()) options |= Option.MULTILINE;
         if (!flags.isMultiLine()) options |= Option.SINGLELINE;
@@ -730,6 +731,40 @@ private static boolean hasControlVerbState(String pattern) {
                 || pattern.contains("(*COMMIT");
     }
 
+    private static boolean hasInlineExtendedOption(String pattern) {
+        boolean escaped = false;
+        boolean inClass = false;
+        for (int i = 0; i + 2 < pattern.length(); i++) {
+            char ch = pattern.charAt(i);
+            if (escaped) {
+                escaped = false;
+                continue;
+            }
+            if (ch == '\\') {
+                escaped = true;
+                continue;
+            }
+            if (ch == '[') {
+                inClass = true;
+                continue;
+            }
+            if (ch == ']' && inClass) {
+                inClass = false;
+                continue;
+            }
+            if (inClass || ch != '(' || pattern.charAt(i + 1) != '?') continue;
+            for (int j = i + 2; j < pattern.length(); j++) {
+                char option = pattern.charAt(j);
+                if (option == ':' || option == ')') break;
+                if (option == 'x') return true;
+                if (option == '-' || option == '^'
+                        || option >= 'a' && option <= 'z') continue;
+                break;
+            }
+        }
+        return false;
+    }
+
     static String translatePattern(String pattern) {
         return translatePattern(pattern, RegexFlags.fromModifiers("", pattern), 0, true);
     }
@@ -747,6 +782,7 @@ private static String translatePattern(String pattern, RegexFlags flags,
         boolean inClass = false;
         boolean atClassStart = false;
         boolean classAllowsLeadingClose = false;
+        boolean inlineExtendedOption = hasInlineExtendedOption(pattern);
         int posixClassDepth = 0;
         for (int i = 0; i < pattern.length(); i++) {
             char ch = pattern.charAt(i);
@@ -822,7 +858,8 @@ private static String translatePattern(String pattern, RegexFlags flags,
                 i += 2;
                 continue;
             }
-            if (inClass && flags.isExtendedWhitespace() && Character.isWhitespace(ch)) {
+            if (inClass && flags.isExtendedWhitespace() && !inlineExtendedOption
+                    && Character.isWhitespace(ch)) {
                 continue;
             }
             if (inClass && atClassStart && ch == '^') {

src/main/java/org/perlonjava/runtime/regex/RegexFlags.java

@@ -223,7 +223,8 @@ public String toFlagString() {
         if (isMultiLine) flagString.append('m');
         if (isDotAll) flagString.append('s');
         if (isCaseInsensitive) flagString.append('i');
-        if (isExtended) flagString.append('x');
+        if (isExtendedWhitespace) flagString.append("xx");
+        else if (isExtended) flagString.append('x');
         if (isNonCapturing) flagString.append('n');
         if (isNonDestructive) flagString.append('r');
         if (taintResults) flagString.append('T');
@@ -243,7 +244,8 @@ public String toModifierString() {
         if (isMultiLine) sb.append('m');
         if (isDotAll) sb.append('s');
         if (isCaseInsensitive) sb.append('i');
-        if (isExtended) sb.append('x');
+        if (isExtendedWhitespace) sb.append("xx");
+        else if (isExtended) sb.append('x');
         if (isNonCapturing) sb.append('n');
         return sb.toString();
     }

src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java

@@ -2325,6 +2325,13 @@ private static UnicodeSet resolvePerlBareGeneralCategory(String property) {
         String looseIsValue = looseIsShortcutValue(property);
         String alias = looseIsValue == null ? property : looseIsValue;
 
+        // L_ is Perl's compatibility spelling for LC (cased letters).  Its
+        // trailing underscore is significant even though ordinary Unicode
+        // property aliases otherwise use loose matching.
+        if (alias.trim().equalsIgnoreCase("L_")) {
+            return PerlUnicodeGeneralCategoryData.resolve("LC");
+        }
+
         // Perl's shared bare namespace gives scripts and binary properties
         // precedence over General_Category compatibility names. Blocks are
         // considered afterward by resolvePerlBareBlockShortcut.

src/test/resources/unit/regex/casefold_adjacent_multifold_partition.t

@@ -0,0 +1,14 @@
+use strict;
+use warnings;
+use Test::More;
+
+like("\x{00DF}s", qr/^s\x{00DF}$/iu,
+    'adjacent sharp-s folds can repartition three s characters');
+like("s\x{00DF}", qr/^\x{00DF}s$/iu,
+    'reverse adjacent sharp-s folds can repartition three s characters');
+unlike("\x{00DF}x", qr/^s\x{00DF}$/iu,
+    'adjacent sharp-s partition still rejects a different suffix');
+unlike("x\x{00DF}", qr/^\x{00DF}s$/iu,
+    'reverse adjacent sharp-s partition still rejects a different prefix');
+
+done_testing;

src/test/resources/unit/regex/extended_more_character_class.t

@@ -0,0 +1,17 @@
+use strict;
+use warnings;
+use Test::More;
+
+ok(' ' =~ /(?x:[a b])/xx, 'scoped single x downgrades outer xx');
+ok(' ' !~ /(?xx:[a b])/x, 'scoped xx ignores class space');
+ok(' ' =~ /(?x)[a b]/xx, 'option-only single x downgrades outer xx');
+ok(' ' !~ /(?xx)[a b]/x, 'option-only xx ignores class space');
+ok(' ' =~ /(?-x:[a b])/xx, 'scoped minus x disables both x levels');
+
+ok("\t" !~ /(?xx:[a	b])/, 'xx ignores an unescaped class tab');
+ok("\n" =~ /(?xx:[a
+b])/, 'xx preserves an unescaped class newline');
+ok('#' =~ /(?xx:[a#b])/, 'xx preserves a class hash');
+ok(' ' =~ /(?xx:[a\ b])/, 'xx preserves escaped class space');
+
+done_testing;

src/test/resources/unit/regex/unicode_l_compatibility_alias.t

@@ -0,0 +1,18 @@
+use strict;
+use warnings;
+use Test::More tests => 8;
+
+my $upper = "A";
+my $lower = "a";
+my $title = chr(0x01C5);
+my $ideograph = chr(0x3400);
+
+ok($upper =~ /^\p{L_}$/, 'L_ matches an uppercase letter');
+ok($lower =~ /^\p{L_}$/, 'L_ matches a lowercase letter');
+ok($title =~ /^\p{L_}$/, 'L_ matches a titlecase letter');
+ok($ideograph !~ /^\p{L_}$/, 'L_ rejects an uncased letter');
+
+ok($upper !~ /^\P{L_}$/, 'negated L_ rejects an uppercase letter');
+ok($lower !~ /^\P{L_}$/, 'negated L_ rejects a lowercase letter');
+ok($title !~ /^\P{L_}$/, 'negated L_ rejects a titlecase letter');
+ok($ideograph =~ /^\P{L_}$/, 'negated L_ matches an uncased letter');

src/test/resources/unit/regex/unicode_property_follower_diagnostics.t

@@ -0,0 +1,19 @@
+use strict;
+use warnings;
+use Test::More tests => 7;
+
+for my $case (
+    [ '\\p A', 'p' ],
+    [ '\\P:',  'P' ],
+    [ '\\p^',  'p' ],
+) {
+    my ($pattern, $escape) = @$case;
+    my $ok = eval "qr/$pattern/; 1";
+    ok(!$ok, "$pattern is rejected");
+    my $expected = "Character following \\$escape must be '{' or a single-character Unicode property name";
+    ok(index($@, $expected) >= 0,
+        "$pattern reports its invalid property follower");
+}
+
+my $ok = eval 'qr/\\p/; 1';
+ok(!$ok && $@ =~ /Empty \\p/, 'bare property escape retains its empty diagnostic');

third_party/joni/src/org/joni/Analyser.java

@@ -33,6 +33,7 @@
 import static org.joni.ast.QuantifierNode.isRepeatInfinite;
 
 import java.util.IllegalFormatConversionException;
+import java.util.ArrayList;
 
 import org.jcodings.CaseFoldCodeItem;
 import org.jcodings.Encoding;
@@ -2139,11 +2140,98 @@ private Node expandCaseFoldString(Node node) {
         }
         /* ending */
         Node xnode = topRoot != null ? topRoot : prevNode.p;
+        xnode = addPerlReverseFoldPartitions(sn, xnode);
 
         node.replaceWith(xnode);
         return xnode;
     }
 
+    private Node addPerlReverseFoldPartitions(StringNode source, Node expanded) {
+        if (!syntax.op2OptionPerl() || Option.isPerlAsciiStrict(regex.options)
+                || Option.isPerlBytePattern(regex.options)) {
+            return expanded;
+        }
+
+        int sourceLength = 0;
+        int canonicalLength = 0;
+        boolean hasFullFold = false;
+        for (int p = source.p; p < source.end;) {
+            int codePoint = enc.mbcToCode(source.bytes, p, source.end);
+            int fullLength = PerlCaseFold.fullFoldLength(codePoint);
+            canonicalLength += fullLength == 0 ? 1 : fullLength;
+            hasFullFold |= fullLength > 1;
+            sourceLength++;
+            p += enc.length(source.bytes, p, source.end);
+        }
+        if (!hasFullFold || sourceLength < 2 || canonicalLength <= sourceLength) {
+            return expanded;
+        }
+
+        int[] canonical = new int[canonicalLength];
+        int offset = 0;
+        for (int p = source.p; p < source.end;) {
+            int codePoint = enc.mbcToCode(source.bytes, p, source.end);
+            int fullLength = PerlCaseFold.fullFoldLength(codePoint);
+            if (fullLength == 0) {
+                canonical[offset++] = codePoint < 0x80
+                        ? Character.toLowerCase(codePoint) : codePoint;
+            } else {
+                for (int index = 0; index < fullLength; index++) {
+                    canonical[offset++] = PerlCaseFold.fullFoldCodePoint(
+                            codePoint, index);
+                }
+            }
+            p += enc.length(source.bytes, p, source.end);
+        }
+
+        ArrayList<int[]> variants = new ArrayList<>();
+        collectPerlReverseFoldPartitions(canonical, 0, new int[canonical.length],
+                0, false, variants);
+        if (variants.isEmpty()) return expanded;
+
+        ListNode alternatives = newAlt(expanded, null);
+        ListNode tail = alternatives;
+        for (int[] variant : variants) {
+            StringNode candidate = new StringNode();
+            for (int codePoint : variant) candidate.catCode(codePoint, enc);
+            candidate.setRaw();
+            ListNode alternative = newAlt(candidate, null);
+            tail.setTail(alternative);
+            tail = alternative;
+        }
+        return alternatives;
+    }
+
+    private void collectPerlReverseFoldPartitions(int[] canonical, int offset,
+            int[] path, int pathLength, boolean usedReverse,
+            ArrayList<int[]> variants) {
+        if (variants.size() >= THRESHOLD_CASE_FOLD_ALT_FOR_EXPANSION) return;
+        if (offset == canonical.length) {
+            if (usedReverse) {
+                variants.add(java.util.Arrays.copyOf(path, pathLength));
+            }
+            return;
+        }
+
+        path[pathLength] = canonical[offset];
+        collectPerlReverseFoldPartitions(canonical, offset + 1, path,
+                pathLength + 1, usedReverse, variants);
+        for (int length = 2; length <= 3 && offset + length <= canonical.length;
+                length++) {
+            int count = PerlCaseFold.reverseFullFoldSourceCount(
+                    canonical, offset, length);
+            for (int index = 0; index < count; index++) {
+                path[pathLength] = PerlCaseFold.reverseFullFoldSourceAt(
+                        canonical, offset, length, index);
+                collectPerlReverseFoldPartitions(canonical, offset + length,
+                        path, pathLength + 1, true, variants);
+                if (variants.size() >= THRESHOLD_CASE_FOLD_ALT_FOR_EXPANSION) {
+                    return;
+                }
+            }
+        }
+    }
+
     private Node expandPerlByteAsciiFoldString(StringNode source, int state) {
         ListNode root = null;
         ListNode sequenceTail = null;

third_party/joni/src/org/joni/Lexer.java

@@ -1128,12 +1128,17 @@ protected final TokenType fetchTokenInCC() {
         if (perlVerticalWhitespaceTokenIndex >= 0) {
             return fetchPerlVerticalWhitespaceToken();
         }
-        if (!left()) {
-            token.type = TokenType.EOT;
-            return token.type;
+        while (true) {
+            if (!left()) {
+                token.type = TokenType.EOT;
+                return token.type;
+            }
+            fetch();
+            if (!syntax.op2OptionPerl() || !Option.isPerlExtendMore(env.option)
+                    || c != ' ' && c != '\t') {
+                break;
+            }
         }
-
-        fetch();
         token.type = TokenType.CHAR;
         token.base = 0;
         token.setC(c);
@@ -1635,8 +1640,10 @@ private void fetchTokenFor_charProperty() {
             token.setPropCType(enc.propertyNameToCType(bytes, nameStart, p));
             token.setPropNot(c == 'P');
         } else if (syntax.op2OptionPerl()) {
-            newSyntaxException(PERL_EMPTY_CHARACTER_PROPERTY.replace(
-                    "%n", Character.toString(c)));
+            String message = left()
+                    ? PERL_INVALID_CHARACTER_PROPERTY_FOLLOWER
+                    : PERL_EMPTY_CHARACTER_PROPERTY;
+            newSyntaxException(message.replace("%n", Character.toString(c)));
         } else {
             syntaxWarn("invalid Unicode Property \\<%n>", (char)c);
         }

third_party/joni/src/org/joni/Option.java

@@ -50,8 +50,10 @@ public final class Option {
     public static final int PERL_ASCII_STRICT    = (1 << 19);
     /** Perl /d byte strings use single-character Latin-1 folding only. */
     public static final int PERL_BYTE_PATTERN    = (1 << 20);
+    /** Perl /xx: EXTEND plus unescaped horizontal-space elision in classes. */
+    public static final int PERL_EXTEND_MORE     = (1 << 21);
 
-    public static final int MAXBIT               = (1 << 21); /* limit */
+    public static final int MAXBIT               = (1 << 22); /* limit */
 
     public static final int DEFAULT              = NONE;
 
@@ -72,6 +74,7 @@ public static String toString(int option) {
         if (isCR7Bit(option)) options += "CR_7_BIT";
         if (isPerlAsciiStrict(option)) options += "PERL_ASCII_STRICT";
         if (isPerlBytePattern(option)) options += "PERL_BYTE_PATTERN";
+        if (isPerlExtendMore(option)) options += "PERL_EXTEND_MORE";
         return options;
     }
 
@@ -83,6 +86,10 @@ public static boolean isExtend(int option) {
         return (option & EXTEND) != 0;
     }
 
+    public static boolean isPerlExtendMore(int option) {
+        return (option & PERL_EXTEND_MORE) != 0;
+    }
+
     public static boolean isSingleline(int option) {
         return (option & SINGLELINE) != 0;
     }