refactor(regex): make Joni property folding explicit

docs/reference/feature-matrix.md

@@ -379,7 +379,7 @@ my @copy = @{$z};         # ERROR
 - ✅  **Backreferences to Named Groups**: Using `\k<name>` or `\g{name}` for backreferences to named groups is supported.
 - ✅  **Relative Backreferences**: Using `\g{-n}` for relative backreferences.
 - ✅  **Basic Unicode Properties**: Common `\p{...}` and `\P{...}` forms such as `\p{L}` execute through Joni. General_Category assignments now enter the forked Joni parser unchanged and resolve to pinned Perl ranges there.
-- 🟡  **Perl Unicode Property Syntax**: Perl-specific properties execute through Joni. `Age`, cumulative `In`/`Present_In`, General_Category, Canonical_Combining_Class, Bidi_Class, Decomposition_Type, East_Asian_Width, Numeric_Value, Joining_Group, Block, Script, and Script_Extensions assignments are generated from pinned Perl 5.44 Unicode 17.0 data with loose and wildcard aliases, ordered missing defaults, official compact aliases and shortcuts, Script-versus-Script_Extensions policy, reserved or composite values, exact rationals, and Perl's generated decimal keyword aliases; `ASCII_Hex_Digit`/`AHex` accepts Perl's eight boolean value aliases. General_Category uses Joni's range-resolver API; the other families remain adapter-translated until their family-specific case-fold and wildcard semantics are represented natively. Other generated property/value aliases still have pinned acceptance/rejection gaps.
+- 🟡  **Perl Unicode Property Syntax**: Perl-specific properties execute through Joni. `Age`, cumulative `In`/`Present_In`, General_Category, Canonical_Combining_Class, Bidi_Class, Decomposition_Type, East_Asian_Width, Numeric_Value, Joining_Group, Block, Script, and Script_Extensions assignments are generated from pinned Perl 5.44 Unicode 17.0 data with loose and wildcard aliases, ordered missing defaults, official compact aliases and shortcuts, Script-versus-Script_Extensions policy, reserved or composite values, exact rationals, and Perl's generated decimal keyword aliases; `ASCII_Hex_Digit`/`AHex` accepts Perl's eight boolean value aliases. General_Category and standalone Block, Script, and Script_Extensions assignments use Joni's range-resolver API with explicit per-family case-fold policy. No-fold properties inside composed character classes and wildcard values remain adapter-translated until Joni represents those syntax semantics natively. Other generated property/value aliases still have pinned acceptance/rejection gaps.
 - ✅  **Possessive Quantifiers**: Quantifiers like `*+`, `++`, `?+`, and `{n,m}+`, which disable backtracking, are supported.
 - ✅  **Atomic Grouping**: Use of `(?>...)` for atomic groups is supported.
 - ✅  **`\K` assertion**: Keep left — in `s///`, text before `\K` is preserved; match variables reflect only the portion after `\K`. Ordinary KEEP assertions route through native Joni and no longer use the Java marker rewrite; the adapter still rejects KEEP inside lookaround until the Joni analyser emits Perl's diagnostic directly.

src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java

@@ -6,6 +6,7 @@
 import org.joni.Matcher;
 import org.joni.CalloutHandler;
 import org.joni.CalloutResult;
+import org.joni.CharacterPropertyResolver;
 import org.joni.DynamicPatternResult;
 import org.joni.MatchView;
 import org.joni.NameEntry;
@@ -60,12 +61,13 @@ private static int resolveNamedCharacter(byte[] bytes, int p, int end,
                         ? StandardCharsets.ISO_8859_1 : StandardCharsets.UTF_8));
     }
 
-    private static int[] resolveCharacterProperty(byte[] bytes, int p, int end,
-                                                  Encoding encoding) {
+    private static CharacterPropertyResolver.Result resolveCharacterProperty(
+            byte[] bytes, int p, int end, Encoding encoding,
+            boolean inCharacterClass) {
         String property = new String(bytes, p, end - p,
                 encoding == ISO8859_1Encoding.INSTANCE
                         ? StandardCharsets.ISO_8859_1 : StandardCharsets.UTF_8);
-        return UnicodeResolver.resolveJoniPropertyRanges(property);
+        return UnicodeResolver.resolveJoniProperty(property, inCharacterClass);
     }
 
     private final Regex regex;
@@ -231,8 +233,9 @@ private static UserPropertyTranslation translateUserDefinedProperties(
             boolean frontendProperty = unnegated.matches(
                     "(?i)^(?:script|sc|block|blk|age|in|present[_ ]?in)\\s*(?:=|:(?!:)).*");
             boolean perlBuiltInAlias = UnicodeResolver.isPerlBuiltInPropertyAlias(unnegated);
-            boolean joniResolvedProperty = UnicodeResolver.resolveJoniPropertyRanges(
-                    unnegated) != null;
+            boolean joniResolvedProperty = UnicodeResolver.resolveJoniProperty(
+                    unnegated, extendedClassBracketDepth > 0
+                            || standardClassBracketDepth > 0) != null;
             if (!userDefined && joniResolvedProperty
                     && (frontendProperty || scriptExtensions || perlBuiltInAlias)) {
                 translated.append(pattern, i, end + 1);

src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java

@@ -3,6 +3,7 @@
 import com.ibm.icu.lang.UCharacter;
 import com.ibm.icu.lang.UProperty;
 import com.ibm.icu.text.UnicodeSet;
+import org.joni.CharacterPropertyResolver;
 import org.perlonjava.app.scriptengine.PerlLanguageProvider;
 import org.perlonjava.runtime.runtimetypes.*;
 
@@ -978,14 +979,35 @@ static boolean isPerlBuiltInPropertyAlias(String property) {
                 || resolvePerlBuiltInPropertyAlias(property) != null;
     }
 
-    /** Returns pinned Perl-property ranges in Joni's native range-array format. */
-    static int[] resolveJoniPropertyRanges(String property) {
+    /** Returns pinned Perl-property ranges and their native Joni fold policy. */
+    static CharacterPropertyResolver.Result resolveJoniProperty(
+            String property, boolean inCharacterClass) {
         if (property == null) return null;
         int assignment = propertyValueDelimiter(property);
-        if (assignment <= 0 || assignment == property.length() - 1
-                || !isGeneralCategoryProperty(property.substring(0, assignment))) {
+        if (assignment <= 0 || assignment == property.length() - 1) {
             return null;
         }
+        String name = property.substring(0, assignment);
+        String value = property.substring(assignment + 1);
+        boolean caseFold;
+        if (isGeneralCategoryProperty(name)) {
+            caseFold = true;
+        } else if (PerlUnicodeBlockData.isPropertyAlias(name)) {
+            if (perlBlockWildcardBody(value) != null) return null;
+            caseFold = false;
+        } else if (PerlUnicodeScriptData.isScriptPropertyAlias(name)
+                || PerlUnicodeScriptData.isScriptExtensionsPropertyAlias(name)) {
+            if (perlNumericWildcardBody(value) != null) return null;
+            caseFold = false;
+        } else {
+            return null;
+        }
+
+        // Joni currently folds a complete bracket expression as one class.
+        // Keep no-fold families translated by the adapter inside brackets until
+        // the AST can retain per-property fold policy through class composition.
+        if (!caseFold && inCharacterClass) return null;
+
         UnicodeSet set = resolvePerlBuiltInPropertyAlias(property);
         if (set == null) return null;
 
@@ -995,7 +1017,7 @@ static int[] resolveJoniPropertyRanges(String property) {
             ranges[i * 2 + 1] = set.getRangeStart(i);
             ranges[i * 2 + 2] = set.getRangeEnd(i);
         }
-        return ranges;
+        return new CharacterPropertyResolver.Result(ranges, caseFold);
     }
 
     private static boolean isPerlSpecialPropertyAlias(String property) {

src/test/java/org/perlonjava/runtime/regex/JoniRegexPatternTest.java

@@ -115,6 +115,18 @@ void passesPinnedGeneralCategoriesToTheJoniParserWithoutTextExpansion() {
         assertFalse(foldedNegated.matcher("A", java.util.List.of()).find());
     }
 
+    @Test
+    void preservesStandaloneBlockAndScriptFoldPolicyInsideJoni() {
+        RegexFlags ignoreCase = RegexFlags.fromModifiers("i", "property");
+        JoniRegexPattern block = new JoniRegexPattern("\\p{Block=ASCII}", ignoreCase);
+        JoniRegexPattern script = new JoniRegexPattern("\\p{Script=Common}", ignoreCase);
+
+        assertEquals("\\p{Block=ASCII}", block.patternDescription());
+        assertEquals("\\p{Script=Common}", script.patternDescription());
+        assertFalse(block.matcher("\u212A", java.util.List.of()).find());
+        assertFalse(script.matcher("K", java.util.List.of()).find());
+    }
+
     @Test
     void flattensTranslatedPropertiesInsideOrdinaryCharacterClasses() {
         JoniRegexPattern pattern = new JoniRegexPattern(

src/test/resources/unit/regex_joni_unicode_property_fold_policy.t

@@ -0,0 +1,18 @@
+use strict;
+use warnings;
+use utf8;
+use Test::More tests => 8;
+
+ok("A" =~ /\p{gc=Uppercase_Letter}/, 'general category positive');
+ok("a" !~ /\p{gc=Uppercase_Letter}/, 'general category negative');
+ok("é" =~ /\p{Script=Latin}/, 'script property in pinned Perl data');
+ok("a" =~ /\p{gc=Uppercase_Letter}/i,
+    'general category participates in case folding');
+ok("A" !~ /\P{gc=Uppercase_Letter}/i,
+    'negated category complements after case folding');
+ok("\x{212A}" !~ /\p{Block=ASCII}/i,
+    'block membership does not gain case-fold members');
+ok("K" !~ /\p{Script=Common}/i,
+    'script membership does not gain case-fold members');
+ok("k" =~ /\p{gc=Uppercase_Letter}/i,
+    'general-category membership gains case-fold members');

third_party/joni/src/org/joni/CharacterPropertyResolver.java

@@ -24,9 +24,23 @@
 /** Resolves syntax-specific character properties to inclusive code-point ranges. */
 @FunctionalInterface
 public interface CharacterPropertyResolver {
+    /** A resolved range set and whether ignore-case folding applies to it. */
+    final class Result {
+        public final int[] ranges;
+        public final boolean caseFold;
+
+        public Result(int[] ranges, boolean caseFold) {
+            this.ranges = ranges;
+            this.caseFold = caseFold;
+        }
+    }
+
     /**
-     * Returns {@code [count, from1, to1, ...]} for a resolved property, or
-     * {@code null} to use the encoding's built-in property lookup.
+     * Returns resolved ranges and their ignore-case policy, or {@code null} to
+     * use the encoding's built-in property lookup. The context flag allows a
+     * resolver to defer properties whose class-composition semantics require
+     * frontend handling.
      */
-    int[] resolve(byte[] bytes, int p, int end, Encoding encoding);
+    Result resolve(byte[] bytes, int p, int end, Encoding encoding,
+                   boolean inCharacterClass);
 }

third_party/joni/src/org/joni/Lexer.java

@@ -1698,30 +1698,34 @@ private void possessiveCheck() {
     protected static final class CharProperty {
         final int ctype;
         final int[] ranges;
+        final boolean caseFold;
 
-        CharProperty(int ctype, int[] ranges) {
+        CharProperty(int ctype, int[] ranges, boolean caseFold) {
             this.ctype = ctype;
             this.ranges = ranges;
+            this.caseFold = caseFold;
         }
     }
 
-    protected final CharProperty fetchCharProperty() {
+    protected final CharProperty fetchCharProperty(boolean inCharacterClass) {
         mark();
 
         while (left()) {
             int last = p;
             fetch();
             if (c == '}') {
                 if (syntax.characterPropertyResolver != null) {
-                    int[] ranges = syntax.characterPropertyResolver.resolve(
-                            bytes, _p, last, enc);
-                    if (ranges != null) {
-                        validateCharacterPropertyRanges(ranges);
-                        return new CharProperty(0, ranges);
+                    CharacterPropertyResolver.Result resolved =
+                            syntax.characterPropertyResolver.resolve(
+                                    bytes, _p, last, enc, inCharacterClass);
+                    if (resolved != null) {
+                        validateCharacterPropertyRanges(resolved.ranges);
+                        return new CharProperty(0, resolved.ranges,
+                                resolved.caseFold);
                     }
                 }
                 return new CharProperty(
-                        enc.propertyNameToCType(bytes, _p, last), null);
+                        enc.propertyNameToCType(bytes, _p, last), null, true);
             } else if (c == '(' || c == ')' || c == '{' || c == '|') {
                 throw new CharacterPropertyException(EncodingError.ERR_INVALID_CHAR_PROPERTY_NAME, bytes, _p, last);
             }

third_party/joni/src/org/joni/Parser.java

@@ -256,7 +256,7 @@ private CClassNode parseCharClass(ObjPtr<CClassNode> ascNode) {
                 break;
 
             case CHAR_PROPERTY:
-                CharProperty property = fetchCharProperty();
+                CharProperty property = fetchCharProperty(true);
                 addCharProperty(cc, ascCc, property, token.getPropNot());
                 cc.nextStateClass(arg, ascCc, env); // goto next_class
                 break;
@@ -1629,13 +1629,13 @@ private Node cClassCaseFold(Node node, CClassNode cc, CClassNode ascCc) {
     }
 
     private Node parseCharProperty() {
-        CharProperty property = fetchCharProperty();
+        CharProperty property = fetchCharProperty(false);
         CClassNode cc = new CClassNode();
         Node node = cc;
         addCharProperty(cc, null, property, false);
         if (token.getPropNot()) cc.setNot();
 
-        if (isIgnoreCase(env.option)) {
+        if (isIgnoreCase(env.option) && property.caseFold) {
             if (property.ranges != null || property.ctype != CharacterType.ASCII) {
                 node = cClassCaseFold(node, cc, cc);
             }

third_party/joni/test/org/joni/test/TestCharacterPropertyResolver.java

@@ -34,10 +34,13 @@
 
 public class TestCharacterPropertyResolver {
     private static final CharacterPropertyResolver RESOLVER =
-            (bytes, p, end, encoding) -> {
+            (bytes, p, end, encoding, inCharacterClass) -> {
                 String name = new String(bytes, p, end - p, StandardCharsets.UTF_8);
                 return switch (name) {
-                    case "Fake" -> new int[] {2, 'A', 'A', 0x1f642, 0x1f642};
+                    case "Fake" -> new CharacterPropertyResolver.Result(
+                            new int[] {2, 'A', 'A', 0x1f642, 0x1f642}, true);
+                    case "FakeNoFold" -> new CharacterPropertyResolver.Result(
+                            new int[] {1, 'A', 'A'}, false);
                     default -> null;
                 };
             };
@@ -69,6 +72,8 @@ public void resolvesRangesInsideAndOutsideCharacterClasses() {
         assertEquals(-1, search("[\\P{Fake}]", "A"));
         assertEquals(0, search("(?i)\\p{Fake}", "a"));
         assertEquals(-1, search("(?i)\\P{Fake}", "A"));
+        assertEquals(0, search("(?i)\\p{FakeNoFold}", "A"));
+        assertEquals(-1, search("(?i)\\p{FakeNoFold}", "a"));
     }
 
     @Test
@@ -81,7 +86,9 @@ public void fallsBackToEncodingProperties() {
     public void preservesResolverExceptions() {
         IllegalArgumentException expected = new IllegalArgumentException("failure");
         try {
-            compile("\\p{Fake}", (bytes, p, end, encoding) -> { throw expected; });
+            compile("\\p{Fake}", (bytes, p, end, encoding, inCharacterClass) -> {
+                throw expected;
+            });
             fail("expected resolver exception");
         } catch (IllegalArgumentException error) {
             assertSame(expected, error);
@@ -91,7 +98,8 @@ public void preservesResolverExceptions() {
     @Test
     public void rejectsMalformedRangeResults() {
         try {
-            compile("\\p{Fake}", (bytes, p, end, encoding) -> new int[] {1, 2});
+            compile("\\p{Fake}", (bytes, p, end, encoding, inCharacterClass) ->
+                    new CharacterPropertyResolver.Result(new int[] {1, 2}, true));
             fail("expected invalid range result");
         } catch (IllegalArgumentException error) {
             assertEquals("invalid character property ranges", error.getMessage());