From d862449936bf5454d98553418434e6752e193fea Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Tue, 18 Aug 2026 19:13:45 +0200 Subject: [PATCH 1/9] refactor(regex): make Joni property folding explicit Carry per-property case-fold policy through the forked Joni resolver and parse standalone pinned Block, Script, and Script_Extensions assignments natively. Keep no-fold properties inside composed classes in the adapter until Joni can preserve member policy through class algebra. Generated with [OpenAI Codex](https://openai.com/codex) Co-Authored-By: OpenAI Codex --- docs/reference/feature-matrix.md | 2 +- .../runtime/regex/JoniRegexPattern.java | 13 +++++--- .../runtime/regex/UnicodeResolver.java | 32 ++++++++++++++++--- .../runtime/regex/JoniRegexPatternTest.java | 12 +++++++ .../regex_joni_unicode_property_fold_policy.t | 18 +++++++++++ .../org/joni/CharacterPropertyResolver.java | 20 ++++++++++-- third_party/joni/src/org/joni/Lexer.java | 20 +++++++----- third_party/joni/src/org/joni/Parser.java | 6 ++-- .../test/TestCharacterPropertyResolver.java | 16 +++++++--- 9 files changed, 110 insertions(+), 29 deletions(-) create mode 100644 src/test/resources/unit/regex_joni_unicode_property_fold_policy.t diff --git a/docs/reference/feature-matrix.md b/docs/reference/feature-matrix.md index c7bf5550e..a703f659a 100644 --- a/docs/reference/feature-matrix.md +++ b/docs/reference/feature-matrix.md @@ -379,7 +379,7 @@ my @copy = @{$z}; # ERROR - ✅ **Backreferences to Named Groups**: Using `\k` or `\g{name}` for backreferences to named groups is supported. - ✅ **Relative Backreferences**: Using `\g{-n}` for relative backreferences. - ✅ **Basic Unicode Properties**: Common `\p{...}` and `\P{...}` forms such as `\p{L}` execute through Joni. General_Category assignments now enter the forked Joni parser unchanged and resolve to pinned Perl ranges there. -- 🟡 **Perl Unicode Property Syntax**: Perl-specific properties execute through Joni. `Age`, cumulative `In`/`Present_In`, General_Category, Canonical_Combining_Class, Bidi_Class, Decomposition_Type, East_Asian_Width, Numeric_Value, Joining_Group, Block, Script, and Script_Extensions assignments are generated from pinned Perl 5.44 Unicode 17.0 data with loose and wildcard aliases, ordered missing defaults, official compact aliases and shortcuts, Script-versus-Script_Extensions policy, reserved or composite values, exact rationals, and Perl's generated decimal keyword aliases; `ASCII_Hex_Digit`/`AHex` accepts Perl's eight boolean value aliases. General_Category uses Joni's range-resolver API; the other families remain adapter-translated until their family-specific case-fold and wildcard semantics are represented natively. Other generated property/value aliases still have pinned acceptance/rejection gaps. +- 🟡 **Perl Unicode Property Syntax**: Perl-specific properties execute through Joni. `Age`, cumulative `In`/`Present_In`, General_Category, Canonical_Combining_Class, Bidi_Class, Decomposition_Type, East_Asian_Width, Numeric_Value, Joining_Group, Block, Script, and Script_Extensions assignments are generated from pinned Perl 5.44 Unicode 17.0 data with loose and wildcard aliases, ordered missing defaults, official compact aliases and shortcuts, Script-versus-Script_Extensions policy, reserved or composite values, exact rationals, and Perl's generated decimal keyword aliases; `ASCII_Hex_Digit`/`AHex` accepts Perl's eight boolean value aliases. General_Category and standalone Block, Script, and Script_Extensions assignments use Joni's range-resolver API with explicit per-family case-fold policy. No-fold properties inside composed character classes and wildcard values remain adapter-translated until Joni represents those syntax semantics natively. Other generated property/value aliases still have pinned acceptance/rejection gaps. - ✅ **Possessive Quantifiers**: Quantifiers like `*+`, `++`, `?+`, and `{n,m}+`, which disable backtracking, are supported. - ✅ **Atomic Grouping**: Use of `(?>...)` for atomic groups is supported. - ✅ **`\K` assertion**: Keep left — in `s///`, text before `\K` is preserved; match variables reflect only the portion after `\K`. Ordinary KEEP assertions route through native Joni and no longer use the Java marker rewrite; the adapter still rejects KEEP inside lookaround until the Joni analyser emits Perl's diagnostic directly. diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index a8b8cf0c9..a27b7de9f 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -6,6 +6,7 @@ import org.joni.Matcher; import org.joni.CalloutHandler; import org.joni.CalloutResult; +import org.joni.CharacterPropertyResolver; import org.joni.DynamicPatternResult; import org.joni.MatchView; import org.joni.NameEntry; @@ -60,12 +61,13 @@ private static int resolveNamedCharacter(byte[] bytes, int p, int end, ? StandardCharsets.ISO_8859_1 : StandardCharsets.UTF_8)); } - private static int[] resolveCharacterProperty(byte[] bytes, int p, int end, - Encoding encoding) { + private static CharacterPropertyResolver.Result resolveCharacterProperty( + byte[] bytes, int p, int end, Encoding encoding, + boolean inCharacterClass) { String property = new String(bytes, p, end - p, encoding == ISO8859_1Encoding.INSTANCE ? StandardCharsets.ISO_8859_1 : StandardCharsets.UTF_8); - return UnicodeResolver.resolveJoniPropertyRanges(property); + return UnicodeResolver.resolveJoniProperty(property, inCharacterClass); } private final Regex regex; @@ -231,8 +233,9 @@ private static UserPropertyTranslation translateUserDefinedProperties( boolean frontendProperty = unnegated.matches( "(?i)^(?:script|sc|block|blk|age|in|present[_ ]?in)\\s*(?:=|:(?!:)).*"); boolean perlBuiltInAlias = UnicodeResolver.isPerlBuiltInPropertyAlias(unnegated); - boolean joniResolvedProperty = UnicodeResolver.resolveJoniPropertyRanges( - unnegated) != null; + boolean joniResolvedProperty = UnicodeResolver.resolveJoniProperty( + unnegated, extendedClassBracketDepth > 0 + || standardClassBracketDepth > 0) != null; if (!userDefined && joniResolvedProperty && (frontendProperty || scriptExtensions || perlBuiltInAlias)) { translated.append(pattern, i, end + 1); diff --git a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java index d1505bd9d..f67f368c7 100644 --- a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java +++ b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java @@ -3,6 +3,7 @@ import com.ibm.icu.lang.UCharacter; import com.ibm.icu.lang.UProperty; import com.ibm.icu.text.UnicodeSet; +import org.joni.CharacterPropertyResolver; import org.perlonjava.app.scriptengine.PerlLanguageProvider; import org.perlonjava.runtime.runtimetypes.*; @@ -978,14 +979,35 @@ static boolean isPerlBuiltInPropertyAlias(String property) { || resolvePerlBuiltInPropertyAlias(property) != null; } - /** Returns pinned Perl-property ranges in Joni's native range-array format. */ - static int[] resolveJoniPropertyRanges(String property) { + /** Returns pinned Perl-property ranges and their native Joni fold policy. */ + static CharacterPropertyResolver.Result resolveJoniProperty( + String property, boolean inCharacterClass) { if (property == null) return null; int assignment = propertyValueDelimiter(property); - if (assignment <= 0 || assignment == property.length() - 1 - || !isGeneralCategoryProperty(property.substring(0, assignment))) { + if (assignment <= 0 || assignment == property.length() - 1) { return null; } + String name = property.substring(0, assignment); + String value = property.substring(assignment + 1); + boolean caseFold; + if (isGeneralCategoryProperty(name)) { + caseFold = true; + } else if (PerlUnicodeBlockData.isPropertyAlias(name)) { + if (perlBlockWildcardBody(value) != null) return null; + caseFold = false; + } else if (PerlUnicodeScriptData.isScriptPropertyAlias(name) + || PerlUnicodeScriptData.isScriptExtensionsPropertyAlias(name)) { + if (perlNumericWildcardBody(value) != null) return null; + caseFold = false; + } else { + return null; + } + + // Joni currently folds a complete bracket expression as one class. + // Keep no-fold families translated by the adapter inside brackets until + // the AST can retain per-property fold policy through class composition. + if (!caseFold && inCharacterClass) return null; + UnicodeSet set = resolvePerlBuiltInPropertyAlias(property); if (set == null) return null; @@ -995,7 +1017,7 @@ static int[] resolveJoniPropertyRanges(String property) { ranges[i * 2 + 1] = set.getRangeStart(i); ranges[i * 2 + 2] = set.getRangeEnd(i); } - return ranges; + return new CharacterPropertyResolver.Result(ranges, caseFold); } private static boolean isPerlSpecialPropertyAlias(String property) { diff --git a/src/test/java/org/perlonjava/runtime/regex/JoniRegexPatternTest.java b/src/test/java/org/perlonjava/runtime/regex/JoniRegexPatternTest.java index 865286794..13dbc1e3a 100644 --- a/src/test/java/org/perlonjava/runtime/regex/JoniRegexPatternTest.java +++ b/src/test/java/org/perlonjava/runtime/regex/JoniRegexPatternTest.java @@ -115,6 +115,18 @@ void passesPinnedGeneralCategoriesToTheJoniParserWithoutTextExpansion() { assertFalse(foldedNegated.matcher("A", java.util.List.of()).find()); } + @Test + void preservesStandaloneBlockAndScriptFoldPolicyInsideJoni() { + RegexFlags ignoreCase = RegexFlags.fromModifiers("i", "property"); + JoniRegexPattern block = new JoniRegexPattern("\\p{Block=ASCII}", ignoreCase); + JoniRegexPattern script = new JoniRegexPattern("\\p{Script=Common}", ignoreCase); + + assertEquals("\\p{Block=ASCII}", block.patternDescription()); + assertEquals("\\p{Script=Common}", script.patternDescription()); + assertFalse(block.matcher("\u212A", java.util.List.of()).find()); + assertFalse(script.matcher("K", java.util.List.of()).find()); + } + @Test void flattensTranslatedPropertiesInsideOrdinaryCharacterClasses() { JoniRegexPattern pattern = new JoniRegexPattern( diff --git a/src/test/resources/unit/regex_joni_unicode_property_fold_policy.t b/src/test/resources/unit/regex_joni_unicode_property_fold_policy.t new file mode 100644 index 000000000..cee1f2432 --- /dev/null +++ b/src/test/resources/unit/regex_joni_unicode_property_fold_policy.t @@ -0,0 +1,18 @@ +use strict; +use warnings; +use utf8; +use Test::More tests => 8; + +ok("A" =~ /\p{gc=Uppercase_Letter}/, 'general category positive'); +ok("a" !~ /\p{gc=Uppercase_Letter}/, 'general category negative'); +ok("é" =~ /\p{Script=Latin}/, 'script property in pinned Perl data'); +ok("a" =~ /\p{gc=Uppercase_Letter}/i, + 'general category participates in case folding'); +ok("A" !~ /\P{gc=Uppercase_Letter}/i, + 'negated category complements after case folding'); +ok("\x{212A}" !~ /\p{Block=ASCII}/i, + 'block membership does not gain case-fold members'); +ok("K" !~ /\p{Script=Common}/i, + 'script membership does not gain case-fold members'); +ok("k" =~ /\p{gc=Uppercase_Letter}/i, + 'general-category membership gains case-fold members'); diff --git a/third_party/joni/src/org/joni/CharacterPropertyResolver.java b/third_party/joni/src/org/joni/CharacterPropertyResolver.java index dd31598dd..de7bf3fd0 100644 --- a/third_party/joni/src/org/joni/CharacterPropertyResolver.java +++ b/third_party/joni/src/org/joni/CharacterPropertyResolver.java @@ -24,9 +24,23 @@ /** Resolves syntax-specific character properties to inclusive code-point ranges. */ @FunctionalInterface public interface CharacterPropertyResolver { + /** A resolved range set and whether ignore-case folding applies to it. */ + final class Result { + public final int[] ranges; + public final boolean caseFold; + + public Result(int[] ranges, boolean caseFold) { + this.ranges = ranges; + this.caseFold = caseFold; + } + } + /** - * Returns {@code [count, from1, to1, ...]} for a resolved property, or - * {@code null} to use the encoding's built-in property lookup. + * Returns resolved ranges and their ignore-case policy, or {@code null} to + * use the encoding's built-in property lookup. The context flag allows a + * resolver to defer properties whose class-composition semantics require + * frontend handling. */ - int[] resolve(byte[] bytes, int p, int end, Encoding encoding); + Result resolve(byte[] bytes, int p, int end, Encoding encoding, + boolean inCharacterClass); } diff --git a/third_party/joni/src/org/joni/Lexer.java b/third_party/joni/src/org/joni/Lexer.java index 0e26280a8..ca80215d6 100644 --- a/third_party/joni/src/org/joni/Lexer.java +++ b/third_party/joni/src/org/joni/Lexer.java @@ -1698,14 +1698,16 @@ private void possessiveCheck() { protected static final class CharProperty { final int ctype; final int[] ranges; + final boolean caseFold; - CharProperty(int ctype, int[] ranges) { + CharProperty(int ctype, int[] ranges, boolean caseFold) { this.ctype = ctype; this.ranges = ranges; + this.caseFold = caseFold; } } - protected final CharProperty fetchCharProperty() { + protected final CharProperty fetchCharProperty(boolean inCharacterClass) { mark(); while (left()) { @@ -1713,15 +1715,17 @@ protected final CharProperty fetchCharProperty() { fetch(); if (c == '}') { if (syntax.characterPropertyResolver != null) { - int[] ranges = syntax.characterPropertyResolver.resolve( - bytes, _p, last, enc); - if (ranges != null) { - validateCharacterPropertyRanges(ranges); - return new CharProperty(0, ranges); + CharacterPropertyResolver.Result resolved = + syntax.characterPropertyResolver.resolve( + bytes, _p, last, enc, inCharacterClass); + if (resolved != null) { + validateCharacterPropertyRanges(resolved.ranges); + return new CharProperty(0, resolved.ranges, + resolved.caseFold); } } return new CharProperty( - enc.propertyNameToCType(bytes, _p, last), null); + enc.propertyNameToCType(bytes, _p, last), null, true); } else if (c == '(' || c == ')' || c == '{' || c == '|') { throw new CharacterPropertyException(EncodingError.ERR_INVALID_CHAR_PROPERTY_NAME, bytes, _p, last); } diff --git a/third_party/joni/src/org/joni/Parser.java b/third_party/joni/src/org/joni/Parser.java index af9cbfc07..cbfd5a616 100644 --- a/third_party/joni/src/org/joni/Parser.java +++ b/third_party/joni/src/org/joni/Parser.java @@ -256,7 +256,7 @@ private CClassNode parseCharClass(ObjPtr ascNode) { break; case CHAR_PROPERTY: - CharProperty property = fetchCharProperty(); + CharProperty property = fetchCharProperty(true); addCharProperty(cc, ascCc, property, token.getPropNot()); cc.nextStateClass(arg, ascCc, env); // goto next_class break; @@ -1629,13 +1629,13 @@ private Node cClassCaseFold(Node node, CClassNode cc, CClassNode ascCc) { } private Node parseCharProperty() { - CharProperty property = fetchCharProperty(); + CharProperty property = fetchCharProperty(false); CClassNode cc = new CClassNode(); Node node = cc; addCharProperty(cc, null, property, false); if (token.getPropNot()) cc.setNot(); - if (isIgnoreCase(env.option)) { + if (isIgnoreCase(env.option) && property.caseFold) { if (property.ranges != null || property.ctype != CharacterType.ASCII) { node = cClassCaseFold(node, cc, cc); } diff --git a/third_party/joni/test/org/joni/test/TestCharacterPropertyResolver.java b/third_party/joni/test/org/joni/test/TestCharacterPropertyResolver.java index 974df8eea..e03ff4757 100644 --- a/third_party/joni/test/org/joni/test/TestCharacterPropertyResolver.java +++ b/third_party/joni/test/org/joni/test/TestCharacterPropertyResolver.java @@ -34,10 +34,13 @@ public class TestCharacterPropertyResolver { private static final CharacterPropertyResolver RESOLVER = - (bytes, p, end, encoding) -> { + (bytes, p, end, encoding, inCharacterClass) -> { String name = new String(bytes, p, end - p, StandardCharsets.UTF_8); return switch (name) { - case "Fake" -> new int[] {2, 'A', 'A', 0x1f642, 0x1f642}; + case "Fake" -> new CharacterPropertyResolver.Result( + new int[] {2, 'A', 'A', 0x1f642, 0x1f642}, true); + case "FakeNoFold" -> new CharacterPropertyResolver.Result( + new int[] {1, 'A', 'A'}, false); default -> null; }; }; @@ -69,6 +72,8 @@ public void resolvesRangesInsideAndOutsideCharacterClasses() { assertEquals(-1, search("[\\P{Fake}]", "A")); assertEquals(0, search("(?i)\\p{Fake}", "a")); assertEquals(-1, search("(?i)\\P{Fake}", "A")); + assertEquals(0, search("(?i)\\p{FakeNoFold}", "A")); + assertEquals(-1, search("(?i)\\p{FakeNoFold}", "a")); } @Test @@ -81,7 +86,9 @@ public void fallsBackToEncodingProperties() { public void preservesResolverExceptions() { IllegalArgumentException expected = new IllegalArgumentException("failure"); try { - compile("\\p{Fake}", (bytes, p, end, encoding) -> { throw expected; }); + compile("\\p{Fake}", (bytes, p, end, encoding, inCharacterClass) -> { + throw expected; + }); fail("expected resolver exception"); } catch (IllegalArgumentException error) { assertSame(expected, error); @@ -91,7 +98,8 @@ public void preservesResolverExceptions() { @Test public void rejectsMalformedRangeResults() { try { - compile("\\p{Fake}", (bytes, p, end, encoding) -> new int[] {1, 2}); + compile("\\p{Fake}", (bytes, p, end, encoding, inCharacterClass) -> + new CharacterPropertyResolver.Result(new int[] {1, 2}, true)); fail("expected invalid range result"); } catch (IllegalArgumentException error) { assertEquals("invalid character property ranges", error.getMessage()); From 7b0a6834cf493183d1a8062bc7a46c5979e4bfa5 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Tue, 18 Aug 2026 19:19:14 +0200 Subject: [PATCH 2/9] refactor(regex): parse exact property families in Joni Route standalone exact combining-class, bidi, decomposition, width, numeric, and joining-group assignments through Joni's no-fold range resolver while retaining wildcard and composed-class semantics in the adapter. Generated with [OpenAI Codex](https://openai.com/codex) Co-Authored-By: OpenAI Codex --- docs/reference/feature-matrix.md | 2 +- .../runtime/regex/UnicodeResolver.java | 9 ++++++ .../runtime/regex/JoniRegexPatternTest.java | 18 ++++++++++++ ...egex_joni_unicode_property_family_policy.t | 29 +++++++++++++++++++ 4 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 src/test/resources/unit/regex_joni_unicode_property_family_policy.t diff --git a/docs/reference/feature-matrix.md b/docs/reference/feature-matrix.md index a703f659a..938c5ae4b 100644 --- a/docs/reference/feature-matrix.md +++ b/docs/reference/feature-matrix.md @@ -379,7 +379,7 @@ my @copy = @{$z}; # ERROR - ✅ **Backreferences to Named Groups**: Using `\k` or `\g{name}` for backreferences to named groups is supported. - ✅ **Relative Backreferences**: Using `\g{-n}` for relative backreferences. - ✅ **Basic Unicode Properties**: Common `\p{...}` and `\P{...}` forms such as `\p{L}` execute through Joni. General_Category assignments now enter the forked Joni parser unchanged and resolve to pinned Perl ranges there. -- 🟡 **Perl Unicode Property Syntax**: Perl-specific properties execute through Joni. `Age`, cumulative `In`/`Present_In`, General_Category, Canonical_Combining_Class, Bidi_Class, Decomposition_Type, East_Asian_Width, Numeric_Value, Joining_Group, Block, Script, and Script_Extensions assignments are generated from pinned Perl 5.44 Unicode 17.0 data with loose and wildcard aliases, ordered missing defaults, official compact aliases and shortcuts, Script-versus-Script_Extensions policy, reserved or composite values, exact rationals, and Perl's generated decimal keyword aliases; `ASCII_Hex_Digit`/`AHex` accepts Perl's eight boolean value aliases. General_Category and standalone Block, Script, and Script_Extensions assignments use Joni's range-resolver API with explicit per-family case-fold policy. No-fold properties inside composed character classes and wildcard values remain adapter-translated until Joni represents those syntax semantics natively. Other generated property/value aliases still have pinned acceptance/rejection gaps. +- 🟡 **Perl Unicode Property Syntax**: Perl-specific properties execute through Joni. `Age`, cumulative `In`/`Present_In`, General_Category, Canonical_Combining_Class, Bidi_Class, Decomposition_Type, East_Asian_Width, Numeric_Value, Joining_Group, Block, Script, and Script_Extensions assignments are generated from pinned Perl 5.44 Unicode 17.0 data with loose and wildcard aliases, ordered missing defaults, official compact aliases and shortcuts, Script-versus-Script_Extensions policy, reserved or composite values, exact rationals, and Perl's generated decimal keyword aliases; `ASCII_Hex_Digit`/`AHex` accepts Perl's eight boolean value aliases. General_Category and standalone exact Block, Script, Script_Extensions, Canonical_Combining_Class, Bidi_Class, Decomposition_Type, East_Asian_Width, Numeric_Value, and Joining_Group assignments use Joni's range-resolver API with explicit per-family case-fold policy. No-fold properties inside composed character classes, wildcard values, and Age-family assignments remain adapter-translated until Joni represents those syntax semantics natively. Other generated property/value aliases still have pinned acceptance/rejection gaps. - ✅ **Possessive Quantifiers**: Quantifiers like `*+`, `++`, `?+`, and `{n,m}+`, which disable backtracking, are supported. - ✅ **Atomic Grouping**: Use of `(?>...)` for atomic groups is supported. - ✅ **`\K` assertion**: Keep left — in `s///`, text before `\K` is preserved; match variables reflect only the portion after `\K`. Ordinary KEEP assertions route through native Joni and no longer use the Java marker rewrite; the adapter still rejects KEEP inside lookaround until the Joni analyser emits Perl's diagnostic directly. diff --git a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java index f67f368c7..486a970bf 100644 --- a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java +++ b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java @@ -999,6 +999,15 @@ static CharacterPropertyResolver.Result resolveJoniProperty( || PerlUnicodeScriptData.isScriptExtensionsPropertyAlias(name)) { if (perlNumericWildcardBody(value) != null) return null; caseFold = false; + } else if (isCanonicalCombiningClassProperty(name) + || PerlUnicodeBidiClassData.isPropertyAlias(name) + || PerlUnicodeDecompositionTypeData.isPropertyAlias(name) + || PerlUnicodeEastAsianWidthData.isPropertyAlias(name)) { + caseFold = false; + } else if (PerlUnicodeNumericValueData.isPropertyAlias(name) + || PerlUnicodeJoiningGroupData.isPropertyAlias(name)) { + if (perlNumericWildcardBody(value) != null) return null; + caseFold = false; } else { return null; } diff --git a/src/test/java/org/perlonjava/runtime/regex/JoniRegexPatternTest.java b/src/test/java/org/perlonjava/runtime/regex/JoniRegexPatternTest.java index 13dbc1e3a..2bc11c2f5 100644 --- a/src/test/java/org/perlonjava/runtime/regex/JoniRegexPatternTest.java +++ b/src/test/java/org/perlonjava/runtime/regex/JoniRegexPatternTest.java @@ -127,6 +127,24 @@ void preservesStandaloneBlockAndScriptFoldPolicyInsideJoni() { assertFalse(script.matcher("K", java.util.List.of()).find()); } + @Test + void passesRemainingExactEnumeratedPropertiesToJoni() { + String[][] cases = { + {"\\p{Canonical_Combining_Class=Above}", "\u0301"}, + {"\\p{Bidi_Class=Right_To_Left}", "\u05D0"}, + {"\\p{Decomposition_Type=Canonical}", "\u00C0"}, + {"\\p{East_Asian_Width=Fullwidth}", "\u3000"}, + {"\\p{Numeric_Value=1/2}", "\u00BD"}, + {"\\p{Joining_Group=Alef}", "\u0627"}, + }; + + for (String[] testCase : cases) { + JoniRegexPattern pattern = new JoniRegexPattern(testCase[0], FLAGS); + assertEquals(testCase[0], pattern.patternDescription()); + assertTrue(pattern.matcher(testCase[1], java.util.List.of()).find()); + } + } + @Test void flattensTranslatedPropertiesInsideOrdinaryCharacterClasses() { JoniRegexPattern pattern = new JoniRegexPattern( diff --git a/src/test/resources/unit/regex_joni_unicode_property_family_policy.t b/src/test/resources/unit/regex_joni_unicode_property_family_policy.t new file mode 100644 index 000000000..b93523706 --- /dev/null +++ b/src/test/resources/unit/regex_joni_unicode_property_family_policy.t @@ -0,0 +1,29 @@ +use strict; +use warnings; +use utf8; +use Test::More tests => 12; + +ok("\x{0301}" =~ /\p{Canonical_Combining_Class=Above}/, + 'canonical combining class assignment'); +ok("\x{0300}" !~ /\p{Canonical_Combining_Class=Below}/, + 'canonical combining class exclusion'); +ok("\x{05D0}" =~ /\p{Bidi_Class=Right_To_Left}/, + 'bidi class assignment'); +ok('A' !~ /\p{Bidi_Class=Right_To_Left}/, + 'bidi class exclusion'); +ok("\x{00C0}" =~ /\p{Decomposition_Type=Canonical}/, + 'decomposition type assignment'); +ok('k' !~ /\p{Decomposition_Type=Canonical}/i, + 'decomposition type does not gain fold members'); +ok("\x{3000}" =~ /\p{East_Asian_Width=Fullwidth}/, + 'east Asian width assignment'); +ok('k' !~ /\p{East_Asian_Width=Ambiguous}/i, + 'east Asian width does not gain fold members'); +ok("\x{00BD}" =~ m{\p{Numeric_Value=1/2}}, + 'numeric value rational assignment'); +ok('A' !~ m{\p{Numeric_Value=1/2}}i, + 'numeric value does not gain fold members'); +ok("\x{0627}" =~ /\p{Joining_Group=Alef}/, + 'joining group assignment'); +ok('A' !~ /\p{Joining_Group=Alef}/i, + 'joining group does not gain fold members'); From 42ea23cb47309a75044022b14997bed4c2306038 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Tue, 18 Aug 2026 19:39:04 +0200 Subject: [PATCH 3/9] feat(regex): resolve exact Age properties in Joni Pass exact Age, In, and Present_In assignments to the pinned Joni character property resolver with explicit no-fold policy. Keep wildcard and composed class forms in the frontend until their syntax semantics are represented by the engine. Generated with Codex (https://openai.com/codex) Co-Authored-By: Codex --- docs/reference/feature-matrix.md | 2 +- .../runtime/regex/UnicodeResolver.java | 48 ++++++++++++++----- .../runtime/regex/JoniRegexPatternTest.java | 21 ++++++++ 3 files changed, 58 insertions(+), 13 deletions(-) diff --git a/docs/reference/feature-matrix.md b/docs/reference/feature-matrix.md index 938c5ae4b..3a7bb069c 100644 --- a/docs/reference/feature-matrix.md +++ b/docs/reference/feature-matrix.md @@ -379,7 +379,7 @@ my @copy = @{$z}; # ERROR - ✅ **Backreferences to Named Groups**: Using `\k` or `\g{name}` for backreferences to named groups is supported. - ✅ **Relative Backreferences**: Using `\g{-n}` for relative backreferences. - ✅ **Basic Unicode Properties**: Common `\p{...}` and `\P{...}` forms such as `\p{L}` execute through Joni. General_Category assignments now enter the forked Joni parser unchanged and resolve to pinned Perl ranges there. -- 🟡 **Perl Unicode Property Syntax**: Perl-specific properties execute through Joni. `Age`, cumulative `In`/`Present_In`, General_Category, Canonical_Combining_Class, Bidi_Class, Decomposition_Type, East_Asian_Width, Numeric_Value, Joining_Group, Block, Script, and Script_Extensions assignments are generated from pinned Perl 5.44 Unicode 17.0 data with loose and wildcard aliases, ordered missing defaults, official compact aliases and shortcuts, Script-versus-Script_Extensions policy, reserved or composite values, exact rationals, and Perl's generated decimal keyword aliases; `ASCII_Hex_Digit`/`AHex` accepts Perl's eight boolean value aliases. General_Category and standalone exact Block, Script, Script_Extensions, Canonical_Combining_Class, Bidi_Class, Decomposition_Type, East_Asian_Width, Numeric_Value, and Joining_Group assignments use Joni's range-resolver API with explicit per-family case-fold policy. No-fold properties inside composed character classes, wildcard values, and Age-family assignments remain adapter-translated until Joni represents those syntax semantics natively. Other generated property/value aliases still have pinned acceptance/rejection gaps. +- 🟡 **Perl Unicode Property Syntax**: Perl-specific properties execute through Joni. `Age`, cumulative `In`/`Present_In`, General_Category, Canonical_Combining_Class, Bidi_Class, Decomposition_Type, East_Asian_Width, Numeric_Value, Joining_Group, Block, Script, and Script_Extensions assignments are generated from pinned Perl 5.44 Unicode 17.0 data with loose and wildcard aliases, ordered missing defaults, official compact aliases and shortcuts, Script-versus-Script_Extensions policy, reserved or composite values, exact rationals, and Perl's generated decimal keyword aliases; `ASCII_Hex_Digit`/`AHex` accepts Perl's eight boolean value aliases. General_Category and standalone exact Age, `In`/`Present_In`, Block, Script, Script_Extensions, Canonical_Combining_Class, Bidi_Class, Decomposition_Type, East_Asian_Width, Numeric_Value, and Joining_Group assignments use Joni's range-resolver API with explicit per-family case-fold policy. No-fold properties inside composed character classes and wildcard values remain adapter-translated until Joni represents those syntax semantics natively. Other generated property/value aliases still have pinned acceptance/rejection gaps. - ✅ **Possessive Quantifiers**: Quantifiers like `*+`, `++`, `?+`, and `{n,m}+`, which disable backtracking, are supported. - ✅ **Atomic Grouping**: Use of `(?>...)` for atomic groups is supported. - ✅ **`\K` assertion**: Keep left — in `s///`, text before `\K` is preserved; match variables reflect only the portion after `\K`. Ordinary KEEP assertions route through native Joni and no longer use the Java marker rewrite; the adapter still rejects KEEP inside lookaround until the Joni analyser emits Perl's diagnostic directly. diff --git a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java index 486a970bf..13b452433 100644 --- a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java +++ b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java @@ -983,6 +983,7 @@ static boolean isPerlBuiltInPropertyAlias(String property) { static CharacterPropertyResolver.Result resolveJoniProperty( String property, boolean inCharacterClass) { if (property == null) return null; + property = normalizePerlIsPropertyAssignment(property); int assignment = propertyValueDelimiter(property); if (assignment <= 0 || assignment == property.length() - 1) { return null; @@ -1008,6 +1009,9 @@ static CharacterPropertyResolver.Result resolveJoniProperty( || PerlUnicodeJoiningGroupData.isPropertyAlias(name)) { if (perlNumericWildcardBody(value) != null) return null; caseFold = false; + } else if (isPerlAgeProperty(name)) { + if (isPerlAgeWildcard(value)) return null; + caseFold = false; } else { return null; } @@ -1060,7 +1064,7 @@ private static boolean isPerlSpecialPropertyAlias(String property) { private static UnicodeSet resolvePerlBuiltInPropertyAlias(String property) { if (property == null) return null; - String alias = property.trim(); + String alias = normalizePerlIsPropertyAssignment(property.trim()); int assignment = propertyValueDelimiter(alias); if (assignment == alias.length() - 1 && (PerlUnicodeScriptData.isScriptPropertyAlias( @@ -1081,6 +1085,8 @@ && isGeneralCategoryProperty(alias.substring(0, assignment))) { } return category; } + UnicodeSet age = resolvePerlAgeProperty(alias, true); + if (age != null) return age; if (assignment > 0 && assignment < alias.length() - 1 && isCanonicalCombiningClassProperty(alias.substring(0, assignment))) { UnicodeSet combiningClass = PerlUnicodeCombiningClassData.resolve( @@ -1722,26 +1728,33 @@ private static int unicodePropertyValue(int property, String alias) { } private static String translatePerlAgeProperty(String property, boolean negated) { - int delimiter = property.indexOf('='); - int colon = property.indexOf(':'); - if (delimiter < 0 || colon > 0 && colon < delimiter) delimiter = colon; + UnicodeSet result = resolvePerlAgeProperty(property, true); + return result == null ? null + : wrapCharClass(unicodeSetToJavaPattern(result), negated); + } + + private static UnicodeSet resolvePerlAgeProperty( + String property, boolean allowWildcard) { + property = normalizePerlIsPropertyAssignment(property); + int delimiter = propertyValueDelimiter(property); if (delimiter <= 0 || delimiter == property.length() - 1) return null; - String name = property.substring(0, delimiter) - .replace("_", "").replace("-", "").replace(" ", ""); + String name = property.substring(0, delimiter); boolean exact; - if (name.equalsIgnoreCase("Age")) { + String looseName = loosePropertyName(name); + if (looseName.equals("age")) { exact = true; - } else if (name.equalsIgnoreCase("In") || name.equalsIgnoreCase("PresentIn")) { + } else if (looseName.equals("in") || looseName.equals("presentin")) { exact = false; } else { return null; } - String requested = normalizeUnicodeAgeVersion(property.substring(delimiter + 1)); + String value = property.substring(delimiter + 1); + if (!allowWildcard && isPerlAgeWildcard(value)) return null; + String requested = normalizeUnicodeAgeVersion(value); if (requested.equalsIgnoreCase("NA") || requested.equalsIgnoreCase("Unassigned")) { - return wrapCharClass( - unicodeSetToJavaPattern(PerlUnicodeAgeData.unassignedSet()), negated); + return PerlUnicodeAgeData.unassignedSet(); } UnicodeSet result = exact @@ -1750,7 +1763,18 @@ private static String translatePerlAgeProperty(String property, boolean negated) if (result == null) { throw new IllegalArgumentException("Unsupported Unicode age version: " + requested); } - return wrapCharClass(unicodeSetToJavaPattern(result), negated); + return result; + } + + private static boolean isPerlAgeProperty(String name) { + String looseName = loosePropertyName(name); + return looseName.equals("age") || looseName.equals("in") + || looseName.equals("presentin"); + } + + private static boolean isPerlAgeWildcard(String value) { + String trimmed = value.trim(); + return trimmed.startsWith(":\\A") && trimmed.endsWith("\\z:"); } private static String normalizeUnicodeAgeVersion(String value) { diff --git a/src/test/java/org/perlonjava/runtime/regex/JoniRegexPatternTest.java b/src/test/java/org/perlonjava/runtime/regex/JoniRegexPatternTest.java index 2bc11c2f5..eaeaf0f8e 100644 --- a/src/test/java/org/perlonjava/runtime/regex/JoniRegexPatternTest.java +++ b/src/test/java/org/perlonjava/runtime/regex/JoniRegexPatternTest.java @@ -145,6 +145,27 @@ void passesRemainingExactEnumeratedPropertiesToJoni() { } } + @Test + void passesExactAgePropertiesToJoniWithoutTextExpansion() { + String[][] cases = { + {"\\p{Age=2.1}", "\u20AC"}, + {"\\p{In=3.0}", "\u20AC"}, + {"\\p{Present_In=3.0}", "\u20AC"}, + {"\\p{Is_Age=6.1}", "\uD83D\uDE00"}, + {"\\p{Age=Unassigned}", "\uD88D\uDC7A"}, + }; + + for (String[] testCase : cases) { + JoniRegexPattern pattern = new JoniRegexPattern(testCase[0], FLAGS); + assertEquals(testCase[0], pattern.patternDescription()); + assertTrue(pattern.matcher(testCase[1], java.util.List.of()).find()); + } + + JoniRegexPattern wildcard = new JoniRegexPattern( + "\\p{Age=:\\AV16_0\\z:}", FLAGS); + assertFalse(wildcard.patternDescription().contains("Age=")); + } + @Test void flattensTranslatedPropertiesInsideOrdinaryCharacterClasses() { JoniRegexPattern pattern = new JoniRegexPattern( From 247aba93d8c4d4afd4591edfb854d2f76e14d42a Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Tue, 18 Aug 2026 19:50:18 +0200 Subject: [PATCH 4/9] fix(regex): preserve composed property fold policy Carry resolver-provided case-fold eligibility through character-class union, intersection, negation, and nesting without changing Joni's existing ASCII boundary policy. Gate singleton and multi-character fold expansion with the composed eligibility class. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../joni/src/org/joni/ApplyCaseFold.java | 15 +- .../joni/src/org/joni/ApplyCaseFoldArg.java | 6 +- third_party/joni/src/org/joni/Parser.java | 129 ++++++++++++------ .../joni/src/org/joni/ast/CClassNode.java | 20 ++- .../test/TestCharacterPropertyResolver.java | 38 +++++- 5 files changed, 156 insertions(+), 52 deletions(-) diff --git a/third_party/joni/src/org/joni/ApplyCaseFold.java b/third_party/joni/src/org/joni/ApplyCaseFold.java index c3f01e543..59021b4c8 100644 --- a/third_party/joni/src/org/joni/ApplyCaseFold.java +++ b/third_party/joni/src/org/joni/ApplyCaseFold.java @@ -36,6 +36,7 @@ public void apply(int from, int[]to, int length, Object o) { Encoding enc = env.enc; CClassNode cc = arg.cc; CClassNode ascCc = arg.ascCc; + CClassNode foldCc = arg.foldCc; BitSet bs = cc.bs; boolean addFlag; @@ -44,13 +45,12 @@ public void apply(int from, int[]to, int length, Object o) { return; } - if (ascCc == null) { + if (!isEligible(foldCc, enc, from) || ascCc == null) { addFlag = false; } else if (Encoding.isAscii(from) == Encoding.isAscii(to[0])) { addFlag = true; } else { - addFlag = ascCc.isCodeInCC(enc, from); - if (ascCc.isNot()) addFlag = !addFlag; + addFlag = isEligible(ascCc, enc, from); } if (length == 1) { @@ -84,7 +84,8 @@ public void apply(int from, int[]to, int length, Object o) { } // CASE_FOLD_IS_APPLIED_INSIDE_NEGATIVE_CCLASS } else { - if (cc.isCodeInCC(enc, from) && (!Config.CASE_FOLD_IS_APPLIED_INSIDE_NEGATIVE_CCLASS || !cc.isNot())) { + if (addFlag && cc.isCodeInCC(enc, from) + && (!Config.CASE_FOLD_IS_APPLIED_INSIDE_NEGATIVE_CCLASS || !cc.isNot())) { StringNode node = null; for (int i=0; i ascNode) { + private CClassNode parseCharClass(ObjPtr ascNode, + ObjPtr foldNode) { final boolean neg; - CClassNode cc, prevCc = null, ascCc = null, ascPrevCc = null, workCc = null, ascWorkCc = null; + CClassNode cc, prevCc = null, ascCc = null, ascPrevCc = null, + workCc = null, ascWorkCc = null, foldCc = null, + foldPrevCc = null, foldWorkCc = null; CCStateArg arg = new CCStateArg(); fetchTokenInCC(); @@ -167,6 +174,7 @@ private CClassNode parseCharClass(ObjPtr ascNode) { cc = new CClassNode(); if (isIgnoreCase(env.option)) { ascCc = ascNode.p = new CClassNode(); + foldCc = foldNode.p = new CClassNode(); } boolean andStart = false; @@ -184,7 +192,7 @@ private CClassNode parseCharClass(ObjPtr ascNode) { } arg.to = token.getC(); arg.toIsRaw = false; - parseCharClassValEntry2(cc, ascCc, arg); // goto val_entry2 + parseCharClassValEntry2(cc, ascCc, foldCc, arg); // goto val_entry2 break; case RAW_BYTE: @@ -224,25 +232,25 @@ private CClassNode parseCharClass(ObjPtr ascNode) { arg.inType = CCVALTYPE.SB; // raw_single: } arg.toIsRaw = true; - parseCharClassValEntry2(cc, ascCc, arg); // goto val_entry2 + parseCharClassValEntry2(cc, ascCc, foldCc, arg); // goto val_entry2 break; case CODE_POINT: arg.to = token.getCode(); arg.toIsRaw = true; - parseCharClassValEntry(cc, ascCc, arg); // val_entry:, val_entry2 + parseCharClassValEntry(cc, ascCc, foldCc, arg); // val_entry:, val_entry2 break; case POSIX_BRACKET_OPEN: - if (parsePosixBracket(cc, ascCc)) { /* true: is not POSIX bracket */ + if (parsePosixBracket(cc, ascCc, foldCc)) { /* true: is not POSIX bracket */ env.ccEscWarn("["); p = token.backP; arg.to = token.getC(); arg.toIsRaw = false; - parseCharClassValEntry(cc, ascCc, arg); // goto val_entry + parseCharClassValEntry(cc, ascCc, foldCc, arg); // goto val_entry break; } - cc.nextStateClass(arg, ascCc, env); // goto next_class + cc.nextStateClass(arg, ascCc, foldCc, env); // goto next_class break; case CHAR_TYPE: @@ -252,13 +260,17 @@ private CClassNode parseCharClass(ObjPtr ascNode) { ascCc.addCType(token.getPropCType(), token.getPropNot(), isAsciiRange(env.option), env, this); } } - cc.nextStateClass(arg, ascCc, env); // next_class: + if (foldCc != null) { + foldCc.addCType(token.getPropCType(), token.getPropNot(), + isAsciiRange(env.option), env, this); + } + cc.nextStateClass(arg, ascCc, foldCc, env); // next_class: break; case CHAR_PROPERTY: CharProperty property = fetchCharProperty(true); - addCharProperty(cc, ascCc, property, token.getPropNot()); - cc.nextStateClass(arg, ascCc, env); // goto next_class + addCharProperty(cc, ascCc, foldCc, property, token.getPropNot()); + cc.nextStateClass(arg, ascCc, foldCc, env); // goto next_class break; case CC_RANGE: @@ -266,11 +278,11 @@ private CClassNode parseCharClass(ObjPtr ascNode) { fetchTokenInCC(); fetched = true; if (token.type == TokenType.CC_CLOSE) { /* allow [x-] */ - parseCharClassRangeEndVal(cc, ascCc, arg); // range_end_val:, goto val_entry; + parseCharClassRangeEndVal(cc, ascCc, foldCc, arg); // range_end_val:, goto val_entry; break; } else if (token.type == TokenType.CC_AND) { env.ccEscWarn("-"); - parseCharClassRangeEndVal(cc, ascCc, arg); // goto range_end_val + parseCharClassRangeEndVal(cc, ascCc, foldCc, arg); // goto range_end_val break; } if (arg.type == CCVALTYPE.CLASS) newValueException(UNMATCHED_RANGE_SPECIFIER_IN_CHAR_CLASS); @@ -281,28 +293,28 @@ private CClassNode parseCharClass(ObjPtr ascNode) { fetchTokenInCC(); fetched = true; if (token.type == TokenType.CC_RANGE || andStart) env.ccEscWarn("-"); /* [--x] or [a&&-x] is warned. */ - parseCharClassValEntry(cc, ascCc, arg); // goto val_entry + parseCharClassValEntry(cc, ascCc, foldCc, arg); // goto val_entry break; } else if (arg.state == CCSTATE.RANGE) { env.ccEscWarn("-"); - parseCharClassSbChar(cc, ascCc, arg); // goto sb_char /* [!--x] is allowed */ + parseCharClassSbChar(cc, ascCc, foldCc, arg); // goto sb_char /* [!--x] is allowed */ break; } else { /* CCS_COMPLETE */ fetchTokenInCC(); fetched = true; if (token.type == TokenType.CC_CLOSE) { /* allow [a-b-] */ - parseCharClassRangeEndVal(cc, ascCc, arg); // goto range_end_val + parseCharClassRangeEndVal(cc, ascCc, foldCc, arg); // goto range_end_val break; } else if (token.type == TokenType.CC_AND) { env.ccEscWarn("-"); - parseCharClassRangeEndVal(cc, ascCc, arg); // goto range_end_val + parseCharClassRangeEndVal(cc, ascCc, foldCc, arg); // goto range_end_val break; } if (syntax.allowDoubleRangeOpInCC()) { env.ccEscWarn("-"); // parseCharClassSbChar(cc, ascCc, arg); // goto sb_char /* [0-9-a] is allowed as [0-9\-a] */ - parseCharClassRangeEndVal(cc, ascCc, arg); // goto range_end_val + parseCharClassRangeEndVal(cc, ascCc, foldCc, arg); // goto range_end_val break; } newSyntaxException(UNMATCHED_RANGE_SPECIFIER_IN_CHAR_CLASS); @@ -311,18 +323,22 @@ private CClassNode parseCharClass(ObjPtr ascNode) { case CC_CC_OPEN: /* [ */ ObjPtr ascPtr = new ObjPtr<>(); - CClassNode acc = parseCharClass(ascPtr); + ObjPtr foldPtr = new ObjPtr<>(); + CClassNode acc = parseCharClass(ascPtr, foldPtr); cc.or(acc, env); if (ascPtr.p != null) { ascCc.or(ascPtr.p, env); } + if (foldPtr.p != null) { + foldCc.or(foldPtr.p, env); + } break; case CC_AND: /* && */ if (arg.state == CCSTATE.VALUE) { arg.to = 0; arg.toIsRaw = false; - cc.nextStateValue(arg, ascCc, env); + cc.nextStateValue(arg, ascCc, foldCc, env); } /* initialize local variables */ andStart = true; @@ -332,6 +348,9 @@ private CClassNode parseCharClass(ObjPtr ascNode) { if (ascCc != null) { ascPrevCc.and(ascCc, env); } + if (foldCc != null) { + foldPrevCc.and(foldCc, env); + } } else { prevCc = cc; if (workCc == null) workCc = new CClassNode(); @@ -341,9 +360,15 @@ private CClassNode parseCharClass(ObjPtr ascNode) { if (ascWorkCc == null) ascWorkCc = new CClassNode(); ascCc = ascWorkCc; } + if (foldCc != null) { + foldPrevCc = foldCc; + if (foldWorkCc == null) foldWorkCc = new CClassNode(); + foldCc = foldWorkCc; + } } cc.clear(); if (ascCc != null) ascCc.clear(); + if (foldCc != null) foldCc.clear(); break; case EOT: @@ -360,7 +385,7 @@ private CClassNode parseCharClass(ObjPtr ascNode) { if (arg.state == CCSTATE.VALUE) { arg.to = 0; arg.toIsRaw = false; - cc.nextStateValue(arg, ascCc, env); + cc.nextStateValue(arg, ascCc, foldCc, env); } if (prevCc != null) { @@ -370,14 +395,20 @@ private CClassNode parseCharClass(ObjPtr ascNode) { ascPrevCc.and(ascCc, env); ascCc = ascPrevCc; } + if (foldCc != null) { + foldPrevCc.and(foldCc, env); + foldCc = foldPrevCc; + } } if (neg) { cc.setNot(); if (ascCc != null) ascCc.setNot(); + if (foldCc != null) foldCc.setNot(); } else { cc.clearNot(); if (ascCc != null) ascCc.clearNot(); + if (foldCc != null) foldCc.clearNot(); } if (cc.isNot() && syntax.notNewlineInNegativeCC()) { @@ -396,27 +427,31 @@ private CClassNode parseCharClass(ObjPtr ascNode) { return cc; } - private void parseCharClassSbChar(CClassNode cc, CClassNode ascCc, CCStateArg arg) { + private void parseCharClassSbChar(CClassNode cc, CClassNode ascCc, + CClassNode foldCc, CCStateArg arg) { arg.inType = CCVALTYPE.SB; arg.to = token.getC(); arg.toIsRaw = false; - parseCharClassValEntry2(cc, ascCc, arg); // goto val_entry2 + parseCharClassValEntry2(cc, ascCc, foldCc, arg); // goto val_entry2 } - private void parseCharClassRangeEndVal(CClassNode cc, CClassNode ascCc, CCStateArg arg) { + private void parseCharClassRangeEndVal(CClassNode cc, CClassNode ascCc, + CClassNode foldCc, CCStateArg arg) { arg.to = '-'; arg.toIsRaw = false; - parseCharClassValEntry(cc, ascCc, arg); // goto val_entry + parseCharClassValEntry(cc, ascCc, foldCc, arg); // goto val_entry } - private void parseCharClassValEntry(CClassNode cc, CClassNode ascCc, CCStateArg arg) { + private void parseCharClassValEntry(CClassNode cc, CClassNode ascCc, + CClassNode foldCc, CCStateArg arg) { int len = enc.codeToMbcLength(arg.to); arg.inType = len == 1 ? CCVALTYPE.SB : CCVALTYPE.CODE_POINT; - parseCharClassValEntry2(cc, ascCc, arg); // val_entry2: + parseCharClassValEntry2(cc, ascCc, foldCc, arg); // val_entry2: } - private void parseCharClassValEntry2(CClassNode cc, CClassNode ascCc, CCStateArg arg) { - cc.nextStateValue(arg, ascCc, env); + private void parseCharClassValEntry2(CClassNode cc, CClassNode ascCc, + CClassNode foldCc, CCStateArg arg) { + cc.nextStateValue(arg, ascCc, foldCc, env); } private Node parseEnclose(TokenType term) { @@ -1174,12 +1209,18 @@ private Node parseExp(TokenType term) { case CC_OPEN: { ObjPtr ascPtr = new ObjPtr<>(); - CClassNode cc = parseCharClass(ascPtr); + ObjPtr foldPtr = new ObjPtr<>(); + CClassNode cc = parseCharClass(ascPtr, foldPtr); int code = cc.isOneChar(); - if (code != -1) return parseStringLoop(StringNode.fromCodePoint(code, enc), group); + if (code != -1 && (!isIgnoreCase(env.option) + || ApplyCaseFold.isEligible(foldPtr.p, enc, code))) { + return parseStringLoop(StringNode.fromCodePoint(code, enc), group); + } node = cc; - if (isIgnoreCase(env.option)) node = cClassCaseFold(node, cc, ascPtr.p); + if (isIgnoreCase(env.option)) { + node = cClassCaseFold(node, cc, ascPtr.p, foldPtr.p); + } break; } @@ -1619,8 +1660,9 @@ private Node parseCharType(Node node) { return node; } - private Node cClassCaseFold(Node node, CClassNode cc, CClassNode ascCc) { - ApplyCaseFoldArg arg = new ApplyCaseFoldArg(env, cc, ascCc); + private Node cClassCaseFold(Node node, CClassNode cc, CClassNode ascCc, + CClassNode foldCc) { + ApplyCaseFoldArg arg = new ApplyCaseFoldArg(env, cc, ascCc, foldCc); enc.applyAllCaseFold(env.caseFoldFlagFor(env.option), ApplyCaseFold.INSTANCE, arg); if (arg.altRoot != null) { node = ListNode.newAlt(node, arg.altRoot); @@ -1632,28 +1674,37 @@ private Node parseCharProperty() { CharProperty property = fetchCharProperty(false); CClassNode cc = new CClassNode(); Node node = cc; - addCharProperty(cc, null, property, false); + addCharProperty(cc, null, null, property, false); if (token.getPropNot()) cc.setNot(); if (isIgnoreCase(env.option) && property.caseFold) { if (property.ranges != null || property.ctype != CharacterType.ASCII) { - node = cClassCaseFold(node, cc, cc); + node = cClassCaseFold(node, cc, cc, cc); } } return node; } private void addCharProperty(CClassNode cc, CClassNode ascCc, - CharProperty property, boolean not) { + CClassNode foldCc, CharProperty property, + boolean not) { if (property.ranges == null) { cc.addCType(property.ctype, not, false, env, this); if (ascCc != null && property.ctype != CharacterType.ASCII) { ascCc.addCType(property.ctype, not, false, env, this); } + if (foldCc != null) { + foldCc.addCType(property.ctype, not, false, env, this); + } return; } cc.addCodeRanges(property.ranges, not, env); - if (ascCc != null) ascCc.addCodeRanges(property.ranges, not, env); + if (ascCc != null) { + ascCc.addCodeRanges(property.ranges, not, env); + } + if (foldCc != null && property.caseFold) { + foldCc.addCodeRanges(property.ranges, not, env); + } } private Node parseAnycharAnytime() { diff --git a/third_party/joni/src/org/joni/ast/CClassNode.java b/third_party/joni/src/org/joni/ast/CClassNode.java index 150227116..463676140 100644 --- a/third_party/joni/src/org/joni/ast/CClassNode.java +++ b/third_party/joni/src/org/joni/ast/CClassNode.java @@ -415,31 +415,37 @@ public static final class CCStateArg { public CCSTATE state; } - public void nextStateClass(CCStateArg arg, CClassNode ascCC, ScanEnvironment env) { + public void nextStateClass(CCStateArg arg, CClassNode ascCc, + CClassNode foldCc, ScanEnvironment env) { if (arg.state == CCSTATE.RANGE) throw new SyntaxException(ErrorMessages.CHAR_CLASS_VALUE_AT_END_OF_RANGE); if (arg.state == CCSTATE.VALUE && arg.type != CCVALTYPE.CLASS) { if (arg.type == CCVALTYPE.SB) { bs.set(env, arg.from); - if (ascCC != null) ascCC.bs.set(arg.from); + if (ascCc != null) ascCc.bs.set(arg.from); + if (foldCc != null) foldCc.bs.set(arg.from); } else if (arg.type == CCVALTYPE.CODE_POINT) { addCodeRange(env, arg.from, arg.from); - if (ascCC != null) ascCC.addCodeRange(env, arg.from, arg.from, false); + if (ascCc != null) ascCc.addCodeRange(env, arg.from, arg.from, false); + if (foldCc != null) foldCc.addCodeRange(env, arg.from, arg.from, false); } } arg.state = CCSTATE.VALUE; arg.type = CCVALTYPE.CLASS; } - public void nextStateValue(CCStateArg arg, CClassNode ascCc, ScanEnvironment env) { + public void nextStateValue(CCStateArg arg, CClassNode ascCc, + CClassNode foldCc, ScanEnvironment env) { switch(arg.state) { case VALUE: if (arg.type == CCVALTYPE.SB) { bs.set(env, arg.from); if (ascCc != null) ascCc.bs.set(arg.from); + if (foldCc != null) foldCc.bs.set(arg.from); } else if (arg.type == CCVALTYPE.CODE_POINT) { addCodeRange(env, arg.from, arg.from); if (ascCc != null) ascCc.addCodeRange(env, arg.from, arg.from, false); + if (foldCc != null) foldCc.addCodeRange(env, arg.from, arg.from, false); } break; @@ -459,9 +465,11 @@ public void nextStateValue(CCStateArg arg, CClassNode ascCc, ScanEnvironment env } bs.setRange(env, arg.from, arg.to); if (ascCc != null) ascCc.bs.setRange(null, arg.from, arg.to); + if (foldCc != null) foldCc.bs.setRange(null, arg.from, arg.to); } else { addCodeRange(env, arg.from, arg.to); if (ascCc != null) ascCc.addCodeRange(env, arg.from, arg.to, false); + if (foldCc != null) foldCc.addCodeRange(env, arg.from, arg.to, false); } } else { if (arg.from > arg.to) { @@ -479,6 +487,10 @@ public void nextStateValue(CCStateArg arg, CClassNode ascCc, ScanEnvironment env ascCc.bs.setRange(null, arg.from, arg.to < 0xff ? arg.to : 0xff); ascCc.addCodeRange(env, arg.from, arg.to, false); } + if (foldCc != null) { + foldCc.bs.setRange(null, arg.from, arg.to < 0xff ? arg.to : 0xff); + foldCc.addCodeRange(env, arg.from, arg.to, false); + } } // ccs_range_end: arg.state = CCSTATE.COMPLETE; diff --git a/third_party/joni/test/org/joni/test/TestCharacterPropertyResolver.java b/third_party/joni/test/org/joni/test/TestCharacterPropertyResolver.java index e03ff4757..112f8cdf1 100644 --- a/third_party/joni/test/org/joni/test/TestCharacterPropertyResolver.java +++ b/third_party/joni/test/org/joni/test/TestCharacterPropertyResolver.java @@ -19,6 +19,7 @@ */ package org.joni.test; +import static org.joni.constants.SyntaxProperties.OP2_CCLASS_SET_OP; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertSame; import static org.junit.Assert.fail; @@ -38,16 +39,17 @@ public class TestCharacterPropertyResolver { String name = new String(bytes, p, end - p, StandardCharsets.UTF_8); return switch (name) { case "Fake" -> new CharacterPropertyResolver.Result( - new int[] {2, 'A', 'A', 0x1f642, 0x1f642}, true); + new int[] {3, 'A', 'A', 0xdf, 0xdf, 0x1f642, 0x1f642}, true); case "FakeNoFold" -> new CharacterPropertyResolver.Result( - new int[] {1, 'A', 'A'}, false); + new int[] {2, 'A', 'A', 0xdf, 0xdf}, false); default -> null; }; }; private static Syntax syntax(CharacterPropertyResolver resolver) { return new Syntax("CharacterPropertyResolver", Syntax.PerlNG.op, - Syntax.PerlNG.op2, Syntax.PerlNG.op3, Syntax.PerlNG.behavior, + Syntax.PerlNG.op2 | OP2_CCLASS_SET_OP, Syntax.PerlNG.op3, + Syntax.PerlNG.behavior, Syntax.PerlNG.options, Syntax.PerlNG.metaCharTable, null, resolver); } @@ -76,6 +78,36 @@ public void resolvesRangesInsideAndOutsideCharacterClasses() { assertEquals(-1, search("(?i)\\p{FakeNoFold}", "a")); } + @Test + public void preservesFoldPolicyInsidePositiveAndNegativeClasses() { + assertEquals(0, search("(?i)[\\p{Fake}]", "a")); + assertEquals(0, search("(?i)[\\p{FakeNoFold}]", "A")); + assertEquals(-1, search("(?i)[\\p{FakeNoFold}]", "a")); + assertEquals(-1, search("(?i)[\\P{FakeNoFold}]", "A")); + assertEquals(0, search("(?i)[\\P{FakeNoFold}]", "a")); + assertEquals(-1, search("(?i)[^\\p{FakeNoFold}]", "A")); + assertEquals(0, search("(?i)[^\\p{FakeNoFold}]", "a")); + } + + @Test + public void composesFoldPolicyThroughUnionsIntersectionsAndNestedClasses() { + assertEquals(0, search("(?i)[\\p{FakeNoFold}\\p{Fake}]", "a")); + assertEquals(0, search("(?i)[\\p{Fake}&&\\p{FakeNoFold}]", "A")); + assertEquals(-1, search("(?i)[\\p{Fake}&&\\p{FakeNoFold}]", "a")); + assertEquals(0, search("(?i)[[\\p{FakeNoFold}]B]", "A")); + assertEquals(-1, search("(?i)[[\\p{FakeNoFold}]B]", "a")); + assertEquals(0, search("(?i)[[\\p{FakeNoFold}]B]", "b")); + } + + @Test + public void foldsOnlyEligibleMembersOfMixedClasses() { + assertEquals(0, search("(?i)[B\\p{FakeNoFold}]", "A")); + assertEquals(-1, search("(?i)[B\\p{FakeNoFold}]", "a")); + assertEquals(0, search("(?i)[B\\p{FakeNoFold}]", "b")); + assertEquals(0, search("(?i)[\\p{Fake}]", "ss")); + assertEquals(-1, search("(?i)[\\p{FakeNoFold}]", "ss")); + } + @Test public void fallsBackToEncodingProperties() { assertEquals(0, search("\\p{Digit}", "7")); From 328044a185989601829dcf6c79bcdff70f71d0fa Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Tue, 18 Aug 2026 21:21:44 +0200 Subject: [PATCH 5/9] fix(regex): support colon-delimited Block wildcards Accept Perl's anchored colon wildcard form for Block values and match official short and long value aliases while retaining the pinned Perl Block sets. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../runtime/regex/UnicodeResolver.java | 20 +++++++- .../regex/unicode_block_colon_wildcards.t | 47 +++++++++++++++++++ 2 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 src/test/resources/unit/regex/unicode_block_colon_wildcards.t diff --git a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java index 13b452433..e6ecb9094 100644 --- a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java +++ b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java @@ -1407,8 +1407,20 @@ private static UnicodeSet resolvePerlBlock(String value) { UnicodeSet result = new UnicodeSet(); for (int valueId = 0; valueId < PerlUnicodeBlockData.valueCount(); valueId++) { String candidate = PerlUnicodeBlockData.canonicalValue(valueId); - if (valuePattern.matcher(candidate).matches() - || valuePattern.matcher(loosePropertyName(candidate)).matches()) { + boolean matches = valuePattern.matcher(candidate).matches() + || valuePattern.matcher(loosePropertyName(candidate)).matches(); + int icuValue = unicodePropertyValue(UProperty.BLOCK, candidate); + for (int nameChoice = UProperty.NameChoice.SHORT; + !matches && icuValue >= 0 && nameChoice <= UProperty.NameChoice.LONG; + nameChoice++) { + String officialAlias = UCharacter.getPropertyValueName( + UProperty.BLOCK, icuValue, nameChoice); + matches = officialAlias != null + && (valuePattern.matcher(officialAlias).matches() + || valuePattern.matcher( + loosePropertyName(officialAlias)).matches()); + } + if (matches) { result.addAll(PerlUnicodeBlockData.set(valueId)); } } @@ -1476,6 +1488,10 @@ private static UnicodeSet resolvePerlScript(String value, boolean extensions) { private static String perlBlockWildcardBody(String value) { String trimmed = value.trim(); + if (trimmed.startsWith(":\\A") && trimmed.endsWith("\\z:") + && trimmed.length() > 6) { + return trimmed.substring(3, trimmed.length() - 3); + } if (!trimmed.startsWith("#") || !trimmed.endsWith("#") || trimmed.length() <= 2) { return null; diff --git a/src/test/resources/unit/regex/unicode_block_colon_wildcards.t b/src/test/resources/unit/regex/unicode_block_colon_wildcards.t new file mode 100644 index 000000000..a8a2de8fd --- /dev/null +++ b/src/test/resources/unit/regex/unicode_block_colon_wildcards.t @@ -0,0 +1,47 @@ +use strict; +use warnings; +use Test::More; + +no warnings 'experimental::uniprop_wildcards'; + +sub compile_property { + my ($property) = @_; + my $pattern = eval 'qr/\A\p{' . $property . '}\z/u'; + return ($pattern, $@); +} + +my ($basic, $basic_error) = compile_property('Block=:\ABasic_Latin\z:'); +ok(defined $basic, 'colon-delimited Block wildcard compiles') + or diag($basic_error); +like('A', $basic, 'colon-delimited Block wildcard matches its block'); +unlike(chr(0x03B1), $basic, + 'colon-delimited Block wildcard excludes another block'); + +my ($short, $short_error) = compile_property('Blk=:\AASCII\z:'); +ok(defined $short, 'Blk accepts a colon-delimited wildcard alias') + or diag($short_error); +like('A', $short, 'wildcard value accepts an official Block alias'); + +my ($loose, $loose_error) = compile_property('Block=:\Abasiclatin\z:'); +ok(defined $loose, 'Block wildcard compares loose value spellings') + or diag($loose_error); +like('A', $loose, 'loose wildcard value selects Basic_Latin'); + +my ($union, $union_error) = + compile_property('Block=:\A(?:Basic_Latin|Greek_And_Coptic)\z:'); +ok(defined $union, 'Block wildcard accepts value alternation') + or diag($union_error); +like('A', $union, 'Block wildcard alternation includes Basic_Latin'); +like(chr(0x0378), $union, + 'Block wildcard alternation includes Greek_And_Coptic'); + +for my $rejected ( + ['Block=:\ANever_A_Block\z:', 'wildcard matching no Block value'], + ['Block=:\A.*\z:', 'star quantifier in Block wildcard'], + ['Is_Block=:\ABasic_Latin\z:', 'Is-prefixed Block wildcard'], +) { + my ($pattern, $error) = compile_property($rejected->[0]); + ok(!defined($pattern) && length($error), "$rejected->[1] is rejected"); +} + +done_testing; From 12e74b967859c3de8eee8eff21a66b38536aa85b Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Tue, 18 Aug 2026 22:00:38 +0200 Subject: [PATCH 6/9] fix(regex): normalize Perl POSIX property aliases Canonicalize Perl's pinned XPosix and ASCII Posix aliases using loose property-name matching while preserving user-property precedence. Propagate explicit missing-property diagnostics through both backend adapters instead of converting them into deferred match-any placeholders. Add a system-Perl-validated focused fixture covering canonical and loose spellings, membership, rejection, and defined callback precedence. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../runtime/regex/JoniRegexPattern.java | 4 +- .../regex/RegexPreprocessorHelper.java | 7 +- .../runtime/regex/UnicodeResolver.java | 49 ++++++++++++++ .../regex/unicode_posix_alias_normalization.t | 67 +++++++++++++++++++ 4 files changed, 123 insertions(+), 4 deletions(-) create mode 100644 src/test/resources/unit/regex/unicode_posix_alias_normalization.t diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index a27b7de9f..c21e111d8 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -273,7 +273,9 @@ private static UserPropertyTranslation translateUserDefinedProperties( .append(')'); } catch (IllegalArgumentException error) { String message = error.getMessage(); - if (!userDefined || message != null && message.contains("in expansion of")) { + if (!userDefined || message != null + && (message.contains("in expansion of") + || message.startsWith("Can't find Unicode property definition"))) { throw error; } translated.append("[\\s\\S]"); diff --git a/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessorHelper.java b/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessorHelper.java index 74fe5b135..f14c2c173 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessorHelper.java +++ b/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessorHelper.java @@ -355,11 +355,12 @@ static int handleEscapeSequences(String s, StringBuilder sb, int c, int offset, // Perl allows user-defined properties (InFoo/IsFoo) to be unknown at compile time; // they are resolved at runtime when the property sub is available. // If it's currently undefined, emit a placeholder that compiles in Java and mark for recompilation. - // But if the error already contains "in expansion of", it is a real user-property definition error - // that should be reported (not deferred). + // Explicit missing-property and "in expansion of" diagnostics are + // real user-property errors that should be reported, not deferred. String msg = e.getMessage(); if (UnicodeResolver.isUserDefinedPropertyName(property) - && (msg == null || !msg.contains("in expansion of"))) { + && (msg == null || (!msg.contains("in expansion of") + && !msg.startsWith("Can't find Unicode property definition")))) { RegexPreprocessor.markDeferredUnicodePropertyEncountered(); sb.setLength(sb.length() - 1); // Remove the backslash // Placeholder: match any single character, including newline diff --git a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java index e6ecb9094..b743f492a 100644 --- a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java +++ b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java @@ -359,6 +359,7 @@ private static UnicodeSet resolvePropertyReferenceAsSet(String propRef, Set recursionSet) { + property = canonicalPerlPosixPropertyAlias(property); UnicodeSet perlBuiltInAlias = resolvePerlBuiltInPropertyAlias(property); if (perlBuiltInAlias != null) return perlBuiltInAlias; @@ -713,6 +714,7 @@ private static String translateUnicodeProperty(String property, boolean negated, property = property.substring(1).trim(); negated = !negated; } + property = canonicalPerlPosixPropertyAlias(property); boolean isPrefixedNumericWildcard = isPerlIsPrefixedNumericWildcard(property); boolean isPrefixedJoiningGroupWildcard = @@ -753,6 +755,12 @@ private static String translateUnicodeProperty(String property, boolean negated, if (userProp != null) { return wrapCharClass(userProp, negated); } + String looseUserProperty = loosePropertyName(property); + if (looseUserProperty.startsWith("isxposix") + || looseUserProperty.startsWith("isposix")) { + throw new IllegalArgumentException( + "Can't find Unicode property definition \"" + property + "\""); + } // Property not found - fall through to throw error below } @@ -974,6 +982,7 @@ static boolean isUserDefinedPropertyName(String property) { static boolean isPerlBuiltInPropertyAlias(String property) { if (property == null) return false; + property = canonicalPerlPosixPropertyAlias(property); return isPerlSpecialPropertyAlias(property.trim()) || !normalizePerlIsPropertyAssignment(property).equals(property) || resolvePerlBuiltInPropertyAlias(property) != null; @@ -1061,6 +1070,46 @@ private static boolean isPerlSpecialPropertyAlias(String property) { }; } + private static String canonicalPerlPosixPropertyAlias(String property) { + if (property == null) return null; + String loose = loosePropertyName(property); + if (loose.startsWith("is")) { + String unprefixed = loose.substring(2); + if (unprefixed.startsWith("xposix") || unprefixed.startsWith("posix")) { + loose = unprefixed; + } + } + return switch (loose) { + case "xposixalnum" -> "XPosixAlnum"; + case "xposixalpha" -> "XPosixAlpha"; + case "xposixblank" -> "XPosixBlank"; + case "xposixcntrl" -> "XPosixCntrl"; + case "xposixdigit" -> "XPosixDigit"; + case "xposixgraph" -> "XPosixGraph"; + case "xposixlower" -> "XPosixLower"; + case "xposixprint" -> "XPosixPrint"; + case "xposixpunct" -> "XPosixPunct"; + case "xposixspace" -> "XPosixSpace"; + case "xposixupper" -> "XPosixUpper"; + case "xposixword" -> "XPosixWord"; + case "xposixxdigit" -> "XPosixXDigit"; + case "posixalnum" -> "PosixAlnum"; + case "posixalpha" -> "PosixAlpha"; + case "posixblank" -> "PosixBlank"; + case "posixcntrl" -> "PosixCntrl"; + case "posixdigit" -> "PosixDigit"; + case "posixgraph" -> "PosixGraph"; + case "posixlower" -> "PosixLower"; + case "posixprint" -> "PosixPrint"; + case "posixpunct" -> "PosixPunct"; + case "posixspace" -> "PosixSpace"; + case "posixupper" -> "PosixUpper"; + case "posixword" -> "PosixWord"; + case "posixxdigit" -> "PosixXDigit"; + default -> property; + }; + } + private static UnicodeSet resolvePerlBuiltInPropertyAlias(String property) { if (property == null) return null; diff --git a/src/test/resources/unit/regex/unicode_posix_alias_normalization.t b/src/test/resources/unit/regex/unicode_posix_alias_normalization.t new file mode 100644 index 000000000..ed5d42a90 --- /dev/null +++ b/src/test/resources/unit/regex/unicode_posix_alias_normalization.t @@ -0,0 +1,67 @@ +use strict; +use warnings; +use Test::More; + +sub compile_property { + my ($property) = @_; + my $pattern = eval 'qr/\A\p{' . $property . '}\z/u'; + return ($pattern, $@); +} + +my ($space, $space_error) = compile_property('XPosixSpace'); +ok(defined $space, 'canonical XPosix property compiles') or diag($space_error); +like(chr(0x00A0), $space, 'XPosixSpace includes non-ASCII Unicode space'); +unlike('A', $space, 'XPosixSpace excludes a letter'); + +my ($ascii_alpha, $ascii_alpha_error) = compile_property('PosixAlpha'); +ok(defined $ascii_alpha, 'canonical ASCII Posix property compiles') + or diag($ascii_alpha_error); +like('A', $ascii_alpha, 'PosixAlpha includes an ASCII letter'); +unlike(chr(0x03B1), $ascii_alpha, + 'PosixAlpha remains ASCII-only'); + +my ($loose, $loose_error) = compile_property('x-p_o s_i x alpha'); +ok(defined $loose, 'XPosix property accepts loose internal separators') + or diag($loose_error); +like(chr(0x03B1), $loose, 'loose XPosixAlpha keeps Unicode membership'); + +my ($leading, $leading_error) = compile_property(' _XPosixWord'); +ok(defined $leading, 'XPosix property accepts leading loose separators') + or diag($leading_error); +like('_', $leading, 'leading-separator XPosixWord matches underscore'); + +my ($is_prefixed, $is_prefixed_error) = compile_property('isxposixxdigit'); +ok(defined $is_prefixed, 'XPosix property accepts a loose Is prefix') + or diag($is_prefixed_error); +like('F', $is_prefixed, 'Is-prefixed XPosixXDigit matches a hex digit'); +unlike('G', $is_prefixed, 'Is-prefixed XPosixXDigit excludes a non-hex digit'); + +my ($unknown, $unknown_error) = compile_property('xposixdefinitelynot'); +ok(!defined($unknown) && length($unknown_error), + 'unknown loose XPosix property is rejected'); + +my ($unknown_is, $unknown_is_error) = + compile_property('Is_XPosixDefinitelyNot'); +eval { 'A' =~ $unknown_is } if defined($unknown_is) && !$unknown_is_error; +$unknown_is_error ||= $@; +ok(length($unknown_is_error), 'unknown Is-prefixed XPosix property is rejected'); + +my $callback_calls = 0; +my $callback_defined = eval q{ + sub Is_XPosixDefinitelyNot { + $callback_calls++; + return "0041"; + } + 1; +}; +ok($callback_defined, 'POSIX-shaped user property callback can be defined'); + +my ($callback, $callback_error) = + compile_property('Is_XPosixDefinitelyNot'); +ok(defined $callback, 'defined POSIX-shaped user property callback wins') + or diag($callback_error); +like('A', $callback, 'defined POSIX-shaped user property matches its range'); +unlike('B', $callback, 'defined POSIX-shaped user property excludes other ranges'); +is($callback_calls, 1, 'defined POSIX-shaped user property is called once'); + +done_testing; From 6bb610e7a839d9634cfc52a3c37f13364518ad79 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Tue, 18 Aug 2026 22:55:37 +0200 Subject: [PATCH 7/9] fix(regex): accept leading-loose Block shortcuts Allow Perl's bare In and Is Block shortcuts to ignore leading whitespace, hyphens, and underscores before the prefix. Keep Script, binary, General_Category, and exact user-property precedence unchanged while resolving only against the pinned Perl Block table. Add a system-Perl-validated focused fixture for membership, precedence, callback behavior, and rejection paths. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../runtime/regex/UnicodeResolver.java | 43 ++++++++---- .../regex/unicode_block_leading_shortcuts.t | 65 +++++++++++++++++++ 2 files changed, 96 insertions(+), 12 deletions(-) create mode 100644 src/test/resources/unit/regex/unicode_block_leading_shortcuts.t diff --git a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java index b743f492a..bdb1f68d6 100644 --- a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java +++ b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java @@ -1266,31 +1266,50 @@ && loosePropertyName(alias.substring(0, assignment)) } } - String blockAlias = alias; + String blockShortcutAlias = alias; + if (assignment < 0) { + int prefixStart = 0; + while (prefixStart < alias.length()) { + char separator = alias.charAt(prefixStart); + if (!Character.isWhitespace(separator) + && separator != '-' && separator != '_') break; + prefixStart++; + } + if (prefixStart > 0 && alias.length() - prefixStart > 2 + && (alias.regionMatches(true, prefixStart, "in", 0, 2) + || alias.regionMatches(true, prefixStart, "is", 0, 2))) { + blockShortcutAlias = alias.substring(prefixStart); + } + } + + String blockAlias = blockShortcutAlias; boolean blockShortcut = false; boolean isBlockShortcut = false; - if (alias.length() > 2 && alias.regionMatches(true, 0, "in", 0, 2)) { + if (blockShortcutAlias.length() > 2 + && blockShortcutAlias.regionMatches(true, 0, "in", 0, 2)) { int valueStart = 2; - while (valueStart < alias.length()) { - char separator = alias.charAt(valueStart); + while (valueStart < blockShortcutAlias.length()) { + char separator = blockShortcutAlias.charAt(valueStart); if (!Character.isWhitespace(separator) && separator != '-' && separator != '_') break; valueStart++; } - if (valueStart >= alias.length()) return null; - blockAlias = alias.substring(valueStart); + if (valueStart >= blockShortcutAlias.length()) return null; + blockAlias = blockShortcutAlias.substring(valueStart); blockShortcut = true; - } else if (assignment >= 0 || unicodePropertyValue(UProperty.SCRIPT, alias) >= 0) { + } else if (assignment >= 0 + || unicodePropertyValue(UProperty.SCRIPT, blockShortcutAlias) >= 0) { return null; - } else if (alias.length() > 2 && alias.regionMatches(true, 0, "is", 0, 2)) { + } else if (blockShortcutAlias.length() > 2 + && blockShortcutAlias.regionMatches(true, 0, "is", 0, 2)) { int valueStart = 2; - while (valueStart < alias.length()) { - char separator = alias.charAt(valueStart); + while (valueStart < blockShortcutAlias.length()) { + char separator = blockShortcutAlias.charAt(valueStart); if (!Character.isWhitespace(separator) && separator != '-' && separator != '_') break; valueStart++; } - if (valueStart >= alias.length()) return null; - String candidate = alias.substring(valueStart); + if (valueStart >= blockShortcutAlias.length()) return null; + String candidate = blockShortcutAlias.substring(valueStart); if (unicodePropertyValue(UProperty.SCRIPT, candidate) >= 0) return null; if (isIcuBinaryPropertyAlias(candidate) || isIcuGeneralCategoryAlias(candidate)) return null; diff --git a/src/test/resources/unit/regex/unicode_block_leading_shortcuts.t b/src/test/resources/unit/regex/unicode_block_leading_shortcuts.t new file mode 100644 index 000000000..4d8e263e7 --- /dev/null +++ b/src/test/resources/unit/regex/unicode_block_leading_shortcuts.t @@ -0,0 +1,65 @@ +use strict; +use warnings; +use Test::More; + +sub compile_property { + my ($property) = @_; + my $pattern = eval 'qr/\A\p{' . $property . '}\z/u'; + return ($pattern, $@); +} + +my ($canonical, $canonical_error) = compile_property('In_Emoticons'); +ok(defined $canonical, 'canonical In Block shortcut compiles') + or diag($canonical_error); +like(chr(0x1F600), $canonical, 'canonical In Block shortcut matches its block'); +unlike('A', $canonical, 'canonical In Block shortcut excludes another block'); + +my ($leading_in, $leading_in_error) = + compile_property(' _ IN_Emoticons'); +ok(defined $leading_in, 'In Block shortcut accepts leading loose separators') + or diag($leading_in_error); +like(chr(0x1F600), $leading_in, + 'leading-loose In Block shortcut matches its block'); +unlike('A', $leading_in, + 'leading-loose In Block shortcut excludes another block'); + +my ($leading_is, $leading_is_error) = + compile_property('- Is_Emoticons'); +ok(defined $leading_is, 'Is Block shortcut accepts leading loose separators') + or diag($leading_is_error); +like(chr(0x1F600), $leading_is, + 'leading-loose Is Block shortcut matches its block'); +unlike('A', $leading_is, + 'leading-loose Is Block shortcut excludes another block'); + +my ($script, $script_error) = compile_property('Is_Latin'); +ok(defined $script, 'Script shortcut retains precedence') or diag($script_error); +like('A', $script, 'Script shortcut membership remains intact'); + +my ($binary, $binary_error) = compile_property('Is_Uppercase'); +ok(defined $binary, 'binary shortcut retains precedence') or diag($binary_error); +like('A', $binary, 'binary shortcut membership remains intact'); + +my $callback_calls = 0; +sub Is_BlockShortcutCallback { + $callback_calls++; + return "0041"; +} +my ($callback, $callback_error) = + compile_property('Is_BlockShortcutCallback'); +ok(defined $callback, 'exact user property callback retains precedence') + or diag($callback_error); +like('A', $callback, 'exact user property callback matches its range'); +is($callback_calls, 1, 'exact user property callback is called once'); + +my ($leading_callback, $leading_callback_error) = + compile_property('_ Is_BlockShortcutCallback'); +ok(!defined($leading_callback) && length($leading_callback_error), + 'leading separators do not broaden user property lookup'); +is($callback_calls, 1, 'rejected leading callback spelling is not invoked'); + +my ($unknown, $unknown_error) = compile_property('_ In_DefinitelyNotABlock'); +ok(!defined($unknown) && length($unknown_error), + 'unknown leading-loose Block shortcut is rejected'); + +done_testing; From 90ca6751c3ac758cf1f7d1f83072c39b610ce277 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Tue, 18 Aug 2026 23:16:33 +0200 Subject: [PATCH 8/9] fix(regex): accept leading-loose Script shortcuts Normalize leading loose separators and the Is prefix before resolving bare Script shortcuts, while preserving binary, general-category, and user-defined property precedence. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../runtime/regex/UnicodeResolver.java | 12 +++- .../regex/unicode_script_leading_shortcuts.t | 72 +++++++++++++++++++ 2 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 src/test/resources/unit/regex/unicode_script_leading_shortcuts.t diff --git a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java index bdb1f68d6..d55ccef13 100644 --- a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java +++ b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java @@ -1245,8 +1245,16 @@ && loosePropertyName(alias.substring(0, assignment)) // Perl's bare script-value shortcuts use Script_Extensions semantics. // Keep binary and General_Category names ahead of this value namespace. String scriptShortcut = alias; - if (alias.length() > 2 && alias.startsWith("Is")) { - int valueStart = 2; + int scriptPrefixStart = 0; + while (scriptPrefixStart < alias.length()) { + char separator = alias.charAt(scriptPrefixStart); + if (!Character.isWhitespace(separator) + && separator != '-' && separator != '_') break; + scriptPrefixStart++; + } + if (alias.length() - scriptPrefixStart > 2 + && alias.regionMatches(true, scriptPrefixStart, "is", 0, 2)) { + int valueStart = scriptPrefixStart + 2; while (valueStart < alias.length()) { char separator = alias.charAt(valueStart); if (!Character.isWhitespace(separator) diff --git a/src/test/resources/unit/regex/unicode_script_leading_shortcuts.t b/src/test/resources/unit/regex/unicode_script_leading_shortcuts.t new file mode 100644 index 000000000..855f565df --- /dev/null +++ b/src/test/resources/unit/regex/unicode_script_leading_shortcuts.t @@ -0,0 +1,72 @@ +use strict; +use warnings; +use Test::More; + +sub compile_property { + my ($property) = @_; + my $pattern = eval 'qr/\A\p{' . $property . '}\z/u'; + return ($pattern, $@); +} + +my ($canonical, $canonical_error) = compile_property('Is_Latin'); +ok(defined $canonical, 'canonical Is Script shortcut compiles') + or diag($canonical_error); +like('A', $canonical, 'canonical Is Script shortcut matches its script'); +unlike(chr(0x03B1), $canonical, + 'canonical Is Script shortcut excludes another script'); + +my ($lowercase, $lowercase_error) = compile_property('islatin'); +ok(defined $lowercase, 'Script shortcut accepts a lowercase Is prefix') + or diag($lowercase_error); +like('A', $lowercase, 'lowercase Is Script shortcut matches its script'); + +my ($leading, $leading_error) = compile_property(' -IS_LATN'); +ok(defined $leading, 'Script shortcut accepts leading loose separators') + or diag($leading_error); +like('A', $leading, 'leading-loose Script alias matches its script'); +unlike(chr(0x03B1), $leading, + 'leading-loose Script alias excludes another script'); + +my ($extensions, $extensions_error) = compile_property('_ is_hira'); +ok(defined $extensions, 'short Script alias accepts loose Is spelling') + or diag($extensions_error); +like(chr(0x30FC), $extensions, + 'bare Script shortcut retains Script_Extensions membership'); + +my ($binary, $binary_error) = compile_property('IsUppercase'); +ok(defined $binary, 'binary property retains precedence') or diag($binary_error); +like('A', $binary, 'binary property membership remains intact'); + +my ($category, $category_error) = compile_property('IsL'); +ok(defined $category, 'General_Category retains precedence') + or diag($category_error); +like(chr(0x03B1), $category, + 'General_Category shortcut still matches non-Latin letters'); + +my $callback_calls = 0; +sub IsScriptShortcutCallback { + $callback_calls++; + return "0041"; +} +my ($callback, $callback_error) = + compile_property('IsScriptShortcutCallback'); +ok(defined $callback, 'exact user property callback retains precedence') + or diag($callback_error); +like('A', $callback, 'exact user property callback matches its range'); +is($callback_calls, 1, 'exact user property callback is called once'); + +my ($leading_callback, $leading_callback_error) = + compile_property('_ IsScriptShortcutCallback'); +ok(!defined($leading_callback) && length($leading_callback_error), + 'leading separators do not broaden user property lookup'); +is($callback_calls, 1, 'rejected leading callback spelling is not invoked'); + +my ($hrkt, $hrkt_error) = compile_property('_ Is_Hrkt'); +ok(!defined($hrkt) && length($hrkt_error), + 'Katakana_Or_Hiragana Script shortcut remains rejected'); + +my ($unknown, $unknown_error) = compile_property('_ Is_DefinitelyNotAScript'); +ok(!defined($unknown) && length($unknown_error), + 'unknown leading-loose Script shortcut is rejected'); + +done_testing; From 124f161e00a284f14be68c2fbda8708920ebbfdb Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 00:11:08 +0200 Subject: [PATCH 9/9] fix(regex): resolve loose Is binary shortcuts Route leading-loose bare Is aliases through the native Joni property callback, preserving Script, Block, General_Category, and user-property precedence while inheriting only already-supported binary and special property bases. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../runtime/regex/UnicodeResolver.java | 60 +++++++++++++- .../regex/unicode_binary_leading_shortcuts.t | 78 +++++++++++++++++++ 2 files changed, 136 insertions(+), 2 deletions(-) create mode 100644 src/test/resources/unit/regex/unicode_binary_leading_shortcuts.t diff --git a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java index d55ccef13..991208d79 100644 --- a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java +++ b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java @@ -995,7 +995,14 @@ static CharacterPropertyResolver.Result resolveJoniProperty( property = normalizePerlIsPropertyAssignment(property); int assignment = propertyValueDelimiter(property); if (assignment <= 0 || assignment == property.length() - 1) { - return null; + String looseIsValue = looseIsShortcutValue(property); + if (looseIsValue == null + || PerlUnicodeScriptData.canonicalValue(looseIsValue) != null + || PerlUnicodeBlockData.set(looseIsValue) != null) { + return null; + } + UnicodeSet bareSet = resolvePerlBuiltInPropertyAlias(property); + return bareSet == null ? null : joniPropertyResult(bareSet, true); } String name = property.substring(0, assignment); String value = property.substring(assignment + 1); @@ -1033,6 +1040,11 @@ static CharacterPropertyResolver.Result resolveJoniProperty( UnicodeSet set = resolvePerlBuiltInPropertyAlias(property); if (set == null) return null; + return joniPropertyResult(set, caseFold); + } + + private static CharacterPropertyResolver.Result joniPropertyResult( + UnicodeSet set, boolean caseFold) { int[] ranges = new int[set.getRangeCount() * 2 + 1]; ranges[0] = set.getRangeCount(); for (int i = 0; i < set.getRangeCount(); i++) { @@ -1232,6 +1244,19 @@ && loosePropertyName(alias.substring(0, assignment)) } } } + String looseIsValue = looseIsShortcutValue(alias); + boolean inheritedBareIs = looseIsValue != null + && PerlUnicodeScriptData.canonicalValue(looseIsValue) == null + && PerlUnicodeBlockData.set(looseIsValue) == null; + if (inheritedBareIs) { + // Keep General_Category ahead of binary aliases in the shared Is + // shortcut namespace, before Block's ambiguity guard runs. + UnicodeSet category = PerlUnicodeGeneralCategoryData.resolve(looseIsValue); + if (category != null) return category; + if (isIcuBinaryPropertyAlias(looseIsValue)) { + return new UnicodeSet().applyPropertyAlias(looseIsValue, "True"); + } + } if (alias.equalsIgnoreCase("L&")) { UnicodeSet casedLetters = unicodePropertyValueSet( UProperty.GENERAL_CATEGORY, "UppercaseLetter"); @@ -1334,7 +1359,15 @@ && loosePropertyName(alias.substring(0, assignment)) // representation debt is closed; explicit Block=/In forms remain pinned. return null; } - return block; + if (block != null) return block; + + if (inheritedBareIs) { + // Only inherit aliases whose unprefixed spelling already resolves. + // This preserves user-property lookup and leaves missing bare bases + // (for example All and Unicode) for their owning property slices. + return resolveStandardPropertyAsSet(looseIsValue, new LinkedHashSet<>()); + } + return null; } private static Boolean perlBooleanPropertyValue(String value) { @@ -1363,6 +1396,29 @@ private static boolean isIcuGeneralCategoryAlias(String alias) { } } + private static String looseIsShortcutValue(String property) { + if (propertyValueDelimiter(property) >= 0) return null; + int prefixStart = 0; + while (prefixStart < property.length()) { + char separator = property.charAt(prefixStart); + if (!Character.isWhitespace(separator) + && separator != '-' && separator != '_') break; + prefixStart++; + } + if (property.length() - prefixStart <= 2 + || !property.regionMatches(true, prefixStart, "is", 0, 2)) { + return null; + } + int valueStart = prefixStart + 2; + while (valueStart < property.length()) { + char separator = property.charAt(valueStart); + if (!Character.isWhitespace(separator) + && separator != '-' && separator != '_') break; + valueStart++; + } + return valueStart < property.length() ? property.substring(valueStart) : null; + } + private static boolean isPerlIsPrefixedNumericWildcard(String property) { int assignment = propertyValueDelimiter(property); if (assignment <= 0 || assignment == property.length() - 1) return false; diff --git a/src/test/resources/unit/regex/unicode_binary_leading_shortcuts.t b/src/test/resources/unit/regex/unicode_binary_leading_shortcuts.t new file mode 100644 index 000000000..15d68978a --- /dev/null +++ b/src/test/resources/unit/regex/unicode_binary_leading_shortcuts.t @@ -0,0 +1,78 @@ +use strict; +use warnings; +use Test::More; + +sub compile_property { + my ($property) = @_; + my $pattern = eval 'qr/\A\p{' . $property . '}\z/u'; + return ($pattern, $@); +} + +my ($canonical, $canonical_error) = compile_property('IsUppercase'); +ok(defined $canonical, 'canonical Is binary shortcut compiles') + or diag($canonical_error); +like('A', $canonical, 'canonical Is binary shortcut matches a member'); +unlike('a', $canonical, 'canonical Is binary shortcut excludes a nonmember'); + +my ($lowercase, $lowercase_error) = compile_property('isuppercase'); +ok(defined $lowercase, 'binary shortcut accepts a lowercase Is prefix') + or diag($lowercase_error); +like('A', $lowercase, 'lowercase Is binary shortcut matches a member'); + +my ($leading, $leading_error) = compile_property('__Is_uppercase'); +ok(defined $leading, 'binary shortcut accepts leading loose separators') + or diag($leading_error); +like('A', $leading, 'leading-loose binary shortcut matches a member'); +unlike('a', $leading, 'leading-loose binary shortcut excludes a nonmember'); + +my ($hex, $hex_error) = compile_property('_ is_ASCII_HEX_DIGIT'); +ok(defined $hex, 'short binary alias accepts fully loose Is spelling') + or diag($hex_error); +like('F', $hex, 'loose binary alias matches its member'); +unlike('G', $hex, 'loose binary alias excludes a nonmember'); + +my ($assigned, $assigned_error) = compile_property('- Is_Assigned'); +ok(defined $assigned, 'another binary family accepts a leading-loose Is') + or diag($assigned_error); +like('A', $assigned, 'Assigned shortcut matches an assigned character'); +unlike(chr(0x0378), $assigned, + 'Assigned shortcut excludes an unassigned character'); + +my ($script, $script_error) = compile_property('_ is_Latn'); +ok(defined $script, 'Script shortcut retains precedence') or diag($script_error); +like('A', $script, 'Script shortcut membership remains intact'); +unlike(chr(0x03B1), $script, 'Script shortcut still excludes another script'); + +my ($category, $category_error) = compile_property('IsL'); +ok(defined $category, 'General_Category retains precedence') + or diag($category_error); +like(chr(0x03B1), $category, + 'General_Category shortcut still matches non-Latin letters'); + +my ($block, $block_error) = compile_property('_ Is_Basic_Latin'); +ok(defined $block, 'Block shortcut retains precedence') or diag($block_error); +like('A', $block, 'Block shortcut membership remains intact'); + +my $callback_calls = 0; +sub IsBinaryShortcutCallback { + $callback_calls++; + return "0041"; +} +my ($callback, $callback_error) = compile_property('IsBinaryShortcutCallback'); +ok(defined $callback, 'exact user property callback retains precedence') + or diag($callback_error); +like('A', $callback, 'exact user property callback matches its range'); +is($callback_calls, 1, 'exact user property callback is called once'); + +my ($leading_callback, $leading_callback_error) = + compile_property('_ IsBinaryShortcutCallback'); +ok(!defined($leading_callback) && length($leading_callback_error), + 'leading separators do not broaden user property lookup'); +is($callback_calls, 1, 'rejected leading callback spelling is not invoked'); + +my ($unknown, $unknown_error) = + compile_property('_ Is_DefinitelyNotABinaryProperty'); +ok(!defined($unknown) && length($unknown_error), + 'unknown leading-loose binary shortcut is rejected'); + +done_testing;