From d862449936bf5454d98553418434e6752e193fea Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Tue, 18 Aug 2026 19:13:45 +0200 Subject: [PATCH] refactor(regex): make Joni property folding explicit Carry per-property case-fold policy through the forked Joni resolver and parse standalone pinned Block, Script, and Script_Extensions assignments natively. Keep no-fold properties inside composed classes in the adapter until Joni can preserve member policy through class algebra. Generated with [OpenAI Codex](https://openai.com/codex) Co-Authored-By: OpenAI Codex --- docs/reference/feature-matrix.md | 2 +- .../runtime/regex/JoniRegexPattern.java | 13 +++++--- .../runtime/regex/UnicodeResolver.java | 32 ++++++++++++++++--- .../runtime/regex/JoniRegexPatternTest.java | 12 +++++++ .../regex_joni_unicode_property_fold_policy.t | 18 +++++++++++ .../org/joni/CharacterPropertyResolver.java | 20 ++++++++++-- third_party/joni/src/org/joni/Lexer.java | 20 +++++++----- third_party/joni/src/org/joni/Parser.java | 6 ++-- .../test/TestCharacterPropertyResolver.java | 16 +++++++--- 9 files changed, 110 insertions(+), 29 deletions(-) create mode 100644 src/test/resources/unit/regex_joni_unicode_property_fold_policy.t diff --git a/docs/reference/feature-matrix.md b/docs/reference/feature-matrix.md index c7bf5550e..a703f659a 100644 --- a/docs/reference/feature-matrix.md +++ b/docs/reference/feature-matrix.md @@ -379,7 +379,7 @@ my @copy = @{$z}; # ERROR - ✅ **Backreferences to Named Groups**: Using `\k` or `\g{name}` for backreferences to named groups is supported. - ✅ **Relative Backreferences**: Using `\g{-n}` for relative backreferences. - ✅ **Basic Unicode Properties**: Common `\p{...}` and `\P{...}` forms such as `\p{L}` execute through Joni. General_Category assignments now enter the forked Joni parser unchanged and resolve to pinned Perl ranges there. -- 🟡 **Perl Unicode Property Syntax**: Perl-specific properties execute through Joni. `Age`, cumulative `In`/`Present_In`, General_Category, Canonical_Combining_Class, Bidi_Class, Decomposition_Type, East_Asian_Width, Numeric_Value, Joining_Group, Block, Script, and Script_Extensions assignments are generated from pinned Perl 5.44 Unicode 17.0 data with loose and wildcard aliases, ordered missing defaults, official compact aliases and shortcuts, Script-versus-Script_Extensions policy, reserved or composite values, exact rationals, and Perl's generated decimal keyword aliases; `ASCII_Hex_Digit`/`AHex` accepts Perl's eight boolean value aliases. General_Category uses Joni's range-resolver API; the other families remain adapter-translated until their family-specific case-fold and wildcard semantics are represented natively. Other generated property/value aliases still have pinned acceptance/rejection gaps. +- 🟡 **Perl Unicode Property Syntax**: Perl-specific properties execute through Joni. `Age`, cumulative `In`/`Present_In`, General_Category, Canonical_Combining_Class, Bidi_Class, Decomposition_Type, East_Asian_Width, Numeric_Value, Joining_Group, Block, Script, and Script_Extensions assignments are generated from pinned Perl 5.44 Unicode 17.0 data with loose and wildcard aliases, ordered missing defaults, official compact aliases and shortcuts, Script-versus-Script_Extensions policy, reserved or composite values, exact rationals, and Perl's generated decimal keyword aliases; `ASCII_Hex_Digit`/`AHex` accepts Perl's eight boolean value aliases. General_Category and standalone Block, Script, and Script_Extensions assignments use Joni's range-resolver API with explicit per-family case-fold policy. No-fold properties inside composed character classes and wildcard values remain adapter-translated until Joni represents those syntax semantics natively. Other generated property/value aliases still have pinned acceptance/rejection gaps. - ✅ **Possessive Quantifiers**: Quantifiers like `*+`, `++`, `?+`, and `{n,m}+`, which disable backtracking, are supported. - ✅ **Atomic Grouping**: Use of `(?>...)` for atomic groups is supported. - ✅ **`\K` assertion**: Keep left — in `s///`, text before `\K` is preserved; match variables reflect only the portion after `\K`. Ordinary KEEP assertions route through native Joni and no longer use the Java marker rewrite; the adapter still rejects KEEP inside lookaround until the Joni analyser emits Perl's diagnostic directly. diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index a8b8cf0c9..a27b7de9f 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -6,6 +6,7 @@ import org.joni.Matcher; import org.joni.CalloutHandler; import org.joni.CalloutResult; +import org.joni.CharacterPropertyResolver; import org.joni.DynamicPatternResult; import org.joni.MatchView; import org.joni.NameEntry; @@ -60,12 +61,13 @@ private static int resolveNamedCharacter(byte[] bytes, int p, int end, ? StandardCharsets.ISO_8859_1 : StandardCharsets.UTF_8)); } - private static int[] resolveCharacterProperty(byte[] bytes, int p, int end, - Encoding encoding) { + private static CharacterPropertyResolver.Result resolveCharacterProperty( + byte[] bytes, int p, int end, Encoding encoding, + boolean inCharacterClass) { String property = new String(bytes, p, end - p, encoding == ISO8859_1Encoding.INSTANCE ? StandardCharsets.ISO_8859_1 : StandardCharsets.UTF_8); - return UnicodeResolver.resolveJoniPropertyRanges(property); + return UnicodeResolver.resolveJoniProperty(property, inCharacterClass); } private final Regex regex; @@ -231,8 +233,9 @@ private static UserPropertyTranslation translateUserDefinedProperties( boolean frontendProperty = unnegated.matches( "(?i)^(?:script|sc|block|blk|age|in|present[_ ]?in)\\s*(?:=|:(?!:)).*"); boolean perlBuiltInAlias = UnicodeResolver.isPerlBuiltInPropertyAlias(unnegated); - boolean joniResolvedProperty = UnicodeResolver.resolveJoniPropertyRanges( - unnegated) != null; + boolean joniResolvedProperty = UnicodeResolver.resolveJoniProperty( + unnegated, extendedClassBracketDepth > 0 + || standardClassBracketDepth > 0) != null; if (!userDefined && joniResolvedProperty && (frontendProperty || scriptExtensions || perlBuiltInAlias)) { translated.append(pattern, i, end + 1); diff --git a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java index d1505bd9d..f67f368c7 100644 --- a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java +++ b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java @@ -3,6 +3,7 @@ import com.ibm.icu.lang.UCharacter; import com.ibm.icu.lang.UProperty; import com.ibm.icu.text.UnicodeSet; +import org.joni.CharacterPropertyResolver; import org.perlonjava.app.scriptengine.PerlLanguageProvider; import org.perlonjava.runtime.runtimetypes.*; @@ -978,14 +979,35 @@ static boolean isPerlBuiltInPropertyAlias(String property) { || resolvePerlBuiltInPropertyAlias(property) != null; } - /** Returns pinned Perl-property ranges in Joni's native range-array format. */ - static int[] resolveJoniPropertyRanges(String property) { + /** Returns pinned Perl-property ranges and their native Joni fold policy. */ + static CharacterPropertyResolver.Result resolveJoniProperty( + String property, boolean inCharacterClass) { if (property == null) return null; int assignment = propertyValueDelimiter(property); - if (assignment <= 0 || assignment == property.length() - 1 - || !isGeneralCategoryProperty(property.substring(0, assignment))) { + if (assignment <= 0 || assignment == property.length() - 1) { return null; } + String name = property.substring(0, assignment); + String value = property.substring(assignment + 1); + boolean caseFold; + if (isGeneralCategoryProperty(name)) { + caseFold = true; + } else if (PerlUnicodeBlockData.isPropertyAlias(name)) { + if (perlBlockWildcardBody(value) != null) return null; + caseFold = false; + } else if (PerlUnicodeScriptData.isScriptPropertyAlias(name) + || PerlUnicodeScriptData.isScriptExtensionsPropertyAlias(name)) { + if (perlNumericWildcardBody(value) != null) return null; + caseFold = false; + } else { + return null; + } + + // Joni currently folds a complete bracket expression as one class. + // Keep no-fold families translated by the adapter inside brackets until + // the AST can retain per-property fold policy through class composition. + if (!caseFold && inCharacterClass) return null; + UnicodeSet set = resolvePerlBuiltInPropertyAlias(property); if (set == null) return null; @@ -995,7 +1017,7 @@ static int[] resolveJoniPropertyRanges(String property) { ranges[i * 2 + 1] = set.getRangeStart(i); ranges[i * 2 + 2] = set.getRangeEnd(i); } - return ranges; + return new CharacterPropertyResolver.Result(ranges, caseFold); } private static boolean isPerlSpecialPropertyAlias(String property) { diff --git a/src/test/java/org/perlonjava/runtime/regex/JoniRegexPatternTest.java b/src/test/java/org/perlonjava/runtime/regex/JoniRegexPatternTest.java index 865286794..13dbc1e3a 100644 --- a/src/test/java/org/perlonjava/runtime/regex/JoniRegexPatternTest.java +++ b/src/test/java/org/perlonjava/runtime/regex/JoniRegexPatternTest.java @@ -115,6 +115,18 @@ void passesPinnedGeneralCategoriesToTheJoniParserWithoutTextExpansion() { assertFalse(foldedNegated.matcher("A", java.util.List.of()).find()); } + @Test + void preservesStandaloneBlockAndScriptFoldPolicyInsideJoni() { + RegexFlags ignoreCase = RegexFlags.fromModifiers("i", "property"); + JoniRegexPattern block = new JoniRegexPattern("\\p{Block=ASCII}", ignoreCase); + JoniRegexPattern script = new JoniRegexPattern("\\p{Script=Common}", ignoreCase); + + assertEquals("\\p{Block=ASCII}", block.patternDescription()); + assertEquals("\\p{Script=Common}", script.patternDescription()); + assertFalse(block.matcher("\u212A", java.util.List.of()).find()); + assertFalse(script.matcher("K", java.util.List.of()).find()); + } + @Test void flattensTranslatedPropertiesInsideOrdinaryCharacterClasses() { JoniRegexPattern pattern = new JoniRegexPattern( diff --git a/src/test/resources/unit/regex_joni_unicode_property_fold_policy.t b/src/test/resources/unit/regex_joni_unicode_property_fold_policy.t new file mode 100644 index 000000000..cee1f2432 --- /dev/null +++ b/src/test/resources/unit/regex_joni_unicode_property_fold_policy.t @@ -0,0 +1,18 @@ +use strict; +use warnings; +use utf8; +use Test::More tests => 8; + +ok("A" =~ /\p{gc=Uppercase_Letter}/, 'general category positive'); +ok("a" !~ /\p{gc=Uppercase_Letter}/, 'general category negative'); +ok("é" =~ /\p{Script=Latin}/, 'script property in pinned Perl data'); +ok("a" =~ /\p{gc=Uppercase_Letter}/i, + 'general category participates in case folding'); +ok("A" !~ /\P{gc=Uppercase_Letter}/i, + 'negated category complements after case folding'); +ok("\x{212A}" !~ /\p{Block=ASCII}/i, + 'block membership does not gain case-fold members'); +ok("K" !~ /\p{Script=Common}/i, + 'script membership does not gain case-fold members'); +ok("k" =~ /\p{gc=Uppercase_Letter}/i, + 'general-category membership gains case-fold members'); diff --git a/third_party/joni/src/org/joni/CharacterPropertyResolver.java b/third_party/joni/src/org/joni/CharacterPropertyResolver.java index dd31598dd..de7bf3fd0 100644 --- a/third_party/joni/src/org/joni/CharacterPropertyResolver.java +++ b/third_party/joni/src/org/joni/CharacterPropertyResolver.java @@ -24,9 +24,23 @@ /** Resolves syntax-specific character properties to inclusive code-point ranges. */ @FunctionalInterface public interface CharacterPropertyResolver { + /** A resolved range set and whether ignore-case folding applies to it. */ + final class Result { + public final int[] ranges; + public final boolean caseFold; + + public Result(int[] ranges, boolean caseFold) { + this.ranges = ranges; + this.caseFold = caseFold; + } + } + /** - * Returns {@code [count, from1, to1, ...]} for a resolved property, or - * {@code null} to use the encoding's built-in property lookup. + * Returns resolved ranges and their ignore-case policy, or {@code null} to + * use the encoding's built-in property lookup. The context flag allows a + * resolver to defer properties whose class-composition semantics require + * frontend handling. */ - int[] resolve(byte[] bytes, int p, int end, Encoding encoding); + Result resolve(byte[] bytes, int p, int end, Encoding encoding, + boolean inCharacterClass); } diff --git a/third_party/joni/src/org/joni/Lexer.java b/third_party/joni/src/org/joni/Lexer.java index 0e26280a8..ca80215d6 100644 --- a/third_party/joni/src/org/joni/Lexer.java +++ b/third_party/joni/src/org/joni/Lexer.java @@ -1698,14 +1698,16 @@ private void possessiveCheck() { protected static final class CharProperty { final int ctype; final int[] ranges; + final boolean caseFold; - CharProperty(int ctype, int[] ranges) { + CharProperty(int ctype, int[] ranges, boolean caseFold) { this.ctype = ctype; this.ranges = ranges; + this.caseFold = caseFold; } } - protected final CharProperty fetchCharProperty() { + protected final CharProperty fetchCharProperty(boolean inCharacterClass) { mark(); while (left()) { @@ -1713,15 +1715,17 @@ protected final CharProperty fetchCharProperty() { fetch(); if (c == '}') { if (syntax.characterPropertyResolver != null) { - int[] ranges = syntax.characterPropertyResolver.resolve( - bytes, _p, last, enc); - if (ranges != null) { - validateCharacterPropertyRanges(ranges); - return new CharProperty(0, ranges); + CharacterPropertyResolver.Result resolved = + syntax.characterPropertyResolver.resolve( + bytes, _p, last, enc, inCharacterClass); + if (resolved != null) { + validateCharacterPropertyRanges(resolved.ranges); + return new CharProperty(0, resolved.ranges, + resolved.caseFold); } } return new CharProperty( - enc.propertyNameToCType(bytes, _p, last), null); + enc.propertyNameToCType(bytes, _p, last), null, true); } else if (c == '(' || c == ')' || c == '{' || c == '|') { throw new CharacterPropertyException(EncodingError.ERR_INVALID_CHAR_PROPERTY_NAME, bytes, _p, last); } diff --git a/third_party/joni/src/org/joni/Parser.java b/third_party/joni/src/org/joni/Parser.java index af9cbfc07..cbfd5a616 100644 --- a/third_party/joni/src/org/joni/Parser.java +++ b/third_party/joni/src/org/joni/Parser.java @@ -256,7 +256,7 @@ private CClassNode parseCharClass(ObjPtr ascNode) { break; case CHAR_PROPERTY: - CharProperty property = fetchCharProperty(); + CharProperty property = fetchCharProperty(true); addCharProperty(cc, ascCc, property, token.getPropNot()); cc.nextStateClass(arg, ascCc, env); // goto next_class break; @@ -1629,13 +1629,13 @@ private Node cClassCaseFold(Node node, CClassNode cc, CClassNode ascCc) { } private Node parseCharProperty() { - CharProperty property = fetchCharProperty(); + CharProperty property = fetchCharProperty(false); CClassNode cc = new CClassNode(); Node node = cc; addCharProperty(cc, null, property, false); if (token.getPropNot()) cc.setNot(); - if (isIgnoreCase(env.option)) { + if (isIgnoreCase(env.option) && property.caseFold) { if (property.ranges != null || property.ctype != CharacterType.ASCII) { node = cClassCaseFold(node, cc, cc); } diff --git a/third_party/joni/test/org/joni/test/TestCharacterPropertyResolver.java b/third_party/joni/test/org/joni/test/TestCharacterPropertyResolver.java index 974df8eea..e03ff4757 100644 --- a/third_party/joni/test/org/joni/test/TestCharacterPropertyResolver.java +++ b/third_party/joni/test/org/joni/test/TestCharacterPropertyResolver.java @@ -34,10 +34,13 @@ public class TestCharacterPropertyResolver { private static final CharacterPropertyResolver RESOLVER = - (bytes, p, end, encoding) -> { + (bytes, p, end, encoding, inCharacterClass) -> { String name = new String(bytes, p, end - p, StandardCharsets.UTF_8); return switch (name) { - case "Fake" -> new int[] {2, 'A', 'A', 0x1f642, 0x1f642}; + case "Fake" -> new CharacterPropertyResolver.Result( + new int[] {2, 'A', 'A', 0x1f642, 0x1f642}, true); + case "FakeNoFold" -> new CharacterPropertyResolver.Result( + new int[] {1, 'A', 'A'}, false); default -> null; }; }; @@ -69,6 +72,8 @@ public void resolvesRangesInsideAndOutsideCharacterClasses() { assertEquals(-1, search("[\\P{Fake}]", "A")); assertEquals(0, search("(?i)\\p{Fake}", "a")); assertEquals(-1, search("(?i)\\P{Fake}", "A")); + assertEquals(0, search("(?i)\\p{FakeNoFold}", "A")); + assertEquals(-1, search("(?i)\\p{FakeNoFold}", "a")); } @Test @@ -81,7 +86,9 @@ public void fallsBackToEncodingProperties() { public void preservesResolverExceptions() { IllegalArgumentException expected = new IllegalArgumentException("failure"); try { - compile("\\p{Fake}", (bytes, p, end, encoding) -> { throw expected; }); + compile("\\p{Fake}", (bytes, p, end, encoding, inCharacterClass) -> { + throw expected; + }); fail("expected resolver exception"); } catch (IllegalArgumentException error) { assertSame(expected, error); @@ -91,7 +98,8 @@ public void preservesResolverExceptions() { @Test public void rejectsMalformedRangeResults() { try { - compile("\\p{Fake}", (bytes, p, end, encoding) -> new int[] {1, 2}); + compile("\\p{Fake}", (bytes, p, end, encoding, inCharacterClass) -> + new CharacterPropertyResolver.Result(new int[] {1, 2}, true)); fail("expected invalid range result"); } catch (IllegalArgumentException error) { assertEquals("invalid character property ranges", error.getMessage());