diff --git a/docs/reference/feature-matrix.md b/docs/reference/feature-matrix.md index c7bf5550e..3a7bb069c 100644 --- a/docs/reference/feature-matrix.md +++ b/docs/reference/feature-matrix.md @@ -379,7 +379,7 @@ my @copy = @{$z}; # ERROR - ✅ **Backreferences to Named Groups**: Using `\k` or `\g{name}` for backreferences to named groups is supported. - ✅ **Relative Backreferences**: Using `\g{-n}` for relative backreferences. - ✅ **Basic Unicode Properties**: Common `\p{...}` and `\P{...}` forms such as `\p{L}` execute through Joni. General_Category assignments now enter the forked Joni parser unchanged and resolve to pinned Perl ranges there. -- 🟡 **Perl Unicode Property Syntax**: Perl-specific properties execute through Joni. `Age`, cumulative `In`/`Present_In`, General_Category, Canonical_Combining_Class, Bidi_Class, Decomposition_Type, East_Asian_Width, Numeric_Value, Joining_Group, Block, Script, and Script_Extensions assignments are generated from pinned Perl 5.44 Unicode 17.0 data with loose and wildcard aliases, ordered missing defaults, official compact aliases and shortcuts, Script-versus-Script_Extensions policy, reserved or composite values, exact rationals, and Perl's generated decimal keyword aliases; `ASCII_Hex_Digit`/`AHex` accepts Perl's eight boolean value aliases. General_Category uses Joni's range-resolver API; the other families remain adapter-translated until their family-specific case-fold and wildcard semantics are represented natively. Other generated property/value aliases still have pinned acceptance/rejection gaps. +- 🟡 **Perl Unicode Property Syntax**: Perl-specific properties execute through Joni. `Age`, cumulative `In`/`Present_In`, General_Category, Canonical_Combining_Class, Bidi_Class, Decomposition_Type, East_Asian_Width, Numeric_Value, Joining_Group, Block, Script, and Script_Extensions assignments are generated from pinned Perl 5.44 Unicode 17.0 data with loose and wildcard aliases, ordered missing defaults, official compact aliases and shortcuts, Script-versus-Script_Extensions policy, reserved or composite values, exact rationals, and Perl's generated decimal keyword aliases; `ASCII_Hex_Digit`/`AHex` accepts Perl's eight boolean value aliases. General_Category and standalone exact Age, `In`/`Present_In`, Block, Script, Script_Extensions, Canonical_Combining_Class, Bidi_Class, Decomposition_Type, East_Asian_Width, Numeric_Value, and Joining_Group assignments use Joni's range-resolver API with explicit per-family case-fold policy. No-fold properties inside composed character classes and wildcard values remain adapter-translated until Joni represents those syntax semantics natively. Other generated property/value aliases still have pinned acceptance/rejection gaps. - ✅ **Possessive Quantifiers**: Quantifiers like `*+`, `++`, `?+`, and `{n,m}+`, which disable backtracking, are supported. - ✅ **Atomic Grouping**: Use of `(?>...)` for atomic groups is supported. - ✅ **`\K` assertion**: Keep left — in `s///`, text before `\K` is preserved; match variables reflect only the portion after `\K`. Ordinary KEEP assertions route through native Joni and no longer use the Java marker rewrite; the adapter still rejects KEEP inside lookaround until the Joni analyser emits Perl's diagnostic directly. diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index a8b8cf0c9..c21e111d8 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -6,6 +6,7 @@ import org.joni.Matcher; import org.joni.CalloutHandler; import org.joni.CalloutResult; +import org.joni.CharacterPropertyResolver; import org.joni.DynamicPatternResult; import org.joni.MatchView; import org.joni.NameEntry; @@ -60,12 +61,13 @@ private static int resolveNamedCharacter(byte[] bytes, int p, int end, ? StandardCharsets.ISO_8859_1 : StandardCharsets.UTF_8)); } - private static int[] resolveCharacterProperty(byte[] bytes, int p, int end, - Encoding encoding) { + private static CharacterPropertyResolver.Result resolveCharacterProperty( + byte[] bytes, int p, int end, Encoding encoding, + boolean inCharacterClass) { String property = new String(bytes, p, end - p, encoding == ISO8859_1Encoding.INSTANCE ? StandardCharsets.ISO_8859_1 : StandardCharsets.UTF_8); - return UnicodeResolver.resolveJoniPropertyRanges(property); + return UnicodeResolver.resolveJoniProperty(property, inCharacterClass); } private final Regex regex; @@ -231,8 +233,9 @@ private static UserPropertyTranslation translateUserDefinedProperties( boolean frontendProperty = unnegated.matches( "(?i)^(?:script|sc|block|blk|age|in|present[_ ]?in)\\s*(?:=|:(?!:)).*"); boolean perlBuiltInAlias = UnicodeResolver.isPerlBuiltInPropertyAlias(unnegated); - boolean joniResolvedProperty = UnicodeResolver.resolveJoniPropertyRanges( - unnegated) != null; + boolean joniResolvedProperty = UnicodeResolver.resolveJoniProperty( + unnegated, extendedClassBracketDepth > 0 + || standardClassBracketDepth > 0) != null; if (!userDefined && joniResolvedProperty && (frontendProperty || scriptExtensions || perlBuiltInAlias)) { translated.append(pattern, i, end + 1); @@ -270,7 +273,9 @@ private static UserPropertyTranslation translateUserDefinedProperties( .append(')'); } catch (IllegalArgumentException error) { String message = error.getMessage(); - if (!userDefined || message != null && message.contains("in expansion of")) { + if (!userDefined || message != null + && (message.contains("in expansion of") + || message.startsWith("Can't find Unicode property definition"))) { throw error; } translated.append("[\\s\\S]"); diff --git a/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessorHelper.java b/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessorHelper.java index 74fe5b135..f14c2c173 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessorHelper.java +++ b/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessorHelper.java @@ -355,11 +355,12 @@ static int handleEscapeSequences(String s, StringBuilder sb, int c, int offset, // Perl allows user-defined properties (InFoo/IsFoo) to be unknown at compile time; // they are resolved at runtime when the property sub is available. // If it's currently undefined, emit a placeholder that compiles in Java and mark for recompilation. - // But if the error already contains "in expansion of", it is a real user-property definition error - // that should be reported (not deferred). + // Explicit missing-property and "in expansion of" diagnostics are + // real user-property errors that should be reported, not deferred. String msg = e.getMessage(); if (UnicodeResolver.isUserDefinedPropertyName(property) - && (msg == null || !msg.contains("in expansion of"))) { + && (msg == null || (!msg.contains("in expansion of") + && !msg.startsWith("Can't find Unicode property definition")))) { RegexPreprocessor.markDeferredUnicodePropertyEncountered(); sb.setLength(sb.length() - 1); // Remove the backslash // Placeholder: match any single character, including newline diff --git a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java index d1505bd9d..991208d79 100644 --- a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java +++ b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java @@ -3,6 +3,7 @@ import com.ibm.icu.lang.UCharacter; import com.ibm.icu.lang.UProperty; import com.ibm.icu.text.UnicodeSet; +import org.joni.CharacterPropertyResolver; import org.perlonjava.app.scriptengine.PerlLanguageProvider; import org.perlonjava.runtime.runtimetypes.*; @@ -358,6 +359,7 @@ private static UnicodeSet resolvePropertyReferenceAsSet(String propRef, Set recursionSet) { + property = canonicalPerlPosixPropertyAlias(property); UnicodeSet perlBuiltInAlias = resolvePerlBuiltInPropertyAlias(property); if (perlBuiltInAlias != null) return perlBuiltInAlias; @@ -712,6 +714,7 @@ private static String translateUnicodeProperty(String property, boolean negated, property = property.substring(1).trim(); negated = !negated; } + property = canonicalPerlPosixPropertyAlias(property); boolean isPrefixedNumericWildcard = isPerlIsPrefixedNumericWildcard(property); boolean isPrefixedJoiningGroupWildcard = @@ -752,6 +755,12 @@ private static String translateUnicodeProperty(String property, boolean negated, if (userProp != null) { return wrapCharClass(userProp, negated); } + String looseUserProperty = loosePropertyName(property); + if (looseUserProperty.startsWith("isxposix") + || looseUserProperty.startsWith("isposix")) { + throw new IllegalArgumentException( + "Can't find Unicode property definition \"" + property + "\""); + } // Property not found - fall through to throw error below } @@ -973,29 +982,76 @@ static boolean isUserDefinedPropertyName(String property) { static boolean isPerlBuiltInPropertyAlias(String property) { if (property == null) return false; + property = canonicalPerlPosixPropertyAlias(property); return isPerlSpecialPropertyAlias(property.trim()) || !normalizePerlIsPropertyAssignment(property).equals(property) || resolvePerlBuiltInPropertyAlias(property) != null; } - /** Returns pinned Perl-property ranges in Joni's native range-array format. */ - static int[] resolveJoniPropertyRanges(String property) { + /** Returns pinned Perl-property ranges and their native Joni fold policy. */ + static CharacterPropertyResolver.Result resolveJoniProperty( + String property, boolean inCharacterClass) { if (property == null) return null; + property = normalizePerlIsPropertyAssignment(property); int assignment = propertyValueDelimiter(property); - if (assignment <= 0 || assignment == property.length() - 1 - || !isGeneralCategoryProperty(property.substring(0, assignment))) { + if (assignment <= 0 || assignment == property.length() - 1) { + String looseIsValue = looseIsShortcutValue(property); + if (looseIsValue == null + || PerlUnicodeScriptData.canonicalValue(looseIsValue) != null + || PerlUnicodeBlockData.set(looseIsValue) != null) { + return null; + } + UnicodeSet bareSet = resolvePerlBuiltInPropertyAlias(property); + return bareSet == null ? null : joniPropertyResult(bareSet, true); + } + String name = property.substring(0, assignment); + String value = property.substring(assignment + 1); + boolean caseFold; + if (isGeneralCategoryProperty(name)) { + caseFold = true; + } else if (PerlUnicodeBlockData.isPropertyAlias(name)) { + if (perlBlockWildcardBody(value) != null) return null; + caseFold = false; + } else if (PerlUnicodeScriptData.isScriptPropertyAlias(name) + || PerlUnicodeScriptData.isScriptExtensionsPropertyAlias(name)) { + if (perlNumericWildcardBody(value) != null) return null; + caseFold = false; + } else if (isCanonicalCombiningClassProperty(name) + || PerlUnicodeBidiClassData.isPropertyAlias(name) + || PerlUnicodeDecompositionTypeData.isPropertyAlias(name) + || PerlUnicodeEastAsianWidthData.isPropertyAlias(name)) { + caseFold = false; + } else if (PerlUnicodeNumericValueData.isPropertyAlias(name) + || PerlUnicodeJoiningGroupData.isPropertyAlias(name)) { + if (perlNumericWildcardBody(value) != null) return null; + caseFold = false; + } else if (isPerlAgeProperty(name)) { + if (isPerlAgeWildcard(value)) return null; + caseFold = false; + } else { return null; } + + // Joni currently folds a complete bracket expression as one class. + // Keep no-fold families translated by the adapter inside brackets until + // the AST can retain per-property fold policy through class composition. + if (!caseFold && inCharacterClass) return null; + UnicodeSet set = resolvePerlBuiltInPropertyAlias(property); if (set == null) return null; + return joniPropertyResult(set, caseFold); + } + + private static CharacterPropertyResolver.Result joniPropertyResult( + UnicodeSet set, boolean caseFold) { int[] ranges = new int[set.getRangeCount() * 2 + 1]; ranges[0] = set.getRangeCount(); for (int i = 0; i < set.getRangeCount(); i++) { ranges[i * 2 + 1] = set.getRangeStart(i); ranges[i * 2 + 2] = set.getRangeEnd(i); } - return ranges; + return new CharacterPropertyResolver.Result(ranges, caseFold); } private static boolean isPerlSpecialPropertyAlias(String property) { @@ -1026,10 +1082,50 @@ private static boolean isPerlSpecialPropertyAlias(String property) { }; } + private static String canonicalPerlPosixPropertyAlias(String property) { + if (property == null) return null; + String loose = loosePropertyName(property); + if (loose.startsWith("is")) { + String unprefixed = loose.substring(2); + if (unprefixed.startsWith("xposix") || unprefixed.startsWith("posix")) { + loose = unprefixed; + } + } + return switch (loose) { + case "xposixalnum" -> "XPosixAlnum"; + case "xposixalpha" -> "XPosixAlpha"; + case "xposixblank" -> "XPosixBlank"; + case "xposixcntrl" -> "XPosixCntrl"; + case "xposixdigit" -> "XPosixDigit"; + case "xposixgraph" -> "XPosixGraph"; + case "xposixlower" -> "XPosixLower"; + case "xposixprint" -> "XPosixPrint"; + case "xposixpunct" -> "XPosixPunct"; + case "xposixspace" -> "XPosixSpace"; + case "xposixupper" -> "XPosixUpper"; + case "xposixword" -> "XPosixWord"; + case "xposixxdigit" -> "XPosixXDigit"; + case "posixalnum" -> "PosixAlnum"; + case "posixalpha" -> "PosixAlpha"; + case "posixblank" -> "PosixBlank"; + case "posixcntrl" -> "PosixCntrl"; + case "posixdigit" -> "PosixDigit"; + case "posixgraph" -> "PosixGraph"; + case "posixlower" -> "PosixLower"; + case "posixprint" -> "PosixPrint"; + case "posixpunct" -> "PosixPunct"; + case "posixspace" -> "PosixSpace"; + case "posixupper" -> "PosixUpper"; + case "posixword" -> "PosixWord"; + case "posixxdigit" -> "PosixXDigit"; + default -> property; + }; + } + private static UnicodeSet resolvePerlBuiltInPropertyAlias(String property) { if (property == null) return null; - String alias = property.trim(); + String alias = normalizePerlIsPropertyAssignment(property.trim()); int assignment = propertyValueDelimiter(alias); if (assignment == alias.length() - 1 && (PerlUnicodeScriptData.isScriptPropertyAlias( @@ -1050,6 +1146,8 @@ && isGeneralCategoryProperty(alias.substring(0, assignment))) { } return category; } + UnicodeSet age = resolvePerlAgeProperty(alias, true); + if (age != null) return age; if (assignment > 0 && assignment < alias.length() - 1 && isCanonicalCombiningClassProperty(alias.substring(0, assignment))) { UnicodeSet combiningClass = PerlUnicodeCombiningClassData.resolve( @@ -1146,6 +1244,19 @@ && loosePropertyName(alias.substring(0, assignment)) } } } + String looseIsValue = looseIsShortcutValue(alias); + boolean inheritedBareIs = looseIsValue != null + && PerlUnicodeScriptData.canonicalValue(looseIsValue) == null + && PerlUnicodeBlockData.set(looseIsValue) == null; + if (inheritedBareIs) { + // Keep General_Category ahead of binary aliases in the shared Is + // shortcut namespace, before Block's ambiguity guard runs. + UnicodeSet category = PerlUnicodeGeneralCategoryData.resolve(looseIsValue); + if (category != null) return category; + if (isIcuBinaryPropertyAlias(looseIsValue)) { + return new UnicodeSet().applyPropertyAlias(looseIsValue, "True"); + } + } if (alias.equalsIgnoreCase("L&")) { UnicodeSet casedLetters = unicodePropertyValueSet( UProperty.GENERAL_CATEGORY, "UppercaseLetter"); @@ -1159,8 +1270,16 @@ && loosePropertyName(alias.substring(0, assignment)) // Perl's bare script-value shortcuts use Script_Extensions semantics. // Keep binary and General_Category names ahead of this value namespace. String scriptShortcut = alias; - if (alias.length() > 2 && alias.startsWith("Is")) { - int valueStart = 2; + int scriptPrefixStart = 0; + while (scriptPrefixStart < alias.length()) { + char separator = alias.charAt(scriptPrefixStart); + if (!Character.isWhitespace(separator) + && separator != '-' && separator != '_') break; + scriptPrefixStart++; + } + if (alias.length() - scriptPrefixStart > 2 + && alias.regionMatches(true, scriptPrefixStart, "is", 0, 2)) { + int valueStart = scriptPrefixStart + 2; while (valueStart < alias.length()) { char separator = alias.charAt(valueStart); if (!Character.isWhitespace(separator) @@ -1180,31 +1299,50 @@ && loosePropertyName(alias.substring(0, assignment)) } } - String blockAlias = alias; + String blockShortcutAlias = alias; + if (assignment < 0) { + int prefixStart = 0; + while (prefixStart < alias.length()) { + char separator = alias.charAt(prefixStart); + if (!Character.isWhitespace(separator) + && separator != '-' && separator != '_') break; + prefixStart++; + } + if (prefixStart > 0 && alias.length() - prefixStart > 2 + && (alias.regionMatches(true, prefixStart, "in", 0, 2) + || alias.regionMatches(true, prefixStart, "is", 0, 2))) { + blockShortcutAlias = alias.substring(prefixStart); + } + } + + String blockAlias = blockShortcutAlias; boolean blockShortcut = false; boolean isBlockShortcut = false; - if (alias.length() > 2 && alias.regionMatches(true, 0, "in", 0, 2)) { + if (blockShortcutAlias.length() > 2 + && blockShortcutAlias.regionMatches(true, 0, "in", 0, 2)) { int valueStart = 2; - while (valueStart < alias.length()) { - char separator = alias.charAt(valueStart); + while (valueStart < blockShortcutAlias.length()) { + char separator = blockShortcutAlias.charAt(valueStart); if (!Character.isWhitespace(separator) && separator != '-' && separator != '_') break; valueStart++; } - if (valueStart >= alias.length()) return null; - blockAlias = alias.substring(valueStart); + if (valueStart >= blockShortcutAlias.length()) return null; + blockAlias = blockShortcutAlias.substring(valueStart); blockShortcut = true; - } else if (assignment >= 0 || unicodePropertyValue(UProperty.SCRIPT, alias) >= 0) { + } else if (assignment >= 0 + || unicodePropertyValue(UProperty.SCRIPT, blockShortcutAlias) >= 0) { return null; - } else if (alias.length() > 2 && alias.regionMatches(true, 0, "is", 0, 2)) { + } else if (blockShortcutAlias.length() > 2 + && blockShortcutAlias.regionMatches(true, 0, "is", 0, 2)) { int valueStart = 2; - while (valueStart < alias.length()) { - char separator = alias.charAt(valueStart); + while (valueStart < blockShortcutAlias.length()) { + char separator = blockShortcutAlias.charAt(valueStart); if (!Character.isWhitespace(separator) && separator != '-' && separator != '_') break; valueStart++; } - if (valueStart >= alias.length()) return null; - String candidate = alias.substring(valueStart); + if (valueStart >= blockShortcutAlias.length()) return null; + String candidate = blockShortcutAlias.substring(valueStart); if (unicodePropertyValue(UProperty.SCRIPT, candidate) >= 0) return null; if (isIcuBinaryPropertyAlias(candidate) || isIcuGeneralCategoryAlias(candidate)) return null; @@ -1221,7 +1359,15 @@ && loosePropertyName(alias.substring(0, assignment)) // representation debt is closed; explicit Block=/In forms remain pinned. return null; } - return block; + if (block != null) return block; + + if (inheritedBareIs) { + // Only inherit aliases whose unprefixed spelling already resolves. + // This preserves user-property lookup and leaves missing bare bases + // (for example All and Unicode) for their owning property slices. + return resolveStandardPropertyAsSet(looseIsValue, new LinkedHashSet<>()); + } + return null; } private static Boolean perlBooleanPropertyValue(String value) { @@ -1250,6 +1396,29 @@ private static boolean isIcuGeneralCategoryAlias(String alias) { } } + private static String looseIsShortcutValue(String property) { + if (propertyValueDelimiter(property) >= 0) return null; + int prefixStart = 0; + while (prefixStart < property.length()) { + char separator = property.charAt(prefixStart); + if (!Character.isWhitespace(separator) + && separator != '-' && separator != '_') break; + prefixStart++; + } + if (property.length() - prefixStart <= 2 + || !property.regionMatches(true, prefixStart, "is", 0, 2)) { + return null; + } + int valueStart = prefixStart + 2; + while (valueStart < property.length()) { + char separator = property.charAt(valueStart); + if (!Character.isWhitespace(separator) + && separator != '-' && separator != '_') break; + valueStart++; + } + return valueStart < property.length() ? property.substring(valueStart) : null; + } + private static boolean isPerlIsPrefixedNumericWildcard(String property) { int assignment = propertyValueDelimiter(property); if (assignment <= 0 || assignment == property.length() - 1) return false; @@ -1370,8 +1539,20 @@ private static UnicodeSet resolvePerlBlock(String value) { UnicodeSet result = new UnicodeSet(); for (int valueId = 0; valueId < PerlUnicodeBlockData.valueCount(); valueId++) { String candidate = PerlUnicodeBlockData.canonicalValue(valueId); - if (valuePattern.matcher(candidate).matches() - || valuePattern.matcher(loosePropertyName(candidate)).matches()) { + boolean matches = valuePattern.matcher(candidate).matches() + || valuePattern.matcher(loosePropertyName(candidate)).matches(); + int icuValue = unicodePropertyValue(UProperty.BLOCK, candidate); + for (int nameChoice = UProperty.NameChoice.SHORT; + !matches && icuValue >= 0 && nameChoice <= UProperty.NameChoice.LONG; + nameChoice++) { + String officialAlias = UCharacter.getPropertyValueName( + UProperty.BLOCK, icuValue, nameChoice); + matches = officialAlias != null + && (valuePattern.matcher(officialAlias).matches() + || valuePattern.matcher( + loosePropertyName(officialAlias)).matches()); + } + if (matches) { result.addAll(PerlUnicodeBlockData.set(valueId)); } } @@ -1439,6 +1620,10 @@ private static UnicodeSet resolvePerlScript(String value, boolean extensions) { private static String perlBlockWildcardBody(String value) { String trimmed = value.trim(); + if (trimmed.startsWith(":\\A") && trimmed.endsWith("\\z:") + && trimmed.length() > 6) { + return trimmed.substring(3, trimmed.length() - 3); + } if (!trimmed.startsWith("#") || !trimmed.endsWith("#") || trimmed.length() <= 2) { return null; @@ -1691,26 +1876,33 @@ private static int unicodePropertyValue(int property, String alias) { } private static String translatePerlAgeProperty(String property, boolean negated) { - int delimiter = property.indexOf('='); - int colon = property.indexOf(':'); - if (delimiter < 0 || colon > 0 && colon < delimiter) delimiter = colon; + UnicodeSet result = resolvePerlAgeProperty(property, true); + return result == null ? null + : wrapCharClass(unicodeSetToJavaPattern(result), negated); + } + + private static UnicodeSet resolvePerlAgeProperty( + String property, boolean allowWildcard) { + property = normalizePerlIsPropertyAssignment(property); + int delimiter = propertyValueDelimiter(property); if (delimiter <= 0 || delimiter == property.length() - 1) return null; - String name = property.substring(0, delimiter) - .replace("_", "").replace("-", "").replace(" ", ""); + String name = property.substring(0, delimiter); boolean exact; - if (name.equalsIgnoreCase("Age")) { + String looseName = loosePropertyName(name); + if (looseName.equals("age")) { exact = true; - } else if (name.equalsIgnoreCase("In") || name.equalsIgnoreCase("PresentIn")) { + } else if (looseName.equals("in") || looseName.equals("presentin")) { exact = false; } else { return null; } - String requested = normalizeUnicodeAgeVersion(property.substring(delimiter + 1)); + String value = property.substring(delimiter + 1); + if (!allowWildcard && isPerlAgeWildcard(value)) return null; + String requested = normalizeUnicodeAgeVersion(value); if (requested.equalsIgnoreCase("NA") || requested.equalsIgnoreCase("Unassigned")) { - return wrapCharClass( - unicodeSetToJavaPattern(PerlUnicodeAgeData.unassignedSet()), negated); + return PerlUnicodeAgeData.unassignedSet(); } UnicodeSet result = exact @@ -1719,7 +1911,18 @@ private static String translatePerlAgeProperty(String property, boolean negated) if (result == null) { throw new IllegalArgumentException("Unsupported Unicode age version: " + requested); } - return wrapCharClass(unicodeSetToJavaPattern(result), negated); + return result; + } + + private static boolean isPerlAgeProperty(String name) { + String looseName = loosePropertyName(name); + return looseName.equals("age") || looseName.equals("in") + || looseName.equals("presentin"); + } + + private static boolean isPerlAgeWildcard(String value) { + String trimmed = value.trim(); + return trimmed.startsWith(":\\A") && trimmed.endsWith("\\z:"); } private static String normalizeUnicodeAgeVersion(String value) { diff --git a/src/test/java/org/perlonjava/runtime/regex/JoniRegexPatternTest.java b/src/test/java/org/perlonjava/runtime/regex/JoniRegexPatternTest.java index 865286794..eaeaf0f8e 100644 --- a/src/test/java/org/perlonjava/runtime/regex/JoniRegexPatternTest.java +++ b/src/test/java/org/perlonjava/runtime/regex/JoniRegexPatternTest.java @@ -115,6 +115,57 @@ void passesPinnedGeneralCategoriesToTheJoniParserWithoutTextExpansion() { assertFalse(foldedNegated.matcher("A", java.util.List.of()).find()); } + @Test + void preservesStandaloneBlockAndScriptFoldPolicyInsideJoni() { + RegexFlags ignoreCase = RegexFlags.fromModifiers("i", "property"); + JoniRegexPattern block = new JoniRegexPattern("\\p{Block=ASCII}", ignoreCase); + JoniRegexPattern script = new JoniRegexPattern("\\p{Script=Common}", ignoreCase); + + assertEquals("\\p{Block=ASCII}", block.patternDescription()); + assertEquals("\\p{Script=Common}", script.patternDescription()); + assertFalse(block.matcher("\u212A", java.util.List.of()).find()); + assertFalse(script.matcher("K", java.util.List.of()).find()); + } + + @Test + void passesRemainingExactEnumeratedPropertiesToJoni() { + String[][] cases = { + {"\\p{Canonical_Combining_Class=Above}", "\u0301"}, + {"\\p{Bidi_Class=Right_To_Left}", "\u05D0"}, + {"\\p{Decomposition_Type=Canonical}", "\u00C0"}, + {"\\p{East_Asian_Width=Fullwidth}", "\u3000"}, + {"\\p{Numeric_Value=1/2}", "\u00BD"}, + {"\\p{Joining_Group=Alef}", "\u0627"}, + }; + + for (String[] testCase : cases) { + JoniRegexPattern pattern = new JoniRegexPattern(testCase[0], FLAGS); + assertEquals(testCase[0], pattern.patternDescription()); + assertTrue(pattern.matcher(testCase[1], java.util.List.of()).find()); + } + } + + @Test + void passesExactAgePropertiesToJoniWithoutTextExpansion() { + String[][] cases = { + {"\\p{Age=2.1}", "\u20AC"}, + {"\\p{In=3.0}", "\u20AC"}, + {"\\p{Present_In=3.0}", "\u20AC"}, + {"\\p{Is_Age=6.1}", "\uD83D\uDE00"}, + {"\\p{Age=Unassigned}", "\uD88D\uDC7A"}, + }; + + for (String[] testCase : cases) { + JoniRegexPattern pattern = new JoniRegexPattern(testCase[0], FLAGS); + assertEquals(testCase[0], pattern.patternDescription()); + assertTrue(pattern.matcher(testCase[1], java.util.List.of()).find()); + } + + JoniRegexPattern wildcard = new JoniRegexPattern( + "\\p{Age=:\\AV16_0\\z:}", FLAGS); + assertFalse(wildcard.patternDescription().contains("Age=")); + } + @Test void flattensTranslatedPropertiesInsideOrdinaryCharacterClasses() { JoniRegexPattern pattern = new JoniRegexPattern( diff --git a/src/test/resources/unit/regex/unicode_binary_leading_shortcuts.t b/src/test/resources/unit/regex/unicode_binary_leading_shortcuts.t new file mode 100644 index 000000000..15d68978a --- /dev/null +++ b/src/test/resources/unit/regex/unicode_binary_leading_shortcuts.t @@ -0,0 +1,78 @@ +use strict; +use warnings; +use Test::More; + +sub compile_property { + my ($property) = @_; + my $pattern = eval 'qr/\A\p{' . $property . '}\z/u'; + return ($pattern, $@); +} + +my ($canonical, $canonical_error) = compile_property('IsUppercase'); +ok(defined $canonical, 'canonical Is binary shortcut compiles') + or diag($canonical_error); +like('A', $canonical, 'canonical Is binary shortcut matches a member'); +unlike('a', $canonical, 'canonical Is binary shortcut excludes a nonmember'); + +my ($lowercase, $lowercase_error) = compile_property('isuppercase'); +ok(defined $lowercase, 'binary shortcut accepts a lowercase Is prefix') + or diag($lowercase_error); +like('A', $lowercase, 'lowercase Is binary shortcut matches a member'); + +my ($leading, $leading_error) = compile_property('__Is_uppercase'); +ok(defined $leading, 'binary shortcut accepts leading loose separators') + or diag($leading_error); +like('A', $leading, 'leading-loose binary shortcut matches a member'); +unlike('a', $leading, 'leading-loose binary shortcut excludes a nonmember'); + +my ($hex, $hex_error) = compile_property('_ is_ASCII_HEX_DIGIT'); +ok(defined $hex, 'short binary alias accepts fully loose Is spelling') + or diag($hex_error); +like('F', $hex, 'loose binary alias matches its member'); +unlike('G', $hex, 'loose binary alias excludes a nonmember'); + +my ($assigned, $assigned_error) = compile_property('- Is_Assigned'); +ok(defined $assigned, 'another binary family accepts a leading-loose Is') + or diag($assigned_error); +like('A', $assigned, 'Assigned shortcut matches an assigned character'); +unlike(chr(0x0378), $assigned, + 'Assigned shortcut excludes an unassigned character'); + +my ($script, $script_error) = compile_property('_ is_Latn'); +ok(defined $script, 'Script shortcut retains precedence') or diag($script_error); +like('A', $script, 'Script shortcut membership remains intact'); +unlike(chr(0x03B1), $script, 'Script shortcut still excludes another script'); + +my ($category, $category_error) = compile_property('IsL'); +ok(defined $category, 'General_Category retains precedence') + or diag($category_error); +like(chr(0x03B1), $category, + 'General_Category shortcut still matches non-Latin letters'); + +my ($block, $block_error) = compile_property('_ Is_Basic_Latin'); +ok(defined $block, 'Block shortcut retains precedence') or diag($block_error); +like('A', $block, 'Block shortcut membership remains intact'); + +my $callback_calls = 0; +sub IsBinaryShortcutCallback { + $callback_calls++; + return "0041"; +} +my ($callback, $callback_error) = compile_property('IsBinaryShortcutCallback'); +ok(defined $callback, 'exact user property callback retains precedence') + or diag($callback_error); +like('A', $callback, 'exact user property callback matches its range'); +is($callback_calls, 1, 'exact user property callback is called once'); + +my ($leading_callback, $leading_callback_error) = + compile_property('_ IsBinaryShortcutCallback'); +ok(!defined($leading_callback) && length($leading_callback_error), + 'leading separators do not broaden user property lookup'); +is($callback_calls, 1, 'rejected leading callback spelling is not invoked'); + +my ($unknown, $unknown_error) = + compile_property('_ Is_DefinitelyNotABinaryProperty'); +ok(!defined($unknown) && length($unknown_error), + 'unknown leading-loose binary shortcut is rejected'); + +done_testing; diff --git a/src/test/resources/unit/regex/unicode_block_colon_wildcards.t b/src/test/resources/unit/regex/unicode_block_colon_wildcards.t new file mode 100644 index 000000000..a8a2de8fd --- /dev/null +++ b/src/test/resources/unit/regex/unicode_block_colon_wildcards.t @@ -0,0 +1,47 @@ +use strict; +use warnings; +use Test::More; + +no warnings 'experimental::uniprop_wildcards'; + +sub compile_property { + my ($property) = @_; + my $pattern = eval 'qr/\A\p{' . $property . '}\z/u'; + return ($pattern, $@); +} + +my ($basic, $basic_error) = compile_property('Block=:\ABasic_Latin\z:'); +ok(defined $basic, 'colon-delimited Block wildcard compiles') + or diag($basic_error); +like('A', $basic, 'colon-delimited Block wildcard matches its block'); +unlike(chr(0x03B1), $basic, + 'colon-delimited Block wildcard excludes another block'); + +my ($short, $short_error) = compile_property('Blk=:\AASCII\z:'); +ok(defined $short, 'Blk accepts a colon-delimited wildcard alias') + or diag($short_error); +like('A', $short, 'wildcard value accepts an official Block alias'); + +my ($loose, $loose_error) = compile_property('Block=:\Abasiclatin\z:'); +ok(defined $loose, 'Block wildcard compares loose value spellings') + or diag($loose_error); +like('A', $loose, 'loose wildcard value selects Basic_Latin'); + +my ($union, $union_error) = + compile_property('Block=:\A(?:Basic_Latin|Greek_And_Coptic)\z:'); +ok(defined $union, 'Block wildcard accepts value alternation') + or diag($union_error); +like('A', $union, 'Block wildcard alternation includes Basic_Latin'); +like(chr(0x0378), $union, + 'Block wildcard alternation includes Greek_And_Coptic'); + +for my $rejected ( + ['Block=:\ANever_A_Block\z:', 'wildcard matching no Block value'], + ['Block=:\A.*\z:', 'star quantifier in Block wildcard'], + ['Is_Block=:\ABasic_Latin\z:', 'Is-prefixed Block wildcard'], +) { + my ($pattern, $error) = compile_property($rejected->[0]); + ok(!defined($pattern) && length($error), "$rejected->[1] is rejected"); +} + +done_testing; diff --git a/src/test/resources/unit/regex/unicode_block_leading_shortcuts.t b/src/test/resources/unit/regex/unicode_block_leading_shortcuts.t new file mode 100644 index 000000000..4d8e263e7 --- /dev/null +++ b/src/test/resources/unit/regex/unicode_block_leading_shortcuts.t @@ -0,0 +1,65 @@ +use strict; +use warnings; +use Test::More; + +sub compile_property { + my ($property) = @_; + my $pattern = eval 'qr/\A\p{' . $property . '}\z/u'; + return ($pattern, $@); +} + +my ($canonical, $canonical_error) = compile_property('In_Emoticons'); +ok(defined $canonical, 'canonical In Block shortcut compiles') + or diag($canonical_error); +like(chr(0x1F600), $canonical, 'canonical In Block shortcut matches its block'); +unlike('A', $canonical, 'canonical In Block shortcut excludes another block'); + +my ($leading_in, $leading_in_error) = + compile_property(' _ IN_Emoticons'); +ok(defined $leading_in, 'In Block shortcut accepts leading loose separators') + or diag($leading_in_error); +like(chr(0x1F600), $leading_in, + 'leading-loose In Block shortcut matches its block'); +unlike('A', $leading_in, + 'leading-loose In Block shortcut excludes another block'); + +my ($leading_is, $leading_is_error) = + compile_property('- Is_Emoticons'); +ok(defined $leading_is, 'Is Block shortcut accepts leading loose separators') + or diag($leading_is_error); +like(chr(0x1F600), $leading_is, + 'leading-loose Is Block shortcut matches its block'); +unlike('A', $leading_is, + 'leading-loose Is Block shortcut excludes another block'); + +my ($script, $script_error) = compile_property('Is_Latin'); +ok(defined $script, 'Script shortcut retains precedence') or diag($script_error); +like('A', $script, 'Script shortcut membership remains intact'); + +my ($binary, $binary_error) = compile_property('Is_Uppercase'); +ok(defined $binary, 'binary shortcut retains precedence') or diag($binary_error); +like('A', $binary, 'binary shortcut membership remains intact'); + +my $callback_calls = 0; +sub Is_BlockShortcutCallback { + $callback_calls++; + return "0041"; +} +my ($callback, $callback_error) = + compile_property('Is_BlockShortcutCallback'); +ok(defined $callback, 'exact user property callback retains precedence') + or diag($callback_error); +like('A', $callback, 'exact user property callback matches its range'); +is($callback_calls, 1, 'exact user property callback is called once'); + +my ($leading_callback, $leading_callback_error) = + compile_property('_ Is_BlockShortcutCallback'); +ok(!defined($leading_callback) && length($leading_callback_error), + 'leading separators do not broaden user property lookup'); +is($callback_calls, 1, 'rejected leading callback spelling is not invoked'); + +my ($unknown, $unknown_error) = compile_property('_ In_DefinitelyNotABlock'); +ok(!defined($unknown) && length($unknown_error), + 'unknown leading-loose Block shortcut is rejected'); + +done_testing; diff --git a/src/test/resources/unit/regex/unicode_posix_alias_normalization.t b/src/test/resources/unit/regex/unicode_posix_alias_normalization.t new file mode 100644 index 000000000..ed5d42a90 --- /dev/null +++ b/src/test/resources/unit/regex/unicode_posix_alias_normalization.t @@ -0,0 +1,67 @@ +use strict; +use warnings; +use Test::More; + +sub compile_property { + my ($property) = @_; + my $pattern = eval 'qr/\A\p{' . $property . '}\z/u'; + return ($pattern, $@); +} + +my ($space, $space_error) = compile_property('XPosixSpace'); +ok(defined $space, 'canonical XPosix property compiles') or diag($space_error); +like(chr(0x00A0), $space, 'XPosixSpace includes non-ASCII Unicode space'); +unlike('A', $space, 'XPosixSpace excludes a letter'); + +my ($ascii_alpha, $ascii_alpha_error) = compile_property('PosixAlpha'); +ok(defined $ascii_alpha, 'canonical ASCII Posix property compiles') + or diag($ascii_alpha_error); +like('A', $ascii_alpha, 'PosixAlpha includes an ASCII letter'); +unlike(chr(0x03B1), $ascii_alpha, + 'PosixAlpha remains ASCII-only'); + +my ($loose, $loose_error) = compile_property('x-p_o s_i x alpha'); +ok(defined $loose, 'XPosix property accepts loose internal separators') + or diag($loose_error); +like(chr(0x03B1), $loose, 'loose XPosixAlpha keeps Unicode membership'); + +my ($leading, $leading_error) = compile_property(' _XPosixWord'); +ok(defined $leading, 'XPosix property accepts leading loose separators') + or diag($leading_error); +like('_', $leading, 'leading-separator XPosixWord matches underscore'); + +my ($is_prefixed, $is_prefixed_error) = compile_property('isxposixxdigit'); +ok(defined $is_prefixed, 'XPosix property accepts a loose Is prefix') + or diag($is_prefixed_error); +like('F', $is_prefixed, 'Is-prefixed XPosixXDigit matches a hex digit'); +unlike('G', $is_prefixed, 'Is-prefixed XPosixXDigit excludes a non-hex digit'); + +my ($unknown, $unknown_error) = compile_property('xposixdefinitelynot'); +ok(!defined($unknown) && length($unknown_error), + 'unknown loose XPosix property is rejected'); + +my ($unknown_is, $unknown_is_error) = + compile_property('Is_XPosixDefinitelyNot'); +eval { 'A' =~ $unknown_is } if defined($unknown_is) && !$unknown_is_error; +$unknown_is_error ||= $@; +ok(length($unknown_is_error), 'unknown Is-prefixed XPosix property is rejected'); + +my $callback_calls = 0; +my $callback_defined = eval q{ + sub Is_XPosixDefinitelyNot { + $callback_calls++; + return "0041"; + } + 1; +}; +ok($callback_defined, 'POSIX-shaped user property callback can be defined'); + +my ($callback, $callback_error) = + compile_property('Is_XPosixDefinitelyNot'); +ok(defined $callback, 'defined POSIX-shaped user property callback wins') + or diag($callback_error); +like('A', $callback, 'defined POSIX-shaped user property matches its range'); +unlike('B', $callback, 'defined POSIX-shaped user property excludes other ranges'); +is($callback_calls, 1, 'defined POSIX-shaped user property is called once'); + +done_testing; diff --git a/src/test/resources/unit/regex/unicode_script_leading_shortcuts.t b/src/test/resources/unit/regex/unicode_script_leading_shortcuts.t new file mode 100644 index 000000000..855f565df --- /dev/null +++ b/src/test/resources/unit/regex/unicode_script_leading_shortcuts.t @@ -0,0 +1,72 @@ +use strict; +use warnings; +use Test::More; + +sub compile_property { + my ($property) = @_; + my $pattern = eval 'qr/\A\p{' . $property . '}\z/u'; + return ($pattern, $@); +} + +my ($canonical, $canonical_error) = compile_property('Is_Latin'); +ok(defined $canonical, 'canonical Is Script shortcut compiles') + or diag($canonical_error); +like('A', $canonical, 'canonical Is Script shortcut matches its script'); +unlike(chr(0x03B1), $canonical, + 'canonical Is Script shortcut excludes another script'); + +my ($lowercase, $lowercase_error) = compile_property('islatin'); +ok(defined $lowercase, 'Script shortcut accepts a lowercase Is prefix') + or diag($lowercase_error); +like('A', $lowercase, 'lowercase Is Script shortcut matches its script'); + +my ($leading, $leading_error) = compile_property(' -IS_LATN'); +ok(defined $leading, 'Script shortcut accepts leading loose separators') + or diag($leading_error); +like('A', $leading, 'leading-loose Script alias matches its script'); +unlike(chr(0x03B1), $leading, + 'leading-loose Script alias excludes another script'); + +my ($extensions, $extensions_error) = compile_property('_ is_hira'); +ok(defined $extensions, 'short Script alias accepts loose Is spelling') + or diag($extensions_error); +like(chr(0x30FC), $extensions, + 'bare Script shortcut retains Script_Extensions membership'); + +my ($binary, $binary_error) = compile_property('IsUppercase'); +ok(defined $binary, 'binary property retains precedence') or diag($binary_error); +like('A', $binary, 'binary property membership remains intact'); + +my ($category, $category_error) = compile_property('IsL'); +ok(defined $category, 'General_Category retains precedence') + or diag($category_error); +like(chr(0x03B1), $category, + 'General_Category shortcut still matches non-Latin letters'); + +my $callback_calls = 0; +sub IsScriptShortcutCallback { + $callback_calls++; + return "0041"; +} +my ($callback, $callback_error) = + compile_property('IsScriptShortcutCallback'); +ok(defined $callback, 'exact user property callback retains precedence') + or diag($callback_error); +like('A', $callback, 'exact user property callback matches its range'); +is($callback_calls, 1, 'exact user property callback is called once'); + +my ($leading_callback, $leading_callback_error) = + compile_property('_ IsScriptShortcutCallback'); +ok(!defined($leading_callback) && length($leading_callback_error), + 'leading separators do not broaden user property lookup'); +is($callback_calls, 1, 'rejected leading callback spelling is not invoked'); + +my ($hrkt, $hrkt_error) = compile_property('_ Is_Hrkt'); +ok(!defined($hrkt) && length($hrkt_error), + 'Katakana_Or_Hiragana Script shortcut remains rejected'); + +my ($unknown, $unknown_error) = compile_property('_ Is_DefinitelyNotAScript'); +ok(!defined($unknown) && length($unknown_error), + 'unknown leading-loose Script shortcut is rejected'); + +done_testing; diff --git a/src/test/resources/unit/regex_joni_unicode_property_family_policy.t b/src/test/resources/unit/regex_joni_unicode_property_family_policy.t new file mode 100644 index 000000000..b93523706 --- /dev/null +++ b/src/test/resources/unit/regex_joni_unicode_property_family_policy.t @@ -0,0 +1,29 @@ +use strict; +use warnings; +use utf8; +use Test::More tests => 12; + +ok("\x{0301}" =~ /\p{Canonical_Combining_Class=Above}/, + 'canonical combining class assignment'); +ok("\x{0300}" !~ /\p{Canonical_Combining_Class=Below}/, + 'canonical combining class exclusion'); +ok("\x{05D0}" =~ /\p{Bidi_Class=Right_To_Left}/, + 'bidi class assignment'); +ok('A' !~ /\p{Bidi_Class=Right_To_Left}/, + 'bidi class exclusion'); +ok("\x{00C0}" =~ /\p{Decomposition_Type=Canonical}/, + 'decomposition type assignment'); +ok('k' !~ /\p{Decomposition_Type=Canonical}/i, + 'decomposition type does not gain fold members'); +ok("\x{3000}" =~ /\p{East_Asian_Width=Fullwidth}/, + 'east Asian width assignment'); +ok('k' !~ /\p{East_Asian_Width=Ambiguous}/i, + 'east Asian width does not gain fold members'); +ok("\x{00BD}" =~ m{\p{Numeric_Value=1/2}}, + 'numeric value rational assignment'); +ok('A' !~ m{\p{Numeric_Value=1/2}}i, + 'numeric value does not gain fold members'); +ok("\x{0627}" =~ /\p{Joining_Group=Alef}/, + 'joining group assignment'); +ok('A' !~ /\p{Joining_Group=Alef}/i, + 'joining group does not gain fold members'); diff --git a/src/test/resources/unit/regex_joni_unicode_property_fold_policy.t b/src/test/resources/unit/regex_joni_unicode_property_fold_policy.t new file mode 100644 index 000000000..cee1f2432 --- /dev/null +++ b/src/test/resources/unit/regex_joni_unicode_property_fold_policy.t @@ -0,0 +1,18 @@ +use strict; +use warnings; +use utf8; +use Test::More tests => 8; + +ok("A" =~ /\p{gc=Uppercase_Letter}/, 'general category positive'); +ok("a" !~ /\p{gc=Uppercase_Letter}/, 'general category negative'); +ok("é" =~ /\p{Script=Latin}/, 'script property in pinned Perl data'); +ok("a" =~ /\p{gc=Uppercase_Letter}/i, + 'general category participates in case folding'); +ok("A" !~ /\P{gc=Uppercase_Letter}/i, + 'negated category complements after case folding'); +ok("\x{212A}" !~ /\p{Block=ASCII}/i, + 'block membership does not gain case-fold members'); +ok("K" !~ /\p{Script=Common}/i, + 'script membership does not gain case-fold members'); +ok("k" =~ /\p{gc=Uppercase_Letter}/i, + 'general-category membership gains case-fold members'); diff --git a/third_party/joni/src/org/joni/ApplyCaseFold.java b/third_party/joni/src/org/joni/ApplyCaseFold.java index c3f01e543..59021b4c8 100644 --- a/third_party/joni/src/org/joni/ApplyCaseFold.java +++ b/third_party/joni/src/org/joni/ApplyCaseFold.java @@ -36,6 +36,7 @@ public void apply(int from, int[]to, int length, Object o) { Encoding enc = env.enc; CClassNode cc = arg.cc; CClassNode ascCc = arg.ascCc; + CClassNode foldCc = arg.foldCc; BitSet bs = cc.bs; boolean addFlag; @@ -44,13 +45,12 @@ public void apply(int from, int[]to, int length, Object o) { return; } - if (ascCc == null) { + if (!isEligible(foldCc, enc, from) || ascCc == null) { addFlag = false; } else if (Encoding.isAscii(from) == Encoding.isAscii(to[0])) { addFlag = true; } else { - addFlag = ascCc.isCodeInCC(enc, from); - if (ascCc.isNot()) addFlag = !addFlag; + addFlag = isEligible(ascCc, enc, from); } if (length == 1) { @@ -84,7 +84,8 @@ public void apply(int from, int[]to, int length, Object o) { } // CASE_FOLD_IS_APPLIED_INSIDE_NEGATIVE_CCLASS } else { - if (cc.isCodeInCC(enc, from) && (!Config.CASE_FOLD_IS_APPLIED_INSIDE_NEGATIVE_CCLASS || !cc.isNot())) { + if (addFlag && cc.isCodeInCC(enc, from) + && (!Config.CASE_FOLD_IS_APPLIED_INSIDE_NEGATIVE_CCLASS || !cc.isNot())) { StringNode node = null; for (int i=0; i ascNode) { + private CClassNode parseCharClass(ObjPtr ascNode, + ObjPtr foldNode) { final boolean neg; - CClassNode cc, prevCc = null, ascCc = null, ascPrevCc = null, workCc = null, ascWorkCc = null; + CClassNode cc, prevCc = null, ascCc = null, ascPrevCc = null, + workCc = null, ascWorkCc = null, foldCc = null, + foldPrevCc = null, foldWorkCc = null; CCStateArg arg = new CCStateArg(); fetchTokenInCC(); @@ -167,6 +174,7 @@ private CClassNode parseCharClass(ObjPtr ascNode) { cc = new CClassNode(); if (isIgnoreCase(env.option)) { ascCc = ascNode.p = new CClassNode(); + foldCc = foldNode.p = new CClassNode(); } boolean andStart = false; @@ -184,7 +192,7 @@ private CClassNode parseCharClass(ObjPtr ascNode) { } arg.to = token.getC(); arg.toIsRaw = false; - parseCharClassValEntry2(cc, ascCc, arg); // goto val_entry2 + parseCharClassValEntry2(cc, ascCc, foldCc, arg); // goto val_entry2 break; case RAW_BYTE: @@ -224,25 +232,25 @@ private CClassNode parseCharClass(ObjPtr ascNode) { arg.inType = CCVALTYPE.SB; // raw_single: } arg.toIsRaw = true; - parseCharClassValEntry2(cc, ascCc, arg); // goto val_entry2 + parseCharClassValEntry2(cc, ascCc, foldCc, arg); // goto val_entry2 break; case CODE_POINT: arg.to = token.getCode(); arg.toIsRaw = true; - parseCharClassValEntry(cc, ascCc, arg); // val_entry:, val_entry2 + parseCharClassValEntry(cc, ascCc, foldCc, arg); // val_entry:, val_entry2 break; case POSIX_BRACKET_OPEN: - if (parsePosixBracket(cc, ascCc)) { /* true: is not POSIX bracket */ + if (parsePosixBracket(cc, ascCc, foldCc)) { /* true: is not POSIX bracket */ env.ccEscWarn("["); p = token.backP; arg.to = token.getC(); arg.toIsRaw = false; - parseCharClassValEntry(cc, ascCc, arg); // goto val_entry + parseCharClassValEntry(cc, ascCc, foldCc, arg); // goto val_entry break; } - cc.nextStateClass(arg, ascCc, env); // goto next_class + cc.nextStateClass(arg, ascCc, foldCc, env); // goto next_class break; case CHAR_TYPE: @@ -252,13 +260,17 @@ private CClassNode parseCharClass(ObjPtr ascNode) { ascCc.addCType(token.getPropCType(), token.getPropNot(), isAsciiRange(env.option), env, this); } } - cc.nextStateClass(arg, ascCc, env); // next_class: + if (foldCc != null) { + foldCc.addCType(token.getPropCType(), token.getPropNot(), + isAsciiRange(env.option), env, this); + } + cc.nextStateClass(arg, ascCc, foldCc, env); // next_class: break; case CHAR_PROPERTY: - CharProperty property = fetchCharProperty(); - addCharProperty(cc, ascCc, property, token.getPropNot()); - cc.nextStateClass(arg, ascCc, env); // goto next_class + CharProperty property = fetchCharProperty(true); + addCharProperty(cc, ascCc, foldCc, property, token.getPropNot()); + cc.nextStateClass(arg, ascCc, foldCc, env); // goto next_class break; case CC_RANGE: @@ -266,11 +278,11 @@ private CClassNode parseCharClass(ObjPtr ascNode) { fetchTokenInCC(); fetched = true; if (token.type == TokenType.CC_CLOSE) { /* allow [x-] */ - parseCharClassRangeEndVal(cc, ascCc, arg); // range_end_val:, goto val_entry; + parseCharClassRangeEndVal(cc, ascCc, foldCc, arg); // range_end_val:, goto val_entry; break; } else if (token.type == TokenType.CC_AND) { env.ccEscWarn("-"); - parseCharClassRangeEndVal(cc, ascCc, arg); // goto range_end_val + parseCharClassRangeEndVal(cc, ascCc, foldCc, arg); // goto range_end_val break; } if (arg.type == CCVALTYPE.CLASS) newValueException(UNMATCHED_RANGE_SPECIFIER_IN_CHAR_CLASS); @@ -281,28 +293,28 @@ private CClassNode parseCharClass(ObjPtr ascNode) { fetchTokenInCC(); fetched = true; if (token.type == TokenType.CC_RANGE || andStart) env.ccEscWarn("-"); /* [--x] or [a&&-x] is warned. */ - parseCharClassValEntry(cc, ascCc, arg); // goto val_entry + parseCharClassValEntry(cc, ascCc, foldCc, arg); // goto val_entry break; } else if (arg.state == CCSTATE.RANGE) { env.ccEscWarn("-"); - parseCharClassSbChar(cc, ascCc, arg); // goto sb_char /* [!--x] is allowed */ + parseCharClassSbChar(cc, ascCc, foldCc, arg); // goto sb_char /* [!--x] is allowed */ break; } else { /* CCS_COMPLETE */ fetchTokenInCC(); fetched = true; if (token.type == TokenType.CC_CLOSE) { /* allow [a-b-] */ - parseCharClassRangeEndVal(cc, ascCc, arg); // goto range_end_val + parseCharClassRangeEndVal(cc, ascCc, foldCc, arg); // goto range_end_val break; } else if (token.type == TokenType.CC_AND) { env.ccEscWarn("-"); - parseCharClassRangeEndVal(cc, ascCc, arg); // goto range_end_val + parseCharClassRangeEndVal(cc, ascCc, foldCc, arg); // goto range_end_val break; } if (syntax.allowDoubleRangeOpInCC()) { env.ccEscWarn("-"); // parseCharClassSbChar(cc, ascCc, arg); // goto sb_char /* [0-9-a] is allowed as [0-9\-a] */ - parseCharClassRangeEndVal(cc, ascCc, arg); // goto range_end_val + parseCharClassRangeEndVal(cc, ascCc, foldCc, arg); // goto range_end_val break; } newSyntaxException(UNMATCHED_RANGE_SPECIFIER_IN_CHAR_CLASS); @@ -311,18 +323,22 @@ private CClassNode parseCharClass(ObjPtr ascNode) { case CC_CC_OPEN: /* [ */ ObjPtr ascPtr = new ObjPtr<>(); - CClassNode acc = parseCharClass(ascPtr); + ObjPtr foldPtr = new ObjPtr<>(); + CClassNode acc = parseCharClass(ascPtr, foldPtr); cc.or(acc, env); if (ascPtr.p != null) { ascCc.or(ascPtr.p, env); } + if (foldPtr.p != null) { + foldCc.or(foldPtr.p, env); + } break; case CC_AND: /* && */ if (arg.state == CCSTATE.VALUE) { arg.to = 0; arg.toIsRaw = false; - cc.nextStateValue(arg, ascCc, env); + cc.nextStateValue(arg, ascCc, foldCc, env); } /* initialize local variables */ andStart = true; @@ -332,6 +348,9 @@ private CClassNode parseCharClass(ObjPtr ascNode) { if (ascCc != null) { ascPrevCc.and(ascCc, env); } + if (foldCc != null) { + foldPrevCc.and(foldCc, env); + } } else { prevCc = cc; if (workCc == null) workCc = new CClassNode(); @@ -341,9 +360,15 @@ private CClassNode parseCharClass(ObjPtr ascNode) { if (ascWorkCc == null) ascWorkCc = new CClassNode(); ascCc = ascWorkCc; } + if (foldCc != null) { + foldPrevCc = foldCc; + if (foldWorkCc == null) foldWorkCc = new CClassNode(); + foldCc = foldWorkCc; + } } cc.clear(); if (ascCc != null) ascCc.clear(); + if (foldCc != null) foldCc.clear(); break; case EOT: @@ -360,7 +385,7 @@ private CClassNode parseCharClass(ObjPtr ascNode) { if (arg.state == CCSTATE.VALUE) { arg.to = 0; arg.toIsRaw = false; - cc.nextStateValue(arg, ascCc, env); + cc.nextStateValue(arg, ascCc, foldCc, env); } if (prevCc != null) { @@ -370,14 +395,20 @@ private CClassNode parseCharClass(ObjPtr ascNode) { ascPrevCc.and(ascCc, env); ascCc = ascPrevCc; } + if (foldCc != null) { + foldPrevCc.and(foldCc, env); + foldCc = foldPrevCc; + } } if (neg) { cc.setNot(); if (ascCc != null) ascCc.setNot(); + if (foldCc != null) foldCc.setNot(); } else { cc.clearNot(); if (ascCc != null) ascCc.clearNot(); + if (foldCc != null) foldCc.clearNot(); } if (cc.isNot() && syntax.notNewlineInNegativeCC()) { @@ -396,27 +427,31 @@ private CClassNode parseCharClass(ObjPtr ascNode) { return cc; } - private void parseCharClassSbChar(CClassNode cc, CClassNode ascCc, CCStateArg arg) { + private void parseCharClassSbChar(CClassNode cc, CClassNode ascCc, + CClassNode foldCc, CCStateArg arg) { arg.inType = CCVALTYPE.SB; arg.to = token.getC(); arg.toIsRaw = false; - parseCharClassValEntry2(cc, ascCc, arg); // goto val_entry2 + parseCharClassValEntry2(cc, ascCc, foldCc, arg); // goto val_entry2 } - private void parseCharClassRangeEndVal(CClassNode cc, CClassNode ascCc, CCStateArg arg) { + private void parseCharClassRangeEndVal(CClassNode cc, CClassNode ascCc, + CClassNode foldCc, CCStateArg arg) { arg.to = '-'; arg.toIsRaw = false; - parseCharClassValEntry(cc, ascCc, arg); // goto val_entry + parseCharClassValEntry(cc, ascCc, foldCc, arg); // goto val_entry } - private void parseCharClassValEntry(CClassNode cc, CClassNode ascCc, CCStateArg arg) { + private void parseCharClassValEntry(CClassNode cc, CClassNode ascCc, + CClassNode foldCc, CCStateArg arg) { int len = enc.codeToMbcLength(arg.to); arg.inType = len == 1 ? CCVALTYPE.SB : CCVALTYPE.CODE_POINT; - parseCharClassValEntry2(cc, ascCc, arg); // val_entry2: + parseCharClassValEntry2(cc, ascCc, foldCc, arg); // val_entry2: } - private void parseCharClassValEntry2(CClassNode cc, CClassNode ascCc, CCStateArg arg) { - cc.nextStateValue(arg, ascCc, env); + private void parseCharClassValEntry2(CClassNode cc, CClassNode ascCc, + CClassNode foldCc, CCStateArg arg) { + cc.nextStateValue(arg, ascCc, foldCc, env); } private Node parseEnclose(TokenType term) { @@ -1174,12 +1209,18 @@ private Node parseExp(TokenType term) { case CC_OPEN: { ObjPtr ascPtr = new ObjPtr<>(); - CClassNode cc = parseCharClass(ascPtr); + ObjPtr foldPtr = new ObjPtr<>(); + CClassNode cc = parseCharClass(ascPtr, foldPtr); int code = cc.isOneChar(); - if (code != -1) return parseStringLoop(StringNode.fromCodePoint(code, enc), group); + if (code != -1 && (!isIgnoreCase(env.option) + || ApplyCaseFold.isEligible(foldPtr.p, enc, code))) { + return parseStringLoop(StringNode.fromCodePoint(code, enc), group); + } node = cc; - if (isIgnoreCase(env.option)) node = cClassCaseFold(node, cc, ascPtr.p); + if (isIgnoreCase(env.option)) { + node = cClassCaseFold(node, cc, ascPtr.p, foldPtr.p); + } break; } @@ -1619,8 +1660,9 @@ private Node parseCharType(Node node) { return node; } - private Node cClassCaseFold(Node node, CClassNode cc, CClassNode ascCc) { - ApplyCaseFoldArg arg = new ApplyCaseFoldArg(env, cc, ascCc); + private Node cClassCaseFold(Node node, CClassNode cc, CClassNode ascCc, + CClassNode foldCc) { + ApplyCaseFoldArg arg = new ApplyCaseFoldArg(env, cc, ascCc, foldCc); enc.applyAllCaseFold(env.caseFoldFlagFor(env.option), ApplyCaseFold.INSTANCE, arg); if (arg.altRoot != null) { node = ListNode.newAlt(node, arg.altRoot); @@ -1629,31 +1671,40 @@ private Node cClassCaseFold(Node node, CClassNode cc, CClassNode ascCc) { } private Node parseCharProperty() { - CharProperty property = fetchCharProperty(); + CharProperty property = fetchCharProperty(false); CClassNode cc = new CClassNode(); Node node = cc; - addCharProperty(cc, null, property, false); + addCharProperty(cc, null, null, property, false); if (token.getPropNot()) cc.setNot(); - if (isIgnoreCase(env.option)) { + if (isIgnoreCase(env.option) && property.caseFold) { if (property.ranges != null || property.ctype != CharacterType.ASCII) { - node = cClassCaseFold(node, cc, cc); + node = cClassCaseFold(node, cc, cc, cc); } } return node; } private void addCharProperty(CClassNode cc, CClassNode ascCc, - CharProperty property, boolean not) { + CClassNode foldCc, CharProperty property, + boolean not) { if (property.ranges == null) { cc.addCType(property.ctype, not, false, env, this); if (ascCc != null && property.ctype != CharacterType.ASCII) { ascCc.addCType(property.ctype, not, false, env, this); } + if (foldCc != null) { + foldCc.addCType(property.ctype, not, false, env, this); + } return; } cc.addCodeRanges(property.ranges, not, env); - if (ascCc != null) ascCc.addCodeRanges(property.ranges, not, env); + if (ascCc != null) { + ascCc.addCodeRanges(property.ranges, not, env); + } + if (foldCc != null && property.caseFold) { + foldCc.addCodeRanges(property.ranges, not, env); + } } private Node parseAnycharAnytime() { diff --git a/third_party/joni/src/org/joni/ast/CClassNode.java b/third_party/joni/src/org/joni/ast/CClassNode.java index 150227116..463676140 100644 --- a/third_party/joni/src/org/joni/ast/CClassNode.java +++ b/third_party/joni/src/org/joni/ast/CClassNode.java @@ -415,31 +415,37 @@ public static final class CCStateArg { public CCSTATE state; } - public void nextStateClass(CCStateArg arg, CClassNode ascCC, ScanEnvironment env) { + public void nextStateClass(CCStateArg arg, CClassNode ascCc, + CClassNode foldCc, ScanEnvironment env) { if (arg.state == CCSTATE.RANGE) throw new SyntaxException(ErrorMessages.CHAR_CLASS_VALUE_AT_END_OF_RANGE); if (arg.state == CCSTATE.VALUE && arg.type != CCVALTYPE.CLASS) { if (arg.type == CCVALTYPE.SB) { bs.set(env, arg.from); - if (ascCC != null) ascCC.bs.set(arg.from); + if (ascCc != null) ascCc.bs.set(arg.from); + if (foldCc != null) foldCc.bs.set(arg.from); } else if (arg.type == CCVALTYPE.CODE_POINT) { addCodeRange(env, arg.from, arg.from); - if (ascCC != null) ascCC.addCodeRange(env, arg.from, arg.from, false); + if (ascCc != null) ascCc.addCodeRange(env, arg.from, arg.from, false); + if (foldCc != null) foldCc.addCodeRange(env, arg.from, arg.from, false); } } arg.state = CCSTATE.VALUE; arg.type = CCVALTYPE.CLASS; } - public void nextStateValue(CCStateArg arg, CClassNode ascCc, ScanEnvironment env) { + public void nextStateValue(CCStateArg arg, CClassNode ascCc, + CClassNode foldCc, ScanEnvironment env) { switch(arg.state) { case VALUE: if (arg.type == CCVALTYPE.SB) { bs.set(env, arg.from); if (ascCc != null) ascCc.bs.set(arg.from); + if (foldCc != null) foldCc.bs.set(arg.from); } else if (arg.type == CCVALTYPE.CODE_POINT) { addCodeRange(env, arg.from, arg.from); if (ascCc != null) ascCc.addCodeRange(env, arg.from, arg.from, false); + if (foldCc != null) foldCc.addCodeRange(env, arg.from, arg.from, false); } break; @@ -459,9 +465,11 @@ public void nextStateValue(CCStateArg arg, CClassNode ascCc, ScanEnvironment env } bs.setRange(env, arg.from, arg.to); if (ascCc != null) ascCc.bs.setRange(null, arg.from, arg.to); + if (foldCc != null) foldCc.bs.setRange(null, arg.from, arg.to); } else { addCodeRange(env, arg.from, arg.to); if (ascCc != null) ascCc.addCodeRange(env, arg.from, arg.to, false); + if (foldCc != null) foldCc.addCodeRange(env, arg.from, arg.to, false); } } else { if (arg.from > arg.to) { @@ -479,6 +487,10 @@ public void nextStateValue(CCStateArg arg, CClassNode ascCc, ScanEnvironment env ascCc.bs.setRange(null, arg.from, arg.to < 0xff ? arg.to : 0xff); ascCc.addCodeRange(env, arg.from, arg.to, false); } + if (foldCc != null) { + foldCc.bs.setRange(null, arg.from, arg.to < 0xff ? arg.to : 0xff); + foldCc.addCodeRange(env, arg.from, arg.to, false); + } } // ccs_range_end: arg.state = CCSTATE.COMPLETE; diff --git a/third_party/joni/test/org/joni/test/TestCharacterPropertyResolver.java b/third_party/joni/test/org/joni/test/TestCharacterPropertyResolver.java index 974df8eea..112f8cdf1 100644 --- a/third_party/joni/test/org/joni/test/TestCharacterPropertyResolver.java +++ b/third_party/joni/test/org/joni/test/TestCharacterPropertyResolver.java @@ -19,6 +19,7 @@ */ package org.joni.test; +import static org.joni.constants.SyntaxProperties.OP2_CCLASS_SET_OP; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertSame; import static org.junit.Assert.fail; @@ -34,17 +35,21 @@ public class TestCharacterPropertyResolver { private static final CharacterPropertyResolver RESOLVER = - (bytes, p, end, encoding) -> { + (bytes, p, end, encoding, inCharacterClass) -> { String name = new String(bytes, p, end - p, StandardCharsets.UTF_8); return switch (name) { - case "Fake" -> new int[] {2, 'A', 'A', 0x1f642, 0x1f642}; + case "Fake" -> new CharacterPropertyResolver.Result( + new int[] {3, 'A', 'A', 0xdf, 0xdf, 0x1f642, 0x1f642}, true); + case "FakeNoFold" -> new CharacterPropertyResolver.Result( + new int[] {2, 'A', 'A', 0xdf, 0xdf}, false); default -> null; }; }; private static Syntax syntax(CharacterPropertyResolver resolver) { return new Syntax("CharacterPropertyResolver", Syntax.PerlNG.op, - Syntax.PerlNG.op2, Syntax.PerlNG.op3, Syntax.PerlNG.behavior, + Syntax.PerlNG.op2 | OP2_CCLASS_SET_OP, Syntax.PerlNG.op3, + Syntax.PerlNG.behavior, Syntax.PerlNG.options, Syntax.PerlNG.metaCharTable, null, resolver); } @@ -69,6 +74,38 @@ public void resolvesRangesInsideAndOutsideCharacterClasses() { assertEquals(-1, search("[\\P{Fake}]", "A")); assertEquals(0, search("(?i)\\p{Fake}", "a")); assertEquals(-1, search("(?i)\\P{Fake}", "A")); + assertEquals(0, search("(?i)\\p{FakeNoFold}", "A")); + assertEquals(-1, search("(?i)\\p{FakeNoFold}", "a")); + } + + @Test + public void preservesFoldPolicyInsidePositiveAndNegativeClasses() { + assertEquals(0, search("(?i)[\\p{Fake}]", "a")); + assertEquals(0, search("(?i)[\\p{FakeNoFold}]", "A")); + assertEquals(-1, search("(?i)[\\p{FakeNoFold}]", "a")); + assertEquals(-1, search("(?i)[\\P{FakeNoFold}]", "A")); + assertEquals(0, search("(?i)[\\P{FakeNoFold}]", "a")); + assertEquals(-1, search("(?i)[^\\p{FakeNoFold}]", "A")); + assertEquals(0, search("(?i)[^\\p{FakeNoFold}]", "a")); + } + + @Test + public void composesFoldPolicyThroughUnionsIntersectionsAndNestedClasses() { + assertEquals(0, search("(?i)[\\p{FakeNoFold}\\p{Fake}]", "a")); + assertEquals(0, search("(?i)[\\p{Fake}&&\\p{FakeNoFold}]", "A")); + assertEquals(-1, search("(?i)[\\p{Fake}&&\\p{FakeNoFold}]", "a")); + assertEquals(0, search("(?i)[[\\p{FakeNoFold}]B]", "A")); + assertEquals(-1, search("(?i)[[\\p{FakeNoFold}]B]", "a")); + assertEquals(0, search("(?i)[[\\p{FakeNoFold}]B]", "b")); + } + + @Test + public void foldsOnlyEligibleMembersOfMixedClasses() { + assertEquals(0, search("(?i)[B\\p{FakeNoFold}]", "A")); + assertEquals(-1, search("(?i)[B\\p{FakeNoFold}]", "a")); + assertEquals(0, search("(?i)[B\\p{FakeNoFold}]", "b")); + assertEquals(0, search("(?i)[\\p{Fake}]", "ss")); + assertEquals(-1, search("(?i)[\\p{FakeNoFold}]", "ss")); } @Test @@ -81,7 +118,9 @@ public void fallsBackToEncodingProperties() { public void preservesResolverExceptions() { IllegalArgumentException expected = new IllegalArgumentException("failure"); try { - compile("\\p{Fake}", (bytes, p, end, encoding) -> { throw expected; }); + compile("\\p{Fake}", (bytes, p, end, encoding, inCharacterClass) -> { + throw expected; + }); fail("expected resolver exception"); } catch (IllegalArgumentException error) { assertSame(expected, error); @@ -91,7 +130,8 @@ public void preservesResolverExceptions() { @Test public void rejectsMalformedRangeResults() { try { - compile("\\p{Fake}", (bytes, p, end, encoding) -> new int[] {1, 2}); + compile("\\p{Fake}", (bytes, p, end, encoding, inCharacterClass) -> + new CharacterPropertyResolver.Result(new int[] {1, 2}, true)); fail("expected invalid range result"); } catch (IllegalArgumentException error) { assertEquals("invalid character property ranges", error.getMessage());