From f65c8cf41c117426c10aaf8e53912d0099a11974 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Tue, 18 Aug 2026 22:55:37 +0200 Subject: [PATCH 1/5] fix(regex): accept leading-loose Block shortcuts Allow Perl's bare In and Is Block shortcuts to ignore leading whitespace, hyphens, and underscores before the prefix. Keep Script, binary, General_Category, and exact user-property precedence unchanged while resolving only against the pinned Perl Block table. Add a system-Perl-validated focused fixture for membership, precedence, callback behavior, and rejection paths. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../runtime/regex/UnicodeResolver.java | 43 ++++++++---- .../regex/unicode_block_leading_shortcuts.t | 65 +++++++++++++++++++ 2 files changed, 96 insertions(+), 12 deletions(-) create mode 100644 src/test/resources/unit/regex/unicode_block_leading_shortcuts.t diff --git a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java index b743f492a..bdb1f68d6 100644 --- a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java +++ b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java @@ -1266,31 +1266,50 @@ && loosePropertyName(alias.substring(0, assignment)) } } - String blockAlias = alias; + String blockShortcutAlias = alias; + if (assignment < 0) { + int prefixStart = 0; + while (prefixStart < alias.length()) { + char separator = alias.charAt(prefixStart); + if (!Character.isWhitespace(separator) + && separator != '-' && separator != '_') break; + prefixStart++; + } + if (prefixStart > 0 && alias.length() - prefixStart > 2 + && (alias.regionMatches(true, prefixStart, "in", 0, 2) + || alias.regionMatches(true, prefixStart, "is", 0, 2))) { + blockShortcutAlias = alias.substring(prefixStart); + } + } + + String blockAlias = blockShortcutAlias; boolean blockShortcut = false; boolean isBlockShortcut = false; - if (alias.length() > 2 && alias.regionMatches(true, 0, "in", 0, 2)) { + if (blockShortcutAlias.length() > 2 + && blockShortcutAlias.regionMatches(true, 0, "in", 0, 2)) { int valueStart = 2; - while (valueStart < alias.length()) { - char separator = alias.charAt(valueStart); + while (valueStart < blockShortcutAlias.length()) { + char separator = blockShortcutAlias.charAt(valueStart); if (!Character.isWhitespace(separator) && separator != '-' && separator != '_') break; valueStart++; } - if (valueStart >= alias.length()) return null; - blockAlias = alias.substring(valueStart); + if (valueStart >= blockShortcutAlias.length()) return null; + blockAlias = blockShortcutAlias.substring(valueStart); blockShortcut = true; - } else if (assignment >= 0 || unicodePropertyValue(UProperty.SCRIPT, alias) >= 0) { + } else if (assignment >= 0 + || unicodePropertyValue(UProperty.SCRIPT, blockShortcutAlias) >= 0) { return null; - } else if (alias.length() > 2 && alias.regionMatches(true, 0, "is", 0, 2)) { + } else if (blockShortcutAlias.length() > 2 + && blockShortcutAlias.regionMatches(true, 0, "is", 0, 2)) { int valueStart = 2; - while (valueStart < alias.length()) { - char separator = alias.charAt(valueStart); + while (valueStart < blockShortcutAlias.length()) { + char separator = blockShortcutAlias.charAt(valueStart); if (!Character.isWhitespace(separator) && separator != '-' && separator != '_') break; valueStart++; } - if (valueStart >= alias.length()) return null; - String candidate = alias.substring(valueStart); + if (valueStart >= blockShortcutAlias.length()) return null; + String candidate = blockShortcutAlias.substring(valueStart); if (unicodePropertyValue(UProperty.SCRIPT, candidate) >= 0) return null; if (isIcuBinaryPropertyAlias(candidate) || isIcuGeneralCategoryAlias(candidate)) return null; diff --git a/src/test/resources/unit/regex/unicode_block_leading_shortcuts.t b/src/test/resources/unit/regex/unicode_block_leading_shortcuts.t new file mode 100644 index 000000000..4d8e263e7 --- /dev/null +++ b/src/test/resources/unit/regex/unicode_block_leading_shortcuts.t @@ -0,0 +1,65 @@ +use strict; +use warnings; +use Test::More; + +sub compile_property { + my ($property) = @_; + my $pattern = eval 'qr/\A\p{' . $property . '}\z/u'; + return ($pattern, $@); +} + +my ($canonical, $canonical_error) = compile_property('In_Emoticons'); +ok(defined $canonical, 'canonical In Block shortcut compiles') + or diag($canonical_error); +like(chr(0x1F600), $canonical, 'canonical In Block shortcut matches its block'); +unlike('A', $canonical, 'canonical In Block shortcut excludes another block'); + +my ($leading_in, $leading_in_error) = + compile_property(' _ IN_Emoticons'); +ok(defined $leading_in, 'In Block shortcut accepts leading loose separators') + or diag($leading_in_error); +like(chr(0x1F600), $leading_in, + 'leading-loose In Block shortcut matches its block'); +unlike('A', $leading_in, + 'leading-loose In Block shortcut excludes another block'); + +my ($leading_is, $leading_is_error) = + compile_property('- Is_Emoticons'); +ok(defined $leading_is, 'Is Block shortcut accepts leading loose separators') + or diag($leading_is_error); +like(chr(0x1F600), $leading_is, + 'leading-loose Is Block shortcut matches its block'); +unlike('A', $leading_is, + 'leading-loose Is Block shortcut excludes another block'); + +my ($script, $script_error) = compile_property('Is_Latin'); +ok(defined $script, 'Script shortcut retains precedence') or diag($script_error); +like('A', $script, 'Script shortcut membership remains intact'); + +my ($binary, $binary_error) = compile_property('Is_Uppercase'); +ok(defined $binary, 'binary shortcut retains precedence') or diag($binary_error); +like('A', $binary, 'binary shortcut membership remains intact'); + +my $callback_calls = 0; +sub Is_BlockShortcutCallback { + $callback_calls++; + return "0041"; +} +my ($callback, $callback_error) = + compile_property('Is_BlockShortcutCallback'); +ok(defined $callback, 'exact user property callback retains precedence') + or diag($callback_error); +like('A', $callback, 'exact user property callback matches its range'); +is($callback_calls, 1, 'exact user property callback is called once'); + +my ($leading_callback, $leading_callback_error) = + compile_property('_ Is_BlockShortcutCallback'); +ok(!defined($leading_callback) && length($leading_callback_error), + 'leading separators do not broaden user property lookup'); +is($callback_calls, 1, 'rejected leading callback spelling is not invoked'); + +my ($unknown, $unknown_error) = compile_property('_ In_DefinitelyNotABlock'); +ok(!defined($unknown) && length($unknown_error), + 'unknown leading-loose Block shortcut is rejected'); + +done_testing; From 933e35c38b361c0f4d33eb2511a8ebad8319ca00 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Tue, 18 Aug 2026 23:16:33 +0200 Subject: [PATCH 2/5] fix(regex): accept leading-loose Script shortcuts Normalize leading loose separators and the Is prefix before resolving bare Script shortcuts, while preserving binary, general-category, and user-defined property precedence. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../runtime/regex/UnicodeResolver.java | 12 +++- .../regex/unicode_script_leading_shortcuts.t | 72 +++++++++++++++++++ 2 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 src/test/resources/unit/regex/unicode_script_leading_shortcuts.t diff --git a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java index bdb1f68d6..d55ccef13 100644 --- a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java +++ b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java @@ -1245,8 +1245,16 @@ && loosePropertyName(alias.substring(0, assignment)) // Perl's bare script-value shortcuts use Script_Extensions semantics. // Keep binary and General_Category names ahead of this value namespace. String scriptShortcut = alias; - if (alias.length() > 2 && alias.startsWith("Is")) { - int valueStart = 2; + int scriptPrefixStart = 0; + while (scriptPrefixStart < alias.length()) { + char separator = alias.charAt(scriptPrefixStart); + if (!Character.isWhitespace(separator) + && separator != '-' && separator != '_') break; + scriptPrefixStart++; + } + if (alias.length() - scriptPrefixStart > 2 + && alias.regionMatches(true, scriptPrefixStart, "is", 0, 2)) { + int valueStart = scriptPrefixStart + 2; while (valueStart < alias.length()) { char separator = alias.charAt(valueStart); if (!Character.isWhitespace(separator) diff --git a/src/test/resources/unit/regex/unicode_script_leading_shortcuts.t b/src/test/resources/unit/regex/unicode_script_leading_shortcuts.t new file mode 100644 index 000000000..855f565df --- /dev/null +++ b/src/test/resources/unit/regex/unicode_script_leading_shortcuts.t @@ -0,0 +1,72 @@ +use strict; +use warnings; +use Test::More; + +sub compile_property { + my ($property) = @_; + my $pattern = eval 'qr/\A\p{' . $property . '}\z/u'; + return ($pattern, $@); +} + +my ($canonical, $canonical_error) = compile_property('Is_Latin'); +ok(defined $canonical, 'canonical Is Script shortcut compiles') + or diag($canonical_error); +like('A', $canonical, 'canonical Is Script shortcut matches its script'); +unlike(chr(0x03B1), $canonical, + 'canonical Is Script shortcut excludes another script'); + +my ($lowercase, $lowercase_error) = compile_property('islatin'); +ok(defined $lowercase, 'Script shortcut accepts a lowercase Is prefix') + or diag($lowercase_error); +like('A', $lowercase, 'lowercase Is Script shortcut matches its script'); + +my ($leading, $leading_error) = compile_property(' -IS_LATN'); +ok(defined $leading, 'Script shortcut accepts leading loose separators') + or diag($leading_error); +like('A', $leading, 'leading-loose Script alias matches its script'); +unlike(chr(0x03B1), $leading, + 'leading-loose Script alias excludes another script'); + +my ($extensions, $extensions_error) = compile_property('_ is_hira'); +ok(defined $extensions, 'short Script alias accepts loose Is spelling') + or diag($extensions_error); +like(chr(0x30FC), $extensions, + 'bare Script shortcut retains Script_Extensions membership'); + +my ($binary, $binary_error) = compile_property('IsUppercase'); +ok(defined $binary, 'binary property retains precedence') or diag($binary_error); +like('A', $binary, 'binary property membership remains intact'); + +my ($category, $category_error) = compile_property('IsL'); +ok(defined $category, 'General_Category retains precedence') + or diag($category_error); +like(chr(0x03B1), $category, + 'General_Category shortcut still matches non-Latin letters'); + +my $callback_calls = 0; +sub IsScriptShortcutCallback { + $callback_calls++; + return "0041"; +} +my ($callback, $callback_error) = + compile_property('IsScriptShortcutCallback'); +ok(defined $callback, 'exact user property callback retains precedence') + or diag($callback_error); +like('A', $callback, 'exact user property callback matches its range'); +is($callback_calls, 1, 'exact user property callback is called once'); + +my ($leading_callback, $leading_callback_error) = + compile_property('_ IsScriptShortcutCallback'); +ok(!defined($leading_callback) && length($leading_callback_error), + 'leading separators do not broaden user property lookup'); +is($callback_calls, 1, 'rejected leading callback spelling is not invoked'); + +my ($hrkt, $hrkt_error) = compile_property('_ Is_Hrkt'); +ok(!defined($hrkt) && length($hrkt_error), + 'Katakana_Or_Hiragana Script shortcut remains rejected'); + +my ($unknown, $unknown_error) = compile_property('_ Is_DefinitelyNotAScript'); +ok(!defined($unknown) && length($unknown_error), + 'unknown leading-loose Script shortcut is rejected'); + +done_testing; From b826b3863952032eadf770f4d95c8d3d84871a46 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 00:11:08 +0200 Subject: [PATCH 3/5] fix(regex): resolve loose Is binary shortcuts Route leading-loose bare Is aliases through the native Joni property callback, preserving Script, Block, General_Category, and user-property precedence while inheriting only already-supported binary and special property bases. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../runtime/regex/UnicodeResolver.java | 60 +++++++++++++- .../regex/unicode_binary_leading_shortcuts.t | 78 +++++++++++++++++++ 2 files changed, 136 insertions(+), 2 deletions(-) create mode 100644 src/test/resources/unit/regex/unicode_binary_leading_shortcuts.t diff --git a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java index d55ccef13..991208d79 100644 --- a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java +++ b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java @@ -995,7 +995,14 @@ static CharacterPropertyResolver.Result resolveJoniProperty( property = normalizePerlIsPropertyAssignment(property); int assignment = propertyValueDelimiter(property); if (assignment <= 0 || assignment == property.length() - 1) { - return null; + String looseIsValue = looseIsShortcutValue(property); + if (looseIsValue == null + || PerlUnicodeScriptData.canonicalValue(looseIsValue) != null + || PerlUnicodeBlockData.set(looseIsValue) != null) { + return null; + } + UnicodeSet bareSet = resolvePerlBuiltInPropertyAlias(property); + return bareSet == null ? null : joniPropertyResult(bareSet, true); } String name = property.substring(0, assignment); String value = property.substring(assignment + 1); @@ -1033,6 +1040,11 @@ static CharacterPropertyResolver.Result resolveJoniProperty( UnicodeSet set = resolvePerlBuiltInPropertyAlias(property); if (set == null) return null; + return joniPropertyResult(set, caseFold); + } + + private static CharacterPropertyResolver.Result joniPropertyResult( + UnicodeSet set, boolean caseFold) { int[] ranges = new int[set.getRangeCount() * 2 + 1]; ranges[0] = set.getRangeCount(); for (int i = 0; i < set.getRangeCount(); i++) { @@ -1232,6 +1244,19 @@ && loosePropertyName(alias.substring(0, assignment)) } } } + String looseIsValue = looseIsShortcutValue(alias); + boolean inheritedBareIs = looseIsValue != null + && PerlUnicodeScriptData.canonicalValue(looseIsValue) == null + && PerlUnicodeBlockData.set(looseIsValue) == null; + if (inheritedBareIs) { + // Keep General_Category ahead of binary aliases in the shared Is + // shortcut namespace, before Block's ambiguity guard runs. + UnicodeSet category = PerlUnicodeGeneralCategoryData.resolve(looseIsValue); + if (category != null) return category; + if (isIcuBinaryPropertyAlias(looseIsValue)) { + return new UnicodeSet().applyPropertyAlias(looseIsValue, "True"); + } + } if (alias.equalsIgnoreCase("L&")) { UnicodeSet casedLetters = unicodePropertyValueSet( UProperty.GENERAL_CATEGORY, "UppercaseLetter"); @@ -1334,7 +1359,15 @@ && loosePropertyName(alias.substring(0, assignment)) // representation debt is closed; explicit Block=/In forms remain pinned. return null; } - return block; + if (block != null) return block; + + if (inheritedBareIs) { + // Only inherit aliases whose unprefixed spelling already resolves. + // This preserves user-property lookup and leaves missing bare bases + // (for example All and Unicode) for their owning property slices. + return resolveStandardPropertyAsSet(looseIsValue, new LinkedHashSet<>()); + } + return null; } private static Boolean perlBooleanPropertyValue(String value) { @@ -1363,6 +1396,29 @@ private static boolean isIcuGeneralCategoryAlias(String alias) { } } + private static String looseIsShortcutValue(String property) { + if (propertyValueDelimiter(property) >= 0) return null; + int prefixStart = 0; + while (prefixStart < property.length()) { + char separator = property.charAt(prefixStart); + if (!Character.isWhitespace(separator) + && separator != '-' && separator != '_') break; + prefixStart++; + } + if (property.length() - prefixStart <= 2 + || !property.regionMatches(true, prefixStart, "is", 0, 2)) { + return null; + } + int valueStart = prefixStart + 2; + while (valueStart < property.length()) { + char separator = property.charAt(valueStart); + if (!Character.isWhitespace(separator) + && separator != '-' && separator != '_') break; + valueStart++; + } + return valueStart < property.length() ? property.substring(valueStart) : null; + } + private static boolean isPerlIsPrefixedNumericWildcard(String property) { int assignment = propertyValueDelimiter(property); if (assignment <= 0 || assignment == property.length() - 1) return false; diff --git a/src/test/resources/unit/regex/unicode_binary_leading_shortcuts.t b/src/test/resources/unit/regex/unicode_binary_leading_shortcuts.t new file mode 100644 index 000000000..15d68978a --- /dev/null +++ b/src/test/resources/unit/regex/unicode_binary_leading_shortcuts.t @@ -0,0 +1,78 @@ +use strict; +use warnings; +use Test::More; + +sub compile_property { + my ($property) = @_; + my $pattern = eval 'qr/\A\p{' . $property . '}\z/u'; + return ($pattern, $@); +} + +my ($canonical, $canonical_error) = compile_property('IsUppercase'); +ok(defined $canonical, 'canonical Is binary shortcut compiles') + or diag($canonical_error); +like('A', $canonical, 'canonical Is binary shortcut matches a member'); +unlike('a', $canonical, 'canonical Is binary shortcut excludes a nonmember'); + +my ($lowercase, $lowercase_error) = compile_property('isuppercase'); +ok(defined $lowercase, 'binary shortcut accepts a lowercase Is prefix') + or diag($lowercase_error); +like('A', $lowercase, 'lowercase Is binary shortcut matches a member'); + +my ($leading, $leading_error) = compile_property('__Is_uppercase'); +ok(defined $leading, 'binary shortcut accepts leading loose separators') + or diag($leading_error); +like('A', $leading, 'leading-loose binary shortcut matches a member'); +unlike('a', $leading, 'leading-loose binary shortcut excludes a nonmember'); + +my ($hex, $hex_error) = compile_property('_ is_ASCII_HEX_DIGIT'); +ok(defined $hex, 'short binary alias accepts fully loose Is spelling') + or diag($hex_error); +like('F', $hex, 'loose binary alias matches its member'); +unlike('G', $hex, 'loose binary alias excludes a nonmember'); + +my ($assigned, $assigned_error) = compile_property('- Is_Assigned'); +ok(defined $assigned, 'another binary family accepts a leading-loose Is') + or diag($assigned_error); +like('A', $assigned, 'Assigned shortcut matches an assigned character'); +unlike(chr(0x0378), $assigned, + 'Assigned shortcut excludes an unassigned character'); + +my ($script, $script_error) = compile_property('_ is_Latn'); +ok(defined $script, 'Script shortcut retains precedence') or diag($script_error); +like('A', $script, 'Script shortcut membership remains intact'); +unlike(chr(0x03B1), $script, 'Script shortcut still excludes another script'); + +my ($category, $category_error) = compile_property('IsL'); +ok(defined $category, 'General_Category retains precedence') + or diag($category_error); +like(chr(0x03B1), $category, + 'General_Category shortcut still matches non-Latin letters'); + +my ($block, $block_error) = compile_property('_ Is_Basic_Latin'); +ok(defined $block, 'Block shortcut retains precedence') or diag($block_error); +like('A', $block, 'Block shortcut membership remains intact'); + +my $callback_calls = 0; +sub IsBinaryShortcutCallback { + $callback_calls++; + return "0041"; +} +my ($callback, $callback_error) = compile_property('IsBinaryShortcutCallback'); +ok(defined $callback, 'exact user property callback retains precedence') + or diag($callback_error); +like('A', $callback, 'exact user property callback matches its range'); +is($callback_calls, 1, 'exact user property callback is called once'); + +my ($leading_callback, $leading_callback_error) = + compile_property('_ IsBinaryShortcutCallback'); +ok(!defined($leading_callback) && length($leading_callback_error), + 'leading separators do not broaden user property lookup'); +is($callback_calls, 1, 'rejected leading callback spelling is not invoked'); + +my ($unknown, $unknown_error) = + compile_property('_ Is_DefinitelyNotABinaryProperty'); +ok(!defined($unknown) && length($unknown_error), + 'unknown leading-loose binary shortcut is rejected'); + +done_testing; From 6f53e0661d024e581444f1a45a8e57d04de8c5ff Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Tue, 18 Aug 2026 23:30:20 +0200 Subject: [PATCH 4/5] test(regex): isolate ASCII-strict negative sibling folds Add a system-Perl-first reducer for negated classes that must exclude safe non-ASCII fold siblings under /aa while retaining ASCII and unrelated members. Generated with [OpenAI Codex](https://openai.com/codex) Co-Authored-By: OpenAI Codex --- .../joni_ascii_strict_negative_sibling_fold.t | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 src/test/resources/unit/regex/joni_ascii_strict_negative_sibling_fold.t diff --git a/src/test/resources/unit/regex/joni_ascii_strict_negative_sibling_fold.t b/src/test/resources/unit/regex/joni_ascii_strict_negative_sibling_fold.t new file mode 100644 index 000000000..30bffbba5 --- /dev/null +++ b/src/test/resources/unit/regex/joni_ascii_strict_negative_sibling_fold.t @@ -0,0 +1,31 @@ +use strict; +use warnings; +use utf8; +use Test::More tests => 12; + +ok("\x{1E9E}" !~ /^[^\x{00DF}]$/iaa, + 'aa negative class excludes capital sharp s sibling'); +ok("\x{00DF}" !~ /^[^\x{1E9E}]$/iaa, + 'aa negative class excludes lowercase sharp s sibling'); +ok("\x{FB06}" !~ /^[^\x{FB05}]$/iaa, + 'aa negative class excludes second st ligature sibling'); +ok("\x{FB05}" !~ /^[^\x{FB06}]$/iaa, + 'aa negative class excludes first st ligature sibling'); + +ok("s" =~ /^[^\x{00DF}]$/iaa, + 'aa negative sharp s class retains ASCII s'); +ok("ä" =~ /^[^\x{00DF}]$/iaa, + 'aa negative sharp s class retains unrelated non-ASCII'); +ok("s" =~ /^[^\x{FB05}]$/iaa, + 'aa negative ligature class retains ASCII s'); +ok("ä" =~ /^[^\x{FB05}]$/iaa, + 'aa negative ligature class retains unrelated non-ASCII'); + +ok("x\x{1E9E}y" !~ /^x(?iaa:[^\x{00DF}])y$/u, + 'scoped aa negative class excludes sharp s sibling'); +ok("xäy" =~ /^x(?iaa:[^\x{00DF}])y$/u, + 'scoped aa negative class retains unrelated member'); +ok("_\x{FB06}_" !~ /^_(?iaa:[^\x{FB05}])_$/u, + 'anchored aa negative class excludes ligature sibling'); +ok("_ä_" =~ /^_(?iaa:[^\x{FB05}])_$/u, + 'anchored aa negative class retains unrelated member'); From ba6f12c019e87de2d275483900a97bcc7b360459 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 00:11:03 +0200 Subject: [PATCH 5/5] fix(regex): retain safe strict folds in negative classes Enumerate complete character-class fold relations, then reject strict-mode callbacks whenever any output code point crosses the ASCII boundary. Generated with [OpenAI Codex](https://openai.com/codex) Co-Authored-By: OpenAI Codex --- .../joni/src/org/joni/ApplyCaseFold.java | 11 ++- .../joni/src/org/joni/ScanEnvironment.java | 7 +- ...estPerlAsciiStrictNegativeSafeSibling.java | 67 +++++++++++++++++++ 3 files changed, 81 insertions(+), 4 deletions(-) create mode 100644 third_party/joni/test/org/joni/test/TestPerlAsciiStrictNegativeSafeSibling.java diff --git a/third_party/joni/src/org/joni/ApplyCaseFold.java b/third_party/joni/src/org/joni/ApplyCaseFold.java index 59021b4c8..a4122684e 100644 --- a/third_party/joni/src/org/joni/ApplyCaseFold.java +++ b/third_party/joni/src/org/joni/ApplyCaseFold.java @@ -41,7 +41,7 @@ public void apply(int from, int[]to, int length, Object o) { boolean addFlag; if (Option.isPerlAsciiStrict(env.option) - && Encoding.isAscii(from) != Encoding.isAscii(to[0])) { + && perlAsciiStrictRelationCrossesAscii(from, to, length)) { return; } @@ -111,6 +111,15 @@ public void apply(int from, int[]to, int length, Object o) { } + private static boolean perlAsciiStrictRelationCrossesAscii(int from, int[] to, + int length) { + boolean fromAscii = Encoding.isAscii(from); + for (int i = 0; i < length; i++) { + if (fromAscii != Encoding.isAscii(to[i])) return true; + } + return false; + } + static boolean isEligible(CClassNode ascCc, Encoding enc, int code) { if (ascCc == null) return false; boolean eligible = ascCc.isCodeInCC(enc, code); diff --git a/third_party/joni/src/org/joni/ScanEnvironment.java b/third_party/joni/src/org/joni/ScanEnvironment.java index cc88fc071..4405782f6 100644 --- a/third_party/joni/src/org/joni/ScanEnvironment.java +++ b/third_party/joni/src/org/joni/ScanEnvironment.java @@ -68,9 +68,10 @@ public final class ScanEnvironment { } int caseFoldFlagFor(int option) { - return Option.isPerlAsciiStrict(option) - ? caseFoldFlag & ~Config.INTERNAL_ENC_CASE_FOLD_MULTI_CHAR - : caseFoldFlag; + // Character classes need the complete fold relation so + // ApplyCaseFold can retain safe non-ASCII siblings under /aa while + // filtering relations that cross into ASCII. + return caseFoldFlag; } int addMemEntry() { diff --git a/third_party/joni/test/org/joni/test/TestPerlAsciiStrictNegativeSafeSibling.java b/third_party/joni/test/org/joni/test/TestPerlAsciiStrictNegativeSafeSibling.java new file mode 100644 index 000000000..c3ee7b021 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestPerlAsciiStrictNegativeSafeSibling.java @@ -0,0 +1,67 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import static org.junit.Assert.assertEquals; + +import java.nio.charset.StandardCharsets; + +import org.jcodings.specific.UTF8Encoding; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.junit.Test; + +public class TestPerlAsciiStrictNegativeSafeSibling { + private static int search(String pattern, String input) { + byte[] patternBytes = pattern.getBytes(StandardCharsets.UTF_8); + byte[] inputBytes = input.getBytes(StandardCharsets.UTF_8); + Regex regex = new Regex(patternBytes, 0, patternBytes.length, + Option.IGNORECASE | Option.PERL_ASCII_STRICT, + UTF8Encoding.INSTANCE, Syntax.PerlNG); + return regex.matcher(inputBytes).search(0, inputBytes.length, Option.NONE); + } + + private static void matches(String pattern, String input) { + assertEquals(0, search(pattern, input)); + } + + private static void misses(String pattern, String input) { + assertEquals(-1, search(pattern, input)); + } + + @Test + public void appliesSafeSiblingsToOrdinaryPositiveAndNegativeClasses() { + matches("^[ß]$", "ẞ"); + misses("^[^ß]$", "ẞ"); + matches("^[ſt]$", "st"); + misses("^[^ſt]$", "st"); + matches("^[^ß]$", "ä"); + matches("^[^ſt]$", "ä"); + } + + @Test + public void rejectsAsciiAtEveryPositionOfAMultiCharacterFold() { + misses("^[ʼn]$", "ʼn"); + misses("^[ʼn]$", "ʼN"); + matches("^[ʼn]$", "ʼn"); + matches("^[^ʼn]$", "ʼ"); + } +}