From b2bcef36b736af41138f2d57b9e5aadb55837c7a Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 20:49:07 +0200 Subject: [PATCH 1/5] fix(regex): implement native Perl extended-more mode Give forked Joni a distinct /xx option, preserve exact inline x nesting levels, and apply horizontal-space elision inside character classes without changing ordinary /x behavior. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../runtime/regex/JoniRegexPattern.java | 39 +++++++++++- .../perlonjava/runtime/regex/RegexFlags.java | 6 +- .../regex/extended_more_character_class.t | 17 ++++++ third_party/joni/src/org/joni/Lexer.java | 15 +++-- third_party/joni/src/org/joni/Option.java | 9 ++- third_party/joni/src/org/joni/Parser.java | 25 +++++++- .../org/joni/test/TestPerlExtendMore.java | 59 +++++++++++++++++++ 7 files changed, 160 insertions(+), 10 deletions(-) create mode 100644 src/test/resources/unit/regex/extended_more_character_class.t create mode 100644 third_party/joni/test/org/joni/test/TestPerlExtendMore.java diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index 66065f5979..6c27d524dc 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -482,6 +482,7 @@ private static int toJoniOptions(RegexFlags flags, boolean forceAsciiClasses) { int options = Option.NONE; if (flags.isCaseInsensitive()) options |= Option.IGNORECASE; if (flags.isExtended()) options |= Option.EXTEND; + if (flags.isExtendedWhitespace()) options |= Option.EXTEND | Option.PERL_EXTEND_MORE; // Oniguruma's MULTILINE option controls whether dot matches newline. if (flags.isDotAll()) options |= Option.MULTILINE; if (!flags.isMultiLine()) options |= Option.SINGLELINE; @@ -730,6 +731,40 @@ private static boolean hasControlVerbState(String pattern) { || pattern.contains("(*COMMIT"); } + private static boolean hasInlineExtendedOption(String pattern) { + boolean escaped = false; + boolean inClass = false; + for (int i = 0; i + 2 < pattern.length(); i++) { + char ch = pattern.charAt(i); + if (escaped) { + escaped = false; + continue; + } + if (ch == '\\') { + escaped = true; + continue; + } + if (ch == '[') { + inClass = true; + continue; + } + if (ch == ']' && inClass) { + inClass = false; + continue; + } + if (inClass || ch != '(' || pattern.charAt(i + 1) != '?') continue; + for (int j = i + 2; j < pattern.length(); j++) { + char option = pattern.charAt(j); + if (option == ':' || option == ')') break; + if (option == 'x') return true; + if (option == '-' || option == '^' + || option >= 'a' && option <= 'z') continue; + break; + } + } + return false; + } + static String translatePattern(String pattern) { return translatePattern(pattern, RegexFlags.fromModifiers("", pattern), 0, true); } @@ -747,6 +782,7 @@ private static String translatePattern(String pattern, RegexFlags flags, boolean inClass = false; boolean atClassStart = false; boolean classAllowsLeadingClose = false; + boolean inlineExtendedOption = hasInlineExtendedOption(pattern); int posixClassDepth = 0; for (int i = 0; i < pattern.length(); i++) { char ch = pattern.charAt(i); @@ -822,7 +858,8 @@ private static String translatePattern(String pattern, RegexFlags flags, i += 2; continue; } - if (inClass && flags.isExtendedWhitespace() && Character.isWhitespace(ch)) { + if (inClass && flags.isExtendedWhitespace() && !inlineExtendedOption + && Character.isWhitespace(ch)) { continue; } if (inClass && atClassStart && ch == '^') { diff --git a/src/main/java/org/perlonjava/runtime/regex/RegexFlags.java b/src/main/java/org/perlonjava/runtime/regex/RegexFlags.java index 6571bd2c81..d07c13f602 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RegexFlags.java +++ b/src/main/java/org/perlonjava/runtime/regex/RegexFlags.java @@ -223,7 +223,8 @@ public String toFlagString() { if (isMultiLine) flagString.append('m'); if (isDotAll) flagString.append('s'); if (isCaseInsensitive) flagString.append('i'); - if (isExtended) flagString.append('x'); + if (isExtendedWhitespace) flagString.append("xx"); + else if (isExtended) flagString.append('x'); if (isNonCapturing) flagString.append('n'); if (isNonDestructive) flagString.append('r'); if (taintResults) flagString.append('T'); @@ -243,7 +244,8 @@ public String toModifierString() { if (isMultiLine) sb.append('m'); if (isDotAll) sb.append('s'); if (isCaseInsensitive) sb.append('i'); - if (isExtended) sb.append('x'); + if (isExtendedWhitespace) sb.append("xx"); + else if (isExtended) sb.append('x'); if (isNonCapturing) sb.append('n'); return sb.toString(); } diff --git a/src/test/resources/unit/regex/extended_more_character_class.t b/src/test/resources/unit/regex/extended_more_character_class.t new file mode 100644 index 0000000000..f3e847388e --- /dev/null +++ b/src/test/resources/unit/regex/extended_more_character_class.t @@ -0,0 +1,17 @@ +use strict; +use warnings; +use Test::More; + +ok(' ' =~ /(?x:[a b])/xx, 'scoped single x downgrades outer xx'); +ok(' ' !~ /(?xx:[a b])/x, 'scoped xx ignores class space'); +ok(' ' =~ /(?x)[a b]/xx, 'option-only single x downgrades outer xx'); +ok(' ' !~ /(?xx)[a b]/x, 'option-only xx ignores class space'); +ok(' ' =~ /(?-x:[a b])/xx, 'scoped minus x disables both x levels'); + +ok("\t" !~ /(?xx:[a b])/, 'xx ignores an unescaped class tab'); +ok("\n" =~ /(?xx:[a +b])/, 'xx preserves an unescaped class newline'); +ok('#' =~ /(?xx:[a#b])/, 'xx preserves a class hash'); +ok(' ' =~ /(?xx:[a\ b])/, 'xx preserves escaped class space'); + +done_testing; diff --git a/third_party/joni/src/org/joni/Lexer.java b/third_party/joni/src/org/joni/Lexer.java index f0599750eb..2990436b8d 100644 --- a/third_party/joni/src/org/joni/Lexer.java +++ b/third_party/joni/src/org/joni/Lexer.java @@ -1128,12 +1128,17 @@ protected final TokenType fetchTokenInCC() { if (perlVerticalWhitespaceTokenIndex >= 0) { return fetchPerlVerticalWhitespaceToken(); } - if (!left()) { - token.type = TokenType.EOT; - return token.type; + while (true) { + if (!left()) { + token.type = TokenType.EOT; + return token.type; + } + fetch(); + if (!syntax.op2OptionPerl() || !Option.isPerlExtendMore(env.option) + || c != ' ' && c != '\t') { + break; + } } - - fetch(); token.type = TokenType.CHAR; token.base = 0; token.setC(c); diff --git a/third_party/joni/src/org/joni/Option.java b/third_party/joni/src/org/joni/Option.java index a4392cd1f0..38d4214e8c 100644 --- a/third_party/joni/src/org/joni/Option.java +++ b/third_party/joni/src/org/joni/Option.java @@ -50,8 +50,10 @@ public final class Option { public static final int PERL_ASCII_STRICT = (1 << 19); /** Perl /d byte strings use single-character Latin-1 folding only. */ public static final int PERL_BYTE_PATTERN = (1 << 20); + /** Perl /xx: EXTEND plus unescaped horizontal-space elision in classes. */ + public static final int PERL_EXTEND_MORE = (1 << 21); - public static final int MAXBIT = (1 << 21); /* limit */ + public static final int MAXBIT = (1 << 22); /* limit */ public static final int DEFAULT = NONE; @@ -72,6 +74,7 @@ public static String toString(int option) { if (isCR7Bit(option)) options += "CR_7_BIT"; if (isPerlAsciiStrict(option)) options += "PERL_ASCII_STRICT"; if (isPerlBytePattern(option)) options += "PERL_BYTE_PATTERN"; + if (isPerlExtendMore(option)) options += "PERL_EXTEND_MORE"; return options; } @@ -83,6 +86,10 @@ public static boolean isExtend(int option) { return (option & EXTEND) != 0; } + public static boolean isPerlExtendMore(int option) { + return (option & PERL_EXTEND_MORE) != 0; + } + public static boolean isSingleline(int option) { return (option & SINGLELINE) != 0; } diff --git a/third_party/joni/src/org/joni/Parser.java b/third_party/joni/src/org/joni/Parser.java index 6044be6b24..0ee1339eb5 100644 --- a/third_party/joni/src/org/joni/Parser.java +++ b/third_party/joni/src/org/joni/Parser.java @@ -992,6 +992,7 @@ && left() && enc.isDigit(peek())) { option = bsOnOff(option, Option.SINGLELINE, false); option = bsOnOff(option, Option.MULTILINE, true); option = bsOnOff(option, Option.EXTEND, true); + option = bsOnOff(option, Option.PERL_EXTEND_MORE, true); option = bsOnOff(option, Option.DONT_CAPTURE_GROUP, true); option = bsOnOff(option, Option.CAPTURE_GROUP, false); option = bsOnOff(option, Option.PERL_ASCII_STRICT, true); @@ -1021,6 +1022,7 @@ && left() && enc.isDigit(peek())) { case 'g': case 'o': boolean neg = false; + int positiveXCount = 0; PerlCharsetOptionState charsetOptions = new PerlCharsetOptionState(); boolean sawContinueModifier = false; while (true) { @@ -1032,7 +1034,15 @@ && left() && enc.isDigit(peek())) { neg = true; break; case 'x': - option = bsOnOff(option, Option.EXTEND, neg); + if (neg) { + option = bsOnOff(option, Option.EXTEND, true); + option = bsOnOff(option, Option.PERL_EXTEND_MORE, true); + } else { + positiveXCount++; + option = bsOnOff(option, Option.EXTEND, false); + option = bsOnOff(option, Option.PERL_EXTEND_MORE, + positiveXCount < 2); + } break; case 'i': option = bsOnOff(option, Option.IGNORECASE, neg); @@ -1669,12 +1679,14 @@ private PerlExtendedClassPrimary parsePerlExtendedClassPrimary() { nestedOption = bsOnOff(nestedOption, Option.SINGLELINE, false); nestedOption = bsOnOff(nestedOption, Option.MULTILINE, true); nestedOption = bsOnOff(nestedOption, Option.EXTEND, true); + nestedOption = bsOnOff(nestedOption, Option.PERL_EXTEND_MORE, true); nestedOption = bsOnOff(nestedOption, Option.DONT_CAPTURE_GROUP, true); nestedOption = bsOnOff(nestedOption, Option.CAPTURE_GROUP, false); nestedOption = bsOnOff(nestedOption, Option.PERL_ASCII_STRICT, true); inc(); } boolean negateOption = false; + int positiveXCount = 0; PerlCharsetOptionState charsetOptions = new PerlCharsetOptionState(); while (left() && !extendedClassAt(':')) { int option = extendedClassCode(); @@ -1687,6 +1699,17 @@ private PerlExtendedClassPrimary parsePerlExtendedClassPrimary() { if (option == '-') negateOption = true; else if (option == 'i') { nestedOption = bsOnOff(nestedOption, Option.IGNORECASE, negateOption); + } else if (option == 'x') { + if (negateOption) { + nestedOption = bsOnOff(nestedOption, Option.EXTEND, true); + nestedOption = bsOnOff( + nestedOption, Option.PERL_EXTEND_MORE, true); + } else { + positiveXCount++; + nestedOption = bsOnOff(nestedOption, Option.EXTEND, false); + nestedOption = bsOnOff(nestedOption, + Option.PERL_EXTEND_MORE, positiveXCount < 2); + } } else if (option == 'a' || option == 'd' || option == 'l' || option == 'u') { nestedOption = charsetOptions.apply( diff --git a/third_party/joni/test/org/joni/test/TestPerlExtendMore.java b/third_party/joni/test/org/joni/test/TestPerlExtendMore.java new file mode 100644 index 0000000000..635dbe1a1b --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestPerlExtendMore.java @@ -0,0 +1,59 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import static org.junit.Assert.assertEquals; + +import java.nio.charset.StandardCharsets; + +import org.jcodings.specific.ASCIIEncoding; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.junit.Test; + +public class TestPerlExtendMore { + private static int search(String pattern, String input, int options) { + byte[] patternBytes = pattern.getBytes(StandardCharsets.US_ASCII); + byte[] inputBytes = input.getBytes(StandardCharsets.US_ASCII); + Regex regex = new Regex(patternBytes, 0, patternBytes.length, options, + ASCIIEncoding.INSTANCE, Syntax.PerlNG); + return regex.matcher(inputBytes).search(0, inputBytes.length, Option.NONE); + } + + @Test + public void topLevelExtendMoreIgnoresOnlyHorizontalClassSpace() { + int xx = Option.EXTEND | Option.PERL_EXTEND_MORE; + assertEquals(-1, search("[a b]", " ", xx)); + assertEquals(-1, search("[a\tb]", "\t", xx)); + assertEquals(0, search("[a\nb]", "\n", xx)); + assertEquals(0, search("[a#b]", "#", xx)); + assertEquals(0, search("[a\\ b]", " ", xx)); + } + + @Test + public void inlineXSelectsAnExactExtendLevel() { + int x = Option.EXTEND; + int xx = x | Option.PERL_EXTEND_MORE; + assertEquals(0, search("(?x:[a b])", " ", xx)); + assertEquals(-1, search("(?xx:[a b])", " ", x)); + assertEquals(0, search("(?-x:[a b])", " ", xx)); + } +} From bbd5fe41ec45800e6f565b7d54fe8190e0b33acc Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 20:59:17 +0200 Subject: [PATCH 2/5] fix(regex): repartition adjacent full folds Generate bounded reverse full-fold partitions across adjacent Perl literal boundaries so equivalent multi-character fold sequences can backtrack without changing single-literal lookbehind width behavior. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../casefold_adjacent_multifold_partition.t | 14 +++ third_party/joni/src/org/joni/Analyser.java | 88 +++++++++++++++++++ .../joni/test/TestPerlAdjacentMultiFold.java | 47 ++++++++++ 3 files changed, 149 insertions(+) create mode 100644 src/test/resources/unit/regex/casefold_adjacent_multifold_partition.t create mode 100644 third_party/joni/test/org/joni/test/TestPerlAdjacentMultiFold.java diff --git a/src/test/resources/unit/regex/casefold_adjacent_multifold_partition.t b/src/test/resources/unit/regex/casefold_adjacent_multifold_partition.t new file mode 100644 index 0000000000..c6ea5afa21 --- /dev/null +++ b/src/test/resources/unit/regex/casefold_adjacent_multifold_partition.t @@ -0,0 +1,14 @@ +use strict; +use warnings; +use Test::More; + +like("\x{00DF}s", qr/^s\x{00DF}$/iu, + 'adjacent sharp-s folds can repartition three s characters'); +like("s\x{00DF}", qr/^\x{00DF}s$/iu, + 'reverse adjacent sharp-s folds can repartition three s characters'); +unlike("\x{00DF}x", qr/^s\x{00DF}$/iu, + 'adjacent sharp-s partition still rejects a different suffix'); +unlike("x\x{00DF}", qr/^\x{00DF}s$/iu, + 'reverse adjacent sharp-s partition still rejects a different prefix'); + +done_testing; diff --git a/third_party/joni/src/org/joni/Analyser.java b/third_party/joni/src/org/joni/Analyser.java index 4f01bb1a1e..d5ec4bfa50 100644 --- a/third_party/joni/src/org/joni/Analyser.java +++ b/third_party/joni/src/org/joni/Analyser.java @@ -33,6 +33,7 @@ import static org.joni.ast.QuantifierNode.isRepeatInfinite; import java.util.IllegalFormatConversionException; +import java.util.ArrayList; import org.jcodings.CaseFoldCodeItem; import org.jcodings.Encoding; @@ -2139,11 +2140,98 @@ private Node expandCaseFoldString(Node node) { } /* ending */ Node xnode = topRoot != null ? topRoot : prevNode.p; + xnode = addPerlReverseFoldPartitions(sn, xnode); node.replaceWith(xnode); return xnode; } + private Node addPerlReverseFoldPartitions(StringNode source, Node expanded) { + if (!syntax.op2OptionPerl() || Option.isPerlAsciiStrict(regex.options) + || Option.isPerlBytePattern(regex.options)) { + return expanded; + } + + int sourceLength = 0; + int canonicalLength = 0; + boolean hasFullFold = false; + for (int p = source.p; p < source.end;) { + int codePoint = enc.mbcToCode(source.bytes, p, source.end); + int fullLength = PerlCaseFold.fullFoldLength(codePoint); + canonicalLength += fullLength == 0 ? 1 : fullLength; + hasFullFold |= fullLength > 1; + sourceLength++; + p += enc.length(source.bytes, p, source.end); + } + if (!hasFullFold || sourceLength < 2 || canonicalLength <= sourceLength) { + return expanded; + } + + int[] canonical = new int[canonicalLength]; + int offset = 0; + for (int p = source.p; p < source.end;) { + int codePoint = enc.mbcToCode(source.bytes, p, source.end); + int fullLength = PerlCaseFold.fullFoldLength(codePoint); + if (fullLength == 0) { + canonical[offset++] = codePoint < 0x80 + ? Character.toLowerCase(codePoint) : codePoint; + } else { + for (int index = 0; index < fullLength; index++) { + canonical[offset++] = PerlCaseFold.fullFoldCodePoint( + codePoint, index); + } + } + p += enc.length(source.bytes, p, source.end); + } + + ArrayList variants = new ArrayList<>(); + collectPerlReverseFoldPartitions(canonical, 0, new int[canonical.length], + 0, false, variants); + if (variants.isEmpty()) return expanded; + + ListNode alternatives = newAlt(expanded, null); + ListNode tail = alternatives; + for (int[] variant : variants) { + StringNode candidate = new StringNode(); + for (int codePoint : variant) candidate.catCode(codePoint, enc); + candidate.setRaw(); + ListNode alternative = newAlt(candidate, null); + tail.setTail(alternative); + tail = alternative; + } + return alternatives; + } + + private void collectPerlReverseFoldPartitions(int[] canonical, int offset, + int[] path, int pathLength, boolean usedReverse, + ArrayList variants) { + if (variants.size() >= THRESHOLD_CASE_FOLD_ALT_FOR_EXPANSION) return; + if (offset == canonical.length) { + if (usedReverse) { + variants.add(java.util.Arrays.copyOf(path, pathLength)); + } + return; + } + + path[pathLength] = canonical[offset]; + collectPerlReverseFoldPartitions(canonical, offset + 1, path, + pathLength + 1, usedReverse, variants); + for (int length = 2; length <= 3 && offset + length <= canonical.length; + length++) { + int count = PerlCaseFold.reverseFullFoldSourceCount( + canonical, offset, length); + for (int index = 0; index < count; index++) { + path[pathLength] = PerlCaseFold.reverseFullFoldSourceAt( + canonical, offset, length, index); + collectPerlReverseFoldPartitions(canonical, offset + length, + path, pathLength + 1, true, variants); + if (variants.size() >= THRESHOLD_CASE_FOLD_ALT_FOR_EXPANSION) { + return; + } + } + } + } + private Node expandPerlByteAsciiFoldString(StringNode source, int state) { ListNode root = null; ListNode sequenceTail = null; diff --git a/third_party/joni/test/org/joni/test/TestPerlAdjacentMultiFold.java b/third_party/joni/test/org/joni/test/TestPerlAdjacentMultiFold.java new file mode 100644 index 0000000000..f88d45cc97 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestPerlAdjacentMultiFold.java @@ -0,0 +1,47 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import static org.junit.Assert.assertEquals; + +import java.nio.charset.StandardCharsets; + +import org.jcodings.specific.UTF8Encoding; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.junit.Test; + +public class TestPerlAdjacentMultiFold { + private static int search(String pattern, String input) { + byte[] patternBytes = pattern.getBytes(StandardCharsets.UTF_8); + byte[] inputBytes = input.getBytes(StandardCharsets.UTF_8); + Regex regex = new Regex(patternBytes, 0, patternBytes.length, + Option.IGNORECASE, UTF8Encoding.INSTANCE, Syntax.PerlNG); + return regex.matcher(inputBytes).search(0, inputBytes.length, Option.NONE); + } + + @Test + public void repartitionsAdjacentSharpSFolds() { + assertEquals(0, search("s\u00df", "\u00dfs")); + assertEquals(0, search("\u00dfs", "s\u00df")); + assertEquals(-1, search("s\u00df", "\u00dfx")); + } +} From c85c1108fe7b7354a3ed39132b5d4f37b407cc32 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 21:07:16 +0200 Subject: [PATCH 3/5] fix(regex): preserve Perl L_ property semantics Resolve Perl's exact L_ compatibility spelling as LC before loose Unicode property normalization, so it matches only cased letters rather than all letters. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../runtime/regex/UnicodeResolver.java | 7 +++++++ .../unit/regex/unicode_l_compatibility_alias.t | 18 ++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 src/test/resources/unit/regex/unicode_l_compatibility_alias.t diff --git a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java index 86d4f38a2f..039f614db6 100644 --- a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java +++ b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java @@ -2325,6 +2325,13 @@ private static UnicodeSet resolvePerlBareGeneralCategory(String property) { String looseIsValue = looseIsShortcutValue(property); String alias = looseIsValue == null ? property : looseIsValue; + // L_ is Perl's compatibility spelling for LC (cased letters). Its + // trailing underscore is significant even though ordinary Unicode + // property aliases otherwise use loose matching. + if (alias.trim().equalsIgnoreCase("L_")) { + return PerlUnicodeGeneralCategoryData.resolve("LC"); + } + // Perl's shared bare namespace gives scripts and binary properties // precedence over General_Category compatibility names. Blocks are // considered afterward by resolvePerlBareBlockShortcut. diff --git a/src/test/resources/unit/regex/unicode_l_compatibility_alias.t b/src/test/resources/unit/regex/unicode_l_compatibility_alias.t new file mode 100644 index 0000000000..986c5ce0eb --- /dev/null +++ b/src/test/resources/unit/regex/unicode_l_compatibility_alias.t @@ -0,0 +1,18 @@ +use strict; +use warnings; +use Test::More tests => 8; + +my $upper = "A"; +my $lower = "a"; +my $title = chr(0x01C5); +my $ideograph = chr(0x3400); + +ok($upper =~ /^\p{L_}$/, 'L_ matches an uppercase letter'); +ok($lower =~ /^\p{L_}$/, 'L_ matches a lowercase letter'); +ok($title =~ /^\p{L_}$/, 'L_ matches a titlecase letter'); +ok($ideograph !~ /^\p{L_}$/, 'L_ rejects an uncased letter'); + +ok($upper !~ /^\P{L_}$/, 'negated L_ rejects an uppercase letter'); +ok($lower !~ /^\P{L_}$/, 'negated L_ rejects a lowercase letter'); +ok($title !~ /^\P{L_}$/, 'negated L_ rejects a titlecase letter'); +ok($ideograph =~ /^\P{L_}$/, 'negated L_ matches an uncased letter'); From 6352b6d4e22856b77816d266009417f96e5fc381 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 21:10:50 +0200 Subject: [PATCH 4/5] docs(regex): align Phase 36 next steps Record the current native extended-more, adjacent full-fold, and L_ semantic position and order the dynamic-pattern and remaining fold work accordingly. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- dev/design/phase36-regex-parity.md | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index 752169ff24..0e93bf7851 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -125,6 +125,17 @@ affected corpus before taking another slice. - Negative lookbehind accepts capture enclosures and uses ACCEPT-aware width analysis in native Joni; named cut errors remain authoritative before an unnamed FAIL. The combined exact `regexp.t` differential has no introductions. +- Perl `/xx` character-class whitespace and nested inline `x`/`xx` mode changes + are native Joni lexer/parser behavior. The corresponding `regexp.t` identities + pass without introductions on both execution backends. +- Reverse full-fold alternatives can repartition across adjacent source + literals without changing single-literal lookbehind width. The targeted + `regexp.t` identity and the existing literal/backreference fold contract pass + on default and forced Joni for JVM and interpreter. +- Perl's exact `L_` General_Category compatibility spelling resolves as `LC` + before loose alias normalization, so uncased letters no longer enter that + class. The focused system-Perl oracle, four runtime legs, and imported + `regexp.t` identity agree. ## Execution Phases @@ -236,18 +247,19 @@ behavior. ## Ordered Next Steps -1. Run one warning-free full build and affected-corpus differential on the - integrated nested-quantifier, named-control-verb, and negative-lookbehind - batch, then open its review PR and require exact-head Ubuntu/Windows CI. -2. Complete byte/Unicode pattern provenance through runtime interpolation and - template composition, then finish `/d`/`u`/`a`/`aa` forward/reverse literal - and backreference folding from generated data. Require direct Joni plus - ordinary/forced JVM/interpreter zero-introduction gates. -3. Implement recursive and runtime `(??{...})` as native nested Joni execution: +1. Integrate the native `/xx`, adjacent full-fold partition, and `L_` alias + batch on the exact merged predecessor. Run one warning-free full build plus + zero-introduction `regexp.t` and fold/property gates, then publish a review + PR and require exact-head Ubuntu/Windows CI. +2. Complete recursive and runtime `(??{...})` as native nested Joni execution: preserve captures, `$^R`, `pos`, modes, byte/Unicode provenance, callback unwind, backtracking re-evaluation, and recursion safety. Route every embedded closure to Joni and delete constant inlining, progressive errors, and the dynamic Java adapter as their gates pass. +3. Complete byte/Unicode pattern provenance through runtime interpolation and + template composition, then finish `/d`/`u`/`a`/`aa` forward/reverse literal, + class, property, and backreference folding from generated data. Require + direct Joni plus ordinary/forced JVM/interpreter zero-introduction gates. 4. Finish the remaining lexical `use re 'strict'`, unescaped-brace, and non-hex diagnostic families. Refresh complete `reg_mesg.t`, `pat.t`, and `pat_advanced.t` maps after each combined batch. From a3b84e02a19bb73724a4675155d49afb377f2d1d Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 21:16:28 +0200 Subject: [PATCH 5/5] fix(regex): report invalid Unicode property followers Distinguish bare property escapes at end of pattern from invalid non-braced followers and emit Perl's native character-property diagnostic from Joni. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../unicode_property_follower_diagnostics.t | 19 +++++++ third_party/joni/src/org/joni/Lexer.java | 6 +- .../src/org/joni/exception/ErrorMessages.java | 2 + .../TestPerlPropertyFollowerDiagnostics.java | 55 +++++++++++++++++++ 4 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 src/test/resources/unit/regex/unicode_property_follower_diagnostics.t create mode 100644 third_party/joni/test/org/joni/test/TestPerlPropertyFollowerDiagnostics.java diff --git a/src/test/resources/unit/regex/unicode_property_follower_diagnostics.t b/src/test/resources/unit/regex/unicode_property_follower_diagnostics.t new file mode 100644 index 0000000000..31ad5eea9f --- /dev/null +++ b/src/test/resources/unit/regex/unicode_property_follower_diagnostics.t @@ -0,0 +1,19 @@ +use strict; +use warnings; +use Test::More tests => 7; + +for my $case ( + [ '\\p A', 'p' ], + [ '\\P:', 'P' ], + [ '\\p^', 'p' ], +) { + my ($pattern, $escape) = @$case; + my $ok = eval "qr/$pattern/; 1"; + ok(!$ok, "$pattern is rejected"); + my $expected = "Character following \\$escape must be '{' or a single-character Unicode property name"; + ok(index($@, $expected) >= 0, + "$pattern reports its invalid property follower"); +} + +my $ok = eval 'qr/\\p/; 1'; +ok(!$ok && $@ =~ /Empty \\p/, 'bare property escape retains its empty diagnostic'); diff --git a/third_party/joni/src/org/joni/Lexer.java b/third_party/joni/src/org/joni/Lexer.java index 2990436b8d..033a38a62b 100644 --- a/third_party/joni/src/org/joni/Lexer.java +++ b/third_party/joni/src/org/joni/Lexer.java @@ -1640,8 +1640,10 @@ private void fetchTokenFor_charProperty() { token.setPropCType(enc.propertyNameToCType(bytes, nameStart, p)); token.setPropNot(c == 'P'); } else if (syntax.op2OptionPerl()) { - newSyntaxException(PERL_EMPTY_CHARACTER_PROPERTY.replace( - "%n", Character.toString(c))); + String message = left() + ? PERL_INVALID_CHARACTER_PROPERTY_FOLLOWER + : PERL_EMPTY_CHARACTER_PROPERTY; + newSyntaxException(message.replace("%n", Character.toString(c))); } else { syntaxWarn("invalid Unicode Property \\<%n>", (char)c); } diff --git a/third_party/joni/src/org/joni/exception/ErrorMessages.java b/third_party/joni/src/org/joni/exception/ErrorMessages.java index 9956d6c9bf..a9833c1f96 100644 --- a/third_party/joni/src/org/joni/exception/ErrorMessages.java +++ b/third_party/joni/src/org/joni/exception/ErrorMessages.java @@ -116,6 +116,8 @@ public interface ErrorMessages extends org.jcodings.exception.ErrorMessages { String PERL_UNKNOWN_BOUND_TYPE = "'%n' is an unknown bound type"; String PERL_EMPTY_CHARACTER_PROPERTY = "Empty \\%n"; String PERL_EMPTY_CHARACTER_PROPERTY_BRACES = "Empty \\%n{}"; + String PERL_INVALID_CHARACTER_PROPERTY_FOLLOWER = + "Character following \\%n must be '{' or a single-character Unicode property name"; String PERL_UNTERMINATED_G_PATTERN = "Unterminated \\g... pattern"; String PERL_UNTERMINATED_G_BRACE_PATTERN = "Unterminated \\g{...} pattern"; String PERL_G_SEQUENCE_NOT_TERMINATED = "Sequence \\g{... not terminated"; diff --git a/third_party/joni/test/org/joni/test/TestPerlPropertyFollowerDiagnostics.java b/third_party/joni/test/org/joni/test/TestPerlPropertyFollowerDiagnostics.java new file mode 100644 index 0000000000..a432650743 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestPerlPropertyFollowerDiagnostics.java @@ -0,0 +1,55 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.nio.charset.StandardCharsets; + +import org.jcodings.specific.UTF8Encoding; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.joni.WarnCallback; +import org.joni.exception.SyntaxException; +import org.junit.Test; + +public class TestPerlPropertyFollowerDiagnostics { + private static void assertInvalidFollower(String pattern, String escape) { + byte[] bytes = pattern.getBytes(StandardCharsets.UTF_8); + try { + new Regex(bytes, 0, bytes.length, Option.NONE, UTF8Encoding.INSTANCE, + Syntax.PerlNG, WarnCallback.NONE); + fail("expected invalid Perl property follower for " + pattern); + } catch (SyntaxException error) { + assertEquals("Character following \\" + escape + + " must be '{' or a single-character Unicode property name", + error.getMessage()); + } + } + + @Test + public void reportsInvalidNonBracedPropertyFollowers() { + assertInvalidFollower("\\p A", "p"); + assertInvalidFollower("\\P:", "P"); + assertInvalidFollower("\\p^", "p"); + } +}