From f11f28a67b252434b2ef4033d698ad7c4d1abb3b Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Tue, 18 Aug 2026 23:47:34 +0200 Subject: [PATCH] fix(regex): retain safe ASCII-strict fold siblings Keep non-ASCII fold siblings available when Perl ASCII-strict matching protects a source character from fold alternatives that cross into ASCII. Generated with [OpenAI Codex](https://openai.com/codex) Co-Authored-By: OpenAI Codex --- .../regex/joni_ascii_strict_sibling_fold.t | 36 +++++++++ third_party/joni/src/org/joni/Analyser.java | 33 ++++++++- .../test/TestPerlAsciiStrictSafeSibling.java | 74 +++++++++++++++++++ 3 files changed, 140 insertions(+), 3 deletions(-) create mode 100644 src/test/resources/unit/regex/joni_ascii_strict_sibling_fold.t create mode 100644 third_party/joni/test/org/joni/test/TestPerlAsciiStrictSafeSibling.java diff --git a/src/test/resources/unit/regex/joni_ascii_strict_sibling_fold.t b/src/test/resources/unit/regex/joni_ascii_strict_sibling_fold.t new file mode 100644 index 000000000..ade8aaec4 --- /dev/null +++ b/src/test/resources/unit/regex/joni_ascii_strict_sibling_fold.t @@ -0,0 +1,36 @@ +use strict; +use warnings; +use utf8; +use Test::More tests => 14; + +ok("\x{1E9E}" =~ /^\x{00DF}$/iaa, + 'aa keeps capital sharp s as lowercase sharp s sibling'); +ok("\x{00DF}" =~ /^\x{1E9E}$/iaa, + 'aa keeps lowercase sharp s as capital sharp s sibling'); +ok("ss" !~ /^\x{00DF}$/iaa, + 'aa blocks sharp s pattern from expanding to ASCII'); +ok("\x{00DF}" !~ /^ss$/iaa, + 'aa blocks ASCII pattern from matching sharp s'); + +ok("\x{FB06}" =~ /^\x{FB05}$/iaa, + 'aa keeps the two non-ASCII st ligatures as siblings'); +ok("\x{FB05}" =~ /^\x{FB06}$/iaa, + 'aa keeps the reverse non-ASCII st ligature sibling'); +ok("st" !~ /^\x{FB05}$/iaa, + 'aa blocks first st ligature pattern from expanding to ASCII'); +ok("\x{FB05}" !~ /^st$/iaa, + 'aa blocks ASCII pattern from matching first st ligature'); + +ok("\x{1E9E}" =~ /^[\x{00DF}]$/iaa, + 'aa keeps sharp s siblings in a positive class'); +ok("\x{FB06}" =~ /^[\x{FB05}]$/iaa, + 'aa keeps ligature siblings in a positive class'); + +ok("x\x{1E9E}y" =~ /^x(?iaa:\x{00DF})y$/u, + 'scoped aa keeps sharp s siblings'); +ok("xssy" !~ /^x(?iaa:\x{00DF})y$/u, + 'scoped aa still blocks sharp s ASCII expansion'); +ok("_\x{FB06}_" =~ /^_(?iaa:\x{FB05})_$/u, + 'anchored aa keeps ligature siblings with surrounding literals'); +ok("_st_" !~ /^_(?iaa:\x{FB05})_$/u, + 'anchored aa blocks ligature ASCII expansion'); diff --git a/third_party/joni/src/org/joni/Analyser.java b/third_party/joni/src/org/joni/Analyser.java index 3e0703738..2b939f1f2 100644 --- a/third_party/joni/src/org/joni/Analyser.java +++ b/third_party/joni/src/org/joni/Analyser.java @@ -1921,6 +1921,34 @@ private boolean perlAsciiStrictSourceCrossesIntoAscii(byte[] bytes, int p, int e return false; } + private Node perlAsciiStrictProtectedFoldNode(byte[] bytes, int p, int end) { + StringNode exact = new StringNode(bytes, p, end); + exact.setRaw(); + ListNode alternatives = newAlt(exact, null); + ListNode tail = alternatives; + + CaseFoldCodeItem[] items = enc.caseFoldCodesByString( + regex.caseFoldFlag, bytes, p, end); + for (CaseFoldCodeItem item : items) { + boolean allNonAscii = true; + for (int foldedCodePoint : item.code) { + if (Encoding.isAscii(foldedCodePoint)) { + allNonAscii = false; + break; + } + } + if (!allNonAscii) continue; + + StringNode folded = new StringNode(); + for (int foldedCodePoint : item.code) folded.catCode(foldedCodePoint, enc); + folded.setRaw(); + ListNode alternative = newAlt(folded, null); + tail.setTail(alternative); + tail = alternative; + } + return alternatives.tail == null ? exact : alternatives; + } + private Node protectPerlAsciiStrictCrossings(StringNode source, int state) { byte[] bytes = source.bytes; int segmentStart = source.p; @@ -1941,9 +1969,8 @@ private Node protectPerlAsciiStrictCrossings(StringNode source, int state) { tail = segment; } - StringNode exact = new StringNode(bytes, p, next); - exact.setRaw(); - ListNode segment = ListNode.newList(exact, null); + ListNode segment = ListNode.newList( + perlAsciiStrictProtectedFoldNode(bytes, p, next), null); if (root == null) root = segment; else tail.setTail(segment); tail = segment; diff --git a/third_party/joni/test/org/joni/test/TestPerlAsciiStrictSafeSibling.java b/third_party/joni/test/org/joni/test/TestPerlAsciiStrictSafeSibling.java new file mode 100644 index 000000000..b2ad144dc --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestPerlAsciiStrictSafeSibling.java @@ -0,0 +1,74 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import static org.junit.Assert.assertEquals; + +import java.nio.charset.StandardCharsets; + +import org.jcodings.specific.UTF8Encoding; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.junit.Test; + +public class TestPerlAsciiStrictSafeSibling { + private static int search(String pattern, String input, int options) { + byte[] patternBytes = pattern.getBytes(StandardCharsets.UTF_8); + byte[] inputBytes = input.getBytes(StandardCharsets.UTF_8); + Regex regex = new Regex(patternBytes, 0, patternBytes.length, options, + UTF8Encoding.INSTANCE, Syntax.PerlNG); + return regex.matcher(inputBytes).search(0, inputBytes.length, Option.NONE); + } + + private static void matches(String pattern, String input) { + assertEquals(0, search(pattern, input, + Option.IGNORECASE | Option.PERL_ASCII_STRICT)); + } + + private static void misses(String pattern, String input) { + assertEquals(-1, search(pattern, input, + Option.IGNORECASE | Option.PERL_ASCII_STRICT)); + } + + @Test + public void retainsOnlySafeNonAsciiSharpSSiblings() { + matches("^ß$", "ẞ"); + matches("^ẞ$", "ß"); + misses("^ß$", "ss"); + misses("^ss$", "ß"); + } + + @Test + public void retainsOnlySafeNonAsciiLigatureSiblings() { + matches("^ſt$", "st"); + matches("^st$", "ſt"); + misses("^ſt$", "st"); + misses("^st$", "ſt"); + } + + @Test + public void appliesToOptimizedPositiveClassesAndLiteralSegments() { + matches("^[ß]$", "ẞ"); + matches("^[ſt]$", "st"); + matches("^xßy$", "xẞy"); + misses("^xßy$", "xssy"); + } +}