fix(regex): integrate ASCII-strict safe siblings

src/test/resources/unit/regex/joni_ascii_strict_sibling_fold.t

@@ -0,0 +1,36 @@
+use strict;
+use warnings;
+use utf8;
+use Test::More tests => 14;
+
+ok("\x{1E9E}" =~ /^\x{00DF}$/iaa,
+    'aa keeps capital sharp s as lowercase sharp s sibling');
+ok("\x{00DF}" =~ /^\x{1E9E}$/iaa,
+    'aa keeps lowercase sharp s as capital sharp s sibling');
+ok("ss" !~ /^\x{00DF}$/iaa,
+    'aa blocks sharp s pattern from expanding to ASCII');
+ok("\x{00DF}" !~ /^ss$/iaa,
+    'aa blocks ASCII pattern from matching sharp s');
+
+ok("\x{FB06}" =~ /^\x{FB05}$/iaa,
+    'aa keeps the two non-ASCII st ligatures as siblings');
+ok("\x{FB05}" =~ /^\x{FB06}$/iaa,
+    'aa keeps the reverse non-ASCII st ligature sibling');
+ok("st" !~ /^\x{FB05}$/iaa,
+    'aa blocks first st ligature pattern from expanding to ASCII');
+ok("\x{FB05}" !~ /^st$/iaa,
+    'aa blocks ASCII pattern from matching first st ligature');
+
+ok("\x{1E9E}" =~ /^[\x{00DF}]$/iaa,
+    'aa keeps sharp s siblings in a positive class');
+ok("\x{FB06}" =~ /^[\x{FB05}]$/iaa,
+    'aa keeps ligature siblings in a positive class');
+
+ok("x\x{1E9E}y" =~ /^x(?iaa:\x{00DF})y$/u,
+    'scoped aa keeps sharp s siblings');
+ok("xssy" !~ /^x(?iaa:\x{00DF})y$/u,
+    'scoped aa still blocks sharp s ASCII expansion');
+ok("_\x{FB06}_" =~ /^_(?iaa:\x{FB05})_$/u,
+    'anchored aa keeps ligature siblings with surrounding literals');
+ok("_st_" !~ /^_(?iaa:\x{FB05})_$/u,
+    'anchored aa blocks ligature ASCII expansion');

third_party/joni/src/org/joni/Analyser.java

@@ -1921,6 +1921,34 @@ private boolean perlAsciiStrictSourceCrossesIntoAscii(byte[] bytes, int p, int e
         return false;
     }
 
+    private Node perlAsciiStrictProtectedFoldNode(byte[] bytes, int p, int end) {
+        StringNode exact = new StringNode(bytes, p, end);
+        exact.setRaw();
+        ListNode alternatives = newAlt(exact, null);
+        ListNode tail = alternatives;
+
+        CaseFoldCodeItem[] items = enc.caseFoldCodesByString(
+                regex.caseFoldFlag, bytes, p, end);
+        for (CaseFoldCodeItem item : items) {
+            boolean allNonAscii = true;
+            for (int foldedCodePoint : item.code) {
+                if (Encoding.isAscii(foldedCodePoint)) {
+                    allNonAscii = false;
+                    break;
+                }
+            }
+            if (!allNonAscii) continue;
+
+            StringNode folded = new StringNode();
+            for (int foldedCodePoint : item.code) folded.catCode(foldedCodePoint, enc);
+            folded.setRaw();
+            ListNode alternative = newAlt(folded, null);
+            tail.setTail(alternative);
+            tail = alternative;
+        }
+        return alternatives.tail == null ? exact : alternatives;
+    }
+
     private Node protectPerlAsciiStrictCrossings(StringNode source, int state) {
         byte[] bytes = source.bytes;
         int segmentStart = source.p;
@@ -1941,9 +1969,8 @@ private Node protectPerlAsciiStrictCrossings(StringNode source, int state) {
                     tail = segment;
                 }
 
-                StringNode exact = new StringNode(bytes, p, next);
-                exact.setRaw();
-                ListNode segment = ListNode.newList(exact, null);
+                ListNode segment = ListNode.newList(
+                        perlAsciiStrictProtectedFoldNode(bytes, p, next), null);
                 if (root == null) root = segment;
                 else tail.setTail(segment);
                 tail = segment;

third_party/joni/test/org/joni/test/TestPerlAsciiStrictSafeSibling.java

@@ -0,0 +1,74 @@
+/*
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of
+ * this software and associated documentation files (the "Software"), to deal in
+ * the Software without restriction, including without limitation the rights to
+ * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+ * of the Software, and to permit persons to whom the Software is furnished to do
+ * so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+package org.joni.test;
+
+import static org.junit.Assert.assertEquals;
+
+import java.nio.charset.StandardCharsets;
+
+import org.jcodings.specific.UTF8Encoding;
+import org.joni.Option;
+import org.joni.Regex;
+import org.joni.Syntax;
+import org.junit.Test;
+
+public class TestPerlAsciiStrictSafeSibling {
+    private static int search(String pattern, String input, int options) {
+        byte[] patternBytes = pattern.getBytes(StandardCharsets.UTF_8);
+        byte[] inputBytes = input.getBytes(StandardCharsets.UTF_8);
+        Regex regex = new Regex(patternBytes, 0, patternBytes.length, options,
+                UTF8Encoding.INSTANCE, Syntax.PerlNG);
+        return regex.matcher(inputBytes).search(0, inputBytes.length, Option.NONE);
+    }
+
+    private static void matches(String pattern, String input) {
+        assertEquals(0, search(pattern, input,
+                Option.IGNORECASE | Option.PERL_ASCII_STRICT));
+    }
+
+    private static void misses(String pattern, String input) {
+        assertEquals(-1, search(pattern, input,
+                Option.IGNORECASE | Option.PERL_ASCII_STRICT));
+    }
+
+    @Test
+    public void retainsOnlySafeNonAsciiSharpSSiblings() {
+        matches("^ß$", "ẞ");
+        matches("^ẞ$", "ß");
+        misses("^ß$", "ss");
+        misses("^ss$", "ß");
+    }
+
+    @Test
+    public void retainsOnlySafeNonAsciiLigatureSiblings() {
+        matches("^ſt$", "st");
+        matches("^st$", "ſt");
+        misses("^ſt$", "st");
+        misses("^st$", "ſt");
+    }
+
+    @Test
+    public void appliesToOptimizedPositiveClassesAndLiteralSegments() {
+        matches("^[ß]$", "ẞ");
+        matches("^[ſt]$", "st");
+        matches("^xßy$", "xẞy");
+        misses("^xßy$", "xssy");
+    }
+}