fix(regex): generalize Perl ASCII-strict fold crossings

AGENTS.md

@@ -201,6 +201,7 @@
 | 2026-08-06 | (no source work lost — build recovered) A process cleanup killed the active Gradle test workers, producing exit 137 failures in two shards. | Agent selected Java PIDs from a broad CPU list without first constraining them to stale processes. Recovery: rerun `make` without killing workers; subsequent build completed successfully. Fix: never kill by CPU list alone; identify the exact command and build ownership first. |
 | 2026-08-17 | (no source work lost — stale workers removed) Failed `make` runs were interrupted after their known Joni failures, but their Gradle unit-shard workers survived and competed with later builds. | Agent sent Ctrl-C to the parent build session before all parallel workers had exited. Recovery: identified stale workers by PID, start time, and shard work directory, terminated only those exact PIDs, and left the current build and sibling repositories untouched. Fix: let failed parallel `make` runs finish naturally, or verify and clean up their exact child PIDs before starting another build. |
 | 2026-08-17 | (no source work lost — CPAN run rerun) A concurrent `make` replaced the development shadow JAR while an active `jcpan` process was spawning a child JVM, causing a transient `ClassNotFoundException`. | Agent waited for another worktree's build but did not wait for the same worktree's bounded CPAN runs before rebuilding `target/perlonjava-5.44.0.jar`. Recovery: let `make` finish and rerun the affected CPAN target. Fix: never rebuild a worktree's development JAR while that worktree has active `jperl` or `jcpan` processes. |
+| 2026-08-18 | (no source work lost — wrong local WIP ref recovered) A test-snapshot cherry-pick landed in the original checkout instead of its newly created continuation worktree. | Agent chained `git worktree add` and `git cherry-pick` while the shell remained in the original working directory. Recovery: preserved the mistaken commit on a recovery branch, restored the original WIP ref to its exact prior commit without reset, then cherry-picked in the intended worktree. Fix: run post-creation Git commands with the new worktree as the explicit working directory and verify `git branch --show-current` before committing. |
 
 When you cause a new incident, append a row here in the same commit
 that fixes it. Future agents need to see that these warnings are real.

src/test/resources/unit/regex/joni_multifold_matrix.t

@@ -0,0 +1,45 @@
+use strict;
+use warnings;
+use utf8;
+use Test::More tests => 33;
+
+ok("ss" =~ /^\x{00DF}$/iu, 'sharp s forward');
+ok("\x{00DF}" =~ /^ss$/iu, 'sharp s reverse');
+ok("\x{017F}\x{017F}" =~ /^\x{00DF}$/iu, 'sharp s through long s');
+ok("st" =~ /^[\x{FB06}]$/iu, 'ligature class forward');
+ok("\x{FB06}" =~ /^st$/iu, 'ligature reverse');
+ok("ffi" =~ /^\x{FB03}$/iu, 'three-codepoint ligature forward');
+ok("\x{FB03}" =~ /^ffi$/iu, 'three-codepoint ligature reverse');
+ok("\x{01F0}" =~ /^\x{006A}\x{030C}$/iu, 'two-codepoint forward');
+ok("\x{006A}\x{030C}" =~ /^\x{01F0}$/iu, 'two-codepoint reverse');
+ok("\x{0390}" =~ /^\x{03B9}\x{0308}\x{0301}$/iu, 'three-codepoint forward');
+ok("\x{03B9}\x{0308}\x{0301}" =~ /^\x{0390}$/iu, 'three-codepoint reverse');
+ok("\x{1E9E}" =~ /^\x{00DF}$/iu, 'capital sharp s sibling forward');
+ok("\x{00DF}" =~ /^\x{1E9E}$/iu, 'capital sharp s sibling reverse');
+
+ok("xssy" =~ /^x(?i:\x{00DF})y$/u, 'scoped i enables full fold');
+ok("xssy" !~ /^x(?-i:\x{00DF})y$/iu, 'scoped minus i disables full fold');
+ok("\x{017F}s\x{017F}" =~ /^(?i:s)(?iaa:s)(?i:s)$/u,
+    'scoped aa preserves outer fold state');
+
+ok("\x{00DF}" =~ /^ss$/ia, 'a allows Unicode to ASCII full fold');
+ok("ss" =~ /^\x{00DF}$/ia, 'a allows ASCII to Unicode full fold');
+ok("\x{00DF}" !~ /^ss$/iaa, 'aa blocks Unicode to ASCII full fold');
+ok("ss" !~ /^\x{00DF}$/iaa, 'aa blocks ASCII to Unicode full fold');
+ok("Ä" =~ /^ä$/iaa, 'aa preserves non-ASCII simple fold');
+ok("ffi" !~ /^\x{FB03}$/iaa, 'aa blocks three-codepoint ASCII ligature fold');
+ok("st" !~ /^\x{FB06}$/iaa, 'aa blocks two-codepoint ASCII ligature fold');
+ok("\x{03B9}\x{0308}\x{0301}" =~ /^\x{0390}$/iaa,
+    'aa preserves all-non-ASCII multi fold');
+
+ok("\x{212A}" =~ /^[k]$/iu, 'positive class gains Kelvin fold');
+ok("\x{212A}" !~ /^[^k]$/iu, 'negative class excludes Kelvin fold');
+ok("x" =~ /^[^k]$/iu, 'negative class keeps unrelated member');
+ok("st" =~ /^[\x{FB06}]$/iu, 'positive class expands multi fold');
+ok("\x{FB06}" !~ /^[^\x{FB06}]$/iu, 'negative class excludes source member');
+
+my $set = qr/(?[ [ksä] - [x] ])/i;
+ok("k" =~ /^$set$/u, 'composed set keeps first member');
+ok("s" =~ /^$set$/u, 'composed set keeps second member');
+ok("ä" =~ /^$set$/u, 'composed set keeps non-ASCII member');
+ok("x" !~ /^$set$/u, 'composed set subtraction remains excluded');

third_party/joni/src/org/joni/Analyser.java

@@ -35,6 +35,7 @@
 import java.util.IllegalFormatConversionException;
 
 import org.jcodings.CaseFoldCodeItem;
+import org.jcodings.Encoding;
 import org.jcodings.ObjPtr;
 import org.jcodings.Ptr;
 import org.jcodings.constants.CharacterType;
@@ -1902,6 +1903,20 @@ private boolean expandCaseFoldStringAlt(int itemNum, CaseFoldCodeItem[]items,
 
     private static final int THRESHOLD_CASE_FOLD_ALT_FOR_EXPANSION = 8;
 
+    private boolean perlAsciiStrictSourceCrossesIntoAscii(byte[] bytes, int p, int end,
+                                                           int codePoint) {
+        if (Encoding.isAscii(codePoint)) return false;
+
+        CaseFoldCodeItem[] items = enc.caseFoldCodesByString(
+                regex.caseFoldFlag, bytes, p, end);
+        for (CaseFoldCodeItem item : items) {
+            for (int foldedCodePoint : item.code) {
+                if (Encoding.isAscii(foldedCodePoint)) return true;
+            }
+        }
+        return false;
+    }
+
     private Node protectPerlAsciiStrictCrossings(StringNode source, int state) {
         byte[] bytes = source.bytes;
         int segmentStart = source.p;
@@ -1912,7 +1927,8 @@ private Node protectPerlAsciiStrictCrossings(StringNode source, int state) {
         while (p < source.end) {
             int codePoint = enc.mbcToCode(bytes, p, source.end);
             int next = p + enc.length(bytes, p, source.end);
-            if (codePoint == 0x017f || codePoint == 0x212a) {
+            if (perlAsciiStrictSourceCrossesIntoAscii(
+                    bytes, p, source.end, codePoint)) {
                 if (segmentStart < p) {
                     ListNode segment = ListNode.newList(
                             new StringNode(bytes, segmentStart, p), null);

third_party/joni/test/org/joni/test/TestPerlMultiFoldMatrix.java

@@ -0,0 +1,123 @@
+/*
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of
+ * this software and associated documentation files (the "Software"), to deal in
+ * the Software without restriction, including without limitation the rights to
+ * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+ * of the Software, and to permit persons to whom the Software is furnished to do
+ * so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+package org.joni.test;
+
+import static org.junit.Assert.assertEquals;
+
+import java.nio.charset.StandardCharsets;
+
+import org.jcodings.specific.UTF8Encoding;
+import org.joni.Matcher;
+import org.joni.Option;
+import org.joni.Regex;
+import org.joni.Syntax;
+import org.junit.Test;
+
+public class TestPerlMultiFoldMatrix {
+    private static int search(String pattern, String input, int options) {
+        return search(pattern, input, options, Syntax.PerlNG);
+    }
+
+    private static int search(String pattern, String input, int options, Syntax syntax) {
+        byte[] patternBytes = pattern.getBytes(StandardCharsets.UTF_8);
+        byte[] inputBytes = input.getBytes(StandardCharsets.UTF_8);
+        Regex regex = new Regex(patternBytes, 0, patternBytes.length, options,
+                UTF8Encoding.INSTANCE, syntax);
+        Matcher matcher = regex.matcher(inputBytes);
+        return matcher.search(0, inputBytes.length, Option.NONE);
+    }
+
+    private static void matches(String pattern, String input) {
+        assertEquals(0, search(pattern, input, Option.IGNORECASE));
+    }
+
+    private static void misses(String pattern, String input) {
+        assertEquals(-1, search(pattern, input, Option.IGNORECASE));
+    }
+
+    @Test
+    public void foldsSharpSAndLigaturesInBothDirections() {
+        matches("^ß$", "ss");
+        matches("^ss$", "ß");
+        matches("^ß$", "ſſ");
+        matches("^[st]$", "st");
+        matches("^st$", "st");
+        matches("^ffi$", "ffi");
+        matches("^ffi$", "ffi");
+    }
+
+    @Test
+    public void foldsTwoAndThreeCodepointSequencesInBothDirections() {
+        matches("^ǰ$", "ǰ");
+        matches("^ǰ$", "ǰ");
+        matches("^ΐ$", "ΐ");
+        matches("^ΐ$", "ΐ");
+    }
+
+    @Test
+    public void foldsCapitalAndLowercaseSharpSSiblings() {
+        matches("^ẞ$", "ß");
+        matches("^ß$", "ẞ");
+    }
+
+    @Test
+    public void scopesIgnoreCaseAndAsciiStrictOptions() {
+        assertEquals(0, search("^x(?i:ß)y$", "xssy", Option.NONE));
+        assertEquals(-1, search("^x(?-i:ß)y$", "xssy", Option.IGNORECASE));
+        assertEquals(0, search("^(?i:s)(?iaa:s)(?i:s)$", "ſsſ", Option.NONE));
+    }
+
+    @Test
+    public void distinguishesPerlAsciiAndAsciiStrictFolding() {
+        assertEquals(0, search("^ss$", "ß", Option.IGNORECASE | Option.ASCII_RANGE));
+        assertEquals(0, search("^ß$", "ss", Option.IGNORECASE | Option.ASCII_RANGE));
+        assertEquals(-1, search("^ss$", "ß",
+                Option.IGNORECASE | Option.PERL_ASCII_STRICT));
+        assertEquals(-1, search("^ß$", "ss",
+                Option.IGNORECASE | Option.PERL_ASCII_STRICT));
+        assertEquals(0, search("^ä$", "Ä",
+                Option.IGNORECASE | Option.PERL_ASCII_STRICT));
+        assertEquals(-1, search("^ffi$", "ffi",
+                Option.IGNORECASE | Option.PERL_ASCII_STRICT));
+        assertEquals(-1, search("^st$", "st",
+                Option.IGNORECASE | Option.PERL_ASCII_STRICT));
+        assertEquals(0, search("^ΐ$", "ΐ",
+                Option.IGNORECASE | Option.PERL_ASCII_STRICT));
+    }
+
+    @Test
+    public void appliesFoldsToPositiveAndNegativeClasses() {
+        matches("^[k]$", "K");
+        misses("^[^k]$", "K");
+        matches("^[^k]$", "x");
+        matches("^[st]$", "st");
+        misses("^[^st]$", "st");
+    }
+
+    @Test
+    public void appliesEligibleFoldsInsideComposedClasses() {
+        String pattern = "^[[ksä]&&[^x]]$";
+        int options = Option.IGNORECASE;
+        assertEquals(0, search(pattern, "K", options, Syntax.DEFAULT));
+        assertEquals(0, search(pattern, "ſ", options, Syntax.DEFAULT));
+        assertEquals(0, search(pattern, "Ä", options, Syntax.DEFAULT));
+        assertEquals(-1, search(pattern, "x", options, Syntax.DEFAULT));
+    }
+}