diff --git a/AGENTS.md b/AGENTS.md index 4c3e61c9f..5324d3b6b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -201,6 +201,7 @@ | 2026-08-06 | (no source work lost — build recovered) A process cleanup killed the active Gradle test workers, producing exit 137 failures in two shards. | Agent selected Java PIDs from a broad CPU list without first constraining them to stale processes. Recovery: rerun `make` without killing workers; subsequent build completed successfully. Fix: never kill by CPU list alone; identify the exact command and build ownership first. | | 2026-08-17 | (no source work lost — stale workers removed) Failed `make` runs were interrupted after their known Joni failures, but their Gradle unit-shard workers survived and competed with later builds. | Agent sent Ctrl-C to the parent build session before all parallel workers had exited. Recovery: identified stale workers by PID, start time, and shard work directory, terminated only those exact PIDs, and left the current build and sibling repositories untouched. Fix: let failed parallel `make` runs finish naturally, or verify and clean up their exact child PIDs before starting another build. | | 2026-08-17 | (no source work lost — CPAN run rerun) A concurrent `make` replaced the development shadow JAR while an active `jcpan` process was spawning a child JVM, causing a transient `ClassNotFoundException`. | Agent waited for another worktree's build but did not wait for the same worktree's bounded CPAN runs before rebuilding `target/perlonjava-5.44.0.jar`. Recovery: let `make` finish and rerun the affected CPAN target. Fix: never rebuild a worktree's development JAR while that worktree has active `jperl` or `jcpan` processes. | +| 2026-08-18 | (no source work lost — wrong local WIP ref recovered) A test-snapshot cherry-pick landed in the original checkout instead of its newly created continuation worktree. | Agent chained `git worktree add` and `git cherry-pick` while the shell remained in the original working directory. Recovery: preserved the mistaken commit on a recovery branch, restored the original WIP ref to its exact prior commit without reset, then cherry-picked in the intended worktree. Fix: run post-creation Git commands with the new worktree as the explicit working directory and verify `git branch --show-current` before committing. | When you cause a new incident, append a row here in the same commit that fixes it. Future agents need to see that these warnings are real. diff --git a/src/test/resources/unit/regex/joni_multifold_matrix.t b/src/test/resources/unit/regex/joni_multifold_matrix.t new file mode 100644 index 000000000..d40f0c2fe --- /dev/null +++ b/src/test/resources/unit/regex/joni_multifold_matrix.t @@ -0,0 +1,45 @@ +use strict; +use warnings; +use utf8; +use Test::More tests => 33; + +ok("ss" =~ /^\x{00DF}$/iu, 'sharp s forward'); +ok("\x{00DF}" =~ /^ss$/iu, 'sharp s reverse'); +ok("\x{017F}\x{017F}" =~ /^\x{00DF}$/iu, 'sharp s through long s'); +ok("st" =~ /^[\x{FB06}]$/iu, 'ligature class forward'); +ok("\x{FB06}" =~ /^st$/iu, 'ligature reverse'); +ok("ffi" =~ /^\x{FB03}$/iu, 'three-codepoint ligature forward'); +ok("\x{FB03}" =~ /^ffi$/iu, 'three-codepoint ligature reverse'); +ok("\x{01F0}" =~ /^\x{006A}\x{030C}$/iu, 'two-codepoint forward'); +ok("\x{006A}\x{030C}" =~ /^\x{01F0}$/iu, 'two-codepoint reverse'); +ok("\x{0390}" =~ /^\x{03B9}\x{0308}\x{0301}$/iu, 'three-codepoint forward'); +ok("\x{03B9}\x{0308}\x{0301}" =~ /^\x{0390}$/iu, 'three-codepoint reverse'); +ok("\x{1E9E}" =~ /^\x{00DF}$/iu, 'capital sharp s sibling forward'); +ok("\x{00DF}" =~ /^\x{1E9E}$/iu, 'capital sharp s sibling reverse'); + +ok("xssy" =~ /^x(?i:\x{00DF})y$/u, 'scoped i enables full fold'); +ok("xssy" !~ /^x(?-i:\x{00DF})y$/iu, 'scoped minus i disables full fold'); +ok("\x{017F}s\x{017F}" =~ /^(?i:s)(?iaa:s)(?i:s)$/u, + 'scoped aa preserves outer fold state'); + +ok("\x{00DF}" =~ /^ss$/ia, 'a allows Unicode to ASCII full fold'); +ok("ss" =~ /^\x{00DF}$/ia, 'a allows ASCII to Unicode full fold'); +ok("\x{00DF}" !~ /^ss$/iaa, 'aa blocks Unicode to ASCII full fold'); +ok("ss" !~ /^\x{00DF}$/iaa, 'aa blocks ASCII to Unicode full fold'); +ok("Ä" =~ /^ä$/iaa, 'aa preserves non-ASCII simple fold'); +ok("ffi" !~ /^\x{FB03}$/iaa, 'aa blocks three-codepoint ASCII ligature fold'); +ok("st" !~ /^\x{FB06}$/iaa, 'aa blocks two-codepoint ASCII ligature fold'); +ok("\x{03B9}\x{0308}\x{0301}" =~ /^\x{0390}$/iaa, + 'aa preserves all-non-ASCII multi fold'); + +ok("\x{212A}" =~ /^[k]$/iu, 'positive class gains Kelvin fold'); +ok("\x{212A}" !~ /^[^k]$/iu, 'negative class excludes Kelvin fold'); +ok("x" =~ /^[^k]$/iu, 'negative class keeps unrelated member'); +ok("st" =~ /^[\x{FB06}]$/iu, 'positive class expands multi fold'); +ok("\x{FB06}" !~ /^[^\x{FB06}]$/iu, 'negative class excludes source member'); + +my $set = qr/(?[ [ksä] - [x] ])/i; +ok("k" =~ /^$set$/u, 'composed set keeps first member'); +ok("s" =~ /^$set$/u, 'composed set keeps second member'); +ok("ä" =~ /^$set$/u, 'composed set keeps non-ASCII member'); +ok("x" !~ /^$set$/u, 'composed set subtraction remains excluded'); diff --git a/third_party/joni/src/org/joni/Analyser.java b/third_party/joni/src/org/joni/Analyser.java index a1536d062..2ed5b9bd5 100644 --- a/third_party/joni/src/org/joni/Analyser.java +++ b/third_party/joni/src/org/joni/Analyser.java @@ -35,6 +35,7 @@ import java.util.IllegalFormatConversionException; import org.jcodings.CaseFoldCodeItem; +import org.jcodings.Encoding; import org.jcodings.ObjPtr; import org.jcodings.Ptr; import org.jcodings.constants.CharacterType; @@ -1902,6 +1903,20 @@ private boolean expandCaseFoldStringAlt(int itemNum, CaseFoldCodeItem[]items, private static final int THRESHOLD_CASE_FOLD_ALT_FOR_EXPANSION = 8; + private boolean perlAsciiStrictSourceCrossesIntoAscii(byte[] bytes, int p, int end, + int codePoint) { + if (Encoding.isAscii(codePoint)) return false; + + CaseFoldCodeItem[] items = enc.caseFoldCodesByString( + regex.caseFoldFlag, bytes, p, end); + for (CaseFoldCodeItem item : items) { + for (int foldedCodePoint : item.code) { + if (Encoding.isAscii(foldedCodePoint)) return true; + } + } + return false; + } + private Node protectPerlAsciiStrictCrossings(StringNode source, int state) { byte[] bytes = source.bytes; int segmentStart = source.p; @@ -1912,7 +1927,8 @@ private Node protectPerlAsciiStrictCrossings(StringNode source, int state) { while (p < source.end) { int codePoint = enc.mbcToCode(bytes, p, source.end); int next = p + enc.length(bytes, p, source.end); - if (codePoint == 0x017f || codePoint == 0x212a) { + if (perlAsciiStrictSourceCrossesIntoAscii( + bytes, p, source.end, codePoint)) { if (segmentStart < p) { ListNode segment = ListNode.newList( new StringNode(bytes, segmentStart, p), null); diff --git a/third_party/joni/test/org/joni/test/TestPerlMultiFoldMatrix.java b/third_party/joni/test/org/joni/test/TestPerlMultiFoldMatrix.java new file mode 100644 index 000000000..8729579d0 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestPerlMultiFoldMatrix.java @@ -0,0 +1,123 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import static org.junit.Assert.assertEquals; + +import java.nio.charset.StandardCharsets; + +import org.jcodings.specific.UTF8Encoding; +import org.joni.Matcher; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.junit.Test; + +public class TestPerlMultiFoldMatrix { + private static int search(String pattern, String input, int options) { + return search(pattern, input, options, Syntax.PerlNG); + } + + private static int search(String pattern, String input, int options, Syntax syntax) { + byte[] patternBytes = pattern.getBytes(StandardCharsets.UTF_8); + byte[] inputBytes = input.getBytes(StandardCharsets.UTF_8); + Regex regex = new Regex(patternBytes, 0, patternBytes.length, options, + UTF8Encoding.INSTANCE, syntax); + Matcher matcher = regex.matcher(inputBytes); + return matcher.search(0, inputBytes.length, Option.NONE); + } + + private static void matches(String pattern, String input) { + assertEquals(0, search(pattern, input, Option.IGNORECASE)); + } + + private static void misses(String pattern, String input) { + assertEquals(-1, search(pattern, input, Option.IGNORECASE)); + } + + @Test + public void foldsSharpSAndLigaturesInBothDirections() { + matches("^ß$", "ss"); + matches("^ss$", "ß"); + matches("^ß$", "ſſ"); + matches("^[st]$", "st"); + matches("^st$", "st"); + matches("^ffi$", "ffi"); + matches("^ffi$", "ffi"); + } + + @Test + public void foldsTwoAndThreeCodepointSequencesInBothDirections() { + matches("^ǰ$", "ǰ"); + matches("^ǰ$", "ǰ"); + matches("^ΐ$", "ΐ"); + matches("^ΐ$", "ΐ"); + } + + @Test + public void foldsCapitalAndLowercaseSharpSSiblings() { + matches("^ẞ$", "ß"); + matches("^ß$", "ẞ"); + } + + @Test + public void scopesIgnoreCaseAndAsciiStrictOptions() { + assertEquals(0, search("^x(?i:ß)y$", "xssy", Option.NONE)); + assertEquals(-1, search("^x(?-i:ß)y$", "xssy", Option.IGNORECASE)); + assertEquals(0, search("^(?i:s)(?iaa:s)(?i:s)$", "ſsſ", Option.NONE)); + } + + @Test + public void distinguishesPerlAsciiAndAsciiStrictFolding() { + assertEquals(0, search("^ss$", "ß", Option.IGNORECASE | Option.ASCII_RANGE)); + assertEquals(0, search("^ß$", "ss", Option.IGNORECASE | Option.ASCII_RANGE)); + assertEquals(-1, search("^ss$", "ß", + Option.IGNORECASE | Option.PERL_ASCII_STRICT)); + assertEquals(-1, search("^ß$", "ss", + Option.IGNORECASE | Option.PERL_ASCII_STRICT)); + assertEquals(0, search("^ä$", "Ä", + Option.IGNORECASE | Option.PERL_ASCII_STRICT)); + assertEquals(-1, search("^ffi$", "ffi", + Option.IGNORECASE | Option.PERL_ASCII_STRICT)); + assertEquals(-1, search("^st$", "st", + Option.IGNORECASE | Option.PERL_ASCII_STRICT)); + assertEquals(0, search("^ΐ$", "ΐ", + Option.IGNORECASE | Option.PERL_ASCII_STRICT)); + } + + @Test + public void appliesFoldsToPositiveAndNegativeClasses() { + matches("^[k]$", "K"); + misses("^[^k]$", "K"); + matches("^[^k]$", "x"); + matches("^[st]$", "st"); + misses("^[^st]$", "st"); + } + + @Test + public void appliesEligibleFoldsInsideComposedClasses() { + String pattern = "^[[ksä]&&[^x]]$"; + int options = Option.IGNORECASE; + assertEquals(0, search(pattern, "K", options, Syntax.DEFAULT)); + assertEquals(0, search(pattern, "ſ", options, Syntax.DEFAULT)); + assertEquals(0, search(pattern, "Ä", options, Syntax.DEFAULT)); + assertEquals(-1, search(pattern, "x", options, Syntax.DEFAULT)); + } +}