From d870b9fd6746055e2225f88525edee48354f736e Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Tue, 18 Aug 2026 14:16:20 +0200 Subject: [PATCH] fix(regex): protect internal scalar property boundaries Prevent generated Unicode property classes from inspecting visible internal scalar marker payloads, while preserving true surrogate membership by splitting crossing ranges into explicit segments. Update the Phase 36 plan with the focused and full-build evidence. Generated with Codex (https://openai.com/codex) Co-Authored-By: Codex --- dev/design/phase36-regex-parity.md | 22 ++++++++---- .../runtime/regex/JoniRegexPattern.java | 33 +++++++++++++----- .../runtime/regex/UnicodeResolver.java | 25 +++++++++++--- .../regex/native_property_marker_boundary.t | 34 +++++++++++++++++++ 4 files changed, 96 insertions(+), 18 deletions(-) create mode 100644 src/test/resources/unit/regex/native_property_marker_boundary.t diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index af7fead39..d0215ffc7 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -306,6 +306,16 @@ while `shadowJar` replaced it and the next exceeded its resource bound while overlapping the full-tree acceptance suite. The rerun must use the stable JAR at lower parallelism after competing heavy workers finish. +The stacked native-property marker-boundary slice prevents generated property +classes from beginning inside a visible marker payload and splits every +generated range crossing U+D800..U+DFFF into explicit pre-surrogate, surrogate, +and post-surrogate segments. That keeps real default-property membership while +removing accidental hexadecimal-payload matches. Its unchanged standard-Perl +oracle passes 10/10 on standard Perl, JVM, and interpreter; the protected HST/ +quick-check and surrogate-renderer gates pass 113/113 and 27/27 respectively on +both execution backends. Direct Joni, packaging, and warning-free `make` pass +in 4m02s. + Lexical `use bytes` now compiles non-ASCII substitution patterns with a single-byte Joni encoding while preserving upgraded, byte-backed, and compiled `qr//` source provenance. The focused oracle passes 12/12 on system Perl, JVM, @@ -846,12 +856,12 @@ is retained for now. - [ ] Complete the authoritative stable-JAR chunks 01–04 map for the surrogate renderer with exact JVM/interpreter identity, zero losses, and no missing numbered assertions before updating aggregate totals. - - [ ] Prevent unanchored native Joni properties/classes that contain no - translated surrogate range from beginning inside the visible payload of - an internal scalar marker. The isolated reducer at - `/tmp/phase36-native-property-marker-boundary.t` passes 2/2 on standard - Perl and 1/2 on PerlOnJava; this is separate from PR #1049's - surrogate-bearing translated-class renderer. + - [x] Prevented generated Joni property classes from beginning inside the + visible payload of an internal scalar marker, while splitting generated + ranges crossing U+D800..U+DFFF so true surrogate membership remains + atomic. The standard-Perl-first oracle passes 10/10; protected HST/quick- + check and surrogate-renderer gates pass 113/113 and 27/27 on JVM and + interpreter; warning-free `make`, direct Joni, and packaging pass. - [x] Integrated native Perl `\v`/`\V` dispatch inside and outside character classes (`1eff1db97`, integrated as `6328935cd`). The focused oracle passes 92/92 and unchanged `reg_posixcc.t` passes 2,560/2,560 on both backends. diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index 48affee20..358b8a46a 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -232,8 +232,11 @@ private static UserPropertyTranslation translateUserDefinedProperties( try { String propertyClass = UnicodeResolver.translateUnicodeProperty( property, pattern.charAt(i + 1) == 'P', flags.isCaseInsensitive()); + String joniPropertyClass = normalizeGeneratedPropertyClassForJoni( + propertyClass); translated.append("(?-i:") - .append(normalizeGeneratedPropertyClassForJoni(propertyClass)) + .append(translateInternalScalarClassMembers( + joniPropertyClass, true)) .append(')'); } catch (IllegalArgumentException error) { String message = error.getMessage(); @@ -509,10 +512,17 @@ private static String translatePattern(String pattern, RegexFlags flags, * They are represented internally as {@code U+FFFD}, which Joni can * match as ordinary text, but not as one logical character. Lift those * members and surrogate ranges into alternatives that consume a complete - * internal marker. For complemented classes, exclude selected marker + * internal marker. Generated Unicode-property classes additionally guard + * ordinary members so an unanchored search cannot begin inside the visible + * marker payload. For complemented classes, exclude selected marker * payloads while still consuming every other marker atomically. */ private static String translateInternalScalarClassMembers(String pattern) { + return translateInternalScalarClassMembers(pattern, false); + } + + private static String translateInternalScalarClassMembers( + String pattern, boolean guardOrdinaryClasses) { StringBuilder translated = new StringBuilder(pattern.length()); boolean escaped = false; for (int i = 0; i < pattern.length(); i++) { @@ -538,7 +548,7 @@ private static String translateInternalScalarClassMembers(String pattern) { continue; } String replacement = translateInternalScalarClassContent( - pattern.substring(i + 1, close)); + pattern.substring(i + 1, close), guardOrdinaryClasses); if (replacement == null) { translated.append(pattern, i, close + 1); } else { @@ -589,7 +599,8 @@ private static int findStandardClassClose(String pattern, int start) { private record HexEscape(long value, int endExclusive) {} - private static String translateInternalScalarClassContent(String content) { + private static String translateInternalScalarClassContent( + String content, boolean guardOrdinaryClass) { boolean negated = content.startsWith("^"); int contentStart = negated ? 1 : 0; StringBuilder retained = new StringBuilder(content.length()); @@ -642,7 +653,16 @@ private static String translateInternalScalarClassContent(String content) { } retained.append(content.charAt(i++)); } - if (exactMarkers.isEmpty() && surrogateRanges.isEmpty()) return null; + String anyMarker = "\\x{FFFD}<[0-9A-F]+>"; + String markerBoundary = INTERNAL_SCALAR_BOUNDARY_GUARD; + if (exactMarkers.isEmpty() && surrogateRanges.isEmpty()) { + if (!guardOrdinaryClass) return null; + if (negated) { + return "(?:" + anyMarker + "|" + markerBoundary + + "(?!" + anyMarker + ")[" + content + "])"; + } + return markerBoundary + "(?!" + anyMarker + ")[" + content + "]"; + } List payloads = new ArrayList<>( exactMarkers.size() + surrogateRanges.size()); @@ -652,9 +672,6 @@ private static String translateInternalScalarClassContent(String content) { } String payload = payloads.size() == 1 ? payloads.get(0) : "(?:" + String.join("|", payloads) + ")"; - String anyMarker = "\\x{FFFD}<[0-9A-F]+>"; - String markerBoundary = INTERNAL_SCALAR_BOUNDARY_GUARD; - List alternatives = new ArrayList<>(2); if (negated) { alternatives.add("\\x{FFFD}<(?!(?:" + payload + ")>)[0-9A-F]+>"); diff --git a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java index fda93ed22..68cb128fd 100644 --- a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java +++ b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java @@ -2507,15 +2507,32 @@ static String unicodeSetToJavaPattern(UnicodeSet set) { for (int i = 0; i < set.getRangeCount(); i++) { int start = set.getRangeStart(i); int end = set.getRangeEnd(i); - appendJavaPatternChar(sb, start); - if (start != end) { - sb.append('-'); - appendJavaPatternChar(sb, end); + if (start <= Character.MAX_SURROGATE + && end >= Character.MIN_SURROGATE) { + if (start < Character.MIN_SURROGATE) { + appendJavaPatternRange(sb, start, Character.MIN_SURROGATE - 1); + } + appendJavaPatternRange(sb, + Math.max(start, Character.MIN_SURROGATE), + Math.min(end, Character.MAX_SURROGATE)); + if (end > Character.MAX_SURROGATE) { + appendJavaPatternRange(sb, Character.MAX_SURROGATE + 1, end); + } + } else { + appendJavaPatternRange(sb, start, end); } } return sb.toString(); } + private static void appendJavaPatternRange(StringBuilder sb, int start, int end) { + appendJavaPatternChar(sb, start); + if (start != end) { + sb.append('-'); + appendJavaPatternChar(sb, end); + } + } + private static void appendJavaPatternChar(StringBuilder sb, int codePoint) { if (codePoint >= 0x10000 || (codePoint >= Character.MIN_SURROGATE diff --git a/src/test/resources/unit/regex/native_property_marker_boundary.t b/src/test/resources/unit/regex/native_property_marker_boundary.t new file mode 100644 index 000000000..4a2b7f5d5 --- /dev/null +++ b/src/test/resources/unit/regex/native_property_marker_boundary.t @@ -0,0 +1,34 @@ +use strict; +use warnings; +use Test::More tests => 10; + +my $surrogate = chr(0xD800); +my $above_unicode = chr(0x110000); + +ok($surrogate !~ /\p{ASCII}/, + 'positive native property cannot inspect a surrogate marker payload'); +ok($surrogate =~ /\P{ASCII}/, + 'complemented native property accepts a surrogate'); +is($&, $surrogate, + 'complemented native property consumes a surrogate as one scalar'); + +ok($above_unicode !~ /\p{ASCII}/, + 'positive native property cannot inspect an above-Unicode marker payload'); +ok($above_unicode =~ /\P{ASCII}/, + 'complemented native property accepts an above-Unicode scalar'); +is($&, $above_unicode, + 'complemented native property consumes an above-Unicode value atomically'); + +my $mixed = "A${surrogate}B${above_unicode}C"; +my @non_ascii = $mixed =~ /(\P{ASCII})/g; +is_deeply(\@non_ascii, [$surrogate, $above_unicode], + 'global complemented properties return complete internal scalars'); + +my $copy = $mixed; +is($copy =~ s/\P{ASCII}/X/g, 2, + 'global substitution counts internal scalars once each'); +is($copy, 'AXBXC', + 'global substitution leaves no marker payload behind'); + +ok("Q${surrogate}R" !~ /\p{ASCII}{2}$/, + 'property quantifier cannot begin inside an internal marker');