diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index af7fead39..d0215ffc7 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -306,6 +306,16 @@ while `shadowJar` replaced it and the next exceeded its resource bound while overlapping the full-tree acceptance suite. The rerun must use the stable JAR at lower parallelism after competing heavy workers finish. +The stacked native-property marker-boundary slice prevents generated property +classes from beginning inside a visible marker payload and splits every +generated range crossing U+D800..U+DFFF into explicit pre-surrogate, surrogate, +and post-surrogate segments. That keeps real default-property membership while +removing accidental hexadecimal-payload matches. Its unchanged standard-Perl +oracle passes 10/10 on standard Perl, JVM, and interpreter; the protected HST/ +quick-check and surrogate-renderer gates pass 113/113 and 27/27 respectively on +both execution backends. Direct Joni, packaging, and warning-free `make` pass +in 4m02s. + Lexical `use bytes` now compiles non-ASCII substitution patterns with a single-byte Joni encoding while preserving upgraded, byte-backed, and compiled `qr//` source provenance. The focused oracle passes 12/12 on system Perl, JVM, @@ -846,12 +856,12 @@ is retained for now. - [ ] Complete the authoritative stable-JAR chunks 01–04 map for the surrogate renderer with exact JVM/interpreter identity, zero losses, and no missing numbered assertions before updating aggregate totals. - - [ ] Prevent unanchored native Joni properties/classes that contain no - translated surrogate range from beginning inside the visible payload of - an internal scalar marker. The isolated reducer at - `/tmp/phase36-native-property-marker-boundary.t` passes 2/2 on standard - Perl and 1/2 on PerlOnJava; this is separate from PR #1049's - surrogate-bearing translated-class renderer. + - [x] Prevented generated Joni property classes from beginning inside the + visible payload of an internal scalar marker, while splitting generated + ranges crossing U+D800..U+DFFF so true surrogate membership remains + atomic. The standard-Perl-first oracle passes 10/10; protected HST/quick- + check and surrogate-renderer gates pass 113/113 and 27/27 on JVM and + interpreter; warning-free `make`, direct Joni, and packaging pass. - [x] Integrated native Perl `\v`/`\V` dispatch inside and outside character classes (`1eff1db97`, integrated as `6328935cd`). The focused oracle passes 92/92 and unchanged `reg_posixcc.t` passes 2,560/2,560 on both backends. diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index 48affee20..358b8a46a 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -232,8 +232,11 @@ private static UserPropertyTranslation translateUserDefinedProperties( try { String propertyClass = UnicodeResolver.translateUnicodeProperty( property, pattern.charAt(i + 1) == 'P', flags.isCaseInsensitive()); + String joniPropertyClass = normalizeGeneratedPropertyClassForJoni( + propertyClass); translated.append("(?-i:") - .append(normalizeGeneratedPropertyClassForJoni(propertyClass)) + .append(translateInternalScalarClassMembers( + joniPropertyClass, true)) .append(')'); } catch (IllegalArgumentException error) { String message = error.getMessage(); @@ -509,10 +512,17 @@ private static String translatePattern(String pattern, RegexFlags flags, * They are represented internally as {@code U+FFFD}, which Joni can * match as ordinary text, but not as one logical character. Lift those * members and surrogate ranges into alternatives that consume a complete - * internal marker. For complemented classes, exclude selected marker + * internal marker. Generated Unicode-property classes additionally guard + * ordinary members so an unanchored search cannot begin inside the visible + * marker payload. For complemented classes, exclude selected marker * payloads while still consuming every other marker atomically. */ private static String translateInternalScalarClassMembers(String pattern) { + return translateInternalScalarClassMembers(pattern, false); + } + + private static String translateInternalScalarClassMembers( + String pattern, boolean guardOrdinaryClasses) { StringBuilder translated = new StringBuilder(pattern.length()); boolean escaped = false; for (int i = 0; i < pattern.length(); i++) { @@ -538,7 +548,7 @@ private static String translateInternalScalarClassMembers(String pattern) { continue; } String replacement = translateInternalScalarClassContent( - pattern.substring(i + 1, close)); + pattern.substring(i + 1, close), guardOrdinaryClasses); if (replacement == null) { translated.append(pattern, i, close + 1); } else { @@ -589,7 +599,8 @@ private static int findStandardClassClose(String pattern, int start) { private record HexEscape(long value, int endExclusive) {} - private static String translateInternalScalarClassContent(String content) { + private static String translateInternalScalarClassContent( + String content, boolean guardOrdinaryClass) { boolean negated = content.startsWith("^"); int contentStart = negated ? 1 : 0; StringBuilder retained = new StringBuilder(content.length()); @@ -642,7 +653,16 @@ private static String translateInternalScalarClassContent(String content) { } retained.append(content.charAt(i++)); } - if (exactMarkers.isEmpty() && surrogateRanges.isEmpty()) return null; + String anyMarker = "\\x{FFFD}<[0-9A-F]+>"; + String markerBoundary = INTERNAL_SCALAR_BOUNDARY_GUARD; + if (exactMarkers.isEmpty() && surrogateRanges.isEmpty()) { + if (!guardOrdinaryClass) return null; + if (negated) { + return "(?:" + anyMarker + "|" + markerBoundary + + "(?!" + anyMarker + ")[" + content + "])"; + } + return markerBoundary + "(?!" + anyMarker + ")[" + content + "]"; + } List payloads = new ArrayList<>( exactMarkers.size() + surrogateRanges.size()); @@ -652,9 +672,6 @@ private static String translateInternalScalarClassContent(String content) { } String payload = payloads.size() == 1 ? payloads.get(0) : "(?:" + String.join("|", payloads) + ")"; - String anyMarker = "\\x{FFFD}<[0-9A-F]+>"; - String markerBoundary = INTERNAL_SCALAR_BOUNDARY_GUARD; - List alternatives = new ArrayList<>(2); if (negated) { alternatives.add("\\x{FFFD}<(?!(?:" + payload + ")>)[0-9A-F]+>"); diff --git a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java index fda93ed22..68cb128fd 100644 --- a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java +++ b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java @@ -2507,15 +2507,32 @@ static String unicodeSetToJavaPattern(UnicodeSet set) { for (int i = 0; i < set.getRangeCount(); i++) { int start = set.getRangeStart(i); int end = set.getRangeEnd(i); - appendJavaPatternChar(sb, start); - if (start != end) { - sb.append('-'); - appendJavaPatternChar(sb, end); + if (start <= Character.MAX_SURROGATE + && end >= Character.MIN_SURROGATE) { + if (start < Character.MIN_SURROGATE) { + appendJavaPatternRange(sb, start, Character.MIN_SURROGATE - 1); + } + appendJavaPatternRange(sb, + Math.max(start, Character.MIN_SURROGATE), + Math.min(end, Character.MAX_SURROGATE)); + if (end > Character.MAX_SURROGATE) { + appendJavaPatternRange(sb, Character.MAX_SURROGATE + 1, end); + } + } else { + appendJavaPatternRange(sb, start, end); } } return sb.toString(); } + private static void appendJavaPatternRange(StringBuilder sb, int start, int end) { + appendJavaPatternChar(sb, start); + if (start != end) { + sb.append('-'); + appendJavaPatternChar(sb, end); + } + } + private static void appendJavaPatternChar(StringBuilder sb, int codePoint) { if (codePoint >= 0x10000 || (codePoint >= Character.MIN_SURROGATE diff --git a/src/test/resources/unit/regex/native_property_marker_boundary.t b/src/test/resources/unit/regex/native_property_marker_boundary.t new file mode 100644 index 000000000..4a2b7f5d5 --- /dev/null +++ b/src/test/resources/unit/regex/native_property_marker_boundary.t @@ -0,0 +1,34 @@ +use strict; +use warnings; +use Test::More tests => 10; + +my $surrogate = chr(0xD800); +my $above_unicode = chr(0x110000); + +ok($surrogate !~ /\p{ASCII}/, + 'positive native property cannot inspect a surrogate marker payload'); +ok($surrogate =~ /\P{ASCII}/, + 'complemented native property accepts a surrogate'); +is($&, $surrogate, + 'complemented native property consumes a surrogate as one scalar'); + +ok($above_unicode !~ /\p{ASCII}/, + 'positive native property cannot inspect an above-Unicode marker payload'); +ok($above_unicode =~ /\P{ASCII}/, + 'complemented native property accepts an above-Unicode scalar'); +is($&, $above_unicode, + 'complemented native property consumes an above-Unicode value atomically'); + +my $mixed = "A${surrogate}B${above_unicode}C"; +my @non_ascii = $mixed =~ /(\P{ASCII})/g; +is_deeply(\@non_ascii, [$surrogate, $above_unicode], + 'global complemented properties return complete internal scalars'); + +my $copy = $mixed; +is($copy =~ s/\P{ASCII}/X/g, 2, + 'global substitution counts internal scalars once each'); +is($copy, 'AXBXC', + 'global substitution leaves no marker payload behind'); + +ok("Q${surrogate}R" !~ /\p{ASCII}{2}$/, + 'property quantifier cannot begin inside an internal marker');