From 1dc870b6d8e7eccd23703e007ad85346d95668bb Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 16:58:21 +0200 Subject: [PATCH 01/16] fix(regex): preserve byte provenance through regex templates Carry byte/Unicode backing through composed regex templates, recompilation, cloning, and nested callback templates so /d can select the correct Joni input variant without inferring encoding from decoded Java strings. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../runtime/regex/JoniRegexPattern.java | 14 +++-- .../runtime/regex/RuntimeRegex.java | 54 ++++++++++++++----- .../regex/RuntimeRegexSourceCompiler.java | 5 +- .../runtime/regex/RuntimeRegexTemplate.java | 31 +++++++++-- .../RuntimeRegexTemplateProvenanceTest.java | 34 ++++++++++++ 5 files changed, 115 insertions(+), 23 deletions(-) create mode 100644 src/test/java/org/perlonjava/runtime/regex/RuntimeRegexTemplateProvenanceTest.java diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index 4ea903c22d..739527760a 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -1063,7 +1063,7 @@ private boolean find(int option, boolean anchored) { matcher = regex.matcher(bytes); if (!callbacks.isEmpty()) { calloutHandler = new PerlCalloutHandler( - input, byteToChar, callbacks, flags, hasControlVerbState, subject); + input, byteToChar, callbacks, flags, hasControlVerbState, byteMode, subject); matcher.setCalloutHandler(calloutHandler); } int result; @@ -1281,6 +1281,7 @@ static CaptureSnapshot of(MatchView match) { private final List callbacks; private final RegexFlags outerFlags; private final boolean publishesControlVerbState; + private final boolean byteMode; private final RuntimeScalar subject; private final int initialLocalLevel; private final RegexState initialRegexState; @@ -1295,20 +1296,21 @@ static CaptureSnapshot of(MatchView match) { PerlCalloutHandler(String input, int[] byteToChar, List callbacks, RegexFlags outerFlags, boolean publishesControlVerbState, - RuntimeScalar subject) { + boolean byteMode, RuntimeScalar subject) { this(input, byteToChar, callbacks, outerFlags, publishesControlVerbState, - subject, null); + byteMode, subject, null); } private PerlCalloutHandler( String input, int[] byteToChar, List callbacks, RegexFlags outerFlags, boolean publishesControlVerbState, - RuntimeScalar subject, PerlCalloutHandler parent) { + boolean byteMode, RuntimeScalar subject, PerlCalloutHandler parent) { this.input = input; this.byteToChar = byteToChar; this.callbacks = callbacks; this.outerFlags = outerFlags; this.publishesControlVerbState = publishesControlVerbState; + this.byteMode = byteMode; this.subject = subject; this.parent = parent; this.nestedDepth = parent == null ? 0 : parent.nestedDepth + 1; @@ -1352,7 +1354,8 @@ public DynamicPatternResult executeDynamic(int id, MatchView match) { nestedCallbacks = runtimeRegex.executableCallbacks; } else if (value.value instanceof RuntimeRegexTemplate template) { nestedPattern = new JoniRegexPattern(template.pattern(), outerFlags, - template.callbacks().size()); + template.callbacks().size(), false, + byteMode && template.byteBackedPattern(), template.byteBackedPattern()); nestedCallbacks = template.callbacks(); } else { String dynamicSource = value.toString(); @@ -1380,6 +1383,7 @@ public DynamicPatternResult executeDynamic(int id, MatchView match) { && runtimeRegex.getRegexFlags() != null ? runtimeRegex.getRegexFlags() : outerFlags, nestedPattern.hasControlVerbState, + byteMode, subject, this); if (nestedHandler != null) executedNestedCallbackPattern = true; diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java index 62308c93e3..44804c5bdb 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java @@ -273,6 +273,10 @@ public RegexFlags getRegexFlags() { return regexFlags; } + boolean isPatternByteBacked() { + return patternByteBacked; + } + private void setExecutableCallbacks(List callbacks) { List retained = List.copyOf(callbacks); for (RuntimeRegexCallback callback : retained) callback.retainOwner(); @@ -315,8 +319,10 @@ public RegexMatcher matcher(RuntimeScalar string, String input) { } private JoniRegexPattern selectRecursivePattern(RuntimeScalar string) { - if (bytesSubstitution && recursivePatternBytes != null - && string.type == RuntimeScalarType.BYTE_STRING) { + boolean byteDefaultSemantics = patternByteBacked && regexFlags != null + && !regexFlags.isUnicode() && !regexFlags.isAscii(); + if ((bytesSubstitution || byteDefaultSemantics) && recursivePatternBytes != null + && !Utf8.isUtf8(string)) { return recursivePatternBytes; } if (recursivePatternUnicode != null && recursivePatternUnicode != recursivePattern @@ -499,11 +505,16 @@ public static RuntimeRegex compile(String patternString, String modifiers) { } private static RuntimeRegex compile(String patternString, String modifiers, int lexicalDebugMode) { - return compile(patternString, modifiers, lexicalDebugMode, 0); + return compile(patternString, modifiers, lexicalDebugMode, 0, false); } private static RuntimeRegex compile(String patternString, String modifiers, int lexicalDebugMode, int trustedCalloutCount) { + return compile(patternString, modifiers, lexicalDebugMode, trustedCalloutCount, false); + } + + private static RuntimeRegex compile(String patternString, String modifiers, int lexicalDebugMode, + int trustedCalloutCount, boolean patternByteBacked) { RuntimeScalar namedCharacterTranslator = org.perlonjava.runtime.HintHashRegistry.getCompileTimeHint("charnames"); modifiers = stripDebugMarkers(modifiers); @@ -516,7 +527,7 @@ private static RuntimeRegex compile(String patternString, String modifiers, int UnicodeResolver.preloadUserDefinedProperties( patternString, preloadFlags.isCaseInsensitive()); return compileSynchronized(patternString, modifiers, lexicalDebugMode, - trustedCalloutCount, false, namedCharacterTranslator); + trustedCalloutCount, false, patternByteBacked, namedCharacterTranslator); } /** User properties execute Perl code and therefore cannot be validated while compiling a CV. */ @@ -538,7 +549,7 @@ public static boolean requiresRuntimeUnicodePropertyResolution(String patternStr public static void validateLiteralSyntax(String patternString, String modifiers) { try { compileSynchronized(patternString, stripDebugMarkers(modifiers), - debugMode(modifiers), 0, true, + debugMode(modifiers), 0, true, false, org.perlonjava.runtime.HintHashRegistry.getCompileTimeHint("charnames")); } catch (PerlJavaUnimplementedException unsupported) { String message = unsupported.getMessage(); @@ -559,6 +570,7 @@ public static void validateLiteralSyntax(String patternString, String modifiers) private static synchronized RuntimeRegex compileSynchronized( String patternString, String modifiers, int lexicalDebugMode, int trustedCalloutCount, boolean literalSyntaxValidation, + boolean patternByteBacked, RuntimeScalar namedCharacterTranslator) { // Debug logging if (DEBUG_REGEX) { @@ -608,6 +620,7 @@ private static synchronized RuntimeRegex compileSynchronized( + "#debug=" + lexicalDebugMode + "#callouts=" + trustedCalloutCount + "#backend=" + RegexBackendPolicy.cacheTag() + + "#bytepattern=" + patternByteBacked + (namedCharacterTranslator == null ? "" : "#charnames=" + namedCharacterTranslator.toString()) + (hasDynamicPattern ? (warnOnUnimplemented ? "\0warn" : "\0defer") : ""); @@ -620,6 +633,7 @@ private static synchronized RuntimeRegex compileSynchronized( System.err.println(" cache miss, compiling new regex"); } regex = new RuntimeRegex(); + regex.patternByteBacked = patternByteBacked; regex.namedCharacterCache = new JoniRegexPattern.NamedCharacterCache(namedCharacterTranslator); regex.lexicalDebugMode = lexicalDebugMode; @@ -687,6 +701,11 @@ private static synchronized RuntimeRegex compileSynchronized( : new JoniRegexPattern(compilePatternString, regex.regexFlags, trustedCalloutCount, false, false, false, regex.namedCharacterCache); + if (patternByteBacked) { + regex.recursivePatternBytes = new JoniRegexPattern( + compilePatternString, regex.regexFlags, trustedCalloutCount, + true, true, true, regex.namedCharacterCache); + } regex.deferredUserDefinedUnicodeProperties = regex.recursivePattern.hasDeferredUserDefinedUnicodeProperty() || regex.recursivePatternUnicode @@ -966,7 +985,8 @@ private static RuntimeRegex ensureCompiledForRuntime(RuntimeRegex regex) { // in sync with compileSynchronized(): leaving the placeholder cached // makes a later qr/\\p{Property}/ reuse its match-any stand-in. state().compiledRegexCache.remove(cacheKey + "#debug=" + regex.lexicalDebugMode - + "#callouts=0#backend=" + RegexBackendPolicy.cacheTag()); + + "#callouts=0#backend=" + RegexBackendPolicy.cacheTag() + + "#bytepattern=" + regex.patternByteBacked); // User property subs can execute arbitrary Perl and block. Resolve them // before compile() takes its process-wide monitor; only simultaneous @@ -975,11 +995,12 @@ private static RuntimeRegex ensureCompiledForRuntime(RuntimeRegex regex) { regex.regexFlags != null && regex.regexFlags.isCaseInsensitive()); RuntimeRegex recompiled = compile(regex.patternString, regex.regexFlags == null ? "" : regex.regexFlags.toFlagString(), - regex.lexicalDebugMode); + regex.lexicalDebugMode, 0, regex.patternByteBacked); regex.pattern = recompiled.pattern; regex.patternUnicode = recompiled.patternUnicode; regex.recursivePattern = recompiled.recursivePattern; regex.recursivePatternUnicode = recompiled.recursivePatternUnicode; + regex.recursivePatternBytes = recompiled.recursivePatternBytes; regex.patternNoInternalMarkers = recompiled.patternNoInternalMarkers; regex.patternUnicodeNoInternalMarkers = recompiled.patternUnicodeNoInternalMarkers; regex.patternFlags = recompiled.patternFlags; @@ -1396,7 +1417,7 @@ && containsExecutableSource(patternString.toString(), modifierStr.indexOf('x') > if (patternString.value instanceof RuntimeRegexTemplate template) { RuntimeRegex regex = compile(template.pattern(), modifierStr, callSiteDebugMode, - template.callbacks().size()).cloneTracked(); + template.callbacks().size(), template.byteBackedPattern()).cloneTracked(); regex.setExecutableCallbacks(template.callbacks()); return new RuntimeScalar(regex).propagateTaint(patternString); } @@ -1416,6 +1437,7 @@ && containsExecutableSource(patternString.toString(), modifierStr.indexOf('x') > regex.patternUnicode = originalRegex.patternUnicode; regex.recursivePattern = originalRegex.recursivePattern; regex.recursivePatternUnicode = originalRegex.recursivePatternUnicode; + regex.recursivePatternBytes = originalRegex.recursivePatternBytes; regex.setExecutableCallbacks(originalRegex.executableCallbacks); regex.patternNoInternalMarkers = originalRegex.patternNoInternalMarkers; regex.patternUnicodeNoInternalMarkers = originalRegex.patternUnicodeNoInternalMarkers; @@ -1459,6 +1481,7 @@ && containsExecutableSource(patternString.toString(), modifierStr.indexOf('x') > regex.patternUnicode = originalRegex.patternUnicode; regex.recursivePattern = originalRegex.recursivePattern; regex.recursivePatternUnicode = originalRegex.recursivePatternUnicode; + regex.recursivePatternBytes = originalRegex.recursivePatternBytes; regex.setExecutableCallbacks(originalRegex.executableCallbacks); regex.patternNoInternalMarkers = originalRegex.patternNoInternalMarkers; regex.patternUnicodeNoInternalMarkers = originalRegex.patternUnicodeNoInternalMarkers; @@ -1496,8 +1519,9 @@ && containsExecutableSource(patternString.toString(), modifierStr.indexOf('x') > // Default: compile as string (cloneTracked() creates a tracked copy // so the cached RuntimeRegex is not corrupted by refCount changes) RuntimeRegex compiled = compile(patternString.toString(), modifierStr, - callSiteDebugMode).cloneTracked(); - compiled.patternByteBacked = patternString.type == RuntimeScalarType.BYTE_STRING; + callSiteDebugMode, 0, + patternString.type == RuntimeScalarType.BYTE_STRING + && containsNonAscii(patternString.toString())).cloneTracked(); return new RuntimeScalar(compiled).propagateTaint(patternString); } @@ -1565,10 +1589,11 @@ static boolean containsExecutableSource(String pattern, boolean extended) { static RuntimeScalar compileExecutableTemplate( String executablePattern, String modifiers, - List callbacks, RuntimeScalar original) { + List callbacks, RuntimeScalar original, + boolean patternByteBacked) { int lexicalDebugMode = debugMode(modifiers); RuntimeRegex regex = compile(executablePattern, stripDebugMarkers(modifiers), - lexicalDebugMode, callbacks.size()).cloneTracked(); + lexicalDebugMode, callbacks.size(), patternByteBacked).cloneTracked(); regex.setExecutableCallbacks(callbacks); return new RuntimeScalar(regex).propagateTaint(original); } @@ -1677,6 +1702,7 @@ public static RuntimeScalar getReplacementRegex(RuntimeScalar patternString, Run regex.patternUnicode = resolvedRegex.patternUnicode; regex.recursivePattern = resolvedRegex.recursivePattern; regex.recursivePatternUnicode = resolvedRegex.recursivePatternUnicode; + regex.recursivePatternBytes = resolvedRegex.recursivePatternBytes; regex.executableCallbacks = resolvedRegex.executableCallbacks; regex.patternNoInternalMarkers = resolvedRegex.patternNoInternalMarkers; regex.patternUnicodeNoInternalMarkers = resolvedRegex.patternUnicodeNoInternalMarkers; @@ -1713,11 +1739,13 @@ public static RuntimeScalar getReplacementRegex(RuntimeScalar patternString, Run if (flagsChanged && !resolved.fromCompiledRegex()) { RuntimeRegex recompiledRegex = compile(resolvedRegex.patternString, newFlags.toFlagString(), regex.lexicalDebugMode, - resolvedRegex.executableCallbacks.size()); + resolvedRegex.executableCallbacks.size(), + resolvedRegex.patternByteBacked); regex.pattern = recompiledRegex.pattern; regex.patternUnicode = recompiledRegex.patternUnicode; regex.recursivePattern = recompiledRegex.recursivePattern; regex.recursivePatternUnicode = recompiledRegex.recursivePatternUnicode; + regex.recursivePatternBytes = recompiledRegex.recursivePatternBytes; regex.executableCallbacks = resolvedRegex.executableCallbacks; regex.patternNoInternalMarkers = recompiledRegex.patternNoInternalMarkers; regex.patternUnicodeNoInternalMarkers = recompiledRegex.patternUnicodeNoInternalMarkers; diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegexSourceCompiler.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegexSourceCompiler.java index 15ccb8e9a1..50b3b0851b 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegexSourceCompiler.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegexSourceCompiler.java @@ -143,7 +143,8 @@ static RuntimeScalar compileTemplate(RuntimeScalar original, RuntimeRegexTemplate template, String modifiers) { RuntimeRegexTemplate.MaskedCallouts masked = template.maskCallouts(); - RuntimeScalar compiled = compile(new RuntimeScalar(masked.pattern()), modifiers); + RuntimeScalar compiled = compile(RuntimeRegexTemplate.patternScalar( + masked.pattern(), template.byteBackedPattern()), modifiers); if (!(compiled.value instanceof RuntimeRegex sourceRegex)) { throw new IllegalStateException("runtime regex source did not compile to qr//"); } @@ -156,7 +157,7 @@ static RuntimeScalar compileTemplate(RuntimeScalar original, List callbacks = new ArrayList<>(template.callbacks()); callbacks.addAll(sourceRegex.executableCallbacks); RuntimeScalar result = RuntimeRegex.compileExecutableTemplate( - executablePattern, modifiers, callbacks, original); + executablePattern, modifiers, callbacks, original, template.byteBackedPattern()); if (result.value != sourceRegex) { sourceRegex.releaseExecutableCallbacks(); } diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegexTemplate.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegexTemplate.java index 8d0e60884c..3b5c6978a0 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegexTemplate.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegexTemplate.java @@ -9,7 +9,9 @@ import org.perlonjava.runtime.runtimetypes.RuntimeList; import org.perlonjava.runtime.runtimetypes.RuntimeScalar; import org.perlonjava.runtime.runtimetypes.RuntimeScalarType; +import org.perlonjava.runtime.perlmodule.Utf8; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; @@ -23,6 +25,7 @@ public final class RuntimeRegexTemplate { "\\(\\?\\{=(CALL|DYNAMIC):(\\d+)\\}\\)"); private final String pattern; private final List callbacks; + private final boolean byteBackedPattern; /** * Deferred array interpolation. Keeping the joined operands separate until @@ -52,9 +55,11 @@ String restore(String compiledPattern) { } } - private RuntimeRegexTemplate(String pattern, List callbacks) { + private RuntimeRegexTemplate(String pattern, List callbacks, + boolean byteBackedPattern) { this.pattern = pattern; this.callbacks = List.copyOf(callbacks); + this.byteBackedPattern = byteBackedPattern; } public static RuntimeScalar build(RuntimeList parts) { @@ -85,9 +90,11 @@ public static RuntimeScalar build(RuntimeList parts) { StringBuilder pattern = new StringBuilder(); List callbacks = new ArrayList<>(); boolean tainted = false; + boolean byteBackedPattern = true; for (RuntimeBase part : parts.elements) { RuntimeScalar scalar = part.scalar(); tainted |= scalar.isTainted(); + byteBackedPattern &= isByteCompatiblePatternPart(scalar); if (scalar.value instanceof RuntimeRegexCallback callback) { int id = callbacks.size(); callbacks.add(callback); @@ -108,12 +115,26 @@ public static RuntimeScalar build(RuntimeList parts) { } } RuntimeScalar result = callbacks.isEmpty() - ? new RuntimeScalar(pattern.toString()) - : new RuntimeScalar(new RuntimeRegexTemplate(pattern.toString(), callbacks)); + ? patternScalar(pattern.toString(), byteBackedPattern) + : new RuntimeScalar(new RuntimeRegexTemplate( + pattern.toString(), callbacks, byteBackedPattern)); result.tainted = tainted; return result; } + private static boolean isByteCompatiblePatternPart(RuntimeScalar scalar) { + if (scalar.value instanceof RuntimeRegexCallback) return true; + if (scalar.value instanceof RuntimeRegex regex) return regex.isPatternByteBacked(); + if (scalar.value instanceof RuntimeRegexTemplate template) return template.byteBackedPattern; + return !Utf8.isUtf8(scalar); + } + + static RuntimeScalar patternScalar(String pattern, boolean byteBackedPattern) { + return byteBackedPattern + ? new RuntimeScalar(pattern.getBytes(StandardCharsets.ISO_8859_1)) + : new RuntimeScalar(pattern); + } + private static RuntimeScalar resolveLoneRegexOverload(RuntimeScalar scalar) { int blessId = RuntimeScalarType.blessedId(scalar); if (blessId >= 0) return null; @@ -285,6 +306,10 @@ List callbacks() { return callbacks; } + boolean byteBackedPattern() { + return byteBackedPattern; + } + /** * Hide parser-created callout markers while runtime-interpolated Perl * source is compiled. The source parser must see raw {@code (?{...})} and diff --git a/src/test/java/org/perlonjava/runtime/regex/RuntimeRegexTemplateProvenanceTest.java b/src/test/java/org/perlonjava/runtime/regex/RuntimeRegexTemplateProvenanceTest.java new file mode 100644 index 0000000000..fce01c6e68 --- /dev/null +++ b/src/test/java/org/perlonjava/runtime/regex/RuntimeRegexTemplateProvenanceTest.java @@ -0,0 +1,34 @@ +package org.perlonjava.runtime.regex; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.perlonjava.runtime.runtimetypes.RuntimeList; +import org.perlonjava.runtime.runtimetypes.RuntimeScalar; +import org.perlonjava.runtime.runtimetypes.RuntimeScalarType; + +@Tag("unit") +class RuntimeRegexTemplateProvenanceTest { + @Test + public void retainsByteProvenanceAcrossByteCompatibleParts() { + RuntimeScalar result = RuntimeRegexTemplate.build(new RuntimeList( + new RuntimeScalar(new byte[] {'^'}), + new RuntimeScalar(new byte[] {(byte) 0xdf}), + new RuntimeScalar(new byte[] {'$'}))); + + assertEquals(RuntimeScalarType.BYTE_STRING, result.type); + assertEquals("^\u00df$", result.toString()); + } + + @Test + public void upgradesWhenAnyPartIsACharacterString() { + RuntimeScalar result = RuntimeRegexTemplate.build(new RuntimeList( + new RuntimeScalar(new byte[] {'^'}), + new RuntimeScalar("\u00df"), + new RuntimeScalar(new byte[] {'$'}))); + + assertEquals(RuntimeScalarType.STRING, result.type); + assertEquals("^\u00df$", result.toString()); + } +} From e701ca8c5053f080f437d415cbbb1c4f31b1018f Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 16:59:24 +0200 Subject: [PATCH 02/16] fix(joni): keep byte /d folds single-character Mark byte-backed Joni patterns explicitly, suppress Unicode multi-character case-fold expansion for them, and retain pinned Latin-1 simple-fold siblings. Add direct and Perl-facing regression coverage for literal and backreference sharp-s behavior. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../runtime/regex/JoniRegexPattern.java | 4 +- .../regex/casefold_literal_backreference.t | 33 +++++++++++++ .../unit/regex/casefold_provenance_modes.t | 32 +++++++++++++ third_party/joni/src/org/joni/Analyser.java | 39 ++++++++++++++- third_party/joni/src/org/joni/Option.java | 9 +++- .../test/TestPerlBytePatternCaseFold.java | 47 +++++++++++++++++++ 6 files changed, 161 insertions(+), 3 deletions(-) create mode 100644 src/test/resources/unit/regex/casefold_literal_backreference.t create mode 100644 src/test/resources/unit/regex/casefold_provenance_modes.t create mode 100644 third_party/joni/test/org/joni/test/TestPerlBytePatternCaseFold.java diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index 739527760a..ee576b6177 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -249,7 +249,9 @@ public boolean supportsPositions() { Syntax syntax = syntaxForNamedCharacters( namedCharacterCache, namedCharacterSourceMode, flags.isCaseInsensitive()); - regex = new Regex(bytes, 0, bytes.length, toJoniOptions(flags, forceAsciiClasses), + int options = toJoniOptions(flags, forceAsciiClasses); + if (byteMode && byteBackedPattern) options |= Option.PERL_BYTE_PATTERN; + regex = new Regex(bytes, 0, bytes.length, options, byteMode ? ISO8859_1Encoding.INSTANCE : UTF8Encoding.INSTANCE, syntax, warningCollector); NamedGroupMaps groupMaps = collectNamedGroups(regex); diff --git a/src/test/resources/unit/regex/casefold_literal_backreference.t b/src/test/resources/unit/regex/casefold_literal_backreference.t new file mode 100644 index 0000000000..5a09e205bd --- /dev/null +++ b/src/test/resources/unit/regex/casefold_literal_backreference.t @@ -0,0 +1,33 @@ +use strict; +use warnings; +use utf8; +use Test::More; + +like('ss', qr/^\x{00DF}$/iu, 'sharp-s forward full fold'); +like("\x{00DF}", qr/^ss$/iu, 'sharp-s reverse full fold'); +like("\x{FB03}", qr/^ffi$/i, 'ligature reverse full fold'); +like("\x{017F}\x{017F}", qr/^\x{00DF}$/i, 'long-s components fold to sharp s'); +like("\x{212A}", qr/^k$/i, 'Kelvin folds with ASCII k'); +unlike('I', qr/^\x{0131}$/i, 'Turkic dotless I remains excluded'); + +like("\x{1E9E}\x{1E9E}", qr/^(\x{00DF})\1$/i, + 'backreference consumes folded non-ASCII siblings'); +unlike('ssss', qr/^(\x{00DF})\1$/iaa, + 'aa prevents ASCII-crossing backreference folds'); +like("\x{1E9E}\x{1E9E}", qr/^(\x{00DF})\1$/iaa, + 'aa retains non-ASCII sibling backreference folds'); + +like('xssy', qr/^x(?i:\x{00DF})y$/u, 'scoped i enables fold locally'); +unlike('xssy', qr/^x(?-i:\x{00DF})y$/i, 'scoped minus i restores outer policy'); + +my $byte_sharp = chr 0xDF; +utf8::downgrade($byte_sharp, 1); +unlike('ss', qr/^$byte_sharp$/di, 'byte d literal blocks full fold'); +my $unicode_sharp = $byte_sharp; +utf8::upgrade($unicode_sharp); +like('ss', qr/^$unicode_sharp$/di, 'upgraded d literal enables full fold'); + +like('affi', qr/^aff\x{0069}$/i, 'positive adjacent reverse-fold boundary'); +unlike('affx', qr/^aff\x{0069}$/i, 'negative adjacent reverse-fold boundary'); + +done_testing; diff --git a/src/test/resources/unit/regex/casefold_provenance_modes.t b/src/test/resources/unit/regex/casefold_provenance_modes.t new file mode 100644 index 0000000000..14a9c862a4 --- /dev/null +++ b/src/test/resources/unit/regex/casefold_provenance_modes.t @@ -0,0 +1,32 @@ +use strict; +use warnings; +use utf8; +use Test::More; + +my $sharp = chr 0xDF; +utf8::downgrade($sharp, 1); +my $ss = 'ss'; +utf8::downgrade($ss, 1); + +unlike($ss, qr/$sharp/di, 'd byte pattern and subject do not full-fold sharp s'); +like($ss, qr/$sharp/ui, 'u permits sharp-s full folding'); +like($ss, qr/$sharp/ai, 'a permits sharp-s full folding'); +unlike($ss, qr/$sharp/aai, 'aa rejects ASCII-crossing sharp-s folding'); + +my $upgraded_sharp = $sharp; +utf8::upgrade($upgraded_sharp); +like($ss, qr/$upgraded_sharp/di, 'd upgraded pattern permits sharp-s full folding'); +my $upgraded_ss = $ss; +utf8::upgrade($upgraded_ss); +like($upgraded_ss, qr/$sharp/di, 'd upgraded subject permits sharp-s full folding'); + +like('I', qr/i/i, 'ordinary I folds with ASCII i'); +unlike('I', qr/\x{0131}/i, 'ordinary folding excludes Turkic dotless i'); +unlike("\x{0130}", qr/i/i, 'ordinary folding excludes Turkic dotted I'); + +like("\x{1E9E}\x{1E9E}", qr/(\x{00DF})\1/iaa, + 'aa backreference preserves non-ASCII sharp-s siblings'); +unlike('ssss', qr/(\x{00DF})\1/iaa, + 'aa backreference rejects captured ASCII-crossing folds'); + +done_testing; diff --git a/third_party/joni/src/org/joni/Analyser.java b/third_party/joni/src/org/joni/Analyser.java index f7768b88f3..c45160f08d 100644 --- a/third_party/joni/src/org/joni/Analyser.java +++ b/third_party/joni/src/org/joni/Analyser.java @@ -2145,6 +2145,41 @@ private Node expandCaseFoldString(Node node) { return xnode; } + private Node expandPerlByteSimpleFoldString(StringNode source, int state) { + ListNode root = null; + boolean expanded = false; + + for (int p = source.p; p < source.end;) { + int next = p + enc.length(source.bytes, p, source.end); + int codePoint = enc.mbcToCode(source.bytes, p, source.end); + StringNode exact = new StringNode(source.bytes, p, next); + exact.setRaw(); + ListNode alternatives = newAlt(exact, null); + ListNode tail = alternatives; + + int foldLength = PerlCaseFold.simpleFoldClassLength(codePoint); + for (int index = 0; index < foldLength; index++) { + int folded = PerlCaseFold.simpleFoldClassCodePoint(codePoint, index); + if (folded == codePoint || folded > 0xff) continue; + StringNode sibling = new StringNode(); + sibling.catCode(folded, enc); + sibling.setRaw(); + ListNode alternative = newAlt(sibling, null); + tail.setTail(alternative); + tail = alternative; + expanded = true; + } + + root = ListNode.listAdd(root, alternatives.tail == null ? exact : alternatives); + p = next; + } + + if (!expanded) return source; + source.replaceWith(root); + setupTree(root, state); + return root; + } + private boolean isUnsafePerlMultiFoldOptimizationBoundary(StringNode node) { if (!syntax.op2OptionPerl() || node.length() == 0) return false; @@ -2344,7 +2379,9 @@ protected final Node setupTree(Node node, int state) { case NodeType.STR: if (isIgnoreCase(regex.options) && !((StringNode)node).isRaw()) { - if (Option.isPerlAsciiStrict(regex.options)) { + if (Option.isPerlBytePattern(regex.options)) { + node = expandPerlByteSimpleFoldString((StringNode)node, state); + } else if (Option.isPerlAsciiStrict(regex.options)) { Node protectedNode = protectPerlAsciiStrictCrossings( (StringNode)node, state); node = protectedNode == node diff --git a/third_party/joni/src/org/joni/Option.java b/third_party/joni/src/org/joni/Option.java index 6e5a9d7742..a4392cd1f0 100644 --- a/third_party/joni/src/org/joni/Option.java +++ b/third_party/joni/src/org/joni/Option.java @@ -48,8 +48,10 @@ public final class Option { public static final int CR_7_BIT = (1 << 18); /** Perl /aa: forbid case-fold crossings between ASCII and non-ASCII. */ public static final int PERL_ASCII_STRICT = (1 << 19); + /** Perl /d byte strings use single-character Latin-1 folding only. */ + public static final int PERL_BYTE_PATTERN = (1 << 20); - public static final int MAXBIT = (1 << 20); /* limit */ + public static final int MAXBIT = (1 << 21); /* limit */ public static final int DEFAULT = NONE; @@ -69,6 +71,7 @@ public static String toString(int option) { if (isPosixRegion(option)) options += "POSIX_REGION"; if (isCR7Bit(option)) options += "CR_7_BIT"; if (isPerlAsciiStrict(option)) options += "PERL_ASCII_STRICT"; + if (isPerlBytePattern(option)) options += "PERL_BYTE_PATTERN"; return options; } @@ -148,6 +151,10 @@ public static boolean isPerlAsciiStrict(int option) { return (option & PERL_ASCII_STRICT) != 0; } + public static boolean isPerlBytePattern(int option) { + return (option & PERL_BYTE_PATTERN) != 0; + } + public static boolean isDynamic(int option) { // Ignore-case and multibyte status are included in compiled code. // Perl ASCII-strict folding also needs a match-time option because it diff --git a/third_party/joni/test/org/joni/test/TestPerlBytePatternCaseFold.java b/third_party/joni/test/org/joni/test/TestPerlBytePatternCaseFold.java new file mode 100644 index 0000000000..40d62f660c --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestPerlBytePatternCaseFold.java @@ -0,0 +1,47 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + */ +package org.joni.test; + +import static org.junit.Assert.assertEquals; + +import java.nio.charset.StandardCharsets; + +import org.jcodings.specific.ISO8859_1Encoding; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.junit.Test; + +public class TestPerlBytePatternCaseFold { + private static int search(String pattern, String input) { + byte[] patternBytes = pattern.getBytes(StandardCharsets.ISO_8859_1); + byte[] inputBytes = input.getBytes(StandardCharsets.ISO_8859_1); + Regex regex = new Regex(patternBytes, 0, patternBytes.length, + Option.IGNORECASE | Option.PERL_BYTE_PATTERN, + ISO8859_1Encoding.INSTANCE, Syntax.PerlNG); + return regex.matcher(inputBytes).search(0, inputBytes.length, Option.NONE); + } + + @Test + public void bytePatternsRejectUnicodeMultiCharacterFolds() { + assertEquals(-1, search("^\u00df$", "ss")); + assertEquals(-1, search("^ss$", "\u00df")); + assertEquals(-1, search("^(\u00df)\\1$", "ssss")); + assertEquals(-1, search("^(ss)\\1$", "\u00df\u00df")); + } + + @Test + public void bytePatternsKeepLatinOneSimpleFolds() { + assertEquals(0, search("^\u00e4$", "\u00c4")); + assertEquals(0, search("^(\u00e4)\\1$", "\u00c4\u00e4")); + } +} From 2810d0c8fff8fd3626a876ed1191b227b1a94989 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 16:59:20 +0200 Subject: [PATCH 03/16] fix(regex): preserve dotted U+ sequence diagnostics Resolve dotted U+ named-character sequences only in regex source while retaining Perl's scalar-string error. Report an unmatched character class before the valid sequence and attach the exact compile-site source location. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../backend/bytecode/CompileOperator.java | 8 ++++- .../frontend/parser/StringSegmentParser.java | 4 +++ .../runtime/NamedCharacterExpansion.java | 13 +++++++ .../runtime/regex/RuntimeRegex.java | 19 +++++++--- .../unicode_named_uplus_sequence_priority.t | 36 +++++++++++++++++++ 5 files changed, 74 insertions(+), 6 deletions(-) create mode 100644 src/test/resources/unit/regex/unicode_named_uplus_sequence_priority.t diff --git a/src/main/java/org/perlonjava/backend/bytecode/CompileOperator.java b/src/main/java/org/perlonjava/backend/bytecode/CompileOperator.java index 9db294d412..a902dffb35 100644 --- a/src/main/java/org/perlonjava/backend/bytecode/CompileOperator.java +++ b/src/main/java/org/perlonjava/backend/bytecode/CompileOperator.java @@ -1128,7 +1128,13 @@ public static void visitOperator(BytecodeCompiler bytecodeCompiler, OperatorNode && !modifiers.contains("u")) { modifiers += "u"; } - RuntimeRegex.validateLiteralSyntax(literalPattern, modifiers); + try { + RuntimeRegex.validateLiteralSyntax(literalPattern, modifiers); + } catch (PerlCompilerException exception) { + throw PerlCompilerException.withSourceLocation( + node.tokenIndex, exception.getMessage(), + bytecodeCompiler.errorUtil); + } } boolean needsCallsiteCache = false; Node flagsNode = operand.elements.get(1); diff --git a/src/main/java/org/perlonjava/frontend/parser/StringSegmentParser.java b/src/main/java/org/perlonjava/frontend/parser/StringSegmentParser.java index 0b2ecef3f3..7686c5479d 100644 --- a/src/main/java/org/perlonjava/frontend/parser/StringSegmentParser.java +++ b/src/main/java/org/perlonjava/frontend/parser/StringSegmentParser.java @@ -1531,6 +1531,10 @@ void handleUnicodeNameEscape() { appendToCurrentSegment("\\N{" + name + "}"); return; } + if (name.matches("(?i)U\\+[0-9A-F]+(?:\\.[0-9A-F]+)+")) { + throwNamedCharacterDiagnostic( + "Invalid hexadecimal number in \\N{U+...}"); + } NamedCharacterExpansion expansion = NamedCharacterExpansion.resolve(name, sourceMode); if (expansion.resolved()) { diff --git a/src/main/java/org/perlonjava/runtime/NamedCharacterExpansion.java b/src/main/java/org/perlonjava/runtime/NamedCharacterExpansion.java index 84d6eabae9..341313b9a9 100644 --- a/src/main/java/org/perlonjava/runtime/NamedCharacterExpansion.java +++ b/src/main/java/org/perlonjava/runtime/NamedCharacterExpansion.java @@ -66,6 +66,19 @@ public static NamedCharacterExpansion resolve( if (name.matches("(?i)U\\+[0-9A-F]+")) { return resolveStandard(name); } + if (name.matches("(?i)U\\+[0-9A-F]+(?:\\.[0-9A-F]+)+")) { + try { + StringBuilder sequence = new StringBuilder(); + for (String scalar : name.substring(2).split("\\.")) { + sequence.appendCodePoint(Integer.parseInt(scalar, 16)); + } + return new NamedCharacterExpansion( + sequence.toString(), SourceMode.UNICODE, + true, Status.RESOLVED, null); + } catch (IllegalArgumentException failure) { + // Fall through to Perl's common malformed-U+ diagnostic. + } + } return new NamedCharacterExpansion( "", SourceMode.UNICODE, true, Status.INVALID, "Invalid hexadecimal number in \\N{U+...}"); diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java index 44804c5bdb..669546a58d 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java @@ -555,8 +555,8 @@ public static void validateLiteralSyntax(String patternString, String modifiers) String message = unsupported.getMessage(); if (message != null && (message.contains("premature end of char-class") || message.contains("Unclosed character class"))) { - throw new PerlCompilerException("Unmatched [ in regex m/" - + patternString + "/"); + throw new PerlCompilerException( + unmatchedCharacterClassDiagnostic(patternString) + "\n"); } if (message != null && (message.contains("Unclosed group") || message.contains("Dangling meta character") @@ -810,11 +810,14 @@ && containsExecutableSource(originalPatternString, if ("Empty \\N{}".equals(message)) { throw new PerlCompilerException("Unknown charname ''"); } - if (literalSyntaxValidation && message != null + if (message != null && (message.contains("premature end of char-class") || message.contains("Unclosed character class"))) { - throw new PerlCompilerException("Unmatched [ in regex m/" - + originalPatternString + "/"); + String diagnostic = unmatchedCharacterClassDiagnostic(originalPatternString); + if (literalSyntaxValidation) { + diagnostic += "\n"; + } + throw new PerlCompilerException(diagnostic); } int bytePosition = ((SyntaxException) e).getPatternPosition(); if (bytePosition != SyntaxException.UNKNOWN_PATTERN_POSITION) { @@ -884,6 +887,12 @@ && containsExecutableSource(originalPatternString, return regex; } + private static String unmatchedCharacterClassDiagnostic(String pattern) { + int open = pattern == null ? -1 : pattern.indexOf('['); + return RegexDiagnosticFormatter.markedPerl( + pattern, open < 0 ? 0 : open + 1, "Unmatched ["); + } + private static String invalidUnicodePropertyName(String message) { if (message == null) return null; String prefix = "invalid character property name <"; diff --git a/src/test/resources/unit/regex/unicode_named_uplus_sequence_priority.t b/src/test/resources/unit/regex/unicode_named_uplus_sequence_priority.t new file mode 100644 index 0000000000..14390f475c --- /dev/null +++ b/src/test/resources/unit/regex/unicode_named_uplus_sequence_priority.t @@ -0,0 +1,36 @@ +use strict; +use warnings; +use Test::More; + +my @warnings; +my $value; +{ + local $SIG{__WARN__} = sub { push @warnings, join '', @_ }; + $value = eval "#line 1 unicode_named_uplus_sequence_priority.t\n" + . q!"\N{U+41.42}"!; +} +my ($scalar_error) = split /\n/, $@; +is($scalar_error, + 'Invalid hexadecimal number in \N{U+...} at unicode_named_uplus_sequence_priority.t line 1, within string', + 'dotted U+ form remains invalid in a string'); +is(scalar @warnings, 0, 'invalid string form has no preceding warning'); + +my $regex = eval "#line 1 unicode_named_uplus_sequence_priority.t\n" + . q!qr/\N{U+41.42}/!; +is($@, '', 'dotted U+ sequence is legal in a regex'); +is("$regex", q!(?^:\N{U+41.42})!, 'qr stringification preserves dotted U+ source'); +ok('AB' =~ $regex, 'dotted U+ sequence matches its code points'); + +$regex = eval "#line 1 unicode_named_uplus_sequence_priority.t\n" + . q!qr/[\N{U+0.00}]/!; +is($@, '', 'dotted U+ sequence is legal in a closed class'); +is("$regex", q!(?^:[\N{U+0.00}])!, 'closed-class stringification preserves source'); + +eval "#line 1 unicode_named_uplus_sequence_priority.t\n" + . q!qr/0000000000000000[\N{U+0.00}0000/!; +my ($priority_error) = split /\n/, $@; +is($priority_error, + 'Unmatched [ in regex; marked by <-- HERE in m/0000000000000000[ <-- HERE \N{U+0.00}0000/ at unicode_named_uplus_sequence_priority.t line 1.', + 'unmatched class diagnostic precedes valid dotted U+ resolution'); + +done_testing; From bcf94e811878f3a3df61e079d8a965d8699c845d Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 17:07:03 +0200 Subject: [PATCH 04/16] test(regex): cover ordinary matcher parity Add a system-Perl-validated matrix for ordinary constant patterns across captures, classes, assertions, modifiers, reuse, substitution, and match state. This freezes explicit Java and Joni behavior before changing the default route. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../regex/ordinary_constant_joni_matrix.t | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 src/test/resources/unit/regex/ordinary_constant_joni_matrix.t diff --git a/src/test/resources/unit/regex/ordinary_constant_joni_matrix.t b/src/test/resources/unit/regex/ordinary_constant_joni_matrix.t new file mode 100644 index 0000000000..d23df12030 --- /dev/null +++ b/src/test/resources/unit/regex/ordinary_constant_joni_matrix.t @@ -0,0 +1,65 @@ +use strict; +use warnings; +use Test::More; + +ok('alphabet' =~ /pha/, 'literal'); +ok('green' =~ /red|green|blue/, 'alternation'); + +my $captured = 'name=perl'; +ok($captured =~ /name=(?\w+)/, 'numbered and named capture match'); +is($1, 'perl', 'numbered capture value'); +is($+{language}, 'perl', 'named capture value'); + +ok("A\x{3b1}9" =~ /[A-Z]\p{Greek}\d/, 'class and Unicode property'); +ok("first\nsecond" =~ /^second$/m, 'multiline anchors'); +ok("a\nlevel\nz" =~ /a.*z/s, 'dot-all modifier'); +ok('item42' =~ /item\d{2}/, 'bounded quantifier'); +ok('colour' =~ /colou?r/, 'optional quantifier'); + +ok('foobar' =~ /foo(?=bar)/, 'positive lookahead'); +ok('foobar' =~ /(?<=foo)bar/, 'fixed positive lookbehind'); +ok('foobaz' =~ /foo(?!bar)/, 'negative lookahead'); + +my $substitution = 'red green red'; +my $replacements = ($substitution =~ s/red/blue/g); +is($replacements, 2, 'global substitution count'); +is($substitution, 'blue green blue', 'global substitution result'); + +my $word = qr/[a-z]+/i; +ok('Perl' =~ $word, 'qr reuse'); +my $suffix = qr/\d+/; +ok('item42' =~ /item$suffix/, 'qr interpolation'); + +my @global = ('a1b22c333' =~ /(\d+)/g); +is_deeply(\@global, [qw(1 22 333)], 'global match list'); + +my $continued = '12ab'; +pos($continued) = 0; +ok($continued =~ /\d+/gc, '/gc first match'); +is(pos($continued), 2, '/gc advances pos'); +ok(!($continued =~ /\d+/gc), '/gc failed continuation'); +is(pos($continued), 2, '/c preserves pos after failure'); + +my $once_text = 'first'; +my $once_pattern = 'first'; +sub once_matches { + my ($candidate) = @_; + return $candidate =~ /$once_pattern/o; +} +ok(once_matches($once_text), '/o initial compilation'); +$once_pattern = 'second'; +ok(once_matches('first'), '/o reuses the initial pattern'); + +my $byte = pack('C', 0xe9); +utf8::downgrade($byte, 1); +my $unicode = "\x{e9}"; +utf8::upgrade($unicode); +ok($byte =~ /\x{e9}/, 'byte scalar match'); +ok($unicode =~ /\x{e9}/u, 'Unicode scalar match'); + +ok('MiXeD' =~ /mixed/i, 'case-insensitive modifier'); +ok('a b' =~ /a \s+ b/x, 'extended modifier'); +ok('ab' =~ /(?:a)(b)/n, 'non-capturing-default modifier'); +is($1, undef, '/n suppresses unnamed captures'); + +done_testing; From 606228c22ef3bce37253dd344bf8fc9cc2da3ac3 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 17:18:48 +0200 Subject: [PATCH 05/16] feat(regex): make Joni the ordinary default Route default and auto regex compilation through Joni while retaining explicit Java as a temporary differential baseline. Update the cache mode identity and the obsolete policy expectations to match the permanent production route. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- dev/implementation/regex.md | 10 ++++----- .../runtime/regex/RegexBackendPolicy.java | 21 +++++++++---------- .../runtime/regex/RegexBackendPolicyTest.java | 14 ++++++------- 3 files changed, 22 insertions(+), 23 deletions(-) diff --git a/dev/implementation/regex.md b/dev/implementation/regex.md index bd8f4f6adf..9f0317c910 100644 --- a/dev/implementation/regex.md +++ b/dev/implementation/regex.md @@ -8,11 +8,11 @@ backtracking, captures, conditions, recursion, control verbs, Unicode matching, case folding, and matcher-visible callbacks. PerlOnJava owns Perl source policy, runtime integration, lexical warnings, diagnostics, and callback closures. -Migration is not complete. Automatic routing still uses Java for ordinary -patterns unless a Joni-only construct is present. Setting -`JPERL_REGEX_BACKEND=joni` or the `jperl.regex.backend=joni` system property -forces Joni and is the compatibility gate used by the unit corpus. The Java -matcher, selector, and Java-only rewrites are temporary migration scaffolding. +Migration is not complete. Automatic routing uses Joni for ordinary patterns. +Setting `JPERL_REGEX_BACKEND=java` or the `jperl.regex.backend=java` system +property retains the legacy matcher solely as a differential baseline; explicit +`joni` selects the production route. The Java matcher, selector, and Java-only +rewrites are temporary migration scaffolding. ## Compilation and routing diff --git a/src/main/java/org/perlonjava/runtime/regex/RegexBackendPolicy.java b/src/main/java/org/perlonjava/runtime/regex/RegexBackendPolicy.java index fd4121b5d8..d341c3c42d 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RegexBackendPolicy.java +++ b/src/main/java/org/perlonjava/runtime/regex/RegexBackendPolicy.java @@ -1,12 +1,11 @@ package org.perlonjava.runtime.regex; /** - * Temporary migration policy for comparing the legacy Java-first routing with - * the canonical Joni matcher. Ordinary lookbehind also remains on Java until - * Joni's nested-lookahead admission is complete. Branch-reset subroutine calls - * also temporarily use Java pending Joni's native named-call patch. Other - * Joni-only constructs in the same pattern still force Joni. This class and its - * controls are removed when the Java matching backend is retired. + * Temporary migration policy for comparing the canonical Joni matcher with + * the legacy Java matcher. Default and auto modes use Joni; explicit Java mode + * remains only for differential diagnosis. Constructs unavailable in Java may + * still force Joni even in explicit Java mode. This class and its controls are + * removed when the Java matching backend is retired. */ final class RegexBackendPolicy { static final String PROPERTY = "jperl.regex.backend"; @@ -27,14 +26,14 @@ static Mode current() { } if (configured == null || configured.isBlank() || configured.equalsIgnoreCase("auto") - || configured.equalsIgnoreCase("java")) { - return Mode.JAVA; - } - if (configured.equalsIgnoreCase("joni")) { + || configured.equalsIgnoreCase("joni")) { return Mode.JONI; } + if (configured.equalsIgnoreCase("java")) { + return Mode.JAVA; + } throw new IllegalArgumentException("Invalid " + ENVIRONMENT + " value '" - + configured + "' (expected java or joni)"); + + configured + "' (expected auto, java, or joni)"); } static boolean useJoni(String pattern) { diff --git a/src/test/java/org/perlonjava/runtime/regex/RegexBackendPolicyTest.java b/src/test/java/org/perlonjava/runtime/regex/RegexBackendPolicyTest.java index f856366382..f8a03c6ed6 100644 --- a/src/test/java/org/perlonjava/runtime/regex/RegexBackendPolicyTest.java +++ b/src/test/java/org/perlonjava/runtime/regex/RegexBackendPolicyTest.java @@ -29,20 +29,20 @@ void restoreBackendProperty() { } @Test - void defaultModeRoutesOrdinaryLookbehindToJoni() { - assertFalse(RegexBackendPolicy.useJoni("ordinary")); + void defaultModeUsesJoniForOrdinaryPatterns() { + assertTrue(RegexBackendPolicy.useJoni("ordinary")); assertTrue(RegexBackendPolicy.useJoni("(?<=x)y")); assertTrue(RegexBackendPolicy.useJoni("(?1)|(?2))(?&digit)")); From e1ffa40bf6c281bca41053833b35ab226c1aa268 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 17:19:29 +0200 Subject: [PATCH 06/16] fix(joni): align byte and property case-fold policies Preserve ASCII byte-pattern provenance for reverse folds, restrict byte /d folding to ASCII, and retain direct Unicode property membership under /aa property closure. Add a compact system-Perl-derived matrix across modes and backreference forms. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../runtime/regex/RuntimeRegex.java | 3 +- .../unit/regex/casefold_generated_matrix.t | 54 +++++++++++++++++++ third_party/joni/src/org/joni/Analyser.java | 33 +++++++----- .../joni/src/org/joni/ApplyCaseFold.java | 2 +- .../joni/src/org/joni/ApplyCaseFoldArg.java | 7 +++ third_party/joni/src/org/joni/Parser.java | 10 +++- .../test/TestPerlBytePatternCaseFold.java | 18 +++++-- 7 files changed, 105 insertions(+), 22 deletions(-) create mode 100644 src/test/resources/unit/regex/casefold_generated_matrix.t diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java index 669546a58d..d01cfe74d7 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java @@ -1529,8 +1529,7 @@ && containsExecutableSource(patternString.toString(), modifierStr.indexOf('x') > // so the cached RuntimeRegex is not corrupted by refCount changes) RuntimeRegex compiled = compile(patternString.toString(), modifierStr, callSiteDebugMode, 0, - patternString.type == RuntimeScalarType.BYTE_STRING - && containsNonAscii(patternString.toString())).cloneTracked(); + patternString.type == RuntimeScalarType.BYTE_STRING).cloneTracked(); return new RuntimeScalar(compiled).propagateTaint(patternString); } diff --git a/src/test/resources/unit/regex/casefold_generated_matrix.t b/src/test/resources/unit/regex/casefold_generated_matrix.t new file mode 100644 index 0000000000..bf7038b633 --- /dev/null +++ b/src/test/resources/unit/regex/casefold_generated_matrix.t @@ -0,0 +1,54 @@ +use strict; +use warnings; +use utf8; +use Test::More; + +my $byte_sharp = chr 0xDF; +utf8::downgrade($byte_sharp, 1); +my $byte_pair = $byte_sharp . $byte_sharp; +utf8::downgrade($byte_pair, 1); +my $byte_a_umlaut = chr 0xE4; +my $byte_A_umlaut = chr 0xC4; +utf8::downgrade($byte_a_umlaut, 1); +utf8::downgrade($byte_A_umlaut, 1); +my $unicode_sharp = chr 0xDF; +utf8::upgrade($unicode_sharp); + +unlike('ss', qr/^$byte_sharp$/di, + 'byte /d literal rejects sharp-s full fold'); +unlike($byte_sharp, qr/^ss$/di, + 'byte /d reverse literal rejects sharp-s full fold'); +unlike($byte_A_umlaut, qr/^$byte_a_umlaut$/di, + 'byte /d rejects Latin-1 character folding'); +like($byte_pair, qr/^($byte_sharp)\1$/di, + 'byte /d numbered backreference retains byte capture'); +like($byte_pair, qr/^(?$byte_sharp)\k$/di, + 'byte /d named backreference retains byte capture'); + +like('ss', qr/^$unicode_sharp$/di, + 'character /d literal retains sharp-s full fold'); +like($unicode_sharp, qr/^ss$/di, + 'character /d reverse literal retains sharp-s full fold'); +like('ss', qr/^\x{DF}$/ui, + '/u retains sharp-s full fold'); +like('ss', qr/^\x{DF}$/ai, + '/a retains sharp-s full fold'); +unlike('ss', qr/^\x{DF}$/aai, + '/aa rejects sharp-s ASCII crossing'); + +like('ss', qr/^(?i:$unicode_sharp)$/, + 'scoped /i enables full fold'); +unlike('ss', qr/^(?aa-i:\x{DF})$/i, + 'scoped minus i disables outer folding'); +unlike('ss', qr/^(?aa:\x{DF})$/i, + 'scoped /aa blocks outer full fold'); + +my $kelvin = "\N{KELVIN SIGN}"; +like($kelvin, qr/^\p{Lowercase}$/i, + 'ignore-case property closure includes Kelvin sign'); +like($kelvin, qr/^\p{Lowercase}$/aai, + 'ascii-strict property closure retains Unicode property membership'); +like('A', qr/^[\p{Lowercase}]$/i, + 'ignore-case class property closure includes uppercase sibling'); + +done_testing; diff --git a/third_party/joni/src/org/joni/Analyser.java b/third_party/joni/src/org/joni/Analyser.java index c45160f08d..f3e4ae2f1d 100644 --- a/third_party/joni/src/org/joni/Analyser.java +++ b/third_party/joni/src/org/joni/Analyser.java @@ -2145,7 +2145,7 @@ private Node expandCaseFoldString(Node node) { return xnode; } - private Node expandPerlByteSimpleFoldString(StringNode source, int state) { + private Node expandPerlByteAsciiFoldString(StringNode source, int state) { ListNode root = null; boolean expanded = false; @@ -2157,24 +2157,29 @@ private Node expandPerlByteSimpleFoldString(StringNode source, int state) { ListNode alternatives = newAlt(exact, null); ListNode tail = alternatives; - int foldLength = PerlCaseFold.simpleFoldClassLength(codePoint); - for (int index = 0; index < foldLength; index++) { - int folded = PerlCaseFold.simpleFoldClassCodePoint(codePoint, index); - if (folded == codePoint || folded > 0xff) continue; - StringNode sibling = new StringNode(); - sibling.catCode(folded, enc); - sibling.setRaw(); - ListNode alternative = newAlt(sibling, null); - tail.setTail(alternative); - tail = alternative; - expanded = true; + if (Encoding.isAscii(codePoint)) { + int foldLength = PerlCaseFold.simpleFoldClassLength(codePoint); + for (int index = 0; index < foldLength; index++) { + int folded = PerlCaseFold.simpleFoldClassCodePoint(codePoint, index); + if (folded == codePoint || !Encoding.isAscii(folded)) continue; + StringNode sibling = new StringNode(); + sibling.catCode(folded, enc); + sibling.setRaw(); + ListNode alternative = newAlt(sibling, null); + tail.setTail(alternative); + tail = alternative; + expanded = true; + } } root = ListNode.listAdd(root, alternatives.tail == null ? exact : alternatives); p = next; } - if (!expanded) return source; + if (!expanded) { + source.setRaw(); + return source; + } source.replaceWith(root); setupTree(root, state); return root; @@ -2380,7 +2385,7 @@ protected final Node setupTree(Node node, int state) { case NodeType.STR: if (isIgnoreCase(regex.options) && !((StringNode)node).isRaw()) { if (Option.isPerlBytePattern(regex.options)) { - node = expandPerlByteSimpleFoldString((StringNode)node, state); + node = expandPerlByteAsciiFoldString((StringNode)node, state); } else if (Option.isPerlAsciiStrict(regex.options)) { Node protectedNode = protectPerlAsciiStrictCrossings( (StringNode)node, state); diff --git a/third_party/joni/src/org/joni/ApplyCaseFold.java b/third_party/joni/src/org/joni/ApplyCaseFold.java index 557cc018b9..e5a250664e 100644 --- a/third_party/joni/src/org/joni/ApplyCaseFold.java +++ b/third_party/joni/src/org/joni/ApplyCaseFold.java @@ -40,7 +40,7 @@ public void apply(int from, int[]to, int length, Object o) { BitSet bs = cc.bs; boolean addFlag; - if (Option.isPerlAsciiStrict(env.option) + if (!arg.preservePropertyAsciiCrossings && Option.isPerlAsciiStrict(env.option) && perlAsciiStrictRelationCrossesAscii(from, to, length)) { return; } diff --git a/third_party/joni/src/org/joni/ApplyCaseFoldArg.java b/third_party/joni/src/org/joni/ApplyCaseFoldArg.java index c3de87823f..54ad870ef5 100644 --- a/third_party/joni/src/org/joni/ApplyCaseFoldArg.java +++ b/third_party/joni/src/org/joni/ApplyCaseFoldArg.java @@ -25,14 +25,21 @@ final class ApplyCaseFoldArg { final ScanEnvironment env; final CClassNode cc, ascCc, foldCc; + final boolean preservePropertyAsciiCrossings; ListNode altRoot; ListNode tail; ApplyCaseFoldArg(ScanEnvironment env, CClassNode cc, CClassNode ascCc, CClassNode foldCc) { + this(env, cc, ascCc, foldCc, false); + } + + ApplyCaseFoldArg(ScanEnvironment env, CClassNode cc, CClassNode ascCc, + CClassNode foldCc, boolean preservePropertyAsciiCrossings) { this.env = env; this.cc = cc; this.ascCc = ascCc; this.foldCc = foldCc; + this.preservePropertyAsciiCrossings = preservePropertyAsciiCrossings; } } diff --git a/third_party/joni/src/org/joni/Parser.java b/third_party/joni/src/org/joni/Parser.java index 6d03aaf61b..b7a37e523a 100644 --- a/third_party/joni/src/org/joni/Parser.java +++ b/third_party/joni/src/org/joni/Parser.java @@ -2230,7 +2230,13 @@ private Node parseCharType(Node node) { private Node cClassCaseFold(Node node, CClassNode cc, CClassNode ascCc, CClassNode foldCc) { - ApplyCaseFoldArg arg = new ApplyCaseFoldArg(env, cc, ascCc, foldCc); + return cClassCaseFold(node, cc, ascCc, foldCc, false); + } + + private Node cClassCaseFold(Node node, CClassNode cc, CClassNode ascCc, + CClassNode foldCc, boolean preservePropertyAsciiCrossings) { + ApplyCaseFoldArg arg = new ApplyCaseFoldArg( + env, cc, ascCc, foldCc, preservePropertyAsciiCrossings); enc.applyAllCaseFold(env.caseFoldFlagFor(env.option), ApplyCaseFold.INSTANCE, arg); if (syntax.op2OptionPerl()) { ApplyCaseFold.applyPerlSimpleClassClosure(arg); @@ -2258,7 +2264,7 @@ private Node parseCharProperty() { if (isIgnoreCase(env.option) && property.caseFold) { if (property.ranges != null || property.wideRanges != null || property.ctype != CharacterType.ASCII) { - node = cClassCaseFold(node, cc, cc, cc); + node = cClassCaseFold(node, cc, cc, cc, true); } } return node; diff --git a/third_party/joni/test/org/joni/test/TestPerlBytePatternCaseFold.java b/third_party/joni/test/org/joni/test/TestPerlBytePatternCaseFold.java index 40d62f660c..0d72ce5b84 100644 --- a/third_party/joni/test/org/joni/test/TestPerlBytePatternCaseFold.java +++ b/third_party/joni/test/org/joni/test/TestPerlBytePatternCaseFold.java @@ -16,6 +16,7 @@ import java.nio.charset.StandardCharsets; import org.jcodings.specific.ISO8859_1Encoding; +import org.jcodings.specific.UTF8Encoding; import org.joni.Option; import org.joni.Regex; import org.joni.Syntax; @@ -40,8 +41,19 @@ public void bytePatternsRejectUnicodeMultiCharacterFolds() { } @Test - public void bytePatternsKeepLatinOneSimpleFolds() { - assertEquals(0, search("^\u00e4$", "\u00c4")); - assertEquals(0, search("^(\u00e4)\\1$", "\u00c4\u00e4")); + public void bytePatternsKeepAsciiButRejectLatinOneFolds() { + assertEquals(0, search("^a$", "A")); + assertEquals(-1, search("^\u00e4$", "\u00c4")); + assertEquals(-1, search("^(\u00e4)\\1$", "\u00c4\u00e4")); + } + + @Test + public void asciiStrictPropertyClosureRetainsUnicodePropertyMembership() { + byte[] pattern = "^\\p{Lowercase}$".getBytes(StandardCharsets.UTF_8); + byte[] kelvin = "\u212a".getBytes(StandardCharsets.UTF_8); + Regex regex = new Regex(pattern, 0, pattern.length, + Option.IGNORECASE | Option.PERL_ASCII_STRICT, + UTF8Encoding.INSTANCE, Syntax.PerlNG); + assertEquals(0, regex.matcher(kelvin).search(0, kelvin.length, Option.NONE)); } } From 679fdc496f3e5660ec2e56f054f6ac48f165c804 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 17:25:04 +0200 Subject: [PATCH 07/16] fix(regex): limit byte matcher variants to ignore-case patterns Avoid changing callback global-match state for byte-backed patterns that have no case-folding semantics, while retaining byte /d selection for /i patterns. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java index d01cfe74d7..fcf230c6e0 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java @@ -320,7 +320,8 @@ public RegexMatcher matcher(RuntimeScalar string, String input) { private JoniRegexPattern selectRecursivePattern(RuntimeScalar string) { boolean byteDefaultSemantics = patternByteBacked && regexFlags != null - && !regexFlags.isUnicode() && !regexFlags.isAscii(); + && regexFlags.isCaseInsensitive() && !regexFlags.isUnicode() + && !regexFlags.isAscii(); if ((bytesSubstitution || byteDefaultSemantics) && recursivePatternBytes != null && !Utf8.isUtf8(string)) { return recursivePatternBytes; @@ -701,7 +702,7 @@ private static synchronized RuntimeRegex compileSynchronized( : new JoniRegexPattern(compilePatternString, regex.regexFlags, trustedCalloutCount, false, false, false, regex.namedCharacterCache); - if (patternByteBacked) { + if (patternByteBacked && regex.regexFlags.isCaseInsensitive()) { regex.recursivePatternBytes = new JoniRegexPattern( compilePatternString, regex.regexFlags, trustedCalloutCount, true, true, true, regex.namedCharacterCache); From c5faf8940ac857c90c0c19c985a6f3cec90a6410 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 17:34:14 +0200 Subject: [PATCH 08/16] test(regex): freeze dynamic pattern edge contract Add a system-Perl-derived contract for undefined, empty, quantified, modifier, position, callback-result, exception, and invalid-source dynamic patterns. Render an invalid returned character class with Perl's marked diagnostic. Generated with Codex (https://openai.com/codex) Co-Authored-By: OpenAI Codex --- .../runtime/regex/JoniRegexPattern.java | 14 +++++- .../unit/regex/dynamic_contract_edges.t | 46 +++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 src/test/resources/unit/regex/dynamic_contract_edges.t diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index ee576b6177..27fdd3c19f 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -17,6 +17,7 @@ import org.joni.Syntax; import org.joni.WarnCallback; import org.joni.WideScalarCodec; +import org.joni.exception.SyntaxException; import static org.joni.constants.SyntaxProperties.ALLOW_MULTIPLEX_DEFINITION_NAME_CALL; import static org.joni.constants.SyntaxProperties.OP2_ESC_H_HORIZONTAL_WHITESPACE; @@ -1376,7 +1377,18 @@ public DynamicPatternResult executeDynamic(int id, MatchView match) { runtimeRegex.executableCallbacks.size()); nestedCallbacks = runtimeRegex.executableCallbacks; } else { - nestedPattern = new JoniRegexPattern(dynamicSource, outerFlags); + try { + nestedPattern = new JoniRegexPattern(dynamicSource, outerFlags); + } catch (SyntaxException exception) { + String message = exception.getMessage(); + if (message != null && (message.contains("premature end of char-class") + || message.contains("Unclosed character class"))) { + int open = dynamicSource.indexOf('['); + throw new PerlCompilerException(RegexDiagnosticFormatter.markedPerl( + dynamicSource, open < 0 ? 0 : open + 1, "Unmatched [")); + } + throw exception; + } } } CalloutHandler nestedHandler = nestedCallbacks.isEmpty() ? null diff --git a/src/test/resources/unit/regex/dynamic_contract_edges.t b/src/test/resources/unit/regex/dynamic_contract_edges.t new file mode 100644 index 0000000000..59b01c57f4 --- /dev/null +++ b/src/test/resources/unit/regex/dynamic_contract_edges.t @@ -0,0 +1,46 @@ +use strict; +use warnings; +use Test::More; + +my @warnings; +my $undef_matches; +{ + local $SIG{__WARN__} = sub { push @warnings, @_ }; + $undef_matches = 'a' =~ /^a(??{ undef })$/; +} +ok($undef_matches, 'undef dynamic result is an empty pattern'); +like(join('', @warnings), qr/^Use of uninitialized value/, 'undef warns at match time'); + +ok('a' =~ /^a(??{ '' })$/, 'empty dynamic result matches empty'); + +my $position; +ok('ab' =~ /^a(??{ $position = pos; 'b' })$/, 'dynamic result matches at current position'); +is($position, 1, 'pos is the current dynamic entry offset'); + +my $repeat_count = 0; +ok('aa' =~ /^(?:(??{ ++$repeat_count; 'a' })){2}$/, + 'dynamic result executes inside a quantifier'); +is($repeat_count, 2, 'dynamic expression executes for each quantifier entry'); + +my $choice_count = 0; +ok('ab' =~ /^(??{ ++$choice_count; 'ab|a' })b$/, + 'nested alternatives can backtrack before the outer suffix'); +is($choice_count, 1, 'nested alternative backtracking does not reevaluate expression'); + +ok('a' =~ /^(??{ 'A' })$/i, 'returned string inherits outer modifiers'); +my $strict_qr = qr/A/; +ok(!('a' =~ /^(??{ $strict_qr })$/i), 'returned qr retains its own modifiers'); + +our $seed; +ok('seed' =~ /^(?{ $seed = 'seed' })(??{ $^R })$/, + 'dynamic expression sees prior callback result'); + +my $error = eval { 'x' =~ /^(??{ die "dynamic boom\n" })$/; 1 }; +ok(!$error, 'exception aborts matching'); +is($@, "dynamic boom\n", 'dynamic exception propagates unchanged'); + +my $invalid = eval { 'x' =~ /^(??{ '[' })$/; 1 }; +ok(!$invalid, 'invalid returned pattern fails at match time'); +like($@, qr/Unmatched \[/, 'invalid returned pattern reports compile error'); + +done_testing; From 73e14257b87172337cc2802ac9196e50575e5d75 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 17:50:23 +0200 Subject: [PATCH 09/16] fix(joni): retain all byte fold string segments Keep a stable sequence head while expanding multi-character ASCII byte-pattern folds. ListNode.listAdd returns the appended tail, so assigning it back to the root discarded every preceding literal segment. Add direct exact, folded, and negative anchored coverage. Generated with Codex (https://openai.com/codex) Co-Authored-By: OpenAI Codex --- third_party/joni/src/org/joni/Analyser.java | 7 ++++++- .../test/org/joni/test/TestPerlBytePatternCaseFold.java | 7 +++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/third_party/joni/src/org/joni/Analyser.java b/third_party/joni/src/org/joni/Analyser.java index f3e4ae2f1d..924e835c36 100644 --- a/third_party/joni/src/org/joni/Analyser.java +++ b/third_party/joni/src/org/joni/Analyser.java @@ -2147,6 +2147,7 @@ private Node expandCaseFoldString(Node node) { private Node expandPerlByteAsciiFoldString(StringNode source, int state) { ListNode root = null; + ListNode sequenceTail = null; boolean expanded = false; for (int p = source.p; p < source.end;) { @@ -2172,7 +2173,11 @@ private Node expandPerlByteAsciiFoldString(StringNode source, int state) { } } - root = ListNode.listAdd(root, alternatives.tail == null ? exact : alternatives); + Node part = alternatives.tail == null ? exact : alternatives; + ListNode segment = ListNode.newList(part, null); + if (root == null) root = segment; + else sequenceTail.setTail(segment); + sequenceTail = segment; p = next; } diff --git a/third_party/joni/test/org/joni/test/TestPerlBytePatternCaseFold.java b/third_party/joni/test/org/joni/test/TestPerlBytePatternCaseFold.java index 0d72ce5b84..b3886985f9 100644 --- a/third_party/joni/test/org/joni/test/TestPerlBytePatternCaseFold.java +++ b/third_party/joni/test/org/joni/test/TestPerlBytePatternCaseFold.java @@ -47,6 +47,13 @@ public void bytePatternsKeepAsciiButRejectLatinOneFolds() { assertEquals(-1, search("^(\u00e4)\\1$", "\u00c4\u00e4")); } + @Test + public void bytePatternsRetainEverySegmentOfAsciiStrings() { + assertEquals(0, search("^dbi:$", "dbi:")); + assertEquals(0, search("^dbi:$", "DBI:")); + assertEquals(-1, search("^dbi:$", "dbx:")); + } + @Test public void asciiStrictPropertyClosureRetainsUnicodePropertyMembership() { byte[] pattern = "^\\p{Lowercase}$".getBytes(StandardCharsets.UTF_8); From 8b00c7019bf628747e1d4e161a79c82e95315bc4 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 17:27:28 +0200 Subject: [PATCH 10/16] chore(import-perl5): drop redundant pat.t import The bulk perl5/t import already restores re/pat.t byte-for-byte now that the historical workaround patch is retired. Remove the duplicate file row after verifying two-pass sync idempotence and the unpatched upstream Joni gates. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- dev/import-perl5/config.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/dev/import-perl5/config.yaml b/dev/import-perl5/config.yaml index 60ecd866e4..f45961dd22 100644 --- a/dev/import-perl5/config.yaml +++ b/dev/import-perl5/config.yaml @@ -144,9 +144,6 @@ imports: target: perl5_t/t/test.pl patch: test.pl.patch - - source: perl5/t/re/pat.t - target: perl5_t/t/re/pat.t - - source: perl5/t/porting/manifest.t target: perl5_t/t/porting/manifest.t patch: manifest.t.patch From faefa75fb9fa7dd874af0a9e04a8709765e7aa4c Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 18:01:13 +0200 Subject: [PATCH 11/16] fix(regex): build byte fold variants only for default mode Align byte-pattern variant construction with selection. Unicode, ASCII, and ASCII-strict modes never select the ISO byte variant, so compiling it eagerly caused property and class fold failures before the intended matcher was used. Generated with Codex (https://openai.com/codex) Co-Authored-By: OpenAI Codex --- src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java index fcf230c6e0..0600df8ef3 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java @@ -702,7 +702,8 @@ private static synchronized RuntimeRegex compileSynchronized( : new JoniRegexPattern(compilePatternString, regex.regexFlags, trustedCalloutCount, false, false, false, regex.namedCharacterCache); - if (patternByteBacked && regex.regexFlags.isCaseInsensitive()) { + if (patternByteBacked && regex.regexFlags.isCaseInsensitive() + && !regex.regexFlags.isUnicode() && !regex.regexFlags.isAscii()) { regex.recursivePatternBytes = new JoniRegexPattern( compilePatternString, regex.regexFlags, trustedCalloutCount, true, true, true, regex.namedCharacterCache); From 3e51a63891472142825760db7824ded56bd42cdb Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 18:08:38 +0200 Subject: [PATCH 12/16] fix(regex): keep Unicode-promoting source out of byte variants Do not build the default ISO byte matcher for named characters, wide numeric escapes, or literal wide code points. These constructs promote otherwise ASCII source to Unicode semantics and byte-variant compilation duplicated lexical translator calls or admitted excluded Turkic folds. Generated with Codex (https://openai.com/codex) Co-Authored-By: OpenAI Codex --- .../runtime/regex/RuntimeRegex.java | 51 ++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java index 0600df8ef3..07d43dbea6 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java @@ -703,7 +703,8 @@ private static synchronized RuntimeRegex compileSynchronized( regex.regexFlags, trustedCalloutCount, false, false, false, regex.namedCharacterCache); if (patternByteBacked && regex.regexFlags.isCaseInsensitive() - && !regex.regexFlags.isUnicode() && !regex.regexFlags.isAscii()) { + && !regex.regexFlags.isUnicode() && !regex.regexFlags.isAscii() + && !hasUnicodePromotingPatternSyntax(compilePatternString)) { regex.recursivePatternBytes = new JoniRegexPattern( compilePatternString, regex.regexFlags, trustedCalloutCount, true, true, true, regex.namedCharacterCache); @@ -1828,6 +1829,54 @@ private static boolean containsNonAscii(String value) { return false; } + /** Perl named characters and wide numeric escapes make an ASCII source Unicode. */ + private static boolean hasUnicodePromotingPatternSyntax(String pattern) { + if (pattern == null) return false; + boolean quoted = false; + for (int i = 0; i < pattern.length(); i++) { + char ch = pattern.charAt(i); + if (ch > 0xff) return true; + if (ch != '\\' || i + 1 >= pattern.length()) continue; + + char escape = pattern.charAt(++i); + if (escape == 'Q' && !quoted) { + quoted = true; + continue; + } + if (escape == 'E' && quoted) { + quoted = false; + continue; + } + if (quoted || escape == '\\') continue; + if (escape == 'N' && i + 1 < pattern.length() + && pattern.charAt(i + 1) == '{') { + return true; + } + if ((escape != 'x' && escape != 'o') || i + 1 >= pattern.length() + || pattern.charAt(i + 1) != '{') { + continue; + } + + int radix = escape == 'x' ? 16 : 8; + boolean sawDigit = false; + long value = 0; + int cursor = i + 2; + for (; cursor < pattern.length() && pattern.charAt(cursor) != '}'; cursor++) { + char digitCharacter = pattern.charAt(cursor); + if (digitCharacter == '_') continue; + int digit = Character.digit(digitCharacter, radix); + if (digit < 0) break; + sawDigit = true; + value = value * radix + digit; + if (value > 0xff) return true; + } + if (sawDigit && cursor < pattern.length() && pattern.charAt(cursor) == '}') { + i = cursor; + } + } + return false; + } + /** * Applies a Perl "qr" object on a string; returns true/false or a list, * and produces side-effects. From 66f224b287caabbacd52ec47d960ea8e110b42e8 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 18:13:05 +0200 Subject: [PATCH 13/16] fix(regex): align Unicode-promoting cache provenance Derive effective byte provenance before constructing the regex cache key and storing the compiled pattern. Named characters, wide numeric escapes, and wide literal source now share one Unicode-backed cache entry between literal validation and runtime compilation, preserving one lexical charname expansion. Generated with Codex (https://openai.com/codex) Co-Authored-By: OpenAI Codex --- .../org/perlonjava/runtime/regex/RuntimeRegex.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java index 07d43dbea6..3f55d89987 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java @@ -617,11 +617,13 @@ private static synchronized RuntimeRegex compileSynchronized( .usesCustomTranslator(namedCharacterTranslator) && patternString != null && patternString.contains("\\N{"); + boolean effectivePatternByteBacked = patternByteBacked + && !hasUnicodePromotingPatternSyntax(patternString); String cacheKey = patternString + "/" + modifiers + "#debug=" + lexicalDebugMode + "#callouts=" + trustedCalloutCount + "#backend=" + RegexBackendPolicy.cacheTag() - + "#bytepattern=" + patternByteBacked + + "#bytepattern=" + effectivePatternByteBacked + (namedCharacterTranslator == null ? "" : "#charnames=" + namedCharacterTranslator.toString()) + (hasDynamicPattern ? (warnOnUnimplemented ? "\0warn" : "\0defer") : ""); @@ -634,7 +636,7 @@ private static synchronized RuntimeRegex compileSynchronized( System.err.println(" cache miss, compiling new regex"); } regex = new RuntimeRegex(); - regex.patternByteBacked = patternByteBacked; + regex.patternByteBacked = effectivePatternByteBacked; regex.namedCharacterCache = new JoniRegexPattern.NamedCharacterCache(namedCharacterTranslator); regex.lexicalDebugMode = lexicalDebugMode; @@ -702,9 +704,8 @@ private static synchronized RuntimeRegex compileSynchronized( : new JoniRegexPattern(compilePatternString, regex.regexFlags, trustedCalloutCount, false, false, false, regex.namedCharacterCache); - if (patternByteBacked && regex.regexFlags.isCaseInsensitive() - && !regex.regexFlags.isUnicode() && !regex.regexFlags.isAscii() - && !hasUnicodePromotingPatternSyntax(compilePatternString)) { + if (effectivePatternByteBacked && regex.regexFlags.isCaseInsensitive() + && !regex.regexFlags.isUnicode() && !regex.regexFlags.isAscii()) { regex.recursivePatternBytes = new JoniRegexPattern( compilePatternString, regex.regexFlags, trustedCalloutCount, true, true, true, regex.namedCharacterCache); From de4b6ef68aac020681b87e9202d90c58c7658b31 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 17:56:51 +0200 Subject: [PATCH 14/16] fix(regex): restore escaped group diagnostics Translate Joni's generic undefined-group-option error only when its marker follows a literal escaped group token, preserving Perl's exact diagnostic and source marker across ordinary and forced-Joni execution. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../perlonjava/runtime/regex/RuntimeRegex.java | 7 +++++++ .../regex_escaped_group_option_diagnostics.t | 18 ++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 src/test/resources/unit/regex/regex_escaped_group_option_diagnostics.t diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java index 3f55d89987..56ea679c5b 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java @@ -827,6 +827,13 @@ && containsExecutableSource(originalPatternString, if (bytePosition != SyntaxException.UNKNOWN_PATTERN_POSITION) { int characterPosition = utf8ByteOffsetToCharacterOffset( compilePatternString, bytePosition); + if ("undefined group option".equals(message) + && originalPatternString != null + && characterPosition >= 3 + && originalPatternString.regionMatches( + characterPosition - 3, "(?\\", 0, 3)) { + message = "Sequence (?\\...) not recognized"; + } throw new PerlCompilerException(RegexDiagnosticFormatter.markedPerl( originalPatternString, characterPosition, message)); } diff --git a/src/test/resources/unit/regex/regex_escaped_group_option_diagnostics.t b/src/test/resources/unit/regex/regex_escaped_group_option_diagnostics.t new file mode 100644 index 0000000000..a0baf703ff --- /dev/null +++ b/src/test/resources/unit/regex/regex_escaped_group_option_diagnostics.t @@ -0,0 +1,18 @@ +use strict; +use warnings; +use Test::More; + +my @cases = ( + [q{(?\ix}, q{Sequence (?\...) not recognized in regex; marked by <-- HERE in m/(?\ <-- HERE ix/}], + [q{(?\:x}, q{Sequence (?\...) not recognized in regex; marked by <-- HERE in m/(?\ <-- HERE :x/}], + [q{(?\<=x}, q{Sequence (?\...) not recognized in regex; marked by <-- HERE in m/(?\ <-- HERE <=x/}], +); + +for my $case (@cases) { + my ($pattern, $expected) = @$case; + eval "#line 1 regex_escaped_group_option_diagnostics.t\nqr/$pattern/"; + my ($error) = split /\n/, $@; + like($error, qr/^\Q$expected\E at /, "escaped group option $pattern"); +} + +done_testing; From ca3e3b8a5944752b536e59fabeba1ac601cc5258 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 18:02:06 +0200 Subject: [PATCH 15/16] fix(regex): report oversized quantifiers Translate Joni's oversized repeat-range diagnostic to Perl's stable quantifier error family while preserving the engine-provided source marker. Add a compact system-Perl-validated parity matrix for exact, open-ended, and bounded oversized quantifiers. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../runtime/regex/RuntimeRegex.java | 2 ++ .../regex_oversized_quantifier_diagnostics.t | 20 +++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 src/test/resources/unit/regex/regex_oversized_quantifier_diagnostics.t diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java index 56ea679c5b..0d4c50f403 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java @@ -833,6 +833,8 @@ && containsExecutableSource(originalPatternString, && originalPatternString.regionMatches( characterPosition - 3, "(?\\", 0, 3)) { message = "Sequence (?\\...) not recognized"; + } else if ("too big number for repeat range".equals(message)) { + message = "Quantifier in {,} bigger than 2147483646"; } throw new PerlCompilerException(RegexDiagnosticFormatter.markedPerl( originalPatternString, characterPosition, message)); diff --git a/src/test/resources/unit/regex/regex_oversized_quantifier_diagnostics.t b/src/test/resources/unit/regex/regex_oversized_quantifier_diagnostics.t new file mode 100644 index 0000000000..7fbe17af82 --- /dev/null +++ b/src/test/resources/unit/regex/regex_oversized_quantifier_diagnostics.t @@ -0,0 +1,20 @@ +use strict; +use warnings; +use Test::More; + +my @cases = ( + [q{x{2147483648}}, q{x{2147483648 <-- HERE }}], + [q{x{2147483648,}}, q{x{2147483648 <-- HERE ,}}], + [q{x{2147483648,2147483649}}, q{x{2147483648 <-- HERE ,2147483649}}], +); + +for my $case (@cases) { + my ($pattern, $marked) = @$case; + eval "#line 1 regex_oversized_quantifier_diagnostics.t\nqr/$pattern/"; + my ($error) = split /\n/, $@; + like($error, + qr/^Quantifier in \{,\} bigger than \d+ in regex; marked by <-- HERE in m\/\Q$marked\E\/ at /, + "oversized quantifier $pattern"); +} + +done_testing; From 1938c1672bef7adec0ff2e755dce56f8a8411dea Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 18:15:56 +0200 Subject: [PATCH 16/16] fix(regex): locate unmatched opening groups Translate Joni's generic end-of-pattern parenthesis error for ordinary groups and place Perl's diagnostic marker at the unmatched opener. Keep specialized (?...) constructs on their dedicated diagnostic paths. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../runtime/regex/RuntimeRegex.java | 47 +++++++++++++++++++ .../regex_unmatched_open_paren_diagnostics.t | 21 +++++++++ 2 files changed, 68 insertions(+) create mode 100644 src/test/resources/unit/regex/regex_unmatched_open_paren_diagnostics.t diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java index 0d4c50f403..d3c965cba1 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java @@ -835,6 +835,13 @@ && containsExecutableSource(originalPatternString, message = "Sequence (?\\...) not recognized"; } else if ("too big number for repeat range".equals(message)) { message = "Quantifier in {,} bigger than 2147483646"; + } else if ("end pattern with unmatched parenthesis".equals(message)) { + int unmatched = ordinaryUnmatchedOpeningParenthesis( + originalPatternString); + if (unmatched >= 0) { + message = "Unmatched ("; + characterPosition = unmatched + 1; + } } throw new PerlCompilerException(RegexDiagnosticFormatter.markedPerl( originalPatternString, characterPosition, message)); @@ -923,6 +930,46 @@ private static int utf8ByteOffsetToCharacterOffset(String pattern, int byteOffse return new String(bytes, 0, boundedOffset, StandardCharsets.UTF_8).length(); } + /** Locate an unmatched ordinary group opener, excluding Perl's specialized {@code (?...} forms. */ + private static int ordinaryUnmatchedOpeningParenthesis(String pattern) { + if (pattern == null) return -1; + Deque openings = new ArrayDeque<>(); + boolean escaped = false; + boolean inClass = false; + for (int i = 0; i < pattern.length(); i++) { + char ch = pattern.charAt(i); + if (escaped) { + escaped = false; + continue; + } + if (ch == '\\') { + escaped = true; + continue; + } + if (ch == '[') { + inClass = true; + continue; + } + if (ch == ']' && inClass) { + inClass = false; + continue; + } + if (inClass) continue; + if (ch == '(') { + openings.push(i); + } else if (ch == ')' && !openings.isEmpty()) { + openings.pop(); + } + } + while (!openings.isEmpty()) { + int opening = openings.removeLast(); + if (opening + 1 >= pattern.length() || pattern.charAt(opening + 1) != '?') { + return opening; + } + } + return -1; + } + private static int debugMode(String modifiers) { if (modifiers == null) return 0; if (modifiers.indexOf(INTERNAL_DEBUGCOLOR_MARKER) >= 0) return 2; diff --git a/src/test/resources/unit/regex/regex_unmatched_open_paren_diagnostics.t b/src/test/resources/unit/regex/regex_unmatched_open_paren_diagnostics.t new file mode 100644 index 0000000000..38e6314fc5 --- /dev/null +++ b/src/test/resources/unit/regex/regex_unmatched_open_paren_diagnostics.t @@ -0,0 +1,21 @@ +use strict; +use warnings; +use utf8; +use Test::More; + +my @cases = ( + [q{((x)}, q{Unmatched ( in regex; marked by <-- HERE in m/( <-- HERE (x)/}], + [q|{(}|, q|Unmatched ( in regex; marked by <-- HERE in m/{( <-- HERE }/|], + [q{ネ((ネ)}, q{Unmatched ( in regex; marked by <-- HERE in m/ネ( <-- HERE (ネ)/}], +); + +my $case_number = 0; +for my $case (@cases) { + $case_number++; + my ($pattern, $expected) = @$case; + eval "#line 1 regex_unmatched_open_paren_diagnostics.t\nqr/$pattern/"; + my ($error) = split /\n/, $@; + like($error, qr/^\Q$expected\E at /, "unmatched opening parenthesis case $case_number"); +} + +done_testing;