From 82f72ae82a3b9344f59f702c2ee9048ee254678f Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 18:47:10 +0200 Subject: [PATCH 01/10] fix(joni): match Perl empty boundary semantics Treat empty input as having no grapheme, sentence, word, or line boundary, so the positive assertions fail and their negations match. Accept Perl's surrounding whitespace in boundary names without weakening unknown-name diagnostics. Generated with Codex (https://openai.com/codex) Co-Authored-By: OpenAI Codex --- .../unit/regex/boundary_empty_whitespace.t | 17 +++++++++++++++++ .../joni/src/org/joni/ByteCodeMachine.java | 3 +++ third_party/joni/src/org/joni/Lexer.java | 2 +- .../joni/test/TestPerlBoundaryDiagnostics.java | 2 ++ 4 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 src/test/resources/unit/regex/boundary_empty_whitespace.t diff --git a/src/test/resources/unit/regex/boundary_empty_whitespace.t b/src/test/resources/unit/regex/boundary_empty_whitespace.t new file mode 100644 index 000000000..4f3eead29 --- /dev/null +++ b/src/test/resources/unit/regex/boundary_empty_whitespace.t @@ -0,0 +1,17 @@ +use strict; +use warnings; +use Test::More; + +for my $name (qw(gcb sb wb lb)) { + my $positive = qr/\b{$name}/; + my $negative = qr/\B{$name}/; + my $spaced_positive = qr/\b{ $name }/; + my $spaced_negative = qr/\B{ $name }/; + + unlike('', $positive, "empty text has no $name boundary"); + like('', $negative, "empty text satisfies negated $name boundary"); + unlike('', $spaced_positive, "whitespace is accepted around $name"); + like('', $spaced_negative, "spaced negated $name retains empty semantics"); +} + +done_testing; diff --git a/third_party/joni/src/org/joni/ByteCodeMachine.java b/third_party/joni/src/org/joni/ByteCodeMachine.java index 33020e4d9..f38475ab5 100644 --- a/third_party/joni/src/org/joni/ByteCodeMachine.java +++ b/third_party/joni/src/org/joni/ByteCodeMachine.java @@ -1399,6 +1399,7 @@ private void opWordBreakBoundary(boolean negated) { } private boolean isWordBreakBoundary() { + if (str == end) return false; if (s <= str || s >= end) return true; // WB1, WB2 int leftPosition = enc.prevCharHead(bytes, str, s, end); @@ -1858,6 +1859,7 @@ private int precedingLineRun(int position, short value) { } private boolean isSentenceBoundary() { + if (str == end) return false; if (s <= str || s >= end) return true; // SB1, SB2 int leftPosition = enc.prevCharHead(bytes, str, s, end); @@ -1981,6 +1983,7 @@ private boolean isSentenceTerminal(byte property) { } private boolean isGraphemeBoundary() { + if (str == end) return false; if (s <= str || s >= end) return true; // GB1, GB2 int leftPosition = enc.prevCharHead(bytes, str, s, end); diff --git a/third_party/joni/src/org/joni/Lexer.java b/third_party/joni/src/org/joni/Lexer.java index 76b65e415..0596502fe 100644 --- a/third_party/joni/src/org/joni/Lexer.java +++ b/third_party/joni/src/org/joni/Lexer.java @@ -1991,7 +1991,7 @@ private boolean fetchTokenForPerlBoundary(boolean negated) { int characterStart = p; fetch(); if (c == '}') { - String boundaryName = name.toString(); + String boundaryName = name.toString().trim(); if (boundaryName.equals("gcb")) { fetchTokenFor_anchor(negated ? AnchorType.NOT_GRAPHEME_BOUNDARY diff --git a/third_party/joni/test/org/joni/test/TestPerlBoundaryDiagnostics.java b/third_party/joni/test/org/joni/test/TestPerlBoundaryDiagnostics.java index fc7ff78e9..dc559a9f3 100644 --- a/third_party/joni/test/org/joni/test/TestPerlBoundaryDiagnostics.java +++ b/third_party/joni/test/org/joni/test/TestPerlBoundaryDiagnostics.java @@ -66,6 +66,8 @@ public void retainsSupportedPerlBoundaryNames() { for (String name : new String[] {"gcb", "sb", "wb", "lb"}) { compile("\\b{" + name + "}", Syntax.PerlNG); compile("\\B{" + name + "}", Syntax.PerlNG); + compile("\\b{ " + name + " }", Syntax.PerlNG); + compile("\\B{ " + name + " }", Syntax.PerlNG); } } From 7387954a6ce337363e7154ee93df53bf34b5d6c0 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 18:59:34 +0200 Subject: [PATCH 02/10] fix(joni): align Perl closing bracket warning policy Suppress Ruby-style unescaped closing-bracket warnings under PerlNG syntax while retaining Joni's existing parsing semantics. Add a system-Perl-validated fixture covering literal, class, negated-class, and backreference contexts. This closes nine unchanged regexp.t rows with no introduced failures. Generated with [OpenAI Codex](https://openai.com/codex/) Co-Authored-By: OpenAI Codex --- .../regex/character_class_closing_bracket.t | 26 +++++++++++++++++++ .../joni/src/org/joni/ScanEnvironment.java | 8 ++++++ 2 files changed, 34 insertions(+) create mode 100644 src/test/resources/unit/regex/character_class_closing_bracket.t diff --git a/src/test/resources/unit/regex/character_class_closing_bracket.t b/src/test/resources/unit/regex/character_class_closing_bracket.t new file mode 100644 index 000000000..318b39ffd --- /dev/null +++ b/src/test/resources/unit/regex/character_class_closing_bracket.t @@ -0,0 +1,26 @@ +use strict; +use warnings; +use Test::More; + +my @cases = ( + [ 'a]', 'a]', 1, 'closing bracket outside a class is literal' ], + [ 'a[]]b', 'a]b', 1, 'leading closing bracket in a class is literal' ], + [ 'a[^]b]c', 'a]c', 0, 'negated class excludes its leading bracket' ], + [ 'a[^]b]c', 'adc', 1, 'negated class retains its ordinary members' ], + [ '2(]*)?$\\1', '2', 1, 'closing bracket class composes with backreference' ], +); + +for my $case (@cases) { + my ($pattern, $subject, $expected, $name) = @$case; + my @warnings; + my $regex; + { + local $SIG{__WARN__} = sub { push @warnings, @_ }; + $regex = eval { qr/$pattern/ }; + } + is($@, '', "$name compiles"); + is(scalar @warnings, 0, "$name has no warning"); + is(($subject =~ $regex) ? 1 : 0, $expected, $name); +} + +done_testing; diff --git a/third_party/joni/src/org/joni/ScanEnvironment.java b/third_party/joni/src/org/joni/ScanEnvironment.java index 4405782f6..ba6a204bd 100644 --- a/third_party/joni/src/org/joni/ScanEnvironment.java +++ b/third_party/joni/src/org/joni/ScanEnvironment.java @@ -162,6 +162,10 @@ int convertBackslashValue(int c) { void ccEscWarn(String s) { if (warnings != WarnCallback.NONE) { + // Perl accepts a leading ']' as a literal character in a class + // without warning. Ruby/Oniguruma warns for this spelling, but + // PerlNG must preserve Perl's warning policy as well as its parse. + if (syntax.op2OptionPerl() && "]".equals(s)) return; if (syntax.warnCCOpNotEscaped() && syntax.backSlashEscapeInCC()) { warnings.warn("character class has '" + s + "' without escape"); } @@ -176,6 +180,10 @@ void unknownEscWarn(String s) { void closeBracketWithoutEscapeWarn(String s) { if (warnings != WarnCallback.NONE) { + // A closing bracket outside a character class is an ordinary + // literal in Perl (except at the start, which the lexer already + // permits separately) and does not produce a regexp warning. + if (syntax.op2OptionPerl() && "]".equals(s)) return; if (syntax.warnCCOpNotEscaped()) { warnings.warn("regular expression has '" + s + "' without escape"); } From d5fb1e61c03c2e68a8096458acc14f85be591d51 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 19:09:56 +0200 Subject: [PATCH 03/10] fix(joni): report descending ranges like Perl Give PerlNG and PerlOnJava syntax descending character ranges Perl's diagnostic while preserving upstream Joni behavior for other syntaxes. Align the temporary Java differential route and add system-Perl and direct-Joni coverage for byte and wide ranges. This closes two unchanged regexp.t rows with no introduced failures. Generated with [OpenAI Codex](https://openai.com/codex/) Co-Authored-By: OpenAI Codex --- .../runtime/regex/RuntimeRegex.java | 5 +++ .../regex/character_class_range_diagnostic.t | 11 +++++ .../joni/src/org/joni/CodeRangeBuffer.java | 2 +- .../joni/src/org/joni/ScanEnvironment.java | 6 +++ .../joni/src/org/joni/ast/CClassNode.java | 6 +-- .../src/org/joni/exception/ErrorMessages.java | 1 + .../TestPerlCharacterClassDiagnostics.java | 44 +++++++++++++++++++ 7 files changed, 71 insertions(+), 4 deletions(-) create mode 100644 src/test/resources/unit/regex/character_class_range_diagnostic.t create mode 100644 third_party/joni/test/org/joni/test/TestPerlCharacterClassDiagnostics.java diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java index d3c965cba..8df75ec1d 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java @@ -20,6 +20,7 @@ import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -790,6 +791,10 @@ private static synchronized RuntimeRegex compileSynchronized( } } } catch (Exception e) { + if (e instanceof PatternSyntaxException syntaxError + && "Illegal character range".equals(syntaxError.getDescription())) { + throw new PerlCompilerException("Invalid [] range"); + } if ("invalid backref number/name".equals(e.getMessage()) || "invalid backref number".equals(e.getMessage())) { throw new PerlCompilerException("Reference to nonexistent group"); diff --git a/src/test/resources/unit/regex/character_class_range_diagnostic.t b/src/test/resources/unit/regex/character_class_range_diagnostic.t new file mode 100644 index 000000000..4bc611af7 --- /dev/null +++ b/src/test/resources/unit/regex/character_class_range_diagnostic.t @@ -0,0 +1,11 @@ +use strict; +use warnings; +use Test::More; + +for my $pattern ('[b-a]', '(?i:a[b-a])', '[\x{100}-\x{ff}]') { + my $regex = eval { qr/$pattern/ }; + ok(!defined($regex), "$pattern is rejected"); + like($@, qr/^Invalid \[\] range/, "$pattern uses Perl range diagnostic"); +} + +done_testing; diff --git a/third_party/joni/src/org/joni/CodeRangeBuffer.java b/third_party/joni/src/org/joni/CodeRangeBuffer.java index c5d0e6ca4..47af86360 100644 --- a/third_party/joni/src/org/joni/CodeRangeBuffer.java +++ b/third_party/joni/src/org/joni/CodeRangeBuffer.java @@ -186,7 +186,7 @@ public static CodeRangeBuffer addCodeRange(CodeRangeBuffer pbuf, ScanEnvironment if (env.syntax.allowEmptyRangeInCC()) { return pbuf; } else { - throw new ValueException(ErrorMessages.EMPTY_RANGE_IN_CHAR_CLASS); + throw new ValueException(env.emptyRangeError()); } } return addCodeRangeToBuff(pbuf, env, from, to, checkDup); diff --git a/third_party/joni/src/org/joni/ScanEnvironment.java b/third_party/joni/src/org/joni/ScanEnvironment.java index ba6a204bd..fce109f31 100644 --- a/third_party/joni/src/org/joni/ScanEnvironment.java +++ b/third_party/joni/src/org/joni/ScanEnvironment.java @@ -178,6 +178,12 @@ void unknownEscWarn(String s) { } } + public String emptyRangeError() { + return ("PerlNG".equals(syntax.name) || "PERLONJAVA".equals(syntax.name)) + ? ErrorMessages.PERL_INVALID_RANGE_IN_CHAR_CLASS + : ErrorMessages.EMPTY_RANGE_IN_CHAR_CLASS; + } + void closeBracketWithoutEscapeWarn(String s) { if (warnings != WarnCallback.NONE) { // A closing bracket outside a character class is an ordinary diff --git a/third_party/joni/src/org/joni/ast/CClassNode.java b/third_party/joni/src/org/joni/ast/CClassNode.java index 30ec08a6a..9cdc69ada 100644 --- a/third_party/joni/src/org/joni/ast/CClassNode.java +++ b/third_party/joni/src/org/joni/ast/CClassNode.java @@ -684,7 +684,7 @@ public void nextStateValue(CCStateArg arg, CClassNode ascCc, arg.state = CCSTATE.COMPLETE; break; } else { - throw new ValueException(ErrorMessages.EMPTY_RANGE_IN_CHAR_CLASS); + throw new ValueException(env.emptyRangeError()); } } bs.setRange(env, (int)arg.from, (int)arg.to); @@ -696,7 +696,7 @@ public void nextStateValue(CCStateArg arg, CClassNode ascCc, arg.state = CCSTATE.COMPLETE; break; } - throw new ValueException(ErrorMessages.EMPTY_RANGE_IN_CHAR_CLASS); + throw new ValueException(env.emptyRangeError()); } addWideScalarRange(arg.from, arg.to); } else { @@ -711,7 +711,7 @@ public void nextStateValue(CCStateArg arg, CClassNode ascCc, arg.state = CCSTATE.COMPLETE; break; } else { - throw new ValueException(ErrorMessages.EMPTY_RANGE_IN_CHAR_CLASS); + throw new ValueException(env.emptyRangeError()); } } long normalTo = Math.min(arg.to, 0x10ffffL); diff --git a/third_party/joni/src/org/joni/exception/ErrorMessages.java b/third_party/joni/src/org/joni/exception/ErrorMessages.java index 25109cc58..5146eebbd 100644 --- a/third_party/joni/src/org/joni/exception/ErrorMessages.java +++ b/third_party/joni/src/org/joni/exception/ErrorMessages.java @@ -147,6 +147,7 @@ public interface ErrorMessages extends org.jcodings.exception.ErrorMessages { String TOO_BIG_NUMBER_FOR_REPEAT_RANGE = "too big number for repeat range"; String UPPER_SMALLER_THAN_LOWER_IN_REPEAT_RANGE = "upper is smaller than lower in repeat range"; String EMPTY_RANGE_IN_CHAR_CLASS = "empty range in char class"; + String PERL_INVALID_RANGE_IN_CHAR_CLASS = "Invalid [] range"; String MISMATCH_CODE_LENGTH_IN_CLASS_RANGE = "mismatch multibyte code length in char-class range"; String TOO_MANY_MULTI_BYTE_RANGES = "too many multibyte code ranges are specified"; String TOO_SHORT_MULTI_BYTE_STRING = "too short multibyte code string"; diff --git a/third_party/joni/test/org/joni/test/TestPerlCharacterClassDiagnostics.java b/third_party/joni/test/org/joni/test/TestPerlCharacterClassDiagnostics.java new file mode 100644 index 000000000..b64b7506d --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestPerlCharacterClassDiagnostics.java @@ -0,0 +1,44 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +import java.nio.charset.StandardCharsets; + +import org.jcodings.specific.UTF8Encoding; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.joni.WarnCallback; +import org.joni.exception.SyntaxException; +import org.junit.Test; + +public class TestPerlCharacterClassDiagnostics { + @Test + public void descendingRangeUsesPerlDiagnostic() { + byte[] bytes = "[b-a]".getBytes(StandardCharsets.UTF_8); + SyntaxException error = assertThrows(SyntaxException.class, + () -> new Regex(bytes, 0, bytes.length, Option.NONE, + UTF8Encoding.INSTANCE, Syntax.PerlNG, WarnCallback.NONE)); + assertEquals("Invalid [] range", error.getMessage()); + } +} From d8026a9cf1c9ff58168f2af4d5b9eac443f0c165 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 19:13:46 +0200 Subject: [PATCH 04/10] fix(joni): report unknown POSIX classes like Perl Retain unknown POSIX class spelling in PerlNG diagnostics, including negated class names, while preserving upstream Joni messages for other syntaxes. Add system-Perl and direct-Joni coverage. This closes four unchanged regexp.t rows with no introduced failures. Generated with [OpenAI Codex](https://openai.com/codex/) Co-Authored-By: OpenAI Codex --- .../unit/regex/posix_class_unknown_diagnostic.t | 13 +++++++++++++ third_party/joni/src/org/joni/Parser.java | 12 +++++++++++- third_party/joni/src/org/joni/ScanEnvironment.java | 6 +++++- .../test/TestPerlCharacterClassDiagnostics.java | 14 ++++++++++++++ 4 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 src/test/resources/unit/regex/posix_class_unknown_diagnostic.t diff --git a/src/test/resources/unit/regex/posix_class_unknown_diagnostic.t b/src/test/resources/unit/regex/posix_class_unknown_diagnostic.t new file mode 100644 index 000000000..0d0d3e8fb --- /dev/null +++ b/src/test/resources/unit/regex/posix_class_unknown_diagnostic.t @@ -0,0 +1,13 @@ +use strict; +use warnings; +use Test::More; + +for my $name ('foo', '^foo', 'xyz') { + my $pattern = "[[:$name:]]"; + my $regex = eval { qr/$pattern/ }; + ok(!defined($regex), "$pattern is rejected"); + like($@, qr/^POSIX class \[:\Q$name\E:\] unknown/, + "$pattern uses Perl POSIX-class diagnostic"); +} + +done_testing; diff --git a/third_party/joni/src/org/joni/Parser.java b/third_party/joni/src/org/joni/Parser.java index b7a37e523..f25175eb6 100644 --- a/third_party/joni/src/org/joni/Parser.java +++ b/third_party/joni/src/org/joni/Parser.java @@ -195,6 +195,7 @@ private boolean parsePosixBracket(CClassNode cc, CClassNode ascCc, } else { not = false; } + int nameStart = p; if (enc.strLength(bytes, p, stop) >= POSIX_BRACKET_NAME_MIN_LEN + 3) { // else goto not_posix_bracket boolean asciiRange = isAsciiRange(env.option) && !isPosixBracketAllRange(env.option); @@ -233,10 +234,19 @@ private boolean parsePosixBracket(CClassNode cc, CClassNode ascCc, } if (c == ':' && left()) { + int nameEnd = p; inc(); if (left()) { fetch(); - if (c == ']') newSyntaxException(INVALID_POSIX_BRACKET_TYPE); + if (c == ']') { + if (env.usesPerlDiagnostics()) { + String name = new String(bytes, nameStart, + nameEnd - nameStart, StandardCharsets.US_ASCII); + newSyntaxException("POSIX class [:" + (not ? "^" : "") + + name + ":] unknown"); + } + newSyntaxException(INVALID_POSIX_BRACKET_TYPE); + } } } restore(); diff --git a/third_party/joni/src/org/joni/ScanEnvironment.java b/third_party/joni/src/org/joni/ScanEnvironment.java index fce109f31..886eb2cab 100644 --- a/third_party/joni/src/org/joni/ScanEnvironment.java +++ b/third_party/joni/src/org/joni/ScanEnvironment.java @@ -179,11 +179,15 @@ void unknownEscWarn(String s) { } public String emptyRangeError() { - return ("PerlNG".equals(syntax.name) || "PERLONJAVA".equals(syntax.name)) + return usesPerlDiagnostics() ? ErrorMessages.PERL_INVALID_RANGE_IN_CHAR_CLASS : ErrorMessages.EMPTY_RANGE_IN_CHAR_CLASS; } + public boolean usesPerlDiagnostics() { + return "PerlNG".equals(syntax.name) || "PERLONJAVA".equals(syntax.name); + } + void closeBracketWithoutEscapeWarn(String s) { if (warnings != WarnCallback.NONE) { // A closing bracket outside a character class is an ordinary diff --git a/third_party/joni/test/org/joni/test/TestPerlCharacterClassDiagnostics.java b/third_party/joni/test/org/joni/test/TestPerlCharacterClassDiagnostics.java index b64b7506d..f3b20da08 100644 --- a/third_party/joni/test/org/joni/test/TestPerlCharacterClassDiagnostics.java +++ b/third_party/joni/test/org/joni/test/TestPerlCharacterClassDiagnostics.java @@ -41,4 +41,18 @@ public void descendingRangeUsesPerlDiagnostic() { UTF8Encoding.INSTANCE, Syntax.PerlNG, WarnCallback.NONE)); assertEquals("Invalid [] range", error.getMessage()); } + + @Test + public void unknownPosixClassNamesUsePerlDiagnostic() { + assertPosixError("[[:foo:]]", "POSIX class [:foo:] unknown"); + assertPosixError("[[:^foo:]]", "POSIX class [:^foo:] unknown"); + } + + private static void assertPosixError(String pattern, String message) { + byte[] bytes = pattern.getBytes(StandardCharsets.UTF_8); + SyntaxException error = assertThrows(SyntaxException.class, + () -> new Regex(bytes, 0, bytes.length, Option.NONE, + UTF8Encoding.INSTANCE, Syntax.PerlNG, WarnCallback.NONE)); + assertEquals(message, error.getMessage()); + } } From c8cfe4eb49d569f3bf49a4f0b120d4908574948f Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 18:34:57 +0200 Subject: [PATCH 05/10] fix(regex): preserve lexical re strict policy Carry lexical use re 'strict' context through literal validation, runtime compilation, regex caching, cloning, and deferred recompilation. Apply that context to Perl's unescaped-left-brace warning-versus-fatal policy while preserving always-fatal and explicitly allowed brace contexts. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../frontend/parser/StringParser.java | 4 + .../org/perlonjava/runtime/perlmodule/Re.java | 2 + .../perlonjava/runtime/perlmodule/Strict.java | 1 + .../runtime/regex/RuntimeRegex.java | 168 ++++++++++++++++-- .../regex/regex_re_strict_left_brace_policy.t | 66 +++++++ 5 files changed, 224 insertions(+), 17 deletions(-) create mode 100644 src/test/resources/unit/regex/regex_re_strict_left_brace_policy.t diff --git a/src/main/java/org/perlonjava/frontend/parser/StringParser.java b/src/main/java/org/perlonjava/frontend/parser/StringParser.java index 550e474b1..9706ddebc 100644 --- a/src/main/java/org/perlonjava/frontend/parser/StringParser.java +++ b/src/main/java/org/perlonjava/frontend/parser/StringParser.java @@ -25,6 +25,7 @@ import static org.perlonjava.runtime.perlmodule.Strict.HINT_RE_TAINT; import static org.perlonjava.runtime.perlmodule.Strict.HINT_RE_DEBUG; import static org.perlonjava.runtime.perlmodule.Strict.HINT_RE_DEBUGCOLOR; +import static org.perlonjava.runtime.perlmodule.Strict.HINT_RE_STRICT; import static org.perlonjava.runtime.perlmodule.Strict.HINT_LOCALE; import static org.perlonjava.runtime.runtimetypes.NameNormalizer.normalizeVariableName; import static org.perlonjava.runtime.runtimetypes.ScalarUtils.printable; @@ -710,6 +711,9 @@ static String addLexicalRegexContext(EmitterContext ctx, String modifiers) { if (ctx.symbolTable.isStrictOptionEnabled(HINT_RE_TAINT) && !result.contains("T")) { result = "T" + result; } + if (ctx.symbolTable.isStrictOptionEnabled(HINT_RE_STRICT)) { + result += RuntimeRegex.INTERNAL_RE_STRICT_MARKER; + } return addLexicalRegexDebugMarker(ctx, result); } diff --git a/src/main/java/org/perlonjava/runtime/perlmodule/Re.java b/src/main/java/org/perlonjava/runtime/perlmodule/Re.java index ebf26ff78..3ea47f509 100644 --- a/src/main/java/org/perlonjava/runtime/perlmodule/Re.java +++ b/src/main/java/org/perlonjava/runtime/perlmodule/Re.java @@ -181,6 +181,7 @@ public static RuntimeList importRe(RuntimeArray args, int ctx) { RuntimeScalar targetCode = getGlobalCodeRef(caller + "::regexp_pattern"); targetCode.set(sourceCode); } else if (opt.equalsIgnoreCase("strict")) { + symbolTable.enableStrictOption(Strict.HINT_RE_STRICT); // Enable categories used by our preprocessor warnings Warnings.warningManager.enableWarning("experimental::re_strict"); Warnings.warningManager.enableWarning("experimental::uniprop_wildcards"); @@ -227,6 +228,7 @@ public static RuntimeList unimportRe(RuntimeArray args, int ctx) { opt = opt.replace("\"", "").replace("'", "").trim(); if (opt.equalsIgnoreCase("strict")) { + symbolTable.disableStrictOption(Strict.HINT_RE_STRICT); Warnings.warningManager.disableWarning("experimental::re_strict"); Warnings.warningManager.disableWarning("experimental::uniprop_wildcards"); Warnings.warningManager.disableWarning("experimental::vlb"); diff --git a/src/main/java/org/perlonjava/runtime/perlmodule/Strict.java b/src/main/java/org/perlonjava/runtime/perlmodule/Strict.java index 8122edc0d..427e759b5 100644 --- a/src/main/java/org/perlonjava/runtime/perlmodule/Strict.java +++ b/src/main/java/org/perlonjava/runtime/perlmodule/Strict.java @@ -51,6 +51,7 @@ private static void propagatePragmaFlags(ScopedSymbolTable source) { public static final int HINT_RE_TAINT = 0x00002000; // use re 'taint' public static final int HINT_RE_DEBUG = 0x00004000; // use re 'debug' public static final int HINT_RE_DEBUGCOLOR = 0x00008000; // use re 'debugcolor' + public static final int HINT_RE_STRICT = 0x00010000; // use re 'strict' /** * Constructor for Strict. diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java index 8df75ec1d..93db78be2 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java @@ -59,6 +59,7 @@ public static RuntimeScalar stabilizeLiteralTarget(RuntimeScalar literal, int ca /** Private AST/runtime modifier markers; removed before Perl modifier parsing. */ public static final char INTERNAL_DEBUG_MARKER = '\u0001'; public static final char INTERNAL_DEBUGCOLOR_MARKER = '\u0002'; + public static final char INTERNAL_RE_STRICT_MARKER = '\u0003'; // Debug flag for regex compilation (set at class load time) private static final boolean DEBUG_REGEX = System.getenv("DEBUG_REGEX") != null; @@ -221,6 +222,7 @@ static void updateControlVerbVariables(String mark, String error) { private List inlineModifierWarnings = new ArrayList<>(); // 0 = off, 1 = debug, 2 = debugcolor. Captured at the regex call site. private int lexicalDebugMode; + private boolean lexicalReStrict; private static final String DYNAMIC_PATTERN_ERROR = "\u0000(??{...}) recursive regex patterns not implemented (dynamic pattern)"; @@ -263,6 +265,7 @@ public RuntimeRegex cloneTracked() { copy.warningsOnUse = new ArrayList<>(this.warningsOnUse); copy.inlineModifierWarnings = new ArrayList<>(this.inlineModifierWarnings); copy.lexicalDebugMode = this.lexicalDebugMode; + copy.lexicalReStrict = this.lexicalReStrict; // replacement and callerArgs are not copied — they are set per-substitution // matched is not copied — each qr// object tracks its own m?PAT? state copy.refCount = 0; // Enable refCount tracking @@ -517,9 +520,16 @@ private static RuntimeRegex compile(String patternString, String modifiers, int private static RuntimeRegex compile(String patternString, String modifiers, int lexicalDebugMode, int trustedCalloutCount, boolean patternByteBacked) { + return compile(patternString, modifiers, lexicalDebugMode, trustedCalloutCount, + patternByteBacked, reStrictMode(modifiers)); + } + + private static RuntimeRegex compile(String patternString, String modifiers, int lexicalDebugMode, + int trustedCalloutCount, boolean patternByteBacked, + boolean lexicalReStrict) { RuntimeScalar namedCharacterTranslator = org.perlonjava.runtime.HintHashRegistry.getCompileTimeHint("charnames"); - modifiers = stripDebugMarkers(modifiers); + modifiers = stripInternalMarkers(modifiers); // Dynamic/interpolated qr// compilation can begin during ordinary // execution, outside the Perl compiler lock. User-defined Unicode // properties execute arbitrary Perl and may block, so resolve them @@ -529,7 +539,8 @@ private static RuntimeRegex compile(String patternString, String modifiers, int UnicodeResolver.preloadUserDefinedProperties( patternString, preloadFlags.isCaseInsensitive()); return compileSynchronized(patternString, modifiers, lexicalDebugMode, - trustedCalloutCount, false, patternByteBacked, namedCharacterTranslator); + trustedCalloutCount, false, patternByteBacked, lexicalReStrict, + namedCharacterTranslator); } /** User properties execute Perl code and therefore cannot be validated while compiling a CV. */ @@ -550,8 +561,8 @@ public static boolean requiresRuntimeUnicodePropertyResolution(String patternStr */ public static void validateLiteralSyntax(String patternString, String modifiers) { try { - compileSynchronized(patternString, stripDebugMarkers(modifiers), - debugMode(modifiers), 0, true, false, + compileSynchronized(patternString, stripInternalMarkers(modifiers), + debugMode(modifiers), 0, true, false, reStrictMode(modifiers), org.perlonjava.runtime.HintHashRegistry.getCompileTimeHint("charnames")); } catch (PerlJavaUnimplementedException unsupported) { String message = unsupported.getMessage(); @@ -572,7 +583,7 @@ public static void validateLiteralSyntax(String patternString, String modifiers) private static synchronized RuntimeRegex compileSynchronized( String patternString, String modifiers, int lexicalDebugMode, int trustedCalloutCount, boolean literalSyntaxValidation, - boolean patternByteBacked, + boolean patternByteBacked, boolean lexicalReStrict, RuntimeScalar namedCharacterTranslator) { // Debug logging if (DEBUG_REGEX) { @@ -625,6 +636,7 @@ private static synchronized RuntimeRegex compileSynchronized( + "#callouts=" + trustedCalloutCount + "#backend=" + RegexBackendPolicy.cacheTag() + "#bytepattern=" + effectivePatternByteBacked + + "#strict=" + lexicalReStrict + (namedCharacterTranslator == null ? "" : "#charnames=" + namedCharacterTranslator.toString()) + (hasDynamicPattern ? (warnOnUnimplemented ? "\0warn" : "\0defer") : ""); @@ -641,6 +653,7 @@ private static synchronized RuntimeRegex compileSynchronized( regex.namedCharacterCache = new JoniRegexPattern.NamedCharacterCache(namedCharacterTranslator); regex.lexicalDebugMode = lexicalDebugMode; + regex.lexicalReStrict = lexicalReStrict; // Note: flags /e /ee are processed at parse time, in parseRegexReplace() @@ -649,6 +662,21 @@ private static synchronized RuntimeRegex compileSynchronized( regex.regexFlags = fromModifiers(modifiers, compilePatternString); regex.useGAssertion = regex.regexFlags.useGAssertion(); regex.patternFlags = regex.regexFlags.toPatternFlags(); + + LeftBraceIssue leftBraceIssue = unescapedLeftBraceIssue( + originalPatternString); + String sourcePolicyWarning = null; + if (leftBraceIssue != null) { + String message = leftBraceIssue.alwaysFatal || lexicalReStrict + ? "Unescaped left brace in regex is illegal here" + : "Unescaped left brace in regex is passed through"; + String diagnostic = RegexDiagnosticFormatter.markedPerl( + originalPatternString, leftBraceIssue.offset + 1, message); + if (leftBraceIssue.alwaysFatal || lexicalReStrict) { + throw new PerlCompilerException(diagnostic); + } + sourcePolicyWarning = diagnostic; + } // Always compute Unicode flags - we need the Unicode variant for when // the input string contains non-ASCII characters (auto-Unicode detection) @@ -689,6 +717,10 @@ private static synchronized RuntimeRegex compileSynchronized( // Track if preprocessing deferred user-defined Unicode properties. // These need to be resolved later, once the corresponding Perl subs are defined. regex.warningsOnUse = new ArrayList<>(quoteMetaWarningsOnUse); + if (sourcePolicyWarning != null) { + regex.inlineModifierWarnings.add(sourcePolicyWarning); + regex.warningsOnUse.add(sourcePolicyWarning); + } if (hasDeferredDynamicPattern) { regex.warningsOnUse.add(DYNAMIC_PATTERN_ERROR); } else if (hasWarnDynamicFallback) { @@ -975,16 +1007,109 @@ private static int ordinaryUnmatchedOpeningParenthesis(String pattern) { return -1; } + private static LeftBraceIssue unescapedLeftBraceIssue(String pattern) { + if (pattern == null || pattern.isEmpty()) return null; + boolean escaped = false; + boolean inClass = false; + for (int i = 0; i < pattern.length(); i++) { + char ch = pattern.charAt(i); + if (escaped) { + escaped = false; + continue; + } + if (ch == '\\') { + escaped = true; + continue; + } + if (ch == '[') { + inClass = true; + continue; + } + if (ch == ']' && inClass) { + inClass = false; + continue; + } + if (inClass || ch != '{') continue; + if (isEscapeArgumentBrace(pattern, i) + || isValidQuantifier(pattern, i) + || isAllowedLiteralLeftBrace(pattern, i)) { + continue; + } + boolean followsAlphanumericEscape = i >= 2 + && Character.isLetterOrDigit(pattern.charAt(i - 1)) + && pattern.charAt(i - 2) == '\\' + && (i < 3 || pattern.charAt(i - 3) != '\\'); + return new LeftBraceIssue(i, followsAlphanumericEscape); + } + return null; + } + + private static boolean isEscapeArgumentBrace(String pattern, int offset) { + if (offset < 2 || pattern.charAt(offset - 2) != '\\' + || (offset >= 3 && pattern.charAt(offset - 3) == '\\')) { + return false; + } + return "pPxXoONkgb".indexOf(pattern.charAt(offset - 1)) >= 0; + } + + private static boolean isValidQuantifier(String pattern, int offset) { + int cursor = offset + 1; + int digitStart = cursor; + while (cursor < pattern.length() && Character.isDigit(pattern.charAt(cursor))) { + cursor++; + } + if (cursor == digitStart) return false; + if (cursor < pattern.length() && pattern.charAt(cursor) == ',') { + cursor++; + while (cursor < pattern.length() && Character.isDigit(pattern.charAt(cursor))) { + cursor++; + } + } + return cursor < pattern.length() && pattern.charAt(cursor) == '}'; + } + + private static boolean isAllowedLiteralLeftBrace(String pattern, int offset) { + if (offset == 0) return true; + char previous = pattern.charAt(offset - 1); + if (previous == '^' || previous == '|' || previous == '(' || previous == '*' + || previous == '+' || previous == '?') { + return true; + } + if (offset >= 3 && (pattern.regionMatches(offset - 3, "(?:", 0, 3) + || pattern.regionMatches(offset - 3, "(:?", 0, 3))) { + return true; + } + if (previous != '}') return false; + int opening = pattern.lastIndexOf('{', offset - 2); + return opening >= 0 && isValidQuantifier(pattern, opening) + && pattern.indexOf('}', opening + 1) == offset - 1; + } + + private static final class LeftBraceIssue { + final int offset; + final boolean alwaysFatal; + + LeftBraceIssue(int offset, boolean alwaysFatal) { + this.offset = offset; + this.alwaysFatal = alwaysFatal; + } + } + private static int debugMode(String modifiers) { if (modifiers == null) return 0; if (modifiers.indexOf(INTERNAL_DEBUGCOLOR_MARKER) >= 0) return 2; return modifiers.indexOf(INTERNAL_DEBUG_MARKER) >= 0 ? 1 : 0; } - private static String stripDebugMarkers(String modifiers) { + private static boolean reStrictMode(String modifiers) { + return modifiers != null && modifiers.indexOf(INTERNAL_RE_STRICT_MARKER) >= 0; + } + + private static String stripInternalMarkers(String modifiers) { if (modifiers == null || modifiers.isEmpty()) return modifiers == null ? "" : modifiers; return modifiers.replace(String.valueOf(INTERNAL_DEBUG_MARKER), "") - .replace(String.valueOf(INTERNAL_DEBUGCOLOR_MARKER), ""); + .replace(String.valueOf(INTERNAL_DEBUGCOLOR_MARKER), "") + .replace(String.valueOf(INTERNAL_RE_STRICT_MARKER), ""); } private void emitCompileDebugTrace() { @@ -1060,7 +1185,8 @@ private static RuntimeRegex ensureCompiledForRuntime(RuntimeRegex regex) { // makes a later qr/\\p{Property}/ reuse its match-any stand-in. state().compiledRegexCache.remove(cacheKey + "#debug=" + regex.lexicalDebugMode + "#callouts=0#backend=" + RegexBackendPolicy.cacheTag() - + "#bytepattern=" + regex.patternByteBacked); + + "#bytepattern=" + regex.patternByteBacked + + "#strict=" + regex.lexicalReStrict); // User property subs can execute arbitrary Perl and block. Resolve them // before compile() takes its process-wide monitor; only simultaneous @@ -1069,7 +1195,8 @@ private static RuntimeRegex ensureCompiledForRuntime(RuntimeRegex regex) { regex.regexFlags != null && regex.regexFlags.isCaseInsensitive()); RuntimeRegex recompiled = compile(regex.patternString, regex.regexFlags == null ? "" : regex.regexFlags.toFlagString(), - regex.lexicalDebugMode, 0, regex.patternByteBacked); + regex.lexicalDebugMode, 0, regex.patternByteBacked, + regex.lexicalReStrict); regex.pattern = recompiled.pattern; regex.patternUnicode = recompiled.patternUnicode; regex.recursivePattern = recompiled.recursivePattern; @@ -1085,6 +1212,7 @@ private static RuntimeRegex ensureCompiledForRuntime(RuntimeRegex regex) { regex.warningsOnUse = new ArrayList<>(recompiled.warningsOnUse); regex.inlineModifierWarnings = new ArrayList<>(recompiled.inlineModifierWarnings); regex.lexicalDebugMode = recompiled.lexicalDebugMode; + regex.lexicalReStrict = recompiled.lexicalReStrict; return regex; } @@ -1462,7 +1590,7 @@ public static RuntimeScalar applyUnicodeStringsFeatureToModifiers(RuntimeScalar public static RuntimeScalar getQuotedRegex(RuntimeScalar patternString, RuntimeScalar modifiers) { String rawModifierStr = modifiers.toString(); int callSiteDebugMode = debugMode(rawModifierStr); - String modifierStr = stripDebugMarkers(rawModifierStr); + String modifierStr = stripInternalMarkers(rawModifierStr); // Unwrap readonly scalar if (patternString.type == RuntimeScalarType.READONLY_SCALAR) patternString = (RuntimeScalar) patternString.value; @@ -1490,7 +1618,7 @@ && containsExecutableSource(patternString.toString(), modifierStr.indexOf('x') > } if (patternString.value instanceof RuntimeRegexTemplate template) { - RuntimeRegex regex = compile(template.pattern(), modifierStr, callSiteDebugMode, + RuntimeRegex regex = compile(template.pattern(), rawModifierStr, callSiteDebugMode, template.callbacks().size(), template.byteBackedPattern()).cloneTracked(); regex.setExecutableCallbacks(template.callbacks()); return new RuntimeScalar(regex).propagateTaint(patternString); @@ -1525,6 +1653,7 @@ && containsExecutableSource(patternString.toString(), modifierStr.indexOf('x') > regex.inlineModifierWarnings = new ArrayList<>(originalRegex.inlineModifierWarnings); regex.lexicalDebugMode = callSiteDebugMode != 0 ? callSiteDebugMode : originalRegex.lexicalDebugMode; + regex.lexicalReStrict = originalRegex.lexicalReStrict; regex.regexFlags = mergeRegexFlags(originalRegex.regexFlags, modifierStr, originalRegex.patternString); regex.hasPreservesMatch = regex.hasPreservesMatch || regex.regexFlags.preservesMatch(); regex.useGAssertion = regex.regexFlags.useGAssertion(); @@ -1569,6 +1698,7 @@ && containsExecutableSource(patternString.toString(), modifierStr.indexOf('x') > regex.inlineModifierWarnings = new ArrayList<>(originalRegex.inlineModifierWarnings); regex.lexicalDebugMode = callSiteDebugMode != 0 ? callSiteDebugMode : originalRegex.lexicalDebugMode; + regex.lexicalReStrict = originalRegex.lexicalReStrict; regex.regexFlags = mergeRegexFlags(originalRegex.regexFlags, modifierStr, originalRegex.patternString); regex.hasPreservesMatch = regex.hasPreservesMatch || regex.regexFlags.preservesMatch(); regex.useGAssertion = regex.regexFlags.useGAssertion(); @@ -1584,7 +1714,8 @@ && containsExecutableSource(patternString.toString(), modifierStr.indexOf('x') > // Try fallback to string conversion RuntimeScalar fallbackResult = overloadCtx.tryOverloadFallback(patternString, "(\"\""); if (fallbackResult != null) { - return new RuntimeScalar(compile(fallbackResult.toString(), modifierStr, callSiteDebugMode).cloneTracked()) + return new RuntimeScalar(compile(fallbackResult.toString(), rawModifierStr, + callSiteDebugMode).cloneTracked()) .propagateTaint(patternString, fallbackResult); } } @@ -1592,7 +1723,7 @@ && containsExecutableSource(patternString.toString(), modifierStr.indexOf('x') > // Default: compile as string (cloneTracked() creates a tracked copy // so the cached RuntimeRegex is not corrupted by refCount changes) - RuntimeRegex compiled = compile(patternString.toString(), modifierStr, + RuntimeRegex compiled = compile(patternString.toString(), rawModifierStr, callSiteDebugMode, 0, patternString.type == RuntimeScalarType.BYTE_STRING).cloneTracked(); return new RuntimeScalar(compiled).propagateTaint(patternString); @@ -1665,7 +1796,7 @@ static RuntimeScalar compileExecutableTemplate( List callbacks, RuntimeScalar original, boolean patternByteBacked) { int lexicalDebugMode = debugMode(modifiers); - RuntimeRegex regex = compile(executablePattern, stripDebugMarkers(modifiers), + RuntimeRegex regex = compile(executablePattern, modifiers, lexicalDebugMode, callbacks.size(), patternByteBacked).cloneTracked(); regex.setExecutableCallbacks(callbacks); return new RuntimeScalar(regex).propagateTaint(original); @@ -1728,7 +1859,7 @@ private static void validateTaintedPatternSecurity(RuntimeScalar patternString) */ public static RuntimeScalar getQuotedRegex(RuntimeScalar patternString, RuntimeScalar modifiers, int callsiteId) { String rawModifierStr = modifiers.toString(); - String modifierStr = stripDebugMarkers(rawModifierStr); + String modifierStr = stripInternalMarkers(rawModifierStr); // Check if /o or m?PAT? modifier is present (both need per-callsite caching // to preserve state: /o caches the compiled pattern, m?PAT? preserves the @@ -1765,7 +1896,7 @@ public static RuntimeScalar getReplacementRegex(RuntimeScalar patternString, Run RuntimeRegex resolvedRegex = resolved.regex(); String rawModifierStr = modifiers.toString(); int callSiteDebugMode = debugMode(rawModifierStr); - String modifierStr = stripDebugMarkers(rawModifierStr); + String modifierStr = stripInternalMarkers(rawModifierStr); // Create a new regex instance with the replacement RuntimeRegex regex = new RuntimeRegex(); @@ -1793,6 +1924,7 @@ public static RuntimeScalar getReplacementRegex(RuntimeScalar patternString, Run regex.inlineModifierWarnings = new ArrayList<>(resolvedRegex.inlineModifierWarnings); regex.lexicalDebugMode = callSiteDebugMode != 0 ? callSiteDebugMode : resolvedRegex.lexicalDebugMode; + regex.lexicalReStrict = resolvedRegex.lexicalReStrict; // Only recompile if we have new modifiers that actually change the flags if (!modifierStr.isEmpty()) { @@ -1813,7 +1945,7 @@ public static RuntimeScalar getReplacementRegex(RuntimeScalar patternString, Run RuntimeRegex recompiledRegex = compile(resolvedRegex.patternString, newFlags.toFlagString(), regex.lexicalDebugMode, resolvedRegex.executableCallbacks.size(), - resolvedRegex.patternByteBacked); + resolvedRegex.patternByteBacked, regex.lexicalReStrict); regex.pattern = recompiledRegex.pattern; regex.patternUnicode = recompiledRegex.patternUnicode; regex.recursivePattern = recompiledRegex.recursivePattern; @@ -2122,6 +2254,8 @@ private static RuntimeBase matchRegexDirect(RuntimeScalar quotedRegex, RuntimeSc tempRegex.lexicalDebugMode = regex.lexicalDebugMode != 0 ? regex.lexicalDebugMode : regexState.lastSuccessfulPattern.lexicalDebugMode; + tempRegex.lexicalReStrict = regex.lexicalReStrict + || regexState.lastSuccessfulPattern.lexicalReStrict; tempRegex.regexFlags = originalFlags; tempRegex.useGAssertion = originalFlags != null && originalFlags.useGAssertion(); regex = tempRegex; diff --git a/src/test/resources/unit/regex/regex_re_strict_left_brace_policy.t b/src/test/resources/unit/regex/regex_re_strict_left_brace_policy.t new file mode 100644 index 000000000..48916c904 --- /dev/null +++ b/src/test/resources/unit/regex/regex_re_strict_left_brace_policy.t @@ -0,0 +1,66 @@ +use strict; +use warnings; +use Test::More; + +sub capture_eval_string { + my ($source) = @_; + my @warnings; + local $SIG{__WARN__} = sub { push @warnings, @_ }; + my $value = eval $source; + return ($value, $@, \@warnings); +} + +sub compile_default { + my ($pattern) = @_; + my @warnings; + local $SIG{__WARN__} = sub { push @warnings, @_ }; + my $value = eval { qr/$pattern/ }; + return ($value, $@, \@warnings); +} + +{ + no warnings 'experimental::re_strict'; + use re 'strict'; + + sub compile_strict { + my ($pattern) = @_; + my @warnings; + local $SIG{__WARN__} = sub { push @warnings, @_ }; + my $value = eval { qr/$pattern/ }; + return ($value, $@, \@warnings); + } +} + +my ($value, $error, $warnings) = compile_default('\\w{'); +ok(!defined($value), 'ambiguous brace after escape is fatal by default'); +like($error, qr/^Unescaped left brace in regex is illegal here in regex; marked by <-- HERE in m\/\\w\{ <-- HERE \/ at /, + 'default fatal marker follows brace'); +is(scalar(@$warnings), 0, 'default fatal emits no warning'); + +($value, $error, $warnings) = compile_default(':{4,a}'); +ok(defined($value) && $error eq '', 'malformed quantifier-like brace passes by default'); +like($warnings->[0] // '', qr/^Unescaped left brace in regex is passed through in regex; marked by <-- HERE in m\/:\{ <-- HERE 4,a\}\/ at /, + 'default warning marker follows brace'); + +($value, $error, $warnings) = compile_strict(':{4,a}'); +ok(!defined($value), 'malformed quantifier-like brace is fatal under lexical strict'); +like($error, qr/^Unescaped left brace in regex is illegal here in regex; marked by <-- HERE in m\/:\{ <-- HERE 4,a\}\/ at /, + 'strict fatal marker follows brace'); +is(scalar(@$warnings), 0, 'strict fatal emits no warning'); + +($value, $error, $warnings) = capture_eval_string(q{qr/:{4,a}/}); +ok(defined($value) && $error eq '' && @$warnings == 1, + 'literal malformed brace warns outside strict'); + +($value, $error, $warnings) = capture_eval_string( + q{no warnings 'experimental::re_strict'; use re 'strict'; qr/:{4,a}/}); +ok(!defined($value) && $error =~ /^Unescaped left brace in regex is illegal here/, + 'literal malformed brace is fatal inside strict'); + +for my $pattern ('^{', 'foo|{', '\\s*{', 'a{3,4}{', 'foo(:?{bar)') { + ($value, $error, $warnings) = compile_strict($pattern); + ok(defined($value) && $error eq '' && @$warnings == 0, + "allowed brace context remains quiet: $pattern"); +} + +done_testing; From 1a55a2130b281463dc9cffb6d977fc59e96bf137 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 18:54:02 +0200 Subject: [PATCH 06/10] fix(regex): enforce strict non-hex escapes Make malformed non-hex escapes fatal under lexical use re 'strict', retain exactly one default warning for qr construction and match use, and preserve the source spelling of resolved braced hex digits. Exclude extended-class grammar from this source-policy correction. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../runtime/regex/RuntimeRegex.java | 101 +++++++++++++++++- .../regex/regex_re_strict_nonhex_policy.t | 63 +++++++++++ 2 files changed, 162 insertions(+), 2 deletions(-) create mode 100644 src/test/resources/unit/regex/regex_re_strict_nonhex_policy.t diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java index 93db78be2..38317e3ce 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java @@ -666,6 +666,7 @@ private static synchronized RuntimeRegex compileSynchronized( LeftBraceIssue leftBraceIssue = unescapedLeftBraceIssue( originalPatternString); String sourcePolicyWarning = null; + String constructionPolicyWarning = null; if (leftBraceIssue != null) { String message = leftBraceIssue.alwaysFatal || lexicalReStrict ? "Unescaped left brace in regex is illegal here" @@ -677,6 +678,23 @@ private static synchronized RuntimeRegex compileSynchronized( } sourcePolicyWarning = diagnostic; } + NonHexIssue nonHexIssue = nonHexEscapeIssue(originalPatternString); + if (nonHexIssue != null) { + if (lexicalReStrict) { + throw new PerlCompilerException(RegexDiagnosticFormatter.markedPerl( + originalPatternString, nonHexIssue.offset + 1, + "Non-hex character")); + } + if (!nonHexIssue.braced) { + char invalid = originalPatternString.charAt(nonHexIssue.offset); + char digit = originalPatternString.charAt(nonHexIssue.offset - 1); + String message = "Non-hex character '" + invalid + + "' terminates \\x early. Resolved as \"\\x0" + + digit + invalid + "\""; + constructionPolicyWarning = RegexDiagnosticFormatter.markedPerl( + originalPatternString, nonHexIssue.offset, message); + } + } // Always compute Unicode flags - we need the Unicode variant for when // the input string contains non-ASCII characters (auto-Unicode detection) @@ -721,6 +739,9 @@ private static synchronized RuntimeRegex compileSynchronized( regex.inlineModifierWarnings.add(sourcePolicyWarning); regex.warningsOnUse.add(sourcePolicyWarning); } + if (constructionPolicyWarning != null) { + regex.inlineModifierWarnings.add(constructionPolicyWarning); + } if (hasDeferredDynamicPattern) { regex.warningsOnUse.add(DYNAMIC_PATTERN_ERROR); } else if (hasWarnDynamicFallback) { @@ -747,8 +768,9 @@ private static synchronized RuntimeRegex compileSynchronized( regex.recursivePattern.hasDeferredUserDefinedUnicodeProperty() || regex.recursivePatternUnicode .hasDeferredUserDefinedUnicodeProperty(); - regex.inlineModifierWarnings.addAll( - regex.recursivePattern.compileWarnings()); + regex.inlineModifierWarnings.addAll(normalizeNonHexWarningCase( + regex.recursivePattern.compileWarnings(), + originalPatternString, nonHexIssue)); regex.warningsOnUse.addAll(regex.inlineModifierWarnings); regex.hasPreservesMatch = regex.regexFlags.preservesMatch() || RegexFlags.hasInlinePreserveModifier(compilePatternString); @@ -1095,6 +1117,81 @@ private static final class LeftBraceIssue { } } + private static NonHexIssue nonHexEscapeIssue(String pattern) { + if (pattern == null || pattern.contains("(?[")) return null; + for (int i = 0; i + 2 < pattern.length(); i++) { + if (pattern.charAt(i) != '\\' || pattern.charAt(i + 1) != 'x' + || (i > 0 && pattern.charAt(i - 1) == '\\')) { + continue; + } + int cursor = i + 2; + if (pattern.charAt(cursor) != '{') { + if (isHexDigit(pattern.charAt(cursor)) + && cursor + 1 < pattern.length() + && Character.isLetterOrDigit(pattern.charAt(cursor + 1)) + && !isHexDigit(pattern.charAt(cursor + 1))) { + return new NonHexIssue(cursor + 1, false); + } + continue; + } + int close = pattern.indexOf('}', cursor + 1); + if (close < 0) continue; + cursor++; + while (cursor < close && pattern.charAt(cursor) == ' ') cursor++; + int digits = cursor; + while (cursor < close && isHexDigit(pattern.charAt(cursor))) cursor++; + if (cursor == digits || cursor == close) { + i = close; + continue; + } + if (pattern.charAt(cursor) != ' ') return new NonHexIssue(cursor, true); + int whitespace = cursor; + while (cursor < close && pattern.charAt(cursor) == ' ') cursor++; + if (cursor < close) return new NonHexIssue(whitespace, true); + i = close; + } + return null; + } + + private static final class NonHexIssue { + final int offset; + final boolean braced; + + NonHexIssue(int offset, boolean braced) { + this.offset = offset; + this.braced = braced; + } + } + + private static List normalizeNonHexWarningCase( + List warnings, String pattern, NonHexIssue issue) { + if (issue == null || !issue.braced || warnings.isEmpty()) return warnings; + int opening = pattern.lastIndexOf('{', issue.offset); + if (opening < 0) return warnings; + String sourceDigits = pattern.substring(opening + 1, issue.offset).trim(); + if (sourceDigits.isEmpty()) return warnings; + List normalized = new ArrayList<>(warnings.size()); + String prefix = "Resolved as \"\\x{"; + for (String warning : warnings) { + int valueStart = warning.indexOf(prefix); + int valueEnd = valueStart < 0 ? -1 : warning.indexOf('}', valueStart + prefix.length()); + if (valueEnd >= 0) { + String resolved = warning.substring(valueStart + prefix.length(), valueEnd); + if (resolved.equalsIgnoreCase(sourceDigits)) { + warning = warning.substring(0, valueStart + prefix.length()) + + sourceDigits + warning.substring(valueEnd); + } + } + normalized.add(warning); + } + return normalized; + } + + private static boolean isHexDigit(char ch) { + return ch >= '0' && ch <= '9' || ch >= 'a' && ch <= 'f' + || ch >= 'A' && ch <= 'F'; + } + private static int debugMode(String modifiers) { if (modifiers == null) return 0; if (modifiers.indexOf(INTERNAL_DEBUGCOLOR_MARKER) >= 0) return 2; diff --git a/src/test/resources/unit/regex/regex_re_strict_nonhex_policy.t b/src/test/resources/unit/regex/regex_re_strict_nonhex_policy.t new file mode 100644 index 000000000..07470fe08 --- /dev/null +++ b/src/test/resources/unit/regex/regex_re_strict_nonhex_policy.t @@ -0,0 +1,63 @@ +use strict; +use warnings; +use Test::More; + +sub capture_eval_string { + my ($source) = @_; + my @warnings; + local $SIG{__WARN__} = sub { push @warnings, @_ }; + my $value = eval $source; + return ($value, $@, \@warnings); +} + +sub compile_default { + my ($pattern) = @_; + my @warnings; + local $SIG{__WARN__} = sub { push @warnings, @_ }; + my $value = eval { qr/$pattern/ }; + return ($value, $@, \@warnings); +} + +{ + no warnings 'experimental::re_strict'; + use re 'strict'; + + sub compile_strict { + my ($pattern) = @_; + my @warnings; + local $SIG{__WARN__} = sub { push @warnings, @_ }; + my $value = eval { qr/$pattern/ }; + return ($value, $@, \@warnings); + } +} + +my @cases = ( + ['\\xAG', 'm/\\xAG <-- HERE /'], + ['[\\xAG]', 'm/[\\xAG <-- HERE ]/'], + ['\\x{ABCDEFG}', 'm/\\x{ABCDEFG <-- HERE }/'], + ['[\\x{ABCDEFG}]', 'm/[\\x{ABCDEFG <-- HERE }]/'], + ['\\x{ 5 0 }', 'm/\\x{ 5 <-- HERE 0 }/'], +); + +for my $case (@cases) { + my ($pattern, $marked_pattern) = @$case; + my ($value, $error, $warnings) = compile_default($pattern); + ok(defined($value) && $error eq '', "non-hex escape passes by default: $pattern"); + is(scalar(@$warnings), 1, "default non-hex escape warns exactly once: $pattern"); + like($warnings->[0] // '', qr/^Non-hex character '.+' terminates \\x early\. Resolved as /, + "default non-hex warning retained: $pattern"); + + ($value, $error, $warnings) = compile_strict($pattern); + ok(!defined($value), "non-hex escape is fatal under lexical strict: $pattern"); + like($error, qr/^Non-hex character in regex; marked by <-- HERE in \Q$marked_pattern\E at /, + "strict non-hex marker: $pattern"); + is(scalar(@$warnings), 0, "strict non-hex fatal emits no warning: $pattern"); +} + +my ($literal, $literal_error, $literal_warnings) = capture_eval_string(q!qr/\xAG/!); +ok(defined($literal) && $literal_error eq '', 'literal unbraced non-hex escape compiles'); +is(scalar(@$literal_warnings), 1, 'literal unbraced non-hex escape warns exactly once'); +like($literal_warnings->[0], qr/^Non-hex character 'G' terminates \\x early/, + 'literal unbraced non-hex warning keeps Perl text'); + +done_testing; From 4392251261439ce3b5aa7cb6cda0ee5017c141f2 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 19:01:47 +0200 Subject: [PATCH 07/10] fix(regex): preserve boundary brace diagnostics Exclude uppercase \\B{...} boundary constructs from literal-left-brace classification so their dedicated empty, missing-brace, unknown-boundary, and /a warning diagnostics remain authoritative. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- .../org/perlonjava/runtime/regex/RuntimeRegex.java | 2 +- .../unit/regex/regex_re_strict_left_brace_policy.t | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java index 38317e3ce..a27049ef3 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java @@ -1071,7 +1071,7 @@ private static boolean isEscapeArgumentBrace(String pattern, int offset) { || (offset >= 3 && pattern.charAt(offset - 3) == '\\')) { return false; } - return "pPxXoONkgb".indexOf(pattern.charAt(offset - 1)) >= 0; + return "pPxXoONkgbB".indexOf(pattern.charAt(offset - 1)) >= 0; } private static boolean isValidQuantifier(String pattern, int offset) { diff --git a/src/test/resources/unit/regex/regex_re_strict_left_brace_policy.t b/src/test/resources/unit/regex/regex_re_strict_left_brace_policy.t index 48916c904..97ec7c967 100644 --- a/src/test/resources/unit/regex/regex_re_strict_left_brace_policy.t +++ b/src/test/resources/unit/regex/regex_re_strict_left_brace_policy.t @@ -63,4 +63,18 @@ for my $pattern ('^{', 'foo|{', '\\s*{', 'a{3,4}{', 'foo(:?{bar)') { "allowed brace context remains quiet: $pattern"); } +my @boundary_cases = ( + ['\\B{gc}', qr/^'gc' is an unknown bound type in regex/], + ['\\B{}', qr/^Empty \\B\{\} in regex/], + ['a\\B{cde', qr/^Missing right brace on \\B\{\} in regex/], +); +for my $case (@boundary_cases) { + my ($pattern, $expected) = @$case; + for my $compiler (\&compile_default, \&compile_strict) { + ($value, $error, $warnings) = $compiler->($pattern); + ok(!defined($value) && $error =~ $expected && @$warnings == 0, + "boundary brace keeps its dedicated diagnostic: $pattern"); + } +} + done_testing; From 7b6abe9399b673fc5d2bdae8c6b16b99fde9d257 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 19:28:43 +0200 Subject: [PATCH 08/10] fix(joni): report Perl empty character classes Give PerlNG empty classes Perl's unmatched-opening-bracket diagnostic and source position. Route this Perl spelling to Joni even in temporary explicit Java differential mode, and add system-Perl and direct-Joni coverage. This closes two unchanged regexp.t rows with no introduced failures. Generated with [OpenAI Codex](https://openai.com/codex/) Co-Authored-By: OpenAI Codex --- .../runtime/regex/JoniRegexPattern.java | 17 ++++++++++++++ .../regex/NativeEmptyClassRoutingTest.java | 22 +++++++++++++++++++ .../regex/empty_character_class_diagnostic.t | 11 ++++++++++ third_party/joni/src/org/joni/Parser.java | 8 ++++++- .../src/org/joni/exception/ErrorMessages.java | 1 + .../TestPerlCharacterClassDiagnostics.java | 10 +++++++++ 6 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 src/test/java/org/perlonjava/runtime/regex/NativeEmptyClassRoutingTest.java create mode 100644 src/test/resources/unit/regex/empty_character_class_diagnostic.t diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index 27fdd3c19..51ed7f54b 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -525,9 +525,26 @@ static boolean requiresJoniBackend(String pattern, RegexFlags flags) { || pattern.contains("(*COMMIT") || pattern.contains("(*MARK") || pattern.contains("(*:") + || containsPerlEmptyCharacterClass(pattern) || hasSubroutineCall; } + private static boolean containsPerlEmptyCharacterClass(String pattern) { + boolean quoted = false; + for (int i = 0; i + 1 < pattern.length(); i++) { + char ch = pattern.charAt(i); + if (ch == '\\') { + char next = pattern.charAt(i + 1); + if (quoted && next == 'E') quoted = false; + else if (!quoted && next == 'Q') quoted = true; + i++; + continue; + } + if (!quoted && ch == '[' && pattern.charAt(i + 1) == ']') return true; + } + return false; + } + static boolean containsNamedCharacterEscape(String pattern) { if (pattern == null) return false; for (int i = 0; i + 2 < pattern.length(); i++) { diff --git a/src/test/java/org/perlonjava/runtime/regex/NativeEmptyClassRoutingTest.java b/src/test/java/org/perlonjava/runtime/regex/NativeEmptyClassRoutingTest.java new file mode 100644 index 000000000..f0e6b403d --- /dev/null +++ b/src/test/java/org/perlonjava/runtime/regex/NativeEmptyClassRoutingTest.java @@ -0,0 +1,22 @@ +package org.perlonjava.runtime.regex; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +@Tag("unit") +class NativeEmptyClassRoutingTest { + @Test + void routesPerlEmptyClassToJoni() { + assertTrue(JoniRegexPattern.requiresJoniBackend("a[]b")); + assertTrue(JoniRegexPattern.requiresJoniBackend("(?i:a[]b)")); + } + + @Test + void ignoresEscapedAndQuotedBracketPairs() { + assertFalse(JoniRegexPattern.requiresJoniBackend("a\\[]b")); + assertFalse(JoniRegexPattern.requiresJoniBackend("\\Q[]\\E")); + } +} diff --git a/src/test/resources/unit/regex/empty_character_class_diagnostic.t b/src/test/resources/unit/regex/empty_character_class_diagnostic.t new file mode 100644 index 000000000..5cc70d35f --- /dev/null +++ b/src/test/resources/unit/regex/empty_character_class_diagnostic.t @@ -0,0 +1,11 @@ +use strict; +use warnings; +use Test::More; + +for my $pattern ('a[]b', '(?i:a[]b)') { + my $regex = eval { qr/$pattern/ }; + ok(!defined($regex), "$pattern is rejected"); + like($@, qr/^Unmatched \[/, "$pattern uses Perl unmatched-class diagnostic"); +} + +done_testing; diff --git a/third_party/joni/src/org/joni/Parser.java b/third_party/joni/src/org/joni/Parser.java index f25175eb6..9abfd1424 100644 --- a/third_party/joni/src/org/joni/Parser.java +++ b/third_party/joni/src/org/joni/Parser.java @@ -351,6 +351,7 @@ int apply(int option, int modifier, boolean neg, boolean rejectLocale) { private ParsedCharClass parseCharClass(ObjPtr ascNode, ObjPtr foldNode) { + int classContentStart = p - getBegin(); final boolean neg; CClassNode cc, prevCc = null, ascCc = null, ascPrevCc = null, workCc = null, ascWorkCc = null, foldCc = null, @@ -367,7 +368,12 @@ private ParsedCharClass parseCharClass(ObjPtr ascNode, } if (token.type == TokenType.CC_CLOSE && !syntax.op3OptionECMAScript()) { - if (!codeExistCheck(']', true)) newSyntaxException(EMPTY_CHAR_CLASS); + if (!codeExistCheck(']', true)) { + if (env.usesPerlDiagnostics()) { + newSyntaxException(PERL_UNMATCHED_OPEN_BRACKET, classContentStart); + } + newSyntaxException(EMPTY_CHAR_CLASS); + } env.ccEscWarn("]"); token.type = TokenType.CHAR; /* allow []...] */ } diff --git a/third_party/joni/src/org/joni/exception/ErrorMessages.java b/third_party/joni/src/org/joni/exception/ErrorMessages.java index 5146eebbd..3fbaed7be 100644 --- a/third_party/joni/src/org/joni/exception/ErrorMessages.java +++ b/third_party/joni/src/org/joni/exception/ErrorMessages.java @@ -36,6 +36,7 @@ public interface ErrorMessages extends org.jcodings.exception.ErrorMessages { String END_PATTERN_AT_LEFT_BRACE = "end pattern at left brace"; String END_PATTERN_AT_LEFT_BRACKET = "end pattern at left bracket"; String EMPTY_CHAR_CLASS = "empty char-class"; + String PERL_UNMATCHED_OPEN_BRACKET = "Unmatched ["; String PREMATURE_END_OF_CHAR_CLASS = "premature end of char-class"; String END_PATTERN_AT_ESCAPE = "end pattern at escape"; String END_PATTERN_AT_META = "end pattern at meta"; diff --git a/third_party/joni/test/org/joni/test/TestPerlCharacterClassDiagnostics.java b/third_party/joni/test/org/joni/test/TestPerlCharacterClassDiagnostics.java index f3b20da08..fa4ce7f33 100644 --- a/third_party/joni/test/org/joni/test/TestPerlCharacterClassDiagnostics.java +++ b/third_party/joni/test/org/joni/test/TestPerlCharacterClassDiagnostics.java @@ -55,4 +55,14 @@ private static void assertPosixError(String pattern, String message) { UTF8Encoding.INSTANCE, Syntax.PerlNG, WarnCallback.NONE)); assertEquals(message, error.getMessage()); } + + @Test + public void emptyClassUsesPerlUnmatchedBracketDiagnostic() { + byte[] bytes = "a[]b".getBytes(StandardCharsets.UTF_8); + SyntaxException error = assertThrows(SyntaxException.class, + () -> new Regex(bytes, 0, bytes.length, Option.NONE, + UTF8Encoding.INSTANCE, Syntax.PerlNG, WarnCallback.NONE)); + assertEquals("Unmatched [", error.getMessage()); + assertEquals(2, error.getPatternPosition()); + } } From c0db1f56d6d3933d0efb079c268db18d099a7c34 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 19:28:44 +0200 Subject: [PATCH 09/10] fix(regex): preserve legal strict escape and quantifier forms Keep underscore-separated braced hexadecimal escapes and omitted-lower-bound quantifiers out of lexical re-strict warning detection. Match Joni's accepted interval whitespace and add a system-Perl-validated quantifier fixture. This restores the full-build wide-scalar gate and prevents seven unchanged regexp.t regressions introduced by the lexical-policy tranche. Generated with [OpenAI Codex](https://openai.com/codex/) Co-Authored-By: OpenAI Codex --- .../runtime/regex/RuntimeRegex.java | 34 +++++++++++++++-- .../regex/quantifier_omitted_lower_bound.t | 38 +++++++++++++++++++ 2 files changed, 69 insertions(+), 3 deletions(-) create mode 100644 src/test/resources/unit/regex/quantifier_omitted_lower_bound.t diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java index a27049ef3..733637924 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java @@ -1076,18 +1076,34 @@ private static boolean isEscapeArgumentBrace(String pattern, int offset) { private static boolean isValidQuantifier(String pattern, int offset) { int cursor = offset + 1; + while (cursor < pattern.length() + && isPerlIntervalWhitespace(pattern.charAt(cursor))) cursor++; int digitStart = cursor; while (cursor < pattern.length() && Character.isDigit(pattern.charAt(cursor))) { cursor++; } - if (cursor == digitStart) return false; + boolean hasLow = cursor > digitStart; + while (cursor < pattern.length() + && isPerlIntervalWhitespace(pattern.charAt(cursor))) cursor++; if (cursor < pattern.length() && pattern.charAt(cursor) == ',') { cursor++; + while (cursor < pattern.length() + && isPerlIntervalWhitespace(pattern.charAt(cursor))) cursor++; + int highStart = cursor; while (cursor < pattern.length() && Character.isDigit(pattern.charAt(cursor))) { cursor++; } + boolean hasHigh = cursor > highStart; + while (cursor < pattern.length() + && isPerlIntervalWhitespace(pattern.charAt(cursor))) cursor++; + return (hasLow || hasHigh) && cursor < pattern.length() + && pattern.charAt(cursor) == '}'; } - return cursor < pattern.length() && pattern.charAt(cursor) == '}'; + return hasLow && cursor < pattern.length() && pattern.charAt(cursor) == '}'; + } + + private static boolean isPerlIntervalWhitespace(char ch) { + return ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' || ch == '\f'; } private static boolean isAllowedLiteralLeftBrace(String pattern, int offset) { @@ -1139,7 +1155,19 @@ private static NonHexIssue nonHexEscapeIssue(String pattern) { cursor++; while (cursor < close && pattern.charAt(cursor) == ' ') cursor++; int digits = cursor; - while (cursor < close && isHexDigit(pattern.charAt(cursor))) cursor++; + while (cursor < close) { + if (isHexDigit(pattern.charAt(cursor))) { + cursor++; + continue; + } + if (pattern.charAt(cursor) == '_' && cursor > digits + && cursor + 1 < close + && isHexDigit(pattern.charAt(cursor + 1))) { + cursor++; + continue; + } + break; + } if (cursor == digits || cursor == close) { i = close; continue; diff --git a/src/test/resources/unit/regex/quantifier_omitted_lower_bound.t b/src/test/resources/unit/regex/quantifier_omitted_lower_bound.t new file mode 100644 index 000000000..4872d5666 --- /dev/null +++ b/src/test/resources/unit/regex/quantifier_omitted_lower_bound.t @@ -0,0 +1,38 @@ +use strict; +use warnings; +use Test::More; + +sub compile_pattern { + my ($pattern, $strict) = @_; + my (@warnings, $regex, $error); + { + local $SIG{__WARN__} = sub { push @warnings, @_ }; + if ($strict) { + no warnings 'experimental::re_strict'; + use re 'strict'; + $regex = eval { qr/$pattern/ }; + } + else { + $regex = eval { qr/$pattern/ }; + } + $error = $@; + } + return ($regex, $error, \@warnings); +} + +for my $case ( + [ 'a{,2}', 'aa' ], + [ 'a{, 2 }', 'aa' ], + [ 'a{ , 2 }', 'aa' ], + [ '[x]{, 2}', 'xx' ], + [ '\p{Latin}{ , 2 }', 'a' ], +) { + my ($pattern, $subject) = @$case; + for my $strict (0, 1) { + my ($regex, $error, $warnings) = compile_pattern($pattern, $strict); + ok(defined($regex) && $error eq '' && !@$warnings && $subject =~ /\A$regex\z/, + "$pattern is a quiet quantifier" . ($strict ? ' under re strict' : '')); + } +} + +done_testing; From 4da58459cf5bfc7ee6274e22ac89c00ba14ff43b Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 19:35:23 +0200 Subject: [PATCH 10/10] fix(joni): report leading quantifiers like Perl Use Perl's leading-quantifier diagnostic under PerlNG syntax while preserving upstream Joni wording for other syntaxes. Add system-Perl and direct-Joni coverage for top-level, alternation, and case-insensitive forms. This closes four unchanged regexp.t rows with no introduced failures. Generated with [OpenAI Codex](https://openai.com/codex/) Co-Authored-By: OpenAI Codex --- .../regex/quantifier_follows_nothing_diagnostic.t | 12 ++++++++++++ third_party/joni/src/org/joni/Parser.java | 4 +++- .../joni/src/org/joni/exception/ErrorMessages.java | 1 + .../joni/test/TestPerlCharacterClassDiagnostics.java | 10 ++++++++++ 4 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 src/test/resources/unit/regex/quantifier_follows_nothing_diagnostic.t diff --git a/src/test/resources/unit/regex/quantifier_follows_nothing_diagnostic.t b/src/test/resources/unit/regex/quantifier_follows_nothing_diagnostic.t new file mode 100644 index 000000000..3230e1f48 --- /dev/null +++ b/src/test/resources/unit/regex/quantifier_follows_nothing_diagnostic.t @@ -0,0 +1,12 @@ +use strict; +use warnings; +use Test::More; + +for my $pattern ('*a', '(|*)b', '(?i:*a)', '(?i:(|*)b)') { + my $regex = eval { qr/$pattern/ }; + ok(!defined($regex), "$pattern is rejected"); + like($@, qr/^Quantifier follows nothing/, + "$pattern uses Perl leading-quantifier diagnostic"); +} + +done_testing; diff --git a/third_party/joni/src/org/joni/Parser.java b/third_party/joni/src/org/joni/Parser.java index 9abfd1424..070c8f4d8 100644 --- a/third_party/joni/src/org/joni/Parser.java +++ b/third_party/joni/src/org/joni/Parser.java @@ -1545,7 +1545,9 @@ private Node parseExp(TokenType term) { case INTERVAL: if (syntax.contextIndepRepeatOps()) { if (syntax.contextInvalidRepeatOps()) { - newSyntaxException(TARGET_OF_REPEAT_OPERATOR_NOT_SPECIFIED); + newSyntaxException(env.usesPerlDiagnostics() + ? PERL_QUANTIFIER_FOLLOWS_NOTHING + : TARGET_OF_REPEAT_OPERATOR_NOT_SPECIFIED); } else { node = StringNode.EMPTY; // node_new_empty } diff --git a/third_party/joni/src/org/joni/exception/ErrorMessages.java b/third_party/joni/src/org/joni/exception/ErrorMessages.java index 3fbaed7be..80d5f29ab 100644 --- a/third_party/joni/src/org/joni/exception/ErrorMessages.java +++ b/third_party/joni/src/org/joni/exception/ErrorMessages.java @@ -47,6 +47,7 @@ public interface ErrorMessages extends org.jcodings.exception.ErrorMessages { String CHAR_CLASS_VALUE_AT_START_OF_RANGE = "char-class value at start of range"; String UNMATCHED_RANGE_SPECIFIER_IN_CHAR_CLASS = "unmatched range specifier in char-class"; String TARGET_OF_REPEAT_OPERATOR_NOT_SPECIFIED = "target of repeat operator is not specified"; + String PERL_QUANTIFIER_FOLLOWS_NOTHING = "Quantifier follows nothing"; String TARGET_OF_REPEAT_OPERATOR_INVALID = "target of repeat operator is invalid"; String NESTED_REPEAT_NOT_ALLOWED = "nested repeat is not allowed"; String NESTED_REPEAT_OPERATOR = "nested repeat operator"; diff --git a/third_party/joni/test/org/joni/test/TestPerlCharacterClassDiagnostics.java b/third_party/joni/test/org/joni/test/TestPerlCharacterClassDiagnostics.java index fa4ce7f33..cc1b5621b 100644 --- a/third_party/joni/test/org/joni/test/TestPerlCharacterClassDiagnostics.java +++ b/third_party/joni/test/org/joni/test/TestPerlCharacterClassDiagnostics.java @@ -65,4 +65,14 @@ public void emptyClassUsesPerlUnmatchedBracketDiagnostic() { assertEquals("Unmatched [", error.getMessage()); assertEquals(2, error.getPatternPosition()); } + + @Test + public void leadingQuantifierUsesPerlDiagnostic() { + byte[] bytes = "*a".getBytes(StandardCharsets.UTF_8); + SyntaxException error = assertThrows(SyntaxException.class, + () -> new Regex(bytes, 0, bytes.length, Option.NONE, + UTF8Encoding.INSTANCE, Syntax.PerlNG, WarnCallback.NONE)); + assertEquals("Quantifier follows nothing", error.getMessage()); + assertEquals(1, error.getPatternPosition()); + } }