diff --git a/docs/reference/feature-matrix.md b/docs/reference/feature-matrix.md index 3a7bb069c..ecb0b961c 100644 --- a/docs/reference/feature-matrix.md +++ b/docs/reference/feature-matrix.md @@ -386,7 +386,7 @@ my @copy = @{$z}; # ERROR - ✅ **Preprocessor**: `\Q`, `\L`, `\U`, `\l`, `\u`, `\E` are preprocessed in regex. - ✅ **Overloading**: `qr` overloading is implemented. See also [overload pragma](#pragmas). - ✅ **Python-style named groups**: `(?P...)` and `(?P=name)` are parsed natively by Joni with Perl capture numbering, duplicate-name behavior, and malformed/unknown-name diagnostics. -- ✅ **Alpha assertion aliases**: `(*pla:...)`, `(*plb:...)`, `(*nla:...)`, `(*nlb:...)`, and `(*atomic:...)` are parsed natively by Joni with Perl nesting, capture numbering, backtracking, and malformed-form diagnostics. +- ✅ **Alpha assertion aliases**: `(*pla:...)`, `(*plb:...)`, `(*nla:...)`, `(*nlb:...)`, `(*atomic:...)`, and the corresponding long spellings are parsed natively by Joni with Perl nesting, capture numbering, backtracking, assertion-condition predicates, and malformed-form diagnostics. - 🟡 **Underscored numeric regex escapes**: Joni natively parses Perl spellings such as `\x{0_0_4_1}` and `\o{0_0_1_0_1}` through U+10FFFF, including literal/class forms, bare high-octal UTF-8 code points, truncation behavior, and structural diagnostics. The frontend normalization remains for forced-Java compatibility; exact `use re 'strict'` diagnostics and Perl code points above U+10FFFF through signed IV max remain source-policy/representation debt. - ✅ **Dynamically-scoped regex variables**: Provisional captures, `$^R`, `$^N`, match positions, and callback locals follow matcher paths and unwind on backtracking. diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index 348a17d38..a5f82224f 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -16,6 +16,7 @@ import org.joni.Syntax; import static org.joni.constants.SyntaxProperties.ALLOW_MULTIPLEX_DEFINITION_NAME_CALL; +import static org.joni.constants.SyntaxProperties.OP2_ESC_H_HORIZONTAL_WHITESPACE; import static org.joni.constants.SyntaxProperties.OP2_OPTION_PERL; import static org.joni.constants.SyntaxProperties.OP2_OPTION_RUBY; import static org.joni.constants.SyntaxProperties.OP2_PLUS_POSSESSIVE_INTERVAL; @@ -45,7 +46,8 @@ final class JoniRegexPattern { // by callouts and control verbs while changing only that default policy. private static final Syntax PERLONJAVA_SYNTAX = new Syntax( "PERLONJAVA", Syntax.RUBY.op, - (Syntax.RUBY.op2 & ~OP2_OPTION_RUBY) | OP2_OPTION_PERL | OP2_PLUS_POSSESSIVE_INTERVAL, + (Syntax.RUBY.op2 & ~OP2_OPTION_RUBY) | OP2_OPTION_PERL + | OP2_PLUS_POSSESSIVE_INTERVAL | OP2_ESC_H_HORIZONTAL_WHITESPACE, Syntax.RUBY.op3, Syntax.RUBY.behavior | ALLOW_MULTIPLEX_DEFINITION_NAME_CALL, Syntax.RUBY.options & ~(Option.ASCII_RANGE @@ -348,6 +350,7 @@ static boolean requiresJoniBackend(String pattern, RegexFlags flags) { pattern, flags != null && flags.isExtended()); return syntaxFeatures.keepPresent() || syntaxFeatures.conditionalPresent() + || syntaxFeatures.alphaAssertionPresent() || pattern.contains("(?{=CALL:") || pattern.contains("(?{=DYNAMIC:") || pattern.contains("(*ACCEPT)") @@ -362,7 +365,8 @@ static boolean requiresJoniBackend(String pattern, RegexFlags flags) { private record PerlSyntaxFeatures(boolean keepPresent, boolean keepInLookaround, - boolean conditionalPresent) {} + boolean conditionalPresent, + boolean alphaAssertionPresent) {} private static PerlSyntaxFeatures analyzePerlSyntax(String pattern, boolean extended) { boolean quoted = false; @@ -373,6 +377,7 @@ private static PerlSyntaxFeatures analyzePerlSyntax(String pattern, boolean exte java.util.ArrayDeque groups = new java.util.ArrayDeque<>(); boolean keepPresent = false; boolean conditionalPresent = false; + boolean alphaAssertionPresent = false; for (int i = 0; i < pattern.length(); i++) { char ch = pattern.charAt(i); @@ -436,7 +441,8 @@ private static PerlSyntaxFeatures analyzePerlSyntax(String pattern, boolean exte } else if (escaped == 'K') { keepPresent = true; if (lookaroundDepth > 0) { - return new PerlSyntaxFeatures(true, true, conditionalPresent); + return new PerlSyntaxFeatures(true, true, conditionalPresent, + alphaAssertionPresent); } } continue; @@ -449,6 +455,24 @@ private static PerlSyntaxFeatures analyzePerlSyntax(String pattern, boolean exte continue; } if (pattern.startsWith("(?(", i)) conditionalPresent = true; + if (pattern.startsWith("(*", i)) { + int nameEnd = i + 2; + while (nameEnd < pattern.length()) { + char nameChar = pattern.charAt(nameEnd); + if (!Character.isLetter(nameChar) && nameChar != '_') break; + nameEnd++; + } + String name = pattern.substring(i + 2, nameEnd); + alphaAssertionPresent |= name.equals("pla") + || name.equals("positive_lookahead") + || name.equals("plb") + || name.equals("positive_lookbehind") + || name.equals("nla") + || name.equals("negative_lookahead") + || name.equals("nlb") + || name.equals("negative_lookbehind") + || name.equals("atomic"); + } boolean lookaround = pattern.startsWith("(?=", i) || pattern.startsWith("(?!", i) || pattern.startsWith("(?<=", i) @@ -459,7 +483,8 @@ private static PerlSyntaxFeatures analyzePerlSyntax(String pattern, boolean exte if (groups.pop()) lookaroundDepth--; } } - return new PerlSyntaxFeatures(keepPresent, false, conditionalPresent); + return new PerlSyntaxFeatures(keepPresent, false, conditionalPresent, + alphaAssertionPresent); } private static boolean hasControlVerbState(String pattern) { diff --git a/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessor.java b/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessor.java index 4897b45d7..d1dfac45c 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessor.java +++ b/src/main/java/org/perlonjava/runtime/regex/RegexPreprocessor.java @@ -991,8 +991,8 @@ private static int handleParentheses(String s, int offset, int length, StringBui // Check for (*...) verb patterns FIRST, before checking (? if (c2 == '*') { - // (*...) control verbs like (*ACCEPT), (*FAIL), (*COMMIT), etc. - // Also handles alpha assertion aliases: (*pla:...), (*plb:...), etc. + // Java-backend compatibility for (*...) control verbs such as + // (*FAIL). Alpha assertions are routed to Joni before preprocessing. // Find the verb name (up to ':' or ')') int verbNameEnd = offset + 2; @@ -1012,42 +1012,26 @@ private static int handleParentheses(String s, int offset, int length, StringBui return verbNameEnd; } - // Check for alpha assertion aliases (Perl 5.28+) - String replacement = switch (verbName) { - case "pla", "positive_lookahead" -> "(?="; - case "plb", "positive_lookbehind" -> "(?<="; - case "nla", "negative_lookahead" -> "(?!"; - case "nlb", "negative_lookbehind" -> "(? "(?>"; - default -> null; - }; - - if (replacement != null && verbNameEnd < length && s.codePointAt(verbNameEnd) == ':') { - // Alpha assertion with content: (*pla:...) -> (?=...) - sb.append(replacement); - offset = handleRegex(s, verbNameEnd + 1, sb, regexFlags, true); - // Fall through to common ')' handling at end of handleParentheses - } else { - // Find the end of the verb for error reporting - int verbEnd = offset + 2; - while (verbEnd < length && s.codePointAt(verbEnd) != ')') { - verbEnd++; - } - if (verbEnd < length) { - verbEnd++; // Include the closing paren - } + // Find the end of the verb for error reporting + int verbEnd = offset + 2; + while (verbEnd < length && s.codePointAt(verbEnd) != ')') { + verbEnd++; + } + if (verbEnd < length) { + verbEnd++; // Include the closing paren + } - // Extract the verb name for error reporting - String verb = s.substring(offset, Math.min(verbEnd, length)); + // Extract the verb name for error reporting + String verb = s.substring(offset, Math.min(verbEnd, length)); - // Replace with empty non-capturing group as placeholder - sb.append("(?:)"); + // Replace with empty non-capturing group as placeholder + sb.append("(?:)"); - // Throw error that can be caught by JPERL_UNIMPLEMENTED=warn - regexUnimplemented(s, offset + 2, "Regex control verb " + verb + " not implemented"); + // Throw error that can be caught by JPERL_UNIMPLEMENTED=warn + regexUnimplemented(s, offset + 2, + "Regex control verb " + verb + " not implemented"); - return verbEnd; // Skip past the entire verb construct - } + return verbEnd; // Skip past the entire verb construct } else if (c2 == '?') { if (offset + 2 >= length) { // Marker should be after the ? diff --git a/src/test/java/org/perlonjava/runtime/regex/NativeAlphaAssertionRoutingTest.java b/src/test/java/org/perlonjava/runtime/regex/NativeAlphaAssertionRoutingTest.java new file mode 100644 index 000000000..2a7f75422 --- /dev/null +++ b/src/test/java/org/perlonjava/runtime/regex/NativeAlphaAssertionRoutingTest.java @@ -0,0 +1,34 @@ +package org.perlonjava.runtime.regex; + +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@Tag("unit") +class NativeAlphaAssertionRoutingTest { + @Test + void routesShortAndLongAlphaAssertionsToJoni() { + assertTrue(JoniRegexPattern.requiresJoniBackend("a(*pla:b)b")); + assertTrue(JoniRegexPattern.requiresJoniBackend("a(*positive_lookahead:b)b")); + assertTrue(JoniRegexPattern.requiresJoniBackend("a(*plb:a)b")); + assertTrue(JoniRegexPattern.requiresJoniBackend("a(*positive_lookbehind:a)b")); + assertTrue(JoniRegexPattern.requiresJoniBackend("a(*nla:c)b")); + assertTrue(JoniRegexPattern.requiresJoniBackend("a(*negative_lookahead:c)b")); + assertTrue(JoniRegexPattern.requiresJoniBackend("a(*nlb:c)b")); + assertTrue(JoniRegexPattern.requiresJoniBackend("a(*negative_lookbehind:c)b")); + assertTrue(JoniRegexPattern.requiresJoniBackend("(*atomic:a|ab)c")); + assertTrue(JoniRegexPattern.requiresJoniBackend("(*pla)")); + assertTrue(JoniRegexPattern.requiresJoniBackend("(*positive_lookahead")); + } + + @Test + void ignoresAlphaAssertionLookalikes() { + assertFalse(JoniRegexPattern.requiresJoniBackend("\\(\\*pla:a\\)")); + assertFalse(JoniRegexPattern.requiresJoniBackend("[(?*pla:)]")); + assertFalse(JoniRegexPattern.requiresJoniBackend("\\Q(*pla:a)\\E")); + assertFalse(JoniRegexPattern.requiresJoniBackend("(?# (*pla:a))ordinary")); + assertFalse(JoniRegexPattern.requiresJoniBackend("(*planet:a)")); + } +} diff --git a/src/test/resources/unit/regex/alpha_assertion_native_routing.t b/src/test/resources/unit/regex/alpha_assertion_native_routing.t new file mode 100644 index 000000000..4785365d1 --- /dev/null +++ b/src/test/resources/unit/regex/alpha_assertion_native_routing.t @@ -0,0 +1,39 @@ +use strict; +use warnings; +use Test::More; + +ok('ab' =~ /a(*pla:b)b/, 'short positive lookahead alias'); +ok('ab' =~ /a(*positive_lookahead:b)b/, 'long positive lookahead alias'); +ok('ab' =~ /a(*plb:a)b/, 'short positive lookbehind alias'); +ok('ab' =~ /a(*positive_lookbehind:a)b/, 'long positive lookbehind alias'); +ok('ab' =~ /a(*nla:c)b/, 'short negative lookahead alias'); +ok('ab' =~ /a(*negative_lookahead:c)b/, 'long negative lookahead alias'); +ok('ab' =~ /a(*nlb:c)b/, 'short negative lookbehind alias'); +ok('ab' =~ /a(*negative_lookbehind:c)b/, 'long negative lookbehind alias'); + +ok('abc' !~ /(*atomic:a|ab)c/, 'atomic alias prevents alternative retry'); +ok('ab' =~ /a(*pla:(*nla:c)b)b/, 'nested alpha assertions'); + +my $captured = 'ab'; +ok($captured =~ /a(*pla:(b))b/, 'capture inside alpha assertion participates'); +is($1, 'b', 'alpha assertion publishes its capture'); + +ok('a' =~ /(?(*pla:a)a|b)/, + 'positive alpha assertion works as a conditional predicate'); +ok('b' =~ /(?(*pla:a)a|b)/, + 'positive alpha assertion conditional takes its alternate'); +ok('a' =~ /(?(*nla:a)b|a)/, + 'negative alpha assertion conditional takes its alternate'); +ok('b' =~ /(?(*nla:a)b|a)/, + 'negative alpha assertion works as a conditional predicate'); + +for my $invalid ( + '(*positive_lookahead)', + '(*positive_lookahead:a', + '(*positive_lookaround:a)', +) { + my $compiled = eval "qr/$invalid/"; + ok(!defined($compiled) && length($@), "malformed long alias is rejected: $invalid"); +} + +done_testing; diff --git a/src/test/resources/unit/regex/horizontal_whitespace_escape.t b/src/test/resources/unit/regex/horizontal_whitespace_escape.t new file mode 100644 index 000000000..1a14c759f --- /dev/null +++ b/src/test/resources/unit/regex/horizontal_whitespace_escape.t @@ -0,0 +1,83 @@ +use strict; +use warnings; +use feature 'unicode_strings'; +use Test::More; + +my @horizontal = (0x09, 0x20, 0xA0, 0x1680, 0x2000, 0x200A, + 0x202F, 0x205F, 0x3000); +my @other = (0x0A, 0x0B, 0x41, 0x85, 0x180E, 0x2028, 0x2029); + +sub check_code_point { + my ($code, $expected, $upgraded) = @_; + my $character = chr($code); + if ($upgraded) { + utf8::upgrade($character); + } + else { + utf8::downgrade($character, 1) + or die sprintf "U+%04X cannot be represented as bytes", $code; + } + my $mode = $upgraded ? 'Unicode' : 'byte'; + my $label = sprintf 'U+%04X %s', $code, $mode; + + is($character =~ /\A\h\z/ ? 1 : 0, $expected, + "direct h $label"); + is($character =~ /\A\H\z/ ? 1 : 0, 1 - $expected, + "direct H $label"); + is($character =~ /\A[\h]\z/ ? 1 : 0, $expected, + "class h $label"); + is($character =~ /\A[\H]\z/ ? 1 : 0, 1 - $expected, + "class H $label"); +} + +for my $code (@horizontal) { + check_code_point($code, 1, 1); + check_code_point($code, 1, 0) if $code <= 0xFF; +} +for my $code (@other) { + check_code_point($code, 0, 1); + check_code_point($code, 0, 0) if $code <= 0xFF; +} + +my $horizontal_run = join '', map chr, @horizontal; +my $other_run = join '', map chr, @other; +ok($horizontal_run =~ /\A\h+\z/, + 'direct h matches a run of horizontal whitespace'); +ok($horizontal_run =~ /\A[\h]+\z/, + 'class h matches a run of horizontal whitespace'); +ok($other_run =~ /\A\H+\z/, + 'direct H matches a run without horizontal whitespace'); +ok($other_run =~ /\A[\H]+\z/, + 'class H matches a run without horizontal whitespace'); + +my $ideographic_space = chr 0x3000; +my $line_feed = "\n"; +ok($ideographic_space =~ /\A(?a:\h)\z/, + 'scoped a keeps direct h Unicode-aware'); +ok($ideographic_space =~ /\A(?aa:\h)\z/, + 'scoped aa keeps direct h Unicode-aware'); +ok($ideographic_space =~ /\A(?a:[\h])\z/, + 'scoped a keeps class h Unicode-aware'); +ok($ideographic_space =~ /\A(?aa:[\h])\z/, + 'scoped aa keeps class h Unicode-aware'); +ok($line_feed =~ /\A(?a:\H)\z/, + 'scoped a keeps direct H complement semantics'); +ok($line_feed =~ /\A(?aa:\H)\z/, + 'scoped aa keeps direct H complement semantics'); +ok($line_feed =~ /\A(?a:[\H])\z/, + 'scoped a keeps class H complement semantics'); +ok($line_feed =~ /\A(?aa:[\H])\z/, + 'scoped aa keeps class H complement semantics'); + +my $byte_nbsp = chr 0xA0; +utf8::downgrade($byte_nbsp, 1); +ok($byte_nbsp =~ /\A(?a:\h)\z/, + 'scoped a keeps direct h byte semantics'); +ok($byte_nbsp =~ /\A(?aa:\h)\z/, + 'scoped aa keeps direct h byte semantics'); +ok($byte_nbsp =~ /\A(?a:[\h])\z/, + 'scoped a keeps class h byte semantics'); +ok($byte_nbsp =~ /\A(?aa:[\h])\z/, + 'scoped aa keeps class h byte semantics'); + +done_testing(); diff --git a/third_party/joni/src/org/joni/Lexer.java b/third_party/joni/src/org/joni/Lexer.java index ca80215d6..cfceac4e5 100644 --- a/third_party/joni/src/org/joni/Lexer.java +++ b/third_party/joni/src/org/joni/Lexer.java @@ -44,6 +44,9 @@ class Lexer extends ScannerSupport { protected final ScanEnvironment env; protected final Syntax syntax; // fast access to syntax protected final Token token = new Token(); // current token + private int perlHorizontalWhitespaceTokenIndex = -1; + private boolean perlHorizontalWhitespaceNegated; + private boolean perlHorizontalWhitespaceSingleByte; private int perlVerticalWhitespaceTokenIndex = -1; private boolean perlVerticalWhitespaceNegated; @@ -552,6 +555,84 @@ private void fetchTokenInCCFor_charType(boolean flag, int type) { token.setPropNot(flag); } + private void startPerlHorizontalWhitespace(boolean negated, TokenType openType) { + perlHorizontalWhitespaceTokenIndex = 0; + perlHorizontalWhitespaceNegated = negated; + perlHorizontalWhitespaceSingleByte = enc.isSingleByte(); + token.type = openType; + } + + private TokenType fetchPerlHorizontalWhitespaceToken() { + token.base = 0; + token.escaped = false; + + int index = perlHorizontalWhitespaceTokenIndex++; + if (perlHorizontalWhitespaceNegated) { + if (index == 0) { + token.type = TokenType.CHAR; + token.setC('^'); + return token.type; + } + index--; + } + + if (perlHorizontalWhitespaceSingleByte && index >= 3) { + token.type = TokenType.CC_CLOSE; + token.setC(']'); + perlHorizontalWhitespaceTokenIndex = -1; + return token.type; + } + + switch (index) { + case 0: + token.type = TokenType.CODE_POINT; + token.setCode(0x09); + break; + case 1: + token.type = TokenType.CODE_POINT; + token.setCode(0x20); + break; + case 2: + token.type = TokenType.CODE_POINT; + token.setCode(0xa0); + break; + case 3: + token.type = TokenType.CODE_POINT; + token.setCode(0x1680); + break; + case 4: + token.type = TokenType.CODE_POINT; + token.setCode(0x2000); + break; + case 5: + token.type = TokenType.CC_RANGE; + token.setC('-'); + break; + case 6: + token.type = TokenType.CODE_POINT; + token.setCode(0x200a); + break; + case 7: + token.type = TokenType.CODE_POINT; + token.setCode(0x202f); + break; + case 8: + token.type = TokenType.CODE_POINT; + token.setCode(0x205f); + break; + case 9: + token.type = TokenType.CODE_POINT; + token.setCode(0x3000); + break; + default: + token.type = TokenType.CC_CLOSE; + token.setC(']'); + perlHorizontalWhitespaceTokenIndex = -1; + break; + } + return token.type; + } + private boolean usesPerlVerticalWhitespaceEscape() { return syntax.op2EscVVerticalWhiteSpace() || syntax.op2OptionPerl() && (syntax == Syntax.Perl @@ -914,6 +995,9 @@ private void fetchTokenInCCFor_and() { } protected final TokenType fetchTokenInCC() { + if (perlHorizontalWhitespaceTokenIndex >= 0) { + return fetchPerlHorizontalWhitespaceToken(); + } if (perlVerticalWhitespaceTokenIndex >= 0) { return fetchPerlVerticalWhitespaceToken(); } @@ -959,10 +1043,26 @@ protected final TokenType fetchTokenInCC() { fetchTokenInCCFor_charType(true, CharacterType.SPACE); break; case 'h': - if (syntax.op2EscHXDigit()) fetchTokenInCCFor_charType(false, CharacterType.XDIGIT); + if (syntax.op2EscHHorizontalWhiteSpace()) { + if (enc.isSingleByte() || isAsciiRange(env.option)) { + startPerlHorizontalWhitespace(false, TokenType.CC_CC_OPEN); + } else { + fetchTokenInCCFor_charType(false, CharacterType.BLANK); + } + } else if (syntax.op2EscHXDigit()) { + fetchTokenInCCFor_charType(false, CharacterType.XDIGIT); + } break; case 'H': - if (syntax.op2EscHXDigit()) fetchTokenInCCFor_charType(true, CharacterType.XDIGIT); + if (syntax.op2EscHHorizontalWhiteSpace()) { + if (enc.isSingleByte() || isAsciiRange(env.option)) { + startPerlHorizontalWhitespace(true, TokenType.CC_CC_OPEN); + } else { + fetchTokenInCCFor_charType(true, CharacterType.BLANK); + } + } else if (syntax.op2EscHXDigit()) { + fetchTokenInCCFor_charType(true, CharacterType.XDIGIT); + } break; case 'p': case 'P': @@ -1418,10 +1518,26 @@ protected final void fetchToken() { if (syntax.opEscDDigit()) fetchTokenInCCFor_charType(true, CharacterType.DIGIT); break; case 'h': - if (syntax.op2EscHXDigit()) fetchTokenInCCFor_charType(false, CharacterType.XDIGIT); + if (syntax.op2EscHHorizontalWhiteSpace()) { + if (enc.isSingleByte()) { + startPerlHorizontalWhitespace(false, TokenType.CC_OPEN); + } else { + fetchTokenInCCFor_charType(false, CharacterType.BLANK); + } + } else if (syntax.op2EscHXDigit()) { + fetchTokenInCCFor_charType(false, CharacterType.XDIGIT); + } break; case 'H': - if (syntax.op2EscHXDigit()) fetchTokenInCCFor_charType(true, CharacterType.XDIGIT); + if (syntax.op2EscHHorizontalWhiteSpace()) { + if (enc.isSingleByte()) { + startPerlHorizontalWhitespace(true, TokenType.CC_OPEN); + } else { + fetchTokenInCCFor_charType(true, CharacterType.BLANK); + } + } else if (syntax.op2EscHXDigit()) { + fetchTokenInCCFor_charType(true, CharacterType.XDIGIT); + } break; case 'A': if (syntax.opEscAZBufAnchor()) fetchTokenFor_anchor(AnchorType.BEGIN_BUF); diff --git a/third_party/joni/src/org/joni/Parser.java b/third_party/joni/src/org/joni/Parser.java index cfcee0f48..d42d906c3 100644 --- a/third_party/joni/src/org/joni/Parser.java +++ b/third_party/joni/src/org/joni/Parser.java @@ -636,6 +636,12 @@ private Node parseEnclose(TokenType term) { ? AnchorType.PREC_READ : AnchorType.PREC_READ_NOT); fetchToken(); assertionCondition.setTarget(parseSubExp(term)); + } else if (c == '*') { + Node alphaCondition = parseAlphaAssertion(); + if (!(alphaCondition instanceof AnchorNode)) { + newSyntaxException(INVALID_CONDITION_PATTERN); + } + assertionCondition = (AnchorNode)alphaCondition; } else if (c == 'R') { recursionConditionGroup = 0; if (!left()) newSyntaxException(INVALID_CONDITION_PATTERN); @@ -997,12 +1003,15 @@ private Node parseAlphaAssertion() { int cursor = p; while (cursor < stop) { int code = enc.mbcToCode(bytes, cursor, stop); - if (!Character.isLetter(code)) break; + if (!Character.isLetter(code) && code != '_') break; cursor += enc.length(bytes, cursor, stop); } String name = new String(bytes, nameStart, cursor - nameStart, StandardCharsets.UTF_8); - if (!name.equals("pla") && !name.equals("plb") && !name.equals("nla") - && !name.equals("nlb") && !name.equals("atomic")) { + if (!name.equals("pla") && !name.equals("positive_lookahead") + && !name.equals("plb") && !name.equals("positive_lookbehind") + && !name.equals("nla") && !name.equals("negative_lookahead") + && !name.equals("nlb") && !name.equals("negative_lookbehind") + && !name.equals("atomic")) { return null; } if (cursor >= stop || enc.mbcToCode(bytes, cursor, stop) != ':') { @@ -1012,16 +1021,16 @@ private Node parseAlphaAssertion() { Node node; switch (name) { - case "pla": + case "pla", "positive_lookahead": node = new AnchorNode(AnchorType.PREC_READ); break; - case "plb": + case "plb", "positive_lookbehind": node = new AnchorNode(AnchorType.LOOK_BEHIND); break; - case "nla": + case "nla", "negative_lookahead": node = new AnchorNode(AnchorType.PREC_READ_NOT); break; - case "nlb": + case "nlb", "negative_lookbehind": node = new AnchorNode(AnchorType.LOOK_BEHIND_NOT); break; default: diff --git a/third_party/joni/test/org/joni/test/TestPerlAlphaAssertionConditions.java b/third_party/joni/test/org/joni/test/TestPerlAlphaAssertionConditions.java new file mode 100644 index 000000000..e67e88462 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestPerlAlphaAssertionConditions.java @@ -0,0 +1,49 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import static org.junit.Assert.assertEquals; + +import java.nio.charset.StandardCharsets; + +import org.jcodings.specific.UTF8Encoding; +import org.joni.Matcher; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.junit.Test; + +public class TestPerlAlphaAssertionConditions { + private static Matcher matcher(String pattern, String input) { + byte[] patternBytes = pattern.getBytes(StandardCharsets.UTF_8); + byte[] inputBytes = input.getBytes(StandardCharsets.UTF_8); + Regex regex = new Regex(patternBytes, 0, patternBytes.length, Option.NONE, + UTF8Encoding.INSTANCE, Syntax.RUBY); + return regex.matcher(inputBytes); + } + + @Test + public void alphaAssertionConditionsSelectTheMatchingBranch() { + assertEquals(0, matcher("(?(*pla:a)a|b)", "a").search(0, 1, Option.NONE)); + assertEquals(0, matcher("(?(*pla:a)a|b)", "b").search(0, 1, Option.NONE)); + assertEquals(0, matcher("(?(*nla:a)b|a)", "a").search(0, 1, Option.NONE)); + assertEquals(0, matcher("(?(*nla:a)b|a)", "b").search(0, 1, Option.NONE)); + } +} diff --git a/third_party/joni/test/org/joni/test/TestPerlAlphaAssertionLongNames.java b/third_party/joni/test/org/joni/test/TestPerlAlphaAssertionLongNames.java new file mode 100644 index 000000000..7e277bce8 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestPerlAlphaAssertionLongNames.java @@ -0,0 +1,59 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import org.jcodings.Encoding; +import org.jcodings.specific.ASCIIEncoding; +import org.joni.Option; +import org.joni.Syntax; +import org.joni.exception.ErrorMessages; + +public class TestPerlAlphaAssertionLongNames extends Test { + @Override + public int option() { + return Option.DEFAULT; + } + + @Override + public Encoding encoding() { + return ASCIIEncoding.INSTANCE; + } + + @Override + public String testEncoding() { + return "iso-8859-2"; + } + + @Override + public Syntax syntax() { + return Syntax.PerlNG; + } + + @Override + public void test() throws Exception { + x2s("a(*positive_lookahead:b)b", "ab", 0, 2); + x2s("a(*positive_lookbehind:a)b", "ab", 0, 2); + x2s("a(*negative_lookahead:c)b", "ab", 0, 2); + x2s("a(*negative_lookbehind:c)b", "ab", 0, 2); + xerrs("(*positive_lookahead)", + "'(*positive_lookahead' requires a terminating ':'"); + xerrs("(*positive_lookahead:a", ErrorMessages.PERL_UNTERMINATED_CONTROL_ARGUMENT); + } +} diff --git a/third_party/joni/test/org/joni/test/TestPerlHorizontalWhitespace.java b/third_party/joni/test/org/joni/test/TestPerlHorizontalWhitespace.java new file mode 100644 index 000000000..568dbb60d --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestPerlHorizontalWhitespace.java @@ -0,0 +1,150 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import static org.joni.constants.SyntaxProperties.OP2_ESC_H_HORIZONTAL_WHITESPACE; +import static org.junit.Assert.assertEquals; + +import java.nio.charset.StandardCharsets; +import java.util.HashSet; +import java.util.Set; + +import org.jcodings.Encoding; +import org.jcodings.constants.CharacterType; +import org.jcodings.specific.ISO8859_1Encoding; +import org.jcodings.specific.UTF8Encoding; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.junit.Test; + +public class TestPerlHorizontalWhitespace { + private static final int[] HORIZONTAL = { + 0x09, 0x20, 0xa0, 0x1680, 0x2000, 0x200a, 0x202f, 0x205f, 0x3000 + }; + private static final int[] OTHER = { + 0x0a, 0x0b, 0x41, 0x85, 0x180e, 0x2028, 0x2029 + }; + private static final Syntax PERL_HORIZONTAL = new Syntax( + "PERL_HORIZONTAL", Syntax.PerlNG.op, + Syntax.PerlNG.op2 | OP2_ESC_H_HORIZONTAL_WHITESPACE, + Syntax.PerlNG.op3, Syntax.PerlNG.behavior, Syntax.PerlNG.options, + Syntax.PerlNG.metaCharTable); + + private static byte[] encode(String value, Encoding encoding) { + return value.getBytes(encoding == ISO8859_1Encoding.INSTANCE + ? StandardCharsets.ISO_8859_1 : StandardCharsets.UTF_8); + } + + private static int search(String pattern, String input, Encoding encoding, + Syntax syntax) { + byte[] patternBytes = encode(pattern, encoding); + byte[] inputBytes = encode(input, encoding); + Regex regex = new Regex(patternBytes, 0, patternBytes.length, Option.NONE, + encoding, syntax); + return regex.matcher(inputBytes).search(0, inputBytes.length, Option.NONE); + } + + private static void assertMatch(String pattern, String input, Encoding encoding, + Syntax syntax) { + assertEquals(0, search("\\A(?:" + pattern + ")\\z", input, encoding, syntax)); + } + + private static void assertNoMatch(String pattern, String input, Encoding encoding, + Syntax syntax) { + assertEquals(-1, search("\\A(?:" + pattern + ")\\z", input, encoding, syntax)); + } + + @Test + public void jcodingsBlankExactlyMatchesPerlHorizontalWhitespace() { + Set expected = new HashSet<>(); + for (int codePoint : new int[] { + 0x09, 0x20, 0xa0, 0x1680, 0x202f, 0x205f, 0x3000}) { + expected.add(codePoint); + } + for (int codePoint = 0x2000; codePoint <= 0x200a; codePoint++) { + expected.add(codePoint); + } + + Set actual = new HashSet<>(); + for (int codePoint = 0; codePoint <= 0x10ffff; codePoint++) { + if (UTF8Encoding.INSTANCE.isCodeCType(codePoint, CharacterType.BLANK)) { + actual.add(codePoint); + } + } + assertEquals(expected, actual); + } + + @Test + public void implementsPerlHorizontalWhitespaceInsideAndOutsideClasses() { + for (int codePoint : HORIZONTAL) { + String character = new String(Character.toChars(codePoint)); + assertMatch("\\h", character, UTF8Encoding.INSTANCE, PERL_HORIZONTAL); + assertMatch("[\\h]", character, UTF8Encoding.INSTANCE, PERL_HORIZONTAL); + assertNoMatch("\\H", character, UTF8Encoding.INSTANCE, PERL_HORIZONTAL); + assertNoMatch("[\\H]", character, UTF8Encoding.INSTANCE, PERL_HORIZONTAL); + if (codePoint <= 0xff) { + assertMatch("\\h", character, ISO8859_1Encoding.INSTANCE, PERL_HORIZONTAL); + assertMatch("[\\h]", character, ISO8859_1Encoding.INSTANCE, PERL_HORIZONTAL); + assertNoMatch("\\H", character, ISO8859_1Encoding.INSTANCE, PERL_HORIZONTAL); + assertNoMatch("[\\H]", character, ISO8859_1Encoding.INSTANCE, PERL_HORIZONTAL); + } + } + + for (int codePoint : OTHER) { + String character = new String(Character.toChars(codePoint)); + assertNoMatch("\\h", character, UTF8Encoding.INSTANCE, PERL_HORIZONTAL); + assertNoMatch("[\\h]", character, UTF8Encoding.INSTANCE, PERL_HORIZONTAL); + assertMatch("\\H", character, UTF8Encoding.INSTANCE, PERL_HORIZONTAL); + assertMatch("[\\H]", character, UTF8Encoding.INSTANCE, PERL_HORIZONTAL); + if (codePoint <= 0xff) { + assertNoMatch("\\h", character, ISO8859_1Encoding.INSTANCE, PERL_HORIZONTAL); + assertNoMatch("[\\h]", character, ISO8859_1Encoding.INSTANCE, PERL_HORIZONTAL); + assertMatch("\\H", character, ISO8859_1Encoding.INSTANCE, PERL_HORIZONTAL); + assertMatch("[\\H]", character, ISO8859_1Encoding.INSTANCE, PERL_HORIZONTAL); + } + } + } + + @Test + public void asciiModifiersDoNotNarrowPerlHorizontalWhitespace() { + String ideographicSpace = new String(Character.toChars(0x3000)); + for (String pattern : new String[] { + "(?a:\\h)", "(?aa:\\h)", "(?a:[\\h])", "(?aa:[\\h])"}) { + assertMatch(pattern, ideographicSpace, UTF8Encoding.INSTANCE, PERL_HORIZONTAL); + } + for (String pattern : new String[] { + "(?a:\\H)", "(?aa:\\H)", "(?a:[\\H])", "(?aa:[\\H])"}) { + assertMatch(pattern, "\n", UTF8Encoding.INSTANCE, PERL_HORIZONTAL); + } + for (String pattern : new String[] { + "(?a:\\h)", "(?aa:\\h)", "(?a:[\\h])", "(?aa:[\\h])"}) { + assertMatch(pattern, "\u00a0", ISO8859_1Encoding.INSTANCE, PERL_HORIZONTAL); + } + } + + @Test + public void preservesRubyHexDigitEscapes() { + assertMatch("\\h+", "09AFaf", UTF8Encoding.INSTANCE, Syntax.RUBY); + assertMatch("[\\h]+", "09AFaf", UTF8Encoding.INSTANCE, Syntax.RUBY); + assertNoMatch("\\h", " ", UTF8Encoding.INSTANCE, Syntax.RUBY); + assertNoMatch("[\\h]", " ", UTF8Encoding.INSTANCE, Syntax.RUBY); + } +}