From 72861cf70f453053cf99c88bf813e7e758ad8dde Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Tue, 18 Aug 2026 23:18:44 +0200 Subject: [PATCH] fix: implement Perl horizontal whitespace escapes in Joni Enable the Perl-only horizontal-whitespace syntax bit and teach the lexer to emit exact \h and \H semantics without changing Ruby's hex-digit escapes. Preserve the full Perl set under /a and /aa, including byte-mode classes. Add direct Joni and standard-Perl-validated product coverage for positive, complement, class, Unicode, byte, and scoped-modifier behavior. Generated with [Codex](https://openai.com/codex/) Co-Authored-By: Codex --- .../runtime/regex/JoniRegexPattern.java | 4 +- .../unit/regex/horizontal_whitespace_escape.t | 83 ++++++++++ third_party/joni/src/org/joni/Lexer.java | 124 ++++++++++++++- .../test/TestPerlHorizontalWhitespace.java | 150 ++++++++++++++++++ 4 files changed, 356 insertions(+), 5 deletions(-) create mode 100644 src/test/resources/unit/regex/horizontal_whitespace_escape.t create mode 100644 third_party/joni/test/org/joni/test/TestPerlHorizontalWhitespace.java diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index 348a17d38..69e8a36b0 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -16,6 +16,7 @@ import org.joni.Syntax; import static org.joni.constants.SyntaxProperties.ALLOW_MULTIPLEX_DEFINITION_NAME_CALL; +import static org.joni.constants.SyntaxProperties.OP2_ESC_H_HORIZONTAL_WHITESPACE; import static org.joni.constants.SyntaxProperties.OP2_OPTION_PERL; import static org.joni.constants.SyntaxProperties.OP2_OPTION_RUBY; import static org.joni.constants.SyntaxProperties.OP2_PLUS_POSSESSIVE_INTERVAL; @@ -45,7 +46,8 @@ final class JoniRegexPattern { // by callouts and control verbs while changing only that default policy. private static final Syntax PERLONJAVA_SYNTAX = new Syntax( "PERLONJAVA", Syntax.RUBY.op, - (Syntax.RUBY.op2 & ~OP2_OPTION_RUBY) | OP2_OPTION_PERL | OP2_PLUS_POSSESSIVE_INTERVAL, + (Syntax.RUBY.op2 & ~OP2_OPTION_RUBY) | OP2_OPTION_PERL + | OP2_PLUS_POSSESSIVE_INTERVAL | OP2_ESC_H_HORIZONTAL_WHITESPACE, Syntax.RUBY.op3, Syntax.RUBY.behavior | ALLOW_MULTIPLEX_DEFINITION_NAME_CALL, Syntax.RUBY.options & ~(Option.ASCII_RANGE diff --git a/src/test/resources/unit/regex/horizontal_whitespace_escape.t b/src/test/resources/unit/regex/horizontal_whitespace_escape.t new file mode 100644 index 000000000..1a14c759f --- /dev/null +++ b/src/test/resources/unit/regex/horizontal_whitespace_escape.t @@ -0,0 +1,83 @@ +use strict; +use warnings; +use feature 'unicode_strings'; +use Test::More; + +my @horizontal = (0x09, 0x20, 0xA0, 0x1680, 0x2000, 0x200A, + 0x202F, 0x205F, 0x3000); +my @other = (0x0A, 0x0B, 0x41, 0x85, 0x180E, 0x2028, 0x2029); + +sub check_code_point { + my ($code, $expected, $upgraded) = @_; + my $character = chr($code); + if ($upgraded) { + utf8::upgrade($character); + } + else { + utf8::downgrade($character, 1) + or die sprintf "U+%04X cannot be represented as bytes", $code; + } + my $mode = $upgraded ? 'Unicode' : 'byte'; + my $label = sprintf 'U+%04X %s', $code, $mode; + + is($character =~ /\A\h\z/ ? 1 : 0, $expected, + "direct h $label"); + is($character =~ /\A\H\z/ ? 1 : 0, 1 - $expected, + "direct H $label"); + is($character =~ /\A[\h]\z/ ? 1 : 0, $expected, + "class h $label"); + is($character =~ /\A[\H]\z/ ? 1 : 0, 1 - $expected, + "class H $label"); +} + +for my $code (@horizontal) { + check_code_point($code, 1, 1); + check_code_point($code, 1, 0) if $code <= 0xFF; +} +for my $code (@other) { + check_code_point($code, 0, 1); + check_code_point($code, 0, 0) if $code <= 0xFF; +} + +my $horizontal_run = join '', map chr, @horizontal; +my $other_run = join '', map chr, @other; +ok($horizontal_run =~ /\A\h+\z/, + 'direct h matches a run of horizontal whitespace'); +ok($horizontal_run =~ /\A[\h]+\z/, + 'class h matches a run of horizontal whitespace'); +ok($other_run =~ /\A\H+\z/, + 'direct H matches a run without horizontal whitespace'); +ok($other_run =~ /\A[\H]+\z/, + 'class H matches a run without horizontal whitespace'); + +my $ideographic_space = chr 0x3000; +my $line_feed = "\n"; +ok($ideographic_space =~ /\A(?a:\h)\z/, + 'scoped a keeps direct h Unicode-aware'); +ok($ideographic_space =~ /\A(?aa:\h)\z/, + 'scoped aa keeps direct h Unicode-aware'); +ok($ideographic_space =~ /\A(?a:[\h])\z/, + 'scoped a keeps class h Unicode-aware'); +ok($ideographic_space =~ /\A(?aa:[\h])\z/, + 'scoped aa keeps class h Unicode-aware'); +ok($line_feed =~ /\A(?a:\H)\z/, + 'scoped a keeps direct H complement semantics'); +ok($line_feed =~ /\A(?aa:\H)\z/, + 'scoped aa keeps direct H complement semantics'); +ok($line_feed =~ /\A(?a:[\H])\z/, + 'scoped a keeps class H complement semantics'); +ok($line_feed =~ /\A(?aa:[\H])\z/, + 'scoped aa keeps class H complement semantics'); + +my $byte_nbsp = chr 0xA0; +utf8::downgrade($byte_nbsp, 1); +ok($byte_nbsp =~ /\A(?a:\h)\z/, + 'scoped a keeps direct h byte semantics'); +ok($byte_nbsp =~ /\A(?aa:\h)\z/, + 'scoped aa keeps direct h byte semantics'); +ok($byte_nbsp =~ /\A(?a:[\h])\z/, + 'scoped a keeps class h byte semantics'); +ok($byte_nbsp =~ /\A(?aa:[\h])\z/, + 'scoped aa keeps class h byte semantics'); + +done_testing(); diff --git a/third_party/joni/src/org/joni/Lexer.java b/third_party/joni/src/org/joni/Lexer.java index ca80215d6..cfceac4e5 100644 --- a/third_party/joni/src/org/joni/Lexer.java +++ b/third_party/joni/src/org/joni/Lexer.java @@ -44,6 +44,9 @@ class Lexer extends ScannerSupport { protected final ScanEnvironment env; protected final Syntax syntax; // fast access to syntax protected final Token token = new Token(); // current token + private int perlHorizontalWhitespaceTokenIndex = -1; + private boolean perlHorizontalWhitespaceNegated; + private boolean perlHorizontalWhitespaceSingleByte; private int perlVerticalWhitespaceTokenIndex = -1; private boolean perlVerticalWhitespaceNegated; @@ -552,6 +555,84 @@ private void fetchTokenInCCFor_charType(boolean flag, int type) { token.setPropNot(flag); } + private void startPerlHorizontalWhitespace(boolean negated, TokenType openType) { + perlHorizontalWhitespaceTokenIndex = 0; + perlHorizontalWhitespaceNegated = negated; + perlHorizontalWhitespaceSingleByte = enc.isSingleByte(); + token.type = openType; + } + + private TokenType fetchPerlHorizontalWhitespaceToken() { + token.base = 0; + token.escaped = false; + + int index = perlHorizontalWhitespaceTokenIndex++; + if (perlHorizontalWhitespaceNegated) { + if (index == 0) { + token.type = TokenType.CHAR; + token.setC('^'); + return token.type; + } + index--; + } + + if (perlHorizontalWhitespaceSingleByte && index >= 3) { + token.type = TokenType.CC_CLOSE; + token.setC(']'); + perlHorizontalWhitespaceTokenIndex = -1; + return token.type; + } + + switch (index) { + case 0: + token.type = TokenType.CODE_POINT; + token.setCode(0x09); + break; + case 1: + token.type = TokenType.CODE_POINT; + token.setCode(0x20); + break; + case 2: + token.type = TokenType.CODE_POINT; + token.setCode(0xa0); + break; + case 3: + token.type = TokenType.CODE_POINT; + token.setCode(0x1680); + break; + case 4: + token.type = TokenType.CODE_POINT; + token.setCode(0x2000); + break; + case 5: + token.type = TokenType.CC_RANGE; + token.setC('-'); + break; + case 6: + token.type = TokenType.CODE_POINT; + token.setCode(0x200a); + break; + case 7: + token.type = TokenType.CODE_POINT; + token.setCode(0x202f); + break; + case 8: + token.type = TokenType.CODE_POINT; + token.setCode(0x205f); + break; + case 9: + token.type = TokenType.CODE_POINT; + token.setCode(0x3000); + break; + default: + token.type = TokenType.CC_CLOSE; + token.setC(']'); + perlHorizontalWhitespaceTokenIndex = -1; + break; + } + return token.type; + } + private boolean usesPerlVerticalWhitespaceEscape() { return syntax.op2EscVVerticalWhiteSpace() || syntax.op2OptionPerl() && (syntax == Syntax.Perl @@ -914,6 +995,9 @@ private void fetchTokenInCCFor_and() { } protected final TokenType fetchTokenInCC() { + if (perlHorizontalWhitespaceTokenIndex >= 0) { + return fetchPerlHorizontalWhitespaceToken(); + } if (perlVerticalWhitespaceTokenIndex >= 0) { return fetchPerlVerticalWhitespaceToken(); } @@ -959,10 +1043,26 @@ protected final TokenType fetchTokenInCC() { fetchTokenInCCFor_charType(true, CharacterType.SPACE); break; case 'h': - if (syntax.op2EscHXDigit()) fetchTokenInCCFor_charType(false, CharacterType.XDIGIT); + if (syntax.op2EscHHorizontalWhiteSpace()) { + if (enc.isSingleByte() || isAsciiRange(env.option)) { + startPerlHorizontalWhitespace(false, TokenType.CC_CC_OPEN); + } else { + fetchTokenInCCFor_charType(false, CharacterType.BLANK); + } + } else if (syntax.op2EscHXDigit()) { + fetchTokenInCCFor_charType(false, CharacterType.XDIGIT); + } break; case 'H': - if (syntax.op2EscHXDigit()) fetchTokenInCCFor_charType(true, CharacterType.XDIGIT); + if (syntax.op2EscHHorizontalWhiteSpace()) { + if (enc.isSingleByte() || isAsciiRange(env.option)) { + startPerlHorizontalWhitespace(true, TokenType.CC_CC_OPEN); + } else { + fetchTokenInCCFor_charType(true, CharacterType.BLANK); + } + } else if (syntax.op2EscHXDigit()) { + fetchTokenInCCFor_charType(true, CharacterType.XDIGIT); + } break; case 'p': case 'P': @@ -1418,10 +1518,26 @@ protected final void fetchToken() { if (syntax.opEscDDigit()) fetchTokenInCCFor_charType(true, CharacterType.DIGIT); break; case 'h': - if (syntax.op2EscHXDigit()) fetchTokenInCCFor_charType(false, CharacterType.XDIGIT); + if (syntax.op2EscHHorizontalWhiteSpace()) { + if (enc.isSingleByte()) { + startPerlHorizontalWhitespace(false, TokenType.CC_OPEN); + } else { + fetchTokenInCCFor_charType(false, CharacterType.BLANK); + } + } else if (syntax.op2EscHXDigit()) { + fetchTokenInCCFor_charType(false, CharacterType.XDIGIT); + } break; case 'H': - if (syntax.op2EscHXDigit()) fetchTokenInCCFor_charType(true, CharacterType.XDIGIT); + if (syntax.op2EscHHorizontalWhiteSpace()) { + if (enc.isSingleByte()) { + startPerlHorizontalWhitespace(true, TokenType.CC_OPEN); + } else { + fetchTokenInCCFor_charType(true, CharacterType.BLANK); + } + } else if (syntax.op2EscHXDigit()) { + fetchTokenInCCFor_charType(true, CharacterType.XDIGIT); + } break; case 'A': if (syntax.opEscAZBufAnchor()) fetchTokenFor_anchor(AnchorType.BEGIN_BUF); diff --git a/third_party/joni/test/org/joni/test/TestPerlHorizontalWhitespace.java b/third_party/joni/test/org/joni/test/TestPerlHorizontalWhitespace.java new file mode 100644 index 000000000..568dbb60d --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestPerlHorizontalWhitespace.java @@ -0,0 +1,150 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import static org.joni.constants.SyntaxProperties.OP2_ESC_H_HORIZONTAL_WHITESPACE; +import static org.junit.Assert.assertEquals; + +import java.nio.charset.StandardCharsets; +import java.util.HashSet; +import java.util.Set; + +import org.jcodings.Encoding; +import org.jcodings.constants.CharacterType; +import org.jcodings.specific.ISO8859_1Encoding; +import org.jcodings.specific.UTF8Encoding; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.junit.Test; + +public class TestPerlHorizontalWhitespace { + private static final int[] HORIZONTAL = { + 0x09, 0x20, 0xa0, 0x1680, 0x2000, 0x200a, 0x202f, 0x205f, 0x3000 + }; + private static final int[] OTHER = { + 0x0a, 0x0b, 0x41, 0x85, 0x180e, 0x2028, 0x2029 + }; + private static final Syntax PERL_HORIZONTAL = new Syntax( + "PERL_HORIZONTAL", Syntax.PerlNG.op, + Syntax.PerlNG.op2 | OP2_ESC_H_HORIZONTAL_WHITESPACE, + Syntax.PerlNG.op3, Syntax.PerlNG.behavior, Syntax.PerlNG.options, + Syntax.PerlNG.metaCharTable); + + private static byte[] encode(String value, Encoding encoding) { + return value.getBytes(encoding == ISO8859_1Encoding.INSTANCE + ? StandardCharsets.ISO_8859_1 : StandardCharsets.UTF_8); + } + + private static int search(String pattern, String input, Encoding encoding, + Syntax syntax) { + byte[] patternBytes = encode(pattern, encoding); + byte[] inputBytes = encode(input, encoding); + Regex regex = new Regex(patternBytes, 0, patternBytes.length, Option.NONE, + encoding, syntax); + return regex.matcher(inputBytes).search(0, inputBytes.length, Option.NONE); + } + + private static void assertMatch(String pattern, String input, Encoding encoding, + Syntax syntax) { + assertEquals(0, search("\\A(?:" + pattern + ")\\z", input, encoding, syntax)); + } + + private static void assertNoMatch(String pattern, String input, Encoding encoding, + Syntax syntax) { + assertEquals(-1, search("\\A(?:" + pattern + ")\\z", input, encoding, syntax)); + } + + @Test + public void jcodingsBlankExactlyMatchesPerlHorizontalWhitespace() { + Set expected = new HashSet<>(); + for (int codePoint : new int[] { + 0x09, 0x20, 0xa0, 0x1680, 0x202f, 0x205f, 0x3000}) { + expected.add(codePoint); + } + for (int codePoint = 0x2000; codePoint <= 0x200a; codePoint++) { + expected.add(codePoint); + } + + Set actual = new HashSet<>(); + for (int codePoint = 0; codePoint <= 0x10ffff; codePoint++) { + if (UTF8Encoding.INSTANCE.isCodeCType(codePoint, CharacterType.BLANK)) { + actual.add(codePoint); + } + } + assertEquals(expected, actual); + } + + @Test + public void implementsPerlHorizontalWhitespaceInsideAndOutsideClasses() { + for (int codePoint : HORIZONTAL) { + String character = new String(Character.toChars(codePoint)); + assertMatch("\\h", character, UTF8Encoding.INSTANCE, PERL_HORIZONTAL); + assertMatch("[\\h]", character, UTF8Encoding.INSTANCE, PERL_HORIZONTAL); + assertNoMatch("\\H", character, UTF8Encoding.INSTANCE, PERL_HORIZONTAL); + assertNoMatch("[\\H]", character, UTF8Encoding.INSTANCE, PERL_HORIZONTAL); + if (codePoint <= 0xff) { + assertMatch("\\h", character, ISO8859_1Encoding.INSTANCE, PERL_HORIZONTAL); + assertMatch("[\\h]", character, ISO8859_1Encoding.INSTANCE, PERL_HORIZONTAL); + assertNoMatch("\\H", character, ISO8859_1Encoding.INSTANCE, PERL_HORIZONTAL); + assertNoMatch("[\\H]", character, ISO8859_1Encoding.INSTANCE, PERL_HORIZONTAL); + } + } + + for (int codePoint : OTHER) { + String character = new String(Character.toChars(codePoint)); + assertNoMatch("\\h", character, UTF8Encoding.INSTANCE, PERL_HORIZONTAL); + assertNoMatch("[\\h]", character, UTF8Encoding.INSTANCE, PERL_HORIZONTAL); + assertMatch("\\H", character, UTF8Encoding.INSTANCE, PERL_HORIZONTAL); + assertMatch("[\\H]", character, UTF8Encoding.INSTANCE, PERL_HORIZONTAL); + if (codePoint <= 0xff) { + assertNoMatch("\\h", character, ISO8859_1Encoding.INSTANCE, PERL_HORIZONTAL); + assertNoMatch("[\\h]", character, ISO8859_1Encoding.INSTANCE, PERL_HORIZONTAL); + assertMatch("\\H", character, ISO8859_1Encoding.INSTANCE, PERL_HORIZONTAL); + assertMatch("[\\H]", character, ISO8859_1Encoding.INSTANCE, PERL_HORIZONTAL); + } + } + } + + @Test + public void asciiModifiersDoNotNarrowPerlHorizontalWhitespace() { + String ideographicSpace = new String(Character.toChars(0x3000)); + for (String pattern : new String[] { + "(?a:\\h)", "(?aa:\\h)", "(?a:[\\h])", "(?aa:[\\h])"}) { + assertMatch(pattern, ideographicSpace, UTF8Encoding.INSTANCE, PERL_HORIZONTAL); + } + for (String pattern : new String[] { + "(?a:\\H)", "(?aa:\\H)", "(?a:[\\H])", "(?aa:[\\H])"}) { + assertMatch(pattern, "\n", UTF8Encoding.INSTANCE, PERL_HORIZONTAL); + } + for (String pattern : new String[] { + "(?a:\\h)", "(?aa:\\h)", "(?a:[\\h])", "(?aa:[\\h])"}) { + assertMatch(pattern, "\u00a0", ISO8859_1Encoding.INSTANCE, PERL_HORIZONTAL); + } + } + + @Test + public void preservesRubyHexDigitEscapes() { + assertMatch("\\h+", "09AFaf", UTF8Encoding.INSTANCE, Syntax.RUBY); + assertMatch("[\\h]+", "09AFaf", UTF8Encoding.INSTANCE, Syntax.RUBY); + assertNoMatch("\\h", " ", UTF8Encoding.INSTANCE, Syntax.RUBY); + assertNoMatch("[\\h]", " ", UTF8Encoding.INSTANCE, Syntax.RUBY); + } +}