From ce9bfdea9f240536789d067f5d693bb2b17b3953 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 10:25:55 +0200 Subject: [PATCH 01/43] docs(regex): require fresh checkpoint acceptance Keep the retained A16 corpus as planning evidence while requiring the user's fresh 622-file run, strict PR-958 comparison, additional checks, and exact-head CI before PR 1087 merges. Generated with [OpenAI Codex](https://openai.com/codex/) Co-Authored-By: OpenAI Codex --- dev/design/phase36-regex-parity.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index e1a13b9c2..1c243eec6 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -187,7 +187,7 @@ incremental release PR; update it only from an exact warning-free integration head whose complete comparator has no pass-count decrease from PR 958 and no new invalid, missing, timeout, truncated, incomplete, or zero-TAP row. -The authoritative 622-file, jobs-5, timeout-300 gate at `d3a4bb074` completed +The planning 622-file, jobs-5, timeout-300 gate at `d3a4bb074` completed with 677871/695187 assertions passing and 95 improved rows. Fresh private-tree isolation found no regex pass decrease: both `pat.t` variants improve by 147 passing assertions. Plan/platform drift explains four raw decreases; missing @@ -201,8 +201,10 @@ return 32768. There is no remaining reproduced non-regex pass-count blocker. The complete runner, comparator, isolation, dossier, 622-file list, reducers, and verified checksum manifest are durably retained under -`../PerlOnJava/logs/test_20260821_080900_d3a4bb074_a16_*`; do not repeat the -full corpus merely to rediscover this evidence. +`../PerlOnJava/logs/test_20260821_080900_d3a4bb074_a16_*`. Use it for residual +root planning, but do not treat it as release acceptance: the user must run a +fresh 622-file gate and strict PR-958 comparison at the exact PR 1087 head, +plus the additional release checks, before that checkpoint may merge. ### Execution Tracker @@ -243,11 +245,10 @@ Active ownership: `regexp.t`, `regex_sets.t`, `charset.t`, `pat.t`, `pat_advanced.t`, `pat_re_eval.t`, and `reg_mesg.t` gates on that exact immutable head. -3. Preserve the documented fixture-invalid and sandbox-environment dispositions. - Refresh the comparator evidence for the exact integration - head without repeating the already-retained discovery run. Push a head with no PR-958 - pass-count regression to PR 1087, run CI, and make the incremental PR - reviewable. +3. Keep PR 1087 frozen at exact locally-green head `cb0926a5c`. Preserve the + documented fixture-invalid and sandbox-environment dispositions. Require the + user's fresh 622-file run, strict PR-958 comparison, additional release + checks, and exact-head CI before merging the incremental checkpoint. 4. Continue P4 documentation, P5 generated Indic properties, P6 pat/diagnostic work, and remaining native Unicode/runtime roots in a new WIP PR. Repeat focused tests per semantic tranche and one combined build per From ed2d9cc2aa2d1b741c58c1bc7ab047274601679e Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 10:36:45 +0200 Subject: [PATCH 02/43] feat(regex): generate current Perl Indic property data Generate Unicode 17 InSC and InPC partitions, aliases, defaults, and wildcard value lookup from the current Perl development tables. Resolve the pinned data through Joni ahead of ICU fallback on both execution backends. Generated with [OpenAI Codex](https://openai.com/codex/) Co-Authored-By: OpenAI Codex --- ...nerate_perl_unicode_indic_category_data.pl | 347 ++++++++++ dev/tools/perl_unicode_data_generators.json | 12 +- .../regex/PerlUnicodeIndicCategoryData.java | 622 ++++++++++++++++++ .../runtime/regex/UnicodeResolver.java | 45 +- .../PerlUnicodeIndicCategoryDataTest.java | 123 ++++ .../unit/regex/indic_category_properties.t | 65 ++ 6 files changed, 1212 insertions(+), 2 deletions(-) create mode 100644 dev/tools/generate_perl_unicode_indic_category_data.pl create mode 100644 src/main/java/org/perlonjava/runtime/regex/PerlUnicodeIndicCategoryData.java create mode 100644 src/test/java/org/perlonjava/runtime/regex/PerlUnicodeIndicCategoryDataTest.java create mode 100644 src/test/resources/unit/regex/indic_category_properties.t diff --git a/dev/tools/generate_perl_unicode_indic_category_data.pl b/dev/tools/generate_perl_unicode_indic_category_data.pl new file mode 100644 index 000000000..e20490dc5 --- /dev/null +++ b/dev/tools/generate_perl_unicode_indic_category_data.pl @@ -0,0 +1,347 @@ +#!/usr/bin/env perl +use strict; +use warnings; +use File::Spec; +use FindBin; +use lib File::Spec->catdir($FindBin::Bin, 'lib'); +use PerlOnJava::UnicodeGenerator qw( + emit_java_range_triples emit_unicode_source_notices loose_name parse_range + read_pinned_source read_unicode_version repo_root select_unicode_root trim +); + +my $expected_version = '17.0.0'; +my $root = repo_root($FindBin::Bin); +my $unicore = select_unicode_root( + repo_root => $root, + version => $expected_version, + required => [qw(version IndicSyllabicCategory.txt + IndicPositionalCategory.txt PropValueAliases.txt PropertyAliases.txt)], +); +my @sources = ( + { + name => 'IndicSyllabicCategory-17.0.0.txt', + path => File::Spec->catfile($unicore, 'IndicSyllabicCategory.txt'), + hash => '3fc122f4cf58b0c19268d5f810263b04ab4e1e67743386ec0e0ada9c76aec5be', + version => qr/^# IndicSyllabicCategory-\Q$expected_version\E\.txt$/m, + }, + { + name => 'IndicPositionalCategory-17.0.0.txt', + path => File::Spec->catfile($unicore, 'IndicPositionalCategory.txt'), + hash => '68cedc29a7e57f984d90fe2c7712f2e6d0c717e253db219607daea8997d6c480', + version => qr/^# IndicPositionalCategory-\Q$expected_version\E\.txt$/m, + }, + { + name => 'PropertyValueAliases-17.0.0.txt', + path => File::Spec->catfile($unicore, 'PropValueAliases.txt'), + hash => '670d2bebb48649c04fabfbf033308073dcff47946324a8033237254c048b3b01', + version => qr/^# PropertyValueAliases-\Q$expected_version\E\.txt$/m, + }, + { + name => 'PropertyAliases-17.0.0.txt', + path => File::Spec->catfile($unicore, 'PropertyAliases.txt'), + hash => '4441f573caf952ffece1d7c892e7715bd7136dfc26f96eb6f268bf1e474715fb', + version => qr/^# PropertyAliases-\Q$expected_version\E\.txt$/m, + }, +); + +for my $source (@sources) { + $source->{text} = read_pinned_source( + path => $source->{path}, + sha256 => $source->{hash}, + version_pattern => $source->{version}, + unicode_version => $expected_version, + ); +} + +my $unicode_version = read_unicode_version( + path => File::Spec->catfile($unicore, 'version'), expected => $expected_version); + +my %property = ( + InSC => { + java => 'INSC', + long => 'Indic_Syllabic_Category', + source => $sources[0], + expected_values => 37, + expected_ranges => 967, + default => 'Other', + short_values => [], + long_values => [], + alias_index => {}, + property_aliases => {}, + ranges => [], + }, + InPC => { + java => 'INPC', + long => 'Indic_Positional_Category', + source => $sources[1], + expected_values => 16, + expected_ranges => 653, + default => 'Not_Applicable', + short_values => [], + long_values => [], + alias_index => {}, + property_aliases => {}, + ranges => [], + }, +); + +for my $line (split /\n/, $sources[2]{text}) { + next if $line =~ /^\s*#/; + $line =~ s/#.*$//; + my @fields = map { trim($_) } split /;/, $line; + next unless @fields >= 3 && exists $property{$fields[0]}; + my $spec = $property{$fields[0]}; + my $index = scalar @{$spec->{short_values}}; + push @{$spec->{short_values}}, $fields[1]; + push @{$spec->{long_values}}, $fields[2]; + for my $alias (@fields[1 .. $#fields]) { + next unless length $alias; + my $loose = loose_name($alias); + die "$fields[0] value alias collision for '$alias'\n" + if exists $spec->{alias_index}{$loose} + && $spec->{alias_index}{$loose} != $index; + $spec->{alias_index}{$loose} = $index; + } +} + +for my $line (split /\n/, $sources[3]{text}) { + next if $line =~ /^\s*#/; + $line =~ s/#.*$//; + my @fields = map { trim($_) } split /;/, $line; + next unless @fields >= 2; + for my $short (keys %property) { + my $spec = $property{$short}; + next unless grep { $_ eq $short || $_ eq $spec->{long} } @fields; + $spec->{property_aliases}{loose_name($_)} = 1 + for grep { length } @fields; + } +} + +for my $short (qw(InSC InPC)) { + my $spec = $property{$short}; + die "$short expected $spec->{expected_values} values, found " + . scalar(@{$spec->{short_values}}) . "\n" + unless @{$spec->{short_values}} == $spec->{expected_values}; + die "$short property aliases missing\n" + unless $spec->{property_aliases}{loose_name($short)} + && $spec->{property_aliases}{loose_name($spec->{long})}; + + my ($default_index) = grep { + loose_name($spec->{long_values}[$_]) eq loose_name($spec->{default}) + } 0 .. $#{$spec->{long_values}}; + die "$short default $spec->{default} is absent from aliases\n" + unless defined $default_index; + $spec->{default_index} = $default_index; + + my $declared_default; + for my $line (split /\n/, $spec->{source}{text}) { + if ($line =~ /^#\s*\@missing:\s*0000\.\.10FFFF\s*;\s*([A-Za-z_]+)/) { + $declared_default = $1; + next; + } + next unless $line =~ /^([0-9A-F]+(?:\.\.[0-9A-F]+)?)\s*;\s*([A-Za-z_]+)/; + my ($start, $end) = parse_range($1); + my $index = $spec->{alias_index}{loose_name($2)}; + die "$short unknown explicit value '$2'\n" unless defined $index; + push @{$spec->{ranges}}, [$start, $end, $index]; + } + die "$short expected default $spec->{default}, found " + . (defined $declared_default ? $declared_default : '') . "\n" + unless defined $declared_default + && loose_name($declared_default) eq loose_name($spec->{default}); + die "$short expected $spec->{expected_ranges} ranges, found " + . scalar(@{$spec->{ranges}}) . "\n" + unless @{$spec->{ranges}} == $spec->{expected_ranges}; +} + +sub emit_string_array { + my ($name, $values) = @_; + print " private static final String[] $name = {\n "; + print join(', ', map { qq{"$_"} } @$values); + print "\n };\n"; +} + +sub emit_property { + my ($short) = @_; + my $spec = $property{$short}; + my @aliases = sort keys %{$spec->{alias_index}}; + my @property_aliases = sort keys %{$spec->{property_aliases}}; + my $prefix = $spec->{java}; + + emit_string_array("${prefix}_SHORT_VALUES", $spec->{short_values}); + emit_string_array("${prefix}_LONG_VALUES", $spec->{long_values}); + emit_string_array("${prefix}_VALUE_ALIASES", \@aliases); + print " private static final byte[] ${prefix}_VALUE_ALIAS_INDEX = {\n "; + print join(', ', map { $spec->{alias_index}{$_} } @aliases); + print "\n };\n"; + emit_string_array("${prefix}_PROPERTY_ALIASES", \@property_aliases); + print " private static final int[] ${prefix}_RANGES = {\n"; + emit_java_range_triples($spec->{ranges}); + print " };\n"; + print " private static final Property $prefix = new Property(\n"; + print " ${prefix}_SHORT_VALUES, ${prefix}_LONG_VALUES,\n"; + print " ${prefix}_VALUE_ALIASES, ${prefix}_VALUE_ALIAS_INDEX,\n"; + print " ${prefix}_PROPERTY_ALIASES, ${prefix}_RANGES,\n"; + print " $spec->{default_index});\n\n"; +} + +print <<'HEADER'; +/* + * Generated from Perl 5.44's current Unicode Character Database. Do not edit manually. + * +HEADER +emit_unicode_source_notices(\@sources); +print <<'HEADER_END'; + */ +package org.perlonjava.runtime.regex; + +import com.ibm.icu.text.UnicodeSet; + +final class PerlUnicodeIndicCategoryData { +HEADER_END + +print " static final String UNICODE_VERSION = \"$unicode_version\";\n"; +print " static final String INDIC_SYLLABIC_CATEGORY_SHA256 = \"$sources[0]{hash}\";\n"; +print " static final String INDIC_POSITIONAL_CATEGORY_SHA256 = \"$sources[1]{hash}\";\n"; +print " static final String PROP_VALUE_ALIASES_SHA256 = \"$sources[2]{hash}\";\n"; +print " static final String PROPERTY_ALIASES_SHA256 = \"$sources[3]{hash}\";\n\n"; + +emit_property($_) for qw(InSC InPC); + +print <<'FOOTER'; + static boolean isPropertyAlias(String alias) { + return property(alias) != null; + } + + static UnicodeSet valueSet(String propertyAlias, String valueAlias) { + Property property = property(propertyAlias); + return property == null ? null : property.valueSet(valueAlias); + } + + static String canonicalValue(String propertyAlias, String valueAlias) { + Property property = property(propertyAlias); + return property == null ? null : property.canonicalValue(valueAlias); + } + + static String[] canonicalValues(String propertyAlias) { + Property property = property(propertyAlias); + return property == null ? null : property.longValues.clone(); + } + + static String defaultValue(String propertyAlias) { + Property property = property(propertyAlias); + return property == null ? null : property.longValues[property.defaultIndex]; + } + + static int valueCount(String propertyAlias) { + Property property = property(propertyAlias); + return property == null ? 0 : property.longValues.length; + } + + static String shortValue(String propertyAlias, int index) { + Property property = property(propertyAlias); + return property == null || index < 0 || index >= property.shortValues.length + ? null : property.shortValues[index]; + } + + static String canonicalValue(String propertyAlias, int index) { + Property property = property(propertyAlias); + return property == null || index < 0 || index >= property.longValues.length + ? null : property.longValues[index]; + } + + static UnicodeSet valueSet(String propertyAlias, int index) { + Property property = property(propertyAlias); + return property == null || index < 0 || index >= property.valueSets.length + ? null : property.valueSets[index]; + } + + private static Property property(String alias) { + String loose = looseName(alias); + if (loose == null) return null; + if (INSC.hasPropertyAlias(loose)) return INSC; + if (INPC.hasPropertyAlias(loose)) return INPC; + return null; + } + + private static String looseName(String name) { + if (name == null) return null; + StringBuilder loose = new StringBuilder(name.length()); + for (int i = 0; i < name.length(); i++) { + char character = name.charAt(i); + if (character == '_' || character == '-' || Character.isWhitespace(character)) continue; + loose.append(Character.toLowerCase(character)); + } + return loose.toString(); + } + + private static final class Property { + private final String[] shortValues; + private final String[] longValues; + private final String[] valueAliases; + private final byte[] valueAliasIndex; + private final String[] propertyAliases; + private final int[] ranges; + private final int defaultIndex; + private final UnicodeSet[] valueSets; + + private Property(String[] shortValues, String[] longValues, + String[] valueAliases, byte[] valueAliasIndex, + String[] propertyAliases, int[] ranges, + int defaultIndex) { + this.shortValues = shortValues; + this.longValues = longValues; + this.valueAliases = valueAliases; + this.valueAliasIndex = valueAliasIndex; + this.propertyAliases = propertyAliases; + this.ranges = ranges; + this.defaultIndex = defaultIndex; + this.valueSets = buildValueSets(); + } + + private boolean hasPropertyAlias(String loose) { + for (String alias : propertyAliases) { + if (alias.equals(loose)) return true; + } + return false; + } + + private UnicodeSet valueSet(String alias) { + int index = valueIndex(alias); + return index < 0 ? null : valueSets[index]; + } + + private String canonicalValue(String alias) { + int index = valueIndex(alias); + return index < 0 ? null : longValues[index]; + } + + private int valueIndex(String alias) { + String loose = looseName(alias); + if (loose == null) return -1; + for (int i = 0; i < valueAliases.length; i++) { + if (valueAliases[i].equals(loose)) return valueAliasIndex[i]; + } + return -1; + } + + private UnicodeSet[] buildValueSets() { + UnicodeSet[] sets = new UnicodeSet[longValues.length]; + for (int i = 0; i < sets.length; i++) sets[i] = new UnicodeSet(); + sets[defaultIndex].add(0, 0x10FFFF); + for (int i = 0; i < ranges.length; i += 3) { + int valueIndex = ranges[i + 2]; + if (valueIndex != defaultIndex) { + sets[defaultIndex].remove(ranges[i], ranges[i + 1]); + sets[valueIndex].add(ranges[i], ranges[i + 1]); + } + } + for (UnicodeSet set : sets) set.freeze(); + return sets; + } + } + + private PerlUnicodeIndicCategoryData() { + } +} +FOOTER diff --git a/dev/tools/perl_unicode_data_generators.json b/dev/tools/perl_unicode_data_generators.json index 436eae6f2..b4bd11172 100644 --- a/dev/tools/perl_unicode_data_generators.json +++ b/dev/tools/perl_unicode_data_generators.json @@ -81,6 +81,16 @@ "extracted/DGeneralCategory.txt" : "d62e5bab70ca74f099343f71224fa051cb1fdd61a1ab45c0488c44cfc0b6102e" } }, + { + "generator" : "dev/tools/generate_perl_unicode_indic_category_data.pl", + "name" : "indic-category", + "output" : "src/main/java/org/perlonjava/runtime/regex/PerlUnicodeIndicCategoryData.java", + "output_sha256" : "2362ba0552cdcc065d0fa5fe9266d557cebc41ed63aff2046b6b0595e3bdbe03", + "sources" : { + "IndicPositionalCategory.txt" : "68cedc29a7e57f984d90fe2c7712f2e6d0c717e253db219607daea8997d6c480", + "IndicSyllabicCategory.txt" : "3fc122f4cf58b0c19268d5f810263b04ab4e1e67743386ec0e0ada9c76aec5be" + } + }, { "generator" : "dev/tools/generate_perl_unicode_joining_group_data.pl", "name" : "joining-group", @@ -119,7 +129,7 @@ } } ], - "perl_commit" : "fa00284182335e5fed578460346efb88843f08da", + "perl_commit" : "fd584474cf36982e2f2ef85b042ea51bb02fffaa", "perl_source_policy" : "current-checkout", "perl_version" : "5.45.2", "schema_version" : 2, diff --git a/src/main/java/org/perlonjava/runtime/regex/PerlUnicodeIndicCategoryData.java b/src/main/java/org/perlonjava/runtime/regex/PerlUnicodeIndicCategoryData.java new file mode 100644 index 000000000..74ba00c34 --- /dev/null +++ b/src/main/java/org/perlonjava/runtime/regex/PerlUnicodeIndicCategoryData.java @@ -0,0 +1,622 @@ +/* + * Generated from Perl 5.44's current Unicode Character Database. Do not edit manually. + * + * Source: IndicSyllabicCategory-17.0.0.txt + * © 2025 Unicode®, Inc. + * Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. + * For terms of use and license, see https://www.unicode.org/terms_of_use.html + * + * Source: IndicPositionalCategory-17.0.0.txt + * © 2025 Unicode®, Inc. + * Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. + * For terms of use and license, see https://www.unicode.org/terms_of_use.html + * + * Source: PropertyValueAliases-17.0.0.txt + * © 2025 Unicode®, Inc. + * Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. + * For terms of use and license, see https://www.unicode.org/terms_of_use.html + * + * Source: PropertyAliases-17.0.0.txt + * © 2025 Unicode®, Inc. + * Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. + * For terms of use and license, see https://www.unicode.org/terms_of_use.html + * + */ +package org.perlonjava.runtime.regex; + +import com.ibm.icu.text.UnicodeSet; + +final class PerlUnicodeIndicCategoryData { + static final String UNICODE_VERSION = "17.0.0"; + static final String INDIC_SYLLABIC_CATEGORY_SHA256 = "3fc122f4cf58b0c19268d5f810263b04ab4e1e67743386ec0e0ada9c76aec5be"; + static final String INDIC_POSITIONAL_CATEGORY_SHA256 = "68cedc29a7e57f984d90fe2c7712f2e6d0c717e253db219607daea8997d6c480"; + static final String PROP_VALUE_ALIASES_SHA256 = "670d2bebb48649c04fabfbf033308073dcff47946324a8033237254c048b3b01"; + static final String PROPERTY_ALIASES_SHA256 = "4441f573caf952ffece1d7c892e7715bd7136dfc26f96eb6f268bf1e474715fb"; + + private static final String[] INSC_SHORT_VALUES = { + "Avagraha", "Bindu", "Brahmi_Joining_Number", "Cantillation_Mark", "Consonant", "Consonant_Dead", "Consonant_Final", "Consonant_Head_Letter", "Consonant_Initial_Postfixed", "Consonant_Killer", "Consonant_Medial", "Consonant_Placeholder", "Consonant_Preceding_Repha", "Consonant_Prefixed", "Consonant_Subjoined", "Consonant_Succeeding_Repha", "Consonant_With_Stacker", "Gemination_Mark", "Invisible_Stacker", "Joiner", "Modifying_Letter", "Non_Joiner", "Nukta", "Number", "Number_Joiner", "Other", "Pure_Killer", "Register_Shifter", "Reordering_Killer", "Syllable_Modifier", "Tone_Letter", "Tone_Mark", "Virama", "Visarga", "Vowel", "Vowel_Dependent", "Vowel_Independent" + }; + private static final String[] INSC_LONG_VALUES = { + "Avagraha", "Bindu", "Brahmi_Joining_Number", "Cantillation_Mark", "Consonant", "Consonant_Dead", "Consonant_Final", "Consonant_Head_Letter", "Consonant_Initial_Postfixed", "Consonant_Killer", "Consonant_Medial", "Consonant_Placeholder", "Consonant_Preceding_Repha", "Consonant_Prefixed", "Consonant_Subjoined", "Consonant_Succeeding_Repha", "Consonant_With_Stacker", "Gemination_Mark", "Invisible_Stacker", "Joiner", "Modifying_Letter", "Non_Joiner", "Nukta", "Number", "Number_Joiner", "Other", "Pure_Killer", "Register_Shifter", "Reordering_Killer", "Syllable_Modifier", "Tone_Letter", "Tone_Mark", "Virama", "Visarga", "Vowel", "Vowel_Dependent", "Vowel_Independent" + }; + private static final String[] INSC_VALUE_ALIASES = { + "avagraha", "bindu", "brahmijoiningnumber", "cantillationmark", "consonant", "consonantdead", "consonantfinal", "consonantheadletter", "consonantinitialpostfixed", "consonantkiller", "consonantmedial", "consonantplaceholder", "consonantprecedingrepha", "consonantprefixed", "consonantsubjoined", "consonantsucceedingrepha", "consonantwithstacker", "geminationmark", "invisiblestacker", "joiner", "modifyingletter", "nonjoiner", "nukta", "number", "numberjoiner", "other", "purekiller", "registershifter", "reorderingkiller", "syllablemodifier", "toneletter", "tonemark", "virama", "visarga", "vowel", "voweldependent", "vowelindependent" + }; + private static final byte[] INSC_VALUE_ALIAS_INDEX = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36 + }; + private static final String[] INSC_PROPERTY_ALIASES = { + "indicsyllabiccategory", "insc" + }; + private static final int[] INSC_RANGES = { + 0x900, 0x902, 1, 0x981, 0x981, 1, 0x982, 0x982, 1, 0x9FC, 0x9FC, 1, + 0xA01, 0xA02, 1, 0xA70, 0xA70, 1, 0xA81, 0xA82, 1, 0xB01, 0xB01, 1, + 0xB02, 0xB02, 1, 0xB82, 0xB82, 1, 0xC00, 0xC00, 1, 0xC01, 0xC02, 1, + 0xC04, 0xC04, 1, 0xC80, 0xC80, 1, 0xC81, 0xC81, 1, 0xC82, 0xC82, 1, + 0xCF3, 0xCF3, 1, 0xD00, 0xD01, 1, 0xD02, 0xD02, 1, 0xD04, 0xD04, 1, + 0xD81, 0xD81, 1, 0xD82, 0xD82, 1, 0xE4D, 0xE4D, 1, 0xECD, 0xECD, 1, + 0xF7E, 0xF7E, 1, 0xF82, 0xF83, 1, 0x1036, 0x1036, 1, 0x17C6, 0x17C6, 1, + 0x1932, 0x1932, 1, 0x1A74, 0x1A74, 1, 0x1B00, 0x1B02, 1, 0x1B80, 0x1B80, 1, + 0x1C34, 0x1C35, 1, 0xA80B, 0xA80B, 1, 0xA873, 0xA873, 1, 0xA880, 0xA880, 1, + 0xA8C5, 0xA8C5, 1, 0xA8F2, 0xA8F3, 1, 0xA980, 0xA981, 1, 0x10A0E, 0x10A0E, 1, + 0x11000, 0x11000, 1, 0x11001, 0x11001, 1, 0x11080, 0x11081, 1, 0x11100, 0x11101, 1, + 0x11180, 0x11181, 1, 0x111CF, 0x111CF, 1, 0x11234, 0x11234, 1, 0x112DF, 0x112DF, 1, + 0x11300, 0x11301, 1, 0x11302, 0x11302, 1, 0x1135E, 0x1135F, 1, 0x113CA, 0x113CA, 1, + 0x113CC, 0x113CC, 1, 0x11443, 0x11444, 1, 0x1145F, 0x1145F, 1, 0x114BF, 0x114C0, 1, + 0x115BC, 0x115BD, 1, 0x1163D, 0x1163D, 1, 0x116AB, 0x116AB, 1, 0x11837, 0x11837, 1, + 0x1193B, 0x1193C, 1, 0x119DE, 0x119DE, 1, 0x11A35, 0x11A38, 1, 0x11A96, 0x11A96, 1, + 0x11C3C, 0x11C3D, 1, 0x11CB5, 0x11CB6, 1, 0x11D40, 0x11D40, 1, 0x11D95, 0x11D95, 1, + 0x11F00, 0x11F01, 1, 0x1612D, 0x1612D, 1, 0x16D40, 0x16D41, 1, 0x903, 0x903, 33, + 0x983, 0x983, 33, 0xA03, 0xA03, 33, 0xA83, 0xA83, 33, 0xB03, 0xB03, 33, + 0xC03, 0xC03, 33, 0xC83, 0xC83, 33, 0xD03, 0xD03, 33, 0xD83, 0xD83, 33, + 0xF7F, 0xF7F, 33, 0x1038, 0x1038, 33, 0x17C7, 0x17C7, 33, 0x1B04, 0x1B04, 33, + 0x1B82, 0x1B82, 33, 0xA881, 0xA881, 33, 0xA983, 0xA983, 33, 0xAAF5, 0xAAF5, 33, + 0x10A0F, 0x10A0F, 33, 0x11002, 0x11002, 33, 0x11082, 0x11082, 33, 0x11102, 0x11102, 33, + 0x11182, 0x11182, 33, 0x11303, 0x11303, 33, 0x113CD, 0x113CD, 33, 0x11445, 0x11445, 33, + 0x114C1, 0x114C1, 33, 0x115BE, 0x115BE, 33, 0x1163E, 0x1163E, 33, 0x116AC, 0x116AC, 33, + 0x11838, 0x11838, 33, 0x119DF, 0x119DF, 33, 0x11A39, 0x11A39, 33, 0x11A97, 0x11A97, 33, + 0x11C3E, 0x11C3E, 33, 0x11D41, 0x11D41, 33, 0x11D96, 0x11D96, 33, 0x11F03, 0x11F03, 33, + 0x16D42, 0x16D42, 33, 0x93D, 0x93D, 0, 0x9BD, 0x9BD, 0, 0xABD, 0xABD, 0, + 0xB3D, 0xB3D, 0, 0xC3D, 0xC3D, 0, 0xCBD, 0xCBD, 0, 0xD3D, 0xD3D, 0, + 0xF85, 0xF85, 0, 0x17DC, 0x17DC, 0, 0x1BBA, 0x1BBA, 0, 0x111C1, 0x111C1, 0, + 0x1133D, 0x1133D, 0, 0x113B7, 0x113B7, 0, 0x11447, 0x11447, 0, 0x114C4, 0x114C4, 0, + 0x119E1, 0x119E1, 0, 0x11A9D, 0x11A9D, 0, 0x11C40, 0x11C40, 0, 0x93C, 0x93C, 22, + 0x9BC, 0x9BC, 22, 0xA3C, 0xA3C, 22, 0xABC, 0xABC, 22, 0xAFD, 0xAFF, 22, + 0xB3C, 0xB3C, 22, 0xC3C, 0xC3C, 22, 0xCBC, 0xCBC, 22, 0xF39, 0xF39, 22, + 0x1B34, 0x1B34, 22, 0x1BE6, 0x1BE6, 22, 0x1C37, 0x1C37, 22, 0xA9B3, 0xA9B3, 22, + 0x10A38, 0x10A3A, 22, 0x110BA, 0x110BA, 22, 0x11173, 0x11173, 22, 0x111CA, 0x111CA, 22, + 0x11236, 0x11236, 22, 0x112E9, 0x112E9, 22, 0x1133B, 0x1133C, 22, 0x11446, 0x11446, 22, + 0x114C3, 0x114C3, 22, 0x115C0, 0x115C0, 22, 0x116B7, 0x116B7, 22, 0x1183A, 0x1183A, 22, + 0x11943, 0x11943, 22, 0x11D42, 0x11D42, 22, 0x11F5A, 0x11F5A, 22, 0x94D, 0x94D, 32, + 0x9CD, 0x9CD, 32, 0xA4D, 0xA4D, 32, 0xACD, 0xACD, 32, 0xB4D, 0xB4D, 32, + 0xBCD, 0xBCD, 32, 0xC4D, 0xC4D, 32, 0xCCD, 0xCCD, 32, 0xD4D, 0xD4D, 32, + 0xDCA, 0xDCA, 32, 0x1B44, 0x1B44, 32, 0xA806, 0xA806, 32, 0xA8C4, 0xA8C4, 32, + 0xA9C0, 0xA9C0, 32, 0x11046, 0x11046, 32, 0x110B9, 0x110B9, 32, 0x111C0, 0x111C0, 32, + 0x11235, 0x11235, 32, 0x1134D, 0x1134D, 32, 0x11442, 0x11442, 32, 0x114C2, 0x114C2, 32, + 0x115BF, 0x115BF, 32, 0x1163F, 0x1163F, 32, 0x116B6, 0x116B6, 32, 0x11839, 0x11839, 32, + 0x119E0, 0x119E0, 32, 0x11C3F, 0x11C3F, 32, 0xD3B, 0xD3C, 26, 0xE3A, 0xE3A, 26, + 0xE4E, 0xE4E, 26, 0xEBA, 0xEBA, 26, 0xF84, 0xF84, 26, 0x103A, 0x103A, 26, + 0x1714, 0x1714, 26, 0x1715, 0x1715, 26, 0x1734, 0x1734, 26, 0x17D1, 0x17D1, 26, + 0x1A7A, 0x1A7A, 26, 0x1BAA, 0x1BAA, 26, 0xA82C, 0xA82C, 26, 0xA953, 0xA953, 26, + 0xABED, 0xABED, 26, 0x11070, 0x11070, 26, 0x11134, 0x11134, 26, 0x112EA, 0x112EA, 26, + 0x113CE, 0x113CE, 26, 0x113CF, 0x113CF, 26, 0x1172B, 0x1172B, 26, 0x1193D, 0x1193D, 26, + 0x11A34, 0x11A34, 26, 0x11D44, 0x11D44, 26, 0x11F41, 0x11F41, 26, 0x1612F, 0x1612F, 26, + 0x16D6B, 0x16D6C, 26, 0x1BF2, 0x1BF3, 28, 0x1039, 0x1039, 18, 0x17D2, 0x17D2, 18, + 0x1A60, 0x1A60, 18, 0x1BAB, 0x1BAB, 18, 0xAAF6, 0xAAF6, 18, 0x10A3F, 0x10A3F, 18, + 0x11133, 0x11133, 18, 0x113D0, 0x113D0, 18, 0x1193E, 0x1193E, 18, 0x11A47, 0x11A47, 18, + 0x11A99, 0x11A99, 18, 0x11D45, 0x11D45, 18, 0x11D97, 0x11D97, 18, 0x11F42, 0x11F42, 18, + 0x904, 0x914, 36, 0x960, 0x961, 36, 0x972, 0x977, 36, 0x985, 0x98C, 36, + 0x98F, 0x990, 36, 0x993, 0x994, 36, 0x9E0, 0x9E1, 36, 0xA05, 0xA0A, 36, + 0xA0F, 0xA10, 36, 0xA13, 0xA14, 36, 0xA85, 0xA8D, 36, 0xA8F, 0xA91, 36, + 0xA93, 0xA94, 36, 0xAE0, 0xAE1, 36, 0xB05, 0xB0C, 36, 0xB0F, 0xB10, 36, + 0xB13, 0xB14, 36, 0xB60, 0xB61, 36, 0xB85, 0xB8A, 36, 0xB8E, 0xB90, 36, + 0xB92, 0xB94, 36, 0xC05, 0xC0C, 36, 0xC0E, 0xC10, 36, 0xC12, 0xC14, 36, + 0xC60, 0xC61, 36, 0xC85, 0xC8C, 36, 0xC8E, 0xC90, 36, 0xC92, 0xC94, 36, + 0xCE0, 0xCE1, 36, 0xD05, 0xD0C, 36, 0xD0E, 0xD10, 36, 0xD12, 0xD14, 36, + 0xD5F, 0xD61, 36, 0xD85, 0xD96, 36, 0x1021, 0x102A, 36, 0x1052, 0x1055, 36, + 0x1700, 0x1702, 36, 0x1720, 0x1722, 36, 0x1740, 0x1742, 36, 0x1760, 0x1762, 36, + 0x17A3, 0x17B3, 36, 0x1A4D, 0x1A52, 36, 0x1B05, 0x1B12, 36, 0x1B83, 0x1B89, 36, + 0x1BE4, 0x1BE5, 36, 0xA800, 0xA801, 36, 0xA803, 0xA805, 36, 0xA882, 0xA891, 36, + 0xA8FE, 0xA8FE, 36, 0xA984, 0xA988, 36, 0xA98C, 0xA98E, 36, 0xAA00, 0xAA05, 36, + 0xAAE0, 0xAAE1, 36, 0xABCE, 0xABCF, 36, 0xABD1, 0xABD1, 36, 0x11005, 0x11012, 36, + 0x11071, 0x11072, 36, 0x11083, 0x1108C, 36, 0x11103, 0x11106, 36, 0x11183, 0x11190, 36, + 0x11200, 0x11207, 36, 0x11240, 0x11240, 36, 0x11280, 0x11283, 36, 0x112B0, 0x112B9, 36, + 0x11305, 0x1130C, 36, 0x1130F, 0x11310, 36, 0x11313, 0x11314, 36, 0x11360, 0x11361, 36, + 0x11380, 0x11389, 36, 0x1138B, 0x1138B, 36, 0x1138E, 0x1138E, 36, 0x11390, 0x11391, 36, + 0x11400, 0x1140D, 36, 0x11481, 0x1148E, 36, 0x11580, 0x1158D, 36, 0x115D8, 0x115DB, 36, + 0x11600, 0x1160D, 36, 0x11680, 0x11689, 36, 0x11800, 0x11809, 36, 0x11900, 0x11906, 36, + 0x11909, 0x11909, 36, 0x119A0, 0x119A7, 36, 0x119AA, 0x119AD, 36, 0x11C00, 0x11C08, 36, + 0x11C0A, 0x11C0D, 36, 0x11D00, 0x11D06, 36, 0x11D08, 0x11D09, 36, 0x11D0B, 0x11D0B, 36, + 0x11D60, 0x11D65, 36, 0x11D67, 0x11D68, 36, 0x11D6A, 0x11D6B, 36, 0x11F04, 0x11F10, 36, + 0x16100, 0x16100, 36, 0x93A, 0x93A, 35, 0x93B, 0x93B, 35, 0x93E, 0x940, 35, + 0x941, 0x948, 35, 0x949, 0x94C, 35, 0x94E, 0x94F, 35, 0x955, 0x957, 35, + 0x962, 0x963, 35, 0x9BE, 0x9C0, 35, 0x9C1, 0x9C4, 35, 0x9C7, 0x9C8, 35, + 0x9CB, 0x9CC, 35, 0x9D7, 0x9D7, 35, 0x9E2, 0x9E3, 35, 0xA3E, 0xA40, 35, + 0xA41, 0xA42, 35, 0xA47, 0xA48, 35, 0xA4B, 0xA4C, 35, 0xABE, 0xAC0, 35, + 0xAC1, 0xAC5, 35, 0xAC7, 0xAC8, 35, 0xAC9, 0xAC9, 35, 0xACB, 0xACC, 35, + 0xAE2, 0xAE3, 35, 0xB3E, 0xB3E, 35, 0xB3F, 0xB3F, 35, 0xB40, 0xB40, 35, + 0xB41, 0xB44, 35, 0xB47, 0xB48, 35, 0xB4B, 0xB4C, 35, 0xB55, 0xB56, 35, + 0xB57, 0xB57, 35, 0xB62, 0xB63, 35, 0xBBE, 0xBBF, 35, 0xBC0, 0xBC0, 35, + 0xBC1, 0xBC2, 35, 0xBC6, 0xBC8, 35, 0xBCA, 0xBCC, 35, 0xBD7, 0xBD7, 35, + 0xC3E, 0xC40, 35, 0xC41, 0xC44, 35, 0xC46, 0xC48, 35, 0xC4A, 0xC4C, 35, + 0xC55, 0xC56, 35, 0xC62, 0xC63, 35, 0xCBE, 0xCBE, 35, 0xCBF, 0xCBF, 35, + 0xCC0, 0xCC4, 35, 0xCC6, 0xCC6, 35, 0xCC7, 0xCC8, 35, 0xCCA, 0xCCB, 35, + 0xCCC, 0xCCC, 35, 0xCD5, 0xCD6, 35, 0xCE2, 0xCE3, 35, 0xD3E, 0xD40, 35, + 0xD41, 0xD44, 35, 0xD46, 0xD48, 35, 0xD4A, 0xD4C, 35, 0xD57, 0xD57, 35, + 0xD62, 0xD63, 35, 0xDCF, 0xDD1, 35, 0xDD2, 0xDD4, 35, 0xDD6, 0xDD6, 35, + 0xDD8, 0xDDF, 35, 0xDF2, 0xDF3, 35, 0xE30, 0xE30, 35, 0xE31, 0xE31, 35, + 0xE32, 0xE33, 35, 0xE34, 0xE39, 35, 0xE40, 0xE45, 35, 0xE47, 0xE47, 35, + 0xEB0, 0xEB0, 35, 0xEB1, 0xEB1, 35, 0xEB2, 0xEB3, 35, 0xEB4, 0xEB9, 35, + 0xEBB, 0xEBB, 35, 0xEC0, 0xEC4, 35, 0xF71, 0xF7D, 35, 0xF80, 0xF81, 35, + 0x102B, 0x102C, 35, 0x102D, 0x1030, 35, 0x1031, 0x1031, 35, 0x1032, 0x1035, 35, + 0x1056, 0x1057, 35, 0x1058, 0x1059, 35, 0x1062, 0x1062, 35, 0x1067, 0x1068, 35, + 0x1071, 0x1074, 35, 0x1083, 0x1084, 35, 0x1085, 0x1086, 35, 0x109C, 0x109C, 35, + 0x109D, 0x109D, 35, 0x1712, 0x1713, 35, 0x1732, 0x1733, 35, 0x1752, 0x1753, 35, + 0x1772, 0x1773, 35, 0x17B6, 0x17B6, 35, 0x17B7, 0x17BD, 35, 0x17BE, 0x17C5, 35, + 0x17C8, 0x17C8, 35, 0x1920, 0x1922, 35, 0x1923, 0x1926, 35, 0x1927, 0x1928, 35, + 0x193A, 0x193A, 35, 0x19B0, 0x19C0, 35, 0x1A17, 0x1A18, 35, 0x1A19, 0x1A1A, 35, + 0x1A1B, 0x1A1B, 35, 0x1A61, 0x1A61, 35, 0x1A62, 0x1A62, 35, 0x1A63, 0x1A64, 35, + 0x1A65, 0x1A6C, 35, 0x1A6D, 0x1A72, 35, 0x1A73, 0x1A73, 35, 0x1B35, 0x1B35, 35, + 0x1B36, 0x1B3A, 35, 0x1B3B, 0x1B3B, 35, 0x1B3C, 0x1B3C, 35, 0x1B3D, 0x1B41, 35, + 0x1B42, 0x1B42, 35, 0x1B43, 0x1B43, 35, 0x1BA4, 0x1BA5, 35, 0x1BA6, 0x1BA7, 35, + 0x1BA8, 0x1BA9, 35, 0x1BE7, 0x1BE7, 35, 0x1BE8, 0x1BE9, 35, 0x1BEA, 0x1BEC, 35, + 0x1BED, 0x1BED, 35, 0x1BEE, 0x1BEE, 35, 0x1BEF, 0x1BEF, 35, 0x1C26, 0x1C2B, 35, + 0x1C2C, 0x1C2C, 35, 0xA802, 0xA802, 35, 0xA823, 0xA824, 35, 0xA825, 0xA826, 35, + 0xA827, 0xA827, 35, 0xA8B5, 0xA8C3, 35, 0xA8FF, 0xA8FF, 35, 0xA947, 0xA94E, 35, + 0xA9B4, 0xA9B5, 35, 0xA9B6, 0xA9B9, 35, 0xA9BA, 0xA9BB, 35, 0xA9BC, 0xA9BC, 35, + 0xA9E5, 0xA9E5, 35, 0xAA29, 0xAA2E, 35, 0xAA2F, 0xAA30, 35, 0xAA31, 0xAA32, 35, + 0xAAB0, 0xAAB0, 35, 0xAAB1, 0xAAB1, 35, 0xAAB2, 0xAAB4, 35, 0xAAB5, 0xAAB6, 35, + 0xAAB7, 0xAAB8, 35, 0xAAB9, 0xAABD, 35, 0xAABE, 0xAABE, 35, 0xAAEB, 0xAAEB, 35, + 0xAAEC, 0xAAED, 35, 0xAAEE, 0xAAEF, 35, 0xABE3, 0xABE4, 35, 0xABE5, 0xABE5, 35, + 0xABE6, 0xABE7, 35, 0xABE8, 0xABE8, 35, 0xABE9, 0xABEA, 35, 0x10A01, 0x10A03, 35, + 0x10A05, 0x10A06, 35, 0x10A0C, 0x10A0D, 35, 0x11038, 0x11045, 35, 0x11073, 0x11074, 35, + 0x110B0, 0x110B2, 35, 0x110B3, 0x110B6, 35, 0x110B7, 0x110B8, 35, 0x110C2, 0x110C2, 35, + 0x11127, 0x1112B, 35, 0x1112C, 0x1112C, 35, 0x1112D, 0x11132, 35, 0x11145, 0x11146, 35, + 0x111B3, 0x111B5, 35, 0x111B6, 0x111BE, 35, 0x111BF, 0x111BF, 35, 0x111CB, 0x111CC, 35, + 0x111CE, 0x111CE, 35, 0x1122C, 0x1122E, 35, 0x1122F, 0x11231, 35, 0x11232, 0x11233, 35, + 0x11241, 0x11241, 35, 0x112E0, 0x112E2, 35, 0x112E3, 0x112E8, 35, 0x1133E, 0x1133F, 35, + 0x11340, 0x11340, 35, 0x11341, 0x11344, 35, 0x11347, 0x11348, 35, 0x1134B, 0x1134C, 35, + 0x11357, 0x11357, 35, 0x11362, 0x11363, 35, 0x113B8, 0x113BA, 35, 0x113BB, 0x113C0, 35, + 0x113C2, 0x113C2, 35, 0x113C5, 0x113C5, 35, 0x113C7, 0x113C9, 35, 0x11435, 0x11437, 35, + 0x11438, 0x1143F, 35, 0x11440, 0x11441, 35, 0x114B0, 0x114B2, 35, 0x114B3, 0x114B8, 35, + 0x114B9, 0x114B9, 35, 0x114BA, 0x114BA, 35, 0x114BB, 0x114BE, 35, 0x115AF, 0x115B1, 35, + 0x115B2, 0x115B5, 35, 0x115B8, 0x115BB, 35, 0x115DC, 0x115DD, 35, 0x11630, 0x11632, 35, + 0x11633, 0x1163A, 35, 0x1163B, 0x1163C, 35, 0x11640, 0x11640, 35, 0x116AD, 0x116AD, 35, + 0x116AE, 0x116AF, 35, 0x116B0, 0x116B5, 35, 0x11720, 0x11721, 35, 0x11722, 0x11725, 35, + 0x11726, 0x11726, 35, 0x11727, 0x1172A, 35, 0x1182C, 0x1182E, 35, 0x1182F, 0x11836, 35, + 0x11930, 0x11935, 35, 0x11937, 0x11938, 35, 0x119D1, 0x119D3, 35, 0x119D4, 0x119D7, 35, + 0x119DA, 0x119DB, 35, 0x119DC, 0x119DD, 35, 0x119E4, 0x119E4, 35, 0x11A01, 0x11A0A, 35, + 0x11A51, 0x11A56, 35, 0x11A57, 0x11A58, 35, 0x11A59, 0x11A5B, 35, 0x11B60, 0x11B60, 35, + 0x11B61, 0x11B61, 35, 0x11B62, 0x11B64, 35, 0x11B65, 0x11B65, 35, 0x11B66, 0x11B66, 35, + 0x11B67, 0x11B67, 35, 0x11C2F, 0x11C2F, 35, 0x11C30, 0x11C36, 35, 0x11C38, 0x11C3B, 35, + 0x11CB0, 0x11CB0, 35, 0x11CB1, 0x11CB1, 35, 0x11CB2, 0x11CB3, 35, 0x11CB4, 0x11CB4, 35, + 0x11D31, 0x11D36, 35, 0x11D3A, 0x11D3A, 35, 0x11D3C, 0x11D3D, 35, 0x11D3F, 0x11D3F, 35, + 0x11D43, 0x11D43, 35, 0x11D8A, 0x11D8E, 35, 0x11D90, 0x11D91, 35, 0x11D93, 0x11D94, 35, + 0x11EF3, 0x11EF4, 35, 0x11EF5, 0x11EF6, 35, 0x11F34, 0x11F35, 35, 0x11F36, 0x11F3A, 35, + 0x11F3E, 0x11F3F, 35, 0x11F40, 0x11F40, 35, 0x1611E, 0x16129, 35, 0x16D63, 0x16D6A, 35, + 0x1963, 0x196D, 34, 0xA85E, 0xA861, 34, 0xA866, 0xA866, 34, 0xA922, 0xA925, 34, + 0xA926, 0xA92A, 34, 0x11150, 0x11154, 34, 0x2D, 0x2D, 11, 0xA0, 0xA0, 11, + 0xD7, 0xD7, 11, 0x980, 0x980, 11, 0xA72, 0xA73, 11, 0x104B, 0x104B, 11, + 0x104E, 0x104E, 11, 0x1CFA, 0x1CFA, 11, 0x2010, 0x2014, 11, 0x25CC, 0x25CC, 11, + 0xAA74, 0xAA76, 11, 0x11A3F, 0x11A3F, 11, 0x11A45, 0x11A45, 11, 0x11EF2, 0x11EF2, 11, + 0x915, 0x939, 4, 0x958, 0x95F, 4, 0x978, 0x97F, 4, 0x995, 0x9A8, 4, + 0x9AA, 0x9B0, 4, 0x9B2, 0x9B2, 4, 0x9B6, 0x9B9, 4, 0x9DC, 0x9DD, 4, + 0x9DF, 0x9DF, 4, 0x9F0, 0x9F1, 4, 0xA15, 0xA28, 4, 0xA2A, 0xA30, 4, + 0xA32, 0xA33, 4, 0xA35, 0xA36, 4, 0xA38, 0xA39, 4, 0xA59, 0xA5C, 4, + 0xA5E, 0xA5E, 4, 0xA95, 0xAA8, 4, 0xAAA, 0xAB0, 4, 0xAB2, 0xAB3, 4, + 0xAB5, 0xAB9, 4, 0xAF9, 0xAF9, 4, 0xB15, 0xB28, 4, 0xB2A, 0xB30, 4, + 0xB32, 0xB33, 4, 0xB35, 0xB39, 4, 0xB5C, 0xB5D, 4, 0xB5F, 0xB5F, 4, + 0xB71, 0xB71, 4, 0xB95, 0xB95, 4, 0xB99, 0xB9A, 4, 0xB9C, 0xB9C, 4, + 0xB9E, 0xB9F, 4, 0xBA3, 0xBA4, 4, 0xBA8, 0xBAA, 4, 0xBAE, 0xBB9, 4, + 0xC15, 0xC28, 4, 0xC2A, 0xC39, 4, 0xC58, 0xC5A, 4, 0xC95, 0xCA8, 4, + 0xCAA, 0xCB3, 4, 0xCB5, 0xCB9, 4, 0xCDE, 0xCDE, 4, 0xD15, 0xD3A, 4, + 0xD9A, 0xDB1, 4, 0xDB3, 0xDBB, 4, 0xDBD, 0xDBD, 4, 0xDC0, 0xDC6, 4, + 0xE01, 0xE2E, 4, 0xE81, 0xE82, 4, 0xE84, 0xE84, 4, 0xE86, 0xE8A, 4, + 0xE8C, 0xEA3, 4, 0xEA5, 0xEA5, 4, 0xEA7, 0xEAE, 4, 0xEDC, 0xEDF, 4, + 0xF40, 0xF47, 4, 0xF49, 0xF6C, 4, 0x1000, 0x1020, 4, 0x103F, 0x103F, 4, + 0x1050, 0x1051, 4, 0x105A, 0x105D, 4, 0x1061, 0x1061, 4, 0x1065, 0x1066, 4, + 0x106E, 0x1070, 4, 0x1075, 0x1081, 4, 0x108E, 0x108E, 4, 0x1703, 0x1711, 4, + 0x171F, 0x171F, 4, 0x1723, 0x1731, 4, 0x1743, 0x1751, 4, 0x1763, 0x176C, 4, + 0x176E, 0x1770, 4, 0x1780, 0x17A2, 4, 0x1900, 0x191E, 4, 0x1950, 0x1962, 4, + 0x1980, 0x19AB, 4, 0x1A00, 0x1A16, 4, 0x1A20, 0x1A4C, 4, 0x1A53, 0x1A54, 4, + 0x1B13, 0x1B33, 4, 0x1B45, 0x1B4C, 4, 0x1B8A, 0x1BA0, 4, 0x1BAE, 0x1BAF, 4, + 0x1BBB, 0x1BBD, 4, 0x1BC0, 0x1BE3, 4, 0x1C00, 0x1C23, 4, 0x1C4D, 0x1C4F, 4, + 0xA807, 0xA80A, 4, 0xA80C, 0xA822, 4, 0xA840, 0xA85D, 4, 0xA862, 0xA865, 4, + 0xA869, 0xA870, 4, 0xA872, 0xA872, 4, 0xA892, 0xA8B3, 4, 0xA90A, 0xA921, 4, + 0xA930, 0xA946, 4, 0xA989, 0xA98B, 4, 0xA98F, 0xA9B2, 4, 0xA9E0, 0xA9E4, 4, + 0xA9E7, 0xA9EF, 4, 0xA9FA, 0xA9FE, 4, 0xAA06, 0xAA28, 4, 0xAA60, 0xAA6F, 4, + 0xAA71, 0xAA73, 4, 0xAA7A, 0xAA7A, 4, 0xAA7E, 0xAA7F, 4, 0xAA80, 0xAAAF, 4, + 0xAAE2, 0xAAEA, 4, 0xABC0, 0xABCD, 4, 0xABD0, 0xABD0, 4, 0xABD2, 0xABDA, 4, + 0x10A00, 0x10A00, 4, 0x10A10, 0x10A13, 4, 0x10A15, 0x10A17, 4, 0x10A19, 0x10A35, 4, + 0x11013, 0x11037, 4, 0x11075, 0x11075, 4, 0x1108D, 0x110AF, 4, 0x11107, 0x11126, 4, + 0x11144, 0x11144, 4, 0x11147, 0x11147, 4, 0x11155, 0x11172, 4, 0x11191, 0x111B2, 4, + 0x11208, 0x11211, 4, 0x11213, 0x1122B, 4, 0x1123F, 0x1123F, 4, 0x11284, 0x11286, 4, + 0x11288, 0x11288, 4, 0x1128A, 0x1128D, 4, 0x1128F, 0x1129D, 4, 0x1129F, 0x112A8, 4, + 0x112BA, 0x112DE, 4, 0x11315, 0x11328, 4, 0x1132A, 0x11330, 4, 0x11332, 0x11333, 4, + 0x11335, 0x11339, 4, 0x11392, 0x113B5, 4, 0x1140E, 0x11434, 4, 0x1148F, 0x114AF, 4, + 0x1158E, 0x115AE, 4, 0x1160E, 0x1162F, 4, 0x1168A, 0x116AA, 4, 0x116B8, 0x116B8, 4, + 0x11700, 0x1171A, 4, 0x11740, 0x11746, 4, 0x1180A, 0x1182B, 4, 0x1190C, 0x11913, 4, + 0x11915, 0x11916, 4, 0x11918, 0x1192F, 4, 0x119AE, 0x119D0, 4, 0x11A00, 0x11A00, 4, + 0x11A0B, 0x11A32, 4, 0x11A50, 0x11A50, 4, 0x11A5C, 0x11A83, 4, 0x11C0E, 0x11C2E, 4, + 0x11C72, 0x11C8F, 4, 0x11D0C, 0x11D30, 4, 0x11D6C, 0x11D89, 4, 0x11EE0, 0x11EF1, 4, + 0x11F12, 0x11F33, 4, 0x16101, 0x1611D, 4, 0x16D43, 0x16D62, 4, 0x9CE, 0x9CE, 5, + 0xC5D, 0xC5D, 5, 0xCDD, 0xCDD, 5, 0xD54, 0xD56, 5, 0xD7A, 0xD7F, 5, + 0x1CF2, 0x1CF3, 5, 0xCF1, 0xCF2, 16, 0x1CF5, 0x1CF6, 16, 0x11003, 0x11004, 16, + 0x11460, 0x11461, 16, 0x11A3A, 0x11A3A, 16, 0x111C2, 0x111C3, 13, 0x1193F, 0x1193F, 13, + 0x11A84, 0x11A85, 13, 0x11A87, 0x11A89, 13, 0xD4E, 0xD4E, 12, 0x113D1, 0x113D1, 12, + 0x11941, 0x11941, 12, 0x11A86, 0x11A86, 12, 0x11D46, 0x11D46, 12, 0x11F02, 0x11F02, 12, + 0x1A5A, 0x1A5A, 8, 0x17CC, 0x17CC, 15, 0xF8D, 0xF97, 14, 0xF99, 0xFBC, 14, + 0x1929, 0x192B, 14, 0x1A57, 0x1A57, 14, 0x1A5B, 0x1A5E, 14, 0x1BA1, 0x1BA1, 14, + 0x1BA2, 0x1BA3, 14, 0x1BAC, 0x1BAD, 14, 0x1C24, 0x1C25, 14, 0xA867, 0xA868, 14, + 0xA871, 0xA871, 14, 0x11C92, 0x11CA7, 14, 0x11CA9, 0x11CA9, 14, 0x11CAA, 0x11CAF, 14, + 0xA75, 0xA75, 10, 0xEBC, 0xEBC, 10, 0xEBD, 0xEBD, 10, 0x103B, 0x103C, 10, + 0x103D, 0x103E, 10, 0x105E, 0x1060, 10, 0x1082, 0x1082, 10, 0x1A55, 0x1A55, 10, + 0x1A56, 0x1A56, 10, 0xA8B4, 0xA8B4, 10, 0xA9BD, 0xA9BD, 10, 0xA9BE, 0xA9BF, 10, + 0xAA33, 0xAA34, 10, 0xAA35, 0xAA36, 10, 0x1171D, 0x1171D, 10, 0x1171E, 0x1171E, 10, + 0x1171F, 0x1171F, 10, 0x11940, 0x11940, 10, 0x11942, 0x11942, 10, 0x11A3B, 0x11A3E, 10, + 0x11D47, 0x11D47, 10, 0x1612A, 0x1612C, 10, 0x1612E, 0x1612E, 10, 0x1930, 0x1931, 6, + 0x1933, 0x1938, 6, 0x1939, 0x1939, 6, 0x19C1, 0x19C7, 6, 0x1A58, 0x1A59, 6, + 0x1B03, 0x1B03, 6, 0x1B81, 0x1B81, 6, 0x1BBE, 0x1BBF, 6, 0x1BF0, 0x1BF1, 6, + 0x1C2D, 0x1C33, 6, 0xA94F, 0xA951, 6, 0xA952, 0xA952, 6, 0xA982, 0xA982, 6, + 0xAA40, 0xAA42, 6, 0xAA43, 0xAA43, 6, 0xAA44, 0xAA4B, 6, 0xAA4C, 0xAA4C, 6, + 0xAA4D, 0xAA4D, 6, 0xABDB, 0xABE2, 6, 0x11A8A, 0x11A95, 6, 0xF88, 0xF8C, 7, + 0xB83, 0xB83, 20, 0x1970, 0x1974, 30, 0xAAC0, 0xAAC0, 30, 0xAAC2, 0xAAC2, 30, + 0xE48, 0xE4B, 31, 0xEC8, 0xECB, 31, 0x1037, 0x1037, 31, 0x1063, 0x1064, 31, + 0x1069, 0x106D, 31, 0x1087, 0x108C, 31, 0x108D, 0x108D, 31, 0x108F, 0x108F, 31, + 0x109A, 0x109B, 31, 0x19C8, 0x19C9, 31, 0x1A75, 0x1A79, 31, 0xA92B, 0xA92D, 31, + 0xAA7B, 0xAA7B, 31, 0xAA7C, 0xAA7C, 31, 0xAA7D, 0xAA7D, 31, 0xAABF, 0xAABF, 31, + 0xAAC1, 0xAAC1, 31, 0xABEC, 0xABEC, 31, 0xA71, 0xA71, 17, 0xAFB, 0xAFB, 17, + 0x11237, 0x11237, 17, 0x113D2, 0x113D2, 17, 0x11A98, 0x11A98, 17, 0x951, 0x952, 3, + 0xA51, 0xA51, 3, 0xAFA, 0xAFA, 3, 0xAFC, 0xAFC, 3, 0x1CD0, 0x1CD2, 3, + 0x1CD4, 0x1CE0, 3, 0x1CE1, 0x1CE1, 3, 0x1CF4, 0x1CF4, 3, 0x1CF7, 0x1CF7, 3, + 0x1CF8, 0x1CF9, 3, 0x20F0, 0x20F0, 3, 0xA8E0, 0xA8F1, 3, 0x1123E, 0x1123E, 3, + 0x11366, 0x1136C, 3, 0x11370, 0x11374, 3, 0x113E1, 0x113E2, 3, 0x17C9, 0x17CA, 27, + 0xB2, 0xB3, 29, 0x9FE, 0x9FE, 29, 0xECE, 0xECE, 29, 0xF35, 0xF35, 29, + 0xF37, 0xF37, 29, 0xFC6, 0xFC6, 29, 0x17CB, 0x17CB, 29, 0x17CE, 0x17D0, 29, + 0x17D3, 0x17D3, 29, 0x17DD, 0x17DD, 29, 0x193B, 0x193B, 29, 0x1A7B, 0x1A7C, 29, + 0x1A7F, 0x1A7F, 29, 0x1C36, 0x1C36, 29, 0x1DFB, 0x1DFB, 29, 0x2074, 0x2074, 29, + 0x2082, 0x2084, 29, 0x111C9, 0x111C9, 29, 0x1145E, 0x1145E, 29, 0x11A33, 0x11A33, 29, + 0xE4C, 0xE4C, 9, 0x17CD, 0x17CD, 9, 0x200C, 0x200C, 21, 0x200D, 0x200D, 19, + 0x1107F, 0x1107F, 24, 0x30, 0x39, 23, 0x966, 0x96F, 23, 0x9E6, 0x9EF, 23, + 0xA66, 0xA6F, 23, 0xAE6, 0xAEF, 23, 0xB66, 0xB6F, 23, 0xBE6, 0xBEF, 23, + 0xC66, 0xC6F, 23, 0xCE6, 0xCEF, 23, 0xD66, 0xD6F, 23, 0xDE6, 0xDEF, 23, + 0xE50, 0xE59, 23, 0xED0, 0xED9, 23, 0xF20, 0xF29, 23, 0xF2A, 0xF33, 23, + 0x1040, 0x1049, 23, 0x1090, 0x1099, 23, 0x17E0, 0x17E9, 23, 0x1946, 0x194F, 23, + 0x19D0, 0x19D9, 23, 0x19DA, 0x19DA, 23, 0x1A80, 0x1A89, 23, 0x1A90, 0x1A99, 23, + 0x1B50, 0x1B59, 23, 0x1BB0, 0x1BB9, 23, 0x1C40, 0x1C49, 23, 0xA8D0, 0xA8D9, 23, + 0xA900, 0xA909, 23, 0xA9D0, 0xA9D9, 23, 0xA9F0, 0xA9F9, 23, 0xAA50, 0xAA59, 23, + 0xABF0, 0xABF9, 23, 0x10A40, 0x10A48, 23, 0x11066, 0x1106F, 23, 0x11136, 0x1113F, 23, + 0x111D0, 0x111D9, 23, 0x111E1, 0x111F4, 23, 0x112F0, 0x112F9, 23, 0x11450, 0x11459, 23, + 0x114D0, 0x114D9, 23, 0x11650, 0x11659, 23, 0x116C0, 0x116C9, 23, 0x116D0, 0x116E3, 23, + 0x11730, 0x11739, 23, 0x1173A, 0x1173B, 23, 0x11950, 0x11959, 23, 0x11C50, 0x11C59, 23, + 0x11C5A, 0x11C6C, 23, 0x11D50, 0x11D59, 23, 0x11DA0, 0x11DA9, 23, 0x11F50, 0x11F59, 23, + 0x16130, 0x16139, 23, 0x16D70, 0x16D79, 23, 0x11052, 0x11065, 2, + }; + private static final Property INSC = new Property( + INSC_SHORT_VALUES, INSC_LONG_VALUES, + INSC_VALUE_ALIASES, INSC_VALUE_ALIAS_INDEX, + INSC_PROPERTY_ALIASES, INSC_RANGES, + 25); + + private static final String[] INPC_SHORT_VALUES = { + "Bottom", "Bottom_And_Left", "Bottom_And_Right", "Left", "Left_And_Right", "NA", "Overstruck", "Right", "Top", "Top_And_Bottom", "Top_And_Bottom_And_Left", "Top_And_Bottom_And_Right", "Top_And_Left", "Top_And_Left_And_Right", "Top_And_Right", "Visual_Order_Left" + }; + private static final String[] INPC_LONG_VALUES = { + "Bottom", "Bottom_And_Left", "Bottom_And_Right", "Left", "Left_And_Right", "Not_Applicable", "Overstruck", "Right", "Top", "Top_And_Bottom", "Top_And_Bottom_And_Left", "Top_And_Bottom_And_Right", "Top_And_Left", "Top_And_Left_And_Right", "Top_And_Right", "Visual_Order_Left" + }; + private static final String[] INPC_VALUE_ALIASES = { + "bottom", "bottomandleft", "bottomandright", "left", "leftandright", "na", "notapplicable", "overstruck", "right", "top", "topandbottom", "topandbottomandleft", "topandbottomandright", "topandleft", "topandleftandright", "topandright", "visualorderleft" + }; + private static final byte[] INPC_VALUE_ALIAS_INDEX = { + 0, 1, 2, 3, 4, 5, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + }; + private static final String[] INPC_PROPERTY_ALIASES = { + "indicpositionalcategory", "inpc" + }; + private static final int[] INPC_RANGES = { + 0x903, 0x903, 7, 0x93B, 0x93B, 7, 0x93E, 0x93E, 7, 0x940, 0x940, 7, + 0x949, 0x94C, 7, 0x94F, 0x94F, 7, 0x982, 0x983, 7, 0x9BE, 0x9BE, 7, + 0x9C0, 0x9C0, 7, 0x9D7, 0x9D7, 7, 0xA03, 0xA03, 7, 0xA3E, 0xA3E, 7, + 0xA40, 0xA40, 7, 0xA83, 0xA83, 7, 0xABE, 0xABE, 7, 0xAC0, 0xAC0, 7, + 0xACB, 0xACC, 7, 0xB02, 0xB03, 7, 0xB3E, 0xB3E, 7, 0xB40, 0xB40, 7, + 0xBBE, 0xBBF, 7, 0xBC1, 0xBC2, 7, 0xBD7, 0xBD7, 7, 0xC01, 0xC03, 7, + 0xC41, 0xC44, 7, 0xC82, 0xC83, 7, 0xCBE, 0xCBE, 7, 0xCC1, 0xCC4, 7, + 0xCD5, 0xCD6, 7, 0xCF3, 0xCF3, 7, 0xD02, 0xD03, 7, 0xD3E, 0xD40, 7, + 0xD57, 0xD57, 7, 0xD82, 0xD83, 7, 0xDCF, 0xDD1, 7, 0xDD8, 0xDD8, 7, + 0xDDF, 0xDDF, 7, 0xDF2, 0xDF3, 7, 0xE30, 0xE30, 7, 0xE32, 0xE33, 7, + 0xE45, 0xE45, 7, 0xEB0, 0xEB0, 7, 0xEB2, 0xEB3, 7, 0xF3E, 0xF3E, 7, + 0xF7F, 0xF7F, 7, 0x102B, 0x102C, 7, 0x1038, 0x1038, 7, 0x103B, 0x103B, 7, + 0x1056, 0x1057, 7, 0x1062, 0x1064, 7, 0x1067, 0x106D, 7, 0x1083, 0x1083, 7, + 0x1087, 0x108C, 7, 0x108F, 0x108F, 7, 0x109A, 0x109C, 7, 0x1715, 0x1715, 7, + 0x1734, 0x1734, 7, 0x17B6, 0x17B6, 7, 0x17C7, 0x17C8, 7, 0x1923, 0x1924, 7, + 0x1929, 0x192B, 7, 0x1930, 0x1931, 7, 0x1933, 0x1938, 7, 0x19B0, 0x19B4, 7, + 0x19B8, 0x19B9, 7, 0x19BB, 0x19C0, 7, 0x19C8, 0x19C9, 7, 0x1A1A, 0x1A1A, 7, + 0x1A57, 0x1A57, 7, 0x1A61, 0x1A61, 7, 0x1A63, 0x1A64, 7, 0x1A6D, 0x1A6D, 7, + 0x1B04, 0x1B04, 7, 0x1B35, 0x1B35, 7, 0x1B44, 0x1B44, 7, 0x1B82, 0x1B82, 7, + 0x1BA1, 0x1BA1, 7, 0x1BA7, 0x1BA7, 7, 0x1BAA, 0x1BAA, 7, 0x1BE7, 0x1BE7, 7, + 0x1BEA, 0x1BEC, 7, 0x1BEE, 0x1BEE, 7, 0x1BF2, 0x1BF3, 7, 0x1C24, 0x1C26, 7, + 0x1C2A, 0x1C2B, 7, 0x1CE1, 0x1CE1, 7, 0x1CF7, 0x1CF7, 7, 0xA823, 0xA824, 7, + 0xA827, 0xA827, 7, 0xA880, 0xA881, 7, 0xA8B4, 0xA8C3, 7, 0xA952, 0xA953, 7, + 0xA983, 0xA983, 7, 0xA9B4, 0xA9B5, 7, 0xAA33, 0xAA33, 7, 0xAA4D, 0xAA4D, 7, + 0xAA7B, 0xAA7B, 7, 0xAA7D, 0xAA7D, 7, 0xAAB1, 0xAAB1, 7, 0xAABA, 0xAABA, 7, + 0xAABD, 0xAABD, 7, 0xAAEF, 0xAAEF, 7, 0xAAF5, 0xAAF5, 7, 0xABE3, 0xABE4, 7, + 0xABE6, 0xABE7, 7, 0xABE9, 0xABEA, 7, 0xABEC, 0xABEC, 7, 0x11000, 0x11000, 7, + 0x11002, 0x11002, 7, 0x11082, 0x11082, 7, 0x110B0, 0x110B0, 7, 0x110B2, 0x110B2, 7, + 0x110B7, 0x110B8, 7, 0x11145, 0x11146, 7, 0x11182, 0x11182, 7, 0x111B3, 0x111B3, 7, + 0x111B5, 0x111B5, 7, 0x111C0, 0x111C0, 7, 0x1122C, 0x1122E, 7, 0x11235, 0x11235, 7, + 0x112E0, 0x112E0, 7, 0x112E2, 0x112E2, 7, 0x11302, 0x11303, 7, 0x1133E, 0x1133F, 7, + 0x11341, 0x11344, 7, 0x1134D, 0x1134D, 7, 0x11357, 0x11357, 7, 0x11362, 0x11363, 7, + 0x113B8, 0x113B8, 7, 0x113C9, 0x113CA, 7, 0x113CC, 0x113CD, 7, 0x113CF, 0x113CF, 7, + 0x11435, 0x11435, 7, 0x11437, 0x11437, 7, 0x11440, 0x11441, 7, 0x11445, 0x11445, 7, + 0x114B0, 0x114B0, 7, 0x114B2, 0x114B2, 7, 0x114BD, 0x114BD, 7, 0x114C1, 0x114C1, 7, + 0x115AF, 0x115AF, 7, 0x115B1, 0x115B1, 7, 0x115BE, 0x115BE, 7, 0x11630, 0x11632, 7, + 0x1163B, 0x1163C, 7, 0x1163E, 0x1163E, 7, 0x116AC, 0x116AC, 7, 0x116AF, 0x116AF, 7, + 0x116B6, 0x116B6, 7, 0x11720, 0x11721, 7, 0x1182C, 0x1182C, 7, 0x1182E, 0x1182E, 7, + 0x11838, 0x11838, 7, 0x11930, 0x11934, 7, 0x1193D, 0x1193D, 7, 0x11940, 0x11940, 7, + 0x119D1, 0x119D1, 7, 0x119D3, 0x119D3, 7, 0x119DC, 0x119DF, 7, 0x11A39, 0x11A39, 7, + 0x11A57, 0x11A58, 7, 0x11A97, 0x11A97, 7, 0x11B61, 0x11B61, 7, 0x11B65, 0x11B65, 7, + 0x11B67, 0x11B67, 7, 0x11C2F, 0x11C2F, 7, 0x11C3E, 0x11C3E, 7, 0x11CA9, 0x11CA9, 7, + 0x11CB4, 0x11CB4, 7, 0x11D46, 0x11D46, 7, 0x11D8A, 0x11D8E, 7, 0x11D93, 0x11D94, 7, + 0x11D96, 0x11D96, 7, 0x11EF6, 0x11EF6, 7, 0x11F03, 0x11F03, 7, 0x11F34, 0x11F35, 7, + 0x11F41, 0x11F41, 7, 0x1612C, 0x1612C, 7, 0x16D40, 0x16D42, 7, 0x16D63, 0x16D6A, 7, + 0x16D6B, 0x16D6C, 7, 0x93F, 0x93F, 3, 0x94E, 0x94E, 3, 0x9BF, 0x9BF, 3, + 0x9C7, 0x9C8, 3, 0xA3F, 0xA3F, 3, 0xABF, 0xABF, 3, 0xB47, 0xB47, 3, + 0xBC6, 0xBC8, 3, 0xD46, 0xD48, 3, 0xDD9, 0xDD9, 3, 0xDDB, 0xDDB, 3, + 0xF3F, 0xF3F, 3, 0x1031, 0x1031, 3, 0x1084, 0x1084, 3, 0x17C1, 0x17C3, 3, + 0x1A19, 0x1A19, 3, 0x1A55, 0x1A55, 3, 0x1A6E, 0x1A72, 3, 0x1B3E, 0x1B3F, 3, + 0x1BA6, 0x1BA6, 3, 0x1C27, 0x1C28, 3, 0x1C34, 0x1C35, 3, 0xA9BA, 0xA9BB, 3, + 0xAA2F, 0xAA30, 3, 0xAA34, 0xAA34, 3, 0xAAEB, 0xAAEB, 3, 0xAAEE, 0xAAEE, 3, + 0x110B1, 0x110B1, 3, 0x1112C, 0x1112C, 3, 0x111B4, 0x111B4, 3, 0x111CE, 0x111CE, 3, + 0x112E1, 0x112E1, 3, 0x11347, 0x11348, 3, 0x113C2, 0x113C2, 3, 0x113C5, 0x113C5, 3, + 0x11436, 0x11436, 3, 0x114B1, 0x114B1, 3, 0x114B9, 0x114B9, 3, 0x115B0, 0x115B0, 3, + 0x115B8, 0x115B8, 3, 0x116AE, 0x116AE, 3, 0x11726, 0x11726, 3, 0x1182D, 0x1182D, 3, + 0x11935, 0x11935, 3, 0x11937, 0x11937, 3, 0x119D2, 0x119D2, 3, 0x119E4, 0x119E4, 3, + 0x11CB1, 0x11CB1, 3, 0x11EF5, 0x11EF5, 3, 0x11F3E, 0x11F3F, 3, 0x1612A, 0x1612B, 3, + 0xE40, 0xE44, 15, 0xEC0, 0xEC4, 15, 0x19B5, 0x19B7, 15, 0x19BA, 0x19BA, 15, + 0xAAB5, 0xAAB6, 15, 0xAAB9, 0xAAB9, 15, 0xAABB, 0xAABC, 15, 0x9CB, 0x9CC, 4, + 0xB4B, 0xB4B, 4, 0xBCA, 0xBCC, 4, 0xD4A, 0xD4C, 4, 0xDDC, 0xDDC, 4, + 0xDDE, 0xDDE, 4, 0x17C0, 0x17C0, 4, 0x17C4, 0x17C5, 4, 0x1B40, 0x1B41, 4, + 0x1134B, 0x1134C, 4, 0x113C7, 0x113C8, 4, 0x114BC, 0x114BC, 4, 0x114BE, 0x114BE, 4, + 0x115BA, 0x115BA, 4, 0x11938, 0x11938, 4, 0x900, 0x902, 8, 0x93A, 0x93A, 8, + 0x945, 0x948, 8, 0x951, 0x951, 8, 0x955, 0x955, 8, 0x981, 0x981, 8, + 0x9FE, 0x9FE, 8, 0xA01, 0xA02, 8, 0xA47, 0xA48, 8, 0xA4B, 0xA4C, 8, + 0xA70, 0xA71, 8, 0xA81, 0xA82, 8, 0xAC5, 0xAC5, 8, 0xAC7, 0xAC8, 8, + 0xAFA, 0xAFF, 8, 0xB01, 0xB01, 8, 0xB3F, 0xB3F, 8, 0xB55, 0xB56, 8, + 0xB82, 0xB82, 8, 0xBC0, 0xBC0, 8, 0xBCD, 0xBCD, 8, 0xC00, 0xC00, 8, + 0xC04, 0xC04, 8, 0xC3E, 0xC40, 8, 0xC46, 0xC47, 8, 0xC4A, 0xC4D, 8, + 0xC55, 0xC55, 8, 0xC81, 0xC81, 8, 0xCBF, 0xCBF, 8, 0xCC6, 0xCC6, 8, + 0xCCC, 0xCCD, 8, 0xD00, 0xD01, 8, 0xD3B, 0xD3C, 8, 0xD4D, 0xD4D, 8, + 0xD4E, 0xD4E, 8, 0xD81, 0xD81, 8, 0xDCA, 0xDCA, 8, 0xDD2, 0xDD3, 8, + 0xE31, 0xE31, 8, 0xE34, 0xE37, 8, 0xE47, 0xE4E, 8, 0xEB1, 0xEB1, 8, + 0xEB4, 0xEB7, 8, 0xEBB, 0xEBB, 8, 0xEC8, 0xECE, 8, 0xF39, 0xF39, 8, + 0xF72, 0xF72, 8, 0xF7A, 0xF7E, 8, 0xF80, 0xF80, 8, 0xF82, 0xF83, 8, + 0xF86, 0xF87, 8, 0x102D, 0x102E, 8, 0x1032, 0x1036, 8, 0x103A, 0x103A, 8, + 0x1071, 0x1074, 8, 0x1085, 0x1086, 8, 0x109D, 0x109D, 8, 0x1712, 0x1712, 8, + 0x1732, 0x1732, 8, 0x1752, 0x1752, 8, 0x1772, 0x1772, 8, 0x17B7, 0x17BA, 8, + 0x17C6, 0x17C6, 8, 0x17C9, 0x17D1, 8, 0x17D3, 0x17D3, 8, 0x17DD, 0x17DD, 8, + 0x1920, 0x1921, 8, 0x1927, 0x1928, 8, 0x193A, 0x193A, 8, 0x1A17, 0x1A17, 8, + 0x1A1B, 0x1A1B, 8, 0x1A58, 0x1A5A, 8, 0x1A62, 0x1A62, 8, 0x1A65, 0x1A68, 8, + 0x1A6B, 0x1A6B, 8, 0x1A73, 0x1A7C, 8, 0x1B00, 0x1B03, 8, 0x1B34, 0x1B34, 8, + 0x1B36, 0x1B37, 8, 0x1B42, 0x1B42, 8, 0x1B6B, 0x1B6B, 8, 0x1B6D, 0x1B73, 8, + 0x1B80, 0x1B81, 8, 0x1BA4, 0x1BA4, 8, 0x1BA8, 0x1BA9, 8, 0x1BE6, 0x1BE6, 8, + 0x1BE8, 0x1BE9, 8, 0x1BED, 0x1BED, 8, 0x1BEF, 0x1BF1, 8, 0x1C2D, 0x1C33, 8, + 0x1C36, 0x1C36, 8, 0x1CD0, 0x1CD2, 8, 0x1CDA, 0x1CDB, 8, 0x1CE0, 0x1CE0, 8, + 0x1CF4, 0x1CF4, 8, 0x1DFB, 0x1DFB, 8, 0x20F0, 0x20F0, 8, 0xA802, 0xA802, 8, + 0xA806, 0xA806, 8, 0xA80B, 0xA80B, 8, 0xA826, 0xA826, 8, 0xA8C5, 0xA8C5, 8, + 0xA8E0, 0xA8F1, 8, 0xA8FF, 0xA8FF, 8, 0xA926, 0xA92A, 8, 0xA94A, 0xA94A, 8, + 0xA94F, 0xA951, 8, 0xA980, 0xA982, 8, 0xA9B3, 0xA9B3, 8, 0xA9B6, 0xA9B7, 8, + 0xA9BC, 0xA9BC, 8, 0xA9E5, 0xA9E5, 8, 0xAA29, 0xAA2C, 8, 0xAA2E, 0xAA2E, 8, + 0xAA31, 0xAA31, 8, 0xAA43, 0xAA43, 8, 0xAA4C, 0xAA4C, 8, 0xAA7C, 0xAA7C, 8, + 0xAAB0, 0xAAB0, 8, 0xAAB2, 0xAAB3, 8, 0xAAB7, 0xAAB8, 8, 0xAABE, 0xAABF, 8, + 0xAAC1, 0xAAC1, 8, 0xAAED, 0xAAED, 8, 0xABE5, 0xABE5, 8, 0x10A05, 0x10A05, 8, + 0x10A0F, 0x10A0F, 8, 0x10A38, 0x10A38, 8, 0x11001, 0x11001, 8, 0x11038, 0x1103B, 8, + 0x11042, 0x11046, 8, 0x11070, 0x11070, 8, 0x11073, 0x11074, 8, 0x11080, 0x11081, 8, + 0x110B5, 0x110B6, 8, 0x11100, 0x11102, 8, 0x11127, 0x11129, 8, 0x1112D, 0x1112D, 8, + 0x11130, 0x11130, 8, 0x11134, 0x11134, 8, 0x11180, 0x11181, 8, 0x111BC, 0x111BE, 8, + 0x111C2, 0x111C3, 8, 0x111CB, 0x111CB, 8, 0x111CF, 0x111CF, 8, 0x11230, 0x11231, 8, + 0x11234, 0x11234, 8, 0x11236, 0x11237, 8, 0x1123E, 0x1123E, 8, 0x112DF, 0x112DF, 8, + 0x112E5, 0x112E8, 8, 0x11300, 0x11301, 8, 0x11340, 0x11340, 8, 0x11366, 0x1136C, 8, + 0x11370, 0x11374, 8, 0x113CE, 0x113CE, 8, 0x113D1, 0x113D1, 8, 0x113E1, 0x113E1, 8, + 0x1143E, 0x1143F, 8, 0x11443, 0x11444, 8, 0x1145E, 0x1145E, 8, 0x114BA, 0x114BA, 8, + 0x114BF, 0x114C0, 8, 0x115BC, 0x115BD, 8, 0x11639, 0x1163A, 8, 0x1163D, 0x1163D, 8, + 0x11640, 0x11640, 8, 0x116AB, 0x116AB, 8, 0x116AD, 0x116AD, 8, 0x116B2, 0x116B5, 8, + 0x1171F, 0x1171F, 8, 0x11722, 0x11723, 8, 0x11727, 0x11727, 8, 0x11729, 0x1172B, 8, + 0x11833, 0x11837, 8, 0x1193B, 0x1193C, 8, 0x1193F, 0x1193F, 8, 0x11941, 0x11941, 8, + 0x119DA, 0x119DB, 8, 0x11A01, 0x11A01, 8, 0x11A04, 0x11A09, 8, 0x11A35, 0x11A38, 8, + 0x11A51, 0x11A51, 8, 0x11A54, 0x11A56, 8, 0x11A84, 0x11A89, 8, 0x11A96, 0x11A96, 8, + 0x11A98, 0x11A98, 8, 0x11B60, 0x11B60, 8, 0x11B64, 0x11B64, 8, 0x11B66, 0x11B66, 8, + 0x11C30, 0x11C31, 8, 0x11C38, 0x11C3D, 8, 0x11CB3, 0x11CB3, 8, 0x11CB5, 0x11CB6, 8, + 0x11D31, 0x11D35, 8, 0x11D3A, 0x11D3A, 8, 0x11D3C, 0x11D3D, 8, 0x11D3F, 0x11D41, 8, + 0x11D43, 0x11D43, 8, 0x11D90, 0x11D91, 8, 0x11D95, 0x11D95, 8, 0x11EF3, 0x11EF3, 8, + 0x11F00, 0x11F01, 8, 0x11F02, 0x11F02, 8, 0x11F36, 0x11F37, 8, 0x11F40, 0x11F40, 8, + 0x11F5A, 0x11F5A, 8, 0x1611E, 0x16129, 8, 0x1612D, 0x1612D, 8, 0x93C, 0x93C, 0, + 0x941, 0x944, 0, 0x94D, 0x94D, 0, 0x952, 0x952, 0, 0x956, 0x957, 0, + 0x962, 0x963, 0, 0x9BC, 0x9BC, 0, 0x9C1, 0x9C4, 0, 0x9CD, 0x9CD, 0, + 0x9E2, 0x9E3, 0, 0xA3C, 0xA3C, 0, 0xA41, 0xA42, 0, 0xA4D, 0xA4D, 0, + 0xA51, 0xA51, 0, 0xA75, 0xA75, 0, 0xABC, 0xABC, 0, 0xAC1, 0xAC4, 0, + 0xACD, 0xACD, 0, 0xAE2, 0xAE3, 0, 0xB3C, 0xB3C, 0, 0xB41, 0xB44, 0, + 0xB4D, 0xB4D, 0, 0xB62, 0xB63, 0, 0xC3C, 0xC3C, 0, 0xC56, 0xC56, 0, + 0xC62, 0xC63, 0, 0xCBC, 0xCBC, 0, 0xCE2, 0xCE3, 0, 0xD41, 0xD44, 0, + 0xD62, 0xD63, 0, 0xDD4, 0xDD4, 0, 0xDD6, 0xDD6, 0, 0xE38, 0xE3A, 0, + 0xEB8, 0xEBA, 0, 0xEBC, 0xEBC, 0, 0xF18, 0xF19, 0, 0xF35, 0xF35, 0, + 0xF37, 0xF37, 0, 0xF71, 0xF71, 0, 0xF74, 0xF75, 0, 0xF84, 0xF84, 0, + 0xF8D, 0xF97, 0, 0xF99, 0xFBC, 0, 0xFC6, 0xFC6, 0, 0x102F, 0x1030, 0, + 0x1037, 0x1037, 0, 0x103D, 0x103E, 0, 0x1058, 0x1059, 0, 0x105E, 0x1060, 0, + 0x1082, 0x1082, 0, 0x108D, 0x108D, 0, 0x1713, 0x1714, 0, 0x1733, 0x1733, 0, + 0x1753, 0x1753, 0, 0x1773, 0x1773, 0, 0x17BB, 0x17BD, 0, 0x1922, 0x1922, 0, + 0x1932, 0x1932, 0, 0x1939, 0x1939, 0, 0x193B, 0x193B, 0, 0x1A18, 0x1A18, 0, + 0x1A56, 0x1A56, 0, 0x1A5B, 0x1A5E, 0, 0x1A69, 0x1A6A, 0, 0x1A6C, 0x1A6C, 0, + 0x1A7F, 0x1A7F, 0, 0x1B38, 0x1B3A, 0, 0x1B6C, 0x1B6C, 0, 0x1BA2, 0x1BA3, 0, + 0x1BA5, 0x1BA5, 0, 0x1BAC, 0x1BAD, 0, 0x1C2C, 0x1C2C, 0, 0x1C37, 0x1C37, 0, + 0x1CD5, 0x1CD9, 0, 0x1CDC, 0x1CDF, 0, 0x1CED, 0x1CED, 0, 0xA825, 0xA825, 0, + 0xA82C, 0xA82C, 0, 0xA8C4, 0xA8C4, 0, 0xA92B, 0xA92D, 0, 0xA947, 0xA949, 0, + 0xA94B, 0xA94E, 0, 0xA9B8, 0xA9B9, 0, 0xA9BD, 0xA9BD, 0, 0xAA2D, 0xAA2D, 0, + 0xAA32, 0xAA32, 0, 0xAA35, 0xAA36, 0, 0xAAB4, 0xAAB4, 0, 0xAAEC, 0xAAEC, 0, + 0xABE8, 0xABE8, 0, 0xABED, 0xABED, 0, 0x10A02, 0x10A03, 0, 0x10A0C, 0x10A0E, 0, + 0x10A39, 0x10A3A, 0, 0x1103C, 0x11041, 0, 0x110B3, 0x110B4, 0, 0x110B9, 0x110BA, 0, + 0x110C2, 0x110C2, 0, 0x1112A, 0x1112B, 0, 0x11131, 0x11132, 0, 0x11173, 0x11173, 0, + 0x111B6, 0x111BB, 0, 0x111C9, 0x111CA, 0, 0x111CC, 0x111CC, 0, 0x1122F, 0x1122F, 0, + 0x11241, 0x11241, 0, 0x112E3, 0x112E4, 0, 0x112E9, 0x112EA, 0, 0x1133B, 0x1133C, 0, + 0x113BB, 0x113C0, 0, 0x113D2, 0x113D2, 0, 0x113E2, 0x113E2, 0, 0x11438, 0x1143D, 0, + 0x11442, 0x11442, 0, 0x11446, 0x11446, 0, 0x114B3, 0x114B8, 0, 0x114C2, 0x114C3, 0, + 0x115B2, 0x115B5, 0, 0x115BF, 0x115C0, 0, 0x115DC, 0x115DD, 0, 0x11633, 0x11638, 0, + 0x1163F, 0x1163F, 0, 0x116B0, 0x116B1, 0, 0x116B7, 0x116B7, 0, 0x1171D, 0x1171D, 0, + 0x11724, 0x11725, 0, 0x11728, 0x11728, 0, 0x1182F, 0x11832, 0, 0x11839, 0x1183A, 0, + 0x11943, 0x11943, 0, 0x119D4, 0x119D7, 0, 0x119E0, 0x119E0, 0, 0x11A02, 0x11A03, 0, + 0x11A0A, 0x11A0A, 0, 0x11A33, 0x11A34, 0, 0x11A3B, 0x11A3E, 0, 0x11A52, 0x11A53, 0, + 0x11A59, 0x11A5B, 0, 0x11A8A, 0x11A95, 0, 0x11B62, 0x11B63, 0, 0x11C32, 0x11C36, 0, + 0x11C3F, 0x11C3F, 0, 0x11C92, 0x11CA7, 0, 0x11CAA, 0x11CB0, 0, 0x11CB2, 0x11CB2, 0, + 0x11D36, 0x11D36, 0, 0x11D42, 0x11D42, 0, 0x11D44, 0x11D44, 0, 0x11D47, 0x11D47, 0, + 0x11EF4, 0x11EF4, 0, 0x11F38, 0x11F3A, 0, 0x1612E, 0x1612F, 0, 0xC48, 0xC48, 9, + 0xF73, 0xF73, 9, 0xF76, 0xF79, 9, 0xF81, 0xF81, 9, 0x1B3C, 0x1B3C, 9, + 0x1112E, 0x1112F, 9, 0xAC9, 0xAC9, 14, 0xB57, 0xB57, 14, 0xCC0, 0xCC0, 14, + 0xCC7, 0xCC8, 14, 0xCCA, 0xCCB, 14, 0x1925, 0x1926, 14, 0x1B43, 0x1B43, 14, + 0x111BF, 0x111BF, 14, 0x11232, 0x11233, 14, 0x113B9, 0x113BA, 14, 0xB48, 0xB48, 12, + 0xDDA, 0xDDA, 12, 0x17BE, 0x17BE, 12, 0x1C29, 0x1C29, 12, 0x114BB, 0x114BB, 12, + 0x115B9, 0x115B9, 12, 0xB4C, 0xB4C, 13, 0xDDD, 0xDDD, 13, 0x17BF, 0x17BF, 13, + 0x115BB, 0x115BB, 13, 0x1B3B, 0x1B3B, 2, 0xA9BE, 0xA9BE, 2, 0xA9C0, 0xA9C0, 2, + 0x11942, 0x11942, 2, 0xA9BF, 0xA9BF, 1, 0x1B3D, 0x1B3D, 11, 0x103C, 0x103C, 10, + 0x1171E, 0x1171E, 10, 0x1CD4, 0x1CD4, 6, 0x1CE2, 0x1CE8, 6, 0x10A01, 0x10A01, 6, + 0x10A06, 0x10A06, 6, + }; + private static final Property INPC = new Property( + INPC_SHORT_VALUES, INPC_LONG_VALUES, + INPC_VALUE_ALIASES, INPC_VALUE_ALIAS_INDEX, + INPC_PROPERTY_ALIASES, INPC_RANGES, + 5); + + static boolean isPropertyAlias(String alias) { + return property(alias) != null; + } + + static UnicodeSet valueSet(String propertyAlias, String valueAlias) { + Property property = property(propertyAlias); + return property == null ? null : property.valueSet(valueAlias); + } + + static String canonicalValue(String propertyAlias, String valueAlias) { + Property property = property(propertyAlias); + return property == null ? null : property.canonicalValue(valueAlias); + } + + static String[] canonicalValues(String propertyAlias) { + Property property = property(propertyAlias); + return property == null ? null : property.longValues.clone(); + } + + static String defaultValue(String propertyAlias) { + Property property = property(propertyAlias); + return property == null ? null : property.longValues[property.defaultIndex]; + } + + static int valueCount(String propertyAlias) { + Property property = property(propertyAlias); + return property == null ? 0 : property.longValues.length; + } + + static String shortValue(String propertyAlias, int index) { + Property property = property(propertyAlias); + return property == null || index < 0 || index >= property.shortValues.length + ? null : property.shortValues[index]; + } + + static String canonicalValue(String propertyAlias, int index) { + Property property = property(propertyAlias); + return property == null || index < 0 || index >= property.longValues.length + ? null : property.longValues[index]; + } + + static UnicodeSet valueSet(String propertyAlias, int index) { + Property property = property(propertyAlias); + return property == null || index < 0 || index >= property.valueSets.length + ? null : property.valueSets[index]; + } + + private static Property property(String alias) { + String loose = looseName(alias); + if (loose == null) return null; + if (INSC.hasPropertyAlias(loose)) return INSC; + if (INPC.hasPropertyAlias(loose)) return INPC; + return null; + } + + private static String looseName(String name) { + if (name == null) return null; + StringBuilder loose = new StringBuilder(name.length()); + for (int i = 0; i < name.length(); i++) { + char character = name.charAt(i); + if (character == '_' || character == '-' || Character.isWhitespace(character)) continue; + loose.append(Character.toLowerCase(character)); + } + return loose.toString(); + } + + private static final class Property { + private final String[] shortValues; + private final String[] longValues; + private final String[] valueAliases; + private final byte[] valueAliasIndex; + private final String[] propertyAliases; + private final int[] ranges; + private final int defaultIndex; + private final UnicodeSet[] valueSets; + + private Property(String[] shortValues, String[] longValues, + String[] valueAliases, byte[] valueAliasIndex, + String[] propertyAliases, int[] ranges, + int defaultIndex) { + this.shortValues = shortValues; + this.longValues = longValues; + this.valueAliases = valueAliases; + this.valueAliasIndex = valueAliasIndex; + this.propertyAliases = propertyAliases; + this.ranges = ranges; + this.defaultIndex = defaultIndex; + this.valueSets = buildValueSets(); + } + + private boolean hasPropertyAlias(String loose) { + for (String alias : propertyAliases) { + if (alias.equals(loose)) return true; + } + return false; + } + + private UnicodeSet valueSet(String alias) { + int index = valueIndex(alias); + return index < 0 ? null : valueSets[index]; + } + + private String canonicalValue(String alias) { + int index = valueIndex(alias); + return index < 0 ? null : longValues[index]; + } + + private int valueIndex(String alias) { + String loose = looseName(alias); + if (loose == null) return -1; + for (int i = 0; i < valueAliases.length; i++) { + if (valueAliases[i].equals(loose)) return valueAliasIndex[i]; + } + return -1; + } + + private UnicodeSet[] buildValueSets() { + UnicodeSet[] sets = new UnicodeSet[longValues.length]; + for (int i = 0; i < sets.length; i++) sets[i] = new UnicodeSet(); + sets[defaultIndex].add(0, 0x10FFFF); + for (int i = 0; i < ranges.length; i += 3) { + int valueIndex = ranges[i + 2]; + if (valueIndex != defaultIndex) { + sets[defaultIndex].remove(ranges[i], ranges[i + 1]); + sets[valueIndex].add(ranges[i], ranges[i + 1]); + } + } + for (UnicodeSet set : sets) set.freeze(); + return sets; + } + } + + private PerlUnicodeIndicCategoryData() { + } +} diff --git a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java index 8ab6b5da4..265c1b54c 100644 --- a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java +++ b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java @@ -1620,7 +1620,8 @@ static CharacterPropertyResolver.Result resolveJoniProperty( } else if (isCanonicalCombiningClassProperty(name) || PerlUnicodeBidiClassData.isPropertyAlias(name) || PerlUnicodeDecompositionTypeData.isPropertyAlias(name) - || PerlUnicodeEastAsianWidthData.isPropertyAlias(name)) { + || PerlUnicodeEastAsianWidthData.isPropertyAlias(name) + || PerlUnicodeIndicCategoryData.isPropertyAlias(name)) { caseFold = false; } else if (PerlUnicodeNumericValueData.isPropertyAlias(name) || PerlUnicodeJoiningGroupData.isPropertyAlias(name)) { @@ -1992,6 +1993,18 @@ && isCanonicalCombiningClassProperty(alias.substring(0, assignment))) { } return eastAsianWidth; } + if (assignment > 0 && assignment < alias.length() - 1 + && PerlUnicodeIndicCategoryData.isPropertyAlias( + alias.substring(0, assignment))) { + String propertyName = alias.substring(0, assignment); + UnicodeSet indicCategory = PerlUnicodeIndicCategoryData.valueSet( + propertyName, alias.substring(assignment + 1)); + if (indicCategory == null) { + throw new PerlCompilerException( + "Can't find Unicode property definition \"" + alias + "\""); + } + return indicCategory; + } if (assignment > 0 && assignment < alias.length() - 1 && PerlUnicodeNumericValueData.isPropertyAlias( alias.substring(0, assignment))) { @@ -2292,6 +2305,9 @@ private static PerlUnicodePropertyWildcard resolvePerlUnicodePropertyWildcard( if (resolvePerlMissingBaseAlias(name) != null) { return resolvePerlBinaryPropertyWildcard(name, wildcard); } + if (PerlUnicodeIndicCategoryData.isPropertyAlias(name)) { + return resolvePerlIndicCategoryWildcard(name, wildcard); + } if (isPerlWordBreakProperty(name)) { return resolvePerlEnumeratedPropertyWildcard( UProperty.WORD_BREAK, wildcard, "Word_Break", @@ -2311,6 +2327,33 @@ private static PerlUnicodePropertyWildcard resolvePerlUnicodePropertyWildcard( return null; } + private static PerlUnicodePropertyWildcard resolvePerlIndicCategoryWildcard( + String propertyName, PerlPropertyValueMatcher wildcard) { + UnicodeSet result = new UnicodeSet(); + boolean matched = false; + for (int index = 0; + index < PerlUnicodeIndicCategoryData.valueCount(propertyName); + index++) { + String shortValue = PerlUnicodeIndicCategoryData.shortValue( + propertyName, index); + String canonicalValue = PerlUnicodeIndicCategoryData.canonicalValue( + propertyName, index); + if (!matchesPerlUnicodePropertyWildcard( + wildcard, shortValue, canonicalValue)) { + continue; + } + matched = true; + result.addAll(PerlUnicodeIndicCategoryData.valueSet( + propertyName, index)); + } + if (!matched) { + throw new IllegalArgumentException( + "No Unicode property value wildcard matches " + + propertyName.trim()); + } + return new PerlUnicodePropertyWildcard(result.freeze(), null, false); + } + private static PerlUnicodePropertyWildcard resolvePerlBinaryPropertyWildcard( String propertyName, PerlPropertyValueMatcher wildcard) { boolean positive = matchesPerlUnicodePropertyWildcard( diff --git a/src/test/java/org/perlonjava/runtime/regex/PerlUnicodeIndicCategoryDataTest.java b/src/test/java/org/perlonjava/runtime/regex/PerlUnicodeIndicCategoryDataTest.java new file mode 100644 index 000000000..1dab2f36e --- /dev/null +++ b/src/test/java/org/perlonjava/runtime/regex/PerlUnicodeIndicCategoryDataTest.java @@ -0,0 +1,123 @@ +package org.perlonjava.runtime.regex; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.ibm.icu.text.UnicodeSet; +import org.joni.CharacterPropertyResolver; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +@Tag("unit") +class PerlUnicodeIndicCategoryDataTest { + @Test + void usesCurrentPerl544Unicode17Sources() { + assertEquals("17.0.0", PerlUnicodeIndicCategoryData.UNICODE_VERSION); + assertEquals("3fc122f4cf58b0c19268d5f810263b04ab4e1e67743386ec0e0ada9c76aec5be", + PerlUnicodeIndicCategoryData.INDIC_SYLLABIC_CATEGORY_SHA256); + assertEquals("68cedc29a7e57f984d90fe2c7712f2e6d0c717e253db219607daea8997d6c480", + PerlUnicodeIndicCategoryData.INDIC_POSITIONAL_CATEGORY_SHA256); + assertEquals("670d2bebb48649c04fabfbf033308073dcff47946324a8033237254c048b3b01", + PerlUnicodeIndicCategoryData.PROP_VALUE_ALIASES_SHA256); + assertEquals("4441f573caf952ffece1d7c892e7715bd7136dfc26f96eb6f268bf1e474715fb", + PerlUnicodeIndicCategoryData.PROPERTY_ALIASES_SHA256); + } + + @Test + void acceptsPropertyAndLooseValueAliases() { + assertTrue(PerlUnicodeIndicCategoryData.isPropertyAlias("InSC")); + assertTrue(PerlUnicodeIndicCategoryData.isPropertyAlias( + "indic syllabic-category")); + assertTrue(PerlUnicodeIndicCategoryData.isPropertyAlias("InPC")); + assertTrue(PerlUnicodeIndicCategoryData.isPropertyAlias( + "INDIC_POSITIONAL CATEGORY")); + assertFalse(PerlUnicodeIndicCategoryData.isPropertyAlias("Script")); + + UnicodeSet vowel = PerlUnicodeIndicCategoryData.valueSet( + "InSC", "Vowel_Dependent"); + assertSame(vowel, PerlUnicodeIndicCategoryData.valueSet( + "Indic-Syllabic Category", "vowel dependent")); + assertEquals("Vowel_Dependent", + PerlUnicodeIndicCategoryData.canonicalValue( + "InSC", "v-o_w e l dependent")); + assertEquals("Not_Applicable", + PerlUnicodeIndicCategoryData.canonicalValue("InPC", "NA")); + assertNull(PerlUnicodeIndicCategoryData.valueSet("InSC", "unknown")); + assertNull(PerlUnicodeIndicCategoryData.valueSet("unknown", "Other")); + } + + @Test + void retainsEveryValueAndAppliesMissingDefaults() { + assertEquals(37, + PerlUnicodeIndicCategoryData.canonicalValues("InSC").length); + assertEquals(16, + PerlUnicodeIndicCategoryData.canonicalValues("InPC").length); + assertEquals("Other", + PerlUnicodeIndicCategoryData.defaultValue("InSC")); + assertEquals("Not_Applicable", + PerlUnicodeIndicCategoryData.defaultValue("InPC")); + + assertTrue(PerlUnicodeIndicCategoryData.valueSet( + "InSC", "Vowel_Dependent").contains(0x093E)); + assertTrue(PerlUnicodeIndicCategoryData.valueSet( + "InPC", "Bottom").contains(0x093C)); + assertTrue(PerlUnicodeIndicCategoryData.valueSet( + "InSC", "Other").contains('A')); + assertTrue(PerlUnicodeIndicCategoryData.valueSet( + "InPC", "NA").contains('A')); + } + + @Test + void eachPropertyFormsOneDisjointUnicodePartition() { + assertPartition("InSC"); + assertPartition("InPC"); + } + + @Test + void resolvesThroughTheNativeJoniHookWithoutCaseFolding() { + CharacterPropertyResolver.Result syllabic = + UnicodeResolver.resolveJoniProperty( + "InSC=Vowel_Dependent", false, true); + CharacterPropertyResolver.Result positional = + UnicodeResolver.resolveJoniProperty( + "Indic_Positional_Category=Bottom", true, true); + assertNotNull(syllabic); + assertNotNull(positional); + assertFalse(syllabic.caseFold); + assertFalse(positional.caseFold); + assertTrue(contains(syllabic.ranges, 0x093E)); + assertTrue(contains(positional.ranges, 0x093C)); + assertFalse(contains(syllabic.ranges, 'A')); + assertFalse(contains(positional.ranges, 'A')); + } + + private static void assertPartition(String property) { + String[] values = PerlUnicodeIndicCategoryData.canonicalValues(property); + UnicodeSet union = new UnicodeSet(); + for (int i = 0; i < values.length; i++) { + UnicodeSet current = PerlUnicodeIndicCategoryData.valueSet(property, i); + assertNotNull(current, values[i]); + assertTrue(current.isFrozen(), values[i]); + union.addAll(current); + for (int j = i + 1; j < values.length; j++) { + UnicodeSet overlap = new UnicodeSet(current).retainAll( + PerlUnicodeIndicCategoryData.valueSet(property, j)); + assertTrue(overlap.isEmpty(), values[i] + " overlaps " + values[j]); + } + } + assertEquals(0x110000, union.size()); + assertTrue(union.contains(0, 0x10FFFF)); + } + + private static boolean contains(int[] ranges, int codePoint) { + if (ranges == null) return false; + for (int i = 1; i + 1 < ranges.length; i += 2) { + if (ranges[i] <= codePoint && codePoint <= ranges[i + 1]) return true; + } + return false; + } +} diff --git a/src/test/resources/unit/regex/indic_category_properties.t b/src/test/resources/unit/regex/indic_category_properties.t new file mode 100644 index 000000000..0f96b33a7 --- /dev/null +++ b/src/test/resources/unit/regex/indic_category_properties.t @@ -0,0 +1,65 @@ +use strict; +use warnings; +use utf8; +no warnings 'experimental::regex_sets'; +no warnings 'experimental::uniprop_wildcards'; +use Test::More; + +my $vowel_dependent = "\x{093E}"; +my $bottom = "\x{093C}"; + +ok($vowel_dependent =~ /\p{InSC=Vowel_Dependent}/, + 'InSC short property matches outside a class'); +ok('A' !~ /\p{InSC=Vowel_Dependent}/, + 'InSC excludes a default-value scalar'); +ok($vowel_dependent =~ /[\p{Indic_Syllabic_Category=Vowel_Dependent}]/, + 'long InSC property matches in a standard class'); +ok($vowel_dependent =~ /\p{indic syllabic-category = vowel dependent}/, + 'InSC accepts loose property and value spelling'); +ok($bottom =~ /\p{InPC=Bottom}/, + 'InPC short property matches outside a class'); +ok('A' !~ /\p{InPC=Bottom}/, + 'InPC excludes a default-value scalar'); +ok($bottom =~ /[\p{Indic_Positional_Category=Bottom}]/, + 'long InPC property matches in a standard class'); +ok($bottom =~ /\p{indic positional-category = bottom}/, + 'InPC accepts loose property and value spelling'); + +ok('A' =~ /\p{InSC=Other}/, 'InSC applies the Other missing default'); +ok($vowel_dependent !~ /\p{InSC=Other}/, + 'explicit InSC range overrides the missing default'); +ok('A' =~ /\p{InPC=NA}/, 'InPC accepts the short default value alias'); +ok($bottom !~ /\p{InPC=NA}/, + 'explicit InPC range overrides the missing default'); + +ok($vowel_dependent =~ /(?[ \p{InSC=Vowel_Dependent} & \P{InPC=Bottom} ])/, + 'Indic properties compose in an extended class'); +ok($bottom =~ /(?[ \p{InPC=Bottom} & \P{InSC=Vowel_Dependent} ])/, + 'extended class selects the positional property'); +ok('A' !~ /(?[ \p{InSC=Vowel_Dependent} | \p{InPC=Bottom} ])/, + 'extended union excludes nonmembers'); + +ok($vowel_dependent =~ /\p{InSC=Vowel_Dependent}/i, + 'InSC resolves under ignore case'); +ok('A' !~ /\p{InSC=Vowel_Dependent}/i, + 'ignore case does not add letters to InSC'); +ok($bottom =~ /[\p{InPC=Bottom}]/i, + 'InPC resolves in an ignore-case standard class'); +ok('A' !~ /[\p{InPC=Bottom}]/i, + 'ignore case does not add letters to InPC'); +ok($vowel_dependent =~ /\p{InSC=:\AVowel_Dependent\z:}/, + 'InSC property-value wildcard selects a canonical value'); +ok($bottom =~ /\p{InPC=:\ABottom\z:}/, + 'InPC property-value wildcard selects a canonical value'); + +{ + use bytes; + ok('A' =~ /\p{InSC=Other}/, 'InSC default resolves in byte mode'); + ok("\xE9" =~ /[\p{InSC=Other}]/, + 'InSC default covers a high byte'); + ok('A' =~ /\p{InPC=NA}/, 'InPC default resolves in byte mode'); + ok("\xE9" =~ /[\p{InPC=NA}]/, + 'InPC default covers a high byte'); +} + +done_testing; From 9b6e9ad5d3a61323b5fb63da439e910f925075bf Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 10:42:58 +0200 Subject: [PATCH 03/43] docs(regex): track refreshed checkpoint ownership Keep PR 1087 user-owned during the latest-Perl sync and route ongoing implementation into the separate post-checkpoint WIP. Generated with Codex (https://openai.com/codex) Co-Authored-By: Codex --- dev/design/phase36-regex-parity.md | 72 ++++++++++++++---------------- 1 file changed, 33 insertions(+), 39 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index 1c243eec6..1ed468fb0 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -169,23 +169,18 @@ Exit: release evidence and public/internal documentation match the code. ## Current Release Gate -The current integration source composition ends at `702bb2853`. Its -constituent source tranches are each -warning-free full-build green. Raw property, warning, escaped-class callback, -byte-NEL line break, and word-boundary whitespace coverage pass on JVM and -interpreter. The byte-NEL tranche makes unchanged `regexp.t` 2210/2210 on both -backends. The native word-boundary tranches make `uniprops10.t` -20,070/20,070 on both backends, recovering 1,190 assertions from their exact -parents with no introduced assertion IDs; direct Joni and focused boundary -oracles are green. One exact combined build of this composition is the current -checkpoint barrier. The integrated overload-copy increment correction -passes unchanged `lib/overload_fallback.t` 4/4 on JVM and interpreter, -adjacent overload gates, and a warning-free full build. The preceding exact -combined code barrier `18b9a0133` passes packaging, licensed Joni, all unit -shards, and the shadow JAR. PR 1087 remains the next -incremental release PR; update it only from an exact warning-free integration -head whose complete comparator has no pass-count decrease from PR 958 and no -new invalid, missing, timeout, truncated, incomplete, or zero-TAP row. +PR 1087 is the next incremental release checkpoint. The user owns its +latest-Perl refresh with `dev/import-perl5/sync.pl`; workers and the coordinator +must not mutate, rebase, push, or reinterpret acceptance against an older PR +head while that refresh is in progress. The refreshed exact head must pass a +warning-free combined build, the fresh 622-file gate, strict PR-958 comparison, +the additional release checks, and complete CI with no new invalid, missing, +timeout, truncated, incomplete, or zero-TAP row before merge. + +Independent implementation continues on `integrate/phase36-post1087-wip`. +Its current preserved tranche generates current-Perl InSC/InPC data and closes +all 4,240 labelled `uniprops02.t` assertions through native Joni property +resolution. It will be rebased only after the refreshed checkpoint merges. The planning 622-file, jobs-5, timeout-300 gate at `d3a4bb074` completed with 677871/695187 assertions passing and 95 improved rows. Fresh private-tree @@ -226,33 +221,32 @@ plus the additional release checks, before that checkpoint may merge. Active ownership: -- P3: close the ordinary native matcher checklist and implement the largest - remaining disjoint matcher-state root, if any. -- P4: implement the general frontend array-match/substitution compile warning - root that accounts for the missing planned `pat.t`/`pat_thr.t` assertion. -- P5: generate and wire current-Perl InSC/InPC data covering 2,120 residual - assertions; the warn-retirement dossier is complete. -- P6: classify and close the largest remaining native `reg_mesg.t` - Parser/Lexer diagnostic family. +- P3: finish the native `FIND_LONGEST` capture-region correction and deliver + its ordinary-matcher closure matrix. +- P4: finish immutable `pat.t`/`pat_thr.t` gates and deliver the frontend + aggregate regex-binding warning correction. +- P5: close the complete remaining 420-assertion generated-property batch + after the delivered InSC/InPC tranche. +- P6: finish no-warning retirement qualification, then close the native + infinite-recursion diagnostic and prepare extended-property scanner removal. - Coordinator: integration, conflict resolution, immutable acceptance, PR/CI, plan state, combined build, and release evidence. ## Ordered Next Steps -1. Run one exact combined warning-free `make` at the current clean integration - head, including P3 a18 and P6 a35. -2. Refresh focused Unicode, - `regexp.t`, `regex_sets.t`, `charset.t`, `pat.t`, - `pat_advanced.t`, `pat_re_eval.t`, and `reg_mesg.t` gates on that exact - immutable head. -3. Keep PR 1087 frozen at exact locally-green head `cb0926a5c`. Preserve the - documented fixture-invalid and sandbox-environment dispositions. Require the - user's fresh 622-file run, strict PR-958 comparison, additional release - checks, and exact-head CI before merging the incremental checkpoint. -4. Continue P4 documentation, P5 generated Indic properties, P6 pat/diagnostic - work, and remaining native Unicode/runtime roots in a new WIP - PR. Repeat focused tests per semantic tranche and one combined build per - integration batch. +1. Let the user refresh PR 1087 from latest Perl with + `dev/import-perl5/sync.pl`; do not race that branch or reuse acceptance from + its former head. +2. On the refreshed exact PR head, run one warning-free `make`, focused Unicode, + `regexp.t`, `regex_sets.t`, `charset.t`, `pat.t`, `pat_advanced.t`, + `pat_re_eval.t`, and `reg_mesg.t` gates, then the user's fresh 622-file run, + strict PR-958 comparison, additional release checks, and exact-head CI. +3. Merge PR 1087 only when every checkpoint gate is green or has a documented + baseline-equivalent environmental disposition. +4. Meanwhile integrate P3 matcher-state, P4 warning, P5 generated-property, + and P6 diagnostic/scanner-removal deliveries only into the separate + post-1087 WIP. Batch focused tests and one combined full build, then rebase + this WIP onto merged refreshed `master`. 5. Delete remaining production migration scaffolding and prove all constants, closures, conditions, verbs, recursion, dynamic source, byte strings, and Unicode strings execute through Joni. From 1ca58c4cce5cd075a6d3513f5c6c5fb9f50ff8a6 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 10:47:25 +0200 Subject: [PATCH 04/43] fix(regex): retain captures for longest Joni match Keep the first winning capture region when FIND_LONGEST encounters a later equal or shorter alternative. Add direct Joni coverage for both shorter and equal alternatives. Generated with [OpenAI Codex](https://openai.com/codex/) Co-Authored-By: OpenAI Codex --- .../joni/src/org/joni/ByteCodeMachine.java | 7 -- .../joni/test/TestPerlFindLongestRegion.java | 68 +++++++++++++++++++ 2 files changed, 68 insertions(+), 7 deletions(-) create mode 100644 third_party/joni/test/org/joni/test/TestPerlFindLongestRegion.java diff --git a/third_party/joni/src/org/joni/ByteCodeMachine.java b/third_party/joni/src/org/joni/ByteCodeMachine.java index d5771aba3..c0dcff526 100644 --- a/third_party/joni/src/org/joni/ByteCodeMachine.java +++ b/third_party/joni/src/org/joni/ByteCodeMachine.java @@ -750,13 +750,6 @@ private boolean opEnd() { msaBegin = visibleMatchStart() - str; msaEnd = s - str; } - } else { - Region region = msaRegion; - if (region != null) { - region.clear(); - } else { - msaBegin = msaEnd = 0; - } } // end_best_len: /* default behavior: return first-matching result. */ diff --git a/third_party/joni/test/org/joni/test/TestPerlFindLongestRegion.java b/third_party/joni/test/org/joni/test/TestPerlFindLongestRegion.java new file mode 100644 index 000000000..daa1f9f0c --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestPerlFindLongestRegion.java @@ -0,0 +1,68 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import static org.junit.Assert.assertEquals; + +import java.nio.charset.StandardCharsets; + +import org.jcodings.specific.ASCIIEncoding; +import org.joni.Matcher; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.junit.Test; + +public class TestPerlFindLongestRegion { + private static Matcher matcher(String pattern, String input) { + byte[] patternBytes = pattern.getBytes(StandardCharsets.US_ASCII); + byte[] inputBytes = input.getBytes(StandardCharsets.US_ASCII); + Regex regex = new Regex(patternBytes, 0, patternBytes.length, + Option.CAPTURE_GROUP | Option.FIND_LONGEST, + ASCIIEncoding.INSTANCE, Syntax.PerlNG); + return regex.matcher(inputBytes); + } + + @Test + public void shorterAlternativeDoesNotClearTheLongestCaptureRegion() { + byte[] input = "aax".getBytes(StandardCharsets.US_ASCII); + Matcher matcher = matcher("(aa|a)", "aax"); + + assertEquals(0, matcher.search(0, input.length, Option.NONE)); + assertEquals(0, matcher.getRegion().getBeg(0)); + assertEquals(2, matcher.getRegion().getEnd(0)); + assertEquals(0, matcher.getRegion().getBeg(1)); + assertEquals(2, matcher.getRegion().getEnd(1)); + } + + @Test + public void equalAlternativeRetainsTheFirstWinningCaptureRegion() { + byte[] input = "a".getBytes(StandardCharsets.US_ASCII); + Matcher matcher = matcher("(a|(a))", "a"); + + assertEquals(0, matcher.search(0, input.length, Option.NONE)); + assertEquals(0, matcher.getRegion().getBeg(0)); + assertEquals(1, matcher.getRegion().getEnd(0)); + assertEquals(0, matcher.getRegion().getBeg(1)); + assertEquals(1, matcher.getRegion().getEnd(1)); + assertEquals(-1, matcher.getRegion().getBeg(2)); + assertEquals(-1, matcher.getRegion().getEnd(2)); + } +} From 359ee4f1a9d9e618bf455005aa1512b5f7e0ab62 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 10:49:27 +0200 Subject: [PATCH 05/43] docs(regex): close ordinary matcher phase Record the complete ordinary-pattern closure matrix and update active post-checkpoint ownership after integrating the matcher and warning fixes. Generated with Codex (https://openai.com/codex) Co-Authored-By: Codex --- dev/design/phase36-regex-parity.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index 1ed468fb0..8b32f5d22 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -82,11 +82,11 @@ repeatable baseline. ### Phase 1 — Ordinary-pattern Joni parity -- [ ] Close all remaining PR 958 ordinary-pattern regressions. -- [ ] Prove capture, duplicate-name, branch-reset, region/bounds, zero-width +- [x] Close all remaining PR 958 ordinary-pattern regressions. +- [x] Prove capture, duplicate-name, branch-reset, region/bounds, zero-width progression, `\G`, `/g`, `/c`, `/o`, substitution, reuse, and nested match-state behavior. -- [ ] Prove byte/Unicode pattern and subject variant selection. +- [x] Prove byte/Unicode pattern and subject variant selection. - [x] Delete the disconnected production backend selector while preserving its historical compatibility tests under test scope. @@ -221,12 +221,12 @@ plus the additional release checks, before that checkpoint may merge. Active ownership: -- P3: finish the native `FIND_LONGEST` capture-region correction and deliver - its ordinary-matcher closure matrix. -- P4: finish immutable `pat.t`/`pat_thr.t` gates and deliver the frontend - aggregate regex-binding warning correction. -- P5: close the complete remaining 420-assertion generated-property batch - after the delivered InSC/InPC tranche. +- P3: close the complete stable regex debug-trace contract after the integrated + native `FIND_LONGEST` capture-region correction and closure matrix. +- P4: close ordered aggregation of multiple fatal/nonfatal regex compile + diagnostics after the delivered aggregate-binding warning correction. +- P5: close the complete remaining 420-assertion generated-property batch, + then the disjoint 5,064-assertion Script bare-alias tranche. - P6: finish no-warning retirement qualification, then close the native infinite-recursion diagnostic and prepare extended-property scanner removal. - Coordinator: integration, conflict resolution, immutable acceptance, PR/CI, From 240c0aa784edacfa8d1664de579851d364b04995 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 10:58:35 +0200 Subject: [PATCH 06/43] docs(regex): bind checkpoint gates to refreshed head Record the exact user-owned PR 1087 acceptance head and require Unicode generators to treat latest-Perl source hashes as provenance rather than pins. Generated with Codex (https://openai.com/codex) Co-Authored-By: Codex --- dev/design/phase36-regex-parity.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index 8b32f5d22..c7b4aab3a 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -169,10 +169,10 @@ Exit: release evidence and public/internal documentation match the code. ## Current Release Gate -PR 1087 is the next incremental release checkpoint. The user owns its -latest-Perl refresh with `dev/import-perl5/sync.pl`; workers and the coordinator -must not mutate, rebase, push, or reinterpret acceptance against an older PR -head while that refresh is in progress. The refreshed exact head must pass a +PR 1087 is the next incremental release checkpoint at exact refreshed head +`5a1e0c15e`. The user owns its fresh 622-file acceptance run; workers and the +coordinator must not mutate, rebase, push, or reinterpret acceptance against an +older PR head while that run is in progress. The refreshed exact head must pass a warning-free combined build, the fresh 622-file gate, strict PR-958 comparison, the additional release checks, and complete CI with no new invalid, missing, timeout, truncated, incomplete, or zero-TAP row before merge. @@ -289,6 +289,10 @@ Active ownership: absent or pull latest upstream when present, then run `dev/import-perl5/sync.pl`. Sync must include required generated inputs such as `unicore/Name.pl`, preserve needed non-regex patches, and be idempotent. +- Unicode generators must consume the latest checked-out `perl5/` tables and + derive their version and source hashes as provenance. Historical source + hashes or Perl/Unicode versions must not act as pins that block a valid latest + upstream refresh; checked-in output hashes remain reproducibility gates. ## Final Acceptance From 5c016fd7e9117991146022b41b6ae5f0737d6d65 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 11:17:10 +0200 Subject: [PATCH 07/43] docs(regex): record refreshed acceptance blockers Keep the phase plan current with the exact latest-Perl sync boundary, the five blocking PR 1087 rows, and their active remediation owners. Generated with Codex (https://openai.com/codex) Co-Authored-By: Codex --- dev/design/phase36-regex-parity.md | 64 ++++++++++++++---------------- 1 file changed, 29 insertions(+), 35 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index c7b4aab3a..c796f4276 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -170,36 +170,28 @@ Exit: release evidence and public/internal documentation match the code. ## Current Release Gate PR 1087 is the next incremental release checkpoint at exact refreshed head -`5a1e0c15e`. The user owns its fresh 622-file acceptance run; workers and the -coordinator must not mutate, rebase, push, or reinterpret acceptance against an -older PR head while that run is in progress. The refreshed exact head must pass a -warning-free combined build, the fresh 622-file gate, strict PR-958 comparison, -the additional release checks, and complete CI with no new invalid, missing, -timeout, truncated, incomplete, or zero-TAP row before merge. +`5a1e0c15e`, after a fresh pull of the latest `perl5/` source and a complete +`dev/import-perl5/sync.pl` run. Those imported fixtures are authoritative and +must not be reverted or patched to recover old counts. Its combined build and +exact-head Ubuntu/Windows CI are green. The +fresh complete corpus gate is red and blocks merge until these rows are repaired +and rerun: `re/pat.t` and `re/pat_thr.t` time out after TAP 829; `op/do.t` drops +25 passing assertions despite a byte-identical fixture; `class/accessor.t` +drops eight; and `japh/abigail.t` drops one. Acceptance must have no new +invalid, missing, timeout, truncated, incomplete, or zero-TAP row and no +unresolved PR-958 pass-count decrease. The user owns the release checkout; +workers use private exact-head worktrees and must not mutate, rebase, or push +the user's branch. Independent implementation continues on `integrate/phase36-post1087-wip`. Its current preserved tranche generates current-Perl InSC/InPC data and closes all 4,240 labelled `uniprops02.t` assertions through native Joni property resolution. It will be rebased only after the refreshed checkpoint merges. -The planning 622-file, jobs-5, timeout-300 gate at `d3a4bb074` completed -with 677871/695187 assertions passing and 95 improved rows. Fresh private-tree -isolation found no regex pass decrease: both `pat.t` variants improve by 147 -passing assertions. Plan/platform drift explains four raw decreases; missing -private-import inputs explain two new porting errors. The -`lib/overload_fallback.t` 3/4 decrease is fixed at `cbee747f1`. -`io/socket.t` is environmental rather than a product decrease: the exact -one-file runner outside the restricted sandbox passes 25/25 with zero -errors/timeouts, while the sandbox denies localhost bind and clamps -`SO_SNDBUF` to 816; focused JVM/interpreter option probes outside it both -return 32768. There is no remaining reproduced non-regex pass-count blocker. -The complete runner, comparator, isolation, dossier, 622-file list, reducers, -and verified checksum manifest are -durably retained under -`../PerlOnJava/logs/test_20260821_080900_d3a4bb074_a16_*`. Use it for residual -root planning, but do not treat it as release acceptance: the user must run a -fresh 622-file gate and strict PR-958 comparison at the exact PR 1087 head, -plus the additional release checks, before that checkpoint may merge. +Repair the five blocking rows with targeted, baseline-equivalent reruns before +repeating the complete corpus. Once targeted evidence is green, rerun the +complete latest-Perl corpus, strict PR-958 comparison, and additional release +checks at the same immutable head. ### Execution Tracker @@ -223,24 +215,26 @@ Active ownership: - P3: close the complete stable regex debug-trace contract after the integrated native `FIND_LONGEST` capture-region correction and closure matrix. -- P4: close ordered aggregation of multiple fatal/nonfatal regex compile - diagnostics after the delivered aggregate-binding warning correction. +- P4: deliver the green fatal/nonfatal diagnostic aggregation change, then fix + the exact-head `pat.t` and `pat_thr.t` timeout root. - P5: close the complete remaining 420-assertion generated-property batch, then the disjoint 5,064-assertion Script bare-alias tranche. -- P6: finish no-warning retirement qualification, then close the native - infinite-recursion diagnostic and prepare extended-property scanner removal. +- P6: publish no-warning retirement qualification, then fix/classify the + exact-head `op/do.t`, `class/accessor.t`, and `japh/abigail.t` decreases; + resume infinite-recursion diagnostics afterward. - Coordinator: integration, conflict resolution, immutable acceptance, PR/CI, plan state, combined build, and release evidence. ## Ordered Next Steps -1. Let the user refresh PR 1087 from latest Perl with - `dev/import-perl5/sync.pl`; do not race that branch or reuse acceptance from - its former head. -2. On the refreshed exact PR head, run one warning-free `make`, focused Unicode, - `regexp.t`, `regex_sets.t`, `charset.t`, `pat.t`, `pat_advanced.t`, - `pat_re_eval.t`, and `reg_mesg.t` gates, then the user's fresh 622-file run, - strict PR-958 comparison, additional release checks, and exact-head CI. +1. Fix and rerun the five exact-head release-blocking rows in private worktrees; + preserve latest-Perl tests and separate changed test cardinality from runtime + regressions with exact assertion-ID evidence. +2. Integrate the fixes into PR 1087, run one warning-free `make`, focused + Unicode, `regexp.t`, `regex_sets.t`, `charset.t`, `pat.t`, `pat_thr.t`, + `pat_advanced.t`, `pat_re_eval.t`, and `reg_mesg.t` gates, then repeat the + complete corpus, strict PR-958 comparison, additional release checks, and + exact-head CI. 3. Merge PR 1087 only when every checkpoint gate is green or has a documented baseline-equivalent environmental disposition. 4. Meanwhile integrate P3 matcher-state, P4 warning, P5 generated-property, From 7369d33d177f58f3935ff51e80d626b1f406ec9b Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 11:30:53 +0200 Subject: [PATCH 08/43] docs(regex): classify refreshed acceptance rows Record latest-corpus cardinality, the isolated pat contention evidence, and normalized standard-Perl handling for the op/do acceptance row. Generated with Codex (https://openai.com/codex) Co-Authored-By: Codex --- dev/design/phase36-regex-parity.md | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index c796f4276..a6fed9c0a 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -151,7 +151,9 @@ outside it. ### Phase 6 — Release and documentation -- [ ] Pass immutable complete 622-file JVM acceptance against PR 958. +- [ ] Pass immutable complete latest-Perl JVM acceptance against PR 958 by + matched path, including every newly synced test (currently 623 files versus + the 622-file baseline). - [ ] Pass the complete regex-bearing ledger on the interpreter and reconcile every JVM/interpreter semantic difference. - [ ] Pass direct/thread parity, bounded `pat_psycho*` and `speed*`, @@ -174,10 +176,12 @@ PR 1087 is the next incremental release checkpoint at exact refreshed head `dev/import-perl5/sync.pl` run. Those imported fixtures are authoritative and must not be reverted or patched to recover old counts. Its combined build and exact-head Ubuntu/Windows CI are green. The -fresh complete corpus gate is red and blocks merge until these rows are repaired -and rerun: `re/pat.t` and `re/pat_thr.t` time out after TAP 829; `op/do.t` drops -25 passing assertions despite a byte-identical fixture; `class/accessor.t` -drops eight; and `japh/abigail.t` drops one. Acceptance must have no new +fresh complete corpus gate is red and blocks merge. Its two `pat*` rows timed +out under resource contention: isolated single-job probes execute all 1,302 +assertions with 1,249 passes each, but equivalent exact-head quiet reruns remain +required. `op/do.t` needs ID-normalized disposition against standard Perl's +66/66 because PR958 over-executed it as 94/99; `class/accessor.t` drops eight; +and `japh/abigail.t` drops one nondeterministically. Acceptance must have no new invalid, missing, timeout, truncated, incomplete, or zero-TAP row and no unresolved PR-958 pass-count decrease. The user owns the release checkout; workers use private exact-head worktrees and must not mutate, rebase, or push @@ -213,10 +217,11 @@ checks at the same immutable head. Active ownership: -- P3: close the complete stable regex debug-trace contract after the integrated - native `FIND_LONGEST` capture-region correction and closure matrix. -- P4: deliver the green fatal/nonfatal diagnostic aggregation change, then fix - the exact-head `pat.t` and `pat_thr.t` timeout root. +- P3: close the complete stable regex debug-trace contract and interpreter eval + propagation; no speculative production change is needed for the `pat*` + contention timeout. +- P4: deliver the green fatal/nonfatal diagnostic aggregation change, then close + the 200-row warning tranche. - P5: close the complete remaining 420-assertion generated-property batch, then the disjoint 5,064-assertion Script bare-alias tranche. - P6: publish no-warning retirement qualification, then fix/classify the From a2423a1edba764a54be98be948ec5bde4cfcc211 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 11:32:12 +0200 Subject: [PATCH 09/43] fix(regex): match Perl infinite recursion diagnostics Translate Joni's exact never-ending-recursion syntax exception to Perl's Infinite recursion diagnostic identity while preserving Joni's native classification and position metadata. Generated with [OpenAI Codex](https://openai.com/codex/) Co-Authored-By: OpenAI Codex --- .../runtime/regex/RuntimeRegex.java | 3 + .../regex/infinite_recursion_diagnostics.t | 32 ++++++++++ .../TestPerlInfiniteRecursionDiagnostic.java | 58 +++++++++++++++++++ 3 files changed, 93 insertions(+) create mode 100644 src/test/resources/unit/regex/infinite_recursion_diagnostics.t create mode 100644 third_party/joni/test/org/joni/test/TestPerlInfiniteRecursionDiagnostic.java diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java index dfd83dff0..75d0ac3b0 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java @@ -934,6 +934,9 @@ && containsExecutableSource(originalPatternString, throw new PerlCompilerException(message + " m/" + originalPatternString + "/"); } + if ("never ending recursion".equals(message)) { + throw new PerlCompilerException("Infinite recursion in regex"); + } int bytePosition = ((SyntaxException) e).getPatternPosition(); if (bytePosition != SyntaxException.UNKNOWN_PATTERN_POSITION) { int characterPosition = utf8ByteOffsetToCharacterOffset( diff --git a/src/test/resources/unit/regex/infinite_recursion_diagnostics.t b/src/test/resources/unit/regex/infinite_recursion_diagnostics.t new file mode 100644 index 000000000..ef3537250 --- /dev/null +++ b/src/test/resources/unit/regex/infinite_recursion_diagnostics.t @@ -0,0 +1,32 @@ +use strict; +use warnings; +use Test::More; + +my @cases = ( + ['lookahead left recursion', q{q(a) =~ /(.(?2))((?<=(?=(?1)).))/}], + ['whole-pattern left recursion', q{q(aa) =~ /(?R)a/}], + ['inter-cyclic optional recursion', + q{q(bbaa) =~ /(?&x)(?(DEFINE)(?(?&y)*a)(?(?&x)*b))/}], + ['optional left recursion', q{q(abc) =~ /a((?1)?)c/}], + ['minimal optional left recursion', q{q(abc) =~ /a((?1)??)c/}], + ['star left recursion', q{q(abc) =~ /a((?1)*)c/}], + ['plus left recursion', q{q(abc) =~ /a((?1)+)c/}], + ['bounded left recursion', q{q(abc) =~ /a((?1){0,3})c/}], +); + +for my $case (@cases) { + my ($name, $source) = @$case; + my @warnings; + local $SIG{__WARN__} = sub { push @warnings, @_ }; + my $value = eval $source; + ok(!defined($value), "$name dies"); + like($@, qr/^Infinite recursion in regex at /, + "$name uses Perl's diagnostic identity"); + is(scalar(@warnings), 0, "$name does not warn before dying"); +} + +$@ = 'sentinel'; +ok(q(aaabbb) =~ /a(?R)?b/, 'productive optional self recursion still matches'); +is($@, 'sentinel', 'productive recursion preserves the existing error scalar'); + +done_testing; diff --git a/third_party/joni/test/org/joni/test/TestPerlInfiniteRecursionDiagnostic.java b/third_party/joni/test/org/joni/test/TestPerlInfiniteRecursionDiagnostic.java new file mode 100644 index 000000000..1a5d7389b --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestPerlInfiniteRecursionDiagnostic.java @@ -0,0 +1,58 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +import java.nio.charset.StandardCharsets; + +import org.jcodings.specific.UTF8Encoding; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.joni.WarnCallback; +import org.joni.exception.SyntaxException; +import org.junit.Test; + +public class TestPerlInfiniteRecursionDiagnostic { + private static final String[] PATTERNS = { + "(.(?2))((?<=(?=(?1)).))", + "(?R)a", + "a((?1)?)c", + "a((?1)??)c", + "a((?1)*)c", + "a((?1)+)c", + "a((?1){0,3})c" + }; + + @Test + public void rejectsNeverEndingRecursionBeforeMatching() { + for (String pattern : PATTERNS) { + byte[] bytes = pattern.getBytes(StandardCharsets.UTF_8); + SyntaxException error = assertThrows(SyntaxException.class, + () -> new Regex(bytes, 0, bytes.length, Option.NONE, + UTF8Encoding.INSTANCE, Syntax.PerlNG, + WarnCallback.NONE)); + assertEquals("never ending recursion", error.getDiagnosticMessage()); + assertEquals(bytes.length, error.getPatternPosition()); + } + } +} From ebf3143d19ef54cd857c1e44c4fc14ebc3e2d531 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 11:38:25 +0200 Subject: [PATCH 10/43] feat(regex): generate remaining current Unicode properties Generate the residual Unicode properties needed by the current Perl tables, resolve bare Script aliases with Perl precedence, and make schema-2 generator metadata provenance-only while retaining generated-output reproducibility. Generated with [OpenAI Codex](https://openai.com/codex/) Co-Authored-By: OpenAI Codex --- dev/tools/generate_perl_unicode_data.pl | 56 +- ...nerate_perl_unicode_indic_category_data.pl | 58 +- ...ate_perl_unicode_residual_property_data.pl | 283 ++ .../generate_perl_unicode_script_data.pl | 50 +- dev/tools/perl_unicode_data_generators.json | 17 +- .../tests/perl_unicode_generator_pipeline.t | 35 +- .../perl_unicode_latest_source_generator.t | 91 + .../regex/PerlUnicodeIndicCategoryData.java | 2 +- .../PerlUnicodeResidualPropertyData.java | 2618 +++++++++++++++++ .../runtime/regex/PerlUnicodeScriptData.java | 2 +- .../runtime/regex/UnicodeResolver.java | 96 +- .../PerlUnicodeResidualPropertyDataTest.java | 58 + .../unit/regex/residual_unicode_properties.t | 57 + .../unit/regex/script_bare_is_aliases.t | 34 + 14 files changed, 3359 insertions(+), 98 deletions(-) create mode 100644 dev/tools/generate_perl_unicode_residual_property_data.pl create mode 100644 dev/tools/tests/perl_unicode_latest_source_generator.t create mode 100644 src/main/java/org/perlonjava/runtime/regex/PerlUnicodeResidualPropertyData.java create mode 100644 src/test/java/org/perlonjava/runtime/regex/PerlUnicodeResidualPropertyDataTest.java create mode 100644 src/test/resources/unit/regex/residual_unicode_properties.t create mode 100644 src/test/resources/unit/regex/script_bare_is_aliases.t diff --git a/dev/tools/generate_perl_unicode_data.pl b/dev/tools/generate_perl_unicode_data.pl index a558c2ffd..338fc69b1 100644 --- a/dev/tools/generate_perl_unicode_data.pl +++ b/dev/tools/generate_perl_unicode_data.pl @@ -10,7 +10,8 @@ use JSON::PP qw(decode_json); use lib File::Spec->catdir($FindBin::Bin, 'lib'); use PerlOnJava::UnicodeGenerator qw( - read_pinned_source read_raw read_unicode_version repo_root select_perl_root select_unicode_root + read_pinned_source read_raw read_unicode_version repo_root select_perl_root + select_unicode_root verify_unicode_notice ); my $check = 0; @@ -108,11 +109,26 @@ my @required = ('version', sort keys %source_hash); $unicode_root //= select_unicode_root( repo_root => $root, - version => $manifest->{unicode_version}, + version => $current_checkout ? 'current' : $manifest->{unicode_version}, required => \@required, ); +my $actual_unicode_version; +if ($current_checkout) { + my $version_text = read_raw(File::Spec->catfile($unicode_root, 'version')); + $actual_unicode_version = $version_text; + $actual_unicode_version =~ s/\s+\z//; + die "Malformed current Unicode version '$actual_unicode_version'\n" + unless $actual_unicode_version =~ /\A\d+\.\d+\.\d+\z/; + for my $relative (sort keys %source_hash) { + my $path = File::Spec->catfile($unicode_root, split m{/}, $relative); + my $text = read_raw($path); + verify_unicode_notice($path, $text); + } +} if ($refresh) { - $manifest->{version_sha256} = sha256_hex(read_raw(File::Spec->catfile($unicode_root, 'version'))); + my $version_text = read_raw(File::Spec->catfile($unicode_root, 'version')); + $manifest->{unicode_version} = $actual_unicode_version; + $manifest->{version_sha256} = sha256_hex($version_text); for my $entry (@entries) { for my $field ('sources', 'perl_sources') { my $base = $field eq 'sources' ? $unicode_root : ($perl_root // select_perl_root(repo_root => $root, unicode_root => $unicode_root, required => [keys %{$entry->{$field} // {}}])); @@ -131,14 +147,16 @@ $perl_source_hash{$_} = $entry->{perl_sources}{$_} for keys %{$entry->{perl_sources} // {}}; } } -read_unicode_version( - path => File::Spec->catfile($unicode_root, 'version'), - expected => $manifest->{unicode_version}, - sha256 => $manifest->{version_sha256}, -); -for my $relative (sort keys %source_hash) { - read_pinned_source(path => File::Spec->catfile($unicode_root, split m{/}, $relative), - sha256 => $source_hash{$relative}); +if (!$current_checkout) { + read_unicode_version( + path => File::Spec->catfile($unicode_root, 'version'), + expected => $manifest->{unicode_version}, + sha256 => $manifest->{version_sha256}, + ); + for my $relative (sort keys %source_hash) { + read_pinned_source(path => File::Spec->catfile($unicode_root, split m{/}, $relative), + sha256 => $source_hash{$relative}); + } } if (%perl_source_hash) { $perl_root //= select_perl_root( @@ -146,22 +164,22 @@ unicode_root => $unicode_root, required => [sort keys %perl_source_hash], ); - for my $relative (sort keys %perl_source_hash) { - read_pinned_source(path => File::Spec->catfile($perl_root, split m{/}, $relative), - sha256 => $perl_source_hash{$relative}); + if (!$current_checkout) { + for my $relative (sort keys %perl_source_hash) { + read_pinned_source(path => File::Spec->catfile($perl_root, split m{/}, $relative), + sha256 => $perl_source_hash{$relative}); + } } } if ($current_checkout) { + $perl_root //= select_perl_root( + repo_root => $root, unicode_root => $unicode_root, + required => ['patchlevel.h']); my $actual_commit = checkout_identity($perl_root, 'rev-parse', 'HEAD'); my $actual_version = perl_language_version($perl_root); if ($refresh) { $manifest->{perl_commit} = $actual_commit; $manifest->{perl_version} = $actual_version; - } else { - die "Perl checkout revision mismatch: expected $manifest->{perl_commit}, found $actual_commit\n" - unless ($manifest->{perl_commit} // '') eq $actual_commit; - die "Perl checkout version mismatch: expected $manifest->{perl_version}, found $actual_version\n" - unless ($manifest->{perl_version} // '') eq $actual_version; } } diff --git a/dev/tools/generate_perl_unicode_indic_category_data.pl b/dev/tools/generate_perl_unicode_indic_category_data.pl index e20490dc5..0f6288fa5 100644 --- a/dev/tools/generate_perl_unicode_indic_category_data.pl +++ b/dev/tools/generate_perl_unicode_indic_category_data.pl @@ -1,68 +1,62 @@ #!/usr/bin/env perl use strict; use warnings; +use Digest::SHA qw(sha256_hex); use File::Spec; use FindBin; use lib File::Spec->catdir($FindBin::Bin, 'lib'); use PerlOnJava::UnicodeGenerator qw( emit_java_range_triples emit_unicode_source_notices loose_name parse_range - read_pinned_source read_unicode_version repo_root select_unicode_root trim + read_raw repo_root select_unicode_root trim verify_unicode_notice ); -my $expected_version = '17.0.0'; my $root = repo_root($FindBin::Bin); my $unicore = select_unicode_root( repo_root => $root, - version => $expected_version, + version => 'current', required => [qw(version IndicSyllabicCategory.txt IndicPositionalCategory.txt PropValueAliases.txt PropertyAliases.txt)], ); +my $unicode_version = read_raw(File::Spec->catfile($unicore, 'version')); +$unicode_version =~ s/\s+\z//; +die "Malformed current Unicode version '$unicode_version'\n" + unless $unicode_version =~ /\A\d+\.\d+\.\d+\z/; my @sources = ( { - name => 'IndicSyllabicCategory-17.0.0.txt', + name => "IndicSyllabicCategory-$unicode_version.txt", path => File::Spec->catfile($unicore, 'IndicSyllabicCategory.txt'), - hash => '3fc122f4cf58b0c19268d5f810263b04ab4e1e67743386ec0e0ada9c76aec5be', - version => qr/^# IndicSyllabicCategory-\Q$expected_version\E\.txt$/m, + version => qr/^# IndicSyllabicCategory-\Q$unicode_version\E\.txt$/m, }, { - name => 'IndicPositionalCategory-17.0.0.txt', + name => "IndicPositionalCategory-$unicode_version.txt", path => File::Spec->catfile($unicore, 'IndicPositionalCategory.txt'), - hash => '68cedc29a7e57f984d90fe2c7712f2e6d0c717e253db219607daea8997d6c480', - version => qr/^# IndicPositionalCategory-\Q$expected_version\E\.txt$/m, + version => qr/^# IndicPositionalCategory-\Q$unicode_version\E\.txt$/m, }, { - name => 'PropertyValueAliases-17.0.0.txt', + name => "PropertyValueAliases-$unicode_version.txt", path => File::Spec->catfile($unicore, 'PropValueAliases.txt'), - hash => '670d2bebb48649c04fabfbf033308073dcff47946324a8033237254c048b3b01', - version => qr/^# PropertyValueAliases-\Q$expected_version\E\.txt$/m, + version => qr/^# PropertyValueAliases-\Q$unicode_version\E\.txt$/m, }, { - name => 'PropertyAliases-17.0.0.txt', + name => "PropertyAliases-$unicode_version.txt", path => File::Spec->catfile($unicore, 'PropertyAliases.txt'), - hash => '4441f573caf952ffece1d7c892e7715bd7136dfc26f96eb6f268bf1e474715fb', - version => qr/^# PropertyAliases-\Q$expected_version\E\.txt$/m, + version => qr/^# PropertyAliases-\Q$unicode_version\E\.txt$/m, }, ); for my $source (@sources) { - $source->{text} = read_pinned_source( - path => $source->{path}, - sha256 => $source->{hash}, - version_pattern => $source->{version}, - unicode_version => $expected_version, - ); + $source->{text} = read_raw($source->{path}); + $source->{hash} = sha256_hex($source->{text}); + die "$source->{path} is inconsistent with Unicode $unicode_version\n" + unless $source->{text} =~ $source->{version}; + verify_unicode_notice($source->{path}, $source->{text}); } -my $unicode_version = read_unicode_version( - path => File::Spec->catfile($unicore, 'version'), expected => $expected_version); - my %property = ( InSC => { java => 'INSC', long => 'Indic_Syllabic_Category', source => $sources[0], - expected_values => 37, - expected_ranges => 967, default => 'Other', short_values => [], long_values => [], @@ -74,8 +68,6 @@ java => 'INPC', long => 'Indic_Positional_Category', source => $sources[1], - expected_values => 16, - expected_ranges => 653, default => 'Not_Applicable', short_values => [], long_values => [], @@ -119,9 +111,7 @@ for my $short (qw(InSC InPC)) { my $spec = $property{$short}; - die "$short expected $spec->{expected_values} values, found " - . scalar(@{$spec->{short_values}}) . "\n" - unless @{$spec->{short_values}} == $spec->{expected_values}; + die "$short defines no values\n" unless @{$spec->{short_values}}; die "$short property aliases missing\n" unless $spec->{property_aliases}{loose_name($short)} && $spec->{property_aliases}{loose_name($spec->{long})}; @@ -149,9 +139,7 @@ . (defined $declared_default ? $declared_default : '') . "\n" unless defined $declared_default && loose_name($declared_default) eq loose_name($spec->{default}); - die "$short expected $spec->{expected_ranges} ranges, found " - . scalar(@{$spec->{ranges}}) . "\n" - unless @{$spec->{ranges}} == $spec->{expected_ranges}; + die "$short defines no explicit ranges\n" unless @{$spec->{ranges}}; } sub emit_string_array { @@ -187,7 +175,7 @@ sub emit_property { print <<'HEADER'; /* - * Generated from Perl 5.44's current Unicode Character Database. Do not edit manually. + * Generated from the current Perl checkout's Unicode Character Database. Do not edit manually. * HEADER emit_unicode_source_notices(\@sources); diff --git a/dev/tools/generate_perl_unicode_residual_property_data.pl b/dev/tools/generate_perl_unicode_residual_property_data.pl new file mode 100644 index 000000000..f79b2d88d --- /dev/null +++ b/dev/tools/generate_perl_unicode_residual_property_data.pl @@ -0,0 +1,283 @@ +#!/usr/bin/env perl +use strict; +use warnings; +use Digest::SHA qw(sha256_hex); +use File::Spec; +use FindBin; +use lib File::Spec->catdir($FindBin::Bin, 'lib'); +use PerlOnJava::UnicodeGenerator qw( + emit_unicode_source_notices loose_name parse_range read_raw repo_root + select_unicode_root trim verify_unicode_notice +); + +my $root = repo_root($FindBin::Bin); +my $unicore = select_unicode_root( + repo_root => $root, version => 'current', + required => [qw(version IdType.txt DCoreProperties.txt PropList.txt + PropValueAliases.txt PropertyAliases.txt Unikemet.txt + auxiliary/GraphemeBreakProperty.txt)], +); +my $unicode_version = read_raw(File::Spec->catfile($unicore, 'version')); +$unicode_version =~ s/\s+\z//; +die "Malformed current Unicode version '$unicode_version'\n" + unless $unicode_version =~ /\A\d+\.\d+\.\d+\z/; +my @sources = ( + ['IdType.txt', qr/^# Version:\s*\Q$unicode_version\E$/m], + ['DCoreProperties.txt', qr/^# DerivedCoreProperties-\Q$unicode_version\E\.txt$/m], + ['PropList.txt', qr/^# PropList-\Q$unicode_version\E\.txt$/m], + ['PropValueAliases.txt', qr/^# PropertyValueAliases-\Q$unicode_version\E\.txt$/m], + ['PropertyAliases.txt', qr/^# PropertyAliases-\Q$unicode_version\E\.txt$/m], + ['Unikemet.txt', qr/^# Unikemet-\Q$unicode_version\E\.txt$/m], + ['auxiliary/GraphemeBreakProperty.txt', qr/^# GraphemeBreakProperty-\Q$unicode_version\E\.txt$/m], +); +for my $source (@sources) { + my ($name, $version_pattern) = @$source; + my $path = File::Spec->catfile($unicore, split m{/}, $name); + my $text = read_raw($path); + die "$path is inconsistent with Unicode $unicode_version\n" + unless $text =~ $version_pattern; + verify_unicode_notice($path, $text); + @$source = ($name, sha256_hex($text), $text); +} +my %text = map { $_->[0] => $_->[2] } @sources; + +my %property = ( + GCB => { aliases => [qw(GCB Grapheme_Cluster_Break)], default => 'Other' }, + InCB => { aliases => [qw(InCB Indic_Conjunct_Break)], default => 'None' }, + IDTYPE => { + aliases => [qw(ID_Type Identifier_Type)], default => 'Not_Character', + values => [qw(Not_Character Deprecated Default_Ignorable Not_NFKC Not_XID + Exclusion Obsolete Technical Uncommon_Use Limited_Use Inclusion Recommended)], + }, + KEHCORE => { aliases => ['kEH_Core'], default => 'N', values => [qw(C L N)] }, +); + +for my $line (split /\n/, $text{'PropValueAliases.txt'}) { + next if $line =~ /^\s*#/; + $line =~ s/#.*$//; + my @field = map { trim($_) } split /;/, $line; + next unless @field >= 3 && ($field[0] eq 'GCB' || $field[0] eq 'InCB'); + my $spec = $property{$field[0]}; + push @{$spec->{values}}, $field[2]; + push @{$spec->{short_values}}, $field[1]; + my $index = $#{$spec->{values}}; + $spec->{value_alias}{loose_name($_)} = $index for grep { length } @field[1 .. $#field]; +} +for my $key (qw(IDTYPE KEHCORE)) { + my $spec = $property{$key}; + $spec->{short_values} = [@{$spec->{values}}]; + for my $index (0 .. $#{$spec->{values}}) { + $spec->{value_alias}{loose_name($spec->{values}[$index])} = $index; + } +} +for my $spec (values %property) { + $spec->{property_alias}{loose_name($_)} = 1 for @{$spec->{aliases}}; + my ($default) = grep { + loose_name($spec->{values}[$_]) eq loose_name($spec->{default}) + } 0 .. $#{$spec->{values}}; + die "Missing default $spec->{default}\n" unless defined $default; + $spec->{default_index} = $default; + $spec->{ranges} = [map { [] } @{$spec->{values}}]; +} + +sub add_range { + my ($key, $range, $value) = @_; + my $spec = $property{$key}; + my $index = $spec->{value_alias}{loose_name($value)}; + die "$key unknown value '$value'\n" unless defined $index; + push @{$spec->{ranges}[$index]}, [parse_range($range)]; +} + +for my $line (split /\n/, $text{'auxiliary/GraphemeBreakProperty.txt'}) { + add_range('GCB', $1, $2) if $line =~ /^([0-9A-F]+(?:\.\.[0-9A-F]+)?)\s*;\s*([A-Za-z_]+)/; +} +for my $line (split /\n/, $text{'DCoreProperties.txt'}) { + add_range('InCB', $1, $2) if $line =~ /^([0-9A-F]+(?:\.\.[0-9A-F]+)?)\s*;\s*InCB\s*;\s*([A-Za-z_]+)/; +} +for my $line (split /\n/, $text{'IdType.txt'}) { + next unless $line =~ /^([0-9A-F]+(?:\.\.[0-9A-F]+)?)\s*;\s*([^#]+)/; + my ($range, $values) = ($1, trim($2)); + add_range('IDTYPE', $range, $_) for split /\s+/, $values; +} +for my $line (split /\n/, $text{'Unikemet.txt'}) { + add_range('KEHCORE', $1, $2) if $line =~ /^U\+([0-9A-F]+)\tkEH_Core\t([CL])$/; +} +my @hex_ranges; +for my $line (split /\n/, $text{'PropList.txt'}) { + push @hex_ranges, [parse_range($1)] + if $line =~ /^([0-9A-F]+(?:\.\.[0-9A-F]+)?)\s*;\s*Hex_Digit\b/; +} + +sub coalesce { + my ($ranges) = @_; + my @out; + for my $range (sort { $a->[0] <=> $b->[0] || $a->[1] <=> $b->[1] } @$ranges) { + if (@out && $range->[0] <= $out[-1][1] + 1) { + $out[-1][1] = $range->[1] if $range->[1] > $out[-1][1]; + } else { + push @out, [@$range]; + } + } + return \@out; +} +sub complement { + my ($ranges) = @_; + my $merged = coalesce($ranges); + my @out; + my $next = 0; + for my $range (@$merged) { + push @out, [$next, $range->[0] - 1] if $next < $range->[0]; + $next = $range->[1] + 1 if $next <= $range->[1]; + } + push @out, [$next, 0x10FFFF] if $next <= 0x10FFFF; + return \@out; +} +for my $spec (values %property) { + my @explicit = map { @$_ } @{$spec->{ranges}}; + $spec->{ranges}[$spec->{default_index}] = complement(\@explicit); + $spec->{ranges}[$_] = coalesce($spec->{ranges}[$_]) + for 0 .. $#{$spec->{ranges}}; +} + +sub emit_strings { + my ($name, $values) = @_; + print " private static final String[] $name = {\n ", + join(', ', map { qq{"$_"} } @$values), "\n };\n"; +} +sub emit_pairs { + my ($ranges, $indent) = @_; + $indent //= ' '; + for (my $i = 0; $i < @$ranges; $i += 4) { + my $end = $i + 3 < $#$ranges ? $i + 3 : $#$ranges; + print $indent, join(', ', map { sprintf '0x%X, 0x%X', @{$ranges->[$_]} } $i .. $end), ",\n"; + } +} +sub emit_property { + my ($key) = @_; + my $spec = $property{$key}; + my @value_alias = sort keys %{$spec->{value_alias}}; + my @property_alias = sort keys %{$spec->{property_alias}}; + emit_strings("${key}_VALUES", $spec->{values}); + emit_strings("${key}_SHORT_VALUES", $spec->{short_values}); + emit_strings("${key}_VALUE_ALIASES", \@value_alias); + print " private static final byte[] ${key}_VALUE_ALIAS_INDEX = {\n ", + join(', ', map { $spec->{value_alias}{$_} } @value_alias), "\n };\n"; + emit_strings("${key}_PROPERTY_ALIASES", \@property_alias); + for my $index (0 .. $#{$spec->{ranges}}) { + my $ranges = $spec->{ranges}[$index]; + print " private static int[] ${key}_ranges_$index() {\n", + " return new int[] {\n"; + emit_pairs($ranges); + print " };\n }\n"; + } + print " private static final int[][] ${key}_RANGES = {\n ", + join(', ', map { "${key}_ranges_$_()" } 0 .. $#{$spec->{ranges}}), + "\n };\n"; + print " private static final Property $key = new Property(\n", + " ${key}_VALUES, ${key}_SHORT_VALUES,\n", + " ${key}_VALUE_ALIASES, ${key}_VALUE_ALIAS_INDEX,\n", + " ${key}_PROPERTY_ALIASES, ${key}_RANGES);\n\n"; +} + +print <<'HEADER'; +/* + * Generated from the current Perl checkout's Unicode Character Database. Do not edit manually. + * +HEADER +emit_unicode_source_notices([map { { name => $_->[0], text => $_->[2] } } @sources]); +print <<'JAVA'; + */ +package org.perlonjava.runtime.regex; + +import com.ibm.icu.text.UnicodeSet; + +final class PerlUnicodeResidualPropertyData { +JAVA +print " static final String UNICODE_VERSION = \"$unicode_version\";\n"; +for my $source (@sources) { + my $constant = uc $source->[0]; $constant =~ s/[^A-Z0-9]+/_/g; + print " static final String ${constant}_SHA256 = \"$source->[1]\";\n"; +} +print "\n"; +emit_property($_) for qw(GCB InCB IDTYPE KEHCORE); +print " private static final int[] HEX_RANGES = {\n"; emit_pairs(\@hex_ranges, ' '); print " };\n"; +print <<'JAVA'; + private static final UnicodeSet HEX = buildSet(HEX_RANGES); + + static boolean isPropertyAlias(String alias) { return property(alias) != null; } + static boolean isBinaryPropertyAlias(String alias) { + String loose = loose(alias); + return loose.equals("hex") || loose.equals("hexdigit"); + } + static UnicodeSet binarySet(String alias) { + return isBinaryPropertyAlias(alias) ? HEX : null; + } + static UnicodeSet valueSet(String propertyAlias, String valueAlias) { + Property property = property(propertyAlias); + return property == null ? null : property.valueSet(valueAlias); + } + static int valueCount(String propertyAlias) { + Property property = property(propertyAlias); + return property == null ? 0 : property.values.length; + } + static String value(String propertyAlias, int index) { + Property property = property(propertyAlias); + return property == null ? null : property.values[index]; + } + static String shortValue(String propertyAlias, int index) { + Property property = property(propertyAlias); + return property == null ? null : property.shortValues[index]; + } + private static Property property(String alias) { + String loose = loose(alias); + if (GCB.hasAlias(loose)) return GCB; + if (InCB.hasAlias(loose)) return InCB; + if (IDTYPE.hasAlias(loose)) return IDTYPE; + if (KEHCORE.hasAlias(loose)) return KEHCORE; + return null; + } + private static String loose(String value) { + return value == null ? "" : value.toLowerCase(java.util.Locale.ROOT) + .replaceAll("[\\t-\\r _-]+", ""); + } + private static UnicodeSet buildSet(int[] ranges) { + UnicodeSet set = new UnicodeSet(); + for (int index = 0; index < ranges.length; index += 2) { + set.add(ranges[index], ranges[index + 1]); + } + return set.freeze(); + } + private static final class Property { + private final String[] values; + private final String[] shortValues; + private final String[] valueAliases; + private final byte[] valueAliasIndex; + private final String[] propertyAliases; + private final int[][] ranges; + private final UnicodeSet[] sets; + private Property(String[] values, String[] shortValues, + String[] valueAliases, byte[] valueAliasIndex, + String[] propertyAliases, int[][] ranges) { + this.values = values; + this.shortValues = shortValues; + this.valueAliases = valueAliases; + this.valueAliasIndex = valueAliasIndex; + this.propertyAliases = propertyAliases; + this.ranges = ranges; + this.sets = new UnicodeSet[values.length]; + } + private boolean hasAlias(String alias) { + return java.util.Arrays.binarySearch(propertyAliases, alias) >= 0; + } + private UnicodeSet valueSet(String alias) { + int found = java.util.Arrays.binarySearch(valueAliases, loose(alias)); + if (found < 0) return null; + int index = valueAliasIndex[found]; + UnicodeSet result = sets[index]; + if (result == null) sets[index] = result = buildSet(ranges[index]); + return result; + } + } + private PerlUnicodeResidualPropertyData() {} +} +JAVA diff --git a/dev/tools/generate_perl_unicode_script_data.pl b/dev/tools/generate_perl_unicode_script_data.pl index 37040c18b..cfcb530a2 100644 --- a/dev/tools/generate_perl_unicode_script_data.pl +++ b/dev/tools/generate_perl_unicode_script_data.pl @@ -1,60 +1,59 @@ #!/usr/bin/env perl use strict; use warnings; +use Digest::SHA qw(sha256_hex); use File::Spec; use FindBin; use lib File::Spec->catdir($FindBin::Bin, 'lib'); use PerlOnJava::UnicodeGenerator qw( emit_java_range_triples emit_unicode_source_notices loose_name parse_range - read_pinned_source read_unicode_version repo_root select_unicode_root trim + read_raw repo_root select_unicode_root trim verify_unicode_notice ); -my $expected_version = '17.0.0'; my $root = repo_root($FindBin::Bin); my $unicore = select_unicode_root( repo_root => $root, - version => $expected_version, + version => 'current', required => [qw(version Scripts.txt ScriptExtensions.txt PropValueAliases.txt PropertyAliases.txt)], ); +my $unicode_version = read_raw(File::Spec->catfile($unicore, 'version')); +$unicode_version =~ s/\s+\z//; +die "Malformed current Unicode version '$unicode_version'\n" + unless $unicode_version =~ /\A\d+\.\d+\.\d+\z/; my @sources = ( { - name => 'Scripts-17.0.0.txt', + name => "Scripts-$unicode_version.txt", path => File::Spec->catfile($unicore, 'Scripts.txt'), - hash => '9f5e50d3abaee7d6ce09480f325c706f485ae3240912527e651954d2d6b035bf', - version => qr/^# Scripts-\Q$expected_version\E\.txt$/m, + version => qr/^# Scripts-\Q$unicode_version\E\.txt$/m, }, { - name => 'ScriptExtensions-17.0.0.txt', + name => "ScriptExtensions-$unicode_version.txt", path => File::Spec->catfile($unicore, 'ScriptExtensions.txt'), - hash => 'ec2107e58825a1586acee8e0911ce18260394ac8b87e535ca325f1ccbeb06bc6', - version => qr/^# ScriptExtensions-\Q$expected_version\E\.txt$/m, + version => qr/^# ScriptExtensions-\Q$unicode_version\E\.txt$/m, }, { - name => 'PropertyValueAliases-17.0.0.txt', + name => "PropertyValueAliases-$unicode_version.txt", path => File::Spec->catfile($unicore, 'PropValueAliases.txt'), - hash => '670d2bebb48649c04fabfbf033308073dcff47946324a8033237254c048b3b01', - version => qr/^# PropertyValueAliases-\Q$expected_version\E\.txt$/m, + version => qr/^# PropertyValueAliases-\Q$unicode_version\E\.txt$/m, }, { - name => 'PropertyAliases-17.0.0.txt', + name => "PropertyAliases-$unicode_version.txt", path => File::Spec->catfile($unicore, 'PropertyAliases.txt'), - hash => '4441f573caf952ffece1d7c892e7715bd7136dfc26f96eb6f268bf1e474715fb', - version => qr/^# PropertyAliases-\Q$expected_version\E\.txt$/m, + version => qr/^# PropertyAliases-\Q$unicode_version\E\.txt$/m, }, ); sub read_source { my ($source) = @_; - $source->{text} = read_pinned_source( - path => $source->{path}, sha256 => $source->{hash}, - version_pattern => $source->{version}, unicode_version => $expected_version); + $source->{text} = read_raw($source->{path}); + $source->{hash} = sha256_hex($source->{text}); + die "$source->{path} is inconsistent with Unicode $unicode_version\n" + unless $source->{text} =~ $source->{version}; + verify_unicode_notice($source->{path}, $source->{text}); } read_source($_) for @sources; -my $unicode_version = read_unicode_version( - path => File::Spec->catfile($unicore, 'version'), expected => $expected_version); - my (@short_values, @long_values, %alias_index, %wildcard_value_index); for my $line (split /\n/, $sources[2]{text}) { next if $line =~ /^\s*#/; @@ -76,8 +75,7 @@ sub read_source { $wildcard_value_index{$alias} = $index; } } -die "Expected 176 Script values, found " . scalar(@short_values) . "\n" - unless @short_values == 176; +die "Current Script aliases define no values\n" unless @short_values; my (%script_property_aliases, %script_extensions_property_aliases); for my $line (split /\n/, $sources[3]{text}) { @@ -93,9 +91,9 @@ sub read_source { for grep { length } @fields; } } -die "Pinned property aliases do not define sc and Script\n" +die "Current property aliases do not define sc and Script\n" unless $script_property_aliases{sc} && $script_property_aliases{script}; -die "Pinned property aliases do not define scx and Script_Extensions\n" +die "Current property aliases do not define scx and Script_Extensions\n" unless $script_extensions_property_aliases{scx} && $script_extensions_property_aliases{scriptextensions}; @@ -172,7 +170,7 @@ sub read_source { print <<'HEADER'; /* - * Generated from Perl 5.44's pinned Unicode Character Database. Do not edit manually. + * Generated from the current Perl checkout's Unicode Character Database. Do not edit manually. * HEADER emit_unicode_source_notices(\@sources); diff --git a/dev/tools/perl_unicode_data_generators.json b/dev/tools/perl_unicode_data_generators.json index b4bd11172..63a2350ca 100644 --- a/dev/tools/perl_unicode_data_generators.json +++ b/dev/tools/perl_unicode_data_generators.json @@ -85,7 +85,7 @@ "generator" : "dev/tools/generate_perl_unicode_indic_category_data.pl", "name" : "indic-category", "output" : "src/main/java/org/perlonjava/runtime/regex/PerlUnicodeIndicCategoryData.java", - "output_sha256" : "2362ba0552cdcc065d0fa5fe9266d557cebc41ed63aff2046b6b0595e3bdbe03", + "output_sha256" : "090dbaf4abf205f48483716e5cb7ca244b8ed93fcd2b965320814bdea5c4c1ae", "sources" : { "IndicPositionalCategory.txt" : "68cedc29a7e57f984d90fe2c7712f2e6d0c717e253db219607daea8997d6c480", "IndicSyllabicCategory.txt" : "3fc122f4cf58b0c19268d5f810263b04ab4e1e67743386ec0e0ada9c76aec5be" @@ -100,6 +100,19 @@ "extracted/DJoinGroup.txt" : "bb67e0c00b88acfa5be633967b66b23326844a86e49c6fde7b57960d3af66cae" } }, + { + "generator" : "dev/tools/generate_perl_unicode_residual_property_data.pl", + "name" : "residual-property", + "output" : "src/main/java/org/perlonjava/runtime/regex/PerlUnicodeResidualPropertyData.java", + "output_sha256" : "2cf43255dbb114a4f0a215b7a0fa3031d28f9a243e3d6e6b573c64c6b6e932d4", + "sources" : { + "DCoreProperties.txt" : "24c7fed1195c482faaefd5c1e7eb821c5ee1fb6de07ecdbaa64b56a99da22c08", + "IdType.txt" : "924ac63faa97ed73420d6ac48d08279d90968c7da0502ab701e08bfbb9683c22", + "PropList.txt" : "130dcddcaadaf071008bdfce1e7743e04fdfbc910886f017d9f9ac931d8c64dd", + "Unikemet.txt" : "76a3081265e6eb673873f9c93d6f36062e82c7ed027c5c1a592accfbe48c20a5", + "auxiliary/GraphemeBreakProperty.txt" : "d6b51d1d2ae5c33b451b7ed994b48f1f4dc62b2272a5831e7fd418514a6bae89" + } + }, { "generator" : "dev/tools/generate_perl_unicode_numeric_value_data.pl", "name" : "numeric-value", @@ -113,7 +126,7 @@ "generator" : "dev/tools/generate_perl_unicode_script_data.pl", "name" : "script", "output" : "src/main/java/org/perlonjava/runtime/regex/PerlUnicodeScriptData.java", - "output_sha256" : "e27079c728d4817a7edd5f3ec7f7c2a009d96909187a84b6a2f15c53933904f6", + "output_sha256" : "565badc0e91896dca9ca4a6d065343b9bab73e1004e913568671125778969e80", "sources" : { "ScriptExtensions.txt" : "ec2107e58825a1586acee8e0911ce18260394ac8b87e535ca325f1ccbeb06bc6", "Scripts.txt" : "9f5e50d3abaee7d6ce09480f325c706f485ae3240912527e651954d2d6b035bf" diff --git a/dev/tools/tests/perl_unicode_generator_pipeline.t b/dev/tools/tests/perl_unicode_generator_pipeline.t index b14d858f2..17a3c6890 100644 --- a/dev/tools/tests/perl_unicode_generator_pipeline.t +++ b/dev/tools/tests/perl_unicode_generator_pipeline.t @@ -20,8 +20,11 @@ my $output_dir = File::Spec->catdir($temporary, 'generated'); make_path($unicode, $perl_root, $tools, $output_dir); my $version = "17.0.0\n"; -my $source = "# test Unicode table\n0041 ; Example\n"; -my $generated = "# test Unicode table\n0041 ; Example\n# pinned Perl generator source\n"; +my $notice = "# © 2025 Unicode®, Inc.\n" + . "# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.\n" + . "# For terms of use and license, see https://www.unicode.org/terms_of_use.html\n"; +my $source = "# test Unicode table\n" . $notice . "0041 ; Example\n"; +my $generated = $source . "# pinned Perl generator source\n"; write_file(File::Spec->catfile($unicode, 'version'), $version); write_file(File::Spec->catfile($unicode, 'table.txt'), $source); my $perl_source = "# pinned Perl generator source\n"; @@ -77,16 +80,32 @@ my $current_manifest = File::Spec->catfile($temporary, 'current-manifest.json'); my $current_value = decode_json(read_file($manifest)); $current_value->{schema_version} = 2; $current_value->{perl_source_policy} = 'current-checkout'; -delete $current_value->{perl_commit}; +$current_value->{perl_commit} = 'stale-current-provenance'; write_manifest($current_manifest, $current_value); -write_file(File::Spec->catfile($unicode, 'table.txt'), "moving current source\n"); +my $moving_source = "# moving current source\n" . $notice; +write_file(File::Spec->catfile($unicode, 'table.txt'), $moving_source); +write_file(File::Spec->catfile($unicode, 'version'), "18.0.0\n"); ($status, $log) = run_pipeline_with_manifest($current_manifest, '--check'); -is($status, 255, 'current-checkout check rejects stale source provenance'); -like($log, qr/table\.txt SHA-256 mismatch/, 'stale current source is named'); +is($status, 255, 'current-checkout check reaches generated-output comparison'); +like($log, qr/generated SHA-256 mismatch/, + 'stale provenance does not mask the reproducible output mismatch'); +write_file(File::Spec->catfile($unicode, 'table.txt'), "missing notice\n"); +($status, $log) = run_pipeline_with_manifest($current_manifest, '--check'); +is($status, 255, 'current-checkout check rejects a missing Unicode notice'); +like($log, qr/does not preserve the Unicode copyright notice/, + 'missing current-source notice is diagnosed'); +write_file(File::Spec->catfile($unicode, 'table.txt'), $moving_source); ($status, $log) = run_pipeline_with_manifest($current_manifest, '--refresh'); -is($status, 0, 'refresh updates source provenance and dependent output'); +is($status, 0, 'refresh accepts a valid newer current Unicode version'); +is(decode_json(read_file($current_manifest))->{unicode_version}, '18.0.0', + 'refresh records the newer Unicode version as provenance'); ($status, $log) = run_pipeline_with_manifest($current_manifest, '--check'); -is($status, 0, 'refreshed current checkout is checkable'); +is($status, 0, 'newer refreshed current checkout is deterministic and checkable'); +write_file(File::Spec->catfile($unicode, 'version'), "not-a-version\n"); +($status, $log) = run_pipeline_with_manifest($current_manifest, '--refresh'); +is($status, 255, 'refresh rejects a malformed current Unicode version'); +like($log, qr/Malformed current Unicode version/, 'malformed version is diagnosed'); +write_file(File::Spec->catfile($unicode, 'version'), $version); write_file(File::Spec->catfile($unicode, 'table.txt'), $source); write_file($output, "stale\n"); diff --git a/dev/tools/tests/perl_unicode_latest_source_generator.t b/dev/tools/tests/perl_unicode_latest_source_generator.t new file mode 100644 index 000000000..a69edcc53 --- /dev/null +++ b/dev/tools/tests/perl_unicode_latest_source_generator.t @@ -0,0 +1,91 @@ +use strict; +use warnings; +use File::Path qw(make_path); +use File::Spec; +use File::Temp qw(tempdir); +use FindBin; +use IPC::Open3; +use Symbol qw(gensym); +use Test::More; + +my $repo = File::Spec->rel2abs(File::Spec->catdir($FindBin::Bin, '..', '..', '..')); +my $generator = File::Spec->catfile( + $repo, 'dev', 'tools', 'generate_perl_unicode_indic_category_data.pl'); +my $root = tempdir(CLEANUP => 1); +make_path($root); + +write_sources('18.0.0'); +my ($status, $first) = run_generator(); +is($status, 0, 'generator accepts a valid newer current Unicode source tree'); +like($first, qr/UNICODE_VERSION = "18\.0\.0"/, + 'generated provenance derives the newer Unicode version'); +my ($second_status, $second) = run_generator(); +is($second_status, 0, 'second newer-source generation succeeds'); +is($second, $first, 'newer current-source generation is deterministic'); + +my $syllabic = File::Spec->catfile($root, 'IndicSyllabicCategory.txt'); +my $valid = read_file($syllabic); +(my $missing_notice = $valid) =~ s/^# © 2025 Unicode®, Inc\.\n//m; +write_file($syllabic, $missing_notice); +my ($notice_status, $notice_log) = run_generator(); +isnt($notice_status, 0, 'generator rejects a current source missing its notice'); +like($notice_log, qr/does not preserve the Unicode copyright notice/, + 'missing notice has a focused diagnostic'); + +write_file($syllabic, $valid =~ s/; Other/; Unknown/r); +my ($malformed_status, $malformed_log) = run_generator(); +isnt($malformed_status, 0, 'generator rejects malformed current property data'); +like($malformed_log, qr/(?:unknown explicit value 'Unknown'|expected default Other, found Unknown)/, + 'malformed property value is named'); + +done_testing; + +sub notice { + return "# © 2025 Unicode®, Inc.\n" + . "# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.\n" + . "# For terms of use and license, see https://www.unicode.org/terms_of_use.html\n"; +} + +sub write_sources { + my ($version) = @_; + write_file(File::Spec->catfile($root, 'version'), "$version\n"); + write_file(File::Spec->catfile($root, 'IndicSyllabicCategory.txt'), + "# IndicSyllabicCategory-$version.txt\n" . notice() + . "# \@missing: 0000..10FFFF; Other\n0041 ; Other\n"); + write_file(File::Spec->catfile($root, 'IndicPositionalCategory.txt'), + "# IndicPositionalCategory-$version.txt\n" . notice() + . "# \@missing: 0000..10FFFF; Not_Applicable\n0042 ; Not_Applicable\n"); + write_file(File::Spec->catfile($root, 'PropValueAliases.txt'), + "# PropertyValueAliases-$version.txt\n" . notice() + . "InSC ; Other ; Other\nInPC ; NA ; Not_Applicable\n"); + write_file(File::Spec->catfile($root, 'PropertyAliases.txt'), + "# PropertyAliases-$version.txt\n" . notice() + . "InSC ; Indic_Syllabic_Category\nInPC ; Indic_Positional_Category\n"); +} + +sub run_generator { + local %ENV = %ENV; + $ENV{PERLONJAVA_UNICODE_ROOT} = $root; + my $error = gensym; + my $pid = open3(undef, my $stdout, $error, $^X, $generator); + local $/; + my $bytes = (<$stdout> // '') . (<$error> // ''); + waitpid($pid, 0); + return ($? >> 8, $bytes); +} + +sub write_file { + my ($path, $bytes) = @_; + open my $output, '>:raw', $path or die "Cannot write $path: $!"; + print {$output} $bytes or die "Cannot write $path: $!"; + close $output or die "Cannot close $path: $!"; +} + +sub read_file { + my ($path) = @_; + open my $input, '<:raw', $path or die "Cannot read $path: $!"; + local $/; + my $bytes = <$input>; + close $input or die "Cannot close $path: $!"; + return $bytes; +} diff --git a/src/main/java/org/perlonjava/runtime/regex/PerlUnicodeIndicCategoryData.java b/src/main/java/org/perlonjava/runtime/regex/PerlUnicodeIndicCategoryData.java index 74ba00c34..5c3a14834 100644 --- a/src/main/java/org/perlonjava/runtime/regex/PerlUnicodeIndicCategoryData.java +++ b/src/main/java/org/perlonjava/runtime/regex/PerlUnicodeIndicCategoryData.java @@ -1,5 +1,5 @@ /* - * Generated from Perl 5.44's current Unicode Character Database. Do not edit manually. + * Generated from the current Perl checkout's Unicode Character Database. Do not edit manually. * * Source: IndicSyllabicCategory-17.0.0.txt * © 2025 Unicode®, Inc. diff --git a/src/main/java/org/perlonjava/runtime/regex/PerlUnicodeResidualPropertyData.java b/src/main/java/org/perlonjava/runtime/regex/PerlUnicodeResidualPropertyData.java new file mode 100644 index 000000000..1a19669ec --- /dev/null +++ b/src/main/java/org/perlonjava/runtime/regex/PerlUnicodeResidualPropertyData.java @@ -0,0 +1,2618 @@ +/* + * Generated from the current Perl checkout's Unicode Character Database. Do not edit manually. + * + * Source: IdType.txt + * © 2025 Unicode®, Inc. + * Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. + * For terms of use and license, see https://www.unicode.org/terms_of_use.html + * + * Source: DCoreProperties.txt + * © 2025 Unicode®, Inc. + * Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. + * For terms of use and license, see https://www.unicode.org/terms_of_use.html + * + * Source: PropList.txt + * © 2025 Unicode®, Inc. + * Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. + * For terms of use and license, see https://www.unicode.org/terms_of_use.html + * + * Source: PropValueAliases.txt + * © 2025 Unicode®, Inc. + * Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. + * For terms of use and license, see https://www.unicode.org/terms_of_use.html + * + * Source: PropertyAliases.txt + * © 2025 Unicode®, Inc. + * Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. + * For terms of use and license, see https://www.unicode.org/terms_of_use.html + * + * Source: Unikemet.txt + * © 2025 Unicode®, Inc. + * Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. + * For terms of use and license, see https://www.unicode.org/terms_of_use.html + * + * Source: auxiliary/GraphemeBreakProperty.txt + * © 2025 Unicode®, Inc. + * Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. + * For terms of use and license, see https://www.unicode.org/terms_of_use.html + * + */ +package org.perlonjava.runtime.regex; + +import com.ibm.icu.text.UnicodeSet; + +final class PerlUnicodeResidualPropertyData { + static final String UNICODE_VERSION = "17.0.0"; + static final String IDTYPE_TXT_SHA256 = "924ac63faa97ed73420d6ac48d08279d90968c7da0502ab701e08bfbb9683c22"; + static final String DCOREPROPERTIES_TXT_SHA256 = "24c7fed1195c482faaefd5c1e7eb821c5ee1fb6de07ecdbaa64b56a99da22c08"; + static final String PROPLIST_TXT_SHA256 = "130dcddcaadaf071008bdfce1e7743e04fdfbc910886f017d9f9ac931d8c64dd"; + static final String PROPVALUEALIASES_TXT_SHA256 = "670d2bebb48649c04fabfbf033308073dcff47946324a8033237254c048b3b01"; + static final String PROPERTYALIASES_TXT_SHA256 = "4441f573caf952ffece1d7c892e7715bd7136dfc26f96eb6f268bf1e474715fb"; + static final String UNIKEMET_TXT_SHA256 = "76a3081265e6eb673873f9c93d6f36062e82c7ed027c5c1a592accfbe48c20a5"; + static final String AUXILIARY_GRAPHEMEBREAKPROPERTY_TXT_SHA256 = "d6b51d1d2ae5c33b451b7ed994b48f1f4dc62b2272a5831e7fd418514a6bae89"; + + private static final String[] GCB_VALUES = { + "Control", "CR", "E_Base", "E_Base_GAZ", "E_Modifier", "Extend", "Glue_After_Zwj", "L", "LF", "LV", "LVT", "Prepend", "Regional_Indicator", "SpacingMark", "T", "V", "Other", "ZWJ" + }; + private static final String[] GCB_SHORT_VALUES = { + "CN", "CR", "EB", "EBG", "EM", "EX", "GAZ", "L", "LF", "LV", "LVT", "PP", "RI", "SM", "T", "V", "XX", "ZWJ" + }; + private static final String[] GCB_VALUE_ALIASES = { + "cn", "control", "cr", "eb", "ebase", "ebasegaz", "ebg", "em", "emodifier", "ex", "extend", "gaz", "glueafterzwj", "l", "lf", "lv", "lvt", "other", "pp", "prepend", "regionalindicator", "ri", "sm", "spacingmark", "t", "v", "xx", "zwj" + }; + private static final byte[] GCB_VALUE_ALIAS_INDEX = { + 0, 0, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 8, 9, 10, 16, 11, 11, 12, 12, 13, 13, 14, 15, 16, 17 + }; + private static final String[] GCB_PROPERTY_ALIASES = { + "gcb", "graphemeclusterbreak" + }; + private static int[] GCB_ranges_0() { + return new int[] { + 0x0, 0x9, 0xB, 0xC, 0xE, 0x1F, 0x7F, 0x9F, + 0xAD, 0xAD, 0x61C, 0x61C, 0x180E, 0x180E, 0x200B, 0x200B, + 0x200E, 0x200F, 0x2028, 0x202E, 0x2060, 0x206F, 0xFEFF, 0xFEFF, + 0xFFF0, 0xFFFB, 0x13430, 0x1343F, 0x1BCA0, 0x1BCA3, 0x1D173, 0x1D17A, + 0xE0000, 0xE001F, 0xE0080, 0xE00FF, 0xE01F0, 0xE0FFF, + }; + } + private static int[] GCB_ranges_1() { + return new int[] { + 0xD, 0xD, + }; + } + private static int[] GCB_ranges_2() { + return new int[] { + }; + } + private static int[] GCB_ranges_3() { + return new int[] { + }; + } + private static int[] GCB_ranges_4() { + return new int[] { + }; + } + private static int[] GCB_ranges_5() { + return new int[] { + 0x300, 0x36F, 0x483, 0x489, 0x591, 0x5BD, 0x5BF, 0x5BF, + 0x5C1, 0x5C2, 0x5C4, 0x5C5, 0x5C7, 0x5C7, 0x610, 0x61A, + 0x64B, 0x65F, 0x670, 0x670, 0x6D6, 0x6DC, 0x6DF, 0x6E4, + 0x6E7, 0x6E8, 0x6EA, 0x6ED, 0x711, 0x711, 0x730, 0x74A, + 0x7A6, 0x7B0, 0x7EB, 0x7F3, 0x7FD, 0x7FD, 0x816, 0x819, + 0x81B, 0x823, 0x825, 0x827, 0x829, 0x82D, 0x859, 0x85B, + 0x897, 0x89F, 0x8CA, 0x8E1, 0x8E3, 0x902, 0x93A, 0x93A, + 0x93C, 0x93C, 0x941, 0x948, 0x94D, 0x94D, 0x951, 0x957, + 0x962, 0x963, 0x981, 0x981, 0x9BC, 0x9BC, 0x9BE, 0x9BE, + 0x9C1, 0x9C4, 0x9CD, 0x9CD, 0x9D7, 0x9D7, 0x9E2, 0x9E3, + 0x9FE, 0x9FE, 0xA01, 0xA02, 0xA3C, 0xA3C, 0xA41, 0xA42, + 0xA47, 0xA48, 0xA4B, 0xA4D, 0xA51, 0xA51, 0xA70, 0xA71, + 0xA75, 0xA75, 0xA81, 0xA82, 0xABC, 0xABC, 0xAC1, 0xAC5, + 0xAC7, 0xAC8, 0xACD, 0xACD, 0xAE2, 0xAE3, 0xAFA, 0xAFF, + 0xB01, 0xB01, 0xB3C, 0xB3C, 0xB3E, 0xB3F, 0xB41, 0xB44, + 0xB4D, 0xB4D, 0xB55, 0xB57, 0xB62, 0xB63, 0xB82, 0xB82, + 0xBBE, 0xBBE, 0xBC0, 0xBC0, 0xBCD, 0xBCD, 0xBD7, 0xBD7, + 0xC00, 0xC00, 0xC04, 0xC04, 0xC3C, 0xC3C, 0xC3E, 0xC40, + 0xC46, 0xC48, 0xC4A, 0xC4D, 0xC55, 0xC56, 0xC62, 0xC63, + 0xC81, 0xC81, 0xCBC, 0xCBC, 0xCBF, 0xCC0, 0xCC2, 0xCC2, + 0xCC6, 0xCC8, 0xCCA, 0xCCD, 0xCD5, 0xCD6, 0xCE2, 0xCE3, + 0xD00, 0xD01, 0xD3B, 0xD3C, 0xD3E, 0xD3E, 0xD41, 0xD44, + 0xD4D, 0xD4D, 0xD57, 0xD57, 0xD62, 0xD63, 0xD81, 0xD81, + 0xDCA, 0xDCA, 0xDCF, 0xDCF, 0xDD2, 0xDD4, 0xDD6, 0xDD6, + 0xDDF, 0xDDF, 0xE31, 0xE31, 0xE34, 0xE3A, 0xE47, 0xE4E, + 0xEB1, 0xEB1, 0xEB4, 0xEBC, 0xEC8, 0xECE, 0xF18, 0xF19, + 0xF35, 0xF35, 0xF37, 0xF37, 0xF39, 0xF39, 0xF71, 0xF7E, + 0xF80, 0xF84, 0xF86, 0xF87, 0xF8D, 0xF97, 0xF99, 0xFBC, + 0xFC6, 0xFC6, 0x102D, 0x1030, 0x1032, 0x1037, 0x1039, 0x103A, + 0x103D, 0x103E, 0x1058, 0x1059, 0x105E, 0x1060, 0x1071, 0x1074, + 0x1082, 0x1082, 0x1085, 0x1086, 0x108D, 0x108D, 0x109D, 0x109D, + 0x135D, 0x135F, 0x1712, 0x1715, 0x1732, 0x1734, 0x1752, 0x1753, + 0x1772, 0x1773, 0x17B4, 0x17B5, 0x17B7, 0x17BD, 0x17C6, 0x17C6, + 0x17C9, 0x17D3, 0x17DD, 0x17DD, 0x180B, 0x180D, 0x180F, 0x180F, + 0x1885, 0x1886, 0x18A9, 0x18A9, 0x1920, 0x1922, 0x1927, 0x1928, + 0x1932, 0x1932, 0x1939, 0x193B, 0x1A17, 0x1A18, 0x1A1B, 0x1A1B, + 0x1A56, 0x1A56, 0x1A58, 0x1A5E, 0x1A60, 0x1A60, 0x1A62, 0x1A62, + 0x1A65, 0x1A6C, 0x1A73, 0x1A7C, 0x1A7F, 0x1A7F, 0x1AB0, 0x1ADD, + 0x1AE0, 0x1AEB, 0x1B00, 0x1B03, 0x1B34, 0x1B3D, 0x1B42, 0x1B44, + 0x1B6B, 0x1B73, 0x1B80, 0x1B81, 0x1BA2, 0x1BA5, 0x1BA8, 0x1BAD, + 0x1BE6, 0x1BE6, 0x1BE8, 0x1BE9, 0x1BED, 0x1BED, 0x1BEF, 0x1BF3, + 0x1C2C, 0x1C33, 0x1C36, 0x1C37, 0x1CD0, 0x1CD2, 0x1CD4, 0x1CE0, + 0x1CE2, 0x1CE8, 0x1CED, 0x1CED, 0x1CF4, 0x1CF4, 0x1CF8, 0x1CF9, + 0x1DC0, 0x1DFF, 0x200C, 0x200C, 0x20D0, 0x20F0, 0x2CEF, 0x2CF1, + 0x2D7F, 0x2D7F, 0x2DE0, 0x2DFF, 0x302A, 0x302F, 0x3099, 0x309A, + 0xA66F, 0xA672, 0xA674, 0xA67D, 0xA69E, 0xA69F, 0xA6F0, 0xA6F1, + 0xA802, 0xA802, 0xA806, 0xA806, 0xA80B, 0xA80B, 0xA825, 0xA826, + 0xA82C, 0xA82C, 0xA8C4, 0xA8C5, 0xA8E0, 0xA8F1, 0xA8FF, 0xA8FF, + 0xA926, 0xA92D, 0xA947, 0xA951, 0xA953, 0xA953, 0xA980, 0xA982, + 0xA9B3, 0xA9B3, 0xA9B6, 0xA9B9, 0xA9BC, 0xA9BD, 0xA9C0, 0xA9C0, + 0xA9E5, 0xA9E5, 0xAA29, 0xAA2E, 0xAA31, 0xAA32, 0xAA35, 0xAA36, + 0xAA43, 0xAA43, 0xAA4C, 0xAA4C, 0xAA7C, 0xAA7C, 0xAAB0, 0xAAB0, + 0xAAB2, 0xAAB4, 0xAAB7, 0xAAB8, 0xAABE, 0xAABF, 0xAAC1, 0xAAC1, + 0xAAEC, 0xAAED, 0xAAF6, 0xAAF6, 0xABE5, 0xABE5, 0xABE8, 0xABE8, + 0xABED, 0xABED, 0xFB1E, 0xFB1E, 0xFE00, 0xFE0F, 0xFE20, 0xFE2F, + 0xFF9E, 0xFF9F, 0x101FD, 0x101FD, 0x102E0, 0x102E0, 0x10376, 0x1037A, + 0x10A01, 0x10A03, 0x10A05, 0x10A06, 0x10A0C, 0x10A0F, 0x10A38, 0x10A3A, + 0x10A3F, 0x10A3F, 0x10AE5, 0x10AE6, 0x10D24, 0x10D27, 0x10D69, 0x10D6D, + 0x10EAB, 0x10EAC, 0x10EFA, 0x10EFF, 0x10F46, 0x10F50, 0x10F82, 0x10F85, + 0x11001, 0x11001, 0x11038, 0x11046, 0x11070, 0x11070, 0x11073, 0x11074, + 0x1107F, 0x11081, 0x110B3, 0x110B6, 0x110B9, 0x110BA, 0x110C2, 0x110C2, + 0x11100, 0x11102, 0x11127, 0x1112B, 0x1112D, 0x11134, 0x11173, 0x11173, + 0x11180, 0x11181, 0x111B6, 0x111BE, 0x111C0, 0x111C0, 0x111C9, 0x111CC, + 0x111CF, 0x111CF, 0x1122F, 0x11231, 0x11234, 0x11237, 0x1123E, 0x1123E, + 0x11241, 0x11241, 0x112DF, 0x112DF, 0x112E3, 0x112EA, 0x11300, 0x11301, + 0x1133B, 0x1133C, 0x1133E, 0x1133E, 0x11340, 0x11340, 0x1134D, 0x1134D, + 0x11357, 0x11357, 0x11366, 0x1136C, 0x11370, 0x11374, 0x113B8, 0x113B8, + 0x113BB, 0x113C0, 0x113C2, 0x113C2, 0x113C5, 0x113C5, 0x113C7, 0x113C9, + 0x113CE, 0x113D0, 0x113D2, 0x113D2, 0x113E1, 0x113E2, 0x11438, 0x1143F, + 0x11442, 0x11444, 0x11446, 0x11446, 0x1145E, 0x1145E, 0x114B0, 0x114B0, + 0x114B3, 0x114B8, 0x114BA, 0x114BA, 0x114BD, 0x114BD, 0x114BF, 0x114C0, + 0x114C2, 0x114C3, 0x115AF, 0x115AF, 0x115B2, 0x115B5, 0x115BC, 0x115BD, + 0x115BF, 0x115C0, 0x115DC, 0x115DD, 0x11633, 0x1163A, 0x1163D, 0x1163D, + 0x1163F, 0x11640, 0x116AB, 0x116AB, 0x116AD, 0x116AD, 0x116B0, 0x116B7, + 0x1171D, 0x1171D, 0x1171F, 0x1171F, 0x11722, 0x11725, 0x11727, 0x1172B, + 0x1182F, 0x11837, 0x11839, 0x1183A, 0x11930, 0x11930, 0x1193B, 0x1193E, + 0x11943, 0x11943, 0x119D4, 0x119D7, 0x119DA, 0x119DB, 0x119E0, 0x119E0, + 0x11A01, 0x11A0A, 0x11A33, 0x11A38, 0x11A3B, 0x11A3E, 0x11A47, 0x11A47, + 0x11A51, 0x11A56, 0x11A59, 0x11A5B, 0x11A8A, 0x11A96, 0x11A98, 0x11A99, + 0x11B60, 0x11B60, 0x11B62, 0x11B64, 0x11B66, 0x11B66, 0x11C30, 0x11C36, + 0x11C38, 0x11C3D, 0x11C3F, 0x11C3F, 0x11C92, 0x11CA7, 0x11CAA, 0x11CB0, + 0x11CB2, 0x11CB3, 0x11CB5, 0x11CB6, 0x11D31, 0x11D36, 0x11D3A, 0x11D3A, + 0x11D3C, 0x11D3D, 0x11D3F, 0x11D45, 0x11D47, 0x11D47, 0x11D90, 0x11D91, + 0x11D95, 0x11D95, 0x11D97, 0x11D97, 0x11EF3, 0x11EF4, 0x11F00, 0x11F01, + 0x11F36, 0x11F3A, 0x11F40, 0x11F42, 0x11F5A, 0x11F5A, 0x13440, 0x13440, + 0x13447, 0x13455, 0x1611E, 0x16129, 0x1612D, 0x1612F, 0x16AF0, 0x16AF4, + 0x16B30, 0x16B36, 0x16F4F, 0x16F4F, 0x16F8F, 0x16F92, 0x16FE4, 0x16FE4, + 0x16FF0, 0x16FF1, 0x1BC9D, 0x1BC9E, 0x1CF00, 0x1CF2D, 0x1CF30, 0x1CF46, + 0x1D165, 0x1D169, 0x1D16D, 0x1D172, 0x1D17B, 0x1D182, 0x1D185, 0x1D18B, + 0x1D1AA, 0x1D1AD, 0x1D242, 0x1D244, 0x1DA00, 0x1DA36, 0x1DA3B, 0x1DA6C, + 0x1DA75, 0x1DA75, 0x1DA84, 0x1DA84, 0x1DA9B, 0x1DA9F, 0x1DAA1, 0x1DAAF, + 0x1E000, 0x1E006, 0x1E008, 0x1E018, 0x1E01B, 0x1E021, 0x1E023, 0x1E024, + 0x1E026, 0x1E02A, 0x1E08F, 0x1E08F, 0x1E130, 0x1E136, 0x1E2AE, 0x1E2AE, + 0x1E2EC, 0x1E2EF, 0x1E4EC, 0x1E4EF, 0x1E5EE, 0x1E5EF, 0x1E6E3, 0x1E6E3, + 0x1E6E6, 0x1E6E6, 0x1E6EE, 0x1E6EF, 0x1E6F5, 0x1E6F5, 0x1E8D0, 0x1E8D6, + 0x1E944, 0x1E94A, 0x1F3FB, 0x1F3FF, 0xE0020, 0xE007F, 0xE0100, 0xE01EF, + }; + } + private static int[] GCB_ranges_6() { + return new int[] { + }; + } + private static int[] GCB_ranges_7() { + return new int[] { + 0x1100, 0x115F, 0xA960, 0xA97C, + }; + } + private static int[] GCB_ranges_8() { + return new int[] { + 0xA, 0xA, + }; + } + private static int[] GCB_ranges_9() { + return new int[] { + 0xAC00, 0xAC00, 0xAC1C, 0xAC1C, 0xAC38, 0xAC38, 0xAC54, 0xAC54, + 0xAC70, 0xAC70, 0xAC8C, 0xAC8C, 0xACA8, 0xACA8, 0xACC4, 0xACC4, + 0xACE0, 0xACE0, 0xACFC, 0xACFC, 0xAD18, 0xAD18, 0xAD34, 0xAD34, + 0xAD50, 0xAD50, 0xAD6C, 0xAD6C, 0xAD88, 0xAD88, 0xADA4, 0xADA4, + 0xADC0, 0xADC0, 0xADDC, 0xADDC, 0xADF8, 0xADF8, 0xAE14, 0xAE14, + 0xAE30, 0xAE30, 0xAE4C, 0xAE4C, 0xAE68, 0xAE68, 0xAE84, 0xAE84, + 0xAEA0, 0xAEA0, 0xAEBC, 0xAEBC, 0xAED8, 0xAED8, 0xAEF4, 0xAEF4, + 0xAF10, 0xAF10, 0xAF2C, 0xAF2C, 0xAF48, 0xAF48, 0xAF64, 0xAF64, + 0xAF80, 0xAF80, 0xAF9C, 0xAF9C, 0xAFB8, 0xAFB8, 0xAFD4, 0xAFD4, + 0xAFF0, 0xAFF0, 0xB00C, 0xB00C, 0xB028, 0xB028, 0xB044, 0xB044, + 0xB060, 0xB060, 0xB07C, 0xB07C, 0xB098, 0xB098, 0xB0B4, 0xB0B4, + 0xB0D0, 0xB0D0, 0xB0EC, 0xB0EC, 0xB108, 0xB108, 0xB124, 0xB124, + 0xB140, 0xB140, 0xB15C, 0xB15C, 0xB178, 0xB178, 0xB194, 0xB194, + 0xB1B0, 0xB1B0, 0xB1CC, 0xB1CC, 0xB1E8, 0xB1E8, 0xB204, 0xB204, + 0xB220, 0xB220, 0xB23C, 0xB23C, 0xB258, 0xB258, 0xB274, 0xB274, + 0xB290, 0xB290, 0xB2AC, 0xB2AC, 0xB2C8, 0xB2C8, 0xB2E4, 0xB2E4, + 0xB300, 0xB300, 0xB31C, 0xB31C, 0xB338, 0xB338, 0xB354, 0xB354, + 0xB370, 0xB370, 0xB38C, 0xB38C, 0xB3A8, 0xB3A8, 0xB3C4, 0xB3C4, + 0xB3E0, 0xB3E0, 0xB3FC, 0xB3FC, 0xB418, 0xB418, 0xB434, 0xB434, + 0xB450, 0xB450, 0xB46C, 0xB46C, 0xB488, 0xB488, 0xB4A4, 0xB4A4, + 0xB4C0, 0xB4C0, 0xB4DC, 0xB4DC, 0xB4F8, 0xB4F8, 0xB514, 0xB514, + 0xB530, 0xB530, 0xB54C, 0xB54C, 0xB568, 0xB568, 0xB584, 0xB584, + 0xB5A0, 0xB5A0, 0xB5BC, 0xB5BC, 0xB5D8, 0xB5D8, 0xB5F4, 0xB5F4, + 0xB610, 0xB610, 0xB62C, 0xB62C, 0xB648, 0xB648, 0xB664, 0xB664, + 0xB680, 0xB680, 0xB69C, 0xB69C, 0xB6B8, 0xB6B8, 0xB6D4, 0xB6D4, + 0xB6F0, 0xB6F0, 0xB70C, 0xB70C, 0xB728, 0xB728, 0xB744, 0xB744, + 0xB760, 0xB760, 0xB77C, 0xB77C, 0xB798, 0xB798, 0xB7B4, 0xB7B4, + 0xB7D0, 0xB7D0, 0xB7EC, 0xB7EC, 0xB808, 0xB808, 0xB824, 0xB824, + 0xB840, 0xB840, 0xB85C, 0xB85C, 0xB878, 0xB878, 0xB894, 0xB894, + 0xB8B0, 0xB8B0, 0xB8CC, 0xB8CC, 0xB8E8, 0xB8E8, 0xB904, 0xB904, + 0xB920, 0xB920, 0xB93C, 0xB93C, 0xB958, 0xB958, 0xB974, 0xB974, + 0xB990, 0xB990, 0xB9AC, 0xB9AC, 0xB9C8, 0xB9C8, 0xB9E4, 0xB9E4, + 0xBA00, 0xBA00, 0xBA1C, 0xBA1C, 0xBA38, 0xBA38, 0xBA54, 0xBA54, + 0xBA70, 0xBA70, 0xBA8C, 0xBA8C, 0xBAA8, 0xBAA8, 0xBAC4, 0xBAC4, + 0xBAE0, 0xBAE0, 0xBAFC, 0xBAFC, 0xBB18, 0xBB18, 0xBB34, 0xBB34, + 0xBB50, 0xBB50, 0xBB6C, 0xBB6C, 0xBB88, 0xBB88, 0xBBA4, 0xBBA4, + 0xBBC0, 0xBBC0, 0xBBDC, 0xBBDC, 0xBBF8, 0xBBF8, 0xBC14, 0xBC14, + 0xBC30, 0xBC30, 0xBC4C, 0xBC4C, 0xBC68, 0xBC68, 0xBC84, 0xBC84, + 0xBCA0, 0xBCA0, 0xBCBC, 0xBCBC, 0xBCD8, 0xBCD8, 0xBCF4, 0xBCF4, + 0xBD10, 0xBD10, 0xBD2C, 0xBD2C, 0xBD48, 0xBD48, 0xBD64, 0xBD64, + 0xBD80, 0xBD80, 0xBD9C, 0xBD9C, 0xBDB8, 0xBDB8, 0xBDD4, 0xBDD4, + 0xBDF0, 0xBDF0, 0xBE0C, 0xBE0C, 0xBE28, 0xBE28, 0xBE44, 0xBE44, + 0xBE60, 0xBE60, 0xBE7C, 0xBE7C, 0xBE98, 0xBE98, 0xBEB4, 0xBEB4, + 0xBED0, 0xBED0, 0xBEEC, 0xBEEC, 0xBF08, 0xBF08, 0xBF24, 0xBF24, + 0xBF40, 0xBF40, 0xBF5C, 0xBF5C, 0xBF78, 0xBF78, 0xBF94, 0xBF94, + 0xBFB0, 0xBFB0, 0xBFCC, 0xBFCC, 0xBFE8, 0xBFE8, 0xC004, 0xC004, + 0xC020, 0xC020, 0xC03C, 0xC03C, 0xC058, 0xC058, 0xC074, 0xC074, + 0xC090, 0xC090, 0xC0AC, 0xC0AC, 0xC0C8, 0xC0C8, 0xC0E4, 0xC0E4, + 0xC100, 0xC100, 0xC11C, 0xC11C, 0xC138, 0xC138, 0xC154, 0xC154, + 0xC170, 0xC170, 0xC18C, 0xC18C, 0xC1A8, 0xC1A8, 0xC1C4, 0xC1C4, + 0xC1E0, 0xC1E0, 0xC1FC, 0xC1FC, 0xC218, 0xC218, 0xC234, 0xC234, + 0xC250, 0xC250, 0xC26C, 0xC26C, 0xC288, 0xC288, 0xC2A4, 0xC2A4, + 0xC2C0, 0xC2C0, 0xC2DC, 0xC2DC, 0xC2F8, 0xC2F8, 0xC314, 0xC314, + 0xC330, 0xC330, 0xC34C, 0xC34C, 0xC368, 0xC368, 0xC384, 0xC384, + 0xC3A0, 0xC3A0, 0xC3BC, 0xC3BC, 0xC3D8, 0xC3D8, 0xC3F4, 0xC3F4, + 0xC410, 0xC410, 0xC42C, 0xC42C, 0xC448, 0xC448, 0xC464, 0xC464, + 0xC480, 0xC480, 0xC49C, 0xC49C, 0xC4B8, 0xC4B8, 0xC4D4, 0xC4D4, + 0xC4F0, 0xC4F0, 0xC50C, 0xC50C, 0xC528, 0xC528, 0xC544, 0xC544, + 0xC560, 0xC560, 0xC57C, 0xC57C, 0xC598, 0xC598, 0xC5B4, 0xC5B4, + 0xC5D0, 0xC5D0, 0xC5EC, 0xC5EC, 0xC608, 0xC608, 0xC624, 0xC624, + 0xC640, 0xC640, 0xC65C, 0xC65C, 0xC678, 0xC678, 0xC694, 0xC694, + 0xC6B0, 0xC6B0, 0xC6CC, 0xC6CC, 0xC6E8, 0xC6E8, 0xC704, 0xC704, + 0xC720, 0xC720, 0xC73C, 0xC73C, 0xC758, 0xC758, 0xC774, 0xC774, + 0xC790, 0xC790, 0xC7AC, 0xC7AC, 0xC7C8, 0xC7C8, 0xC7E4, 0xC7E4, + 0xC800, 0xC800, 0xC81C, 0xC81C, 0xC838, 0xC838, 0xC854, 0xC854, + 0xC870, 0xC870, 0xC88C, 0xC88C, 0xC8A8, 0xC8A8, 0xC8C4, 0xC8C4, + 0xC8E0, 0xC8E0, 0xC8FC, 0xC8FC, 0xC918, 0xC918, 0xC934, 0xC934, + 0xC950, 0xC950, 0xC96C, 0xC96C, 0xC988, 0xC988, 0xC9A4, 0xC9A4, + 0xC9C0, 0xC9C0, 0xC9DC, 0xC9DC, 0xC9F8, 0xC9F8, 0xCA14, 0xCA14, + 0xCA30, 0xCA30, 0xCA4C, 0xCA4C, 0xCA68, 0xCA68, 0xCA84, 0xCA84, + 0xCAA0, 0xCAA0, 0xCABC, 0xCABC, 0xCAD8, 0xCAD8, 0xCAF4, 0xCAF4, + 0xCB10, 0xCB10, 0xCB2C, 0xCB2C, 0xCB48, 0xCB48, 0xCB64, 0xCB64, + 0xCB80, 0xCB80, 0xCB9C, 0xCB9C, 0xCBB8, 0xCBB8, 0xCBD4, 0xCBD4, + 0xCBF0, 0xCBF0, 0xCC0C, 0xCC0C, 0xCC28, 0xCC28, 0xCC44, 0xCC44, + 0xCC60, 0xCC60, 0xCC7C, 0xCC7C, 0xCC98, 0xCC98, 0xCCB4, 0xCCB4, + 0xCCD0, 0xCCD0, 0xCCEC, 0xCCEC, 0xCD08, 0xCD08, 0xCD24, 0xCD24, + 0xCD40, 0xCD40, 0xCD5C, 0xCD5C, 0xCD78, 0xCD78, 0xCD94, 0xCD94, + 0xCDB0, 0xCDB0, 0xCDCC, 0xCDCC, 0xCDE8, 0xCDE8, 0xCE04, 0xCE04, + 0xCE20, 0xCE20, 0xCE3C, 0xCE3C, 0xCE58, 0xCE58, 0xCE74, 0xCE74, + 0xCE90, 0xCE90, 0xCEAC, 0xCEAC, 0xCEC8, 0xCEC8, 0xCEE4, 0xCEE4, + 0xCF00, 0xCF00, 0xCF1C, 0xCF1C, 0xCF38, 0xCF38, 0xCF54, 0xCF54, + 0xCF70, 0xCF70, 0xCF8C, 0xCF8C, 0xCFA8, 0xCFA8, 0xCFC4, 0xCFC4, + 0xCFE0, 0xCFE0, 0xCFFC, 0xCFFC, 0xD018, 0xD018, 0xD034, 0xD034, + 0xD050, 0xD050, 0xD06C, 0xD06C, 0xD088, 0xD088, 0xD0A4, 0xD0A4, + 0xD0C0, 0xD0C0, 0xD0DC, 0xD0DC, 0xD0F8, 0xD0F8, 0xD114, 0xD114, + 0xD130, 0xD130, 0xD14C, 0xD14C, 0xD168, 0xD168, 0xD184, 0xD184, + 0xD1A0, 0xD1A0, 0xD1BC, 0xD1BC, 0xD1D8, 0xD1D8, 0xD1F4, 0xD1F4, + 0xD210, 0xD210, 0xD22C, 0xD22C, 0xD248, 0xD248, 0xD264, 0xD264, + 0xD280, 0xD280, 0xD29C, 0xD29C, 0xD2B8, 0xD2B8, 0xD2D4, 0xD2D4, + 0xD2F0, 0xD2F0, 0xD30C, 0xD30C, 0xD328, 0xD328, 0xD344, 0xD344, + 0xD360, 0xD360, 0xD37C, 0xD37C, 0xD398, 0xD398, 0xD3B4, 0xD3B4, + 0xD3D0, 0xD3D0, 0xD3EC, 0xD3EC, 0xD408, 0xD408, 0xD424, 0xD424, + 0xD440, 0xD440, 0xD45C, 0xD45C, 0xD478, 0xD478, 0xD494, 0xD494, + 0xD4B0, 0xD4B0, 0xD4CC, 0xD4CC, 0xD4E8, 0xD4E8, 0xD504, 0xD504, + 0xD520, 0xD520, 0xD53C, 0xD53C, 0xD558, 0xD558, 0xD574, 0xD574, + 0xD590, 0xD590, 0xD5AC, 0xD5AC, 0xD5C8, 0xD5C8, 0xD5E4, 0xD5E4, + 0xD600, 0xD600, 0xD61C, 0xD61C, 0xD638, 0xD638, 0xD654, 0xD654, + 0xD670, 0xD670, 0xD68C, 0xD68C, 0xD6A8, 0xD6A8, 0xD6C4, 0xD6C4, + 0xD6E0, 0xD6E0, 0xD6FC, 0xD6FC, 0xD718, 0xD718, 0xD734, 0xD734, + 0xD750, 0xD750, 0xD76C, 0xD76C, 0xD788, 0xD788, + }; + } + private static int[] GCB_ranges_10() { + return new int[] { + 0xAC01, 0xAC1B, 0xAC1D, 0xAC37, 0xAC39, 0xAC53, 0xAC55, 0xAC6F, + 0xAC71, 0xAC8B, 0xAC8D, 0xACA7, 0xACA9, 0xACC3, 0xACC5, 0xACDF, + 0xACE1, 0xACFB, 0xACFD, 0xAD17, 0xAD19, 0xAD33, 0xAD35, 0xAD4F, + 0xAD51, 0xAD6B, 0xAD6D, 0xAD87, 0xAD89, 0xADA3, 0xADA5, 0xADBF, + 0xADC1, 0xADDB, 0xADDD, 0xADF7, 0xADF9, 0xAE13, 0xAE15, 0xAE2F, + 0xAE31, 0xAE4B, 0xAE4D, 0xAE67, 0xAE69, 0xAE83, 0xAE85, 0xAE9F, + 0xAEA1, 0xAEBB, 0xAEBD, 0xAED7, 0xAED9, 0xAEF3, 0xAEF5, 0xAF0F, + 0xAF11, 0xAF2B, 0xAF2D, 0xAF47, 0xAF49, 0xAF63, 0xAF65, 0xAF7F, + 0xAF81, 0xAF9B, 0xAF9D, 0xAFB7, 0xAFB9, 0xAFD3, 0xAFD5, 0xAFEF, + 0xAFF1, 0xB00B, 0xB00D, 0xB027, 0xB029, 0xB043, 0xB045, 0xB05F, + 0xB061, 0xB07B, 0xB07D, 0xB097, 0xB099, 0xB0B3, 0xB0B5, 0xB0CF, + 0xB0D1, 0xB0EB, 0xB0ED, 0xB107, 0xB109, 0xB123, 0xB125, 0xB13F, + 0xB141, 0xB15B, 0xB15D, 0xB177, 0xB179, 0xB193, 0xB195, 0xB1AF, + 0xB1B1, 0xB1CB, 0xB1CD, 0xB1E7, 0xB1E9, 0xB203, 0xB205, 0xB21F, + 0xB221, 0xB23B, 0xB23D, 0xB257, 0xB259, 0xB273, 0xB275, 0xB28F, + 0xB291, 0xB2AB, 0xB2AD, 0xB2C7, 0xB2C9, 0xB2E3, 0xB2E5, 0xB2FF, + 0xB301, 0xB31B, 0xB31D, 0xB337, 0xB339, 0xB353, 0xB355, 0xB36F, + 0xB371, 0xB38B, 0xB38D, 0xB3A7, 0xB3A9, 0xB3C3, 0xB3C5, 0xB3DF, + 0xB3E1, 0xB3FB, 0xB3FD, 0xB417, 0xB419, 0xB433, 0xB435, 0xB44F, + 0xB451, 0xB46B, 0xB46D, 0xB487, 0xB489, 0xB4A3, 0xB4A5, 0xB4BF, + 0xB4C1, 0xB4DB, 0xB4DD, 0xB4F7, 0xB4F9, 0xB513, 0xB515, 0xB52F, + 0xB531, 0xB54B, 0xB54D, 0xB567, 0xB569, 0xB583, 0xB585, 0xB59F, + 0xB5A1, 0xB5BB, 0xB5BD, 0xB5D7, 0xB5D9, 0xB5F3, 0xB5F5, 0xB60F, + 0xB611, 0xB62B, 0xB62D, 0xB647, 0xB649, 0xB663, 0xB665, 0xB67F, + 0xB681, 0xB69B, 0xB69D, 0xB6B7, 0xB6B9, 0xB6D3, 0xB6D5, 0xB6EF, + 0xB6F1, 0xB70B, 0xB70D, 0xB727, 0xB729, 0xB743, 0xB745, 0xB75F, + 0xB761, 0xB77B, 0xB77D, 0xB797, 0xB799, 0xB7B3, 0xB7B5, 0xB7CF, + 0xB7D1, 0xB7EB, 0xB7ED, 0xB807, 0xB809, 0xB823, 0xB825, 0xB83F, + 0xB841, 0xB85B, 0xB85D, 0xB877, 0xB879, 0xB893, 0xB895, 0xB8AF, + 0xB8B1, 0xB8CB, 0xB8CD, 0xB8E7, 0xB8E9, 0xB903, 0xB905, 0xB91F, + 0xB921, 0xB93B, 0xB93D, 0xB957, 0xB959, 0xB973, 0xB975, 0xB98F, + 0xB991, 0xB9AB, 0xB9AD, 0xB9C7, 0xB9C9, 0xB9E3, 0xB9E5, 0xB9FF, + 0xBA01, 0xBA1B, 0xBA1D, 0xBA37, 0xBA39, 0xBA53, 0xBA55, 0xBA6F, + 0xBA71, 0xBA8B, 0xBA8D, 0xBAA7, 0xBAA9, 0xBAC3, 0xBAC5, 0xBADF, + 0xBAE1, 0xBAFB, 0xBAFD, 0xBB17, 0xBB19, 0xBB33, 0xBB35, 0xBB4F, + 0xBB51, 0xBB6B, 0xBB6D, 0xBB87, 0xBB89, 0xBBA3, 0xBBA5, 0xBBBF, + 0xBBC1, 0xBBDB, 0xBBDD, 0xBBF7, 0xBBF9, 0xBC13, 0xBC15, 0xBC2F, + 0xBC31, 0xBC4B, 0xBC4D, 0xBC67, 0xBC69, 0xBC83, 0xBC85, 0xBC9F, + 0xBCA1, 0xBCBB, 0xBCBD, 0xBCD7, 0xBCD9, 0xBCF3, 0xBCF5, 0xBD0F, + 0xBD11, 0xBD2B, 0xBD2D, 0xBD47, 0xBD49, 0xBD63, 0xBD65, 0xBD7F, + 0xBD81, 0xBD9B, 0xBD9D, 0xBDB7, 0xBDB9, 0xBDD3, 0xBDD5, 0xBDEF, + 0xBDF1, 0xBE0B, 0xBE0D, 0xBE27, 0xBE29, 0xBE43, 0xBE45, 0xBE5F, + 0xBE61, 0xBE7B, 0xBE7D, 0xBE97, 0xBE99, 0xBEB3, 0xBEB5, 0xBECF, + 0xBED1, 0xBEEB, 0xBEED, 0xBF07, 0xBF09, 0xBF23, 0xBF25, 0xBF3F, + 0xBF41, 0xBF5B, 0xBF5D, 0xBF77, 0xBF79, 0xBF93, 0xBF95, 0xBFAF, + 0xBFB1, 0xBFCB, 0xBFCD, 0xBFE7, 0xBFE9, 0xC003, 0xC005, 0xC01F, + 0xC021, 0xC03B, 0xC03D, 0xC057, 0xC059, 0xC073, 0xC075, 0xC08F, + 0xC091, 0xC0AB, 0xC0AD, 0xC0C7, 0xC0C9, 0xC0E3, 0xC0E5, 0xC0FF, + 0xC101, 0xC11B, 0xC11D, 0xC137, 0xC139, 0xC153, 0xC155, 0xC16F, + 0xC171, 0xC18B, 0xC18D, 0xC1A7, 0xC1A9, 0xC1C3, 0xC1C5, 0xC1DF, + 0xC1E1, 0xC1FB, 0xC1FD, 0xC217, 0xC219, 0xC233, 0xC235, 0xC24F, + 0xC251, 0xC26B, 0xC26D, 0xC287, 0xC289, 0xC2A3, 0xC2A5, 0xC2BF, + 0xC2C1, 0xC2DB, 0xC2DD, 0xC2F7, 0xC2F9, 0xC313, 0xC315, 0xC32F, + 0xC331, 0xC34B, 0xC34D, 0xC367, 0xC369, 0xC383, 0xC385, 0xC39F, + 0xC3A1, 0xC3BB, 0xC3BD, 0xC3D7, 0xC3D9, 0xC3F3, 0xC3F5, 0xC40F, + 0xC411, 0xC42B, 0xC42D, 0xC447, 0xC449, 0xC463, 0xC465, 0xC47F, + 0xC481, 0xC49B, 0xC49D, 0xC4B7, 0xC4B9, 0xC4D3, 0xC4D5, 0xC4EF, + 0xC4F1, 0xC50B, 0xC50D, 0xC527, 0xC529, 0xC543, 0xC545, 0xC55F, + 0xC561, 0xC57B, 0xC57D, 0xC597, 0xC599, 0xC5B3, 0xC5B5, 0xC5CF, + 0xC5D1, 0xC5EB, 0xC5ED, 0xC607, 0xC609, 0xC623, 0xC625, 0xC63F, + 0xC641, 0xC65B, 0xC65D, 0xC677, 0xC679, 0xC693, 0xC695, 0xC6AF, + 0xC6B1, 0xC6CB, 0xC6CD, 0xC6E7, 0xC6E9, 0xC703, 0xC705, 0xC71F, + 0xC721, 0xC73B, 0xC73D, 0xC757, 0xC759, 0xC773, 0xC775, 0xC78F, + 0xC791, 0xC7AB, 0xC7AD, 0xC7C7, 0xC7C9, 0xC7E3, 0xC7E5, 0xC7FF, + 0xC801, 0xC81B, 0xC81D, 0xC837, 0xC839, 0xC853, 0xC855, 0xC86F, + 0xC871, 0xC88B, 0xC88D, 0xC8A7, 0xC8A9, 0xC8C3, 0xC8C5, 0xC8DF, + 0xC8E1, 0xC8FB, 0xC8FD, 0xC917, 0xC919, 0xC933, 0xC935, 0xC94F, + 0xC951, 0xC96B, 0xC96D, 0xC987, 0xC989, 0xC9A3, 0xC9A5, 0xC9BF, + 0xC9C1, 0xC9DB, 0xC9DD, 0xC9F7, 0xC9F9, 0xCA13, 0xCA15, 0xCA2F, + 0xCA31, 0xCA4B, 0xCA4D, 0xCA67, 0xCA69, 0xCA83, 0xCA85, 0xCA9F, + 0xCAA1, 0xCABB, 0xCABD, 0xCAD7, 0xCAD9, 0xCAF3, 0xCAF5, 0xCB0F, + 0xCB11, 0xCB2B, 0xCB2D, 0xCB47, 0xCB49, 0xCB63, 0xCB65, 0xCB7F, + 0xCB81, 0xCB9B, 0xCB9D, 0xCBB7, 0xCBB9, 0xCBD3, 0xCBD5, 0xCBEF, + 0xCBF1, 0xCC0B, 0xCC0D, 0xCC27, 0xCC29, 0xCC43, 0xCC45, 0xCC5F, + 0xCC61, 0xCC7B, 0xCC7D, 0xCC97, 0xCC99, 0xCCB3, 0xCCB5, 0xCCCF, + 0xCCD1, 0xCCEB, 0xCCED, 0xCD07, 0xCD09, 0xCD23, 0xCD25, 0xCD3F, + 0xCD41, 0xCD5B, 0xCD5D, 0xCD77, 0xCD79, 0xCD93, 0xCD95, 0xCDAF, + 0xCDB1, 0xCDCB, 0xCDCD, 0xCDE7, 0xCDE9, 0xCE03, 0xCE05, 0xCE1F, + 0xCE21, 0xCE3B, 0xCE3D, 0xCE57, 0xCE59, 0xCE73, 0xCE75, 0xCE8F, + 0xCE91, 0xCEAB, 0xCEAD, 0xCEC7, 0xCEC9, 0xCEE3, 0xCEE5, 0xCEFF, + 0xCF01, 0xCF1B, 0xCF1D, 0xCF37, 0xCF39, 0xCF53, 0xCF55, 0xCF6F, + 0xCF71, 0xCF8B, 0xCF8D, 0xCFA7, 0xCFA9, 0xCFC3, 0xCFC5, 0xCFDF, + 0xCFE1, 0xCFFB, 0xCFFD, 0xD017, 0xD019, 0xD033, 0xD035, 0xD04F, + 0xD051, 0xD06B, 0xD06D, 0xD087, 0xD089, 0xD0A3, 0xD0A5, 0xD0BF, + 0xD0C1, 0xD0DB, 0xD0DD, 0xD0F7, 0xD0F9, 0xD113, 0xD115, 0xD12F, + 0xD131, 0xD14B, 0xD14D, 0xD167, 0xD169, 0xD183, 0xD185, 0xD19F, + 0xD1A1, 0xD1BB, 0xD1BD, 0xD1D7, 0xD1D9, 0xD1F3, 0xD1F5, 0xD20F, + 0xD211, 0xD22B, 0xD22D, 0xD247, 0xD249, 0xD263, 0xD265, 0xD27F, + 0xD281, 0xD29B, 0xD29D, 0xD2B7, 0xD2B9, 0xD2D3, 0xD2D5, 0xD2EF, + 0xD2F1, 0xD30B, 0xD30D, 0xD327, 0xD329, 0xD343, 0xD345, 0xD35F, + 0xD361, 0xD37B, 0xD37D, 0xD397, 0xD399, 0xD3B3, 0xD3B5, 0xD3CF, + 0xD3D1, 0xD3EB, 0xD3ED, 0xD407, 0xD409, 0xD423, 0xD425, 0xD43F, + 0xD441, 0xD45B, 0xD45D, 0xD477, 0xD479, 0xD493, 0xD495, 0xD4AF, + 0xD4B1, 0xD4CB, 0xD4CD, 0xD4E7, 0xD4E9, 0xD503, 0xD505, 0xD51F, + 0xD521, 0xD53B, 0xD53D, 0xD557, 0xD559, 0xD573, 0xD575, 0xD58F, + 0xD591, 0xD5AB, 0xD5AD, 0xD5C7, 0xD5C9, 0xD5E3, 0xD5E5, 0xD5FF, + 0xD601, 0xD61B, 0xD61D, 0xD637, 0xD639, 0xD653, 0xD655, 0xD66F, + 0xD671, 0xD68B, 0xD68D, 0xD6A7, 0xD6A9, 0xD6C3, 0xD6C5, 0xD6DF, + 0xD6E1, 0xD6FB, 0xD6FD, 0xD717, 0xD719, 0xD733, 0xD735, 0xD74F, + 0xD751, 0xD76B, 0xD76D, 0xD787, 0xD789, 0xD7A3, + }; + } + private static int[] GCB_ranges_11() { + return new int[] { + 0x600, 0x605, 0x6DD, 0x6DD, 0x70F, 0x70F, 0x890, 0x891, + 0x8E2, 0x8E2, 0xD4E, 0xD4E, 0x110BD, 0x110BD, 0x110CD, 0x110CD, + 0x111C2, 0x111C3, 0x113D1, 0x113D1, 0x1193F, 0x1193F, 0x11941, 0x11941, + 0x11A84, 0x11A89, 0x11D46, 0x11D46, 0x11F02, 0x11F02, + }; + } + private static int[] GCB_ranges_12() { + return new int[] { + 0x1F1E6, 0x1F1FF, + }; + } + private static int[] GCB_ranges_13() { + return new int[] { + 0x903, 0x903, 0x93B, 0x93B, 0x93E, 0x940, 0x949, 0x94C, + 0x94E, 0x94F, 0x982, 0x983, 0x9BF, 0x9C0, 0x9C7, 0x9C8, + 0x9CB, 0x9CC, 0xA03, 0xA03, 0xA3E, 0xA40, 0xA83, 0xA83, + 0xABE, 0xAC0, 0xAC9, 0xAC9, 0xACB, 0xACC, 0xB02, 0xB03, + 0xB40, 0xB40, 0xB47, 0xB48, 0xB4B, 0xB4C, 0xBBF, 0xBBF, + 0xBC1, 0xBC2, 0xBC6, 0xBC8, 0xBCA, 0xBCC, 0xC01, 0xC03, + 0xC41, 0xC44, 0xC82, 0xC83, 0xCBE, 0xCBE, 0xCC1, 0xCC1, + 0xCC3, 0xCC4, 0xCF3, 0xCF3, 0xD02, 0xD03, 0xD3F, 0xD40, + 0xD46, 0xD48, 0xD4A, 0xD4C, 0xD82, 0xD83, 0xDD0, 0xDD1, + 0xDD8, 0xDDE, 0xDF2, 0xDF3, 0xE33, 0xE33, 0xEB3, 0xEB3, + 0xF3E, 0xF3F, 0xF7F, 0xF7F, 0x1031, 0x1031, 0x103B, 0x103C, + 0x1056, 0x1057, 0x1084, 0x1084, 0x17B6, 0x17B6, 0x17BE, 0x17C5, + 0x17C7, 0x17C8, 0x1923, 0x1926, 0x1929, 0x192B, 0x1930, 0x1931, + 0x1933, 0x1938, 0x1A19, 0x1A1A, 0x1A55, 0x1A55, 0x1A57, 0x1A57, + 0x1A6D, 0x1A72, 0x1B04, 0x1B04, 0x1B3E, 0x1B41, 0x1B82, 0x1B82, + 0x1BA1, 0x1BA1, 0x1BA6, 0x1BA7, 0x1BE7, 0x1BE7, 0x1BEA, 0x1BEC, + 0x1BEE, 0x1BEE, 0x1C24, 0x1C2B, 0x1C34, 0x1C35, 0x1CE1, 0x1CE1, + 0x1CF7, 0x1CF7, 0xA823, 0xA824, 0xA827, 0xA827, 0xA880, 0xA881, + 0xA8B4, 0xA8C3, 0xA952, 0xA952, 0xA983, 0xA983, 0xA9B4, 0xA9B5, + 0xA9BA, 0xA9BB, 0xA9BE, 0xA9BF, 0xAA2F, 0xAA30, 0xAA33, 0xAA34, + 0xAA4D, 0xAA4D, 0xAAEB, 0xAAEB, 0xAAEE, 0xAAEF, 0xAAF5, 0xAAF5, + 0xABE3, 0xABE4, 0xABE6, 0xABE7, 0xABE9, 0xABEA, 0xABEC, 0xABEC, + 0x11000, 0x11000, 0x11002, 0x11002, 0x11082, 0x11082, 0x110B0, 0x110B2, + 0x110B7, 0x110B8, 0x1112C, 0x1112C, 0x11145, 0x11146, 0x11182, 0x11182, + 0x111B3, 0x111B5, 0x111BF, 0x111BF, 0x111CE, 0x111CE, 0x1122C, 0x1122E, + 0x11232, 0x11233, 0x112E0, 0x112E2, 0x11302, 0x11303, 0x1133F, 0x1133F, + 0x11341, 0x11344, 0x11347, 0x11348, 0x1134B, 0x1134C, 0x11362, 0x11363, + 0x113B9, 0x113BA, 0x113CA, 0x113CA, 0x113CC, 0x113CD, 0x11435, 0x11437, + 0x11440, 0x11441, 0x11445, 0x11445, 0x114B1, 0x114B2, 0x114B9, 0x114B9, + 0x114BB, 0x114BC, 0x114BE, 0x114BE, 0x114C1, 0x114C1, 0x115B0, 0x115B1, + 0x115B8, 0x115BB, 0x115BE, 0x115BE, 0x11630, 0x11632, 0x1163B, 0x1163C, + 0x1163E, 0x1163E, 0x116AC, 0x116AC, 0x116AE, 0x116AF, 0x1171E, 0x1171E, + 0x11726, 0x11726, 0x1182C, 0x1182E, 0x11838, 0x11838, 0x11931, 0x11935, + 0x11937, 0x11938, 0x11940, 0x11940, 0x11942, 0x11942, 0x119D1, 0x119D3, + 0x119DC, 0x119DF, 0x119E4, 0x119E4, 0x11A39, 0x11A39, 0x11A57, 0x11A58, + 0x11A97, 0x11A97, 0x11B61, 0x11B61, 0x11B65, 0x11B65, 0x11B67, 0x11B67, + 0x11C2F, 0x11C2F, 0x11C3E, 0x11C3E, 0x11CA9, 0x11CA9, 0x11CB1, 0x11CB1, + 0x11CB4, 0x11CB4, 0x11D8A, 0x11D8E, 0x11D93, 0x11D94, 0x11D96, 0x11D96, + 0x11EF5, 0x11EF6, 0x11F03, 0x11F03, 0x11F34, 0x11F35, 0x11F3E, 0x11F3F, + 0x1612A, 0x1612C, 0x16F51, 0x16F87, + }; + } + private static int[] GCB_ranges_14() { + return new int[] { + 0x11A8, 0x11FF, 0xD7CB, 0xD7FB, + }; + } + private static int[] GCB_ranges_15() { + return new int[] { + 0x1160, 0x11A7, 0xD7B0, 0xD7C6, 0x16D63, 0x16D63, 0x16D67, 0x16D6A, + }; + } + private static int[] GCB_ranges_16() { + return new int[] { + 0x20, 0x7E, 0xA0, 0xAC, 0xAE, 0x2FF, 0x370, 0x482, + 0x48A, 0x590, 0x5BE, 0x5BE, 0x5C0, 0x5C0, 0x5C3, 0x5C3, + 0x5C6, 0x5C6, 0x5C8, 0x5FF, 0x606, 0x60F, 0x61B, 0x61B, + 0x61D, 0x64A, 0x660, 0x66F, 0x671, 0x6D5, 0x6DE, 0x6DE, + 0x6E5, 0x6E6, 0x6E9, 0x6E9, 0x6EE, 0x70E, 0x710, 0x710, + 0x712, 0x72F, 0x74B, 0x7A5, 0x7B1, 0x7EA, 0x7F4, 0x7FC, + 0x7FE, 0x815, 0x81A, 0x81A, 0x824, 0x824, 0x828, 0x828, + 0x82E, 0x858, 0x85C, 0x88F, 0x892, 0x896, 0x8A0, 0x8C9, + 0x904, 0x939, 0x93D, 0x93D, 0x950, 0x950, 0x958, 0x961, + 0x964, 0x980, 0x984, 0x9BB, 0x9BD, 0x9BD, 0x9C5, 0x9C6, + 0x9C9, 0x9CA, 0x9CE, 0x9D6, 0x9D8, 0x9E1, 0x9E4, 0x9FD, + 0x9FF, 0xA00, 0xA04, 0xA3B, 0xA3D, 0xA3D, 0xA43, 0xA46, + 0xA49, 0xA4A, 0xA4E, 0xA50, 0xA52, 0xA6F, 0xA72, 0xA74, + 0xA76, 0xA80, 0xA84, 0xABB, 0xABD, 0xABD, 0xAC6, 0xAC6, + 0xACA, 0xACA, 0xACE, 0xAE1, 0xAE4, 0xAF9, 0xB00, 0xB00, + 0xB04, 0xB3B, 0xB3D, 0xB3D, 0xB45, 0xB46, 0xB49, 0xB4A, + 0xB4E, 0xB54, 0xB58, 0xB61, 0xB64, 0xB81, 0xB83, 0xBBD, + 0xBC3, 0xBC5, 0xBC9, 0xBC9, 0xBCE, 0xBD6, 0xBD8, 0xBFF, + 0xC05, 0xC3B, 0xC3D, 0xC3D, 0xC45, 0xC45, 0xC49, 0xC49, + 0xC4E, 0xC54, 0xC57, 0xC61, 0xC64, 0xC80, 0xC84, 0xCBB, + 0xCBD, 0xCBD, 0xCC5, 0xCC5, 0xCC9, 0xCC9, 0xCCE, 0xCD4, + 0xCD7, 0xCE1, 0xCE4, 0xCF2, 0xCF4, 0xCFF, 0xD04, 0xD3A, + 0xD3D, 0xD3D, 0xD45, 0xD45, 0xD49, 0xD49, 0xD4F, 0xD56, + 0xD58, 0xD61, 0xD64, 0xD80, 0xD84, 0xDC9, 0xDCB, 0xDCE, + 0xDD5, 0xDD5, 0xDD7, 0xDD7, 0xDE0, 0xDF1, 0xDF4, 0xE30, + 0xE32, 0xE32, 0xE3B, 0xE46, 0xE4F, 0xEB0, 0xEB2, 0xEB2, + 0xEBD, 0xEC7, 0xECF, 0xF17, 0xF1A, 0xF34, 0xF36, 0xF36, + 0xF38, 0xF38, 0xF3A, 0xF3D, 0xF40, 0xF70, 0xF85, 0xF85, + 0xF88, 0xF8C, 0xF98, 0xF98, 0xFBD, 0xFC5, 0xFC7, 0x102C, + 0x1038, 0x1038, 0x103F, 0x1055, 0x105A, 0x105D, 0x1061, 0x1070, + 0x1075, 0x1081, 0x1083, 0x1083, 0x1087, 0x108C, 0x108E, 0x109C, + 0x109E, 0x10FF, 0x1200, 0x135C, 0x1360, 0x1711, 0x1716, 0x1731, + 0x1735, 0x1751, 0x1754, 0x1771, 0x1774, 0x17B3, 0x17D4, 0x17DC, + 0x17DE, 0x180A, 0x1810, 0x1884, 0x1887, 0x18A8, 0x18AA, 0x191F, + 0x192C, 0x192F, 0x193C, 0x1A16, 0x1A1C, 0x1A54, 0x1A5F, 0x1A5F, + 0x1A61, 0x1A61, 0x1A63, 0x1A64, 0x1A7D, 0x1A7E, 0x1A80, 0x1AAF, + 0x1ADE, 0x1ADF, 0x1AEC, 0x1AFF, 0x1B05, 0x1B33, 0x1B45, 0x1B6A, + 0x1B74, 0x1B7F, 0x1B83, 0x1BA0, 0x1BAE, 0x1BE5, 0x1BF4, 0x1C23, + 0x1C38, 0x1CCF, 0x1CD3, 0x1CD3, 0x1CE9, 0x1CEC, 0x1CEE, 0x1CF3, + 0x1CF5, 0x1CF6, 0x1CFA, 0x1DBF, 0x1E00, 0x200A, 0x2010, 0x2027, + 0x202F, 0x205F, 0x2070, 0x20CF, 0x20F1, 0x2CEE, 0x2CF2, 0x2D7E, + 0x2D80, 0x2DDF, 0x2E00, 0x3029, 0x3030, 0x3098, 0x309B, 0xA66E, + 0xA673, 0xA673, 0xA67E, 0xA69D, 0xA6A0, 0xA6EF, 0xA6F2, 0xA801, + 0xA803, 0xA805, 0xA807, 0xA80A, 0xA80C, 0xA822, 0xA828, 0xA82B, + 0xA82D, 0xA87F, 0xA882, 0xA8B3, 0xA8C6, 0xA8DF, 0xA8F2, 0xA8FE, + 0xA900, 0xA925, 0xA92E, 0xA946, 0xA954, 0xA95F, 0xA97D, 0xA97F, + 0xA984, 0xA9B2, 0xA9C1, 0xA9E4, 0xA9E6, 0xAA28, 0xAA37, 0xAA42, + 0xAA44, 0xAA4B, 0xAA4E, 0xAA7B, 0xAA7D, 0xAAAF, 0xAAB1, 0xAAB1, + 0xAAB5, 0xAAB6, 0xAAB9, 0xAABD, 0xAAC0, 0xAAC0, 0xAAC2, 0xAAEA, + 0xAAF0, 0xAAF4, 0xAAF7, 0xABE2, 0xABEB, 0xABEB, 0xABEE, 0xABFF, + 0xD7A4, 0xD7AF, 0xD7C7, 0xD7CA, 0xD7FC, 0xFB1D, 0xFB1F, 0xFDFF, + 0xFE10, 0xFE1F, 0xFE30, 0xFEFE, 0xFF00, 0xFF9D, 0xFFA0, 0xFFEF, + 0xFFFC, 0x101FC, 0x101FE, 0x102DF, 0x102E1, 0x10375, 0x1037B, 0x10A00, + 0x10A04, 0x10A04, 0x10A07, 0x10A0B, 0x10A10, 0x10A37, 0x10A3B, 0x10A3E, + 0x10A40, 0x10AE4, 0x10AE7, 0x10D23, 0x10D28, 0x10D68, 0x10D6E, 0x10EAA, + 0x10EAD, 0x10EF9, 0x10F00, 0x10F45, 0x10F51, 0x10F81, 0x10F86, 0x10FFF, + 0x11003, 0x11037, 0x11047, 0x1106F, 0x11071, 0x11072, 0x11075, 0x1107E, + 0x11083, 0x110AF, 0x110BB, 0x110BC, 0x110BE, 0x110C1, 0x110C3, 0x110CC, + 0x110CE, 0x110FF, 0x11103, 0x11126, 0x11135, 0x11144, 0x11147, 0x11172, + 0x11174, 0x1117F, 0x11183, 0x111B2, 0x111C1, 0x111C1, 0x111C4, 0x111C8, + 0x111CD, 0x111CD, 0x111D0, 0x1122B, 0x11238, 0x1123D, 0x1123F, 0x11240, + 0x11242, 0x112DE, 0x112EB, 0x112FF, 0x11304, 0x1133A, 0x1133D, 0x1133D, + 0x11345, 0x11346, 0x11349, 0x1134A, 0x1134E, 0x11356, 0x11358, 0x11361, + 0x11364, 0x11365, 0x1136D, 0x1136F, 0x11375, 0x113B7, 0x113C1, 0x113C1, + 0x113C3, 0x113C4, 0x113C6, 0x113C6, 0x113CB, 0x113CB, 0x113D3, 0x113E0, + 0x113E3, 0x11434, 0x11447, 0x1145D, 0x1145F, 0x114AF, 0x114C4, 0x115AE, + 0x115B6, 0x115B7, 0x115C1, 0x115DB, 0x115DE, 0x1162F, 0x11641, 0x116AA, + 0x116B8, 0x1171C, 0x11720, 0x11721, 0x1172C, 0x1182B, 0x1183B, 0x1192F, + 0x11936, 0x11936, 0x11939, 0x1193A, 0x11944, 0x119D0, 0x119D8, 0x119D9, + 0x119E1, 0x119E3, 0x119E5, 0x11A00, 0x11A0B, 0x11A32, 0x11A3A, 0x11A3A, + 0x11A3F, 0x11A46, 0x11A48, 0x11A50, 0x11A5C, 0x11A83, 0x11A9A, 0x11B5F, + 0x11B68, 0x11C2E, 0x11C37, 0x11C37, 0x11C40, 0x11C91, 0x11CA8, 0x11CA8, + 0x11CB7, 0x11D30, 0x11D37, 0x11D39, 0x11D3B, 0x11D3B, 0x11D3E, 0x11D3E, + 0x11D48, 0x11D89, 0x11D8F, 0x11D8F, 0x11D92, 0x11D92, 0x11D98, 0x11EF2, + 0x11EF7, 0x11EFF, 0x11F04, 0x11F33, 0x11F3B, 0x11F3D, 0x11F43, 0x11F59, + 0x11F5B, 0x1342F, 0x13441, 0x13446, 0x13456, 0x1611D, 0x16130, 0x16AEF, + 0x16AF5, 0x16B2F, 0x16B37, 0x16D62, 0x16D64, 0x16D66, 0x16D6B, 0x16F4E, + 0x16F50, 0x16F50, 0x16F88, 0x16F8E, 0x16F93, 0x16FE3, 0x16FE5, 0x16FEF, + 0x16FF2, 0x1BC9C, 0x1BC9F, 0x1BC9F, 0x1BCA4, 0x1CEFF, 0x1CF2E, 0x1CF2F, + 0x1CF47, 0x1D164, 0x1D16A, 0x1D16C, 0x1D183, 0x1D184, 0x1D18C, 0x1D1A9, + 0x1D1AE, 0x1D241, 0x1D245, 0x1D9FF, 0x1DA37, 0x1DA3A, 0x1DA6D, 0x1DA74, + 0x1DA76, 0x1DA83, 0x1DA85, 0x1DA9A, 0x1DAA0, 0x1DAA0, 0x1DAB0, 0x1DFFF, + 0x1E007, 0x1E007, 0x1E019, 0x1E01A, 0x1E022, 0x1E022, 0x1E025, 0x1E025, + 0x1E02B, 0x1E08E, 0x1E090, 0x1E12F, 0x1E137, 0x1E2AD, 0x1E2AF, 0x1E2EB, + 0x1E2F0, 0x1E4EB, 0x1E4F0, 0x1E5ED, 0x1E5F0, 0x1E6E2, 0x1E6E4, 0x1E6E5, + 0x1E6E7, 0x1E6ED, 0x1E6F0, 0x1E6F4, 0x1E6F6, 0x1E8CF, 0x1E8D7, 0x1E943, + 0x1E94B, 0x1F1E5, 0x1F200, 0x1F3FA, 0x1F400, 0xDFFFF, 0xE1000, 0x10FFFF, + }; + } + private static int[] GCB_ranges_17() { + return new int[] { + 0x200D, 0x200D, + }; + } + private static final int[][] GCB_RANGES = { + GCB_ranges_0(), GCB_ranges_1(), GCB_ranges_2(), GCB_ranges_3(), GCB_ranges_4(), GCB_ranges_5(), GCB_ranges_6(), GCB_ranges_7(), GCB_ranges_8(), GCB_ranges_9(), GCB_ranges_10(), GCB_ranges_11(), GCB_ranges_12(), GCB_ranges_13(), GCB_ranges_14(), GCB_ranges_15(), GCB_ranges_16(), GCB_ranges_17() + }; + private static final Property GCB = new Property( + GCB_VALUES, GCB_SHORT_VALUES, + GCB_VALUE_ALIASES, GCB_VALUE_ALIAS_INDEX, + GCB_PROPERTY_ALIASES, GCB_RANGES); + + private static final String[] InCB_VALUES = { + "Consonant", "Extend", "Linker", "None" + }; + private static final String[] InCB_SHORT_VALUES = { + "Consonant", "Extend", "Linker", "None" + }; + private static final String[] InCB_VALUE_ALIASES = { + "consonant", "extend", "linker", "none" + }; + private static final byte[] InCB_VALUE_ALIAS_INDEX = { + 0, 1, 2, 3 + }; + private static final String[] InCB_PROPERTY_ALIASES = { + "incb", "indicconjunctbreak" + }; + private static int[] InCB_ranges_0() { + return new int[] { + 0x915, 0x939, 0x958, 0x95F, 0x978, 0x97F, 0x995, 0x9A8, + 0x9AA, 0x9B0, 0x9B2, 0x9B2, 0x9B6, 0x9B9, 0x9DC, 0x9DD, + 0x9DF, 0x9DF, 0x9F0, 0x9F1, 0xA95, 0xAA8, 0xAAA, 0xAB0, + 0xAB2, 0xAB3, 0xAB5, 0xAB9, 0xAF9, 0xAF9, 0xB15, 0xB28, + 0xB2A, 0xB30, 0xB32, 0xB33, 0xB35, 0xB39, 0xB5C, 0xB5D, + 0xB5F, 0xB5F, 0xB71, 0xB71, 0xC15, 0xC28, 0xC2A, 0xC39, + 0xC58, 0xC5A, 0xD15, 0xD3A, 0x1000, 0x102A, 0x103F, 0x103F, + 0x1050, 0x1055, 0x105A, 0x105D, 0x1061, 0x1061, 0x1065, 0x1066, + 0x106E, 0x1070, 0x1075, 0x1081, 0x108E, 0x108E, 0x1780, 0x17B3, + 0x1A20, 0x1A54, 0x1B0B, 0x1B0C, 0x1B13, 0x1B33, 0x1B45, 0x1B4C, + 0x1B83, 0x1BA0, 0x1BAE, 0x1BAF, 0x1BBB, 0x1BBD, 0xA989, 0xA98B, + 0xA98F, 0xA9B2, 0xA9E0, 0xA9E4, 0xA9E7, 0xA9EF, 0xA9FA, 0xA9FE, + 0xAA60, 0xAA6F, 0xAA71, 0xAA73, 0xAA7A, 0xAA7A, 0xAA7E, 0xAA7F, + 0xAAE0, 0xAAEA, 0xABC0, 0xABDA, 0x10A00, 0x10A00, 0x10A10, 0x10A13, + 0x10A15, 0x10A17, 0x10A19, 0x10A35, 0x11103, 0x11126, 0x11144, 0x11144, + 0x11147, 0x11147, 0x11380, 0x11389, 0x1138B, 0x1138B, 0x1138E, 0x1138E, + 0x11390, 0x113B5, 0x11900, 0x11906, 0x11909, 0x11909, 0x1190C, 0x11913, + 0x11915, 0x11916, 0x11918, 0x1192F, 0x11A00, 0x11A00, 0x11A0B, 0x11A32, + 0x11A50, 0x11A50, 0x11A5C, 0x11A83, 0x11F04, 0x11F10, 0x11F12, 0x11F33, + }; + } + private static int[] InCB_ranges_1() { + return new int[] { + 0x300, 0x36F, 0x483, 0x489, 0x591, 0x5BD, 0x5BF, 0x5BF, + 0x5C1, 0x5C2, 0x5C4, 0x5C5, 0x5C7, 0x5C7, 0x610, 0x61A, + 0x64B, 0x65F, 0x670, 0x670, 0x6D6, 0x6DC, 0x6DF, 0x6E4, + 0x6E7, 0x6E8, 0x6EA, 0x6ED, 0x711, 0x711, 0x730, 0x74A, + 0x7A6, 0x7B0, 0x7EB, 0x7F3, 0x7FD, 0x7FD, 0x816, 0x819, + 0x81B, 0x823, 0x825, 0x827, 0x829, 0x82D, 0x859, 0x85B, + 0x897, 0x89F, 0x8CA, 0x8E1, 0x8E3, 0x902, 0x93A, 0x93A, + 0x93C, 0x93C, 0x941, 0x948, 0x951, 0x957, 0x962, 0x963, + 0x981, 0x981, 0x9BC, 0x9BC, 0x9BE, 0x9BE, 0x9C1, 0x9C4, + 0x9D7, 0x9D7, 0x9E2, 0x9E3, 0x9FE, 0x9FE, 0xA01, 0xA02, + 0xA3C, 0xA3C, 0xA41, 0xA42, 0xA47, 0xA48, 0xA4B, 0xA4D, + 0xA51, 0xA51, 0xA70, 0xA71, 0xA75, 0xA75, 0xA81, 0xA82, + 0xABC, 0xABC, 0xAC1, 0xAC5, 0xAC7, 0xAC8, 0xAE2, 0xAE3, + 0xAFA, 0xAFF, 0xB01, 0xB01, 0xB3C, 0xB3C, 0xB3E, 0xB3F, + 0xB41, 0xB44, 0xB55, 0xB57, 0xB62, 0xB63, 0xB82, 0xB82, + 0xBBE, 0xBBE, 0xBC0, 0xBC0, 0xBCD, 0xBCD, 0xBD7, 0xBD7, + 0xC00, 0xC00, 0xC04, 0xC04, 0xC3C, 0xC3C, 0xC3E, 0xC40, + 0xC46, 0xC48, 0xC4A, 0xC4C, 0xC55, 0xC56, 0xC62, 0xC63, + 0xC81, 0xC81, 0xCBC, 0xCBC, 0xCBF, 0xCC0, 0xCC2, 0xCC2, + 0xCC6, 0xCC8, 0xCCA, 0xCCD, 0xCD5, 0xCD6, 0xCE2, 0xCE3, + 0xD00, 0xD01, 0xD3B, 0xD3C, 0xD3E, 0xD3E, 0xD41, 0xD44, + 0xD57, 0xD57, 0xD62, 0xD63, 0xD81, 0xD81, 0xDCA, 0xDCA, + 0xDCF, 0xDCF, 0xDD2, 0xDD4, 0xDD6, 0xDD6, 0xDDF, 0xDDF, + 0xE31, 0xE31, 0xE34, 0xE3A, 0xE47, 0xE4E, 0xEB1, 0xEB1, + 0xEB4, 0xEBC, 0xEC8, 0xECE, 0xF18, 0xF19, 0xF35, 0xF35, + 0xF37, 0xF37, 0xF39, 0xF39, 0xF71, 0xF7E, 0xF80, 0xF84, + 0xF86, 0xF87, 0xF8D, 0xF97, 0xF99, 0xFBC, 0xFC6, 0xFC6, + 0x102D, 0x1030, 0x1032, 0x1037, 0x103A, 0x103A, 0x103D, 0x103E, + 0x1058, 0x1059, 0x105E, 0x1060, 0x1071, 0x1074, 0x1082, 0x1082, + 0x1085, 0x1086, 0x108D, 0x108D, 0x109D, 0x109D, 0x135D, 0x135F, + 0x1712, 0x1715, 0x1732, 0x1734, 0x1752, 0x1753, 0x1772, 0x1773, + 0x17B4, 0x17B5, 0x17B7, 0x17BD, 0x17C6, 0x17C6, 0x17C9, 0x17D1, + 0x17D3, 0x17D3, 0x17DD, 0x17DD, 0x180B, 0x180D, 0x180F, 0x180F, + 0x1885, 0x1886, 0x18A9, 0x18A9, 0x1920, 0x1922, 0x1927, 0x1928, + 0x1932, 0x1932, 0x1939, 0x193B, 0x1A17, 0x1A18, 0x1A1B, 0x1A1B, + 0x1A56, 0x1A56, 0x1A58, 0x1A5E, 0x1A62, 0x1A62, 0x1A65, 0x1A6C, + 0x1A73, 0x1A7C, 0x1A7F, 0x1A7F, 0x1AB0, 0x1ADD, 0x1AE0, 0x1AEB, + 0x1B00, 0x1B03, 0x1B34, 0x1B3D, 0x1B42, 0x1B43, 0x1B6B, 0x1B73, + 0x1B80, 0x1B81, 0x1BA2, 0x1BA5, 0x1BA8, 0x1BAA, 0x1BAC, 0x1BAD, + 0x1BE6, 0x1BE6, 0x1BE8, 0x1BE9, 0x1BED, 0x1BED, 0x1BEF, 0x1BF3, + 0x1C2C, 0x1C33, 0x1C36, 0x1C37, 0x1CD0, 0x1CD2, 0x1CD4, 0x1CE0, + 0x1CE2, 0x1CE8, 0x1CED, 0x1CED, 0x1CF4, 0x1CF4, 0x1CF8, 0x1CF9, + 0x1DC0, 0x1DFF, 0x200D, 0x200D, 0x20D0, 0x20F0, 0x2CEF, 0x2CF1, + 0x2D7F, 0x2D7F, 0x2DE0, 0x2DFF, 0x302A, 0x302F, 0x3099, 0x309A, + 0xA66F, 0xA672, 0xA674, 0xA67D, 0xA69E, 0xA69F, 0xA6F0, 0xA6F1, + 0xA802, 0xA802, 0xA806, 0xA806, 0xA80B, 0xA80B, 0xA825, 0xA826, + 0xA82C, 0xA82C, 0xA8C4, 0xA8C5, 0xA8E0, 0xA8F1, 0xA8FF, 0xA8FF, + 0xA926, 0xA92D, 0xA947, 0xA951, 0xA953, 0xA953, 0xA980, 0xA982, + 0xA9B3, 0xA9B3, 0xA9B6, 0xA9B9, 0xA9BC, 0xA9BD, 0xA9E5, 0xA9E5, + 0xAA29, 0xAA2E, 0xAA31, 0xAA32, 0xAA35, 0xAA36, 0xAA43, 0xAA43, + 0xAA4C, 0xAA4C, 0xAA7C, 0xAA7C, 0xAAB0, 0xAAB0, 0xAAB2, 0xAAB4, + 0xAAB7, 0xAAB8, 0xAABE, 0xAABF, 0xAAC1, 0xAAC1, 0xAAEC, 0xAAED, + 0xABE5, 0xABE5, 0xABE8, 0xABE8, 0xABED, 0xABED, 0xFB1E, 0xFB1E, + 0xFE00, 0xFE0F, 0xFE20, 0xFE2F, 0xFF9E, 0xFF9F, 0x101FD, 0x101FD, + 0x102E0, 0x102E0, 0x10376, 0x1037A, 0x10A01, 0x10A03, 0x10A05, 0x10A06, + 0x10A0C, 0x10A0F, 0x10A38, 0x10A3A, 0x10AE5, 0x10AE6, 0x10D24, 0x10D27, + 0x10D69, 0x10D6D, 0x10EAB, 0x10EAC, 0x10EFA, 0x10EFF, 0x10F46, 0x10F50, + 0x10F82, 0x10F85, 0x11001, 0x11001, 0x11038, 0x11046, 0x11070, 0x11070, + 0x11073, 0x11074, 0x1107F, 0x11081, 0x110B3, 0x110B6, 0x110B9, 0x110BA, + 0x110C2, 0x110C2, 0x11100, 0x11102, 0x11127, 0x1112B, 0x1112D, 0x11132, + 0x11134, 0x11134, 0x11173, 0x11173, 0x11180, 0x11181, 0x111B6, 0x111BE, + 0x111C0, 0x111C0, 0x111C9, 0x111CC, 0x111CF, 0x111CF, 0x1122F, 0x11231, + 0x11234, 0x11237, 0x1123E, 0x1123E, 0x11241, 0x11241, 0x112DF, 0x112DF, + 0x112E3, 0x112EA, 0x11300, 0x11301, 0x1133B, 0x1133C, 0x1133E, 0x1133E, + 0x11340, 0x11340, 0x1134D, 0x1134D, 0x11357, 0x11357, 0x11366, 0x1136C, + 0x11370, 0x11374, 0x113B8, 0x113B8, 0x113BB, 0x113C0, 0x113C2, 0x113C2, + 0x113C5, 0x113C5, 0x113C7, 0x113C9, 0x113CE, 0x113CF, 0x113D2, 0x113D2, + 0x113E1, 0x113E2, 0x11438, 0x1143F, 0x11442, 0x11444, 0x11446, 0x11446, + 0x1145E, 0x1145E, 0x114B0, 0x114B0, 0x114B3, 0x114B8, 0x114BA, 0x114BA, + 0x114BD, 0x114BD, 0x114BF, 0x114C0, 0x114C2, 0x114C3, 0x115AF, 0x115AF, + 0x115B2, 0x115B5, 0x115BC, 0x115BD, 0x115BF, 0x115C0, 0x115DC, 0x115DD, + 0x11633, 0x1163A, 0x1163D, 0x1163D, 0x1163F, 0x11640, 0x116AB, 0x116AB, + 0x116AD, 0x116AD, 0x116B0, 0x116B7, 0x1171D, 0x1171D, 0x1171F, 0x1171F, + 0x11722, 0x11725, 0x11727, 0x1172B, 0x1182F, 0x11837, 0x11839, 0x1183A, + 0x11930, 0x11930, 0x1193B, 0x1193D, 0x11943, 0x11943, 0x119D4, 0x119D7, + 0x119DA, 0x119DB, 0x119E0, 0x119E0, 0x11A01, 0x11A0A, 0x11A33, 0x11A38, + 0x11A3B, 0x11A3E, 0x11A51, 0x11A56, 0x11A59, 0x11A5B, 0x11A8A, 0x11A96, + 0x11A98, 0x11A98, 0x11B60, 0x11B60, 0x11B62, 0x11B64, 0x11B66, 0x11B66, + 0x11C30, 0x11C36, 0x11C38, 0x11C3D, 0x11C3F, 0x11C3F, 0x11C92, 0x11CA7, + 0x11CAA, 0x11CB0, 0x11CB2, 0x11CB3, 0x11CB5, 0x11CB6, 0x11D31, 0x11D36, + 0x11D3A, 0x11D3A, 0x11D3C, 0x11D3D, 0x11D3F, 0x11D45, 0x11D47, 0x11D47, + 0x11D90, 0x11D91, 0x11D95, 0x11D95, 0x11D97, 0x11D97, 0x11EF3, 0x11EF4, + 0x11F00, 0x11F01, 0x11F36, 0x11F3A, 0x11F40, 0x11F41, 0x11F5A, 0x11F5A, + 0x13440, 0x13440, 0x13447, 0x13455, 0x1611E, 0x16129, 0x1612D, 0x1612F, + 0x16AF0, 0x16AF4, 0x16B30, 0x16B36, 0x16F4F, 0x16F4F, 0x16F8F, 0x16F92, + 0x16FE4, 0x16FE4, 0x16FF0, 0x16FF1, 0x1BC9D, 0x1BC9E, 0x1CF00, 0x1CF2D, + 0x1CF30, 0x1CF46, 0x1D165, 0x1D169, 0x1D16D, 0x1D172, 0x1D17B, 0x1D182, + 0x1D185, 0x1D18B, 0x1D1AA, 0x1D1AD, 0x1D242, 0x1D244, 0x1DA00, 0x1DA36, + 0x1DA3B, 0x1DA6C, 0x1DA75, 0x1DA75, 0x1DA84, 0x1DA84, 0x1DA9B, 0x1DA9F, + 0x1DAA1, 0x1DAAF, 0x1E000, 0x1E006, 0x1E008, 0x1E018, 0x1E01B, 0x1E021, + 0x1E023, 0x1E024, 0x1E026, 0x1E02A, 0x1E08F, 0x1E08F, 0x1E130, 0x1E136, + 0x1E2AE, 0x1E2AE, 0x1E2EC, 0x1E2EF, 0x1E4EC, 0x1E4EF, 0x1E5EE, 0x1E5EF, + 0x1E6E3, 0x1E6E3, 0x1E6E6, 0x1E6E6, 0x1E6EE, 0x1E6EF, 0x1E6F5, 0x1E6F5, + 0x1E8D0, 0x1E8D6, 0x1E944, 0x1E94A, 0x1F3FB, 0x1F3FF, 0xE0020, 0xE007F, + 0xE0100, 0xE01EF, + }; + } + private static int[] InCB_ranges_2() { + return new int[] { + 0x94D, 0x94D, 0x9CD, 0x9CD, 0xACD, 0xACD, 0xB4D, 0xB4D, + 0xC4D, 0xC4D, 0xD4D, 0xD4D, 0x1039, 0x1039, 0x17D2, 0x17D2, + 0x1A60, 0x1A60, 0x1B44, 0x1B44, 0x1BAB, 0x1BAB, 0xA9C0, 0xA9C0, + 0xAAF6, 0xAAF6, 0x10A3F, 0x10A3F, 0x11133, 0x11133, 0x113D0, 0x113D0, + 0x1193E, 0x1193E, 0x11A47, 0x11A47, 0x11A99, 0x11A99, 0x11F42, 0x11F42, + }; + } + private static int[] InCB_ranges_3() { + return new int[] { + 0x0, 0x2FF, 0x370, 0x482, 0x48A, 0x590, 0x5BE, 0x5BE, + 0x5C0, 0x5C0, 0x5C3, 0x5C3, 0x5C6, 0x5C6, 0x5C8, 0x60F, + 0x61B, 0x64A, 0x660, 0x66F, 0x671, 0x6D5, 0x6DD, 0x6DE, + 0x6E5, 0x6E6, 0x6E9, 0x6E9, 0x6EE, 0x710, 0x712, 0x72F, + 0x74B, 0x7A5, 0x7B1, 0x7EA, 0x7F4, 0x7FC, 0x7FE, 0x815, + 0x81A, 0x81A, 0x824, 0x824, 0x828, 0x828, 0x82E, 0x858, + 0x85C, 0x896, 0x8A0, 0x8C9, 0x8E2, 0x8E2, 0x903, 0x914, + 0x93B, 0x93B, 0x93D, 0x940, 0x949, 0x94C, 0x94E, 0x950, + 0x960, 0x961, 0x964, 0x977, 0x980, 0x980, 0x982, 0x994, + 0x9A9, 0x9A9, 0x9B1, 0x9B1, 0x9B3, 0x9B5, 0x9BA, 0x9BB, + 0x9BD, 0x9BD, 0x9BF, 0x9C0, 0x9C5, 0x9CC, 0x9CE, 0x9D6, + 0x9D8, 0x9DB, 0x9DE, 0x9DE, 0x9E0, 0x9E1, 0x9E4, 0x9EF, + 0x9F2, 0x9FD, 0x9FF, 0xA00, 0xA03, 0xA3B, 0xA3D, 0xA40, + 0xA43, 0xA46, 0xA49, 0xA4A, 0xA4E, 0xA50, 0xA52, 0xA6F, + 0xA72, 0xA74, 0xA76, 0xA80, 0xA83, 0xA94, 0xAA9, 0xAA9, + 0xAB1, 0xAB1, 0xAB4, 0xAB4, 0xABA, 0xABB, 0xABD, 0xAC0, + 0xAC6, 0xAC6, 0xAC9, 0xACC, 0xACE, 0xAE1, 0xAE4, 0xAF8, + 0xB00, 0xB00, 0xB02, 0xB14, 0xB29, 0xB29, 0xB31, 0xB31, + 0xB34, 0xB34, 0xB3A, 0xB3B, 0xB3D, 0xB3D, 0xB40, 0xB40, + 0xB45, 0xB4C, 0xB4E, 0xB54, 0xB58, 0xB5B, 0xB5E, 0xB5E, + 0xB60, 0xB61, 0xB64, 0xB70, 0xB72, 0xB81, 0xB83, 0xBBD, + 0xBBF, 0xBBF, 0xBC1, 0xBCC, 0xBCE, 0xBD6, 0xBD8, 0xBFF, + 0xC01, 0xC03, 0xC05, 0xC14, 0xC29, 0xC29, 0xC3A, 0xC3B, + 0xC3D, 0xC3D, 0xC41, 0xC45, 0xC49, 0xC49, 0xC4E, 0xC54, + 0xC57, 0xC57, 0xC5B, 0xC61, 0xC64, 0xC80, 0xC82, 0xCBB, + 0xCBD, 0xCBE, 0xCC1, 0xCC1, 0xCC3, 0xCC5, 0xCC9, 0xCC9, + 0xCCE, 0xCD4, 0xCD7, 0xCE1, 0xCE4, 0xCFF, 0xD02, 0xD14, + 0xD3D, 0xD3D, 0xD3F, 0xD40, 0xD45, 0xD4C, 0xD4E, 0xD56, + 0xD58, 0xD61, 0xD64, 0xD80, 0xD82, 0xDC9, 0xDCB, 0xDCE, + 0xDD0, 0xDD1, 0xDD5, 0xDD5, 0xDD7, 0xDDE, 0xDE0, 0xE30, + 0xE32, 0xE33, 0xE3B, 0xE46, 0xE4F, 0xEB0, 0xEB2, 0xEB3, + 0xEBD, 0xEC7, 0xECF, 0xF17, 0xF1A, 0xF34, 0xF36, 0xF36, + 0xF38, 0xF38, 0xF3A, 0xF70, 0xF7F, 0xF7F, 0xF85, 0xF85, + 0xF88, 0xF8C, 0xF98, 0xF98, 0xFBD, 0xFC5, 0xFC7, 0xFFF, + 0x102B, 0x102C, 0x1031, 0x1031, 0x1038, 0x1038, 0x103B, 0x103C, + 0x1040, 0x104F, 0x1056, 0x1057, 0x1062, 0x1064, 0x1067, 0x106D, + 0x1083, 0x1084, 0x1087, 0x108C, 0x108F, 0x109C, 0x109E, 0x135C, + 0x1360, 0x1711, 0x1716, 0x1731, 0x1735, 0x1751, 0x1754, 0x1771, + 0x1774, 0x177F, 0x17B6, 0x17B6, 0x17BE, 0x17C5, 0x17C7, 0x17C8, + 0x17D4, 0x17DC, 0x17DE, 0x180A, 0x180E, 0x180E, 0x1810, 0x1884, + 0x1887, 0x18A8, 0x18AA, 0x191F, 0x1923, 0x1926, 0x1929, 0x1931, + 0x1933, 0x1938, 0x193C, 0x1A16, 0x1A19, 0x1A1A, 0x1A1C, 0x1A1F, + 0x1A55, 0x1A55, 0x1A57, 0x1A57, 0x1A5F, 0x1A5F, 0x1A61, 0x1A61, + 0x1A63, 0x1A64, 0x1A6D, 0x1A72, 0x1A7D, 0x1A7E, 0x1A80, 0x1AAF, + 0x1ADE, 0x1ADF, 0x1AEC, 0x1AFF, 0x1B04, 0x1B0A, 0x1B0D, 0x1B12, + 0x1B3E, 0x1B41, 0x1B4D, 0x1B6A, 0x1B74, 0x1B7F, 0x1B82, 0x1B82, + 0x1BA1, 0x1BA1, 0x1BA6, 0x1BA7, 0x1BB0, 0x1BBA, 0x1BBE, 0x1BE5, + 0x1BE7, 0x1BE7, 0x1BEA, 0x1BEC, 0x1BEE, 0x1BEE, 0x1BF4, 0x1C2B, + 0x1C34, 0x1C35, 0x1C38, 0x1CCF, 0x1CD3, 0x1CD3, 0x1CE1, 0x1CE1, + 0x1CE9, 0x1CEC, 0x1CEE, 0x1CF3, 0x1CF5, 0x1CF7, 0x1CFA, 0x1DBF, + 0x1E00, 0x200C, 0x200E, 0x20CF, 0x20F1, 0x2CEE, 0x2CF2, 0x2D7E, + 0x2D80, 0x2DDF, 0x2E00, 0x3029, 0x3030, 0x3098, 0x309B, 0xA66E, + 0xA673, 0xA673, 0xA67E, 0xA69D, 0xA6A0, 0xA6EF, 0xA6F2, 0xA801, + 0xA803, 0xA805, 0xA807, 0xA80A, 0xA80C, 0xA824, 0xA827, 0xA82B, + 0xA82D, 0xA8C3, 0xA8C6, 0xA8DF, 0xA8F2, 0xA8FE, 0xA900, 0xA925, + 0xA92E, 0xA946, 0xA952, 0xA952, 0xA954, 0xA97F, 0xA983, 0xA988, + 0xA98C, 0xA98E, 0xA9B4, 0xA9B5, 0xA9BA, 0xA9BB, 0xA9BE, 0xA9BF, + 0xA9C1, 0xA9DF, 0xA9E6, 0xA9E6, 0xA9F0, 0xA9F9, 0xA9FF, 0xAA28, + 0xAA2F, 0xAA30, 0xAA33, 0xAA34, 0xAA37, 0xAA42, 0xAA44, 0xAA4B, + 0xAA4D, 0xAA5F, 0xAA70, 0xAA70, 0xAA74, 0xAA79, 0xAA7B, 0xAA7B, + 0xAA7D, 0xAA7D, 0xAA80, 0xAAAF, 0xAAB1, 0xAAB1, 0xAAB5, 0xAAB6, + 0xAAB9, 0xAABD, 0xAAC0, 0xAAC0, 0xAAC2, 0xAADF, 0xAAEB, 0xAAEB, + 0xAAEE, 0xAAF5, 0xAAF7, 0xABBF, 0xABDB, 0xABE4, 0xABE6, 0xABE7, + 0xABE9, 0xABEC, 0xABEE, 0xFB1D, 0xFB1F, 0xFDFF, 0xFE10, 0xFE1F, + 0xFE30, 0xFF9D, 0xFFA0, 0x101FC, 0x101FE, 0x102DF, 0x102E1, 0x10375, + 0x1037B, 0x109FF, 0x10A04, 0x10A04, 0x10A07, 0x10A0B, 0x10A14, 0x10A14, + 0x10A18, 0x10A18, 0x10A36, 0x10A37, 0x10A3B, 0x10A3E, 0x10A40, 0x10AE4, + 0x10AE7, 0x10D23, 0x10D28, 0x10D68, 0x10D6E, 0x10EAA, 0x10EAD, 0x10EF9, + 0x10F00, 0x10F45, 0x10F51, 0x10F81, 0x10F86, 0x11000, 0x11002, 0x11037, + 0x11047, 0x1106F, 0x11071, 0x11072, 0x11075, 0x1107E, 0x11082, 0x110B2, + 0x110B7, 0x110B8, 0x110BB, 0x110C1, 0x110C3, 0x110FF, 0x1112C, 0x1112C, + 0x11135, 0x11143, 0x11145, 0x11146, 0x11148, 0x11172, 0x11174, 0x1117F, + 0x11182, 0x111B5, 0x111BF, 0x111BF, 0x111C1, 0x111C8, 0x111CD, 0x111CE, + 0x111D0, 0x1122E, 0x11232, 0x11233, 0x11238, 0x1123D, 0x1123F, 0x11240, + 0x11242, 0x112DE, 0x112E0, 0x112E2, 0x112EB, 0x112FF, 0x11302, 0x1133A, + 0x1133D, 0x1133D, 0x1133F, 0x1133F, 0x11341, 0x1134C, 0x1134E, 0x11356, + 0x11358, 0x11365, 0x1136D, 0x1136F, 0x11375, 0x1137F, 0x1138A, 0x1138A, + 0x1138C, 0x1138D, 0x1138F, 0x1138F, 0x113B6, 0x113B7, 0x113B9, 0x113BA, + 0x113C1, 0x113C1, 0x113C3, 0x113C4, 0x113C6, 0x113C6, 0x113CA, 0x113CD, + 0x113D1, 0x113D1, 0x113D3, 0x113E0, 0x113E3, 0x11437, 0x11440, 0x11441, + 0x11445, 0x11445, 0x11447, 0x1145D, 0x1145F, 0x114AF, 0x114B1, 0x114B2, + 0x114B9, 0x114B9, 0x114BB, 0x114BC, 0x114BE, 0x114BE, 0x114C1, 0x114C1, + 0x114C4, 0x115AE, 0x115B0, 0x115B1, 0x115B6, 0x115BB, 0x115BE, 0x115BE, + 0x115C1, 0x115DB, 0x115DE, 0x11632, 0x1163B, 0x1163C, 0x1163E, 0x1163E, + 0x11641, 0x116AA, 0x116AC, 0x116AC, 0x116AE, 0x116AF, 0x116B8, 0x1171C, + 0x1171E, 0x1171E, 0x11720, 0x11721, 0x11726, 0x11726, 0x1172C, 0x1182E, + 0x11838, 0x11838, 0x1183B, 0x118FF, 0x11907, 0x11908, 0x1190A, 0x1190B, + 0x11914, 0x11914, 0x11917, 0x11917, 0x11931, 0x1193A, 0x1193F, 0x11942, + 0x11944, 0x119D3, 0x119D8, 0x119D9, 0x119DC, 0x119DF, 0x119E1, 0x119FF, + 0x11A39, 0x11A3A, 0x11A3F, 0x11A46, 0x11A48, 0x11A4F, 0x11A57, 0x11A58, + 0x11A84, 0x11A89, 0x11A97, 0x11A97, 0x11A9A, 0x11B5F, 0x11B61, 0x11B61, + 0x11B65, 0x11B65, 0x11B67, 0x11C2F, 0x11C37, 0x11C37, 0x11C3E, 0x11C3E, + 0x11C40, 0x11C91, 0x11CA8, 0x11CA9, 0x11CB1, 0x11CB1, 0x11CB4, 0x11CB4, + 0x11CB7, 0x11D30, 0x11D37, 0x11D39, 0x11D3B, 0x11D3B, 0x11D3E, 0x11D3E, + 0x11D46, 0x11D46, 0x11D48, 0x11D8F, 0x11D92, 0x11D94, 0x11D96, 0x11D96, + 0x11D98, 0x11EF2, 0x11EF5, 0x11EFF, 0x11F02, 0x11F03, 0x11F11, 0x11F11, + 0x11F34, 0x11F35, 0x11F3B, 0x11F3F, 0x11F43, 0x11F59, 0x11F5B, 0x1343F, + 0x13441, 0x13446, 0x13456, 0x1611D, 0x1612A, 0x1612C, 0x16130, 0x16AEF, + 0x16AF5, 0x16B2F, 0x16B37, 0x16F4E, 0x16F50, 0x16F8E, 0x16F93, 0x16FE3, + 0x16FE5, 0x16FEF, 0x16FF2, 0x1BC9C, 0x1BC9F, 0x1CEFF, 0x1CF2E, 0x1CF2F, + 0x1CF47, 0x1D164, 0x1D16A, 0x1D16C, 0x1D173, 0x1D17A, 0x1D183, 0x1D184, + 0x1D18C, 0x1D1A9, 0x1D1AE, 0x1D241, 0x1D245, 0x1D9FF, 0x1DA37, 0x1DA3A, + 0x1DA6D, 0x1DA74, 0x1DA76, 0x1DA83, 0x1DA85, 0x1DA9A, 0x1DAA0, 0x1DAA0, + 0x1DAB0, 0x1DFFF, 0x1E007, 0x1E007, 0x1E019, 0x1E01A, 0x1E022, 0x1E022, + 0x1E025, 0x1E025, 0x1E02B, 0x1E08E, 0x1E090, 0x1E12F, 0x1E137, 0x1E2AD, + 0x1E2AF, 0x1E2EB, 0x1E2F0, 0x1E4EB, 0x1E4F0, 0x1E5ED, 0x1E5F0, 0x1E6E2, + 0x1E6E4, 0x1E6E5, 0x1E6E7, 0x1E6ED, 0x1E6F0, 0x1E6F4, 0x1E6F6, 0x1E8CF, + 0x1E8D7, 0x1E943, 0x1E94B, 0x1F3FA, 0x1F400, 0xE001F, 0xE0080, 0xE00FF, + 0xE01F0, 0x10FFFF, + }; + } + private static final int[][] InCB_RANGES = { + InCB_ranges_0(), InCB_ranges_1(), InCB_ranges_2(), InCB_ranges_3() + }; + private static final Property InCB = new Property( + InCB_VALUES, InCB_SHORT_VALUES, + InCB_VALUE_ALIASES, InCB_VALUE_ALIAS_INDEX, + InCB_PROPERTY_ALIASES, InCB_RANGES); + + private static final String[] IDTYPE_VALUES = { + "Not_Character", "Deprecated", "Default_Ignorable", "Not_NFKC", "Not_XID", "Exclusion", "Obsolete", "Technical", "Uncommon_Use", "Limited_Use", "Inclusion", "Recommended" + }; + private static final String[] IDTYPE_SHORT_VALUES = { + "Not_Character", "Deprecated", "Default_Ignorable", "Not_NFKC", "Not_XID", "Exclusion", "Obsolete", "Technical", "Uncommon_Use", "Limited_Use", "Inclusion", "Recommended" + }; + private static final String[] IDTYPE_VALUE_ALIASES = { + "defaultignorable", "deprecated", "exclusion", "inclusion", "limiteduse", "notcharacter", "notnfkc", "notxid", "obsolete", "recommended", "technical", "uncommonuse" + }; + private static final byte[] IDTYPE_VALUE_ALIAS_INDEX = { + 2, 1, 5, 10, 9, 0, 3, 4, 6, 11, 7, 8 + }; + private static final String[] IDTYPE_PROPERTY_ALIASES = { + "identifiertype", "idtype" + }; + private static int[] IDTYPE_ranges_0() { + return new int[] { + 0x0, 0x8, 0xE, 0x1F, 0x7F, 0x84, 0x86, 0x9F, + 0x378, 0x379, 0x380, 0x383, 0x38B, 0x38B, 0x38D, 0x38D, + 0x3A2, 0x3A2, 0x530, 0x530, 0x557, 0x558, 0x58B, 0x58C, + 0x590, 0x590, 0x5C8, 0x5CF, 0x5EB, 0x5EE, 0x5F5, 0x5FF, + 0x70E, 0x70E, 0x74B, 0x74C, 0x7B2, 0x7BF, 0x7FB, 0x7FC, + 0x82E, 0x82F, 0x83F, 0x83F, 0x85C, 0x85D, 0x85F, 0x85F, + 0x86B, 0x86F, 0x892, 0x896, 0x984, 0x984, 0x98D, 0x98E, + 0x991, 0x992, 0x9A9, 0x9A9, 0x9B1, 0x9B1, 0x9B3, 0x9B5, + 0x9BA, 0x9BB, 0x9C5, 0x9C6, 0x9C9, 0x9CA, 0x9CF, 0x9D6, + 0x9D8, 0x9DB, 0x9DE, 0x9DE, 0x9E4, 0x9E5, 0x9FF, 0xA00, + 0xA04, 0xA04, 0xA0B, 0xA0E, 0xA11, 0xA12, 0xA29, 0xA29, + 0xA31, 0xA31, 0xA34, 0xA34, 0xA37, 0xA37, 0xA3A, 0xA3B, + 0xA3D, 0xA3D, 0xA43, 0xA46, 0xA49, 0xA4A, 0xA4E, 0xA50, + 0xA52, 0xA58, 0xA5D, 0xA5D, 0xA5F, 0xA65, 0xA77, 0xA80, + 0xA84, 0xA84, 0xA8E, 0xA8E, 0xA92, 0xA92, 0xAA9, 0xAA9, + 0xAB1, 0xAB1, 0xAB4, 0xAB4, 0xABA, 0xABB, 0xAC6, 0xAC6, + 0xACA, 0xACA, 0xACE, 0xACF, 0xAD1, 0xADF, 0xAE4, 0xAE5, + 0xAF2, 0xAF8, 0xB00, 0xB00, 0xB04, 0xB04, 0xB0D, 0xB0E, + 0xB11, 0xB12, 0xB29, 0xB29, 0xB31, 0xB31, 0xB34, 0xB34, + 0xB3A, 0xB3B, 0xB45, 0xB46, 0xB49, 0xB4A, 0xB4E, 0xB54, + 0xB58, 0xB5B, 0xB5E, 0xB5E, 0xB64, 0xB65, 0xB78, 0xB81, + 0xB84, 0xB84, 0xB8B, 0xB8D, 0xB91, 0xB91, 0xB96, 0xB98, + 0xB9B, 0xB9B, 0xB9D, 0xB9D, 0xBA0, 0xBA2, 0xBA5, 0xBA7, + 0xBAB, 0xBAD, 0xBBA, 0xBBD, 0xBC3, 0xBC5, 0xBC9, 0xBC9, + 0xBCE, 0xBCF, 0xBD1, 0xBD6, 0xBD8, 0xBE5, 0xBFB, 0xBFF, + 0xC0D, 0xC0D, 0xC11, 0xC11, 0xC29, 0xC29, 0xC3A, 0xC3B, + 0xC45, 0xC45, 0xC49, 0xC49, 0xC4E, 0xC54, 0xC57, 0xC57, + 0xC5B, 0xC5B, 0xC5E, 0xC5F, 0xC64, 0xC65, 0xC70, 0xC76, + 0xC8D, 0xC8D, 0xC91, 0xC91, 0xCA9, 0xCA9, 0xCB4, 0xCB4, + 0xCBA, 0xCBB, 0xCC5, 0xCC5, 0xCC9, 0xCC9, 0xCCE, 0xCD4, + 0xCD7, 0xCDB, 0xCDF, 0xCDF, 0xCE4, 0xCE5, 0xCF0, 0xCF0, + 0xCF4, 0xCFF, 0xD0D, 0xD0D, 0xD11, 0xD11, 0xD45, 0xD45, + 0xD49, 0xD49, 0xD50, 0xD53, 0xD64, 0xD65, 0xD80, 0xD80, + 0xD84, 0xD84, 0xD97, 0xD99, 0xDB2, 0xDB2, 0xDBC, 0xDBC, + 0xDBE, 0xDBF, 0xDC7, 0xDC9, 0xDCB, 0xDCE, 0xDD5, 0xDD5, + 0xDD7, 0xDD7, 0xDE0, 0xDE5, 0xDF0, 0xDF1, 0xDF5, 0xE00, + 0xE3B, 0xE3E, 0xE5C, 0xE80, 0xE83, 0xE83, 0xE85, 0xE85, + 0xE8B, 0xE8B, 0xEA4, 0xEA4, 0xEA6, 0xEA6, 0xEBE, 0xEBF, + 0xEC5, 0xEC5, 0xEC7, 0xEC7, 0xECF, 0xECF, 0xEDA, 0xEDB, + 0xEE0, 0xEFF, 0xF48, 0xF48, 0xF6D, 0xF70, 0xF98, 0xF98, + 0xFBD, 0xFBD, 0xFCD, 0xFCD, 0xFDB, 0xFFF, 0x10C6, 0x10C6, + 0x10C8, 0x10CC, 0x10CE, 0x10CF, 0x1249, 0x1249, 0x124E, 0x124F, + 0x1257, 0x1257, 0x1259, 0x1259, 0x125E, 0x125F, 0x1289, 0x1289, + 0x128E, 0x128F, 0x12B1, 0x12B1, 0x12B6, 0x12B7, 0x12BF, 0x12BF, + 0x12C1, 0x12C1, 0x12C6, 0x12C7, 0x12D7, 0x12D7, 0x1311, 0x1311, + 0x1316, 0x1317, 0x135B, 0x135C, 0x137D, 0x137F, 0x139A, 0x139F, + 0x13F6, 0x13F7, 0x13FE, 0x13FF, 0x169D, 0x169F, 0x16F9, 0x16FF, + 0x1716, 0x171E, 0x1737, 0x173F, 0x1754, 0x175F, 0x176D, 0x176D, + 0x1771, 0x1771, 0x1774, 0x177F, 0x17DE, 0x17DF, 0x17EA, 0x17EF, + 0x17FA, 0x17FF, 0x181A, 0x181F, 0x1879, 0x187F, 0x18AB, 0x18AF, + 0x18F6, 0x18FF, 0x191F, 0x191F, 0x192C, 0x192F, 0x193C, 0x193F, + 0x1941, 0x1943, 0x196E, 0x196F, 0x1975, 0x197F, 0x19AC, 0x19AF, + 0x19CA, 0x19CF, 0x19DB, 0x19DD, 0x1A1C, 0x1A1D, 0x1A5F, 0x1A5F, + 0x1A7D, 0x1A7E, 0x1A8A, 0x1A8F, 0x1A9A, 0x1A9F, 0x1AAE, 0x1AAF, + 0x1ADE, 0x1ADF, 0x1AEC, 0x1AFF, 0x1B4D, 0x1B4D, 0x1BF4, 0x1BFB, + 0x1C38, 0x1C3A, 0x1C4A, 0x1C4C, 0x1C8B, 0x1C8F, 0x1CBB, 0x1CBC, + 0x1CC8, 0x1CCF, 0x1CFB, 0x1CFF, 0x1F16, 0x1F17, 0x1F1E, 0x1F1F, + 0x1F46, 0x1F47, 0x1F4E, 0x1F4F, 0x1F58, 0x1F58, 0x1F5A, 0x1F5A, + 0x1F5C, 0x1F5C, 0x1F5E, 0x1F5E, 0x1F7E, 0x1F7F, 0x1FB5, 0x1FB5, + 0x1FC5, 0x1FC5, 0x1FD4, 0x1FD5, 0x1FDC, 0x1FDC, 0x1FF0, 0x1FF1, + 0x1FF5, 0x1FF5, 0x1FFF, 0x1FFF, 0x2065, 0x2065, 0x2072, 0x2073, + 0x208F, 0x208F, 0x209D, 0x209F, 0x20C2, 0x20CF, 0x20F1, 0x20FF, + 0x218C, 0x218F, 0x242A, 0x243F, 0x244B, 0x245F, 0x2B74, 0x2B75, + 0x2CF4, 0x2CF8, 0x2D26, 0x2D26, 0x2D28, 0x2D2C, 0x2D2E, 0x2D2F, + 0x2D68, 0x2D6E, 0x2D71, 0x2D7E, 0x2D97, 0x2D9F, 0x2DA7, 0x2DA7, + 0x2DAF, 0x2DAF, 0x2DB7, 0x2DB7, 0x2DBF, 0x2DBF, 0x2DC7, 0x2DC7, + 0x2DCF, 0x2DCF, 0x2DD7, 0x2DD7, 0x2DDF, 0x2DDF, 0x2E5E, 0x2E7F, + 0x2E9A, 0x2E9A, 0x2EF4, 0x2EFF, 0x2FD6, 0x2FEF, 0x3040, 0x3040, + 0x3097, 0x3098, 0x3100, 0x3104, 0x3130, 0x3130, 0x318F, 0x318F, + 0x31E6, 0x31EE, 0x321F, 0x321F, 0xA48D, 0xA48F, 0xA4C7, 0xA4CF, + 0xA62C, 0xA63F, 0xA6F8, 0xA6FF, 0xA7DD, 0xA7F0, 0xA82D, 0xA82F, + 0xA83A, 0xA83F, 0xA878, 0xA87F, 0xA8C6, 0xA8CD, 0xA8DA, 0xA8DF, + 0xA954, 0xA95E, 0xA97D, 0xA97F, 0xA9CE, 0xA9CE, 0xA9DA, 0xA9DD, + 0xA9FF, 0xA9FF, 0xAA37, 0xAA3F, 0xAA4E, 0xAA4F, 0xAA5A, 0xAA5B, + 0xAAC3, 0xAADA, 0xAAF7, 0xAB00, 0xAB07, 0xAB08, 0xAB0F, 0xAB10, + 0xAB17, 0xAB1F, 0xAB27, 0xAB27, 0xAB2F, 0xAB2F, 0xAB6C, 0xAB6F, + 0xABEE, 0xABEF, 0xABFA, 0xABFF, 0xD7A4, 0xD7AF, 0xD7C7, 0xD7CA, + 0xD7FC, 0xF8FF, 0xFA6E, 0xFA6F, 0xFADA, 0xFAFF, 0xFB07, 0xFB12, + 0xFB18, 0xFB1C, 0xFB37, 0xFB37, 0xFB3D, 0xFB3D, 0xFB3F, 0xFB3F, + 0xFB42, 0xFB42, 0xFB45, 0xFB45, 0xFDD0, 0xFDEF, 0xFE1A, 0xFE1F, + 0xFE53, 0xFE53, 0xFE67, 0xFE67, 0xFE6C, 0xFE6F, 0xFE75, 0xFE75, + 0xFEFD, 0xFEFE, 0xFF00, 0xFF00, 0xFFBF, 0xFFC1, 0xFFC8, 0xFFC9, + 0xFFD0, 0xFFD1, 0xFFD8, 0xFFD9, 0xFFDD, 0xFFDF, 0xFFE7, 0xFFE7, + 0xFFEF, 0xFFF8, 0xFFFE, 0xFFFF, 0x1000C, 0x1000C, 0x10027, 0x10027, + 0x1003B, 0x1003B, 0x1003E, 0x1003E, 0x1004E, 0x1004F, 0x1005E, 0x1007F, + 0x100FB, 0x100FF, 0x10103, 0x10106, 0x10134, 0x10136, 0x1018F, 0x1018F, + 0x1019D, 0x1019F, 0x101A1, 0x101CF, 0x101FE, 0x1027F, 0x1029D, 0x1029F, + 0x102D1, 0x102DF, 0x102FC, 0x102FF, 0x10324, 0x1032C, 0x1034B, 0x1034F, + 0x1037B, 0x1037F, 0x1039E, 0x1039E, 0x103C4, 0x103C7, 0x103D6, 0x103FF, + 0x1049E, 0x1049F, 0x104AA, 0x104AF, 0x104D4, 0x104D7, 0x104FC, 0x104FF, + 0x10528, 0x1052F, 0x10564, 0x1056E, 0x1057B, 0x1057B, 0x1058B, 0x1058B, + 0x10593, 0x10593, 0x10596, 0x10596, 0x105A2, 0x105A2, 0x105B2, 0x105B2, + 0x105BA, 0x105BA, 0x105BD, 0x105BF, 0x105F4, 0x105FF, 0x10737, 0x1073F, + 0x10756, 0x1075F, 0x10768, 0x1077F, 0x10786, 0x10786, 0x107B1, 0x107B1, + 0x107BB, 0x107FF, 0x10806, 0x10807, 0x10809, 0x10809, 0x10836, 0x10836, + 0x10839, 0x1083B, 0x1083D, 0x1083E, 0x10856, 0x10856, 0x1089F, 0x108A6, + 0x108B0, 0x108DF, 0x108F3, 0x108F3, 0x108F6, 0x108FA, 0x1091C, 0x1091E, + 0x1093A, 0x1093E, 0x1095A, 0x1097F, 0x109B8, 0x109BB, 0x109D0, 0x109D1, + 0x10A04, 0x10A04, 0x10A07, 0x10A0B, 0x10A14, 0x10A14, 0x10A18, 0x10A18, + 0x10A36, 0x10A37, 0x10A3B, 0x10A3E, 0x10A49, 0x10A4F, 0x10A59, 0x10A5F, + 0x10AA0, 0x10ABF, 0x10AE7, 0x10AEA, 0x10AF7, 0x10AFF, 0x10B36, 0x10B38, + 0x10B56, 0x10B57, 0x10B73, 0x10B77, 0x10B92, 0x10B98, 0x10B9D, 0x10BA8, + 0x10BB0, 0x10BFF, 0x10C49, 0x10C7F, 0x10CB3, 0x10CBF, 0x10CF3, 0x10CF9, + 0x10D28, 0x10D2F, 0x10D3A, 0x10D3F, 0x10D66, 0x10D68, 0x10D86, 0x10D8D, + 0x10D90, 0x10E5F, 0x10E7F, 0x10E7F, 0x10EAA, 0x10EAA, 0x10EAE, 0x10EAF, + 0x10EB2, 0x10EC1, 0x10EC8, 0x10ECF, 0x10ED9, 0x10EF9, 0x10F28, 0x10F2F, + 0x10F5A, 0x10F6F, 0x10F8A, 0x10FAF, 0x10FCC, 0x10FDF, 0x10FF7, 0x10FFF, + 0x1104E, 0x11051, 0x11076, 0x1107E, 0x110C3, 0x110CC, 0x110CE, 0x110CF, + 0x110E9, 0x110EF, 0x110FA, 0x110FF, 0x11135, 0x11135, 0x11148, 0x1114F, + 0x11177, 0x1117F, 0x111E0, 0x111E0, 0x111F5, 0x111FF, 0x11212, 0x11212, + 0x11242, 0x1127F, 0x11287, 0x11287, 0x11289, 0x11289, 0x1128E, 0x1128E, + 0x1129E, 0x1129E, 0x112AA, 0x112AF, 0x112EB, 0x112EF, 0x112FA, 0x112FF, + 0x11304, 0x11304, 0x1130D, 0x1130E, 0x11311, 0x11312, 0x11329, 0x11329, + 0x11331, 0x11331, 0x11334, 0x11334, 0x1133A, 0x1133A, 0x11345, 0x11346, + 0x11349, 0x1134A, 0x1134E, 0x1134F, 0x11351, 0x11356, 0x11358, 0x1135C, + 0x11364, 0x11365, 0x1136D, 0x1136F, 0x11375, 0x1137F, 0x1138A, 0x1138A, + 0x1138C, 0x1138D, 0x1138F, 0x1138F, 0x113B6, 0x113B6, 0x113C1, 0x113C1, + 0x113C3, 0x113C4, 0x113C6, 0x113C6, 0x113CB, 0x113CB, 0x113D6, 0x113D6, + 0x113D9, 0x113E0, 0x113E3, 0x113FF, 0x1145C, 0x1145C, 0x11462, 0x1147F, + 0x114C8, 0x114CF, 0x114DA, 0x1157F, 0x115B6, 0x115B7, 0x115DE, 0x115FF, + 0x11645, 0x1164F, 0x1165A, 0x1165F, 0x1166D, 0x1167F, 0x116BA, 0x116BF, + 0x116CA, 0x116CF, 0x116E4, 0x116FF, 0x1171B, 0x1171C, 0x1172C, 0x1172F, + 0x11747, 0x117FF, 0x1183C, 0x1189F, 0x118F3, 0x118FE, 0x11907, 0x11908, + 0x1190A, 0x1190B, 0x11914, 0x11914, 0x11917, 0x11917, 0x11936, 0x11936, + 0x11939, 0x1193A, 0x11947, 0x1194F, 0x1195A, 0x1199F, 0x119A8, 0x119A9, + 0x119D8, 0x119D9, 0x119E5, 0x119FF, 0x11A48, 0x11A4F, 0x11AA3, 0x11AAF, + 0x11AF9, 0x11AFF, 0x11B0A, 0x11B5F, 0x11B68, 0x11BBF, 0x11BE2, 0x11BEF, + 0x11BFA, 0x11BFF, 0x11C09, 0x11C09, 0x11C37, 0x11C37, 0x11C46, 0x11C4F, + 0x11C6D, 0x11C6F, 0x11C90, 0x11C91, 0x11CA8, 0x11CA8, 0x11CB7, 0x11CFF, + 0x11D07, 0x11D07, 0x11D0A, 0x11D0A, 0x11D37, 0x11D39, 0x11D3B, 0x11D3B, + 0x11D3E, 0x11D3E, 0x11D48, 0x11D4F, 0x11D5A, 0x11D5F, 0x11D66, 0x11D66, + 0x11D69, 0x11D69, 0x11D8F, 0x11D8F, 0x11D92, 0x11D92, 0x11D99, 0x11D9F, + 0x11DAA, 0x11DAF, 0x11DDC, 0x11DDF, 0x11DEA, 0x11EDF, 0x11EF9, 0x11EFF, + 0x11F11, 0x11F11, 0x11F3B, 0x11F3D, 0x11F5B, 0x11FAF, 0x11FB1, 0x11FBF, + 0x11FF2, 0x11FFE, 0x1239A, 0x123FF, 0x1246F, 0x1246F, 0x12475, 0x1247F, + 0x12544, 0x12F8F, 0x12FF3, 0x12FFF, 0x13456, 0x1345F, 0x143FB, 0x143FF, + 0x14647, 0x160FF, 0x1613A, 0x167FF, 0x16A39, 0x16A3F, 0x16A5F, 0x16A5F, + 0x16A6A, 0x16A6D, 0x16ABF, 0x16ABF, 0x16ACA, 0x16ACF, 0x16AEE, 0x16AEF, + 0x16AF6, 0x16AFF, 0x16B46, 0x16B4F, 0x16B5A, 0x16B5A, 0x16B62, 0x16B62, + 0x16B78, 0x16B7C, 0x16B90, 0x16D3F, 0x16D7A, 0x16E3F, 0x16E9B, 0x16E9F, + 0x16EB9, 0x16EBA, 0x16ED4, 0x16EFF, 0x16F4B, 0x16F4E, 0x16F88, 0x16F8E, + 0x16FA0, 0x16FDF, 0x16FE5, 0x16FEF, 0x16FF7, 0x16FFF, 0x18CD6, 0x18CFE, + 0x18D1F, 0x18D7F, 0x18DF3, 0x1AFEF, 0x1AFF4, 0x1AFF4, 0x1AFFC, 0x1AFFC, + 0x1AFFF, 0x1AFFF, 0x1B123, 0x1B131, 0x1B133, 0x1B14F, 0x1B153, 0x1B154, + 0x1B156, 0x1B163, 0x1B168, 0x1B16F, 0x1B2FC, 0x1BBFF, 0x1BC6B, 0x1BC6F, + 0x1BC7D, 0x1BC7F, 0x1BC89, 0x1BC8F, 0x1BC9A, 0x1BC9B, 0x1BCA4, 0x1CBFF, + 0x1CCFD, 0x1CCFF, 0x1CEB4, 0x1CEB9, 0x1CED1, 0x1CEDF, 0x1CEF1, 0x1CEFF, + 0x1CF2E, 0x1CF2F, 0x1CF47, 0x1CF4F, 0x1CFC4, 0x1CFFF, 0x1D0F6, 0x1D0FF, + 0x1D127, 0x1D128, 0x1D1EB, 0x1D1FF, 0x1D246, 0x1D2BF, 0x1D2D4, 0x1D2DF, + 0x1D2F4, 0x1D2FF, 0x1D357, 0x1D35F, 0x1D379, 0x1D3FF, 0x1D455, 0x1D455, + 0x1D49D, 0x1D49D, 0x1D4A0, 0x1D4A1, 0x1D4A3, 0x1D4A4, 0x1D4A7, 0x1D4A8, + 0x1D4AD, 0x1D4AD, 0x1D4BA, 0x1D4BA, 0x1D4BC, 0x1D4BC, 0x1D4C4, 0x1D4C4, + 0x1D506, 0x1D506, 0x1D50B, 0x1D50C, 0x1D515, 0x1D515, 0x1D51D, 0x1D51D, + 0x1D53A, 0x1D53A, 0x1D53F, 0x1D53F, 0x1D545, 0x1D545, 0x1D547, 0x1D549, + 0x1D551, 0x1D551, 0x1D6A6, 0x1D6A7, 0x1D7CC, 0x1D7CD, 0x1DA8C, 0x1DA9A, + 0x1DAA0, 0x1DAA0, 0x1DAB0, 0x1DEFF, 0x1DF1F, 0x1DF24, 0x1DF2B, 0x1DFFF, + 0x1E007, 0x1E007, 0x1E019, 0x1E01A, 0x1E022, 0x1E022, 0x1E025, 0x1E025, + 0x1E02B, 0x1E02F, 0x1E06E, 0x1E08E, 0x1E090, 0x1E0FF, 0x1E12D, 0x1E12F, + 0x1E13E, 0x1E13F, 0x1E14A, 0x1E14D, 0x1E150, 0x1E28F, 0x1E2AF, 0x1E2BF, + 0x1E2FA, 0x1E2FE, 0x1E300, 0x1E4CF, 0x1E4FA, 0x1E5CF, 0x1E5FB, 0x1E5FE, + 0x1E600, 0x1E6BF, 0x1E6DF, 0x1E6DF, 0x1E6F6, 0x1E6FD, 0x1E700, 0x1E7DF, + 0x1E7E7, 0x1E7E7, 0x1E7EC, 0x1E7EC, 0x1E7EF, 0x1E7EF, 0x1E7FF, 0x1E7FF, + 0x1E8C5, 0x1E8C6, 0x1E8D7, 0x1E8FF, 0x1E94C, 0x1E94F, 0x1E95A, 0x1E95D, + 0x1E960, 0x1EC70, 0x1ECB5, 0x1ED00, 0x1ED3E, 0x1EDFF, 0x1EE04, 0x1EE04, + 0x1EE20, 0x1EE20, 0x1EE23, 0x1EE23, 0x1EE25, 0x1EE26, 0x1EE28, 0x1EE28, + 0x1EE33, 0x1EE33, 0x1EE38, 0x1EE38, 0x1EE3A, 0x1EE3A, 0x1EE3C, 0x1EE41, + 0x1EE43, 0x1EE46, 0x1EE48, 0x1EE48, 0x1EE4A, 0x1EE4A, 0x1EE4C, 0x1EE4C, + 0x1EE50, 0x1EE50, 0x1EE53, 0x1EE53, 0x1EE55, 0x1EE56, 0x1EE58, 0x1EE58, + 0x1EE5A, 0x1EE5A, 0x1EE5C, 0x1EE5C, 0x1EE5E, 0x1EE5E, 0x1EE60, 0x1EE60, + 0x1EE63, 0x1EE63, 0x1EE65, 0x1EE66, 0x1EE6B, 0x1EE6B, 0x1EE73, 0x1EE73, + 0x1EE78, 0x1EE78, 0x1EE7D, 0x1EE7D, 0x1EE7F, 0x1EE7F, 0x1EE8A, 0x1EE8A, + 0x1EE9C, 0x1EEA0, 0x1EEA4, 0x1EEA4, 0x1EEAA, 0x1EEAA, 0x1EEBC, 0x1EEEF, + 0x1EEF2, 0x1EFFF, 0x1F02C, 0x1F02F, 0x1F094, 0x1F09F, 0x1F0AF, 0x1F0B0, + 0x1F0C0, 0x1F0C0, 0x1F0D0, 0x1F0D0, 0x1F0F6, 0x1F0FF, 0x1F1AE, 0x1F1E5, + 0x1F203, 0x1F20F, 0x1F23C, 0x1F23F, 0x1F249, 0x1F24F, 0x1F252, 0x1F25F, + 0x1F266, 0x1F2FF, 0x1F6D9, 0x1F6DB, 0x1F6ED, 0x1F6EF, 0x1F6FD, 0x1F6FF, + 0x1F7DA, 0x1F7DF, 0x1F7EC, 0x1F7EF, 0x1F7F1, 0x1F7FF, 0x1F80C, 0x1F80F, + 0x1F848, 0x1F84F, 0x1F85A, 0x1F85F, 0x1F888, 0x1F88F, 0x1F8AE, 0x1F8AF, + 0x1F8BC, 0x1F8BF, 0x1F8C2, 0x1F8CF, 0x1F8D9, 0x1F8FF, 0x1FA58, 0x1FA5F, + 0x1FA6E, 0x1FA6F, 0x1FA7D, 0x1FA7F, 0x1FA8B, 0x1FA8D, 0x1FAC7, 0x1FAC7, + 0x1FAC9, 0x1FACC, 0x1FADD, 0x1FADE, 0x1FAEB, 0x1FAEE, 0x1FAF9, 0x1FAFF, + 0x1FB93, 0x1FB93, 0x1FBFB, 0x1FFFF, 0x2A6E0, 0x2A6FF, 0x2B81E, 0x2B81F, + 0x2CEAE, 0x2CEAF, 0x2EBE1, 0x2EBEF, 0x2EE5E, 0x2F7FF, 0x2FA1E, 0x2FFFF, + 0x3134B, 0x3134F, 0x3347A, 0xE0000, 0xE0002, 0xE001F, 0xE0080, 0xE00FF, + 0xE01F0, 0x10FFFF, + }; + } + private static int[] IDTYPE_ranges_1() { + return new int[] { + 0x149, 0x149, 0x673, 0x673, 0xF77, 0xF77, 0xF79, 0xF79, + 0x17A3, 0x17A4, 0x206A, 0x206F, 0x2329, 0x232A, 0xE0001, 0xE0001, + }; + } + private static int[] IDTYPE_ranges_2() { + return new int[] { + 0xAD, 0xAD, 0x34F, 0x34F, 0x61C, 0x61C, 0x115F, 0x1160, + 0x17B4, 0x17B5, 0x180B, 0x180F, 0x200B, 0x200F, 0x202A, 0x202E, + 0x2060, 0x2064, 0x2066, 0x2069, 0x3164, 0x3164, 0xFE00, 0xFE0F, + 0xFEFF, 0xFEFF, 0xFFA0, 0xFFA0, 0x1BCA0, 0x1BCA3, 0x1D173, 0x1D17A, + 0xE0020, 0xE007F, 0xE0100, 0xE01EF, + }; + } + private static int[] IDTYPE_ranges_3() { + return new int[] { + 0xA0, 0xA0, 0xA8, 0xA8, 0xAA, 0xAA, 0xAF, 0xAF, + 0xB2, 0xB5, 0xB8, 0xBA, 0xBC, 0xBE, 0x132, 0x133, + 0x13F, 0x140, 0x17F, 0x17F, 0x1C4, 0x1CC, 0x1F1, 0x1F3, + 0x2B0, 0x2B8, 0x2D8, 0x2DD, 0x2E0, 0x2E4, 0x340, 0x341, + 0x343, 0x344, 0x374, 0x374, 0x37A, 0x37A, 0x37E, 0x37E, + 0x384, 0x385, 0x387, 0x387, 0x3D0, 0x3D6, 0x3F0, 0x3F2, + 0x3F4, 0x3F5, 0x3F9, 0x3F9, 0x587, 0x587, 0x675, 0x678, + 0x958, 0x95F, 0x9DC, 0x9DD, 0x9DF, 0x9DF, 0xA33, 0xA33, + 0xA36, 0xA36, 0xA59, 0xA5B, 0xA5E, 0xA5E, 0xB5C, 0xB5D, + 0xE33, 0xE33, 0xEB3, 0xEB3, 0xEDC, 0xEDD, 0xF0C, 0xF0C, + 0xF43, 0xF43, 0xF4D, 0xF4D, 0xF52, 0xF52, 0xF57, 0xF57, + 0xF5C, 0xF5C, 0xF69, 0xF69, 0xF73, 0xF73, 0xF75, 0xF76, + 0xF78, 0xF78, 0xF81, 0xF81, 0xF93, 0xF93, 0xF9D, 0xF9D, + 0xFA2, 0xFA2, 0xFA7, 0xFA7, 0xFAC, 0xFAC, 0xFB9, 0xFB9, + 0x10FC, 0x10FC, 0x1D2C, 0x1D2E, 0x1D30, 0x1D3A, 0x1D3C, 0x1D4D, + 0x1D4F, 0x1D6A, 0x1D78, 0x1D78, 0x1D9B, 0x1DBF, 0x1E9A, 0x1E9B, + 0x1F71, 0x1F71, 0x1F73, 0x1F73, 0x1F75, 0x1F75, 0x1F77, 0x1F77, + 0x1F79, 0x1F79, 0x1F7B, 0x1F7B, 0x1F7D, 0x1F7D, 0x1FBB, 0x1FBB, + 0x1FBD, 0x1FC1, 0x1FC9, 0x1FC9, 0x1FCB, 0x1FCB, 0x1FCD, 0x1FCF, + 0x1FD3, 0x1FD3, 0x1FDB, 0x1FDB, 0x1FDD, 0x1FDF, 0x1FE3, 0x1FE3, + 0x1FEB, 0x1FEB, 0x1FED, 0x1FEF, 0x1FF9, 0x1FF9, 0x1FFB, 0x1FFB, + 0x1FFD, 0x1FFE, 0x2000, 0x200A, 0x2011, 0x2011, 0x2017, 0x2017, + 0x2024, 0x2026, 0x202F, 0x202F, 0x2033, 0x2034, 0x2036, 0x2037, + 0x203C, 0x203C, 0x203E, 0x203E, 0x2047, 0x2049, 0x2057, 0x2057, + 0x205F, 0x205F, 0x2070, 0x2071, 0x2074, 0x208E, 0x2090, 0x209C, + 0x20A8, 0x20A8, 0x2100, 0x2103, 0x2105, 0x2107, 0x2109, 0x2113, + 0x2115, 0x2116, 0x2119, 0x211D, 0x2120, 0x2122, 0x2124, 0x2124, + 0x2126, 0x2126, 0x2128, 0x2128, 0x212A, 0x212D, 0x212F, 0x2131, + 0x2133, 0x2139, 0x213B, 0x2140, 0x2145, 0x2149, 0x2150, 0x217F, + 0x2189, 0x2189, 0x222C, 0x222D, 0x222F, 0x2230, 0x2460, 0x24EA, + 0x2A0C, 0x2A0C, 0x2A74, 0x2A76, 0x2ADC, 0x2ADC, 0x2C7C, 0x2C7D, + 0x2D6F, 0x2D6F, 0x2E9F, 0x2E9F, 0x2EF3, 0x2EF3, 0x2F00, 0x2FD5, + 0x3000, 0x3000, 0x3036, 0x3036, 0x3038, 0x303A, 0x309B, 0x309C, + 0x309F, 0x309F, 0x30FF, 0x30FF, 0x3131, 0x3163, 0x3165, 0x318E, + 0x3192, 0x319F, 0x3200, 0x321E, 0x3220, 0x3247, 0x3250, 0x327E, + 0x3280, 0x33FF, 0xA69C, 0xA69D, 0xA770, 0xA770, 0xA7F1, 0xA7F4, + 0xA7F8, 0xA7F9, 0xAB5C, 0xAB5F, 0xAB69, 0xAB69, 0xF900, 0xFA0D, + 0xFA10, 0xFA10, 0xFA12, 0xFA12, 0xFA15, 0xFA1E, 0xFA20, 0xFA20, + 0xFA22, 0xFA22, 0xFA25, 0xFA26, 0xFA2A, 0xFA6D, 0xFA70, 0xFAD9, + 0xFB00, 0xFB06, 0xFB13, 0xFB17, 0xFB1D, 0xFB1D, 0xFB1F, 0xFB36, + 0xFB38, 0xFB3C, 0xFB3E, 0xFB3E, 0xFB40, 0xFB41, 0xFB43, 0xFB44, + 0xFB46, 0xFBB1, 0xFBD3, 0xFD3D, 0xFD50, 0xFD8F, 0xFD92, 0xFDC7, + 0xFDF0, 0xFDFC, 0xFE10, 0xFE19, 0xFE30, 0xFE44, 0xFE47, 0xFE52, + 0xFE54, 0xFE66, 0xFE68, 0xFE6B, 0xFE70, 0xFE72, 0xFE74, 0xFE74, + 0xFE76, 0xFEFC, 0xFF01, 0xFF9F, 0xFFA1, 0xFFBE, 0xFFC2, 0xFFC7, + 0xFFCA, 0xFFCF, 0xFFD2, 0xFFD7, 0xFFDA, 0xFFDC, 0xFFE0, 0xFFE6, + 0xFFE8, 0xFFEE, 0x10781, 0x10785, 0x10787, 0x107B0, 0x107B2, 0x107BA, + 0x1CCD6, 0x1CCF9, 0x1D15E, 0x1D164, 0x1D1BB, 0x1D1C0, 0x1D400, 0x1D454, + 0x1D456, 0x1D49C, 0x1D49E, 0x1D49F, 0x1D4A2, 0x1D4A2, 0x1D4A5, 0x1D4A6, + 0x1D4A9, 0x1D4AC, 0x1D4AE, 0x1D4B9, 0x1D4BB, 0x1D4BB, 0x1D4BD, 0x1D4C3, + 0x1D4C5, 0x1D505, 0x1D507, 0x1D50A, 0x1D50D, 0x1D514, 0x1D516, 0x1D51C, + 0x1D51E, 0x1D539, 0x1D53B, 0x1D53E, 0x1D540, 0x1D544, 0x1D546, 0x1D546, + 0x1D54A, 0x1D550, 0x1D552, 0x1D6A5, 0x1D6A8, 0x1D7CB, 0x1D7CE, 0x1D7FF, + 0x1E030, 0x1E06D, 0x1EE00, 0x1EE03, 0x1EE05, 0x1EE1F, 0x1EE21, 0x1EE22, + 0x1EE24, 0x1EE24, 0x1EE27, 0x1EE27, 0x1EE29, 0x1EE32, 0x1EE34, 0x1EE37, + 0x1EE39, 0x1EE39, 0x1EE3B, 0x1EE3B, 0x1EE42, 0x1EE42, 0x1EE47, 0x1EE47, + 0x1EE49, 0x1EE49, 0x1EE4B, 0x1EE4B, 0x1EE4D, 0x1EE4F, 0x1EE51, 0x1EE52, + 0x1EE54, 0x1EE54, 0x1EE57, 0x1EE57, 0x1EE59, 0x1EE59, 0x1EE5B, 0x1EE5B, + 0x1EE5D, 0x1EE5D, 0x1EE5F, 0x1EE5F, 0x1EE61, 0x1EE62, 0x1EE64, 0x1EE64, + 0x1EE67, 0x1EE6A, 0x1EE6C, 0x1EE72, 0x1EE74, 0x1EE77, 0x1EE79, 0x1EE7C, + 0x1EE7E, 0x1EE7E, 0x1EE80, 0x1EE89, 0x1EE8B, 0x1EE9B, 0x1EEA1, 0x1EEA3, + 0x1EEA5, 0x1EEA9, 0x1EEAB, 0x1EEBB, 0x1F100, 0x1F10A, 0x1F110, 0x1F12E, + 0x1F130, 0x1F14F, 0x1F16A, 0x1F16C, 0x1F190, 0x1F190, 0x1F200, 0x1F202, + 0x1F210, 0x1F23B, 0x1F240, 0x1F248, 0x1F250, 0x1F251, 0x1FBF0, 0x1FBF9, + 0x2F800, 0x2FA1D, + }; + } + private static int[] IDTYPE_ranges_4() { + return new int[] { + 0x9, 0xD, 0x20, 0x26, 0x28, 0x2C, 0x2F, 0x2F, + 0x3B, 0x40, 0x5B, 0x5E, 0x60, 0x60, 0x7B, 0x7E, + 0x85, 0x85, 0xA1, 0xA7, 0xA9, 0xA9, 0xAB, 0xAC, + 0xAE, 0xAE, 0xB0, 0xB1, 0xB6, 0xB6, 0xBB, 0xBB, + 0xBF, 0xBF, 0xD7, 0xD7, 0xF7, 0xF7, 0x2C2, 0x2C5, + 0x2D2, 0x2D7, 0x2DE, 0x2DF, 0x2E5, 0x2EB, 0x2ED, 0x2ED, + 0x2EF, 0x2FF, 0x375, 0x375, 0x3F6, 0x3F6, 0x482, 0x482, + 0x488, 0x489, 0x55A, 0x55F, 0x589, 0x589, 0x58D, 0x58F, + 0x5BE, 0x5BE, 0x5C0, 0x5C0, 0x5C3, 0x5C3, 0x5C6, 0x5C6, + 0x600, 0x60F, 0x61B, 0x61B, 0x61D, 0x61F, 0x66A, 0x66D, + 0x6D4, 0x6D4, 0x6DD, 0x6DE, 0x6E9, 0x6E9, 0x700, 0x70D, + 0x70F, 0x70F, 0x7F6, 0x7F9, 0x7FE, 0x7FF, 0x830, 0x83E, + 0x85E, 0x85E, 0x888, 0x888, 0x890, 0x891, 0x8E2, 0x8E2, + 0x964, 0x965, 0x970, 0x970, 0x9F2, 0x9FB, 0x9FD, 0x9FD, + 0xA76, 0xA76, 0xAF0, 0xAF1, 0xB70, 0xB70, 0xB72, 0xB77, + 0xBF0, 0xBFA, 0xC77, 0xC7F, 0xC84, 0xC84, 0xD4F, 0xD4F, + 0xD58, 0xD5E, 0xD70, 0xD79, 0xDF4, 0xDF4, 0xE3F, 0xE3F, + 0xE4F, 0xE4F, 0xE5A, 0xE5B, 0xF01, 0xF0A, 0xF0D, 0xF17, + 0xF1A, 0xF1F, 0xF2A, 0xF34, 0xF36, 0xF36, 0xF38, 0xF38, + 0xF3A, 0xF3D, 0xF85, 0xF85, 0xFBE, 0xFC5, 0xFC7, 0xFCC, + 0xFCE, 0xFDA, 0x104A, 0x104F, 0x109E, 0x109F, 0x10FB, 0x10FB, + 0x1360, 0x1368, 0x1372, 0x137C, 0x1390, 0x1399, 0x1400, 0x1400, + 0x166D, 0x166E, 0x1680, 0x1680, 0x169B, 0x169C, 0x16EB, 0x16ED, + 0x1735, 0x1736, 0x17D4, 0x17D6, 0x17D8, 0x17DB, 0x17F0, 0x17F9, + 0x1800, 0x180A, 0x1940, 0x1940, 0x1944, 0x1945, 0x19DE, 0x19FF, + 0x1A1E, 0x1A1F, 0x1AA0, 0x1AA6, 0x1AA8, 0x1AAD, 0x1ABE, 0x1ABE, + 0x1B4E, 0x1B4F, 0x1B5A, 0x1B6A, 0x1B74, 0x1B7F, 0x1BFC, 0x1BFF, + 0x1C3B, 0x1C3F, 0x1C7E, 0x1C7F, 0x1CC0, 0x1CC7, 0x1CD3, 0x1CD3, + 0x2012, 0x2016, 0x2018, 0x2018, 0x201A, 0x2023, 0x2028, 0x2029, + 0x2030, 0x2032, 0x2035, 0x2035, 0x2038, 0x203B, 0x203D, 0x203D, + 0x2041, 0x2046, 0x204A, 0x2053, 0x2055, 0x2056, 0x2058, 0x205E, + 0x20A0, 0x20A7, 0x20A9, 0x20C1, 0x20DD, 0x20E0, 0x20E2, 0x20E4, + 0x2104, 0x2104, 0x2108, 0x2108, 0x2114, 0x2114, 0x2117, 0x2117, + 0x211E, 0x211F, 0x2123, 0x2123, 0x2125, 0x2125, 0x2127, 0x2127, + 0x2129, 0x2129, 0x213A, 0x213A, 0x2141, 0x2144, 0x214A, 0x214D, + 0x214F, 0x214F, 0x218A, 0x218B, 0x2190, 0x222B, 0x222E, 0x222E, + 0x2231, 0x2328, 0x232B, 0x2429, 0x2440, 0x244A, 0x24EB, 0x2A0B, + 0x2A0D, 0x2A73, 0x2A77, 0x2ADB, 0x2ADD, 0x2B73, 0x2B76, 0x2BFF, + 0x2CE5, 0x2CEA, 0x2CF9, 0x2CFF, 0x2D70, 0x2D70, 0x2E00, 0x2E5D, + 0x2E80, 0x2E99, 0x2E9B, 0x2E9E, 0x2EA0, 0x2EF2, 0x2FF0, 0x2FFF, + 0x3001, 0x3004, 0x3008, 0x3020, 0x3030, 0x3030, 0x3037, 0x3037, + 0x303D, 0x303F, 0x3190, 0x3191, 0x31C0, 0x31E5, 0x31EF, 0x31EF, + 0x3248, 0x324F, 0x327F, 0x327F, 0x4DC0, 0x4DFF, 0xA490, 0xA4C6, + 0xA4FE, 0xA4FF, 0xA60D, 0xA60F, 0xA670, 0xA673, 0xA67E, 0xA67E, + 0xA6F2, 0xA6F7, 0xA700, 0xA716, 0xA720, 0xA721, 0xA789, 0xA78A, + 0xA828, 0xA82B, 0xA830, 0xA839, 0xA874, 0xA877, 0xA8CE, 0xA8CF, + 0xA8F8, 0xA8FA, 0xA8FC, 0xA8FC, 0xA92E, 0xA92F, 0xA95F, 0xA95F, + 0xA9C1, 0xA9CD, 0xA9DE, 0xA9DF, 0xAA5C, 0xAA5F, 0xAA77, 0xAA79, + 0xAADE, 0xAADF, 0xAAF0, 0xAAF1, 0xAB5B, 0xAB5B, 0xAB6A, 0xAB6B, + 0xABEB, 0xABEB, 0xFBB2, 0xFBD2, 0xFD3E, 0xFD4F, 0xFD90, 0xFD91, + 0xFDC8, 0xFDCF, 0xFDFD, 0xFDFF, 0xFE45, 0xFE46, 0xFFF9, 0xFFFD, + 0x10100, 0x10102, 0x10107, 0x10133, 0x10137, 0x1013F, 0x10175, 0x1018E, + 0x10190, 0x1019C, 0x101A0, 0x101A0, 0x101D0, 0x101FC, 0x102E1, 0x102FB, + 0x10320, 0x10323, 0x1039F, 0x1039F, 0x103D0, 0x103D0, 0x1056F, 0x1056F, + 0x10857, 0x1085F, 0x10877, 0x1087F, 0x108A7, 0x108AF, 0x108FB, 0x108FF, + 0x10916, 0x1091B, 0x1091F, 0x1091F, 0x1093F, 0x1093F, 0x109BC, 0x109BD, + 0x109C0, 0x109CF, 0x109D2, 0x109FF, 0x10A40, 0x10A48, 0x10A50, 0x10A58, + 0x10A7D, 0x10A7F, 0x10A9D, 0x10A9F, 0x10AC8, 0x10AC8, 0x10AEB, 0x10AF6, + 0x10B39, 0x10B3F, 0x10B58, 0x10B5F, 0x10B78, 0x10B7F, 0x10B99, 0x10B9C, + 0x10BA9, 0x10BAF, 0x10CFA, 0x10CFF, 0x10D6E, 0x10D6E, 0x10D8E, 0x10D8F, + 0x10E60, 0x10E7E, 0x10EAD, 0x10EAD, 0x10ED0, 0x10ED8, 0x10F1D, 0x10F26, + 0x10F51, 0x10F59, 0x10F86, 0x10F89, 0x10FC5, 0x10FCB, 0x11047, 0x1104D, + 0x11052, 0x11065, 0x110BB, 0x110C1, 0x110CD, 0x110CD, 0x11140, 0x11143, + 0x11174, 0x11175, 0x111C5, 0x111C8, 0x111CD, 0x111CD, 0x111DB, 0x111DB, + 0x111DD, 0x111DF, 0x111E1, 0x111F4, 0x11238, 0x1123D, 0x112A9, 0x112A9, + 0x113D4, 0x113D5, 0x113D7, 0x113D8, 0x1144B, 0x1144F, 0x1145A, 0x1145B, + 0x1145D, 0x1145D, 0x114C6, 0x114C6, 0x115C1, 0x115D7, 0x11641, 0x11643, + 0x11660, 0x1166C, 0x116B9, 0x116B9, 0x1173A, 0x1173F, 0x1183B, 0x1183B, + 0x118EA, 0x118F2, 0x11944, 0x11946, 0x119E2, 0x119E2, 0x11A3F, 0x11A46, + 0x11A9A, 0x11A9C, 0x11A9E, 0x11AA2, 0x11B00, 0x11B09, 0x11BE1, 0x11BE1, + 0x11C41, 0x11C45, 0x11C5A, 0x11C6C, 0x11C70, 0x11C71, 0x11EF7, 0x11EF8, + 0x11F43, 0x11F4F, 0x11FC0, 0x11FF1, 0x11FFF, 0x11FFF, 0x12470, 0x12474, + 0x12FF1, 0x12FF2, 0x13430, 0x1343F, 0x16A6E, 0x16A6F, 0x16AF5, 0x16AF5, + 0x16B37, 0x16B3F, 0x16B44, 0x16B45, 0x16B5B, 0x16B61, 0x16D6D, 0x16D6F, + 0x16E80, 0x16E9A, 0x16FE2, 0x16FE2, 0x1BC9C, 0x1BC9C, 0x1BC9F, 0x1BC9F, + 0x1CC00, 0x1CCD5, 0x1CCFA, 0x1CCFC, 0x1CD00, 0x1CEB3, 0x1CEBA, 0x1CED0, + 0x1CEE0, 0x1CEF0, 0x1CF50, 0x1CFC3, 0x1D000, 0x1D0F5, 0x1D100, 0x1D126, + 0x1D129, 0x1D15D, 0x1D16A, 0x1D16C, 0x1D183, 0x1D184, 0x1D18C, 0x1D1A9, + 0x1D1AE, 0x1D1BA, 0x1D1C1, 0x1D1EA, 0x1D200, 0x1D241, 0x1D245, 0x1D245, + 0x1D2C0, 0x1D2D3, 0x1D2E0, 0x1D2F3, 0x1D300, 0x1D356, 0x1D360, 0x1D378, + 0x1D800, 0x1D9FF, 0x1DA37, 0x1DA3A, 0x1DA6D, 0x1DA74, 0x1DA76, 0x1DA83, + 0x1DA85, 0x1DA8B, 0x1E14F, 0x1E14F, 0x1E2FF, 0x1E2FF, 0x1E5FF, 0x1E5FF, + 0x1E8C7, 0x1E8CF, 0x1E95E, 0x1E95F, 0x1EC71, 0x1ECB4, 0x1ED01, 0x1ED3D, + 0x1EEF0, 0x1EEF1, 0x1F000, 0x1F02B, 0x1F030, 0x1F093, 0x1F0A0, 0x1F0AE, + 0x1F0B1, 0x1F0BF, 0x1F0C1, 0x1F0CF, 0x1F0D1, 0x1F0F5, 0x1F10B, 0x1F10F, + 0x1F12F, 0x1F12F, 0x1F150, 0x1F169, 0x1F16D, 0x1F18F, 0x1F191, 0x1F1AD, + 0x1F1E6, 0x1F1FF, 0x1F260, 0x1F265, 0x1F300, 0x1F6D8, 0x1F6DC, 0x1F6EC, + 0x1F6F0, 0x1F6FC, 0x1F700, 0x1F7D9, 0x1F7E0, 0x1F7EB, 0x1F7F0, 0x1F7F0, + 0x1F800, 0x1F80B, 0x1F810, 0x1F847, 0x1F850, 0x1F859, 0x1F860, 0x1F887, + 0x1F890, 0x1F8AD, 0x1F8B0, 0x1F8BB, 0x1F8C0, 0x1F8C1, 0x1F8D0, 0x1F8D8, + 0x1F900, 0x1FA57, 0x1FA60, 0x1FA6D, 0x1FA70, 0x1FA7C, 0x1FA80, 0x1FA8A, + 0x1FA8E, 0x1FAC6, 0x1FAC8, 0x1FAC8, 0x1FACD, 0x1FADC, 0x1FADF, 0x1FAEA, + 0x1FAEF, 0x1FAF8, 0x1FB00, 0x1FB92, 0x1FB94, 0x1FBEF, 0x1FBFA, 0x1FBFA, + }; + } + private static int[] IDTYPE_ranges_5() { + return new int[] { + 0x3E2, 0x3EF, 0x800, 0x82D, 0x830, 0x83E, 0x1680, 0x169C, + 0x16A0, 0x16F8, 0x1700, 0x1715, 0x171F, 0x1736, 0x1740, 0x1753, + 0x1760, 0x176C, 0x176E, 0x1770, 0x1772, 0x1773, 0x1800, 0x180A, + 0x1810, 0x1819, 0x1820, 0x1878, 0x1880, 0x18AA, 0x1A00, 0x1A1B, + 0x1A1E, 0x1A1F, 0x1CFA, 0x1CFA, 0x2C00, 0x2C5F, 0x2C80, 0x2CF3, + 0x2CF9, 0x2CFF, 0x2E30, 0x2E30, 0x2E3C, 0x2E3C, 0xA840, 0xA877, + 0xA930, 0xA953, 0xA95F, 0xA95F, 0x10000, 0x1000B, 0x1000D, 0x10026, + 0x10028, 0x1003A, 0x1003C, 0x1003D, 0x1003F, 0x1004D, 0x10050, 0x1005D, + 0x10080, 0x100FA, 0x10100, 0x10102, 0x10107, 0x10133, 0x10137, 0x1013F, + 0x10280, 0x1029C, 0x102A0, 0x102D0, 0x10300, 0x10323, 0x1032D, 0x1034A, + 0x10350, 0x1037A, 0x10380, 0x1039D, 0x1039F, 0x103C3, 0x103C8, 0x103D5, + 0x10400, 0x1049D, 0x104A0, 0x104A9, 0x10500, 0x10527, 0x10530, 0x10563, + 0x1056F, 0x1057A, 0x1057C, 0x1058A, 0x1058C, 0x10592, 0x10594, 0x10595, + 0x10597, 0x105A1, 0x105A3, 0x105B1, 0x105B3, 0x105B9, 0x105BB, 0x105BC, + 0x105C0, 0x105F3, 0x10600, 0x10736, 0x10740, 0x10755, 0x10760, 0x10767, + 0x10800, 0x10805, 0x10808, 0x10808, 0x1080A, 0x10835, 0x10837, 0x10838, + 0x1083C, 0x1083C, 0x1083F, 0x10855, 0x10857, 0x1089E, 0x108A7, 0x108AF, + 0x108E0, 0x108F2, 0x108F4, 0x108F5, 0x108FB, 0x1091B, 0x1091F, 0x10939, + 0x1093F, 0x10959, 0x10980, 0x109B7, 0x109BC, 0x109CF, 0x109D2, 0x10A03, + 0x10A05, 0x10A06, 0x10A0C, 0x10A13, 0x10A15, 0x10A17, 0x10A19, 0x10A35, + 0x10A38, 0x10A3A, 0x10A3F, 0x10A48, 0x10A50, 0x10A58, 0x10A60, 0x10A9F, + 0x10AC0, 0x10AE6, 0x10AEB, 0x10AF6, 0x10B00, 0x10B35, 0x10B39, 0x10B55, + 0x10B58, 0x10B72, 0x10B78, 0x10B91, 0x10B99, 0x10B9C, 0x10BA9, 0x10BAF, + 0x10C00, 0x10C48, 0x10C80, 0x10CB2, 0x10CC0, 0x10CF2, 0x10CFA, 0x10CFF, + 0x10D40, 0x10D65, 0x10D69, 0x10D85, 0x10D8E, 0x10D8F, 0x10E80, 0x10EA9, + 0x10EAB, 0x10EAD, 0x10EB0, 0x10EB1, 0x10F00, 0x10F27, 0x10F30, 0x10F59, + 0x10F70, 0x10F89, 0x10FB0, 0x10FCB, 0x10FE0, 0x10FF6, 0x11000, 0x1104D, + 0x11052, 0x11075, 0x1107F, 0x110C2, 0x110CD, 0x110CD, 0x110D0, 0x110E8, + 0x110F0, 0x110F9, 0x11150, 0x11176, 0x11180, 0x111DF, 0x11200, 0x11211, + 0x11213, 0x11241, 0x11280, 0x11286, 0x11288, 0x11288, 0x1128A, 0x1128D, + 0x1128F, 0x1129D, 0x1129F, 0x112A9, 0x112B0, 0x112EA, 0x112F0, 0x112F9, + 0x11300, 0x11300, 0x11302, 0x11302, 0x11305, 0x1130C, 0x1130F, 0x11310, + 0x11313, 0x11328, 0x1132A, 0x11330, 0x11332, 0x11333, 0x11335, 0x11339, + 0x1133D, 0x11344, 0x11347, 0x11348, 0x1134B, 0x1134D, 0x11350, 0x11350, + 0x11357, 0x11357, 0x1135D, 0x11363, 0x11366, 0x1136C, 0x11370, 0x11374, + 0x11380, 0x11389, 0x1138B, 0x1138B, 0x1138E, 0x1138E, 0x11390, 0x113B5, + 0x113B7, 0x113C0, 0x113C2, 0x113C2, 0x113C5, 0x113C5, 0x113C7, 0x113CA, + 0x113CC, 0x113D5, 0x113D7, 0x113D8, 0x113E1, 0x113E2, 0x11480, 0x114C7, + 0x114D0, 0x114D9, 0x11580, 0x115B5, 0x115B8, 0x115DD, 0x11600, 0x11644, + 0x11650, 0x11659, 0x11660, 0x1166C, 0x11680, 0x116B9, 0x116C0, 0x116C9, + 0x11700, 0x1171A, 0x1171D, 0x1172B, 0x11730, 0x11746, 0x11800, 0x1183B, + 0x118A0, 0x118F2, 0x118FF, 0x11906, 0x11909, 0x11909, 0x1190C, 0x11913, + 0x11915, 0x11916, 0x11918, 0x11935, 0x11937, 0x11938, 0x1193B, 0x11946, + 0x11950, 0x11959, 0x119A0, 0x119A7, 0x119AA, 0x119D7, 0x119DA, 0x119E4, + 0x11A00, 0x11A47, 0x11A50, 0x11AA2, 0x11AC0, 0x11AF8, 0x11B60, 0x11B67, + 0x11BC0, 0x11BE1, 0x11BF0, 0x11BF9, 0x11C00, 0x11C08, 0x11C0A, 0x11C36, + 0x11C38, 0x11C45, 0x11C50, 0x11C6C, 0x11C70, 0x11C8F, 0x11C92, 0x11CA7, + 0x11CA9, 0x11CB6, 0x11D00, 0x11D06, 0x11D08, 0x11D09, 0x11D0B, 0x11D36, + 0x11D3A, 0x11D3A, 0x11D3C, 0x11D3D, 0x11D3F, 0x11D47, 0x11D50, 0x11D59, + 0x11D60, 0x11D65, 0x11D67, 0x11D68, 0x11D6A, 0x11D8E, 0x11D90, 0x11D91, + 0x11D93, 0x11D98, 0x11DA0, 0x11DA9, 0x11DB0, 0x11DDB, 0x11DE0, 0x11DE9, + 0x11EE0, 0x11EF8, 0x11F00, 0x11F10, 0x11F12, 0x11F3A, 0x11F3E, 0x11F5A, + 0x12000, 0x12399, 0x12400, 0x1246E, 0x12470, 0x12474, 0x12480, 0x12543, + 0x12F90, 0x12FF2, 0x13000, 0x13455, 0x13460, 0x143FA, 0x14400, 0x14646, + 0x16100, 0x16139, 0x16A40, 0x16A5E, 0x16A60, 0x16A69, 0x16A6E, 0x16ABE, + 0x16AC0, 0x16AC9, 0x16AD0, 0x16AED, 0x16AF0, 0x16AF5, 0x16B00, 0x16B45, + 0x16B50, 0x16B59, 0x16B5B, 0x16B61, 0x16B63, 0x16B77, 0x16B7D, 0x16B8F, + 0x16D40, 0x16D79, 0x16E40, 0x16E9A, 0x16EA0, 0x16EB8, 0x16EBB, 0x16ED3, + 0x16FE0, 0x16FE1, 0x16FE4, 0x16FE4, 0x17000, 0x18CD5, 0x18CFF, 0x18D1E, + 0x18D80, 0x18DF2, 0x1B170, 0x1B2FB, 0x1BC00, 0x1BC6A, 0x1BC70, 0x1BC7C, + 0x1BC80, 0x1BC88, 0x1BC90, 0x1BC99, 0x1BC9C, 0x1BC9F, 0x1D800, 0x1DA8B, + 0x1DA9B, 0x1DA9F, 0x1DAA1, 0x1DAAF, 0x1E000, 0x1E006, 0x1E008, 0x1E018, + 0x1E01B, 0x1E021, 0x1E023, 0x1E024, 0x1E026, 0x1E02A, 0x1E290, 0x1E2AE, + 0x1E4D0, 0x1E4F9, 0x1E5D0, 0x1E5FA, 0x1E5FF, 0x1E5FF, 0x1E6C0, 0x1E6DE, + 0x1E6E0, 0x1E6F5, 0x1E6FE, 0x1E6FF, 0x1E800, 0x1E8C4, 0x1E8C7, 0x1E8D6, + }; + } + private static int[] IDTYPE_ranges_6() { + return new int[] { + 0x138, 0x138, 0x18D, 0x18D, 0x1AA, 0x1AB, 0x1B9, 0x1BB, + 0x1BE, 0x1BF, 0x1F6, 0x1F7, 0x21C, 0x21D, 0x277, 0x277, + 0x27C, 0x27C, 0x29E, 0x29E, 0x345, 0x345, 0x363, 0x373, + 0x376, 0x377, 0x37B, 0x37D, 0x37F, 0x37F, 0x3D8, 0x3E1, + 0x3F3, 0x3F3, 0x3F7, 0x3F8, 0x3FA, 0x3FF, 0x460, 0x489, + 0x49C, 0x49D, 0x4A6, 0x4A7, 0x4B8, 0x4B9, 0x500, 0x50F, + 0x514, 0x523, 0x526, 0x52F, 0x5A2, 0x5A2, 0x5C5, 0x5C6, + 0x63B, 0x63C, 0x63E, 0x640, 0x66E, 0x66F, 0x690, 0x690, + 0x6AC, 0x6AC, 0x77E, 0x77F, 0x7E8, 0x7EA, 0x7FA, 0x7FA, + 0x88E, 0x88E, 0x8AD, 0x8B1, 0x8B5, 0x8B5, 0x90C, 0x90C, + 0x93D, 0x93D, 0x94E, 0x94E, 0x951, 0x952, 0x960, 0x963, + 0x971, 0x971, 0x978, 0x978, 0x980, 0x980, 0x9BD, 0x9BD, + 0x9E0, 0x9E3, 0x9FC, 0x9FC, 0xABD, 0xABD, 0xAE0, 0xAE3, + 0xB3D, 0xB3D, 0xB60, 0xB61, 0xC00, 0xC00, 0xC34, 0xC34, + 0xC3D, 0xC3D, 0xC58, 0xC59, 0xC5C, 0xC5C, 0xC60, 0xC61, + 0xC81, 0xC81, 0xC8C, 0xC8C, 0xCB1, 0xCB1, 0xCBD, 0xCBD, + 0xCDC, 0xCDC, 0xCDE, 0xCDE, 0xCE0, 0xCE3, 0xCF1, 0xCF2, + 0xD01, 0xD01, 0xD04, 0xD04, 0xD3A, 0xD3D, 0xD4C, 0xD4C, + 0xD4E, 0xD4E, 0xD5F, 0xD61, 0xD9E, 0xD9E, 0xF6A, 0xF6A, + 0xF82, 0xF83, 0xF86, 0xF8F, 0x1050, 0x1059, 0x10A0, 0x10C5, + 0x10F1, 0x10F6, 0x1100, 0x115E, 0x1161, 0x11FF, 0x1369, 0x1371, + 0x17A8, 0x17A8, 0x17D1, 0x17D1, 0x17D3, 0x17D3, 0x17D8, 0x17D8, + 0x17DC, 0x17DD, 0x1AB0, 0x1ABD, 0x1C80, 0x1C88, 0x1CD0, 0x1CF9, + 0x1DC0, 0x1DC3, 0x1DCE, 0x1DCE, 0x1DD1, 0x1DE6, 0x1F00, 0x1F15, + 0x1F18, 0x1F1D, 0x1F20, 0x1F45, 0x1F48, 0x1F4D, 0x1F50, 0x1F57, + 0x1F59, 0x1F59, 0x1F5B, 0x1F5B, 0x1F5D, 0x1F5D, 0x1F5F, 0x1F70, + 0x1F72, 0x1F72, 0x1F74, 0x1F74, 0x1F76, 0x1F76, 0x1F78, 0x1F78, + 0x1F7A, 0x1F7A, 0x1F7C, 0x1F7C, 0x1F80, 0x1F9F, 0x1FB0, 0x1FB1, + 0x1FB6, 0x1FBA, 0x1FBC, 0x1FBC, 0x1FC2, 0x1FC4, 0x1FC6, 0x1FC8, + 0x1FCA, 0x1FCA, 0x1FCC, 0x1FCC, 0x1FD0, 0x1FD2, 0x1FD6, 0x1FDA, + 0x1FE0, 0x1FE2, 0x1FE4, 0x1FEA, 0x1FF2, 0x1FF4, 0x1FF6, 0x1FF8, + 0x1FFA, 0x1FFA, 0x1FFC, 0x1FFC, 0x2056, 0x2056, 0x2058, 0x205E, + 0x2127, 0x2127, 0x2132, 0x2132, 0x214E, 0x214F, 0x2180, 0x2188, + 0x2C6D, 0x2C76, 0x2C7E, 0x2C7F, 0x2D00, 0x2D25, 0x2DE0, 0x2E16, + 0x2E2A, 0x2E2F, 0x2E31, 0x2E32, 0x2E35, 0x2E35, 0x2E39, 0x2E39, + 0x301E, 0x301E, 0x302E, 0x302F, 0x312E, 0x312E, 0x31F0, 0x31FF, + 0xA610, 0xA612, 0xA62A, 0xA62B, 0xA640, 0xA66E, 0xA670, 0xA67B, + 0xA67F, 0xA69B, 0xA69E, 0xA69F, 0xA700, 0xA707, 0xA722, 0xA76F, + 0xA771, 0xA787, 0xA790, 0xA791, 0xA794, 0xA7A9, 0xA7AB, 0xA7AD, + 0xA7B0, 0xA7B1, 0xA7C0, 0xA7C1, 0xA7C4, 0xA7C4, 0xA7C7, 0xA7CA, + 0xA7D0, 0xA7D9, 0xA7F5, 0xA7F7, 0xA7FB, 0xA7FF, 0xA8E0, 0xA8FF, + 0xA960, 0xA97C, 0xA9E0, 0xA9E6, 0xAB30, 0xAB5A, 0xAB64, 0xAB65, + 0xD7B0, 0xD7C6, 0xD7CB, 0xD7FB, 0x10140, 0x10174, 0x101D0, 0x101FD, + 0x102E0, 0x102FB, 0x16FE3, 0x16FE3, 0x16FF0, 0x16FF1, 0x1B000, 0x1B122, + 0x1B132, 0x1B132, 0x1B150, 0x1B152, 0x1B155, 0x1B155, 0x1B164, 0x1B167, + 0x1D200, 0x1D245, 0x1E08F, 0x1E08F, + }; + } + private static int[] IDTYPE_ranges_7() { + return new int[] { + 0x180, 0x180, 0x18D, 0x18D, 0x1AA, 0x1AB, 0x1BA, 0x1BB, + 0x1BE, 0x1BE, 0x1C0, 0x1C3, 0x200, 0x217, 0x234, 0x236, + 0x250, 0x252, 0x255, 0x255, 0x258, 0x258, 0x25A, 0x25A, + 0x25C, 0x262, 0x264, 0x267, 0x26A, 0x271, 0x273, 0x288, + 0x28A, 0x28A, 0x28C, 0x291, 0x293, 0x2AF, 0x2B9, 0x2BA, + 0x2BD, 0x2C1, 0x2C6, 0x2D1, 0x2EC, 0x2EC, 0x2EE, 0x2EE, + 0x30E, 0x315, 0x317, 0x31A, 0x31C, 0x320, 0x324, 0x325, + 0x329, 0x330, 0x333, 0x333, 0x335, 0x335, 0x337, 0x33F, + 0x342, 0x342, 0x346, 0x34E, 0x350, 0x357, 0x359, 0x362, + 0x375, 0x375, 0x3CF, 0x3CF, 0x3D7, 0x3D7, 0x3F3, 0x3F3, + 0x3FC, 0x3FC, 0x484, 0x487, 0x559, 0x559, 0x560, 0x560, + 0x588, 0x588, 0x5C7, 0x5C7, 0x653, 0x653, 0x671, 0x671, + 0x6E5, 0x6E6, 0x740, 0x74A, 0x870, 0x888, 0x8C9, 0x8C9, + 0x950, 0x950, 0x953, 0x954, 0x97D, 0x97D, 0xA74, 0xA74, + 0xAD0, 0xAD0, 0xB82, 0xB82, 0xBD0, 0xBD0, 0xD04, 0xD04, + 0xD81, 0xD81, 0xD8F, 0xD90, 0xDDF, 0xDDF, 0xDF3, 0xDF3, + 0xEAF, 0xEAF, 0xF00, 0xF00, 0xF18, 0xF19, 0xF35, 0xF35, + 0xF37, 0xF37, 0xF3E, 0xF3F, 0xFC6, 0xFC6, 0x10F9, 0x10FA, + 0x17CE, 0x17CF, 0x17D1, 0x17D1, 0x17DD, 0x17DD, 0x1ABF, 0x1AC0, + 0x1ACF, 0x1ADD, 0x1AE0, 0x1AEB, 0x1B6B, 0x1B73, 0x1D00, 0x1D2B, + 0x1D2F, 0x1D2F, 0x1D3B, 0x1D3B, 0x1D4E, 0x1D4E, 0x1D6B, 0x1D77, + 0x1D79, 0x1D9A, 0x1DC0, 0x1E01, 0x1E18, 0x1E1B, 0x1E2A, 0x1E2D, + 0x1E72, 0x1E77, 0x1E9C, 0x1E9D, 0x1E9F, 0x1E9F, 0x1EFA, 0x1EFF, + 0x1FB0, 0x1FB1, 0x203F, 0x2040, 0x20D0, 0x20F0, 0x2118, 0x2118, + 0x212E, 0x212E, 0x2180, 0x2183, 0x24EB, 0x24FF, 0x2800, 0x28FF, + 0x2C60, 0x2C67, 0x2C77, 0x2C7B, 0x2CF0, 0x2CF1, 0x2D27, 0x2D27, + 0x2D2D, 0x2D2D, 0x2E00, 0x2E0D, 0x3021, 0x302F, 0x3031, 0x3035, + 0x303B, 0x303C, 0x327F, 0x327F, 0x4DC0, 0x4DFF, 0xA708, 0xA71F, + 0xA722, 0xA72F, 0xA788, 0xA788, 0xA78E, 0xA78E, 0xA7AE, 0xA7AF, + 0xA7BA, 0xA7BF, 0xA7C5, 0xA7C6, 0xA7FA, 0xA7FA, 0xAB68, 0xAB68, + 0xFB1E, 0xFB1E, 0xFBB2, 0xFBD2, 0xFD3E, 0xFD4F, 0xFD90, 0xFD91, + 0xFDC8, 0xFDCF, 0xFDFD, 0xFDFF, 0xFE20, 0xFE2F, 0xFE45, 0xFE46, + 0xFE73, 0xFE73, 0x10EC5, 0x10EC6, 0x10EFB, 0x10EFB, 0x16FF2, 0x16FF6, + 0x1CF00, 0x1CF2D, 0x1CF30, 0x1CF46, 0x1CF50, 0x1CFC3, 0x1D000, 0x1D0F5, + 0x1D100, 0x1D126, 0x1D129, 0x1D15D, 0x1D165, 0x1D172, 0x1D17B, 0x1D1BA, + 0x1D1C1, 0x1D1EA, 0x1D242, 0x1D244, 0x1D300, 0x1D356, 0x1DF00, 0x1DF1E, + 0x1DF25, 0x1DF2A, + }; + } + private static int[] IDTYPE_ranges_8() { + return new int[] { + 0x114, 0x115, 0x12C, 0x12D, 0x14E, 0x14F, 0x156, 0x157, + 0x162, 0x163, 0x182, 0x185, 0x187, 0x188, 0x18B, 0x18C, + 0x193, 0x193, 0x195, 0x195, 0x19A, 0x19C, 0x19E, 0x19F, + 0x1A2, 0x1A9, 0x1AC, 0x1AE, 0x1B1, 0x1B1, 0x1B5, 0x1B6, + 0x1B8, 0x1B8, 0x1BC, 0x1BD, 0x1D5, 0x1DC, 0x1DE, 0x1E5, + 0x1EA, 0x1ED, 0x1F0, 0x1F0, 0x1F4, 0x1F5, 0x1FA, 0x1FF, + 0x21E, 0x233, 0x237, 0x243, 0x245, 0x24B, 0x24E, 0x24F, + 0x305, 0x305, 0x30D, 0x30D, 0x316, 0x316, 0x321, 0x322, + 0x332, 0x332, 0x334, 0x334, 0x336, 0x336, 0x358, 0x358, + 0x400, 0x400, 0x40D, 0x40D, 0x450, 0x450, 0x45D, 0x45D, + 0x48A, 0x48F, 0x4C1, 0x4CE, 0x4DA, 0x4DB, 0x4EA, 0x4ED, + 0x4F6, 0x4F7, 0x4FA, 0x4FF, 0x510, 0x513, 0x591, 0x5BD, + 0x5BF, 0x5BF, 0x5C1, 0x5C2, 0x5C4, 0x5C5, 0x5C7, 0x5C7, + 0x5EF, 0x5F2, 0x610, 0x61A, 0x653, 0x653, 0x656, 0x65F, + 0x69B, 0x69E, 0x6A1, 0x6A1, 0x6A3, 0x6A3, 0x6B2, 0x6B2, + 0x6B4, 0x6B4, 0x6B8, 0x6B9, 0x6BF, 0x6BF, 0x6D6, 0x6DC, + 0x6DF, 0x6E4, 0x6E7, 0x6E8, 0x6EA, 0x6ED, 0x6FA, 0x6FC, + 0x750, 0x750, 0x753, 0x755, 0x757, 0x75F, 0x761, 0x761, + 0x764, 0x765, 0x769, 0x769, 0x76B, 0x76D, 0x772, 0x77D, + 0x889, 0x88D, 0x897, 0x89F, 0x8A1, 0x8A1, 0x8AA, 0x8AC, + 0x8B2, 0x8B4, 0x8B6, 0x8BA, 0x8C3, 0x8C6, 0x8C8, 0x8C8, + 0x8CA, 0x8E1, 0x8E3, 0x900, 0x904, 0x904, 0x929, 0x929, + 0x934, 0x934, 0x944, 0x944, 0x955, 0x955, 0x979, 0x97A, + 0x98C, 0x98C, 0x9D7, 0x9D7, 0x9FE, 0x9FE, 0xA01, 0xA01, + 0xA03, 0xA03, 0xA51, 0xA51, 0xA66, 0xA6F, 0xA72, 0xA73, + 0xA75, 0xA75, 0xA81, 0xA81, 0xAF9, 0xAFF, 0xB0C, 0xB0C, + 0xB35, 0xB35, 0xB44, 0xB44, 0xB55, 0xB55, 0xB57, 0xB57, + 0xB62, 0xB63, 0xB66, 0xB6F, 0xBD7, 0xBD7, 0xBE6, 0xBEF, + 0xC01, 0xC01, 0xC04, 0xC04, 0xC0C, 0xC0C, 0xC31, 0xC31, + 0xC3C, 0xC3C, 0xC55, 0xC56, 0xC5A, 0xC5A, 0xC5D, 0xC5D, + 0xC62, 0xC63, 0xC66, 0xC6F, 0xC80, 0xC80, 0xCBC, 0xCBC, + 0xCC4, 0xCC4, 0xCD5, 0xCD6, 0xCDD, 0xCDD, 0xCF3, 0xCF3, + 0xD00, 0xD00, 0xD0C, 0xD0C, 0xD29, 0xD29, 0xD44, 0xD44, + 0xD54, 0xD56, 0xD62, 0xD63, 0xD66, 0xD6F, 0xD8E, 0xD90, + 0xDDF, 0xDDF, 0xDE6, 0xDEF, 0xDF3, 0xDF3, 0xE4E, 0xE4E, + 0xE86, 0xE86, 0xE89, 0xE89, 0xE8C, 0xE8C, 0xE8E, 0xE93, + 0xE98, 0xE98, 0xEA0, 0xEA0, 0xEA8, 0xEA9, 0xEAC, 0xEAC, + 0xEBA, 0xEBA, 0xECE, 0xECE, 0xEDE, 0xEDF, 0xF39, 0xF39, + 0xF6A, 0xF6C, 0xF82, 0xF83, 0xFAE, 0xFB0, 0xFC6, 0xFC6, + 0x1050, 0x1059, 0x1065, 0x1074, 0x108B, 0x108E, 0x1090, 0x109D, + 0x10F7, 0x10FA, 0x10FD, 0x10FF, 0x1207, 0x1207, 0x1287, 0x1287, + 0x12AF, 0x12AF, 0x12F8, 0x12FF, 0x130F, 0x130F, 0x131F, 0x131F, + 0x1347, 0x1347, 0x135A, 0x135A, 0x135D, 0x135F, 0x1380, 0x138F, + 0x179D, 0x179E, 0x17A9, 0x17A9, 0x17D7, 0x17D7, 0x18A9, 0x18A9, + 0x1AC1, 0x1ACE, 0x1C89, 0x1C8A, 0x1E02, 0x1E0B, 0x1E0E, 0x1E11, + 0x1E14, 0x1E17, 0x1E1C, 0x1E1F, 0x1E22, 0x1E23, 0x1E26, 0x1E29, + 0x1E2E, 0x1E35, 0x1E38, 0x1E3B, 0x1E40, 0x1E41, 0x1E4C, 0x1E59, + 0x1E5C, 0x1E61, 0x1E64, 0x1E6B, 0x1E6E, 0x1E6F, 0x1E78, 0x1E8B, + 0x1E8E, 0x1E91, 0x1E94, 0x1E99, 0x2054, 0x2054, 0x218A, 0x218B, + 0x2BEC, 0x2BEF, 0x2C68, 0x2C6C, 0x2D80, 0x2D96, 0x2DA0, 0x2DA6, + 0x2DA8, 0x2DAE, 0x2DB0, 0x2DB6, 0x2DB8, 0x2DBE, 0x2DC0, 0x2DC6, + 0x2DC8, 0x2DCE, 0x2DD0, 0x2DD6, 0x2DD8, 0x2DDE, 0x3099, 0x309A, + 0x3400, 0x3446, 0x3448, 0x3472, 0x3474, 0x34E3, 0x34E5, 0x3576, + 0x3578, 0x359D, 0x359F, 0x35A0, 0x35A2, 0x35AC, 0x35AE, 0x35BE, + 0x35C0, 0x35CD, 0x35CF, 0x35F2, 0x35F4, 0x35FD, 0x35FF, 0x360D, + 0x360F, 0x3619, 0x361B, 0x3917, 0x3919, 0x395F, 0x3961, 0x396D, + 0x396F, 0x39CE, 0x39D1, 0x39DA, 0x39DC, 0x39DE, 0x39E0, 0x39F7, + 0x39F9, 0x39FD, 0x39FF, 0x3A17, 0x3A19, 0x3A51, 0x3A53, 0x3A5B, + 0x3A5D, 0x3A66, 0x3A68, 0x3A72, 0x3A74, 0x3B38, 0x3B3A, 0x3B4D, + 0x3B4F, 0x3BA2, 0x3BA4, 0x3C6D, 0x3C6F, 0x3CDF, 0x3CE1, 0x3DE6, + 0x3DE8, 0x3DEA, 0x3DEC, 0x3E73, 0x3E75, 0x3ECF, 0x3ED1, 0x4055, + 0x4057, 0x4064, 0x4066, 0x4069, 0x406B, 0x40BA, 0x40BC, 0x40DE, + 0x40E0, 0x4136, 0x4138, 0x415E, 0x4160, 0x4336, 0x4338, 0x43AB, + 0x43AD, 0x43B0, 0x43B2, 0x43D2, 0x43D4, 0x43DC, 0x43DE, 0x4442, + 0x4444, 0x44D5, 0x44D7, 0x44E9, 0x44EB, 0x4605, 0x4607, 0x464B, + 0x464D, 0x4660, 0x4662, 0x4722, 0x4724, 0x4728, 0x472A, 0x477B, + 0x477D, 0x478C, 0x478E, 0x47F3, 0x47F5, 0x4881, 0x4883, 0x4946, + 0x4948, 0x4979, 0x497B, 0x497C, 0x497E, 0x4981, 0x4984, 0x4984, + 0x4987, 0x499A, 0x499C, 0x499E, 0x49A0, 0x49B5, 0x49B8, 0x4A11, + 0x4A13, 0x4AB7, 0x4AB9, 0x4C76, 0x4C78, 0x4C7C, 0x4C7E, 0x4C80, + 0x4C82, 0x4C84, 0x4C86, 0x4C9C, 0x4CA4, 0x4D12, 0x4D1A, 0x4DAD, + 0x4DAF, 0x4DBF, 0x4E12, 0x4E12, 0x4E29, 0x4E29, 0x4E68, 0x4E68, + 0x4E79, 0x4E79, 0x4E96, 0x4E96, 0x4EA3, 0x4EA3, 0x4EBC, 0x4EBC, + 0x4ECC, 0x4ECC, 0x4EE7, 0x4EE7, 0x4EF8, 0x4EFA, 0x4EFC, 0x4EFC, + 0x4EFE, 0x4EFE, 0x4F07, 0x4F07, 0x4F16, 0x4F16, 0x4F28, 0x4F28, + 0x4F31, 0x4F31, 0x4F35, 0x4F35, 0x4F37, 0x4F37, 0x4F40, 0x4F40, + 0x4F44, 0x4F44, 0x4F71, 0x4F71, 0x4F8C, 0x4F8C, 0x4F8E, 0x4F8E, + 0x4FA2, 0x4FA2, 0x4FBD, 0x4FBD, 0x4FC6, 0x4FC6, 0x4FC8, 0x4FC8, + 0x4FCC, 0x4FCC, 0x4FE2, 0x4FE2, 0x4FFC, 0x4FFD, 0x5010, 0x5010, + 0x5034, 0x5034, 0x5038, 0x5038, 0x503D, 0x503D, 0x5042, 0x5042, + 0x5052, 0x5052, 0x5058, 0x5058, 0x507C, 0x507C, 0x5081, 0x5081, + 0x5093, 0x5093, 0x5097, 0x5097, 0x509F, 0x50A1, 0x50B9, 0x50B9, + 0x50C3, 0x50C3, 0x50D8, 0x50D8, 0x50DF, 0x50DF, 0x50E1, 0x50E2, + 0x50EB, 0x50EB, 0x50F4, 0x50F4, 0x50F7, 0x50F7, 0x511B, 0x511B, + 0x5128, 0x5128, 0x512B, 0x512B, 0x5142, 0x5142, 0x514A, 0x514A, + 0x514F, 0x514F, 0x5153, 0x5153, 0x5158, 0x5158, 0x5160, 0x5160, + 0x5164, 0x5164, 0x5172, 0x5172, 0x517E, 0x517E, 0x5183, 0x5184, + 0x518E, 0x518E, 0x51A1, 0x51A1, 0x51A3, 0x51A3, 0x51AD, 0x51AD, + 0x51B8, 0x51B8, 0x51BA, 0x51BA, 0x51C2, 0x51C2, 0x51D2, 0x51D3, + 0x51DF, 0x51DF, 0x51EC, 0x51EC, 0x51EE, 0x51EE, 0x51F2, 0x51F2, + 0x5253, 0x5253, 0x5266, 0x5266, 0x5279, 0x5279, 0x5285, 0x5285, + 0x528E, 0x528E, 0x52C4, 0x52C4, 0x52C8, 0x52C8, 0x52CC, 0x52CC, + 0x52CE, 0x52CE, 0x52D1, 0x52D1, 0x52D4, 0x52D4, 0x52E1, 0x52E1, + 0x52E5, 0x52E5, 0x52EE, 0x52EE, 0x5303, 0x5304, 0x5318, 0x5318, + 0x531B, 0x531B, 0x531E, 0x531E, 0x5327, 0x5327, 0x5329, 0x5329, + 0x5332, 0x5332, 0x5335, 0x5336, 0x5342, 0x5342, 0x535B, 0x535B, + 0x535D, 0x535D, 0x536A, 0x536A, 0x536D, 0x536D, 0x5380, 0x5380, + 0x53A1, 0x53A1, 0x53AA, 0x53AB, 0x53AF, 0x53AF, 0x53BA, 0x53BA, + 0x53C5, 0x53C5, 0x53CF, 0x53CF, 0x53DD, 0x53DE, 0x53E7, 0x53E7, + 0x53FF, 0x5400, 0x541A, 0x541A, 0x5422, 0x5422, 0x544C, 0x544C, + 0x545D, 0x545D, 0x5469, 0x5469, 0x548A, 0x548A, 0x54B5, 0x54B5, + 0x54F6, 0x54F6, 0x5515, 0x5515, 0x5518, 0x5519, 0x5547, 0x5547, + 0x5560, 0x5560, 0x557A, 0x557A, 0x55E0, 0x55E0, 0x55F8, 0x55F8, + 0x560A, 0x560B, 0x5620, 0x5620, 0x562B, 0x562B, 0x5637, 0x5637, + 0x563C, 0x563C, 0x5644, 0x5644, 0x564B, 0x564B, 0x5651, 0x5651, + 0x5656, 0x5656, 0x565F, 0x565F, 0x5661, 0x5661, 0x5675, 0x5675, + 0x567D, 0x567D, 0x5688, 0x5688, 0x568B, 0x568B, 0x5696, 0x5696, + 0x569E, 0x569E, 0x56BA, 0x56BA, 0x56CF, 0x56CF, 0x56D9, 0x56D9, + 0x56E6, 0x56E6, 0x56F6, 0x56F6, 0x56F8, 0x56F8, 0x56FB, 0x56FC, + 0x5705, 0x5705, 0x5711, 0x5711, 0x5717, 0x5717, 0x5721, 0x5721, + 0x5724, 0x5724, 0x573D, 0x573D, 0x5743, 0x5743, 0x5748, 0x5748, + 0x5755, 0x5756, 0x5758, 0x5758, 0x5763, 0x5763, 0x5778, 0x5778, + 0x5781, 0x5781, 0x5787, 0x5787, 0x5796, 0x5796, 0x57A8, 0x57A8, + 0x57CA, 0x57CA, 0x57D1, 0x57D1, 0x57DB, 0x57DB, 0x5817, 0x5818, + 0x5850, 0x5850, 0x5856, 0x5856, 0x5860, 0x5860, 0x5866, 0x5867, + 0x5877, 0x5877, 0x5895, 0x5895, 0x58AA, 0x58AA, 0x58B6, 0x58B6, + 0x58C0, 0x58C0, 0x58C3, 0x58C4, 0x58CD, 0x58CD, 0x58D0, 0x58D0, + 0x58E1, 0x58E1, 0x58E6, 0x58E6, 0x58F5, 0x58F5, 0x5901, 0x5901, + 0x5905, 0x5905, 0x5908, 0x5908, 0x5911, 0x5911, 0x5913, 0x5913, + 0x5923, 0x5923, 0x5933, 0x5933, 0x5936, 0x5936, 0x5959, 0x5959, + 0x595B, 0x595B, 0x59B7, 0x59B7, 0x59E7, 0x59E7, 0x5A24, 0x5A24, + 0x5A26, 0x5A26, 0x5A2C, 0x5A2C, 0x5A30, 0x5A30, 0x5A54, 0x5A54, + 0x5A59, 0x5A59, 0x5A6F, 0x5A6F, 0x5A71, 0x5A71, 0x5A87, 0x5A87, + 0x5A8D, 0x5A8D, 0x5AAB, 0x5AAB, 0x5AD3, 0x5AD3, 0x5AEF, 0x5AF0, + 0x5B0A, 0x5B0A, 0x5B0D, 0x5B0D, 0x5B39, 0x5B39, 0x5B46, 0x5B46, + 0x5B4F, 0x5B4F, 0x5B52, 0x5B52, 0x5B60, 0x5B61, 0x5B6F, 0x5B6F, + 0x5B79, 0x5B79, 0x5B7E, 0x5B7E, 0x5B86, 0x5B86, 0x5B90, 0x5B90, + 0x5BA9, 0x5BA9, 0x5BB2, 0x5BB2, 0x5BB7, 0x5BB7, 0x5BBC, 0x5BBC, + 0x5BC8, 0x5BC8, 0x5BDA, 0x5BDA, 0x5C00, 0x5C00, 0x5C1B, 0x5C1B, + 0x5C23, 0x5C23, 0x5C26, 0x5C26, 0x5C29, 0x5C29, 0x5C36, 0x5C36, + 0x5C5A, 0x5C5A, 0x5C85, 0x5C85, 0x5CB4, 0x5CB4, 0x5CB9, 0x5CB9, + 0x5CD5, 0x5CD5, 0x5CDD, 0x5CDD, 0x5CF5, 0x5CF5, 0x5D2B, 0x5D2B, + 0x5D2F, 0x5D2F, 0x5D3B, 0x5D3B, 0x5D53, 0x5D53, 0x5D57, 0x5D57, + 0x5D60, 0x5D60, 0x5D83, 0x5D83, 0x5D96, 0x5D96, 0x5DA3, 0x5DA4, + 0x5DAB, 0x5DAB, 0x5DB3, 0x5DB3, 0x5DB9, 0x5DB9, 0x5DC4, 0x5DC4, + 0x5DD7, 0x5DD7, 0x5DDA, 0x5DDA, 0x5DDC, 0x5DDC, 0x5DF6, 0x5DF6, + 0x5E12, 0x5E12, 0x5E48, 0x5E48, 0x5E51, 0x5E51, 0x5E92, 0x5E92, + 0x5EBA, 0x5EBA, 0x5EC0, 0x5EC0, 0x5EEB, 0x5EEB, 0x5EF9, 0x5EF9, + 0x5F0E, 0x5F0E, 0x5F3B, 0x5F3B, 0x5F3D, 0x5F3D, 0x5F8F, 0x5F8F, + 0x5F9A, 0x5F9A, 0x5FA3, 0x5FA4, 0x5FB0, 0x5FB0, 0x5FC2, 0x5FC2, + 0x5FCE, 0x5FCE, 0x5FDB, 0x5FDB, 0x5FE2, 0x5FE2, 0x5FEC, 0x5FEC, + 0x5FFC, 0x5FFC, 0x6023, 0x6023, 0x6056, 0x6056, 0x6061, 0x6061, + 0x6071, 0x6071, 0x6074, 0x6074, 0x6091, 0x6091, 0x6093, 0x6093, + 0x60A5, 0x60A5, 0x60D2, 0x60D2, 0x60D6, 0x60D6, 0x60DE, 0x60DE, + 0x60E5, 0x60E5, 0x60FD, 0x60FD, 0x6102, 0x6102, 0x6107, 0x6107, + 0x6111, 0x6111, 0x611E, 0x611E, 0x6131, 0x6131, 0x6133, 0x6133, + 0x6135, 0x6135, 0x6138, 0x6139, 0x6160, 0x6160, 0x617B, 0x617B, + 0x617F, 0x617F, 0x6186, 0x6186, 0x6197, 0x6197, 0x619C, 0x619C, + 0x61B9, 0x61B9, 0x61BB, 0x61BB, 0x61D3, 0x61D3, 0x61D5, 0x61D5, + 0x61EC, 0x61EC, 0x61EF, 0x61EF, 0x6205, 0x6205, 0x6235, 0x6235, + 0x6239, 0x6239, 0x6257, 0x6257, 0x628D, 0x628D, 0x629D, 0x629D, + 0x62DE, 0x62DE, 0x62EA, 0x62EA, 0x630A, 0x630A, 0x6317, 0x6317, + 0x6331, 0x6331, 0x6337, 0x6337, 0x635B, 0x635B, 0x638B, 0x638B, + 0x6393, 0x6393, 0x63D1, 0x63D1, 0x643B, 0x643C, 0x6449, 0x6449, + 0x645A, 0x645A, 0x647E, 0x647E, 0x6486, 0x6486, 0x64A1, 0x64A1, + 0x64AF, 0x64AF, 0x64B6, 0x64B6, 0x64C8, 0x64C8, 0x64D5, 0x64D5, + 0x64EE, 0x64EE, 0x64F5, 0x64F5, 0x64F9, 0x64F9, 0x6502, 0x6502, + 0x650A, 0x650A, 0x651F, 0x651F, 0x6528, 0x6528, 0x6540, 0x6540, + 0x6542, 0x6542, 0x655A, 0x655A, 0x655F, 0x655F, 0x657D, 0x657D, + 0x658A, 0x658A, 0x659A, 0x659A, 0x65B5, 0x65B5, 0x65BE, 0x65BE, + 0x65C8, 0x65C9, 0x65D1, 0x65D1, 0x65D8, 0x65D8, 0x65DC, 0x65DC, + 0x65E4, 0x65E4, 0x65EA, 0x65EA, 0x65F9, 0x65F9, 0x65FE, 0x65FE, + 0x6617, 0x6617, 0x662C, 0x662C, 0x6637, 0x6638, 0x6648, 0x6648, + 0x664D, 0x664D, 0x6660, 0x6660, 0x6663, 0x6663, 0x6692, 0x6692, + 0x669C, 0x669C, 0x669E, 0x669E, 0x66AC, 0x66AD, 0x66D0, 0x66D0, + 0x66D3, 0x66D3, 0x66D7, 0x66D7, 0x66DF, 0x66DF, 0x66EF, 0x66EF, + 0x6702, 0x6702, 0x6707, 0x6707, 0x6719, 0x6719, 0x6724, 0x6724, + 0x6729, 0x6729, 0x6767, 0x6767, 0x6788, 0x6788, 0x6796, 0x6796, + 0x67BD, 0x67BD, 0x67BF, 0x67BF, 0x67D5, 0x67D5, 0x67D7, 0x67D7, + 0x67F9, 0x67F9, 0x6801, 0x6801, 0x6815, 0x6815, 0x6827, 0x6827, + 0x6830, 0x6830, 0x6858, 0x6858, 0x685A, 0x685A, 0x685E, 0x685E, + 0x687A, 0x687A, 0x6895, 0x6895, 0x6899, 0x6899, 0x68A5, 0x68A5, + 0x68B8, 0x68B8, 0x68C3, 0x68C3, 0x68D9, 0x68D9, 0x68E2, 0x68E2, + 0x68E5, 0x68E5, 0x6909, 0x6909, 0x693E, 0x693E, 0x694D, 0x694D, + 0x699F, 0x699F, 0x69A2, 0x69A2, 0x69C0, 0x69C0, 0x69D1, 0x69D2, + 0x69D5, 0x69D7, 0x6A03, 0x6A03, 0x6A1C, 0x6A1C, 0x6A24, 0x6A24, + 0x6A37, 0x6A37, 0x6A4A, 0x6A4A, 0x6A5C, 0x6A5C, 0x6A6E, 0x6A6E, + 0x6A70, 0x6A70, 0x6A86, 0x6A86, 0x6A8A, 0x6A8A, 0x6A8F, 0x6A8F, + 0x6A99, 0x6A99, 0x6A9D, 0x6A9D, 0x6AB1, 0x6AB1, 0x6ABE, 0x6ABE, + 0x6AC0, 0x6AC0, 0x6AC4, 0x6AC4, 0x6AC9, 0x6AC9, 0x6AD8, 0x6AD8, + 0x6AE9, 0x6AE9, 0x6B0E, 0x6B0E, 0x6B1B, 0x6B1B, 0x6B2E, 0x6B2E, + 0x6B35, 0x6B35, 0x6B40, 0x6B40, 0x6B57, 0x6B58, 0x6B5D, 0x6B5D, + 0x6B68, 0x6B68, 0x6B6C, 0x6B6C, 0x6B6E, 0x6B6E, 0x6B71, 0x6B71, + 0x6B75, 0x6B75, 0x6B7D, 0x6B7D, 0x6BB8, 0x6BB8, 0x6BE9, 0x6BE9, + 0x6BF1, 0x6BF1, 0x6BF4, 0x6BF4, 0x6BFA, 0x6BFA, 0x6C0A, 0x6C0A, + 0x6C1C, 0x6C1C, 0x6C2D, 0x6C2D, 0x6C3C, 0x6C3C, 0x6C45, 0x6C45, + 0x6C6C, 0x6C6C, 0x6C6E, 0x6C6E, 0x6CA0, 0x6CA0, 0x6CD8, 0x6CD8, + 0x6CF4, 0x6CF4, 0x6D02, 0x6D02, 0x6D1C, 0x6D1C, 0x6D24, 0x6D24, + 0x6D71, 0x6D71, 0x6D81, 0x6D81, 0x6D96, 0x6D96, 0x6DB0, 0x6DB1, + 0x6DB6, 0x6DB6, 0x6DFE, 0x6DFF, 0x6E01, 0x6E02, 0x6E06, 0x6E06, + 0x6E12, 0x6E12, 0x6E18, 0x6E18, 0x6E2A, 0x6E2A, 0x6E4C, 0x6E4C, + 0x6E6C, 0x6E6C, 0x6E7B, 0x6E7D, 0x6E8B, 0x6E8B, 0x6E95, 0x6E95, + 0x6EDB, 0x6EDB, 0x6EE3, 0x6EE3, 0x6F04, 0x6F04, 0x6F0B, 0x6F0B, + 0x6F42, 0x6F42, 0x6F48, 0x6F48, 0x6F4A, 0x6F4A, 0x6F79, 0x6F79, + 0x6F98, 0x6F98, 0x6F9A, 0x6F9A, 0x6F9F, 0x6F9F, 0x6FB7, 0x6FB7, + 0x6FC5, 0x6FC5, 0x6FD0, 0x6FD0, 0x6FD3, 0x6FD3, 0x6FF5, 0x6FF5, + 0x6FFD, 0x6FFD, 0x7010, 0x7010, 0x7013, 0x7013, 0x7047, 0x7047, + 0x704B, 0x704B, 0x704E, 0x704E, 0x7072, 0x7073, 0x707B, 0x707B, + 0x7081, 0x7081, 0x708D, 0x708D, 0x7097, 0x7097, 0x709B, 0x709B, + 0x70AA, 0x70AA, 0x70B2, 0x70B2, 0x70B6, 0x70B6, 0x70D5, 0x70D5, + 0x70FE, 0x70FE, 0x7108, 0x7108, 0x7124, 0x7124, 0x7133, 0x7134, + 0x7157, 0x7157, 0x716B, 0x716B, 0x716D, 0x716D, 0x718D, 0x718D, + 0x7196, 0x7196, 0x71A6, 0x71A6, 0x71AB, 0x71AB, 0x71B6, 0x71B6, + 0x71CC, 0x71CC, 0x71D3, 0x71D3, 0x71F3, 0x71F3, 0x71FA, 0x71FA, + 0x720B, 0x720B, 0x7211, 0x7211, 0x7215, 0x7215, 0x7217, 0x7217, + 0x7220, 0x7220, 0x7224, 0x7225, 0x722F, 0x722F, 0x7234, 0x7234, + 0x7245, 0x7245, 0x724E, 0x724E, 0x7250, 0x7250, 0x7255, 0x7255, + 0x72AB, 0x72AB, 0x72BE, 0x72BE, 0x7302, 0x7302, 0x7310, 0x7310, + 0x7328, 0x7328, 0x7353, 0x7353, 0x739C, 0x739C, 0x73C1, 0x73C1, + 0x73F3, 0x73F3, 0x73FB, 0x73FB, 0x7418, 0x7418, 0x7439, 0x7439, + 0x743E, 0x743E, 0x7447, 0x7447, 0x7449, 0x7449, 0x7458, 0x7458, + 0x747B, 0x747B, 0x7484, 0x7484, 0x7496, 0x7496, 0x749D, 0x749D, + 0x74C7, 0x74C7, 0x74C9, 0x74C9, 0x74CC, 0x74CC, 0x74EB, 0x74EB, + 0x7520, 0x7520, 0x7541, 0x7541, 0x7552, 0x7552, 0x7555, 0x7555, + 0x755E, 0x755E, 0x7561, 0x7561, 0x7571, 0x7571, 0x757B, 0x757B, + 0x7585, 0x7585, 0x75A9, 0x75A9, 0x75B7, 0x75B7, 0x75DC, 0x75DC, + 0x75EE, 0x75EE, 0x762C, 0x762C, 0x7644, 0x7645, 0x7651, 0x7651, + 0x7655, 0x7655, 0x7673, 0x7673, 0x768D, 0x768D, 0x76A1, 0x76A2, + 0x76A5, 0x76A5, 0x76A8, 0x76A8, 0x76B3, 0x76B3, 0x76B6, 0x76B6, + 0x76C1, 0x76C1, 0x76CB, 0x76CB, 0x76D9, 0x76D9, 0x76EB, 0x76EB, + 0x7700, 0x7700, 0x7702, 0x7702, 0x770E, 0x770E, 0x7721, 0x7721, + 0x772B, 0x772B, 0x773F, 0x773F, 0x7742, 0x7742, 0x7764, 0x7764, + 0x7796, 0x7796, 0x77A4, 0x77A4, 0x77BE, 0x77BE, 0x77C1, 0x77C1, + 0x77D2, 0x77D2, 0x77DD, 0x77DD, 0x77E4, 0x77E4, 0x77E6, 0x77E6, + 0x77F4, 0x77F5, 0x7824, 0x7824, 0x7836, 0x7836, 0x7842, 0x7842, + 0x7846, 0x7846, 0x784B, 0x784B, 0x7876, 0x7876, 0x7888, 0x7888, + 0x78C2, 0x78C2, 0x78C7, 0x78C7, 0x78D2, 0x78D2, 0x78F0, 0x78F0, + 0x78F8, 0x78F8, 0x7900, 0x7900, 0x7908, 0x7908, 0x790D, 0x790D, + 0x7915, 0x7915, 0x791F, 0x7920, 0x7932, 0x7932, 0x7936, 0x7936, + 0x7959, 0x7959, 0x796C, 0x796C, 0x796E, 0x796E, 0x7975, 0x7976, + 0x7986, 0x7987, 0x799E, 0x799E, 0x79A9, 0x79A9, 0x79BC, 0x79BC, + 0x79C4, 0x79C4, 0x79C7, 0x79C7, 0x79CC, 0x79CC, 0x79D4, 0x79D4, + 0x79D7, 0x79D7, 0x7A01, 0x7A01, 0x7A07, 0x7A07, 0x7A09, 0x7A09, + 0x7A2C, 0x7A2C, 0x7A38, 0x7A38, 0x7A3A, 0x7A3A, 0x7A64, 0x7A64, + 0x7A6A, 0x7A6A, 0x7A6F, 0x7A6F, 0x7A82, 0x7A82, 0x7A9A, 0x7A9B, + 0x7AB9, 0x7AB9, 0x7ABB, 0x7ABD, 0x7AC2, 0x7AC2, 0x7AC6, 0x7AC6, + 0x7AE9, 0x7AE9, 0x7AF5, 0x7AF5, 0x7AFC, 0x7AFC, 0x7B07, 0x7B07, + 0x7B1F, 0x7B1F, 0x7B27, 0x7B27, 0x7B29, 0x7B29, 0x7B42, 0x7B42, + 0x7B53, 0x7B53, 0x7BA3, 0x7BA3, 0x7BA5, 0x7BA5, 0x7BB0, 0x7BB0, + 0x7BB2, 0x7BB2, 0x7BFA, 0x7BFA, 0x7C1B, 0x7C1B, 0x7C2E, 0x7C2F, + 0x7C52, 0x7C52, 0x7C55, 0x7C55, 0x7C5D, 0x7C5D, 0x7C76, 0x7C76, + 0x7C87, 0x7C87, 0x7C93, 0x7C93, 0x7C9A, 0x7C9A, 0x7CAC, 0x7CAC, + 0x7CD3, 0x7CD3, 0x7CDA, 0x7CDB, 0x7CE1, 0x7CE1, 0x7CE3, 0x7CE3, + 0x7CE5, 0x7CE6, 0x7CFC, 0x7CFC, 0x7CFF, 0x7CFF, 0x7D23, 0x7D23, + 0x7D2A, 0x7D2A, 0x7D2D, 0x7D2D, 0x7D48, 0x7D48, 0x7D4D, 0x7D4D, + 0x7D5A, 0x7D5A, 0x7D64, 0x7D64, 0x7D78, 0x7D78, 0x7D82, 0x7D82, + 0x7D95, 0x7D95, 0x7D98, 0x7D98, 0x7DA4, 0x7DA4, 0x7DA8, 0x7DA8, + 0x7DCD, 0x7DCD, 0x7DD3, 0x7DD3, 0x7DE5, 0x7DE5, 0x7DEB, 0x7DEB, + 0x7DFD, 0x7DFF, 0x7E18, 0x7E18, 0x7E5B, 0x7E5B, 0x7E64, 0x7E64, + 0x7E9D, 0x7E9D, 0x7F3B, 0x7F3C, 0x7F41, 0x7F41, 0x7F46, 0x7F46, + 0x7F59, 0x7F59, 0x7F84, 0x7F84, 0x7F90, 0x7F90, 0x7F97, 0x7F97, + 0x7F99, 0x7F99, 0x7FB4, 0x7FB4, 0x7FD6, 0x7FD6, 0x7FDD, 0x7FDD, + 0x7FE4, 0x7FE4, 0x800A, 0x800A, 0x802F, 0x802F, 0x803C, 0x803C, + 0x8040, 0x8040, 0x8066, 0x8066, 0x8088, 0x8088, 0x808E, 0x808E, + 0x8094, 0x8094, 0x80A6, 0x80A8, 0x80B3, 0x80B3, 0x80B9, 0x80B9, + 0x80DF, 0x80DF, 0x8103, 0x8104, 0x8134, 0x8135, 0x8184, 0x8184, + 0x8190, 0x8190, 0x8196, 0x8196, 0x81CB, 0x81CB, 0x81E4, 0x81E4, + 0x81EF, 0x81F0, 0x8213, 0x8213, 0x8224, 0x8224, 0x8241, 0x8241, + 0x8265, 0x8265, 0x828C, 0x828C, 0x82B2, 0x82B2, 0x82E2, 0x82E2, + 0x82FC, 0x82FC, 0x830A, 0x830A, 0x8310, 0x8310, 0x8330, 0x8330, + 0x8355, 0x8355, 0x83BE, 0x83BE, 0x83E6, 0x83E6, 0x83ED, 0x83ED, + 0x8414, 0x8414, 0x8416, 0x8417, 0x841F, 0x841F, 0x8458, 0x8458, + 0x8483, 0x8483, 0x8495, 0x8495, 0x84B7, 0x84B7, 0x84C3, 0x84C3, + 0x84ED, 0x84ED, 0x8505, 0x8505, 0x8510, 0x8510, 0x8532, 0x8533, + 0x854C, 0x854C, 0x8550, 0x8550, 0x857F, 0x857F, 0x8593, 0x8593, + 0x85B2, 0x85B2, 0x85BB, 0x85BB, 0x85CC, 0x85CC, 0x85EE, 0x85EE, + 0x85F3, 0x85F3, 0x85FC, 0x85FC, 0x8603, 0x8603, 0x860D, 0x860E, + 0x8610, 0x8610, 0x8615, 0x8615, 0x861D, 0x861D, 0x8637, 0x8637, + 0x8657, 0x8657, 0x8675, 0x8675, 0x8689, 0x8689, 0x8692, 0x8692, + 0x86A0, 0x86A0, 0x86A6, 0x86A6, 0x86D5, 0x86D5, 0x86E0, 0x86E0, + 0x86E7, 0x86E7, 0x86FD, 0x86FD, 0x871D, 0x871D, 0x872F, 0x872F, + 0x873D, 0x873D, 0x8745, 0x8745, 0x8771, 0x8771, 0x878E, 0x878E, + 0x8799, 0x8799, 0x87DA, 0x87DA, 0x87F0, 0x87F0, 0x8807, 0x8807, + 0x8812, 0x8812, 0x882D, 0x882D, 0x883A, 0x883A, 0x8847, 0x8847, + 0x8858, 0x8858, 0x885C, 0x885C, 0x885F, 0x885F, 0x887A, 0x887A, + 0x88E6, 0x88E6, 0x88E9, 0x88E9, 0x88ED, 0x88ED, 0x8903, 0x8903, + 0x890F, 0x890F, 0x8924, 0x8924, 0x8965, 0x8965, 0x8975, 0x8975, + 0x897D, 0x897D, 0x898D, 0x898D, 0x8990, 0x8990, 0x8994, 0x8994, + 0x8999, 0x8999, 0x89B0, 0x89B0, 0x89B4, 0x89B4, 0x89BB, 0x89BC, + 0x89EE, 0x89EE, 0x89F5, 0x89F5, 0x89F9, 0x89F9, 0x89FD, 0x89FD, + 0x8A05, 0x8A06, 0x8A14, 0x8A14, 0x8A19, 0x8A19, 0x8A20, 0x8A21, + 0x8A2B, 0x8A2B, 0x8A3D, 0x8A3D, 0x8A4B, 0x8A4B, 0x8A64, 0x8A64, + 0x8A78, 0x8A78, 0x8A7D, 0x8A7D, 0x8A88, 0x8A88, 0x8A9F, 0x8A9F, + 0x8AAF, 0x8AAF, 0x8AB7, 0x8AB7, 0x8AD0, 0x8AD0, 0x8AEC, 0x8AEC, + 0x8B29, 0x8B29, 0x8B32, 0x8B32, 0x8B38, 0x8B38, 0x8B3F, 0x8B3F, + 0x8B61, 0x8B62, 0x8B69, 0x8B69, 0x8B75, 0x8B75, 0x8B7C, 0x8B7C, + 0x8B81, 0x8B81, 0x8B87, 0x8B87, 0x8B8D, 0x8B8D, 0x8B8F, 0x8B8F, + 0x8B9B, 0x8B9B, 0x8C38, 0x8C38, 0x8C40, 0x8C40, 0x8C44, 0x8C44, + 0x8C51, 0x8C53, 0x8C58, 0x8C58, 0x8C74, 0x8C74, 0x8C7F, 0x8C7F, + 0x8C83, 0x8C83, 0x8C87, 0x8C87, 0x8C8B, 0x8C8B, 0x8C9B, 0x8C9B, + 0x8CA6, 0x8CA6, 0x8CCB, 0x8CCB, 0x8CD6, 0x8CD6, 0x8CD8, 0x8CD8, + 0x8CE9, 0x8CE9, 0x8CF7, 0x8CF7, 0x8D01, 0x8D01, 0x8D11, 0x8D12, + 0x8D7C, 0x8D7C, 0x8DA6, 0x8DA6, 0x8DC0, 0x8DC0, 0x8DE5, 0x8DE5, + 0x8E01, 0x8E01, 0x8E0B, 0x8E0B, 0x8E32, 0x8E32, 0x8E46, 0x8E46, + 0x8E4F, 0x8E4F, 0x8E6E, 0x8E6E, 0x8E75, 0x8E75, 0x8E77, 0x8E77, + 0x8E79, 0x8E79, 0x8E9B, 0x8E9B, 0x8EA2, 0x8EA2, 0x8EB3, 0x8EB3, + 0x8EB6, 0x8EB7, 0x8EC1, 0x8EC1, 0x8EC4, 0x8EC4, 0x8ED9, 0x8ED9, + 0x8EF0, 0x8EF0, 0x8F0F, 0x8F0F, 0x8F2D, 0x8F2D, 0x8F3A, 0x8F3A, + 0x8F41, 0x8F41, 0x8F9D, 0x8F9D, 0x8FA4, 0x8FA4, 0x8FB3, 0x8FB3, + 0x8FC3, 0x8FC3, 0x8FCA, 0x8FCA, 0x8FE7, 0x8FE7, 0x902A, 0x902A, + 0x902C, 0x902C, 0x9037, 0x9037, 0x9040, 0x9040, 0x9046, 0x9046, + 0x90AB, 0x90AB, 0x90CC, 0x90CD, 0x90D2, 0x90D2, 0x90F6, 0x90F6, + 0x910A, 0x910A, 0x913C, 0x913D, 0x9159, 0x9159, 0x917B, 0x917B, + 0x9195, 0x9195, 0x9198, 0x9198, 0x91A9, 0x91A9, 0x91BF, 0x91BF, + 0x91C4, 0x91C4, 0x91E0, 0x91E0, 0x91EF, 0x91EF, 0x9213, 0x9213, + 0x921F, 0x921F, 0x9222, 0x9222, 0x9243, 0x9243, 0x9269, 0x926A, + 0x9281, 0x9281, 0x9284, 0x9284, 0x929E, 0x929E, 0x92BD, 0x92BD, + 0x92D4, 0x92D4, 0x92DB, 0x92DB, 0x92E2, 0x92E2, 0x931C, 0x931C, + 0x9330, 0x9331, 0x9362, 0x9362, 0x9368, 0x9368, 0x936B, 0x936B, + 0x936F, 0x936F, 0x9373, 0x9373, 0x9378, 0x9378, 0x937F, 0x937F, + 0x9381, 0x9381, 0x938B, 0x938B, 0x939C, 0x939C, 0x93A0, 0x93A0, + 0x93AB, 0x93AB, 0x93BB, 0x93BB, 0x93E0, 0x93E0, 0x93F3, 0x93F3, + 0x9402, 0x9402, 0x9417, 0x9417, 0x941C, 0x941C, 0x941E, 0x941F, + 0x9424, 0x9424, 0x9443, 0x9443, 0x944E, 0x944E, 0x946C, 0x946C, + 0x947B, 0x947B, 0x9578, 0x9579, 0x957E, 0x957E, 0x9585, 0x9585, + 0x9597, 0x9597, 0x95B3, 0x95B4, 0x95B8, 0x95B8, 0x95C1, 0x95C1, + 0x95D9, 0x95D9, 0x95DD, 0x95DD, 0x9625, 0x9626, 0x9629, 0x9629, + 0x963E, 0x963E, 0x9656, 0x9657, 0x9679, 0x9679, 0x967B, 0x967B, + 0x967F, 0x967F, 0x9681, 0x9682, 0x968C, 0x968C, 0x9696, 0x9696, + 0x969A, 0x969A, 0x969D, 0x969D, 0x969F, 0x969F, 0x96AB, 0x96AB, + 0x96AF, 0x96AF, 0x96B5, 0x96B5, 0x96E4, 0x96E4, 0x96E6, 0x96E7, + 0x96FC, 0x96FC, 0x9714, 0x9714, 0x9717, 0x9717, 0x971A, 0x971B, + 0x9733, 0x9734, 0x9737, 0x9737, 0x9740, 0x9741, 0x974D, 0x974D, + 0x9757, 0x9757, 0x9763, 0x9763, 0x9775, 0x9775, 0x9787, 0x9787, + 0x9789, 0x9789, 0x979B, 0x979B, 0x97A9, 0x97A9, 0x97B0, 0x97B1, + 0x97B5, 0x97B5, 0x97BE, 0x97BE, 0x97C0, 0x97C0, 0x97D2, 0x97D2, + 0x97FC, 0x97FC, 0x981F, 0x981F, 0x9825, 0x9825, 0x982A, 0x982A, + 0x9833, 0x9833, 0x983A, 0x983A, 0x983E, 0x983E, 0x9842, 0x9842, + 0x9847, 0x9847, 0x9856, 0x9856, 0x9866, 0x9866, 0x9868, 0x9868, + 0x98B7, 0x98B7, 0x98CA, 0x98CA, 0x98E4, 0x98E4, 0x98EC, 0x98EC, + 0x98F1, 0x98F1, 0x98F8, 0x98F8, 0x98FB, 0x98FB, 0x9919, 0x9919, + 0x993B, 0x993B, 0x9944, 0x9944, 0x995A, 0x995A, 0x995D, 0x995D, + 0x99BF, 0x99BF, 0x99E0, 0x99E0, 0x99E6, 0x99E6, 0x99EB, 0x99EB, + 0x99F5, 0x99F5, 0x9A10, 0x9A10, 0x9A17, 0x9A18, 0x9A3B, 0x9A3B, + 0x9A51, 0x9A51, 0x9A58, 0x9A58, 0x9A5D, 0x9A5D, 0x9A63, 0x9A63, + 0x9AA9, 0x9AA9, 0x9ABD, 0x9ABD, 0x9AC8, 0x9AC8, 0x9AD7, 0x9AD7, + 0x9AE0, 0x9AE0, 0x9AE4, 0x9AE4, 0x9AE8, 0x9AE9, 0x9AF0, 0x9AF0, + 0x9B00, 0x9B00, 0x9B02, 0x9B02, 0x9B09, 0x9B09, 0x9B14, 0x9B14, + 0x9B1B, 0x9B1B, 0x9B34, 0x9B34, 0x9B3D, 0x9B3D, 0x9B40, 0x9B40, + 0x9B50, 0x9B50, 0x9B57, 0x9B57, 0x9B62, 0x9B62, 0x9B72, 0x9B72, + 0x9B89, 0x9B89, 0x9B8C, 0x9B8C, 0x9B99, 0x9B99, 0x9BC2, 0x9BC2, + 0x9BF6, 0x9BF6, 0x9C00, 0x9C01, 0x9C03, 0x9C03, 0x9C42, 0x9C42, + 0x9C4F, 0x9C4F, 0x9C51, 0x9C51, 0x9C61, 0x9C61, 0x9C64, 0x9C64, + 0x9C7B, 0x9C7B, 0x9D0C, 0x9D0D, 0x9D11, 0x9D11, 0x9D27, 0x9D27, + 0x9D35, 0x9D35, 0x9D3C, 0x9D3C, 0x9D6D, 0x9D6D, 0x9D95, 0x9D95, + 0x9DAE, 0x9DAE, 0x9DBD, 0x9DBD, 0x9DC0, 0x9DC0, 0x9DEA, 0x9DEA, + 0x9DFC, 0x9DFC, 0x9E0E, 0x9E0E, 0x9E16, 0x9E16, 0x9E1C, 0x9E1C, + 0x9E7B, 0x9E7B, 0x9E8F, 0x9E90, 0x9E98, 0x9E98, 0x9E9E, 0x9E9E, + 0x9EA2, 0x9EA2, 0x9EAB, 0x9EAC, 0x9EB1, 0x9EB1, 0x9EEC, 0x9EEC, + 0x9EF1, 0x9EF1, 0x9F03, 0x9F03, 0x9F11, 0x9F11, 0x9F14, 0x9F14, + 0x9F26, 0x9F26, 0x9F45, 0x9F45, 0x9F53, 0x9F53, 0x9F6D, 0x9F6D, + 0x9FA1, 0x9FA1, 0x9FA3, 0x9FA3, 0x9FA6, 0x9FFF, 0xA66F, 0xA66F, + 0xA67C, 0xA67D, 0xA69E, 0xA69E, 0xA78B, 0xA78C, 0xA78F, 0xA78F, + 0xA792, 0xA793, 0xA7B2, 0xA7B9, 0xA7C2, 0xA7C3, 0xA7CB, 0xA7CF, + 0xA7DA, 0xA7DC, 0xA8FC, 0xA8FD, 0xA9CF, 0xA9CF, 0xA9E7, 0xA9FE, + 0xAA60, 0xAA76, 0xAA7A, 0xAA7A, 0xAA7C, 0xAA7F, 0xAB01, 0xAB06, + 0xAB09, 0xAB0E, 0xAB11, 0xAB16, 0xAB20, 0xAB26, 0xAB28, 0xAB2E, + 0xAB60, 0xAB63, 0xAB66, 0xAB67, 0xFA0E, 0xFA0F, 0xFA11, 0xFA11, + 0xFA13, 0xFA14, 0xFA1F, 0xFA1F, 0xFA21, 0xFA21, 0xFA23, 0xFA24, + 0xFA27, 0xFA29, 0xFB1E, 0xFB1E, 0xFE2E, 0xFE2F, 0x10780, 0x10780, + 0x10EC2, 0x10EC4, 0x10EC7, 0x10EC7, 0x10EFA, 0x10EFA, 0x10EFC, 0x10EFF, + 0x1133B, 0x1133B, 0x116D0, 0x116E3, 0x16A40, 0x16A5E, 0x16A60, 0x16A69, + 0x1AFF0, 0x1AFF3, 0x1AFF5, 0x1AFFB, 0x1AFFD, 0x1AFFE, 0x1D1DE, 0x1D1E8, + 0x1F54F, 0x1F54F, 0x20000, 0x2070D, 0x2070F, 0x20730, 0x20732, 0x20778, + 0x2077A, 0x20C52, 0x20C54, 0x20C77, 0x20C79, 0x20C95, 0x20C97, 0x20CCE, + 0x20CD0, 0x20CD4, 0x20CD6, 0x20D14, 0x20D16, 0x20D7B, 0x20D7D, 0x20D7E, + 0x20D80, 0x20E0D, 0x20E10, 0x20E76, 0x20E78, 0x20E9C, 0x20E9E, 0x20EA1, + 0x20EA3, 0x20ED6, 0x20ED8, 0x20EF8, 0x20EFB, 0x20F2C, 0x20F2F, 0x20F4B, + 0x20F4D, 0x20FB3, 0x20FB5, 0x20FBB, 0x20FBD, 0x20FE9, 0x20FEB, 0x2105B, + 0x2105D, 0x2106E, 0x21070, 0x21074, 0x21077, 0x2107A, 0x2107C, 0x210C0, + 0x210C2, 0x210C8, 0x210CA, 0x211D8, 0x211DA, 0x220C6, 0x220C8, 0x227B4, + 0x227B6, 0x22AD4, 0x22AD6, 0x22B42, 0x22B44, 0x22BC9, 0x22BCB, 0x22C50, + 0x22C52, 0x22C54, 0x22C56, 0x22CC1, 0x22CC3, 0x22D07, 0x22D09, 0x22D4B, + 0x22D4D, 0x22D66, 0x22D68, 0x22EB2, 0x22EB4, 0x23CB6, 0x23CB8, 0x244D2, + 0x244D4, 0x24DB7, 0x24DB9, 0x24DE9, 0x24DEB, 0x2512A, 0x2512C, 0x26257, + 0x26259, 0x267CB, 0x267CD, 0x269F1, 0x269F3, 0x269F9, 0x269FB, 0x27A3D, + 0x27A3F, 0x2815C, 0x2815E, 0x28206, 0x28208, 0x282E1, 0x282E3, 0x28CC9, + 0x28CCB, 0x28CCC, 0x28CCE, 0x28CD1, 0x28CD3, 0x29D97, 0x29D99, 0x2A6DF, + 0x2A700, 0x2B81D, 0x2B820, 0x2CEAD, 0x2CEB0, 0x2EBE0, 0x2EBF0, 0x2EE5D, + 0x30000, 0x3134A, 0x31350, 0x33479, + }; + } + private static int[] IDTYPE_ranges_9() { + return new int[] { + 0x2EA, 0x2EB, 0x700, 0x70D, 0x70F, 0x74A, 0x74D, 0x74F, + 0x7C0, 0x7FA, 0x7FD, 0x7FF, 0x840, 0x85B, 0x85E, 0x85E, + 0x860, 0x86A, 0x13A0, 0x13F5, 0x13F8, 0x13FD, 0x1400, 0x167F, + 0x18B0, 0x18F5, 0x1900, 0x191E, 0x1920, 0x192B, 0x1930, 0x193B, + 0x1940, 0x1940, 0x1944, 0x196D, 0x1970, 0x1974, 0x1980, 0x19AB, + 0x19B0, 0x19C9, 0x19D0, 0x19DA, 0x19DE, 0x19DF, 0x1A20, 0x1A5E, + 0x1A60, 0x1A7C, 0x1A7F, 0x1A89, 0x1A90, 0x1A99, 0x1AA0, 0x1AAD, + 0x1B00, 0x1B4C, 0x1B4E, 0x1BF3, 0x1BFC, 0x1C37, 0x1C3B, 0x1C49, + 0x1C4D, 0x1C7F, 0x1CC0, 0x1CC7, 0x1DFA, 0x1DFA, 0x2D30, 0x2D67, + 0x2D70, 0x2D70, 0x2D7F, 0x2D7F, 0x3105, 0x312F, 0x31A0, 0x31BF, + 0xA000, 0xA48C, 0xA490, 0xA4C6, 0xA4D0, 0xA62B, 0xA6A0, 0xA6F7, + 0xA800, 0xA82C, 0xA880, 0xA8C5, 0xA8CE, 0xA8D9, 0xA900, 0xA92D, + 0xA92F, 0xA92F, 0xA980, 0xA9CD, 0xA9CF, 0xA9D9, 0xA9DE, 0xA9DF, + 0xAA00, 0xAA36, 0xAA40, 0xAA4D, 0xAA50, 0xAA59, 0xAA5C, 0xAA5F, + 0xAA80, 0xAAC2, 0xAADB, 0xAAF6, 0xAB70, 0xABED, 0xABF0, 0xABF9, + 0x104B0, 0x104D3, 0x104D8, 0x104FB, 0x10D00, 0x10D27, 0x10D30, 0x10D39, + 0x11100, 0x11134, 0x11136, 0x11147, 0x11400, 0x1145B, 0x1145D, 0x11461, + 0x11AB0, 0x11ABF, 0x11FB0, 0x11FB0, 0x16800, 0x16A38, 0x16F00, 0x16F4A, + 0x16F4F, 0x16F87, 0x16F8F, 0x16F9F, 0x1E100, 0x1E12C, 0x1E130, 0x1E13D, + 0x1E140, 0x1E149, 0x1E14E, 0x1E14F, 0x1E2C0, 0x1E2F9, 0x1E2FF, 0x1E2FF, + 0x1E900, 0x1E94B, 0x1E950, 0x1E959, 0x1E95E, 0x1E95F, + }; + } + private static int[] IDTYPE_ranges_10() { + return new int[] { + 0x27, 0x27, 0x2D, 0x2E, 0x3A, 0x3A, 0xB7, 0xB7, + 0x2BB, 0x2BC, 0x58A, 0x58A, 0x5F3, 0x5F4, 0x6FD, 0x6FE, + 0xF0B, 0xF0B, 0x2010, 0x2010, 0x2019, 0x2019, 0x2027, 0x2027, + 0x30A0, 0x30A0, 0x30FB, 0x30FB, + }; + } + private static int[] IDTYPE_ranges_11() { + return new int[] { + 0x30, 0x39, 0x41, 0x5A, 0x5F, 0x5F, 0x61, 0x7A, + 0xC0, 0xD6, 0xD8, 0xF6, 0xF8, 0x113, 0x116, 0x12B, + 0x12E, 0x131, 0x134, 0x137, 0x139, 0x13E, 0x141, 0x148, + 0x14A, 0x14D, 0x150, 0x155, 0x158, 0x161, 0x164, 0x17E, + 0x181, 0x181, 0x186, 0x186, 0x189, 0x18A, 0x18E, 0x192, + 0x194, 0x194, 0x196, 0x199, 0x19D, 0x19D, 0x1A0, 0x1A1, + 0x1AF, 0x1B0, 0x1B2, 0x1B4, 0x1B7, 0x1B7, 0x1CD, 0x1D4, + 0x1DD, 0x1DD, 0x1E6, 0x1E9, 0x1EE, 0x1EF, 0x1F8, 0x1F9, + 0x218, 0x21B, 0x244, 0x244, 0x24C, 0x24D, 0x253, 0x254, + 0x256, 0x257, 0x259, 0x259, 0x25B, 0x25B, 0x263, 0x263, + 0x268, 0x269, 0x272, 0x272, 0x289, 0x289, 0x28B, 0x28B, + 0x292, 0x292, 0x300, 0x304, 0x306, 0x30C, 0x31B, 0x31B, + 0x323, 0x323, 0x326, 0x328, 0x331, 0x331, 0x386, 0x386, + 0x388, 0x38A, 0x38C, 0x38C, 0x38E, 0x3A1, 0x3A3, 0x3CE, + 0x401, 0x40C, 0x40E, 0x44F, 0x451, 0x45C, 0x45E, 0x45F, + 0x490, 0x49B, 0x49E, 0x4A5, 0x4A8, 0x4B7, 0x4BA, 0x4C0, + 0x4CF, 0x4D9, 0x4DC, 0x4E9, 0x4EE, 0x4F5, 0x4F8, 0x4F9, + 0x524, 0x525, 0x531, 0x556, 0x561, 0x586, 0x5D0, 0x5EA, + 0x620, 0x63A, 0x63D, 0x63D, 0x641, 0x652, 0x654, 0x655, + 0x660, 0x669, 0x670, 0x670, 0x672, 0x672, 0x674, 0x674, + 0x679, 0x68F, 0x691, 0x69A, 0x69F, 0x6A0, 0x6A2, 0x6A2, + 0x6A4, 0x6AB, 0x6AD, 0x6B1, 0x6B3, 0x6B3, 0x6B5, 0x6B7, + 0x6BA, 0x6BE, 0x6C0, 0x6D3, 0x6D5, 0x6D5, 0x6EE, 0x6F9, + 0x6FF, 0x6FF, 0x751, 0x752, 0x756, 0x756, 0x760, 0x760, + 0x762, 0x763, 0x766, 0x768, 0x76A, 0x76A, 0x76E, 0x771, + 0x780, 0x7B1, 0x88F, 0x88F, 0x8A0, 0x8A0, 0x8A2, 0x8A9, + 0x8BB, 0x8C2, 0x8C7, 0x8C7, 0x901, 0x903, 0x905, 0x90B, + 0x90D, 0x928, 0x92A, 0x933, 0x935, 0x93C, 0x93E, 0x943, + 0x945, 0x94D, 0x94F, 0x94F, 0x956, 0x957, 0x966, 0x96F, + 0x972, 0x977, 0x97B, 0x97C, 0x97E, 0x97F, 0x981, 0x983, + 0x985, 0x98B, 0x98F, 0x990, 0x993, 0x9A8, 0x9AA, 0x9B0, + 0x9B2, 0x9B2, 0x9B6, 0x9B9, 0x9BC, 0x9BC, 0x9BE, 0x9C4, + 0x9C7, 0x9C8, 0x9CB, 0x9CE, 0x9E6, 0x9F1, 0xA02, 0xA02, + 0xA05, 0xA0A, 0xA0F, 0xA10, 0xA13, 0xA28, 0xA2A, 0xA30, + 0xA32, 0xA32, 0xA35, 0xA35, 0xA38, 0xA39, 0xA3C, 0xA3C, + 0xA3E, 0xA42, 0xA47, 0xA48, 0xA4B, 0xA4D, 0xA5C, 0xA5C, + 0xA70, 0xA71, 0xA82, 0xA83, 0xA85, 0xA8D, 0xA8F, 0xA91, + 0xA93, 0xAA8, 0xAAA, 0xAB0, 0xAB2, 0xAB3, 0xAB5, 0xAB9, + 0xABC, 0xABC, 0xABE, 0xAC5, 0xAC7, 0xAC9, 0xACB, 0xACD, + 0xAE6, 0xAEF, 0xB01, 0xB03, 0xB05, 0xB0B, 0xB0F, 0xB10, + 0xB13, 0xB28, 0xB2A, 0xB30, 0xB32, 0xB33, 0xB36, 0xB39, + 0xB3C, 0xB3C, 0xB3E, 0xB43, 0xB47, 0xB48, 0xB4B, 0xB4D, + 0xB56, 0xB56, 0xB5F, 0xB5F, 0xB71, 0xB71, 0xB83, 0xB83, + 0xB85, 0xB8A, 0xB8E, 0xB90, 0xB92, 0xB95, 0xB99, 0xB9A, + 0xB9C, 0xB9C, 0xB9E, 0xB9F, 0xBA3, 0xBA4, 0xBA8, 0xBAA, + 0xBAE, 0xBB9, 0xBBE, 0xBC2, 0xBC6, 0xBC8, 0xBCA, 0xBCD, + 0xC02, 0xC03, 0xC05, 0xC0B, 0xC0E, 0xC10, 0xC12, 0xC28, + 0xC2A, 0xC30, 0xC32, 0xC33, 0xC35, 0xC39, 0xC3E, 0xC44, + 0xC46, 0xC48, 0xC4A, 0xC4D, 0xC82, 0xC83, 0xC85, 0xC8B, + 0xC8E, 0xC90, 0xC92, 0xCA8, 0xCAA, 0xCB0, 0xCB2, 0xCB3, + 0xCB5, 0xCB9, 0xCBE, 0xCC3, 0xCC6, 0xCC8, 0xCCA, 0xCCD, + 0xCE6, 0xCEF, 0xD02, 0xD03, 0xD05, 0xD0B, 0xD0E, 0xD10, + 0xD12, 0xD28, 0xD2A, 0xD39, 0xD3E, 0xD43, 0xD46, 0xD48, + 0xD4A, 0xD4B, 0xD4D, 0xD4D, 0xD57, 0xD57, 0xD7A, 0xD7F, + 0xD82, 0xD83, 0xD85, 0xD8D, 0xD91, 0xD96, 0xD9A, 0xD9D, + 0xD9F, 0xDB1, 0xDB3, 0xDBB, 0xDBD, 0xDBD, 0xDC0, 0xDC6, + 0xDCA, 0xDCA, 0xDCF, 0xDD4, 0xDD6, 0xDD6, 0xDD8, 0xDDE, + 0xDF2, 0xDF2, 0xE01, 0xE32, 0xE34, 0xE3A, 0xE40, 0xE4D, + 0xE50, 0xE59, 0xE81, 0xE82, 0xE84, 0xE84, 0xE87, 0xE88, + 0xE8A, 0xE8A, 0xE8D, 0xE8D, 0xE94, 0xE97, 0xE99, 0xE9F, + 0xEA1, 0xEA3, 0xEA5, 0xEA5, 0xEA7, 0xEA7, 0xEAA, 0xEAB, + 0xEAD, 0xEAE, 0xEB0, 0xEB2, 0xEB4, 0xEB9, 0xEBB, 0xEBD, + 0xEC0, 0xEC4, 0xEC6, 0xEC6, 0xEC8, 0xECD, 0xED0, 0xED9, + 0xF20, 0xF29, 0xF40, 0xF42, 0xF44, 0xF47, 0xF49, 0xF4C, + 0xF4E, 0xF51, 0xF53, 0xF56, 0xF58, 0xF5B, 0xF5D, 0xF68, + 0xF71, 0xF72, 0xF74, 0xF74, 0xF7A, 0xF80, 0xF84, 0xF84, + 0xF90, 0xF92, 0xF94, 0xF97, 0xF99, 0xF9C, 0xF9E, 0xFA1, + 0xFA3, 0xFA6, 0xFA8, 0xFAB, 0xFAD, 0xFAD, 0xFB1, 0xFB8, + 0xFBA, 0xFBC, 0x1000, 0x1049, 0x105A, 0x1064, 0x1075, 0x108A, + 0x108F, 0x108F, 0x10C7, 0x10C7, 0x10CD, 0x10CD, 0x10D0, 0x10F0, + 0x1200, 0x1206, 0x1208, 0x1248, 0x124A, 0x124D, 0x1250, 0x1256, + 0x1258, 0x1258, 0x125A, 0x125D, 0x1260, 0x1286, 0x1288, 0x1288, + 0x128A, 0x128D, 0x1290, 0x12AE, 0x12B0, 0x12B0, 0x12B2, 0x12B5, + 0x12B8, 0x12BE, 0x12C0, 0x12C0, 0x12C2, 0x12C5, 0x12C8, 0x12D6, + 0x12D8, 0x12F7, 0x1300, 0x130E, 0x1310, 0x1310, 0x1312, 0x1315, + 0x1318, 0x131E, 0x1320, 0x1346, 0x1348, 0x1359, 0x1780, 0x179C, + 0x179F, 0x17A2, 0x17A5, 0x17A7, 0x17AA, 0x17B3, 0x17B6, 0x17CD, + 0x17D0, 0x17D0, 0x17D2, 0x17D2, 0x17E0, 0x17E9, 0x1C90, 0x1CBA, + 0x1CBD, 0x1CBF, 0x1E0C, 0x1E0D, 0x1E12, 0x1E13, 0x1E20, 0x1E21, + 0x1E24, 0x1E25, 0x1E36, 0x1E37, 0x1E3C, 0x1E3F, 0x1E42, 0x1E4B, + 0x1E5A, 0x1E5B, 0x1E62, 0x1E63, 0x1E6C, 0x1E6D, 0x1E70, 0x1E71, + 0x1E8C, 0x1E8D, 0x1E92, 0x1E93, 0x1E9E, 0x1E9E, 0x1EA0, 0x1EF9, + 0x1FA0, 0x1FAF, 0x1FB2, 0x1FB4, 0x1FEC, 0x1FEC, 0x3005, 0x3007, + 0x3041, 0x3096, 0x309D, 0x309E, 0x30A1, 0x30FA, 0x30FC, 0x30FE, + 0x3447, 0x3447, 0x3473, 0x3473, 0x34E4, 0x34E4, 0x3577, 0x3577, + 0x359E, 0x359E, 0x35A1, 0x35A1, 0x35AD, 0x35AD, 0x35BF, 0x35BF, + 0x35CE, 0x35CE, 0x35F3, 0x35F3, 0x35FE, 0x35FE, 0x360E, 0x360E, + 0x361A, 0x361A, 0x3918, 0x3918, 0x3960, 0x3960, 0x396E, 0x396E, + 0x39CF, 0x39D0, 0x39DB, 0x39DB, 0x39DF, 0x39DF, 0x39F8, 0x39F8, + 0x39FE, 0x39FE, 0x3A18, 0x3A18, 0x3A52, 0x3A52, 0x3A5C, 0x3A5C, + 0x3A67, 0x3A67, 0x3A73, 0x3A73, 0x3B39, 0x3B39, 0x3B4E, 0x3B4E, + 0x3BA3, 0x3BA3, 0x3C6E, 0x3C6E, 0x3CE0, 0x3CE0, 0x3DE7, 0x3DE7, + 0x3DEB, 0x3DEB, 0x3E74, 0x3E74, 0x3ED0, 0x3ED0, 0x4056, 0x4056, + 0x4065, 0x4065, 0x406A, 0x406A, 0x40BB, 0x40BB, 0x40DF, 0x40DF, + 0x4137, 0x4137, 0x415F, 0x415F, 0x4337, 0x4337, 0x43AC, 0x43AC, + 0x43B1, 0x43B1, 0x43D3, 0x43D3, 0x43DD, 0x43DD, 0x4443, 0x4443, + 0x44D6, 0x44D6, 0x44EA, 0x44EA, 0x4606, 0x4606, 0x464C, 0x464C, + 0x4661, 0x4661, 0x4723, 0x4723, 0x4729, 0x4729, 0x477C, 0x477C, + 0x478D, 0x478D, 0x47F4, 0x47F4, 0x4882, 0x4882, 0x4947, 0x4947, + 0x497A, 0x497A, 0x497D, 0x497D, 0x4982, 0x4983, 0x4985, 0x4986, + 0x499B, 0x499B, 0x499F, 0x499F, 0x49B6, 0x49B7, 0x4A12, 0x4A12, + 0x4AB8, 0x4AB8, 0x4C77, 0x4C77, 0x4C7D, 0x4C7D, 0x4C81, 0x4C81, + 0x4C85, 0x4C85, 0x4C9D, 0x4CA3, 0x4D13, 0x4D19, 0x4DAE, 0x4DAE, + 0x4E00, 0x4E11, 0x4E13, 0x4E28, 0x4E2A, 0x4E67, 0x4E69, 0x4E78, + 0x4E7A, 0x4E95, 0x4E97, 0x4EA2, 0x4EA4, 0x4EBB, 0x4EBD, 0x4ECB, + 0x4ECD, 0x4EE6, 0x4EE8, 0x4EF7, 0x4EFB, 0x4EFB, 0x4EFD, 0x4EFD, + 0x4EFF, 0x4F06, 0x4F08, 0x4F15, 0x4F17, 0x4F27, 0x4F29, 0x4F30, + 0x4F32, 0x4F34, 0x4F36, 0x4F36, 0x4F38, 0x4F3F, 0x4F41, 0x4F43, + 0x4F45, 0x4F70, 0x4F72, 0x4F8B, 0x4F8D, 0x4F8D, 0x4F8F, 0x4FA1, + 0x4FA3, 0x4FBC, 0x4FBE, 0x4FC5, 0x4FC7, 0x4FC7, 0x4FC9, 0x4FCB, + 0x4FCD, 0x4FE1, 0x4FE3, 0x4FFB, 0x4FFE, 0x500F, 0x5011, 0x5033, + 0x5035, 0x5037, 0x5039, 0x503C, 0x503E, 0x5041, 0x5043, 0x5051, + 0x5053, 0x5057, 0x5059, 0x507B, 0x507D, 0x5080, 0x5082, 0x5092, + 0x5094, 0x5096, 0x5098, 0x509E, 0x50A2, 0x50B8, 0x50BA, 0x50C2, + 0x50C4, 0x50D7, 0x50D9, 0x50DE, 0x50E0, 0x50E0, 0x50E3, 0x50EA, + 0x50EC, 0x50F3, 0x50F5, 0x50F6, 0x50F8, 0x511A, 0x511C, 0x5127, + 0x5129, 0x512A, 0x512C, 0x5141, 0x5143, 0x5149, 0x514B, 0x514E, + 0x5150, 0x5152, 0x5154, 0x5157, 0x5159, 0x515F, 0x5161, 0x5163, + 0x5165, 0x5171, 0x5173, 0x517D, 0x517F, 0x5182, 0x5185, 0x518D, + 0x518F, 0x51A0, 0x51A2, 0x51A2, 0x51A4, 0x51AC, 0x51AE, 0x51B7, + 0x51B9, 0x51B9, 0x51BB, 0x51C1, 0x51C3, 0x51D1, 0x51D4, 0x51DE, + 0x51E0, 0x51EB, 0x51ED, 0x51ED, 0x51EF, 0x51F1, 0x51F3, 0x5252, + 0x5254, 0x5265, 0x5267, 0x5278, 0x527A, 0x5284, 0x5286, 0x528D, + 0x528F, 0x52C3, 0x52C5, 0x52C7, 0x52C9, 0x52CB, 0x52CD, 0x52CD, + 0x52CF, 0x52D0, 0x52D2, 0x52D3, 0x52D5, 0x52E0, 0x52E2, 0x52E4, + 0x52E6, 0x52ED, 0x52EF, 0x5302, 0x5305, 0x5317, 0x5319, 0x531A, + 0x531C, 0x531D, 0x531F, 0x5326, 0x5328, 0x5328, 0x532A, 0x5331, + 0x5333, 0x5334, 0x5337, 0x5341, 0x5343, 0x535A, 0x535C, 0x535C, + 0x535E, 0x5369, 0x536B, 0x536C, 0x536E, 0x537F, 0x5381, 0x53A0, + 0x53A2, 0x53A9, 0x53AC, 0x53AE, 0x53B0, 0x53B9, 0x53BB, 0x53C4, + 0x53C6, 0x53CE, 0x53D0, 0x53DC, 0x53DF, 0x53E6, 0x53E8, 0x53FE, + 0x5401, 0x5419, 0x541B, 0x5421, 0x5423, 0x544B, 0x544D, 0x545C, + 0x545E, 0x5468, 0x546A, 0x5489, 0x548B, 0x54B4, 0x54B6, 0x54F5, + 0x54F7, 0x5514, 0x5516, 0x5517, 0x551A, 0x5546, 0x5548, 0x555F, + 0x5561, 0x5579, 0x557B, 0x55DF, 0x55E1, 0x55F7, 0x55F9, 0x5609, + 0x560C, 0x561F, 0x5621, 0x562A, 0x562C, 0x5636, 0x5638, 0x563B, + 0x563D, 0x5643, 0x5645, 0x564A, 0x564C, 0x5650, 0x5652, 0x5655, + 0x5657, 0x565E, 0x5660, 0x5660, 0x5662, 0x5674, 0x5676, 0x567C, + 0x567E, 0x5687, 0x5689, 0x568A, 0x568C, 0x5695, 0x5697, 0x569D, + 0x569F, 0x56B9, 0x56BB, 0x56CE, 0x56D0, 0x56D8, 0x56DA, 0x56E5, + 0x56E7, 0x56F5, 0x56F7, 0x56F7, 0x56F9, 0x56FA, 0x56FD, 0x5704, + 0x5706, 0x5710, 0x5712, 0x5716, 0x5718, 0x5720, 0x5722, 0x5723, + 0x5725, 0x573C, 0x573E, 0x5742, 0x5744, 0x5747, 0x5749, 0x5754, + 0x5757, 0x5757, 0x5759, 0x5762, 0x5764, 0x5777, 0x5779, 0x5780, + 0x5782, 0x5786, 0x5788, 0x5795, 0x5797, 0x57A7, 0x57A9, 0x57C9, + 0x57CB, 0x57D0, 0x57D2, 0x57DA, 0x57DC, 0x5816, 0x5819, 0x584F, + 0x5851, 0x5855, 0x5857, 0x585F, 0x5861, 0x5865, 0x5868, 0x5876, + 0x5878, 0x5894, 0x5896, 0x58A9, 0x58AB, 0x58B5, 0x58B7, 0x58BF, + 0x58C1, 0x58C2, 0x58C5, 0x58CC, 0x58CE, 0x58CF, 0x58D1, 0x58E0, + 0x58E2, 0x58E5, 0x58E7, 0x58F4, 0x58F6, 0x5900, 0x5902, 0x5904, + 0x5906, 0x5907, 0x5909, 0x5910, 0x5912, 0x5912, 0x5914, 0x5922, + 0x5924, 0x5932, 0x5934, 0x5935, 0x5937, 0x5958, 0x595A, 0x595A, + 0x595C, 0x59B6, 0x59B8, 0x59E6, 0x59E8, 0x5A23, 0x5A25, 0x5A25, + 0x5A27, 0x5A2B, 0x5A2D, 0x5A2F, 0x5A31, 0x5A53, 0x5A55, 0x5A58, + 0x5A5A, 0x5A6E, 0x5A70, 0x5A70, 0x5A72, 0x5A86, 0x5A88, 0x5A8C, + 0x5A8E, 0x5AAA, 0x5AAC, 0x5AD2, 0x5AD4, 0x5AEE, 0x5AF1, 0x5B09, + 0x5B0B, 0x5B0C, 0x5B0E, 0x5B38, 0x5B3A, 0x5B45, 0x5B47, 0x5B4E, + 0x5B50, 0x5B51, 0x5B53, 0x5B5F, 0x5B62, 0x5B6E, 0x5B70, 0x5B78, + 0x5B7A, 0x5B7D, 0x5B7F, 0x5B85, 0x5B87, 0x5B8F, 0x5B91, 0x5BA8, + 0x5BAA, 0x5BB1, 0x5BB3, 0x5BB6, 0x5BB8, 0x5BBB, 0x5BBD, 0x5BC7, + 0x5BC9, 0x5BD9, 0x5BDB, 0x5BFF, 0x5C01, 0x5C1A, 0x5C1C, 0x5C22, + 0x5C24, 0x5C25, 0x5C27, 0x5C28, 0x5C2A, 0x5C35, 0x5C37, 0x5C59, + 0x5C5B, 0x5C84, 0x5C86, 0x5CB3, 0x5CB5, 0x5CB8, 0x5CBA, 0x5CD4, + 0x5CD6, 0x5CDC, 0x5CDE, 0x5CF4, 0x5CF6, 0x5D2A, 0x5D2C, 0x5D2E, + 0x5D30, 0x5D3A, 0x5D3C, 0x5D52, 0x5D54, 0x5D56, 0x5D58, 0x5D5F, + 0x5D61, 0x5D82, 0x5D84, 0x5D95, 0x5D97, 0x5DA2, 0x5DA5, 0x5DAA, + 0x5DAC, 0x5DB2, 0x5DB4, 0x5DB8, 0x5DBA, 0x5DC3, 0x5DC5, 0x5DD6, + 0x5DD8, 0x5DD9, 0x5DDB, 0x5DDB, 0x5DDD, 0x5DF5, 0x5DF7, 0x5E11, + 0x5E13, 0x5E47, 0x5E49, 0x5E50, 0x5E52, 0x5E91, 0x5E93, 0x5EB9, + 0x5EBB, 0x5EBF, 0x5EC1, 0x5EEA, 0x5EEC, 0x5EF8, 0x5EFA, 0x5F0D, + 0x5F0F, 0x5F3A, 0x5F3C, 0x5F3C, 0x5F3E, 0x5F8E, 0x5F90, 0x5F99, + 0x5F9B, 0x5FA2, 0x5FA5, 0x5FAF, 0x5FB1, 0x5FC1, 0x5FC3, 0x5FCD, + 0x5FCF, 0x5FDA, 0x5FDC, 0x5FE1, 0x5FE3, 0x5FEB, 0x5FED, 0x5FFB, + 0x5FFD, 0x6022, 0x6024, 0x6055, 0x6057, 0x6060, 0x6062, 0x6070, + 0x6072, 0x6073, 0x6075, 0x6090, 0x6092, 0x6092, 0x6094, 0x60A4, + 0x60A6, 0x60D1, 0x60D3, 0x60D5, 0x60D7, 0x60DD, 0x60DF, 0x60E4, + 0x60E6, 0x60FC, 0x60FE, 0x6101, 0x6103, 0x6106, 0x6108, 0x6110, + 0x6112, 0x611D, 0x611F, 0x6130, 0x6132, 0x6132, 0x6134, 0x6134, + 0x6136, 0x6137, 0x613A, 0x615F, 0x6161, 0x617A, 0x617C, 0x617E, + 0x6180, 0x6185, 0x6187, 0x6196, 0x6198, 0x619B, 0x619D, 0x61B8, + 0x61BA, 0x61BA, 0x61BC, 0x61D2, 0x61D4, 0x61D4, 0x61D6, 0x61EB, + 0x61ED, 0x61EE, 0x61F0, 0x6204, 0x6206, 0x6234, 0x6236, 0x6238, + 0x623A, 0x6256, 0x6258, 0x628C, 0x628E, 0x629C, 0x629E, 0x62DD, + 0x62DF, 0x62E9, 0x62EB, 0x6309, 0x630B, 0x6316, 0x6318, 0x6330, + 0x6332, 0x6336, 0x6338, 0x635A, 0x635C, 0x638A, 0x638C, 0x6392, + 0x6394, 0x63D0, 0x63D2, 0x643A, 0x643D, 0x6448, 0x644A, 0x6459, + 0x645B, 0x647D, 0x647F, 0x6485, 0x6487, 0x64A0, 0x64A2, 0x64AE, + 0x64B0, 0x64B5, 0x64B7, 0x64C7, 0x64C9, 0x64D4, 0x64D6, 0x64ED, + 0x64EF, 0x64F4, 0x64F6, 0x64F8, 0x64FA, 0x6501, 0x6503, 0x6509, + 0x650B, 0x651E, 0x6520, 0x6527, 0x6529, 0x653F, 0x6541, 0x6541, + 0x6543, 0x6559, 0x655B, 0x655E, 0x6560, 0x657C, 0x657E, 0x6589, + 0x658B, 0x6599, 0x659B, 0x65B4, 0x65B6, 0x65BD, 0x65BF, 0x65C7, + 0x65CA, 0x65D0, 0x65D2, 0x65D7, 0x65D9, 0x65DB, 0x65DD, 0x65E3, + 0x65E5, 0x65E9, 0x65EB, 0x65F8, 0x65FA, 0x65FD, 0x65FF, 0x6616, + 0x6618, 0x662B, 0x662D, 0x6636, 0x6639, 0x6647, 0x6649, 0x664C, + 0x664E, 0x665F, 0x6661, 0x6662, 0x6664, 0x6691, 0x6693, 0x669B, + 0x669D, 0x669D, 0x669F, 0x66AB, 0x66AE, 0x66CF, 0x66D1, 0x66D2, + 0x66D4, 0x66D6, 0x66D8, 0x66DE, 0x66E0, 0x66EE, 0x66F0, 0x6701, + 0x6703, 0x6706, 0x6708, 0x6718, 0x671A, 0x6723, 0x6725, 0x6728, + 0x672A, 0x6766, 0x6768, 0x6787, 0x6789, 0x6795, 0x6797, 0x67BC, + 0x67BE, 0x67BE, 0x67C0, 0x67D4, 0x67D6, 0x67D6, 0x67D8, 0x67F8, + 0x67FA, 0x6800, 0x6802, 0x6814, 0x6816, 0x6826, 0x6828, 0x682F, + 0x6831, 0x6857, 0x6859, 0x6859, 0x685B, 0x685D, 0x685F, 0x6879, + 0x687B, 0x6894, 0x6896, 0x6898, 0x689A, 0x68A4, 0x68A6, 0x68B7, + 0x68B9, 0x68C2, 0x68C4, 0x68D8, 0x68DA, 0x68E1, 0x68E3, 0x68E4, + 0x68E6, 0x6908, 0x690A, 0x693D, 0x693F, 0x694C, 0x694E, 0x699E, + 0x69A0, 0x69A1, 0x69A3, 0x69BF, 0x69C1, 0x69D0, 0x69D3, 0x69D4, + 0x69D8, 0x6A02, 0x6A04, 0x6A1B, 0x6A1D, 0x6A23, 0x6A25, 0x6A36, + 0x6A38, 0x6A49, 0x6A4B, 0x6A5B, 0x6A5D, 0x6A6D, 0x6A6F, 0x6A6F, + 0x6A71, 0x6A85, 0x6A87, 0x6A89, 0x6A8B, 0x6A8E, 0x6A90, 0x6A98, + 0x6A9A, 0x6A9C, 0x6A9E, 0x6AB0, 0x6AB2, 0x6ABD, 0x6ABF, 0x6ABF, + 0x6AC1, 0x6AC3, 0x6AC5, 0x6AC8, 0x6ACA, 0x6AD7, 0x6AD9, 0x6AE8, + 0x6AEA, 0x6B0D, 0x6B0F, 0x6B1A, 0x6B1C, 0x6B2D, 0x6B2F, 0x6B34, + 0x6B36, 0x6B3F, 0x6B41, 0x6B56, 0x6B59, 0x6B5C, 0x6B5E, 0x6B67, + 0x6B69, 0x6B6B, 0x6B6D, 0x6B6D, 0x6B6F, 0x6B70, 0x6B72, 0x6B74, + 0x6B76, 0x6B7C, 0x6B7E, 0x6BB7, 0x6BB9, 0x6BE8, 0x6BEA, 0x6BF0, + 0x6BF2, 0x6BF3, 0x6BF5, 0x6BF9, 0x6BFB, 0x6C09, 0x6C0B, 0x6C1B, + 0x6C1D, 0x6C2C, 0x6C2E, 0x6C3B, 0x6C3D, 0x6C44, 0x6C46, 0x6C6B, + 0x6C6D, 0x6C6D, 0x6C6F, 0x6C9F, 0x6CA1, 0x6CD7, 0x6CD9, 0x6CF3, + 0x6CF5, 0x6D01, 0x6D03, 0x6D1B, 0x6D1D, 0x6D23, 0x6D25, 0x6D70, + 0x6D72, 0x6D80, 0x6D82, 0x6D95, 0x6D97, 0x6DAF, 0x6DB2, 0x6DB5, + 0x6DB7, 0x6DFD, 0x6E00, 0x6E00, 0x6E03, 0x6E05, 0x6E07, 0x6E11, + 0x6E13, 0x6E17, 0x6E19, 0x6E29, 0x6E2B, 0x6E4B, 0x6E4D, 0x6E6B, + 0x6E6D, 0x6E7A, 0x6E7E, 0x6E8A, 0x6E8C, 0x6E94, 0x6E96, 0x6EDA, + 0x6EDC, 0x6EE2, 0x6EE4, 0x6F03, 0x6F05, 0x6F0A, 0x6F0C, 0x6F41, + 0x6F43, 0x6F47, 0x6F49, 0x6F49, 0x6F4B, 0x6F78, 0x6F7A, 0x6F97, + 0x6F99, 0x6F99, 0x6F9B, 0x6F9E, 0x6FA0, 0x6FB6, 0x6FB8, 0x6FC4, + 0x6FC6, 0x6FCF, 0x6FD1, 0x6FD2, 0x6FD4, 0x6FF4, 0x6FF6, 0x6FFC, + 0x6FFE, 0x700F, 0x7011, 0x7012, 0x7014, 0x7046, 0x7048, 0x704A, + 0x704C, 0x704D, 0x704F, 0x7071, 0x7074, 0x707A, 0x707C, 0x7080, + 0x7082, 0x708C, 0x708E, 0x7096, 0x7098, 0x709A, 0x709C, 0x70A9, + 0x70AB, 0x70B1, 0x70B3, 0x70B5, 0x70B7, 0x70D4, 0x70D6, 0x70FD, + 0x70FF, 0x7107, 0x7109, 0x7123, 0x7125, 0x7132, 0x7135, 0x7156, + 0x7158, 0x716A, 0x716C, 0x716C, 0x716E, 0x718C, 0x718E, 0x7195, + 0x7197, 0x71A5, 0x71A7, 0x71AA, 0x71AC, 0x71B5, 0x71B7, 0x71CB, + 0x71CD, 0x71D2, 0x71D4, 0x71F2, 0x71F4, 0x71F9, 0x71FB, 0x720A, + 0x720C, 0x7210, 0x7212, 0x7214, 0x7216, 0x7216, 0x7218, 0x721F, + 0x7221, 0x7223, 0x7226, 0x722E, 0x7230, 0x7233, 0x7235, 0x7244, + 0x7246, 0x724D, 0x724F, 0x724F, 0x7251, 0x7254, 0x7256, 0x72AA, + 0x72AC, 0x72BD, 0x72BF, 0x7301, 0x7303, 0x730F, 0x7311, 0x7327, + 0x7329, 0x7352, 0x7354, 0x739B, 0x739D, 0x73C0, 0x73C2, 0x73F2, + 0x73F4, 0x73FA, 0x73FC, 0x7417, 0x7419, 0x7438, 0x743A, 0x743D, + 0x743F, 0x7446, 0x7448, 0x7448, 0x744A, 0x7457, 0x7459, 0x747A, + 0x747C, 0x7483, 0x7485, 0x7495, 0x7497, 0x749C, 0x749E, 0x74C6, + 0x74C8, 0x74C8, 0x74CA, 0x74CB, 0x74CD, 0x74EA, 0x74EC, 0x751F, + 0x7521, 0x7540, 0x7542, 0x7551, 0x7553, 0x7554, 0x7556, 0x755D, + 0x755F, 0x7560, 0x7562, 0x7570, 0x7572, 0x757A, 0x757C, 0x7584, + 0x7586, 0x75A8, 0x75AA, 0x75B6, 0x75B8, 0x75DB, 0x75DD, 0x75ED, + 0x75EF, 0x762B, 0x762D, 0x7643, 0x7646, 0x7650, 0x7652, 0x7654, + 0x7656, 0x7672, 0x7674, 0x768C, 0x768E, 0x76A0, 0x76A3, 0x76A4, + 0x76A6, 0x76A7, 0x76A9, 0x76B2, 0x76B4, 0x76B5, 0x76B7, 0x76C0, + 0x76C2, 0x76CA, 0x76CC, 0x76D8, 0x76DA, 0x76EA, 0x76EC, 0x76FF, + 0x7701, 0x7701, 0x7703, 0x770D, 0x770F, 0x7720, 0x7722, 0x772A, + 0x772C, 0x773E, 0x7740, 0x7741, 0x7743, 0x7763, 0x7765, 0x7795, + 0x7797, 0x77A3, 0x77A5, 0x77BD, 0x77BF, 0x77C0, 0x77C2, 0x77D1, + 0x77D3, 0x77DC, 0x77DE, 0x77E3, 0x77E5, 0x77E5, 0x77E7, 0x77F3, + 0x77F6, 0x7823, 0x7825, 0x7835, 0x7837, 0x7841, 0x7843, 0x7845, + 0x7847, 0x784A, 0x784C, 0x7875, 0x7877, 0x7887, 0x7889, 0x78C1, + 0x78C3, 0x78C6, 0x78C8, 0x78D1, 0x78D3, 0x78EF, 0x78F1, 0x78F7, + 0x78F9, 0x78FF, 0x7901, 0x7907, 0x7909, 0x790C, 0x790E, 0x7914, + 0x7916, 0x791E, 0x7921, 0x7931, 0x7933, 0x7935, 0x7937, 0x7958, + 0x795A, 0x796B, 0x796D, 0x796D, 0x796F, 0x7974, 0x7977, 0x7985, + 0x7988, 0x799D, 0x799F, 0x79A8, 0x79AA, 0x79BB, 0x79BD, 0x79C3, + 0x79C5, 0x79C6, 0x79C8, 0x79CB, 0x79CD, 0x79D3, 0x79D5, 0x79D6, + 0x79D8, 0x7A00, 0x7A02, 0x7A06, 0x7A08, 0x7A08, 0x7A0A, 0x7A2B, + 0x7A2D, 0x7A37, 0x7A39, 0x7A39, 0x7A3B, 0x7A63, 0x7A65, 0x7A69, + 0x7A6B, 0x7A6E, 0x7A70, 0x7A81, 0x7A83, 0x7A99, 0x7A9C, 0x7AB8, + 0x7ABA, 0x7ABA, 0x7ABE, 0x7AC1, 0x7AC3, 0x7AC5, 0x7AC7, 0x7AE8, + 0x7AEA, 0x7AF4, 0x7AF6, 0x7AFB, 0x7AFD, 0x7B06, 0x7B08, 0x7B1E, + 0x7B20, 0x7B26, 0x7B28, 0x7B28, 0x7B2A, 0x7B41, 0x7B43, 0x7B52, + 0x7B54, 0x7BA2, 0x7BA4, 0x7BA4, 0x7BA6, 0x7BAF, 0x7BB1, 0x7BB1, + 0x7BB3, 0x7BF9, 0x7BFB, 0x7C1A, 0x7C1C, 0x7C2D, 0x7C30, 0x7C51, + 0x7C53, 0x7C54, 0x7C56, 0x7C5C, 0x7C5E, 0x7C75, 0x7C77, 0x7C86, + 0x7C88, 0x7C92, 0x7C94, 0x7C99, 0x7C9B, 0x7CAB, 0x7CAD, 0x7CD2, + 0x7CD4, 0x7CD9, 0x7CDC, 0x7CE0, 0x7CE2, 0x7CE2, 0x7CE4, 0x7CE4, + 0x7CE7, 0x7CFB, 0x7CFD, 0x7CFE, 0x7D00, 0x7D22, 0x7D24, 0x7D29, + 0x7D2B, 0x7D2C, 0x7D2E, 0x7D47, 0x7D49, 0x7D4C, 0x7D4E, 0x7D59, + 0x7D5B, 0x7D63, 0x7D65, 0x7D77, 0x7D79, 0x7D81, 0x7D83, 0x7D94, + 0x7D96, 0x7D97, 0x7D99, 0x7DA3, 0x7DA5, 0x7DA7, 0x7DA9, 0x7DCC, + 0x7DCE, 0x7DD2, 0x7DD4, 0x7DE4, 0x7DE6, 0x7DEA, 0x7DEC, 0x7DFC, + 0x7E00, 0x7E17, 0x7E19, 0x7E5A, 0x7E5C, 0x7E63, 0x7E65, 0x7E9C, + 0x7E9E, 0x7F3A, 0x7F3D, 0x7F40, 0x7F42, 0x7F45, 0x7F47, 0x7F58, + 0x7F5A, 0x7F83, 0x7F85, 0x7F8F, 0x7F91, 0x7F96, 0x7F98, 0x7F98, + 0x7F9A, 0x7FB3, 0x7FB5, 0x7FD5, 0x7FD7, 0x7FDC, 0x7FDE, 0x7FE3, + 0x7FE5, 0x8009, 0x800B, 0x802E, 0x8030, 0x803B, 0x803D, 0x803F, + 0x8041, 0x8065, 0x8067, 0x8087, 0x8089, 0x808D, 0x808F, 0x8093, + 0x8095, 0x80A5, 0x80A9, 0x80B2, 0x80B4, 0x80B8, 0x80BA, 0x80DE, + 0x80E0, 0x8102, 0x8105, 0x8133, 0x8136, 0x8183, 0x8185, 0x818F, + 0x8191, 0x8195, 0x8197, 0x81CA, 0x81CC, 0x81E3, 0x81E5, 0x81EE, + 0x81F1, 0x8212, 0x8214, 0x8223, 0x8225, 0x8240, 0x8242, 0x8264, + 0x8266, 0x828B, 0x828D, 0x82B1, 0x82B3, 0x82E1, 0x82E3, 0x82FB, + 0x82FD, 0x8309, 0x830B, 0x830F, 0x8311, 0x832F, 0x8331, 0x8354, + 0x8356, 0x83BD, 0x83BF, 0x83E5, 0x83E7, 0x83EC, 0x83EE, 0x8413, + 0x8415, 0x8415, 0x8418, 0x841E, 0x8420, 0x8457, 0x8459, 0x8482, + 0x8484, 0x8494, 0x8496, 0x84B6, 0x84B8, 0x84C2, 0x84C4, 0x84EC, + 0x84EE, 0x8504, 0x8506, 0x850F, 0x8511, 0x8531, 0x8534, 0x854B, + 0x854D, 0x854F, 0x8551, 0x857E, 0x8580, 0x8592, 0x8594, 0x85B1, + 0x85B3, 0x85BA, 0x85BC, 0x85CB, 0x85CD, 0x85ED, 0x85EF, 0x85F2, + 0x85F4, 0x85FB, 0x85FD, 0x8602, 0x8604, 0x860C, 0x860F, 0x860F, + 0x8611, 0x8614, 0x8616, 0x861C, 0x861E, 0x8636, 0x8638, 0x8656, + 0x8658, 0x8674, 0x8676, 0x8688, 0x868A, 0x8691, 0x8693, 0x869F, + 0x86A1, 0x86A5, 0x86A7, 0x86D4, 0x86D6, 0x86DF, 0x86E1, 0x86E6, + 0x86E8, 0x86FC, 0x86FE, 0x871C, 0x871E, 0x872E, 0x8730, 0x873C, + 0x873E, 0x8744, 0x8746, 0x8770, 0x8772, 0x878D, 0x878F, 0x8798, + 0x879A, 0x87D9, 0x87DB, 0x87EF, 0x87F1, 0x8806, 0x8808, 0x8811, + 0x8813, 0x882C, 0x882E, 0x8839, 0x883B, 0x8846, 0x8848, 0x8857, + 0x8859, 0x885B, 0x885D, 0x885E, 0x8860, 0x8879, 0x887B, 0x88E5, + 0x88E7, 0x88E8, 0x88EA, 0x88EC, 0x88EE, 0x8902, 0x8904, 0x890E, + 0x8910, 0x8923, 0x8925, 0x8964, 0x8966, 0x8974, 0x8976, 0x897C, + 0x897E, 0x898C, 0x898E, 0x898F, 0x8991, 0x8993, 0x8995, 0x8998, + 0x899A, 0x89AF, 0x89B1, 0x89B3, 0x89B5, 0x89BA, 0x89BD, 0x89ED, + 0x89EF, 0x89F4, 0x89F6, 0x89F8, 0x89FA, 0x89FC, 0x89FE, 0x8A04, + 0x8A07, 0x8A13, 0x8A15, 0x8A18, 0x8A1A, 0x8A1F, 0x8A22, 0x8A2A, + 0x8A2C, 0x8A3C, 0x8A3E, 0x8A4A, 0x8A4C, 0x8A63, 0x8A65, 0x8A77, + 0x8A79, 0x8A7C, 0x8A7E, 0x8A87, 0x8A89, 0x8A9E, 0x8AA0, 0x8AAE, + 0x8AB0, 0x8AB6, 0x8AB8, 0x8ACF, 0x8AD1, 0x8AEB, 0x8AED, 0x8B28, + 0x8B2A, 0x8B31, 0x8B33, 0x8B37, 0x8B39, 0x8B3E, 0x8B40, 0x8B60, + 0x8B63, 0x8B68, 0x8B6A, 0x8B74, 0x8B76, 0x8B7B, 0x8B7D, 0x8B80, + 0x8B82, 0x8B86, 0x8B88, 0x8B8C, 0x8B8E, 0x8B8E, 0x8B90, 0x8B9A, + 0x8B9C, 0x8C37, 0x8C39, 0x8C3F, 0x8C41, 0x8C43, 0x8C45, 0x8C50, + 0x8C54, 0x8C57, 0x8C59, 0x8C73, 0x8C75, 0x8C7E, 0x8C80, 0x8C82, + 0x8C84, 0x8C86, 0x8C88, 0x8C8A, 0x8C8C, 0x8C9A, 0x8C9C, 0x8CA5, + 0x8CA7, 0x8CCA, 0x8CCC, 0x8CD5, 0x8CD7, 0x8CD7, 0x8CD9, 0x8CE8, + 0x8CEA, 0x8CF6, 0x8CF8, 0x8D00, 0x8D02, 0x8D10, 0x8D13, 0x8D7B, + 0x8D7D, 0x8DA5, 0x8DA7, 0x8DBF, 0x8DC1, 0x8DE4, 0x8DE6, 0x8E00, + 0x8E02, 0x8E0A, 0x8E0C, 0x8E31, 0x8E33, 0x8E45, 0x8E47, 0x8E4E, + 0x8E50, 0x8E6D, 0x8E6F, 0x8E74, 0x8E76, 0x8E76, 0x8E78, 0x8E78, + 0x8E7A, 0x8E9A, 0x8E9C, 0x8EA1, 0x8EA3, 0x8EB2, 0x8EB4, 0x8EB5, + 0x8EB8, 0x8EC0, 0x8EC2, 0x8EC3, 0x8EC5, 0x8ED8, 0x8EDA, 0x8EEF, + 0x8EF1, 0x8F0E, 0x8F10, 0x8F2C, 0x8F2E, 0x8F39, 0x8F3B, 0x8F40, + 0x8F42, 0x8F9C, 0x8F9E, 0x8FA3, 0x8FA5, 0x8FB2, 0x8FB4, 0x8FC2, + 0x8FC4, 0x8FC9, 0x8FCB, 0x8FE6, 0x8FE8, 0x9029, 0x902B, 0x902B, + 0x902D, 0x9036, 0x9038, 0x903F, 0x9041, 0x9045, 0x9047, 0x90AA, + 0x90AC, 0x90CB, 0x90CE, 0x90D1, 0x90D3, 0x90F5, 0x90F7, 0x9109, + 0x910B, 0x913B, 0x913E, 0x9158, 0x915A, 0x917A, 0x917C, 0x9194, + 0x9196, 0x9197, 0x9199, 0x91A8, 0x91AA, 0x91BE, 0x91C0, 0x91C3, + 0x91C5, 0x91DF, 0x91E1, 0x91EE, 0x91F0, 0x9212, 0x9214, 0x921E, + 0x9220, 0x9221, 0x9223, 0x9242, 0x9244, 0x9268, 0x926B, 0x9280, + 0x9282, 0x9283, 0x9285, 0x929D, 0x929F, 0x92BC, 0x92BE, 0x92D3, + 0x92D5, 0x92DA, 0x92DC, 0x92E1, 0x92E3, 0x931B, 0x931D, 0x932F, + 0x9332, 0x9361, 0x9363, 0x9367, 0x9369, 0x936A, 0x936C, 0x936E, + 0x9370, 0x9372, 0x9374, 0x9377, 0x9379, 0x937E, 0x9380, 0x9380, + 0x9382, 0x938A, 0x938C, 0x939B, 0x939D, 0x939F, 0x93A1, 0x93AA, + 0x93AC, 0x93BA, 0x93BC, 0x93DF, 0x93E1, 0x93F2, 0x93F4, 0x9401, + 0x9403, 0x9416, 0x9418, 0x941B, 0x941D, 0x941D, 0x9420, 0x9423, + 0x9425, 0x9442, 0x9444, 0x944D, 0x944F, 0x946B, 0x946D, 0x947A, + 0x947C, 0x9577, 0x957A, 0x957D, 0x957F, 0x9584, 0x9586, 0x9596, + 0x9598, 0x95B2, 0x95B5, 0x95B7, 0x95B9, 0x95C0, 0x95C2, 0x95D8, + 0x95DA, 0x95DC, 0x95DE, 0x9624, 0x9627, 0x9628, 0x962A, 0x963D, + 0x963F, 0x9655, 0x9658, 0x9678, 0x967A, 0x967A, 0x967C, 0x967E, + 0x9680, 0x9680, 0x9683, 0x968B, 0x968D, 0x9695, 0x9697, 0x9699, + 0x969B, 0x969C, 0x969E, 0x969E, 0x96A0, 0x96AA, 0x96AC, 0x96AE, + 0x96B0, 0x96B4, 0x96B6, 0x96E3, 0x96E5, 0x96E5, 0x96E8, 0x96FB, + 0x96FD, 0x9713, 0x9715, 0x9716, 0x9718, 0x9719, 0x971C, 0x9732, + 0x9735, 0x9736, 0x9738, 0x973F, 0x9742, 0x974C, 0x974E, 0x9756, + 0x9758, 0x9762, 0x9764, 0x9774, 0x9776, 0x9786, 0x9788, 0x9788, + 0x978A, 0x979A, 0x979C, 0x97A8, 0x97AA, 0x97AF, 0x97B2, 0x97B4, + 0x97B6, 0x97BD, 0x97BF, 0x97BF, 0x97C1, 0x97D1, 0x97D3, 0x97FB, + 0x97FD, 0x981E, 0x9820, 0x9824, 0x9826, 0x9829, 0x982B, 0x9832, + 0x9834, 0x9839, 0x983B, 0x983D, 0x983F, 0x9841, 0x9843, 0x9846, + 0x9848, 0x9855, 0x9857, 0x9865, 0x9867, 0x9867, 0x9869, 0x98B6, + 0x98B8, 0x98C9, 0x98CB, 0x98E3, 0x98E5, 0x98EB, 0x98ED, 0x98F0, + 0x98F2, 0x98F7, 0x98F9, 0x98FA, 0x98FC, 0x9918, 0x991A, 0x993A, + 0x993C, 0x9943, 0x9945, 0x9959, 0x995B, 0x995C, 0x995E, 0x99BE, + 0x99C0, 0x99DF, 0x99E1, 0x99E5, 0x99E7, 0x99EA, 0x99EC, 0x99F4, + 0x99F6, 0x9A0F, 0x9A11, 0x9A16, 0x9A19, 0x9A3A, 0x9A3C, 0x9A50, + 0x9A52, 0x9A57, 0x9A59, 0x9A5C, 0x9A5E, 0x9A62, 0x9A64, 0x9AA8, + 0x9AAA, 0x9ABC, 0x9ABE, 0x9AC7, 0x9AC9, 0x9AD6, 0x9AD8, 0x9ADF, + 0x9AE1, 0x9AE3, 0x9AE5, 0x9AE7, 0x9AEA, 0x9AEF, 0x9AF1, 0x9AFF, + 0x9B01, 0x9B01, 0x9B03, 0x9B08, 0x9B0A, 0x9B13, 0x9B15, 0x9B1A, + 0x9B1C, 0x9B33, 0x9B35, 0x9B3C, 0x9B3E, 0x9B3F, 0x9B41, 0x9B4F, + 0x9B51, 0x9B56, 0x9B58, 0x9B61, 0x9B63, 0x9B71, 0x9B73, 0x9B88, + 0x9B8A, 0x9B8B, 0x9B8D, 0x9B98, 0x9B9A, 0x9BC1, 0x9BC3, 0x9BF5, + 0x9BF7, 0x9BFF, 0x9C02, 0x9C02, 0x9C04, 0x9C41, 0x9C43, 0x9C4E, + 0x9C50, 0x9C50, 0x9C52, 0x9C60, 0x9C62, 0x9C63, 0x9C65, 0x9C7A, + 0x9C7C, 0x9D0B, 0x9D0E, 0x9D10, 0x9D12, 0x9D26, 0x9D28, 0x9D34, + 0x9D36, 0x9D3B, 0x9D3D, 0x9D6C, 0x9D6E, 0x9D94, 0x9D96, 0x9DAD, + 0x9DAF, 0x9DBC, 0x9DBE, 0x9DBF, 0x9DC1, 0x9DE9, 0x9DEB, 0x9DFB, + 0x9DFD, 0x9E0D, 0x9E0F, 0x9E15, 0x9E17, 0x9E1B, 0x9E1D, 0x9E7A, + 0x9E7C, 0x9E8E, 0x9E91, 0x9E97, 0x9E99, 0x9E9D, 0x9E9F, 0x9EA1, + 0x9EA3, 0x9EAA, 0x9EAD, 0x9EB0, 0x9EB2, 0x9EEB, 0x9EED, 0x9EF0, + 0x9EF2, 0x9F02, 0x9F04, 0x9F10, 0x9F12, 0x9F13, 0x9F15, 0x9F25, + 0x9F27, 0x9F44, 0x9F46, 0x9F52, 0x9F54, 0x9F6C, 0x9F6E, 0x9FA0, + 0x9FA2, 0x9FA2, 0x9FA4, 0x9FA5, 0xA78D, 0xA78D, 0xA7AA, 0xA7AA, + 0xAA7B, 0xAA7B, 0xAC00, 0xD7A3, 0x11301, 0x11301, 0x11303, 0x11303, + 0x1133C, 0x1133C, 0x1E7E0, 0x1E7E6, 0x1E7E8, 0x1E7EB, 0x1E7ED, 0x1E7EE, + 0x1E7F0, 0x1E7FE, 0x2070E, 0x2070E, 0x20731, 0x20731, 0x20779, 0x20779, + 0x20C53, 0x20C53, 0x20C78, 0x20C78, 0x20C96, 0x20C96, 0x20CCF, 0x20CCF, + 0x20CD5, 0x20CD5, 0x20D15, 0x20D15, 0x20D7C, 0x20D7C, 0x20D7F, 0x20D7F, + 0x20E0E, 0x20E0F, 0x20E77, 0x20E77, 0x20E9D, 0x20E9D, 0x20EA2, 0x20EA2, + 0x20ED7, 0x20ED7, 0x20EF9, 0x20EFA, 0x20F2D, 0x20F2E, 0x20F4C, 0x20F4C, + 0x20FB4, 0x20FB4, 0x20FBC, 0x20FBC, 0x20FEA, 0x20FEA, 0x2105C, 0x2105C, + 0x2106F, 0x2106F, 0x21075, 0x21076, 0x2107B, 0x2107B, 0x210C1, 0x210C1, + 0x210C9, 0x210C9, 0x211D9, 0x211D9, 0x220C7, 0x220C7, 0x227B5, 0x227B5, + 0x22AD5, 0x22AD5, 0x22B43, 0x22B43, 0x22BCA, 0x22BCA, 0x22C51, 0x22C51, + 0x22C55, 0x22C55, 0x22CC2, 0x22CC2, 0x22D08, 0x22D08, 0x22D4C, 0x22D4C, + 0x22D67, 0x22D67, 0x22EB3, 0x22EB3, 0x23CB7, 0x23CB7, 0x244D3, 0x244D3, + 0x24DB8, 0x24DB8, 0x24DEA, 0x24DEA, 0x2512B, 0x2512B, 0x26258, 0x26258, + 0x267CC, 0x267CC, 0x269F2, 0x269F2, 0x269FA, 0x269FA, 0x27A3E, 0x27A3E, + 0x2815D, 0x2815D, 0x28207, 0x28207, 0x282E2, 0x282E2, 0x28CCA, 0x28CCA, + 0x28CCD, 0x28CCD, 0x28CD2, 0x28CD2, 0x29D98, 0x29D98, + }; + } + private static final int[][] IDTYPE_RANGES = { + IDTYPE_ranges_0(), IDTYPE_ranges_1(), IDTYPE_ranges_2(), IDTYPE_ranges_3(), IDTYPE_ranges_4(), IDTYPE_ranges_5(), IDTYPE_ranges_6(), IDTYPE_ranges_7(), IDTYPE_ranges_8(), IDTYPE_ranges_9(), IDTYPE_ranges_10(), IDTYPE_ranges_11() + }; + private static final Property IDTYPE = new Property( + IDTYPE_VALUES, IDTYPE_SHORT_VALUES, + IDTYPE_VALUE_ALIASES, IDTYPE_VALUE_ALIAS_INDEX, + IDTYPE_PROPERTY_ALIASES, IDTYPE_RANGES); + + private static final String[] KEHCORE_VALUES = { + "C", "L", "N" + }; + private static final String[] KEHCORE_SHORT_VALUES = { + "C", "L", "N" + }; + private static final String[] KEHCORE_VALUE_ALIASES = { + "c", "l", "n" + }; + private static final byte[] KEHCORE_VALUE_ALIAS_INDEX = { + 0, 1, 2 + }; + private static final String[] KEHCORE_PROPERTY_ALIASES = { + "kehcore" + }; + private static int[] KEHCORE_ranges_0() { + return new int[] { + 0x13000, 0x1305C, 0x1305F, 0x13069, 0x1306B, 0x1306B, 0x1306D, 0x13091, + 0x13093, 0x13096, 0x13098, 0x130B8, 0x130BA, 0x130C0, 0x130C2, 0x130F9, + 0x130FB, 0x1310B, 0x1310D, 0x13117, 0x13119, 0x13132, 0x13136, 0x13136, + 0x13139, 0x13139, 0x1313D, 0x13140, 0x13142, 0x13151, 0x13153, 0x13154, + 0x13158, 0x13168, 0x1316A, 0x1316A, 0x1316C, 0x13171, 0x13173, 0x13173, + 0x13177, 0x13195, 0x13197, 0x131A4, 0x131A6, 0x131A9, 0x131AB, 0x131AD, + 0x131B0, 0x131B1, 0x131B3, 0x131C5, 0x131C7, 0x131D7, 0x131D9, 0x131E0, + 0x131E3, 0x13201, 0x13203, 0x13209, 0x1320B, 0x13214, 0x13216, 0x13250, + 0x13253, 0x13255, 0x13257, 0x1325D, 0x13265, 0x13265, 0x13267, 0x1326F, + 0x13271, 0x1327B, 0x1327D, 0x1327D, 0x1327F, 0x13297, 0x13299, 0x132A2, + 0x132A4, 0x132A4, 0x132A6, 0x132DE, 0x132E2, 0x132F0, 0x132F2, 0x132F4, + 0x132F7, 0x132FA, 0x132FC, 0x1330B, 0x1330E, 0x1332C, 0x1332E, 0x1332E, + 0x13330, 0x13334, 0x13338, 0x13338, 0x1333C, 0x13352, 0x13354, 0x13359, + 0x1335B, 0x1339B, 0x1339D, 0x1339D, 0x1339F, 0x1339F, 0x133A1, 0x133A1, + 0x133A3, 0x133B1, 0x133B3, 0x133B8, 0x133BA, 0x133BF, 0x133C1, 0x133CC, + 0x133CE, 0x133D6, 0x133D8, 0x133DB, 0x133DD, 0x133DE, 0x133E0, 0x133E8, + 0x133EA, 0x133EA, 0x133ED, 0x13402, 0x13404, 0x13413, 0x13415, 0x13423, + 0x13425, 0x1342F, 0x13461, 0x1346D, 0x1346F, 0x13474, 0x13476, 0x13487, + 0x13489, 0x13494, 0x13496, 0x13496, 0x13498, 0x1349E, 0x134A0, 0x134A5, + 0x134A7, 0x134A8, 0x134AA, 0x134AD, 0x134AF, 0x134B6, 0x134B8, 0x134BD, + 0x134BF, 0x134C1, 0x134C3, 0x134CB, 0x134CD, 0x134F4, 0x134F6, 0x13502, + 0x13504, 0x13505, 0x13507, 0x13514, 0x13516, 0x1351A, 0x1351C, 0x13520, + 0x13522, 0x1352C, 0x1352E, 0x13531, 0x13533, 0x13543, 0x13546, 0x13549, + 0x1354B, 0x1355E, 0x13560, 0x13569, 0x1356B, 0x1356C, 0x1356E, 0x1357C, + 0x1357E, 0x13580, 0x13582, 0x1359F, 0x135A1, 0x135A2, 0x135A4, 0x135BC, + 0x135BE, 0x135BF, 0x135C1, 0x135CD, 0x135CF, 0x135D0, 0x135D3, 0x135D5, + 0x135D7, 0x135DA, 0x135DC, 0x135E9, 0x135EB, 0x135EB, 0x135ED, 0x1360D, + 0x1360F, 0x1361B, 0x1361D, 0x13623, 0x13625, 0x1362A, 0x1362C, 0x13630, + 0x13632, 0x13634, 0x13636, 0x13639, 0x1363B, 0x13648, 0x1364A, 0x1364E, + 0x13650, 0x13657, 0x13659, 0x13659, 0x1365B, 0x13669, 0x1366C, 0x136A2, + 0x136A4, 0x136A7, 0x136A9, 0x136AA, 0x136AC, 0x136AD, 0x136AF, 0x136AF, + 0x136B1, 0x136B2, 0x136B5, 0x136B9, 0x136BB, 0x136BB, 0x136BD, 0x136D7, + 0x136D9, 0x136DC, 0x136DE, 0x136EB, 0x136ED, 0x136F0, 0x136F2, 0x136F4, + 0x136F6, 0x136FC, 0x136FE, 0x13706, 0x13708, 0x1370A, 0x1370C, 0x13711, + 0x13713, 0x13729, 0x1372B, 0x1372C, 0x1372F, 0x1373D, 0x1373F, 0x13740, + 0x13742, 0x13745, 0x13747, 0x1374B, 0x1374D, 0x1374F, 0x13751, 0x13754, + 0x13758, 0x1375D, 0x1375F, 0x13769, 0x1376C, 0x1376E, 0x13770, 0x13774, + 0x13776, 0x13780, 0x13784, 0x13786, 0x13788, 0x1378D, 0x1378F, 0x13791, + 0x13793, 0x137BA, 0x137BC, 0x137BD, 0x137BF, 0x137BF, 0x137C1, 0x137C9, + 0x137CB, 0x137CE, 0x137D0, 0x137D8, 0x137DA, 0x137DC, 0x137DE, 0x137E0, + 0x137E2, 0x137F0, 0x137F2, 0x137F3, 0x137F5, 0x137FA, 0x137FC, 0x13803, + 0x13805, 0x13805, 0x13807, 0x13813, 0x13815, 0x1381E, 0x13820, 0x13820, + 0x13824, 0x13827, 0x13829, 0x1382B, 0x1382D, 0x13831, 0x13833, 0x13833, + 0x13835, 0x13847, 0x13849, 0x1384B, 0x1384E, 0x13853, 0x13855, 0x13868, + 0x1386A, 0x1386D, 0x1386F, 0x1386F, 0x13871, 0x13878, 0x1387A, 0x13886, + 0x13888, 0x13897, 0x13899, 0x13899, 0x1389B, 0x138A1, 0x138A4, 0x138B0, + 0x138B2, 0x138B2, 0x138B4, 0x138B4, 0x138B6, 0x138BB, 0x138BE, 0x138DA, + 0x138DC, 0x13900, 0x13902, 0x13905, 0x13907, 0x13923, 0x13925, 0x1393A, + 0x1393D, 0x13946, 0x13948, 0x1394B, 0x1394D, 0x1394D, 0x1394F, 0x13952, + 0x13954, 0x1395B, 0x1395D, 0x13960, 0x13962, 0x13967, 0x13969, 0x1396B, + 0x1396D, 0x13980, 0x13982, 0x13982, 0x13984, 0x13992, 0x13994, 0x139A7, + 0x139A9, 0x139BA, 0x139BC, 0x139BC, 0x139BE, 0x139D1, 0x139D3, 0x139D3, + 0x139D5, 0x139D7, 0x139D9, 0x139E2, 0x139E6, 0x13A24, 0x13A26, 0x13A2B, + 0x13A2D, 0x13A2D, 0x13A2F, 0x13A2F, 0x13A32, 0x13A39, 0x13A3B, 0x13A45, + 0x13A47, 0x13A47, 0x13A49, 0x13A49, 0x13A4C, 0x13A4C, 0x13A4E, 0x13A50, + 0x13A54, 0x13A60, 0x13A62, 0x13A6E, 0x13A70, 0x13A78, 0x13A7A, 0x13A8F, + 0x13A91, 0x13A95, 0x13A97, 0x13AA8, 0x13AAA, 0x13AAB, 0x13AAE, 0x13AB1, + 0x13AB3, 0x13ABB, 0x13ABD, 0x13ABF, 0x13AC1, 0x13AC5, 0x13AC7, 0x13AC8, + 0x13ACA, 0x13ADB, 0x13ADD, 0x13AF1, 0x13AF4, 0x13AF5, 0x13AF7, 0x13AFB, + 0x13AFE, 0x13B03, 0x13B05, 0x13B05, 0x13B07, 0x13B08, 0x13B0A, 0x13B0A, + 0x13B0C, 0x13B0F, 0x13B11, 0x13B1E, 0x13B20, 0x13B23, 0x13B25, 0x13B28, + 0x13B2B, 0x13B39, 0x13B3B, 0x13B3C, 0x13B3E, 0x13B4A, 0x13B4C, 0x13B51, + 0x13B53, 0x13B54, 0x13B56, 0x13B5D, 0x13B5F, 0x13B79, 0x13B7B, 0x13B7D, + 0x13B7F, 0x13B7F, 0x13B81, 0x13B83, 0x13B85, 0x13B8C, 0x13B8E, 0x13B95, + 0x13B97, 0x13B98, 0x13B9A, 0x13B9D, 0x13B9F, 0x13BA6, 0x13BA9, 0x13BAB, + 0x13BAE, 0x13BAE, 0x13BB2, 0x13BB9, 0x13BBC, 0x13BC0, 0x13BC2, 0x13BD8, + 0x13BDA, 0x13BE5, 0x13BE7, 0x13BEA, 0x13BEC, 0x13BEC, 0x13BEE, 0x13C05, + 0x13C07, 0x13C0B, 0x13C0D, 0x13C11, 0x13C13, 0x13C1D, 0x13C1F, 0x13C23, + 0x13C25, 0x13C25, 0x13C27, 0x13C2B, 0x13C2D, 0x13C31, 0x13C33, 0x13C34, + 0x13C36, 0x13C38, 0x13C3A, 0x13C3A, 0x13C3C, 0x13C3C, 0x13C3F, 0x13C40, + 0x13C43, 0x13C43, 0x13C45, 0x13C52, 0x13C54, 0x13C60, 0x13C62, 0x13C6A, + 0x13C6C, 0x13C7D, 0x13C7F, 0x13C88, 0x13C8A, 0x13C99, 0x13C9B, 0x13CA7, + 0x13CAA, 0x13CAD, 0x13CAF, 0x13CB0, 0x13CB2, 0x13CB8, 0x13CBA, 0x13CBA, + 0x13CBC, 0x13CBC, 0x13CBE, 0x13CCA, 0x13CCC, 0x13CCC, 0x13CCF, 0x13CD1, + 0x13CD3, 0x13CD6, 0x13CD9, 0x13CDA, 0x13CDC, 0x13CDE, 0x13CE0, 0x13CE3, + 0x13CE5, 0x13CE7, 0x13CE9, 0x13CFC, 0x13CFE, 0x13CFE, 0x13D00, 0x13D00, + 0x13D02, 0x13D0B, 0x13D0E, 0x13D1F, 0x13D23, 0x13D26, 0x13D28, 0x13D4E, + 0x13D51, 0x13D5F, 0x13D61, 0x13D65, 0x13D67, 0x13D7F, 0x13D82, 0x13D84, + 0x13D87, 0x13D9B, 0x13D9D, 0x13D9D, 0x13DA0, 0x13DA5, 0x13DA7, 0x13DAB, + 0x13DAD, 0x13DAF, 0x13DB1, 0x13DB4, 0x13DB6, 0x13DB9, 0x13DBB, 0x13DC1, + 0x13DC3, 0x13DC4, 0x13DCA, 0x13DDA, 0x13DDC, 0x13DDD, 0x13DDF, 0x13DDF, + 0x13DE1, 0x13DE7, 0x13DE9, 0x13DF2, 0x13DF4, 0x13DFB, 0x13DFD, 0x13E03, + 0x13E05, 0x13E0B, 0x13E0D, 0x13E0D, 0x13E0F, 0x13E11, 0x13E13, 0x13E17, + 0x13E19, 0x13E23, 0x13E26, 0x13E27, 0x13E2A, 0x13E2A, 0x13E2D, 0x13E36, + 0x13E38, 0x13E3A, 0x13E3C, 0x13E3C, 0x13E3F, 0x13E41, 0x13E43, 0x13E45, + 0x13E47, 0x13E48, 0x13E4A, 0x13E4C, 0x13E4E, 0x13E4F, 0x13E51, 0x13E55, + 0x13E58, 0x13E67, 0x13E6A, 0x13E70, 0x13E73, 0x13E7E, 0x13E80, 0x13E8C, + 0x13E8E, 0x13E9B, 0x13E9D, 0x13EA3, 0x13EA5, 0x13EA6, 0x13EA8, 0x13EAC, + 0x13EAF, 0x13EB5, 0x13EB7, 0x13EBC, 0x13EBE, 0x13EBE, 0x13EC0, 0x13EC0, + 0x13EC2, 0x13ED1, 0x13ED3, 0x13EDB, 0x13EDD, 0x13EF3, 0x13EF5, 0x13EF6, + 0x13EF8, 0x13F06, 0x13F08, 0x13F08, 0x13F0A, 0x13F0A, 0x13F0C, 0x13F21, + 0x13F23, 0x13F26, 0x13F28, 0x13F2A, 0x13F2C, 0x13F34, 0x13F36, 0x13F3C, + 0x13F3E, 0x13F48, 0x13F4A, 0x13F4A, 0x13F4C, 0x13F4D, 0x13F4F, 0x13F54, + 0x13F56, 0x13F57, 0x13F59, 0x13F67, 0x13F69, 0x13F76, 0x13F78, 0x13F7C, + 0x13F7F, 0x13F96, 0x13F98, 0x13F9A, 0x13F9C, 0x13F9F, 0x13FA1, 0x13FA3, + 0x13FA5, 0x13FA9, 0x13FAB, 0x13FAE, 0x13FB1, 0x13FB1, 0x13FB3, 0x13FC1, + 0x13FC3, 0x13FC6, 0x13FC9, 0x13FCB, 0x13FCD, 0x13FD0, 0x13FD2, 0x13FE2, + 0x13FE4, 0x13FE5, 0x13FE7, 0x13FE7, 0x13FEA, 0x13FEC, 0x13FEE, 0x13FF7, + 0x13FFA, 0x13FFB, 0x13FFD, 0x14000, 0x14002, 0x14003, 0x14005, 0x14012, + 0x14014, 0x14014, 0x14017, 0x14029, 0x1402B, 0x1402B, 0x1402D, 0x1402D, + 0x1402F, 0x1403A, 0x1403F, 0x14046, 0x14048, 0x14049, 0x1404B, 0x14052, + 0x14054, 0x14054, 0x14056, 0x1405A, 0x1405C, 0x1405F, 0x14061, 0x14063, + 0x14065, 0x14071, 0x14073, 0x1407A, 0x1407C, 0x1407E, 0x14080, 0x14082, + 0x14084, 0x14086, 0x14088, 0x1408D, 0x14090, 0x14090, 0x14093, 0x14095, + 0x14097, 0x14097, 0x14099, 0x1409C, 0x1409E, 0x140A1, 0x140A3, 0x140A6, + 0x140A8, 0x140AD, 0x140AF, 0x140BD, 0x140BF, 0x140BF, 0x140C1, 0x140C8, + 0x140CA, 0x140D0, 0x140D6, 0x140D6, 0x140D8, 0x140D8, 0x140DA, 0x140DE, + 0x140E1, 0x140E7, 0x140EA, 0x140EA, 0x140EC, 0x140F1, 0x140F3, 0x140F4, + 0x140F6, 0x140F6, 0x140F9, 0x140F9, 0x140FC, 0x14102, 0x14104, 0x14107, + 0x14109, 0x1410E, 0x14110, 0x14119, 0x1411B, 0x14122, 0x14124, 0x14124, + 0x14126, 0x14126, 0x14128, 0x1412D, 0x1412F, 0x1412F, 0x14131, 0x14132, + 0x14136, 0x14138, 0x1413A, 0x1413C, 0x1413E, 0x14142, 0x14144, 0x14145, + 0x14147, 0x14148, 0x1414C, 0x14162, 0x14164, 0x14166, 0x14168, 0x14169, + 0x1416B, 0x14170, 0x14173, 0x14187, 0x14189, 0x1418E, 0x14190, 0x14192, + 0x14194, 0x14197, 0x14199, 0x1419A, 0x1419C, 0x1419F, 0x141A1, 0x141A2, + 0x141A4, 0x141AD, 0x141AF, 0x141BA, 0x141BD, 0x141C3, 0x141C5, 0x141C9, + 0x141CB, 0x141D0, 0x141D2, 0x141D2, 0x141D4, 0x141D8, 0x141DA, 0x141EE, + 0x141F0, 0x141F7, 0x141FB, 0x14211, 0x14213, 0x14219, 0x1421B, 0x1421D, + 0x1421F, 0x1422B, 0x1422D, 0x14238, 0x1423A, 0x14243, 0x14245, 0x1424D, + 0x14250, 0x14251, 0x14253, 0x14253, 0x14256, 0x14256, 0x14259, 0x14263, + 0x14265, 0x14265, 0x14268, 0x14278, 0x1427A, 0x14280, 0x14282, 0x14283, + 0x14285, 0x14285, 0x14287, 0x14287, 0x14289, 0x1428D, 0x1428F, 0x14294, + 0x14296, 0x1429A, 0x1429C, 0x1429D, 0x1429F, 0x1429F, 0x142A1, 0x142A4, + 0x142A6, 0x142AC, 0x142AE, 0x142B5, 0x142B7, 0x142BF, 0x142C1, 0x142C2, + 0x142C4, 0x142CB, 0x142CD, 0x142CE, 0x142D0, 0x142D1, 0x142D3, 0x142E0, + 0x142E2, 0x142E5, 0x142E8, 0x142EC, 0x142EE, 0x142F4, 0x142F6, 0x14300, + 0x14303, 0x14304, 0x14306, 0x14308, 0x1430B, 0x1430B, 0x1430D, 0x1430D, + 0x1430F, 0x14310, 0x14312, 0x14315, 0x14317, 0x1431D, 0x1431F, 0x14325, + 0x14327, 0x14327, 0x14329, 0x1432F, 0x14331, 0x14334, 0x14336, 0x14339, + 0x1433B, 0x14346, 0x14348, 0x14350, 0x14352, 0x14364, 0x14366, 0x14367, + 0x14369, 0x1437D, 0x1437F, 0x14381, 0x14384, 0x14389, 0x1438B, 0x1438C, + 0x1438F, 0x14393, 0x14395, 0x14397, 0x14399, 0x143A5, 0x143A7, 0x143A9, + 0x143AB, 0x143AB, 0x143AE, 0x143B0, 0x143B3, 0x143B5, 0x143B7, 0x143BE, + 0x143C0, 0x143C2, 0x143C4, 0x143C9, 0x143CB, 0x143CC, 0x143CE, 0x143D5, + 0x143D7, 0x143DD, 0x143DF, 0x143EA, 0x143EE, 0x143F0, 0x143F3, 0x143F8, + 0x143FA, 0x143FA, + }; + } + private static int[] KEHCORE_ranges_1() { + return new int[] { + 0x1305D, 0x1305E, 0x1306A, 0x1306A, 0x1306C, 0x1306C, 0x13092, 0x13092, + 0x13097, 0x13097, 0x130B9, 0x130B9, 0x130C1, 0x130C1, 0x130FA, 0x130FA, + 0x1310C, 0x1310C, 0x13118, 0x13118, 0x13133, 0x13135, 0x13137, 0x13138, + 0x1313A, 0x1313C, 0x13141, 0x13141, 0x13152, 0x13152, 0x13155, 0x13157, + 0x13169, 0x13169, 0x1316B, 0x1316B, 0x13172, 0x13172, 0x13174, 0x13176, + 0x13196, 0x13196, 0x131A5, 0x131A5, 0x131AA, 0x131AA, 0x131AE, 0x131AF, + 0x131B2, 0x131B2, 0x131C6, 0x131C6, 0x131D8, 0x131D8, 0x131E1, 0x131E2, + 0x13202, 0x13202, 0x1320A, 0x1320A, 0x13215, 0x13215, 0x13251, 0x13252, + 0x13256, 0x13256, 0x1325E, 0x13264, 0x13266, 0x13266, 0x13270, 0x13270, + 0x1327C, 0x1327C, 0x1327E, 0x1327E, 0x13298, 0x13298, 0x132A3, 0x132A3, + 0x132A5, 0x132A5, 0x132DF, 0x132E1, 0x132F1, 0x132F1, 0x132F5, 0x132F6, + 0x132FB, 0x132FB, 0x1330C, 0x1330D, 0x1332D, 0x1332D, 0x1332F, 0x1332F, + 0x13335, 0x13337, 0x13339, 0x1333B, 0x13353, 0x13353, 0x1335A, 0x1335A, + 0x1339C, 0x1339C, 0x1339E, 0x1339E, 0x133A0, 0x133A0, 0x133A2, 0x133A2, + 0x133B2, 0x133B2, 0x133B9, 0x133B9, 0x133C0, 0x133C0, 0x133CD, 0x133CD, + 0x133D7, 0x133D7, 0x133DC, 0x133DC, 0x133DF, 0x133DF, 0x133E9, 0x133E9, + 0x133EB, 0x133EC, 0x13403, 0x13403, 0x13414, 0x13414, 0x13424, 0x13424, + }; + } + private static int[] KEHCORE_ranges_2() { + return new int[] { + 0x0, 0x12FFF, 0x13430, 0x13460, 0x1346E, 0x1346E, 0x13475, 0x13475, + 0x13488, 0x13488, 0x13495, 0x13495, 0x13497, 0x13497, 0x1349F, 0x1349F, + 0x134A6, 0x134A6, 0x134A9, 0x134A9, 0x134AE, 0x134AE, 0x134B7, 0x134B7, + 0x134BE, 0x134BE, 0x134C2, 0x134C2, 0x134CC, 0x134CC, 0x134F5, 0x134F5, + 0x13503, 0x13503, 0x13506, 0x13506, 0x13515, 0x13515, 0x1351B, 0x1351B, + 0x13521, 0x13521, 0x1352D, 0x1352D, 0x13532, 0x13532, 0x13544, 0x13545, + 0x1354A, 0x1354A, 0x1355F, 0x1355F, 0x1356A, 0x1356A, 0x1356D, 0x1356D, + 0x1357D, 0x1357D, 0x13581, 0x13581, 0x135A0, 0x135A0, 0x135A3, 0x135A3, + 0x135BD, 0x135BD, 0x135C0, 0x135C0, 0x135CE, 0x135CE, 0x135D1, 0x135D2, + 0x135D6, 0x135D6, 0x135DB, 0x135DB, 0x135EA, 0x135EA, 0x135EC, 0x135EC, + 0x1360E, 0x1360E, 0x1361C, 0x1361C, 0x13624, 0x13624, 0x1362B, 0x1362B, + 0x13631, 0x13631, 0x13635, 0x13635, 0x1363A, 0x1363A, 0x13649, 0x13649, + 0x1364F, 0x1364F, 0x13658, 0x13658, 0x1365A, 0x1365A, 0x1366A, 0x1366B, + 0x136A3, 0x136A3, 0x136A8, 0x136A8, 0x136AB, 0x136AB, 0x136AE, 0x136AE, + 0x136B0, 0x136B0, 0x136B3, 0x136B4, 0x136BA, 0x136BA, 0x136BC, 0x136BC, + 0x136D8, 0x136D8, 0x136DD, 0x136DD, 0x136EC, 0x136EC, 0x136F1, 0x136F1, + 0x136F5, 0x136F5, 0x136FD, 0x136FD, 0x13707, 0x13707, 0x1370B, 0x1370B, + 0x13712, 0x13712, 0x1372A, 0x1372A, 0x1372D, 0x1372E, 0x1373E, 0x1373E, + 0x13741, 0x13741, 0x13746, 0x13746, 0x1374C, 0x1374C, 0x13750, 0x13750, + 0x13755, 0x13757, 0x1375E, 0x1375E, 0x1376A, 0x1376B, 0x1376F, 0x1376F, + 0x13775, 0x13775, 0x13781, 0x13783, 0x13787, 0x13787, 0x1378E, 0x1378E, + 0x13792, 0x13792, 0x137BB, 0x137BB, 0x137BE, 0x137BE, 0x137C0, 0x137C0, + 0x137CA, 0x137CA, 0x137CF, 0x137CF, 0x137D9, 0x137D9, 0x137DD, 0x137DD, + 0x137E1, 0x137E1, 0x137F1, 0x137F1, 0x137F4, 0x137F4, 0x137FB, 0x137FB, + 0x13804, 0x13804, 0x13806, 0x13806, 0x13814, 0x13814, 0x1381F, 0x1381F, + 0x13821, 0x13823, 0x13828, 0x13828, 0x1382C, 0x1382C, 0x13832, 0x13832, + 0x13834, 0x13834, 0x13848, 0x13848, 0x1384C, 0x1384D, 0x13854, 0x13854, + 0x13869, 0x13869, 0x1386E, 0x1386E, 0x13870, 0x13870, 0x13879, 0x13879, + 0x13887, 0x13887, 0x13898, 0x13898, 0x1389A, 0x1389A, 0x138A2, 0x138A3, + 0x138B1, 0x138B1, 0x138B3, 0x138B3, 0x138B5, 0x138B5, 0x138BC, 0x138BD, + 0x138DB, 0x138DB, 0x13901, 0x13901, 0x13906, 0x13906, 0x13924, 0x13924, + 0x1393B, 0x1393C, 0x13947, 0x13947, 0x1394C, 0x1394C, 0x1394E, 0x1394E, + 0x13953, 0x13953, 0x1395C, 0x1395C, 0x13961, 0x13961, 0x13968, 0x13968, + 0x1396C, 0x1396C, 0x13981, 0x13981, 0x13983, 0x13983, 0x13993, 0x13993, + 0x139A8, 0x139A8, 0x139BB, 0x139BB, 0x139BD, 0x139BD, 0x139D2, 0x139D2, + 0x139D4, 0x139D4, 0x139D8, 0x139D8, 0x139E3, 0x139E5, 0x13A25, 0x13A25, + 0x13A2C, 0x13A2C, 0x13A2E, 0x13A2E, 0x13A30, 0x13A31, 0x13A3A, 0x13A3A, + 0x13A46, 0x13A46, 0x13A48, 0x13A48, 0x13A4A, 0x13A4B, 0x13A4D, 0x13A4D, + 0x13A51, 0x13A53, 0x13A61, 0x13A61, 0x13A6F, 0x13A6F, 0x13A79, 0x13A79, + 0x13A90, 0x13A90, 0x13A96, 0x13A96, 0x13AA9, 0x13AA9, 0x13AAC, 0x13AAD, + 0x13AB2, 0x13AB2, 0x13ABC, 0x13ABC, 0x13AC0, 0x13AC0, 0x13AC6, 0x13AC6, + 0x13AC9, 0x13AC9, 0x13ADC, 0x13ADC, 0x13AF2, 0x13AF3, 0x13AF6, 0x13AF6, + 0x13AFC, 0x13AFD, 0x13B04, 0x13B04, 0x13B06, 0x13B06, 0x13B09, 0x13B09, + 0x13B0B, 0x13B0B, 0x13B10, 0x13B10, 0x13B1F, 0x13B1F, 0x13B24, 0x13B24, + 0x13B29, 0x13B2A, 0x13B3A, 0x13B3A, 0x13B3D, 0x13B3D, 0x13B4B, 0x13B4B, + 0x13B52, 0x13B52, 0x13B55, 0x13B55, 0x13B5E, 0x13B5E, 0x13B7A, 0x13B7A, + 0x13B7E, 0x13B7E, 0x13B80, 0x13B80, 0x13B84, 0x13B84, 0x13B8D, 0x13B8D, + 0x13B96, 0x13B96, 0x13B99, 0x13B99, 0x13B9E, 0x13B9E, 0x13BA7, 0x13BA8, + 0x13BAC, 0x13BAD, 0x13BAF, 0x13BB1, 0x13BBA, 0x13BBB, 0x13BC1, 0x13BC1, + 0x13BD9, 0x13BD9, 0x13BE6, 0x13BE6, 0x13BEB, 0x13BEB, 0x13BED, 0x13BED, + 0x13C06, 0x13C06, 0x13C0C, 0x13C0C, 0x13C12, 0x13C12, 0x13C1E, 0x13C1E, + 0x13C24, 0x13C24, 0x13C26, 0x13C26, 0x13C2C, 0x13C2C, 0x13C32, 0x13C32, + 0x13C35, 0x13C35, 0x13C39, 0x13C39, 0x13C3B, 0x13C3B, 0x13C3D, 0x13C3E, + 0x13C41, 0x13C42, 0x13C44, 0x13C44, 0x13C53, 0x13C53, 0x13C61, 0x13C61, + 0x13C6B, 0x13C6B, 0x13C7E, 0x13C7E, 0x13C89, 0x13C89, 0x13C9A, 0x13C9A, + 0x13CA8, 0x13CA9, 0x13CAE, 0x13CAE, 0x13CB1, 0x13CB1, 0x13CB9, 0x13CB9, + 0x13CBB, 0x13CBB, 0x13CBD, 0x13CBD, 0x13CCB, 0x13CCB, 0x13CCD, 0x13CCE, + 0x13CD2, 0x13CD2, 0x13CD7, 0x13CD8, 0x13CDB, 0x13CDB, 0x13CDF, 0x13CDF, + 0x13CE4, 0x13CE4, 0x13CE8, 0x13CE8, 0x13CFD, 0x13CFD, 0x13CFF, 0x13CFF, + 0x13D01, 0x13D01, 0x13D0C, 0x13D0D, 0x13D20, 0x13D22, 0x13D27, 0x13D27, + 0x13D4F, 0x13D50, 0x13D60, 0x13D60, 0x13D66, 0x13D66, 0x13D80, 0x13D81, + 0x13D85, 0x13D86, 0x13D9C, 0x13D9C, 0x13D9E, 0x13D9F, 0x13DA6, 0x13DA6, + 0x13DAC, 0x13DAC, 0x13DB0, 0x13DB0, 0x13DB5, 0x13DB5, 0x13DBA, 0x13DBA, + 0x13DC2, 0x13DC2, 0x13DC5, 0x13DC9, 0x13DDB, 0x13DDB, 0x13DDE, 0x13DDE, + 0x13DE0, 0x13DE0, 0x13DE8, 0x13DE8, 0x13DF3, 0x13DF3, 0x13DFC, 0x13DFC, + 0x13E04, 0x13E04, 0x13E0C, 0x13E0C, 0x13E0E, 0x13E0E, 0x13E12, 0x13E12, + 0x13E18, 0x13E18, 0x13E24, 0x13E25, 0x13E28, 0x13E29, 0x13E2B, 0x13E2C, + 0x13E37, 0x13E37, 0x13E3B, 0x13E3B, 0x13E3D, 0x13E3E, 0x13E42, 0x13E42, + 0x13E46, 0x13E46, 0x13E49, 0x13E49, 0x13E4D, 0x13E4D, 0x13E50, 0x13E50, + 0x13E56, 0x13E57, 0x13E68, 0x13E69, 0x13E71, 0x13E72, 0x13E7F, 0x13E7F, + 0x13E8D, 0x13E8D, 0x13E9C, 0x13E9C, 0x13EA4, 0x13EA4, 0x13EA7, 0x13EA7, + 0x13EAD, 0x13EAE, 0x13EB6, 0x13EB6, 0x13EBD, 0x13EBD, 0x13EBF, 0x13EBF, + 0x13EC1, 0x13EC1, 0x13ED2, 0x13ED2, 0x13EDC, 0x13EDC, 0x13EF4, 0x13EF4, + 0x13EF7, 0x13EF7, 0x13F07, 0x13F07, 0x13F09, 0x13F09, 0x13F0B, 0x13F0B, + 0x13F22, 0x13F22, 0x13F27, 0x13F27, 0x13F2B, 0x13F2B, 0x13F35, 0x13F35, + 0x13F3D, 0x13F3D, 0x13F49, 0x13F49, 0x13F4B, 0x13F4B, 0x13F4E, 0x13F4E, + 0x13F55, 0x13F55, 0x13F58, 0x13F58, 0x13F68, 0x13F68, 0x13F77, 0x13F77, + 0x13F7D, 0x13F7E, 0x13F97, 0x13F97, 0x13F9B, 0x13F9B, 0x13FA0, 0x13FA0, + 0x13FA4, 0x13FA4, 0x13FAA, 0x13FAA, 0x13FAF, 0x13FB0, 0x13FB2, 0x13FB2, + 0x13FC2, 0x13FC2, 0x13FC7, 0x13FC8, 0x13FCC, 0x13FCC, 0x13FD1, 0x13FD1, + 0x13FE3, 0x13FE3, 0x13FE6, 0x13FE6, 0x13FE8, 0x13FE9, 0x13FED, 0x13FED, + 0x13FF8, 0x13FF9, 0x13FFC, 0x13FFC, 0x14001, 0x14001, 0x14004, 0x14004, + 0x14013, 0x14013, 0x14015, 0x14016, 0x1402A, 0x1402A, 0x1402C, 0x1402C, + 0x1402E, 0x1402E, 0x1403B, 0x1403E, 0x14047, 0x14047, 0x1404A, 0x1404A, + 0x14053, 0x14053, 0x14055, 0x14055, 0x1405B, 0x1405B, 0x14060, 0x14060, + 0x14064, 0x14064, 0x14072, 0x14072, 0x1407B, 0x1407B, 0x1407F, 0x1407F, + 0x14083, 0x14083, 0x14087, 0x14087, 0x1408E, 0x1408F, 0x14091, 0x14092, + 0x14096, 0x14096, 0x14098, 0x14098, 0x1409D, 0x1409D, 0x140A2, 0x140A2, + 0x140A7, 0x140A7, 0x140AE, 0x140AE, 0x140BE, 0x140BE, 0x140C0, 0x140C0, + 0x140C9, 0x140C9, 0x140D1, 0x140D5, 0x140D7, 0x140D7, 0x140D9, 0x140D9, + 0x140DF, 0x140E0, 0x140E8, 0x140E9, 0x140EB, 0x140EB, 0x140F2, 0x140F2, + 0x140F5, 0x140F5, 0x140F7, 0x140F8, 0x140FA, 0x140FB, 0x14103, 0x14103, + 0x14108, 0x14108, 0x1410F, 0x1410F, 0x1411A, 0x1411A, 0x14123, 0x14123, + 0x14125, 0x14125, 0x14127, 0x14127, 0x1412E, 0x1412E, 0x14130, 0x14130, + 0x14133, 0x14135, 0x14139, 0x14139, 0x1413D, 0x1413D, 0x14143, 0x14143, + 0x14146, 0x14146, 0x14149, 0x1414B, 0x14163, 0x14163, 0x14167, 0x14167, + 0x1416A, 0x1416A, 0x14171, 0x14172, 0x14188, 0x14188, 0x1418F, 0x1418F, + 0x14193, 0x14193, 0x14198, 0x14198, 0x1419B, 0x1419B, 0x141A0, 0x141A0, + 0x141A3, 0x141A3, 0x141AE, 0x141AE, 0x141BB, 0x141BC, 0x141C4, 0x141C4, + 0x141CA, 0x141CA, 0x141D1, 0x141D1, 0x141D3, 0x141D3, 0x141D9, 0x141D9, + 0x141EF, 0x141EF, 0x141F8, 0x141FA, 0x14212, 0x14212, 0x1421A, 0x1421A, + 0x1421E, 0x1421E, 0x1422C, 0x1422C, 0x14239, 0x14239, 0x14244, 0x14244, + 0x1424E, 0x1424F, 0x14252, 0x14252, 0x14254, 0x14255, 0x14257, 0x14258, + 0x14264, 0x14264, 0x14266, 0x14267, 0x14279, 0x14279, 0x14281, 0x14281, + 0x14284, 0x14284, 0x14286, 0x14286, 0x14288, 0x14288, 0x1428E, 0x1428E, + 0x14295, 0x14295, 0x1429B, 0x1429B, 0x1429E, 0x1429E, 0x142A0, 0x142A0, + 0x142A5, 0x142A5, 0x142AD, 0x142AD, 0x142B6, 0x142B6, 0x142C0, 0x142C0, + 0x142C3, 0x142C3, 0x142CC, 0x142CC, 0x142CF, 0x142CF, 0x142D2, 0x142D2, + 0x142E1, 0x142E1, 0x142E6, 0x142E7, 0x142ED, 0x142ED, 0x142F5, 0x142F5, + 0x14301, 0x14302, 0x14305, 0x14305, 0x14309, 0x1430A, 0x1430C, 0x1430C, + 0x1430E, 0x1430E, 0x14311, 0x14311, 0x14316, 0x14316, 0x1431E, 0x1431E, + 0x14326, 0x14326, 0x14328, 0x14328, 0x14330, 0x14330, 0x14335, 0x14335, + 0x1433A, 0x1433A, 0x14347, 0x14347, 0x14351, 0x14351, 0x14365, 0x14365, + 0x14368, 0x14368, 0x1437E, 0x1437E, 0x14382, 0x14383, 0x1438A, 0x1438A, + 0x1438D, 0x1438E, 0x14394, 0x14394, 0x14398, 0x14398, 0x143A6, 0x143A6, + 0x143AA, 0x143AA, 0x143AC, 0x143AD, 0x143B1, 0x143B2, 0x143B6, 0x143B6, + 0x143BF, 0x143BF, 0x143C3, 0x143C3, 0x143CA, 0x143CA, 0x143CD, 0x143CD, + 0x143D6, 0x143D6, 0x143DE, 0x143DE, 0x143EB, 0x143ED, 0x143F1, 0x143F2, + 0x143F9, 0x143F9, 0x143FB, 0x10FFFF, + }; + } + private static final int[][] KEHCORE_RANGES = { + KEHCORE_ranges_0(), KEHCORE_ranges_1(), KEHCORE_ranges_2() + }; + private static final Property KEHCORE = new Property( + KEHCORE_VALUES, KEHCORE_SHORT_VALUES, + KEHCORE_VALUE_ALIASES, KEHCORE_VALUE_ALIAS_INDEX, + KEHCORE_PROPERTY_ALIASES, KEHCORE_RANGES); + + private static final int[] HEX_RANGES = { + 0x30, 0x39, 0x41, 0x46, 0x61, 0x66, 0xFF10, 0xFF19, + 0xFF21, 0xFF26, 0xFF41, 0xFF46, + }; + private static final UnicodeSet HEX = buildSet(HEX_RANGES); + + static boolean isPropertyAlias(String alias) { return property(alias) != null; } + static boolean isBinaryPropertyAlias(String alias) { + String loose = loose(alias); + return loose.equals("hex") || loose.equals("hexdigit"); + } + static UnicodeSet binarySet(String alias) { + return isBinaryPropertyAlias(alias) ? HEX : null; + } + static UnicodeSet valueSet(String propertyAlias, String valueAlias) { + Property property = property(propertyAlias); + return property == null ? null : property.valueSet(valueAlias); + } + static int valueCount(String propertyAlias) { + Property property = property(propertyAlias); + return property == null ? 0 : property.values.length; + } + static String value(String propertyAlias, int index) { + Property property = property(propertyAlias); + return property == null ? null : property.values[index]; + } + static String shortValue(String propertyAlias, int index) { + Property property = property(propertyAlias); + return property == null ? null : property.shortValues[index]; + } + private static Property property(String alias) { + String loose = loose(alias); + if (GCB.hasAlias(loose)) return GCB; + if (InCB.hasAlias(loose)) return InCB; + if (IDTYPE.hasAlias(loose)) return IDTYPE; + if (KEHCORE.hasAlias(loose)) return KEHCORE; + return null; + } + private static String loose(String value) { + return value == null ? "" : value.toLowerCase(java.util.Locale.ROOT) + .replaceAll("[\\t-\\r _-]+", ""); + } + private static UnicodeSet buildSet(int[] ranges) { + UnicodeSet set = new UnicodeSet(); + for (int index = 0; index < ranges.length; index += 2) { + set.add(ranges[index], ranges[index + 1]); + } + return set.freeze(); + } + private static final class Property { + private final String[] values; + private final String[] shortValues; + private final String[] valueAliases; + private final byte[] valueAliasIndex; + private final String[] propertyAliases; + private final int[][] ranges; + private final UnicodeSet[] sets; + private Property(String[] values, String[] shortValues, + String[] valueAliases, byte[] valueAliasIndex, + String[] propertyAliases, int[][] ranges) { + this.values = values; + this.shortValues = shortValues; + this.valueAliases = valueAliases; + this.valueAliasIndex = valueAliasIndex; + this.propertyAliases = propertyAliases; + this.ranges = ranges; + this.sets = new UnicodeSet[values.length]; + } + private boolean hasAlias(String alias) { + return java.util.Arrays.binarySearch(propertyAliases, alias) >= 0; + } + private UnicodeSet valueSet(String alias) { + int found = java.util.Arrays.binarySearch(valueAliases, loose(alias)); + if (found < 0) return null; + int index = valueAliasIndex[found]; + UnicodeSet result = sets[index]; + if (result == null) sets[index] = result = buildSet(ranges[index]); + return result; + } + } + private PerlUnicodeResidualPropertyData() {} +} diff --git a/src/main/java/org/perlonjava/runtime/regex/PerlUnicodeScriptData.java b/src/main/java/org/perlonjava/runtime/regex/PerlUnicodeScriptData.java index 16b2b20d7..315d11dfd 100644 --- a/src/main/java/org/perlonjava/runtime/regex/PerlUnicodeScriptData.java +++ b/src/main/java/org/perlonjava/runtime/regex/PerlUnicodeScriptData.java @@ -1,5 +1,5 @@ /* - * Generated from Perl 5.44's pinned Unicode Character Database. Do not edit manually. + * Generated from the current Perl checkout's Unicode Character Database. Do not edit manually. * * Source: Scripts-17.0.0.txt * © 2025 Unicode®, Inc. diff --git a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java index 265c1b54c..0c7b4a97e 100644 --- a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java +++ b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java @@ -1530,6 +1530,11 @@ static CharacterPropertyResolver.Result resolveJoniProperty( property = normalizePerlIsPropertyAssignment(property); int assignment = propertyValueDelimiter(property); if (assignment <= 0 || assignment == property.length() - 1) { + UnicodeSet generatedBinary = + PerlUnicodeResidualPropertyData.binarySet(property); + if (generatedBinary != null) { + return joniPropertyResult(generatedBinary, true); + } UnicodeSet foldableBareProperty = resolvePerlCaseFoldableBareProperty( property, caseInsensitive); if (foldableBareProperty != null) { @@ -1559,6 +1564,10 @@ static CharacterPropertyResolver.Result resolveJoniProperty( if (blockShortcut != null) { return joniPropertyResult(blockShortcut, false); } + UnicodeSet scriptShortcut = resolvePerlBareScriptAlias(property); + if (scriptShortcut != null) { + return joniPropertyResult(scriptShortcut, false); + } if (looseIsValue == null) { PerlBarePropertyAlias bareAlias = resolvePerlBarePropertyAlias(property); if (bareAlias != null) { @@ -1603,8 +1612,11 @@ static CharacterPropertyResolver.Result resolveJoniProperty( boolean caseFold; PerlBinaryBooleanAssignment binaryAssignment = perlBinaryBooleanAssignment(property); + UnicodeSet residualAssignment = binaryAssignment == null + ? resolvePerlResidualPropertyAssignment(property) : null; UnicodeSet genericAssignment = binaryAssignment == null - ? resolveIcuPropertyAssignment(property) : null; + && residualAssignment == null + ? resolveIcuPropertyAssignment(property) : residualAssignment; if (binaryAssignment != null) { // A false assignment is a complement after applying /i to the // positive property. Folding the complement itself would add the @@ -1621,7 +1633,8 @@ static CharacterPropertyResolver.Result resolveJoniProperty( || PerlUnicodeBidiClassData.isPropertyAlias(name) || PerlUnicodeDecompositionTypeData.isPropertyAlias(name) || PerlUnicodeEastAsianWidthData.isPropertyAlias(name) - || PerlUnicodeIndicCategoryData.isPropertyAlias(name)) { + || PerlUnicodeIndicCategoryData.isPropertyAlias(name) + || PerlUnicodeResidualPropertyData.isPropertyAlias(name)) { caseFold = false; } else if (PerlUnicodeNumericValueData.isPropertyAlias(name) || PerlUnicodeJoiningGroupData.isPropertyAlias(name)) { @@ -1648,8 +1661,12 @@ static CharacterPropertyResolver.Result resolveJoniProperty( UnicodeSet set; if (binaryAssignment != null) { - set = resolveStandardPropertyAsSet( - binaryAssignment.propertyName, new LinkedHashSet<>()); + set = PerlUnicodeResidualPropertyData.binarySet( + binaryAssignment.propertyName); + if (set == null) { + set = resolveStandardPropertyAsSet( + binaryAssignment.propertyName, new LinkedHashSet<>()); + } if (!binaryAssignment.value) { set = new UnicodeSet(set); if (caseInsensitive) { @@ -1686,6 +1703,20 @@ private static UnicodeSet resolveIcuPropertyAssignment(String property) { } } + private static UnicodeSet resolvePerlResidualPropertyAssignment(String property) { + int assignment = propertyValueDelimiter(property); + if (assignment <= 0 || assignment == property.length() - 1) return null; + String name = property.substring(0, assignment); + if (!PerlUnicodeResidualPropertyData.isPropertyAlias(name)) return null; + UnicodeSet set = PerlUnicodeResidualPropertyData.valueSet( + name, property.substring(assignment + 1)); + if (set == null) { + throw new PerlCompilerException( + "Can't find Unicode property definition \"" + property + "\""); + } + return set; + } + /** Resolves an anchored Perl wildcard across an ICU enumerated property. */ private static UnicodeSet resolveIcuEnumeratedPropertyWildcard( String propertyName, String wildcardValue) { @@ -1910,6 +1941,9 @@ private static UnicodeSet resolvePerlBuiltInPropertyAlias(String property) { resolvePerlUnicodePropertyWildcard(alias); if (propertyWildcard != null) return propertyWildcard.set; if (assignment < 0) { + UnicodeSet generatedBinary = + PerlUnicodeResidualPropertyData.binarySet(alias); + if (generatedBinary != null) return generatedBinary; UnicodeSet blockBinaryPrecedence = resolvePerlBlockBinaryPrecedenceAlias(alias); if (blockBinaryPrecedence != null) return blockBinaryPrecedence; @@ -2005,6 +2039,8 @@ && isCanonicalCombiningClassProperty(alias.substring(0, assignment))) { } return indicCategory; } + UnicodeSet residualProperty = resolvePerlResidualPropertyAssignment(alias); + if (residualProperty != null) return residualProperty; if (assignment > 0 && assignment < alias.length() - 1 && PerlUnicodeNumericValueData.isPropertyAlias( alias.substring(0, assignment))) { @@ -2299,6 +2335,9 @@ private static PerlUnicodePropertyWildcard resolvePerlUnicodePropertyWildcard( compilePerlUnicodePropertyWildcard(value); if (wildcard == null) return null; + if (PerlUnicodeResidualPropertyData.isBinaryPropertyAlias(name)) { + return resolvePerlBinaryPropertyWildcard(name, wildcard); + } if (isIcuBinaryPropertyAlias(name)) { return resolvePerlBinaryPropertyWildcard(name, wildcard); } @@ -2308,6 +2347,9 @@ private static PerlUnicodePropertyWildcard resolvePerlUnicodePropertyWildcard( if (PerlUnicodeIndicCategoryData.isPropertyAlias(name)) { return resolvePerlIndicCategoryWildcard(name, wildcard); } + if (PerlUnicodeResidualPropertyData.isPropertyAlias(name)) { + return resolvePerlResidualPropertyWildcard(name, wildcard); + } if (isPerlWordBreakProperty(name)) { return resolvePerlEnumeratedPropertyWildcard( UProperty.WORD_BREAK, wildcard, "Word_Break", @@ -2354,6 +2396,27 @@ private static PerlUnicodePropertyWildcard resolvePerlIndicCategoryWildcard( return new PerlUnicodePropertyWildcard(result.freeze(), null, false); } + private static PerlUnicodePropertyWildcard resolvePerlResidualPropertyWildcard( + String propertyName, PerlPropertyValueMatcher wildcard) { + UnicodeSet result = new UnicodeSet(); + boolean matched = false; + for (int index = 0; + index < PerlUnicodeResidualPropertyData.valueCount(propertyName); + index++) { + String value = PerlUnicodeResidualPropertyData.value(propertyName, index); + String shortValue = + PerlUnicodeResidualPropertyData.shortValue(propertyName, index); + if (!matchesPerlUnicodePropertyWildcard(wildcard, shortValue, value)) continue; + matched = true; + result.addAll(PerlUnicodeResidualPropertyData.valueSet(propertyName, value)); + } + if (!matched) { + throw new IllegalArgumentException( + "No Unicode property value wildcard matches " + propertyName.trim()); + } + return new PerlUnicodePropertyWildcard(result.freeze(), null, false); + } + private static PerlUnicodePropertyWildcard resolvePerlBinaryPropertyWildcard( String propertyName, PerlPropertyValueMatcher wildcard) { boolean positive = matchesPerlUnicodePropertyWildcard( @@ -2366,8 +2429,11 @@ private static PerlUnicodePropertyWildcard resolvePerlBinaryPropertyWildcard( + propertyName.trim()); } - UnicodeSet propertySet = resolveStandardPropertyAsSet( - propertyName, new LinkedHashSet<>()); + UnicodeSet propertySet = PerlUnicodeResidualPropertyData.binarySet(propertyName); + if (propertySet == null) { + propertySet = resolveStandardPropertyAsSet( + propertyName, new LinkedHashSet<>()); + } if (propertySet == null) { throw new IllegalArgumentException( "Unsupported binary property: " + propertyName.trim()); @@ -2872,6 +2938,24 @@ private static UnicodeSet resolvePerlBareBlockShortcut(String property) { return PerlUnicodeBlockData.set(value); } + private static UnicodeSet resolvePerlBareScriptAlias(String property) { + if (propertyValueDelimiter(property) >= 0) return null; + for (int index = 0; index < property.length(); index++) { + if (property.charAt(index) > 0x7f) return null; + } + String isValue = looseIsShortcutValue(property); + String candidate = isValue == null ? property : isValue; + if (isIcuBinaryPropertyAlias(candidate) + || isIcuGeneralCategoryAlias(candidate)) { + return null; + } + String canonical = PerlUnicodeScriptData.canonicalValue(candidate); + if (canonical == null || "Katakana_Or_Hiragana".equals(canonical)) { + return null; + } + return PerlUnicodeScriptData.scriptExtensionsSet(candidate); + } + private static boolean isPerlIsPrefixedNumericWildcard(String property) { int assignment = propertyValueDelimiter(property); if (assignment <= 0 || assignment == property.length() - 1) return false; diff --git a/src/test/java/org/perlonjava/runtime/regex/PerlUnicodeResidualPropertyDataTest.java b/src/test/java/org/perlonjava/runtime/regex/PerlUnicodeResidualPropertyDataTest.java new file mode 100644 index 000000000..75c0032a9 --- /dev/null +++ b/src/test/java/org/perlonjava/runtime/regex/PerlUnicodeResidualPropertyDataTest.java @@ -0,0 +1,58 @@ +package org.perlonjava.runtime.regex; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.joni.CharacterPropertyResolver; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +@Tag("unit") +class PerlUnicodeResidualPropertyDataTest { + @Test + void resolvesGeneratedEnumeratedPropertiesThroughJoni() { + assertContains("InCB=Linker", 0x094D); + assertContains("GCB=Extend", 0x0300); + assertContains("Identifier_Type=Recommended", 'A'); + assertContains("kEH_Core=C", 0x13000); + assertContains("kEH_Core=L", 0x1305D); + assertContains("kEH_Core=N", 'A'); + } + + @Test + void resolvesWildcardsAndRetainedEmptyValues() { + assertContains("InCB=:^Consonant$:", 0x0915); + assertContains("Identifier_Type=:^Recommended$:", 'A'); + CharacterPropertyResolver.Result empty = + UnicodeResolver.resolveJoniProperty("GCB=:^E_Base$:", false); + assertNotNull(empty); + assertFalse(contains(empty.ranges, 0x1F466)); + } + + @Test + void pinsHexDigitAheadOfTheOlderIcuData() { + assertContains("Hex", 0xFF46); + assertContains("Hex=True", 0xFF46); + CharacterPropertyResolver.Result negative = + UnicodeResolver.resolveJoniProperty("Hex=False", false); + assertNotNull(negative); + assertFalse(contains(negative.ranges, 0xFF46)); + assertTrue(contains(negative.ranges, 'G')); + } + + private static void assertContains(String property, int codePoint) { + CharacterPropertyResolver.Result result = + UnicodeResolver.resolveJoniProperty(property, false, true); + assertNotNull(result, property); + assertTrue(contains(result.ranges, codePoint), property); + } + + private static boolean contains(int[] ranges, int codePoint) { + if (ranges == null) return false; + for (int index = 1; index + 1 < ranges.length; index += 2) { + if (ranges[index] <= codePoint && codePoint <= ranges[index + 1]) return true; + } + return false; + } +} diff --git a/src/test/resources/unit/regex/residual_unicode_properties.t b/src/test/resources/unit/regex/residual_unicode_properties.t new file mode 100644 index 000000000..1923f88b4 --- /dev/null +++ b/src/test/resources/unit/regex/residual_unicode_properties.t @@ -0,0 +1,57 @@ +use strict; +use warnings; +use utf8; +no warnings 'experimental::uniprop_wildcards'; +use Test::More; + +sub property_matches { + my ($scalar, $property) = @_; + return eval '$scalar =~ /\\p{' . $property . '}/'; +} + +ok(property_matches("\x{094D}", 'InCB=Linker'), 'InCB Linker matches'); +ok(property_matches("\x{0915}", 'Indic_Conjunct_Break=Consonant'), + 'long InCB alias matches'); +ok(property_matches("\x{0300}", 'InCB=:\AExtend\z:'), + 'InCB wildcard matches a value'); +ok(property_matches('A', 'InCB=None'), 'InCB missing default matches'); + +ok(property_matches("\x{0300}", 'GCB=Extend'), 'GCB Extend matches'); +ok(property_matches("\x{0600}", 'Grapheme_Cluster_Break=Prepend'), + 'long GCB alias matches'); +ok(eval q{qr/\p{GCB=:\AE_Modifier\z:}/; 1}, + 'GCB wildcard accepts a retained empty value'); +ok(property_matches('A', 'GCB=Other'), 'GCB missing default matches'); + +ok(property_matches('A', 'Identifier_Type=Recommended'), + 'Identifier_Type Recommended matches'); +SKIP: { + skip 'Not_Character alias requires current Identifier_Type data', 2 + unless eval q{qr/\p{ID_Type=Not_Character}/; 1}; + ok(property_matches("\x{0378}", 'ID_Type=Not_Character'), + 'ID_Type default matches an unassigned scalar'); + ok(!property_matches('A', 'Identifier_Type=Not_Character'), + 'Identifier_Type default excludes explicit records'); +} +ok(property_matches('A', 'Identifier_Type=:\ARecommended\z:'), + 'Identifier_Type wildcard matches'); + +ok(property_matches("\x{FF46}", 'Hex=True'), 'Hex includes fullwidth f'); +ok(!property_matches("\x{FF46}", 'Hex=False'), 'Hex false excludes fullwidth f'); +ok(property_matches("\x{FF46}", 'Hex=:\AY\z:'), + 'Hex wildcard uses Perl Hex data'); +ok(!property_matches('G', 'Hex'), 'bare Hex excludes non-hex letters'); + +SKIP: { + my $supported = eval q{qr/\p{kEH_Core=C}/; 1}; + skip 'kEH_Core requires Unicode 17 data', 5 unless $supported; + ok(property_matches("\x{13000}", 'kEH_Core=C'), 'kEH core value matches'); + ok(property_matches("\x{1305D}", 'Is_kEH_Core=L'), 'Is_kEH_Core legacy value matches'); + ok(property_matches('A', 'kEH_Core=N'), 'kEH_Core missing default matches'); + ok(!property_matches("\x{13000}", 'kEH_Core=N'), + 'kEH_Core default excludes explicit core records'); + ok(property_matches("\x{1305D}", 'kEH_Core=:\AL\z:'), + 'kEH_Core wildcard matches'); +} + +done_testing; diff --git a/src/test/resources/unit/regex/script_bare_is_aliases.t b/src/test/resources/unit/regex/script_bare_is_aliases.t new file mode 100644 index 000000000..ea7c7fd12 --- /dev/null +++ b/src/test/resources/unit/regex/script_bare_is_aliases.t @@ -0,0 +1,34 @@ +use strict; +use warnings; +use utf8; +no warnings 'experimental::uniprop_wildcards'; +use Test::More; + +ok("\x{1DFA}" =~ /\p{Syriac}/, + 'bare long Script alias uses Script_Extensions'); +ok("\x{1DFA}" =~ /\p{Syrc}/, + 'bare short Script alias uses Script_Extensions'); +ok("\x{10450}" =~ /\p{ Shavian}/, + 'bare Script accepts leading ASCII loose separators'); +ok("\x{10450}" =~ /\p{-_Shaw}/, + 'bare short Script accepts mixed leading separators'); +ok("\x{1DFA}" =~ /\p{Is_Syriac}/, + 'Is-prefixed long Script alias resolves'); +ok("\x{1DFA}" =~ /\p{--Is_Syrc}/, + 'Is-prefixed short Script alias preserves loose spelling'); + +ok('A' =~ /\p{IsAlpha}/, 'binary alias retains precedence over Script'); +ok('A' =~ /\p{IsL}/, 'General_Category retains precedence over Script'); +ok("\x{03B1}" =~ /\p{IsGreek}/, 'Script wins its ordinary Is shortcut'); +ok("\x{03B1}" =~ /\p{Is_Script=Greek}/, + 'exact Is property assignment remains separate'); + +ok(!eval(q{qr/\p{Is_Script=:\ASyriac\z:}/; 1}), + 'Is-prefixed Script wildcard remains rejected'); +{ + local $SIG{__WARN__} = sub { }; + ok(!eval(qq{qr/\\p{Is\x{A0}Syriac}/; 1}), + 'non-ASCII whitespace is not treated as a loose separator'); +} + +done_testing; From 55d57cc7e870d1dd4a4791d0dca864a6230b9a6c Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 11:46:19 +0200 Subject: [PATCH 11/43] docs(regex): correct op/do acceptance target Record the authoritative current-Perl 71/71 target and retain the normalized PR 958 disposition requirement. Generated with Codex (https://openai.com/codex) Co-Authored-By: Codex --- dev/design/phase36-regex-parity.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index a6fed9c0a..564d472b6 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -179,8 +179,9 @@ exact-head Ubuntu/Windows CI are green. The fresh complete corpus gate is red and blocks merge. Its two `pat*` rows timed out under resource contention: isolated single-job probes execute all 1,302 assertions with 1,249 passes each, but equivalent exact-head quiet reruns remain -required. `op/do.t` needs ID-normalized disposition against standard Perl's -66/66 because PR958 over-executed it as 94/99; `class/accessor.t` drops eight; +required. `op/do.t` needs two fixes to reach standard Perl's 71/71 plus an +ID-normalized disposition because PR958 over-executed it as 94/99; +`class/accessor.t` drops eight; and `japh/abigail.t` drops one nondeterministically. Acceptance must have no new invalid, missing, timeout, truncated, incomplete, or zero-TAP row and no unresolved PR-958 pass-count decrease. The user owns the release checkout; From 60ee25ff7a0743a1792b2a40a3f708e1b895d944 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 11:50:04 +0200 Subject: [PATCH 12/43] docs(regex): refresh active phase ownership Record the current release repair and post-release Joni work slices, including the quiet exclusive policy for the expensive pat tests. Generated with Codex (https://openai.com/codex) Co-Authored-By: Codex --- dev/design/phase36-regex-parity.md | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index 564d472b6..88fa3465b 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -181,8 +181,8 @@ out under resource contention: isolated single-job probes execute all 1,302 assertions with 1,249 passes each, but equivalent exact-head quiet reruns remain required. `op/do.t` needs two fixes to reach standard Perl's 71/71 plus an ID-normalized disposition because PR958 over-executed it as 94/99; -`class/accessor.t` drops eight; -and `japh/abigail.t` drops one nondeterministically. Acceptance must have no new +`class/accessor.t` aborts before the newly imported accessor-name rows; and +`japh/abigail.t` drops one nondeterministically. Acceptance must have no new invalid, missing, timeout, truncated, incomplete, or zero-TAP row and no unresolved PR-958 pass-count decrease. The user owns the release checkout; workers use private exact-head worktrees and must not mutate, rebase, or push @@ -219,15 +219,15 @@ checks at the same immutable head. Active ownership: - P3: close the complete stable regex debug-trace contract and interpreter eval - propagation; no speculative production change is needed for the `pat*` - contention timeout. -- P4: deliver the green fatal/nonfatal diagnostic aggregation change, then close - the 200-row warning tranche. -- P5: close the complete remaining 420-assertion generated-property batch, - then the disjoint 5,064-assertion Script bare-alias tranche. -- P6: publish no-warning retirement qualification, then fix/classify the - exact-head `op/do.t`, `class/accessor.t`, and `japh/abigail.t` decreases; - resume infinite-recursion diagnostics afterward. + propagation, including exact compile/match/free lifecycle parity and imported + reducers. +- P4: finish and deliver the focused-green 200-row deprecated-property warning + tranche after its single full build. +- P5: close the disjoint 1,048-row `In...` Block-alias tranche, then remove + redundant property scanning after the P3 integration reference is published. +- P6: finish exact-head release repairs: both backends at `op/do.t` 71/71, + deterministic `japh/abigail.t`, and the already delivered class accessor-name + correction; validate all three with one combined build. - Coordinator: integration, conflict resolution, immutable acceptance, PR/CI, plan state, combined build, and release evidence. @@ -240,7 +240,10 @@ Active ownership: Unicode, `regexp.t`, `regex_sets.t`, `charset.t`, `pat.t`, `pat_thr.t`, `pat_advanced.t`, `pat_re_eval.t`, and `reg_mesg.t` gates, then repeat the complete corpus, strict PR-958 comparison, additional release checks, and - exact-head CI. + exact-head CI. Run `pat.t` and `pat_thr.t` exclusively with no external + worker builds; the runner's 600-second minimum already exceeds their + isolated 399/446-second measurements. Increase only that documented + per-file minimum if a quiet exact-head run still times out. 3. Merge PR 1087 only when every checkpoint gate is green or has a documented baseline-equivalent environmental disposition. 4. Meanwhile integrate P3 matcher-state, P4 warning, P5 generated-property, From 5776a67984e9131cf6e2baddbbe3c94570093b89 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 11:52:45 +0200 Subject: [PATCH 13/43] fix(regex): preserve deprecated Unicode property warnings Classify deprecated Hyphen values and Line_Break surrogate aliases while their original spelling, lexical warning state, and source location remain available to the quote parser. Generated with [Codex](https://openai.com/codex/) Co-Authored-By: Codex --- .../frontend/parser/StringDoubleQuoted.java | 42 ++++++++--- .../deprecated_unicode_property_warnings.t | 74 +++++++++++++++++++ 2 files changed, 105 insertions(+), 11 deletions(-) create mode 100644 src/test/resources/unit/regex/deprecated_unicode_property_warnings.t diff --git a/src/main/java/org/perlonjava/frontend/parser/StringDoubleQuoted.java b/src/main/java/org/perlonjava/frontend/parser/StringDoubleQuoted.java index 7ee46f3c0..160922ad5 100644 --- a/src/main/java/org/perlonjava/frontend/parser/StringDoubleQuoted.java +++ b/src/main/java/org/perlonjava/frontend/parser/StringDoubleQuoted.java @@ -575,18 +575,35 @@ private void preserveUnicodePropertyEscape(String escape) { int close = remaining.indexOf('}', 1); if (close >= 0) { - warnDeprecatedHyphenProperty(remaining.substring(1, close)); + warnDeprecatedUnicodeProperty(remaining.substring(1, close)); } } - private void warnDeprecatedHyphenProperty(String sourceProperty) { + private void warnDeprecatedUnicodeProperty(String sourceProperty) { String spelling = sourceProperty.replaceFirst("^\\^?\\s*", ""); - String normalized = spelling.replaceAll("[\\s_]", "") - .toLowerCase(java.util.Locale.ROOT); - boolean canonicalIsAlias = spelling.equals("IsHyphen") - || spelling.equals("Is_Hyphen"); - if (!(normalized.equals("hyphen") - || normalized.equals("ishyphen") && !canonicalIsAlias)) { + String[] assignment = spelling.split("[=:]", 2); + String property = normalizeUnicodePropertyPart(assignment[0]); + String value = assignment.length == 2 + ? normalizeUnicodePropertyPart(assignment[1]) : null; + String reason = null; + + if (property.equals("hyphen") || property.equals("ishyphen")) { + boolean canonicalIsAlias = assignment.length == 1 + && (spelling.equals("IsHyphen") || spelling.equals("Is_Hyphen")); + if (!canonicalIsAlias) { + reason = "Supplanted by Line_Break property values; " + + "see www.unicode.org/reports/tr14"; + } + } else if (value != null + && (value.equals("surrogate") || value.equals("sg")) + && (property.equals("linebreak") + || property.equals("islinebreak") + || property.equals("lb") + || property.equals("islb"))) { + reason = "Surrogates should never appear in well-formed text, " + + "and therefore shouldn't be the basis for line breaking"; + } + if (reason == null) { return; } @@ -595,9 +612,7 @@ private void warnDeprecatedHyphenProperty(String sourceProperty) { return; } String message = "Use of '" + spelling - + "' in \\p{} or \\P{} is deprecated because: " - + "Supplanted by Line_Break property values; " - + "see www.unicode.org/reports/tr14"; + + "' in \\p{} or \\P{} is deprecated because: " + reason; RuntimeScalar text = new RuntimeScalar(message); RuntimeScalar location = new RuntimeScalar( ctx.errorUtil.warningLocation(tokenIndex)); @@ -612,6 +627,11 @@ private void warnDeprecatedHyphenProperty(String sourceProperty) { } } + private static String normalizeUnicodePropertyPart(String part) { + return part.replaceAll("[\\s_]", "") + .toLowerCase(java.util.Locale.ROOT); + } + /** * Processes escape sequences for double-quoted strings. * diff --git a/src/test/resources/unit/regex/deprecated_unicode_property_warnings.t b/src/test/resources/unit/regex/deprecated_unicode_property_warnings.t new file mode 100644 index 000000000..0cb6dee1f --- /dev/null +++ b/src/test/resources/unit/regex/deprecated_unicode_property_warnings.t @@ -0,0 +1,74 @@ +use strict; +use warnings; +use Test::More; + +sub compile_probe { + my ($source, $handler_dies) = @_; + my @warnings; + local $SIG{__WARN__} = $handler_dies + ? sub { die "handler died: $_[0]" } + : sub { push @warnings, @_ }; + my $value = eval $source; + return ($value, $@, \@warnings); +} + +my ($value, $error, $warnings) = compile_probe( + q{use warnings; qr/\p{Hyphen=no}/; 1}); +is($value, 1, 'Hyphen value compiles'); +is($error, '', 'Hyphen value has no compile error'); +is(scalar @$warnings, 1, 'Hyphen value warns once'); +like($warnings->[0], + qr{\AUse of 'Hyphen=no'.*Supplanted by Line_Break property values.* at \(eval \d+\) line 1\.\n\z}s, + 'Hyphen warning preserves spelling, reason, and eval location'); + +($value, $error, $warnings) = compile_probe( + q{use warnings; qr/\P{Is_Hyphen= y}/; 1}); +is(scalar @$warnings, 1, 'Is_Hyphen value warns once through P escape'); +like($warnings->[0], qr{Use of 'Is_Hyphen= y'}, + 'Hyphen alias warning preserves whitespace spelling'); + +($value, $error, $warnings) = compile_probe( + q{use warnings; qr/\p{Line_Break=surrogate}/; 1}); +is($value, 1, 'Line_Break surrogate compiles'); +is(scalar @$warnings, 1, 'Line_Break surrogate warns once'); +like($warnings->[0], + qr{\AUse of 'Line_Break=surrogate'.*Surrogates should never appear in well-formed text.* at \(eval \d+\) line 1\.\n\z}s, + 'Line_Break warning preserves spelling, reason, and eval location'); + +($value, $error, $warnings) = compile_probe( + q{use warnings; qr/\p{Is_Lb= SG}/; 1}); +is(scalar @$warnings, 1, 'short Line_Break alias warns once'); +like($warnings->[0], qr{Use of 'Is_Lb= SG'}, + 'short Line_Break alias preserves spelling'); + +($value, $error, $warnings) = compile_probe( + q{use warnings; qr/\p{Hyphen=no}\P{Line_Break=surrogate}/; 1}); +is(scalar @$warnings, 2, 'two deprecated properties warn twice'); +like($warnings->[0], qr{Use of 'Hyphen=no'}, + 'first warning retains source order'); +like($warnings->[1], qr{Use of 'Line_Break=surrogate'}, + 'second warning retains source order'); + +($value, $error, $warnings) = compile_probe( + q{no warnings 'deprecated::unicode_property_name'; qr/\p{Hyphen=no}\p{Line_Break=surrogate}/; 1}); +is($value, 1, 'suppressed deprecated properties compile'); +is($error, '', 'suppressed deprecated properties have no error'); +is(scalar @$warnings, 0, 'lexical suppression emits no warnings'); + +($value, $error, $warnings) = compile_probe( + q{use warnings FATAL => 'deprecated::unicode_property_name'; qr/\p{Hyphen=no}/; 1}); +ok(!defined $value, 'fatal deprecated property rejects eval'); +like($error, + qr{\AUse of 'Hyphen=no'.* at \(eval \d+\) line 1\.\n\z}s, + 'fatal warning becomes located eval error'); +is(scalar @$warnings, 0, 'fatal property warning bypasses warning handler'); + +($value, $error, $warnings) = compile_probe( + q{use warnings; qr/\p{Line_Break=surrogate}/; 1}, 1); +ok(!defined $value, 'dying warning handler rejects eval'); +like($error, + qr{\Ahandler died: Use of 'Line_Break=surrogate'.* line 1\.\n\z}s, + 'warning handler receives complete located property diagnostic'); +is(scalar @$warnings, 0, 'dying handler leaves no captured warning'); + +done_testing; From f7a2011eada58cc3b362d5871f87b00bf5c0ff14 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 11:57:33 +0200 Subject: [PATCH 14/43] docs(regex): track pat performance regression Require profile-guided removal of the measured pat and pat_thr slowdown after the checkpoint correctness gate. Generated with Codex (https://openai.com/codex) Co-Authored-By: Codex --- dev/design/phase36-regex-parity.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index 88fa3465b..fdd7f13f8 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -179,7 +179,10 @@ exact-head Ubuntu/Windows CI are green. The fresh complete corpus gate is red and blocks merge. Its two `pat*` rows timed out under resource contention: isolated single-job probes execute all 1,302 assertions with 1,249 passes each, but equivalent exact-head quiet reruns remain -required. `op/do.t` needs two fixes to reach standard Perl's 71/71 plus an +required. Their isolated 399/446-second runtimes versus roughly 250 seconds at +PR958 are also a post-checkpoint optimization root: profile both revisions with +JFR and remove the measured regression rather than masking it with a larger +timeout. `op/do.t` needs two fixes to reach standard Perl's 71/71 plus an ID-normalized disposition because PR958 over-executed it as 94/99; `class/accessor.t` aborts before the newly imported accessor-name rows; and `japh/abigail.t` drops one nondeterministically. Acceptance must have no new From ce47e8a1fbcfca108b1dccf993a67daaebd09316 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 11:57:10 +0200 Subject: [PATCH 15/43] feat(regex): generate Perl Block shortcut aliases Derive Perl's bare In... Block shortcuts from the current generated keyword table, preserve Block precedence over ambiguous Script aliases, and record the current Unicode and Perl source provenance in the generated data. Generated with [OpenAI Codex](https://openai.com/codex/) Co-Authored-By: OpenAI Codex --- dev/tools/generate_perl_unicode_block_data.pl | 191 ++++++++++++------ dev/tools/perl_unicode_data_generators.json | 5 +- .../runtime/regex/PerlUnicodeBlockData.java | 159 ++++++++++++++- .../runtime/regex/UnicodeResolver.java | 2 + .../unit/regex/block_bare_in_aliases.t | 36 ++++ 5 files changed, 316 insertions(+), 77 deletions(-) create mode 100644 src/test/resources/unit/regex/block_bare_in_aliases.t diff --git a/dev/tools/generate_perl_unicode_block_data.pl b/dev/tools/generate_perl_unicode_block_data.pl index 5b816bef7..405394889 100644 --- a/dev/tools/generate_perl_unicode_block_data.pl +++ b/dev/tools/generate_perl_unicode_block_data.pl @@ -1,49 +1,66 @@ #!/usr/bin/env perl use strict; use warnings; +use Digest::SHA qw(sha256_hex); use File::Spec; use FindBin; use lib File::Spec->catdir($FindBin::Bin, 'lib'); use PerlOnJava::UnicodeGenerator qw( - loose_name parse_range read_pinned_source repo_root select_unicode_root trim verify_unicode_notice + emit_unicode_source_notices loose_name parse_range read_raw repo_root + select_perl_root select_unicode_root trim verify_unicode_notice ); binmode STDOUT, ':raw'; -my $expected_version = '17.0.0'; +my $root = repo_root($FindBin::Bin); my @required_sources = qw(version Blocks.txt PropertyAliases.txt PropValueAliases.txt); my $unicore = select_unicode_root( - repo_root => repo_root($FindBin::Bin), version => $expected_version, + repo_root => $root, version => 'current', required => \@required_sources); -my %sources = ( - Version => [File::Spec->catfile($unicore, 'version'), - '8c30575264b2772c7a69c5bb6069a28f0e0a7a0df735871bde2d99ee674316ac'], - Blocks => [File::Spec->catfile($unicore, 'Blocks.txt'), - 'c0edefaf1a19771e830a82735472716af6bf3c3975f6c2a23ffbe2580fbbcb15'], - Property_Aliases => [File::Spec->catfile($unicore, 'PropertyAliases.txt'), - '4441f573caf952ffece1d7c892e7715bd7136dfc26f96eb6f268bf1e474715fb'], - Property_Value_Aliases => [File::Spec->catfile($unicore, 'PropValueAliases.txt'), - '670d2bebb48649c04fabfbf033308073dcff47946324a8033237254c048b3b01'], +my $unicode_version = read_raw(File::Spec->catfile($unicore, 'version')); +$unicode_version =~ s/\s+\z//; +die "Malformed current Unicode version '$unicode_version'\n" + unless $unicode_version =~ /\A\d+\.\d+\.\d+\z/; +my @sources = ( + { + name => "Blocks-$unicode_version.txt", + path => File::Spec->catfile($unicore, 'Blocks.txt'), + version => qr/^# Blocks-\Q$unicode_version\E\.txt$/m, + }, + { + name => "PropertyAliases-$unicode_version.txt", + path => File::Spec->catfile($unicore, 'PropertyAliases.txt'), + version => qr/^# PropertyAliases-\Q$unicode_version\E\.txt$/m, + }, + { + name => "PropertyValueAliases-$unicode_version.txt", + path => File::Spec->catfile($unicore, 'PropValueAliases.txt'), + version => qr/^# PropertyValueAliases-\Q$unicode_version\E\.txt$/m, + }, ); - -sub source_text { - my ($name) = @_; - my ($path, $expected_hash) = @{$sources{$name}}; - return ($path, read_pinned_source(path => $path, sha256 => $expected_hash)); +for my $source (@sources) { + $source->{text} = read_raw($source->{path}); + $source->{hash} = sha256_hex($source->{text}); + die "$source->{path} is inconsistent with Unicode $unicode_version\n" + unless $source->{text} =~ $source->{version}; + verify_unicode_notice($source->{path}, $source->{text}); } +my ($blocks_source, $property_source, $value_source) = @sources; + +my $perl_root = select_perl_root( + repo_root => $root, unicode_root => $unicore, required => ['uni_keywords.h']); +my $keywords_path = File::Spec->catfile($perl_root, 'uni_keywords.h'); +my $keywords_text = read_raw($keywords_path); +my $keywords_hash = sha256_hex($keywords_text); +die "$keywords_path is not a generated current-Perl Unicode keyword table\n" + unless $keywords_text =~ /This file is built by regen\/mk_invlists\.pl/ + && $keywords_text =~ /generator script:\s*regen\/mk_invlists\.pl/; sub loose { return loose_name(@_); } sub range_from_text { return parse_range(@_); } -my ($version_path, $version_text) = source_text('Version'); -$version_text =~ s/\s+\z//; -die "Expected Unicode $expected_version, found '$version_text' in $version_path\n" - unless $version_text eq $expected_version; - -my ($property_path, $property_text) = source_text('Property_Aliases'); -die "$property_path is not pinned Unicode $expected_version data\n" - unless $property_text =~ /^# PropertyAliases-\Q$expected_version\E\.txt$/m; -verify_unicode_notice($property_path, $property_text); +my ($property_path, $property_text) = + ($property_source->{path}, $property_source->{text}); my @property_aliases; for my $line (split /\n/, $property_text) { $line =~ s/#.*//; @@ -54,10 +71,7 @@ sub source_text { die "Unexpected Block aliases in $property_path: @property_aliases\n" unless join("\0", @property_aliases) eq join("\0", 'blk', 'Block'); -my ($value_path, $value_text) = source_text('Property_Value_Aliases'); -die "$value_path is not pinned Unicode $expected_version data\n" - unless $value_text =~ /^# PropertyValueAliases-\Q$expected_version\E\.txt$/m; -verify_unicode_notice($value_path, $value_text); +my ($value_path, $value_text) = ($value_source->{path}, $value_source->{text}); my (@value_rows, %row_for_long, %row_for_alias); my $in_block_values = 0; for my $line (split /\n/, $value_text) { @@ -84,27 +98,19 @@ sub source_text { $row_for_alias{$key} = $row; } } -die "Expected 347 Block value records, found " . scalar(@value_rows) . "\n" - unless @value_rows == 347; +die "Current Block aliases define no values\n" unless @value_rows; my %exact_aliases; my $alias_field_count = 0; for my $row (@value_rows) { $alias_field_count += @$row; $exact_aliases{$_} = 1 for @$row; } -die "Expected 700 Block alias fields, found $alias_field_count\n" - unless $alias_field_count == 700; -die "Expected 496 exact Block aliases, found " . scalar(keys %exact_aliases) . "\n" - unless keys(%exact_aliases) == 496; -die "Expected 495 loose Block aliases, found " . scalar(keys %row_for_alias) . "\n" - unless keys(%row_for_alias) == 495; +die "Current Block aliases define no exact or loose names\n" + unless $alias_field_count && keys(%exact_aliases) && keys(%row_for_alias); my $no_block_row = $row_for_long{loose('No_Block')}; die "Missing No_Block aliases\n" unless defined $no_block_row; -my ($blocks_path, $blocks_text) = source_text('Blocks'); -die "$blocks_path is not pinned Unicode $expected_version data\n" - unless $blocks_text =~ /^# Blocks-\Q$expected_version\E\.txt$/m; -verify_unicode_notice($blocks_path, $blocks_text); +my ($blocks_path, $blocks_text) = ($blocks_source->{path}, $blocks_source->{text}); my (@missing, @explicit); for my $line (split /\n/, $blocks_text) { if ($line =~ /^\#\s*\@missing:\s*([0-9A-F]+(?:\.\.[0-9A-F]+)?)\s*; @@ -122,12 +128,10 @@ sub source_text { die "Expected one ordered Block \@missing rule\n" unless @missing == 1 && $missing[0][0] == 0 && $missing[0][1] == 0x10ffff && loose($missing[0][2]) eq loose('No_Block'); -die "Expected 346 explicit Block ranges, found " . scalar(@explicit) . "\n" - unless @explicit == 346; +die "Current Block data has no explicit ranges\n" unless @explicit; my @value_names = ('No_Block'); my %value_id_for_row = ($no_block_row => 0); -my $explicit_count = 0; for my $index (0 .. $#explicit) { my ($first, $last, $source_name) = @{$explicit[$index]}; die sprintf("Invalid Block range U+%04X..U+%04X\n", $first, $last) @@ -144,12 +148,9 @@ sub source_text { $value_id_for_row{$row} = $value_id; push @value_names, $value_rows[$row][1]; $explicit[$index][2] = $value_id; - $explicit_count += $last - $first + 1; } -die "Expected 347 reachable Block values, found " . scalar(@value_names) . "\n" - unless @value_names == 347 && keys(%value_id_for_row) == 347; -die "Expected 303,808 named-Block code points, found $explicit_count\n" - unless $explicit_count == 303_808; +die "Not every current Block value is reachable from Blocks.txt\n" + unless @value_names == @value_rows && keys(%value_id_for_row) == @value_rows; my @partition; my $cursor = 0; @@ -159,8 +160,6 @@ sub source_text { $cursor = $range->[1] + 1; } push @partition, [$cursor, 0x10ffff, 0] if $cursor <= 0x10ffff; -die "Expected 397 complete Block intervals, found " . scalar(@partition) . "\n" - unless @partition == 397; my ($partition_count, $no_block_count, $no_block_ranges) = (0, 0, 0); for my $index (0 .. $#partition) { my ($first, $last, $value_id) = @{$partition[$index]}; @@ -180,8 +179,8 @@ sub source_text { unless $partition[-1][1] == 0x10ffff; die "Block partition does not cover the Unicode scalar universe\n" unless $partition_count == 0x110000; -die "Expected 51 No_Block ranges and 810,304 code points\n" - unless $no_block_ranges == 51 && $no_block_count == 810_304; +die "Current Block partition has no No_Block ranges\n" + unless $no_block_ranges && $no_block_count; my %alias_value_id; for my $key (keys %row_for_alias) { @@ -191,26 +190,70 @@ sub source_text { } my @alias_keys = sort keys %alias_value_id; +# Perl adds bare In... compatibility spellings that are not all expressible by +# simply prefixing an official Block value alias. Derive them from the current +# generated keyword table by joining its C property token to a reachable +# block=/blk= spelling from the same table. +my (@keyword_rows, %block_row_for_token); +for my $line (split /\n/, $keywords_text) { + next unless $line =~ /\b(-?UNI_[A-Z0-9_]+)\s*\}\s*\/\*\s*([^*]+?)\s*\*\//; + my ($token, $keyword) = ($1, trim($2)); + next if $token =~ s/^-//; + push @keyword_rows, [$token, $keyword]; + next unless $keyword =~ /\A(?:blk|block)=(.+)\z/; + my $row = $row_for_alias{loose($1)}; + next unless defined $row; + die "Perl keyword token $token maps to multiple Block values\n" + if exists $block_row_for_token{$token} + && $block_row_for_token{$token} != $row; + $block_row_for_token{$token} = $row; +} +my %shortcut_value_id; +for my $keyword_row (@keyword_rows) { + my ($token, $keyword) = @$keyword_row; + next unless $keyword =~ /\Ain[^=]+\z/; + my $row = $block_row_for_token{$token}; + next unless defined $row; + my $value_id = $value_id_for_row{$row}; + die "Perl Block shortcut '$keyword' has no reachable value\n" + unless defined $value_id; + my $key = loose($keyword); + die "Perl Block shortcut '$keyword' collides across values\n" + if exists $shortcut_value_id{$key} + && $shortcut_value_id{$key} != $value_id; + $shortcut_value_id{$key} = $value_id; +} +die "Current Perl keyword table defines no bare In... Block shortcuts\n" + unless keys %shortcut_value_id; +my @shortcut_keys = sort keys %shortcut_value_id; + print <<'HEADER'; +/* + * Generated from the current Perl checkout's Unicode Character Database and + * uni_keywords.h. Do not edit manually. + * +HEADER +emit_unicode_source_notices(\@sources); +print <<'HEADER_END'; + * Perl keyword-generator provenance: + * uni_keywords.h is generated by Perl's regen/mk_invlists.pl. + * Distributed under either the GNU General Public License or the Artistic License, + * as specified by the Perl source-tree README. + * + */ package org.perlonjava.runtime.regex; import com.ibm.icu.text.UnicodeSet; import java.util.Arrays; -/* - * Generated from Perl 5.44's pinned Unicode Character Database by - * dev/tools/generate_perl_unicode_block_data.pl. Do not edit manually. - * - * Unicode data source copyright: - * © 2025 Unicode®, Inc. - * Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in - * the U.S. and other countries. - * For terms of use and license, see https://www.unicode.org/terms_of_use.html - */ final class PerlUnicodeBlockData { -HEADER +HEADER_END -print " static final String UNICODE_VERSION = \"$expected_version\";\n"; +print " static final String UNICODE_VERSION = \"$unicode_version\";\n"; +print " static final String BLOCKS_SHA256 = \"$blocks_source->{hash}\";\n"; +print " static final String PROPERTY_ALIASES_SHA256 = \"$property_source->{hash}\";\n"; +print " static final String PROP_VALUE_ALIASES_SHA256 = \"$value_source->{hash}\";\n"; +print " static final String PERL_UNI_KEYWORDS_SHA256 = \"$keywords_hash\";\n"; print " static final short INVALID = -1;\n"; print " static final short NO_BLOCK = 0;\n\n"; print " private static final String[] VALUE_NAMES = {\n"; @@ -235,6 +278,16 @@ package org.perlonjava.runtime.regex; my $end = $i + 19 < $#alias_keys ? $i + 19 : $#alias_keys; print " ", join(', ', map { $alias_value_id{$alias_keys[$_]} } $i .. $end), ",\n"; } +print " };\n\n private static final String[] SHORTCUT_KEYS = {\n"; +for (my $i = 0; $i < @shortcut_keys; $i += 6) { + my $end = $i + 5 < $#shortcut_keys ? $i + 5 : $#shortcut_keys; + print " ", join(', ', map { qq{\"$shortcut_keys[$_]\"} } $i .. $end), ",\n"; +} +print " };\n\n private static final short[] SHORTCUT_VALUE_IDS = {\n"; +for (my $i = 0; $i < @shortcut_keys; $i += 20) { + my $end = $i + 19 < $#shortcut_keys ? $i + 19 : $#shortcut_keys; + print " ", join(', ', map { $shortcut_value_id{$shortcut_keys[$_]} } $i .. $end), ",\n"; +} print <<'FOOTER'; }; @@ -265,6 +318,12 @@ package org.perlonjava.runtime.regex; return valueId == INVALID ? null : SETS[valueId]; } + static UnicodeSet shortcutSet(String alias) { + String key = loose(alias); + int index = Arrays.binarySearch(SHORTCUT_KEYS, key); + return index < 0 ? null : SETS[SHORTCUT_VALUE_IDS[index]]; + } + static short value(String valueAlias) { String key = loose(valueAlias); int index = Arrays.binarySearch(ALIAS_KEYS, key); diff --git a/dev/tools/perl_unicode_data_generators.json b/dev/tools/perl_unicode_data_generators.json index 63a2350ca..cf5853931 100644 --- a/dev/tools/perl_unicode_data_generators.json +++ b/dev/tools/perl_unicode_data_generators.json @@ -22,7 +22,10 @@ "generator" : "dev/tools/generate_perl_unicode_block_data.pl", "name" : "block", "output" : "src/main/java/org/perlonjava/runtime/regex/PerlUnicodeBlockData.java", - "output_sha256" : "8bba409ec13fe03da3c47545865917fb251136e1bc531c055deb12849c36b255", + "output_sha256" : "fe6cccf1074a7911f50c730957ab70781456346fe8566e662c62b55059627558", + "perl_sources" : { + "uni_keywords.h" : "ccbc1b09be7e393558d3a0ebbd1eb53d5ed1c5d7db6f04effca2977bb622432a" + }, "sources" : { "Blocks.txt" : "c0edefaf1a19771e830a82735472716af6bf3c3975f6c2a23ffbe2580fbbcb15" } diff --git a/src/main/java/org/perlonjava/runtime/regex/PerlUnicodeBlockData.java b/src/main/java/org/perlonjava/runtime/regex/PerlUnicodeBlockData.java index 011c23cd8..3e625bb55 100644 --- a/src/main/java/org/perlonjava/runtime/regex/PerlUnicodeBlockData.java +++ b/src/main/java/org/perlonjava/runtime/regex/PerlUnicodeBlockData.java @@ -1,20 +1,39 @@ -package org.perlonjava.runtime.regex; - -import com.ibm.icu.text.UnicodeSet; -import java.util.Arrays; - /* - * Generated from Perl 5.44's pinned Unicode Character Database by - * dev/tools/generate_perl_unicode_block_data.pl. Do not edit manually. + * Generated from the current Perl checkout's Unicode Character Database and + * uni_keywords.h. Do not edit manually. + * + * Source: Blocks-17.0.0.txt + * © 2025 Unicode®, Inc. + * Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. + * For terms of use and license, see https://www.unicode.org/terms_of_use.html * - * Unicode data source copyright: + * Source: PropertyAliases-17.0.0.txt * © 2025 Unicode®, Inc. - * Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in - * the U.S. and other countries. + * Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. * For terms of use and license, see https://www.unicode.org/terms_of_use.html + * + * Source: PropertyValueAliases-17.0.0.txt + * © 2025 Unicode®, Inc. + * Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. + * For terms of use and license, see https://www.unicode.org/terms_of_use.html + * + * Perl keyword-generator provenance: + * uni_keywords.h is generated by Perl's regen/mk_invlists.pl. + * Distributed under either the GNU General Public License or the Artistic License, + * as specified by the Perl source-tree README. + * */ +package org.perlonjava.runtime.regex; + +import com.ibm.icu.text.UnicodeSet; +import java.util.Arrays; + final class PerlUnicodeBlockData { static final String UNICODE_VERSION = "17.0.0"; + static final String BLOCKS_SHA256 = "c0edefaf1a19771e830a82735472716af6bf3c3975f6c2a23ffbe2580fbbcb15"; + static final String PROPERTY_ALIASES_SHA256 = "4441f573caf952ffece1d7c892e7715bd7136dfc26f96eb6f268bf1e474715fb"; + static final String PROP_VALUE_ALIASES_SHA256 = "670d2bebb48649c04fabfbf033308073dcff47946324a8033237254c048b3b01"; + static final String PERL_UNI_KEYWORDS_SHA256 = "ccbc1b09be7e393558d3a0ebbd1eb53d5ed1c5d7db6f04effca2977bb622432a"; static final short INVALID = -1; static final short NO_BLOCK = 0; @@ -329,6 +348,120 @@ final class PerlUnicodeBlockData { 67, 158, 185, 157, 344, 307, 240, 212, 120, 120, 123, 122, 243, 292, 292, }; + private static final String[] SHORTCUT_KEYS = { + "inadlam", "inaegeannumbers", "inahom", "inalchemical", "inalchemicalsymbols", "inalphabeticpf", + "inalphabeticpresentationforms", "inanatolianhieroglyphs", "inancientgreekmusic", "inancientgreekmusicalnotation", "inancientgreeknumbers", "inancientsymbols", + "inarabic", "inarabicexta", "inarabicextb", "inarabicextc", "inarabicextendeda", "inarabicextendedb", + "inarabicextendedc", "inarabicmath", "inarabicmathematicalalphabeticsymbols", "inarabicpfa", "inarabicpfb", "inarabicpresentationformsa", + "inarabicpresentationformsb", "inarabicsup", "inarabicsupplement", "inarmenian", "inarrows", "inascii", + "inavestan", "inbalinese", "inbamum", "inbamumsup", "inbamumsupplement", "inbasiclatin", + "inbassavah", "inbatak", "inbengali", "inberiaerfe", "inbhaiksuki", "inblockelements", + "inbopomofo", "inbopomofoext", "inbopomofoextended", "inboxdrawing", "inbrahmi", "inbraille", + "inbraillepatterns", "inbuginese", "inbuhid", "inbyzantinemusic", "inbyzantinemusicalsymbols", "incanadiansyllabics", + "incarian", "incaucasianalbanian", "inchakma", "incham", "incherokee", "incherokeesup", + "incherokeesupplement", "inchesssymbols", "inchorasmian", "incjk", "incjkcompat", "incjkcompatforms", + "incjkcompatibility", "incjkcompatibilityforms", "incjkcompatibilityideographs", "incjkcompatibilityideographssupplement", "incjkcompatideographs", "incjkcompatideographssup", + "incjkexta", "incjkextb", "incjkextc", "incjkextd", "incjkexte", "incjkextf", + "incjkextg", "incjkexth", "incjkexti", "incjkextj", "incjkradicalssup", "incjkradicalssupplement", + "incjkstrokes", "incjksymbols", "incjksymbolsandpunctuation", "incjkunifiedideographs", "incjkunifiedideographsextensiona", "incjkunifiedideographsextensionb", + "incjkunifiedideographsextensionc", "incjkunifiedideographsextensiond", "incjkunifiedideographsextensione", "incjkunifiedideographsextensionf", "incjkunifiedideographsextensiong", "incjkunifiedideographsextensionh", + "incjkunifiedideographsextensioni", "incjkunifiedideographsextensionj", "incombiningdiacriticalmarks", "incombiningdiacriticalmarksextended", "incombiningdiacriticalmarksforsymbols", "incombiningdiacriticalmarkssupplement", + "incombininghalfmarks", "incombiningmarksforsymbols", "incommonindicnumberforms", "incompatjamo", "incontrolpictures", "incoptic", + "incopticepactnumbers", "incountingrod", "incountingrodnumerals", "incuneiform", "incuneiformnumbers", "incuneiformnumbersandpunctuation", + "incurrencysymbols", "incypriotsyllabary", "incyprominoan", "incyrillic", "incyrillicexta", "incyrillicextb", + "incyrillicextc", "incyrillicextd", "incyrillicextendeda", "incyrillicextendedb", "incyrillicextendedc", "incyrillicextendedd", + "incyrillicsup", "incyrillicsupplement", "incyrillicsupplementary", "indeseret", "indevanagari", "indevanagariext", + "indevanagariexta", "indevanagariextended", "indevanagariextendeda", "indiacriticals", "indiacriticalsext", "indiacriticalsforsymbols", + "indiacriticalssup", "indicnumberforms", "indicsiyaqnumbers", "indingbats", "indivesakuru", "indogra", + "indomino", "indominotiles", "induployan", "inearlydynasticcuneiform", "inegyptianhieroglyphformatcontrols", "inegyptianhieroglyphs", + "inegyptianhieroglyphsexta", "inegyptianhieroglyphsextendeda", "inelbasan", "inelymaic", "inemoticons", "inenclosedalphanum", + "inenclosedalphanumerics", "inenclosedalphanumericsupplement", "inenclosedalphanumsup", "inenclosedcjk", "inenclosedcjklettersandmonths", "inenclosedideographicsup", + "inenclosedideographicsupplement", "inethiopic", "inethiopicext", "inethiopicexta", "inethiopicextb", "inethiopicextended", + "inethiopicextendeda", "inethiopicextendedb", "inethiopicsup", "inethiopicsupplement", "ingaray", "ingeneralpunctuation", + "ingeometricshapes", "ingeometricshapesext", "ingeometricshapesextended", "ingeorgian", "ingeorgianext", "ingeorgianextended", + "ingeorgiansup", "ingeorgiansupplement", "inglagolitic", "inglagoliticsup", "inglagoliticsupplement", "ingothic", + "ingrantha", "ingreek", "ingreekandcoptic", "ingreekext", "ingreekextended", "ingujarati", + "ingunjalagondi", "ingurmukhi", "ingurungkhema", "inhalfandfullforms", "inhalfmarks", "inhalfwidthandfullwidthforms", + "inhangul", "inhangulcompatibilityjamo", "inhanguljamo", "inhanguljamoextendeda", "inhanguljamoextendedb", "inhangulsyllables", + "inhanifirohingya", "inhanunoo", "inhatran", "inhebrew", "inhighprivateusesurrogates", "inhighpusurrogates", + "inhighsurrogates", "inhiragana", "inidc", "inideographicdescriptioncharacters", "inideographicsymbols", "inideographicsymbolsandpunctuation", + "inimperialaramaic", "inindicnumberforms", "inindicsiyaqnumbers", "ininscriptionalpahlavi", "ininscriptionalparthian", "inipaext", + "inipaextensions", "injamo", "injamoexta", "injamoextb", "injavanese", "inkaithi", + "inkaktoviknumerals", "inkanaexta", "inkanaextb", "inkanaextendeda", "inkanaextendedb", "inkanasup", + "inkanasupplement", "inkanbun", "inkangxi", "inkangxiradicals", "inkannada", "inkatakana", + "inkatakanaext", "inkatakanaphoneticextensions", "inkawi", "inkayahli", "inkharoshthi", "inkhitansmallscript", + "inkhmer", "inkhmersymbols", "inkhojki", "inkhudawadi", "inkiratrai", "inlao", + "inlatin1", "inlatin1sup", "inlatin1supplement", "inlatinexta", "inlatinextadditional", "inlatinextb", + "inlatinextc", "inlatinextd", "inlatinexte", "inlatinextendeda", "inlatinextendedadditional", "inlatinextendedb", + "inlatinextendedc", "inlatinextendedd", "inlatinextendede", "inlatinextendedf", "inlatinextendedg", "inlatinextf", + "inlatinextg", "inlepcha", "inletterlikesymbols", "inlimbu", "inlineara", "inlinearbideograms", + "inlinearbsyllabary", "inlisu", "inlisusup", "inlisusupplement", "inlowsurrogates", "inlycian", + "inlydian", "inmahajani", "inmahjong", "inmahjongtiles", "inmakasar", "inmalayalam", + "inmandaic", "inmanichaean", "inmarchen", "inmasaramgondi", "inmathalphanum", "inmathematicalalphanumericsymbols", + "inmathematicaloperators", "inmathoperators", "inmayannumerals", "inmedefaidrin", "inmeeteimayek", "inmeeteimayekext", + "inmeeteimayekextensions", "inmendekikakui", "inmeroiticcursive", "inmeroitichieroglyphs", "inmiao", "inmiscarrows", + "inmiscellaneousmathematicalsymbolsa", "inmiscellaneousmathematicalsymbolsb", "inmiscellaneoussymbols", "inmiscellaneoussymbolsandarrows", "inmiscellaneoussymbolsandpictographs", "inmiscellaneoussymbolssupplement", + "inmiscellaneoustechnical", "inmiscmathsymbolsa", "inmiscmathsymbolsb", "inmiscpictographs", "inmiscsymbols", "inmiscsymbolssup", + "inmisctechnical", "inmodi", "inmodifierletters", "inmodifiertoneletters", "inmongolian", "inmongoliansup", + "inmongoliansupplement", "inmro", "inmultani", "inmusic", "inmusicalsymbols", "inmyanmar", + "inmyanmarexta", "inmyanmarextb", "inmyanmarextc", "inmyanmarextendeda", "inmyanmarextendedb", "inmyanmarextendedc", + "innabataean", "innagmundari", "innandinagari", "innb", "innewa", "innewtailue", + "innko", "innoblock", "innumberforms", "innushu", "innyiakengpuachuehmong", "inocr", + "inogham", "inolchiki", "inoldhungarian", "inolditalic", "inoldnortharabian", "inoldpermic", + "inoldpersian", "inoldsogdian", "inoldsoutharabian", "inoldturkic", "inolduyghur", "inolonal", + "inopticalcharacterrecognition", "inoriya", "inornamentaldingbats", "inosage", "inosmanya", "inottomansiyaqnumbers", + "inpahawhhmong", "inpalmyrene", "inpaucinhau", "inphagspa", "inphaistos", "inphaistosdisc", + "inphoenician", "inphoneticext", "inphoneticextensions", "inphoneticextensionssupplement", "inphoneticextsup", "inplayingcards", + "inprivateuse", "inprivateusearea", "inpsalterpahlavi", "inpua", "inpunctuation", "inrejang", + "inrumi", "inruminumeralsymbols", "inrunic", "insamaritan", "insaurashtra", "insharada", + "insharadasup", "insharadasupplement", "inshavian", "inshorthandformatcontrols", "insiddham", "insidetic", + "insinhala", "insinhalaarchaicnumbers", "insmallforms", "insmallformvariants", "insmallkanaext", "insmallkanaextension", + "insogdian", "insorasompeng", "insoyombo", "inspacingmodifierletters", "inspecials", "insundanese", + "insundanesesup", "insundanesesupplement", "insunuwar", "insuparrowsa", "insuparrowsb", "insuparrowsc", + "insuperandsub", "insuperscriptsandsubscripts", "insupmathoperators", "insupplementalarrowsa", "insupplementalarrowsb", "insupplementalarrowsc", + "insupplementalmathematicaloperators", "insupplementalpunctuation", "insupplementalsymbolsandpictographs", "insupplementaryprivateuseareaa", "insupplementaryprivateuseareab", "insuppuaa", + "insuppuab", "insuppunctuation", "insupsymbolsandpictographs", "insuttonsignwriting", "insylotinagri", "insymbolsandpictographsexta", + "insymbolsandpictographsextendeda", "insymbolsforlegacycomputing", "insymbolsforlegacycomputingsup", "insymbolsforlegacycomputingsupplement", "insyriac", "insyriacsup", + "insyriacsupplement", "intagalog", "intagbanwa", "intags", "intaile", "intaitham", + "intaiviet", "intaixuanjing", "intaixuanjingsymbols", "intaiyo", "intakri", "intamil", + "intamilsup", "intamilsupplement", "intangsa", "intangut", "intangutcomponents", "intangutcomponentssup", + "intangutcomponentssupplement", "intangutsup", "intangutsupplement", "intelugu", "inthaana", "inthai", + "intibetan", "intifinagh", "intirhuta", "intodhri", "intolongsiki", "intoto", + "intransportandmap", "intransportandmapsymbols", "intulutigalari", "inucas", "inucasext", "inucasexta", + "inugaritic", "inunifiedcanadianaboriginalsyllabics", "inunifiedcanadianaboriginalsyllabicsextended", "inunifiedcanadianaboriginalsyllabicsextendeda", "invai", "invariationselectors", + "invariationselectorssupplement", "invedicext", "invedicextensions", "inverticalforms", "invithkuqi", "invs", + "invssup", "inwancho", "inwarangciti", "inyezidi", "inyijing", "inyijinghexagramsymbols", + "inyiradicals", "inyisyllables", "inzanabazarsquare", "inznamennymusic", "inznamennymusicalnotation", + }; + + private static final short[] SHORTCUT_VALUE_IDS = { + 313, 167, 238, 326, 326, 155, 155, 266, 295, 295, 168, 169, 13, 22, 21, 213, 22, 21, 213, 316, + 316, 156, 162, 156, 162, 15, 15, 11, 79, 1, 203, 59, 127, 268, 268, 1, 271, 61, 24, 275, + 250, 86, 111, 114, 114, 85, 219, 92, 92, 56, 47, 293, 293, 42, 172, 184, 222, 140, 41, 146, + 146, 330, 217, 121, 118, 160, 118, 160, 154, 339, 154, 339, 119, 333, 334, 335, 336, 337, 340, 341, + 338, 342, 105, 105, 115, 108, 108, 121, 119, 333, 334, 335, 336, 337, 340, 341, 338, 342, 7, 58, + 76, 70, 159, 76, 131, 112, 82, 99, 173, 299, 299, 259, 260, 260, 75, 189, 262, 9, 103, 126, + 64, 304, 103, 126, 64, 304, 10, 10, 10, 179, 23, 134, 247, 134, 247, 7, 58, 76, 70, 131, + 314, 89, 241, 239, 318, 318, 288, 261, 264, 263, 265, 265, 183, 218, 323, 84, 84, 320, 320, 117, + 117, 321, 321, 39, 102, 144, 311, 102, 144, 311, 40, 40, 210, 73, 87, 327, 327, 37, 65, 65, + 100, 100, 97, 303, 303, 175, 229, 8, 8, 72, 72, 26, 253, 25, 267, 163, 159, 163, 148, 112, + 38, 137, 149, 148, 209, 46, 193, 12, 151, 151, 150, 109, 107, 107, 277, 277, 190, 131, 314, 205, + 204, 5, 5, 38, 137, 149, 138, 220, 296, 285, 283, 285, 283, 284, 284, 113, 106, 106, 30, 110, + 116, 116, 256, 135, 199, 280, 49, 55, 226, 228, 273, 34, 2, 2, 2, 3, 71, 4, 98, 129, + 145, 3, 71, 4, 98, 129, 145, 188, 302, 188, 302, 62, 77, 52, 187, 166, 165, 124, 257, 257, + 152, 171, 195, 223, 317, 317, 255, 31, 19, 202, 251, 252, 300, 300, 80, 80, 297, 274, 147, 143, + 143, 312, 198, 197, 276, 96, 90, 94, 88, 96, 322, 291, 81, 90, 94, 322, 88, 291, 81, 234, + 6, 128, 50, 235, 235, 269, 227, 294, 294, 36, 141, 139, 237, 141, 139, 237, 192, 308, 242, 0, + 231, 54, 17, 0, 78, 287, 305, 83, 43, 63, 208, 174, 201, 176, 178, 214, 200, 207, 216, 309, + 83, 27, 324, 182, 181, 315, 272, 191, 246, 132, 170, 170, 194, 68, 68, 69, 69, 319, 153, 153, + 206, 153, 73, 136, 211, 211, 44, 18, 133, 224, 248, 248, 180, 289, 233, 196, 32, 225, 161, 161, + 286, 286, 215, 221, 244, 6, 164, 60, 66, 66, 249, 91, 93, 328, 74, 74, 95, 91, 93, 328, + 95, 104, 329, 345, 346, 345, 346, 104, 329, 301, 130, 331, 331, 332, 290, 290, 14, 20, 20, 45, + 48, 343, 53, 57, 142, 298, 298, 310, 236, 28, 258, 258, 270, 278, 279, 282, 282, 281, 281, 29, + 16, 33, 35, 101, 232, 186, 254, 306, 325, 325, 230, 42, 51, 245, 177, 42, 51, 245, 125, 157, + 344, 67, 67, 158, 185, 157, 344, 307, 240, 212, 120, 120, 123, 122, 243, 292, 292, + }; + private static final UnicodeSet[] SETS = buildSets(); static int valueCount() { @@ -356,6 +489,12 @@ static UnicodeSet set(String valueAlias) { return valueId == INVALID ? null : SETS[valueId]; } + static UnicodeSet shortcutSet(String alias) { + String key = loose(alias); + int index = Arrays.binarySearch(SHORTCUT_KEYS, key); + return index < 0 ? null : SETS[SHORTCUT_VALUE_IDS[index]]; + } + static short value(String valueAlias) { String key = loose(valueAlias); int index = Arrays.binarySearch(ALIAS_KEYS, key); diff --git a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java index 0c7b4a97e..5a85990b6 100644 --- a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java +++ b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java @@ -2913,6 +2913,8 @@ private static UnicodeSet resolvePerlBareBlockShortcut(String property) { || isIcuGeneralCategoryAlias(alias)) { return null; } + UnicodeSet shortcut = PerlUnicodeBlockData.shortcutSet(alias); + if (shortcut != null) return shortcut; UnicodeSet direct = PerlUnicodeBlockData.set(alias); if (direct != null) return direct; diff --git a/src/test/resources/unit/regex/block_bare_in_aliases.t b/src/test/resources/unit/regex/block_bare_in_aliases.t new file mode 100644 index 000000000..b684b6ef1 --- /dev/null +++ b/src/test/resources/unit/regex/block_bare_in_aliases.t @@ -0,0 +1,36 @@ +use strict; +use warnings; +use utf8; +use Test::More; + +ok("\x{2800}" =~ /\p{inbraille}/, + 'Perl short InBraille spelling resolves the Braille Patterns block'); +ok("\x{27FF}" !~ /\p{inbraille}/, + 'InBraille keeps the block boundary'); +ok("\x{2000}" =~ /\p{inpunctuation}/, + 'Perl InPunctuation spelling resolves General Punctuation'); +ok("\x{FE00}" =~ /\p{invs}/, + 'Perl InVS spelling resolves Variation Selectors'); +ok("\x{E000}" =~ /\p{inprivateuse}/, + 'Perl InPrivateUse spelling resolves the BMP Private Use Area'); +ok("\x{F0000}" !~ /\p{inprivateuse}/, + 'InPrivateUse does not broaden to supplementary private-use blocks'); +ok("\x{2FF0}" =~ /\p{inidc}/, + 'Perl InIDC spelling resolves Ideographic Description Characters'); +ok("\x{2800}" =~ /\p{ _-In_Braille}/, + 'In Block shortcuts retain Perl loose ASCII separators'); + +ok("\x{0370}" =~ /\p{InGreek}/, + 'InGreek includes Greek and Coptic'); +ok("\x{1F00}" !~ /\p{InGreek}/, + 'InGreek remains a block shortcut rather than Script Greek'); +ok("\x{0600}" =~ /\p{InArabic}/, + 'InArabic includes the Arabic block'); +ok("\x{0750}" !~ /\p{InArabic}/, + 'InArabic remains a block shortcut rather than Script Arabic'); +ok("\x{30A0}" =~ /\p{InKatakana}/, + 'InKatakana includes the Katakana block'); +ok("\x{31F0}" !~ /\p{InKatakana}/, + 'InKatakana remains a block shortcut rather than Script Katakana'); + +done_testing; From cf802639c7063b9e0cadf04fdc20a0c993456743 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 11:59:06 +0200 Subject: [PATCH 16/43] docs(regex): advance Unicode migration ownership Reflect the integrated warning and Block-alias tranches and assign the next XID, extended-class, and scanner-removal work. Generated with Codex (https://openai.com/codex) Co-Authored-By: Codex --- dev/design/phase36-regex-parity.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index fdd7f13f8..264fa1c6e 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -224,10 +224,10 @@ Active ownership: - P3: close the complete stable regex debug-trace contract and interpreter eval propagation, including exact compile/match/free lifecycle parity and imported reducers. -- P4: finish and deliver the focused-green 200-row deprecated-property warning - tranche after its single full build. -- P5: close the disjoint 1,048-row `In...` Block-alias tranche, then remove - redundant property scanning after the P3 integration reference is published. +- P4: close the disjoint XID binary-property root, then complete nested/scoped + extended-character-class interpolation and set-operation parity. +- P5: remove redundant property scanning after the integrated 1,048-row + `In...` Block-alias correction and the P3 integration reference are published. - P6: finish exact-head release repairs: both backends at `op/do.t` 71/71, deterministic `japh/abigail.t`, and the already delivered class accessor-name correction; validate all three with one combined build. From 73100e51af7dd72e4da053d5b400443e3971a2fb Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 12:10:21 +0200 Subject: [PATCH 17/43] docs(regex): require current Perl class oracle Record that the accessor-name release fix must be validated by a private build of the latest pulled Perl source rather than the stale installed executable. Generated with Codex (https://openai.com/codex) Co-Authored-By: Codex --- dev/design/phase36-regex-parity.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index 264fa1c6e..b23905bcb 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -191,6 +191,10 @@ unresolved PR-958 pass-count decrease. The user owns the release checkout; workers use private exact-head worktrees and must not mutate, rebase, or push the user's branch. +The class candidate must also be validated with a private build of the exact +latest pulled Perl source: the available 5.43.9 executable predates the new +leading-underscore accessor rule and cannot serve as its standard-Perl oracle. + Independent implementation continues on `integrate/phase36-post1087-wip`. Its current preserved tranche generates current-Perl InSC/InPC data and closes all 4,240 labelled `uniprops02.t` assertions through native Joni property From b36e341e865b67d5a7035f68f0a863607abacde1 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 12:22:41 +0200 Subject: [PATCH 18/43] fix: complete native regex debug trace foundation Propagate lexical lower-case and named re-debug categories across literal, runtime-source, substitution, string-eval, and interpreter compilation paths. Emit native Joni bytecode and optimizer facts while preserving stable engine identity and per-runtime compile, execute, color, and free lifecycle state. Add standard-Perl, captured-stderr, and direct-Joni coverage for lexical scoping, eval inheritance, lifecycle behavior, diagnostics, and UTF-8 debug descriptions. Generated with [OpenAI Codex](https://openai.com/codex/) Co-Authored-By: OpenAI Codex --- .../backend/bytecode/CompileOperator.java | 6 +- .../backend/bytecode/EvalStringHandler.java | 39 +++++- .../backend/bytecode/InterpretedCode.java | 2 +- .../backend/bytecode/SlowOpcodeHandler.java | 10 +- .../frontend/parser/StringParser.java | 20 ++- .../frontend/semantic/ScopedSymbolTable.java | 26 ++++ .../org/perlonjava/runtime/perlmodule/Re.java | 78 +++++++++-- .../runtime/regex/JoniRegexPattern.java | 23 +++- .../runtime/regex/RuntimeRegex.java | 64 +++++---- .../regex/RuntimeRegexSourceCompiler.java | 12 +- .../runtimetypes/RuntimeRegexState.java | 3 + .../RegexDebugTraceContractTest.java | 128 ++++++++++++++++++ .../unit/regex/re_debug_trace_contract.t | 97 +++++++++++++ .../joni/src/org/joni/ByteCodePrinter.java | 17 ++- third_party/joni/src/org/joni/Regex.java | 5 + .../org/joni/constants/internal/OPCode.java | 10 +- .../joni/test/TestRegexDebugDescription.java | 61 +++++++++ 17 files changed, 535 insertions(+), 66 deletions(-) create mode 100644 src/test/java/org/perlonjava/RegexDebugTraceContractTest.java create mode 100644 src/test/resources/unit/regex/re_debug_trace_contract.t create mode 100644 third_party/joni/test/org/joni/test/TestRegexDebugDescription.java diff --git a/src/main/java/org/perlonjava/backend/bytecode/CompileOperator.java b/src/main/java/org/perlonjava/backend/bytecode/CompileOperator.java index 5f45b0bb6..fea33c96c 100644 --- a/src/main/java/org/perlonjava/backend/bytecode/CompileOperator.java +++ b/src/main/java/org/perlonjava/backend/bytecode/CompileOperator.java @@ -1403,6 +1403,9 @@ public static void visitOperator(BytecodeCompiler bytecodeCompiler, OperatorNode } case "eval", "evalbytes" -> { if (node.operand != null) { + int evalSiteRegexDebugFlags = node instanceof EvalOperatorNode evalNode + ? evalNode.getSymbolTable().getLexicalRegexDebugFlags() + : bytecodeCompiler.symbolTable.getLexicalRegexDebugFlags(); node.operand.accept(bytecodeCompiler); int stringReg = bytecodeCompiler.lastResultReg; int rd = bytecodeCompiler.allocateOutputRegister(); @@ -1413,7 +1416,8 @@ public static void visitOperator(BytecodeCompiler bytecodeCompiler, OperatorNode bytecodeCompiler.symbolTable.featureFlagsStack.peek(), op.equals("evalbytes") ? 1 : 0, bytecodeCompiler.addToStringPool( - bytecodeCompiler.symbolTable.getWarningBitsString()) + bytecodeCompiler.symbolTable.getWarningBitsString()), + evalSiteRegexDebugFlags }); bytecodeCompiler.emitWithToken(Opcodes.EVAL_STRING, node.getIndex()); bytecodeCompiler.emitReg(rd); diff --git a/src/main/java/org/perlonjava/backend/bytecode/EvalStringHandler.java b/src/main/java/org/perlonjava/backend/bytecode/EvalStringHandler.java index 921f60bce..3b9d581d0 100644 --- a/src/main/java/org/perlonjava/backend/bytecode/EvalStringHandler.java +++ b/src/main/java/org/perlonjava/backend/bytecode/EvalStringHandler.java @@ -13,6 +13,7 @@ import org.perlonjava.frontend.semantic.ScopedSymbolTable; import org.perlonjava.runtime.HintHashRegistry; import org.perlonjava.runtime.regex.RegexQuoteMeta; +import org.perlonjava.runtime.regex.RuntimeRegex; import org.perlonjava.runtime.operators.WarnDie; import org.perlonjava.runtime.perlmodule.BHooksEndOfScope; import org.perlonjava.runtime.perlmodule.Strict; @@ -163,7 +164,7 @@ public static RuntimeList evalStringList(String perlCode, boolean isEvalbytes) { return evalStringList(perlCode, RuntimeScalarType.STRING, currentCode, registers, sourceName, sourceLine, callContext, siteRegistry, siteStrictOptions, - siteFeatureFlags, isEvalbytes, null); + siteFeatureFlags, isEvalbytes, null, -1); } public static RuntimeList evalStringList(RuntimeScalar codeScalar, @@ -194,7 +195,7 @@ public static RuntimeList evalStringList(RuntimeScalar codeScalar, RuntimeCode.rejectTaintedEval(codeScalar); return evalStringList(codeScalar.toString(), codeScalar.type, currentCode, registers, sourceName, sourceLine, callContext, siteRegistry, siteStrictOptions, - siteFeatureFlags, isEvalbytes, null); + siteFeatureFlags, isEvalbytes, null, -1); } public static RuntimeList evalStringList(RuntimeScalar codeScalar, @@ -208,10 +209,27 @@ public static RuntimeList evalStringList(RuntimeScalar codeScalar, int siteFeatureFlags, boolean isEvalbytes, String siteWarningBits) { + return evalStringList(codeScalar, currentCode, registers, sourceName, + sourceLine, callContext, siteRegistry, siteStrictOptions, + siteFeatureFlags, isEvalbytes, siteWarningBits, -1); + } + + public static RuntimeList evalStringList(RuntimeScalar codeScalar, + InterpretedCode currentCode, + RuntimeBase[] registers, + String sourceName, + int sourceLine, + int callContext, + Map siteRegistry, + int siteStrictOptions, + int siteFeatureFlags, + boolean isEvalbytes, + String siteWarningBits, + int siteRegexDebugFlags) { RuntimeCode.rejectTaintedEval(codeScalar); return evalStringList(codeScalar.toString(), codeScalar.type, currentCode, registers, sourceName, sourceLine, callContext, siteRegistry, siteStrictOptions, - siteFeatureFlags, isEvalbytes, siteWarningBits); + siteFeatureFlags, isEvalbytes, siteWarningBits, siteRegexDebugFlags); } private static RuntimeList evalStringList(String perlCode, @@ -225,7 +243,8 @@ private static RuntimeList evalStringList(String perlCode, int siteStrictOptions, int siteFeatureFlags, boolean isEvalbytes, - String siteWarningBits) { + String siteWarningBits, + int siteRegexDebugFlags) { try (PerlRuntime.Binding runtimeBinding = PerlRuntime.current().bind()) { PerlLanguageProvider.CompilationLockGuard compilationLock = PerlLanguageProvider.acquireCompilationLock(); @@ -433,6 +452,18 @@ private static RuntimeList evalStringList(String perlCode, symbolTable.warningFlagsStack.push((java.util.BitSet) currentCode.warningFlags.clone()); WarningFlags.setWarningBitsFromString(symbolTable, siteWarningBits); } + int effectiveRegexDebugFlags = siteRegexDebugFlags; + if (effectiveRegexDebugFlags < 0) { + effectiveRegexDebugFlags = 0; + if ((inheritedStrictOptions & Strict.HINT_RE_DEBUG) != 0) { + effectiveRegexDebugFlags |= RuntimeRegex.LEXICAL_DEBUG_COMPILE + | RuntimeRegex.LEXICAL_DEBUG_EXECUTE; + } + if ((inheritedStrictOptions & Strict.HINT_RE_DEBUGCOLOR) != 0) { + effectiveRegexDebugFlags |= RuntimeRegex.LEXICAL_DEBUG_COLOR; + } + } + symbolTable.setLexicalRegexDebugFlags(effectiveRegexDebugFlags); // Use runtime package (maintained by PUSH_PACKAGE/SET_PACKAGE opcodes). // This correctly reflects the current package scope when eval STRING runs diff --git a/src/main/java/org/perlonjava/backend/bytecode/InterpretedCode.java b/src/main/java/org/perlonjava/backend/bytecode/InterpretedCode.java index ca722e8d7..c949efc6c 100644 --- a/src/main/java/org/perlonjava/backend/bytecode/InterpretedCode.java +++ b/src/main/java/org/perlonjava/backend/bytecode/InterpretedCode.java @@ -34,7 +34,7 @@ public class InterpretedCode extends RuntimeCode implements PerlSubroutine { public final RuntimeBase[] capturedVars; // Closure support (captured from outer scope) public final Map variableRegistry; // Variable name → register index (for eval STRING) public final List> evalSiteRegistries; // Per-eval-site variable registries - public final List evalSitePragmaFlags; // Per-eval-site [strict, features, evalbytes, warningBitsPoolIndex] + public final List evalSitePragmaFlags; // [strict, features, evalbytes, warningBitsPoolIndex, regexDebug] // Optimization flags (set by compiler after construction) // If false, we can skip DynamicVariableManager.getLocalLevel/popToLocalLevel calls diff --git a/src/main/java/org/perlonjava/backend/bytecode/SlowOpcodeHandler.java b/src/main/java/org/perlonjava/backend/bytecode/SlowOpcodeHandler.java index 61e5a89b8..06052c90d 100644 --- a/src/main/java/org/perlonjava/backend/bytecode/SlowOpcodeHandler.java +++ b/src/main/java/org/perlonjava/backend/bytecode/SlowOpcodeHandler.java @@ -294,6 +294,7 @@ public static int executeEvalString( int siteFeatureFlags = -1; boolean siteIsEvalbytes = false; String siteWarningBits = null; + int siteRegexDebugFlags = -1; if (evalSiteIndex >= 0 && code.evalSitePragmaFlags != null && evalSiteIndex < code.evalSitePragmaFlags.size()) { int[] pragmaFlags = code.evalSitePragmaFlags.get(evalSiteIndex); @@ -304,6 +305,9 @@ public static int executeEvalString( && pragmaFlags[3] < code.stringPool.length) { siteWarningBits = code.stringPool[pragmaFlags[3]]; } + if (pragmaFlags.length > 4) { + siteRegexDebugFlags = pragmaFlags[4]; + } } RuntimeBase codeValue = registers[stringReg]; @@ -348,7 +352,8 @@ public static int executeEvalString( siteStrictOptions, siteFeatureFlags, siteIsEvalbytes, - siteWarningBits + siteWarningBits, + siteRegexDebugFlags ); registers[rd] = result; evalTrace("EVAL_STRING opcode exit LIST stored=" + (registers[rd] != null ? registers[rd].getClass().getSimpleName() : "null") + @@ -365,7 +370,8 @@ public static int executeEvalString( siteStrictOptions, siteFeatureFlags, siteIsEvalbytes, - siteWarningBits + siteWarningBits, + siteRegexDebugFlags ).scalar(); registers[rd] = result; evalTrace("EVAL_STRING opcode exit SCALAR/VOID stored=" + (registers[rd] != null ? registers[rd].getClass().getSimpleName() : "null") + diff --git a/src/main/java/org/perlonjava/frontend/parser/StringParser.java b/src/main/java/org/perlonjava/frontend/parser/StringParser.java index b7f07bb2d..d02efb699 100644 --- a/src/main/java/org/perlonjava/frontend/parser/StringParser.java +++ b/src/main/java/org/perlonjava/frontend/parser/StringParser.java @@ -29,8 +29,6 @@ import static org.perlonjava.runtime.perlmodule.Strict.HINT_RE_EVAL; import static org.perlonjava.runtime.perlmodule.Strict.HINT_RE_UNICODE; import static org.perlonjava.runtime.perlmodule.Strict.HINT_RE_TAINT; -import static org.perlonjava.runtime.perlmodule.Strict.HINT_RE_DEBUG; -import static org.perlonjava.runtime.perlmodule.Strict.HINT_RE_DEBUGCOLOR; import static org.perlonjava.runtime.perlmodule.Strict.HINT_RE_STRICT; import static org.perlonjava.runtime.perlmodule.Strict.HINT_LOCALE; import static org.perlonjava.runtime.runtimetypes.NameNormalizer.normalizeVariableName; @@ -789,12 +787,22 @@ public static void validateLiteralNamedCharacters( * makes the JVM and interpreter backends capture the same call-site value. */ private static String addLexicalRegexDebugMarker(EmitterContext ctx, String modifiers) { - if (ctx.symbolTable == null || !ctx.symbolTable.isStrictOptionEnabled(HINT_RE_DEBUG)) { + if (ctx.symbolTable == null) { return modifiers; } - return modifiers + (ctx.symbolTable.isStrictOptionEnabled(HINT_RE_DEBUGCOLOR) - ? RuntimeRegex.INTERNAL_DEBUGCOLOR_MARKER - : RuntimeRegex.INTERNAL_DEBUG_MARKER); + int debugFlags = ctx.symbolTable.getLexicalRegexDebugFlags(); + if (debugFlags == 0) return modifiers; + StringBuilder marked = new StringBuilder(modifiers); + if ((debugFlags & RuntimeRegex.LEXICAL_DEBUG_COMPILE) != 0) { + marked.append(RuntimeRegex.INTERNAL_DEBUG_COMPILE_MARKER); + } + if ((debugFlags & RuntimeRegex.LEXICAL_DEBUG_EXECUTE) != 0) { + marked.append(RuntimeRegex.INTERNAL_DEBUG_EXECUTE_MARKER); + } + if ((debugFlags & RuntimeRegex.LEXICAL_DEBUG_COLOR) != 0) { + marked.append(RuntimeRegex.INTERNAL_DEBUG_COLOR_MARKER); + } + return marked.toString(); } static String addLexicalRegexContext(EmitterContext ctx, String modifiers) { diff --git a/src/main/java/org/perlonjava/frontend/semantic/ScopedSymbolTable.java b/src/main/java/org/perlonjava/frontend/semantic/ScopedSymbolTable.java index 03e3eb29a..79bbbec07 100644 --- a/src/main/java/org/perlonjava/frontend/semantic/ScopedSymbolTable.java +++ b/src/main/java/org/perlonjava/frontend/semantic/ScopedSymbolTable.java @@ -93,6 +93,11 @@ public void inheritAllWarningsForRegisteredCategory(String category) { // These do not fit in PerlOnJava's legacy 32-bit strict-options mask and // must follow the same lexical enter/exit/snapshot rules independently. private final Stack regexModifierStack = new Stack<>(); + // Coarse stable re-debug categories captured lexically. Perl exposes many + // engine-specific named flags; the frontend preserves their compile versus + // execute ownership without baking a particular engine's bit layout into + // the AST. + private final Stack regexDebugFlagsStack = new Stack<>(); // A stack to manage nested scopes of symbol tables. private final Stack symbolTableStack = new Stack<>(); private final Stack packageStack = new Stack<>(); @@ -134,6 +139,7 @@ public ScopedSymbolTable() { // Initialize the strict options stack with 0 for the global scope strictOptionsStack.push(0); regexModifierStack.push(""); + regexDebugFlagsStack.push(0); // Initialize the package name packageStack.push(new PackageInfo("main", false, null)); // Initialize the subroutine stack with empty string (no subroutine) @@ -210,6 +216,7 @@ public int enterScope() { // Push a copy of the current strict options onto the stack strictOptionsStack.push(strictOptionsStack.peek()); regexModifierStack.push(regexModifierStack.peek()); + regexDebugFlagsStack.push(regexDebugFlagsStack.peek()); // Return the current size of the symbol table stack as the scope index return symbolTableStack.size() - 1; @@ -251,6 +258,7 @@ public void exitScope(int scopeIndex) { postderefQqStack.pop(); strictOptionsStack.pop(); regexModifierStack.pop(); + regexDebugFlagsStack.pop(); } // Propagate the child scope's index to the parent to prevent slot reuse. // This ensures that local variable slots allocated inside conditional branches @@ -445,6 +453,18 @@ public void disableLexicalRegexModifiers(String modifiers) { regexModifierStack.set(regexModifierStack.size() - 1, current); } + public int getLexicalRegexDebugFlags() { + return regexDebugFlagsStack.peek(); + } + + public void setLexicalRegexDebugFlags(int flags) { + regexDebugFlagsStack.set(regexDebugFlagsStack.size() - 1, flags); + } + + public void disableLexicalRegexDebugFlags(int flags) { + setLexicalRegexDebugFlags(getLexicalRegexDebugFlags() & ~flags); + } + /** * Adds a variable to the current scope. * @@ -799,6 +819,8 @@ public ScopedSymbolTable snapShot() { st.strictOptionsStack.push(this.strictOptionsStack.peek()); st.regexModifierStack.pop(); st.regexModifierStack.push(this.regexModifierStack.peek()); + st.regexDebugFlagsStack.pop(); + st.regexDebugFlagsStack.push(this.regexDebugFlagsStack.peek()); return st; } @@ -1117,6 +1139,10 @@ public void copyFlagsFrom(ScopedSymbolTable source, int sourceScopeIndex) { Math.min(sourceScopeIndex, source.regexModifierStack.size() - 1)); this.regexModifierStack.pop(); this.regexModifierStack.push(source.regexModifierStack.get(regexModifierIndex)); + int regexDebugIndex = Math.max(0, + Math.min(sourceScopeIndex, source.regexDebugFlagsStack.size() - 1)); + this.regexDebugFlagsStack.pop(); + this.regexDebugFlagsStack.push(source.regexDebugFlagsStack.get(regexDebugIndex)); } public record PackageInfo(String packageName, boolean isClass, String version) { diff --git a/src/main/java/org/perlonjava/runtime/perlmodule/Re.java b/src/main/java/org/perlonjava/runtime/perlmodule/Re.java index 9e22438af..61a8f790b 100644 --- a/src/main/java/org/perlonjava/runtime/perlmodule/Re.java +++ b/src/main/java/org/perlonjava/runtime/perlmodule/Re.java @@ -2,6 +2,7 @@ import org.perlonjava.frontend.semantic.ScopedSymbolTable; import org.perlonjava.runtime.regex.RuntimeRegex; +import org.perlonjava.runtime.operators.WarnDie; import org.perlonjava.runtime.runtimetypes.RuntimeArray; import org.perlonjava.runtime.runtimetypes.RuntimeCode; import org.perlonjava.runtime.runtimetypes.RuntimeContextType; @@ -62,6 +63,45 @@ private static String lexicalRegexModifiers(String option) { return flags; } + private static int namedDebugFlag(String option) { + return switch (option) { + case "COMPILE", "PARSE", "OPTIMISE", "TRIEC", "DUMP", "FLAGS", + "TEST", "EXTRA", "DUMP_PRE_OPTIMIZE", "WILDCARD" -> + RuntimeRegex.LEXICAL_DEBUG_COMPILE; + case "EXECUTE", "INTUIT", "MATCH", "TRIEE", "TRIEM", "OFFSETS", + "OFFSETSDBG", "STATE", "OPTIMISEM", "STACK", "BUFFERS", + "GPOS" -> RuntimeRegex.LEXICAL_DEBUG_EXECUTE; + case "ALL", "All", "all", "Extra", "More", "MORE", "State", "TRIE" -> + RuntimeRegex.LEXICAL_DEBUG_COMPILE + | RuntimeRegex.LEXICAL_DEBUG_EXECUTE; + default -> -1; + }; + } + + private static void warnUnknownDebugFlag(String option) { + WarnDie.warn(new RuntimeScalar("Unknown \"re\" Debug flag '" + option + + "', possible flags: ALL, All, BUFFERS, COMPILE, DUMP, " + + "DUMP_PRE_OPTIMIZE, EXECUTE, EXTRA, Extra, FLAGS, GPOS, " + + "INTUIT, MATCH, MORE, More, OFFSETS, OFFSETSDBG, OPTIMISE, " + + "OPTIMISEM, PARSE, STACK, STATE, State, TEST, TRIE, TRIEC, " + + "TRIEE, TRIEM, WILDCARD, all"), new RuntimeScalar()); + } + + private static void setDebugFlags(ScopedSymbolTable symbolTable, int flags) { + symbolTable.setLexicalRegexDebugFlags(flags); + if ((flags & (RuntimeRegex.LEXICAL_DEBUG_COMPILE + | RuntimeRegex.LEXICAL_DEBUG_EXECUTE)) == 0) { + symbolTable.disableStrictOption(Strict.HINT_RE_DEBUG | Strict.HINT_RE_DEBUGCOLOR); + return; + } + symbolTable.enableStrictOption(Strict.HINT_RE_DEBUG); + if ((flags & RuntimeRegex.LEXICAL_DEBUG_COLOR) != 0) { + symbolTable.enableStrictOption(Strict.HINT_RE_DEBUGCOLOR); + } else { + symbolTable.disableStrictOption(Strict.HINT_RE_DEBUGCOLOR); + } + } + /** * Constructor initializes the module. */ @@ -253,10 +293,28 @@ public static RuntimeList importRe(RuntimeArray args, int ctx) { symbolTable.enableStrictOption(Strict.HINT_RE_EVAL); } else if (opt.equalsIgnoreCase("taint")) { symbolTable.enableStrictOption(Strict.HINT_RE_TAINT); - } else if (opt.equalsIgnoreCase("debug")) { - symbolTable.enableStrictOption(Strict.HINT_RE_DEBUG); - } else if (opt.equalsIgnoreCase("debugcolor")) { - symbolTable.enableStrictOption(Strict.HINT_RE_DEBUG | Strict.HINT_RE_DEBUGCOLOR); + } else if (opt.equals("Debug") || opt.equals("Debugcolor")) { + int flags = opt.equals("Debugcolor") + ? RuntimeRegex.LEXICAL_DEBUG_COLOR : 0; + for (int j = i + 1; j < args.size(); j++) { + String namedFlag = args.get(j).toString() + .replace("\"", "").replace("'", "").trim(); + int debugFlag = namedDebugFlag(namedFlag); + if (debugFlag < 0) { + warnUnknownDebugFlag(namedFlag); + } else { + flags |= debugFlag; + } + } + setDebugFlags(symbolTable, flags); + break; + } else if (opt.equals("debug")) { + setDebugFlags(symbolTable, RuntimeRegex.LEXICAL_DEBUG_COMPILE + | RuntimeRegex.LEXICAL_DEBUG_EXECUTE); + } else if (opt.equals("debugcolor")) { + setDebugFlags(symbolTable, RuntimeRegex.LEXICAL_DEBUG_COMPILE + | RuntimeRegex.LEXICAL_DEBUG_EXECUTE + | RuntimeRegex.LEXICAL_DEBUG_COLOR); } else if (opt.equals("/a")) { // use re '/a' - ASCII-restrict regex character classes symbolTable.enableStrictOption(Strict.HINT_RE_ASCII); @@ -299,10 +357,14 @@ public static RuntimeList unimportRe(RuntimeArray args, int ctx) { symbolTable.disableStrictOption(Strict.HINT_RE_EVAL); } else if (opt.equalsIgnoreCase("taint")) { symbolTable.disableStrictOption(Strict.HINT_RE_TAINT); - } else if (opt.equalsIgnoreCase("debug")) { - symbolTable.disableStrictOption(Strict.HINT_RE_DEBUG | Strict.HINT_RE_DEBUGCOLOR); - } else if (opt.equalsIgnoreCase("debugcolor")) { - symbolTable.disableStrictOption(Strict.HINT_RE_DEBUGCOLOR); + } else if (opt.equals("Debug") || opt.equals("Debugcolor")) { + // re.pm resets ${^RE_DEBUG_FLAGS} before processing a named + // unimport. Removing any named Debug selection therefore turns + // the engine hook off for this lexical scope. + setDebugFlags(symbolTable, 0); + break; + } else if (opt.equals("debug") || opt.equals("debugcolor")) { + setDebugFlags(symbolTable, 0); } else if (opt.equals("/a") || opt.equals("/aa")) { symbolTable.disableStrictOption(Strict.HINT_RE_ASCII | Strict.HINT_RE_ASCII_AA); } else if (opt.equals("/u")) { diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index af5833c56..09454dc43 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -505,14 +505,27 @@ Regex engineRegex() { } String optimizerDebugDescription() { + org.joni.Regex.OptimizationInfo info = regex.getOptimizationInfo(); + StringBuilder description = new StringBuilder("optimizer search=") + .append(info.searchAlgorithm()) + .append(" minlen=").append(info.minimumLength()); + if (info.exact() != null) { + description.append(" exact=<").append(info.exact()).append('>') + .append(" offset=").append(info.minimumOffset()).append(".."); + description.append(info.maximumOffset() == null + ? "infinity" : info.maximumOffset()); + } int anchor = regex.getAnchor(); if ((anchor & AnchorType.ANYCHAR_STAR_ML) != 0) { - return "anchored(SBOL) implicit"; - } - if ((anchor & AnchorType.ANYCHAR_STAR) != 0) { - return "anchored(MBOL) implicit"; + description.append(" anchored(SBOL) implicit"); + } else if ((anchor & AnchorType.ANYCHAR_STAR) != 0) { + description.append(" anchored(MBOL) implicit"); } - return ""; + return description.toString(); + } + + String nativeCompileDebugDescription() { + return regex.byteCodeDebugDescription(); } boolean hasOnlyAuthoritativeWideCharacterClasses() { diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java index 75d0ac3b0..51c9dbf97 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java @@ -24,8 +24,6 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; import static org.perlonjava.runtime.regex.RegexFlags.fromModifiers; @@ -65,9 +63,13 @@ public static RuntimeScalar stabilizeLiteralTarget(RuntimeScalar literal, int ca } /** Private AST/runtime modifier markers; removed before Perl modifier parsing. */ - public static final char INTERNAL_DEBUG_MARKER = '\u0001'; - public static final char INTERNAL_DEBUGCOLOR_MARKER = '\u0002'; + public static final char INTERNAL_DEBUG_COMPILE_MARKER = '\u0001'; + public static final char INTERNAL_DEBUG_EXECUTE_MARKER = '\u0002'; public static final char INTERNAL_RE_STRICT_MARKER = '\u0003'; + public static final char INTERNAL_DEBUG_COLOR_MARKER = '\u0004'; + public static final int LEXICAL_DEBUG_COMPILE = 1; + public static final int LEXICAL_DEBUG_EXECUTE = 2; + public static final int LEXICAL_DEBUG_COLOR = 4; // Debug flag for regex compilation (set at class load time) private static final boolean DEBUG_REGEX = System.getenv("DEBUG_REGEX") != null; @@ -80,12 +82,6 @@ public static RuntimeScalar stabilizeLiteralTarget(RuntimeScalar literal, int ca Pattern.compile("\\\\[pP]\\{"); // Maximum size for each runtime's regex cache. private static final int MAX_REGEX_CACHE_SIZE = RuntimeRegexState.MAX_REGEX_CACHE_SIZE; - // Literal CVs are shared by ithreads. A child runtime may populate its own - // regex cache, but that must not look like a second compilation of the - // already-compiled literal in re-debug output. - private static final Set REPORTED_DEBUG_COMPILATIONS = - ConcurrentHashMap.newKeySet(); - private static RuntimeRegexState state() { return PerlRuntime.current().regexState; } @@ -1018,7 +1014,9 @@ && containsExecutableSource(originalPatternString, || state().compiledRegexCache.containsKey(cacheKey)) { state().compiledRegexCache.put(cacheKey, regex); } - if (lexicalDebugMode != 0 && REPORTED_DEBUG_COMPILATIONS.add(cacheKey)) { + if (lexicalDebugMode != 0 + && !regex.deferredUserDefinedUnicodeProperties + && state().reportedDebugCompilations.add(cacheKey)) { regex.emitCompileDebugTrace(); } } else { @@ -1376,10 +1374,19 @@ private static final class LeftBraceIssue { } } - private static int debugMode(String modifiers) { + static int debugMode(String modifiers) { if (modifiers == null) return 0; - if (modifiers.indexOf(INTERNAL_DEBUGCOLOR_MARKER) >= 0) return 2; - return modifiers.indexOf(INTERNAL_DEBUG_MARKER) >= 0 ? 1 : 0; + int mode = 0; + if (modifiers.indexOf(INTERNAL_DEBUG_COMPILE_MARKER) >= 0) { + mode |= LEXICAL_DEBUG_COMPILE; + } + if (modifiers.indexOf(INTERNAL_DEBUG_EXECUTE_MARKER) >= 0) { + mode |= LEXICAL_DEBUG_EXECUTE; + } + if (modifiers.indexOf(INTERNAL_DEBUG_COLOR_MARKER) >= 0) { + mode |= LEXICAL_DEBUG_COLOR; + } + return mode; } private static boolean reStrictMode(String modifiers) { @@ -1388,32 +1395,36 @@ private static boolean reStrictMode(String modifiers) { private static String stripInternalMarkers(String modifiers) { if (modifiers == null || modifiers.isEmpty()) return modifiers == null ? "" : modifiers; - return modifiers.replace(String.valueOf(INTERNAL_DEBUG_MARKER), "") - .replace(String.valueOf(INTERNAL_DEBUGCOLOR_MARKER), "") - .replace(String.valueOf(INTERNAL_RE_STRICT_MARKER), ""); + return modifiers.replace(String.valueOf(INTERNAL_DEBUG_COMPILE_MARKER), "") + .replace(String.valueOf(INTERNAL_DEBUG_EXECUTE_MARKER), "") + .replace(String.valueOf(INTERNAL_RE_STRICT_MARKER), "") + .replace(String.valueOf(INTERNAL_DEBUG_COLOR_MARKER), ""); } private void emitCompileDebugTrace() { - if (lexicalDebugMode == 0) return; + if ((lexicalDebugMode & (LEXICAL_DEBUG_COMPILE + | LEXICAL_DEBUG_EXECUTE)) == 0) return; registerDebugLifecycle(); + if ((lexicalDebugMode & LEXICAL_DEBUG_COMPILE) == 0) return; String patternDescription = patternString == null ? "" : patternString; String optimizerDescription = recursivePattern == null ? "" : recursivePattern.optimizerDebugDescription(); + String nativeProgram = recursivePattern == null + ? "" : recursivePattern.nativeCompileDebugDescription(); debugWrite("Compiling REx \"" + patternDescription + "\"\n" + "Final program:\n" - + " 1: JONI_PATTERN <" + patternDescription + "> (2)\n" - + " 2: END (0)\n" + + "JONI_PATTERN native bytecode:\n" + + nativeProgram + (optimizerDescription.isEmpty() ? "" : optimizerDescription + "\n")); } public void emitExecutionDebugTrace(String input) { - if (lexicalDebugMode == 0) return; - registerDebugLifecycle(); + if ((lexicalDebugMode & LEXICAL_DEBUG_EXECUTE) == 0) return; StringBuilder trace = new StringBuilder(Math.max(96, input.length() * 72)); trace.append("Matching REx \"").append(patternString == null ? "" : patternString) .append("\" against input of length ").append(input.length()).append('\n'); - // Java's matcher does not publish its start-class instruction stream. + // Joni's matcher does not expose its live instruction cursor publicly. // Report the three observable stages of consuming each candidate input // position: reading it, testing it, and advancing to the next position. for (int offset = 0; offset < input.length(); offset++) { @@ -1437,7 +1448,8 @@ private void registerDebugLifecycle() { public static void emitCurrentRuntimeDebugFreeTraces() { List active = state().activeDebugRegexes; for (RuntimeRegex regex : active) { - if (regex.lexicalDebugMode == 0) continue; + if ((regex.lexicalDebugMode & (LEXICAL_DEBUG_COMPILE + | LEXICAL_DEBUG_EXECUTE)) == 0) continue; String patternDescription = regex.patternString == null ? "" : regex.patternString; regex.debugWrite("Freeing REx: \"" + patternDescription + "\"\n"); } @@ -1445,7 +1457,7 @@ public static void emitCurrentRuntimeDebugFreeTraces() { } private void debugWrite(String message) { - if (lexicalDebugMode == 2) { + if ((lexicalDebugMode & LEXICAL_DEBUG_COLOR) != 0) { message = "\u001b[36m" + message + "\u001b[0m"; } RuntimeIO.getStderr().write(message); @@ -1474,6 +1486,7 @@ private static RuntimeRegex ensureCompiledForRuntime(RuntimeRegex regex) { // provenance must both be reused for recompilation. if (regex.compiledRegexCacheKey != null) { state().compiledRegexCache.remove(regex.compiledRegexCacheKey); + state().reportedDebugCompilations.remove(regex.compiledRegexCacheKey); } return compileSynchronized(regex.patternString, regex.regexFlags == null @@ -3138,6 +3151,7 @@ public static RuntimeBase replaceRegex(RuntimeScalar quotedRegex, RuntimeScalar } regex.emitWarningsOnUse(); + regex.emitExecutionDebugTrace(inputStr); RegexMatcher matcher = regex.selectRecursivePattern(inputValue) .matcher(inputStr, regex.executableCallbacks, inputValue); diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegexSourceCompiler.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegexSourceCompiler.java index f6992a83a..a3099e55b 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegexSourceCompiler.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegexSourceCompiler.java @@ -71,8 +71,9 @@ static RuntimeScalar compile(RuntimeScalar pattern, String modifiers) { } String publicModifiers = modifiers.replace("E", "").replace("T", "") - .replace(String.valueOf(RuntimeRegex.INTERNAL_DEBUG_MARKER), "") - .replace(String.valueOf(RuntimeRegex.INTERNAL_DEBUGCOLOR_MARKER), "") + .replace(String.valueOf(RuntimeRegex.INTERNAL_DEBUG_COMPILE_MARKER), "") + .replace(String.valueOf(RuntimeRegex.INTERNAL_DEBUG_EXECUTE_MARKER), "") + .replace(String.valueOf(RuntimeRegex.INTERNAL_DEBUG_COLOR_MARKER), "") .replace(String.valueOf(RuntimeRegex.INTERNAL_RE_STRICT_MARKER), ""); // qr// accepts pattern modifiers, not operation modifiers such as the // trailing marker used for m?PAT?. Reapply the complete operation flag @@ -102,6 +103,7 @@ static RuntimeScalar compile(RuntimeScalar pattern, String modifiers) { // code until the first match executes. symbolTable.setStrictOptions( WarningBitsRegistry.getCallSiteHints() | HINT_RE_EVAL); + symbolTable.setLexicalRegexDebugFlags(RuntimeRegex.debugMode(modifiers)); symbolTable.enableLexicalRegexModifiers( publicModifiers.replaceAll("[^imsx]", "")); String warningBits = RegexQuoteMeta.getCallSiteWarningBits(); @@ -205,8 +207,12 @@ static RuntimeScalar compileTemplate(RuntimeScalar original, RuntimeRegexTemplate template, String modifiers) { RuntimeRegexTemplate.MaskedCallouts masked = template.maskCallouts(); + String sourceModifiers = modifiers + .replace(String.valueOf(RuntimeRegex.INTERNAL_DEBUG_COMPILE_MARKER), "") + .replace(String.valueOf(RuntimeRegex.INTERNAL_DEBUG_EXECUTE_MARKER), "") + .replace(String.valueOf(RuntimeRegex.INTERNAL_DEBUG_COLOR_MARKER), ""); RuntimeScalar compiled = compile(RuntimeRegexTemplate.patternScalar( - masked.pattern(), template.byteBackedPattern()), modifiers); + masked.pattern(), template.byteBackedPattern()), sourceModifiers); if (!(compiled.value instanceof RuntimeRegex sourceRegex)) { throw new IllegalStateException("runtime regex source did not compile to qr//"); } diff --git a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeRegexState.java b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeRegexState.java index f84d1d326..364af84e1 100644 --- a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeRegexState.java +++ b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeRegexState.java @@ -65,6 +65,8 @@ protected boolean removeEldestEntry(Map.Entry eldest) { /** Regex objects whose lexical debug lifecycle is active in this runtime. */ public final List activeDebugRegexes = new ArrayList<>(); + /** Compile traces already emitted in this runtime; inherited by ithreads. */ + public final Set reportedDebugCompilations = new LinkedHashSet<>(); /** Per-runtime {@code pos()} values and zero-length-match bookkeeping. */ final Map positionCache = @@ -105,5 +107,6 @@ public void clearMatchState() { void snapshotInto(RuntimeRegexState target) { target.userUnicodePropertyCache.putAll(userUnicodePropertyCache); target.deferredUserUnicodeProperties.addAll(deferredUserUnicodeProperties); + target.reportedDebugCompilations.addAll(reportedDebugCompilations); } } diff --git a/src/test/java/org/perlonjava/RegexDebugTraceContractTest.java b/src/test/java/org/perlonjava/RegexDebugTraceContractTest.java new file mode 100644 index 000000000..6bca75916 --- /dev/null +++ b/src/test/java/org/perlonjava/RegexDebugTraceContractTest.java @@ -0,0 +1,128 @@ +package org.perlonjava; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.perlonjava.app.cli.ArgumentParser; +import org.perlonjava.app.cli.CompilerOptions; +import org.perlonjava.app.scriptengine.PerlLanguageProvider; +import org.perlonjava.runtime.io.StandardIO; +import org.perlonjava.runtime.runtimetypes.GlobalVariable; +import org.perlonjava.runtime.runtimetypes.RuntimeIO; + +@Tag("unit") +public class RegexDebugTraceContractTest extends PerlRuntimeTestBase { + private RuntimeIO originalStdout; + private RuntimeIO originalStderr; + private ByteArrayOutputStream stderr; + + @BeforeEach + void setUp() { + PerlLanguageProvider.resetAll(); + originalStdout = RuntimeIO.getStdout(); + originalStderr = RuntimeIO.getStderr(); + installCapturedIo(); + } + + @AfterEach + void tearDown() { + RuntimeIO.setStdout(originalStdout); + RuntimeIO.setStderr(originalStderr); + GlobalVariable.getGlobalIO("main::STDOUT").setIO(RuntimeIO.getStdout()); + GlobalVariable.getGlobalIO("main::STDERR").setIO(RuntimeIO.getStderr()); + PerlLanguageProvider.resetAll(); + } + + private void installCapturedIo() { + RuntimeIO.setStdout(new RuntimeIO( + new StandardIO(new ByteArrayOutputStream(), true))); + stderr = new ByteArrayOutputStream(); + RuntimeIO.setStderr(new RuntimeIO(new StandardIO(stderr, false))); + GlobalVariable.getGlobalIO("main::STDOUT").setIO(RuntimeIO.getStdout()); + GlobalVariable.getGlobalIO("main::STDERR").setIO(RuntimeIO.getStderr()); + } + + private String execute(String source) throws Exception { + CompilerOptions options = ArgumentParser.parseArguments( + new String[] {"-e", source}); + PerlLanguageProvider.executePerlCode(options, true); + RuntimeIO.getStderr().flush(); + return stderr.toString(StandardCharsets.ISO_8859_1); + } + + @Test + void selectsCompileAndExecuteFamiliesAndLifecycle() throws Exception { + String trace = execute("{ use re Debug => 'COMPILE'; " + + "qr/phase36_trace_compile/; } " + + "{ use re Debug => 'EXECUTE'; " + + "'phase36_trace_execute' =~ /phase36_trace_execute/; }"); + + assertTrue(trace.contains("Compiling REx \"phase36_trace_compile\""), trace); + assertTrue(trace.contains("Final program:"), trace); + assertTrue(trace.contains("JONI_PATTERN native bytecode:"), trace); + assertTrue(trace.contains("code length:"), trace); + assertFalse(trace.contains("Matching REx \"phase36_trace_compile\""), trace); + assertFalse(trace.contains("Compiling REx \"phase36_trace_execute\""), trace); + assertTrue(trace.contains("Matching REx \"phase36_trace_execute\""), trace); + assertTrue(trace.contains("Freeing REx: \"phase36_trace_compile\""), trace); + assertTrue(trace.contains("Freeing REx: \"phase36_trace_execute\""), trace); + } + + @Test + void preservesLexicalDisableEvalRestorationAndColor() throws Exception { + String trace = execute("{ use re 'debug'; qr/phase36_trace_outer/; " + + "{ no re 'debug'; qr/phase36_trace_muted/; } " + + "eval q{qr/phase36_trace_eval/}; } " + + "qr/phase36_trace_outside/; " + + "{ use re 'debugcolor'; qr/phase36_trace_color/; " + + "{ no re 'debugcolor'; qr/phase36_trace_color_muted/; } }"); + + assertTrue(trace.contains("Compiling REx \"phase36_trace_outer\""), trace); + assertTrue(trace.contains("Compiling REx \"phase36_trace_eval\""), trace); + assertFalse(trace.contains("phase36_trace_muted"), trace); + assertFalse(trace.contains("phase36_trace_outside"), trace); + assertTrue(trace.contains("\u001b[36mCompiling REx \"phase36_trace_color\""), trace); + assertFalse(trace.contains("phase36_trace_color_muted"), trace); + } + + @Test + void tracesRuntimeSourceAndSubstitutionExecution() throws Exception { + String trace = execute("use re 'eval'; " + + "{ use re Debug => 'EXECUTE'; " + + "my $p = '(?{ 1 })phase36_runtime_source'; " + + "'phase36_runtime_source' =~ /$p/; " + + "my $s = 'phase36_substitution'; " + + "$s =~ s/phase36_substitution/replaced/; }"); + + assertTrue(trace.contains("Matching REx \"(?{ 1 })phase36_runtime_source\""), trace); + assertTrue(trace.contains("Matching REx \"phase36_substitution\""), trace); + } + + @Test + void warnsForUnknownNamedDebugFlag() throws Exception { + String trace = execute("use re Debug => 'BOGUS'; qr/phase36_unknown_debug/;"); + assertTrue(trace.contains("Unknown \"re\" Debug flag 'BOGUS'"), trace); + assertFalse(trace.contains("Compiling REx \"phase36_unknown_debug\""), trace); + } + + @Test + void independentRuntimesEachEmitCompileAndFreeLifecycle() throws Exception { + String source = "use re Debug => 'COMPILE'; qr/phase36_runtime_lifecycle/;"; + String first = execute(source); + assertTrue(first.contains("Compiling REx \"phase36_runtime_lifecycle\""), first); + assertTrue(first.contains("Freeing REx: \"phase36_runtime_lifecycle\""), first); + + PerlLanguageProvider.resetAll(); + installCapturedIo(); + String second = execute(source); + assertTrue(second.contains("Compiling REx \"phase36_runtime_lifecycle\""), second); + assertTrue(second.contains("Freeing REx: \"phase36_runtime_lifecycle\""), second); + } +} diff --git a/src/test/resources/unit/regex/re_debug_trace_contract.t b/src/test/resources/unit/regex/re_debug_trace_contract.t new file mode 100644 index 000000000..2045d75e2 --- /dev/null +++ b/src/test/resources/unit/regex/re_debug_trace_contract.t @@ -0,0 +1,97 @@ +use strict; +use warnings; +use Test::More tests => 18; + +{ + use re 'debug'; + ok('phase36_debug_active' =~ /phase36_debug_active/, + q{use re 'debug' enables a matching regex}); + + { + no re 'debug'; + ok('phase36_debug_muted' =~ /phase36_debug_muted/, + q{no re 'debug' disables tracing lexically}); + } + + ok('phase36_debug_restored' =~ /phase36_debug_restored/, + q{debug is restored after a nested scope}); + + my $eval_ok = eval q{'phase36_debug_eval' =~ /phase36_debug_eval/}; + ok($eval_ok, 'string eval inherits lexical debug'); +} + +ok('phase36_debug_outside' =~ /phase36_debug_outside/, + 'debug does not leak out of its lexical scope'); + +{ + use re Debug => 'COMPILE'; + ok('phase36_debug_compile' =~ /phase36_debug_compile/, + q{Debug => 'COMPILE' preserves matching}); +} + +{ + use re Debug => 'EXECUTE'; + ok('phase36_debug_execute' =~ /phase36_debug_execute/, + q{Debug => 'EXECUTE' preserves matching}); +} + +{ + use re Debug => 'PARSE'; + ok('phase36_debug_parse' =~ /phase36_debug_parse/, + q{Debug => 'PARSE' preserves matching}); +} + +{ + use re Debug => 'COMPILE'; + ok(eval q{'phase36_debug_eval_compile' =~ /phase36_debug_eval_compile/}, + 'string eval inherits compile-only debug'); +} + +{ + use re Debug => 'EXECUTE'; + ok(eval q{'phase36_debug_eval_execute' =~ /phase36_debug_eval_execute/}, + 'string eval inherits execute-only debug'); + { + no re Debug => 'EXECUTE'; + ok(eval q{'phase36_debug_eval_muted' =~ /phase36_debug_eval_muted/}, + 'string eval inherits an authoritative named-debug disable'); + } +} + +{ + use re Debug => 'ALL'; + { + no re Debug => 'EXECUTE'; + ok('phase36_debug_no_execute' =~ /phase36_debug_no_execute/, + q{no re Debug => 'EXECUTE' disables named debug tracing}); + } + ok('phase36_debug_all_restored' =~ /phase36_debug_all_restored/, + 'named debug flags are restored after a nested scope'); +} + +{ + use re 'debugcolor'; + ok('phase36_debug_color' =~ /phase36_debug_color/, + q{use re 'debugcolor' enables tracing without changing matching}); + { + no re 'debugcolor'; + ok('phase36_debug_color_muted' =~ /phase36_debug_color_muted/, + q{no re 'debugcolor' disables tracing lexically}); + } +} + +ok('phase36_debug_final_outside' =~ /phase36_debug_final_outside/, + 'debugcolor does not leak out of its lexical scope'); + +{ + use re 'eval'; + use re Debug => 'EXECUTE'; + my $runtime = '(?{ 1 })phase36_debug_runtime_source'; + ok('phase36_debug_runtime_source' =~ /$runtime/, + 'execute tracing preserves runtime regex source'); + + my $substitution = 'phase36_debug_substitution'; + $substitution =~ s/phase36_debug_substitution/replaced/; + is($substitution, 'replaced', + 'execute tracing preserves substitution semantics'); +} diff --git a/third_party/joni/src/org/joni/ByteCodePrinter.java b/third_party/joni/src/org/joni/ByteCodePrinter.java index ace40ed91..c859a1040 100644 --- a/third_party/joni/src/org/joni/ByteCodePrinter.java +++ b/third_party/joni/src/org/joni/ByteCodePrinter.java @@ -44,19 +44,23 @@ public String byteCodeListToString() { private void pString(StringBuilder sb, int len, int s) { sb.append(':'); - while (len-- > 0) sb.append(new String(new byte[]{(byte)code[s++]})); + byte[] bytes = new byte[len]; + for (int i = 0; i < len; i++) bytes[i] = (byte)code[s + i]; + sb.append(new String(bytes, enc.getCharset())); } private void pLenString(StringBuilder sb, int len, int mbLen, int s) { int x = len * mbLen; sb.append(':').append(len).append(':'); - while (x-- > 0) sb.append(new String(new byte[]{(byte)code[s++]})); + byte[] bytes = new byte[x]; + for (int i = 0; i < x; i++) bytes[i] = (byte)code[s + i]; + sb.append(new String(bytes, enc.getCharset())); } private void pLenStringFromTemplate(StringBuilder sb, int len, int mbLen, byte[]tm, int idx) { int x = len * mbLen; sb.append(":T:").append(len).append(':'); - while (x-- > 0) sb.append(new String(tm, idx++, 1)); + sb.append(new String(tm, idx, x, enc.getCharset())); } public int compiledByteCodeToString(StringBuilder sb, int bp) { @@ -204,10 +208,13 @@ public int compiledByteCodeToString(StringBuilder sb, int bp) { idx = code[bp]; bp += OPSize.INDEX; sb.append(":T:").append(mbLen).append(":").append(len).append(":"); - while (n-- > 0) sb.append(new String(templates[tm], idx++, 1)); + sb.append(new String(templates[tm], idx, n, enc.getCharset())); } else { sb.append(":").append(mbLen).append(":").append(len).append(":"); - while (n-- > 0) sb.append(new String(new byte[]{(byte)code[bp++]})); + byte[] bytes = new byte[n]; + for (int i = 0; i < n; i++) bytes[i] = (byte)code[bp + i]; + sb.append(new String(bytes, enc.getCharset())); + bp += n; } break; diff --git a/third_party/joni/src/org/joni/Regex.java b/third_party/joni/src/org/joni/Regex.java index a1a67631f..6511f3848 100644 --- a/third_party/joni/src/org/joni/Regex.java +++ b/third_party/joni/src/org/joni/Regex.java @@ -620,6 +620,11 @@ public OptimizationInfo getOptimizationInfo() { characterMapOptimization, numMem > 0); } + /** Stable textual view of the actual compiled native instruction stream. */ + public String byteCodeDebugDescription() { + return new ByteCodePrinter(this).byteCodeListToString(); + } + public void setUserOptions(int options) { this.userOptions = options; } diff --git a/third_party/joni/src/org/joni/constants/internal/OPCode.java b/third_party/joni/src/org/joni/constants/internal/OPCode.java index 31a3ed8b2..713e9e4a7 100644 --- a/third_party/joni/src/org/joni/constants/internal/OPCode.java +++ b/third_party/joni/src/org/joni/constants/internal/OPCode.java @@ -19,8 +19,6 @@ */ package org.joni.constants.internal; -import org.joni.Config; - public interface OPCode { int FINISH = 0; /* matching process terminator (no more alternative) */ int END = 1; /* pattern code terminator (success end) */ @@ -174,7 +172,7 @@ public interface OPCode { int BACKREFN_PREV_IC = 130; /* case-folded prior repeat self-reference */ int SCRIPT_RUN = 131; /* validate the current (*script_run:...) span */ - String[] OpCodeNames = Config.DEBUG_COMPILE ? new String[] { + String[] OpCodeNames = new String[] { "finish", /*OP_FINISH*/ "end", /*OP_END*/ "exact1", /*OP_EXACT1*/ @@ -308,9 +306,9 @@ public interface OPCode { "backrefn-prev", "backrefn-prev-ic", "script-run", - } : null; + }; - int[] OpCodeArgTypes = Config.DEBUG_COMPILE ? new int[] { + int[] OpCodeArgTypes = new int[] { Arguments.NON, /*OP_FINISH*/ Arguments.NON, /*OP_END*/ Arguments.SPECIAL, /*OP_EXACT1*/ @@ -444,5 +442,5 @@ public interface OPCode { Arguments.MEMNUM, /*OP_BACKREFN_PREV*/ Arguments.MEMNUM, /*OP_BACKREFN_PREV_IC*/ Arguments.NON, /*OP_SCRIPT_RUN*/ - } : null; + }; } diff --git a/third_party/joni/test/org/joni/test/TestRegexDebugDescription.java b/third_party/joni/test/org/joni/test/TestRegexDebugDescription.java new file mode 100644 index 000000000..432f6338a --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestRegexDebugDescription.java @@ -0,0 +1,61 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import static org.junit.Assert.assertTrue; + +import java.nio.charset.StandardCharsets; + +import org.jcodings.specific.ASCIIEncoding; +import org.jcodings.specific.UTF8Encoding; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.junit.Test; + +public class TestRegexDebugDescription { + @Test + public void exposesTheActualCompiledInstructionStream() { + byte[] pattern = "phase36_debug_native".getBytes(StandardCharsets.US_ASCII); + Regex regex = new Regex(pattern, 0, pattern.length, Option.CAPTURE_GROUP, + ASCIIEncoding.INSTANCE, Syntax.PerlNG); + + String description = regex.byteCodeDebugDescription(); + assertTrue(description.startsWith("code length: ")); + assertTrue(description.toLowerCase().contains("[exact")); + assertTrue(description.toLowerCase().contains("[end]")); + } + + @Test + public void describesStructuredAndUtf8OperandsWithoutMetadataGaps() { + byte[] structured = "(ab|c+)[d-f]".getBytes(StandardCharsets.US_ASCII); + Regex structuredRegex = new Regex(structured, 0, structured.length, + Option.CAPTURE_GROUP, ASCIIEncoding.INSTANCE, Syntax.PerlNG); + String structuredDescription = structuredRegex.byteCodeDebugDescription(); + assertTrue(structuredDescription.contains("[end]")); + assertTrue(structuredDescription.contains("[push")); + + String unicodeLiteral = "åβ"; + byte[] utf8 = unicodeLiteral.getBytes(StandardCharsets.UTF_8); + Regex utf8Regex = new Regex(utf8, 0, utf8.length, Option.CAPTURE_GROUP, + UTF8Encoding.INSTANCE, Syntax.PerlNG); + assertTrue(utf8Regex.byteCodeDebugDescription().contains(unicodeLiteral)); + } +} From 2ed94421efed74c63515a78791d172dd5b6384d7 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 12:25:03 +0200 Subject: [PATCH 19/43] docs(regex): refresh active release and migration gates Keep the plan focused on current acceptance state and complete next steps. Record the corrected JAPH warning-hook root, current-Perl class oracle, Unicode residual ledger, and long worker ownership without implementation history. Generated with [OpenAI Codex](https://openai.com/codex/) Co-Authored-By: OpenAI Codex --- dev/design/phase36-regex-parity.md | 51 +++++++++++++++++------------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index b23905bcb..e884587c8 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -183,22 +183,31 @@ required. Their isolated 399/446-second runtimes versus roughly 250 seconds at PR958 are also a post-checkpoint optimization root: profile both revisions with JFR and remove the measured regression rather than masking it with a larger timeout. `op/do.t` needs two fixes to reach standard Perl's 71/71 plus an -ID-normalized disposition because PR958 over-executed it as 94/99; -`class/accessor.t` aborts before the newly imported accessor-name rows; and -`japh/abigail.t` drops one nondeterministically. Acceptance must have no new +ID-normalized disposition because PR958 over-executed it as 94/99. The current +candidate reaches 71/71 on both backends. The class accessor-name candidate is +validated by current Perl 5.45.3 at 8/8 focused and 30/30 authoritative. +`japh/abigail.t` remains the release blocker: private per-run isolation removes +its filesystem race, and an exact assertion map isolates the sole PR-958 delta +at ID 51. Its quoted multi-`-e` switch loses literal `$_` through shell argv +construction; IDs 127–130 are unchanged and also fail on current Perl. +Acceptance must have no new invalid, missing, timeout, truncated, incomplete, or zero-TAP row and no unresolved PR-958 pass-count decrease. The user owns the release checkout; workers use private exact-head worktrees and must not mutate, rebase, or push the user's branch. -The class candidate must also be validated with a private build of the exact -latest pulled Perl source: the available 5.43.9 executable predates the new -leading-underscore accessor rule and cannot serve as its standard-Perl oracle. +The release fixes still require one combined product build after the JAPH +warning-hook correction. Only then run the quiet exact-head `pat.t` and +`pat_thr.t` acceptance without external worker builds. Independent implementation continues on `integrate/phase36-post1087-wip`. -Its current preserved tranche generates current-Perl InSC/InPC data and closes -all 4,240 labelled `uniprops02.t` assertions through native Joni property -resolution. It will be rebased only after the refreshed checkpoint merges. +Its current preserved tranches generate current-Perl InSC/InPC and Block data, +preserve warning/fatal/recursion behavior, and establish a native Joni debug +trace foundation. The complete Unicode ledger covers 167,506 assertions: after +the integrated warning and Block tranches plus pending generated-XID data, +exactly 30 `uniprops01.t` rows remain across word-boundary, `/a` wildcard, +Decomposition_Type, and Cased_Letter families. It will be rebased only after +the refreshed checkpoint merges. Repair the five blocking rows with targeted, baseline-equivalent reruns before repeating the complete corpus. Once targeted evidence is green, rerun the @@ -225,24 +234,22 @@ checks at the same immutable head. Active ownership: -- P3: close the complete stable regex debug-trace contract and interpreter eval - propagation, including exact compile/match/free lifecycle parity and imported - reducers. -- P4: close the disjoint XID binary-property root, then complete nested/scoped - extended-character-class interpolation and set-operation parity. -- P5: remove redundant property scanning after the integrated 1,048-row - `In...` Block-alias correction and the P3 integration reference are published. -- P6: finish exact-head release repairs: both backends at `op/do.t` 71/71, - deterministic `japh/abigail.t`, and the already delivered class accessor-name - correction; validate all three with one combined build. +- P3: complete all 1,187 `re/anyof.t` native debug-description rows, then + profile and remove the measured `pat*` performance regression. +- P4: finish generated current-table XID binary-property parity, then + nested/scoped extended-character-class interpolation and set-operation parity. +- P5: remove redundant built-in property scanning on the published P3 + integration reference, then close the exact 30-row Unicode residual. +- P6: preserve literal argv for JAPH's quoted multi-`-e` assertion 51 and prove + deterministic 110/130 or better; the `op/do.t` and class candidates are ready. - Coordinator: integration, conflict resolution, immutable acceptance, PR/CI, plan state, combined build, and release evidence. ## Ordered Next Steps -1. Fix and rerun the five exact-head release-blocking rows in private worktrees; - preserve latest-Perl tests and separate changed test cardinality from runtime - regressions with exact assertion-ID evidence. +1. Finish and rerun the isolated JAPH argv-preservation root, then integrate the + already-green `op/do.t` and class candidates; preserve latest-Perl tests and + exact assertion-ID evidence. 2. Integrate the fixes into PR 1087, run one warning-free `make`, focused Unicode, `regexp.t`, `regex_sets.t`, `charset.t`, `pat.t`, `pat_thr.t`, `pat_advanced.t`, `pat_re_eval.t`, and `reg_mesg.t` gates, then repeat the From 5ba22530ec8c366b257355e8d5671989cf2555d6 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 12:29:14 +0200 Subject: [PATCH 20/43] fix(regex): generate Perl binary property aliases Derive native binary-property acceptance and rejection from the current Perl Unicode keyword table and authoritative Unicode alias data. Record source hashes and upstream notices, remove handwritten binary inventories, and preserve XID and obsolete-alias behavior with focused tests. Generated with [OpenAI Codex](https://openai.com/codex/) Co-Authored-By: OpenAI Codex --- dev/design/phase36-regex-parity.md | 4 +- ...perl_unicode_binary_property_alias_data.pl | 196 ++++++++++++++++++ dev/tools/perl_unicode_data_generators.json | 13 ++ ..._unicode_binary_property_alias_generator.t | 117 +++++++++++ .../PerlUnicodeBinaryPropertyAliasData.java | 110 ++++++++++ .../runtime/regex/UnicodeResolver.java | 82 ++++---- .../regex/PerlXidPropertyResolverTest.java | 85 ++++++++ .../regex/unicode_xid_binary_properties.t | 51 +++++ 8 files changed, 612 insertions(+), 46 deletions(-) create mode 100644 dev/tools/generate_perl_unicode_binary_property_alias_data.pl create mode 100644 dev/tools/tests/perl_unicode_binary_property_alias_generator.t create mode 100644 src/main/java/org/perlonjava/runtime/regex/PerlUnicodeBinaryPropertyAliasData.java create mode 100644 src/test/java/org/perlonjava/runtime/regex/PerlXidPropertyResolverTest.java create mode 100644 src/test/resources/unit/regex/unicode_xid_binary_properties.t diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index e884587c8..9730eb4e6 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -236,8 +236,8 @@ Active ownership: - P3: complete all 1,187 `re/anyof.t` native debug-description rows, then profile and remove the measured `pat*` performance regression. -- P4: finish generated current-table XID binary-property parity, then - nested/scoped extended-character-class interpolation and set-operation parity. +- P4: close the three highest-count remaining same-source `reg_mesg.t` native + parser/lexer diagnostic families; XID and imported extended-class gates are green. - P5: remove redundant built-in property scanning on the published P3 integration reference, then close the exact 30-row Unicode residual. - P6: preserve literal argv for JAPH's quoted multi-`-e` assertion 51 and prove diff --git a/dev/tools/generate_perl_unicode_binary_property_alias_data.pl b/dev/tools/generate_perl_unicode_binary_property_alias_data.pl new file mode 100644 index 000000000..13a89887f --- /dev/null +++ b/dev/tools/generate_perl_unicode_binary_property_alias_data.pl @@ -0,0 +1,196 @@ +#!/usr/bin/env perl +use strict; +use warnings; +use Digest::SHA qw(sha256_hex); +use File::Spec; +use FindBin; +use lib File::Spec->catdir($FindBin::Bin, 'lib'); +use PerlOnJava::UnicodeGenerator qw( + emit_unicode_source_notices loose_name read_raw repo_root select_perl_root + select_unicode_root trim verify_unicode_notice +); + +my $root = repo_root($FindBin::Bin); +if (defined $ENV{PERLONJAVA_UNICODE_ROOT}) { + for my $required (qw(version PropertyAliases.txt PropValueAliases.txt)) { + my $path = File::Spec->catfile($ENV{PERLONJAVA_UNICODE_ROOT}, $required); + die "Explicit Unicode source root missing $required\n" unless -f $path; + } +} +my $unicore = select_unicode_root( + repo_root => $root, version => 'current', + required => [qw(version PropertyAliases.txt PropValueAliases.txt)], +); +if (defined $ENV{PERLONJAVA_PERL_ROOT}) { + my $path = File::Spec->catfile($ENV{PERLONJAVA_PERL_ROOT}, 'uni_keywords.h'); + die "Explicit Perl source root missing uni_keywords.h\n" unless -f $path; +} +my $perl_root = select_perl_root( + repo_root => $root, unicode_root => $unicore, + required => ['uni_keywords.h'], +); + +my $unicode_version = read_raw(File::Spec->catfile($unicore, 'version')); +$unicode_version =~ s/\s+\z//; +die "Malformed current Unicode version '$unicode_version'\n" + unless $unicode_version =~ /\A\d+\.\d+\.\d+\z/; + +my @unicode_sources = ( + ['PropertyAliases.txt', qr/^# PropertyAliases-\Q$unicode_version\E\.txt$/m], + ['PropValueAliases.txt', qr/^# PropertyValueAliases-\Q$unicode_version\E\.txt$/m], +); +for my $source (@unicode_sources) { + my ($name, $version_pattern) = @$source; + my $path = File::Spec->catfile($unicore, $name); + my $text = read_raw($path); + die "$path is inconsistent with Unicode $unicode_version\n" + unless $text =~ $version_pattern; + verify_unicode_notice($path, $text); + @$source = ($name, sha256_hex($text), $text); +} +my %unicode_text = map { $_->[0] => $_->[2] } @unicode_sources; + +my $keyword_path = File::Spec->catfile($perl_root, 'uni_keywords.h'); +my $keyword_text = read_raw($keyword_path); +die "$keyword_path is not a generated Perl Unicode keyword table\n" + unless $keyword_text =~ m{^/\* -\*- mode: C; buffer-read-only: t -\*-\n} + && $keyword_text =~ /^ \* !!!!!!! DO NOT EDIT THIS FILE !!!!!!!$/m + && $keyword_text =~ /^ \* This file is built by regen\/mk_invlists\.pl from mph\.pl\.$/m + && $keyword_text =~ /^ \* Any changes made here will be lost!$/m; + +my %keyword; +for my $line (split /\n/, $keyword_text) { + next unless $line =~ m{/\* ([a-z0-9_=.+-]+) \*/}; + die "$keyword_path repeats normalized keyword '$1'\n" if $keyword{$1}++; +} +die "$keyword_path contains no generated Unicode keywords\n" unless %keyword; + +my %property_aliases; +for my $line (split /\n/, $unicode_text{'PropertyAliases.txt'}) { + $line =~ s/\s*#.*\z//; + next unless $line =~ /\S/; + my @field = map { trim($_) } split /;/, $line; + die "Malformed PropertyAliases row '$line'\n" unless @field >= 2; + die "PropertyAliases repeats '$field[0]'\n" if exists $property_aliases{$field[0]}; + $property_aliases{$field[0]} = \@field; +} + +my %value_names; +for my $line (split /\n/, $unicode_text{'PropValueAliases.txt'}) { + $line =~ s/\s*#.*\z//; + next unless $line =~ /\S/; + my @field = map { trim($_) } split /;/, $line; + die "Malformed PropValueAliases row '$line'\n" unless @field >= 3; + $value_names{$field[0]}{loose_name($field[1])} = 1; +} + +my (%accepted, %rejected); +for my $property (sort keys %property_aliases) { + my $values = $value_names{$property} // next; + next unless keys(%$values) == 2 && $values->{n} && $values->{y}; + my $aliases = $property_aliases{$property}; + my $canonical = $aliases->[1]; + for my $spelling (@$aliases) { + next unless length $spelling; + my $loose = loose_name($spelling); + if ($keyword{$loose}) { + die "Conflicting accepted binary alias '$loose'\n" + if exists $accepted{$loose} && $accepted{$loose} ne $canonical; + $accepted{$loose} = $canonical; + } else { + $rejected{$loose} = 1; + } + } +} +die "No Perl binary property aliases found\n" unless %accepted; +die "Current Perl keywords omit XIDC\n" + unless $accepted{xidc} && $accepted{xidcontinue}; + +# PerlOnJava historically accepted this non-Unicode abbreviation. Its absence +# from the current generated Perl keyword table is an intentional rejection. +die "Current Perl unexpectedly accepts the compatibility spelling xidcont\n" + if $keyword{xidcont}; +$rejected{xidcont} = 1; +delete $rejected{$_} for keys %accepted; + +my @accepted = sort keys %accepted; +my @rejected = sort keys %rejected; +my $keyword_hash = sha256_hex($keyword_text); + +print <<'HEADER'; +/* + * Generated from the current Perl Unicode keyword table and Unicode property + * alias tables. Do not edit manually. + * +HEADER +print " * Unicode version: $unicode_version\n"; +print " * Source: uni_keywords.h (SHA-256 $keyword_hash)\n"; +print " * Perl source notice: !!!!!!! DO NOT EDIT THIS FILE !!!!!!!\n"; +print " * Perl source notice: This file is built by regen/mk_invlists.pl from mph.pl.\n"; +print " * Perl source notice: Any changes made here will be lost!\n"; +print " *\n"; +emit_unicode_source_notices([ + map { +{name => "$_->[0] (SHA-256 $_->[1])", text => $_->[2]} } + @unicode_sources +]); +print <<'HEADER'; + */ +package org.perlonjava.runtime.regex; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +final class PerlUnicodeBinaryPropertyAliasData { +HEADER +print " static final String UNICODE_VERSION = \"$unicode_version\";\n"; +print " static final String PERL_KEYWORD_SHA256 = \"$keyword_hash\";\n\n"; +print " private static final String[] ACCEPTED = {\n"; +for (my $index = 0; $index < @accepted; $index += 3) { + my $last = $index + 2 < $#accepted ? $index + 2 : $#accepted; + print " ", join(', ', map { + qq{\"$accepted[$_]\", \"$accepted{$accepted[$_]}\"} + } $index .. $last), ",\n"; +} +print " };\n\n"; +print " private static final Set REJECTED = Set.of(\n"; +for (my $index = 0; $index < @rejected; $index += 6) { + my $last = $index + 5 < $#rejected ? $index + 5 : $#rejected; + my $suffix = $last == $#rejected ? '' : ','; + print " ", join(', ', map { qq{\"$rejected[$_]\"} } + $index .. $last), "$suffix\n"; +} +print <<'FOOTER'; + ); + + private static final Map ACCEPTED_BY_ALIAS = buildAccepted(); + + static String canonicalProperty(String alias) { + String normalized = normalize(alias); + String canonical = ACCEPTED_BY_ALIAS.get(normalized); + if (canonical == null && normalized.startsWith("is")) { + canonical = ACCEPTED_BY_ALIAS.get(normalized.substring(2)); + } + return canonical; + } + + static boolean isRejected(String alias) { + return REJECTED.contains(normalize(alias)); + } + + private static Map buildAccepted() { + Map result = new HashMap<>(); + for (int i = 0; i < ACCEPTED.length; i += 2) { + result.put(ACCEPTED[i], ACCEPTED[i + 1]); + } + return Map.copyOf(result); + } + + private static String normalize(String value) { + return value.toLowerCase(java.util.Locale.ROOT) + .replaceAll("[\\x09-\\x0d _-]+", ""); + } + + private PerlUnicodeBinaryPropertyAliasData() {} +} +FOOTER diff --git a/dev/tools/perl_unicode_data_generators.json b/dev/tools/perl_unicode_data_generators.json index cf5853931..245c95599 100644 --- a/dev/tools/perl_unicode_data_generators.json +++ b/dev/tools/perl_unicode_data_generators.json @@ -18,6 +18,19 @@ "extracted/DBidiClass.txt" : "4867b4b7f0731ed1bfcd34cc6251211ff1542541fce0734b6fbda139ee80b3a4" } }, + { + "generator" : "dev/tools/generate_perl_unicode_binary_property_alias_data.pl", + "name" : "binary-property-alias", + "output" : "src/main/java/org/perlonjava/runtime/regex/PerlUnicodeBinaryPropertyAliasData.java", + "output_sha256" : "f04128dd8128ee48da74aa2c3d90d93e3dc1461802ef082b510a991a59ed7f0c", + "perl_sources" : { + "uni_keywords.h" : "ccbc1b09be7e393558d3a0ebbd1eb53d5ed1c5d7db6f04effca2977bb622432a" + }, + "sources" : { + "PropValueAliases.txt" : "670d2bebb48649c04fabfbf033308073dcff47946324a8033237254c048b3b01", + "PropertyAliases.txt" : "4441f573caf952ffece1d7c892e7715bd7136dfc26f96eb6f268bf1e474715fb" + } + }, { "generator" : "dev/tools/generate_perl_unicode_block_data.pl", "name" : "block", diff --git a/dev/tools/tests/perl_unicode_binary_property_alias_generator.t b/dev/tools/tests/perl_unicode_binary_property_alias_generator.t new file mode 100644 index 000000000..0ead3593a --- /dev/null +++ b/dev/tools/tests/perl_unicode_binary_property_alias_generator.t @@ -0,0 +1,117 @@ +use strict; +use warnings; +use File::Path qw(make_path); +use File::Spec; +use File::Temp qw(tempdir); +use FindBin; +use IPC::Open3; +use Symbol qw(gensym); +use Test::More; + +my $repo = File::Spec->rel2abs(File::Spec->catdir($FindBin::Bin, '..', '..', '..')); +my $generator = File::Spec->catfile( + $repo, 'dev', 'tools', 'generate_perl_unicode_binary_property_alias_data.pl'); +my $temporary = tempdir(CLEANUP => 1); +my $unicode = File::Spec->catdir($temporary, 'unicore'); +my $perl_root = File::Spec->catdir($temporary, 'perl5'); +make_path($unicode, $perl_root); + +write_sources(); +my ($status, $first) = run_generator(); +is($status, 0, 'generator accepts complete current Unicode and Perl sources'); +like($first, qr/"xidc", "XID_Continue"/, + 'accepted XID short alias is generated from current Perl keywords'); +like($first, qr/"xidcontinue", "XID_Continue"/, + 'accepted XID long alias is generated from current Perl keywords'); +like($first, qr/"graphemelink".*"grlink".*"xidcont"/s, + 'current Perl omissions and compatibility spelling are explicit rejections'); +my ($second_status, $second) = run_generator(); +is($second_status, 0, 'second generation succeeds'); +is($second, $first, 'generation is byte-for-byte deterministic'); + +my $keywords = File::Spec->catfile($perl_root, 'uni_keywords.h'); +my $valid_keywords = read_file($keywords); +unlink $keywords or die "Cannot remove $keywords: $!"; +my ($missing_status, $missing_log) = run_generator(); +isnt($missing_status, 0, 'generator rejects a missing Perl keyword source'); +like($missing_log, qr/Explicit Perl source root missing uni_keywords\.h/, + 'missing Perl source has a focused diagnostic'); + +write_file($keywords, "/* not a generated table */\n"); +my ($header_status, $header_log) = run_generator(); +isnt($header_status, 0, 'generator rejects malformed Perl source provenance'); +like($header_log, qr/not a generated Perl Unicode keyword table/, + 'malformed Perl provenance has a focused diagnostic'); +write_file($keywords, $valid_keywords); + +my $aliases = File::Spec->catfile($unicode, 'PropertyAliases.txt'); +my $valid_aliases = read_file($aliases); +write_file($aliases, $valid_aliases . "MalformedRow\n"); +my ($row_status, $row_log) = run_generator(); +isnt($row_status, 0, 'generator rejects malformed Unicode property rows'); +like($row_log, qr/Malformed PropertyAliases row 'MalformedRow'/, + 'malformed Unicode row has a focused diagnostic'); + +done_testing; + +sub notice { + return "# © 2025 Unicode®, Inc.\n" + . "# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.\n" + . "# For terms of use and license, see https://www.unicode.org/terms_of_use.html\n"; +} + +sub write_sources { + write_file(File::Spec->catfile($unicode, 'version'), "18.0.0\n"); + write_file(File::Spec->catfile($unicode, 'PropertyAliases.txt'), + "# PropertyAliases-18.0.0.txt\n" . notice() + . "AHex ; ASCII_Hex_Digit\nGr_Link ; Grapheme_Link\n" + . "XIDC ; XID_Continue\n"); + write_file(File::Spec->catfile($unicode, 'PropValueAliases.txt'), + "# PropertyValueAliases-18.0.0.txt\n" . notice() + . "AHex ; N ; No ; F ; False\nAHex ; Y ; Yes ; T ; True\n" + . "Gr_Link ; N ; No ; F ; False\nGr_Link ; Y ; Yes ; T ; True\n" + . "XIDC ; N ; No ; F ; False\nXIDC ; Y ; Yes ; T ; True\n"); + write_file(File::Spec->catfile($perl_root, 'uni_keywords.h'), <<'KEYWORDS'); +/* -*- mode: C; buffer-read-only: t -*- + * !!!!!!! DO NOT EDIT THIS FILE !!!!!!! + * This file is built by regen/mk_invlists.pl from mph.pl. + * Any changes made here will be lost! + */ + { 0 } /* ahex */, + { 0 } /* asciihexdigit */, + { 0 } /* isahex */, + { 0 } /* isasciihexdigit */, + { 0 } /* isxidc */, + { 0 } /* isxidcontinue */, + { 0 } /* xidc */, + { 0 } /* xidcontinue */, +KEYWORDS +} + +sub run_generator { + local %ENV = %ENV; + $ENV{PERLONJAVA_UNICODE_ROOT} = $unicode; + $ENV{PERLONJAVA_PERL_ROOT} = $perl_root; + my $error = gensym; + my $pid = open3(undef, my $stdout, $error, $^X, $generator); + local $/; + my $bytes = (<$stdout> // '') . (<$error> // ''); + waitpid($pid, 0); + return ($? >> 8, $bytes); +} + +sub write_file { + my ($path, $bytes) = @_; + open my $output, '>:raw', $path or die "Cannot write $path: $!"; + print {$output} $bytes or die "Cannot write $path: $!"; + close $output or die "Cannot close $path: $!"; +} + +sub read_file { + my ($path) = @_; + open my $input, '<:raw', $path or die "Cannot read $path: $!"; + local $/; + my $bytes = <$input>; + close $input or die "Cannot close $path: $!"; + return $bytes; +} diff --git a/src/main/java/org/perlonjava/runtime/regex/PerlUnicodeBinaryPropertyAliasData.java b/src/main/java/org/perlonjava/runtime/regex/PerlUnicodeBinaryPropertyAliasData.java new file mode 100644 index 000000000..d6d471d98 --- /dev/null +++ b/src/main/java/org/perlonjava/runtime/regex/PerlUnicodeBinaryPropertyAliasData.java @@ -0,0 +1,110 @@ +/* + * Generated from the current Perl Unicode keyword table and Unicode property + * alias tables. Do not edit manually. + * + * Unicode version: 17.0.0 + * Source: uni_keywords.h (SHA-256 ccbc1b09be7e393558d3a0ebbd1eb53d5ed1c5d7db6f04effca2977bb622432a) + * Perl source notice: !!!!!!! DO NOT EDIT THIS FILE !!!!!!! + * Perl source notice: This file is built by regen/mk_invlists.pl from mph.pl. + * Perl source notice: Any changes made here will be lost! + * + * Source: PropertyAliases.txt (SHA-256 4441f573caf952ffece1d7c892e7715bd7136dfc26f96eb6f268bf1e474715fb) + * © 2025 Unicode®, Inc. + * Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. + * For terms of use and license, see https://www.unicode.org/terms_of_use.html + * + * Source: PropValueAliases.txt (SHA-256 670d2bebb48649c04fabfbf033308073dcff47946324a8033237254c048b3b01) + * © 2025 Unicode®, Inc. + * Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. + * For terms of use and license, see https://www.unicode.org/terms_of_use.html + * + */ +package org.perlonjava.runtime.regex; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +final class PerlUnicodeBinaryPropertyAliasData { + static final String UNICODE_VERSION = "17.0.0"; + static final String PERL_KEYWORD_SHA256 = "ccbc1b09be7e393558d3a0ebbd1eb53d5ed1c5d7db6f04effca2977bb622432a"; + + private static final String[] ACCEPTED = { + "ahex", "ASCII_Hex_Digit", "alpha", "Alphabetic", "alphabetic", "Alphabetic", + "asciihexdigit", "ASCII_Hex_Digit", "bidic", "Bidi_Control", "bidicontrol", "Bidi_Control", + "bidim", "Bidi_Mirrored", "bidimirrored", "Bidi_Mirrored", "cased", "Cased", + "caseignorable", "Case_Ignorable", "ce", "Composition_Exclusion", "changeswhencasefolded", "Changes_When_Casefolded", + "changeswhencasemapped", "Changes_When_Casemapped", "changeswhenlowercased", "Changes_When_Lowercased", "changeswhennfkccasefolded", "Changes_When_NFKC_Casefolded", + "changeswhentitlecased", "Changes_When_Titlecased", "changeswhenuppercased", "Changes_When_Uppercased", "ci", "Case_Ignorable", + "compex", "Full_Composition_Exclusion", "compositionexclusion", "Composition_Exclusion", "cwcf", "Changes_When_Casefolded", + "cwcm", "Changes_When_Casemapped", "cwkcf", "Changes_When_NFKC_Casefolded", "cwl", "Changes_When_Lowercased", + "cwt", "Changes_When_Titlecased", "cwu", "Changes_When_Uppercased", "dash", "Dash", + "defaultignorablecodepoint", "Default_Ignorable_Code_Point", "dep", "Deprecated", "deprecated", "Deprecated", + "di", "Default_Ignorable_Code_Point", "dia", "Diacritic", "diacritic", "Diacritic", + "ebase", "Emoji_Modifier_Base", "ecomp", "Emoji_Component", "emod", "Emoji_Modifier", + "emoji", "Emoji", "emojicomponent", "Emoji_Component", "emojimodifier", "Emoji_Modifier", + "emojimodifierbase", "Emoji_Modifier_Base", "emojipresentation", "Emoji_Presentation", "epres", "Emoji_Presentation", + "ext", "Extender", "extendedpictographic", "Extended_Pictographic", "extender", "Extender", + "extpict", "Extended_Pictographic", "fullcompositionexclusion", "Full_Composition_Exclusion", "graphemebase", "Grapheme_Base", + "graphemeextend", "Grapheme_Extend", "grbase", "Grapheme_Base", "grext", "Grapheme_Extend", + "hex", "Hex_Digit", "hexdigit", "Hex_Digit", "hyphen", "Hyphen", + "idc", "ID_Continue", "idcompatmathcontinue", "ID_Compat_Math_Continue", "idcompatmathstart", "ID_Compat_Math_Start", + "idcontinue", "ID_Continue", "ideo", "Ideographic", "ideographic", "Ideographic", + "ids", "ID_Start", "idsb", "IDS_Binary_Operator", "idsbinaryoperator", "IDS_Binary_Operator", + "idst", "IDS_Trinary_Operator", "idstart", "ID_Start", "idstrinaryoperator", "IDS_Trinary_Operator", + "idsu", "IDS_Unary_Operator", "idsunaryoperator", "IDS_Unary_Operator", "joinc", "Join_Control", + "joincontrol", "Join_Control", "kehnomirror", "kEH_NoMirror", "kehnorotate", "kEH_NoRotate", + "loe", "Logical_Order_Exception", "logicalorderexception", "Logical_Order_Exception", "lower", "Lowercase", + "lowercase", "Lowercase", "math", "Math", "mcm", "Modifier_Combining_Mark", + "modifiercombiningmark", "Modifier_Combining_Mark", "nchar", "Noncharacter_Code_Point", "noncharactercodepoint", "Noncharacter_Code_Point", + "patsyn", "Pattern_Syntax", "patternsyntax", "Pattern_Syntax", "patternwhitespace", "Pattern_White_Space", + "patws", "Pattern_White_Space", "pcm", "Prepended_Concatenation_Mark", "prependedconcatenationmark", "Prepended_Concatenation_Mark", + "qmark", "Quotation_Mark", "quotationmark", "Quotation_Mark", "radical", "Radical", + "regionalindicator", "Regional_Indicator", "ri", "Regional_Indicator", "sd", "Soft_Dotted", + "sentenceterminal", "Sentence_Terminal", "softdotted", "Soft_Dotted", "space", "White_Space", + "sterm", "Sentence_Terminal", "term", "Terminal_Punctuation", "terminalpunctuation", "Terminal_Punctuation", + "uideo", "Unified_Ideograph", "unifiedideograph", "Unified_Ideograph", "upper", "Uppercase", + "uppercase", "Uppercase", "variationselector", "Variation_Selector", "vs", "Variation_Selector", + "whitespace", "White_Space", "wspace", "White_Space", "xidc", "XID_Continue", + "xidcontinue", "XID_Continue", "xids", "XID_Start", "xidstart", "XID_Start", + }; + + private static final Set REJECTED = Set.of( + "expandsonnfc", "expandsonnfd", "expandsonnfkc", "expandsonnfkd", "graphemelink", "grlink", + "nfdqc", "nfdquickcheck", "nfkdqc", "nfkdquickcheck", "oalpha", "odi", + "ogrext", "oidc", "oids", "olower", "omath", "otheralphabetic", + "otherdefaultignorablecodepoint", "othergraphemeextend", "otheridcontinue", "otheridstart", "otherlowercase", "othermath", + "otheruppercase", "oupper", "xidcont", "xonfc", "xonfd", "xonfkc", + "xonfkd" + ); + + private static final Map ACCEPTED_BY_ALIAS = buildAccepted(); + + static String canonicalProperty(String alias) { + String normalized = normalize(alias); + String canonical = ACCEPTED_BY_ALIAS.get(normalized); + if (canonical == null && normalized.startsWith("is")) { + canonical = ACCEPTED_BY_ALIAS.get(normalized.substring(2)); + } + return canonical; + } + + static boolean isRejected(String alias) { + return REJECTED.contains(normalize(alias)); + } + + private static Map buildAccepted() { + Map result = new HashMap<>(); + for (int i = 0; i < ACCEPTED.length; i += 2) { + result.put(ACCEPTED[i], ACCEPTED[i + 1]); + } + return Map.copyOf(result); + } + + private static String normalize(String value) { + return value.toLowerCase(java.util.Locale.ROOT) + .replaceAll("[\\x09-\\x0d _-]+", ""); + } + + private PerlUnicodeBinaryPropertyAliasData() {} +} diff --git a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java index 5a85990b6..e9deedc8e 100644 --- a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java +++ b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java @@ -39,25 +39,11 @@ private record PropertyScanState(boolean caseInsensitive, boolean extended) {} new UnicodeSet(PERL_UNICODE_BASE_SET) .removeAll(PerlUnicodeGeneralCategoryData.resolve("Cn")) .freeze(); - private static final Set PERL_BARE_PROPERTY_ALIASES = Set.of( - "ahex", "alnum", "alpha", "any", "assigned", "bidim", - "bidimirrored", "changeswhencasemapped", "changeswhenlowercased", - "changeswhennfkccasefolded", "closepunctuation", "cntrl", - "combiningmark", "compex", "connectorpunctuation", "cwu", "cwkcf", - "dash", "dashpunctuation", "decimalnumber", "deprecated", "dia", - "emojimodifierbase", "emojipresentation", "epres", "extender", - "fullcompositionexclusion", "grbase", "grext", "hexdigit", - "idcompatmathstart", "ideo", "idst", "idsu", "idsunaryoperator", - "loe", "logicalorderexception", "lowercase", "modifiercombiningmark", - "othernumber", "otherpunctuation", "regionalindicator", - "sentenceterminal", "softdotted", "space", "spaceseparator", "uideo", - "unassigned", "unifiedideograph", "xids", "xidstart"); - private static final Set PERL_REJECTED_OBSOLETE_BINARY_ALIASES = - Set.of("graphemelink", "grlink", "otheralphabetic", "oalpha", - "otherdefaultignorablecodepoint", "odi", - "othergraphemeextend", "ogrext", "otheridcontinue", "oidc", - "otheridstart", "oids", "otherlowercase", "olower", - "othermath", "omath", "otheruppercase", "oupper"); + private static final Set PERL_NON_BINARY_BARE_PROPERTY_ALIASES = Set.of( + "alnum", "any", "assigned", "closepunctuation", "cntrl", + "combiningmark", "connectorpunctuation", "dashpunctuation", + "decimalnumber", "modifiercombiningmark", "othernumber", + "otherpunctuation", "spaceseparator", "unassigned"); private static final String[][] PERL_WORD_BREAK_WILDCARD_VALUES = { {"CR"}, {"DQ", "Double_Quote"}, {"EB", "E_Base"}, {"EBG", "E_Base_GAZ"}, {"EM", "E_Modifier"}, @@ -1330,7 +1316,6 @@ && isPerlAllProperty(property, looseIsValue)) { // Use ICU4J UnicodeSet for accurate XID_Start return getXIDStartPattern(negated); case "XIDC": - case "XIDCont": case "XID_Continue": // Use ICU4J UnicodeSet for accurate XID_Continue return getXIDContinuePattern(negated); @@ -1478,8 +1463,7 @@ static CharacterPropertyResolver.Result resolveJoniProperty( String property, boolean inCharacterClass, boolean caseInsensitive) { if (property == null) return null; property = property.trim(); - if (PERL_REJECTED_OBSOLETE_BINARY_ALIASES.contains( - loosePropertyName(property))) { + if (isPerlRejectedBinaryPropertyExpression(property)) { throw new IllegalArgumentException( "Unsupported obsolete Unicode property: " + property); } @@ -1872,7 +1856,7 @@ private static boolean isPerlSpecialPropertyAlias(String property) { "PosixPunct", "PosixSpace", "PosixUpper", "PosixWord", "PosixXDigit", "XIDS", "XIDStart", "XID_Start", - "XIDC", "XIDCont", "XID_Continue", + "XIDC", "XID_Continue", "_Perl_IDStart", "_Perl_IDCont" -> true; default -> false; }; @@ -1931,8 +1915,7 @@ private static UnicodeSet resolvePerlBuiltInPropertyAlias(String property) { String alias = canonicalPerlPosixPropertyAlias(property.trim()); alias = normalizePerlIsPropertyAssignment(alias); - if (PERL_REJECTED_OBSOLETE_BINARY_ALIASES.contains( - loosePropertyName(alias))) { + if (isPerlRejectedBinaryPropertyExpression(alias)) { throw new IllegalArgumentException( "Unsupported obsolete Unicode property: " + property.trim()); } @@ -2335,13 +2318,7 @@ private static PerlUnicodePropertyWildcard resolvePerlUnicodePropertyWildcard( compilePerlUnicodePropertyWildcard(value); if (wildcard == null) return null; - if (PerlUnicodeResidualPropertyData.isBinaryPropertyAlias(name)) { - return resolvePerlBinaryPropertyWildcard(name, wildcard); - } - if (isIcuBinaryPropertyAlias(name)) { - return resolvePerlBinaryPropertyWildcard(name, wildcard); - } - if (resolvePerlMissingBaseAlias(name) != null) { + if (isPerlAcceptedBinaryPropertyAlias(name)) { return resolvePerlBinaryPropertyWildcard(name, wildcard); } if (PerlUnicodeIndicCategoryData.isPropertyAlias(name)) { @@ -2581,8 +2558,7 @@ private static PerlBinaryBooleanAssignment perlBinaryBooleanAssignment( String name = normalized.substring(0, assignment); Boolean value = perlBooleanPropertyValue( normalized.substring(assignment + 1)); - if (value == null || !isIcuBinaryPropertyAlias(name) - && resolvePerlMissingBaseAlias(name) == null) return null; + if (value == null || !isPerlAcceptedBinaryPropertyAlias(name)) return null; return new PerlBinaryBooleanAssignment(name, value); } @@ -2616,7 +2592,21 @@ private static PerlBarePropertyAlias resolvePerlBarePropertyAlias( String property) { if (propertyValueDelimiter(property) >= 0) return null; String looseAlias = loosePropertyName(property); - if (!PERL_BARE_PROPERTY_ALIASES.contains(looseAlias)) return null; + + String binaryProperty = + PerlUnicodeBinaryPropertyAliasData.canonicalProperty(looseAlias); + if (binaryProperty != null) { + try { + UnicodeSet binary = new UnicodeSet() + .applyPropertyAlias(binaryProperty, "True") + .freeze(); + return new PerlBarePropertyAlias( + binary, looseAlias.equals("lowercase")); + } catch (IllegalArgumentException unsupported) { + return null; + } + } + if (!PERL_NON_BINARY_BARE_PROPERTY_ALIASES.contains(looseAlias)) return null; if (looseAlias.equals("any")) { return new PerlBarePropertyAlias(PERL_UNICODE_BASE_SET, false); @@ -2633,15 +2623,7 @@ private static PerlBarePropertyAlias resolvePerlBarePropertyAlias( if (category != null) { return new PerlBarePropertyAlias(category, true); } - try { - UnicodeSet binary = new UnicodeSet() - .applyPropertyAlias(looseAlias, "True") - .freeze(); - return new PerlBarePropertyAlias( - binary, looseAlias.equals("lowercase")); - } catch (IllegalArgumentException unsupported) { - return null; - } + return null; } private static PerlBarePropertyAlias resolvePerlPosixCompatibilityAlias( @@ -2837,6 +2819,18 @@ private static boolean isIcuBinaryPropertyAlias(String alias) { } } + private static boolean isPerlAcceptedBinaryPropertyAlias(String alias) { + return PerlUnicodeBinaryPropertyAliasData.canonicalProperty(alias) != null + || PerlUnicodeResidualPropertyData.isBinaryPropertyAlias(alias) + || resolvePerlMissingBaseAlias(alias) != null; + } + + private static boolean isPerlRejectedBinaryPropertyExpression(String property) { + int assignment = propertyValueDelimiter(property); + String name = assignment < 0 ? property : property.substring(0, assignment); + return PerlUnicodeBinaryPropertyAliasData.isRejected(name); + } + private static boolean isIcuGeneralCategoryAlias(String alias) { try { new UnicodeSet().applyPropertyAlias("General_Category", alias); diff --git a/src/test/java/org/perlonjava/runtime/regex/PerlXidPropertyResolverTest.java b/src/test/java/org/perlonjava/runtime/regex/PerlXidPropertyResolverTest.java new file mode 100644 index 000000000..95c7c3c01 --- /dev/null +++ b/src/test/java/org/perlonjava/runtime/regex/PerlXidPropertyResolverTest.java @@ -0,0 +1,85 @@ +package org.perlonjava.runtime.regex; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; + +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +@Tag("unit") +class PerlXidPropertyResolverTest { + private static final RegexFlags FLAGS = RegexFlags.fromModifiers("", "XID"); + + @Test + void usesTheGeneratedCurrentPerlAliasInventory() { + assertEquals("XID_Continue", + PerlUnicodeBinaryPropertyAliasData.canonicalProperty("XIDC")); + assertEquals("XID_Continue", + PerlUnicodeBinaryPropertyAliasData.canonicalProperty("IsXID Continue")); + assertTrue(PerlUnicodeBinaryPropertyAliasData.isRejected("XIDCont")); + assertTrue(PerlUnicodeBinaryPropertyAliasData.isRejected("Gr_Link")); + assertFalse(PerlUnicodeBinaryPropertyAliasData.isRejected("XIDC")); + } + + @Test + void resolvesEveryPerlXidAliasNativelyThroughJoni() { + for (String alias : new String[] { + "XIDS", "XIDStart", "XID_Start", "xid start", "IsXID_Start" + }) { + JoniRegexPattern pattern = new JoniRegexPattern("\\p{" + alias + "}", FLAGS); + assertEquals("\\p{" + alias + "}", pattern.patternDescription(), alias); + assertTrue(matches(pattern, "A"), alias); + assertTrue(matches(pattern, "\u2118"), alias); + assertFalse(matches(pattern, "0"), alias); + assertFalse(matches(pattern, "_"), alias); + } + + for (String alias : new String[] { + "XIDC", "XID_Continue", "xid continue", "IsXID_Continue" + }) { + JoniRegexPattern pattern = new JoniRegexPattern("\\p{" + alias + "}", FLAGS); + assertEquals("\\p{" + alias + "}", pattern.patternDescription(), alias); + assertTrue(matches(pattern, "A"), alias); + assertTrue(matches(pattern, "0"), alias); + assertTrue(matches(pattern, "_"), alias); + assertTrue(matches(pattern, "\u0301"), alias); + assertTrue(matches(pattern, "\u200C"), alias); + assertFalse(matches(pattern, "\uD83D\uDE00"), alias); + } + } + + @Test + void composesXidBooleanAssignmentsAndComplements() { + assertTrue(matches("\\p{XID_Start=Yes}", "A")); + assertFalse(matches("\\p{XID_Start=Yes}", "0")); + assertFalse(matches("\\p{XID_Start=No}", "A")); + assertTrue(matches("\\p{XID_Start=No}", "0")); + assertTrue(matches("\\p{XID_Continue=Yes}", "0")); + assertFalse(matches("\\p{XID_Continue=No}", "0")); + assertTrue(matches("\\p{XID_Continue=No}", "\uD83D\uDE00")); + assertFalse(matches("\\p{^XID_Continue}", "0")); + assertTrue(matches("\\P{^XID_Continue}", "0")); + } + + @Test + void rejectsTheNonPerlXidContAbbreviation() { + for (String alias : new String[] { + "XIDCont", "XIDCont=Yes", "Gr_Link", "Gr_Link=Y" + }) { + assertThrows(RuntimeException.class, + () -> new JoniRegexPattern("\\p{" + alias + "}", FLAGS), alias); + } + } + + private static boolean matches(String pattern, String subject) { + return matches(new JoniRegexPattern(pattern, FLAGS), subject); + } + + private static boolean matches(JoniRegexPattern pattern, String subject) { + return pattern.matcher(subject, List.of()).find(); + } +} diff --git a/src/test/resources/unit/regex/unicode_xid_binary_properties.t b/src/test/resources/unit/regex/unicode_xid_binary_properties.t new file mode 100644 index 000000000..cc8795a13 --- /dev/null +++ b/src/test/resources/unit/regex/unicode_xid_binary_properties.t @@ -0,0 +1,51 @@ +use strict; +use warnings; +use Test::More; + +my @start = ('A', "\x{2118}"); +my @not_start = ('0', '_', "\x{301}", "\x{200C}", "\x{1F600}"); +my @continue = ('A', '0', '_', "\x{301}", "\x{2118}", "\x{200C}"); +my @not_continue = ("\x{1F600}"); + +for my $property ('XIDS', 'XIDStart', 'XID_Start', 'xid start', 'IsXID_Start') { + my $re = eval "qr/\\p{$property}/"; + ok(defined $re, "$property compiles"); + is(scalar(grep { $_ !~ $re } @start), 0, + "$property includes XID_Start boundaries"); + is(scalar(grep { $_ =~ $re } @not_start), 0, + "$property excludes non-start boundaries"); +} + +for my $property ('XIDC', 'XID_Continue', 'xid continue', 'IsXID_Continue') { + my $re = eval "qr/\\p{$property}/"; + ok(defined $re, "$property compiles"); + is(scalar(grep { $_ !~ $re } @continue), 0, + "$property includes XID_Continue boundaries"); + is(scalar(grep { $_ =~ $re } @not_continue), 0, + "$property excludes non-continue boundaries"); +} + +ok('A' =~ qr/\p{XID_Start=Yes}/ && '0' !~ qr/\p{XID_Start=Yes}/, + 'XID_Start true assignment'); +ok('A' !~ qr/\p{XID_Start=No}/ && '0' =~ qr/\p{XID_Start=No}/, + 'XID_Start false assignment'); +ok('0' =~ qr/\p{XID_Continue=Yes}/ && "\x{1F600}" !~ qr/\p{XID_Continue=Yes}/, + 'XID_Continue true assignment'); +ok('0' !~ qr/\p{XID_Continue=No}/ && "\x{1F600}" =~ qr/\p{XID_Continue=No}/, + 'XID_Continue false assignment'); + +ok('A' !~ qr/\p{^XID_Start}/ && '0' =~ qr/\p{^XID_Start}/, + 'inner caret complements XID_Start'); +ok('A' !~ qr/\P{XID_Start}/ && '0' =~ qr/\P{XID_Start}/, + 'outer P complements XID_Start'); +ok('A' =~ qr/\P{^XID_Start}/ && '0' !~ qr/\P{^XID_Start}/, + 'inner caret cancels outer P'); +ok('0' !~ qr/\p{^XID_Continue}/ && "\x{1F600}" =~ qr/\p{^XID_Continue}/, + 'inner caret complements XID_Continue'); + +my $bad = eval q{qr/\p{XIDCont}/; 1}; +ok(!defined $bad, 'non-Perl XIDCont abbreviation is rejected'); +like($@, qr{Can't find Unicode property definition "XIDCont"}, + 'invalid abbreviation reports the property name'); + +done_testing; From 5a12b2bd86d216d6c7a0a6b6c394ed934be41d4f Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 12:35:27 +0200 Subject: [PATCH 21/43] test(regex): lock native Unicode property source preservation Cover native property spellings across standalone, standard-class, and extended-class contexts while retaining deferred user-property precedence. Generated with [OpenAI Codex](https://openai.com/codex/) Co-Authored-By: OpenAI Codex --- ...iNativePropertySourcePreservationTest.java | 35 ++++++++++ ...code_native_property_source_preservation.t | 67 +++++++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 src/test/java/org/perlonjava/runtime/regex/JoniNativePropertySourcePreservationTest.java create mode 100644 src/test/resources/unit/regex/unicode_native_property_source_preservation.t diff --git a/src/test/java/org/perlonjava/runtime/regex/JoniNativePropertySourcePreservationTest.java b/src/test/java/org/perlonjava/runtime/regex/JoniNativePropertySourcePreservationTest.java new file mode 100644 index 000000000..837942000 --- /dev/null +++ b/src/test/java/org/perlonjava/runtime/regex/JoniNativePropertySourcePreservationTest.java @@ -0,0 +1,35 @@ +package org.perlonjava.runtime.regex; + +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +@Tag("unit") +class JoniNativePropertySourcePreservationTest { + private static final RegexFlags FLAGS = RegexFlags.fromModifiers("", ""); + + @Test + void preservesNativeUnicodePropertiesByteForByteAcrossScannerContexts() { + String[] exactPatterns = { + "\\p{Block=Basic_Latin}", + "\\p{Script=Latin}", + "\\p{Script_Extensions=Hiragana}", + "\\p{Age=2.1}", + "\\p{Present_In=3.0}", + "\\p{Uppercase_Letter}", + "\\p{XPosixSpace}", + "[\\p{Block=Basic_Latin}_]", + "[^\\p{Block=Basic_Latin}]", + "[\\P{Script=Latin}_]", + "(?[ \\p{Script=Latin} & \\p{Block=Basic_Latin} ])", + "(?[ \\p{Script_Extensions=Hiragana} - [A-Z] ])", + "(?[ ! \\p{Block=Basic_Latin} ])", + }; + + for (String source : exactPatterns) { + JoniRegexPattern pattern = new JoniRegexPattern(source, FLAGS); + assertEquals(source, pattern.patternDescription(), source); + } + } +} diff --git a/src/test/resources/unit/regex/unicode_native_property_source_preservation.t b/src/test/resources/unit/regex/unicode_native_property_source_preservation.t new file mode 100644 index 000000000..ea41db76a --- /dev/null +++ b/src/test/resources/unit/regex/unicode_native_property_source_preservation.t @@ -0,0 +1,67 @@ +use strict; +use warnings; +use utf8; +use Test::More; +no warnings qw(experimental::regex_sets experimental::uniprop_wildcards); + +ok("A" =~ /\p{Block=Basic_Latin}/, + 'Block member matches outside a character class'); +ok("\x{03c0}" !~ /\p{Block=Basic_Latin}/, + 'Block nonmember stays excluded'); +ok("A" =~ /\p{Script=Latin}/, + 'Script member matches outside a character class'); +ok("\x{30fc}" =~ /\p{Script_Extensions=Hiragana}/, + 'Script_Extensions includes a Common-script extension member'); +ok("\x{20ac}" =~ /\p{Age=2.1}/, + 'Age exact value matches its introduced code point'); +ok("\x{20ac}" =~ /\p{Present_In=3.0}/, + 'Present_In includes an earlier introduced code point'); +ok("A" =~ /\p{Uppercase_Letter}/, + 'ordinary general-category alias remains usable'); +ok("\x{a0}" =~ /\p{XPosixSpace}/, + 'ordinary Perl compatibility alias remains usable'); + +ok("A" =~ /[\p{Block=Basic_Latin}_]/, + 'Block remains native in a standard character class'); +ok("\x{03c0}" !~ /[\p{Block=Basic_Latin}_]/, + 'standard class does not gain a Block nonmember'); +ok("A" =~ /(?[ \p{Script=Latin} & \p{Block=Basic_Latin} ])/, + 'Script and Block remain native in an extended-class intersection'); +ok("\x{30fc}" =~ /(?[ \p{Script_Extensions=Hiragana} - [A-Z] ])/, + 'Script_Extensions remains native in extended-class subtraction'); +ok("\x{03c0}" =~ /\P{Block=Basic_Latin}/, + 'token-level property negation remains native'); +ok("\x{03c0}" =~ /[^\p{Block=Basic_Latin}]/, + 'outer standard-class negation remains native'); +ok("\x{03c0}" =~ /(?[ ! \p{Block=Basic_Latin} ])/, + 'extended-class unary complement remains native'); + +our ($deferred, $deferred_class, $extended_collision_error); +BEGIN { + my $property = 'InGreek'; + $deferred = qr/\p{$property}/; + $deferred_class = qr/[\p{$property}_]/; + { + no warnings 'experimental::regex_sets'; + $extended_collision_error = eval q{qr/(?[ \p{InGreek} + [_] ])/}; + $extended_collision_error = $@; + } +} + +our @calls; +sub InGreek { + push @calls, $_[0] ? 'i' : 's'; + return "0600\n"; +} + +ok("\x{0600}" =~ $deferred && "\x{0370}" !~ $deferred, + 'deferred callback outranks colliding built-in outside a class'); +ok("\x{0600}" =~ $deferred_class && "\x{0370}" !~ $deferred_class, + 'deferred callback outranks colliding built-in in a standard class'); +like($extended_collision_error, + qr/Unknown user-defined property name "InGreek"/, + 'forward extended-class collision remains fatal'); +is_deeply(\@calls, ['s'], + 'deferred property is materialized once for its matching mode'); + +done_testing; From 71d322719c16fb3a19050077ed548adffda9a8f3 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 12:42:17 +0200 Subject: [PATCH 22/43] fix(regex): canonicalize named range diagnostics Render named range endpoints as Perl canonical U+ escapes in mixed Unicode range warnings while retaining engine source positions. Add exact standard-Perl and adapter regressions. Generated with Codex (https://openai.com/codex) Co-Authored-By: Codex --- .../runtime/regex/JoniRegexPattern.java | 46 ++++++++++++++++--- .../regex/JoniRegexWarningPositionTest.java | 20 ++++++++ .../named_range_warning_canonical_display.t | 31 +++++++++++++ 3 files changed, 91 insertions(+), 6 deletions(-) create mode 100644 src/test/resources/unit/regex/named_range_warning_canonical_display.t diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index 09454dc43..a3b88ad11 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -50,6 +50,8 @@ final class JoniRegexPattern { private static final String NAMED_SEQUENCE_CLASS_WARNING = "Using just the first character returned by \\N{} in character class"; + private static final String MIXED_NAMED_RANGE_WARNING = + "Both or neither range ends should be Unicode"; private static final Map INPUT_ENCODINGS = new WeakHashMap<>(); private static final Map BYTE_INPUT_ENCODINGS = new WeakHashMap<>(); private static final Map SUBJECT_INPUT_ENCODINGS = @@ -345,6 +347,10 @@ public void warn(String message, int bytePosition) { ? canonicalNamedSequenceWarningDisplay( sourcePattern, characterOffset, namedCharacterCache, namedCharacterSourceMode) + : message.equals(MIXED_NAMED_RANGE_WARNING) + ? canonicalNamedRangeWarningDisplay( + sourcePattern, characterOffset, + namedCharacterCache, namedCharacterSourceMode) : new WarningDisplay(sourcePattern, characterOffset); compileWarnings.add(RegexDiagnosticFormatter.markedPerl( display.pattern(), display.offset(), message)); @@ -399,18 +405,46 @@ private static WarningDisplay canonicalNamedSequenceWarningDisplay( return new WarningDisplay(pattern, offset); } + String canonical = canonicalNamedCharacterEscape(expansion.sequence()); + String displayPattern = pattern.substring(0, escapeStart) + + canonical + pattern.substring(close + 1); + return new WarningDisplay(displayPattern, + escapeStart + canonical.length()); + } + + /** Canonicalizes the named endpoint in Perl's mixed Unicode-range warning. */ + private static WarningDisplay canonicalNamedRangeWarningDisplay( + String pattern, int offset, NamedCharacterCache cache, + NamedCharacterExpansion.SourceMode sourceMode) { + int escapeStart = pattern.lastIndexOf("\\N{", Math.max(0, offset - 1)); + if (escapeStart < 0) return new WarningDisplay(pattern, offset); + int nameStart = escapeStart + 3; + int close = pattern.indexOf('}', nameStart); + if (close < 0 || close >= offset) return new WarningDisplay(pattern, offset); + + NamedCharacterExpansion expansion = cache.resolve( + pattern.substring(nameStart, close), sourceMode); + if (!expansion.resolved() || expansion.sequence().isEmpty()) { + return new WarningDisplay(pattern, offset); + } + + String canonical = canonicalNamedCharacterEscape(expansion.sequence()); + String displayPattern = pattern.substring(0, escapeStart) + + canonical + pattern.substring(close + 1); + return new WarningDisplay(displayPattern, + offset + canonical.length() - (close + 1 - escapeStart)); + } + + private static String canonicalNamedCharacterEscape(String sequence) { StringBuilder canonical = new StringBuilder("\\N{U+"); boolean first = true; - for (int codePoint : expansion.sequence().codePoints().toArray()) { + for (int codePoint : sequence.codePoints().toArray()) { if (!first) canonical.append('.'); + if (codePoint < 0x10) canonical.append('0'); canonical.append(Integer.toHexString(codePoint).toUpperCase()); first = false; } - canonical.append('}'); - String displayPattern = pattern.substring(0, escapeStart) - + canonical + pattern.substring(close + 1); - return new WarningDisplay(displayPattern, - escapeStart + canonical.length()); + return canonical.append('}').toString(); } RegexMatcher matcher(String input, List callbacks) { diff --git a/src/test/java/org/perlonjava/runtime/regex/JoniRegexWarningPositionTest.java b/src/test/java/org/perlonjava/runtime/regex/JoniRegexWarningPositionTest.java index 3b97dab6d..d4fb6c5ff 100644 --- a/src/test/java/org/perlonjava/runtime/regex/JoniRegexWarningPositionTest.java +++ b/src/test/java/org/perlonjava/runtime/regex/JoniRegexWarningPositionTest.java @@ -28,4 +28,24 @@ void convertsJoniBytePositionAfterUnicodePrefix() { + "m/ネ\\o{789} <-- HERE /"), new JoniRegexPattern("ネ\\o{789}", FLAGS).compileWarnings()); } + + @Test + void canonicalizesNamedEndpointsInMixedUnicodeRangeWarnings() { + for (String[] testCase : new String[][] { + {"[\\x00-\\N{SOH}]\\x{100}", + "[\\x00-\\N{U+01} <-- HERE ]\\x{100}"}, + {"[\\N{DEL}-\\o{377}]\\x{100}", + "[\\N{U+7F}-\\o{377} <-- HERE ]\\x{100}"}, + {"[\\N{DEL}-\\377]\\x{100}", + "[\\N{U+7F}-\\377 <-- HERE ]\\x{100}"}, + }) { + JoniRegexPattern pattern = new JoniRegexPattern( + testCase[0], FLAGS, 0, false, false, false, + new NamedCharacterCache(), true); + assertEquals(List.of( + "Both or neither range ends should be Unicode in regex; " + + "marked by <-- HERE in m/" + testCase[1] + "/"), + pattern.compileWarnings(), testCase[0]); + } + } } diff --git a/src/test/resources/unit/regex/named_range_warning_canonical_display.t b/src/test/resources/unit/regex/named_range_warning_canonical_display.t new file mode 100644 index 000000000..660d6a310 --- /dev/null +++ b/src/test/resources/unit/regex/named_range_warning_canonical_display.t @@ -0,0 +1,31 @@ +use strict; +use warnings; +use Test::More tests => 12; + +sub compile_warnings { + my ($source) = @_; + my @warnings; + my $compiled; + { + local $SIG{__WARN__} = sub { push @warnings, @_ }; + $compiled = eval "no warnings 'experimental::re_strict'; " + . "use re 'strict'; qr/$source/"; + } + ok(defined $compiled, "$source compiles under re strict"); + is(scalar @warnings, 1, "$source emits one warning"); + return $warnings[0]; +} + +for my $case ( + [q{[\x00-\N{SOH}]\x{100}}, q{\N{U+01}}, q{\N{SOH}}], + [q{[\N{DEL}-\o{377}]\x{100}}, q{\N{U+7F}}, q{\N{DEL}}], + [q{[\N{DEL}-\377]\x{100}}, q{\N{U+7F}}, q{\N{DEL}}], +) { + my ($source, $canonical, $original) = @$case; + my $warning = compile_warnings($source); + like($warning, + qr/Both or neither range ends should be Unicode.*\Q$canonical\E.*<-- HERE/s, + "$source warning uses the canonical named endpoint"); + unlike($warning, qr/\Q$original\E/, + "$source warning omits the source alias"); +} From 8197faf1ae5671f0e0c2fb6b172a50da04318992 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 13:04:53 +0200 Subject: [PATCH 23/43] docs(regex): record current Perl JAPH disposition Replace the incorrect ID 51 implementation target with the exact latest-Perl oracle and keep the release and worker next steps aligned with active work. Generated with Codex (https://openai.com/codex) Co-Authored-By: Codex --- dev/design/phase36-regex-parity.md | 39 ++++++++++++++++++------------ 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index 9730eb4e6..b99dfc2c6 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -186,19 +186,25 @@ timeout. `op/do.t` needs two fixes to reach standard Perl's 71/71 plus an ID-normalized disposition because PR958 over-executed it as 94/99. The current candidate reaches 71/71 on both backends. The class accessor-name candidate is validated by current Perl 5.45.3 at 8/8 focused and 30/30 authoritative. -`japh/abigail.t` remains the release blocker: private per-run isolation removes -its filesystem race, and an exact assertion map isolates the sole PR-958 delta -at ID 51. Its quoted multi-`-e` switch loses literal `$_` through shell argv -construction; IDs 127–130 are unchanged and also fail on current Perl. +`japh/abigail.t` has a required upstream-regression disposition: private +per-run isolation removes its filesystem race, and an exact assertion map +isolates the sole PR-958 delta at ID 51. Upstream commit +`5355a72c746fa5146541d790aece53b22737afee` changed `_quote_args` to add a +nested double/single wrapper around whitespace-bearing arguments. The exact +latest Perl 5.45.3 checkout also fails ID 51 because its qx shell path expands +`$_` and retains the apostrophes. PerlOnJava must preserve that current-Perl +behavior rather than add an over-parity argv special case. IDs 127–130 are +unchanged and also fail on current Perl. Acceptance must have no new invalid, missing, timeout, truncated, incomplete, or zero-TAP row and no unresolved PR-958 pass-count decrease. The user owns the release checkout; workers use private exact-head worktrees and must not mutate, rebase, or push the user's branch. -The release fixes still require one combined product build after the JAPH -warning-hook correction. Only then run the quiet exact-head `pat.t` and -`pat_thr.t` acceptance without external worker builds. +The release fixes still require one combined product build after integrating +the class, `op/do.t`, and runner-isolation candidates. Then prove the exact +JAPH ID 51 failure matches current Perl and run the quiet exact-head `pat.t` +and `pat_thr.t` acceptance without external worker builds. Independent implementation continues on `integrate/phase36-post1087-wip`. Its current preserved tranches generate current-Perl InSC/InPC and Block data, @@ -236,20 +242,21 @@ Active ownership: - P3: complete all 1,187 `re/anyof.t` native debug-description rows, then profile and remove the measured `pat*` performance regression. -- P4: close the three highest-count remaining same-source `reg_mesg.t` native - parser/lexer diagnostic families; XID and imported extended-class gates are green. -- P5: remove redundant built-in property scanning on the published P3 - integration reference, then close the exact 30-row Unicode residual. -- P6: preserve literal argv for JAPH's quoted multi-`-e` assertion 51 and prove - deterministic 110/130 or better; the `op/do.t` and class candidates are ready. +- P4: map and close remaining non-POSIX native parser/diagnostic families in + `regexp.t`, `regex_sets.t`, `charset.t`, and `pat_advanced.t`; `reg_mesg.t` + is green at 3,390/3,390 on both backends. +- P5: finish the source-less generated-data gate, then close the exact 30-row + Unicode residual; redundant built-in property scanning is removed. +- P6: independently preserve the latest-Perl qx/JAPH oracle matrix and review + the release disposition; no POJ-only ID 51 argv normalization is permitted. - Coordinator: integration, conflict resolution, immutable acceptance, PR/CI, plan state, combined build, and release evidence. ## Ordered Next Steps -1. Finish and rerun the isolated JAPH argv-preservation root, then integrate the - already-green `op/do.t` and class candidates; preserve latest-Perl tests and - exact assertion-ID evidence. +1. Integrate the already-green `op/do.t`, class, and per-run isolation + candidates; prove unmodified JAPH ID 51 matches latest Perl's upstream + failure and preserve exact assertion-ID evidence. 2. Integrate the fixes into PR 1087, run one warning-free `make`, focused Unicode, `regexp.t`, `regex_sets.t`, `charset.t`, `pat.t`, `pat_thr.t`, `pat_advanced.t`, `pat_re_eval.t`, and `reg_mesg.t` gates, then repeat the From 6daf016d9c6d57e3a940e96d05b0142098968a0b Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 13:09:34 +0200 Subject: [PATCH 24/43] fix(regex): validate Perl user property names in Joni Diagnose malformed package-qualified Unicode user-property names in the native Perl lexer while preserving valid callback names and Perl's internal surrogate property for the runtime resolver. Add direct position checks and an oracle-backed product regression. Generated with Codex (https://openai.com/codex) Co-Authored-By: Codex --- .../unit/regex/invalid_user_property_name.t | 19 +++++ third_party/joni/src/org/joni/Lexer.java | 71 +++++++++++++++++ .../src/org/joni/exception/ErrorMessages.java | 2 + .../TestPerlUserPropertyNameDiagnostics.java | 79 +++++++++++++++++++ 4 files changed, 171 insertions(+) create mode 100644 src/test/resources/unit/regex/invalid_user_property_name.t create mode 100644 third_party/joni/test/org/joni/test/TestPerlUserPropertyNameDiagnostics.java diff --git a/src/test/resources/unit/regex/invalid_user_property_name.t b/src/test/resources/unit/regex/invalid_user_property_name.t new file mode 100644 index 000000000..b2cedf4ec --- /dev/null +++ b/src/test/resources/unit/regex/invalid_user_property_name.t @@ -0,0 +1,19 @@ +use strict; +use warnings; +use Test::More tests => 12; + +for my $property ( + 'utf8::perl x', + 'Foo::bar', + 'Foo::9bar', +) { + my $source = 'qr/\\p{' . $property . '}/'; + my $ok = eval "$source; 1"; + ok(!$ok, "$property is rejected"); + like($@, qr/^Illegal user-defined property name \"\Q$property\E\" in regex/, + "$property reports the Perl user-property diagnostic"); + like($@, qr/\\p\{\Q$property\E\} <-- HERE /, + "$property marks immediately after the property"); + unlike($@, qr/^Can't find Unicode property definition/, + "$property is not reported as an unknown built-in property"); +} diff --git a/third_party/joni/src/org/joni/Lexer.java b/third_party/joni/src/org/joni/Lexer.java index beab1e18b..9241c35e4 100644 --- a/third_party/joni/src/org/joni/Lexer.java +++ b/third_party/joni/src/org/joni/Lexer.java @@ -2624,6 +2624,9 @@ protected final CharProperty fetchCharProperty(boolean inCharacterClass) { "%n", Character.toString(perlCharacterPropertyEscape)), _p - getBegin()); } + if (syntax.op2OptionPerl()) { + validatePerlUserDefinedPropertyName(_p, last); + } if (syntax.characterPropertyResolver != null) { CharacterPropertyResolver.Result resolved = syntax.characterPropertyResolver.resolve( @@ -2654,6 +2657,74 @@ protected final CharProperty fetchCharProperty(boolean inCharacterClass) { return null; // not reached } + /** Enforces Perl's package-qualified user-property naming grammar. */ + private void validatePerlUserDefinedPropertyName(int start, int end) { + while (start < end && isAsciiPropertySpace(bytes[start] & 0xff)) start++; + while (end > start && isAsciiPropertySpace(bytes[end - 1] & 0xff)) end--; + + boolean qualified = false; + for (int i = start; i + 1 < end; i++) { + if (bytes[i] == ':' && bytes[i + 1] == ':') { + qualified = true; + break; + } + } + if (!qualified) return; + + // Perl exposes this one internal property despite its deliberately + // reserved local-name spelling. + String name = new String(bytes, start, end - start); + if (name.equals("utf8::_perl_surrogate")) return; + + int segmentStart = start; + int finalStart = start; + for (int i = start; i < end; i++) { + if (i + 1 < end && bytes[i] == ':' && bytes[i + 1] == ':') { + if (segmentStart < i && !isPerlPropertyIdentifier(segmentStart, i)) { + illegalPerlUserDefinedPropertyName(name); + } + i++; + segmentStart = i + 1; + finalStart = segmentStart; + } else if (bytes[i] == ':') { + illegalPerlUserDefinedPropertyName(name); + } + } + if (!isPerlPropertyIdentifier(finalStart, end) + || end - finalStart < 3 + || !((bytes[finalStart] == 'I') + && (bytes[finalStart + 1] == 's' + || bytes[finalStart + 1] == 'n'))) { + illegalPerlUserDefinedPropertyName(name); + } + } + + private boolean isPerlPropertyIdentifier(int start, int end) { + if (start >= end || !isAsciiPropertyIdentifierStart(bytes[start] & 0xff)) { + return false; + } + for (int i = start + 1; i < end; i++) { + int ch = bytes[i] & 0xff; + if (!isAsciiPropertyIdentifierStart(ch) && (ch < '0' || ch > '9')) { + return false; + } + } + return true; + } + + private void illegalPerlUserDefinedPropertyName(String name) { + newSyntaxException(PERL_ILLEGAL_USER_DEFINED_PROPERTY_NAME.replace("%n", name), + p - getBegin()); + } + + private static boolean isAsciiPropertyIdentifierStart(int ch) { + return ch == '_' || ch >= 'A' && ch <= 'Z' || ch >= 'a' && ch <= 'z'; + } + + private static boolean isAsciiPropertySpace(int ch) { + return ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' || ch == '\f'; + } + private static void validateCharacterPropertyRanges(int[] ranges, long[] wideRanges) { if (ranges == null && wideRanges == null) { diff --git a/third_party/joni/src/org/joni/exception/ErrorMessages.java b/third_party/joni/src/org/joni/exception/ErrorMessages.java index 06d84b80a..f778e1991 100644 --- a/third_party/joni/src/org/joni/exception/ErrorMessages.java +++ b/third_party/joni/src/org/joni/exception/ErrorMessages.java @@ -174,6 +174,8 @@ public interface ErrorMessages extends org.jcodings.exception.ErrorMessages { String PERL_UNKNOWN_BOUND_TYPE = "'%n' is an unknown bound type"; String PERL_EMPTY_CHARACTER_PROPERTY = "Empty \\%n"; String PERL_EMPTY_CHARACTER_PROPERTY_BRACES = "Empty \\%n{}"; + String PERL_ILLEGAL_USER_DEFINED_PROPERTY_NAME = + "Illegal user-defined property name \"%n\""; String PERL_INVALID_CHARACTER_PROPERTY_FOLLOWER = "Character following \\%n must be '{' or a single-character Unicode property name"; String PERL_UNTERMINATED_G_PATTERN = "Unterminated \\g... pattern"; diff --git a/third_party/joni/test/org/joni/test/TestPerlUserPropertyNameDiagnostics.java b/third_party/joni/test/org/joni/test/TestPerlUserPropertyNameDiagnostics.java new file mode 100644 index 000000000..5bae23133 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestPerlUserPropertyNameDiagnostics.java @@ -0,0 +1,79 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package org.joni.test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +import java.nio.charset.StandardCharsets; + +import org.jcodings.specific.UTF8Encoding; +import org.joni.CharacterPropertyResolver; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.joni.WarnCallback; +import org.joni.exception.SyntaxException; +import org.junit.Test; + +public class TestPerlUserPropertyNameDiagnostics { + private static void compile(String pattern, Syntax syntax) { + byte[] bytes = pattern.getBytes(StandardCharsets.UTF_8); + new Regex(bytes, 0, bytes.length, Option.NONE, UTF8Encoding.INSTANCE, + syntax, WarnCallback.NONE); + } + + private static void assertInvalid(String pattern, String message, + int bytePosition) { + SyntaxException error = assertThrows(SyntaxException.class, + () -> compile(pattern, Syntax.PerlNG)); + assertEquals(message, error.getMessage()); + assertEquals(bytePosition, error.getPatternPosition()); + } + + private static Syntax acceptingPropertySyntax() { + CharacterPropertyResolver resolver = + (bytes, p, end, encoding, inCharacterClass) -> + new CharacterPropertyResolver.Result( + new int[] {1, 'A', 'A'}, false); + return new Syntax("PerlUserPropertyNameDiagnostics", Syntax.PerlNG.op, + Syntax.PerlNG.op2, Syntax.PerlNG.op3, Syntax.PerlNG.behavior, + Syntax.PerlNG.options, Syntax.PerlNG.metaCharTable, null, + resolver); + } + + @Test + public void rejectsMalformedPackageQualifiedUserProperties() { + assertInvalid("\\p{utf8::perl x}", + "Illegal user-defined property name \"utf8::perl x\"", 16); + assertInvalid("\\P{Foo::bar}", + "Illegal user-defined property name \"Foo::bar\"", 12); + assertInvalid("[\\p{Foo::9bar}]", + "Illegal user-defined property name \"Foo::9bar\"", 14); + } + + @Test + public void preservesValidAndInternalNamesForTheResolver() { + Syntax syntax = acceptingPropertySyntax(); + compile("\\p{main::IsFoo}", syntax); + compile("\\p{Foo::::InBar}", syntax); + compile("\\p{utf8::_perl_surrogate}", syntax); + } +} From 007517d68ffc85a9041b2cb90890d0c2b03189c5 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 13:14:24 +0200 Subject: [PATCH 25/43] fix(regex): skip Block regeneration without current sources Keep explicit Unicode and Perl source selection fail-closed while allowing the byte-reproducibility unit gate to skip in source-less development worktrees. Describe moving Perl source checkouts consistently as current rather than pinned. Generated with [OpenAI Codex](https://openai.com/codex/) Co-Authored-By: OpenAI Codex --- dev/tools/lib/PerlOnJava/UnicodeGenerator.pm | 2 +- .../perl_unicode_current_source_diagnostic.t | 23 +++++++++++++++++++ .../regex/PerlUnicodeBlockDataTest.java | 21 +++++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 dev/tools/tests/perl_unicode_current_source_diagnostic.t diff --git a/dev/tools/lib/PerlOnJava/UnicodeGenerator.pm b/dev/tools/lib/PerlOnJava/UnicodeGenerator.pm index 69cf559d4..fcdb4992f 100644 --- a/dev/tools/lib/PerlOnJava/UnicodeGenerator.pm +++ b/dev/tools/lib/PerlOnJava/UnicodeGenerator.pm @@ -72,7 +72,7 @@ sub select_perl_root { return $candidate unless @missing; push @diagnostics, "$candidate missing " . join(', ', @missing); } - die "No complete pinned Perl source tree: " . join('; ', @diagnostics) . "\n"; + die "No complete current Perl source tree: " . join('; ', @diagnostics) . "\n"; } sub read_raw { diff --git a/dev/tools/tests/perl_unicode_current_source_diagnostic.t b/dev/tools/tests/perl_unicode_current_source_diagnostic.t new file mode 100644 index 000000000..ec20a873b --- /dev/null +++ b/dev/tools/tests/perl_unicode_current_source_diagnostic.t @@ -0,0 +1,23 @@ +use strict; +use warnings; +use File::Spec; +use File::Temp qw(tempdir); +use FindBin; +use Test::More tests => 1; + +use lib File::Spec->catdir($FindBin::Bin, '..', 'lib'); +use PerlOnJava::UnicodeGenerator qw(select_perl_root); + +my $root = tempdir(CLEANUP => 1); +my $explicit = File::Spec->catdir($root, 'explicit-perl'); +local $ENV{PERLONJAVA_PERL_ROOT} = $explicit; + +eval { + select_perl_root( + repo_root => $root, + required => ['uni_keywords.h'], + ); +}; +like($@, + qr/^No complete current Perl source tree: \Q$explicit\E missing uni_keywords\.h/, + 'missing explicit Perl roots use current-source terminology'); diff --git a/src/test/java/org/perlonjava/runtime/regex/PerlUnicodeBlockDataTest.java b/src/test/java/org/perlonjava/runtime/regex/PerlUnicodeBlockDataTest.java index 887ea036f..073f36b3e 100644 --- a/src/test/java/org/perlonjava/runtime/regex/PerlUnicodeBlockDataTest.java +++ b/src/test/java/org/perlonjava/runtime/regex/PerlUnicodeBlockDataTest.java @@ -6,6 +6,7 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; import com.ibm.icu.text.UnicodeSet; import java.nio.charset.StandardCharsets; @@ -134,6 +135,11 @@ void generatorReproducesCheckedInDataByteForByte() throws Exception { .start(); byte[] actual = process.getInputStream().readAllBytes(); assertTrue(process.waitFor(30, TimeUnit.SECONDS), "generator timed out"); + if (process.exitValue() != 0 + && !hasExplicitGeneratorSource() + && reportsMissingDevelopmentSources(actual)) { + assumeTrue(false, new String(actual, StandardCharsets.UTF_8).trim()); + } assertEquals(0, process.exitValue(), new String(actual, StandardCharsets.UTF_8)); assertArrayEquals(Files.readAllBytes(generated), actual); for (byte value : actual) { @@ -141,6 +147,21 @@ void generatorReproducesCheckedInDataByteForByte() throws Exception { } } + private static boolean hasExplicitGeneratorSource() { + return hasText(System.getenv("PERLONJAVA_UNICODE_ROOT")) + || hasText(System.getenv("PERLONJAVA_PERL_ROOT")); + } + + private static boolean hasText(String value) { + return value != null && !value.isBlank(); + } + + private static boolean reportsMissingDevelopmentSources(byte[] output) { + String diagnostic = new String(output, StandardCharsets.UTF_8); + return diagnostic.startsWith("No complete Unicode current source tree:") + || diagnostic.startsWith("No complete current Perl source tree:"); + } + private static UnicodeSet set(String alias) { UnicodeSet set = PerlUnicodeBlockData.set(alias); if (set == null) throw new AssertionError("Unknown Block alias " + alias); From c89158c04afbb43dc9c6882123e96c7918c25f93 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 13:19:17 +0200 Subject: [PATCH 26/43] fix: preserve regex debug lifecycle provenance Emit a truthful deferred-program description without publishing placeholder bytecode, retain callback source spelling across cached and runtime recompilations, and reset top-level regex callsite/debug state coherently. Generated with [OpenAI Codex](https://openai.com/codex/) Co-Authored-By: OpenAI Codex --- .../scriptengine/PerlLanguageProvider.java | 6 + .../runtime/regex/RuntimeRegex.java | 108 ++++++++--- .../runtimetypes/RuntimeRegexState.java | 19 ++ .../RegexDeferredDebugTraceTest.java | 177 ++++++++++++++++++ 4 files changed, 288 insertions(+), 22 deletions(-) create mode 100644 src/test/java/org/perlonjava/RegexDeferredDebugTraceTest.java diff --git a/src/main/java/org/perlonjava/app/scriptengine/PerlLanguageProvider.java b/src/main/java/org/perlonjava/app/scriptengine/PerlLanguageProvider.java index aa7e5cab4..c57fb3bbb 100644 --- a/src/main/java/org/perlonjava/app/scriptengine/PerlLanguageProvider.java +++ b/src/main/java/org/perlonjava/app/scriptengine/PerlLanguageProvider.java @@ -99,6 +99,12 @@ public static void resetAll() { try (PerlRuntime.Binding runtimeBinding = PerlRuntime.bindCurrentOrNew(); CompilationLockGuard ignored = acquireCompilationLock()) { PerlRuntime.current().globalState().resetCoreGlobalsInitialization(); + // resetAll() starts a new top-level interpreter lifecycle even + // when the embedding layer reuses the same PerlRuntime object. + // Reset compiled/debug lifecycle state without discarding + // inherited user-property results prepared for an ithread + // snapshot before lazy runtime initialization. + PerlRuntime.current().regexState.resetForTopLevel(); GlobalContext.setThreadTaintMode(false); resetAllGlobals(); WeakRefRegistry.resetState(); diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java index 51c9dbf97..5252dc38b 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java @@ -21,6 +21,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; @@ -122,6 +123,10 @@ static void updateControlVerbVariables(String mark, String error) { List executableCallbacks = List.of(); private boolean executableCallbacksReleased; public String patternString; + // Perl source spelling used only for diagnostics and debug lifecycle + // output. The executable pattern may contain private structural callout + // slots that must never leak into user-visible traces. + private String debugPatternString; // Source scalar provenance needed when a compiled qr// is reused under // lexical use bytes: byte-backed source characters are already octets. private boolean patternByteBacked; @@ -193,6 +198,7 @@ private RuntimeRegex cloneTrackedWithCallbacks(List callba copy.userPropertyPackage = this.userPropertyPackage; copy.setExecutableCallbacks(callbacks); copy.patternString = this.patternString; + copy.debugPatternString = this.debugPatternString; copy.patternByteBacked = this.patternByteBacked; copy.hasPreservesMatch = this.hasPreservesMatch; copy.useGAssertion = this.useGAssertion; @@ -822,6 +828,8 @@ private static synchronized RuntimeRegex compileSynchronized( regex.hasPreservesMatch = regex.regexFlags.preservesMatch() || RegexFlags.hasInlinePreserveModifier(compilePatternString); regex.patternString = originalPatternString; + regex.debugPatternString = sourceDiagnosticPattern == null + ? originalPatternString : displayDiagnosticPattern; // Check if pattern has code block captures for $^R optimization // Code blocks are encoded as named captures like (?) @@ -1015,7 +1023,6 @@ && containsExecutableSource(originalPatternString, state().compiledRegexCache.put(cacheKey, regex); } if (lexicalDebugMode != 0 - && !regex.deferredUserDefinedUnicodeProperties && state().reportedDebugCompilations.add(cacheKey)) { regex.emitCompileDebugTrace(); } @@ -1406,7 +1413,14 @@ private void emitCompileDebugTrace() { | LEXICAL_DEBUG_EXECUTE)) == 0) return; registerDebugLifecycle(); if ((lexicalDebugMode & LEXICAL_DEBUG_COMPILE) == 0) return; - String patternDescription = patternString == null ? "" : patternString; + String patternDescription = debugPatternDescription(); + if (deferredUserDefinedUnicodeProperties) { + debugWrite("Compiling REx \"" + patternDescription + "\"\n" + + "Final program:\n" + + "JONI_PATTERN deferred user-property placeholder; " + + "native bytecode pending runtime resolution\n"); + return; + } String optimizerDescription = recursivePattern == null ? "" : recursivePattern.optimizerDebugDescription(); String nativeProgram = recursivePattern == null @@ -1421,8 +1435,12 @@ private void emitCompileDebugTrace() { public void emitExecutionDebugTrace(String input) { if ((lexicalDebugMode & LEXICAL_DEBUG_EXECUTE) == 0) return; + // A qr// can acquire execute tracing through a wrapper or enter an + // ithread whose active lifecycle list is intentionally empty. Ensure + // the logical compiled regex is represented before emitting Matching. + registerDebugLifecycle(); StringBuilder trace = new StringBuilder(Math.max(96, input.length() * 72)); - trace.append("Matching REx \"").append(patternString == null ? "" : patternString) + trace.append("Matching REx \"").append(debugPatternDescription()) .append("\" against input of length ").append(input.length()).append('\n'); // Joni's matcher does not expose its live instruction cursor publicly. // Report the three observable stages of consuming each candidate input @@ -1440,6 +1458,12 @@ private void registerDebugLifecycle() { List active = state().activeDebugRegexes; for (RuntimeRegex regex : active) { if (regex == this) return; + if (Objects.equals(regex.debugPatternDescription(), + debugPatternDescription()) + && Objects.equals( + regex.regexFlags == null + ? "" : regex.regexFlags.toFlagString(), + regexFlags == null ? "" : regexFlags.toFlagString())) return; } active.add(this); } @@ -1450,7 +1474,7 @@ public static void emitCurrentRuntimeDebugFreeTraces() { for (RuntimeRegex regex : active) { if ((regex.lexicalDebugMode & (LEXICAL_DEBUG_COMPILE | LEXICAL_DEBUG_EXECUTE)) == 0) continue; - String patternDescription = regex.patternString == null ? "" : regex.patternString; + String patternDescription = regex.debugPatternDescription(); regex.debugWrite("Freeing REx: \"" + patternDescription + "\"\n"); } active.clear(); @@ -1464,6 +1488,13 @@ private void debugWrite(String message) { RuntimeIO.getStderr().flush(); } + private String debugPatternDescription() { + if (debugPatternString != null) return debugPatternString; + return executableCallbacks.isEmpty() + ? patternString == null ? "" : patternString + : RuntimeRegexTemplate.displayPattern(patternString, executableCallbacks); + } + private static RuntimeRegex ensureCompiledForRuntime(RuntimeRegex regex) { if (!regex.deferredUserDefinedUnicodeProperties) { return regex; @@ -1481,21 +1512,13 @@ private static RuntimeRegex ensureCompiledForRuntime(RuntimeRegex regex) { && regex.regexFlags.isCaseInsensitive(), regex.regexFlags != null && regex.regexFlags.isExtended()); - // The placeholder was cached under its original lexical - // compilation context. Package and lexical charnames - // provenance must both be reused for recompilation. - if (regex.compiledRegexCacheKey != null) { - state().compiledRegexCache.remove(regex.compiledRegexCacheKey); - state().reportedDebugCompilations.remove(regex.compiledRegexCacheKey); - } - return compileSynchronized(regex.patternString, - regex.regexFlags == null - ? "" : regex.regexFlags.toFlagString(), - regex.lexicalDebugMode, regex.trustedCalloutCount, false, - regex.patternByteBacked, regex.lexicalReStrict, - regex.namedCharacterTranslator, regex.namedCharacterCache, - regex.namedCharacterSourceMode, true, null); + return recompileDeferredForRuntime(regex); }); + // The forced compilation emits the resolved native program, but the + // temporary cache object is not a second logical regex lifecycle. + // The original deferred object remains registered for the one Freeing + // record emitted at runtime teardown. + state().activeDebugRegexes.removeIf(active -> active == recompiled); if (recompiled.deferredUserDefinedUnicodeProperties) { throw new PerlCompilerException( "Deferred user-defined Unicode property remained unresolved at runtime"); @@ -1523,6 +1546,33 @@ private static RuntimeRegex ensureCompiledForRuntime(RuntimeRegex regex) { return regex; } + private static synchronized RuntimeRegex recompileDeferredForRuntime( + RuntimeRegex regex) { + // Another tracked clone of this placeholder may already have resolved + // the shared cache key. Reuse its native program instead of evicting + // and recompiling it again. + if (regex.compiledRegexCacheKey != null) { + RuntimeRegex cached = state().compiledRegexCache.get( + regex.compiledRegexCacheKey); + if (cached != null && !cached.deferredUserDefinedUnicodeProperties) { + return cached; + } + + // The placeholder was cached under its original lexical + // compilation context. Package and lexical charnames provenance + // must both be reused for recompilation. + state().compiledRegexCache.remove(regex.compiledRegexCacheKey); + state().reportedDebugCompilations.remove(regex.compiledRegexCacheKey); + } + return compileSynchronized(regex.patternString, + regex.regexFlags == null ? "" : regex.regexFlags.toFlagString(), + regex.lexicalDebugMode, regex.trustedCalloutCount, false, + regex.patternByteBacked, regex.lexicalReStrict, + regex.namedCharacterTranslator, regex.namedCharacterCache, + regex.namedCharacterSourceMode, true, + regex.debugPatternDescription()); + } + private static RuntimeScalar copyScalarOrNull(RuntimeScalar value) { return value == null ? null : new RuntimeScalar(value); } @@ -1640,10 +1690,15 @@ && containsExecutableSource(patternString.toString(), modifierStr.indexOf('x') > } if (patternString.value instanceof RuntimeRegexTemplate template) { + String displayPattern = RuntimeRegexTemplate.displayPattern( + template.pattern(), template.callbacks()); RuntimeRegex regex = compile(template.pattern(), rawModifierStr, callSiteDebugMode, template.callbacks().size(), template.byteBackedPattern(), - reStrictMode(rawModifierStr), RuntimeRegexTemplate.displayPattern( - template.pattern(), template.callbacks())).cloneTracked(); + reStrictMode(rawModifierStr), displayPattern).cloneTracked(); + // Native programs are shared by executable slot skeleton and + // callback count. Source display belongs to this construction, + // not to whichever same-shaped template populated the cache. + regex.debugPatternString = displayPattern; regex.setExecutableCallbacks(template.callbacks()); return new RuntimeScalar(regex).propagateTaint(patternString); } @@ -1671,6 +1726,7 @@ && containsExecutableSource(patternString.toString(), modifierStr.indexOf('x') > regex.userPropertyPackage = originalRegex.userPropertyPackage; regex.setExecutableCallbacks(originalRegex.executableCallbacks); regex.patternString = originalRegex.patternString; + regex.debugPatternString = originalRegex.debugPatternString; regex.patternByteBacked = originalRegex.patternByteBacked; regex.deferredUserDefinedUnicodeProperties = originalRegex.deferredUserDefinedUnicodeProperties; @@ -1718,6 +1774,7 @@ && containsExecutableSource(patternString.toString(), modifierStr.indexOf('x') > regex.userPropertyPackage = originalRegex.userPropertyPackage; regex.setExecutableCallbacks(originalRegex.executableCallbacks); regex.patternString = originalRegex.patternString; + regex.debugPatternString = originalRegex.debugPatternString; regex.patternByteBacked = originalRegex.patternByteBacked; regex.deferredUserDefinedUnicodeProperties = originalRegex.deferredUserDefinedUnicodeProperties; @@ -1834,8 +1891,12 @@ static RuntimeScalar compileExecutableTemplate( List callbacks, RuntimeScalar original, boolean patternByteBacked) { int lexicalDebugMode = debugMode(modifiers); + String displayPattern = RuntimeRegexTemplate.displayPattern( + executablePattern, callbacks); RuntimeRegex regex = compile(executablePattern, modifiers, - lexicalDebugMode, callbacks.size(), patternByteBacked).cloneTracked(); + lexicalDebugMode, callbacks.size(), patternByteBacked, + reStrictMode(modifiers), displayPattern).cloneTracked(); + regex.debugPatternString = displayPattern; regex.setExecutableCallbacks(callbacks); return new RuntimeScalar(regex).propagateTaint(original); } @@ -1962,6 +2023,7 @@ public static RuntimeScalar getReplacementRegex(RuntimeScalar patternString, Run regex.userPropertyPackage = resolvedRegex.userPropertyPackage; regex.executableCallbacks = resolvedRegex.executableCallbacks; regex.patternString = resolvedRegex.patternString; + regex.debugPatternString = resolvedRegex.debugPatternString; regex.patternByteBacked = resolvedRegex.patternByteBacked; regex.deferredUserDefinedUnicodeProperties = resolvedRegex.deferredUserDefinedUnicodeProperties; @@ -1995,7 +2057,8 @@ public static RuntimeScalar getReplacementRegex(RuntimeScalar patternString, Run RuntimeRegex recompiledRegex = compile(resolvedRegex.patternString, newFlags.toFlagString(), regex.lexicalDebugMode, resolvedRegex.executableCallbacks.size(), - resolvedRegex.patternByteBacked, regex.lexicalReStrict); + resolvedRegex.patternByteBacked, regex.lexicalReStrict, + resolvedRegex.debugPatternDescription()); regex.recursivePattern = recompiledRegex.recursivePattern; regex.recursivePatternUnicode = recompiledRegex.recursivePatternUnicode; regex.recursivePatternBytes = recompiledRegex.recursivePatternBytes; @@ -2008,6 +2071,7 @@ public static RuntimeScalar getReplacementRegex(RuntimeScalar patternString, Run regex.userPropertyPackage = recompiledRegex.userPropertyPackage; regex.executableCallbacks = resolvedRegex.executableCallbacks; regex.patternString = recompiledRegex.patternString; + regex.debugPatternString = recompiledRegex.debugPatternString; regex.patternByteBacked = recompiledRegex.patternByteBacked; regex.regexFlags = recompiledRegex.regexFlags; regex.deferredUserDefinedUnicodeProperties = diff --git a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeRegexState.java b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeRegexState.java index 364af84e1..eb09aa7ff 100644 --- a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeRegexState.java +++ b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeRegexState.java @@ -103,6 +103,25 @@ public void clearMatchState() { manualCaptureEnds = null; } + /** + * Start a new top-level script in the same managed runtime. + * + *

User-property results are deliberately retained: they can be seeded + * before lazy initialization and are the only regex metadata inherited by + * an ithread snapshot. Callsite identities, match/pos state, compiled + * programs, and debug lifecycle records belong to one top-level script + * and must not collide with the next script's reused integer IDs.

+ */ + public void resetForTopLevel() { + compiledRegexCache.clear(); + optimizedRegexCache.clear(); + literalRegexTargets.clear(); + positionCache.clear(); + activeDebugRegexes.clear(); + reportedDebugCompilations.clear(); + clearMatchState(); + } + /** Copy immutable regex metadata that Perl ithreads inherit at creation. */ void snapshotInto(RuntimeRegexState target) { target.userUnicodePropertyCache.putAll(userUnicodePropertyCache); diff --git a/src/test/java/org/perlonjava/RegexDeferredDebugTraceTest.java b/src/test/java/org/perlonjava/RegexDeferredDebugTraceTest.java new file mode 100644 index 000000000..dbfd8af50 --- /dev/null +++ b/src/test/java/org/perlonjava/RegexDeferredDebugTraceTest.java @@ -0,0 +1,177 @@ +package org.perlonjava; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.perlonjava.app.cli.ArgumentParser; +import org.perlonjava.app.cli.CompilerOptions; +import org.perlonjava.app.scriptengine.PerlLanguageProvider; +import org.perlonjava.runtime.io.StandardIO; +import org.perlonjava.runtime.runtimetypes.GlobalVariable; +import org.perlonjava.runtime.runtimetypes.RuntimeIO; + +@Tag("unit") +public class RegexDeferredDebugTraceTest extends PerlRuntimeTestBase { + private RuntimeIO originalStdout; + private RuntimeIO originalStderr; + private ByteArrayOutputStream stderr; + + @BeforeEach + void setUp() { + PerlLanguageProvider.resetAll(); + originalStdout = RuntimeIO.getStdout(); + originalStderr = RuntimeIO.getStderr(); + RuntimeIO.setStdout(new RuntimeIO( + new StandardIO(new ByteArrayOutputStream(), true))); + stderr = new ByteArrayOutputStream(); + RuntimeIO.setStderr(new RuntimeIO(new StandardIO(stderr, false))); + GlobalVariable.getGlobalIO("main::STDOUT").setIO(RuntimeIO.getStdout()); + GlobalVariable.getGlobalIO("main::STDERR").setIO(RuntimeIO.getStderr()); + } + + @AfterEach + void tearDown() { + RuntimeIO.setStdout(originalStdout); + RuntimeIO.setStderr(originalStderr); + GlobalVariable.getGlobalIO("main::STDOUT").setIO(RuntimeIO.getStdout()); + GlobalVariable.getGlobalIO("main::STDERR").setIO(RuntimeIO.getStderr()); + PerlLanguageProvider.resetAll(); + } + + private String execute(String source) throws Exception { + CompilerOptions options = ArgumentParser.parseArguments( + new String[] {"-e", source}); + PerlLanguageProvider.executePerlCode(options, true); + RuntimeIO.getStderr().flush(); + return stderr.toString(StandardCharsets.ISO_8859_1); + } + + private static int occurrences(String value, String needle) { + int count = 0; + for (int offset = 0; (offset = value.indexOf(needle, offset)) >= 0; + offset += needle.length()) { + count++; + } + return count; + } + + @Test + void describesDeferredProgramWithoutPublishingPlaceholderBytecode() throws Exception { + String trace = execute("use re Debug => 'COMPILE'; " + + "our $r; BEGIN { my $p = " + + "'[^[:^print:][:^ascii:]b\\p{IsPhase36DeferredDebug}]'; " + + "$r = qr/$p/; }"); + + assertTrue(trace.contains( + "Compiling REx \"[^[:^print:][:^ascii:]b" + + "\\p{IsPhase36DeferredDebug}]\""), trace); + assertTrue(trace.contains("Final program:\n" + + "JONI_PATTERN deferred user-property placeholder; " + + "native bytecode pending runtime resolution\n"), trace); + assertFalse(trace.contains("code length:"), trace); + assertFalse(trace.contains("minlen "), trace); + } + + @Test + void reportsOneDeferredProgramPerRuntimeCacheKey() throws Exception { + String trace = execute("use re Debug => 'COMPILE'; " + + "our ($left, $right); BEGIN { " + + "my $p = '\\p{IsPhase36DeferredCache}'; " + + "$left = qr/$p/; $right = qr/$p/; }"); + + assertEquals(1, occurrences(trace, + "Compiling REx \"\\p{IsPhase36DeferredCache}\""), trace); + assertEquals(1, occurrences(trace, + "native bytecode pending runtime resolution"), trace); + } + + @Test + void runtimeResolutionReplacesDeferredReportWithNativeProgram() throws Exception { + String trace = execute("use re Debug => 'COMPILE'; our $r; " + + "our $calls = 0; " + + "BEGIN { $r = qr/\\p{IsPhase36ForwardDebug}/; } " + + "eval q{sub IsPhase36ForwardDebug { ++$calls; qq{0041\\n} }}; " + + "die $@ if $@; die unless 'A' =~ $r; die unless 'A' =~ $r; " + + "die unless $calls == 1;"); + + assertEquals(2, occurrences(trace, + "Compiling REx \"\\p{IsPhase36ForwardDebug}\""), trace); + assertEquals(1, occurrences(trace, + "native bytecode pending runtime resolution"), trace); + assertEquals(1, occurrences(trace, "JONI_PATTERN native bytecode:"), trace); + assertEquals(1, occurrences(trace, "code length:"), trace); + assertEquals(1, occurrences(trace, + "Freeing REx: \"\\p{IsPhase36ForwardDebug}\""), trace); + } + + @Test + void trackedDeferredClonesReuseOneResolvedNativeProgram() throws Exception { + String trace = execute("use re Debug => 'COMPILE'; our $r; " + + "BEGIN { $r = qr/\\p{IsPhase36CloneDebug}/; } " + + "my $left = qr/$r/; my $right = qr/$r/; " + + "eval q{sub IsPhase36CloneDebug { qq{0041\\n} }}; " + + "die $@ if $@; die unless 'A' =~ $left; " + + "die unless 'A' =~ $right;"); + + assertEquals(1, occurrences(trace, + "native bytecode pending runtime resolution"), trace); + assertEquals(1, occurrences(trace, "JONI_PATTERN native bytecode:"), trace); + } + + @Test + void executeOnlyDeferredProgramKeepsOneLogicalLifecycle() throws Exception { + String trace = execute("use re Debug => 'EXECUTE'; our $r; " + + "BEGIN { $r = qr/\\p{IsPhase36ExecuteDebug}/; } " + + "eval q{sub IsPhase36ExecuteDebug { qq{0041\\n} }}; " + + "die $@ if $@; die unless 'A' =~ $r;"); + + assertFalse(trace.contains("Compiling REx"), trace); + assertFalse(trace.contains("native bytecode pending runtime resolution"), trace); + assertTrue(trace.contains( + "Matching REx \"\\p{IsPhase36ExecuteDebug}\""), trace); + assertEquals(1, occurrences(trace, + "Freeing REx: \"\\p{IsPhase36ExecuteDebug}\""), trace); + } + + @Test + void deferredCalloutKeepsOriginalSourceAcrossResolvedLifecycle() throws Exception { + String trace = execute("use re 'eval'; use re Debug => 'ALL'; " + + "our ($r, $calls); BEGIN { $r = qr/" + + "(?{ ++$main::calls })\\p{IsPhase36CalloutDebug}/; } " + + "eval q{sub IsPhase36CalloutDebug { qq{0041\\n} }}; " + + "die $@ if $@; die unless 'A' =~ $r; die unless $calls == 1;"); + String source = "(?{ ++$main::calls })\\p{IsPhase36CalloutDebug}"; + + assertEquals(2, occurrences(trace, "Compiling REx \"" + source + "\""), trace); + assertTrue(trace.contains("Matching REx \"" + source + "\""), trace); + assertEquals(1, occurrences(trace, "Freeing REx: \"" + source + "\""), trace); + assertFalse(trace.contains("\u001e"), trace); + assertFalse(trace.contains("\u001f"), trace); + assertFalse(trace.contains("?B0?"), trace); + } + + @Test + void sameNativeSkeletonKeepsEachConstructionSource() throws Exception { + String trace = execute("use re 'eval'; use re Debug => 'EXECUTE'; " + + "our ($one, $two); " + + "my $first = '(?{ ++$main::one })A'; " + + "my $second = '(?{ ++$main::two })A'; " + + "die unless 'A' =~ /$first/; die unless 'A' =~ /$second/; " + + "die unless $one == 1 && $two == 1;"); + + assertTrue(trace.contains( + "Matching REx \"(?{ ++$main::one })A\""), trace); + assertTrue(trace.contains( + "Matching REx \"(?{ ++$main::two })A\""), trace); + assertFalse(trace.contains("\u001e"), trace); + assertFalse(trace.contains("\u001f"), trace); + } +} From 55d4429f81db29d1d89de9f751cdd9194463c4d6 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 13:22:28 +0200 Subject: [PATCH 27/43] docs(regex): advance the PR 1087 release gate Record the warning-free candidate, exact focused dispositions, active worker ownership, and the remaining quiet pat acceptance boundary. Generated with Codex (https://openai.com/codex) Co-Authored-By: Codex --- dev/design/phase36-regex-parity.md | 37 +++++++++++++++++------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index b99dfc2c6..1536ddf86 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -201,10 +201,14 @@ unresolved PR-958 pass-count decrease. The user owns the release checkout; workers use private exact-head worktrees and must not mutate, rebase, or push the user's branch. -The release fixes still require one combined product build after integrating -the class, `op/do.t`, and runner-isolation candidates. Then prove the exact -JAPH ID 51 failure matches current Perl and run the quiet exact-head `pat.t` -and `pat_thr.t` acceptance without external worker builds. +The four-commit release candidate at `9739104ac` has a clean warning-free full +`make`. In writable current-source state `op/do.t` is 71/71 on both backends; +`class/accessor.t` executes all 30 rows and improves to 17 passes versus the +PR-958 12-pass floor; isolated JAPH is deterministically 109/130 with ID 51 +matching current Perl's upstream regression. The remaining pre-evaluation gate +is a quiet exact-head `pat.t` and `pat_thr.t` run without external worker +builds; an initial timing attempt was discarded when an already-authorized +worker build overlapped it. Independent implementation continues on `integrate/phase36-post1087-wip`. Its current preserved tranches generate current-Perl InSC/InPC and Block data, @@ -241,23 +245,24 @@ checks at the same immutable head. Active ownership: - P3: complete all 1,187 `re/anyof.t` native debug-description rows, then - profile and remove the measured `pat*` performance regression. -- P4: map and close remaining non-POSIX native parser/diagnostic families in - `regexp.t`, `regex_sets.t`, `charset.t`, and `pat_advanced.t`; `reg_mesg.t` - is green at 3,390/3,390 on both backends. -- P5: finish the source-less generated-data gate, then close the exact 30-row - Unicode residual; redundant built-in property scanning is removed. -- P6: independently preserve the latest-Perl qx/JAPH oracle matrix and review - the release disposition; no POJ-only ID 51 argv normalization is permitted. + profile and remove the measured `pat*` performance regression; debug source, + cache, reset, and lifecycle provenance are integrated and green. +- P4: close the ten disjoint `pat_advanced.t` fold/trie/introspection rows; + `regexp.t`, `regex_sets.t`, `charset.t`, and `reg_mesg.t` are green on both + backends and malformed user-property grammar is native in Joni. +- P5: close the exact 30-row Unicode residual; redundant built-in property + scanning is removed and source-present/source-less generator gates are green. +- P6: preserve the release disposition bundle and finish packaging, notices, + licenses, and affected-CPAN acceptance after the quiet `pat*` gate. - Coordinator: integration, conflict resolution, immutable acceptance, PR/CI, plan state, combined build, and release evidence. ## Ordered Next Steps -1. Integrate the already-green `op/do.t`, class, and per-run isolation - candidates; prove unmodified JAPH ID 51 matches latest Perl's upstream - failure and preserve exact assertion-ID evidence. -2. Integrate the fixes into PR 1087, run one warning-free `make`, focused +1. Complete the quiet `pat.t`/`pat_thr.t` run on `9739104ac`, publish the four + green candidate commits for integration into PR 1087, and preserve the + exact source-hash/assertion-ID dispositions. +2. After integration into PR 1087, run focused Unicode, `regexp.t`, `regex_sets.t`, `charset.t`, `pat.t`, `pat_thr.t`, `pat_advanced.t`, `pat_re_eval.t`, and `reg_mesg.t` gates, then repeat the complete corpus, strict PR-958 comparison, additional release checks, and From 5f631e46c1908a5498a8202512713b49d5b559df Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 13:23:15 +0200 Subject: [PATCH 28/43] fix(regex): close remaining Unicode property rows Accept Perl's exact /aa wildcard modifier policy, resolve generated Decomposition_Type aliases through their complete value table, and normalize the General_Category L& compatibility spelling. Lock the final 30-row closure across both regex execution backends. Generated with [OpenAI Codex](https://openai.com/codex/) Co-Authored-By: OpenAI Codex --- ...te_perl_unicode_decomposition_type_data.pl | 15 ++++- .../PerlUnicodeDecompositionTypeData.java | 27 +++++++++ .../runtime/regex/UnicodeResolver.java | 31 +++++++++- .../unit/regex/unicode_remaining_30_closure.t | 60 +++++++++++++++++++ 4 files changed, 130 insertions(+), 3 deletions(-) create mode 100644 src/test/resources/unit/regex/unicode_remaining_30_closure.t diff --git a/dev/tools/generate_perl_unicode_decomposition_type_data.pl b/dev/tools/generate_perl_unicode_decomposition_type_data.pl index 398c570a5..7b48628a6 100644 --- a/dev/tools/generate_perl_unicode_decomposition_type_data.pl +++ b/dev/tools/generate_perl_unicode_decomposition_type_data.pl @@ -84,7 +84,7 @@ sub source_text { die "$value_path is not pinned Unicode $expected_version data\n" unless $value_text =~ /^# PropertyValueAliases-\Q$expected_version\E\.txt$/m; verify_unicode_notice($value_path, $value_text); -my (@values, %value_index, %value_aliases); +my (@values, %value_index, %value_aliases, @value_alias_spellings); for my $line (split /\n/, $value_text) { $line =~ s/#.*//; my @fields = map { trim($_) } split /;/, $line, -1; @@ -100,6 +100,8 @@ sub source_text { die "Conflicting Decomposition_Type alias '$alias'\n" if exists $value_aliases{$normalized} && $value_aliases{$normalized} != $id; $value_aliases{$normalized} = $id; + push @{$value_alias_spellings[$id]}, $alias + unless grep { $_ eq $alias } @{$value_alias_spellings[$id] // []}; } } die "Expected 18 Decomposition_Type values, found " . scalar(@values) . "\n" @@ -113,6 +115,7 @@ sub source_text { $value_index{Non_Canonical} = $non_canonical_id; $value_aliases{loose('Non_Canon')} = $non_canonical_id; $value_aliases{loose('Non_Canonical')} = $non_canonical_id; +$value_alias_spellings[$non_canonical_id] = ['Non_Canon', 'Non_Canonical']; for my $range (@ranges, @missing) { die "Unknown Decomposition_Type value '$range->[2]'\n" @@ -208,6 +211,11 @@ package org.perlonjava.runtime.regex; print " private static final String[] CANONICAL_NAMES = {\n "; print join(', ', map { qq{"$_"} } @values); print "\n };\n\n"; +print " private static final String[][] VALUE_ALIASES = {\n"; +for my $aliases (@value_alias_spellings) { + print " {", join(', ', map { qq{"$_"} } @$aliases), "},\n"; +} +print " };\n\n"; print " private static final int[] STARTS = {\n"; for (my $i = 0; $i < @coalesced_starts; $i += 10) { my $end = $i + 9 < $#coalesced_starts ? $i + 9 : $#coalesced_starts; @@ -294,6 +302,11 @@ package org.perlonjava.runtime.regex; ? CANONICAL_NAMES[value] : null; } + static String[] valueAliases(byte value) { + return value >= 0 && value < VALUE_ALIASES.length + ? VALUE_ALIASES[value] : null; + } + static boolean matches(byte property, byte requested) { if (requested == NON_CANONICAL) { return property != CANONICAL && property != NONE; diff --git a/src/main/java/org/perlonjava/runtime/regex/PerlUnicodeDecompositionTypeData.java b/src/main/java/org/perlonjava/runtime/regex/PerlUnicodeDecompositionTypeData.java index dfe3bd02f..8bd0148f8 100644 --- a/src/main/java/org/perlonjava/runtime/regex/PerlUnicodeDecompositionTypeData.java +++ b/src/main/java/org/perlonjava/runtime/regex/PerlUnicodeDecompositionTypeData.java @@ -37,6 +37,28 @@ final class PerlUnicodeDecompositionTypeData { "Canonical", "Compat", "Circle", "Final", "Font", "Fraction", "Initial", "Isolated", "Medial", "Narrow", "Nobreak", "None", "Small", "Square", "Sub", "Super", "Vertical", "Wide", "Non_Canonical" }; + private static final String[][] VALUE_ALIASES = { + {"Can", "Canonical", "can"}, + {"Com", "Compat", "com"}, + {"Enc", "Circle", "enc"}, + {"Fin", "Final", "fin"}, + {"Font", "font"}, + {"Fra", "Fraction", "fra"}, + {"Init", "Initial", "init"}, + {"Iso", "Isolated", "iso"}, + {"Med", "Medial", "med"}, + {"Nar", "Narrow", "nar"}, + {"Nb", "Nobreak", "nb"}, + {"None", "none"}, + {"Sml", "Small", "sml"}, + {"Sqr", "Square", "sqr"}, + {"Sub", "sub"}, + {"Sup", "Super", "sup"}, + {"Vert", "Vertical", "vert"}, + {"Wide", "wide"}, + {"Non_Canon", "Non_Canonical"}, + }; + private static final int[] STARTS = { 0x0, 0xA0, 0xA1, 0xA8, 0xA9, 0xAA, 0xAB, 0xAF, 0xB0, 0xB2, 0xB4, 0xB6, 0xB8, 0xB9, 0xBB, 0xBC, 0xBF, 0xC0, 0xC6, 0xC7, @@ -344,6 +366,11 @@ static String canonicalValueName(byte value) { ? CANONICAL_NAMES[value] : null; } + static String[] valueAliases(byte value) { + return value >= 0 && value < VALUE_ALIASES.length + ? VALUE_ALIASES[value] : null; + } + static boolean matches(byte property, byte requested) { if (requested == NON_CANONICAL) { return property != CANONICAL && property != NONE; diff --git a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java index e9deedc8e..b1d44bf6e 100644 --- a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java +++ b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java @@ -1952,8 +1952,11 @@ private static UnicodeSet resolvePerlBuiltInPropertyAlias(String property) { } if (assignment > 0 && assignment < alias.length() - 1 && isGeneralCategoryProperty(alias.substring(0, assignment))) { - UnicodeSet category = PerlUnicodeGeneralCategoryData.resolve( - alias.substring(assignment + 1)); + String categoryValue = alias.substring(assignment + 1); + if (loosePropertyName(categoryValue).equals("l&")) { + categoryValue = "LC"; + } + UnicodeSet category = PerlUnicodeGeneralCategoryData.resolve(categoryValue); if (category == null) { throw new IllegalArgumentException( "Unsupported General_Category value: " @@ -2324,6 +2327,9 @@ private static PerlUnicodePropertyWildcard resolvePerlUnicodePropertyWildcard( if (PerlUnicodeIndicCategoryData.isPropertyAlias(name)) { return resolvePerlIndicCategoryWildcard(name, wildcard); } + if (PerlUnicodeDecompositionTypeData.isPropertyAlias(name)) { + return resolvePerlDecompositionTypeWildcard(name, wildcard); + } if (PerlUnicodeResidualPropertyData.isPropertyAlias(name)) { return resolvePerlResidualPropertyWildcard(name, wildcard); } @@ -2373,6 +2379,26 @@ private static PerlUnicodePropertyWildcard resolvePerlIndicCategoryWildcard( return new PerlUnicodePropertyWildcard(result.freeze(), null, false); } + private static PerlUnicodePropertyWildcard resolvePerlDecompositionTypeWildcard( + String propertyName, PerlPropertyValueMatcher wildcard) { + UnicodeSet result = new UnicodeSet(); + boolean matched = false; + for (byte value = PerlUnicodeDecompositionTypeData.CANONICAL; + value <= PerlUnicodeDecompositionTypeData.NON_CANONICAL; + value++) { + String[] aliases = PerlUnicodeDecompositionTypeData.valueAliases(value); + if (!matchesPerlUnicodePropertyWildcard(wildcard, aliases)) continue; + matched = true; + result.addAll(PERL_DECOMPOSITION_TYPE_SETS[value]); + } + if (!matched) { + throw new IllegalArgumentException( + "No Unicode property value wildcard matches " + + propertyName.trim()); + } + return new PerlUnicodePropertyWildcard(result.freeze(), null, false); + } + private static PerlUnicodePropertyWildcard resolvePerlResidualPropertyWildcard( String propertyName, PerlPropertyValueMatcher wildcard) { UnicodeSet result = new UnicodeSet(); @@ -3250,6 +3276,7 @@ private static void rejectForbiddenPerlUnicodePropertyWildcardSyntax( marker = modifier.end() - 1; } else { for (char candidate : new char[] {'g', 'a', 'u', 'd', 'l'}) { + if (candidate == 'a' && modifiers.equals("aa")) continue; if (modifiers.indexOf(candidate) >= 0) { prohibited = String.valueOf(candidate); if (candidate == 'g') marker = modifier.end() - 1; diff --git a/src/test/resources/unit/regex/unicode_remaining_30_closure.t b/src/test/resources/unit/regex/unicode_remaining_30_closure.t new file mode 100644 index 000000000..3e9db026c --- /dev/null +++ b/src/test/resources/unit/regex/unicode_remaining_30_closure.t @@ -0,0 +1,60 @@ +use strict; +use warnings; +use utf8; +use Test::More tests => 30; +no warnings qw(experimental::uniprop_wildcards regexp); + +sub check_case { + my ($pattern, $subject, $expected, $id) = @_; + my $compiled = eval "qr/$pattern/"; + my $error = $@; + my $matched = !$error && $subject =~ $compiled ? 1 : 0; + ok(!$error && $matched == $expected, "$id $pattern"); + diag($error) if $error; +} + +my $word_body = q{\b{wb}\x{20}\B{wb}\x{20}\B{wb}\x{20}\B{wb}\x{20}\B{wb}\x{20}\b{wb}\x{20}\B{wb}\x{308}\b{wb}}; +my @word_ids = qw(20 22 24 26 28); +my @word_modifiers = qw(a aa d u i); +for my $index (0 .. $#word_ids) { + check_case("(?$word_modifiers[$index]:$word_body)", + " \x{308}", 1, "uniprops01:$word_ids[$index]"); +} + +check_case(q{\p{gc=:(?aa)s:}}, " ", 1, 'uniprops01:40'); + +my @decomposition_patterns = ( + q{\p{Decomposition_Type=:\ANon_Canonical\z:}}, + q{\p{Decomposition_Type=:\Anoncanonical\z:}}, + q{\p{Dt=:\ANon_Canon\z:}}, + q{\p{Dt=:\Anoncanon\z:}}, +); +my @decomposition_ids = ( + [38395, 38396], [38405, 38406], [38417, 38418], [38427, 38428], +); +for my $index (0 .. $#decomposition_patterns) { + check_case($decomposition_patterns[$index], "\x{1fbf9}", 1, + "uniprops01:$decomposition_ids[$index][0]"); + check_case($decomposition_patterns[$index], "\x{1fbfa}", 0, + "uniprops01:$decomposition_ids[$index][1]"); +} + +my @category_forms = ( + [q{\p{Is_General_Category%s%s}}, 0], + [q{\p{^Is_General_Category%s%s}}, 1], + [q{\P{Is_General_Category%s%s}}, 1], + [q{\P{^Is_General_Category%s%s}}, 0], +); +my @category_subjects = ("\x{1e943}", "\x{1e944}"); +my @category_member = (1, 0); +my $category_id = 41547; +for my $assignment ([':', 'l&'], ['= ', 'L&']) { + for my $subject_index (0 .. $#category_subjects) { + for my $form (@category_forms) { + my $pattern = sprintf($form->[0], @$assignment); + my $expected = $category_member[$subject_index] ^ $form->[1]; + check_case($pattern, $category_subjects[$subject_index], $expected, + "uniprops01:" . $category_id++); + } + } +} From 8ac8c2109fe635546bc409121dd6474958e31b76 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 13:59:52 +0200 Subject: [PATCH 29/43] docs(regex): record current release gate Mark the completed same-source diagnostic family and record the exact six-commit PR 1087 child candidate, warning-free build, and complete serial pat.t and pat_thr.t acceptance counts. Generated with [OpenAI Codex](https://openai.com/codex/) Co-Authored-By: OpenAI Codex --- dev/design/phase36-regex-parity.md | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index 1536ddf86..468a82d5a 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -128,7 +128,7 @@ Exit: Unicode and native syntax corpora execute without compatibility masking. - [x] Native numeric, named-character, quantifier, group-name, extended-class, range, and selected POSIX diagnostics. - [x] `pat_re_eval.t` semantic contract. -- [ ] Finish all remaining same-source `reg_mesg.t` diagnostic families. +- [x] Finish all remaining same-source `reg_mesg.t` diagnostic families. - [x] Finish analyser warning-policy rows. - [ ] Finish remaining debug-trace rows. - [ ] Refresh complete `regexp.t`, `reg_mesg.t`, and runtime-source gates. @@ -201,14 +201,15 @@ unresolved PR-958 pass-count decrease. The user owns the release checkout; workers use private exact-head worktrees and must not mutate, rebase, or push the user's branch. -The four-commit release candidate at `9739104ac` has a clean warning-free full +The six-commit release candidate at `b587e9195` has a clean warning-free full `make`. In writable current-source state `op/do.t` is 71/71 on both backends; `class/accessor.t` executes all 30 rows and improves to 17 passes versus the PR-958 12-pass floor; isolated JAPH is deterministically 109/130 with ID 51 -matching current Perl's upstream regression. The remaining pre-evaluation gate -is a quiet exact-head `pat.t` and `pat_thr.t` run without external worker -builds; an initial timing attempt was discarded when an already-authorized -worker build overlapped it. +matching current Perl's upstream regression. Aggregate binding warnings and +fatal construction diagnostics cover the two assertions missing from the +first complete `pat*` localization. The clean serial exact-head gate executes +all 1302 rows in each file at 1249/1302 for both `pat.t` and `pat_thr.t`, with +zero errors, timeouts, or incomplete rows. Independent implementation continues on `integrate/phase36-post1087-wip`. Its current preserved tranches generate current-Perl InSC/InPC and Block data, @@ -259,9 +260,9 @@ Active ownership: ## Ordered Next Steps -1. Complete the quiet `pat.t`/`pat_thr.t` run on `9739104ac`, publish the four - green candidate commits for integration into PR 1087, and preserve the - exact source-hash/assertion-ID dispositions. +1. Publish the six green commits at `b587e9195` for integration into PR 1087 + with the preserved source-hash/assertion-ID dispositions and clean serial + 1249/1302 `pat.t`/`pat_thr.t` evidence. 2. After integration into PR 1087, run focused Unicode, `regexp.t`, `regex_sets.t`, `charset.t`, `pat.t`, `pat_thr.t`, `pat_advanced.t`, `pat_re_eval.t`, and `reg_mesg.t` gates, then repeat the From e0ffbe8a0c7a613f5d16c64c76b18f42198d7880 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 14:21:12 +0200 Subject: [PATCH 30/43] test(regex): bind runtime for named warning fixtures Qualify the nested named-character cache after its ownership moved into JoniRegexPattern, and bind the standard test PerlRuntime while resolving named range endpoints through lexical hint state. Generated with [OpenAI Codex](https://openai.com/codex/) Co-Authored-By: OpenAI Codex --- .../runtime/regex/JoniRegexWarningPositionTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/perlonjava/runtime/regex/JoniRegexWarningPositionTest.java b/src/test/java/org/perlonjava/runtime/regex/JoniRegexWarningPositionTest.java index d4fb6c5ff..f724a5d55 100644 --- a/src/test/java/org/perlonjava/runtime/regex/JoniRegexWarningPositionTest.java +++ b/src/test/java/org/perlonjava/runtime/regex/JoniRegexWarningPositionTest.java @@ -6,9 +6,10 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.perlonjava.PerlRuntimeTestBase; @Tag("unit") -class JoniRegexWarningPositionTest { +class JoniRegexWarningPositionTest extends PerlRuntimeTestBase { private static final RegexFlags FLAGS = RegexFlags.fromModifiers("", ""); @Test @@ -41,7 +42,7 @@ void canonicalizesNamedEndpointsInMixedUnicodeRangeWarnings() { }) { JoniRegexPattern pattern = new JoniRegexPattern( testCase[0], FLAGS, 0, false, false, false, - new NamedCharacterCache(), true); + new JoniRegexPattern.NamedCharacterCache(), true); assertEquals(List.of( "Both or neither range ends should be Unicode in regex; " + "marked by <-- HERE in m/" + testCase[1] + "/"), From 11547903955a328d1dfcba45279dfa86e5e6d10e Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 14:06:40 +0200 Subject: [PATCH 31/43] fix(strings): preserve Unicode flag for named characters Emit resolved \N{} string literals with Perl's UTF-8 flag even when their code points fit in a byte. Preserve the invariant across JVM and interpreter literal, stable-target, and reference paths, restoring Latin-1 and sharp-s fold and trie semantics. Generated with Codex (https://openai.com/codex) Co-Authored-By: Codex --- .../backend/bytecode/BytecodeCompiler.java | 11 +++-- .../perlonjava/backend/jvm/EmitLiteral.java | 8 ++-- .../frontend/astnode/StringNode.java | 11 +++++ .../frontend/parser/PrototypeArgs.java | 3 +- .../frontend/parser/StringParser.java | 3 +- .../frontend/parser/StringSegmentParser.java | 11 ++++- .../unit/string/named_character_utf8_flag.t | 48 +++++++++++++++++++ 7 files changed, 84 insertions(+), 11 deletions(-) create mode 100644 src/test/resources/unit/string/named_character_utf8_flag.t diff --git a/src/main/java/org/perlonjava/backend/bytecode/BytecodeCompiler.java b/src/main/java/org/perlonjava/backend/bytecode/BytecodeCompiler.java index ce15802aa..695f6d6a3 100644 --- a/src/main/java/org/perlonjava/backend/bytecode/BytecodeCompiler.java +++ b/src/main/java/org/perlonjava/backend/bytecode/BytecodeCompiler.java @@ -1634,6 +1634,8 @@ public void visit(StringNode node) { short opcode; if (node.isVString) { opcode = Opcodes.LOAD_VSTRING; + } else if (node.forceUnicodeString) { + opcode = Opcodes.LOAD_STRING; } else if (node.forceByteString) { opcode = Opcodes.LOAD_BYTE_STRING; } else if (isAsciiOnly(node.value)) { @@ -1696,9 +1698,9 @@ private static boolean isAsciiOnly(String value) { int compileStableRegexLiteral(StringNode node) { RuntimeScalar literal = new RuntimeScalar(node.value); boolean hasWideChars = node.value.codePoints().anyMatch(cp -> cp > 255); - if (node.forceByteString || isAsciiOnly(node.value) + if (!node.forceUnicodeString && (node.forceByteString || isAsciiOnly(node.value) || (!hasWideChars && emitterContext != null && emitterContext.symbolTable != null - && !emitterContext.symbolTable.isStrictOptionEnabled(Strict.HINT_UTF8))) { + && !emitterContext.symbolTable.isStrictOptionEnabled(Strict.HINT_UTF8)))) { literal.type = RuntimeScalarType.BYTE_STRING; } // Do not value-deduplicate this entry with ordinary read-only literals. @@ -5120,8 +5122,9 @@ void compileVariableReference(OperatorNode node, String op) { : RuntimeContextType.LIST; int valueReg; if (node.operand instanceof StringNode stringNode && !stringNode.isVString) { - boolean byteString = stringNode.forceByteString || isAsciiOnly(stringNode.value); - if (!byteString + boolean byteString = !stringNode.forceUnicodeString + && (stringNode.forceByteString || isAsciiOnly(stringNode.value)); + if (!stringNode.forceUnicodeString && !byteString && emitterContext != null && emitterContext.symbolTable != null && !emitterContext.symbolTable.isStrictOptionEnabled(Strict.HINT_UTF8)) { diff --git a/src/main/java/org/perlonjava/backend/jvm/EmitLiteral.java b/src/main/java/org/perlonjava/backend/jvm/EmitLiteral.java index 4a68a9c1d..a937016d4 100644 --- a/src/main/java/org/perlonjava/backend/jvm/EmitLiteral.java +++ b/src/main/java/org/perlonjava/backend/jvm/EmitLiteral.java @@ -265,7 +265,8 @@ public static void emitString(EmitterContext ctx, StringNode node) { } } - if (node.forceByteString || asciiOnly || !ctx.symbolTable.isStrictOptionEnabled(HINT_UTF8)) { + if (!node.forceUnicodeString && (node.forceByteString || asciiOnly + || !ctx.symbolTable.isStrictOptionEnabled(HINT_UTF8))) { // Under `no utf8` - create an octet string, unless it contains wide characters (> 255) // Wide characters (like \x{100}) force the string to be UTF-8 even without `use utf8` boolean hasWideChars = false; @@ -364,8 +365,9 @@ public static void emitStringForRefToLiteral(EmitterContext ctx, StringNode node return; } - if (node.forceByteString - || (!ctx.symbolTable.isStrictOptionEnabled(HINT_UTF8) && !ctx.compilerOptions.isUnicodeSource)) { + if (!node.forceUnicodeString && (node.forceByteString + || (!ctx.symbolTable.isStrictOptionEnabled(HINT_UTF8) + && !ctx.compilerOptions.isUnicodeSource))) { boolean hasWideChars = false; for (int i = 0; i < node.value.length(); i++) { if (node.value.charAt(i) > 255) { diff --git a/src/main/java/org/perlonjava/frontend/astnode/StringNode.java b/src/main/java/org/perlonjava/frontend/astnode/StringNode.java index cdb210b27..cbdf1b4fd 100644 --- a/src/main/java/org/perlonjava/frontend/astnode/StringNode.java +++ b/src/main/java/org/perlonjava/frontend/astnode/StringNode.java @@ -27,6 +27,9 @@ public class StringNode extends AbstractNode { */ public final boolean forceByteString; + /** Force Perl's UTF-8 flag even when the literal's code points fit in a byte. */ + public final boolean forceUnicodeString; + /** * Constructs a new StringNode with the specified string value. * @@ -37,6 +40,7 @@ public StringNode(String value, int tokenIndex) { this.tokenIndex = tokenIndex; this.isVString = false; this.forceByteString = false; + this.forceUnicodeString = false; } /** @@ -51,13 +55,20 @@ public StringNode(String value, boolean isVString, int tokenIndex) { this.tokenIndex = tokenIndex; this.isVString = isVString; this.forceByteString = false; + this.forceUnicodeString = false; } public StringNode(String value, boolean isVString, boolean forceByteString, int tokenIndex) { + this(value, isVString, forceByteString, false, tokenIndex); + } + + public StringNode(String value, boolean isVString, boolean forceByteString, + boolean forceUnicodeString, int tokenIndex) { this.value = value; this.tokenIndex = tokenIndex; this.isVString = isVString; this.forceByteString = forceByteString; + this.forceUnicodeString = forceUnicodeString; } /** diff --git a/src/main/java/org/perlonjava/frontend/parser/PrototypeArgs.java b/src/main/java/org/perlonjava/frontend/parser/PrototypeArgs.java index f55de855e..921e25f0c 100644 --- a/src/main/java/org/perlonjava/frontend/parser/PrototypeArgs.java +++ b/src/main/java/org/perlonjava/frontend/parser/PrototypeArgs.java @@ -85,7 +85,8 @@ private static Node qualifyTwoArgOpenDupMode(Parser parser, Node modeNode) { if (!handleName.contains("::") && isValidFilehandleName(handleName)) { String qualified = FileHandle.normalizeBarewordHandle(parser, handleName); return new StringNode(prefix + qualified, stringNode.isVString, - stringNode.forceByteString, stringNode.getIndex()); + stringNode.forceByteString, stringNode.forceUnicodeString, + stringNode.getIndex()); } break; } diff --git a/src/main/java/org/perlonjava/frontend/parser/StringParser.java b/src/main/java/org/perlonjava/frontend/parser/StringParser.java index d02efb699..416cc63ec 100644 --- a/src/main/java/org/perlonjava/frontend/parser/StringParser.java +++ b/src/main/java/org/perlonjava/frontend/parser/StringParser.java @@ -745,7 +745,8 @@ static void captureLexicalNamedCharacterTranslator( if (pattern instanceof StringNode string) { operand.elements.set(0, new StringNode( string.value + identity, string.isVString, - string.forceByteString, string.tokenIndex)); + string.forceByteString, string.forceUnicodeString, + string.tokenIndex)); } else { operand.elements.set(0, new BinaryOperatorNode(".", pattern, new StringNode(identity, sourceTokenIndex), sourceTokenIndex)); diff --git a/src/main/java/org/perlonjava/frontend/parser/StringSegmentParser.java b/src/main/java/org/perlonjava/frontend/parser/StringSegmentParser.java index db0bce340..7906effe2 100644 --- a/src/main/java/org/perlonjava/frontend/parser/StringSegmentParser.java +++ b/src/main/java/org/perlonjava/frontend/parser/StringSegmentParser.java @@ -79,6 +79,7 @@ public abstract class StringSegmentParser { */ protected final StringBuilder currentSegment; private boolean currentSegmentHasSourceNonAscii = false; + private boolean currentSegmentForcesUnicode = false; private boolean inRegexCharClass = false; private boolean regexCharClassFirst = false; private boolean regexCharClassEscape = false; @@ -275,10 +276,13 @@ protected void flushCurrentSegment() { } value = octets.toString(); } - boolean forceByteString = shouldForceByteStringLiteral(value); - addStringSegment(new StringNode(value, false, forceByteString, tokenIndex)); + boolean forceByteString = !currentSegmentForcesUnicode + && shouldForceByteStringLiteral(value); + addStringSegment(new StringNode(value, false, forceByteString, + currentSegmentForcesUnicode, tokenIndex)); currentSegment.setLength(0); currentSegmentHasSourceNonAscii = false; + currentSegmentForcesUnicode = false; } } @@ -1698,6 +1702,9 @@ void handleUnicodeNameEscape() { NamedCharacterExpansion expansion = NamedCharacterExpansion.resolve(name, sourceMode); if (expansion.resolved()) { + // Perl marks the complete literal containing a resolved \N{} + // as UTF-8, including when every code point fits in a byte. + currentSegmentForcesUnicode = true; appendToCurrentSegment(expansion.sequence()); } else { throwNamedCharacterDiagnostic(expansion.diagnostic()); diff --git a/src/test/resources/unit/string/named_character_utf8_flag.t b/src/test/resources/unit/string/named_character_utf8_flag.t new file mode 100644 index 000000000..6b9515d24 --- /dev/null +++ b/src/test/resources/unit/string/named_character_utf8_flag.t @@ -0,0 +1,48 @@ +use strict; +use warnings; +use charnames ':full'; +use Test::More tests => 21; + +my $lower = "\N{LATIN SMALL LETTER A WITH GRAVE}"; +my $upper = "\N{LATIN CAPITAL LETTER A WITH GRAVE}"; + +ok(utf8::is_utf8($lower), 'named lower is Unicode-upgraded'); +ok(utf8::is_utf8($upper), 'named upper is Unicode-upgraded'); +is(ord($lower), 0xE0, 'named lower retains its code point'); +is(ord($upper), 0xC0, 'named upper retains its code point'); + +ok($lower =~ qr/$upper/i, 'interpolated named upper matches lower'); +ok($upper =~ qr/$lower/i, 'interpolated named lower matches upper'); +ok($lower =~ qr/[$upper]/i, 'interpolated named upper class matches lower'); +ok($upper =~ qr/[$lower]/i, 'interpolated named lower class matches upper'); + +ok($lower =~ qr/\N{LATIN CAPITAL LETTER A WITH GRAVE}/i, + 'literal named upper matches lower'); +ok($upper =~ qr/\N{LATIN SMALL LETTER A WITH GRAVE}/i, + 'literal named lower matches upper'); + +my $ascii = "\N{LATIN CAPITAL LETTER A}"; +ok(utf8::is_utf8($ascii), 'ASCII named character is Unicode-upgraded'); +is(ord($ascii), 0x41, 'ASCII named character retains its code point'); + +my $literal_ref = \("\N{LATIN CAPITAL LETTER A WITH GRAVE}"); +ok(utf8::is_utf8($$literal_ref), 'reference to named literal stays Unicode-upgraded'); +is(ord($$literal_ref), 0xC0, 'referenced named literal retains its code point'); + +my $sharp_s = "\N{LATIN SMALL LETTER SHARP S}"; +ok($sharp_s =~ qr/ss/i, 'ASCII lower pair folds to named sharp s subject'); +ok($sharp_s =~ qr/SS/i, 'ASCII upper pair folds to named sharp s subject'); + +ok("foba ba$sharp_s" =~ qr/(foo|Bass|bar)/i && $1 eq "ba$sharp_s", + 'trie ASCII lower pair folds to named sharp s subject'); +ok("foba ba$sharp_s" =~ qr/(foo|BaSS|bar)/i && $1 eq "ba$sharp_s", + 'trie ASCII upper pair folds to named sharp s subject'); +ok("foba ba${sharp_s}pxySS$sharp_s$sharp_s" + =~ qr/(b(?:a${sharp_s}t|a${sharp_s}f|a${sharp_s}p)[xy]+$sharp_s*)/i + && $1 eq "ba${sharp_s}pxySS$sharp_s$sharp_s", + 'common-prefix trie preserves named sharp s folds'); + +my $mixed = "\xFF\N{LATIN CAPITAL LETTER A WITH GRAVE}"; +ok(utf8::is_utf8($mixed), 'named escape upgrades an adjacent byte escape'); +is(join(',', map { ord } split //, $mixed), '255,192', + 'adjacent byte and named escapes retain their code points'); From 380799fbc6e437282ac6793884972fdaeecad883 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 14:08:31 +0200 Subject: [PATCH 32/43] fix: expose semantic regex debug facts Classify full, empty, and newline-complement compiled wide classes without parsing debug text, and preserve materialized overload provenance when selecting native match variants and converting match offsets. Generated with [OpenAI Codex](https://openai.com/codex/) Co-Authored-By: OpenAI Codex --- .../runtime/regex/RuntimeRegex.java | 64 +++++--- .../unit/regex/overloaded_subject_utf8.t | 46 ++++++ third_party/joni/src/org/joni/Regex.java | 31 ++++ .../joni/src/org/joni/RegexDebugProgram.java | 67 ++++++++ .../joni/src/org/joni/ast/CClassNode.java | 64 ++++++++ .../test/TestRegexSemanticDebugFacts.java | 155 ++++++++++++++++++ 6 files changed, 401 insertions(+), 26 deletions(-) create mode 100644 src/test/resources/unit/regex/overloaded_subject_utf8.t create mode 100644 third_party/joni/src/org/joni/RegexDebugProgram.java create mode 100644 third_party/joni/test/org/joni/test/TestRegexSemanticDebugFacts.java diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java index 5252dc38b..271695f81 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java @@ -2416,13 +2416,6 @@ private static RuntimeBase matchRegexDirect(RuntimeScalar quotedRegex, RuntimeSc regex.emitWarningsOnUse(); - // Debug logging - if (DEBUG_REGEX) { - String description = regex.recursivePattern.patternDescription(); - System.err.println("matchRegexDirect: pattern=" + description + - " input=" + string.toString() + " ctx=" + ctx); - } - if (regex.regexFlags.isMatchExactlyOnce() && regex.matched) { // m?PAT? already matched once; now return false if (ctx == RuntimeContextType.LIST) { @@ -2434,10 +2427,20 @@ private static RuntimeBase matchRegexDirect(RuntimeScalar quotedRegex, RuntimeSc } } - String inputStr = string.toString(); - regex.emitNonUnicodePropertyWarning(string, inputStr); + // Stringify an overloaded subject once and retain that scalar's byte/UTF-8 + // provenance for native variant selection. Match state and callbacks stay + // attached to the original subject scalar, as Perl's pos()/taint identity does. + RuntimeScalar inputValue = RuntimeScalarType.blessedId(string) != 0 + ? Overload.stringify(string) : string; + String inputStr = inputValue.toString(); + if (DEBUG_REGEX) { + String description = regex.recursivePattern.patternDescription(); + System.err.println("matchRegexDirect: pattern=" + description + + " input=" + inputStr + " ctx=" + ctx); + } + regex.emitNonUnicodePropertyWarning(inputValue, inputStr); regex.emitExecutionDebugTrace(inputStr); - RegexMatcher matcher = regex.selectRecursivePattern(string) + RegexMatcher matcher = regex.selectRecursivePattern(inputValue) .matcher(inputStr, regex.executableCallbacks, string); // hexPrinter(inputStr); @@ -2456,15 +2459,18 @@ private static RuntimeBase matchRegexDirect(RuntimeScalar quotedRegex, RuntimeSc posScalar = RuntimePosLvalue.pos(string); isPosDefined = posScalar.getDefinedBoolean(); startPos = isPosDefined - ? RuntimePosLvalue.toMatcherOffset(string, inputStr, posScalar.getInt()) + ? RuntimePosLvalue.toMatcherOffset( + inputValue, inputStr, posScalar.getInt()) : 0; - RuntimeBase fastXmlElementResult = tryFastXmlElementScan(regex, string, inputStr, startPos, posScalar, ctx); + RuntimeBase fastXmlElementResult = tryFastXmlElementScan( + regex, string, inputValue, inputStr, startPos, posScalar, ctx); if (fastXmlElementResult != null) { return fastXmlElementResult; } - RuntimeBase fastXmpResult = tryFastXmpMetaElementScan(regex, string, inputStr, startPos, posScalar, ctx); + RuntimeBase fastXmpResult = tryFastXmpMetaElementScan( + regex, string, inputValue, inputStr, startPos, posScalar, ctx); if (fastXmpResult != null) { return fastXmpResult; } @@ -2481,7 +2487,7 @@ private static RuntimeBase matchRegexDirect(RuntimeScalar quotedRegex, RuntimeSc string, posScalar.getInt(), patternKey)) { // First, try the notempty variant at the SAME position (Perl behavior) RegexMatcher notemptyMatcher = findNonEmptyGlobalRetry( - regex, string, inputStr, startPos); + regex, inputValue, string, inputStr, startPos); boolean notemptySucceeded = notemptyMatcher != null; if (notemptySucceeded) { matcher = notemptyMatcher; @@ -2500,7 +2506,7 @@ private static RuntimeBase matchRegexDirect(RuntimeScalar quotedRegex, RuntimeSc return RuntimeScalarCache.scalarFalse; } posScalar.set(RuntimePosLvalue.fromMatcherOffset( - string, inputStr, startPos)); + inputValue, inputStr, startPos)); RuntimePosLvalue.recordNonZeroLengthMatch(string); isPosDefined = true; } @@ -2558,7 +2564,8 @@ private static RuntimeBase matchRegexDirect(RuntimeScalar quotedRegex, RuntimeSc regexState.lastMatchResultsTainted = GlobalContext.isTaintModeActive() && (quotedRegex.isTainted() || (regex.regexFlags.taintResults() && string.isTainted())); - regexState.lastMatchWasByteString = (string.type == RuntimeScalarType.BYTE_STRING); + regexState.lastMatchWasByteString = + inputValue.type == RuntimeScalarType.BYTE_STRING; int captureCount = matcher.groupCount(); // Always initialize $1, $2, @+, @-, $`, $&, $' for every successful match @@ -2614,7 +2621,7 @@ private static RuntimeBase matchRegexDirect(RuntimeScalar quotedRegex, RuntimeSc // (only for global matches - posScalar is null for non-global) if (posScalar != null) { int perlMatchEnd = RuntimePosLvalue.fromMatcherOffset( - string, inputStr, matchEnd); + inputValue, inputStr, matchEnd); posScalar.set(perlMatchEnd); // Record zero-length match for cross-call tracking if (matchEnd == matchStart) { @@ -2630,7 +2637,7 @@ private static RuntimeBase matchRegexDirect(RuntimeScalar quotedRegex, RuntimeSc if (nativeGlobalPosition) matcher.setGlobalPosition(startPos); if (posScalar != null) { posScalar.set(RuntimePosLvalue.fromMatcherOffset( - string, inputStr, startPos)); + inputValue, inputStr, startPos)); } // Only redirect the matcher when we forcibly advanced past // a zero-length match. In every other case Java's find() @@ -2645,7 +2652,7 @@ private static RuntimeBase matchRegexDirect(RuntimeScalar quotedRegex, RuntimeSc } if (zeroLengthMatch) { RegexMatcher notemptyMatcher = findNonEmptyGlobalRetry( - regex, string, inputStr, startPos); + regex, inputValue, string, inputStr, startPos); if (notemptyMatcher != null) { matcher = notemptyMatcher; skipFirstFind = true; @@ -2748,6 +2755,7 @@ private static RuntimeBase matchRegexDirect(RuntimeScalar quotedRegex, RuntimeSc */ private static RuntimeBase tryFastXmpMetaElementScan(RuntimeRegex regex, RuntimeScalar string, + RuntimeScalar inputValue, String inputStr, int startPos, RuntimeScalar posScalar, @@ -2799,7 +2807,8 @@ private static RuntimeBase tryFastXmpMetaElementScan(RuntimeRegex regex, String group3 = inputStr.substring(suffixEnd, tagEnd); String group4 = group3.endsWith("/") ? "/" : ""; - state().lastMatchWasByteString = (string.type == RuntimeScalarType.BYTE_STRING); + state().lastMatchWasByteString = + inputValue.type == RuntimeScalarType.BYTE_STRING; state().globalMatcher = null; state().globalMatchString = inputStr; state().lastMatchUsedBackslashK = false; @@ -2832,7 +2841,7 @@ private static RuntimeBase tryFastXmpMetaElementScan(RuntimeRegex regex, if (regex.regexFlags.isGlobalMatch() && posScalar != null) { posScalar.set(RuntimePosLvalue.fromMatcherOffset( - string, inputStr, state().lastMatchEnd)); + inputValue, inputStr, state().lastMatchEnd)); RuntimePosLvalue.recordNonZeroLengthMatch(string); } @@ -2850,6 +2859,7 @@ private static boolean isXmpMetaElementScanPattern(RuntimeRegex regex) { */ private static RuntimeBase tryFastXmlElementScan(RuntimeRegex regex, RuntimeScalar string, + RuntimeScalar inputValue, String inputStr, int startPos, RuntimeScalar posScalar, @@ -2879,7 +2889,8 @@ private static RuntimeBase tryFastXmlElementScan(RuntimeRegex regex, String group2 = inputStr.substring(match.group2Start, match.group2End); String group3 = inputStr.substring(match.group3Start, match.group3End); - state().lastMatchWasByteString = (string.type == RuntimeScalarType.BYTE_STRING); + state().lastMatchWasByteString = + inputValue.type == RuntimeScalarType.BYTE_STRING; state().globalMatcher = null; state().globalMatchString = inputStr; state().lastMatchUsedBackslashK = false; @@ -2902,7 +2913,7 @@ private static RuntimeBase tryFastXmlElementScan(RuntimeRegex regex, if (regex.regexFlags.isGlobalMatch() && posScalar != null) { posScalar.set(RuntimePosLvalue.fromMatcherOffset( - string, inputStr, state().lastMatchEnd)); + inputValue, inputStr, state().lastMatchEnd)); RuntimePosLvalue.recordNonZeroLengthMatch(string); } @@ -3104,11 +3115,12 @@ private static int bumpGlobalMatchPosition(String inputStr, int offset) { } private static RegexMatcher findNonEmptyGlobalRetry(RuntimeRegex regex, - RuntimeScalar string, + RuntimeScalar inputValue, + RuntimeScalar subject, String inputStr, int startPos) { - RegexMatcher retryMatcher = regex.selectRecursivePattern(string) - .matcher(inputStr, regex.executableCallbacks, string); + RegexMatcher retryMatcher = regex.selectRecursivePattern(inputValue) + .matcher(inputStr, regex.executableCallbacks, subject); retryMatcher.region(startPos, inputStr.length()); retryMatcher.useAnchoringBounds(false); diff --git a/src/test/resources/unit/regex/overloaded_subject_utf8.t b/src/test/resources/unit/regex/overloaded_subject_utf8.t new file mode 100644 index 000000000..40f16f295 --- /dev/null +++ b/src/test/resources/unit/regex/overloaded_subject_utf8.t @@ -0,0 +1,46 @@ +use strict; +use warnings; +use Test::More; + +{ + package Phase36::WideString; + use overload q{""} => sub { + ++$main::stringifications; + return ${$_[0]}; + }; + + sub new { + my ($class, $value) = @_; + return bless \$value, $class; + } +} + +our $stringifications = 0; +my $list_subject = Phase36::WideString->new("a\x{100}/\x{100}b"); +my @captures = ($list_subject =~ /\b(.)\x{100}/g); +is_deeply(\@captures, ['a', '/'], + 'global matching selects Unicode engine from overloaded string value'); +is($stringifications, 1, 'list global match stringifies its subject once'); + +$stringifications = 0; +my $scalar_subject = Phase36::WideString->new("a\x{100}/\x{100}b"); +ok($scalar_subject =~ /\b(.)\x{100}/g, + 'scalar global match uses overloaded Unicode value'); +is($1, 'a', 'scalar global match publishes its capture'); +is(pos($scalar_subject), 2, + 'global position remains attached to the original overloaded scalar'); +is($stringifications, 1, 'scalar global match stringifies its subject once'); + +$stringifications = 0; +my $byte_subject = Phase36::WideString->new(pack('C*', 0xC3, 0xA9, 0x58)); +pos($byte_subject) = 1; +$stringifications = 0; +ok($byte_subject =~ /(.)/g, + 'byte-returning overload selects the byte matcher at an existing position'); +is(unpack('C', $1), 0xA9, + 'byte-returning overload publishes one byte rather than one Unicode scalar'); +is(pos($byte_subject), 2, + 'byte offset is converted while pos stays on the original overloaded scalar'); +is($stringifications, 1, 'byte-returning overload is stringified once'); + +done_testing; diff --git a/third_party/joni/src/org/joni/Regex.java b/third_party/joni/src/org/joni/Regex.java index 6511f3848..c20e868d6 100644 --- a/third_party/joni/src/org/joni/Regex.java +++ b/third_party/joni/src/org/joni/Regex.java @@ -625,6 +625,37 @@ public String byteCodeDebugDescription() { return new ByteCodePrinter(this).byteCodeListToString(); } + public enum DebugProgramKind { + FULL_CLASS, + EMPTY_CLASS, + ALL_EXCEPT_NEWLINE_CLASS, + OTHER + } + + public record DebugProgramFact(DebugProgramKind kind) { + static DebugProgramFact other() { + return new DebugProgramFact(DebugProgramKind.OTHER); + } + } + + /** + * Returns a semantic fact when the program begins with a class instruction, + * optionally after one canonical dynamic-option prologue. + */ + public DebugProgramFact firstDebugProgramFact() { + return RegexDebugProgram.firstFact(this); + } + + /** Renders the currently supported Perl program-shape name, if any. */ + public String perlFirstProgramDebugDescription() { + return switch (firstDebugProgramFact().kind()) { + case FULL_CLASS -> "SANY"; + case EMPTY_CLASS -> "OPFAIL"; + case ALL_EXCEPT_NEWLINE_CLASS -> "REG_ANY"; + case OTHER -> ""; + }; + } + public void setUserOptions(int options) { this.userOptions = options; } diff --git a/third_party/joni/src/org/joni/RegexDebugProgram.java b/third_party/joni/src/org/joni/RegexDebugProgram.java new file mode 100644 index 000000000..4d746c3e5 --- /dev/null +++ b/third_party/joni/src/org/joni/RegexDebugProgram.java @@ -0,0 +1,67 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import org.joni.ast.CClassNode.DebugDomainShape; +import org.joni.constants.internal.OPCode; +import org.joni.constants.internal.OPSize; + +final class RegexDebugProgram { + private RegexDebugProgram() { + } + + static Regex.DebugProgramFact firstFact(Regex regex) { + int cursor = skipInitialDynamicOptionWrapper(regex.code, regex.codeLength); + if (cursor < 0 || cursor >= regex.codeLength + || regex.code[cursor] != OPCode.WIDE_SCALAR_CLASS + || cursor + OPSize.WIDE_SCALAR_CLASS > regex.codeLength) { + return Regex.DebugProgramFact.other(); + } + + int classIndex = regex.code[cursor + 1]; + if (regex.wideScalarClasses == null || classIndex < 0 + || classIndex >= regex.wideScalarClasses.length) { + return Regex.DebugProgramFact.other(); + } + + DebugDomainShape shape = regex.wideScalarClasses[classIndex] + .debugDomainShape(regex.enc); + return switch (shape) { + case FULL -> new Regex.DebugProgramFact( + Regex.DebugProgramKind.FULL_CLASS); + case EMPTY -> new Regex.DebugProgramFact( + Regex.DebugProgramKind.EMPTY_CLASS); + case ALL_EXCEPT_NEWLINE -> new Regex.DebugProgramFact( + Regex.DebugProgramKind.ALL_EXCEPT_NEWLINE_CLASS); + case OTHER -> Regex.DebugProgramFact.other(); + }; + } + + private static int skipInitialDynamicOptionWrapper(int[] code, int length) { + if (code == null || length == 0 || code[0] != OPCode.SET_OPTION_PUSH) { + return 0; + } + int cursor = OPSize.SET_OPTION_PUSH; + if (cursor >= length || code[cursor] != OPCode.SET_OPTION) return -1; + cursor += OPSize.SET_OPTION; + if (cursor >= length || code[cursor] != OPCode.FAIL) return -1; + return cursor + OPSize.FAIL; + } +} diff --git a/third_party/joni/src/org/joni/ast/CClassNode.java b/third_party/joni/src/org/joni/ast/CClassNode.java index 42578990e..f9ef4d94c 100644 --- a/third_party/joni/src/org/joni/ast/CClassNode.java +++ b/third_party/joni/src/org/joni/ast/CClassNode.java @@ -32,6 +32,13 @@ import org.joni.exception.ValueException; public final class CClassNode extends Node { + public enum DebugDomainShape { + EMPTY, + FULL, + ALL_EXCEPT_NEWLINE, + OTHER + } + private static final int FLAG_NCCLASS_NOT = 1 << 0; private static final long FIRST_WIDE_SCALAR = 0x110000L; @@ -342,6 +349,63 @@ public boolean isScalarInCC(Encoding enc, long value) { return isWideScalarInCC(value); } + /** Classifies this class over the Unicode and signed-IV scalar domains. */ + public DebugDomainShape debugDomainShape(Encoding enc) { + boolean allLow = true; + boolean noLow = true; + boolean allLowExceptNewline = true; + for (int code = 0; code < BitSet.SINGLE_BYTE_SIZE; code++) { + boolean member = isCodeInCC(enc, code); + allLow &= member; + noLow &= !member; + allLowExceptNewline &= code == '\n' ? !member : member; + } + + boolean rawHighEmpty = !rawRangesIntersect(0x100, 0x10ffff); + boolean rawHighFull = rawRangesCover(0x100, 0x10ffff); + boolean highFull = isNot() ? rawHighEmpty : rawHighFull; + boolean highEmpty = isNot() ? rawHighFull : rawHighEmpty; + boolean rawWideEmpty = wideRangeCount == 0; + boolean rawWideFull = wideRangeCount == 1 + && wideRanges[0] == FIRST_WIDE_SCALAR + && wideRanges[1] == Long.MAX_VALUE; + boolean wideFull = isNot() ? rawWideEmpty : rawWideFull; + boolean wideEmpty = isNot() ? rawWideFull : rawWideEmpty; + + if (noLow && highEmpty && wideEmpty) return DebugDomainShape.EMPTY; + if (allLow && highFull && wideFull) return DebugDomainShape.FULL; + if (allLowExceptNewline && highFull && wideFull) { + return DebugDomainShape.ALL_EXCEPT_NEWLINE; + } + return DebugDomainShape.OTHER; + } + + private boolean rawRangesIntersect(int from, int to) { + if (mbuf == null) return false; + int[] ranges = mbuf.getCodeRange(); + for (int i = 0; i < ranges[0]; i++) { + int rangeFrom = ranges[i * 2 + 1]; + int rangeTo = ranges[i * 2 + 2]; + if (rangeTo >= from && rangeFrom <= to) return true; + if (rangeFrom > to) return false; + } + return false; + } + + private boolean rawRangesCover(int from, int to) { + if (mbuf == null) return false; + int[] ranges = mbuf.getCodeRange(); + long next = from; + for (int i = 0; i < ranges[0] && next <= to; i++) { + int rangeFrom = ranges[i * 2 + 1]; + int rangeTo = ranges[i * 2 + 2]; + if (rangeTo < next) continue; + if (rangeFrom > next) return false; + next = (long)rangeTo + 1; + } + return next > to; + } + private boolean containsWideScalar(long value) { int low = 0; int high = wideRangeCount - 1; diff --git a/third_party/joni/test/org/joni/test/TestRegexSemanticDebugFacts.java b/third_party/joni/test/org/joni/test/TestRegexSemanticDebugFacts.java new file mode 100644 index 000000000..96c2049b4 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestRegexSemanticDebugFacts.java @@ -0,0 +1,155 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.nio.charset.StandardCharsets; +import java.util.Locale; + +import org.jcodings.Encoding; +import org.jcodings.specific.UTF8Encoding; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.joni.WideScalarCodec; +import org.junit.Test; + +public class TestRegexSemanticDebugFacts { + private static final WideScalarCodec CODEC = new WideScalarCodec() { + @Override + public byte[] encode(long value, Encoding encoding) { + return marker(value); + } + + @Override + public Decoded decode(byte[] bytes, int p, int end, Encoding encoding) { + if (p + 4 > end || bytes[p] != '~' || bytes[p + 1] != '<') return null; + long value = 0; + int cursor = p + 2; + int digits = 0; + while (cursor < end && bytes[cursor] != '>') { + int digit = Character.digit((char)(bytes[cursor] & 0xff), 16); + if (digit < 0 || digits == 16 + || value > (Long.MAX_VALUE - digit) / 16) return null; + value = value * 16 + digit; + cursor++; + digits++; + } + return digits == 0 || cursor >= end + ? null : new Decoded(value, cursor + 1); + } + }; + + private static final Syntax SYNTAX = new Syntax( + "RegexSemanticDebugFacts", Syntax.PerlNG.op, Syntax.PerlNG.op2, + Syntax.PerlNG.op3, Syntax.PerlNG.behavior, Syntax.PerlNG.options, + Syntax.PerlNG.metaCharTable, null, null, CODEC); + + @Test + public void classifiesSemanticUnicodeProgramShapesWithoutMutatingMatching() { + Regex full = compile("[\\x{0}-\\x{7FFF_FFFF_FFFF_FFFF}]"); + assertStable(full, "A", true, "SANY", + Regex.DebugProgramKind.FULL_CLASS); + assertStable(full, "\n", true, "SANY", + Regex.DebugProgramKind.FULL_CLASS); + assertStable(full, "Ā", true, "SANY", + Regex.DebugProgramKind.FULL_CLASS); + assertStable(full, new String(Character.toChars(0x10ffff)), true, + "SANY", Regex.DebugProgramKind.FULL_CLASS); + assertStable(full, marker(0x110000L), true, "SANY", + Regex.DebugProgramKind.FULL_CLASS); + + Regex empty = compile("[^\\x{0}-\\x{7FFF_FFFF_FFFF_FFFF}]"); + assertStable(empty, "A", false, "OPFAIL", + Regex.DebugProgramKind.EMPTY_CLASS); + + Regex exceptNewline = compile("[^\\n]"); + assertStable(exceptNewline, "A", true, "REG_ANY", + Regex.DebugProgramKind.ALL_EXCEPT_NEWLINE_CLASS); + assertStable(exceptNewline, "\n", false, "REG_ANY", + Regex.DebugProgramKind.ALL_EXCEPT_NEWLINE_CLASS); + assertStable(exceptNewline, marker(0x110000L), true, "REG_ANY", + Regex.DebugProgramKind.ALL_EXCEPT_NEWLINE_CLASS); + + Regex ordinary = compile("[ab]"); + assertStable(ordinary, "a", true, "", Regex.DebugProgramKind.OTHER); + assertStable(ordinary, "c", false, "", Regex.DebugProgramKind.OTHER); + + assertStable(compile("[\\x{0}-\\x{10ffff}]"), marker(0x110000L), + false, "", Regex.DebugProgramKind.OTHER); + assertStable(compile("[^\\x{0}-\\x{10ffff}]"), marker(0x110000L), + true, "", Regex.DebugProgramKind.OTHER); + assertStable(compile("[^\\n\\x{110000}]"), marker(0x110000L), + false, "", Regex.DebugProgramKind.OTHER); + assertStable(compile("[^\\x{100}]"), "Ā", false, "", + Regex.DebugProgramKind.OTHER); + assertStable(compile("[\\x{100}]"), "Ā", true, "", + Regex.DebugProgramKind.OTHER); + assertStable(compile("[[:alpha:][:^alpha:]]"), marker(0x110000L), + false, "", Regex.DebugProgramKind.OTHER); + } + + @Test + public void crossesTheStaticOptionInstructionWrapperOnlyAtProgramStart() { + Regex wrapped = compile( + "(?aa:[\\x{0}-\\x{7FFF_FFFF_FFFF_FFFF}])"); + assertStable(wrapped, "A", true, "SANY", + Regex.DebugProgramKind.FULL_CLASS); + } + + private static Regex compile(String pattern) { + byte[] bytes = pattern.getBytes(StandardCharsets.UTF_8); + return new Regex(bytes, 0, bytes.length, Option.CAPTURE_GROUP, + UTF8Encoding.INSTANCE, SYNTAX); + } + + private static void assertStable(Regex regex, String input, + boolean expectedMatch, String expectedDescription, + Regex.DebugProgramKind expectedKind) { + assertStable(regex, input.getBytes(StandardCharsets.UTF_8), expectedMatch, + expectedDescription, expectedKind); + } + + private static void assertStable(Regex regex, byte[] bytes, + boolean expectedMatch, String expectedDescription, + Regex.DebugProgramKind expectedKind) { + assertEquals(expectedMatch, matches(regex, bytes)); + assertEquals(regex.byteCodeDebugDescription(), expectedKind, + regex.firstDebugProgramFact().kind()); + assertEquals(expectedDescription, regex.perlFirstProgramDebugDescription()); + if (expectedMatch) { + assertTrue(matches(regex, bytes)); + } else { + assertFalse(matches(regex, bytes)); + } + } + + private static boolean matches(Regex regex, byte[] input) { + return regex.matcher(input).search(0, input.length, Option.NONE) >= 0; + } + + private static byte[] marker(long value) { + return ("~<" + Long.toHexString(value).toUpperCase(Locale.ROOT) + ">") + .getBytes(StandardCharsets.US_ASCII); + } +} From 8853677a0f43605438741a833dba8d7ac3a2cec1 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 14:26:39 +0200 Subject: [PATCH 33/43] docs(regex): promote unified release and live ownership Make PR 1091 the immutable user-acceptance target and record the current post-release integration head, remaining pat_advanced rows, active worker ownership, and ordered merge/finalization sequence. Generated with [OpenAI Codex](https://openai.com/codex/) Co-Authored-By: OpenAI Codex --- dev/design/phase36-regex-parity.md | 120 ++++++++++------------------- 1 file changed, 42 insertions(+), 78 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index 468a82d5a..ca052ef96 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -171,59 +171,34 @@ Exit: release evidence and public/internal documentation match the code. ## Current Release Gate -PR 1087 is the next incremental release checkpoint at exact refreshed head -`5a1e0c15e`, after a fresh pull of the latest `perl5/` source and a complete -`dev/import-perl5/sync.pl` run. Those imported fixtures are authoritative and -must not be reverted or patched to recover old counts. Its combined build and -exact-head Ubuntu/Windows CI are green. The -fresh complete corpus gate is red and blocks merge. Its two `pat*` rows timed -out under resource contention: isolated single-job probes execute all 1,302 -assertions with 1,249 passes each, but equivalent exact-head quiet reruns remain -required. Their isolated 399/446-second runtimes versus roughly 250 seconds at -PR958 are also a post-checkpoint optimization root: profile both revisions with -JFR and remove the measured regression rather than masking it with a larger -timeout. `op/do.t` needs two fixes to reach standard Perl's 71/71 plus an -ID-normalized disposition because PR958 over-executed it as 94/99. The current -candidate reaches 71/71 on both backends. The class accessor-name candidate is -validated by current Perl 5.45.3 at 8/8 focused and 30/30 authoritative. -`japh/abigail.t` has a required upstream-regression disposition: private -per-run isolation removes its filesystem race, and an exact assertion map -isolates the sole PR-958 delta at ID 51. Upstream commit -`5355a72c746fa5146541d790aece53b22737afee` changed `_quote_args` to add a -nested double/single wrapper around whitespace-bearing arguments. The exact -latest Perl 5.45.3 checkout also fails ID 51 because its qx shell path expands -`$_` and retains the apostrophes. PerlOnJava must preserve that current-Perl -behavior rather than add an over-parity argv special case. IDs 127–130 are -unchanged and also fail on current Perl. -Acceptance must have no new -invalid, missing, timeout, truncated, incomplete, or zero-TAP row and no -unresolved PR-958 pass-count decrease. The user owns the release checkout; -workers use private exact-head worktrees and must not mutate, rebase, or push -the user's branch. - -The six-commit release candidate at `b587e9195` has a clean warning-free full -`make`. In writable current-source state `op/do.t` is 71/71 on both backends; +PR 1091 is the sole release and user-acceptance target at exact head +`ff68371af`. It contains all 332 range-diff-equivalent PR 1087 patches rebased +onto current `master`, preserves the user's latest-Perl sync commit, and includes +the six release fixes formerly staged in PR 1090. Imported fixtures remain +authoritative and must not be reverted or patched to recover old counts. The +exact unified head has a clean warning-free full `make`; Ubuntu and Windows CI +and the user's complete acceptance are in progress. Workers must not mutate, +rebase, or push this immutable test branch. Acceptance rejects every new +invalid, missing, timeout, truncated, incomplete, or zero-TAP row and every +unresolved PR-958 pass-count decrease. + +In writable current-source state `op/do.t` is 71/71 on both backends; `class/accessor.t` executes all 30 rows and improves to 17 passes versus the PR-958 12-pass floor; isolated JAPH is deterministically 109/130 with ID 51 matching current Perl's upstream regression. Aggregate binding warnings and fatal construction diagnostics cover the two assertions missing from the -first complete `pat*` localization. The clean serial exact-head gate executes +initial `pat*` localization. The clean serial exact-head gate executes all 1302 rows in each file at 1249/1302 for both `pat.t` and `pat_thr.t`, with zero errors, timeouts, or incomplete rows. -Independent implementation continues on `integrate/phase36-post1087-wip`. -Its current preserved tranches generate current-Perl InSC/InPC and Block data, -preserve warning/fatal/recursion behavior, and establish a native Joni debug -trace foundation. The complete Unicode ledger covers 167,506 assertions: after -the integrated warning and Block tranches plus pending generated-XID data, -exactly 30 `uniprops01.t` rows remain across word-boundary, `/a` wildcard, -Decomposition_Type, and Cased_Letter families. It will be rebased only after -the refreshed checkpoint merges. - -Repair the five blocking rows with targeted, baseline-equivalent reruns before -repeating the complete corpus. Once targeted evidence is green, rerun the -complete latest-Perl corpus, strict PR-958 comparison, and additional release -checks at the same immutable head. +Independent implementation continues on `integrate/phase36-post1087-wip` at +`e136c9b00` or later. Integrated Unicode property closure is complete. P4's +named-character source-mode fix removes all nine owned `pat_advanced.t` fold/ +trie rows, and P3's single-stringification repair closes overloaded-subject row +379 while adding typed native Joni debug facts. The exact remaining +`pat_advanced.t` frontier is property rows 1662–1664 and 1685 plus debug rows +1673–1683. This WIP stays separate until PR 1091 merges, then rebases onto the +new `master` before its combined acceptance and PR update. ### Execution Tracker @@ -245,50 +220,39 @@ checks at the same immutable head. Active ownership: -- P3: complete all 1,187 `re/anyof.t` native debug-description rows, then - profile and remove the measured `pat*` performance regression; debug source, - cache, reset, and lifecycle provenance are integrated and green. -- P4: close the ten disjoint `pat_advanced.t` fold/trie/introspection rows; - `regexp.t`, `regex_sets.t`, `charset.t`, and `reg_mesg.t` are green on both - backends and malformed user-property grammar is native in Joni. -- P5: close the exact 30-row Unicode residual; redundant built-in property - scanning is removed and source-present/source-less generator gates are green. -- P6: preserve the release disposition bundle and finish packaging, notices, - licenses, and affected-CPAN acceptance after the quiet `pat*` gate. +- P3: finish typed native debug descriptions and exact rows 1673–1683 across + the complete 1,187-row `anyof.t` and 1,687-row `pat_advanced.t` maps. +- P4: finish the non-POSIX native range/parser diagnostic inventory, then remove + the first disjoint matcher-semantic source scanner outside P3/P5 ownership. +- P5: finish property rows 1662–1664 and 1685, including JVM lexical package + propagation, wildcard delimiters, surrogate mapping, and scanner cleanup. +- P6: promote packaging, notice/license/SBOM, affected-CPAN, and release + disposition evidence to exact PR 1091 head. - Coordinator: integration, conflict resolution, immutable acceptance, PR/CI, plan state, combined build, and release evidence. ## Ordered Next Steps -1. Publish the six green commits at `b587e9195` for integration into PR 1087 - with the preserved source-hash/assertion-ID dispositions and clean serial - 1249/1302 `pat.t`/`pat_thr.t` evidence. -2. After integration into PR 1087, run focused - Unicode, `regexp.t`, `regex_sets.t`, `charset.t`, `pat.t`, `pat_thr.t`, - `pat_advanced.t`, `pat_re_eval.t`, and `reg_mesg.t` gates, then repeat the - complete corpus, strict PR-958 comparison, additional release checks, and - exact-head CI. Run `pat.t` and `pat_thr.t` exclusively with no external - worker builds; the runner's 600-second minimum already exceeds their - isolated 399/446-second measurements. Increase only that documented - per-file minimum if a quiet exact-head run still times out. -3. Merge PR 1087 only when every checkpoint gate is green or has a documented - baseline-equivalent environmental disposition. -4. Meanwhile integrate P3 matcher-state, P4 warning, P5 generated-property, - and P6 diagnostic/scanner-removal deliveries only into the separate - post-1087 WIP. Batch focused tests and one combined full build, then rebase - this WIP onto merged refreshed `master`. -5. Delete remaining production migration scaffolding and prove all constants, +1. Keep PR 1091 immutable while the user runs the complete latest-Perl corpus, + strict PR-958 comparison, additional release checks, and exact-head CI. +2. Merge PR 1091 directly to `master` only when every checkpoint gate is green + or has a documented baseline-equivalent environmental disposition; then + close PRs 1087 and 1090 as superseded. +3. Meanwhile integrate P3 debug/matcher, P4 parser/scanner, P5 property, and P6 + acceptance deliveries only into the separate post-1091 WIP. Batch focused + tests and one combined full build, then rebase this WIP onto merged `master`. +4. Delete remaining production migration scaffolding and prove all constants, closures, conditions, verbs, recursion, dynamic source, byte strings, and Unicode strings execute through Joni. -6. Run complete JVM/interpreter acceptance, direct/thread parity, affected CPAN +5. Run complete JVM/interpreter acceptance, direct/thread parity, affected CPAN suites, five warmed performance samples, packaging, notices/licenses, warning-free build, and platform CI. -7. Reconcile final documentation and remove redundant design material. -8. After the final implementation PR is merged to `master`, remove automatic +6. Reconcile final documentation and remove redundant design material. +7. After the final implementation PR is merged to `master`, remove automatic regex `JPERL_UNIMPLEMENTED=warn` injection from `dev/tools/perl_test_runner.pl`; rerun the complete corpus; then delete the RuntimeRegex warning-plus-never-match downgrade and obsolete tests/docs. -9. On final `master`, review shipped behavior against +8. On final `master`, review shipped behavior against `pod/perlreref.pod`, `pod/perlrecharclass.pod`, `pod/perlrequick.pod`, `pod/perlrepository.pod`, `pod/perlre.pod`, `pod/perlretut.pod`, and `pod/perlrebackslash.pod`. Record every From 015c1e2a02b85300b118bbfe384d089273cb8312 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 14:14:30 +0200 Subject: [PATCH 34/43] fix: close residual Perl Unicode property gaps Implement current-Perl punctuation delimiters for property-value wildcards, preserve JVM lexical packages for user-property callbacks, expose the internal surrogate range, and remove redundant family-specific delimiter adapters. Document and test the shared native resolver policy across exact regressions, enumerated property families, escaped delimiters, and both scalar domains. Generated with [Codex](https://openai.com/codex/) Co-Authored-By: Codex --- dev/implementation/regex.md | 9 ++ .../org/perlonjava/backend/jvm/EmitRegex.java | 18 ++- .../runtime/regex/RuntimeRegex.java | 28 ++++ .../runtime/regex/UnicodeResolver.java | 127 ++++++++++++------ .../PerlPropertyWildcardDelimiterTest.java | 42 ++++++ .../unit/regex/pat_property_residuals.t | 35 +++++ 6 files changed, 215 insertions(+), 44 deletions(-) create mode 100644 src/test/java/org/perlonjava/runtime/regex/PerlPropertyWildcardDelimiterTest.java create mode 100644 src/test/resources/unit/regex/pat_property_residuals.t diff --git a/dev/implementation/regex.md b/dev/implementation/regex.md index 7ad23fffa..d7f306a13 100644 --- a/dev/implementation/regex.md +++ b/dev/implementation/regex.md @@ -156,6 +156,15 @@ eligibility through Joni's `NamedCharacterCache`; `PerlPropertyValueMatcher` evaluates wildcard value expressions without depending on PerlOnJava runtime classes. +Property-value wildcard delimiters follow current Perl's punctuation grammar: +ASCII punctuation other than `-`, `+`, `_`, and `{` may delimit the value +subpattern; `(`, `[`, and `<` use their paired closing character, and a +backslash-escaped opener requires the same escape before its closer. The same +parser owns Numeric_Value, Block, Name, Age, and the other enumerated property +families. The resolver also exposes Perl's internal +`utf8::_perl_surrogate` property as D800-DFFF in both Joni's encoding-domain +and wide-scalar ranges. + The generator registry is `dev/tools/perl_unicode_data_generators.json`. It records the consumed Perl checkout, Unicode version, input hashes, generated outputs, and output hashes. diff --git a/src/main/java/org/perlonjava/backend/jvm/EmitRegex.java b/src/main/java/org/perlonjava/backend/jvm/EmitRegex.java index 049200425..725a01831 100644 --- a/src/main/java/org/perlonjava/backend/jvm/EmitRegex.java +++ b/src/main/java/org/perlonjava/backend/jvm/EmitRegex.java @@ -372,11 +372,13 @@ static void handleQuoteRegex(EmitterVisitor emitterVisitor, OperatorNode node) { operand.elements.get(1).accept(scalarVisitor); // Flags maybeApplyUnicodeStringsRegexModifiers(emitterVisitor); emitRegexWarningState(emitterVisitor, node); + emitterVisitor.ctx.mv.visitLdcInsn( + emitterVisitor.ctx.symbolTable.getCurrentPackage()); // Create the quoted regex emitterVisitor.ctx.mv.visitMethodInsn(Opcodes.INVOKESTATIC, - "org/perlonjava/runtime/regex/RuntimeRegex", "getQuotedRegex", - "(Lorg/perlonjava/runtime/runtimetypes/RuntimeScalar;Lorg/perlonjava/runtime/runtimetypes/RuntimeScalar;)Lorg/perlonjava/runtime/runtimetypes/RuntimeScalar;", false); + "org/perlonjava/runtime/regex/RuntimeRegex", "getQuotedRegexInPackage", + "(Lorg/perlonjava/runtime/runtimetypes/RuntimeScalar;Lorg/perlonjava/runtime/runtimetypes/RuntimeScalar;Ljava/lang/String;)Lorg/perlonjava/runtime/runtimetypes/RuntimeScalar;", false); String markMethod = node.getBooleanAnnotation("syntacticQuoteRegex") ? "markSyntacticQuoteConstruction" : "markQuoteConstruction"; @@ -451,13 +453,17 @@ static void handleMatchRegex(EmitterVisitor emitterVisitor, OperatorNode node) { if (needsCallsiteCache) { int callsiteId = nextCallsiteId.getAndIncrement(); emitterVisitor.ctx.mv.visitLdcInsn(callsiteId); + emitterVisitor.ctx.mv.visitLdcInsn( + emitterVisitor.ctx.symbolTable.getCurrentPackage()); emitterVisitor.ctx.mv.visitMethodInsn(Opcodes.INVOKESTATIC, - "org/perlonjava/runtime/regex/RuntimeRegex", "getQuotedRegex", - "(Lorg/perlonjava/runtime/runtimetypes/RuntimeScalar;Lorg/perlonjava/runtime/runtimetypes/RuntimeScalar;I)Lorg/perlonjava/runtime/runtimetypes/RuntimeScalar;", false); + "org/perlonjava/runtime/regex/RuntimeRegex", "getQuotedRegexInPackage", + "(Lorg/perlonjava/runtime/runtimetypes/RuntimeScalar;Lorg/perlonjava/runtime/runtimetypes/RuntimeScalar;ILjava/lang/String;)Lorg/perlonjava/runtime/runtimetypes/RuntimeScalar;", false); } else { + emitterVisitor.ctx.mv.visitLdcInsn( + emitterVisitor.ctx.symbolTable.getCurrentPackage()); emitterVisitor.ctx.mv.visitMethodInsn(Opcodes.INVOKESTATIC, - "org/perlonjava/runtime/regex/RuntimeRegex", "getQuotedRegex", - "(Lorg/perlonjava/runtime/runtimetypes/RuntimeScalar;Lorg/perlonjava/runtime/runtimetypes/RuntimeScalar;)Lorg/perlonjava/runtime/runtimetypes/RuntimeScalar;", false); + "org/perlonjava/runtime/regex/RuntimeRegex", "getQuotedRegexInPackage", + "(Lorg/perlonjava/runtime/runtimetypes/RuntimeScalar;Lorg/perlonjava/runtime/runtimetypes/RuntimeScalar;Ljava/lang/String;)Lorg/perlonjava/runtime/runtimetypes/RuntimeScalar;", false); } int regexSlot = emitterVisitor.ctx.javaClassInfo.acquireSpillSlot(); diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java index 271695f81..06b3e48f5 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java @@ -1650,6 +1650,20 @@ public static RuntimeScalar getQuotedRegex(RuntimeScalar patternString, RuntimeS return getQuotedRegex(patternString, modifiers, metadata); } + /** Compile a JVM-emitted regex under its lexical package for user properties. */ + public static RuntimeScalar getQuotedRegexInPackage( + RuntimeScalar patternString, RuntimeScalar modifiers, + String lexicalPackage) { + RuntimeScalar currentPackage = InterpreterState.currentPackage.get(); + String previousPackage = currentPackage.toString(); + currentPackage.set(lexicalPackage); + try { + return getQuotedRegex(patternString, modifiers); + } finally { + currentPackage.set(previousPackage); + } + } + /** Attach CV-owned immutable charname results without changing pattern text. */ public static RuntimeScalar attachNamedCharacterExpansions( RuntimeScalar patternString, NamedCharacterExpansionMap metadata) { @@ -1963,6 +1977,20 @@ public static RuntimeScalar getQuotedRegex(RuntimeScalar patternString, RuntimeS return getQuotedRegex(patternString, modifiers, callsiteId, metadata); } + /** /o and m?PAT? variant retaining the JVM emitter's lexical package. */ + public static RuntimeScalar getQuotedRegexInPackage( + RuntimeScalar patternString, RuntimeScalar modifiers, + int callsiteId, String lexicalPackage) { + RuntimeScalar currentPackage = InterpreterState.currentPackage.get(); + String previousPackage = currentPackage.toString(); + currentPackage.set(lexicalPackage); + try { + return getQuotedRegex(patternString, modifiers, callsiteId); + } finally { + currentPackage.set(previousPackage); + } + } + public static RuntimeScalar getQuotedRegex( RuntimeScalar patternString, RuntimeScalar modifiers, int callsiteId, NamedCharacterExpansionMap preResolvedNamedCharacters) { diff --git a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java index b1d44bf6e..3ce4ded49 100644 --- a/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java +++ b/src/main/java/org/perlonjava/runtime/regex/UnicodeResolver.java @@ -1467,6 +1467,13 @@ static CharacterPropertyResolver.Result resolveJoniProperty( throw new IllegalArgumentException( "Unsupported obsolete Unicode property: " + property); } + if (property.equals("utf8::_perl_surrogate")) { + UnicodeSet surrogateScalars = new UnicodeSet(0xD800, 0xDFFF).freeze(); + return joniPropertyResult( + surrogateScalars, + new long[] {1, 0xD800L, 0xDFFFL}, + false); + } if (property.matches( "(?i)(?:sc|script|scx|script[-_ ]?extensions)\\s*(?:=|:)\\s*")) { throw new IllegalArgumentException( @@ -2520,12 +2527,12 @@ static boolean isPerlStringProperty(String property) { int assignment = propertyValueDelimiter(property); return assignment > 0 && assignment < property.length() - 1 && isPerlNameProperty(property.substring(0, assignment)) - && perlSlashWildcardBody(property.substring(assignment + 1)) == null; + && perlNumericWildcardBody(property.substring(assignment + 1)) == null; } private static PerlUnicodePropertyWildcard resolvePerlNamePropertyWildcard( String propertyName, String value) { - String body = perlSlashWildcardBody(value); + String body = perlNumericWildcardBody(value); if (body == null) return null; rejectForbiddenPerlUnicodePropertyWildcardSyntax(body); @@ -2554,14 +2561,6 @@ private static PerlUnicodePropertyWildcard resolvePerlNamePropertyWildcard( return new PerlUnicodePropertyWildcard(result.freeze(), null, true); } - private static String perlSlashWildcardBody(String value) { - String trimmed = value.trim(); - return trimmed.length() > 2 && trimmed.startsWith("/") - && trimmed.endsWith("/") - ? trimmed.substring(1, trimmed.length() - 1) - : null; - } - private static final class PerlUnicodePropertyWildcard { private final UnicodeSet set; private final long[] wideRanges; @@ -3005,7 +3004,7 @@ private static boolean isPerlIsPrefixedBlockWildcard(String property) { property.substring(0, assignment)); return propertyName != null && PerlUnicodeBlockData.isPropertyAlias(propertyName) - && perlBlockWildcardBody(property.substring(assignment + 1)) != null; + && perlNumericWildcardBody(property.substring(assignment + 1)) != null; } private static boolean isPerlIsPrefixedScriptWildcard(String property) { @@ -3073,7 +3072,7 @@ private static UnicodeSet resolvePerlJoiningGroup(String value) { } private static UnicodeSet resolvePerlBlock(String value) { - String wildcard = perlBlockWildcardBody(value); + String wildcard = perlNumericWildcardBody(value); if (wildcard == null) { UnicodeSet exact = PerlUnicodeBlockData.set(value); if (exact == null) { @@ -3177,19 +3176,6 @@ private static UnicodeSet resolvePerlScript(String value, boolean extensions) { return result.freeze(); } - private static String perlBlockWildcardBody(String value) { - String trimmed = value.trim(); - if (trimmed.startsWith(":\\A") && trimmed.endsWith("\\z:") - && trimmed.length() > 6) { - return "\\A" + trimmed.substring(3, trimmed.length() - 3) + "\\z"; - } - if (!trimmed.startsWith("#") || !trimmed.endsWith("#") - || trimmed.length() <= 2) { - return null; - } - return trimmed.substring(1, trimmed.length() - 1); - } - private static UnicodeSet resolvePerlNumericValue(String value) { String wildcard = perlNumericWildcardBody(value); if (wildcard != null) { @@ -3225,19 +3211,46 @@ private static UnicodeSet resolvePerlNumericValue(String value) { private static String perlNumericWildcardBody(String value) { String trimmed = value.trim(); - if (trimmed.length() > 2 && trimmed.startsWith(":") - && trimmed.endsWith(":")) { - return trimmed.substring(1, trimmed.length() - 1); + if (trimmed.isEmpty()) return null; + + int bodyStart = 1; + boolean escapedDelimiter = trimmed.charAt(0) == '\\'; + char open = trimmed.charAt(0); + if (escapedDelimiter) { + if (trimmed.length() < 2 || !isAsciiPunctuation(trimmed.charAt(1))) { + return null; + } + open = trimmed.charAt(1); + bodyStart = 2; + } else if (!isAsciiPunctuation(open) + || open == '-' || open == '+' || open == '_' || open == '{') { + return null; } - if (trimmed.startsWith("/\\A") && trimmed.endsWith("\\z/") - && trimmed.length() > 6) { - return "\\A" + trimmed.substring(3, trimmed.length() - 3) + "\\z"; + + char close = switch (open) { + case '(' -> ')'; + case '[' -> ']'; + case '<' -> '>'; + default -> open; + }; + int bodyEnd = trimmed.length() - 1; + boolean terminated = bodyEnd >= bodyStart && trimmed.charAt(bodyEnd) == close; + if (terminated && escapedDelimiter) { + terminated = bodyEnd > bodyStart && trimmed.charAt(bodyEnd - 1) == '\\'; + bodyEnd--; } - if (trimmed.startsWith(":\\A") && trimmed.endsWith("\\z:") - && trimmed.length() > 6) { - return "\\A" + trimmed.substring(3, trimmed.length() - 3) + "\\z"; + if (!terminated) { + throw new IllegalArgumentException( + "Unicode property wildcard not terminated"); } - return null; + return trimmed.substring(bodyStart, bodyEnd); + } + + private static boolean isAsciiPunctuation(char value) { + return value >= '!' && value <= '/' + || value >= ':' && value <= '@' + || value >= '[' && value <= '`' + || value >= '{' && value <= '~'; } /** @@ -3247,7 +3260,7 @@ private static String perlNumericWildcardBody(String value) { */ private static void rejectForbiddenPerlUnicodePropertyWildcardSyntax( String body) { - int offset = body.indexOf('*'); + int offset = forbiddenPerlWildcardQuantifierStar(body); if (offset >= 0) { throw propertyWildcardSyntaxError(body, offset + 1, "Use of quantifier '*' is not allowed in Unicode property wildcard subpatterns"); @@ -3292,6 +3305,45 @@ private static void rejectForbiddenPerlUnicodePropertyWildcardSyntax( } } + private static int forbiddenPerlWildcardQuantifierStar(String body) { + boolean escaped = false; + boolean quoted = false; + boolean inCharacterClass = false; + for (int index = 0; index < body.length(); index++) { + char current = body.charAt(index); + if (escaped) { + if (!inCharacterClass && current == 'Q') quoted = true; + else if (!inCharacterClass && current == 'E') quoted = false; + escaped = false; + continue; + } + if (current == '\\') { + escaped = true; + continue; + } + if (quoted) continue; + if (inCharacterClass) { + if (current == ']') inCharacterClass = false; + continue; + } + if (current == '[') { + inCharacterClass = true; + continue; + } + if (current == '(' && index + 2 < body.length() + && body.charAt(index + 1) == '?' + && body.charAt(index + 2) == '#') { + index += 3; + while (index < body.length() && body.charAt(index) != ')') index++; + continue; + } + if (current == '*' && (index == 0 || body.charAt(index - 1) != '(')) { + return index; + } + } + return -1; + } + private static PerlCompilerException propertyWildcardSyntaxError( String body, int offset, String message) { return new PerlCompilerException( @@ -3581,8 +3633,7 @@ private static boolean isPerlAgeProperty(String name) { } private static boolean isPerlAgeWildcard(String value) { - String trimmed = value.trim(); - return trimmed.startsWith(":\\A") && trimmed.endsWith("\\z:"); + return perlNumericWildcardBody(value) != null; } private static String normalizeUnicodeAgeVersion(String value) { diff --git a/src/test/java/org/perlonjava/runtime/regex/PerlPropertyWildcardDelimiterTest.java b/src/test/java/org/perlonjava/runtime/regex/PerlPropertyWildcardDelimiterTest.java new file mode 100644 index 000000000..cc0af45ab --- /dev/null +++ b/src/test/java/org/perlonjava/runtime/regex/PerlPropertyWildcardDelimiterTest.java @@ -0,0 +1,42 @@ +package org.perlonjava.runtime.regex; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.joni.CharacterPropertyResolver; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +@Tag("unit") +class PerlPropertyWildcardDelimiterTest { + @Test + void acceptsPairedParenthesisDelimiterAndReportsNoMatchingValue() { + IllegalArgumentException error = assertThrows( + IllegalArgumentException.class, + () -> UnicodeResolver.resolveJoniProperty( + "nv:(\\B(*COMMIT)C+)", false)); + assertTrue(error.getMessage().startsWith( + "No Unicode property value wildcard matches")); + } + + @Test + void reportsSingleCharacterBracketDelimiterAsUnterminated() { + IllegalArgumentException error = assertThrows( + IllegalArgumentException.class, + () -> UnicodeResolver.resolveJoniProperty("upper:]", false)); + assertEquals("Unicode property wildcard not terminated", error.getMessage()); + } + + @Test + void resolvesPerlInternalSurrogatePropertyInBothScalarDomains() { + CharacterPropertyResolver.Result result = + UnicodeResolver.resolveJoniProperty( + "utf8::_perl_surrogate", false); + assertNotNull(result); + assertArrayEquals(new int[] {1, 0xD800, 0xDFFF}, result.ranges); + assertArrayEquals(new long[] {1, 0xD800L, 0xDFFFL}, result.wideRanges); + } +} diff --git a/src/test/resources/unit/regex/pat_property_residuals.t b/src/test/resources/unit/regex/pat_property_residuals.t new file mode 100644 index 000000000..21ada1ccf --- /dev/null +++ b/src/test/resources/unit/regex/pat_property_residuals.t @@ -0,0 +1,35 @@ +#!/usr/bin/env perl +use strict; +use warnings; +use Test::More tests => 13; +no warnings 'experimental::uniprop_wildcards'; + +eval q{qr/\p{nv:(\B(*COMMIT)C+)}/}; +like($@, qr/No Unicode property value wildcard matches/, + 'pat_advanced:1663 wildcard control verb reports no match'); + +eval q{qr/\p{upper:]}|\337(?|ss)|)(?0/}; +like($@, qr/Unicode property wildcard not terminated/, + 'pat_advanced:1664 single-character wildcard is unterminated'); + +eval q{qr/\p{utf8::_perl_surrogate}/}; +is($@, '', 'pat_advanced:1685 runtime utf8 surrogate property compiles'); + +my @delimiter_matrix = ( + [q{qr/\p{nv=(\A1\z)}/}, '1', 'numeric paired parentheses'], + [q{qr/\p{Block=[\ABasic_Latin\z]}/}, 'A', 'Block paired brackets'], + [q{qr/\p{name=<\ALATIN CAPITAL LETTER A\z>}/}, 'A', + 'Name paired angle brackets'], + [q{qr/\p{Age=#\AV2_1\z#}/}, "\x{20ac}", + 'Age repeated hash delimiter'], +); +for my $case (@delimiter_matrix) { + my ($source, $member, $label) = @$case; + my $pattern = eval $source; + is($@, '', "$label compiles"); + like($member, $pattern, "$label matches"); +} + +my $escaped_delimiter = eval q{qr/\p{nv=\#\A1\z\#}/}; +is($@, '', 'escaped repeated delimiter compiles'); +like('1', $escaped_delimiter, 'escaped repeated delimiter matches'); From 36b1dd184a8affc7e3fbe7349588cc798135c46b Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 14:22:37 +0200 Subject: [PATCH 35/43] fix: report semantic regex class programs Prefer proven semantic class labels before the native Joni bytecode details in compile debug output, while retaining the native fallback for unsupported program shapes. Generated with [OpenAI Codex](https://openai.com/codex/) Co-Authored-By: OpenAI Codex --- .../runtime/regex/RuntimeRegex.java | 4 ++ .../RegexSemanticDebugTraceTest.java | 71 +++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 src/test/java/org/perlonjava/RegexSemanticDebugTraceTest.java diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java index 06b3e48f5..c811edc54 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java @@ -1425,8 +1425,12 @@ private void emitCompileDebugTrace() { ? "" : recursivePattern.optimizerDebugDescription(); String nativeProgram = recursivePattern == null ? "" : recursivePattern.nativeCompileDebugDescription(); + String perlProgram = recursivePattern == null + ? "" : recursivePattern.engineRegex() + .perlFirstProgramDebugDescription(); debugWrite("Compiling REx \"" + patternDescription + "\"\n" + "Final program:\n" + + (perlProgram.isEmpty() ? "" : perlProgram + "\n") + "JONI_PATTERN native bytecode:\n" + nativeProgram + (optimizerDescription.isEmpty() diff --git a/src/test/java/org/perlonjava/RegexSemanticDebugTraceTest.java b/src/test/java/org/perlonjava/RegexSemanticDebugTraceTest.java new file mode 100644 index 000000000..0d4432c43 --- /dev/null +++ b/src/test/java/org/perlonjava/RegexSemanticDebugTraceTest.java @@ -0,0 +1,71 @@ +package org.perlonjava; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.perlonjava.app.cli.ArgumentParser; +import org.perlonjava.app.cli.CompilerOptions; +import org.perlonjava.app.scriptengine.PerlLanguageProvider; +import org.perlonjava.runtime.io.StandardIO; +import org.perlonjava.runtime.runtimetypes.GlobalVariable; +import org.perlonjava.runtime.runtimetypes.RuntimeIO; + +@Tag("unit") +public class RegexSemanticDebugTraceTest extends PerlRuntimeTestBase { + private RuntimeIO originalStdout; + private RuntimeIO originalStderr; + private ByteArrayOutputStream stderr; + + @BeforeEach + void setUp() { + PerlLanguageProvider.resetAll(); + originalStdout = RuntimeIO.getStdout(); + originalStderr = RuntimeIO.getStderr(); + RuntimeIO.setStdout(new RuntimeIO( + new StandardIO(new ByteArrayOutputStream(), true))); + stderr = new ByteArrayOutputStream(); + RuntimeIO.setStderr(new RuntimeIO(new StandardIO(stderr, false))); + GlobalVariable.getGlobalIO("main::STDOUT").setIO(RuntimeIO.getStdout()); + GlobalVariable.getGlobalIO("main::STDERR").setIO(RuntimeIO.getStderr()); + } + + @AfterEach + void tearDown() { + RuntimeIO.setStdout(originalStdout); + RuntimeIO.setStderr(originalStderr); + GlobalVariable.getGlobalIO("main::STDOUT").setIO(RuntimeIO.getStdout()); + GlobalVariable.getGlobalIO("main::STDERR").setIO(RuntimeIO.getStderr()); + PerlLanguageProvider.resetAll(); + } + + private String execute(String source) throws Exception { + CompilerOptions options = ArgumentParser.parseArguments( + new String[] {"-e", source}); + PerlLanguageProvider.executePerlCode(options, true); + RuntimeIO.getStderr().flush(); + return stderr.toString(StandardCharsets.ISO_8859_1); + } + + @Test + void reportsProvenSemanticClassLabelsBeforeNativeDetails() throws Exception { + String trace = execute("use re Debug => 'COMPILE'; " + + "qr/\\p{All}/; qr/\\P{All}/; qr/[^\\n]/; qr/[ab]/;"); + + assertTrue(trace.contains("Final program:\nSANY\n" + + "JONI_PATTERN native bytecode:"), trace); + assertTrue(trace.contains("Final program:\nOPFAIL\n" + + "JONI_PATTERN native bytecode:"), trace); + assertTrue(trace.contains("Final program:\nREG_ANY\n" + + "JONI_PATTERN native bytecode:"), trace); + assertTrue(trace.contains("Compiling REx \"[ab]\"\n" + + "Final program:\nJONI_PATTERN native bytecode:"), trace); + assertFalse(trace.contains("Final program:\n\n"), trace); + } +} From e6ddefc8bf8c48f9103d7e29e9aa39e269861375 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 14:33:03 +0200 Subject: [PATCH 36/43] feat: expose compiled regex class membership facts Decode every native character-class operand into an immutable canonical membership snapshot across byte, Unicode, and signed-wide scalar domains while keeping semantic renderer classification separate from representation facts. Generated with [OpenAI Codex](https://openai.com/codex/) Co-Authored-By: OpenAI Codex --- third_party/joni/src/org/joni/Regex.java | 43 +++- .../joni/src/org/joni/RegexDebugProgram.java | 150 ++++++++++++- .../joni/src/org/joni/ast/CClassNode.java | 99 ++++++++ .../TestRegexCharacterClassDebugFacts.java | 211 ++++++++++++++++++ 4 files changed, 493 insertions(+), 10 deletions(-) create mode 100644 third_party/joni/test/org/joni/test/TestRegexCharacterClassDebugFacts.java diff --git a/third_party/joni/src/org/joni/Regex.java b/third_party/joni/src/org/joni/Regex.java index c20e868d6..69f668bd2 100644 --- a/third_party/joni/src/org/joni/Regex.java +++ b/third_party/joni/src/org/joni/Regex.java @@ -28,6 +28,8 @@ import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.Iterator; +import java.util.List; +import java.util.Objects; import org.jcodings.CaseFoldCodeItem; import org.jcodings.Encoding; @@ -632,15 +634,50 @@ public enum DebugProgramKind { OTHER } - public record DebugProgramFact(DebugProgramKind kind) { + /** Inclusive effective class-membership range in [0, Long.MAX_VALUE]. */ + public record DebugRange(long from, long to) { + public DebugRange { + if (from < 0 || from > to) { + throw new IllegalArgumentException("invalid debug range"); + } + } + } + + /** + * Canonical sorted/coalesced effective membership and the compiled class + * representation's NOT flag (which need not mirror source spelling). + */ + public record DebugCharacterClassFact(boolean storageNegated, + List ranges) { + public DebugCharacterClassFact { + ranges = List.copyOf(ranges); + } + } + + public record DebugProgramFact(DebugProgramKind kind, + DebugCharacterClassFact characterClass) { + public DebugProgramFact { + Objects.requireNonNull(kind, "kind"); + if (kind != DebugProgramKind.OTHER && characterClass == null) { + throw new IllegalArgumentException( + "semantic class fact requires membership"); + } + } + + public DebugProgramFact(DebugProgramKind kind) { + this(kind, null); + } + static DebugProgramFact other() { return new DebugProgramFact(DebugProgramKind.OTHER); } } /** - * Returns a semantic fact when the program begins with a class instruction, - * optionally after one canonical dynamic-option prologue. + * Returns a semantic shape and optional immutable membership when the + * program begins with a class instruction, optionally after one canonical + * dynamic-option prologue. Unsupported programs return OTHER with no + * membership; ordinary classes return OTHER with membership. */ public DebugProgramFact firstDebugProgramFact() { return RegexDebugProgram.firstFact(this); diff --git a/third_party/joni/src/org/joni/RegexDebugProgram.java b/third_party/joni/src/org/joni/RegexDebugProgram.java index 4d746c3e5..bae4b63bf 100644 --- a/third_party/joni/src/org/joni/RegexDebugProgram.java +++ b/third_party/joni/src/org/joni/RegexDebugProgram.java @@ -19,7 +19,11 @@ */ package org.joni; +import java.util.ArrayList; +import java.util.List; + import org.joni.ast.CClassNode.DebugDomainShape; +import org.joni.ast.CClassNode.DebugMembership; import org.joni.constants.internal.OPCode; import org.joni.constants.internal.OPSize; @@ -29,9 +33,14 @@ private RegexDebugProgram() { static Regex.DebugProgramFact firstFact(Regex regex) { int cursor = skipInitialDynamicOptionWrapper(regex.code, regex.codeLength); - if (cursor < 0 || cursor >= regex.codeLength - || regex.code[cursor] != OPCode.WIDE_SCALAR_CLASS - || cursor + OPSize.WIDE_SCALAR_CLASS > regex.codeLength) { + if (cursor < 0 || cursor >= regex.codeLength) { + return Regex.DebugProgramFact.other(); + } + + if (regex.code[cursor] != OPCode.WIDE_SCALAR_CLASS) { + return ordinaryClassFact(regex, cursor); + } + if (cursor + OPSize.WIDE_SCALAR_CLASS > regex.codeLength) { return Regex.DebugProgramFact.other(); } @@ -41,19 +50,146 @@ static Regex.DebugProgramFact firstFact(Regex regex) { return Regex.DebugProgramFact.other(); } + DebugMembership membership = regex.wideScalarClasses[classIndex] + .debugMembership(regex.enc); + List publicRanges = new ArrayList<>( + membership.ranges().size()); + for (org.joni.ast.CClassNode.DebugRange range : membership.ranges()) { + publicRanges.add(new Regex.DebugRange(range.from(), range.to())); + } + Regex.DebugCharacterClassFact characterClass = + new Regex.DebugCharacterClassFact(membership.storageNegated(), + publicRanges); DebugDomainShape shape = regex.wideScalarClasses[classIndex] .debugDomainShape(regex.enc); return switch (shape) { case FULL -> new Regex.DebugProgramFact( - Regex.DebugProgramKind.FULL_CLASS); + Regex.DebugProgramKind.FULL_CLASS, characterClass); case EMPTY -> new Regex.DebugProgramFact( - Regex.DebugProgramKind.EMPTY_CLASS); + Regex.DebugProgramKind.EMPTY_CLASS, characterClass); case ALL_EXCEPT_NEWLINE -> new Regex.DebugProgramFact( - Regex.DebugProgramKind.ALL_EXCEPT_NEWLINE_CLASS); - case OTHER -> Regex.DebugProgramFact.other(); + Regex.DebugProgramKind.ALL_EXCEPT_NEWLINE_CLASS, + characterClass); + case OTHER -> new Regex.DebugProgramFact( + Regex.DebugProgramKind.OTHER, characterClass); }; } + private static Regex.DebugProgramFact ordinaryClassFact(Regex regex, + int cursor) { + int opcode = regex.code[cursor++]; + boolean negated; + boolean hasBitmap; + boolean hasRanges; + switch (opcode) { + case OPCode.CCLASS -> { + negated = false; + hasBitmap = true; + hasRanges = false; + } + case OPCode.CCLASS_NOT -> { + negated = true; + hasBitmap = true; + hasRanges = false; + } + case OPCode.CCLASS_MB -> { + negated = false; + hasBitmap = false; + hasRanges = true; + } + case OPCode.CCLASS_MB_NOT -> { + negated = true; + hasBitmap = false; + hasRanges = true; + } + case OPCode.CCLASS_MIX -> { + negated = false; + hasBitmap = true; + hasRanges = true; + } + case OPCode.CCLASS_MIX_NOT -> { + negated = true; + hasBitmap = true; + hasRanges = true; + } + default -> { + return Regex.DebugProgramFact.other(); + } + } + + List raw = new ArrayList<>(); + if (hasBitmap) { + if (cursor + BitSet.BITSET_SIZE > regex.codeLength) { + return Regex.DebugProgramFact.other(); + } + long start = -1; + for (int codePoint = 0; codePoint < BitSet.SINGLE_BYTE_SIZE; + codePoint++) { + int word = regex.code[cursor + codePoint / BitSet.BITS_IN_ROOM]; + boolean member = (word & (1 << (codePoint % BitSet.BITS_IN_ROOM))) + != 0; + if (member) { + if (start < 0) start = codePoint; + } else if (start >= 0) { + appendRange(raw, start, codePoint - 1L); + start = -1; + } + } + if (start >= 0) appendRange(raw, start, 0xff); + cursor += BitSet.BITSET_SIZE; + } + if (hasRanges) { + if (cursor >= regex.codeLength) return Regex.DebugProgramFact.other(); + int length = regex.code[cursor++]; + if (length < 1 || cursor + length > regex.codeLength) { + return Regex.DebugProgramFact.other(); + } + int count = regex.code[cursor]; + if (count < 0 || 1L + count * 2L > length) { + return Regex.DebugProgramFact.other(); + } + for (int index = 0; index < count; index++) { + int from = regex.code[cursor + index * 2 + 1]; + int to = regex.code[cursor + index * 2 + 2]; + if (from < 0 || from > to) return Regex.DebugProgramFact.other(); + appendRange(raw, from, to); + } + } + + List effective = negated + ? complement(raw, CodeRangeBuffer.LAST_CODE_POINT) + : List.copyOf(raw); + return new Regex.DebugProgramFact(Regex.DebugProgramKind.OTHER, + new Regex.DebugCharacterClassFact(negated, effective)); + } + + private static List complement( + List raw, long maximum) { + List result = new ArrayList<>(); + long next = 0; + for (Regex.DebugRange range : raw) { + if (next < range.from()) appendRange(result, next, range.from() - 1); + next = range.to() + 1; + } + if (next <= maximum) appendRange(result, next, maximum); + return List.copyOf(result); + } + + private static void appendRange(List ranges, + long from, long to) { + if (from > to) return; + if (!ranges.isEmpty()) { + Regex.DebugRange previous = ranges.get(ranges.size() - 1); + if (from <= previous.to() + 1) { + ranges.set(ranges.size() - 1, + new Regex.DebugRange(previous.from(), + Math.max(previous.to(), to))); + return; + } + } + ranges.add(new Regex.DebugRange(from, to)); + } + private static int skipInitialDynamicOptionWrapper(int[] code, int length) { if (code == null || length == 0 || code[0] != OPCode.SET_OPTION_PUSH) { return 0; diff --git a/third_party/joni/src/org/joni/ast/CClassNode.java b/third_party/joni/src/org/joni/ast/CClassNode.java index f9ef4d94c..bdbbc4fbf 100644 --- a/third_party/joni/src/org/joni/ast/CClassNode.java +++ b/third_party/joni/src/org/joni/ast/CClassNode.java @@ -19,6 +19,9 @@ */ package org.joni.ast; +import java.util.ArrayList; +import java.util.List; + import org.jcodings.CodeRange; import org.jcodings.Encoding; import org.jcodings.IntHolder; @@ -39,6 +42,21 @@ public enum DebugDomainShape { OTHER } + public record DebugRange(long from, long to) { + public DebugRange { + if (from < 0 || from > to) { + throw new IllegalArgumentException("invalid debug range"); + } + } + } + + public record DebugMembership(boolean storageNegated, + List ranges) { + public DebugMembership { + ranges = List.copyOf(ranges); + } + } + private static final int FLAG_NCCLASS_NOT = 1 << 0; private static final long FIRST_WIDE_SCALAR = 0x110000L; @@ -380,6 +398,87 @@ public DebugDomainShape debugDomainShape(Encoding enc) { return DebugDomainShape.OTHER; } + /** Returns an immutable snapshot of effective signed-scalar membership. */ + public DebugMembership debugMembership(Encoding enc) { + List ranges = new ArrayList<>(); + long start = -1; + for (int code = 0; code < BitSet.SINGLE_BYTE_SIZE; code++) { + if (isCodeInCC(enc, code)) { + if (start < 0) start = code; + } else if (start >= 0) { + appendDebugRange(ranges, start, code - 1L); + start = -1; + } + } + if (start >= 0) { + appendDebugRange(ranges, start, BitSet.SINGLE_BYTE_SIZE - 1L); + } + + List encoded = rawEncodedDebugRanges(0x100, 0x10ffff); + appendEffectiveDebugRanges(ranges, encoded, 0x100, 0x10ffff, + isNot()); + List wide = rawWideDebugRanges(); + appendEffectiveDebugRanges(ranges, wide, FIRST_WIDE_SCALAR, + Long.MAX_VALUE, isNot()); + return new DebugMembership(isNot(), ranges); + } + + private List rawEncodedDebugRanges(long minimum, long maximum) { + if (mbuf == null) return List.of(); + List ranges = new ArrayList<>(); + int[] raw = mbuf.getCodeRange(); + for (int i = 0; i < raw[0]; i++) { + long from = Math.max(minimum, raw[i * 2 + 1]); + long to = Math.min(maximum, raw[i * 2 + 2]); + if (from <= to) appendDebugRange(ranges, from, to); + } + return ranges; + } + + private List rawWideDebugRanges() { + if (wideRangeCount == 0) return List.of(); + List ranges = new ArrayList<>(wideRangeCount); + for (int i = 0; i < wideRangeCount; i++) { + appendDebugRange(ranges, wideRanges[i * 2], wideRanges[i * 2 + 1]); + } + return ranges; + } + + private static void appendEffectiveDebugRanges(List output, + List raw, long minimum, long maximum, + boolean negated) { + if (!negated) { + for (DebugRange range : raw) { + appendDebugRange(output, range.from(), range.to()); + } + return; + } + + long next = minimum; + for (DebugRange range : raw) { + if (next < range.from()) { + appendDebugRange(output, next, range.from() - 1); + } + if (range.to() == Long.MAX_VALUE) return; + next = range.to() + 1; + } + if (next <= maximum) appendDebugRange(output, next, maximum); + } + + private static void appendDebugRange(List ranges, + long from, long to) { + if (from > to) return; + if (!ranges.isEmpty()) { + DebugRange previous = ranges.get(ranges.size() - 1); + if (previous.to() == Long.MAX_VALUE || from <= previous.to() + 1) { + ranges.set(ranges.size() - 1, + new DebugRange(previous.from(), Math.max(previous.to(), to))); + return; + } + } + ranges.add(new DebugRange(from, to)); + } + private boolean rawRangesIntersect(int from, int to) { if (mbuf == null) return false; int[] ranges = mbuf.getCodeRange(); diff --git a/third_party/joni/test/org/joni/test/TestRegexCharacterClassDebugFacts.java b/third_party/joni/test/org/joni/test/TestRegexCharacterClassDebugFacts.java new file mode 100644 index 000000000..93fe08d63 --- /dev/null +++ b/third_party/joni/test/org/joni/test/TestRegexCharacterClassDebugFacts.java @@ -0,0 +1,211 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni.test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Locale; + +import org.jcodings.Encoding; +import org.jcodings.specific.UTF8Encoding; +import org.joni.Option; +import org.joni.Regex; +import org.joni.Syntax; +import org.joni.WideScalarCodec; +import org.junit.Test; + +public class TestRegexCharacterClassDebugFacts { + private static final WideScalarCodec CODEC = new WideScalarCodec() { + @Override + public byte[] encode(long value, Encoding encoding) { + return marker(value); + } + + @Override + public Decoded decode(byte[] bytes, int p, int end, Encoding encoding) { + if (p + 4 > end || bytes[p] != '~' || bytes[p + 1] != '<') return null; + long value = 0; + int cursor = p + 2; + int digits = 0; + while (cursor < end && bytes[cursor] != '>') { + int digit = Character.digit((char)(bytes[cursor] & 0xff), 16); + if (digit < 0 || digits == 16 + || value > (Long.MAX_VALUE - digit) / 16) return null; + value = value * 16 + digit; + cursor++; + digits++; + } + return digits == 0 || cursor >= end + ? null : new Decoded(value, cursor + 1); + } + }; + + private static final Syntax SYNTAX = new Syntax( + "RegexCharacterClassDebugFacts", Syntax.PerlNG.op, Syntax.PerlNG.op2, + Syntax.PerlNG.op3, Syntax.PerlNG.behavior, Syntax.PerlNG.options, + Syntax.PerlNG.metaCharTable, null, null, CODEC); + + @Test + public void snapshotsCanonicalPositiveMembershipRanges() { + Regex.DebugProgramFact fact = compile( + "[a-c\\x{100}-\\x{102}\\x{110000}]") + .firstDebugProgramFact(); + + assertEquals(Regex.DebugProgramKind.OTHER, fact.kind()); + assertEquals(false, fact.characterClass().storageNegated()); + assertEquals(List.of( + new Regex.DebugRange('a', 'c'), + new Regex.DebugRange(0x100, 0x102), + new Regex.DebugRange(0x110000, 0x110000)), + fact.characterClass().ranges()); + assertThrows(UnsupportedOperationException.class, + () -> fact.characterClass().ranges().add( + new Regex.DebugRange(1, 1))); + assertTrue(matches(regexFor( + "[a-c\\x{100}-\\x{102}\\x{110000}]"), "b")); + assertTrue(matches(regexFor( + "[a-c\\x{100}-\\x{102}\\x{110000}]"), "Ā")); + assertTrue(matches(regexFor( + "[a-c\\x{100}-\\x{102}\\x{110000}]"), marker(0x110000))); + } + + @Test + public void snapshotsComplementAcrossUnicodeAndSignedWideDomains() { + Regex regex = compile("[^a-c\\x{100}-\\x{102}\\x{110000}]"); + Regex.DebugProgramFact fact = regex.firstDebugProgramFact(); + + assertEquals(Regex.DebugProgramKind.OTHER, fact.kind()); + assertTrue(fact.characterClass().storageNegated()); + assertEquals(List.of( + new Regex.DebugRange(0, 'a' - 1), + new Regex.DebugRange('c' + 1, 0xff), + new Regex.DebugRange(0x103, 0x10ffff), + new Regex.DebugRange(0x110001, Long.MAX_VALUE)), + fact.characterClass().ranges()); + assertEquals(fact, regex.firstDebugProgramFact()); + assertEquals("", regex.perlFirstProgramDebugDescription()); + assertTrue(matches(regex, "d")); + assertTrue(!matches(regex, "b")); + assertTrue(!matches(regex, marker(0x110000))); + } + + @Test + public void joinsTheUtf8ByteAndUnicodeBoundariesStructurally() { + assertWide("[\\x{7f}-\\x{101}]", false, + List.of(new Regex.DebugRange(0x7f, 0x101))); + assertWide("[^\\x{7f}-\\x{101}]", true, + List.of(new Regex.DebugRange(0, 0x7e), + new Regex.DebugRange(0x102, Long.MAX_VALUE))); + } + + @Test + public void includesMembershipOnEverySemanticShapeFact() { + Regex.DebugProgramFact full = compile( + "[\\x{0}-\\x{7FFF_FFFF_FFFF_FFFF}]") + .firstDebugProgramFact(); + assertEquals(Regex.DebugProgramKind.FULL_CLASS, full.kind()); + assertEquals(List.of(new Regex.DebugRange(0, Long.MAX_VALUE)), + full.characterClass().ranges()); + + Regex.DebugProgramFact empty = compile( + "[^\\x{0}-\\x{7FFF_FFFF_FFFF_FFFF}]") + .firstDebugProgramFact(); + assertEquals(Regex.DebugProgramKind.EMPTY_CLASS, empty.kind()); + assertEquals(List.of(), empty.characterClass().ranges()); + + Regex.DebugProgramFact any = compile("[^\\n]").firstDebugProgramFact(); + assertEquals(Regex.DebugProgramKind.ALL_EXCEPT_NEWLINE_CLASS, any.kind()); + assertEquals(List.of(new Regex.DebugRange(0, 9), + new Regex.DebugRange(11, Long.MAX_VALUE)), + any.characterClass().ranges()); + + assertWide("[^\\x{110000}-\\x{7FFF_FFFF_FFFF_FFFF}]", true, + List.of(new Regex.DebugRange(0, 0x10ffff))); + Regex.DebugProgramFact exact = compile("a").firstDebugProgramFact(); + assertEquals(Regex.DebugProgramKind.OTHER, exact.kind()); + assertEquals(null, exact.characterClass()); + } + + @Test + public void decodesEveryOrdinaryCompiledClassOperandShape() { + assertOrdinary("[a-c]", false, + List.of(new Regex.DebugRange('a', 'c'))); + assertOrdinary("[^a-c]", true, + List.of(new Regex.DebugRange(0, 'a' - 1), + new Regex.DebugRange('c' + 1, 0x7fffffffL))); + assertOrdinary("[\\x{100}-\\x{102}]", false, + List.of(new Regex.DebugRange(0x100, 0x102))); + assertOrdinary("[^\\x{100}-\\x{102}]", true, + List.of(new Regex.DebugRange(0, 0xff), + new Regex.DebugRange(0x103, 0x7fffffffL))); + assertOrdinary("[a-c\\x{100}-\\x{102}]", false, + List.of(new Regex.DebugRange('a', 'c'), + new Regex.DebugRange(0x100, 0x102))); + assertOrdinary("[^a-c\\x{100}-\\x{102}]", true, + List.of(new Regex.DebugRange(0, 'a' - 1), + new Regex.DebugRange('c' + 1, 0xff), + new Regex.DebugRange(0x103, 0x7fffffffL))); + } + + private static void assertOrdinary(String pattern, boolean negated, + List ranges) { + byte[] bytes = pattern.getBytes(StandardCharsets.UTF_8); + Regex regex = new Regex(bytes, 0, bytes.length, Option.CAPTURE_GROUP, + UTF8Encoding.INSTANCE, Syntax.PerlNG); + Regex.DebugProgramFact fact = regex.firstDebugProgramFact(); + assertEquals(Regex.DebugProgramKind.OTHER, fact.kind()); + assertEquals(negated, fact.characterClass().storageNegated()); + assertEquals(ranges, fact.characterClass().ranges()); + } + + private static Regex compile(String pattern) { + byte[] bytes = pattern.getBytes(StandardCharsets.UTF_8); + return new Regex(bytes, 0, bytes.length, Option.CAPTURE_GROUP, + UTF8Encoding.INSTANCE, SYNTAX); + } + + private static Regex regexFor(String pattern) { + return compile(pattern); + } + + private static void assertWide(String pattern, boolean storageNegated, + List ranges) { + Regex.DebugProgramFact fact = compile(pattern).firstDebugProgramFact(); + assertEquals(storageNegated, fact.characterClass().storageNegated()); + assertEquals(ranges, fact.characterClass().ranges()); + } + + private static boolean matches(Regex regex, String input) { + return matches(regex, input.getBytes(StandardCharsets.UTF_8)); + } + + private static boolean matches(Regex regex, byte[] input) { + return regex.matcher(input).search(0, input.length, Option.NONE) >= 0; + } + + private static byte[] marker(long value) { + return ("~<" + Long.toHexString(value).toUpperCase(Locale.ROOT) + ">") + .getBytes(StandardCharsets.US_ASCII); + } +} From b062324f27dd9e4caa6fbb87886235ca69fdd657 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 14:38:17 +0200 Subject: [PATCH 37/43] docs(regex): refresh active phase 36 ownership Keep PR 1091 immutable during user acceptance, record the integrated Joni property and debug-fact frontier, and reassign the stale parser/scanner lease. Generated with [OpenAI Codex](https://openai.com/codex/) Co-Authored-By: OpenAI Codex --- dev/design/phase36-regex-parity.md | 41 +++++++++++++++++------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index ca052ef96..883ef9666 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -176,9 +176,9 @@ PR 1091 is the sole release and user-acceptance target at exact head onto current `master`, preserves the user's latest-Perl sync commit, and includes the six release fixes formerly staged in PR 1090. Imported fixtures remain authoritative and must not be reverted or patched to recover old counts. The -exact unified head has a clean warning-free full `make`; Ubuntu and Windows CI -and the user's complete acceptance are in progress. Workers must not mutate, -rebase, or push this immutable test branch. Acceptance rejects every new +exact unified head has a clean warning-free full `make`, and Ubuntu and Windows +CI are green. The user's complete production-load acceptance is in progress. +Workers must not mutate, rebase, or push this immutable test branch. Acceptance rejects every new invalid, missing, timeout, truncated, incomplete, or zero-TAP row and every unresolved PR-958 pass-count decrease. @@ -192,13 +192,16 @@ all 1302 rows in each file at 1249/1302 for both `pat.t` and `pat_thr.t`, with zero errors, timeouts, or incomplete rows. Independent implementation continues on `integrate/phase36-post1087-wip` at -`e136c9b00` or later. Integrated Unicode property closure is complete. P4's -named-character source-mode fix removes all nine owned `pat_advanced.t` fold/ -trie rows, and P3's single-stringification repair closes overloaded-subject row -379 while adding typed native Joni debug facts. The exact remaining -`pat_advanced.t` frontier is property rows 1662–1664 and 1685 plus debug rows -1673–1683. This WIP stays separate until PR 1091 merges, then rebases onto the -new `master` before its combined acceptance and PR update. +`9bbd3f18e` or later. Unicode property closure, lexical package propagation, +named-character source mode, and overloaded-subject handling are integrated. +Typed native Joni debug facts now expose semantic instructions and immutable +character-class membership. Property rows 1662–1664 and 1685, the nine named- +character fold/trie rows, and overloaded-subject row 379 are closed in focused +worker maps. The expected remaining `pat_advanced.t` frontier is debug rows +1673–1683; an exact combined-head map must confirm that frontier after the +display-provenance consumer lands. This WIP stays separate until PR 1091 +merges, then rebases onto the new `master` before combined acceptance and its +PR update. ### Execution Tracker @@ -220,14 +223,16 @@ new `master` before its combined acceptance and PR update. Active ownership: -- P3: finish typed native debug descriptions and exact rows 1673–1683 across - the complete 1,187-row `anyof.t` and 1,687-row `pat_advanced.t` maps. -- P4: finish the non-POSIX native range/parser diagnostic inventory, then remove - the first disjoint matcher-semantic source scanner outside P3/P5 ownership. -- P5: finish property rows 1662–1664 and 1685, including JVM lexical package - propagation, wildcard delimiters, surrogate mapping, and scanner cleanup. -- P6: promote packaging, notice/license/SBOM, affected-CPAN, and release - disposition evidence to exact PR 1091 head. +- P3: finish native Perl display provenance for rows 1673–1683 and prove the + complete 1,187-row `anyof.t` and 1,687-row `pat_advanced.t` maps. +- P5: move extended-class string/deferred-property policy and source positions + into Joni, delete `validateExtendedPropertyPolicy`, and close the remaining + property-specific scanner inventory. +- P6: finish exact-PR-1091 packaging/notice/license/SBOM and affected-CPAN + evidence, then own the complete non-property, non-debug parser/diagnostic and + first matcher-semantic scanner-removal tranche formerly assigned to P4. +- P4: fenced from the parser/scanner tranche pending restoration of its mailbox + monitor; it has no authoritative implementation ownership. - Coordinator: integration, conflict resolution, immutable acceptance, PR/CI, plan state, combined build, and release evidence. From 9497ccea890eb3e4cb90259d84dd6dce1fda3a50 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 14:52:33 +0200 Subject: [PATCH 38/43] fix(test-runner): preserve heavy regex gates under load Raise the pat.t and pat_thr.t minimum outer deadline to 900 seconds and the anyof variants to 600 seconds after the ten-worker PR 1091 acceptance proved that the prior limits could truncate otherwise healthy complete gates. Refresh the current plan and acceptance disposition without restarting the full corpus. Generated with [OpenAI Codex](https://openai.com/codex/) Co-Authored-By: OpenAI Codex --- dev/design/phase36-regex-parity.md | 23 ++++++++++++++++++----- dev/tools/perl_test_runner.pl | 9 ++++++++- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index 883ef9666..bdd096861 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -64,6 +64,11 @@ Remaining scanner-removal queue: match-all text substitutions used to represent deferred properties. - [ ] Preserve ordinary/extended-class context and source positions in Joni so the adapter no longer scans bracket depth or generates replacement classes. +- [ ] Replace `hasControlVerbState(String)` with a compiled Joni fact, and move + the `\K`-inside-lookaround diagnostic into Joni before deleting the remaining + production `analyzePerlSyntax` scan. +- [ ] Delete test-only routing scanners (`requiresJoniBackend` and its empty- + class/name helpers) once their assertions are replaced by direct Joni facts. - [ ] Retain only runtime-neutral trusted-callout materialization and documented Perl source-policy checks outside Joni; delete the scanner when these gates pass. @@ -160,7 +165,7 @@ outside it. affected CPAN suites, packaging, notice/license, and warmed performance gates. - [x] Pass the five-run warmed ordinary-regex comparison: candidate median 12.23s versus exact-parent 12.68s under alternating contended runs. -- [ ] Pass warning-free `make`, Ubuntu, Windows, and complete CI. +- [x] Pass warning-free `make`, Ubuntu, Windows, and complete CI. - [ ] Reconcile `docs/reference/feature-matrix.md`, `dev/implementation/regex.md`, and `docs/design/joni-callout-fork.md` with shipped behavior. @@ -178,9 +183,16 @@ the six release fixes formerly staged in PR 1090. Imported fixtures remain authoritative and must not be reverted or patched to recover old counts. The exact unified head has a clean warning-free full `make`, and Ubuntu and Windows CI are green. The user's complete production-load acceptance is in progress. -Workers must not mutate, rebase, or push this immutable test branch. Acceptance rejects every new -invalid, missing, timeout, truncated, incomplete, or zero-TAP row and every -unresolved PR-958 pass-count decrease. +Workers must not mutate, rebase, or push this immutable test branch. Acceptance +rejects every new invalid, missing, timeout, truncated, incomplete, or zero-TAP +row and every unresolved PR-958 pass-count decrease. + +In the live ten-process run, `pat.t` completed at 1249/1302 in 568.99s, +`pat_thr.t` reached the 600s outer watchdog, and `anyof.t` reached the 300s +watchdog. Rerun only `pat_thr.t` and `anyof.t` at exact PR 1091 head with 900s +per-file timeouts; do not restart the full corpus. The post-1091 runner now +gives both `pat.t` variants a 900s production-load-safe minimum and the +`anyof.t` variants a 600s minimum. In writable current-source state `op/do.t` is 71/71 on both backends; `class/accessor.t` executes all 30 rows and improves to 17 passes versus the @@ -296,7 +308,8 @@ Active ownership: ## Final Acceptance - [ ] Every semantic row passing in PR 958 still passes. -- [ ] Complete 622-file JVM output is compared file-by-file with PR 958. +- [ ] Complete latest-Perl JVM output is compared file-by-file with PR 958 + (the PR 1091 sync currently discovers 623 files). - [ ] Complete regex-bearing JVM/interpreter and direct/thread results agree. - [ ] Joni covers constants, closures, conditions, control verbs, recursion, dynamic source, byte strings, and Unicode strings. diff --git a/dev/tools/perl_test_runner.pl b/dev/tools/perl_test_runner.pl index f7275c6ec..2446d493e 100755 --- a/dev/tools/perl_test_runner.pl +++ b/dev/tools/perl_test_runner.pl @@ -687,9 +687,16 @@ sub timeout_for_test { # --timeout values proportional and leaving every other test unchanged. return $timeout * 2 if $test_file =~ m{(?:^|/)perl5_t/t/io/(?:crlf_)?through\.t$}; + # A ten-worker production-load acceptance can push pat.t and pat_thr.t past + # six hundred seconds even when the same build completes both serially. + # Preserve larger caller requests, but do not truncate these complete + # semantic gates under the supported parallel runner configuration. + return 900 if $test_file =~ m{(?:^|/)perl5_t/t/re/pat(?:_thr)?\.t$} + && $timeout < 900; + return 600 if $test_file =~ m{ (?:^|/)perl5_t/t/lib/croak\.t$ - | (?:^|/)perl5_t/t/re/pat(?:_thr)?\.t$ + | (?:^|/)perl5_t/t/re/anyof(?:_thr)?\.t$ | (?:^|/)perl5_t/t/re/pat_psycho(?:_thr)?\.t$ | (?:^|/)perl5_t/t/op/gv\.t$ | (?:^|/)perl5_t/t/re/pat_advanced(?:_thr)?\.t$ From 430c3e6a79d67d4257e4b5db7b52f8d95332fa1e Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 14:56:20 +0200 Subject: [PATCH 39/43] docs(regex): record PR 1091 acceptance position Record the completed platform and packaging gates, the two targeted watchdog supplements, the current 623-file corpus size, and the active scanner-removal ownership while preserving the immutable release target. Generated with [OpenAI Codex](https://openai.com/codex/) Co-Authored-By: OpenAI Codex --- dev/design/phase36-regex-parity.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index bdd096861..1442164ac 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -203,6 +203,12 @@ initial `pat*` localization. The clean serial exact-head gate executes all 1302 rows in each file at 1249/1302 for both `pat.t` and `pat_thr.t`, with zero errors, timeouts, or incomplete rows. +Exact-head packaging acceptance is complete: full `make`, standalone Joni +packaging verification, notices/licenses, merged SBOM, and artifact hashes are +green. The executable tree and ordered 28,518-entry JAR inventory are identical +to the predecessor CPAN-tested artifact, so its four-module offline matrix +transfers without a redundant rerun. + Independent implementation continues on `integrate/phase36-post1087-wip` at `9bbd3f18e` or later. Unicode property closure, lexical package propagation, named-character source mode, and overloaded-subject handling are integrated. @@ -240,9 +246,9 @@ Active ownership: - P5: move extended-class string/deferred-property policy and source positions into Joni, delete `validateExtendedPropertyPolicy`, and close the remaining property-specific scanner inventory. -- P6: finish exact-PR-1091 packaging/notice/license/SBOM and affected-CPAN - evidence, then own the complete non-property, non-debug parser/diagnostic and - first matcher-semantic scanner-removal tranche formerly assigned to P4. +- P6: own the complete non-property, non-debug parser/diagnostic and first + matcher-semantic scanner-removal tranche, starting with replacing the + control-verb source scan by a compiled Joni fact. - P4: fenced from the parser/scanner tranche pending restoration of its mailbox monitor; it has no authoritative implementation ownership. - Coordinator: integration, conflict resolution, immutable acceptance, PR/CI, From 460ee5859d0dbc41e0d1fea460b3023d5081cbe5 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 14:38:11 +0200 Subject: [PATCH 40/43] refactor(regex): use compiled control verb metadata Expose Joni's compiled control-verb inventory and use it for runtime state publication instead of scanning raw pattern text. This avoids treating verb spellings inside regex comments as executable control verbs. Generated with Codex (https://openai.com/codex) Co-Authored-By: Codex --- .../runtime/regex/JoniRegexPattern.java | 11 +---- .../regex/control_verb_compiled_metadata.t | 40 ++++++++++++++++ third_party/joni/src/org/joni/Regex.java | 5 ++ .../org/joni/TestControlVerbMetadata.java | 46 +++++++++++++++++++ 4 files changed, 92 insertions(+), 10 deletions(-) create mode 100644 src/test/resources/unit/regex/control_verb_compiled_metadata.t create mode 100644 third_party/joni/test/org/joni/TestControlVerbMetadata.java diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index a3b88ad11..21c25ea49 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -324,7 +324,6 @@ private static CharacterPropertyResolver.Result resolveCharacterProperty( validateExtendedPropertyPolicy(perlPattern); this.flags = flags; this.byteMode = byteMode; - hasControlVerbState = hasControlVerbState(perlPattern); sourcePattern = RuntimeRegexTemplate.materializeTrustedCallouts( perlPattern, trustedCalloutCount); compatibilityPatternDescription = legacyCompatibilityDescription( @@ -370,6 +369,7 @@ public boolean supportsPositions() { regex = new Regex(bytes, 0, bytes.length, options, byteMode ? ISO8859_1Encoding.INSTANCE : UTF8Encoding.INSTANCE, syntax, warningCollector); + hasControlVerbState = regex.hasControlVerbs(); hasDeferredUserDefinedUnicodeProperty = deferredPropertyState.deferred; hasUserDefinedUnicodeProperty = deferredPropertyState.userDefined; NamedGroupMaps groupMaps = collectNamedGroups(regex); @@ -912,15 +912,6 @@ private static PerlSyntaxFeatures analyzePerlSyntax(String pattern, boolean exte alphaAssertionPresent, asciiStrictPresent); } - private static boolean hasControlVerbState(String pattern) { - return pattern.contains("(*MARK") || pattern.contains("(*:") - || pattern.contains("(*ACCEPT") || pattern.contains("(*FAIL") - || pattern.contains("(*F)") || pattern.contains("(*F:") - || pattern.contains("(*PRUNE") - || pattern.contains("(*SKIP") || pattern.contains("(*THEN") - || pattern.contains("(*COMMIT"); - } - /** * Test-facing compatibility normalizer. Production patterns retain raw * {@code \N{...}} source for Joni's resolver; this helper keeps its diff --git a/src/test/resources/unit/regex/control_verb_compiled_metadata.t b/src/test/resources/unit/regex/control_verb_compiled_metadata.t new file mode 100644 index 000000000..c605e5882 --- /dev/null +++ b/src/test/resources/unit/regex/control_verb_compiled_metadata.t @@ -0,0 +1,40 @@ +use strict; +use warnings; +use Test::More tests => 12; + +our ($REGMARK, $REGERROR); + +{ + local $REGMARK = 'sentinel mark'; + local $REGERROR = 'sentinel error'; + ok('ok' =~ /(*FAIL:backtracked)never|ok/, + 'real control verb can backtrack to ordinary success'); + is($REGMARK, '1', 'controlled success publishes true without a mark'); + is($REGERROR, '', 'controlled success clears a backtracked error'); +} + +{ + local $REGMARK = 'quoted mark'; + local $REGERROR = 'quoted error'; + ok('(*FAIL:quoted)' =~ /\Q(*FAIL:quoted)\E/, + 'quoted control-verb spelling is literal'); + is($REGMARK, 'quoted mark', 'quoted spelling leaves REGMARK untouched'); + is($REGERROR, 'quoted error', 'quoted spelling leaves REGERROR untouched'); +} + +{ + local $REGMARK = 'comment mark'; + local $REGERROR = 'comment error'; + ok('ok' =~ /(?# (*FAIL:commented)ok/, + 'commented control-verb spelling is ignored'); + is($REGMARK, 'comment mark', 'commented spelling leaves REGMARK untouched'); + is($REGERROR, 'comment error', 'commented spelling leaves REGERROR untouched'); +} + +{ + local $REGMARK = 'ordinary mark'; + local $REGERROR = 'ordinary error'; + ok('ok' =~ /ok/, 'ordinary pattern matches'); + is($REGMARK, 'ordinary mark', 'ordinary match leaves REGMARK untouched'); + is($REGERROR, 'ordinary error', 'ordinary match leaves REGERROR untouched'); +} diff --git a/third_party/joni/src/org/joni/Regex.java b/third_party/joni/src/org/joni/Regex.java index 69f668bd2..d55108db5 100644 --- a/third_party/joni/src/org/joni/Regex.java +++ b/third_party/joni/src/org/joni/Regex.java @@ -237,6 +237,11 @@ public boolean hasOnlyAuthoritativeWideCharacterClasses() { return true; } + /** Whether the compiled program contains at least one real control verb. */ + public boolean hasControlVerbs() { + return controlVerbLabels != null && controlVerbLabels.length > 0; + } + public int numberOfCaptureHistories() { if (Config.USE_CAPTURE_HISTORY) { int n = 0; diff --git a/third_party/joni/test/org/joni/TestControlVerbMetadata.java b/third_party/joni/test/org/joni/TestControlVerbMetadata.java new file mode 100644 index 000000000..6b3129a06 --- /dev/null +++ b/third_party/joni/test/org/joni/TestControlVerbMetadata.java @@ -0,0 +1,46 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.nio.charset.StandardCharsets; + +import org.jcodings.specific.UTF8Encoding; +import org.junit.Test; + +public class TestControlVerbMetadata { + private static Regex compile(String source) { + byte[] bytes = source.getBytes(StandardCharsets.UTF_8); + return new Regex(bytes, 0, bytes.length, Option.NONE, + UTF8Encoding.INSTANCE, Syntax.PerlNG); + } + + @Test + public void reportsCompiledControlVerbsInsteadOfTheirSourceSpellings() { + assertTrue(compile("(*FAIL:backtracked)never|ok").hasControlVerbs()); + assertTrue(compile("a(*MARK:seen)b").hasControlVerbs()); + + assertFalse(compile("\\Q(*FAIL:quoted)\\E").hasControlVerbs()); + assertFalse(compile("(?# (*FAIL:commented)ok").hasControlVerbs()); + assertFalse(compile("ordinary").hasControlVerbs()); + } +} From d335b39ee84bd2342e6e59e4bf9f9d416efd24ec Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 14:40:56 +0200 Subject: [PATCH 41/43] refactor(regex): use parsed charset option metadata Expose Joni's parsed positive inline Perl charset options and use that fact for byte-versus-Unicode matcher selection. Remove the raw pattern scanner so /u, /a, and /aa spellings inside comments cannot change case-fold semantics. Generated with Codex (https://openai.com/codex) Co-Authored-By: Codex --- .../runtime/regex/JoniRegexPattern.java | 4 ++ .../runtime/regex/RuntimeRegex.java | 55 ++----------------- .../regex/inline_charset_compiled_metadata.t | 17 ++++++ third_party/joni/src/org/joni/Parser.java | 6 ++ third_party/joni/src/org/joni/Regex.java | 6 ++ .../joni/TestPerlCharsetOptionMetadata.java | 50 +++++++++++++++++ 6 files changed, 87 insertions(+), 51 deletions(-) create mode 100644 src/test/resources/unit/regex/inline_charset_compiled_metadata.t create mode 100644 third_party/joni/test/org/joni/TestPerlCharsetOptionMetadata.java diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index 21c25ea49..80d2feee7 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -566,6 +566,10 @@ boolean hasOnlyAuthoritativeWideCharacterClasses() { return regex.hasOnlyAuthoritativeWideCharacterClasses(); } + boolean hasUnicodeCharsetModifier() { + return regex.hasUnicodeCharsetModifier(); + } + boolean hasDeferredUserDefinedUnicodeProperty() { return hasDeferredUserDefinedUnicodeProperty; } diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java index c811edc54..f2f192762 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java @@ -250,9 +250,10 @@ public RegexMatcher matcher(RuntimeScalar string, String input) { private JoniRegexPattern selectRecursivePattern(RuntimeScalar string) { boolean byteDefaultSemantics = patternByteBacked && regexFlags != null && !regexFlags.isUnicode() - && !regexFlags.isAscii() && !hasInlineUnicodeCaseFoldModifier(patternString); - if ((bytesSubstitution || byteDefaultSemantics) && recursivePatternBytes != null - && !Utf8.isUtf8(string)) { + && !regexFlags.isAscii(); + if (recursivePatternBytes != null && !Utf8.isUtf8(string) + && (bytesSubstitution || byteDefaultSemantics + && !recursivePatternBytes.hasUnicodeCharsetModifier())) { return recursivePatternBytes; } if (recursivePatternUnicode != null && recursivePatternUnicode != recursivePattern @@ -3714,54 +3715,6 @@ private static boolean hasUnicodeChars(String s) { return false; } - /** - * A scoped {@code /u}, {@code /a}, or {@code /aa} group must use the UTF-8 - * Joni variant even when its containing qr// originated in a byte scalar. - * The byte variant carries {@code PERL_BYTE_PATTERN}, whose global - * case-fold policy deliberately excludes Latin-1 folds; that policy cannot - * be relaxed by a nested charset group. - */ - private static boolean hasInlineUnicodeCaseFoldModifier(String pattern) { - if (pattern == null) return false; - - boolean escaped = false; - boolean inClass = false; - for (int i = 0; i + 2 < pattern.length(); i++) { - char current = pattern.charAt(i); - if (escaped) { - escaped = false; - continue; - } - if (current == '\\') { - escaped = true; - continue; - } - if (current == '[') { - inClass = true; - continue; - } - if (current == ']' && inClass) { - inClass = false; - continue; - } - if (inClass || current != '(' || pattern.charAt(i + 1) != '?') continue; - - boolean negated = false; - for (int j = i + 2; j < pattern.length(); j++) { - char modifier = pattern.charAt(j); - if (modifier == ':' || modifier == ')') break; - if (modifier == '-') { - negated = true; - continue; - } - if (modifier == '^') continue; - if (modifier < 'a' || modifier > 'z') break; - if ((modifier == 'u' || modifier == 'a') && !negated) return true; - } - } - return false; - } - private record ResolvedRegex(RuntimeRegex regex, boolean fromCompiledRegex) {} private static RegexFlags mergeOperationFlags(RegexFlags baseFlags, String modifiers, String patternString) { diff --git a/src/test/resources/unit/regex/inline_charset_compiled_metadata.t b/src/test/resources/unit/regex/inline_charset_compiled_metadata.t new file mode 100644 index 000000000..e0f159885 --- /dev/null +++ b/src/test/resources/unit/regex/inline_charset_compiled_metadata.t @@ -0,0 +1,17 @@ +use strict; +use warnings; +use Test::More tests => 8; + +my $upper = "\xC0"; +my $lower = "\xE0"; + +ok(!utf8::is_utf8($upper), 'upper subject is byte-backed'); +ok(!utf8::is_utf8($lower), 'lower subject is byte-backed'); +ok($lower !~ qr/$upper/i, 'byte default suppresses Latin-1 case fold'); +ok($lower !~ qr/$upper(?# (?u:commented)/i, + 'commented Unicode modifier does not change byte folding'); +ok($lower =~ qr/(?u:$upper)/i, 'scoped Unicode modifier enables Latin-1 fold'); +ok($lower =~ qr/(?a:$upper)/i, 'scoped ASCII modifier keeps non-ASCII fold'); +ok($lower =~ qr/(?aa:$upper)/i, 'scoped strict ASCII keeps non-ASCII fold'); +ok($lower !~ qr/\Q(?u:$upper)\E/i, + 'quoted modifier spelling remains literal'); diff --git a/third_party/joni/src/org/joni/Parser.java b/third_party/joni/src/org/joni/Parser.java index 3a4cc527b..6ba0becda 100644 --- a/third_party/joni/src/org/joni/Parser.java +++ b/third_party/joni/src/org/joni/Parser.java @@ -1758,9 +1758,15 @@ && left() && enc.isDigit(peek())) { case 'a': /* limits \d, \s, \w and POSIX brackets to ASCII range */ option = charsetOptions.apply(option, c, neg, false); + if (!neg && syntax.op2OptionPerl()) { + regex.hasUnicodeCharsetModifier = true; + } break; case 'u': option = charsetOptions.apply(option, c, neg, false); + if (!neg && syntax.op2OptionPerl()) { + regex.hasUnicodeCharsetModifier = true; + } break; case 'd': diff --git a/third_party/joni/src/org/joni/Regex.java b/third_party/joni/src/org/joni/Regex.java index d55108db5..ea3de41bb 100644 --- a/third_party/joni/src/org/joni/Regex.java +++ b/third_party/joni/src/org/joni/Regex.java @@ -48,6 +48,7 @@ public final class Regex { int codeLength; boolean requireStack; boolean hasDynamicOptions; + boolean hasUnicodeCharsetModifier; int numMem; /* used memory(...) num counted from 1 */ int numPhysicalNamedCaptures; @@ -242,6 +243,11 @@ public boolean hasControlVerbs() { return controlVerbLabels != null && controlVerbLabels.length > 0; } + /** Whether parsing encountered a real positive inline Perl /u, /a, or /aa. */ + public boolean hasUnicodeCharsetModifier() { + return hasUnicodeCharsetModifier; + } + public int numberOfCaptureHistories() { if (Config.USE_CAPTURE_HISTORY) { int n = 0; diff --git a/third_party/joni/test/org/joni/TestPerlCharsetOptionMetadata.java b/third_party/joni/test/org/joni/TestPerlCharsetOptionMetadata.java new file mode 100644 index 000000000..43f453b05 --- /dev/null +++ b/third_party/joni/test/org/joni/TestPerlCharsetOptionMetadata.java @@ -0,0 +1,50 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.joni; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.nio.charset.StandardCharsets; + +import org.jcodings.specific.ISO8859_1Encoding; +import org.junit.Test; + +public class TestPerlCharsetOptionMetadata { + private static Regex compile(String source) { + byte[] bytes = source.getBytes(StandardCharsets.ISO_8859_1); + return new Regex(bytes, 0, bytes.length, + Option.IGNORECASE | Option.PERL_BYTE_PATTERN, + ISO8859_1Encoding.INSTANCE, Syntax.PerlNG); + } + + @Test + public void reportsParsedCharsetOptionsInsteadOfTheirSourceSpellings() { + assertTrue(compile("(?u:\\xC0)").hasUnicodeCharsetModifier()); + assertTrue(compile("(?a:\\xC0)").hasUnicodeCharsetModifier()); + assertTrue(compile("(?aa:\\xC0)").hasUnicodeCharsetModifier()); + + assertFalse(compile("(?# (?u:commented)\\xC0") + .hasUnicodeCharsetModifier()); + assertFalse(compile("\\Q(?u:quoted)\\E") + .hasUnicodeCharsetModifier()); + assertFalse(compile("ordinary").hasUnicodeCharsetModifier()); + } +} From 18bbbdad74dcfefe710ce6c2ebff2528b91c9dcc Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 14:45:42 +0200 Subject: [PATCH 42/43] refactor(regex): defer lookaround KEEP validation to Joni Remove the adapter's production pre-scan for \K inside lookarounds. The native Joni analyser already owns and tests the same Perl diagnostic, while the legacy syntax scanner remains available only to compatibility routing tests. Generated with Codex (https://openai.com/codex) Co-Authored-By: Codex --- .../java/org/perlonjava/runtime/regex/JoniRegexPattern.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index 80d2feee7..899c63b07 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -317,10 +317,6 @@ private static CharacterPropertyResolver.Result resolveCharacterProperty( boolean byteBackedPattern, NamedCharacterCache namedCharacterCache, NamedCharacterExpansion.SourceMode namedCharacterSourceMode, boolean perlReStrict) { - PerlSyntaxFeatures syntaxFeatures = analyzePerlSyntax(perlPattern, flags.isExtended()); - if (syntaxFeatures.keepInLookaround()) { - throw new PerlCompilerException("\\K not permitted in lookahead/lookbehind in regex"); - } validateExtendedPropertyPolicy(perlPattern); this.flags = flags; this.byteMode = byteMode; From 60d4d35cc2b27acd5f4f0011cf383a1b7c9bcbe9 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Fri, 21 Aug 2026 15:20:39 +0200 Subject: [PATCH 43/43] fix(regex): retain unnamed control verb metadata Record the parser's control-verb fact directly on the compiled Joni Regex so unnamed verbs do not disappear when the named-label table is empty. Add direct Joni and Perl regressions and refresh the post-1091 execution plan. Generated with [OpenAI Codex](https://openai.com/codex/) Co-Authored-By: OpenAI Codex --- dev/design/phase36-regex-parity.md | 112 +++++++----------- .../regex/control_verb_compiled_metadata.t | 11 +- third_party/joni/src/org/joni/Analyser.java | 1 + third_party/joni/src/org/joni/Regex.java | 3 +- .../org/joni/TestControlVerbMetadata.java | 2 + 5 files changed, 56 insertions(+), 73 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index 1442164ac..3863eccfe 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -64,9 +64,8 @@ Remaining scanner-removal queue: match-all text substitutions used to represent deferred properties. - [ ] Preserve ordinary/extended-class context and source positions in Joni so the adapter no longer scans bracket depth or generates replacement classes. -- [ ] Replace `hasControlVerbState(String)` with a compiled Joni fact, and move - the `\K`-inside-lookaround diagnostic into Joni before deleting the remaining - production `analyzePerlSyntax` scan. +- [x] Replace `hasControlVerbState(String)` with a compiled Joni fact, including + unnamed verbs, and move the `\K`-inside-lookaround diagnostic into Joni. - [ ] Delete test-only routing scanners (`requiresJoniBackend` and its empty- class/name helpers) once their assertions are replaced by direct Joni facts. - [ ] Retain only runtime-neutral trusted-callout materialization and documented @@ -176,50 +175,27 @@ Exit: release evidence and public/internal documentation match the code. ## Current Release Gate -PR 1091 is the sole release and user-acceptance target at exact head -`ff68371af`. It contains all 332 range-diff-equivalent PR 1087 patches rebased -onto current `master`, preserves the user's latest-Perl sync commit, and includes -the six release fixes formerly staged in PR 1090. Imported fixtures remain -authoritative and must not be reverted or patched to recover old counts. The -exact unified head has a clean warning-free full `make`, and Ubuntu and Windows -CI are green. The user's complete production-load acceptance is in progress. -Workers must not mutate, rebase, or push this immutable test branch. Acceptance -rejects every new invalid, missing, timeout, truncated, incomplete, or zero-TAP -row and every unresolved PR-958 pass-count decrease. - -In the live ten-process run, `pat.t` completed at 1249/1302 in 568.99s, -`pat_thr.t` reached the 600s outer watchdog, and `anyof.t` reached the 300s -watchdog. Rerun only `pat_thr.t` and `anyof.t` at exact PR 1091 head with 900s -per-file timeouts; do not restart the full corpus. The post-1091 runner now -gives both `pat.t` variants a 900s production-load-safe minimum and the -`anyof.t` variants a 600s minimum. - -In writable current-source state `op/do.t` is 71/71 on both backends; -`class/accessor.t` executes all 30 rows and improves to 17 passes versus the -PR-958 12-pass floor; isolated JAPH is deterministically 109/130 with ID 51 -matching current Perl's upstream regression. Aggregate binding warnings and -fatal construction diagnostics cover the two assertions missing from the -initial `pat*` localization. The clean serial exact-head gate executes -all 1302 rows in each file at 1249/1302 for both `pat.t` and `pat_thr.t`, with -zero errors, timeouts, or incomplete rows. - -Exact-head packaging acceptance is complete: full `make`, standalone Joni -packaging verification, notices/licenses, merged SBOM, and artifact hashes are -green. The executable tree and ordered 28,518-entry JAR inventory are identical -to the predecessor CPAN-tested artifact, so its four-module offline matrix -transfers without a redundant rerun. - -Independent implementation continues on `integrate/phase36-post1087-wip` at -`9bbd3f18e` or later. Unicode property closure, lexical package propagation, -named-character source mode, and overloaded-subject handling are integrated. -Typed native Joni debug facts now expose semantic instructions and immutable -character-class membership. Property rows 1662–1664 and 1685, the nine named- -character fold/trie rows, and overloaded-subject row 379 are closed in focused -worker maps. The expected remaining `pat_advanced.t` frontier is debug rows -1673–1683; an exact combined-head map must confirm that frontier after the -display-provenance consumer lands. This WIP stays separate until PR 1091 -merges, then rebases onto the new `master` before combined acceptance and its -PR update. +PR 1091 is merged into `master` at merge commit `4203bf811`. Its exact accepted +head `ff68371af` is the immutable 623-file release baseline: full `make`, Ubuntu, +Windows, packaging, notices/licenses, and strict PR-958 comparison are green. +Imported fixtures remain authoritative and must not be patched to recover old +counts. + +The production-load run left two mandatory post-merge repairs. `pat_thr.t` +reached row 1192 with exactly the known semantic failures before the 600-second +watchdog; it must complete in reasonable time under ten-process load, with the +new 900-second floor serving only as a safety net. DBIx::Class installation +also exposed two unexpected uninitialized-match warnings in its caller walk; +the repair must be driven by a small system-Perl reducer, not the expensive +distribution suite. + +Development continues in draft PR 1089 on +`integrate/phase36-post1087-wip`, rebased onto merged `master`. Integrated work +includes Unicode property closure, lexical package propagation, named-character +source mode, overloaded-subject handling, typed Joni debug facts, compiled +control-verb/charset metadata, and native `\K`-lookaround diagnostics. The next +published head must pass one combined warning-free `make` before workers +transplant their current deliveries. ### Execution Tracker @@ -241,41 +217,35 @@ PR update. Active ownership: -- P3: finish native Perl display provenance for rows 1673–1683 and prove the - complete 1,187-row `anyof.t` and 1,687-row `pat_advanced.t` maps. -- P5: move extended-class string/deferred-property policy and source positions - into Joni, delete `validateExtendedPropertyPolicy`, and close the remaining - property-specific scanner inventory. -- P6: own the complete non-property, non-debug parser/diagnostic and first - matcher-semantic scanner-removal tranche, starting with replacing the - control-verb source scan by a compiled Joni fact. -- P4: fenced from the parser/scanner tranche pending restoration of its mailbox - monitor; it has no authoritative implementation ownership. +- P3: finish native Perl display provenance and prove complete `anyof.t` and + `pat_advanced.t` maps, then transplant the committed renderer delivery. +- P4: reduce and fix the DBIx caller/warning defect without running the full + distribution suite. +- P5: integrate the extended-property context delivery, repair the four + Quick_Check assigned aliases, and close the property scanner inventory. +- P6: profile and fix loaded `pat.t`/`pat_thr.t` completion, preserving direct + counts and proving representative ten-process behavior. - Coordinator: integration, conflict resolution, immutable acceptance, PR/CI, - plan state, combined build, and release evidence. + plan state, combined builds, worker rebasing, and release evidence. ## Ordered Next Steps -1. Keep PR 1091 immutable while the user runs the complete latest-Perl corpus, - strict PR-958 comparison, additional release checks, and exact-head CI. -2. Merge PR 1091 directly to `master` only when every checkpoint gate is green - or has a documented baseline-equivalent environmental disposition; then - close PRs 1087 and 1090 as superseded. -3. Meanwhile integrate P3 debug/matcher, P4 parser/scanner, P5 property, and P6 - acceptance deliveries only into the separate post-1091 WIP. Batch focused - tests and one combined full build, then rebase this WIP onto merged `master`. -4. Delete remaining production migration scaffolding and prove all constants, +1. Publish the rebased post-1091 WIP after focused compiled-metadata tests and + one warning-free combined `make`; close PRs 1087 and 1090 as superseded. +2. Transplant the P3/P5 deliveries and the P4/P6 post-merge repairs onto that + exact base, resolve overlaps centrally, and refresh complete affected maps. +3. Delete remaining production migration scaffolding and prove all constants, closures, conditions, verbs, recursion, dynamic source, byte strings, and Unicode strings execute through Joni. -5. Run complete JVM/interpreter acceptance, direct/thread parity, affected CPAN +4. Run complete JVM/interpreter acceptance, direct/thread parity, affected CPAN suites, five warmed performance samples, packaging, notices/licenses, warning-free build, and platform CI. -6. Reconcile final documentation and remove redundant design material. -7. After the final implementation PR is merged to `master`, remove automatic +5. Reconcile final documentation and remove redundant design material. +6. After the final implementation PR is merged to `master`, remove automatic regex `JPERL_UNIMPLEMENTED=warn` injection from `dev/tools/perl_test_runner.pl`; rerun the complete corpus; then delete the RuntimeRegex warning-plus-never-match downgrade and obsolete tests/docs. -8. On final `master`, review shipped behavior against +7. On final `master`, review shipped behavior against `pod/perlreref.pod`, `pod/perlrecharclass.pod`, `pod/perlrequick.pod`, `pod/perlrepository.pod`, `pod/perlre.pod`, `pod/perlretut.pod`, and `pod/perlrebackslash.pod`. Record every diff --git a/src/test/resources/unit/regex/control_verb_compiled_metadata.t b/src/test/resources/unit/regex/control_verb_compiled_metadata.t index c605e5882..8ed1e8a46 100644 --- a/src/test/resources/unit/regex/control_verb_compiled_metadata.t +++ b/src/test/resources/unit/regex/control_verb_compiled_metadata.t @@ -1,6 +1,6 @@ use strict; use warnings; -use Test::More tests => 12; +use Test::More tests => 15; our ($REGMARK, $REGERROR); @@ -13,6 +13,15 @@ our ($REGMARK, $REGERROR); is($REGERROR, '', 'controlled success clears a backtracked error'); } +{ + local $REGMARK = 'unnamed mark'; + local $REGERROR = 'unnamed error'; + ok('ok' =~ /ok|(*FAIL)never/, + 'pattern with an untraversed unnamed control verb can succeed'); + is($REGMARK, '1', 'unnamed control-verb metadata publishes true'); + is($REGERROR, '', 'unnamed control-verb metadata clears REGERROR'); +} + { local $REGMARK = 'quoted mark'; local $REGERROR = 'quoted error'; diff --git a/third_party/joni/src/org/joni/Analyser.java b/third_party/joni/src/org/joni/Analyser.java index 36ec09ffe..8a2b00dfb 100644 --- a/third_party/joni/src/org/joni/Analyser.java +++ b/third_party/joni/src/org/joni/Analyser.java @@ -123,6 +123,7 @@ protected final void compile() { regex.captureHistory = env.captureHistory; regex.btMemStart = env.btMemStart; + regex.hasControlVerb = env.hasControlVerb; if (env.hasControlVerb) { // ACCEPT needs stack markers to distinguish captures opened inside // its nearest assertion or call boundary from captures outside it. diff --git a/third_party/joni/src/org/joni/Regex.java b/third_party/joni/src/org/joni/Regex.java index ea3de41bb..37984df01 100644 --- a/third_party/joni/src/org/joni/Regex.java +++ b/third_party/joni/src/org/joni/Regex.java @@ -102,6 +102,7 @@ public final class Regex { byte[][]templates; /* fixed pattern strings not embedded in bytecode */ int templateNum; + boolean hasControlVerb; String[] controlVerbLabels; CClassNode[] wideScalarClasses; final WideScalarCodec wideScalarCodec; @@ -240,7 +241,7 @@ public boolean hasOnlyAuthoritativeWideCharacterClasses() { /** Whether the compiled program contains at least one real control verb. */ public boolean hasControlVerbs() { - return controlVerbLabels != null && controlVerbLabels.length > 0; + return hasControlVerb; } /** Whether parsing encountered a real positive inline Perl /u, /a, or /aa. */ diff --git a/third_party/joni/test/org/joni/TestControlVerbMetadata.java b/third_party/joni/test/org/joni/TestControlVerbMetadata.java index 6b3129a06..d3244e8f5 100644 --- a/third_party/joni/test/org/joni/TestControlVerbMetadata.java +++ b/third_party/joni/test/org/joni/TestControlVerbMetadata.java @@ -38,6 +38,8 @@ private static Regex compile(String source) { public void reportsCompiledControlVerbsInsteadOfTheirSourceSpellings() { assertTrue(compile("(*FAIL:backtracked)never|ok").hasControlVerbs()); assertTrue(compile("a(*MARK:seen)b").hasControlVerbs()); + assertTrue(compile("ok|(*FAIL)never").hasControlVerbs()); + assertTrue(compile("ok|(*ACCEPT)").hasControlVerbs()); assertFalse(compile("\\Q(*FAIL:quoted)\\E").hasControlVerbs()); assertFalse(compile("(?# (*FAIL:commented)ok").hasControlVerbs());