From 471978f9f5344b797acf99282183b7bb9ae59b2b Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Tue, 18 Aug 2026 20:28:30 +0200 Subject: [PATCH 01/31] chore: refresh Perl 5 imports and patch provenance Refresh configured imports from Perl blead revision 35694276f3eb4e8b7b3ce4cb2e9fa8a5cf92a03a. Encode existing PerlOnJava App::Cpan, IPC::Cmd, charname, and overloading adaptations in the import patches so a full sync remains idempotent and does not remove runtime behavior. Generated with [OpenAI Codex](https://openai.com/codex/) Co-Authored-By: OpenAI Codex --- dev/import-perl5/config.yaml | 1 + dev/import-perl5/patches/App-Cpan.pm.patch | 34 ++++-- dev/import-perl5/patches/IPC-Cmd.pm.patch | 82 ++++++++++++- dev/import-perl5/patches/_charnames.pm.patch | 22 +++- dev/import-perl5/patches/overloading.pm.patch | 10 ++ src/main/perl/lib/I18N/LangTags/List.pm | 1 + src/main/perl/lib/Pod/perl5450delta.pod | 4 +- src/main/perl/lib/Pod/perldelta.pod | 5 + src/main/perl/lib/Pod/perlguts.pod | 5 +- src/main/perl/lib/Pod/perlhacktips.pod | 114 +++++++++++++----- 10 files changed, 225 insertions(+), 53 deletions(-) create mode 100644 dev/import-perl5/patches/overloading.pm.patch diff --git a/dev/import-perl5/config.yaml b/dev/import-perl5/config.yaml index 86c60e4871..fa4de75d5c 100644 --- a/dev/import-perl5/config.yaml +++ b/dev/import-perl5/config.yaml @@ -927,6 +927,7 @@ imports: # overloading pragma - Lexically disable overloading - source: perl5/lib/overloading.pm target: src/main/perl/lib/overloading.pm + patch: overloading.pm.patch # Term::ANSIColor - ANSI terminal color support (used by various tests) - source: perl5/cpan/Term-ANSIColor/lib/Term/ANSIColor.pm diff --git a/dev/import-perl5/patches/App-Cpan.pm.patch b/dev/import-perl5/patches/App-Cpan.pm.patch index cd789092d3..06e8291780 100644 --- a/dev/import-perl5/patches/App-Cpan.pm.patch +++ b/dev/import-perl5/patches/App-Cpan.pm.patch @@ -1,14 +1,20 @@ ---- perl5/cpan/CPAN/lib/App/Cpan.pm 2026-04-10 11:04:21 -+++ src/main/perl/lib/App/Cpan.pm 2026-08-10 10:12:19 -@@ -652,6 +652,7 @@ +--- perl5/cpan/CPAN/lib/App/Cpan.pm ++++ src/main/perl/lib/App/Cpan.pm +@@ -647,11 +647,13 @@ + $logger->error( "Skipping $arg because I couldn't find a matching namespace." ); + next; + }; ++ my $requested_distribution_id = eval { $module->distribution->id }; + + _clear_cpanpm_output(); $action->( $arg ); my $error = _cpanpm_output_indicates_failure(); -+ $error ||= _cpanpm_status_indicates_failure(); ++ $error ||= _cpanpm_status_indicates_failure($requested_distribution_id); push @errors, $error if $error; } -@@ -752,7 +753,7 @@ +@@ -752,7 +754,7 @@ BEGIN { my $epic_fail_words = join '|', @@ -17,21 +23,29 @@ fail(?:ed)? Cannot\s+install ); sub _cpanpm_output_indicates_failure -@@ -763,6 +764,17 @@ - return A_MODULE_FAILED_TO_INSTALL if $last_line =~ /\b(?:Cannot\s+install)\b/i; +@@ -764,6 +766,25 @@ $result || (); -+ } + } + +sub _cpanpm_status_indicates_failure + { ++ my $requested_distribution_id = shift; ++ $requested_distribution_id =~ s{^./../}{} ++ if defined $requested_distribution_id; ++ + # CPAN already records structured phase status for every distribution in + # the current command, including recursively installed prerequisites. + # Prefer that state when App::Cpan's legacy last-output-line heuristic is + # fooled by trailing hints or report suggestions. + my @failed = CPAN::Shell->find_failed($CPAN::CurrentCommandId); -+ return A_MODULE_FAILED_TO_INSTALL if grep { $_->[5] } @failed; ++ return A_MODULE_FAILED_TO_INSTALL if grep { ++ $_->[5] ++ || (defined $requested_distribution_id ++ && $_->[1] eq $requested_distribution_id) ++ } @failed; + return; - } ++ } } + sub _cpanpm_output_indicates_success diff --git a/dev/import-perl5/patches/IPC-Cmd.pm.patch b/dev/import-perl5/patches/IPC-Cmd.pm.patch index 9c8115568b..1ae5fbdd1a 100644 --- a/dev/import-perl5/patches/IPC-Cmd.pm.patch +++ b/dev/import-perl5/patches/IPC-Cmd.pm.patch @@ -1,8 +1,14 @@ --- perl5/cpan/IPC-Cmd/lib/IPC/Cmd.pm +++ src/main/perl/lib/IPC/Cmd.pm -@@ -8,5 +8,9 @@ +@@ -7,7 +7,14 @@ + use constant IS_VMS => $^O eq 'VMS' ? 1 : 0; + use constant IS_WIN32 => $^O eq 'MSWin32' ? 1 : 0; use constant IS_HPUX => $^O eq 'hpux' ? 1 : 0; - use constant IS_WIN98 => (IS_WIN32 and !Win32::IsWinNT()) ? 1 : 0; +- use constant IS_WIN98 => (IS_WIN32 and !Win32::IsWinNT()) ? 1 : 0; ++ use constant IS_WIN98 => (IS_WIN32 and do { ++ require Win32; ++ !Win32::IsWinNT(); ++ }) ? 1 : 0; + use constant IS_PERLONJAVA => do { + require Config; + $Config::Config{perlonjava} ? 1 : 0; @@ -10,7 +16,7 @@ use constant ALARM_CLASS => __PACKAGE__ . '::TimeOut'; use constant SPECIAL_CHARS => qw[< > | &]; use constant QUOTE => do { IS_WIN32 ? q["] : q['] }; -@@ -38,7 +42,7 @@ +@@ -38,7 +45,7 @@ require Time::HiRes; Time::HiRes->import(); require Win32 if IS_WIN32; }; @@ -19,3 +25,73 @@ eval { my $wait_start_time = Time::HiRes::clock_gettime(&Time::HiRes::CLOCK_MONOTONIC); +@@ -190,6 +197,7 @@ + sub can_capture_buffer { + my $self = shift; + ++ return 1 if IS_PERLONJAVA; + return 1 if $USE_IPC_RUN && $self->can_use_ipc_run; + return 1 if $USE_IPC_OPEN3 && $self->can_use_ipc_open3; + return; +@@ -1345,9 +1353,18 @@ + qq[: Command '$pp_cmd' aborted by alarm after $timeout seconds] + }, ALARM_CLASS } if $timeout; + alarm $timeout || 0; ++ ++ ### PerlOnJava has no fork(), so forced IPC::Run/Open3 selection must ++ ### still use the JVM ProcessBuilder backend. ++ if (IS_PERLONJAVA) { ++ $self->_debug("# Using PerlOnJava::Process. Have buffer: $have_buffer") ++ if $DEBUG; ++ $ok = $self->_perlonjava_run( ++ $cmd, $_out_handler, $_err_handler, $timeout, $verbose ++ ); + + ### IPC::Run is first choice if $USE_IPC_RUN is set. +- if( !IS_WIN32 and $USE_IPC_RUN and $self->can_use_ipc_run( 1 ) ) { ++ } elsif( !IS_WIN32 and $USE_IPC_RUN and $self->can_use_ipc_run( 1 ) ) { + ### ipc::run handlers needs the command as a string or an array ref + + $self->_debug( "# Using IPC::Run. Have buffer: $have_buffer" ) +@@ -1407,8 +1424,41 @@ + ? ($ok, $err, \@buffer, \@buff_out, \@buff_err) + : ($ok, $err ) + : $ok ++ ++ ++} ++ ++sub _perlonjava_run { ++ my ($self, $cmd, $outhand, $errhand, $timeout, $verbose) = @_; + ++ require Config; ++ require PerlOnJava::Process; ++ my $argv = ref($cmd) eq 'ARRAY' ? [@$cmd] : IS_WIN32 ++ ? [($ENV{COMSPEC} || 'cmd.exe'), '/d', '/s', '/c', $cmd] ++ : [($Config::Config{sh} || '/bin/sh'), '-c', $cmd]; + ++ my $result = PerlOnJava::Process::run_process( ++ argv => $argv, ++ timeout => $timeout, ++ tee => 0, ++ ); ++ $outhand->($result->{stdout}) if length($result->{stdout} // ''); ++ $errhand->($result->{stderr}) if length($result->{stderr} // ''); ++ ++ if ($result->{timed_out}) { ++ $self->error(loc("Command '%1' timed out", ref($cmd) ? "@$cmd" : $cmd)); ++ return $self->ok(0); ++ } ++ if (length($result->{error} // '')) { ++ $self->error($result->{error}); ++ return $self->ok(0); ++ } ++ if (($result->{exit_code} // -1) != 0) { ++ $self->error(loc("Command '%1' exited with value %2", ++ ref($cmd) ? "@$cmd" : $cmd, $result->{exit_code})); ++ return $self->ok(0); ++ } ++ return $self->ok(1); + } + + sub _open3_run_win32 { diff --git a/dev/import-perl5/patches/_charnames.pm.patch b/dev/import-perl5/patches/_charnames.pm.patch index a7e682e0f8..4d40fa4e17 100644 --- a/dev/import-perl5/patches/_charnames.pm.patch +++ b/dev/import-perl5/patches/_charnames.pm.patch @@ -1,5 +1,5 @@ ---- perl5/lib/_charnames.pm 2025-12-11 14:13:49 -+++ src/main/perl/lib/_charnames.pm 2026-04-08 09:40:13 +--- perl5/lib/_charnames.pm ++++ src/main/perl/lib/_charnames.pm @@ -137,6 +137,12 @@ return if $txt; @@ -13,7 +13,23 @@ Internals::SvREADONLY($txt, 1); } -@@ -805,6 +811,16 @@ +@@ -422,6 +428,15 @@ + { + $result = chr $ord; + } ++ # PerlOnJava bundles ICU4J, whose Unicode name database is complete and ++ # current. Use it for strict official-name lookup before falling back to ++ # the generated Perl table, just as viacode() does for reverse lookup. ++ elsif (! $loose && $^H{charnames_full} && defined &_java_vianame ++ && defined(my $java_ord = _java_vianame($lookup_name))) ++ { ++ $result = chr $java_ord; ++ $full_names_cache{$name} = $result; ++ } + else { + + # Not algorithmically determinable; look up in the table. The name +@@ -805,6 +820,16 @@ if (defined $algorithmic) { $viacode{$hex} = $algorithmic; return $algorithmic; diff --git a/dev/import-perl5/patches/overloading.pm.patch b/dev/import-perl5/patches/overloading.pm.patch new file mode 100644 index 0000000000..137a8de422 --- /dev/null +++ b/dev/import-perl5/patches/overloading.pm.patch @@ -0,0 +1,10 @@ +--- perl5/lib/overloading.pm ++++ src/main/perl/lib/overloading.pm +@@ -27,6 +27,7 @@ + delete $^H{overloading}; + $^H &= ~$HINT_NO_AMAGIC; + } ++ + } + + sub unimport ($, @ops) { diff --git a/src/main/perl/lib/I18N/LangTags/List.pm b/src/main/perl/lib/I18N/LangTags/List.pm index 005c2eb9d5..6bbb1980a0 100644 --- a/src/main/perl/lib/I18N/LangTags/List.pm +++ b/src/main/perl/lib/I18N/LangTags/List.pm @@ -28,6 +28,7 @@ our $VERSION = '0.42'; $Is_Disrec{$1} = 1; } } + close DATA; die "No tags read??" unless $count; } #---------------------------------------------------------------------- diff --git a/src/main/perl/lib/Pod/perl5450delta.pod b/src/main/perl/lib/Pod/perl5450delta.pod index 8edcaca4cc..2df22e1a95 100644 --- a/src/main/perl/lib/Pod/perl5450delta.pod +++ b/src/main/perl/lib/Pod/perl5450delta.pod @@ -168,7 +168,7 @@ accuracy will hopefully follow within this development cycle. =head1 Acknowledgements -Perl 5.45.1 represents approximately 1 week of development since Perl 5.44.0 +Perl 5.45.0 represents approximately 1 week of development since Perl 5.44.0 and contains approximately 36,000 lines of changes across 470 files from 19 authors. @@ -177,7 +177,7 @@ approximately 12,000 lines of changes to 310 .pm, .t, .c and .h files. Perl continues to flourish into its fourth decade thanks to a vibrant community of users and developers. The following people are known to have -contributed the improvements that became Perl 5.45.1: +contributed the improvements that became Perl 5.45.0: Andrew Fresh, Chad Granum, Chris 'BinGOs' Williams, Craig A. Berry, Dagfinn Ilmari Mannsåker, David Mitchell, Georgij Tsarin, Graham Knop, James E diff --git a/src/main/perl/lib/Pod/perldelta.pod b/src/main/perl/lib/Pod/perldelta.pod index a31d60e0a0..d486bfec25 100644 --- a/src/main/perl/lib/Pod/perldelta.pod +++ b/src/main/perl/lib/Pod/perldelta.pod @@ -319,6 +319,11 @@ made: =item * +F: Corrected to now check for entries in +F that no longer need to be there. + +=item * + XXX =back diff --git a/src/main/perl/lib/Pod/perlguts.pod b/src/main/perl/lib/Pod/perlguts.pod index 658ce80aa4..c9250d939d 100644 --- a/src/main/perl/lib/Pod/perlguts.pod +++ b/src/main/perl/lib/Pod/perlguts.pod @@ -3240,7 +3240,7 @@ bits. =head2 Background and MULTIPLICITY =for apidoc_section $concurrency -=for apidoc Amnh||PERL_IMPLICIT_CONTEXT +=for apidoc Amnh||MULTIPLICITY The Perl interpreter can be regarded as a closed box: it has an API for feeding it code or otherwise making it do things, but it also has @@ -3258,9 +3258,6 @@ ithreads threading model, related to the macro USE_ITHREADS.) PERL_IMPLICIT_CONTEXT is a legacy synonym for MULTIPLICITY. -=for apidoc_section $concurrency -=for apidoc Amnh||MULTIPLICITY - To see whether you have non-const data you can use a BSD (or GNU) compatible C: diff --git a/src/main/perl/lib/Pod/perlhacktips.pod b/src/main/perl/lib/Pod/perlhacktips.pod index 7b5365e577..9b581dfbb8 100644 --- a/src/main/perl/lib/Pod/perlhacktips.pod +++ b/src/main/perl/lib/Pod/perlhacktips.pod @@ -396,50 +396,100 @@ with C, C, or C, or ending with C<_pl_>. =head3 Symbol visibility For most of its life, Perl made little or no effort to hide its internal -symbols or functions. This has led to programmers using Perl to use -functionality that was dependent on Perl internal implementation -details, breaking when we unknowingly tried to change our -implementation, and thus hindering progress. - -That has been changing in recent releases, and as of v5.44, the -visibility of new symbols is restricted to just the perl core, unless an -explicit declaration is made otherwise. (Except symbol names which -match the pattern C<$names_reserved_for_perl_use_re> found in -F are made visible everywhere.) This means you can add -symbols with whatever C-compliant spelling you want, without fear that -they will be misused by someone. +symbols or functions. This made it easy for programmers to, even +inadvertently, use functionality that depended on Perl's internal +implementation details. When we tried to change those details, user +functionality often broke, thus creating frictions, and slowing down +adding improvements. -Note that symbols not placed in header files have never been visible to -outside code. +That has been changing in recent releases. In 5.38, the default +visibility of functions was changed to automatically be restricted to +just the Perl core. This was extended in 5.44 to newly added macros. +This means you can add these kinds of symbols with whatever C-compliant +spelling you want, without fear that they will clash with a user's +existing symbol or be misused by someone, + +You can override the default to make any symbol more visible as follows: + +=over 4 + +=item * To everyone + +=item * To modules considered to be Perl extensions + +=item * To just the regular expression (C) extension. + +=back + +The methods to do this vary depending on the type of symbol. + +=over 4 + +=item Functions -You should consider several things before making a new symbol visible. -The bottom line is "Who really needs to see it?" +Every function not static to a single file must have an entry in +F. Each entry will have various flags that apply to it, +including some that determine its visibility. They are listed in the +comments at the beginning of the file. -The best method is to document the symbol. How to do this is described -near the top of F, and that file can be used to mark a -symbol's visibility. But the main documentation remains using -C<=for apidoc> lines in the source and various pods. There are several -advantages to doing this +To restrict the visibilty to just the C module, more work is needed, +in the form of C preprocessor conditionals surrounding the entry, like +so: + + # if defined(PERL_EXT_RE_BUILD) + E...|return-type|function-name|arguments + # endif + +The C flag restricts the symbol's visibility to Perl Extensions; and +the #ifdef further to just the C module. + +Static functions are not required to have an F entry, but +doing so is encouraged, and has some advantages, and no downsides: =over =item 1 -People will know how to use your symbol without having to puzzle it out -from the code. That might even be you 6 months from now. +The function can be referred to by its short name, without having to +consider if it needs a thread-context parameter. =item 2 -Various services are automatically generated for symbols naming -functions, such as Cs for parameter input conditions. +Various services are automatically generated for functions, such as +Cs for parameter sanity checking. + +=back + +New services keep getting added, which mainly automatically will be +applied to your function without any effort on your part. + +=item Macros -One service is that specifying the visibility with one of the flags for -the purpose automatically makes sure the symbol has that visibility -without you having to do anything else. +You can put an entry for it in F, just like a function, but +adding the C (for macro) flag. -=item 3 +But it is often more convenient to specify the visibility at the place +where the macro is documented via C<=for apidoc> lines. The same flags +as in F entries are recognized. To restrict the visibility +to just the C module, you also need to use the same C preprocessor +conditional as you would for a function. -Simple test cases can be automatically generated. +There is an exception for macros that don't have such entries. Ones +whose names match the pattern (C<$names_reserved_for_perl_use_re> are +made visible, under the theory that you wouldn't have bothered to add +the clumsier spelling if you didn't want them visible. + +=item Enums, Typedefs, Structs, and Unions + +These unfortunately by default are visible to everyone. To restrict +their visibility, you have to resort to C preprocessor conditionals +surrounding them, like + + # if defined(PERL_CORE) + typedef enum { ... } my_enum; + # endif + +Use C to restrict the visibilty to perl extensions. =back @@ -453,6 +503,8 @@ likely to clash with ones an author might choose, problems don't arise. If you choose to not document a new symbol that needs to be visible everywhere, add it to the array C<@undocumented_always_visible> in F. +Note that symbols not placed in header files have never been visible to +outside code. =head3 Choosing good symbol names @@ -555,7 +607,7 @@ Therefore, if a macro does use variables, their names should be such that it is very unlikely that they would collide with any caller, now or forever. One way to do that, now being used in the perl source, is to include the name of the macro itself as part of the name of each -variable in the macro. Suppose the macro is named C Then we +variable in the macro. Suppose the macro is named C. Then we could have int foo_svpv_ = 0; From fba7dfcada7e5b84b8f9414353c3af6cd28086e7 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Tue, 18 Aug 2026 21:09:24 +0200 Subject: [PATCH 02/31] docs(regex): prioritize remaining Java fallback removal Keep the Phase 36 critical path focused on lookbehind, branch reset, and alphabetic assertions as the next native Joni routing slices. Generated with Codex (https://openai.com/codex) Co-Authored-By: Codex --- dev/design/phase36-regex-parity.md | 766 +++++++---------------------- 1 file changed, 175 insertions(+), 591 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index b0d0e523d7..7265b97967 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -19,10 +19,6 @@ The historical comparison point is: ../PerlOnJava/logs/test_20260815_080000_958.log ``` -The last completed 80-file differential recorded 51,002/94,829 passing -assertions, 729 more passing assertions and 58 more planned assertions than that -baseline, with no per-file pass-count regressions. - ## Architecture ### Final engine boundary @@ -150,15 +146,13 @@ allocate callback state or callback frames. ### Phase 6 — Integration and release -1. Retire regex-test accommodations incrementally. Whenever a PerlOnJava fix - makes a `dev/import-perl5/patches/pat.t.patch` hunk unnecessary, remove that - hunk and rerun `perl dev/import-perl5/sync.pl --only perl5/t/re/pat.t` to - restore the unchanged upstream assertions. Do not hand-edit the imported - test to approximate upstream content. -2. Delete the `pat.t.patch` configuration entry and patch file once its final - hunk is obsolete. Rerun the targeted sync twice and require the second run to - produce no diff, proving that the checked-in test is the unpatched Perl 5.44 - source and the import is idempotent. +1. Keep regex core tests unpatched. The canonical `perl5/t` directory import + owns `re/pat.t`; no duplicate file row or regex-test patch may replace or + weaken upstream assertions. +2. Run `perl dev/import-perl5/sync.pl --only perl5/t` twice, verify the imported + `re/pat.t` hash against the configured upstream source, and require the + second run to produce no content diff. If an upstream assertion fails after + sync, fix PerlOnJava rather than editing the imported test. 3. Run the complete direct and `_thr.t` regex matrix on JVM and interpreter backends and compare it file-by-file with both the Phase 0 result and PR 958. 4. Run unchanged Type::Tiny, Regexp::Common, Object::InsideOut, and every CPAN @@ -184,21 +178,16 @@ the regex corpus is reproduced from `dev/import-perl5/sync.pl` without a regex test patch, and documentation reports optimizer/debug-only exclusions explicitly. -### Upstream patch retirement queue - -`pat.t.patch` is reduced in place as these gates close; the corresponding -upstream hunk is restored by the targeted importer before its result is counted: - -| Upstream section | Gate before restoring the hunk | -|---|---| -| `(*ACCEPT)` capture-close cases | Exact success and capture values pass without converting fatal setup failures to warnings | -| `pos` inside `(?{...})` | Callback-visible `pos`, captures, and unwind behavior pass on JVM and interpreter | -| reference stringification diagnostics | Unqualified `diag` resolves in the original lexical/package context | -| `${^LAST_SUCCESSFUL_PATTERN}` | Dynamic empty-pattern reuse, copying, matching, and substitution pass | -| `(??{...})` code blocks interpolated from arrays | All original runtime-eval and side-effect assertions pass without an enclosing compatibility `eval` | +### Imported-test provenance gate -The queue is complete only when `config.yaml` no longer names `pat.t.patch`, the -patch file is gone, and two consecutive targeted syncs leave a clean tree. +- `dev/import-perl5/config.yaml` imports `perl5/t/re/pat.t` through the + canonical `perl5/t` directory entry, without a duplicate row or patch. +- No regex-specific import patch weakens or skips upstream assertions. +- A targeted `--only perl5/t` sync restores the exact configured upstream + source. +- A second consecutive directory sync is content-idempotent and leaves the + tree clean. +- The synchronized direct and thread tests run unchanged on JVM and interpreter. ## Test Contract @@ -231,573 +220,168 @@ developer backend selector is removed in Phase 5. Existing Perl syntax, variables, warning categories, and regex object behavior are the public compatibility contract. -## Progress Tracking - -### Current Status: Phases 0, 2, and 4 complete; Phases 1 and 3 corpus gates active - -The unified `integration/phase36-regex-parity` branch was assembled on -2026-08-18 from all 35 ready Phase 36 PRs, with one squashed review-unit commit -per PR. Duplicate #1007 ancestry from #1010, duplicate #1008/#1009/#1012 -ancestry from #1016, and temporary integration merge commits were excluded. -The final stacked implementation matches PR #1040 plus the independent #1006, -#1007, and #1010 changes. The exact source head `3e6076a67` passed warning-free -`make` in 6m46s, including direct Joni, packaging, and all five unit shards. -Unified draft PR #1042 is open against `master`; Ubuntu and Windows CI and the -final forced-backend differential remain pending. - -The unified history includes the completed callback/runtime slices, lossless -generated Unicode fixtures, explicit `Is_*` property/value -normalization, fatal Joni syntax diagnostics, native GCB semantics, and the -first 524 lines of retired Java-only preprocessor code. Every source slice has -a warning-free combined `make` checkpoint. PR #1027 adds native sentence -boundaries; PR #1028 adds independently validated alpha assertion aliases and -native word boundaries. PR #1029 integrates corrected global zero-width `/g` -progression and pinned Perl 5.44 Unicode 17.0 Age properties. PR #1030 adds -binary `ASCII_Hex_Digit` values, pinned General_Category sets, and exact native -line boundaries. PR #1031 adds pinned Canonical_Combining_Class sets and valid -empty-property rendering. PR #1032 integrates native numeric escapes through -U+10FFFF; PR #1033 adds pinned Bidi_Class sets; PR #1034 integrates native -vertical-whitespace escapes; PR #1035 adds Decomposition_Type and PR #1036 -adds East_Asian_Width, PR #1037 adds Numeric_Value, and PR #1038 adds -Joining_Group, and PR #1039 adds Block. The current WIP integrates -Script/Script_Extensions; independently validated break-property values, -generic and specialized binary-property data, residual enumerated-property -families, and the first preprocessor dead-state deletion are ready for focused -integration. - -Lexical `use bytes` now compiles non-ASCII substitution patterns with a -single-byte Joni encoding while preserving upgraded, byte-backed, and compiled -`qr//` source provenance. The focused oracle passes 12/12 on system Perl, JVM, -and interpreter, and the exact upstream marker-stage reducer improves from 2/4 -to 4/4 on both execution backends. Generated chunks 05–10 consequently execute -239,843 genuine boundary assertions rather than matching literal UTF-8 marker -text. JVM and interpreter have exact per-file parity at 2,192/239,843 with every -plan complete, exit 0, and no child timeout at the pre-GCB baseline. The runner -classifies zero-pass files as `error`, but their recorded plans, actual counts, -and process exits are complete. - -Native Joni GCB assertions now implement GB1–GB13 and GB999, including Indic -conjunct and emoji-ZWJ context, and `\X` consumes repeated GB9c conjuncts. The -focused 29-assertion oracle passes on system Perl, JVM, and interpreter. -Authoritative chunk 05 improves by 6,324 assertions from 2,192/14,953 to -8,516/14,976 identically on JVM and interpreter: its complete GCB/`\X` section -passes, leaving only the 6,460 sentence-boundary assertions in that chunk. - -Native Joni sentence assertions now implement SB1–SB11 and SB998 with a -reproducibly generated Perl 5.44 Unicode 17.0 `Sentence_Break` table. The -23-assertion focused oracle passes on system Perl, JVM, and interpreter, direct -Joni coverage exercises the same engine path, and authoritative chunk 05 passes -14,976/14,976 identically on JVM and interpreter. This closes all 6,460 -remaining sentence assertions without coupling the Joni fork to ICU or the -PerlOnJava runtime. - -The most recent exact property chunks 01–04 remain 98,092/167,501 on both -execution backends. Native Joni word assertions implement WB1–WB16 and WB999 -from repository-pinned Perl 5.44 Unicode 17.0 Word_Break and -Extended_Pictographic data. The 33-assertion focused oracle passes on system -Perl, JVM, and interpreter, direct Joni exercises the same path, and generated -chunk 10 passes 19,510/19,510 identically on both execution backends. Combined -with the complete boundary chunk 05 and unchanged chunks 06–09, current -generated evidence was 132,578/407,367 before the current property slice. A resource-contended -current-head refresh did not reproduce a complete exact JVM/interpreter pair, -so it does not replace that accepted baseline. - -`Age` now uses exact introduction-version sets and `In`/`Present_In` use -cumulative sets generated from the repository-pinned Perl 5.44 Unicode 17.0 -`DAge.txt`; `Unassigned`/`NA`, colon delimiters, wildcard-value spellings, and -Perl loose version aliases are covered without inheriting the host ICU Unicode -version. The focused oracle passes 14/14 on system Perl, JVM, and interpreter. -Stable chunk 01 validation improves from 30,194 to 30,705 passing assertions -identically on JVM and interpreter, with no numbered regression. This raises -current generated evidence by 511 to 133,089/407,367. - -`General_Category`/`gc`/`Category` assignments now resolve all atomic and -aggregate values from repository-pinned Perl 5.44 Unicode 17.0 data. Short, -long, `Is_`, colon, wildcard, and loose value aliases are generated -reproducibly without the host ICU category table. The focused oracle passes -18/18 on system Perl, JVM, and interpreter; Age remains 14/14 and invalid -property diagnostics remain 39/39. Chunk 01 improves by another 606 assertions -to 31,311/41,843 identically on both execution backends with no numbered -regression, raising property-plus-completed-sentence/word evidence to -133,695/407,367 before line integration. - -Native Joni line assertions now implement Unicode 17 UAX #14 from pinned Perl -5.44 Line_Break, General_Category, East_Asian_Width, and emoji data. The -84-assertion focused oracle passes on both execution backends and chunks 06–09 -pass 205,380/205,380 each on JVM and interpreter. Protected sentence and word -chunks remain exact, making the complete generated boundary corpus -239,866/239,866 and current generated evidence 339,075/407,367. - -`Canonical_Combining_Class`/`ccc` assignments now resolve every pinned Unicode -17 value and alias, including ordered `Not_Reordered` defaults for unassigned -code points and reserved valid values whose sets are empty. Empty properties -render as valid match-none/match-all classes rather than invalid `[]` syntax. -The focused oracle passes 24/24 on system Perl, JVM, and interpreter. Chunk 01 -passes 33,516/41,843 identically on both execution backends: 2,195 CCC -assertions and 10 already-native line preamble assertions improve over the -31,311 baseline with zero numbered regressions. Current generated evidence is -341,280/407,367. - -`Bidi_Class`/`bc` assignments now resolve all 23 values from a complete pinned -Unicode 17 partition. Ordered missing defaults, short/long and loose aliases, -directional controls, noncharacters, and unknown-value rejection are covered. -The focused oracle passes 99/99 on system Perl, JVM, and interpreter; the -combined Unicode/property/boundary smoke is 345/345 per backend. Chunk 01 gains -736 assertions to 34,252/41,843 identically on both execution backends with no -numbered regression, raising current generated evidence to 342,016/407,367. - -Native Joni `\v` now matches Perl's seven vertical-whitespace code points and -`\V` matches their complement, both directly and inside character classes, -without changing non-Perl Joni syntax behavior. The focused oracle passes -92/92 on system Perl, JVM, and interpreter. Unchanged `reg_posixcc.t` improves -from 2,052/2,560 to 2,560/2,560 on both execution backends with zero numbered -regressions, closing its entire 508-assertion Joni gap. - -`Decomposition_Type`/`dt` assignments now resolve all 18 atomic values plus -Perl's composite `Non_Canonical` value from a complete pinned Unicode 17 -partition. Short/long and loose aliases, ordered `None` defaults, the exact -case-sensitive `Is` assignment prefix, and invalid-value rejection are covered. -The focused oracle passes 45/45 on system Perl, JVM, and interpreter. Chunk 01 -gains 640 assertions to 34,892/41,843 identically on both execution backends -with zero numbered regressions, raising current generated evidence to -342,656/407,367. - -`East_Asian_Width`/`ea` assignments now resolve all six values from a complete -pinned Unicode 17 partition, including the ordered CJK `Wide` and general -`Neutral` missing defaults. Short/long and loose aliases are covered, and -surrogate range endpoints render as explicit Joni hex escapes rather than -lossy literal surrogates. The focused oracle passes 31/31 on system Perl, JVM, -and interpreter; protected boundary smoke remains 169/169 per backend. Chunk -01 gains 216 assertions to 35,108/41,843 identically on both execution backends -with zero numbered regressions, raising current generated evidence to -342,872/407,367. - -`Numeric_Value`/`nv` assignments now resolve all 144 exact rational values and -the `NaN` complement from pinned Perl 5.44 Unicode 17 data. Integer, decimal, -exponent, reduced-rational, loose, wildcard, and exact case-sensitive `Is` -forms follow Perl's generated keyword aliases and binary-NV canonicalization, -including four-significant-digit decimal spellings without heuristic tolerance. -The focused oracle passes 50/50 -on system Perl, JVM, and interpreter; protected boundary smoke remains 169/169 -per backend. Chunks 02–03 gain 13,976 assertions with zero numbered regressions -and exact JVM/interpreter success sets, raising current generated evidence to -356,848/407,367. - -`Joining_Group`/`jg` assignments now resolve all 106 values from a complete -pinned Unicode 17 partition. Loose aliases, the ordered `No_Joining_Group` -default, the alternate `Hamza_On_Heh_Goal` wildcard name, canonical and -squeezed wildcard values, exact case-sensitive `Is` policy, and wildcard -diagnostics follow Perl 5.44. The focused oracle passes 49/49 on system Perl, -JVM, and interpreter; protected boundary smoke remains 169/169 per backend. -Chunks 01–04 gain 4,290 assertions with no pass-count regression and exact -JVM/interpreter success sets, raising current generated evidence to -361,138/407,367. - -`Block`/`blk` assignments and `In...`/single-`Is...` shortcuts now resolve all -347 values, including `No_Block`, from a complete pinned Unicode 17 partition. -Official compact aliases, loose forms, `#...#` wildcards, Script and -General_Category/binary precedence, ordered gaps, noncharacters, and exact -compound `Is` policy follow Perl 5.44. The 36-assertion oracle passes standard -Perl; JVM and interpreter pass all 35 Block-specific assertions while retaining -one pre-existing TODO for unresolved deferred `In...` user-property timing. -The two focused precedence reducers pass 12/12 on all runtimes, protected -boundary smoke remains 169/169 per backend, and chunks 01–04 gain 8,324 -assertions with zero numbered regressions and exact backend identity. Current -generated evidence is 369,462/407,367. - -`Script`/`sc` and `Script_Extensions`/`scx` assignments now resolve all 176 -values from pinned Unicode 17 partitions and Script_Extensions overrides. -Explicit `sc` retains strict Script semantics while Perl's bare Script-value -shortcuts use Script_Extensions; the composite `Katakana_Or_Hiragana`/`Hrkt` -pseudo-value is rejected from bare and exact assignments and excluded from -wildcard unions as required by Perl. Loose aliases, `Qaac`/`Qaai`, -wildcards, exact `Is` assignment policy, precedence over Block shortcuts, and -positive or complemented properties inside ordinary character classes are -covered. The 95-assertion oracle passes system Perl, JVM, and interpreter; the -focused precedence, class-negation, and bare-scx reducers pass 7/7, 8/8, and -10/10 respectively on all three runtimes. Protected boundary smoke remains -169/169 per backend. Chunks 01–04 gain 8,140 assertions with zero numbered -regressions and exact JVM/interpreter counts, raising current generated -evidence to 377,602/407,367. - -Joni now accepts Perl's top-level, scoped, combined, and negative inline `p` -syntax as matcher-neutral policy. PerlOnJava publishes that policy while -ordinary and substitution callbacks execute, without misclassifying escaped or -character-class text. The focused 15-assertion oracle passes on system Perl, -JVM, and interpreter, and unchanged `reg_pmod.t` reaches 88/88 on both -execution backends. Regex source scanning also consumes each `\c` operand -before interpolation, so `\c@` cannot be mistaken for `@-`; the focused -4-assertion oracle passes on all three runtimes and unchanged `subst.t` reaches -250/281 on JVM and interpreter. - -The matcher adapter now carries Joni's search start and Perl `\G` position as -independent cursors, including Unicode offset conversion. The focused -12-assertion oracle passes on system Perl, JVM, and interpreter; unchanged -`subst.t` reaches 275/281 on both execution backends with tests 165-188 -restored. The temporary Java backend retains its start-at-`pos` approximation. - -Executable callback source and literal trailing `/x` comments survive canonical -regex-object stringification on both execution backends. Recursive Joni call -frames now preserve the Perl-visible caller capture view for optimistic -callbacks and committed matches, including `$1`, `$^N`, and `$+`. Tied scalar -values returned by dynamic callbacks are materialized before callee regex state -teardown. Joni invalid-backreference errors use Perl's nonexistent-group -diagnostic. Reopened repeated groups expose their preceding closed capture to -dynamic callbacks without altering matching registers. Nested dynamic matcher -completion preserves the last successful block result in `$^R`, including a -runtime `qr` returned by an outer `(??{...})`. Executable-looking groups inside -double-quote case modifiers are deferred until after interpolation and obey -runtime `re 'eval'` permission. Foreach aliases refresh the active lexical-cell -registry on both execution backends, so runtime-compiled callbacks capture each -iteration's cell and retain it after scope exit. Executable runtime pattern -compilation uses independent `(eval N)` source identities for diagnostics. -Runtime source now inherits exact lexical warning masks and reports Unicode -parser names and undefined match operands at the original call site. Failed -callback branches preserve the preceding successful `$&`, `$1`, and related -match state. Dynamic regex-state restoration releases discarded temporary -callback patterns, so captured values stay alive through the enclosing scope -and are destroyed when that scope exits. Recursive callback unwind preserves -the failed nested `$^N` and `$+` state without clobbering numbered captures or -one-level failed callback state. The focused `pat_re_eval.t` gate executes all -555 assertions with 550 semantic assertions passing on both execution backends; -the remaining five inspect Perl's optimizer/debug transcript. - -The last completed forced-backend differential's forced-Java/JVM leg covers all -80 files at -49,923/94,823 versus PR 958's 50,273/94,771. The apparent aggregate regression -is dominated by `pat{,_thr}.t` aborting after test 239 on a runtime eval-group -policy error; that source-policy slice is assigned independently. The completed -forced-Joni/JVM leg is 32,479/77,612, with ten bounded timeout files. Its -largest completed losses against forced Java are `reg_posixcc.t` (-508), -`reg_mesg.t` (-300), both `pat_advanced` variants (-240 each), -`alpha_assertions.t` (-89), and `regex_sets.t` (-84). The completed -forced-Java/interpreter leg covers all 80 files at 50,021/94,823 with no runner -timeouts, 98 more passing assertions than forced-Java/JVM, and an identical -plan. The completed forced-Joni/interpreter leg is 32,483/77,612 with the same -ten timeouts and planned count as Joni/JVM. The final same-binary report is -complete in `dev/design/phase36-regex-differential-20260817.md`; Phase 1's exit -criterion is not met because Joni loses Java-passing assertions and introduces -matcher-specific timeouts on both execution backends. - -The post-PR-#1028 plus `/g` combined forced-Joni refresh executes all 80 files -at 74,603/331,826 on JVM and 74,607/331,826 on interpreter. Four generated -property chunks time out after producing partial TAP and require the narrow -600-second rerun; chunks 05 and 10 are exact while chunks 06–09 expose only the -assigned line-boundary gap. Six regressions versus the preceding Joni result -reduced to two fatal roots. Binary `ASCII_Hex_Digit=True` routing is now closed: -the focused Perl boolean-value oracle passes 16/16 on system Perl, JVM, and -interpreter, and `pat.t` is restored from its zero-TAP abort to the independently -tracked test-239 runtime-eval gate. Native Joni numeric parsing now treats bare -high octal escapes as UTF-8 code points and accepts underscored braced hex and -octal escapes through U+10FFFF. The focused standard-Perl oracle has 14 ordinary -passes plus four explicitly classified TODOs on both execution backends; -`pat_rt_report{,_thr}` advances from 5 executed assertions to 73/72, and -`pat_advanced.t` reaches its later independent `Titlecase` property blocker. -Strict-regex source policy and Perl code points above U+10FFFF remain explicit -frontend/representation debt, so the forced-Java underscore compatibility pass -is retained for now. - -### Completed Phases - -- [x] Phase 0: Reproducible differential baseline (2026-08-17) - - Captured the 80-file regex differential with complete output and JSON. - - Compared every file with the PR 958 baseline at - `../PerlOnJava/logs/test_20260815_080000_958.log`. - - Recorded 51,002/94,829 passing assertions, a net gain of 729 passing and - 58 planned assertions, with no per-file pass-count regressions. - - Added separate parallel handling for CPU-heavy `pat_psycho*` and `speed*` - tests while retaining per-child timeouts. -- [ ] Phase 1: Joni ordinary-pattern parity (implementation substantially - complete; forced Java/Joni corpus comparison remains) - - [x] Added the temporary backend selector and made Joni the default. - - [x] Routed ordinary matching, substitution, and split through the selected - backend without per-operation fallback. - - [x] Completed the forced-Java/JVM 80-file leg and identified the - `pat{,_thr}.t` test-239 source-policy abort as the leading regression. - - [x] Completed the four-leg forced-backend matrix and published its - classification. The exit criterion is explicitly not met; timeout and - semantic remediation remain Phase 1 work. - - [x] Reduced the forced-Joni zero-pass surface to seven shared causes: - catastrophic backtracking, quadratic matcher reconstruction, absent - generated Unicode fixtures, regex-set preprocessing, unsupported compiler - introspection, regexp-object propagation, and three assertion-level - environment/runtime failures. - - [x] Moved immutable Joni UTF-8 input and offset maps out of the scalar - `/g` hot loop. The focused million-match oracle completes in 1.07 seconds - on JVM and 1.41 seconds on interpreter (PR #1008), with exact map and - supplementary-character capture-boundary coverage. - - [x] Separated Joni's search-start and `\G` cursors for ordinary matching - and substitution, including Unicode subjects and code replacements. The - focused oracle passes 12/12 and unchanged `subst.t` passes 275/281 on JVM - and interpreter. - - [x] Added a provenance-aware single-byte Joni pattern/input path for - non-ASCII substitutions under lexical `use bytes`. Upgraded, byte-backed, - and compiled byte-backed patterns pass 12/12 on all runtimes, and the - generated Unicode marker stage passes 4/4 on JVM and interpreter. - - [x] Closed `/g` same-position retry and capture semantics after a zero-width - first alternative (`0703725c8`, integrated as `402102446`). The focused - oracle passes 23/23, the raw omniholder reducer improves from 7/10 to 10/10 - in all six Java/Joni × JVM/interpreter modes, and DBIx::Simple remains 69/69. - - [x] Reran the combined forced-Joni 80-file corpus on JVM and interpreter, - published the complete file-by-file comparison, and reduced its six actual - regressions to two fatal roots with narrow owners and rerun gates. -- [x] Phase 2: Conditions and backtracking-visible state (2026-08-17) - - [x] Implemented executable callback conditions, control verbs including - `(*MARK:NAME)`, and callback-visible recursive capture state in Joni. - - [x] Closed runtime callback capture ownership at final scope teardown. - - [x] Closed failed-path `$^N` and `$+` restoration through recursive - callback unwind. - - [x] Added direct active-localization lookup for runtime control variables; - dynamic `PRUNE`, `SKIP`, and `COMMIT` update package `$REGERROR` without - mutating non-localized `$REGERROR`/`$REGMARK` variables on either backend. - - [x] Propagated `PRUNE`, `SKIP`, `COMMIT`, and `THEN` cuts and search-control - requests from nested `(??{...})` matcher programs. A 9-assertion - standard-Perl oracle passes on JVM and interpreter, and `pat_advanced.t` - test 891 now observes 3 callback executions instead of 9. - - [x] Refreshed the package alias stored for a reused `our` symbol when a - later declaration changes package. The focused package oracle passes on - system Perl, JVM, and interpreter, and `pat_advanced.t` tests 922-933 pass - on both execution backends without a regex-adapter workaround. - - [x] Exposed the actual match subject as callback `$_`, the provisional - callout offset through `pos`, and the in-progress match span through `$&` - plus the pre-match and post-match variables. Callback-bearing substitution - recompilation now preserves trusted callout markers. The 24-assertion - upstream `pos inside (?{})` block - passes on system Perl, JVM, and interpreter; `subst_amp.t` remains 13/13 - on both execution backends. - - [x] Removed the obsolete nested `(*ACCEPT)` and callback-`pos` workarounds - from `pat.t.patch` and resynchronized those original Perl 5.44 assertions. - - [x] Verified reference stringification (5/5) and - `${^LAST_SUCCESSFUL_PATTERN}` dynamic scope and reuse (25/25) on system - Perl, JVM, and interpreter; removed both obsolete `pat.t.patch` wrappers - and resynchronized the original assertions. - - [x] Preserved callback-bearing compiled regexes through one- and multi-item - array interpolation, including Perl's deferred dot-overload composition - with surrounding dynamic callbacks. The focused oracle passes 28/28 on - system Perl, JVM, and interpreter. - - [x] Removed the final `pat.t.patch` hunk, deleted the patch and its importer - configuration, and resynchronized the unmodified Perl 5.44 `pat.t`. -- [ ] Phase 3: Unicode and pattern syntax completion (focused gates complete; - generated full-corpus remediation active) - - [x] Added Perl escape syntax, Unicode-property resolution, scoped ASCII - folds, possessive intervals, and bounded lookbehind support to Joni. - - [x] Converted public regex `pos` values between Perl logical-character - offsets and Java matcher offsets for scalar `/g`, `\G`, fast scanners, and - substitution callbacks. The 11-assertion supplementary-character oracle - passes on system Perl, JVM, and interpreter. - - [x] Restricted user-defined property dispatch to Perl's exact `Is`/`In` - naming convention and made unknown-property diagnostics fatal even in - compatibility warning mode. The focused oracle passes 39/39 on system - Perl, JVM, and interpreter; `regexp_unicode_prop.t` gains 15 assertions. - - [x] Matched user-property definition validation, deterministic recursion - chains, callback-death wrapping, and direct package-name policy. - The focused oracle passes 12/12 on system Perl, JVM, and interpreter; - unchanged upstream coverage gains two assertions. - - [x] Preserved deferred user-property package provenance through implicit - Unicode-flag copies and later literal reuse. The focused oracle passes 8/8 - on system Perl, JVM, and interpreter; `regexp_unicode_prop.t` gains nine - assertions to 1,065/1,110 on both execution backends. - - [x] Accepted inline `p` directly in Joni while retaining match-variable - policy in PerlOnJava, including provisional callback state. The focused - oracle passes 15/15 and unchanged `reg_pmod.t` passes 88/88 on JVM and - interpreter. - - [x] Preserved `\c` control operands through regex source interpolation. - The focused oracle passes 4/4 and unchanged `subst.t` gains test 154 on - both execution backends. - - [x] Completed the built-in Unicode aliases exercised by - `regexp_unicode_prop.t` while preserving deferred user-property precedence. - The focused alias oracle passes 16/16 on system Perl, JVM, and interpreter; - unchanged `regexp_unicode_prop.t` passes 1,110/1,110 on both execution - backends. - - [x] Added a lossless, idempotent importer for Perl's generated TestProp - corpus. The focused importer test passes 66/66, two real generations are - byte-identical, system Perl executes 503,197 TAP, and JVM/interpreter both - execute 290,912 TAP with exact semantic parity and no timeout. - - [x] Classified all 115,144 failures newly exposed by the lossless generated - `uniprops*.t` corpus, including the cross-cutting invalid boundary-harness - evidence in chunks 05–10. - - [x] Normalized explicit `Is_*` property/value assignments and the colon - delimiter (PR #1019), gaining exactly 44,944 generated assertions on both - execution backends without changing any plan. - - [x] Rejected 40 invalid Perl inline option/group-name forms in forked Joni - with exact JVM/interpreter `reg_mesg.t` parity, reducing residual Joni-only - acceptance differences from 198 to 158 (`028602adc`). - - [x] Integrated native Python-style named captures and backreferences plus - removal of their frontend conversion (`afbe2bc34`, integrated as - `cc489bee8`). The 20-case oracle passes on both execution backends with exact - malformed/unknown diagnostics. - - [x] Integrated native braced-octal parsing and missing-close/empty - diagnostics plus fatal unterminated braced-hex diagnostics (`55433291a`, - `913e2b583`) with exact JVM/interpreter `reg_mesg.t` parity. - - [x] Integrated native bare high-octal and underscored braced hex/octal - parsing through U+10FFFF (`f849c2ef9`, integrated as `eb907a10b`). The - focused gate has 14 ordinary passes plus four classified TODOs on both - backends and restores `pat_advanced`/`pat_rt_report` startup. - - [x] Integrated native Joni alpha assertion aliases `pla`, `plb`, `nla`, - `nlb`, and `atomic` (`a6255fbff`, integrated as `49d7d9648`). The focused - 25-case oracle passes on both execution backends and the generated alpha - corpus gains 98 passing assertions per backend with zero regressions. - - [x] Fixed byte-mode substitution of upgraded marker regexes so chunks 05–10 - exercise real boundary subjects with exact JVM/interpreter plans. - - [x] Implemented native Joni GCB assertions for GB1–GB13 and GB999 and aligned - `\X` with repeated GB9c Indic conjunct behavior. The focused oracle passes - 29/29 and generated chunk 05 reaches 8,516/14,976 on both execution backends. - - [x] Implemented native Joni sentence assertions for SB1–SB11 and SB998 from - a reproducible Perl 5.44 Unicode 17.0 table. The focused oracle passes 23/23 - and generated chunk 05 passes 14,976/14,976 on both execution backends. - - [x] Implemented native Joni word assertions for WB1–WB16 and WB999 from - reproducible Perl 5.44 Unicode 17.0 Word_Break and Extended_Pictographic - tables. The focused oracle passes 33/33 and generated chunk 10 passes - 19,510/19,510 on both execution backends. - - [x] Generated exact `Age` and cumulative `In`/`Present_In` sets from pinned - Perl 5.44 Unicode 17.0 data, including loose version, wildcard, and - unassigned aliases. The focused oracle passes 14/14 and chunk 01 gains 511 - assertions with no numbered regression. - - [x] Routed Perl boolean values for the built-in `ASCII_Hex_Digit`/`AHex` - property through the frontend set resolver. All eight true/false aliases - pass 16/16 on system Perl, JVM, and interpreter, restoring `pat.t` startup. - - [x] Generated pinned Unicode 17.0 General_Category atomic and aggregate - sets with Perl property/value aliases. The focused oracle passes 18/18 and - chunk 01 gains 606 assertions on both backends with zero regressions. - - [x] Implemented native Joni line assertions from reproducible pinned Unicode - 17.0 data. The focused oracle passes 84/84 and chunks 06–09 pass - 205,380/205,380 on both execution backends while sentence/word stay exact. - - [x] Generated pinned Unicode 17.0 Canonical_Combining_Class sets with loose - aliases, ordered missing defaults, reserved empty values, and valid - empty/full rendering. The focused oracle passes 24/24 and chunk 01 reaches - 33,516/41,843 on both backends with zero numbered regressions. - - [x] Generated and integrated a complete pinned Unicode 17.0 Bidi_Class - partition with loose aliases and ordered missing defaults. The focused - oracle passes 99/99 and chunk 01 gains 736 assertions to 34,252/41,843 on - both backends with zero numbered regressions. - - [x] Generated and integrated a complete pinned Unicode 17.0 - Decomposition_Type partition, including Perl's composite `Non_Canonical` - value and exact `Is` prefix policy. The focused oracle passes 45/45 and - chunk 01 gains 640 assertions to 34,892/41,843 on both backends with zero - numbered regressions. - - [x] Generated and integrated a complete pinned Unicode 17.0 - East_Asian_Width partition with all ordered missing defaults and lossless - surrogate-range rendering. The focused oracle passes 31/31 and chunk 01 - gains 216 assertions to 35,108/41,843 on both backends with zero numbered - regressions. - - [x] Generated and integrated pinned Perl 5.44 Unicode 17.0 Numeric_Value - sets for all 144 rationals plus NaN, including exact rational reduction, - generated decimal keyword aliases, loose forms, and wildcard policy. The - focused oracle passes 50/50; chunks 02–03 gain 13,976 assertions with zero - numbered regressions and exact backend identity. - - [x] Generated and integrated a complete pinned Unicode 17.0 Joining_Group - partition with loose aliases, ordered defaults, alternate wildcard names, - and exact `Is`/wildcard rejection policy. The focused oracle passes 49/49; - chunks 01–04 gain 4,290 assertions with exact backend identity. - - [x] Generated and integrated a complete pinned Unicode 17.0 Block - partition with official aliases, ordered `No_Block` gaps, wildcard policy, - and Script/category/binary precedence. Chunks 01–04 gain 8,324 assertions - with zero numbered regressions and exact backend identity. - - [x] Generated and integrated complete pinned Unicode 17.0 Script and - Script_Extensions sets, including Perl's bare-scx policy, strict explicit - Script assignments, composite-value rejection and wildcard exclusion, - aliases, precedence, and ordinary character-class complements. The focused oracle - passes 95/95; chunks 01–04 gain 8,140 assertions with zero numbered - regressions and exact backend counts. - - [x] Integrated native Perl `\v`/`\V` dispatch inside and outside character - classes (`1eff1db97`, integrated as `6328935cd`). The focused oracle passes - 92/92 and unchanged `reg_posixcc.t` passes 2,560/2,560 on both backends. - - [ ] Close the remaining property failures before marking Phase 3 complete. -- [x] Phase 4: Runtime source and diagnostics (2026-08-17; semantic gate - complete at 550/555) - - [x] Preserved mixed executable-source provenance, nested dynamic callback - values, foreach lexical cells, and independent `(eval N)` source names. - - [x] Restored lexical warning masks, Unicode source diagnostics, undefined - operand warnings, and prior successful match state across failed callbacks. - - [x] Released callback captures when temporary match state is discarded and - the final owning regex scope exits (test 307). - - [x] Resolved failed-path `$^N`/`$+` tests 85-86 on JVM and interpreter. - - [x] Classified tests 444-448 as optimizer/debug-transcript exclusions. - - [x] Decoded byte-backed eval source according to lexical `use utf8`, - including pragmas activated inside the source, while preserving `no utf8` - byte semantics and fatal malformed-UTF-8 diagnostics. The focused oracle - passes 7/7 on system Perl, JVM, interpreter, and the direct JVM eval - compiler. - - [x] Kept Joni syntax/value exceptions fatal for ordinary user-source - compilation while preserving executable-source validation deferral. The - focused oracle passes 7/7 and unchanged forced-Joni `reg_mesg.t` gains 259 - raw passing assertions; 197 parser-acceptance differences remain classified. -- [ ] Phase 5: Remove the Java matching backend - - [x] Retired the unreachable top-level `(*PRUNE)` text rewrite after native - Joni control-verb gates passed under default and forced-Java policy - (`5760874e4`; 316 preprocessor lines removed). - - [x] Removed the disabled invalid-brace pass and its exclusive helpers - (`4be6a48e3`; 208 preprocessor lines removed), retaining active Perl/Joni - quantifier diagnostics as explicit focused hard/TODO gates. - - [x] Removed the Java-only terminated-whitespace possessification pass - (`c5343aca2`; 80 preprocessor lines removed) after greedy backtracking and - 20,000-character gates passed default and forced-Java policy on both - execution backends. - - [x] Validated removal of the Java-only terminated lazy-negated-class - possessification pass (`625ea97a2`; 252 preprocessor lines removed) with - leftmost-capture and 20,000-character gates in all four backend modes. - - [x] Retired the Java-only DBIx omniholder alternative reorder - (`18e71a532`; 50 lines removed) after exact substitution and bundled - DBIx::Simple gates passed. The independent raw `/g` 7/10 progression gap is - now closed at 10/10 by the shared matcher-adapter fix above. -- [ ] Phase 6: Integration and release - -### Next Steps - -1. Keep unified draft PR #1042 open until final validation completes. Preserve - its 35 source PR commits and use a merge commit after the forced-backend - differential and Ubuntu/Windows CI pass. -2. Preserve the now-complete native Joni boundary corpus at 239,866/239,866: - sentence chunk 05, line chunks 06–09, and word chunk 10 must remain exact on - JVM and interpreter while property and parser work continues. -3. Integrate the independently generated break-property value slice, then the - generic and specialized binary-property families and residual enumerated - families currently advancing in parallel. - Preserve pinned Perl 5.44 - acceptance and rejection semantics rather than inheriting host ICU breadth. - Keep native `\v`/`\V` exact at 2,560/2,560 in `reg_posixcc.t`. -4. Rerun generated property chunks 01–04 on both backends with the classified - 600-second bound and retain complete TAP/JSON. After the two fatal roots and - native line boundaries integrate, refresh the complete forced-Joni 80-file - corpus and apply the no-regression gate against Phase 0 and PR 958. -5. Audit every `RegexPreprocessor` rule against the final ownership boundary. - Move matcher semantics into Joni, retain only source-policy scanning, delete - Java-only rewrites and compiled-pattern variants, and remove the temporary - Java backend selector after the performance gate passes. -6. Reconcile `docs/reference/feature-matrix.md` with the final corpus; update - `dev/implementation/regex.md` and `docs/design/joni-callout-fork.md` to the - as-implemented architecture and review both for clarity and structure. - Audit redundant regex/Joni documents, deleting only wholly redundant text - and summarizing historically useful rationale with links to the canonical - documents. Replace stale Unicode limitations, add any still-missing regex - features, and link each limitation to a reducer or explicit optimizer/debug - exclusion. -7. Run unchanged CPAN consumers, the direct/thread release matrix, packaging - and license checks, and require green Ubuntu and Windows CI on the unified - PR. Merge it with a merge commit so the focused history remains available - and the complete integration can be reverted with `git revert -m 1`. - -### Open Questions and Blockers - -- Exact optimizer/debug transcript assertions are not semantic release blockers; - each exclusion still requires an explicit report entry. -- Resource-sensitive baselines must wait for unrelated Java builds to finish. -- The interpreter does not reliably expose the lexical package through - `InterpreterState.currentPackage` while a regex executes. Localized - `$REGMARK`/`$REGERROR` slots are therefore enumerated from active dynamic - `GlobalRuntimeScalar` bindings rather than inferred from the current package - or scanned from dormant globals. -- Shared parser or `eval` failures are fixed in focused slices when they block a - regex semantic test, rather than being approximated inside the matcher. -- Starting a forced-Joni global match exactly on a supplementary character - also requires PR #1008's high-surrogate offset-map correction. The public - `pos` conversion is independently complete; add that exact-start assertion - when #1008 integrates. +## Execution Tracker + +This tracker records only current plan state. Implementation history, commit +identifiers, dates, and completed repair narratives belong in Git history. + +### Phase status + +- [x] Phase 0 — reproducible differential baseline +- [ ] Phase 1 — Joni ordinary-pattern parity +- [x] Phase 2 — conditions and backtracking-visible state +- [ ] Phase 3 — Unicode and pattern syntax completion +- [x] Phase 4 — runtime source and diagnostics +- [ ] Phase 5 — remove the Java matching backend +- [ ] Phase 6 — integration and release + +A checked phase means its focused semantic implementation is complete. Release +and no-regression gates remain Phase 6 responsibilities and may reopen a phase +if they expose a semantic defect. + +### Current critical path + +1. Validate and merge the rebased native-Joni delivery stack bottom-up. After + each parent merge, rebase its child onto current master, verify the commit + range and expected file set, and require warning-free build plus green CI. + Preserve the negative-file manifest and normalized comparator as mandatory + pre-acceptance gates; no unexplained negative file may be deferred to a long + acceptance run for discovery. +2. Remove temporary ordinary-pattern Java routing as native Joni replacements + become green: + - remove the temporary adapter KEEP-in-lookaround guard after the native + Joni diagnostic stack passes its combined gate; + - remove Java routing immediately after each native reducer and combined + corpus gate are green; + - use the integration report to choose the next fallback whose removal moves + the most assertions to pure Joni; + - retire the next ordinary-pattern fallbacks in this priority order: + 1. route ordinary lookbehind through Joni and delete the Java-only + lookbehind length analyzer after the 27-assertion four-leg reducer and + six imported owner-file pass sets are non-regressing; + 2. route branch-reset groups and delete their exclusive rewrite only after + the named/numeric-call reducers and complete imported branch-reset gate + pass without the temporary automatic-Java guard; + 3. route alphabetic assertions and delete their recursive Java rewrite + after direct Joni tests and the classified `alpha_assertions.t` gate + prove every remaining failure is understood; + - never move callback, condition, control-verb, or dynamic-source patterns + back to Java. + - after the current native stack is integrated, refresh the four-leg + `perl5_t/t/re` matrix (forced Java/Joni × JVM/interpreter) against one + current artifact and establish new exact counts; the stale pre-migration + counts are not a release gate for the progressive backend. +3. Complete the remaining Unicode ownership boundary: + - preserve per-member fold policy in Joni's composed-class AST through union, + intersection, negation, and nested classes, then remove the corresponding + adapter translation; + - represent property-value wildcard parsing and diagnostics with a dedicated + Joni syntax node rather than flattening wildcard behavior into literal + ranges prematurely; + - close the remaining generated property/value alias gaps against the pinned + Perl 5.44 corpus; + - keep Perl lexical/source policy in the adapter; + - prefer bundled Perl Unicode data over duplicating ICU behavior or depending + on the host JDK Unicode version. +4. Reconcile the regex rows in `docs/reference/feature-matrix.md`, including + the expected-Joni features currently listed around lines 390–409. Add missing + Perl regex features with an explicit reducer, owner, and acceptance gate. +5. Retire `RegexPreprocessor` rules and imported-test patches as their native + implementations land. Rerun targeted `dev/import-perl5/sync.pl` imports and + require an idempotent second sync before declaring the upstream tests clean. +6. Complete the full release corpus, performance, documentation, packaging, + license, and cross-platform CI gates; then delete Java matching and the + temporary backend selector. + +### Parallel workstreams + +Workstreams must use isolated branches/worktrees and communicate through the +shared handoff files. Ownership is exclusive at the file/semantic-slice level. +Completed candidates stay on their validated base; the coordinator transplants +them onto the canonical stack and verifies `range-diff`. Independent fixes +targeting master start from current master. Engineers rebase themselves only +when the coordinator assigns an exact new base before implementation begins. + +1. Integration and PR 958 parity: combine validated runtime repairs, run focused + and full comparisons, and own CI/readiness of the current integration PR. +2. Native Joni fallback removal: implement parser/compiler/matcher semantics + that replace temporary Java routing, one independently testable feature at a + time. +3. Differential prioritization: maintain exact Perl 5.44 oracles, map remaining + failures to fallback triggers, and rank pure-Joni slices by recovered corpus + impact. +4. Unicode ownership: identify duplicated ICU/Joni/adapter behavior and propose + the smallest non-overlapping migration into forked Joni. +5. Documentation and feature inventory: keep the feature matrix and final + as-implemented documents aligned with validated behavior, without recording + implementation history in this plan. + +At most two full builds may run concurrently. Timing-sensitive final gates run +serialized. Engineers should continue source-independent analysis and focused +work while a full build runs rather than blocking on it. + +Implementation lanes run their required full `make` once the candidate is +stable. The review stack then runs one combined full build and one combined +focused reducer matrix; unchanged intermediate stacks are not rebuilt merely +for handoff. Handoff messages are event-driven: candidate ready, build started, +build finished, push complete, or blocker. Heartbeats exist only for crash +detection and do not replace implementation work. + +### Delivery checkpoints + +- Keep the current integration PR in draft until the PR 958 parity gate passes. +- Mark it ready only after the focused gate, fresh full comparison, warning-free + build, and required CI are green. +- Once a stable phase is handed to review, continue the next independent slice + on a new draft PR rather than accumulating unrelated risk in the review PR. +- The release manager owns integration-branch mutation; implementation lanes + provide pushed, validated commits and do not modify that branch directly. + +### Final acceptance checklist + +- [ ] Every semantic regex test that passes in the PR 958 baseline still passes. +- [ ] The complete `dev/tools/perl_test_runner.pl` output is compared + file-by-file with + `../PerlOnJava/logs/test_20260815_080000_958.log`. +- [ ] JVM and interpreter results agree for direct and thread regex tests. +- [ ] Forced-Joni runs cover ordinary constants, runtime patterns, embedded + closures, conditions, control verbs, recursion, and dynamic source. +- [ ] `pat_psycho*` and `speed*` complete with the bounded parallel policy. +- [ ] No supported regex test requires `JPERL_UNIMPLEMENTED=warn`. +- [ ] Joni is the sole production matcher; Java routing and selector code are + removed. +- [ ] Matcher-semantic preprocessing is gone; only documented Perl source-policy + scanning remains. +- [ ] Obsolete regex import patches are removed and targeted sync is idempotent. +- [ ] `docs/reference/feature-matrix.md` contains every known missing regex + feature and accurately reports the backend used. +- [ ] `dev/implementation/regex.md` and + `docs/design/joni-callout-fork.md` describe the shipped architecture + clearly and consistently. +- [ ] Redundant design documents are removed or reduced to concise rationale + summaries that point to canonical documentation. +- [ ] Original Joni/JCodings copyright and authorship notices are preserved. +- [ ] Performance remains within the gate defined above. +- [ ] `make` is warning-free; packaging, license checks, Ubuntu CI, and Windows + CI pass. + +### Open decisions and blockers + +- Optimizer/debug transcript assertions are reported separately from semantic + behavior and must never silently alter the raw baseline comparison. +- A runtime or shared-language defect that prevents a regex test from executing + is fixed in its owning subsystem; it is not approximated in the regex engine. +- Any proposed permanent adapter behavior must be classified as Perl source + policy. If it depends on backtracking or match state, it belongs in Joni. +- Resource-sensitive baselines and final timing tests require a quiet, + serialized build slot. ## Related Documents and Skills - `docs/design/joni-callout-fork.md` +- `dev/implementation/regex.md` - `dev/design/executable-regex-callbacks.md` -- `dev/design/regex_parser_integration.md` -- `dev/design/regex_preprocessing_fixes.md` +- `dev/design/regex-foreach-lexical-fix.md` +- `dev/design/regex-script-properties.md` +- `dev/design/regex-property-aliases.md` - `.agents/skills/debug-perlonjava/SKILL.md` From d2353808a24e3cd9b67fd55dc0337ac4bfa6d454 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Tue, 18 Aug 2026 21:40:21 +0200 Subject: [PATCH 03/31] docs(regex): require native Joni lookbehind ownership Make the forward migration dependency explicit: ceiling diagnostics, valid folded and nested forms, and ACCEPT-aware widths must be native before Java lookbehind routing and analysis are removed. Generated with Codex (https://openai.com/codex) Co-Authored-By: Codex --- dev/design/phase36-regex-parity.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index 7265b97967..0803106c0a 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -258,7 +258,11 @@ if they expose a semantic defect. - retire the next ordinary-pattern fallbacks in this priority order: 1. route ordinary lookbehind through Joni and delete the Java-only lookbehind length analyzer after the 27-assertion four-leg reducer and - six imported owner-file pass sets are non-regressing; + six imported owner-file pass sets are non-regressing; before enabling + that routing, Joni must enforce Perl's 255-character ceiling, accept + valid folded-class and nested-assertion lookbehinds, and preserve + `(*ACCEPT)`-reachable effective widths; do not replace these native + prerequisites with a narrower permanent Java fallback; 2. route branch-reset groups and delete their exclusive rewrite only after the named/numeric-call reducers and complete imported branch-reset gate pass without the temporary automatic-Java guard; From 6c3ffe13d09104f7cda8f047155c98df1e5ea083 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Tue, 18 Aug 2026 22:12:56 +0200 Subject: [PATCH 04/31] docs(regex): require lexical branch-reset call targets Make the native-Joni branch-reset gate explicitly cover relative calls whose physical target differs across alternatives sharing one logical capture number. Generated with [Codex](https://openai.com/codex/) Co-Authored-By: Codex --- dev/design/phase36-regex-parity.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index 0803106c0a..3a8383bac8 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -265,7 +265,10 @@ if they expose a semantic defect. prerequisites with a narrower permanent Java fallback; 2. route branch-reset groups and delete their exclusive rewrite only after the named/numeric-call reducers and complete imported branch-reset gate - pass without the temporary automatic-Java guard; + pass without the temporary automatic-Java guard; relative calls such as + `(?-1)` must retain their lexical physical target in each branch-reset + alternative rather than resolving later through the shared logical + capture number; 3. route alphabetic assertions and delete their recursive Java rewrite after direct Joni tests and the classified `alpha_assertions.t` gate prove every remaining failure is understood; From 608cbeb89e9ca6721d5e77b3937b876c79dd7e92 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Tue, 18 Aug 2026 22:45:38 +0200 Subject: [PATCH 05/31] docs(regex): record routing-test policy blocker Keep the forward-only Phase 36 plan explicit about temporary Java-routing assertions that cannot coexist with the final Joni-only architecture under the current no-existing-test-edit rule. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- dev/design/phase36-regex-parity.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index 3a8383bac8..d8648d4900 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -374,6 +374,12 @@ detection and do not replace implementation work. ### Open decisions and blockers +- Temporary backend-policy tests that assert Java routing can conflict with the + required final Joni-only behavior. The repository forbids modifying existing + tests, so a routing-removal slice that makes such an assertion obsolete must + remain preserved and unmerged until the user explicitly approves a test + replacement/update policy; implementation must not disguise the new route to + keep a stale assertion green. - Optimizer/debug transcript assertions are reported separately from semantic behavior and must never silently alter the raw baseline comparison. - A runtime or shared-language defect that prevents a regex test from executing From 5fa95af93129171c47e5bba2662636b97c01d78b Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Tue, 18 Aug 2026 23:10:15 +0200 Subject: [PATCH 06/31] docs(regex): track active Unicode migration gates Add the remaining horizontal-whitespace, leading-loose alias, and ASCII-strict fold gates to the forward-only Phase 36 critical path. References dev/design/phase36-regex-parity.md. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- dev/design/phase36-regex-parity.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index d8648d4900..e572326ea7 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -279,9 +279,17 @@ if they expose a semantic defect. current artifact and establish new exact counts; the stale pre-migration counts are not a release gate for the progressive backend. 3. Complete the remaining Unicode ownership boundary: + - parse `\h` and `\H` natively as Perl's exact horizontal-whitespace set in + direct and character-class forms, including scoped `/a` and `/aa`, without + changing stock Ruby-syntax hexadecimal escapes; + - close the leading-loose Block and Script shortcut families, then classify + the remaining bare binary, enumerated, and General_Category aliases while + preserving user-property and property-family precedence; - preserve per-member fold policy in Joni's composed-class AST through union, intersection, negation, and nested classes, then remove the corresponding adapter translation; + - close ASCII-strict multi-character folds and the remaining `/aa` imported + owner families before deleting their Java fold routing; - represent property-value wildcard parsing and diagnostics with a dedicated Joni syntax node rather than flattening wildcard behavior into literal ranges prematurely; From 0bc089b2cc5a9defbc4078b5cedb2954d36aff53 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 00:15:36 +0200 Subject: [PATCH 07/31] docs(regex): record current forced-Joni acceptance gates Keep the forward-only plan anchored to the current 80-file differential and prioritize the two native fixes that unlock the truncated pat suites. Generated with [OpenAI Codex](https://openai.com/codex) Co-Authored-By: OpenAI Codex --- dev/design/phase36-regex-parity.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index e572326ea7..0f09ccb515 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -274,10 +274,16 @@ if they expose a semantic defect. prove every remaining failure is understood; - never move callback, condition, control-verb, or dynamic-source patterns back to Java. - - after the current native stack is integrated, refresh the four-leg - `perl5_t/t/re` matrix (forced Java/Joni × JVM/interpreter) against one - current artifact and establish new exact counts; the stale pre-migration - counts are not a release gate for the progressive backend. + - use the current forced-Joni JVM gate of 363,164/391,977 assertions across + the same 80 regex files as the active migration manifest; it has 19 + per-file pass-count regressions against PR 958 that must reach zero; + - first remove the fatal loose-binary-property blocker that leaves both + `pat.t` variants at 631/1,302 executed assertions, then remove the wide + scalar fatal that leaves both `pat_advanced.t` variants at zero TAP; + - after those native patches are integrated, refresh all four legs (forced + Java/Joni × JVM/interpreter) against one current artifact. Reject zero-TAP, + timeout, incomplete, or negative-file results before a long acceptance run + is offered for user testing. 3. Complete the remaining Unicode ownership boundary: - parse `\h` and `\H` natively as Perl's exact horizontal-whitespace set in direct and character-class forms, including scoped `/a` and `/aa`, without From 57282505d4f2a66743126eebccd2e2b4c0338a72 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 00:31:50 +0200 Subject: [PATCH 08/31] docs(regex): record pat property unblock Update the forward-only critical path with the current forced-Joni pat gate after the native Unicode alias stack removes the fatal property lookup. Generated with [OpenAI Codex](https://openai.com/codex) Co-Authored-By: OpenAI Codex --- dev/design/phase36-regex-parity.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index 0f09ccb515..f66e868144 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -277,9 +277,11 @@ if they expose a semantic defect. - use the current forced-Joni JVM gate of 363,164/391,977 assertions across the same 80 regex files as the active migration manifest; it has 19 per-file pass-count regressions against PR 958 that must reach zero; - - first remove the fatal loose-binary-property blocker that leaves both - `pat.t` variants at 631/1,302 executed assertions, then remove the wide - scalar fatal that leaves both `pat_advanced.t` variants at zero TAP; + - the current combined candidate clears the loose-binary-property fatal: + each `pat.t` variant executes 1,301/1,302 assertions and passes 1,223; + classify and close the remaining 79 failures plus the one-test plan + shortfall, then remove the wide-scalar fatal that leaves both + `pat_advanced.t` variants at zero TAP; - after those native patches are integrated, refresh all four legs (forced Java/Joni × JVM/interpreter) against one current artifact. Reject zero-TAP, timeout, incomplete, or negative-file results before a long acceptance run From f4c75d0c7d80719c28b8ba63bf72b7caa99ffe04 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 04:20:11 +0200 Subject: [PATCH 09/31] docs(regex): refresh current migration plan Keep the execution tracker and critical path aligned with the current native Joni acceptance gates, remaining diagnostics, `/aa` residuals, and explicit test-policy blockers without adding implementation history to the plan. Generated with [OpenAI Codex](https://openai.com/codex) Co-Authored-By: OpenAI Codex --- dev/design/phase36-regex-parity.md | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index f66e868144..925d6ddea8 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -231,7 +231,7 @@ identifiers, dates, and completed repair narratives belong in Git history. - [ ] Phase 1 — Joni ordinary-pattern parity - [x] Phase 2 — conditions and backtracking-visible state - [ ] Phase 3 — Unicode and pattern syntax completion -- [x] Phase 4 — runtime source and diagnostics +- [ ] Phase 4 — runtime source and diagnostics - [ ] Phase 5 — remove the Java matching backend - [ ] Phase 6 — integration and release @@ -280,8 +280,15 @@ if they expose a semantic defect. - the current combined candidate clears the loose-binary-property fatal: each `pat.t` variant executes 1,301/1,302 assertions and passes 1,223; classify and close the remaining 79 failures plus the one-test plan - shortfall, then remove the wide-scalar fatal that leaves both - `pat_advanced.t` variants at zero TAP; + shortfall; each `pat_advanced.t` variant now executes all 1,687 assertions + and passes 1,570 on both JVM and interpreter backends, so classify and + close the remaining 117 failures without relying on the runner's status + heuristic or treating optimizer/debug transcripts as semantic parity; + - the current forced-Joni `reg_mesg.t` gate executes 2,595 assertions and + passes 1,692 after malformed `\g` diagnostics; finish typed warning + collection, Perl wording/categories, source markers, strict-mode + classification, and fatal-versus-warning behavior in the native frontend + and source-policy renderer; - after those native patches are integrated, refresh all four legs (forced Java/Joni × JVM/interpreter) against one current artifact. Reject zero-TAP, timeout, incomplete, or negative-file results before a long acceptance run @@ -297,12 +304,18 @@ if they expose a semantic defect. intersection, negation, and nested classes, then remove the corresponding adapter translation; - close ASCII-strict multi-character folds and the remaining `/aa` imported - owner families before deleting their Java fold routing; + owner families before deleting their Java fold routing; the current exact + forced-Joni `/aa` envelope executes 837 assertions and passes 833 after + strict literal, mixed-class, and scoped mixed-source fold repairs; the four + remaining failed assertions report 48 concrete backreference records and + 10 aggregate summaries, with JVM/interpreter identity; - represent property-value wildcard parsing and diagnostics with a dedicated Joni syntax node rather than flattening wildcard behavior into literal ranges prematurely; - close the remaining generated property/value alias gaps against the pinned - Perl 5.44 corpus; + Perl 5.44 corpus while retaining native `All` through Perl's signed-IV-wide + scalar domain via the long-range property result rather than truncating to + Java `int`; - keep Perl lexical/source policy in the adapter; - prefer bundled Perl Unicode data over duplicating ICU behavior or depending on the host JDK Unicode version. @@ -396,6 +409,11 @@ detection and do not replace implementation work. remain preserved and unmerged until the user explicitly approves a test replacement/update policy; implementation must not disguise the new route to keep a stale assertion green. +- Production diagnostics use Perl's exact trailing space after an + end-of-pattern `<-- HERE` marker. The obsolete no-trailing-space formatter + entry point remains only because its current unit test asserts that legacy + rendering; deleting it requires the same explicit existing-test update + policy. - Optimizer/debug transcript assertions are reported separately from semantic behavior and must never silently alter the raw baseline comparison. - A runtime or shared-language defect that prevents a regex test from executing From 051c81568a382ab24a1a37dd68e3180debca37b2 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 04:50:29 +0200 Subject: [PATCH 10/31] docs(regex): update current native Joni gates Refresh the forward-only critical path with the completed `/aa` acceptance envelope and latest native diagnostic count. Implementation history remains in commits rather than the design plan. Generated with [OpenAI Codex](https://openai.com/codex) Co-Authored-By: OpenAI Codex --- dev/design/phase36-regex-parity.md | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index 925d6ddea8..a48f6105ce 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -285,8 +285,8 @@ if they expose a semantic defect. close the remaining 117 failures without relying on the runner's status heuristic or treating optimizer/debug transcripts as semantic parity; - the current forced-Joni `reg_mesg.t` gate executes 2,595 assertions and - passes 1,692 after malformed `\g` diagnostics; finish typed warning - collection, Perl wording/categories, source markers, strict-mode + passes 1,694 after malformed backreference diagnostics; finish typed + warning collection, Perl wording/categories, source markers, strict-mode classification, and fatal-versus-warning behavior in the native frontend and source-policy renderer; - after those native patches are integrated, refresh all four legs (forced @@ -303,12 +303,10 @@ if they expose a semantic defect. - preserve per-member fold policy in Joni's composed-class AST through union, intersection, negation, and nested classes, then remove the corresponding adapter translation; - - close ASCII-strict multi-character folds and the remaining `/aa` imported - owner families before deleting their Java fold routing; the current exact - forced-Joni `/aa` envelope executes 837 assertions and passes 833 after - strict literal, mixed-class, and scoped mixed-source fold repairs; the four - remaining failed assertions report 48 concrete backreference records and - 10 aggregate summaries, with JVM/interpreter identity; + - the exact forced-Joni `/aa` envelope now passes 837/837 on JVM and + interpreter, including strict literal, mixed-class, scoped mixed-source, + and backreference folds; remove its temporary Java fold routing after the + combined 80-file gate confirms no owner-file regression; - represent property-value wildcard parsing and diagnostics with a dedicated Joni syntax node rather than flattening wildcard behavior into literal ranges prematurely; From ab87365b8ebc7fa1a1244273600292db85128970 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 06:17:06 +0200 Subject: [PATCH 11/31] docs(regex): refresh the Phase 36 critical path Record the completed native /aa routing and Unicode alias slices, and make lexical named-character sequences plus the pinned Unicode residual the next forward implementation targets. Generated with [OpenAI Codex](https://openai.com/codex/) Co-Authored-By: OpenAI Codex --- dev/design/phase36-regex-parity.md | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index a48f6105ce..ddd31f19d3 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -303,13 +303,22 @@ if they expose a semantic defect. - preserve per-member fold policy in Joni's composed-class AST through union, intersection, negation, and nested classes, then remove the corresponding adapter translation; - - the exact forced-Joni `/aa` envelope now passes 837/837 on JVM and - interpreter, including strict literal, mixed-class, scoped mixed-source, - and backreference folds; remove its temporary Java fold routing after the - combined 80-file gate confirms no owner-file regression; + - the exact `/aa` envelope passes 837/837 on JVM and interpreter, including + strict literal, mixed-class, scoped mixed-source, and backreference folds; + automatic and forced cells are byte-identical on native Joni, and the + temporary Java fold route is gone; retain this as a combined-corpus guard; + - complete native multi-character and empty lexical `\N{name}` atoms, + character-class alternatives, compile caching, lexical scope restoration, + stringification, and exact extended-class diagnostics without moving + matcher semantics back into textual preprocessing; - represent property-value wildcard parsing and diagnostics with a dedicated Joni syntax node rather than flattening wildcard behavior into literal ranges prematurely; + - the native Word_Break, Sentence_Break, and Vertical_Orientation alias + families are closed with zero corpus introductions; continue from the + pinned 1,720/83,648 residual set by closing bare binary-property aliases, + then the remaining Block, General_Category, compatibility, and wildcard + families; - close the remaining generated property/value alias gaps against the pinned Perl 5.44 corpus while retaining native `All` through Perl's signed-IV-wide scalar domain via the long-range property result rather than truncating to From 76b2cad011cb38ebeffffadcb677dbf501018a3f Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 07:32:10 +0200 Subject: [PATCH 12/31] docs(regex): refresh pat advanced baseline Replace the stale combined count with the exact complete 1,687-test pre-named-character differential baseline and keep the next gate forward-looking. Generated with [OpenAI Codex](https://openai.com/codex/) Co-Authored-By: OpenAI Codex --- dev/design/phase36-regex-parity.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index ddd31f19d3..9e850664bf 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -280,10 +280,12 @@ if they expose a semantic defect. - the current combined candidate clears the loose-binary-property fatal: each `pat.t` variant executes 1,301/1,302 assertions and passes 1,223; classify and close the remaining 79 failures plus the one-test plan - shortfall; each `pat_advanced.t` variant now executes all 1,687 assertions - and passes 1,570 on both JVM and interpreter backends, so classify and - close the remaining 117 failures without relying on the runner's status - heuristic or treating optimizer/debug transcripts as semantic parity; + shortfall; the exact pre-named-character `pat_advanced.t` baseline executes + all 1,687 assertions and passes 1,577 on both JVM and interpreter, with + byte-identical 110-row residuals; refresh that complete gate after the + named-character and `(*THEN)` slices, then close the remaining semantic + failures without relying on the runner's status heuristic or treating + optimizer/debug transcripts as semantic parity; - the current forced-Joni `reg_mesg.t` gate executes 2,595 assertions and passes 1,694 after malformed backreference diagnostics; finish typed warning collection, Perl wording/categories, source markers, strict-mode From 15d2cd776438de9d5a2cc7a52f17de7305026811 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 07:40:01 +0200 Subject: [PATCH 13/31] docs(regex): record current complete pat gate Advance the forward critical path to the complete 1,625/1,687 dual-backend gate after native named-character and alternation-boundary repairs. Generated with [OpenAI Codex](https://openai.com/codex/) Co-Authored-By: OpenAI Codex --- dev/design/phase36-regex-parity.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index 9e850664bf..dfb7571f96 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -280,12 +280,12 @@ if they expose a semantic defect. - the current combined candidate clears the loose-binary-property fatal: each `pat.t` variant executes 1,301/1,302 assertions and passes 1,223; classify and close the remaining 79 failures plus the one-test plan - shortfall; the exact pre-named-character `pat_advanced.t` baseline executes - all 1,687 assertions and passes 1,577 on both JVM and interpreter, with - byte-identical 110-row residuals; refresh that complete gate after the - named-character and `(*THEN)` slices, then close the remaining semantic - failures without relying on the runner's status heuristic or treating - optimizer/debug transcripts as semantic parity; + shortfall; the current `pat_advanced.t` gate executes all 1,687 assertions + and passes 1,625 on both JVM and interpreter, with byte-identical 62-row + residuals and zero introductions against the exact 110-row baseline; + close the remaining semantic failures without relying on the runner's + status heuristic or treating optimizer/debug transcripts as semantic + parity; - the current forced-Joni `reg_mesg.t` gate executes 2,595 assertions and passes 1,694 after malformed backreference diagnostics; finish typed warning collection, Perl wording/categories, source markers, strict-mode From 2880f2a629b7f486e1c1c5c703b50255562eff5d Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 08:07:43 +0200 Subject: [PATCH 14/31] docs(regex): refresh forced-Joni parity position Record the current complete pat_advanced gate after native unbraced relative subpattern calls, with identical JVM/interpreter residuals and no introduced failures. Generated with [OpenAI Codex](https://openai.com/codex/) Co-Authored-By: OpenAI Codex --- dev/design/phase36-regex-parity.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index dfb7571f96..ddc7146e5b 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -281,7 +281,7 @@ if they expose a semantic defect. each `pat.t` variant executes 1,301/1,302 assertions and passes 1,223; classify and close the remaining 79 failures plus the one-test plan shortfall; the current `pat_advanced.t` gate executes all 1,687 assertions - and passes 1,625 on both JVM and interpreter, with byte-identical 62-row + and passes 1,629 on both JVM and interpreter, with byte-identical 58-row residuals and zero introductions against the exact 110-row baseline; close the remaining semantic failures without relying on the runner's status heuristic or treating optimizer/debug transcripts as semantic From 1e6defd7d76496975d1c69dcf85f7d2e5f674bdb Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 08:42:20 +0200 Subject: [PATCH 15/31] docs(regex): advance Unicode migration position Record the closed bare binary-property alias family and the current identical JVM/interpreter Unicode residual gate. Generated with [OpenAI Codex](https://openai.com/codex/) Co-Authored-By: OpenAI Codex --- dev/design/phase36-regex-parity.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index ddc7146e5b..f6f3c35f70 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -316,10 +316,10 @@ if they expose a semantic defect. - represent property-value wildcard parsing and diagnostics with a dedicated Joni syntax node rather than flattening wildcard behavior into literal ranges prematurely; - - the native Word_Break, Sentence_Break, and Vertical_Orientation alias - families are closed with zero corpus introductions; continue from the - pinned 1,720/83,648 residual set by closing bare binary-property aliases, - then the remaining Block, General_Category, compatibility, and wildcard + - the native Word_Break, Sentence_Break, Vertical_Orientation, and bare + binary-property alias families are closed with zero corpus introductions; + continue from the byte-identical 1,188/83,648 JVM/interpreter residual set + with the remaining Block, General_Category, compatibility, and wildcard families; - close the remaining generated property/value alias gaps against the pinned Perl 5.44 corpus while retaining native `All` through Perl's signed-IV-wide From 8b90436a828380fae0df36be5d0663c5b9cc6cf7 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 09:08:36 +0200 Subject: [PATCH 16/31] docs(regex): advance current parity count Record the current forward-looking forced-Joni pat_advanced gate after recursive callout capture visibility reached parity. Generated with [OpenAI Codex](https://openai.com/codex) Co-Authored-By: OpenAI Codex --- dev/design/phase36-regex-parity.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index f6f3c35f70..91c58aede6 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -281,7 +281,7 @@ if they expose a semantic defect. each `pat.t` variant executes 1,301/1,302 assertions and passes 1,223; classify and close the remaining 79 failures plus the one-test plan shortfall; the current `pat_advanced.t` gate executes all 1,687 assertions - and passes 1,629 on both JVM and interpreter, with byte-identical 58-row + and passes 1,632 on both JVM and interpreter, with byte-identical 55-row residuals and zero introductions against the exact 110-row baseline; close the remaining semantic failures without relying on the runner's status heuristic or treating optimizer/debug transcripts as semantic From 1965a96f5495442ecef84815b7b677b2ca0766f1 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 09:35:23 +0200 Subject: [PATCH 17/31] docs(regex): compress the Phase 36 execution plan Replace repeated implementation detail with a forward-only current position, final architecture, ordered next steps, parallel ownership model, and complete acceptance gates. Generated with [OpenAI Codex](https://openai.com/codex) Co-Authored-By: OpenAI Codex --- dev/design/phase36-regex-parity.md | 681 ++++++++++++----------------- 1 file changed, 274 insertions(+), 407 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index 91c58aede6..ede690a0f5 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -2,442 +2,309 @@ ## Goal -Complete Perl 5.44 regular-expression semantics on both PerlOnJava execution -backends and converge on the vendored, namespaced Joni engine for all matching. -Java `Pattern` remains available only as a temporary differential backend while -the migration is being proved. +Implement Perl 5.44 regular-expression semantics on both PerlOnJava execution +backends and make the vendored Joni fork the sole production matcher. Java +`Pattern` and the backend selector are temporary differential tools and must be +removed before completion. -The acceptance target includes matching, captures, match state, callbacks, -dynamic patterns, errors, warning categories and locations, byte and Unicode -behavior, direct/thread parity, and unchanged-source CPAN consumers. Assertions -that inspect Perl's internal optimizer program or debug transcript are reported -separately from language-semantic failures. +The contract includes matching, captures, match state, callbacks, dynamic +patterns, diagnostics, warnings, source locations, byte/Unicode behavior, +direct/thread parity, unchanged Perl core tests, and unchanged CPAN consumers. +Optimizer programs and debug transcripts are reported separately from language +semantics. -The historical comparison point is: +The immutable no-regression comparison point is: ```text ../PerlOnJava/logs/test_20260815_080000_958.log ``` -## Architecture +## Final Architecture -### Final engine boundary - -- The vendored Joni fork is the sole production matcher. -- PerlOnJava owns Perl source policy: interpolation provenance, `use re 'eval'`, - lexical warning and modifier state, executable callback closures, user-defined - Unicode properties, source locations, and Perl diagnostics. -- Joni owns regex parsing and matcher semantics: captures, conditions, +- The forked Joni engine is implemented and maintained in this repository. +- Joni owns regex grammar and matcher semantics: captures, conditions, recursion, lookarounds, case folding, control verbs, backtracking-visible - state, and byte/Unicode matching. -- The fork remains runtime-neutral. It receives internal callback IDs and a - matcher-local handler API, never Perl source or PerlOnJava runtime objects. -- Upstream packages and notices stay unchanged in `third_party/joni`; standalone - packaging relocates Joni and JCodings into `org.perlonjava.internal`. - -### Migration controls - -A temporary developer-only backend selector supports separate Java and Joni -corpus runs. It must never run both matchers for one operation because callbacks, -tied variables, `pos()`, and substitutions may have observable side effects. -Joni is the default matcher; explicit Java mode remains only for differential -measurement. The selector and Java matching fields are removed at the end of the -migration. - -### Preprocessing boundary - -Every current `RegexPreprocessor` rule is classified before it moves: - -1. Perl source policy remains outside Joni. -2. Backend-neutral spelling normalization moves into a small frontend scanner. -3. Matcher semantics move into Joni's parser, compiler, and matcher. -4. Java-only syntax rewrites and stack workarounds are deleted with the Java - matching backend. - -Text rewriting must not emulate behavior that depends on backtracking, capture -close order, matcher regions, or encoding. - -## Implementation Phases + state, byte/Unicode matching, and property membership. +- PerlOnJava owns source policy: interpolation provenance, lexical hints and + warnings, `use re 'eval'`, executable Perl closures, user-defined properties, + source locations, and final Perl diagnostic rendering. +- The fork remains runtime-neutral. Callouts use internal IDs and a matcher-local + handler API; Joni never receives Perl source or PerlOnJava runtime objects. +- Upstream Joni/JCodings package names and all copyright/authorship notices stay + intact under `third_party/`. Standalone packaging relocates them to + `org.perlonjava.internal`, avoiding public namespace collisions without + rewriting the maintained source namespace. +- Joni is the default for every closure-bearing pattern throughout migration. + No callback, condition, control verb, recursive program, or dynamic source may + fall back to Java. Ordinary constants move to Joni as their native gates pass. +- Final production code has one matcher. Ordinary patterns allocate no callout + state, and a match operation never runs two engines because regex side effects + are observable. + +## Preprocessing Boundary + +Classify every current regex preprocessing rule as one of: + +1. Perl source policy retained in a small frontend scanner. +2. Backend-neutral spelling normalization retained outside the matcher. +3. Matcher semantics moved into Joni parser/compiler/matcher internals. +4. Java-only translation or stack workaround deleted with the Java backend. + +Text rewriting must not emulate behavior dependent on capture close order, +backtracking, matcher regions, encoding, or callback execution. As native fixes +land, remove corresponding `RegexPreprocessor` code immediately and rerun the +affected corpus before taking another slice. + +## Current Validated Position + +- The reproducible differential baseline and conditions/control-verb phase are + complete. +- `(*MARK:NAME)`, named `(*SKIP:NAME)`, `$REGMARK`, `$REGERROR`, callback unwind, + and native `(*THEN)` branch boundaries are in the Joni path. +- Absolute, forward, backward, whole-pattern, and signed-relative numbered + subpattern calls parse natively in Joni; the adapter rewrite is gone. +- Recursive callouts observe captures from the just-completed recursive frame. +- Folded exact-search candidate bounds are safe for long ASCII-strict literals, + and native numbered-call diagnostics retain the Perl source location. +- Forced-Joni `pat_advanced.t` executes 1,687/1,687 and passes 1,644 on JVM and + interpreter, with identical 43-row residuals and zero introductions against + the exact 110-row reference set. +- Forced-Joni `reg_mesg.t` passes 1,694/2,595. +- Forced-Joni Unicode property comparison has 1,124/83,648 residual records on + each backend, with byte-identical normalized output. +- Each `pat.t` variant executes 1,301/1,302 and passes 1,223. +- The current 80-file forced-Joni gate passes 363,164/391,977 and has 19 + per-file pass-count regressions against PR 958. These figures must be refreshed + after the current native stack is integrated. +- Exact `/aa` routing/folding gates pass on native Joni, and the Java `/aa` + workaround is removed. + +## Execution Phases ### Phase 0 — Reproducible differential baseline -1. Run all 80 `perl5_t/t/re` files on JVM and interpreter backends from the same - clean commit after confirming that no unrelated PerlOnJava builds are active. -2. Save complete output and JSON outside the source tree, compare every file - against the PR 958 baseline, and reject any unexplained zero-TAP or timeout - result. -3. Classify every failure as matcher semantics, source policy, diagnostics, - shared non-regex behavior, or optimizer/debug transcript. -4. Give `pat_psycho*` and `speed*` a configurable two-worker CPU-heavy lane. - Keep `pat*`, `pat_advanced*`, and memory-sensitive fixtures in a one-worker - exclusive lane. Every child retains its own hard timeout and process group. - -Exit criteria: the baseline is repeatable, direct/thread and JVM/interpreter -differences are visible, and the report identifies the next semantic slice. - -### Phase 1 — Joni ordinary-pattern parity - -1. Add the temporary forced-backend selector and route ordinary patterns through - Joni by default in focused tests. -2. Compile separate byte and Unicode variants using the source and target scalar - metadata; preserve raw byte offsets and convert UTF-8 offsets only at the Perl - match-variable boundary. -3. Complete matcher regions, anchoring and transparent bounds, zero-width search - progression, `\G`, `/g`, `/c`, `/o`, captures, duplicate names, branch reset, - regex-object reuse, substitution, and nested match-state restoration. -4. Run the same corpus once with each forced backend. Do not hide unsupported - Joni behavior by falling back within a match. - -Exit criteria: every assertion previously passing on Java also passes on Joni, -with no direct/thread or JVM/interpreter regression. +- Run the same 80 `perl5_t/t/re` files on JVM/interpreter and forced Java/Joni. +- Save complete logs and machine-readable results outside the tree. +- Reject zero-TAP, timeout, truncated, or incomplete records before comparing. +- Classify failures as matcher semantics, source policy, diagnostics, shared + runtime behavior, or optimizer/debug-only output. -### Phase 2 — Conditions and backtracking-visible state +Exit: the baseline is repeatable and every next slice has exact rows and an +oracle. -1. Implement numbered and named capture conditions, assertion conditions, - recursion conditions, `(DEFINE)`, and executable callback conditions in Joni. -2. Complete `(*MARK:name)`, named `(*SKIP:name)`, `$REGMARK`, `$REGERROR`, cut - boundaries, recursion limits, and interactions with lookarounds, subpattern - calls, dynamic programs, and callback unwind. -3. Preserve exact capture-close order, provisional match variables, dynamic - locals, and callback side effects along the selected matcher path. +### Phase 1 — Ordinary-pattern Joni parity -Exit criteria: focused standard-Perl oracles and applicable `pat_advanced.t`, -`rxcode.t`, `reg_eval_scope.t`, and callback sections agree on both backends. +- Complete byte/Unicode variant selection from pattern and subject provenance. +- Close captures, duplicate names, branch reset, regions, bounds, zero-width + search progression, `\G`, `/g`, `/c`, `/o`, substitution, reuse, and nested + match-state restoration. +- Route each ordinary-pattern family to Joni only after focused and complete + differential gates show zero introductions. -### Phase 3 — Unicode and pattern syntax completion +Exit: every semantic assertion passing on Java passes on forced Joni, with JVM +and interpreter agreement. -1. Implement Perl property aliases, versioned `Age` forms, script extensions, - `\N{name}`, extended classes, user-defined property recursion and errors, and - byte-versus-Unicode warning behavior. -2. Complete remaining case-folding, grapheme, lookbehind, and invalid-pattern - diagnostics in the Joni frontend and engine. -3. Derive property names and aliases from the bundled Perl 5.44 Unicode data so - behavior does not depend on the host JDK Unicode version. +### Phase 2 — Conditions and backtracking-visible state -Exit criteria: semantic assertions in `regexp_unicode_prop.t`, `pat.t`, and -`pat_advanced.t` complete without `JPERL_UNIMPLEMENTED=warn` masking supported -syntax. +- Numbered/named/assertion/recursion/callback conditions and `(DEFINE)`. +- `(*MARK:NAME)`, named `(*SKIP:NAME)`, `(*PRUNE)`, `(*COMMIT)`, `(*THEN)`, + `$REGMARK`, `$REGERROR`, cut boundaries, recursion limits, and unwind. +- Capture-close order, provisional match variables, dynamic locals, and callback + side effects on the selected path. + +Exit: focused Perl oracles and relevant `pat_advanced.t`, `rxcode.t`, and +`reg_eval_scope.t` sections agree. + +### Phase 3 — Unicode and native pattern syntax + +- Add a development generator that reads and analyzes the repository's pinned + Perl 5.44 Unicode tables, then emits checked-in Java source for Joni property + names, loose aliases, value families, ranges, and case-fold metadata. The same + script emits resolver oracle fixtures plus input/output checksums. Generated + Java is reproducible and its second consecutive generation must be diff-free; + hand-written Java is limited to reviewed precedence and behavior that cannot + be derived from the source tables. +- Finish Block, Script, Script_Extensions, General_Category, binary, + compatibility, wildcard, versioned `Age`, POSIX, `\h`/`\H`, and user-property + behavior from pinned Perl 5.44 Unicode data. +- Preserve property-family and user-callback precedence, signed-IV-wide scalar + domains, and byte/Unicode warnings; do not duplicate ICU behavior when the + pinned data or ICU/JCodings already provides it. +- Complete `\N{name}` single/multi/empty atoms, classes, lexical translators, + caching, and exact extended-class diagnostics. +- Complete `/d`, `/u`, `/a`, `/aa`, literal/backreference/class/property case + folding and optimizer search safety. +- Complete Perl escapes, subpattern calls, lookbehind width/255-character rules, + branch-reset lexical targets, `\K`, recursion safety, and invalid-pattern + diagnostics natively in Joni. + +Exit: semantic `regexp_unicode_prop.t`, `pat.t`, and `pat_advanced.t` assertions +complete without masking supported syntax through `JPERL_UNIMPLEMENTED=warn`. ### Phase 4 — Runtime source and diagnostics -1. Preserve runtime-eval source names, package and lexical context, warning - masks, line numbers, syntax errors, and Unicode/byte source identity. -2. Complete recursive and nested `(??{...})`, mixed literal/runtime executable - source, tied and localized interpolation, regex-object stringification, and - `/g`, `/c`, `/o` state across callback and exception boundaries. -3. Close the semantic assertions in `pat_re_eval.t`. Track shared non-regex - `eval` failures separately, but fix them when they prevent regex source from - executing with standard Perl behavior. - -Exit criteria: all 555 `pat_re_eval.t` assertions execute and every semantic -assertion passes on both execution backends. - -### Phase 5 — Remove the Java matching backend - -1. Move every remaining matcher-semantic preprocessor rule into Joni. -2. Delete Java compiled-pattern variants, feature routing, Java-only rewrites, - and the temporary backend selector. -3. Retain only the small Perl source-policy/frontend layer described above. -4. Remove stale parser and preprocessing plans or rewrite them to describe the - final ownership boundary. - -Exit criteria: Joni is the only production matcher and ordinary patterns do not -allocate callback state or callback frames. - -### Phase 6 — Integration and release - -1. Keep regex core tests unpatched. The canonical `perl5/t` directory import - owns `re/pat.t`; no duplicate file row or regex-test patch may replace or - weaken upstream assertions. -2. Run `perl dev/import-perl5/sync.pl --only perl5/t` twice, verify the imported - `re/pat.t` hash against the configured upstream source, and require the - second run to produce no content diff. If an upstream assertion fails after - sync, fix PerlOnJava rather than editing the imported test. -3. Run the complete direct and `_thr.t` regex matrix on JVM and interpreter - backends and compare it file-by-file with both the Phase 0 result and PR 958. -4. Run unchanged Type::Tiny, Regexp::Common, Object::InsideOut, and every CPAN - suite whose regex capability policy is removed. -5. Run warning-free `make`, Joni upstream tests, packaging and license checks, - and the thread release matrix. -6. Rewrite `dev/implementation/regex.md` to describe the final as-implemented - matcher architecture and ownership boundaries, and update - `docs/design/joni-callout-fork.md` to match the shipped fork API, namespace, - packaging, callback/unwind contract, and Unicode responsibilities. Review - both documents for a clear reader path, consistent terminology, and removal - of superseded proposals or predictions. Audit the remaining regex/Joni - design documents: delete only content that is wholly redundant and retains - no useful rationale; otherwise replace historical implementation plans with - concise summaries that preserve decisions and point to the canonical - implementation and fork documents. Preserve copyright and authorship - notices in every retained or consolidated third-party description. -7. Rebase each focused delivery slice onto current master. Require green Ubuntu - and Windows CI before merging and beginning the next slice. - -Exit criteria: all semantic gates pass, no previously passing file regresses, -the regex corpus is reproduced from `dev/import-perl5/sync.pl` without a regex -test patch, and documentation reports optimizer/debug-only exclusions -explicitly. - -### Imported-test provenance gate - -- `dev/import-perl5/config.yaml` imports `perl5/t/re/pat.t` through the - canonical `perl5/t` directory entry, without a duplicate row or patch. -- No regex-specific import patch weakens or skips upstream assertions. -- A targeted `--only perl5/t` sync restores the exact configured upstream - source. -- A second consecutive directory sync is content-idempotent and leaves the - tree clean. -- The synchronized direct and thread tests run unchanged on JVM and interpreter. - -## Test Contract - -- Validate every new or changed Perl unit test with system `perl` or `prove` - before running it with PerlOnJava. -- Run JVM and interpreter tests under `timeout`, capture complete output in - files, and inspect the saved files rather than truncated terminal output. -- Use `perl dev/tools/perl_test_runner.pl`; the runner requires process `fork` - and must not run under `jperl`. -- Run direct tests before thread wrappers. A wrapper must preserve the direct - result and may change only resources and ownership context. -- Unsupported syntax remains fatal until its complete semantic gate passes. -- Do not alter existing Perl core tests to fit PerlOnJava behavior. -- Treat the import manifest and its patch files as temporary compatibility debt: - remove each regex-test patch hunk as soon as its guarded behavior passes, then - use a targeted `sync.pl` run to recover the exact upstream test source. -- `make` must pass without warning output before every push. +- Preserve package, lexical context, warning masks/categories, filename, line, + syntax position, and byte/Unicode identity through literal, runtime, and eval + compilation. +- Complete recursive/nested `(??{...})`, mixed literal/runtime executable + source, tied/localized interpolation, object stringification, and `/g`/`/c`/ + `/o` behavior across callbacks and exceptions. +- Finish native warning collection plus exact Perl wording, markers, + fatal-versus-warning behavior, and source suffixes. +- Close all 555 `pat_re_eval.t` assertions on both execution backends; classify + shared non-regex eval defects separately but fix any that block regex source. + +Exit: runtime-generated regexes and diagnostics agree with standard Perl. + +### Phase 5 — Remove migration scaffolding + +- Delete every Java matcher field, route, syntax rewrite, fallback, and the + backend selector. +- Delete matcher-semantic preprocessing after each replacement gate is green. +- Keep only the documented Perl source-policy/frontend layer. +- Remove regex patches introduced by `dev/import-perl5/sync.pl`, then restore + exact upstream files with targeted sync. + +Exit: Joni is the only production matcher and imported regex tests are +unchanged upstream files. + +### Phase 6 — Release and documentation + +- Run all direct and `_thr.t` regex files on JVM/interpreter and compare the + complete `dev/tools/perl_test_runner.pl` output file-by-file with PR 958. +- Run unchanged Type::Tiny, Regexp::Common, Object::InsideOut, and every CPAN + suite affected by removed regex capability policy. +- Run performance, packaging, notices/licenses, warning-free build, Ubuntu, + Windows, and full CI gates. +- Update `docs/reference/feature-matrix.md`, including the expected-Joni feature + set around lines 390–409 and every currently missing Perl regex feature. +- Rewrite `dev/implementation/regex.md` as the clear as-implemented architecture. +- Update `docs/design/joni-callout-fork.md` for the shipped fork API, packaging, + namespace, callback/unwind contract, and Unicode ownership. +- Delete wholly redundant documents; reduce rationale-bearing older documents + to concise summaries pointing to the canonical implementation documents. + +Exit: all semantic and release gates pass and documentation matches shipped +behavior. + +## Ordered Next Steps + +1. Keep the canonical native-Joni PR and this plan branch durable. Require exact + commit/file review, warning-free `make`, and green stacked CI before moving a + PR from draft to user acceptance. +2. Finish the current independent slices and integrate only zero-introduction + commits: + - deterministic generated Java for all pinned Perl Unicode data families; + - the complete General_Category compatibility and wildcard family from the + 1,124-record Unicode residual; + - native named-call grammar, malformed-pattern diagnostics, and recursion + safety from the remaining parser tail; + - byte `/d` fold provenance only after named-character Unicode provenance is + correct; reject the broad ASCII-strict policy if it regresses named folds. +3. Consolidate Unicode generation before adding more alias families: use one + pinned development script to parse/analyze the Perl tables and emit compiled, + checked-in Java resolver/fold classes, generated oracle fixtures, and + checksums. Require deterministic, diff-free regeneration; keep runtime + provenance and matcher algorithms in reviewed hand-written code. +4. Refresh `pat_advanced.t` on the combined head and classify every remaining + semantic row. Prioritize `\N{}` extended-class failure, `\K`, property `/i` + closure, multi-character folds, recursion safety, wildcard/property errors, + then source/debug-only rows. +5. Refresh `reg_mesg.t`, `pat.t`, and the 83,648-record Unicode corpus on both + backends. Close the largest semantically uniform native-Joni groups with a + system-Perl-first reducer and zero-introduction complete gate for each. +6. Refresh all four 80-file legs on one combined artifact. Resolve all 19 + per-file PR 958 regressions; do not offer a long user acceptance run while + any negative, zero-TAP, timeout, or incomplete file is unexplained. +7. Use the integration report to retire ordinary Java fallbacks in impact order: + lookbehind, branch reset, alphabetic assertions, then remaining constant + patterns. Delete each route and its semantic preprocessor rule in the same + validated slice. +8. Complete runtime source/eval semantics and diagnostics, then close + `pat_re_eval.t`. +9. Remove obsolete regex import patches. Run + `perl dev/import-perl5/sync.pl --only perl5/t` twice; verify the configured + upstream `re/pat.t` hash and require the second sync to be content-idempotent. +10. Remove Java matching and the selector, rerun the complete semantic and CPAN + matrix, then execute performance and release gates. +11. Finish feature-matrix and as-implemented documentation, consolidate + redundant plans, rebase the final stack onto current master, and require + green Ubuntu/Windows CI before merge. + +## Parallel Work + +- Coordinator/integration: canonical stack, PR 958 comparison, PR/CI readiness, + plan state, ownership, and conflict resolution. +- Native syntax/matcher: one non-overlapping Joni grammar or matcher feature per + branch with direct fork tests and Perl reducers. +- Unicode: one classified property family per branch from the exact residual + artifact. +- Differential/release: immutable row sets, normalized comparators, fallback + impact ranking, import sync, CPAN and platform gates. +- Documentation: feature inventory and final architecture documents after the + corresponding behavior is validated. + +Workers use isolated worktrees and append-only handoff mailboxes. Assignments +state exact base, owned/excluded files, oracle, complete gates, correction +budget, and delivery evidence. Workers self-monitor CPU and may admit at most +three concurrent expensive jobs globally; timing-sensitive final gates are +serial. Workers normally stop after focused and complete affected-corpus gates +and deliver local commits without pushing. The coordinator batches two to four +non-overlapping deliveries, runs one warning-free full `make` on the combined +head, and only then pushes or updates the PR. A worker-local full build is +reserved for build-system changes or focused evidence of broad cross-suite risk. +`pat_psycho*` and `speed*` may use two CPU-heavy workers, while `pat*`, +`pat_advanced*`, and memory-sensitive fixtures remain one-worker exclusive. +Every `jperl`, `jcpan`, and `prove` process has a hard timeout. + +## Test and Delivery Contract + +- Never modify or delete existing tests. Validate every NEW Perl fixture with + system Perl before PerlOnJava. +- Capture complete output in files. Use `perl dev/tools/perl_test_runner.pl`, + never `jperl`, to drive the fork-based core runner. +- Compare JVM and interpreter results, direct and thread wrappers, and forced + backends where the temporary selector still exists. +- A semantic slice needs its focused oracle, direct Joni tests where applicable, + and complete affected corpus with zero introductions. Each combined + integration batch needs one warning-free `make` before push or PR update. +- Preserve original Joni/JCodings notices and verify relocated packaging. +- Never push master. Use focused branches, attributed commits, PRs, and current + master rebases after upstream merges. ## Performance Gate -Before removing Java matching, run five warmed ordinary-pattern measurements on -each backend. Joni's median runtime must be within 25% of the Java baseline, -must introduce no new timeout, and must not materially increase steady-state -allocation. A failure blocks backend removal, not semantic fixes. - -## Public Interfaces - -No permanent public regex API or command-line option is added. The temporary -developer backend selector is removed in Phase 5. Existing Perl syntax, -variables, warning categories, and regex object behavior are the public -compatibility contract. +Before deleting Java matching, run five warmed ordinary-pattern measurements on +each backend. Joni median runtime must be within 25% of the Java baseline, add +no timeout, and not materially increase steady-state allocation. Performance +failure blocks backend removal, not semantic fixes. ## Execution Tracker -This tracker records only current plan state. Implementation history, commit -identifiers, dates, and completed repair narratives belong in Git history. - -### Phase status - - [x] Phase 0 — reproducible differential baseline -- [ ] Phase 1 — Joni ordinary-pattern parity +- [ ] Phase 1 — ordinary-pattern Joni parity - [x] Phase 2 — conditions and backtracking-visible state -- [ ] Phase 3 — Unicode and pattern syntax completion +- [ ] Phase 3 — Unicode and native pattern syntax - [ ] Phase 4 — runtime source and diagnostics -- [ ] Phase 5 — remove the Java matching backend -- [ ] Phase 6 — integration and release +- [ ] Phase 5 — remove migration scaffolding +- [ ] Phase 6 — release and documentation A checked phase means its focused semantic implementation is complete. Release -and no-regression gates remain Phase 6 responsibilities and may reopen a phase -if they expose a semantic defect. - -### Current critical path - -1. Validate and merge the rebased native-Joni delivery stack bottom-up. After - each parent merge, rebase its child onto current master, verify the commit - range and expected file set, and require warning-free build plus green CI. - Preserve the negative-file manifest and normalized comparator as mandatory - pre-acceptance gates; no unexplained negative file may be deferred to a long - acceptance run for discovery. -2. Remove temporary ordinary-pattern Java routing as native Joni replacements - become green: - - remove the temporary adapter KEEP-in-lookaround guard after the native - Joni diagnostic stack passes its combined gate; - - remove Java routing immediately after each native reducer and combined - corpus gate are green; - - use the integration report to choose the next fallback whose removal moves - the most assertions to pure Joni; - - retire the next ordinary-pattern fallbacks in this priority order: - 1. route ordinary lookbehind through Joni and delete the Java-only - lookbehind length analyzer after the 27-assertion four-leg reducer and - six imported owner-file pass sets are non-regressing; before enabling - that routing, Joni must enforce Perl's 255-character ceiling, accept - valid folded-class and nested-assertion lookbehinds, and preserve - `(*ACCEPT)`-reachable effective widths; do not replace these native - prerequisites with a narrower permanent Java fallback; - 2. route branch-reset groups and delete their exclusive rewrite only after - the named/numeric-call reducers and complete imported branch-reset gate - pass without the temporary automatic-Java guard; relative calls such as - `(?-1)` must retain their lexical physical target in each branch-reset - alternative rather than resolving later through the shared logical - capture number; - 3. route alphabetic assertions and delete their recursive Java rewrite - after direct Joni tests and the classified `alpha_assertions.t` gate - prove every remaining failure is understood; - - never move callback, condition, control-verb, or dynamic-source patterns - back to Java. - - use the current forced-Joni JVM gate of 363,164/391,977 assertions across - the same 80 regex files as the active migration manifest; it has 19 - per-file pass-count regressions against PR 958 that must reach zero; - - the current combined candidate clears the loose-binary-property fatal: - each `pat.t` variant executes 1,301/1,302 assertions and passes 1,223; - classify and close the remaining 79 failures plus the one-test plan - shortfall; the current `pat_advanced.t` gate executes all 1,687 assertions - and passes 1,632 on both JVM and interpreter, with byte-identical 55-row - residuals and zero introductions against the exact 110-row baseline; - close the remaining semantic failures without relying on the runner's - status heuristic or treating optimizer/debug transcripts as semantic - parity; - - the current forced-Joni `reg_mesg.t` gate executes 2,595 assertions and - passes 1,694 after malformed backreference diagnostics; finish typed - warning collection, Perl wording/categories, source markers, strict-mode - classification, and fatal-versus-warning behavior in the native frontend - and source-policy renderer; - - after those native patches are integrated, refresh all four legs (forced - Java/Joni × JVM/interpreter) against one current artifact. Reject zero-TAP, - timeout, incomplete, or negative-file results before a long acceptance run - is offered for user testing. -3. Complete the remaining Unicode ownership boundary: - - parse `\h` and `\H` natively as Perl's exact horizontal-whitespace set in - direct and character-class forms, including scoped `/a` and `/aa`, without - changing stock Ruby-syntax hexadecimal escapes; - - close the leading-loose Block and Script shortcut families, then classify - the remaining bare binary, enumerated, and General_Category aliases while - preserving user-property and property-family precedence; - - preserve per-member fold policy in Joni's composed-class AST through union, - intersection, negation, and nested classes, then remove the corresponding - adapter translation; - - the exact `/aa` envelope passes 837/837 on JVM and interpreter, including - strict literal, mixed-class, scoped mixed-source, and backreference folds; - automatic and forced cells are byte-identical on native Joni, and the - temporary Java fold route is gone; retain this as a combined-corpus guard; - - complete native multi-character and empty lexical `\N{name}` atoms, - character-class alternatives, compile caching, lexical scope restoration, - stringification, and exact extended-class diagnostics without moving - matcher semantics back into textual preprocessing; - - represent property-value wildcard parsing and diagnostics with a dedicated - Joni syntax node rather than flattening wildcard behavior into literal - ranges prematurely; - - the native Word_Break, Sentence_Break, Vertical_Orientation, and bare - binary-property alias families are closed with zero corpus introductions; - continue from the byte-identical 1,188/83,648 JVM/interpreter residual set - with the remaining Block, General_Category, compatibility, and wildcard - families; - - close the remaining generated property/value alias gaps against the pinned - Perl 5.44 corpus while retaining native `All` through Perl's signed-IV-wide - scalar domain via the long-range property result rather than truncating to - Java `int`; - - keep Perl lexical/source policy in the adapter; - - prefer bundled Perl Unicode data over duplicating ICU behavior or depending - on the host JDK Unicode version. -4. Reconcile the regex rows in `docs/reference/feature-matrix.md`, including - the expected-Joni features currently listed around lines 390–409. Add missing - Perl regex features with an explicit reducer, owner, and acceptance gate. -5. Retire `RegexPreprocessor` rules and imported-test patches as their native - implementations land. Rerun targeted `dev/import-perl5/sync.pl` imports and - require an idempotent second sync before declaring the upstream tests clean. -6. Complete the full release corpus, performance, documentation, packaging, - license, and cross-platform CI gates; then delete Java matching and the - temporary backend selector. - -### Parallel workstreams - -Workstreams must use isolated branches/worktrees and communicate through the -shared handoff files. Ownership is exclusive at the file/semantic-slice level. -Completed candidates stay on their validated base; the coordinator transplants -them onto the canonical stack and verifies `range-diff`. Independent fixes -targeting master start from current master. Engineers rebase themselves only -when the coordinator assigns an exact new base before implementation begins. - -1. Integration and PR 958 parity: combine validated runtime repairs, run focused - and full comparisons, and own CI/readiness of the current integration PR. -2. Native Joni fallback removal: implement parser/compiler/matcher semantics - that replace temporary Java routing, one independently testable feature at a - time. -3. Differential prioritization: maintain exact Perl 5.44 oracles, map remaining - failures to fallback triggers, and rank pure-Joni slices by recovered corpus - impact. -4. Unicode ownership: identify duplicated ICU/Joni/adapter behavior and propose - the smallest non-overlapping migration into forked Joni. -5. Documentation and feature inventory: keep the feature matrix and final - as-implemented documents aligned with validated behavior, without recording - implementation history in this plan. - -At most two full builds may run concurrently. Timing-sensitive final gates run -serialized. Engineers should continue source-independent analysis and focused -work while a full build runs rather than blocking on it. - -Implementation lanes run their required full `make` once the candidate is -stable. The review stack then runs one combined full build and one combined -focused reducer matrix; unchanged intermediate stacks are not rebuilt merely -for handoff. Handoff messages are event-driven: candidate ready, build started, -build finished, push complete, or blocker. Heartbeats exist only for crash -detection and do not replace implementation work. - -### Delivery checkpoints - -- Keep the current integration PR in draft until the PR 958 parity gate passes. -- Mark it ready only after the focused gate, fresh full comparison, warning-free - build, and required CI are green. -- Once a stable phase is handed to review, continue the next independent slice - on a new draft PR rather than accumulating unrelated risk in the review PR. -- The release manager owns integration-branch mutation; implementation lanes - provide pushed, validated commits and do not modify that branch directly. - -### Final acceptance checklist - -- [ ] Every semantic regex test that passes in the PR 958 baseline still passes. -- [ ] The complete `dev/tools/perl_test_runner.pl` output is compared - file-by-file with - `../PerlOnJava/logs/test_20260815_080000_958.log`. -- [ ] JVM and interpreter results agree for direct and thread regex tests. -- [ ] Forced-Joni runs cover ordinary constants, runtime patterns, embedded - closures, conditions, control verbs, recursion, and dynamic source. -- [ ] `pat_psycho*` and `speed*` complete with the bounded parallel policy. -- [ ] No supported regex test requires `JPERL_UNIMPLEMENTED=warn`. -- [ ] Joni is the sole production matcher; Java routing and selector code are - removed. -- [ ] Matcher-semantic preprocessing is gone; only documented Perl source-policy - scanning remains. -- [ ] Obsolete regex import patches are removed and targeted sync is idempotent. -- [ ] `docs/reference/feature-matrix.md` contains every known missing regex - feature and accurately reports the backend used. -- [ ] `dev/implementation/regex.md` and - `docs/design/joni-callout-fork.md` describe the shipped architecture - clearly and consistently. -- [ ] Redundant design documents are removed or reduced to concise rationale - summaries that point to canonical documentation. -- [ ] Original Joni/JCodings copyright and authorship notices are preserved. -- [ ] Performance remains within the gate defined above. -- [ ] `make` is warning-free; packaging, license checks, Ubuntu CI, and Windows - CI pass. - -### Open decisions and blockers - -- Temporary backend-policy tests that assert Java routing can conflict with the - required final Joni-only behavior. The repository forbids modifying existing - tests, so a routing-removal slice that makes such an assertion obsolete must - remain preserved and unmerged until the user explicitly approves a test - replacement/update policy; implementation must not disguise the new route to - keep a stale assertion green. -- Production diagnostics use Perl's exact trailing space after an - end-of-pattern `<-- HERE` marker. The obsolete no-trailing-space formatter - entry point remains only because its current unit test asserts that legacy - rendering; deleting it requires the same explicit existing-test update - policy. -- Optimizer/debug transcript assertions are reported separately from semantic - behavior and must never silently alter the raw baseline comparison. -- A runtime or shared-language defect that prevents a regex test from executing - is fixed in its owning subsystem; it is not approximated in the regex engine. -- Any proposed permanent adapter behavior must be classified as Perl source - policy. If it depends on backtracking or match state, it belongs in Joni. -- Resource-sensitive baselines and final timing tests require a quiet, - serialized build slot. - -## Related Documents and Skills - -- `docs/design/joni-callout-fork.md` -- `dev/implementation/regex.md` -- `dev/design/executable-regex-callbacks.md` -- `dev/design/regex-foreach-lexical-fix.md` -- `dev/design/regex-script-properties.md` -- `dev/design/regex-property-aliases.md` -- `.agents/skills/debug-perlonjava/SKILL.md` +gates may reopen it if a semantic regression appears. + +## Final Acceptance + +- [ ] Every semantic regex test passing in PR 958 still passes. +- [ ] Complete runner output is compared file-by-file with the PR 958 log. +- [ ] JVM/interpreter and direct/thread results agree. +- [ ] Forced Joni covers constants, closures, conditions, control verbs, + recursion, dynamic source, byte strings, and Unicode strings. +- [ ] `pat_psycho*` and `speed*` pass under bounded parallelism. +- [ ] No supported regex test needs `JPERL_UNIMPLEMENTED=warn`. +- [ ] Joni is the sole production matcher; Java routing/selector code is gone. +- [ ] Matcher-semantic preprocessing is gone. +- [ ] Obsolete import patches are removed and targeted sync is idempotent. +- [ ] Feature matrix and final architecture documents match implementation. +- [ ] Original copyright/authorship notices are preserved. +- [ ] Performance, warning-free `make`, packaging, license, Ubuntu, Windows, + and CI gates pass. From 09a2b5a29d59396e2b6d7c714c9da310ba4261c9 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 11:17:55 +0200 Subject: [PATCH 18/31] docs(regex): refresh the Phase 36 execution position Record the combined native-Joni gates and replace completed work with the current Unicode, case-fold, and extended-class implementation sequence. Generated with [OpenAI Codex](https://openai.com/codex) Co-Authored-By: OpenAI Codex --- dev/design/phase36-regex-parity.md | 56 ++++++++++++++++-------------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index ede690a0f5..39c743bf26 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -66,12 +66,18 @@ affected corpus before taking another slice. - Recursive callouts observe captures from the just-completed recursive frame. - Folded exact-search candidate bounds are safe for long ASCII-strict literals, and native numbered-call diagnostics retain the Perl source location. -- Forced-Joni `pat_advanced.t` executes 1,687/1,687 and passes 1,644 on JVM and - interpreter, with identical 43-row residuals and zero introductions against +- Forced-Joni `pat_advanced.t` executes 1,687/1,687 and passes 1,646 on JVM and + interpreter, with identical 41-row residuals and zero introductions against the exact 110-row reference set. -- Forced-Joni `reg_mesg.t` passes 1,694/2,595. -- Forced-Joni Unicode property comparison has 1,124/83,648 residual records on - each backend, with byte-identical normalized output. +- The current imported `reg_mesg.t` passes 1,710/2,603 on each backend with an + identical status/test-number vector. +- Forced-Joni Unicode property comparison has 900/83,648 residual records on + each backend, with an identical residual identity set. A corpus-proven + property-value-wildcard slice reduces this to 332 and awaits integration. +- The shared deterministic pinned-Perl Unicode generator covers all 11 current + families. General_Category compatibility aliases and native named-call/parser + safety are integrated; generated case-fold metadata, signed-IV user-property + ranges, and native extended classes are active independent slices. - Each `pat.t` variant executes 1,301/1,302 and passes 1,223. - The current 80-file forced-Joni gate passes 363,164/391,977 and has 19 per-file pass-count regressions against PR 958. These figures must be refreshed @@ -192,27 +198,25 @@ behavior. 1. Keep the canonical native-Joni PR and this plan branch durable. Require exact commit/file review, warning-free `make`, and green stacked CI before moving a PR from draft to user acceptance. -2. Finish the current independent slices and integrate only zero-introduction - commits: - - deterministic generated Java for all pinned Perl Unicode data families; - - the complete General_Category compatibility and wildcard family from the - 1,124-record Unicode residual; - - native named-call grammar, malformed-pattern diagnostics, and recursion - safety from the remaining parser tail; - - byte `/d` fold provenance only after named-character Unicode provenance is - correct; reject the broad ASCII-strict policy if it regresses named folds. -3. Consolidate Unicode generation before adding more alias families: use one - pinned development script to parse/analyze the Perl tables and emit compiled, - checked-in Java resolver/fold classes, generated oracle fixtures, and - checksums. Require deterministic, diff-free regeneration; keep runtime - provenance and matcher algorithms in reviewed hand-written code. -4. Refresh `pat_advanced.t` on the combined head and classify every remaining - semantic row. Prioritize `\N{}` extended-class failure, `\K`, property `/i` - closure, multi-character folds, recursion safety, wildcard/property errors, - then source/debug-only rows. -5. Refresh `reg_mesg.t`, `pat.t`, and the 83,648-record Unicode corpus on both - backends. Close the largest semantically uniform native-Joni groups with a - system-Perl-first reducer and zero-introduction complete gate for each. +2. Integrate the proven Unicode value-wildcard slice, then batch its follow-ups: + signed-IV user-property ranges, the exact 152-row POSIX/Perl compatibility + family, and the remaining 96 Block plus 84 binary/diagnostic rows. Replace + all temporary `java.util.regex.Pattern` value-wildcard execution—including + older Age/Block/Script/Numeric helpers—with one vendored-Joni evaluator + before Java matcher removal. +3. Complete generated Perl case-fold metadata through the shared deterministic + pipeline, then wire the reviewed Joni fold API for literals, classes, + properties, backreferences, and optimizer search. Keep `/d`, `/u`, `/a`, + `/aa`, locale, Turkic, and byte/Unicode provenance policy hand-written. +4. Implement native Joni `(?[...])` grammar/AST/evaluation and delete the + corresponding textual lowering. Close empty/multi-code-point `\N{}` legality, + nesting, set algebra, ranges, strict warnings, and exact diagnostics across + the affected `pat_advanced.t`, `pat.t`, and `reg_mesg.t` rows. +5. Refresh `reg_mesg.t`, `pat.t`, `pat_advanced.t`, and the 83,648-record Unicode + corpus on each combined batch. Close the largest semantically uniform + native-Joni groups with a system-Perl-first reducer and zero-introduction + complete gate for each; compare stable test identities when diagnostics + contain backend-specific source-location or binary rendering. 6. Refresh all four 80-file legs on one combined artifact. Resolve all 19 per-file PR 958 regressions; do not offer a long user acceptance run while any negative, zero-TAP, timeout, or incomplete file is unexplained. From 4b42e6570bd25265257d010a37f32838fe083d50 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 11:35:05 +0200 Subject: [PATCH 19/31] docs(regex): record integrated Unicode wildcards Advance the forward-only plan to the 332-row Unicode residual and make native Joni wildcard wiring the next cleanup step. Generated with [OpenAI Codex](https://openai.com/codex) Co-Authored-By: OpenAI Codex --- dev/design/phase36-regex-parity.md | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index 39c743bf26..f33497774d 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -71,13 +71,14 @@ affected corpus before taking another slice. the exact 110-row reference set. - The current imported `reg_mesg.t` passes 1,710/2,603 on each backend with an identical status/test-number vector. -- Forced-Joni Unicode property comparison has 900/83,648 residual records on - each backend, with an identical residual identity set. A corpus-proven - property-value-wildcard slice reduces this to 332 and awaits integration. +- Forced-Joni Unicode property comparison has 332/83,648 residual records on + each backend, with a byte-identical residual set. The integrated wildcard + slice removed 568 rows without introductions. - The shared deterministic pinned-Perl Unicode generator covers all 11 current - families. General_Category compatibility aliases and native named-call/parser - safety are integrated; generated case-fold metadata, signed-IV user-property - ranges, and native extended classes are active independent slices. + families. General_Category compatibility aliases, native named-call/parser + safety, and a runtime-neutral Joni property-value matcher are integrated; + generated case-fold metadata, signed-IV user-property ranges, native extended + classes, and removal of Java wildcard execution are active independent slices. - Each `pat.t` variant executes 1,301/1,302 and passes 1,223. - The current 80-file forced-Joni gate passes 363,164/391,977 and has 19 per-file pass-count regressions against PR 958. These figures must be refreshed @@ -198,12 +199,12 @@ behavior. 1. Keep the canonical native-Joni PR and this plan branch durable. Require exact commit/file review, warning-free `make`, and green stacked CI before moving a PR from draft to user acceptance. -2. Integrate the proven Unicode value-wildcard slice, then batch its follow-ups: - signed-IV user-property ranges, the exact 152-row POSIX/Perl compatibility - family, and the remaining 96 Block plus 84 binary/diagnostic rows. Replace - all temporary `java.util.regex.Pattern` value-wildcard execution—including - older Age/Block/Script/Numeric helpers—with one vendored-Joni evaluator - before Java matcher removal. +2. Wire every property-value wildcard family to the integrated vendored-Joni + evaluator and remove all temporary `java.util.regex.Pattern` wildcard + execution, including older Age/Block/Script/Numeric helpers. In parallel, + finish signed-IV user-property ranges, the exact 152-row POSIX/Perl + compatibility family, and the remaining 96 Block plus 84 binary/diagnostic + rows. 3. Complete generated Perl case-fold metadata through the shared deterministic pipeline, then wire the reviewed Joni fold API for literals, classes, properties, backreferences, and optimizer search. Keep `/d`, `/u`, `/a`, From db65be86790b3834f9e11011b3a1120c027871ee Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 12:25:23 +0200 Subject: [PATCH 20/31] docs(regex): refresh Phase 36 execution plan Keep the plan forward-only while recording the current validated integration head, Unicode residual, active native case-fold slices, and exact next gates. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- dev/design/phase36-regex-parity.md | 60 +++++++++++++++++------------- 1 file changed, 35 insertions(+), 25 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index f33497774d..f6840c22ab 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -66,19 +66,28 @@ affected corpus before taking another slice. - Recursive callouts observe captures from the just-completed recursive frame. - Folded exact-search candidate bounds are safe for long ASCII-strict literals, and native numbered-call diagnostics retain the Perl source location. -- Forced-Joni `pat_advanced.t` executes 1,687/1,687 and passes 1,646 on JVM and - interpreter, with identical 41-row residuals and zero introductions against - the exact 110-row reference set. +- The last complete forced-Joni `pat_advanced.t` gate executes 1,687/1,687 and + passes 1,646 on JVM and interpreter, with identical 41-row residuals. The + integrated signed-IV range fix removes row 1651 with zero introductions in + exact A/B evidence; a fresh combined serial gate remains required because a + later run stopped before the complete plan under concurrent CPAN load. - The current imported `reg_mesg.t` passes 1,710/2,603 on each backend with an identical status/test-number vector. -- Forced-Joni Unicode property comparison has 332/83,648 residual records on - each backend, with a byte-identical residual set. The integrated wildcard - slice removed 568 rows without introductions. -- The shared deterministic pinned-Perl Unicode generator covers all 11 current - families. General_Category compatibility aliases, native named-call/parser - safety, and a runtime-neutral Joni property-value matcher are integrated; - generated case-fold metadata, signed-IV user-property ranges, native extended - classes, and removal of Java wildcard execution are active independent slices. +- Forced-Joni Unicode property comparison has 180/83,648 residual records on + each backend after the wildcard and POSIX/Perl compatibility slices. The + wildcard slice removed 568 rows and the POSIX slice removed the exact assigned + 152 rows, each with zero introductions; the remaining 96 Block and 84 binary/ + diagnostic rows are active consecutive slices. +- The shared deterministic pinned-Perl Unicode generator covers all current + property families plus compact Perl default simple/full/reverse case-fold + metadata. General_Category compatibility aliases, native named-call/parser + safety, the runtime-neutral Joni property-value matcher, signed-IV user- + property ranges, POSIX compatibility, and generated fold data are integrated. + Native extended classes, analyser fold safety, the final property aliases, + and removal of Java wildcard execution are active independent slices. +- Draft PR 1078 is durable at `e50c667d7`; that exact head passes warning-free + `make`, all 17 tasks, Joni tests, five unit shards, packaging, and generated- + data checks. - Each `pat.t` variant executes 1,301/1,302 and passes 1,223. - The current 80-file forced-Joni gate passes 363,164/391,977 and has 19 per-file pass-count regressions against PR 958. These figures must be refreshed @@ -199,20 +208,21 @@ behavior. 1. Keep the canonical native-Joni PR and this plan branch durable. Require exact commit/file review, warning-free `make`, and green stacked CI before moving a PR from draft to user acceptance. -2. Wire every property-value wildcard family to the integrated vendored-Joni - evaluator and remove all temporary `java.util.regex.Pattern` wildcard - execution, including older Age/Block/Script/Numeric helpers. In parallel, - finish signed-IV user-property ranges, the exact 152-row POSIX/Perl - compatibility family, and the remaining 96 Block plus 84 binary/diagnostic - rows. -3. Complete generated Perl case-fold metadata through the shared deterministic - pipeline, then wire the reviewed Joni fold API for literals, classes, - properties, backreferences, and optimizer search. Keep `/d`, `/u`, `/a`, - `/aa`, locale, Turkic, and byte/Unicode provenance policy hand-written. -4. Implement native Joni `(?[...])` grammar/AST/evaluation and delete the - corresponding textual lowering. Close empty/multi-code-point `\N{}` legality, - nesting, set algebra, ranges, strict warnings, and exact diagnostics across - the affected `pat_advanced.t`, `pat.t`, and `reg_mesg.t` rows. +2. Finish the exact remaining 96 Block and 84 binary/diagnostic Unicode rows. + Then wire every property-value wildcard family to the integrated vendored- + Joni evaluator in one conflict-free commit and remove all temporary + `java.util.regex.Pattern` wildcard execution, including Age/Block/Script/ + Numeric helpers. +3. Integrate the generated fold table through bounded native slices: package- + local adapter and analyser optimizer safety; property/class closure; explicit + fold/provenance context; literal forward/reverse expansion; backreferences; + and final optimizer proof. Keep `/d`, `/u`, `/a`, `/aa`, locale, Turkic, and + byte/Unicode provenance policy explicit and hand-reviewed. +4. Finish native Joni `(?[...])` grammar/AST/evaluation and delete the textual + lowering. Require operand-local `/i`, scoped modifier isolation, wide-domain + algebra, literal/comment scanning, empty/multi-code-point `\N{}` legality, + nesting, and exact diagnostics. Then replace the `(?(DEFINE)...)` adapter + rewrite with a native non-executing definition container. 5. Refresh `reg_mesg.t`, `pat.t`, `pat_advanced.t`, and the 83,648-record Unicode corpus on each combined batch. Close the largest semantically uniform native-Joni groups with a system-Perl-first reducer and zero-introduction From 21b618ff247ea3dfb9654e7c0c5b392b48b565ec Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 12:44:52 +0200 Subject: [PATCH 21/31] docs: refresh Phase 36 active migration slices Keep the forward plan aligned with the integrated Block and analyser work, the binary residual, native extended-class gates, wildcard migration, and generated named-sequence follow-up. Generated with [Codex](https://openai.com/codex/) Co-Authored-By: Codex --- dev/design/phase36-regex-parity.md | 35 ++++++++++++++++++------------ 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index f6840c22ab..c9f98ac861 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -73,18 +73,21 @@ affected corpus before taking another slice. later run stopped before the complete plan under concurrent CPAN load. - The current imported `reg_mesg.t` passes 1,710/2,603 on each backend with an identical status/test-number vector. -- Forced-Joni Unicode property comparison has 180/83,648 residual records on - each backend after the wildcard and POSIX/Perl compatibility slices. The - wildcard slice removed 568 rows and the POSIX slice removed the exact assigned - 152 rows, each with zero introductions; the remaining 96 Block and 84 binary/ - diagnostic rows are active consecutive slices. +- Forced-Joni Unicode property comparison has 84/83,648 residual records on + each backend after the wildcard, POSIX/Perl compatibility, and Block slices. + The Block slice removed its exact assigned 96 rows with zero introductions. + The binary-alias candidate removes another exact 52 rows on the complete JVM + corpus, leaving only 32 Hyphen diagnostic rows; matching interpreter evidence + and integration remain required. - The shared deterministic pinned-Perl Unicode generator covers all current property families plus compact Perl default simple/full/reverse case-fold metadata. General_Category compatibility aliases, native named-call/parser safety, the runtime-neutral Joni property-value matcher, signed-IV user- property ranges, POSIX compatibility, and generated fold data are integrated. - Native extended classes, analyser fold safety, the final property aliases, - and removal of Java wildcard execution are active independent slices. + The analyser fold-safety slice is integrated in local staging. Native extended + classes, the final binary aliases, generated named-sequence lookup, property/ + class fold closure, and removal of Java wildcard execution are active + independent slices. - Draft PR 1078 is durable at `e50c667d7`; that exact head passes warning-free `make`, all 17 tasks, Joni tests, five unit shards, packaging, and generated- data checks. @@ -208,21 +211,25 @@ behavior. 1. Keep the canonical native-Joni PR and this plan branch durable. Require exact commit/file review, warning-free `make`, and green stacked CI before moving a PR from draft to user acceptance. -2. Finish the exact remaining 96 Block and 84 binary/diagnostic Unicode rows. - Then wire every property-value wildcard family to the integrated vendored- +2. Finish the exact remaining binary aliases and 32 Hyphen diagnostic Unicode + rows. Then wire every property-value wildcard family to the integrated vendored- Joni evaluator in one conflict-free commit and remove all temporary `java.util.regex.Pattern` wildcard execution, including Age/Block/Script/ - Numeric helpers. + Numeric helpers. Generate the complete named-sequence lookup from Perl's + pinned `NamedSequences.txt` in parallel and route standard `\N{name}` through + it without reimplementing the table by hand. 3. Integrate the generated fold table through bounded native slices: package- local adapter and analyser optimizer safety; property/class closure; explicit fold/provenance context; literal forward/reverse expansion; backreferences; and final optimizer proof. Keep `/d`, `/u`, `/a`, `/aa`, locale, Turkic, and byte/Unicode provenance policy explicit and hand-reviewed. 4. Finish native Joni `(?[...])` grammar/AST/evaluation and delete the textual - lowering. Require operand-local `/i`, scoped modifier isolation, wide-domain - algebra, literal/comment scanning, empty/multi-code-point `\N{}` legality, - nesting, and exact diagnostics. Then replace the `(?(DEFINE)...)` adapter - rewrite with a native non-executing definition container. + lowering. Require operand-local `/i`, scoped `^`/`a`/`aa`/`d`/`u` modifier + isolation, wide-domain algebra, literal/comment scanning, exact-three-digit + octal handling, nested-POSIX boundaries, empty/multi-code-point `\N{}` + legality, nesting, and exact diagnostics with zero `reg_mesg.t` + introductions. Then replace the `(?(DEFINE)...)` adapter rewrite with a + native non-executing definition container. 5. Refresh `reg_mesg.t`, `pat.t`, `pat_advanced.t`, and the 83,648-record Unicode corpus on each combined batch. Close the largest semantically uniform native-Joni groups with a system-Perl-first reducer and zero-introduction From f3a97d45d2a9ec226fd2f45e6b21ff17017c9c28 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 12:56:07 +0200 Subject: [PATCH 22/31] docs: expose active Phase 36 milestone progress Add a forward-looking subphase tracker so completed Unicode and case-fold milestones are visible while the broader phases remain correctly open. Generated with [Codex](https://openai.com/codex/) Co-Authored-By: Codex --- dev/design/phase36-regex-parity.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index c9f98ac861..33860e4e35 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -314,6 +314,25 @@ failure blocks backend removal, not semantic fixes. A checked phase means its focused semantic implementation is complete. Release gates may reopen it if a semantic regression appears. +### Active phase detail + +- [x] Pinned Perl Unicode property-data generators and freshness gates +- [x] General Category, Script, Block, POSIX, binary-membership, and signed-wide + property ranges +- [x] Runtime-neutral Joni property-value matcher +- [ ] Replace every Java property-wildcard execution site with the Joni matcher +- [ ] Complete Hyphen warning/category/source-position diagnostics +- [x] Pinned Perl simple/full/reverse case-fold data +- [x] Native fold adapter and unsafe optimizer-boundary suppression +- [ ] Property/class fold closure +- [ ] Fold-mode and byte/Unicode provenance context +- [ ] Forward/reverse literal expansion and backreference folding +- [ ] Generated Perl named-sequence lookup and native `\N{name}` completion +- [ ] Native `(?[...])` with zero diagnostic regressions +- [ ] Native `(?(DEFINE)...)` and removal of its adapter rewrite +- [ ] Refresh the complete Unicode, `pat.t`, `pat_advanced.t`, `reg_mesg.t`, and + 80-file forced-Joni gates on one integrated artifact + ## Final Acceptance - [ ] Every semantic regex test passing in PR 958 still passes. From 428cbbeea9a09cef9c90d2e03deb8cb69886f901 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 13:32:02 +0200 Subject: [PATCH 23/31] docs(regex): refresh Phase 36 execution tracker Record completed Joni wildcard execution and named-sequence resolution while keeping native property parsing and diagnostic cleanup as explicit next steps. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- dev/design/phase36-regex-parity.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index 33860e4e35..56e9b684e8 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -320,14 +320,18 @@ gates may reopen it if a semantic regression appears. - [x] General Category, Script, Block, POSIX, binary-membership, and signed-wide property ranges - [x] Runtime-neutral Joni property-value matcher -- [ ] Replace every Java property-wildcard execution site with the Joni matcher +- [x] Replace every Java property-wildcard execution site with the Joni matcher +- [ ] Parse nested property-value regex syntax in Joni and remove adapter + materialization of the selected ranges - [ ] Complete Hyphen warning/category/source-position diagnostics - [x] Pinned Perl simple/full/reverse case-fold data - [x] Native fold adapter and unsafe optimizer-boundary suppression - [ ] Property/class fold closure - [ ] Fold-mode and byte/Unicode provenance context - [ ] Forward/reverse literal expansion and backreference folding -- [ ] Generated Perl named-sequence lookup and native `\N{name}` completion +- [x] Generated Perl named-sequence lookup and native sequence resolution +- [ ] Remove temporary named-sequence encoding from native Joni pattern source +- [ ] Restore Perl diagnostics for unknown and encoded named sequences - [ ] Native `(?[...])` with zero diagnostic regressions - [ ] Native `(?(DEFINE)...)` and removal of its adapter rewrite - [ ] Refresh the complete Unicode, `pat.t`, `pat_advanced.t`, `reg_mesg.t`, and From f536dcd29611b9f58104e700c180a48f40ef7cba Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 14:10:47 +0200 Subject: [PATCH 24/31] docs(regex): record complete Unicode property corpus Mark the Hyphen diagnostic slice complete after both forced-Joni backends reached 83,648/83,648 with zero introduced identities. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- dev/design/phase36-regex-parity.md | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index 56e9b684e8..9ac1a8db3e 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -73,21 +73,18 @@ affected corpus before taking another slice. later run stopped before the complete plan under concurrent CPAN load. - The current imported `reg_mesg.t` passes 1,710/2,603 on each backend with an identical status/test-number vector. -- Forced-Joni Unicode property comparison has 84/83,648 residual records on - each backend after the wildcard, POSIX/Perl compatibility, and Block slices. - The Block slice removed its exact assigned 96 rows with zero introductions. - The binary-alias candidate removes another exact 52 rows on the complete JVM - corpus, leaving only 32 Hyphen diagnostic rows; matching interpreter evidence - and integration remain required. +- Forced-Joni Unicode property comparison is 83,648/83,648 on both backends. + The Hyphen/IsHyphen warning correction removes the final exact 32 residuals + with zero introductions while preserving dynamic property interpolation. - The shared deterministic pinned-Perl Unicode generator covers all current property families plus compact Perl default simple/full/reverse case-fold metadata. General_Category compatibility aliases, native named-call/parser safety, the runtime-neutral Joni property-value matcher, signed-IV user- property ranges, POSIX compatibility, and generated fold data are integrated. - The analyser fold-safety slice is integrated in local staging. Native extended - classes, the final binary aliases, generated named-sequence lookup, property/ - class fold closure, and removal of Java wildcard execution are active - independent slices. + The analyser fold-safety slice, final binary aliases, generated named-sequence + lookup, property-wildcard execution, and Hyphen diagnostics are integrated in + local staging. Native extended classes, property/class fold closure, and + nested property-wildcard lexing are active independent slices. - Draft PR 1078 is durable at `e50c667d7`; that exact head passes warning-free `make`, all 17 tasks, Joni tests, five unit shards, packaging, and generated- data checks. @@ -323,7 +320,7 @@ gates may reopen it if a semantic regression appears. - [x] Replace every Java property-wildcard execution site with the Joni matcher - [ ] Parse nested property-value regex syntax in Joni and remove adapter materialization of the selected ranges -- [ ] Complete Hyphen warning/category/source-position diagnostics +- [x] Complete Hyphen warning/category/source-position diagnostics - [x] Pinned Perl simple/full/reverse case-fold data - [x] Native fold adapter and unsafe optimizer-boundary suppression - [ ] Property/class fold closure From 257d20b151eb891a44a9f76ea3ee836d730ae077 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 14:44:26 +0200 Subject: [PATCH 25/31] docs(regex): mark native syntax milestones complete Record native extended classes, nested property-wildcard lexing, and canonical named-sequence diagnostics after their zero-introduction gates. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- dev/design/phase36-regex-parity.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index 9ac1a8db3e..a54ab54ae0 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -85,9 +85,10 @@ affected corpus before taking another slice. lookup, property-wildcard execution, and Hyphen diagnostics are integrated in local staging. Native extended classes, property/class fold closure, and nested property-wildcard lexing are active independent slices. -- Draft PR 1078 is durable at `e50c667d7`; that exact head passes warning-free - `make`, all 17 tasks, Joni tests, five unit shards, packaging, and generated- - data checks. +- Draft PR 1078 is durable at `29a6de3fd`. Its integration source through + `29e4a1b7e` passes warning-free `make`, all 17 tasks, Joni tests, five unit + shards, packaging, and generated-data checks; the final isolated named- + diagnostic commit passes its focused all-17-task make. - Each `pat.t` variant executes 1,301/1,302 and passes 1,223. - The current 80-file forced-Joni gate passes 363,164/391,977 and has 19 per-file pass-count regressions against PR 958. These figures must be refreshed @@ -318,7 +319,7 @@ gates may reopen it if a semantic regression appears. property ranges - [x] Runtime-neutral Joni property-value matcher - [x] Replace every Java property-wildcard execution site with the Joni matcher -- [ ] Parse nested property-value regex syntax in Joni and remove adapter +- [x] Parse nested property-value regex syntax in Joni and remove adapter materialization of the selected ranges - [x] Complete Hyphen warning/category/source-position diagnostics - [x] Pinned Perl simple/full/reverse case-fold data @@ -328,8 +329,9 @@ gates may reopen it if a semantic regression appears. - [ ] Forward/reverse literal expansion and backreference folding - [x] Generated Perl named-sequence lookup and native sequence resolution - [ ] Remove temporary named-sequence encoding from native Joni pattern source -- [ ] Restore Perl diagnostics for unknown and encoded named sequences -- [ ] Native `(?[...])` with zero diagnostic regressions +- [x] Restore canonical multi-code-point named-sequence extended-class diagnostics +- [ ] Restore remaining Perl diagnostics for unknown named sequences +- [x] Native `(?[...])` with zero diagnostic regressions - [ ] Native `(?(DEFINE)...)` and removal of its adapter rewrite - [ ] Refresh the complete Unicode, `pat.t`, `pat_advanced.t`, `reg_mesg.t`, and 80-file forced-Joni gates on one integrated artifact From f235da05b949b99b6e7c5f6a29ccd6a9f0d8a2a7 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 15:35:42 +0200 Subject: [PATCH 26/31] docs: refresh Phase 36 forward execution plan Compress completed implementation state into the current validated position and keep only the remaining ordered migration work in next steps. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- dev/design/phase36-regex-parity.md | 86 +++++++++++++++--------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index a54ab54ae0..197e62c2f2 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -82,13 +82,23 @@ affected corpus before taking another slice. safety, the runtime-neutral Joni property-value matcher, signed-IV user- property ranges, POSIX compatibility, and generated fold data are integrated. The analyser fold-safety slice, final binary aliases, generated named-sequence - lookup, property-wildcard execution, and Hyphen diagnostics are integrated in - local staging. Native extended classes, property/class fold closure, and - nested property-wildcard lexing are active independent slices. -- Draft PR 1078 is durable at `29a6de3fd`. Its integration source through - `29e4a1b7e` passes warning-free `make`, all 17 tasks, Joni tests, five unit - shards, packaging, and generated-data checks; the final isolated named- - diagnostic commit passes its focused all-17-task make. + lookup, property-wildcard execution, Hyphen diagnostics, native extended + classes, property/class fold closure, and nested property-wildcard lexing are + integrated in local staging. +- Raw `\N{name}` source now survives frontend and matcher compilation without + the temporary `=POJSEQ=` transport. Generated and lexical multi-code-point + names resolve through Joni; ordinary scalar names retain one explicit routing + gate only until the native `/aa` Kelvin fold correction lands. +- Draft PR 1078 is durable remotely through `29a6de3fd`; current local staging + is `00d461a71`. That exact head passes warning-free `make`, all 17 tasks, + Joni tests, five unit shards, packaging, and generated-data checks. The new + missing-brace diagnostic fixture passes system Perl, JVM, and interpreter + 8/8. The named-scalar Kelvin fixture passes system Perl 9/9 but ordinary JVM + and interpreter 8/9 until the explicit temporary ordinary-name routing gate + is deleted; PR publication waits for that already-owned correction. +- Forced-Joni `pat_re_eval.t` now executes and passes 555/555 on both JVM and + interpreter. Perl's release-build `-D` diagnostic is preserved without + enabling PerlOnJava's unrelated internal compiler trace. - Each `pat.t` variant executes 1,301/1,302 and passes 1,223. - The current 80-file forced-Joni gate passes 363,164/391,977 and has 19 per-file pass-count regressions against PR 958. These figures must be refreshed @@ -209,45 +219,35 @@ behavior. 1. Keep the canonical native-Joni PR and this plan branch durable. Require exact commit/file review, warning-free `make`, and green stacked CI before moving a PR from draft to user acceptance. -2. Finish the exact remaining binary aliases and 32 Hyphen diagnostic Unicode - rows. Then wire every property-value wildcard family to the integrated vendored- - Joni evaluator in one conflict-free commit and remove all temporary - `java.util.regex.Pattern` wildcard execution, including Age/Block/Script/ - Numeric helpers. Generate the complete named-sequence lookup from Perl's - pinned `NamedSequences.txt` in parallel and route standard `\N{name}` through - it without reimplementing the table by hand. -3. Integrate the generated fold table through bounded native slices: package- - local adapter and analyser optimizer safety; property/class closure; explicit - fold/provenance context; literal forward/reverse expansion; backreferences; - and final optimizer proof. Keep `/d`, `/u`, `/a`, `/aa`, locale, Turkic, and - byte/Unicode provenance policy explicit and hand-reviewed. -4. Finish native Joni `(?[...])` grammar/AST/evaluation and delete the textual - lowering. Require operand-local `/i`, scoped `^`/`a`/`aa`/`d`/`u` modifier - isolation, wide-domain algebra, literal/comment scanning, exact-three-digit - octal handling, nested-POSIX boundaries, empty/multi-code-point `\N{}` - legality, nesting, and exact diagnostics with zero `reg_mesg.t` - introductions. Then replace the `(?(DEFINE)...)` adapter rewrite with a - native non-executing definition container. -5. Refresh `reg_mesg.t`, `pat.t`, `pat_advanced.t`, and the 83,648-record Unicode - corpus on each combined batch. Close the largest semantically uniform - native-Joni groups with a system-Perl-first reducer and zero-introduction - complete gate for each; compare stable test identities when diagnostics - contain backend-specific source-location or binary rendering. -6. Refresh all four 80-file legs on one combined artifact. Resolve all 19 +2. Finish fold provenance in isolated native slices: `/d`/`u`/`a`/`aa`, locale, + Turkic, byte/Unicode source identity, forward/reverse literal expansion, and + backreferences. Correct named-scalar Kelvin `/aa`, then delete the temporary + generated-sequence routing distinction so every ordinary `\N{name}` pattern + uses Joni. +3. Land native `(?(DEFINE)...)` with exact enclosing-capture publication and + delete its adapter. Reuse the validated variable-lookbehind candidate, finish + Perl's character/byte 255/256-width and diagnostic rules, then route ordinary + lookbehind through Joni and delete its Java translation. +4. Restore the remaining unknown/empty/malformed named-character diagnostics. + Refresh `reg_mesg.t`, `pat.t`, `pat_advanced.t`, the 83,648-record Unicode + corpus, and stable fold files after each combined native batch with zero + introduced identities. +5. Refresh all four 80-file legs on one combined artifact. Resolve all 19 per-file PR 958 regressions; do not offer a long user acceptance run while any negative, zero-TAP, timeout, or incomplete file is unexplained. -7. Use the integration report to retire ordinary Java fallbacks in impact order: - lookbehind, branch reset, alphabetic assertions, then remaining constant - patterns. Delete each route and its semantic preprocessor rule in the same - validated slice. -8. Complete runtime source/eval semantics and diagnostics, then close - `pat_re_eval.t`. -9. Remove obsolete regex import patches. Run +6. Use the integration report to retire remaining Java fallbacks in impact order: + lookbehind, branch reset, then ordinary constants. Alphabetic assertions are + already native and their focused 19-case oracle is green on system Perl and + both forced-Joni execution backends. Delete each route and its semantic + preprocessor rule in the same validated slice. +7. Keep `pat_re_eval.t` at 555/555 on both backends while completing remaining + runtime source, warning/source-position, and eval diagnostic matrices. +8. Remove obsolete regex import patches. Run `perl dev/import-perl5/sync.pl --only perl5/t` twice; verify the configured upstream `re/pat.t` hash and require the second sync to be content-idempotent. -10. Remove Java matching and the selector, rerun the complete semantic and CPAN +9. Remove Java matching and the selector, rerun the complete semantic and CPAN matrix, then execute performance and release gates. -11. Finish feature-matrix and as-implemented documentation, consolidate +10. Finish feature-matrix and as-implemented documentation, consolidate redundant plans, rebase the final stack onto current master, and require green Ubuntu/Windows CI before merge. @@ -324,11 +324,11 @@ gates may reopen it if a semantic regression appears. - [x] Complete Hyphen warning/category/source-position diagnostics - [x] Pinned Perl simple/full/reverse case-fold data - [x] Native fold adapter and unsafe optimizer-boundary suppression -- [ ] Property/class fold closure +- [x] Property/class fold closure - [ ] Fold-mode and byte/Unicode provenance context - [ ] Forward/reverse literal expansion and backreference folding - [x] Generated Perl named-sequence lookup and native sequence resolution -- [ ] Remove temporary named-sequence encoding from native Joni pattern source +- [x] Remove temporary named-sequence encoding from native Joni pattern source - [x] Restore canonical multi-code-point named-sequence extended-class diagnostics - [ ] Restore remaining Perl diagnostics for unknown named sequences - [x] Native `(?[...])` with zero diagnostic regressions From bb36ceebbbc17f4f606effcd056f6fd6563802fb Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 15:49:05 +0200 Subject: [PATCH 27/31] docs: record ready Phase 36 consolidation Record the green ordinary named-character Joni route, PR 1078 review boundary, and the isolated successor integration branch. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- dev/design/phase36-regex-parity.md | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index 197e62c2f2..3afda5cdc2 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -87,15 +87,15 @@ affected corpus before taking another slice. integrated in local staging. - Raw `\N{name}` source now survives frontend and matcher compilation without the temporary `=POJSEQ=` transport. Generated and lexical multi-code-point - names resolve through Joni; ordinary scalar names retain one explicit routing - gate only until the native `/aa` Kelvin fold correction lands. -- Draft PR 1078 is durable remotely through `29a6de3fd`; current local staging - is `00d461a71`. That exact head passes warning-free `make`, all 17 tasks, - Joni tests, five unit shards, packaging, and generated-data checks. The new - missing-brace diagnostic fixture passes system Perl, JVM, and interpreter - 8/8. The named-scalar Kelvin fixture passes system Perl 9/9 but ordinary JVM - and interpreter 8/9 until the explicit temporary ordinary-name routing gate - is deleted; PR publication waits for that already-owned correction. + names and ordinary scalar names resolve through Joni; the temporary + generated-sequence-only routing distinction is deleted. +- PR 1078 is ready for review against `master` at `d5d733982`. That exact head + passes warning-free `make`, all 17 tasks, Joni tests, five unit shards, + packaging, and generated-data checks. The named-scalar Kelvin fixture passes + system Perl and ordinary JVM/interpreter 9/9; named transport passes 11/11, + cache 9/9, and missing-brace diagnostics 8/8 on both execution backends. + Ubuntu and Windows CI are in progress. New worker deliveries target the + isolated `integration/phase36-native-syntax-v8` successor branch. - Forced-Joni `pat_re_eval.t` now executes and passes 555/555 on both JVM and interpreter. Perl's release-build `-D` diagnostic is preserved without enabling PerlOnJava's unrelated internal compiler trace. @@ -221,9 +221,8 @@ behavior. PR from draft to user acceptance. 2. Finish fold provenance in isolated native slices: `/d`/`u`/`a`/`aa`, locale, Turkic, byte/Unicode source identity, forward/reverse literal expansion, and - backreferences. Correct named-scalar Kelvin `/aa`, then delete the temporary - generated-sequence routing distinction so every ordinary `\N{name}` pattern - uses Joni. + backreferences. Preserve the now-green ordinary named-character Joni route + across byte/upgraded, scoped, qr/interpolation, and substitution paths. 3. Land native `(?(DEFINE)...)` with exact enclosing-capture publication and delete its adapter. Reuse the validated variable-lookbehind candidate, finish Perl's character/byte 255/256-width and diagnostic rules, then route ordinary From bb698ea8d9fc3c80c383210d07f322a160356077 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 16:39:58 +0200 Subject: [PATCH 28/31] docs(regex): align Phase 36 plan with native integration Replace completed DEFINE, lookbehind, branch-reset, named-diagnostic, and plain non-newline work with the current PR gates and the remaining provenance, dynamic-pattern, differential, adapter-retirement, and release sequence. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- dev/design/phase36-regex-parity.md | 96 +++++++++++++++++------------- 1 file changed, 54 insertions(+), 42 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index 3afda5cdc2..a15276ee1d 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -89,13 +89,23 @@ affected corpus before taking another slice. the temporary `=POJSEQ=` transport. Generated and lexical multi-code-point names and ordinary scalar names resolve through Joni; the temporary generated-sequence-only routing distinction is deleted. -- PR 1078 is ready for review against `master` at `d5d733982`. That exact head - passes warning-free `make`, all 17 tasks, Joni tests, five unit shards, - packaging, and generated-data checks. The named-scalar Kelvin fixture passes - system Perl and ordinary JVM/interpreter 9/9; named transport passes 11/11, - cache 9/9, and missing-brace diagnostics 8/8 on both execution backends. - Ubuntu and Windows CI are in progress. New worker deliveries target the - isolated `integration/phase36-native-syntax-v8` successor branch. +- PR 1078 targets `master` at `daca5545a`. Its warning-free local `make` passes + all 17 tasks, including Joni tests, five unit shards, packaging, and generated + data. Named-scalar Kelvin passes 9/9; named transport 11/11, cache 9/9, and + missing-brace diagnostics 8/8 on JVM/interpreter. Ubuntu and Windows CI are + running on the exact head; merge remains gated on both checks. +- Draft PR 1079 is the successor integration batch. Its current local head + contains native `(?(DEFINE)...)`, ordinary lookbehind, branch reset, plain + `\N`, and the PR 1078 platform correction on top of the validated malformed/ + unknown named-character diagnostics. The combined warning-free build is the + next durability gate before push. +- Native DEFINE, ordinary lookbehind, and branch reset now route through Joni; + their feature-specific Java rewrites and branch-reset capture-map adapter are + deleted. Plain Perl `\N` is a native Joni non-line-feed atom, including + intervals, `/s` independence, and Perl's character-class diagnostic. +- Unknown, empty, malformed U+, and missing-brace named-character groups have + native frontend/runtime diagnostics. The remaining dotted-U+ unmatched-class + identity is an active bounded runtime diagnostic correction. - Forced-Joni `pat_re_eval.t` now executes and passes 555/555 on both JVM and interpreter. Perl's release-build `-D` diagnostic is preserved without enabling PerlOnJava's unrelated internal compiler trace. @@ -216,39 +226,36 @@ behavior. ## Ordered Next Steps -1. Keep the canonical native-Joni PR and this plan branch durable. Require exact - commit/file review, warning-free `make`, and green stacked CI before moving a - PR from draft to user acceptance. -2. Finish fold provenance in isolated native slices: `/d`/`u`/`a`/`aa`, locale, - Turkic, byte/Unicode source identity, forward/reverse literal expansion, and - backreferences. Preserve the now-green ordinary named-character Joni route - across byte/upgraded, scoped, qr/interpolation, and substitution paths. -3. Land native `(?(DEFINE)...)` with exact enclosing-capture publication and - delete its adapter. Reuse the validated variable-lookbehind candidate, finish - Perl's character/byte 255/256-width and diagnostic rules, then route ordinary - lookbehind through Joni and delete its Java translation. -4. Restore the remaining unknown/empty/malformed named-character diagnostics. - Refresh `reg_mesg.t`, `pat.t`, `pat_advanced.t`, the 83,648-record Unicode - corpus, and stable fold files after each combined native batch with zero - introduced identities. -5. Refresh all four 80-file legs on one combined artifact. Resolve all 19 - per-file PR 958 regressions; do not offer a long user acceptance run while - any negative, zero-TAP, timeout, or incomplete file is unexplained. -6. Use the integration report to retire remaining Java fallbacks in impact order: - lookbehind, branch reset, then ordinary constants. Alphabetic assertions are - already native and their focused 19-case oracle is green on system Perl and - both forced-Joni execution backends. Delete each route and its semantic - preprocessor rule in the same validated slice. -7. Keep `pat_re_eval.t` at 555/555 on both backends while completing remaining - runtime source, warning/source-position, and eval diagnostic matrices. -8. Remove obsolete regex import patches. Run - `perl dev/import-perl5/sync.pl --only perl5/t` twice; verify the configured - upstream `re/pat.t` hash and require the second sync to be content-idempotent. -9. Remove Java matching and the selector, rerun the complete semantic and CPAN - matrix, then execute performance and release gates. -10. Finish feature-matrix and as-implemented documentation, consolidate - redundant plans, rebase the final stack onto current master, and require - green Ubuntu/Windows CI before merge. +1. Merge PR 1078 only after exact-head Ubuntu and Windows CI pass. Rebase PR + 1079 onto the resulting `master`, retain its independent semantic commits, + run one combined warning-free `make`, push, and obtain stacked CI. +2. Complete byte/Unicode pattern provenance through runtime interpolation and + template composition, then finish `/d`/`u`/`a`/`aa` forward/reverse literal + and backreference folding from generated data. Require direct Joni plus + ordinary/forced JVM/interpreter zero-introduction gates. +3. Implement recursive and runtime `(??{...})` as native nested Joni execution: + preserve captures, `$^R`, `pos`, modes, byte/Unicode provenance, callback + unwind, backtracking re-evaluation, and recursion safety. Route every embedded + closure to Joni and delete constant inlining, progressive errors, and the + dynamic Java adapter as their gates pass. +4. Finish the dotted-U+ runtime unmatched-class correction and the next uniform + `reg_mesg.t` warning/source-position groups. Refresh complete `reg_mesg.t`, + `pat.t`, and `pat_advanced.t` maps after each combined diagnostics batch. +5. Complete the four-leg 80-file Java/Joni × JVM/interpreter matrix on one exact + artifact and compare every file with the PR 958 log. Resolve every regression, + zero-TAP record, timeout, truncation, or incomplete file before user acceptance. +6. Remove each proven-obsolete regex transformation from `dev/import-perl5` + sync sources, regenerate a private unpatched corpus twice, prove byte-for-byte + idempotence, and run the affected upstream tests without editing them. +7. Use the refreshed impact report to move all remaining ordinary constants to + native Joni, deleting their Java routes and matcher-semantic preprocessing in + the same validated slices. Keep `pat_re_eval.t` at 555/555 throughout. +8. Delete Java matching, selector, fallback state, and unreachable preprocessors; + then run direct/thread regex, CPAN, performance, packaging, notice/license, + warning-free build, Ubuntu, Windows, and full CI gates. +9. Update the feature matrix and final as-implemented/fork documents, remove or + summarize redundant design documents, rebase the final stack on `master`, and + run the complete PR 958 parity audit before declaring Phase 36 complete. ## Parallel Work @@ -329,9 +336,14 @@ gates may reopen it if a semantic regression appears. - [x] Generated Perl named-sequence lookup and native sequence resolution - [x] Remove temporary named-sequence encoding from native Joni pattern source - [x] Restore canonical multi-code-point named-sequence extended-class diagnostics -- [ ] Restore remaining Perl diagnostics for unknown named sequences +- [x] Restore Perl diagnostics for unknown/empty/malformed named sequences - [x] Native `(?[...])` with zero diagnostic regressions -- [ ] Native `(?(DEFINE)...)` and removal of its adapter rewrite +- [x] Native `(?(DEFINE)...)` and removal of its adapter rewrite +- [x] Native ordinary lookbehind and removal of its Java translation +- [x] Native branch reset and removal of its capture-map adapter +- [x] Native plain `\N` non-newline atom and interval forms +- [ ] Native recursive/runtime `(??{...})` and removal of dynamic adapters +- [ ] Retire proven-obsolete `dev/import-perl5` regex patches - [ ] Refresh the complete Unicode, `pat.t`, `pat_advanced.t`, `reg_mesg.t`, and 80-file forced-Joni gates on one integrated artifact From 0b360cfc4239a200daa358c6a2ced063c09b2ede Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 16:54:46 +0200 Subject: [PATCH 29/31] docs(regex): propose embeddable Perl regex library Describe a post-Phase-36 standalone JVM API, compatibility contract, execution tiers, packaging, security boundaries, and release prerequisites. The RFC is explicitly non-implementing and does not expand Phase 36 scope. Generated with Codex (https://openai.com/codex) Co-Authored-By: OpenAI Codex --- dev/design/perl-regex-library-rfc.md | 182 +++++++++++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 dev/design/perl-regex-library-rfc.md diff --git a/dev/design/perl-regex-library-rfc.md b/dev/design/perl-regex-library-rfc.md new file mode 100644 index 0000000000..da9973b573 --- /dev/null +++ b/dev/design/perl-regex-library-rfc.md @@ -0,0 +1,182 @@ +# RFC: Embeddable Perl-Compatible Regex Library for the JVM + +## Status + +Proposal only. This RFC does not authorize implementation and is not part of +the Phase 36 completion criteria. + +## Summary + +After the full Joni migration is complete, PerlOnJava could publish its regex +engine as a standalone JVM library. The library would offer Perl-compatible +regular expressions through a small Java API, in the same broad product space +as PCRE, without requiring applications to run general Perl programs. + +The public contract would be Perl regex semantics implemented by the forked +Joni engine and the minimum PerlOnJava compatibility runtime needed by it. It +must not expose Phase 36's temporary Java-regex routing or transitional +preprocessing as permanent behavior. + +## Motivation + +Java applications currently have no lightweight way to request Perl regex +semantics when `java.util.regex` is insufficient. PerlOnJava's Joni fork is +gaining capabilities that are useful independently of the language runtime: + +- Perl syntax and capture behavior +- Perl-compatible Unicode and byte-string semantics +- named groups, subroutine calls, recursion, and control verbs +- Perl-style diagnostics +- a differential corpus against system Perl + +A standalone artifact would make that work reusable by JVM applications and +would give the regex implementation a narrow, testable public boundary. + +## Proposed Product Boundary + +The default artifact should compile and execute data-only patterns. A familiar +Java-facing API is preferable to exposing Joni internals: + +```java +PerlPattern pattern = PerlPattern.compile("(?\\w+)", PerlFlags.UNICODE); +PerlMatcher matcher = pattern.matcher(input); + +if (matcher.find()) { + String word = matcher.group("word"); +} +``` + +The initial public surface should cover: + +- immutable compiled patterns +- stateful matchers +- numeric and named captures +- `find`, anchored match, replacement, and split operations +- explicit Perl flags and byte-versus-Unicode input modes +- structured compile and match exceptions +- configurable resource limits + +The API should document Perl semantics directly. Similarity to +`java.util.regex.Pattern` and `Matcher` is useful for discoverability, but it +must not imply Java-regex behavior where Perl differs. + +## Executable Pattern Tiers + +Executable constructs require a deliberately separate contract: + +1. **Data-only engine**: Ordinary patterns, recursion, subroutine calls, + conditionals, control verbs, and other constructs that do not execute host + language code. This is the safe default artifact and API. +2. **Host-callout engine**: Callouts invoke explicitly registered Java + callbacks through a constrained interface. Applications control the + registry and policy. +3. **Perl-execution integration**: Constructs such as `(?{ ... })` and + `(??{ ... })` execute Perl code in a PerlOnJava runtime context. This belongs + in a separate opt-in integration artifact, not in the default library. + +The data-only library must reject executable constructs unless the caller has +selected and configured the corresponding execution tier. + +## Architecture + +The standalone library should depend on a narrow regex-runtime module rather +than the complete PerlOnJava compiler and runtime. The preferred dependency +direction is: + +```text +public Perl regex API + | +minimal Perl regex compatibility layer + | +PerlOnJava Joni fork +``` + +The compatibility layer may contain generated Unicode/property data, +byte/Unicode provenance, diagnostics, replacement semantics, and other logic +that is genuinely part of Perl regex behavior. General Perl parsing, +bytecode generation, global variables, and runtime operators should remain +outside the data-only artifact. + +Joni classes are implementation details. Applications should not depend on +fork-specific packages or internal syntax nodes, so the fork can evolve +without breaking the public API. + +## Compatibility Contract + +Releases should identify a target Perl version and publish measured +compatibility rather than claim unqualified "Perl compatible" behavior. The +release evidence should include: + +- the exact upstream Perl regex corpus revision +- selected, executed, passed, failed, and skipped test counts +- byte, Unicode, JVM, and interpreter dimensions where applicable +- known unsupported or intentionally different behavior +- results relative to the maintained PerlOnJava baseline + +Compatibility changes should follow semantic versioning at the API level. +Corrections that make matching behavior agree with the declared Perl version +may still affect applications and must be called out in release notes. + +## Packaging and Attribution + +Publish the data-only API and Perl-execution integration as separate Maven +artifacts. Avoid split packages with upstream Joni and other Joni forks. The +fork should remain in the collision-resistant PerlOnJava namespace selected by +the Joni fork design. + +All original Joni copyright, license, and authorship notices must be retained. +Generated data and derived sources must record their input source, applicable +license, generator, and reproducible generation command. + +## Security and Resource Control + +Regex matching can consume substantial CPU, memory, and stack even without +callbacks. The API should support match deadlines or operation budgets, +backtracking/stack limits where technically possible, and cancellation. It +must define whether compiled patterns and inputs retain caller-owned data. + +Executable tiers require stronger isolation guidance. Callback and Perl-code +execution must be disabled by default, explicit at construction time, and +documented as execution of trusted code rather than ordinary regex matching. + +## Release Prerequisites + +Implementation should not begin until Phase 36 establishes all of the +following: + +- ordinary constant patterns use Joni by default +- temporary Java backend routing is removed from the advertised path +- temporary PerlOnJava preprocessors have either moved into justified Joni + internals or been removed +- the four-leg differential matrix has no unexplained regressions +- the final upstream Perl regex comparison meets the project parity gate +- fork packaging, licensing, attribution, and generated-data provenance are + audited +- the internal regex API is separable without depending on general Perl + execution + +## Open Questions + +- Which Perl release defines the first compatibility target? +- Should byte input use `byte[]`, a dedicated immutable value, or both? +- Should replacement templates be part of the first release? +- Which resource limits can the Joni fork enforce reliably? +- Is the host-callout tier useful enough to publish before the full + Perl-execution integration? +- Should the artifact name emphasize Perl compatibility, PerlOnJava, or the + Joni fork while avoiding confusion with PCRE? + +## Relationship to Current Work + +Phase 36 remains focused on completing and validating PerlOnJava's own full +Joni migration. This RFC is a possible follow-on productization step. It must +consume the completed implementation rather than introduce a second regex +behavior or stabilize transitional routing decisions. + +Related documents: + +- `dev/design/phase36-regex-parity.md` +- `dev/design/executable-regex-callbacks.md` +- `dev/design/regex_engines.md` +- `dev/design/regex_jruby_joni.md` +- `dev/implementation/regex.md` From cd08ab97949baefe7b6f75e9c2755a3a73ed23b9 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 17:18:00 +0200 Subject: [PATCH 30/31] docs(regex): advance Phase 36 integration gate Record PR 1078's cross-platform merge and make PR 1079's rebased Ubuntu and Windows checks the current integration gate. Keep the ordered plan forward-only. Generated with Codex (https://openai.com/codex) Co-Authored-By: OpenAI Codex --- dev/design/phase36-regex-parity.md | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index a15276ee1d..8a26f9d09d 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -89,16 +89,11 @@ affected corpus before taking another slice. the temporary `=POJSEQ=` transport. Generated and lexical multi-code-point names and ordinary scalar names resolve through Joni; the temporary generated-sequence-only routing distinction is deleted. -- PR 1078 targets `master` at `daca5545a`. Its warning-free local `make` passes - all 17 tasks, including Joni tests, five unit shards, packaging, and generated - data. Named-scalar Kelvin passes 9/9; named transport 11/11, cache 9/9, and - missing-brace diagnostics 8/8 on JVM/interpreter. Ubuntu and Windows CI are - running on the exact head; merge remains gated on both checks. -- Draft PR 1079 is the successor integration batch. Its current local head - contains native `(?(DEFINE)...)`, ordinary lookbehind, branch reset, plain - `\N`, and the PR 1078 platform correction on top of the validated malformed/ - unknown named-character diagnostics. The combined warning-free build is the - next durability gate before push. +- PR 1078 passed exact-head Ubuntu and Windows CI and is merged into `master`. +- PR 1079 is rebased onto that master and contains native `(?(DEFINE)...)`, + ordinary lookbehind, branch reset, and plain `\N` on top of the validated + malformed/unknown named-character diagnostics. Its rebased warning-free + `make` passes all 17 tasks; Ubuntu and Windows CI are the merge gate. - Native DEFINE, ordinary lookbehind, and branch reset now route through Joni; their feature-specific Java rewrites and branch-reset capture-map adapter are deleted. Plain Perl `\N` is a native Joni non-line-feed atom, including @@ -226,9 +221,9 @@ behavior. ## Ordered Next Steps -1. Merge PR 1078 only after exact-head Ubuntu and Windows CI pass. Rebase PR - 1079 onto the resulting `master`, retain its independent semantic commits, - run one combined warning-free `make`, push, and obtain stacked CI. +1. Merge PR 1079 only after exact-head Ubuntu and Windows CI pass, then rebase + the next integration batch onto the resulting `master` without combining its + independent semantic commits. 2. Complete byte/Unicode pattern provenance through runtime interpolation and template composition, then finish `/d`/`u`/`a`/`aa` forward/reverse literal and backreference folding from generated data. Require direct Joni plus From a2c272456cdc233da116833a0a34a15d931f0d02 Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Wed, 19 Aug 2026 18:23:23 +0200 Subject: [PATCH 31/31] docs(regex): advance to successor integration gate Record the cross-platform merged native-syntax state and make the successor fold, provenance, ordinary-default, dynamic, diagnostic, and import batch the current forward-only gate. Generated with Codex (https://openai.com/codex) Co-Authored-By: OpenAI Codex --- dev/design/phase36-regex-parity.md | 54 +++++++++++++++++------------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index 8a26f9d09d..9beb329587 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -71,7 +71,7 @@ affected corpus before taking another slice. integrated signed-IV range fix removes row 1651 with zero introductions in exact A/B evidence; a fresh combined serial gate remains required because a later run stopped before the complete plan under concurrent CPAN load. -- The current imported `reg_mesg.t` passes 1,710/2,603 on each backend with an +- The current imported `reg_mesg.t` passes 1,794/2,613 on each backend with an identical status/test-number vector. - Forced-Joni Unicode property comparison is 83,648/83,648 on both backends. The Hyphen/IsHyphen warning correction removes the final exact 32 residuals @@ -89,25 +89,33 @@ affected corpus before taking another slice. the temporary `=POJSEQ=` transport. Generated and lexical multi-code-point names and ordinary scalar names resolve through Joni; the temporary generated-sequence-only routing distinction is deleted. -- PR 1078 passed exact-head Ubuntu and Windows CI and is merged into `master`. -- PR 1079 is rebased onto that master and contains native `(?(DEFINE)...)`, - ordinary lookbehind, branch reset, and plain `\N` on top of the validated - malformed/unknown named-character diagnostics. Its rebased warning-free - `make` passes all 17 tasks; Ubuntu and Windows CI are the merge gate. +- `master` contains the validated named-character diagnostics plus native + `(?(DEFINE)...)`, ordinary lookbehind, branch reset, and plain `\N`; its exact + head passed warning-free local, Ubuntu, and Windows gates. +- The successor integration batch carries byte/Unicode provenance and fold + policy, dotted-U+ diagnostics, the ordinary-pattern Joni default, a dynamic- + pattern edge contract, and the first obsolete import retirement. Its fold, + property, and resolver-cache residuals are closed; the exact semantic head + passes a warning-free 17-task `make`. The prospective PR head also includes + three independent runtime-diagnostic corrections and passes the same full + combined gate. - Native DEFINE, ordinary lookbehind, and branch reset now route through Joni; their feature-specific Java rewrites and branch-reset capture-map adapter are deleted. Plain Perl `\N` is a native Joni non-line-feed atom, including intervals, `/s` independence, and Perl's character-class diagnostic. -- Unknown, empty, malformed U+, and missing-brace named-character groups have - native frontend/runtime diagnostics. The remaining dotted-U+ unmatched-class - identity is an active bounded runtime diagnostic correction. +- Unknown, empty, malformed/dotted U+, and missing-brace named-character groups + have native frontend/runtime diagnostics. - Forced-Joni `pat_re_eval.t` now executes and passes 555/555 on both JVM and interpreter. Perl's release-build `-D` diagnostic is preserved without enabling PerlOnJava's unrelated internal compiler trace. -- Each `pat.t` variant executes 1,301/1,302 and passes 1,223. -- The current 80-file forced-Joni gate passes 363,164/391,977 and has 19 - per-file pass-count regressions against PR 958. These figures must be refreshed - after the current native stack is integrated. +- Each `pat.t` variant executes 1,301/1,302; JVM passes 1,225 and interpreter + passes 1,234. The remaining JVM-only dynamic code-array rows are active work. +- The four Java/Joni × JVM/interpreter legs now have one 80-file comparison + ledger on the pre-successor artifact. It identifies stable extended-class, + regexp, charset, fold-grind, and bounded-speed negative clusters plus several + zero-TAP/execution records. The complete matrix must be repeated on the exact + successor artifact before acceptance; older aggregate figures are not release + evidence. - Exact `/aa` routing/folding gates pass on native Joni, and the Java `/aa` workaround is removed. @@ -221,9 +229,8 @@ behavior. ## Ordered Next Steps -1. Merge PR 1079 only after exact-head Ubuntu and Windows CI pass, then rebase - the next integration batch onto the resulting `master` without combining its - independent semantic commits. +1. Open the validated successor review PR against `master` and require + exact-head Ubuntu/Windows CI. 2. Complete byte/Unicode pattern provenance through runtime interpolation and template composition, then finish `/d`/`u`/`a`/`aa` forward/reverse literal and backreference folding from generated data. Require direct Joni plus @@ -233,12 +240,13 @@ behavior. unwind, backtracking re-evaluation, and recursion safety. Route every embedded closure to Joni and delete constant inlining, progressive errors, and the dynamic Java adapter as their gates pass. -4. Finish the dotted-U+ runtime unmatched-class correction and the next uniform - `reg_mesg.t` warning/source-position groups. Refresh complete `reg_mesg.t`, - `pat.t`, and `pat_advanced.t` maps after each combined diagnostics batch. -5. Complete the four-leg 80-file Java/Joni × JVM/interpreter matrix on one exact - artifact and compare every file with the PR 958 log. Resolve every regression, - zero-TAP record, timeout, truncation, or incomplete file before user acceptance. +4. Carry lexical `use re 'strict'` policy through regex compilation and close + the unescaped-brace/non-hex diagnostic families. Refresh complete + `reg_mesg.t`, `pat.t`, and `pat_advanced.t` maps after each combined batch. +5. Repeat the four-leg 80-file Java/Joni × JVM/interpreter matrix on the exact + successor artifact and compare every file with the PR 958 log. Resolve every + regression, zero-TAP record, timeout, truncation, or incomplete file before + user acceptance. 6. Remove each proven-obsolete regex transformation from `dev/import-perl5` sync sources, regenerate a private unpatched corpus twice, prove byte-for-byte idempotence, and run the affected upstream tests without editing them. @@ -326,7 +334,7 @@ gates may reopen it if a semantic regression appears. - [x] Pinned Perl simple/full/reverse case-fold data - [x] Native fold adapter and unsafe optimizer-boundary suppression - [x] Property/class fold closure -- [ ] Fold-mode and byte/Unicode provenance context +- [x] Fold-mode and byte/Unicode provenance context - [ ] Forward/reverse literal expansion and backreference folding - [x] Generated Perl named-sequence lookup and native sequence resolution - [x] Remove temporary named-sequence encoding from native Joni pattern source