Commit fd1d59c1be for perl
commit fd1d59c1bed36cc2553da44c0338f05e81c069cb
Author: David Mitchell <davem@iabyn.nospamdeletethisbit.com>
Date: Tue Sep 15 12:00:57 2026 +0100
regex SLC: better ignore non-regular tails
This is the second of two commits to change the implementation of the
super-linear cache when it comes to non-regular items like \1 and
(??{...}).
See the previous commit message for an overview.
In contrast to that previous commit which disabled the cache while a
sub-pattern was being executed, this commit changes how the
rest-of-pattern tail is handled. Formerly when encountering something
like \1 or (?(cond)yes|no), the SLC countdown was reset so that the SLC
wouldn't be engaged for a while. More strictly, when encountering
sub-patterns like (?1) or (??{ $sub_qr }}), maxiter was set to zero so
that both a new countdown was triggered and the existing cache would be
zeroed out. Trying to prove that that approach is correct and efficient
is hard.
This commit takes a different approach. A new local boolean variable
seen_nonregular is added, which is set whenever a non-regular node such
as \1 is seen. It is used to veto setting cache entries. See the
update to perlreguts.pod in this commit for the full technical details.
This commit also removes all the setting of poscache_iter or
poscache_maxiter to zero, which was the previous crude way of
temporarily disabling the cache. This commit and the previous one
collectively have replaced that mechanism.
diff --git a/pod/perlreguts.pod b/pod/perlreguts.pod
index 266097a426..9a48d47eb5 100644
--- a/pod/perlreguts.pod
+++ b/pod/perlreguts.pod
@@ -1239,14 +1239,6 @@ exponential behaviour.
=item *
-Non-regular pattern items such as backreferences (C<\g{1}>) and evals
-(C<(??{...})/>) break the assumption that running the same rest-of-pattern
-from the same position will give the same result each time. If these occur
-in the rest-of-pattern, the cache is reset. This is currently done at
-runtime.
-
-=item *
-
Nested quantifiers can sometimes the break the assumption that running the
rest-of-pattern from the same string position will always give the same
result. To understand this issue, first consider a non-nested quantifier
@@ -1321,6 +1313,67 @@ complicate the code.
=item *
+More generally, non-regular node types (including not only C<(?1)>
+and C<(?{{ $inner_qr })> but also back-references like C<\g{1}> and
+conditionals like C<(?{cond|yes|no)>) break the assumption that running
+the same rest-of-pattern from the same position will give the same result
+each time.
+
+This is detected by setting the boolean variable C<seen_nonregular>
+whenever such a non-regular node is executed. Then during backtracking,
+skipping the marking of the cache if the variable is true.
+
+In the presence of multiple quantifiers such as
+
+ /A (B)* C (D)* E/
+
+a non-regular node encountered while executing a D iteration or E should
+disable cache setting in both C<(B)*> and C<(D)*> when backtracking on
+failure. Conversely, a non-regular node encountered in B or C should only
+affect C<(B)*>. Finally, such a node in A shouldn't affect either
+quantifier. This is because such a node should only disable a quantifier's
+cache if that node appears within that quantifier's rest-of-pattern.
+
+To support this per-quantifier selective behaviour, at the start of each
+new quantifier (i.e. when processing the C<CURLYX>), the current value of
+C<seen_nonregular> is saved and then the variable is set to false. On
+return (via either success or failure), the saved value is restored but
+also ORed with the current value.
+
+The variable is similarly saved, set to false and ORed on return when
+entering the 'rest of pattern' after a qualifier, i.e. C<WHILEM_B_min,max>
+states. Or to put it another way, the variable is saved/restored at each
+of the spaces in the example pattern above.
+
+Currently the value is not saved/restored between individual iterations of
+a quantifier; that is, the C<WHILEM_A_*> states don't process
+C<seen_nonregular>. This is based on the assumption that each iteration of
+B or D is likely to have similar results: they I<all> contain non-regular
+nodes, or none of them do. This won't be universal of course: B could
+represent the physical pattern C<(abc|\g{1})> and some iterations might
+match one and others the other. But where this occurs, the only effect is
+that the cache isn't set when in theory it was safe to do so; so more
+backtracking occurs than is strictly necessary. It's always possible at a
+later date to add C<WHILEM_A_min,max> branches and save/restore
+C<seen_nonregular> on a more fine-grained basis.
+
+This run-time mechanism has an advantage over a compile-time one in that
+the cache is disabled only if a non-regular node is actually executed,
+rather than just being on the possible execution path. For example.
+C<...(aaa|bbb\1)...> might never encounter the \1 if the string doesn't
+contain C<bbb>, while the regex compiler would have to disable the cache
+on the grounds that the C<\1> I<might> be executed.
+
+Note that a simple C<(?{...})> shouldn't affect the SLC, because in theory
+the eval can't have any side-effects on the execution of the
+rest-of-pattern if it doesn't also have non-regular nodes.
+[ DAPM 2026 - actually the eval can modify C<$_> etc which under some
+circumstances can change the string currently being matched against. But
+that is a more general bug in eval which should be addressed at some
+point. ]
+
+=item *
+
There is currently a hard limit of 15 cache-participating C<WHILEM> nodes
per regex.
diff --git a/regexec.c b/regexec.c
index 89aa71fd80..b75af679a0 100644
--- a/regexec.c
+++ b/regexec.c
@@ -6483,7 +6483,7 @@ S_backup_one_WB_but_over_Extend_FO(pTHX_ WB_enum * previous,
/* we don't use STMT_START/END here because it leads to
"unreachable code" warnings, which are bogus, but distracting. */
#define CACHEsayNO \
- if (ST.cache_mask) { \
+ if (ST.cache_mask &&!seen_nonregular) { \
DEBUG_EXECUTE_r({ \
regnode *whilem = \
REGNODE_BEFORE(regnext(cur_curlyx->u.curlyx.me)); \
@@ -6765,6 +6765,10 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog)
false: plain (?=foo)
true: used as a condition: (?(?=foo))
*/
+ bool seen_nonregular = false; /* we've encountered a non-regular node
+ type such as \1 or (??{...}. For more
+ details, see
+ L<perlreguts/The super-linear cache> */
PAD* last_pad = NULL;
dMULTICALL;
U8 gimme = G_SCALAR;
@@ -8288,7 +8292,8 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog)
}
do_nref_ref_common:
- reginfo->poscache_iter = reginfo->poscache_maxiter; /* Void cache */
+ seen_nonregular = true;
+
if (RXp_LASTPAREN(rex) < n)
sayNO;
@@ -8733,7 +8738,7 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog)
* At this point we expect the stack context to be
* set up correctly */
- reginfo->poscache_maxiter = 0;
+ seen_nonregular = true;
/* the new regexp might have a different is_utf8_pat than we do */
is_utf8_pat = reginfo->is_utf8_pat = cBOOL(RX_UTF8(re_sv));
@@ -8791,8 +8796,6 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog)
cur_eval = ST.prev_eval;
cur_curlyx = ST.prev_curlyx;
- /* Invalidate cache. See "invalidate" comment above. */
- reginfo->poscache_maxiter = 0;
if ( nochange_depth )
nochange_depth--;
@@ -8827,8 +8830,6 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog)
cur_eval = ST.prev_eval;
cur_curlyx = ST.prev_curlyx;
- /* Invalidate cache. See "invalidate" comment above. */
- reginfo->poscache_maxiter = 0;
if ( nochange_depth )
nochange_depth--;
@@ -8936,7 +8937,8 @@ S_regmatch(pTHX_ regmatch_info *reginfo, char *startpos, regnode *prog)
break;
case IFTHEN: /* (?(cond)A|B) */
- reginfo->poscache_iter = reginfo->poscache_maxiter; /* Void cache */
+ seen_nonregular = true;
+
if (sw)
next = REGNODE_AFTER_type(scan,tregnode_IFTHEN);
else {
@@ -9062,6 +9064,8 @@ NULL
minmod = 0;
ST.count = -1; /* this will be updated by WHILEM */
ST.lastloc = NULL; /* this will be updated by WHILEM */
+ ST.saved_seen_nonregular = seen_nonregular;
+ seen_nonregular = false;
PUSH_YES_STATE_GOTO(CURLYX_end, REGNODE_BEFORE(next), locinput, loceol,
script_run_begin);
@@ -9069,11 +9073,13 @@ NULL
}
case CURLYX_end: /* just finished matching all of A*B */
+ seen_nonregular |= ST.saved_seen_nonregular;
cur_curlyx = ST.prev_curlyx;
sayYES;
NOT_REACHED; /* NOTREACHED */
case CURLYX_end_fail: /* just failed to match all of A*B */
+ seen_nonregular |= ST.saved_seen_nonregular;
REGCP_UNWIND(ST.cp); /* LEAVE in disguise */
cur_curlyx = ST.prev_curlyx;
sayNO;
@@ -9251,6 +9257,8 @@ NULL
if (cur_curlyx->u.curlyx.minmod) {
ST.save_curlyx = cur_curlyx;
cur_curlyx = cur_curlyx->u.curlyx.prev_curlyx;
+ ST.saved_seen_nonregular = seen_nonregular;
+ seen_nonregular = false;
PUSH_YES_STATE_GOTO(WHILEM_B_min, ST.save_curlyx->u.curlyx.B,
locinput, loceol, script_run_begin);
NOT_REACHED; /* NOTREACHED */
@@ -9273,11 +9281,13 @@ NULL
case WHILEM_B_min: /* just matched B in a minimal match */
case WHILEM_B_max: /* just matched B in a maximal match */
+ seen_nonregular |= ST.saved_seen_nonregular;
cur_curlyx = ST.save_curlyx;
sayYES;
NOT_REACHED; /* NOTREACHED */
case WHILEM_B_max_fail: /* just failed to match B in a maximal match */
+ seen_nonregular |= ST.saved_seen_nonregular;
cur_curlyx = ST.save_curlyx;
cur_curlyx->u.curlyx.lastloc = ST.save_lastloc;
cur_curlyx->u.curlyx.count--;
@@ -9305,11 +9315,14 @@ NULL
/* now try B */
ST.save_curlyx = cur_curlyx;
cur_curlyx = cur_curlyx->u.curlyx.prev_curlyx;
+ ST.saved_seen_nonregular = seen_nonregular;
+ seen_nonregular = false;
PUSH_YES_STATE_GOTO(WHILEM_B_max, ST.save_curlyx->u.curlyx.B,
locinput, loceol, script_run_begin);
NOT_REACHED; /* NOTREACHED */
case WHILEM_B_min_fail: /* just failed to match B in a minimal match */
+ seen_nonregular |= ST.saved_seen_nonregular;
cur_curlyx = ST.save_curlyx;
if (cur_curlyx->u.curlyx.count >= /*max*/ARG2i(cur_curlyx->u.curlyx.me)) {
diff --git a/regexp.h b/regexp.h
index bea3b0fa19..cfb8dc68e4 100644
--- a/regexp.h
+++ b/regexp.h
@@ -1039,6 +1039,7 @@ typedef struct regmatch_state {
CHECKPOINT cp; /* see note above "struct branchlike" */
CHECKPOINT lastcp; /* see note above "struct branchlike" */
bool minmod;
+ bool saved_seen_nonregular; /* previous seen_nonregular */
int parenfloor; /* how far back to strip paren data */
/* these two are modified by WHILEM */
@@ -1055,6 +1056,7 @@ typedef struct regmatch_state {
char *save_lastloc; /* previous curlyx.lastloc */
I32 cache_offset;
I32 cache_mask;
+ bool saved_seen_nonregular; /* previous seen_nonregular */
} whilem;
struct {
diff --git a/t/re/pat.t b/t/re/pat.t
index e9dfab022b..3299727b34 100644
--- a/t/re/pat.t
+++ b/t/re/pat.t
@@ -28,7 +28,7 @@ skip_all_without_unicode_tables();
my $has_locales = locales_enabled('LC_CTYPE');
my $utf8_locale = find_utf8_ctype_locale();
-plan tests => 1312; # Update this when adding/deleting tests.
+plan tests => 1313; # Update this when adding/deleting tests.
run_tests() unless caller;
@@ -2697,6 +2697,23 @@ SKIP:
ok("xayxay" =~ /^.*(??{ $inner_qr }){2,3}/, 'SLC nested qr');
is($&, "xayxay", 'SLC nested qr $&');
+ # Non-regular items such as back-references
+
+ ok( "aaabbbaa"
+ =~ /^
+ (a+) # a
+ ([ab]+)* # aabbb
+ (
+ \g{1} # a
+ |
+ cccc
+ )
+ [az] # a
+ $/x,
+ "SLC backref"
+ );
+
+
}
{