diff --git a/third_party/joni/src/org/joni/ApplyCaseFold.java b/third_party/joni/src/org/joni/ApplyCaseFold.java index c3f01e543..59021b4c8 100644 --- a/third_party/joni/src/org/joni/ApplyCaseFold.java +++ b/third_party/joni/src/org/joni/ApplyCaseFold.java @@ -36,6 +36,7 @@ public void apply(int from, int[]to, int length, Object o) { Encoding enc = env.enc; CClassNode cc = arg.cc; CClassNode ascCc = arg.ascCc; + CClassNode foldCc = arg.foldCc; BitSet bs = cc.bs; boolean addFlag; @@ -44,13 +45,12 @@ public void apply(int from, int[]to, int length, Object o) { return; } - if (ascCc == null) { + if (!isEligible(foldCc, enc, from) || ascCc == null) { addFlag = false; } else if (Encoding.isAscii(from) == Encoding.isAscii(to[0])) { addFlag = true; } else { - addFlag = ascCc.isCodeInCC(enc, from); - if (ascCc.isNot()) addFlag = !addFlag; + addFlag = isEligible(ascCc, enc, from); } if (length == 1) { @@ -84,7 +84,8 @@ public void apply(int from, int[]to, int length, Object o) { } // CASE_FOLD_IS_APPLIED_INSIDE_NEGATIVE_CCLASS } else { - if (cc.isCodeInCC(enc, from) && (!Config.CASE_FOLD_IS_APPLIED_INSIDE_NEGATIVE_CCLASS || !cc.isNot())) { + if (addFlag && cc.isCodeInCC(enc, from) + && (!Config.CASE_FOLD_IS_APPLIED_INSIDE_NEGATIVE_CCLASS || !cc.isNot())) { StringNode node = null; for (int i=0; i ascNode) { + private CClassNode parseCharClass(ObjPtr ascNode, + ObjPtr foldNode) { final boolean neg; - CClassNode cc, prevCc = null, ascCc = null, ascPrevCc = null, workCc = null, ascWorkCc = null; + CClassNode cc, prevCc = null, ascCc = null, ascPrevCc = null, + workCc = null, ascWorkCc = null, foldCc = null, + foldPrevCc = null, foldWorkCc = null; CCStateArg arg = new CCStateArg(); fetchTokenInCC(); @@ -167,6 +174,7 @@ private CClassNode parseCharClass(ObjPtr ascNode) { cc = new CClassNode(); if (isIgnoreCase(env.option)) { ascCc = ascNode.p = new CClassNode(); + foldCc = foldNode.p = new CClassNode(); } boolean andStart = false; @@ -184,7 +192,7 @@ private CClassNode parseCharClass(ObjPtr ascNode) { } arg.to = token.getC(); arg.toIsRaw = false; - parseCharClassValEntry2(cc, ascCc, arg); // goto val_entry2 + parseCharClassValEntry2(cc, ascCc, foldCc, arg); // goto val_entry2 break; case RAW_BYTE: @@ -224,25 +232,25 @@ private CClassNode parseCharClass(ObjPtr ascNode) { arg.inType = CCVALTYPE.SB; // raw_single: } arg.toIsRaw = true; - parseCharClassValEntry2(cc, ascCc, arg); // goto val_entry2 + parseCharClassValEntry2(cc, ascCc, foldCc, arg); // goto val_entry2 break; case CODE_POINT: arg.to = token.getCode(); arg.toIsRaw = true; - parseCharClassValEntry(cc, ascCc, arg); // val_entry:, val_entry2 + parseCharClassValEntry(cc, ascCc, foldCc, arg); // val_entry:, val_entry2 break; case POSIX_BRACKET_OPEN: - if (parsePosixBracket(cc, ascCc)) { /* true: is not POSIX bracket */ + if (parsePosixBracket(cc, ascCc, foldCc)) { /* true: is not POSIX bracket */ env.ccEscWarn("["); p = token.backP; arg.to = token.getC(); arg.toIsRaw = false; - parseCharClassValEntry(cc, ascCc, arg); // goto val_entry + parseCharClassValEntry(cc, ascCc, foldCc, arg); // goto val_entry break; } - cc.nextStateClass(arg, ascCc, env); // goto next_class + cc.nextStateClass(arg, ascCc, foldCc, env); // goto next_class break; case CHAR_TYPE: @@ -252,13 +260,17 @@ private CClassNode parseCharClass(ObjPtr ascNode) { ascCc.addCType(token.getPropCType(), token.getPropNot(), isAsciiRange(env.option), env, this); } } - cc.nextStateClass(arg, ascCc, env); // next_class: + if (foldCc != null) { + foldCc.addCType(token.getPropCType(), token.getPropNot(), + isAsciiRange(env.option), env, this); + } + cc.nextStateClass(arg, ascCc, foldCc, env); // next_class: break; case CHAR_PROPERTY: CharProperty property = fetchCharProperty(true); - addCharProperty(cc, ascCc, property, token.getPropNot()); - cc.nextStateClass(arg, ascCc, env); // goto next_class + addCharProperty(cc, ascCc, foldCc, property, token.getPropNot()); + cc.nextStateClass(arg, ascCc, foldCc, env); // goto next_class break; case CC_RANGE: @@ -266,11 +278,11 @@ private CClassNode parseCharClass(ObjPtr ascNode) { fetchTokenInCC(); fetched = true; if (token.type == TokenType.CC_CLOSE) { /* allow [x-] */ - parseCharClassRangeEndVal(cc, ascCc, arg); // range_end_val:, goto val_entry; + parseCharClassRangeEndVal(cc, ascCc, foldCc, arg); // range_end_val:, goto val_entry; break; } else if (token.type == TokenType.CC_AND) { env.ccEscWarn("-"); - parseCharClassRangeEndVal(cc, ascCc, arg); // goto range_end_val + parseCharClassRangeEndVal(cc, ascCc, foldCc, arg); // goto range_end_val break; } if (arg.type == CCVALTYPE.CLASS) newValueException(UNMATCHED_RANGE_SPECIFIER_IN_CHAR_CLASS); @@ -281,28 +293,28 @@ private CClassNode parseCharClass(ObjPtr ascNode) { fetchTokenInCC(); fetched = true; if (token.type == TokenType.CC_RANGE || andStart) env.ccEscWarn("-"); /* [--x] or [a&&-x] is warned. */ - parseCharClassValEntry(cc, ascCc, arg); // goto val_entry + parseCharClassValEntry(cc, ascCc, foldCc, arg); // goto val_entry break; } else if (arg.state == CCSTATE.RANGE) { env.ccEscWarn("-"); - parseCharClassSbChar(cc, ascCc, arg); // goto sb_char /* [!--x] is allowed */ + parseCharClassSbChar(cc, ascCc, foldCc, arg); // goto sb_char /* [!--x] is allowed */ break; } else { /* CCS_COMPLETE */ fetchTokenInCC(); fetched = true; if (token.type == TokenType.CC_CLOSE) { /* allow [a-b-] */ - parseCharClassRangeEndVal(cc, ascCc, arg); // goto range_end_val + parseCharClassRangeEndVal(cc, ascCc, foldCc, arg); // goto range_end_val break; } else if (token.type == TokenType.CC_AND) { env.ccEscWarn("-"); - parseCharClassRangeEndVal(cc, ascCc, arg); // goto range_end_val + parseCharClassRangeEndVal(cc, ascCc, foldCc, arg); // goto range_end_val break; } if (syntax.allowDoubleRangeOpInCC()) { env.ccEscWarn("-"); // parseCharClassSbChar(cc, ascCc, arg); // goto sb_char /* [0-9-a] is allowed as [0-9\-a] */ - parseCharClassRangeEndVal(cc, ascCc, arg); // goto range_end_val + parseCharClassRangeEndVal(cc, ascCc, foldCc, arg); // goto range_end_val break; } newSyntaxException(UNMATCHED_RANGE_SPECIFIER_IN_CHAR_CLASS); @@ -311,18 +323,22 @@ private CClassNode parseCharClass(ObjPtr ascNode) { case CC_CC_OPEN: /* [ */ ObjPtr ascPtr = new ObjPtr<>(); - CClassNode acc = parseCharClass(ascPtr); + ObjPtr foldPtr = new ObjPtr<>(); + CClassNode acc = parseCharClass(ascPtr, foldPtr); cc.or(acc, env); if (ascPtr.p != null) { ascCc.or(ascPtr.p, env); } + if (foldPtr.p != null) { + foldCc.or(foldPtr.p, env); + } break; case CC_AND: /* && */ if (arg.state == CCSTATE.VALUE) { arg.to = 0; arg.toIsRaw = false; - cc.nextStateValue(arg, ascCc, env); + cc.nextStateValue(arg, ascCc, foldCc, env); } /* initialize local variables */ andStart = true; @@ -332,6 +348,9 @@ private CClassNode parseCharClass(ObjPtr ascNode) { if (ascCc != null) { ascPrevCc.and(ascCc, env); } + if (foldCc != null) { + foldPrevCc.and(foldCc, env); + } } else { prevCc = cc; if (workCc == null) workCc = new CClassNode(); @@ -341,9 +360,15 @@ private CClassNode parseCharClass(ObjPtr ascNode) { if (ascWorkCc == null) ascWorkCc = new CClassNode(); ascCc = ascWorkCc; } + if (foldCc != null) { + foldPrevCc = foldCc; + if (foldWorkCc == null) foldWorkCc = new CClassNode(); + foldCc = foldWorkCc; + } } cc.clear(); if (ascCc != null) ascCc.clear(); + if (foldCc != null) foldCc.clear(); break; case EOT: @@ -360,7 +385,7 @@ private CClassNode parseCharClass(ObjPtr ascNode) { if (arg.state == CCSTATE.VALUE) { arg.to = 0; arg.toIsRaw = false; - cc.nextStateValue(arg, ascCc, env); + cc.nextStateValue(arg, ascCc, foldCc, env); } if (prevCc != null) { @@ -370,14 +395,20 @@ private CClassNode parseCharClass(ObjPtr ascNode) { ascPrevCc.and(ascCc, env); ascCc = ascPrevCc; } + if (foldCc != null) { + foldPrevCc.and(foldCc, env); + foldCc = foldPrevCc; + } } if (neg) { cc.setNot(); if (ascCc != null) ascCc.setNot(); + if (foldCc != null) foldCc.setNot(); } else { cc.clearNot(); if (ascCc != null) ascCc.clearNot(); + if (foldCc != null) foldCc.clearNot(); } if (cc.isNot() && syntax.notNewlineInNegativeCC()) { @@ -396,27 +427,31 @@ private CClassNode parseCharClass(ObjPtr ascNode) { return cc; } - private void parseCharClassSbChar(CClassNode cc, CClassNode ascCc, CCStateArg arg) { + private void parseCharClassSbChar(CClassNode cc, CClassNode ascCc, + CClassNode foldCc, CCStateArg arg) { arg.inType = CCVALTYPE.SB; arg.to = token.getC(); arg.toIsRaw = false; - parseCharClassValEntry2(cc, ascCc, arg); // goto val_entry2 + parseCharClassValEntry2(cc, ascCc, foldCc, arg); // goto val_entry2 } - private void parseCharClassRangeEndVal(CClassNode cc, CClassNode ascCc, CCStateArg arg) { + private void parseCharClassRangeEndVal(CClassNode cc, CClassNode ascCc, + CClassNode foldCc, CCStateArg arg) { arg.to = '-'; arg.toIsRaw = false; - parseCharClassValEntry(cc, ascCc, arg); // goto val_entry + parseCharClassValEntry(cc, ascCc, foldCc, arg); // goto val_entry } - private void parseCharClassValEntry(CClassNode cc, CClassNode ascCc, CCStateArg arg) { + private void parseCharClassValEntry(CClassNode cc, CClassNode ascCc, + CClassNode foldCc, CCStateArg arg) { int len = enc.codeToMbcLength(arg.to); arg.inType = len == 1 ? CCVALTYPE.SB : CCVALTYPE.CODE_POINT; - parseCharClassValEntry2(cc, ascCc, arg); // val_entry2: + parseCharClassValEntry2(cc, ascCc, foldCc, arg); // val_entry2: } - private void parseCharClassValEntry2(CClassNode cc, CClassNode ascCc, CCStateArg arg) { - cc.nextStateValue(arg, ascCc, env); + private void parseCharClassValEntry2(CClassNode cc, CClassNode ascCc, + CClassNode foldCc, CCStateArg arg) { + cc.nextStateValue(arg, ascCc, foldCc, env); } private Node parseEnclose(TokenType term) { @@ -1174,12 +1209,18 @@ private Node parseExp(TokenType term) { case CC_OPEN: { ObjPtr ascPtr = new ObjPtr<>(); - CClassNode cc = parseCharClass(ascPtr); + ObjPtr foldPtr = new ObjPtr<>(); + CClassNode cc = parseCharClass(ascPtr, foldPtr); int code = cc.isOneChar(); - if (code != -1) return parseStringLoop(StringNode.fromCodePoint(code, enc), group); + if (code != -1 && (!isIgnoreCase(env.option) + || ApplyCaseFold.isEligible(foldPtr.p, enc, code))) { + return parseStringLoop(StringNode.fromCodePoint(code, enc), group); + } node = cc; - if (isIgnoreCase(env.option)) node = cClassCaseFold(node, cc, ascPtr.p); + if (isIgnoreCase(env.option)) { + node = cClassCaseFold(node, cc, ascPtr.p, foldPtr.p); + } break; } @@ -1619,8 +1660,9 @@ private Node parseCharType(Node node) { return node; } - private Node cClassCaseFold(Node node, CClassNode cc, CClassNode ascCc) { - ApplyCaseFoldArg arg = new ApplyCaseFoldArg(env, cc, ascCc); + private Node cClassCaseFold(Node node, CClassNode cc, CClassNode ascCc, + CClassNode foldCc) { + ApplyCaseFoldArg arg = new ApplyCaseFoldArg(env, cc, ascCc, foldCc); enc.applyAllCaseFold(env.caseFoldFlagFor(env.option), ApplyCaseFold.INSTANCE, arg); if (arg.altRoot != null) { node = ListNode.newAlt(node, arg.altRoot); @@ -1632,28 +1674,37 @@ private Node parseCharProperty() { CharProperty property = fetchCharProperty(false); CClassNode cc = new CClassNode(); Node node = cc; - addCharProperty(cc, null, property, false); + addCharProperty(cc, null, null, property, false); if (token.getPropNot()) cc.setNot(); if (isIgnoreCase(env.option) && property.caseFold) { if (property.ranges != null || property.ctype != CharacterType.ASCII) { - node = cClassCaseFold(node, cc, cc); + node = cClassCaseFold(node, cc, cc, cc); } } return node; } private void addCharProperty(CClassNode cc, CClassNode ascCc, - CharProperty property, boolean not) { + CClassNode foldCc, CharProperty property, + boolean not) { if (property.ranges == null) { cc.addCType(property.ctype, not, false, env, this); if (ascCc != null && property.ctype != CharacterType.ASCII) { ascCc.addCType(property.ctype, not, false, env, this); } + if (foldCc != null) { + foldCc.addCType(property.ctype, not, false, env, this); + } return; } cc.addCodeRanges(property.ranges, not, env); - if (ascCc != null) ascCc.addCodeRanges(property.ranges, not, env); + if (ascCc != null) { + ascCc.addCodeRanges(property.ranges, not, env); + } + if (foldCc != null && property.caseFold) { + foldCc.addCodeRanges(property.ranges, not, env); + } } private Node parseAnycharAnytime() { diff --git a/third_party/joni/src/org/joni/ast/CClassNode.java b/third_party/joni/src/org/joni/ast/CClassNode.java index 150227116..463676140 100644 --- a/third_party/joni/src/org/joni/ast/CClassNode.java +++ b/third_party/joni/src/org/joni/ast/CClassNode.java @@ -415,31 +415,37 @@ public static final class CCStateArg { public CCSTATE state; } - public void nextStateClass(CCStateArg arg, CClassNode ascCC, ScanEnvironment env) { + public void nextStateClass(CCStateArg arg, CClassNode ascCc, + CClassNode foldCc, ScanEnvironment env) { if (arg.state == CCSTATE.RANGE) throw new SyntaxException(ErrorMessages.CHAR_CLASS_VALUE_AT_END_OF_RANGE); if (arg.state == CCSTATE.VALUE && arg.type != CCVALTYPE.CLASS) { if (arg.type == CCVALTYPE.SB) { bs.set(env, arg.from); - if (ascCC != null) ascCC.bs.set(arg.from); + if (ascCc != null) ascCc.bs.set(arg.from); + if (foldCc != null) foldCc.bs.set(arg.from); } else if (arg.type == CCVALTYPE.CODE_POINT) { addCodeRange(env, arg.from, arg.from); - if (ascCC != null) ascCC.addCodeRange(env, arg.from, arg.from, false); + if (ascCc != null) ascCc.addCodeRange(env, arg.from, arg.from, false); + if (foldCc != null) foldCc.addCodeRange(env, arg.from, arg.from, false); } } arg.state = CCSTATE.VALUE; arg.type = CCVALTYPE.CLASS; } - public void nextStateValue(CCStateArg arg, CClassNode ascCc, ScanEnvironment env) { + public void nextStateValue(CCStateArg arg, CClassNode ascCc, + CClassNode foldCc, ScanEnvironment env) { switch(arg.state) { case VALUE: if (arg.type == CCVALTYPE.SB) { bs.set(env, arg.from); if (ascCc != null) ascCc.bs.set(arg.from); + if (foldCc != null) foldCc.bs.set(arg.from); } else if (arg.type == CCVALTYPE.CODE_POINT) { addCodeRange(env, arg.from, arg.from); if (ascCc != null) ascCc.addCodeRange(env, arg.from, arg.from, false); + if (foldCc != null) foldCc.addCodeRange(env, arg.from, arg.from, false); } break; @@ -459,9 +465,11 @@ public void nextStateValue(CCStateArg arg, CClassNode ascCc, ScanEnvironment env } bs.setRange(env, arg.from, arg.to); if (ascCc != null) ascCc.bs.setRange(null, arg.from, arg.to); + if (foldCc != null) foldCc.bs.setRange(null, arg.from, arg.to); } else { addCodeRange(env, arg.from, arg.to); if (ascCc != null) ascCc.addCodeRange(env, arg.from, arg.to, false); + if (foldCc != null) foldCc.addCodeRange(env, arg.from, arg.to, false); } } else { if (arg.from > arg.to) { @@ -479,6 +487,10 @@ public void nextStateValue(CCStateArg arg, CClassNode ascCc, ScanEnvironment env ascCc.bs.setRange(null, arg.from, arg.to < 0xff ? arg.to : 0xff); ascCc.addCodeRange(env, arg.from, arg.to, false); } + if (foldCc != null) { + foldCc.bs.setRange(null, arg.from, arg.to < 0xff ? arg.to : 0xff); + foldCc.addCodeRange(env, arg.from, arg.to, false); + } } // ccs_range_end: arg.state = CCSTATE.COMPLETE; diff --git a/third_party/joni/test/org/joni/test/TestCharacterPropertyResolver.java b/third_party/joni/test/org/joni/test/TestCharacterPropertyResolver.java index e03ff4757..112f8cdf1 100644 --- a/third_party/joni/test/org/joni/test/TestCharacterPropertyResolver.java +++ b/third_party/joni/test/org/joni/test/TestCharacterPropertyResolver.java @@ -19,6 +19,7 @@ */ package org.joni.test; +import static org.joni.constants.SyntaxProperties.OP2_CCLASS_SET_OP; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertSame; import static org.junit.Assert.fail; @@ -38,16 +39,17 @@ public class TestCharacterPropertyResolver { String name = new String(bytes, p, end - p, StandardCharsets.UTF_8); return switch (name) { case "Fake" -> new CharacterPropertyResolver.Result( - new int[] {2, 'A', 'A', 0x1f642, 0x1f642}, true); + new int[] {3, 'A', 'A', 0xdf, 0xdf, 0x1f642, 0x1f642}, true); case "FakeNoFold" -> new CharacterPropertyResolver.Result( - new int[] {1, 'A', 'A'}, false); + new int[] {2, 'A', 'A', 0xdf, 0xdf}, false); default -> null; }; }; private static Syntax syntax(CharacterPropertyResolver resolver) { return new Syntax("CharacterPropertyResolver", Syntax.PerlNG.op, - Syntax.PerlNG.op2, Syntax.PerlNG.op3, Syntax.PerlNG.behavior, + Syntax.PerlNG.op2 | OP2_CCLASS_SET_OP, Syntax.PerlNG.op3, + Syntax.PerlNG.behavior, Syntax.PerlNG.options, Syntax.PerlNG.metaCharTable, null, resolver); } @@ -76,6 +78,36 @@ public void resolvesRangesInsideAndOutsideCharacterClasses() { assertEquals(-1, search("(?i)\\p{FakeNoFold}", "a")); } + @Test + public void preservesFoldPolicyInsidePositiveAndNegativeClasses() { + assertEquals(0, search("(?i)[\\p{Fake}]", "a")); + assertEquals(0, search("(?i)[\\p{FakeNoFold}]", "A")); + assertEquals(-1, search("(?i)[\\p{FakeNoFold}]", "a")); + assertEquals(-1, search("(?i)[\\P{FakeNoFold}]", "A")); + assertEquals(0, search("(?i)[\\P{FakeNoFold}]", "a")); + assertEquals(-1, search("(?i)[^\\p{FakeNoFold}]", "A")); + assertEquals(0, search("(?i)[^\\p{FakeNoFold}]", "a")); + } + + @Test + public void composesFoldPolicyThroughUnionsIntersectionsAndNestedClasses() { + assertEquals(0, search("(?i)[\\p{FakeNoFold}\\p{Fake}]", "a")); + assertEquals(0, search("(?i)[\\p{Fake}&&\\p{FakeNoFold}]", "A")); + assertEquals(-1, search("(?i)[\\p{Fake}&&\\p{FakeNoFold}]", "a")); + assertEquals(0, search("(?i)[[\\p{FakeNoFold}]B]", "A")); + assertEquals(-1, search("(?i)[[\\p{FakeNoFold}]B]", "a")); + assertEquals(0, search("(?i)[[\\p{FakeNoFold}]B]", "b")); + } + + @Test + public void foldsOnlyEligibleMembersOfMixedClasses() { + assertEquals(0, search("(?i)[B\\p{FakeNoFold}]", "A")); + assertEquals(-1, search("(?i)[B\\p{FakeNoFold}]", "a")); + assertEquals(0, search("(?i)[B\\p{FakeNoFold}]", "b")); + assertEquals(0, search("(?i)[\\p{Fake}]", "ss")); + assertEquals(-1, search("(?i)[\\p{FakeNoFold}]", "ss")); + } + @Test public void fallsBackToEncodingProperties() { assertEquals(0, search("\\p{Digit}", "7"));