/[pcre]/code/branches/pcre16/pcre_compile.c
ViewVC logotype

Diff of /code/branches/pcre16/pcre_compile.c

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 530 by ph10, Tue Jun 1 13:42:06 2010 UTC revision 618 by ph10, Sat Jul 16 17:24:16 2011 UTC
# Line 6  Line 6 
6  and semantics are as close as possible to those of the Perl 5 language.  and semantics are as close as possible to those of the Perl 5 language.
7    
8                         Written by Philip Hazel                         Written by Philip Hazel
9             Copyright (c) 1997-2010 University of Cambridge             Copyright (c) 1997-2011 University of Cambridge
10    
11  -----------------------------------------------------------------------------  -----------------------------------------------------------------------------
12  Redistribution and use in source and binary forms, with or without  Redistribution and use in source and binary forms, with or without
# Line 261  static const int posix_class_maps[] = { Line 261  static const int posix_class_maps[] = {
261    cbit_xdigit,-1,          0              /* xdigit */    cbit_xdigit,-1,          0              /* xdigit */
262  };  };
263    
264  /* Table of substitutes for \d etc when PCRE_UCP is set. The POSIX class  /* Table of substitutes for \d etc when PCRE_UCP is set. The POSIX class
265  substitutes must be in the order of the names, defined above, and there are  substitutes must be in the order of the names, defined above, and there are
266  both positive and negative cases. NULL means no substitute. */  both positive and negative cases. NULL means no substitute. */
267    
268  #ifdef SUPPORT_UCP  #ifdef SUPPORT_UCP
# Line 272  static const uschar *substitutes[] = { Line 272  static const uschar *substitutes[] = {
272    (uschar *)"\\P{Xsp}",   /* \S */       /* NOTE: Xsp is Perl space */    (uschar *)"\\P{Xsp}",   /* \S */       /* NOTE: Xsp is Perl space */
273    (uschar *)"\\p{Xsp}",   /* \s */    (uschar *)"\\p{Xsp}",   /* \s */
274    (uschar *)"\\P{Xwd}",   /* \W */    (uschar *)"\\P{Xwd}",   /* \W */
275    (uschar *)"\\p{Xwd}"    /* \w */    (uschar *)"\\p{Xwd}"    /* \w */
276  };  };
277    
278  static const uschar *posix_substitutes[] = {  static const uschar *posix_substitutes[] = {
279    (uschar *)"\\p{L}",     /* alpha */    (uschar *)"\\p{L}",     /* alpha */
280    (uschar *)"\\p{Ll}",    /* lower */    (uschar *)"\\p{Ll}",    /* lower */
281    (uschar *)"\\p{Lu}",    /* upper */    (uschar *)"\\p{Lu}",    /* upper */
282    (uschar *)"\\p{Xan}",   /* alnum */    (uschar *)"\\p{Xan}",   /* alnum */
283    NULL,                   /* ascii */    NULL,                   /* ascii */
284    (uschar *)"\\h",        /* blank */    (uschar *)"\\h",        /* blank */
285    NULL,                   /* cntrl */    NULL,                   /* cntrl */
# Line 289  static const uschar *posix_substitutes[] Line 289  static const uschar *posix_substitutes[]
289    NULL,                   /* punct */    NULL,                   /* punct */
290    (uschar *)"\\p{Xps}",   /* space */    /* NOTE: Xps is POSIX space */    (uschar *)"\\p{Xps}",   /* space */    /* NOTE: Xps is POSIX space */
291    (uschar *)"\\p{Xwd}",   /* word */    (uschar *)"\\p{Xwd}",   /* word */
292    NULL,                   /* xdigit */    NULL,                   /* xdigit */
293    /* Negated cases */    /* Negated cases */
294    (uschar *)"\\P{L}",     /* ^alpha */    (uschar *)"\\P{L}",     /* ^alpha */
295    (uschar *)"\\P{Ll}",    /* ^lower */    (uschar *)"\\P{Ll}",    /* ^lower */
296    (uschar *)"\\P{Lu}",    /* ^upper */    (uschar *)"\\P{Lu}",    /* ^upper */
297    (uschar *)"\\P{Xan}",   /* ^alnum */    (uschar *)"\\P{Xan}",   /* ^alnum */
298    NULL,                   /* ^ascii */    NULL,                   /* ^ascii */
299    (uschar *)"\\H",        /* ^blank */    (uschar *)"\\H",        /* ^blank */
300    NULL,                   /* ^cntrl */    NULL,                   /* ^cntrl */
# Line 304  static const uschar *posix_substitutes[] Line 304  static const uschar *posix_substitutes[]
304    NULL,                   /* ^punct */    NULL,                   /* ^punct */
305    (uschar *)"\\P{Xps}",   /* ^space */   /* NOTE: Xps is POSIX space */    (uschar *)"\\P{Xps}",   /* ^space */   /* NOTE: Xps is POSIX space */
306    (uschar *)"\\P{Xwd}",   /* ^word */    (uschar *)"\\P{Xwd}",   /* ^word */
307    NULL                    /* ^xdigit */    NULL                    /* ^xdigit */
308  };  };
309  #define POSIX_SUBSIZE (sizeof(posix_substitutes)/sizeof(uschar *))  #define POSIX_SUBSIZE (sizeof(posix_substitutes)/sizeof(uschar *))
310  #endif  #endif
311    
312  #define STRING(a)  # a  #define STRING(a)  # a
313  #define XSTRING(s) STRING(s)  #define XSTRING(s) STRING(s)
# Line 407  static const char error_texts[] = Line 407  static const char error_texts[] =
407    /* 65 */    /* 65 */
408    "different names for subpatterns of the same number are not allowed\0"    "different names for subpatterns of the same number are not allowed\0"
409    "(*MARK) must have an argument\0"    "(*MARK) must have an argument\0"
410    "this version of PCRE is not compiled with PCRE_UCP support\0"    "this version of PCRE is not compiled with PCRE_UCP support\0"
411      "\\c must be followed by an ASCII character\0"
412    ;    ;
413    
414  /* Table to identify digits and hex digits. This is used when compiling  /* Table to identify digits and hex digits. This is used when compiling
# Line 544  static const unsigned char ebcdic_charta Line 545  static const unsigned char ebcdic_charta
545  /* Definition to allow mutual recursion */  /* Definition to allow mutual recursion */
546    
547  static BOOL  static BOOL
548    compile_regex(int, int, uschar **, const uschar **, int *, BOOL, BOOL, int,    compile_regex(int, uschar **, const uschar **, int *, BOOL, BOOL, int, int *,
549      int *, int *, branch_chain *, compile_data *, int *);      int *, branch_chain *, compile_data *, int *);
550    
551    
552    
# Line 841  else Line 842  else
842      break;      break;
843    
844      /* For \c, a following letter is upper-cased; then the 0x40 bit is flipped.      /* For \c, a following letter is upper-cased; then the 0x40 bit is flipped.
845      This coding is ASCII-specific, but then the whole concept of \cx is      An error is given if the byte following \c is not an ASCII character. This
846        coding is ASCII-specific, but then the whole concept of \cx is
847      ASCII-specific. (However, an EBCDIC equivalent has now been added.) */      ASCII-specific. (However, an EBCDIC equivalent has now been added.) */
848    
849      case CHAR_c:      case CHAR_c:
# Line 851  else Line 853  else
853        *errorcodeptr = ERR2;        *errorcodeptr = ERR2;
854        break;        break;
855        }        }
856    #ifndef EBCDIC    /* ASCII/UTF-8 coding */
857  #ifndef EBCDIC  /* ASCII/UTF-8 coding */      if (c > 127)  /* Excludes all non-ASCII in either mode */
858          {
859          *errorcodeptr = ERR68;
860          break;
861          }
862      if (c >= CHAR_a && c <= CHAR_z) c -= 32;      if (c >= CHAR_a && c <= CHAR_z) c -= 32;
863      c ^= 0x40;      c ^= 0x40;
864  #else           /* EBCDIC coding */  #else             /* EBCDIC coding */
865      if (c >= CHAR_a && c <= CHAR_z) c += 64;      if (c >= CHAR_a && c <= CHAR_z) c += 64;
866      c ^= 0xC0;      c ^= 0xC0;
867  #endif  #endif
# Line 1099  top-level call starts at the beginning o Line 1105  top-level call starts at the beginning o
1105  start at a parenthesis. It scans along a pattern's text looking for capturing  start at a parenthesis. It scans along a pattern's text looking for capturing
1106  subpatterns, and counting them. If it finds a named pattern that matches the  subpatterns, and counting them. If it finds a named pattern that matches the
1107  name it is given, it returns its number. Alternatively, if the name is NULL, it  name it is given, it returns its number. Alternatively, if the name is NULL, it
1108  returns when it reaches a given numbered subpattern. We know that if (?P< is  returns when it reaches a given numbered subpattern. Recursion is used to keep
1109  encountered, the name will be terminated by '>' because that is checked in the  track of subpatterns that reset the capturing group numbers - the (?| feature.
1110  first pass. Recursion is used to keep track of subpatterns that reset the  
1111  capturing group numbers - the (?| feature.  This function was originally called only from the second pass, in which we know
1112    that if (?< or (?' or (?P< is encountered, the name will be correctly
1113    terminated because that is checked in the first pass. There is now one call to
1114    this function in the first pass, to check for a recursive back reference by
1115    name (so that we can make the whole group atomic). In this case, we need check
1116    only up to the current position in the pattern, and that is still OK because
1117    and previous occurrences will have been checked. To make this work, the test
1118    for "end of pattern" is a check against cd->end_pattern in the main loop,
1119    instead of looking for a binary zero. This means that the special first-pass
1120    call can adjust cd->end_pattern temporarily. (Checks for binary zero while
1121    processing items within the loop are OK, because afterwards the main loop will
1122    terminate.)
1123    
1124  Arguments:  Arguments:
1125    ptrptr       address of the current character pointer (updated)    ptrptr       address of the current character pointer (updated)
# Line 1110  Arguments: Line 1127  Arguments:
1127    name         name to seek, or NULL if seeking a numbered subpattern    name         name to seek, or NULL if seeking a numbered subpattern
1128    lorn         name length, or subpattern number if name is NULL    lorn         name length, or subpattern number if name is NULL
1129    xmode        TRUE if we are in /x mode    xmode        TRUE if we are in /x mode
1130      utf8         TRUE if we are in UTF-8 mode
1131    count        pointer to the current capturing subpattern number (updated)    count        pointer to the current capturing subpattern number (updated)
1132    
1133  Returns:       the number of the named subpattern, or -1 if not found  Returns:       the number of the named subpattern, or -1 if not found
# Line 1117  Returns: the number of the named s Line 1135  Returns: the number of the named s
1135    
1136  static int  static int
1137  find_parens_sub(uschar **ptrptr, compile_data *cd, const uschar *name, int lorn,  find_parens_sub(uschar **ptrptr, compile_data *cd, const uschar *name, int lorn,
1138    BOOL xmode, int *count)    BOOL xmode, BOOL utf8, int *count)
1139  {  {
1140  uschar *ptr = *ptrptr;  uschar *ptr = *ptrptr;
1141  int start_count = *count;  int start_count = *count;
# Line 1129  dealing with. The very first call may no Line 1147  dealing with. The very first call may no
1147    
1148  if (ptr[0] == CHAR_LEFT_PARENTHESIS)  if (ptr[0] == CHAR_LEFT_PARENTHESIS)
1149    {    {
1150    if (ptr[1] == CHAR_QUESTION_MARK &&    /* Handle specials such as (*SKIP) or (*UTF8) etc. */
1151        ptr[2] == CHAR_VERTICAL_LINE)  
1152      if (ptr[1] == CHAR_ASTERISK) ptr += 2;
1153    
1154      /* Handle a normal, unnamed capturing parenthesis. */
1155    
1156      else if (ptr[1] != CHAR_QUESTION_MARK)
1157        {
1158        *count += 1;
1159        if (name == NULL && *count == lorn) return *count;
1160        ptr++;
1161        }
1162    
1163      /* All cases now have (? at the start. Remember when we are in a group
1164      where the parenthesis numbers are duplicated. */
1165    
1166      else if (ptr[2] == CHAR_VERTICAL_LINE)
1167      {      {
1168      ptr += 3;      ptr += 3;
1169      dup_parens = TRUE;      dup_parens = TRUE;
1170      }      }
1171    
1172    /* Handle a normal, unnamed capturing parenthesis */    /* Handle comments; all characters are allowed until a ket is reached. */
1173    
1174    else if (ptr[1] != CHAR_QUESTION_MARK && ptr[1] != CHAR_ASTERISK)    else if (ptr[2] == CHAR_NUMBER_SIGN)
1175      {      {
1176      *count += 1;      for (ptr += 3; *ptr != 0; ptr++) if (*ptr == CHAR_RIGHT_PARENTHESIS) break;
1177      if (name == NULL && *count == lorn) return *count;      goto FAIL_EXIT;
     ptr++;  
1178      }      }
1179    
1180    /* Handle a condition. If it is an assertion, just carry on so that it    /* Handle a condition. If it is an assertion, just carry on so that it
1181    is processed as normal. If not, skip to the closing parenthesis of the    is processed as normal. If not, skip to the closing parenthesis of the
1182    condition (there can't be any nested parens. */    condition (there can't be any nested parens). */
1183    
1184    else if (ptr[2] == CHAR_LEFT_PARENTHESIS)    else if (ptr[2] == CHAR_LEFT_PARENTHESIS)
1185      {      {
# Line 1159  if (ptr[0] == CHAR_LEFT_PARENTHESIS) Line 1191  if (ptr[0] == CHAR_LEFT_PARENTHESIS)
1191        }        }
1192      }      }
1193    
1194    /* We have either (? or (* and not a condition */    /* Start with (? but not a condition. */
1195    
1196    else    else
1197      {      {
# Line 1188  if (ptr[0] == CHAR_LEFT_PARENTHESIS) Line 1220  if (ptr[0] == CHAR_LEFT_PARENTHESIS)
1220    }    }
1221    
1222  /* Past any initial parenthesis handling, scan for parentheses or vertical  /* Past any initial parenthesis handling, scan for parentheses or vertical
1223  bars. */  bars. Stop if we get to cd->end_pattern. Note that this is important for the
1224    first-pass call when this value is temporarily adjusted to stop at the current
1225    position. So DO NOT change this to a test for binary zero. */
1226    
1227  for (; *ptr != 0; ptr++)  for (; ptr < cd->end_pattern; ptr++)
1228    {    {
1229    /* Skip over backslashed characters and also entire \Q...\E */    /* Skip over backslashed characters and also entire \Q...\E */
1230    
# Line 1264  for (; *ptr != 0; ptr++) Line 1298  for (; *ptr != 0; ptr++)
1298    
1299    if (xmode && *ptr == CHAR_NUMBER_SIGN)    if (xmode && *ptr == CHAR_NUMBER_SIGN)
1300      {      {
1301      while (*(++ptr) != 0 && *ptr != CHAR_NL) {};      ptr++;
1302        while (*ptr != 0)
1303          {
1304          if (IS_NEWLINE(ptr)) { ptr += cd->nllen - 1; break; }
1305          ptr++;
1306    #ifdef SUPPORT_UTF8
1307          if (utf8) while ((*ptr & 0xc0) == 0x80) ptr++;
1308    #endif
1309          }
1310      if (*ptr == 0) goto FAIL_EXIT;      if (*ptr == 0) goto FAIL_EXIT;
1311      continue;      continue;
1312      }      }
# Line 1273  for (; *ptr != 0; ptr++) Line 1315  for (; *ptr != 0; ptr++)
1315    
1316    if (*ptr == CHAR_LEFT_PARENTHESIS)    if (*ptr == CHAR_LEFT_PARENTHESIS)
1317      {      {
1318      int rc = find_parens_sub(&ptr, cd, name, lorn, xmode, count);      int rc = find_parens_sub(&ptr, cd, name, lorn, xmode, utf8, count);
1319      if (rc > 0) return rc;      if (rc > 0) return rc;
1320      if (*ptr == 0) goto FAIL_EXIT;      if (*ptr == 0) goto FAIL_EXIT;
1321      }      }
# Line 1281  for (; *ptr != 0; ptr++) Line 1323  for (; *ptr != 0; ptr++)
1323    else if (*ptr == CHAR_RIGHT_PARENTHESIS)    else if (*ptr == CHAR_RIGHT_PARENTHESIS)
1324      {      {
1325      if (dup_parens && *count < hwm_count) *count = hwm_count;      if (dup_parens && *count < hwm_count) *count = hwm_count;
1326      *ptrptr = ptr;      goto FAIL_EXIT;
     return -1;  
1327      }      }
1328    
1329    else if (*ptr == CHAR_VERTICAL_LINE && dup_parens)    else if (*ptr == CHAR_VERTICAL_LINE && dup_parens)
# Line 1320  Arguments: Line 1361  Arguments:
1361    name         name to seek, or NULL if seeking a numbered subpattern    name         name to seek, or NULL if seeking a numbered subpattern
1362    lorn         name length, or subpattern number if name is NULL    lorn         name length, or subpattern number if name is NULL
1363    xmode        TRUE if we are in /x mode    xmode        TRUE if we are in /x mode
1364      utf8         TRUE if we are in UTF-8 mode
1365    
1366  Returns:       the number of the found subpattern, or -1 if not found  Returns:       the number of the found subpattern, or -1 if not found
1367  */  */
1368    
1369  static int  static int
1370  find_parens(compile_data *cd, const uschar *name, int lorn, BOOL xmode)  find_parens(compile_data *cd, const uschar *name, int lorn, BOOL xmode,
1371      BOOL utf8)
1372  {  {
1373  uschar *ptr = (uschar *)cd->start_pattern;  uschar *ptr = (uschar *)cd->start_pattern;
1374  int count = 0;  int count = 0;
# Line 1338  matching closing parens. That is why we Line 1381  matching closing parens. That is why we
1381    
1382  for (;;)  for (;;)
1383    {    {
1384    rc = find_parens_sub(&ptr, cd, name, lorn, xmode, &count);    rc = find_parens_sub(&ptr, cd, name, lorn, xmode, utf8, &count);
1385    if (rc > 0 || *ptr++ == 0) break;    if (rc > 0 || *ptr++ == 0) break;
1386    }    }
1387    
# Line 1354  return rc; Line 1397  return rc;
1397    
1398  /* This is called by several functions that scan a compiled expression looking  /* This is called by several functions that scan a compiled expression looking
1399  for a fixed first character, or an anchoring op code etc. It skips over things  for a fixed first character, or an anchoring op code etc. It skips over things
1400  that do not influence this. For some calls, a change of option is important.  that do not influence this. For some calls, it makes sense to skip negative
1401  For some calls, it makes sense to skip negative forward and all backward  forward and all backward assertions, and also the \b assertion; for others it
1402  assertions, and also the \b assertion; for others it does not.  does not.
1403    
1404  Arguments:  Arguments:
1405    code         pointer to the start of the group    code         pointer to the start of the group
   options      pointer to external options  
   optbit       the option bit whose changing is significant, or  
                  zero if none are  
1406    skipassert   TRUE if certain assertions are to be skipped    skipassert   TRUE if certain assertions are to be skipped
1407    
1408  Returns:       pointer to the first significant opcode  Returns:       pointer to the first significant opcode
1409  */  */
1410    
1411  static const uschar*  static const uschar*
1412  first_significant_code(const uschar *code, int *options, int optbit,  first_significant_code(const uschar *code, BOOL skipassert)
   BOOL skipassert)  
1413  {  {
1414  for (;;)  for (;;)
1415    {    {
1416    switch ((int)*code)    switch ((int)*code)
1417      {      {
     case OP_OPT:  
     if (optbit > 0 && ((int)code[1] & optbit) != (*options & optbit))  
       *options = (int)code[1];  
     code += 2;  
     break;  
   
1418      case OP_ASSERT_NOT:      case OP_ASSERT_NOT:
1419      case OP_ASSERTBACK:      case OP_ASSERTBACK:
1420      case OP_ASSERTBACK_NOT:      case OP_ASSERTBACK_NOT:
# Line 1431  and doing the check at the end; a flag s Line 1464  and doing the check at the end; a flag s
1464    
1465  Arguments:  Arguments:
1466    code     points to the start of the pattern (the bracket)    code     points to the start of the pattern (the bracket)
1467    options  the compiling options    utf8     TRUE in UTF-8 mode
1468    atend    TRUE if called when the pattern is complete    atend    TRUE if called when the pattern is complete
1469    cd       the "compile data" structure    cd       the "compile data" structure
1470    
# Line 1442  Returns: the fixed length, Line 1475  Returns: the fixed length,
1475  */  */
1476    
1477  static int  static int
1478  find_fixedlength(uschar *code, int options, BOOL atend, compile_data *cd)  find_fixedlength(uschar *code, BOOL utf8, BOOL atend, compile_data *cd)
1479  {  {
1480  int length = -1;  int length = -1;
1481    
# Line 1459  for (;;) Line 1492  for (;;)
1492    register int op = *cc;    register int op = *cc;
1493    switch (op)    switch (op)
1494      {      {
1495        /* We only need to continue for OP_CBRA (normal capturing bracket) and
1496        OP_BRA (normal non-capturing bracket) because the other variants of these
1497        opcodes are all concerned with unlimited repeated groups, which of course
1498        are not of fixed length. They will cause a -1 response from the default
1499        case of this switch. */
1500    
1501      case OP_CBRA:      case OP_CBRA:
1502      case OP_BRA:      case OP_BRA:
1503      case OP_ONCE:      case OP_ONCE:
1504      case OP_COND:      case OP_COND:
1505      d = find_fixedlength(cc + ((op == OP_CBRA)? 2:0), options, atend, cd);      d = find_fixedlength(cc + ((op == OP_CBRA)? 2:0), utf8, atend, cd);
1506      if (d < 0) return d;      if (d < 0) return d;
1507      branchlength += d;      branchlength += d;
1508      do cc += GET(cc, 1); while (*cc == OP_ALT);      do cc += GET(cc, 1); while (*cc == OP_ALT);
# Line 1472  for (;;) Line 1511  for (;;)
1511    
1512      /* Reached end of a branch; if it's a ket it is the end of a nested      /* Reached end of a branch; if it's a ket it is the end of a nested
1513      call. If it's ALT it is an alternation in a nested call. If it is      call. If it's ALT it is an alternation in a nested call. If it is
1514      END it's the end of the outer call. All can be handled by the same code. */      END it's the end of the outer call. All can be handled by the same code.
1515        Note that we must not include the OP_KETRxxx opcodes here, because they
1516        all imply an unlimited repeat. */
1517    
1518      case OP_ALT:      case OP_ALT:
1519      case OP_KET:      case OP_KET:
     case OP_KETRMAX:  
     case OP_KETRMIN:  
1520      case OP_END:      case OP_END:
1521      if (length < 0) length = branchlength;      if (length < 0) length = branchlength;
1522        else if (length != branchlength) return -1;        else if (length != branchlength) return -1;
# Line 1495  for (;;) Line 1534  for (;;)
1534      cs = ce = (uschar *)cd->start_code + GET(cc, 1);  /* Start subpattern */      cs = ce = (uschar *)cd->start_code + GET(cc, 1);  /* Start subpattern */
1535      do ce += GET(ce, 1); while (*ce == OP_ALT);       /* End subpattern */      do ce += GET(ce, 1); while (*ce == OP_ALT);       /* End subpattern */
1536      if (cc > cs && cc < ce) return -1;                /* Recursion */      if (cc > cs && cc < ce) return -1;                /* Recursion */
1537      d = find_fixedlength(cs + 2, options, atend, cd);      d = find_fixedlength(cs + 2, utf8, atend, cd);
1538      if (d < 0) return d;      if (d < 0) return d;
1539      branchlength += d;      branchlength += d;
1540      cc += 1 + LINK_SIZE;      cc += 1 + LINK_SIZE;
# Line 1518  for (;;) Line 1557  for (;;)
1557      case OP_RREF:      case OP_RREF:
1558      case OP_NRREF:      case OP_NRREF:
1559      case OP_DEF:      case OP_DEF:
     case OP_OPT:  
1560      case OP_CALLOUT:      case OP_CALLOUT:
1561      case OP_SOD:      case OP_SOD:
1562      case OP_SOM:      case OP_SOM:
# Line 1526  for (;;) Line 1564  for (;;)
1564      case OP_EOD:      case OP_EOD:
1565      case OP_EODN:      case OP_EODN:
1566      case OP_CIRC:      case OP_CIRC:
1567        case OP_CIRCM:
1568      case OP_DOLL:      case OP_DOLL:
1569        case OP_DOLLM:
1570      case OP_NOT_WORD_BOUNDARY:      case OP_NOT_WORD_BOUNDARY:
1571      case OP_WORD_BOUNDARY:      case OP_WORD_BOUNDARY:
1572      cc += _pcre_OP_lengths[*cc];      cc += _pcre_OP_lengths[*cc];
# Line 1535  for (;;) Line 1575  for (;;)
1575      /* Handle literal characters */      /* Handle literal characters */
1576    
1577      case OP_CHAR:      case OP_CHAR:
1578      case OP_CHARNC:      case OP_CHARI:
1579      case OP_NOT:      case OP_NOT:
1580        case OP_NOTI:
1581      branchlength++;      branchlength++;
1582      cc += 2;      cc += 2;
1583  #ifdef SUPPORT_UTF8  #ifdef SUPPORT_UTF8
1584      if ((options & PCRE_UTF8) != 0 && cc[-1] >= 0xc0)      if (utf8 && cc[-1] >= 0xc0) cc += _pcre_utf8_table4[cc[-1] & 0x3f];
       cc += _pcre_utf8_table4[cc[-1] & 0x3f];  
1585  #endif  #endif
1586      break;      break;
1587    
# Line 1552  for (;;) Line 1592  for (;;)
1592      branchlength += GET2(cc,1);      branchlength += GET2(cc,1);
1593      cc += 4;      cc += 4;
1594  #ifdef SUPPORT_UTF8  #ifdef SUPPORT_UTF8
1595      if ((options & PCRE_UTF8) != 0 && cc[-1] >= 0xc0)      if (utf8 && cc[-1] >= 0xc0) cc += _pcre_utf8_table4[cc[-1] & 0x3f];
       cc += _pcre_utf8_table4[cc[-1] & 0x3f];  
1596  #endif  #endif
1597      break;      break;
1598    
# Line 1655  _pcre_find_bracket(const uschar *code, B Line 1694  _pcre_find_bracket(const uschar *code, B
1694  for (;;)  for (;;)
1695    {    {
1696    register int c = *code;    register int c = *code;
1697    
1698    if (c == OP_END) return NULL;    if (c == OP_END) return NULL;
1699    
1700    /* XCLASS is used for classes that cannot be represented just by a bit    /* XCLASS is used for classes that cannot be represented just by a bit
# Line 1673  for (;;) Line 1713  for (;;)
1713    
1714    /* Handle capturing bracket */    /* Handle capturing bracket */
1715    
1716    else if (c == OP_CBRA)    else if (c == OP_CBRA || c == OP_SCBRA ||
1717               c == OP_CBRAPOS || c == OP_SCBRAPOS)
1718      {      {
1719      int n = GET2(code, 1+LINK_SIZE);      int n = GET2(code, 1+LINK_SIZE);
1720      if (n == number) return (uschar *)code;      if (n == number) return (uschar *)code;
# Line 1711  for (;;) Line 1752  for (;;)
1752        case OP_MARK:        case OP_MARK:
1753        case OP_PRUNE_ARG:        case OP_PRUNE_ARG:
1754        case OP_SKIP_ARG:        case OP_SKIP_ARG:
       case OP_THEN_ARG:  
1755        code += code[1];        code += code[1];
1756        break;        break;
1757    
1758          case OP_THEN_ARG:
1759          code += code[1+LINK_SIZE];
1760          break;
1761        }        }
1762    
1763      /* Add in the fixed length from the table */      /* Add in the fixed length from the table */
# Line 1728  for (;;) Line 1772  for (;;)
1772      if (utf8) switch(c)      if (utf8) switch(c)
1773        {        {
1774        case OP_CHAR:        case OP_CHAR:
1775        case OP_CHARNC:        case OP_CHARI:
1776        case OP_EXACT:        case OP_EXACT:
1777          case OP_EXACTI:
1778        case OP_UPTO:        case OP_UPTO:
1779          case OP_UPTOI:
1780        case OP_MINUPTO:        case OP_MINUPTO:
1781          case OP_MINUPTOI:
1782        case OP_POSUPTO:        case OP_POSUPTO:
1783          case OP_POSUPTOI:
1784        case OP_STAR:        case OP_STAR:
1785          case OP_STARI:
1786        case OP_MINSTAR:        case OP_MINSTAR:
1787          case OP_MINSTARI:
1788        case OP_POSSTAR:        case OP_POSSTAR:
1789          case OP_POSSTARI:
1790        case OP_PLUS:        case OP_PLUS:
1791          case OP_PLUSI:
1792        case OP_MINPLUS:        case OP_MINPLUS:
1793          case OP_MINPLUSI:
1794        case OP_POSPLUS:        case OP_POSPLUS:
1795          case OP_POSPLUSI:
1796        case OP_QUERY:        case OP_QUERY:
1797          case OP_QUERYI:
1798        case OP_MINQUERY:        case OP_MINQUERY:
1799          case OP_MINQUERYI:
1800        case OP_POSQUERY:        case OP_POSQUERY:
1801          case OP_POSQUERYI:
1802        if (code[-1] >= 0xc0) code += _pcre_utf8_table4[code[-1] & 0x3f];        if (code[-1] >= 0xc0) code += _pcre_utf8_table4[code[-1] & 0x3f];
1803        break;        break;
1804        }        }
# Line 1814  for (;;) Line 1871  for (;;)
1871        case OP_MARK:        case OP_MARK:
1872        case OP_PRUNE_ARG:        case OP_PRUNE_ARG:
1873        case OP_SKIP_ARG:        case OP_SKIP_ARG:
       case OP_THEN_ARG:  
1874        code += code[1];        code += code[1];
1875        break;        break;
1876    
1877          case OP_THEN_ARG:
1878          code += code[1+LINK_SIZE];
1879          break;
1880        }        }
1881    
1882      /* Add in the fixed length from the table */      /* Add in the fixed length from the table */
# Line 1831  for (;;) Line 1891  for (;;)
1891      if (utf8) switch(c)      if (utf8) switch(c)
1892        {        {
1893        case OP_CHAR:        case OP_CHAR:
1894        case OP_CHARNC:        case OP_CHARI:
1895        case OP_EXACT:        case OP_EXACT:
1896          case OP_EXACTI:
1897        case OP_UPTO:        case OP_UPTO:
1898          case OP_UPTOI:
1899        case OP_MINUPTO:        case OP_MINUPTO:
1900          case OP_MINUPTOI:
1901        case OP_POSUPTO:        case OP_POSUPTO:
1902          case OP_POSUPTOI:
1903        case OP_STAR:        case OP_STAR:
1904          case OP_STARI:
1905        case OP_MINSTAR:        case OP_MINSTAR:
1906          case OP_MINSTARI:
1907        case OP_POSSTAR:        case OP_POSSTAR:
1908          case OP_POSSTARI:
1909        case OP_PLUS:        case OP_PLUS:
1910          case OP_PLUSI:
1911        case OP_MINPLUS:        case OP_MINPLUS:
1912          case OP_MINPLUSI:
1913        case OP_POSPLUS:        case OP_POSPLUS:
1914          case OP_POSPLUSI:
1915        case OP_QUERY:        case OP_QUERY:
1916          case OP_QUERYI:
1917        case OP_MINQUERY:        case OP_MINQUERY:
1918          case OP_MINQUERYI:
1919        case OP_POSQUERY:        case OP_POSQUERY:
1920          case OP_POSQUERYI:
1921        if (code[-1] >= 0xc0) code += _pcre_utf8_table4[code[-1] & 0x3f];        if (code[-1] >= 0xc0) code += _pcre_utf8_table4[code[-1] & 0x3f];
1922        break;        break;
1923        }        }
# Line 1883  could_be_empty_branch(const uschar *code Line 1956  could_be_empty_branch(const uschar *code
1956    compile_data *cd)    compile_data *cd)
1957  {  {
1958  register int c;  register int c;
1959  for (code = first_significant_code(code + _pcre_OP_lengths[*code], NULL, 0, TRUE);  for (code = first_significant_code(code + _pcre_OP_lengths[*code], TRUE);
1960       code < endcode;       code < endcode;
1961       code = first_significant_code(code + _pcre_OP_lengths[c], NULL, 0, TRUE))       code = first_significant_code(code + _pcre_OP_lengths[c], TRUE))
1962    {    {
1963    const uschar *ccode;    const uschar *ccode;
1964    
# Line 1901  for (code = first_significant_code(code Line 1974  for (code = first_significant_code(code
1974      continue;      continue;
1975      }      }
1976    
   /* Groups with zero repeats can of course be empty; skip them. */  
   
   if (c == OP_BRAZERO || c == OP_BRAMINZERO || c == OP_SKIPZERO)  
     {  
     code += _pcre_OP_lengths[c];  
     do code += GET(code, 1); while (*code == OP_ALT);  
     c = *code;  
     continue;  
     }  
   
1977    /* For a recursion/subroutine call, if its end has been reached, which    /* For a recursion/subroutine call, if its end has been reached, which
1978    implies a subroutine call, we can scan it. */    implies a subroutine call, we can scan it. */
1979    
# Line 1933  for (code = first_significant_code(code Line 1996  for (code = first_significant_code(code
1996      continue;      continue;
1997      }      }
1998    
1999      /* Groups with zero repeats can of course be empty; skip them. */
2000    
2001      if (c == OP_BRAZERO || c == OP_BRAMINZERO || c == OP_SKIPZERO ||
2002          c == OP_BRAPOSZERO)
2003        {
2004        code += _pcre_OP_lengths[c];
2005        do code += GET(code, 1); while (*code == OP_ALT);
2006        c = *code;
2007        continue;
2008        }
2009    
2010      /* A nested group that is already marked as "could be empty" can just be
2011      skipped. */
2012    
2013      if (c == OP_SBRA  || c == OP_SBRAPOS ||
2014          c == OP_SCBRA || c == OP_SCBRAPOS)
2015        {
2016        do code += GET(code, 1); while (*code == OP_ALT);
2017        c = *code;
2018        continue;
2019        }
2020    
2021    /* For other groups, scan the branches. */    /* For other groups, scan the branches. */
2022    
2023    if (c == OP_BRA || c == OP_CBRA || c == OP_ONCE || c == OP_COND)    if (c == OP_BRA  || c == OP_BRAPOS ||
2024          c == OP_CBRA || c == OP_CBRAPOS ||
2025          c == OP_ONCE || c == OP_COND)
2026      {      {
2027      BOOL empty_branch;      BOOL empty_branch;
2028      if (GET(code, 1) == 0) return TRUE;    /* Hit unclosed bracket */      if (GET(code, 1) == 0) return TRUE;    /* Hit unclosed bracket */
# Line 2022  for (code = first_significant_code(code Line 2109  for (code = first_significant_code(code
2109      case OP_ALLANY:      case OP_ALLANY:
2110      case OP_ANYBYTE:      case OP_ANYBYTE:
2111      case OP_CHAR:      case OP_CHAR:
2112      case OP_CHARNC:      case OP_CHARI:
2113      case OP_NOT:      case OP_NOT:
2114        case OP_NOTI:
2115      case OP_PLUS:      case OP_PLUS:
2116      case OP_MINPLUS:      case OP_MINPLUS:
2117      case OP_POSPLUS:      case OP_POSPLUS:
# Line 2063  for (code = first_significant_code(code Line 2151  for (code = first_significant_code(code
2151      case OP_KET:      case OP_KET:
2152      case OP_KETRMAX:      case OP_KETRMAX:
2153      case OP_KETRMIN:      case OP_KETRMIN:
2154        case OP_KETRPOS:
2155      case OP_ALT:      case OP_ALT:
2156      return TRUE;      return TRUE;
2157    
# Line 2071  for (code = first_significant_code(code Line 2160  for (code = first_significant_code(code
2160    
2161  #ifdef SUPPORT_UTF8  #ifdef SUPPORT_UTF8
2162      case OP_STAR:      case OP_STAR:
2163        case OP_STARI:
2164      case OP_MINSTAR:      case OP_MINSTAR:
2165        case OP_MINSTARI:
2166      case OP_POSSTAR:      case OP_POSSTAR:
2167        case OP_POSSTARI:
2168      case OP_QUERY:      case OP_QUERY:
2169        case OP_QUERYI:
2170      case OP_MINQUERY:      case OP_MINQUERY:
2171        case OP_MINQUERYI:
2172      case OP_POSQUERY:      case OP_POSQUERY:
2173        case OP_POSQUERYI:
2174      if (utf8 && code[1] >= 0xc0) code += _pcre_utf8_table4[code[1] & 0x3f];      if (utf8 && code[1] >= 0xc0) code += _pcre_utf8_table4[code[1] & 0x3f];
2175      break;      break;
2176    
2177      case OP_UPTO:      case OP_UPTO:
2178        case OP_UPTOI:
2179      case OP_MINUPTO:      case OP_MINUPTO:
2180        case OP_MINUPTOI:
2181      case OP_POSUPTO:      case OP_POSUPTO:
2182        case OP_POSUPTOI:
2183      if (utf8 && code[3] >= 0xc0) code += _pcre_utf8_table4[code[3] & 0x3f];      if (utf8 && code[3] >= 0xc0) code += _pcre_utf8_table4[code[3] & 0x3f];
2184      break;      break;
2185  #endif  #endif
# Line 2092  for (code = first_significant_code(code Line 2190  for (code = first_significant_code(code
2190      case OP_MARK:      case OP_MARK:
2191      case OP_PRUNE_ARG:      case OP_PRUNE_ARG:
2192      case OP_SKIP_ARG:      case OP_SKIP_ARG:
     case OP_THEN_ARG:  
2193      code += code[1];      code += code[1];
2194      break;      break;
2195    
2196        case OP_THEN_ARG:
2197        code += code[1+LINK_SIZE];
2198        break;
2199    
2200      /* None of the remaining opcodes are required to match a character. */      /* None of the remaining opcodes are required to match a character. */
2201    
2202      default:      default:
# Line 2392  for (++c; c <= d; c++) Line 2493  for (++c; c <= d; c++)
2493    
2494  return TRUE;  return TRUE;
2495  }  }
2496    
2497    
2498    
2499    /*************************************************
2500    *        Check a character and a property        *
2501    *************************************************/
2502    
2503    /* This function is called by check_auto_possessive() when a property item
2504    is adjacent to a fixed character.
2505    
2506    Arguments:
2507      c            the character
2508      ptype        the property type
2509      pdata        the data for the type
2510      negated      TRUE if it's a negated property (\P or \p{^)
2511    
2512    Returns:       TRUE if auto-possessifying is OK
2513    */
2514    
2515    static BOOL
2516    check_char_prop(int c, int ptype, int pdata, BOOL negated)
2517    {
2518    const ucd_record *prop = GET_UCD(c);
2519    switch(ptype)
2520      {
2521      case PT_LAMP:
2522      return (prop->chartype == ucp_Lu ||
2523              prop->chartype == ucp_Ll ||
2524              prop->chartype == ucp_Lt) == negated;
2525    
2526      case PT_GC:
2527      return (pdata == _pcre_ucp_gentype[prop->chartype]) == negated;
2528    
2529      case PT_PC:
2530      return (pdata == prop->chartype) == negated;
2531    
2532      case PT_SC:
2533      return (pdata == prop->script) == negated;
2534    
2535      /* These are specials */
2536    
2537      case PT_ALNUM:
2538      return (_pcre_ucp_gentype[prop->chartype] == ucp_L ||
2539              _pcre_ucp_gentype[prop->chartype] == ucp_N) == negated;
2540    
2541      case PT_SPACE:    /* Perl space */
2542      return (_pcre_ucp_gentype[prop->chartype] == ucp_Z ||
2543              c == CHAR_HT || c == CHAR_NL || c == CHAR_FF || c == CHAR_CR)
2544              == negated;
2545    
2546      case PT_PXSPACE:  /* POSIX space */
2547      return (_pcre_ucp_gentype[prop->chartype] == ucp_Z ||
2548              c == CHAR_HT || c == CHAR_NL || c == CHAR_VT ||
2549              c == CHAR_FF || c == CHAR_CR)
2550              == negated;
2551    
2552      case PT_WORD:
2553      return (_pcre_ucp_gentype[prop->chartype] == ucp_L ||
2554              _pcre_ucp_gentype[prop->chartype] == ucp_N ||
2555              c == CHAR_UNDERSCORE) == negated;
2556      }
2557    return FALSE;
2558    }
2559  #endif  /* SUPPORT_UCP */  #endif  /* SUPPORT_UCP */
2560    
2561    
# Line 2405  whether the next thing could possibly ma Line 2569  whether the next thing could possibly ma
2569  sense to automatically possessify the repeated item.  sense to automatically possessify the repeated item.
2570    
2571  Arguments:  Arguments:
2572    op_code       the repeated op code    previous      pointer to the repeated opcode
   this          data for this item, depends on the opcode  
2573    utf8          TRUE in UTF-8 mode    utf8          TRUE in UTF-8 mode
   utf8_char     used for utf8 character bytes, NULL if not relevant  
2574    ptr           next character in pattern    ptr           next character in pattern
2575    options       options bits    options       options bits
2576    cd            contains pointers to tables etc.    cd            contains pointers to tables etc.
# Line 2417  Returns: TRUE if possessifying is Line 2579  Returns: TRUE if possessifying is
2579  */  */
2580    
2581  static BOOL  static BOOL
2582  check_auto_possessive(int op_code, int item, BOOL utf8, uschar *utf8_char,  check_auto_possessive(const uschar *previous, BOOL utf8, const uschar *ptr,
2583    const uschar *ptr, int options, compile_data *cd)    int options, compile_data *cd)
2584  {  {
2585  int next;  int c, next;
2586    int op_code = *previous++;
2587    
2588  /* Skip whitespace and comments in extended mode */  /* Skip whitespace and comments in extended mode */
2589    
# Line 2431  if ((options & PCRE_EXTENDED) != 0) Line 2594  if ((options & PCRE_EXTENDED) != 0)
2594      while ((cd->ctypes[*ptr] & ctype_space) != 0) ptr++;      while ((cd->ctypes[*ptr] & ctype_space) != 0) ptr++;
2595      if (*ptr == CHAR_NUMBER_SIGN)      if (*ptr == CHAR_NUMBER_SIGN)
2596        {        {
2597        while (*(++ptr) != 0)        ptr++;
2598          while (*ptr != 0)
2599            {
2600          if (IS_NEWLINE(ptr)) { ptr += cd->nllen; break; }          if (IS_NEWLINE(ptr)) { ptr += cd->nllen; break; }
2601            ptr++;
2602    #ifdef SUPPORT_UTF8
2603            if (utf8) while ((*ptr & 0xc0) == 0x80) ptr++;
2604    #endif
2605            }
2606        }        }
2607      else break;      else break;
2608      }      }
# Line 2468  if ((options & PCRE_EXTENDED) != 0) Line 2638  if ((options & PCRE_EXTENDED) != 0)
2638      while ((cd->ctypes[*ptr] & ctype_space) != 0) ptr++;      while ((cd->ctypes[*ptr] & ctype_space) != 0) ptr++;
2639      if (*ptr == CHAR_NUMBER_SIGN)      if (*ptr == CHAR_NUMBER_SIGN)
2640        {        {
2641        while (*(++ptr) != 0)        ptr++;
2642          while (*ptr != 0)
2643            {
2644          if (IS_NEWLINE(ptr)) { ptr += cd->nllen; break; }          if (IS_NEWLINE(ptr)) { ptr += cd->nllen; break; }
2645            ptr++;
2646    #ifdef SUPPORT_UTF8
2647            if (utf8) while ((*ptr & 0xc0) == 0x80) ptr++;
2648    #endif
2649            }
2650        }        }
2651      else break;      else break;
2652      }      }
# Line 2481  if (*ptr == CHAR_ASTERISK || *ptr == CHA Line 2658  if (*ptr == CHAR_ASTERISK || *ptr == CHA
2658    strncmp((char *)ptr, STR_LEFT_CURLY_BRACKET STR_0 STR_COMMA, 3) == 0)    strncmp((char *)ptr, STR_LEFT_CURLY_BRACKET STR_0 STR_COMMA, 3) == 0)
2659      return FALSE;      return FALSE;
2660    
2661  /* Now compare the next item with the previous opcode. If the previous is a  /* Now compare the next item with the previous opcode. First, handle cases when
2662  positive single character match, "item" either contains the character or, if  the next item is a character. */
 "item" is greater than 127 in utf8 mode, the character's bytes are in  
 utf8_char. */  
   
   
 /* Handle cases when the next item is a character. */  
2663    
2664  if (next >= 0) switch(op_code)  if (next >= 0) switch(op_code)
2665    {    {
2666    case OP_CHAR:    case OP_CHAR:
2667  #ifdef SUPPORT_UTF8  #ifdef SUPPORT_UTF8
2668    if (utf8 && item > 127) { GETCHAR(item, utf8_char); }    GETCHARTEST(c, previous);
2669  #else  #else
2670    (void)(utf8_char);  /* Keep compiler happy by referencing function argument */    c = *previous;
2671  #endif  #endif
2672    return item != next;    return c != next;
2673    
2674    /* For CHARNC (caseless character) we must check the other case. If we have    /* For CHARI (caseless character) we must check the other case. If we have
2675    Unicode property support, we can use it to test the other case of    Unicode property support, we can use it to test the other case of
2676    high-valued characters. */    high-valued characters. */
2677    
2678    case OP_CHARNC:    case OP_CHARI:
2679  #ifdef SUPPORT_UTF8  #ifdef SUPPORT_UTF8
2680    if (utf8 && item > 127) { GETCHAR(item, utf8_char); }    GETCHARTEST(c, previous);
2681    #else
2682      c = *previous;
2683  #endif  #endif
2684    if (item == next) return FALSE;    if (c == next) return FALSE;
2685  #ifdef SUPPORT_UTF8  #ifdef SUPPORT_UTF8
2686    if (utf8)    if (utf8)
2687      {      {
# Line 2518  if (next >= 0) switch(op_code) Line 2692  if (next >= 0) switch(op_code)
2692  #else  #else
2693      othercase = NOTACHAR;      othercase = NOTACHAR;
2694  #endif  #endif
2695      return (unsigned int)item != othercase;      return (unsigned int)c != othercase;
2696      }      }
2697    else    else
2698  #endif  /* SUPPORT_UTF8 */  #endif  /* SUPPORT_UTF8 */
2699    return (item != cd->fcc[next]);  /* Non-UTF-8 mode */    return (c != cd->fcc[next]);  /* Non-UTF-8 mode */
2700    
2701    /* For OP_NOT, "item" must be a single-byte character. */    /* For OP_NOT and OP_NOTI, the data is always a single-byte character. These
2702      opcodes are not used for multi-byte characters, because they are coded using
2703      an XCLASS instead. */
2704    
2705    case OP_NOT:    case OP_NOT:
2706    if (item == next) return TRUE;    return (c = *previous) == next;
2707    if ((options & PCRE_CASELESS) == 0) return FALSE;  
2708      case OP_NOTI:
2709      if ((c = *previous) == next) return TRUE;
2710  #ifdef SUPPORT_UTF8  #ifdef SUPPORT_UTF8
2711    if (utf8)    if (utf8)
2712      {      {
# Line 2539  if (next >= 0) switch(op_code) Line 2717  if (next >= 0) switch(op_code)
2717  #else  #else
2718      othercase = NOTACHAR;      othercase = NOTACHAR;
2719  #endif  #endif
2720      return (unsigned int)item == othercase;      return (unsigned int)c == othercase;
2721      }      }
2722    else    else
2723  #endif  /* SUPPORT_UTF8 */  #endif  /* SUPPORT_UTF8 */
2724    return (item == cd->fcc[next]);  /* Non-UTF-8 mode */    return (c == cd->fcc[next]);  /* Non-UTF-8 mode */
2725    
2726    /* Note that OP_DIGIT etc. are generated only when PCRE_UCP is *not* set.    /* Note that OP_DIGIT etc. are generated only when PCRE_UCP is *not* set.
2727    When it is set, \d etc. are converted into OP_(NOT_)PROP codes. */    When it is set, \d etc. are converted into OP_(NOT_)PROP codes. */
2728    
2729    case OP_DIGIT:    case OP_DIGIT:
2730    return next > 127 || (cd->ctypes[next] & ctype_digit) == 0;    return next > 127 || (cd->ctypes[next] & ctype_digit) == 0;
# Line 2611  if (next >= 0) switch(op_code) Line 2789  if (next >= 0) switch(op_code)
2789      return op_code != OP_NOT_VSPACE;      return op_code != OP_NOT_VSPACE;
2790      }      }
2791    
2792    #ifdef SUPPORT_UCP
2793      case OP_PROP:
2794      return check_char_prop(next, previous[0], previous[1], FALSE);
2795    
2796      case OP_NOTPROP:
2797      return check_char_prop(next, previous[0], previous[1], TRUE);
2798    #endif
2799    
2800    default:    default:
2801    return FALSE;    return FALSE;
2802    }    }
2803    
2804    
2805  /* Handle the case when the next item is \d, \s, etc. Note that when PCRE_UCP  /* Handle the case when the next item is \d, \s, etc. Note that when PCRE_UCP
2806  is set, \d turns into ESC_du rather than ESC_d, etc., so ESC_d etc. are  is set, \d turns into ESC_du rather than ESC_d, etc., so ESC_d etc. are
2807  generated only when PCRE_UCP is *not* set, that is, when only ASCII  generated only when PCRE_UCP is *not* set, that is, when only ASCII
2808  characteristics are recognized. */  characteristics are recognized. Similarly, the opcodes OP_DIGIT etc. are
2809    replaced by OP_PROP codes when PCRE_UCP is set. */
2810    
2811  switch(op_code)  switch(op_code)
2812    {    {
2813    case OP_CHAR:    case OP_CHAR:
2814    case OP_CHARNC:    case OP_CHARI:
2815  #ifdef SUPPORT_UTF8  #ifdef SUPPORT_UTF8
2816    if (utf8 && item > 127) { GETCHAR(item, utf8_char); }    GETCHARTEST(c, previous);
2817    #else
2818      c = *previous;
2819  #endif  #endif
2820    switch(-next)    switch(-next)
2821      {      {
2822      case ESC_d:      case ESC_d:
2823      return item > 127 || (cd->ctypes[item] & ctype_digit) == 0;      return c > 127 || (cd->ctypes[c] & ctype_digit) == 0;
2824    
2825      case ESC_D:      case ESC_D:
2826      return item <= 127 && (cd->ctypes[item] & ctype_digit) != 0;      return c <= 127 && (cd->ctypes[c] & ctype_digit) != 0;
2827    
2828      case ESC_s:      case ESC_s:
2829      return item > 127 || (cd->ctypes[item] & ctype_space) == 0;      return c > 127 || (cd->ctypes[c] & ctype_space) == 0;
2830    
2831      case ESC_S:      case ESC_S:
2832      return item <= 127 && (cd->ctypes[item] & ctype_space) != 0;      return c <= 127 && (cd->ctypes[c] & ctype_space) != 0;
2833    
2834      case ESC_w:      case ESC_w:
2835      return item > 127 || (cd->ctypes[item] & ctype_word) == 0;      return c > 127 || (cd->ctypes[c] & ctype_word) == 0;
2836    
2837      case ESC_W:      case ESC_W:
2838      return item <= 127 && (cd->ctypes[item] & ctype_word) != 0;      return c <= 127 && (cd->ctypes[c] & ctype_word) != 0;
2839    
2840      case ESC_h:      case ESC_h:
2841      case ESC_H:      case ESC_H:
2842      switch(item)      switch(c)
2843        {        {
2844        case 0x09:        case 0x09:
2845        case 0x20:        case 0x20:
# Line 2678  switch(op_code) Line 2867  switch(op_code)
2867    
2868      case ESC_v:      case ESC_v:
2869      case ESC_V:      case ESC_V:
2870      switch(item)      switch(c)
2871        {        {
2872        case 0x0a:        case 0x0a:
2873        case 0x0b:        case 0x0b:
# Line 2692  switch(op_code) Line 2881  switch(op_code)
2881        return -next == ESC_v;        return -next == ESC_v;
2882        }        }
2883    
2884        /* When PCRE_UCP is set, these values get generated for \d etc. Find
2885        their substitutions and process them. The result will always be either
2886        -ESC_p or -ESC_P. Then fall through to process those values. */
2887    
2888    #ifdef SUPPORT_UCP
2889        case ESC_du:
2890        case ESC_DU:
2891        case ESC_wu:
2892        case ESC_WU:
2893        case ESC_su:
2894        case ESC_SU:
2895          {
2896          int temperrorcode = 0;
2897          ptr = substitutes[-next - ESC_DU];
2898          next = check_escape(&ptr, &temperrorcode, 0, options, FALSE);
2899          if (temperrorcode != 0) return FALSE;
2900          ptr++;    /* For compatibility */
2901          }
2902        /* Fall through */
2903    
2904        case ESC_p:
2905        case ESC_P:
2906          {
2907          int ptype, pdata, errorcodeptr;
2908          BOOL negated;
2909    
2910          ptr--;      /* Make ptr point at the p or P */
2911          ptype = get_ucp(&ptr, &negated, &pdata, &errorcodeptr);
2912          if (ptype < 0) return FALSE;
2913          ptr++;      /* Point past the final curly ket */
2914    
2915          /* If the property item is optional, we have to give up. (When generated
2916          from \d etc by PCRE_UCP, this test will have been applied much earlier,
2917          to the original \d etc. At this point, ptr will point to a zero byte. */
2918    
2919          if (*ptr == CHAR_ASTERISK || *ptr == CHAR_QUESTION_MARK ||
2920            strncmp((char *)ptr, STR_LEFT_CURLY_BRACKET STR_0 STR_COMMA, 3) == 0)
2921              return FALSE;
2922    
2923          /* Do the property check. */
2924    
2925          return check_char_prop(c, ptype, pdata, (next == -ESC_P) != negated);
2926          }
2927    #endif
2928    
2929      default:      default:
2930      return FALSE;      return FALSE;
2931      }      }
2932    
2933      /* In principle, support for Unicode properties should be integrated here as
2934      well. It means re-organizing the above code so as to get hold of the property
2935      values before switching on the op-code. However, I wonder how many patterns
2936      combine ASCII \d etc with Unicode properties? (Note that if PCRE_UCP is set,
2937      these op-codes are never generated.) */
2938    
2939    case OP_DIGIT:    case OP_DIGIT:
2940    return next == -ESC_D || next == -ESC_s || next == -ESC_W ||    return next == -ESC_D || next == -ESC_s || next == -ESC_W ||
2941           next == -ESC_h || next == -ESC_v || next == -ESC_R;           next == -ESC_h || next == -ESC_v || next == -ESC_R;
# Line 2710  switch(op_code) Line 2950  switch(op_code)
2950    return next == -ESC_s || next == -ESC_h || next == -ESC_v;    return next == -ESC_s || next == -ESC_h || next == -ESC_v;
2951    
2952    case OP_HSPACE:    case OP_HSPACE:
2953    return next == -ESC_S || next == -ESC_H || next == -ESC_d ||    return next == -ESC_S || next == -ESC_H || next == -ESC_d ||
2954           next == -ESC_w || next == -ESC_v || next == -ESC_R;           next == -ESC_w || next == -ESC_v || next == -ESC_R;
2955    
2956    case OP_NOT_HSPACE:    case OP_NOT_HSPACE:
2957    return next == -ESC_h;    return next == -ESC_h;
2958    
2959    /* Can't have \S in here because VT matches \S (Perl anomaly) */    /* Can't have \S in here because VT matches \S (Perl anomaly) */
2960    case OP_ANYNL:    case OP_ANYNL:
2961    case OP_VSPACE:    case OP_VSPACE:
2962    return next == -ESC_V || next == -ESC_d || next == -ESC_w;    return next == -ESC_V || next == -ESC_d || next == -ESC_w;
2963    
# Line 2725  switch(op_code) Line 2965  switch(op_code)
2965    return next == -ESC_v || next == -ESC_R;    return next == -ESC_v || next == -ESC_R;
2966    
2967    case OP_WORDCHAR:    case OP_WORDCHAR:
2968    return next == -ESC_W || next == -ESC_s || next == -ESC_h ||    return next == -ESC_W || next == -ESC_s || next == -ESC_h ||
2969           next == -ESC_v || next == -ESC_R;           next == -ESC_v || next == -ESC_R;
2970    
2971    case OP_NOT_WORDCHAR:    case OP_NOT_WORDCHAR:
# Line 2861  for (;; ptr++) Line 3101  for (;; ptr++)
3101    
3102    c = *ptr;    c = *ptr;
3103    
3104    /* If we are at the end of a nested substitution, revert to the outer level    /* If we are at the end of a nested substitution, revert to the outer level
3105    string. Nesting only happens one level deep. */    string. Nesting only happens one level deep. */
3106    
3107    if (c == 0 && nestptr != NULL)    if (c == 0 && nestptr != NULL)
# Line 2983  for (;; ptr++) Line 3223  for (;; ptr++)
3223      if ((cd->ctypes[c] & ctype_space) != 0) continue;      if ((cd->ctypes[c] & ctype_space) != 0) continue;
3224      if (c == CHAR_NUMBER_SIGN)      if (c == CHAR_NUMBER_SIGN)
3225        {        {
3226        while (*(++ptr) != 0)        ptr++;
3227          while (*ptr != 0)
3228          {          {
3229          if (IS_NEWLINE(ptr)) { ptr += cd->nllen - 1; break; }          if (IS_NEWLINE(ptr)) { ptr += cd->nllen - 1; break; }
3230            ptr++;
3231    #ifdef SUPPORT_UTF8
3232            if (utf8) while ((*ptr & 0xc0) == 0x80) ptr++;
3233    #endif
3234          }          }
3235        if (*ptr != 0) continue;        if (*ptr != 0) continue;
3236    
# Line 3030  for (;; ptr++) Line 3275  for (;; ptr++)
3275      the setting of any following char as a first character. */      the setting of any following char as a first character. */
3276    
3277      case CHAR_CIRCUMFLEX_ACCENT:      case CHAR_CIRCUMFLEX_ACCENT:
3278        previous = NULL;
3279      if ((options & PCRE_MULTILINE) != 0)      if ((options & PCRE_MULTILINE) != 0)
3280        {        {
3281        if (firstbyte == REQ_UNSET) firstbyte = REQ_NONE;        if (firstbyte == REQ_UNSET) firstbyte = REQ_NONE;
3282          *code++ = OP_CIRCM;
3283        }        }
3284      previous = NULL;      else *code++ = OP_CIRC;
     *code++ = OP_CIRC;  
3285      break;      break;
3286    
3287      case CHAR_DOLLAR_SIGN:      case CHAR_DOLLAR_SIGN:
3288      previous = NULL;      previous = NULL;
3289      *code++ = OP_DOLL;      *code++ = ((options & PCRE_MULTILINE) != 0)? OP_DOLLM : OP_DOLL;
3290      break;      break;
3291    
3292      /* There can never be a first char if '.' is first, whatever happens about      /* There can never be a first char if '.' is first, whatever happens about
# Line 3168  for (;; ptr++) Line 3414  for (;; ptr++)
3414          {                           /* Braces are required because the */          {                           /* Braces are required because the */
3415          GETCHARLEN(c, ptr, ptr);    /* macro generates multiple statements */          GETCHARLEN(c, ptr, ptr);    /* macro generates multiple statements */
3416          }          }
3417    
3418        /* In the pre-compile phase, accumulate the length of any UTF-8 extra        /* In the pre-compile phase, accumulate the length of any UTF-8 extra
3419        data and reset the pointer. This is so that very large classes that        data and reset the pointer. This is so that very large classes that
3420        contain a zillion UTF-8 characters no longer overwrite the work space        contain a zillion UTF-8 characters no longer overwrite the work space
# Line 3237  for (;; ptr++) Line 3483  for (;; ptr++)
3483    
3484          if ((options & PCRE_CASELESS) != 0 && posix_class <= 2)          if ((options & PCRE_CASELESS) != 0 && posix_class <= 2)
3485            posix_class = 0;            posix_class = 0;
3486    
3487          /* When PCRE_UCP is set, some of the POSIX classes are converted to          /* When PCRE_UCP is set, some of the POSIX classes are converted to
3488          different escape sequences that use Unicode properties. */          different escape sequences that use Unicode properties. */
3489    
3490  #ifdef SUPPORT_UCP  #ifdef SUPPORT_UCP
3491          if ((options & PCRE_UCP) != 0)          if ((options & PCRE_UCP) != 0)
3492            {            {
3493            int pc = posix_class + ((local_negate)? POSIX_SUBSIZE/2 : 0);            int pc = posix_class + ((local_negate)? POSIX_SUBSIZE/2 : 0);
3494            if (posix_substitutes[pc] != NULL)            if (posix_substitutes[pc] != NULL)
3495              {              {
3496              nestptr = tempptr + 1;              nestptr = tempptr + 1;
3497              ptr = posix_substitutes[pc] - 1;              ptr = posix_substitutes[pc] - 1;
3498              continue;              continue;
3499              }              }
3500            }            }
3501  #endif  #endif
3502          /* In the non-UCP case, we build the bit map for the POSIX class in a          /* In the non-UCP case, we build the bit map for the POSIX class in a
3503          chunk of local store because we may be adding and subtracting from it,          chunk of local store because we may be adding and subtracting from it,
3504          and we don't want to subtract bits that may be in the main map already.          and we don't want to subtract bits that may be in the main map already.
# Line 3339  for (;; ptr++) Line 3585  for (;; ptr++)
3585              case ESC_SU:              case ESC_SU:
3586              nestptr = ptr;              nestptr = ptr;
3587              ptr = substitutes[-c - ESC_DU] - 1;  /* Just before substitute */              ptr = substitutes[-c - ESC_DU] - 1;  /* Just before substitute */
3588              class_charcount -= 2;                /* Undo! */              class_charcount -= 2;                /* Undo! */
3589              continue;              continue;
3590  #endif  #endif
3591              case ESC_d:              case ESC_d:
# Line 3360  for (;; ptr++) Line 3606  for (;; ptr++)
3606              for (c = 0; c < 32; c++) classbits[c] |= ~cbits[c+cbit_word];              for (c = 0; c < 32; c++) classbits[c] |= ~cbits[c+cbit_word];
3607              continue;              continue;
3608    
3609                /* Perl 5.004 onwards omits VT from \s, but we must preserve it
3610                if it was previously set by something earlier in the character
3611                class. */
3612    
3613              case ESC_s:              case ESC_s:
3614              for (c = 0; c < 32; c++) classbits[c] |= cbits[c+cbit_space];              classbits[0] |= cbits[cbit_space];
3615              classbits[1] &= ~0x08;   /* Perl 5.004 onwards omits VT from \s */              classbits[1] |= cbits[cbit_space+1] & ~0x08;
3616                for (c = 2; c < 32; c++) classbits[c] |= cbits[c+cbit_space];
3617              continue;              continue;
3618    
3619              case ESC_S:              case ESC_S:
# Line 3781  for (;; ptr++) Line 4032  for (;; ptr++)
4032    
4033      In UTF-8 mode, we can optimize the negative case only if there were no      In UTF-8 mode, we can optimize the negative case only if there were no
4034      characters >= 128 because OP_NOT and the related opcodes like OP_NOTSTAR      characters >= 128 because OP_NOT and the related opcodes like OP_NOTSTAR
4035      operate on single-bytes only. This is an historical hangover. Maybe one day      operate on single-bytes characters only. This is an historical hangover.
4036      we can tidy these opcodes to handle multi-byte characters.      Maybe one day we can tidy these opcodes to handle multi-byte characters.
4037    
4038      The optimization throws away the bit map. We turn the item into a      The optimization throws away the bit map. We turn the item into a
4039      1-character OP_CHAR[NC] if it's positive, or OP_NOT if it's negative. Note      1-character OP_CHAR[I] if it's positive, or OP_NOT[I] if it's negative.
4040      that OP_NOT does not support multibyte characters. In the positive case, it      Note that OP_NOT[I] does not support multibyte characters. In the positive
4041      can cause firstbyte to be set. Otherwise, there can be no first char if      case, it can cause firstbyte to be set. Otherwise, there can be no first
4042      this item is first, whatever repeat count may follow. In the case of      char if this item is first, whatever repeat count may follow. In the case
4043      reqbyte, save the previous value for reinstating. */      of reqbyte, save the previous value for reinstating. */
4044    
4045  #ifdef SUPPORT_UTF8  #ifdef SUPPORT_UTF8
4046      if (class_charcount == 1 && !class_utf8 &&      if (class_charcount == 1 && !class_utf8 &&
4047        (!utf8 || !negate_class || class_lastchar < 128))        (!utf8 || !negate_class || class_lastchar < 128))
# Line 3800  for (;; ptr++) Line 4051  for (;; ptr++)
4051        {        {
4052        zeroreqbyte = reqbyte;        zeroreqbyte = reqbyte;
4053    
4054        /* The OP_NOT opcode works on one-byte characters only. */        /* The OP_NOT[I] opcodes work on one-byte characters only. */
4055    
4056        if (negate_class)        if (negate_class)
4057          {          {
4058          if (firstbyte == REQ_UNSET) firstbyte = REQ_NONE;          if (firstbyte == REQ_UNSET) firstbyte = REQ_NONE;
4059          zerofirstbyte = firstbyte;          zerofirstbyte = firstbyte;
4060          *code++ = OP_NOT;          *code++ = ((options & PCRE_CASELESS) != 0)? OP_NOTI: OP_NOT;
4061          *code++ = class_lastchar;          *code++ = class_lastchar;
4062          break;          break;
4063          }          }
# Line 3870  for (;; ptr++) Line 4121  for (;; ptr++)
4121        }        }
4122  #endif  #endif
4123    
4124      /* If there are no characters > 255, or they are all to be included or      /* If there are no characters > 255, or they are all to be included or
4125      excluded, set the opcode to OP_CLASS or OP_NCLASS, depending on whether the      excluded, set the opcode to OP_CLASS or OP_NCLASS, depending on whether the
4126      whole class was negated and whether there were negative specials such as \S      whole class was negated and whether there were negative specials such as \S
4127      (non-UCP) in the class. Then copy the 32-byte map into the code vector,      (non-UCP) in the class. Then copy the 32-byte map into the code vector,
# Line 3964  for (;; ptr++) Line 4215  for (;; ptr++)
4215      the first thing in a branch because the x will have gone into firstbyte      the first thing in a branch because the x will have gone into firstbyte
4216      instead.  */      instead.  */
4217    
4218      if (*previous == OP_CHAR || *previous == OP_CHARNC)      if (*previous == OP_CHAR || *previous == OP_CHARI)
4219        {        {
4220          op_type = (*previous == OP_CHAR)? 0 : OP_STARI - OP_STAR;
4221    
4222        /* Deal with UTF-8 characters that take up more than one byte. It's        /* Deal with UTF-8 characters that take up more than one byte. It's
4223        easier to write this out separately than try to macrify it. Use c to        easier to write this out separately than try to macrify it. Use c to
4224        hold the length of the character in bytes, plus 0x80 to flag that it's a        hold the length of the character in bytes, plus 0x80 to flag that it's a
# Line 3998  for (;; ptr++) Line 4251  for (;; ptr++)
4251    
4252        if (!possessive_quantifier &&        if (!possessive_quantifier &&
4253            repeat_max < 0 &&            repeat_max < 0 &&
4254            check_auto_possessive(*previous, c, utf8, utf8_char, ptr + 1,            check_auto_possessive(previous, utf8, ptr + 1, options, cd))
             options, cd))  
4255          {          {
4256          repeat_type = 0;    /* Force greedy */          repeat_type = 0;    /* Force greedy */
4257          possessive_quantifier = TRUE;          possessive_quantifier = TRUE;
# Line 4011  for (;; ptr++) Line 4263  for (;; ptr++)
4263      /* If previous was a single negated character ([^a] or similar), we use      /* If previous was a single negated character ([^a] or similar), we use
4264      one of the special opcodes, replacing it. The code is shared with single-      one of the special opcodes, replacing it. The code is shared with single-
4265      character repeats by setting opt_type to add a suitable offset into      character repeats by setting opt_type to add a suitable offset into
4266      repeat_type. We can also test for auto-possessification. OP_NOT is      repeat_type. We can also test for auto-possessification. OP_NOT and OP_NOTI
4267      currently used only for single-byte chars. */      are currently used only for single-byte chars. */
4268    
4269      else if (*previous == OP_NOT)      else if (*previous == OP_NOT || *previous == OP_NOTI)
4270        {        {
4271        op_type = OP_NOTSTAR - OP_STAR;  /* Use "not" opcodes */        op_type = ((*previous == OP_NOT)? OP_NOTSTAR : OP_NOTSTARI) - OP_STAR;
4272        c = previous[1];        c = previous[1];
4273        if (!possessive_quantifier &&        if (!possessive_quantifier &&
4274            repeat_max < 0 &&            repeat_max < 0 &&
4275            check_auto_possessive(OP_NOT, c, utf8, NULL, ptr + 1, options, cd))            check_auto_possessive(previous, utf8, ptr + 1, options, cd))
4276          {          {
4277          repeat_type = 0;    /* Force greedy */          repeat_type = 0;    /* Force greedy */
4278          possessive_quantifier = TRUE;          possessive_quantifier = TRUE;
# Line 4044  for (;; ptr++) Line 4296  for (;; ptr++)
4296    
4297        if (!possessive_quantifier &&        if (!possessive_quantifier &&
4298            repeat_max < 0 &&            repeat_max < 0 &&
4299            check_auto_possessive(c, 0, utf8, NULL, ptr + 1, options, cd))            check_auto_possessive(previous, utf8, ptr + 1, options, cd))
4300          {          {
4301          repeat_type = 0;    /* Force greedy */          repeat_type = 0;    /* Force greedy */
4302          possessive_quantifier = TRUE;          possessive_quantifier = TRUE;
# Line 4213  for (;; ptr++) Line 4465  for (;; ptr++)
4465  #ifdef SUPPORT_UTF8  #ifdef SUPPORT_UTF8
4466               *previous == OP_XCLASS ||               *previous == OP_XCLASS ||
4467  #endif  #endif
4468               *previous == OP_REF)               *previous == OP_REF ||
4469                 *previous == OP_REFI)
4470        {        {
4471        if (repeat_max == 0)        if (repeat_max == 0)
4472          {          {
# Line 4247  for (;; ptr++) Line 4500  for (;; ptr++)
4500        }        }
4501    
4502      /* If previous was a bracket group, we may have to replicate it in certain      /* If previous was a bracket group, we may have to replicate it in certain
4503      cases. */      cases. Note that at this point we can encounter only the "basic" BRA and
4504        KET opcodes, as this is the place where they get converted into the more
4505        special varieties. */
4506    
4507      else if (*previous == OP_BRA  || *previous == OP_CBRA ||      else if (*previous == OP_BRA  || *previous == OP_CBRA ||
4508               *previous == OP_ONCE || *previous == OP_COND)               *previous == OP_ONCE || *previous == OP_COND)
4509        {        {
4510        register int i;        register int i;
       int ketoffset = 0;  
4511        int len = (int)(code - previous);        int len = (int)(code - previous);
4512        uschar *bralink = NULL;        uschar *bralink = NULL;
4513          uschar *brazeroptr = NULL;
4514    
4515        /* Repeating a DEFINE group is pointless */        /* Repeating a DEFINE group is pointless */
4516    
4517        if (*previous == OP_COND && previous[LINK_SIZE+1] == OP_DEF)        if (*previous == OP_COND && previous[LINK_SIZE+1] == OP_DEF)
# Line 4265  for (;; ptr++) Line 4520  for (;; ptr++)
4520          goto FAILED;          goto FAILED;
4521          }          }
4522    
       /* If the maximum repeat count is unlimited, find the end of the bracket  
       by scanning through from the start, and compute the offset back to it  
       from the current code pointer. There may be an OP_OPT setting following  
       the final KET, so we can't find the end just by going back from the code  
       pointer. */  
   
       if (repeat_max == -1)  
         {  
         register uschar *ket = previous;  
         do ket += GET(ket, 1); while (*ket != OP_KET);  
         ketoffset = (int)(code - ket);  
         }  
   
4523        /* The case of a zero minimum is special because of the need to stick        /* The case of a zero minimum is special because of the need to stick
4524        OP_BRAZERO in front of it, and because the group appears once in the        OP_BRAZERO in front of it, and because the group appears once in the
4525        data, whereas in other cases it appears the minimum number of times. For        data, whereas in other cases it appears the minimum number of times. For
# Line 4319  for (;; ptr++) Line 4561  for (;; ptr++)
4561              *previous++ = OP_SKIPZERO;              *previous++ = OP_SKIPZERO;
4562              goto END_REPEAT;              goto END_REPEAT;
4563              }              }
4564              brazeroptr = previous;    /* Save for possessive optimizing */
4565            *previous++ = OP_BRAZERO + repeat_type;            *previous++ = OP_BRAZERO + repeat_type;
4566            }            }
4567    
# Line 4483  for (;; ptr++) Line 4726  for (;; ptr++)
4726            }            }
4727          }          }
4728    
4729        /* If the maximum is unlimited, set a repeater in the final copy. We        /* If the maximum is unlimited, set a repeater in the final copy. For
4730        can't just offset backwards from the current code point, because we        ONCE brackets, that's all we need to do.
4731        don't know if there's been an options resetting after the ket. The  
4732        correct offset was computed above.        (To be done next, after recursion adjusted)
4733          However, possessively repeated
4734          ONCE brackets can be converted into non-capturing brackets, as the
4735          behaviour of (?:xx)++ is the same as (?>xx)++ and this saves having to
4736          deal with possessive ONCEs specially.
4737          (....)
4738    
4739          Otherwise, if the quantifier was possessive, we convert the BRA code to
4740          the POS form, and the KET code to KETRPOS. (It turns out to be convenient
4741          at runtime to detect this kind of subpattern at both the start and at the
4742          end.) The use of special opcodes makes it possible to reduce greatly the
4743          stack usage in pcre_exec(). If the group is preceded by OP_BRAZERO,
4744          convert this to OP_BRAPOSZERO. Then cancel the possessive flag so that
4745          the default action below, of wrapping everything inside atomic brackets,
4746          does not happen.
4747    
4748        Then, when we are doing the actual compile phase, check to see whether        Then, when we are doing the actual compile phase, check to see whether
4749        this group is a non-atomic one that could match an empty string. If so,        this group is one that could match an empty string. If so, convert the
4750        convert the initial operator to the S form (e.g. OP_BRA -> OP_SBRA) so        initial operator to the S form (e.g. OP_BRA -> OP_SBRA) so that runtime
4751        that runtime checking can be done. [This check is also applied to        checking can be done. [This check is also applied to ONCE groups at
4752        atomic groups at runtime, but in a different way.] */        runtime, but in a different way.] */
4753    
4754        else        else
4755          {          {
4756          uschar *ketcode = code - ketoffset;          uschar *ketcode = code - 1 - LINK_SIZE;
4757          uschar *bracode = ketcode - GET(ketcode, 1);          uschar *bracode = ketcode - GET(ketcode, 1);
4758          *ketcode = OP_KETRMAX + repeat_type;  
4759          if (lengthptr == NULL && *bracode != OP_ONCE)  /****
4760            if (*bracode == OP_ONCE && possessive_quantifier)
4761              *bracode = OP_BRA;
4762    ****/
4763    
4764            if (*bracode == OP_ONCE)
4765              *ketcode = OP_KETRMAX + repeat_type;
4766            else
4767            {            {
4768            uschar *scode = bracode;            if (possessive_quantifier)
           do  
4769              {              {
4770              if (could_be_empty_branch(scode, ketcode, utf8, cd))              *bracode += 1;                   /* Switch to xxxPOS opcodes */
4771                *ketcode = OP_KETRPOS;
4772                if (brazeroptr != NULL) *brazeroptr = OP_BRAPOSZERO;
4773                possessive_quantifier = FALSE;
4774                }
4775              else *ketcode = OP_KETRMAX + repeat_type;
4776    
4777              if (lengthptr == NULL)
4778                {
4779                uschar *scode = bracode;
4780                do
4781                {                {
4782                *bracode += OP_SBRA - OP_BRA;                if (could_be_empty_branch(scode, ketcode, utf8, cd))
4783                break;                  {
4784                    *bracode += OP_SBRA - OP_BRA;
4785                    break;
4786                    }
4787                  scode += GET(scode, 1);
4788                }                }
4789              scode += GET(scode, 1);              while (*scode == OP_ALT);
4790              }              }
           while (*scode == OP_ALT);  
4791            }            }
4792          }          }
4793        }        }
# Line 4532  for (;; ptr++) Line 4808  for (;; ptr++)
4808        }        }
4809    
4810      /* If the character following a repeat is '+', or if certain optimization      /* If the character following a repeat is '+', or if certain optimization
4811      tests above succeeded, possessive_quantifier is TRUE. For some of the      tests above succeeded, possessive_quantifier is TRUE. For some opcodes,
4812      simpler opcodes, there is an special alternative opcode for this. For      there are special alternative opcodes for this case. For anything else, we
4813      anything else, we wrap the entire repeated item inside OP_ONCE brackets.      wrap the entire repeated item inside OP_ONCE brackets. Logically, the '+'
4814      The '+' notation is just syntactic sugar, taken from Sun's Java package,      notation is just syntactic sugar, taken from Sun's Java package, but the
4815      but the special opcodes can optimize it a bit. The repeated item starts at      special opcodes can optimize it.
4816      tempcode, not at previous, which might be the first part of a string whose  
4817      (former) last char we repeated.      Possessively repeated subpatterns have already been handled in the code
4818        just above, so possessive_quantifier is always FALSE for them at this
4819        stage.
4820    
4821        Note that the repeated item starts at tempcode, not at previous, which
4822        might be the first part of a string whose (former) last char we repeated.
4823    
4824      Possessifying an 'exact' quantifier has no effect, so we can ignore it. But      Possessifying an 'exact' quantifier has no effect, so we can ignore it. But
4825      an 'upto' may follow. We skip over an 'exact' item, and then test the      an 'upto' may follow. We skip over an 'exact' item, and then test the
# Line 4569  for (;; ptr++) Line 4850  for (;; ptr++)
4850          case OP_QUERY: *tempcode = OP_POSQUERY; break;          case OP_QUERY: *tempcode = OP_POSQUERY; break;
4851          case OP_UPTO:  *tempcode = OP_POSUPTO; break;          case OP_UPTO:  *tempcode = OP_POSUPTO; break;
4852    
4853          case OP_TYPESTAR:  *tempcode = OP_TYPEPOSSTAR; break;          case OP_STARI:  *tempcode = OP_POSSTARI; break;
4854          case OP_TYPEPLUS:  *tempcode = OP_TYPEPOSPLUS; break;          case OP_PLUSI:  *tempcode = OP_POSPLUSI; break;
4855          case OP_TYPEQUERY: *tempcode = OP_TYPEPOSQUERY; break;          case OP_QUERYI: *tempcode = OP_POSQUERYI; break;
4856          case OP_TYPEUPTO:  *tempcode = OP_TYPEPOSUPTO; break;          case OP_UPTOI:  *tempcode = OP_POSUPTOI; break;
4857    
4858          case OP_NOTSTAR:  *tempcode = OP_NOTPOSSTAR; break;          case OP_NOTSTAR:  *tempcode = OP_NOTPOSSTAR; break;
4859          case OP_NOTPLUS:  *tempcode = OP_NOTPOSPLUS; break;          case OP_NOTPLUS:  *tempcode = OP_NOTPOSPLUS; break;
4860          case OP_NOTQUERY: *tempcode = OP_NOTPOSQUERY; break;          case OP_NOTQUERY: *tempcode = OP_NOTPOSQUERY; break;
4861          case OP_NOTUPTO:  *tempcode = OP_NOTPOSUPTO; break;          case OP_NOTUPTO:  *tempcode = OP_NOTPOSUPTO; break;
4862    
4863            case OP_NOTSTARI:  *tempcode = OP_NOTPOSSTARI; break;
4864            case OP_NOTPLUSI:  *tempcode = OP_NOTPOSPLUSI; break;
4865            case OP_NOTQUERYI: *tempcode = OP_NOTPOSQUERYI; break;
4866            case OP_NOTUPTOI:  *tempcode = OP_NOTPOSUPTOI; break;
4867    
4868            case OP_TYPESTAR:  *tempcode = OP_TYPEPOSSTAR; break;
4869            case OP_TYPEPLUS:  *tempcode = OP_TYPEPOSPLUS; break;
4870            case OP_TYPEQUERY: *tempcode = OP_TYPEPOSQUERY; break;
4871            case OP_TYPEUPTO:  *tempcode = OP_TYPEPOSUPTO; break;
4872    
4873          /* Because we are moving code along, we must ensure that any          /* Because we are moving code along, we must ensure that any
4874          pending recursive references are updated. */          pending recursive references are updated. */
4875    
# Line 4637  for (;; ptr++) Line 4928  for (;; ptr++)
4928          arg = ++ptr;          arg = ++ptr;
4929          while ((cd->ctypes[*ptr] & (ctype_letter|ctype_digit)) != 0          while ((cd->ctypes[*ptr] & (ctype_letter|ctype_digit)) != 0
4930            || *ptr == '_') ptr++;            || *ptr == '_') ptr++;
4931          arglen = ptr - arg;          arglen = (int)(ptr - arg);
4932          }          }
4933    
4934        if (*ptr != CHAR_RIGHT_PARENTHESIS)        if (*ptr != CHAR_RIGHT_PARENTHESIS)
# Line 4653  for (;; ptr++) Line 4944  for (;; ptr++)
4944          if (namelen == verbs[i].len &&          if (namelen == verbs[i].len &&
4945              strncmp((char *)name, vn, namelen) == 0)              strncmp((char *)name, vn, namelen) == 0)
4946            {            {
4947            /* Check for open captures before ACCEPT */            /* Check for open captures before ACCEPT and convert it to
4948              ASSERT_ACCEPT if in an assertion. */
4949    
4950            if (verbs[i].op == OP_ACCEPT)            if (verbs[i].op == OP_ACCEPT)
4951              {              {
4952              open_capitem *oc;              open_capitem *oc;
4953                if (arglen != 0)
4954                  {
4955                  *errorcodeptr = ERR59;
4956                  goto FAILED;
4957                  }
4958              cd->had_accept = TRUE;              cd->had_accept = TRUE;
4959              for (oc = cd->open_caps; oc != NULL; oc = oc->next)              for (oc = cd->open_caps; oc != NULL; oc = oc->next)
4960                {                {
4961                *code++ = OP_CLOSE;                *code++ = OP_CLOSE;
4962                PUT2INC(code, 0, oc->number);                PUT2INC(code, 0, oc->number);
4963                }                }
4964                *code++ = (cd->assert_depth > 0)? OP_ASSERT_ACCEPT : OP_ACCEPT;
4965              }              }
4966    
4967            /* Handle the cases with/without an argument */            /* Handle other cases with/without an argument */
4968    
4969            if (arglen == 0)            else if (arglen == 0)
4970              {              {
4971              if (verbs[i].op < 0)   /* Argument is mandatory */              if (verbs[i].op < 0)   /* Argument is mandatory */
4972                {                {
4973                *errorcodeptr = ERR66;                *errorcodeptr = ERR66;
4974                goto FAILED;                goto FAILED;
4975                }                }
4976              *code++ = verbs[i].op;              *code = verbs[i].op;
4977                if (*code++ == OP_THEN)
4978                  {
4979                  PUT(code, 0, code - bcptr->current_branch - 1);
4980                  code += LINK_SIZE;
4981                  }
4982              }              }
4983    
4984            else            else
# Line 4685  for (;; ptr++) Line 4988  for (;; ptr++)
4988                *errorcodeptr = ERR59;                *errorcodeptr = ERR59;
4989                goto FAILED;                goto FAILED;
4990                }                }
4991              *code++ = verbs[i].op_arg;              *code = verbs[i].op_arg;
4992                if (*code++ == OP_THEN_ARG)
4993                  {
4994                  PUT(code, 0, code - bcptr->current_branch - 1);
4995                  code += LINK_SIZE;
4996                  }
4997              *code++ = arglen;              *code++ = arglen;
4998              memcpy(code, arg, arglen);              memcpy(code, arg, arglen);
4999              code += arglen;              code += arglen;
# Line 4879  for (;; ptr++) Line 5187  for (;; ptr++)
5187          /* Search the pattern for a forward reference */          /* Search the pattern for a forward reference */
5188    
5189          else if ((i = find_parens(cd, name, namelen,          else if ((i = find_parens(cd, name, namelen,
5190                          (options & PCRE_EXTENDED) != 0)) > 0)                          (options & PCRE_EXTENDED) != 0, utf8)) > 0)
5191            {            {
5192            PUT2(code, 2+LINK_SIZE, i);            PUT2(code, 2+LINK_SIZE, i);
5193            code[1+LINK_SIZE]++;            code[1+LINK_SIZE]++;
# Line 4947  for (;; ptr++) Line 5255  for (;; ptr++)
5255          /* ------------------------------------------------------------ */          /* ------------------------------------------------------------ */
5256          case CHAR_EQUALS_SIGN:                 /* Positive lookahead */          case CHAR_EQUALS_SIGN:                 /* Positive lookahead */
5257          bravalue = OP_ASSERT;          bravalue = OP_ASSERT;
5258            cd->assert_depth += 1;
5259          ptr++;          ptr++;
5260          break;          break;
5261    
# Line 4961  for (;; ptr++) Line 5270  for (;; ptr++)
5270            continue;            continue;
5271            }            }
5272          bravalue = OP_ASSERT_NOT;          bravalue = OP_ASSERT_NOT;
5273            cd->assert_depth += 1;
5274          break;          break;
5275    
5276    
# Line 4970  for (;; ptr++) Line 5280  for (;; ptr++)
5280            {            {
5281            case CHAR_EQUALS_SIGN:               /* Positive lookbehind */            case CHAR_EQUALS_SIGN:               /* Positive lookbehind */
5282            bravalue = OP_ASSERTBACK;            bravalue = OP_ASSERTBACK;
5283              cd->assert_depth += 1;
5284            ptr += 2;            ptr += 2;
5285            break;            break;
5286    
5287            case CHAR_EXCLAMATION_MARK:          /* Negative lookbehind */            case CHAR_EXCLAMATION_MARK:          /* Negative lookbehind */
5288            bravalue = OP_ASSERTBACK_NOT;            bravalue = OP_ASSERTBACK_NOT;
5289              cd->assert_depth += 1;
5290            ptr += 2;            ptr += 2;
5291            break;            break;
5292    
# Line 5180  for (;; ptr++) Line 5492  for (;; ptr++)
5492          while ((cd->ctypes[*ptr] & ctype_word) != 0) ptr++;          while ((cd->ctypes[*ptr] & ctype_word) != 0) ptr++;
5493          namelen = (int)(ptr - name);          namelen = (int)(ptr - name);
5494    
5495          /* In the pre-compile phase, do a syntax check and set a dummy          /* In the pre-compile phase, do a syntax check. We used to just set
5496          reference number. */          a dummy reference number, because it was not used in the first pass.
5497            However, with the change of recursive back references to be atomic,
5498            we have to look for the number so that this state can be identified, as
5499            otherwise the incorrect length is computed. If it's not a backwards
5500            reference, the dummy number will do. */
5501    
5502          if (lengthptr != NULL)          if (lengthptr != NULL)
5503            {            {
5504              const uschar *temp;
5505    
5506            if (namelen == 0)            if (namelen == 0)
5507              {              {
5508              *errorcodeptr = ERR62;              *errorcodeptr = ERR62;
# Line 5200  for (;; ptr++) Line 5518  for (;; ptr++)
5518              *errorcodeptr = ERR48;              *errorcodeptr = ERR48;
5519              goto FAILED;              goto FAILED;
5520              }              }
5521            recno = 0;  
5522              /* The name table does not exist in the first pass, so we cannot
5523              do a simple search as in the code below. Instead, we have to scan the
5524              pattern to find the number. It is important that we scan it only as
5525              far as we have got because the syntax of named subpatterns has not
5526              been checked for the rest of the pattern, and find_parens() assumes
5527              correct syntax. In any case, it's a waste of resources to scan
5528              further. We stop the scan at the current point by temporarily
5529              adjusting the value of cd->endpattern. */
5530    
5531              temp = cd->end_pattern;
5532              cd->end_pattern = ptr;
5533              recno = find_parens(cd, name, namelen,
5534                (options & PCRE_EXTENDED) != 0, utf8);
5535              cd->end_pattern = temp;
5536              if (recno < 0) recno = 0;    /* Forward ref; set dummy number */
5537            }            }
5538    
5539          /* In the real compile, seek the name in the table. We check the name          /* In the real compile, seek the name in the table. We check the name
# Line 5225  for (;; ptr++) Line 5558  for (;; ptr++)
5558              }              }
5559            else if ((recno =                /* Forward back reference */            else if ((recno =                /* Forward back reference */
5560                      find_parens(cd, name, namelen,                      find_parens(cd, name, namelen,
5561                        (options & PCRE_EXTENDED) != 0)) <= 0)                        (options & PCRE_EXTENDED) != 0, utf8)) <= 0)
5562              {              {
5563              *errorcodeptr = ERR15;              *errorcodeptr = ERR15;
5564              goto FAILED;              goto FAILED;
# Line 5336  for (;; ptr++) Line 5669  for (;; ptr++)
5669              if (called == NULL)              if (called == NULL)
5670                {                {
5671                if (find_parens(cd, NULL, recno,                if (find_parens(cd, NULL, recno,
5672                      (options & PCRE_EXTENDED) != 0) < 0)                      (options & PCRE_EXTENDED) != 0, utf8) < 0)
5673                  {                  {
5674                  *errorcodeptr = ERR15;                  *errorcodeptr = ERR15;
5675                  goto FAILED;                  goto FAILED;
# Line 5365  for (;; ptr++) Line 5698  for (;; ptr++)
5698            /* Insert the recursion/subroutine item, automatically wrapped inside            /* Insert the recursion/subroutine item, automatically wrapped inside
5699            "once" brackets. Set up a "previous group" length so that a            "once" brackets. Set up a "previous group" length so that a
5700            subsequent quantifier will work. */            subsequent quantifier will work. */
5701    
5702            *code = OP_ONCE;            *code = OP_ONCE;
5703            PUT(code, 1, 2 + 2*LINK_SIZE);            PUT(code, 1, 2 + 2*LINK_SIZE);
5704            code += 1 + LINK_SIZE;            code += 1 + LINK_SIZE;
# Line 5439  for (;; ptr++) Line 5772  for (;; ptr++)
5772          is necessary to ensure we correctly detect the start of the pattern in          is necessary to ensure we correctly detect the start of the pattern in
5773          both phases.          both phases.
5774    
5775          If we are not at the pattern start, compile code to change the ims          If we are not at the pattern start, reset the greedy defaults and the
5776          options if this setting actually changes any of them, and reset the          case value for firstbyte and reqbyte. */
         greedy defaults and the case value for firstbyte and reqbyte. */  
5777    
5778          if (*ptr == CHAR_RIGHT_PARENTHESIS)          if (*ptr == CHAR_RIGHT_PARENTHESIS)
5779            {            {
# Line 5452  for (;; ptr++) Line 5784  for (;; ptr++)
5784              }              }
5785            else            else
5786              {              {
             if ((options & PCRE_IMS) != (newoptions & PCRE_IMS))  
               {  
               *code++ = OP_OPT;  
               *code++ = newoptions & PCRE_IMS;  
               }  
5787              greedy_default = ((newoptions & PCRE_UNGREEDY) != 0);              greedy_default = ((newoptions & PCRE_UNGREEDY) != 0);
5788              greedy_non_default = greedy_default ^ 1;              greedy_non_default = greedy_default ^ 1;
5789              req_caseopt = ((newoptions & PCRE_CASELESS) != 0)? REQ_CASELESS : 0;              req_caseopt = ((newoptions & PCRE_CASELESS) != 0)? REQ_CASELESS : 0;
5790              }              }
5791    
5792            /* Change options at this level, and pass them back for use            /* Change options at this level, and pass them back for use
5793            in subsequent branches. When not at the start of the pattern, this            in subsequent branches. */
           information is also necessary so that a resetting item can be  
           compiled at the end of a group (if we are in a group). */  
5794    
5795            *optionsptr = options = newoptions;            *optionsptr = options = newoptions;
5796            previous = NULL;       /* This item can't be repeated */            previous = NULL;       /* This item can't be repeated */
# Line 5503  for (;; ptr++) Line 5828  for (;; ptr++)
5828    
5829      /* Process nested bracketed regex. Assertions may not be repeated, but      /* Process nested bracketed regex. Assertions may not be repeated, but
5830      other kinds can be. All their opcodes are >= OP_ONCE. We copy code into a      other kinds can be. All their opcodes are >= OP_ONCE. We copy code into a
5831      non-register variable in order to be able to pass its address because some      non-register variable (tempcode) in order to be able to pass its address
5832      compilers complain otherwise. Pass in a new setting for the ims options if      because some compilers complain otherwise. */
     they have changed. */  
5833    
5834      previous = (bravalue >= OP_ONCE)? code : NULL;      previous = (bravalue >= OP_ONCE)? code : NULL;
5835      *code = bravalue;      *code = bravalue;
# Line 5515  for (;; ptr++) Line 5839  for (;; ptr++)
5839    
5840      if (!compile_regex(      if (!compile_regex(
5841           newoptions,                   /* The complete new option state */           newoptions,                   /* The complete new option state */
          options & PCRE_IMS,           /* The previous ims option state */  
5842           &tempcode,                    /* Where to put code (updated) */           &tempcode,                    /* Where to put code (updated) */
5843           &ptr,                         /* Input pointer (updated) */           &ptr,                         /* Input pointer (updated) */
5844           errorcodeptr,                 /* Where to put an error message */           errorcodeptr,                 /* Where to put an error message */
# Line 5531  for (;; ptr++) Line 5854  for (;; ptr++)
5854             &length_prevgroup           /* Pre-compile phase */             &length_prevgroup           /* Pre-compile phase */
5855           ))           ))
5856        goto FAILED;        goto FAILED;
5857    
5858        if (bravalue >= OP_ASSERT && bravalue <= OP_ASSERTBACK_NOT)
5859          cd->assert_depth -= 1;
5860    
5861      /* At the end of compiling, code is still pointing to the start of the      /* At the end of compiling, code is still pointing to the start of the
5862      group, while tempcode has been updated to point past the end of the group      group, while tempcode has been updated to point past the end of the group
# Line 5602  for (;; ptr++) Line 5928  for (;; ptr++)
5928          goto FAILED;          goto FAILED;
5929          }          }
5930        *lengthptr += length_prevgroup - 2 - 2*LINK_SIZE;        *lengthptr += length_prevgroup - 2 - 2*LINK_SIZE;
5931        *code++ = OP_BRA;        code++;   /* This already contains bravalue */
5932        PUTINC(code, 0, 1 + LINK_SIZE);        PUTINC(code, 0, 1 + LINK_SIZE);
5933        *code++ = OP_KET;        *code++ = OP_KET;
5934        PUTINC(code, 0, 1 + LINK_SIZE);        PUTINC(code, 0, 1 + LINK_SIZE);
# Line 5675  for (;; ptr++) Line 6001  for (;; ptr++)
6001    
6002      /* ===================================================================*/      /* ===================================================================*/
6003      /* Handle metasequences introduced by \. For ones like \d, the ESC_ values      /* Handle metasequences introduced by \. For ones like \d, the ESC_ values
6004      are arranged to be the negation of the corresponding OP_values in the      are arranged to be the negation of the corresponding OP_values in the
6005      default case when PCRE_UCP is not set. For the back references, the values      default case when PCRE_UCP is not set. For the back references, the values
6006      are ESC_REF plus the reference number. Only back references and those types      are ESC_REF plus the reference number. Only back references and those types
6007      that consume a character may be repeated. We can test for values between      that consume a character may be repeated. We can test for values between
# Line 5794  for (;; ptr++) Line 6120  for (;; ptr++)
6120          HANDLE_REFERENCE:    /* Come here from named backref handling */          HANDLE_REFERENCE:    /* Come here from named backref handling */
6121          if (firstbyte == REQ_UNSET) firstbyte = REQ_NONE;          if (firstbyte == REQ_UNSET) firstbyte = REQ_NONE;
6122          previous = code;          previous = code;
6123          *code++ = OP_REF;          *code++ = ((options & PCRE_CASELESS) != 0)? OP_REFI : OP_REF;
6124          PUT2INC(code, 0, recno);          PUT2INC(code, 0, recno);
6125          cd->backref_map |= (recno < 32)? (1 << recno) : 1;          cd->backref_map |= (recno < 32)? (1 << recno) : 1;
6126          if (recno > cd->top_backref) cd->top_backref = recno;          if (recno > cd->top_backref) cd->top_backref = recno;
# Line 5853  for (;; ptr++) Line 6179  for (;; ptr++)
6179            ptr = substitutes[-c - ESC_DU] - 1;  /* Just before substitute */            ptr = substitutes[-c - ESC_DU] - 1;  /* Just before substitute */
6180            }            }
6181          else          else
6182  #endif  #endif
6183            {            {
6184            previous = (-c > ESC_b && -c < ESC_Z)? code : NULL;            previous = (-c > ESC_b && -c < ESC_Z)? code : NULL;
6185            *code++ = -c;            *code++ = -c;
6186            }            }
6187          }          }
6188        continue;        continue;
6189        }        }
# Line 5902  for (;; ptr++) Line 6228  for (;; ptr++)
6228    
6229      ONE_CHAR:      ONE_CHAR:
6230      previous = code;      previous = code;
6231      *code++ = ((options & PCRE_CASELESS) != 0)? OP_CHARNC : OP_CHAR;      *code++ = ((options & PCRE_CASELESS) != 0)? OP_CHARI : OP_CHAR;
6232      for (c = 0; c < mclength; c++) *code++ = mcbuffer[c];      for (c = 0; c < mclength; c++) *code++ = mcbuffer[c];
6233    
6234      /* Remember if \r or \n were seen */      /* Remember if \r or \n were seen */
# Line 5966  return FALSE; Line 6292  return FALSE;
6292  /* On entry, ptr is pointing past the bracket character, but on return it  /* On entry, ptr is pointing past the bracket character, but on return it
6293  points to the closing bracket, or vertical bar, or end of string. The code  points to the closing bracket, or vertical bar, or end of string. The code
6294  variable is pointing at the byte into which the BRA operator has been stored.  variable is pointing at the byte into which the BRA operator has been stored.
 If the ims options are changed at the start (for a (?ims: group) or during any  
 branch, we need to insert an OP_OPT item at the start of every following branch  
 to ensure they get set correctly at run time, and also pass the new options  
 into every subsequent branch compile.  
   
6295  This function is used during the pre-compile phase when we are trying to find  This function is used during the pre-compile phase when we are trying to find
6296  out the amount of memory needed, as well as during the real compile phase. The  out the amount of memory needed, as well as during the real compile phase. The
6297  value of lengthptr distinguishes the two phases.  value of lengthptr distinguishes the two phases.
6298    
6299  Arguments:  Arguments:
6300    options        option bits, including any changes for this subpattern    options        option bits, including any changes for this subpattern
   oldims         previous settings of ims option bits  
6301    codeptr        -> the address of the current code pointer    codeptr        -> the address of the current code pointer
6302    ptrptr         -> the address of the current pattern pointer    ptrptr         -> the address of the current pattern pointer
6303    errorcodeptr   -> pointer to error code variable    errorcodeptr   -> pointer to error code variable
# Line 5995  Returns: TRUE on success Line 6315  Returns: TRUE on success
6315  */  */
6316    
6317  static BOOL  static BOOL
6318  compile_regex(int options, int oldims, uschar **codeptr, const uschar **ptrptr,  compile_regex(int options, uschar **codeptr, const uschar **ptrptr,
6319    int *errorcodeptr, BOOL lookbehind, BOOL reset_bracount, int skipbytes,    int *errorcodeptr, BOOL lookbehind, BOOL reset_bracount, int skipbytes,
6320    int *firstbyteptr, int *reqbyteptr, branch_chain *bcptr, compile_data *cd,    int *firstbyteptr, int *reqbyteptr, branch_chain *bcptr, compile_data *cd,
6321    int *lengthptr)    int *lengthptr)
# Line 6012  int branchfirstbyte, branchreqbyte; Line 6332  int branchfirstbyte, branchreqbyte;
6332  int length;  int length;
6333  int orig_bracount;  int orig_bracount;
6334  int max_bracount;  int max_bracount;
 int old_external_options = cd->external_options;  
6335  branch_chain bc;  branch_chain bc;
6336    
6337  bc.outer = bcptr;  bc.outer = bcptr;
# Line 6036  pre-compile phase to find out whether an Line 6355  pre-compile phase to find out whether an
6355    
6356  /* If this is a capturing subpattern, add to the chain of open capturing items  /* If this is a capturing subpattern, add to the chain of open capturing items
6357  so that we can detect them if (*ACCEPT) is encountered. This is also used to  so that we can detect them if (*ACCEPT) is encountered. This is also used to
6358  detect groups that contain recursive back references to themselves. */  detect groups that contain recursive back references to themselves. Note that
6359    only OP_CBRA need be tested here; changing this opcode to one of its variants,
6360    e.g. OP_SCBRAPOS, happens later, after the group has been compiled. */
6361    
6362  if (*code == OP_CBRA)  if (*code == OP_CBRA)
6363    {    {
# Line 6062  for (;;) Line 6383  for (;;)
6383    
6384    if (reset_bracount) cd->bracount = orig_bracount;    if (reset_bracount) cd->bracount = orig_bracount;
6385    
   /* Handle a change of ims options at the start of the branch */  
   
   if ((options & PCRE_IMS) != oldims)  
     {  
     *code++ = OP_OPT;  
     *code++ = options & PCRE_IMS;  
     length += 2;  
     }  
   
6386    /* Set up dummy OP_REVERSE if lookbehind assertion */    /* Set up dummy OP_REVERSE if lookbehind assertion */
6387    
6388    if (lookbehind)    if (lookbehind)
# Line 6091  for (;;) Line 6403  for (;;)
6403      return FALSE;      return FALSE;
6404      }      }
6405    
   /* If the external options have changed during this branch, it means that we  
   are at the top level, and a leading option setting has been encountered. We  
   need to re-set the original option values to take account of this so that,  
   during the pre-compile phase, we know to allow for a re-set at the start of  
   subsequent branches. */  
   
   if (old_external_options != cd->external_options)  
     oldims = cd->external_options & PCRE_IMS;  
   
6406    /* Keep the highest bracket count in case (?| was used and some branch    /* Keep the highest bracket count in case (?| was used and some branch
6407    has fewer than the rest. */    has fewer than the rest. */
6408    
# Line 6160  for (;;) Line 6463  for (;;)
6463        {        {
6464        int fixed_length;        int fixed_length;
6465        *code = OP_END;        *code = OP_END;
6466        fixed_length = find_fixedlength(last_branch, options, FALSE, cd);        fixed_length = find_fixedlength(last_branch,  (options & PCRE_UTF8) != 0,
6467            FALSE, cd);
6468        DPRINTF(("fixed length = %d\n", fixed_length));        DPRINTF(("fixed length = %d\n", fixed_length));
6469        if (fixed_length == -3)        if (fixed_length == -3)
6470          {          {
# Line 6181  for (;;) Line 6485  for (;;)
6485    of offsets, with the field in the BRA item now becoming an offset to the    of offsets, with the field in the BRA item now becoming an offset to the
6486    first alternative. If there are no alternatives, it points to the end of the    first alternative. If there are no alternatives, it points to the end of the
6487    group. The length in the terminating ket is always the length of the whole    group. The length in the terminating ket is always the length of the whole
6488    bracketed item. If any of the ims options were changed inside the group,    bracketed item. Return leaving the pointer at the terminating char. */
   compile a resetting op-code following, except at the very end of the pattern.  
   Return leaving the pointer at the terminating char. */  
6489    
6490    if (*ptr != CHAR_VERTICAL_LINE)    if (*ptr != CHAR_VERTICAL_LINE)
6491      {      {
# Line 6227  for (;;) Line 6529  for (;;)
6529        cd->open_caps = cd->open_caps->next;        cd->open_caps = cd->open_caps->next;
6530        }        }
6531    
     /* Reset options if needed. */  
   
     if ((options & PCRE_IMS) != oldims && *ptr == CHAR_RIGHT_PARENTHESIS)  
       {  
       *code++ = OP_OPT;  
       *code++ = oldims;  
       length += 2;  
       }  
   
6532      /* Retain the highest bracket number, in case resetting was used. */      /* Retain the highest bracket number, in case resetting was used. */
6533    
6534      cd->bracount = max_bracount;      cd->bracount = max_bracount;
# Line 6295  for (;;) Line 6588  for (;;)
6588  /* Try to find out if this is an anchored regular expression. Consider each  /* Try to find out if this is an anchored regular expression. Consider each
6589  alternative branch. If they all start with OP_SOD or OP_CIRC, or with a bracket  alternative branch. If they all start with OP_SOD or OP_CIRC, or with a bracket
6590  all of whose alternatives start with OP_SOD or OP_CIRC (recurse ad lib), then  all of whose alternatives start with OP_SOD or OP_CIRC (recurse ad lib), then
6591  it's anchored. However, if this is a multiline pattern, then only OP_SOD  it's anchored. However, if this is a multiline pattern, then only OP_SOD will
6592  counts, since OP_CIRC can match in the middle.  be found, because ^ generates OP_CIRCM in that mode.
6593    
6594  We can also consider a regex to be anchored if OP_SOM starts all its branches.  We can also consider a regex to be anchored if OP_SOM starts all its branches.
6595  This is the code for \G, which means "match at start of match position, taking  This is the code for \G, which means "match at start of match position, taking
# Line 6317  of the more common cases more precisely. Line 6610  of the more common cases more precisely.
6610    
6611  Arguments:  Arguments:
6612    code           points to start of expression (the bracket)    code           points to start of expression (the bracket)
   options        points to the options setting  
6613    bracket_map    a bitmap of which brackets we are inside while testing; this    bracket_map    a bitmap of which brackets we are inside while testing; this
6614                    handles up to substring 31; after that we just have to take                    handles up to substring 31; after that we just have to take
6615                    the less precise approach                    the less precise approach
# Line 6327  Returns: TRUE or FALSE Line 6619  Returns: TRUE or FALSE
6619  */  */
6620    
6621  static BOOL  static BOOL
6622  is_anchored(register const uschar *code, int *options, unsigned int bracket_map,  is_anchored(register const uschar *code, unsigned int bracket_map,
6623    unsigned int backref_map)    unsigned int backref_map)
6624  {  {
6625  do {  do {
6626     const uschar *scode = first_significant_code(code + _pcre_OP_lengths[*code],     const uschar *scode = first_significant_code(code + _pcre_OP_lengths[*code],
6627       options, PCRE_MULTILINE, FALSE);       FALSE);
6628     register int op = *scode;     register int op = *scode;
6629    
6630     /* Non-capturing brackets */     /* Non-capturing brackets */
6631    
6632     if (op == OP_BRA)     if (op == OP_BRA  || op == OP_BRAPOS ||
6633           op == OP_SBRA || op == OP_SBRAPOS)
6634       {       {
6635       if (!is_anchored(scode, options, bracket_map, backref_map)) return FALSE;       if (!is_anchored(scode, bracket_map, backref_map)) return FALSE;
6636       }       }
6637    
6638     /* Capturing brackets */     /* Capturing brackets */
6639    
6640     else if (op == OP_CBRA)     else if (op == OP_CBRA  || op == OP_CBRAPOS ||
6641                op == OP_SCBRA || op == OP_SCBRAPOS)
6642       {       {
6643       int n = GET2(scode, 1+LINK_SIZE);       int n = GET2(scode, 1+LINK_SIZE);
6644       int new_map = bracket_map | ((n < 32)? (1 << n) : 1);       int new_map = bracket_map | ((n < 32)? (1 << n) : 1);
6645       if (!is_anchored(scode, options, new_map, backref_map)) return FALSE;       if (!is_anchored(scode, new_map, backref_map)) return FALSE;
6646       }       }
6647    
6648     /* Other brackets */     /* Other brackets */
6649    
6650     else if (op == OP_ASSERT || op == OP_ONCE || op == OP_COND)     else if (op == OP_ASSERT || op == OP_ONCE || op == OP_COND)
6651       {       {
6652       if (!is_anchored(scode, options, bracket_map, backref_map)) return FALSE;       if (!is_anchored(scode, bracket_map, backref_map)) return FALSE;
6653       }       }
6654    
6655     /* .* is not anchored unless DOTALL is set (which generates OP_ALLANY) and     /* .* is not anchored unless DOTALL is set (which generates OP_ALLANY) and
# Line 6370  do { Line 6664  do {
6664    
6665     /* Check for explicit anchoring */     /* Check for explicit anchoring */
6666    
6667     else if (op != OP_SOD && op != OP_SOM &&     else if (op != OP_SOD && op != OP_SOM && op != OP_CIRC) return FALSE;
            ((*options & PCRE_MULTILINE) != 0 || op != OP_CIRC))  
      return FALSE;  
6668     code += GET(code, 1);     code += GET(code, 1);
6669     }     }
6670  while (*code == OP_ALT);   /* Loop for each alternative */  while (*code == OP_ALT);   /* Loop for each alternative */
# Line 6408  is_startline(const uschar *code, unsigne Line 6700  is_startline(const uschar *code, unsigne
6700  {  {
6701  do {  do {
6702     const uschar *scode = first_significant_code(code + _pcre_OP_lengths[*code],     const uschar *scode = first_significant_code(code + _pcre_OP_lengths[*code],
6703       NULL, 0, FALSE);       FALSE);
6704     register int op = *scode;     register int op = *scode;
6705    
6706     /* If we are at the start of a conditional assertion group, *both* the     /* If we are at the start of a conditional assertion group, *both* the
# Line 6435  do { Line 6727  do {
6727         scode += 1 + LINK_SIZE;         scode += 1 + LINK_SIZE;
6728         break;         break;
6729         }         }
6730       scode = first_significant_code(scode, NULL, 0, FALSE);       scode = first_significant_code(scode, FALSE);
6731       op = *scode;       op = *scode;
6732       }       }
6733    
6734     /* Non-capturing brackets */     /* Non-capturing brackets */
6735    
6736     if (op == OP_BRA)     if (op == OP_BRA  || op == OP_BRAPOS ||
6737           op == OP_SBRA || op == OP_SBRAPOS)
6738       {       {
6739       if (!is_startline(scode, bracket_map, backref_map)) return FALSE;       if (!is_startline(scode, bracket_map, backref_map)) return FALSE;
6740       }       }
6741    
6742     /* Capturing brackets */     /* Capturing brackets */
6743    
6744     else if (op == OP_CBRA)     else if (op == OP_CBRA  || op == OP_CBRAPOS ||
6745                op == OP_SCBRA || op == OP_SCBRAPOS)
6746       {       {
6747       int n = GET2(scode, 1+LINK_SIZE);       int n = GET2(scode, 1+LINK_SIZE);
6748       int new_map = bracket_map | ((n < 32)? (1 << n) : 1);       int new_map = bracket_map | ((n < 32)? (1 << n) : 1);
# Line 6472  do { Line 6766  do {
6766    
6767     /* Check for explicit circumflex */     /* Check for explicit circumflex */
6768    
6769     else if (op != OP_CIRC) return FALSE;     else if (op != OP_CIRC && op != OP_CIRCM) return FALSE;
6770    
6771     /* Move on to the next alternative */     /* Move on to the next alternative */
6772    
# Line 6498  we return that char, otherwise -1. Line 6792  we return that char, otherwise -1.
6792    
6793  Arguments:  Arguments:
6794    code       points to start of expression (the bracket)    code       points to start of expression (the bracket)
   options    pointer to the options (used to check casing changes)  
6795    inassert   TRUE if in an assertion    inassert   TRUE if in an assertion
6796    
6797  Returns:     -1 or the fixed first char  Returns:     -1 or the fixed first char
6798  */  */
6799    
6800  static int  static int
6801  find_firstassertedchar(const uschar *code, int *options, BOOL inassert)  find_firstassertedchar(const uschar *code, BOOL inassert)
6802  {  {
6803  register int c = -1;  register int c = -1;
6804  do {  do {
6805     int d;     int d;
6806     const uschar *scode =     int xl = (*code == OP_CBRA || *code == OP_SCBRA ||
6807       first_significant_code(code + 1+LINK_SIZE, options, PCRE_CASELESS, TRUE);               *code == OP_CBRAPOS || *code == OP_SCBRAPOS)? 2:0;
6808       const uschar *scode = first_significant_code(code + 1+LINK_SIZE + xl, TRUE);
6809     register int op = *scode;     register int op = *scode;
6810    
6811     switch(op)     switch(op)
# Line 6520  do { Line 6814  do {
6814       return -1;       return -1;
6815    
6816       case OP_BRA:       case OP_BRA:
6817         case OP_BRAPOS:
6818       case OP_CBRA:       case OP_CBRA:
6819         case OP_SCBRA:
6820         case OP_CBRAPOS:
6821         case OP_SCBRAPOS:
6822       case OP_ASSERT:       case OP_ASSERT:
6823       case OP_ONCE:       case OP_ONCE:
6824       case OP_COND:       case OP_COND:
6825       if ((d = find_firstassertedchar(scode, options, op == OP_ASSERT)) < 0)       if ((d = find_firstassertedchar(scode, op == OP_ASSERT)) < 0)
6826         return -1;         return -1;
6827       if (c < 0) c = d; else if (c != d) return -1;       if (c < 0) c = d; else if (c != d) return -1;
6828       break;       break;
6829    
6830       case OP_EXACT:       /* Fall through */       case OP_EXACT:
6831       scode += 2;       scode += 2;
6832         /* Fall through */
6833    
6834       case OP_CHAR:       case OP_CHAR:
      case OP_CHARNC:  
6835       case OP_PLUS:       case OP_PLUS:
6836       case OP_MINPLUS:       case OP_MINPLUS:
6837       case OP_POSPLUS:       case OP_POSPLUS:
6838       if (!inassert) return -1;       if (!inassert) return -1;
6839       if (c < 0)       if (c < 0) c = scode[1];
6840         {         else if (c != scode[1]) return -1;
6841         c = scode[1];       break;
6842         if ((*options & PCRE_CASELESS) != 0) c |= REQ_CASELESS;  
6843         }       case OP_EXACTI:
6844       else if (c != scode[1]) return -1;       scode += 2;
6845         /* Fall through */
6846    
6847         case OP_CHARI:
6848         case OP_PLUSI:
6849         case OP_MINPLUSI:
6850         case OP_POSPLUSI:
6851         if (!inassert) return -1;
6852         if (c < 0) c = scode[1] | REQ_CASELESS;
6853           else if (c != scode[1]) return -1;
6854       break;       break;
6855       }       }
6856    
# Line 6666  while (ptr[skipatstart] == CHAR_LEFT_PAR Line 6973  while (ptr[skipatstart] == CHAR_LEFT_PAR
6973      { skipatstart += 7; options |= PCRE_UTF8; continue; }      { skipatstart += 7; options |= PCRE_UTF8; continue; }
6974    else if (strncmp((char *)(ptr+skipatstart+2), STRING_UCP_RIGHTPAR, 4) == 0)    else if (strncmp((char *)(ptr+skipatstart+2), STRING_UCP_RIGHTPAR, 4) == 0)
6975      { skipatstart += 6; options |= PCRE_UCP; continue; }      { skipatstart += 6; options |= PCRE_UCP; continue; }
6976      else if (strncmp((char *)(ptr+skipatstart+2), STRING_NO_START_OPT_RIGHTPAR, 13) == 0)
6977        { skipatstart += 15; options |= PCRE_NO_START_OPTIMIZE; continue; }
6978    
6979    if (strncmp((char *)(ptr+skipatstart+2), STRING_CR_RIGHTPAR, 3) == 0)    if (strncmp((char *)(ptr+skipatstart+2), STRING_CR_RIGHTPAR, 3) == 0)
6980      { skipatstart += 5; newnl = PCRE_NEWLINE_CR; }      { skipatstart += 5; newnl = PCRE_NEWLINE_CR; }
# Line 6689  while (ptr[skipatstart] == CHAR_LEFT_PAR Line 6998  while (ptr[skipatstart] == CHAR_LEFT_PAR
6998      options = (options & ~(PCRE_BSR_ANYCRLF|PCRE_BSR_UNICODE)) | newbsr;      options = (options & ~(PCRE_BSR_ANYCRLF|PCRE_BSR_UNICODE)) | newbsr;
6999    else break;    else break;
7000    }    }
7001    
7002  utf8 = (options & PCRE_UTF8) != 0;  utf8 = (options & PCRE_UTF8) != 0;
7003    
7004  /* Can't support UTF8 unless PCRE has been compiled to include the code. */  /* Can't support UTF8 unless PCRE has been compiled to include the code. The
7005    return of an error code from _pcre_valid_utf8() is a new feature, introduced in
7006    release 8.13. It is passed back from pcre_[dfa_]exec(), but at the moment is
7007    not used here. */
7008    
7009  #ifdef SUPPORT_UTF8  #ifdef SUPPORT_UTF8
7010  if (utf8 && (options & PCRE_NO_UTF8_CHECK) == 0 &&  if (utf8 && (options & PCRE_NO_UTF8_CHECK) == 0 &&
7011       (*erroroffset = _pcre_valid_utf8((USPTR)pattern, -1)) >= 0)       (errorcode = _pcre_valid_utf8((USPTR)pattern, -1, erroroffset)) != 0)
7012    {    {
7013    errorcode = ERR44;    errorcode = ERR44;
7014    goto PCRE_EARLY_ERROR_RETURN2;    goto PCRE_EARLY_ERROR_RETURN2;
# Line 6715  if (utf8) Line 7027  if (utf8)
7027  if ((options & PCRE_UCP) != 0)  if ((options & PCRE_UCP) != 0)
7028    {    {
7029    errorcode = ERR67;    errorcode = ERR67;
7030    goto PCRE_EARLY_ERROR_RETURN;    goto PCRE_EARLY_ERROR_RETURN;
7031    }    }
7032  #endif  #endif
7033    
7034  /* Check validity of \R options. */  /* Check validity of \R options. */
# Line 6812  outside can help speed up starting point Line 7124  outside can help speed up starting point
7124  ptr += skipatstart;  ptr += skipatstart;
7125  code = cworkspace;  code = cworkspace;
7126  *code = OP_BRA;  *code = OP_BRA;
7127  (void)compile_regex(cd->external_options, cd->external_options & PCRE_IMS,  (void)compile_regex(cd->external_options, &code, &ptr, &errorcode, FALSE,
7128    &code, &ptr, &errorcode, FALSE, FALSE, 0, &firstbyte, &reqbyte, NULL, cd,    FALSE, 0, &firstbyte, &reqbyte, NULL, cd, &length);
   &length);  
7129  if (errorcode != 0) goto PCRE_EARLY_ERROR_RETURN;  if (errorcode != 0) goto PCRE_EARLY_ERROR_RETURN;
7130    
7131  DPRINTF(("end pre-compile: length=%d workspace=%d\n", length,  DPRINTF(("end pre-compile: length=%d workspace=%d\n", length,
# Line 6868  field; this time it's used for rememberi Line 7179  field; this time it's used for rememberi
7179  */  */
7180    
7181  cd->final_bracount = cd->bracount;  /* Save for checking forward references */  cd->final_bracount = cd->bracount;  /* Save for checking forward references */
7182    cd->assert_depth = 0;
7183  cd->bracount = 0;  cd->bracount = 0;
7184  cd->names_found = 0;  cd->names_found = 0;
7185  cd->name_table = (uschar *)re + re->name_table_offset;  cd->name_table = (uschar *)re + re->name_table_offset;
# Line 6886  of the function here. */ Line 7198  of the function here. */
7198  ptr = (const uschar *)pattern + skipatstart;  ptr = (const uschar *)pattern + skipatstart;
7199  code = (uschar *)codestart;  code = (uschar *)codestart;
7200  *code = OP_BRA;  *code = OP_BRA;
7201  (void)compile_regex(re->options, re->options & PCRE_IMS, &code, &ptr,  (void)compile_regex(re->options, &code, &ptr, &errorcode, FALSE, FALSE, 0,
7202    &errorcode, FALSE, FALSE, 0, &firstbyte, &reqbyte, NULL, cd, NULL);    &firstbyte, &reqbyte, NULL, cd, NULL);
7203  re->top_bracket = cd->bracount;  re->top_bracket = cd->bracount;
7204  re->top_backref = cd->top_backref;  re->top_backref = cd->top_backref;
7205  re->flags = cd->external_flags;  re->flags = cd->external_flags;
# Line 6953  if (cd->check_lookbehind) Line 7265  if (cd->check_lookbehind)
7265        uschar *be = cc - 1 - LINK_SIZE + GET(cc, -LINK_SIZE);        uschar *be = cc - 1 - LINK_SIZE + GET(cc, -LINK_SIZE);
7266        int end_op = *be;        int end_op = *be;
7267        *be = OP_END;        *be = OP_END;
7268        fixed_length = find_fixedlength(cc, re->options, TRUE, cd);        fixed_length = find_fixedlength(cc, (re->options & PCRE_UTF8) != 0, TRUE,
7269            cd);
7270        *be = end_op;        *be = end_op;
7271        DPRINTF(("fixed length = %d\n", fixed_length));        DPRINTF(("fixed length = %d\n", fixed_length));
7272        if (fixed_length < 0)        if (fixed_length < 0)
# Line 6992  start with ^. and also when all branches Line 7305  start with ^. and also when all branches
7305    
7306  if ((re->options & PCRE_ANCHORED) == 0)  if ((re->options & PCRE_ANCHORED) == 0)
7307    {    {
7308    int temp_options = re->options;   /* May get changed during these scans */    if (is_anchored(codestart, 0, cd->backref_map))
   if (is_anchored(codestart, &temp_options, 0, cd->backref_map))  
7309      re->options |= PCRE_ANCHORED;      re->options |= PCRE_ANCHORED;
7310    else    else
7311      {      {
7312      if (firstbyte < 0)      if (firstbyte < 0)
7313        firstbyte = find_firstassertedchar(codestart, &temp_options, FALSE);        firstbyte = find_firstassertedchar(codestart, FALSE);
7314      if (firstbyte >= 0)   /* Remove caseless flag for non-caseable chars */      if (firstbyte >= 0)   /* Remove caseless flag for non-caseable chars */
7315        {        {
7316        int ch = firstbyte & 255;        int ch = firstbyte & 255;

Legend:
Removed from v.530  
changed lines
  Added in v.618

webmaster@exim.org
ViewVC Help
Powered by ViewVC 1.1.12