/[pcre]/code/tags/pcre-4.5/pcre.c

Diff of /code/tags/pcre-4.5/pcre.c

Parent Directory | Revision Log | View Patch Patch

-revision 43 by nigel,
Sat Feb 24 21:39:21 2007 UTC
+revision 63 by nigel,
Sat Feb 24 21:40:03 2007 UTC
 Line 9 
 the file Tech.Notes for some information
  Written by: Philip Hazel <ph10@cam.ac.uk>
-            Copyright (c) 1997-2000 University of Cambridge
+            Copyright (c) 1997-2003 University of Cambridge
  -----------------------------------------------------------------------------
  Permission is granted to anyone to use this software for any purpose on any
 Line 32 
 restrictions:
  -----------------------------------------------------------------------------
  */
  /* Define DEBUG to get debugging output on stdout. */
  /* #define DEBUG */
-Line 60 
 the external pcre header. */
+Line 59 
 the external pcre header. */
  #endif
- /* Number of items on the nested bracket stacks at compile time. This should
+ /* Maximum number of items on the nested bracket stacks at compile time. This
- not be set greater than 200. */
+ applies to the nesting of all kinds of parentheses. It does not limit
+ un-nested, non-capturing parentheses. This number can be made bigger if
+ necessary - it is used to dimension one int and one unsigned char vector at
+ compile time. */
  #define BRASTACK_SIZE 200
+ /* Maximum number of ints of offset to save on the stack for recursive calls.
+ If the offset vector is bigger, malloc is used. This should be a multiple of 3,
+ because the offset vector is always a multiple of 3 long. */
+ #define REC_STACK_SAVE_MAX 30
+ /* The number of bytes in a literal character string above which we can't add
+ any more is set at 250 in order to allow for UTF-8 characters. (In theory it
+ could be 255 when UTF-8 support is excluded, but that means that some of the
+ test output would be different, which just complicates things.) */
+ #define MAXLIT 250
+ /* Table of sizes for the fixed-length opcodes. It's defined in a macro so that
+ the definition is next to the definition of the opcodes in internal.h. */
+ static uschar OP_lengths[] = { OP_LENGTHS };
  /* Min and max values for the common repeats; for the maxima, 0 => infinity */
  static const char rep_min[] = { 0, 0, 1, 1, 0, 0 };
  static const char rep_max[] = { 0, 0, 0, 0, 1, 1 };
- /* Text forms of OP_ values and things, for debugging (not all used) */
- #ifdef DEBUG
- static const char *OP_names[] = {
-   "End", "\\A", "\\B", "\\b", "\\D", "\\d",
-   "\\S", "\\s", "\\W", "\\w", "\\Z", "\\z",
-   "Opt", "^", "$", "Any", "chars", "not",
-   "*", "*?", "+", "+?", "?", "??", "{", "{", "{",
-   "*", "*?", "+", "+?", "?", "??", "{", "{", "{",
-   "*", "*?", "+", "+?", "?", "??", "{", "{", "{",
-   "*", "*?", "+", "+?", "?", "??", "{", "{",
-   "class", "Ref", "Recurse",
-   "Alt", "Ket", "KetRmax", "KetRmin", "Assert", "Assert not",
-   "AssertB", "AssertB not", "Reverse", "Once", "Cond", "Cref",
-   "Brazero", "Braminzero", "Bra"
- };
- #endif
  /* Table for handling escaped characters in the range '0'-'z'. Positive returns
  are simple data values; negative values are for special things like \d and so
  on. Zero means further processing is needed (for things like \x), or the escape
-Line 97 
 is invalid. */
+Line 102 
 is invalid. */
  static const short int escapes[] = {
 ,      0,      0,      0,      0,      0,      0,      0,   /* 0 - 7 */
 ,      0,    ':',    ';',    '<',    '=',    '>',    '?',   /* 8 - ? */
-   '@', -ESC_A, -ESC_B,      0, -ESC_D,      0,      0,      0,   /* @ - G */
+   '@', -ESC_A, -ESC_B, -ESC_C, -ESC_D, -ESC_E,      0, -ESC_G,   /* @ - G */
 ,      0,      0,      0,      0,      0,      0,      0,   /* H - O */
-,      0,      0, -ESC_S,      0,      0,      0, -ESC_W,   /* P - W */
+, -ESC_Q,      0, -ESC_S,      0,      0,      0, -ESC_W,   /* P - W */
 ,      0, -ESC_Z,    '[',   '\\',    ']',    '^',    '_',   /* X - _ */
-   '`',      7, -ESC_b,      0, -ESC_d,     27,   '\f',      0,   /* ` - g */
+   '`',      7, -ESC_b,      0, -ESC_d,  ESC_e,  ESC_f,      0,   /* ` - g */
-,      0,      0,      0,      0,      0,   '\n',      0,   /* h - o */
+,      0,      0,      0,      0,      0,  ESC_n,      0,   /* h - o */
-,      0,   '\r', -ESC_s,   '\t',      0,      0, -ESC_w,   /* p - w */
+,      0,  ESC_r, -ESC_s,  ESC_t,      0,      0, -ESC_w,   /* p - w */
 ,      0, -ESC_z                                            /* x - z */
  };
-Line 113 
 as this is assumed for handling case ind
+Line 118 
 as this is assumed for handling case ind
  static const char *posix_names[] = {
    "alpha", "lower", "upper",
-   "alnum", "ascii", "cntrl", "digit", "graph",
+   "alnum", "ascii", "blank", "cntrl", "digit", "graph",
    "print", "punct", "space", "word",  "xdigit" };
  static const uschar posix_name_lengths[] = {
-, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 4, 6, 0 };
+, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 4, 6, 0 };
  /* Table of class bit maps for each POSIX class; up to three may be combined
- to form the class. */
+ to form the class. The table for [:blank:] is dynamically modified to remove
+ the vertical space characters. */
  static const int posix_class_maps[] = {
    cbit_lower, cbit_upper, -1,             /* alpha */
-Line 128 
 static const int posix_class_maps[] = {
+Line 134 
 static const int posix_class_maps[] = {
    cbit_upper, -1,         -1,             /* upper */
    cbit_digit, cbit_lower, cbit_upper,     /* alnum */
    cbit_print, cbit_cntrl, -1,             /* ascii */
+   cbit_space, -1,         -1,             /* blank - a GNU extension */
    cbit_cntrl, -1,         -1,             /* cntrl */
    cbit_digit, -1,         -1,             /* digit */
    cbit_graph, -1,         -1,             /* graph */
    cbit_print, -1,         -1,             /* print */
    cbit_punct, -1,         -1,             /* punct */
    cbit_space, -1,         -1,             /* space */
-   cbit_word,  -1,         -1,             /* word */
+   cbit_word,  -1,         -1,             /* word - a Perl extension */
    cbit_xdigit,-1,         -1              /* xdigit */
  };
-Line 143 
 static const int posix_class_maps[] = {
+Line 150 
 static const int posix_class_maps[] = {
  static BOOL
    compile_regex(int, int, int *, uschar **, const uschar **, const char **,
-     BOOL, int, int *, int *, compile_data *);
+     BOOL, int, int *, int *, branch_chain *, compile_data *);
+ /* Structure for building a chain of data that actually lives on the
+ stack, for holding the values of the subject pointer at the start of each
+ subpattern, so as to detect when an empty string has been matched by a
+ subpattern - to break infinite loops. */
+ typedef struct eptrblock {
+   struct eptrblock *prev;
+   const uschar *saved_eptr;
+ } eptrblock;
+ /* Flag bits for the match() function */
+ #define match_condassert   0x01    /* Called to check a condition assertion */
+ #define match_isgroup      0x02    /* Set if start of bracketed group */
+ /* Non-error returns from the match() function. Error returns are externally
+ defined PCRE_ERROR_xxx codes, which are all negative. */
+ #define MATCH_MATCH        1
+ #define MATCH_NOMATCH      0
-Line 153 
 static BOOL
+Line 181 
 static BOOL
  /* PCRE is thread-clean and doesn't use any global variables in the normal
  sense. However, it calls memory allocation and free functions via the two
- indirections below, which are can be changed by the caller, but are shared
+ indirections below, and it can optionally do callouts. These values can be
- between all threads. */
+ changed by the caller, but are shared between all threads. However, when
+ compiling for Virtual Pascal, things are done differently (see pcre.in). */
+ #ifndef VPCOMPAT
  void *(*pcre_malloc)(size_t) = malloc;
  void  (*pcre_free)(void *) = free;
+ int   (*pcre_callout)(pcre_callout_block *) = NULL;
+ #endif
+ /*************************************************
+ *    Macros and tables for character handling    *
+ *************************************************/
+ /* When UTF-8 encoding is being used, a character is no longer just a single
+ byte. The macros for character handling generate simple sequences when used in
+ byte-mode, and more complicated ones for UTF-8 characters. */
+ #ifndef SUPPORT_UTF8
+ #define GETCHAR(c, eptr) c = *eptr;
+ #define GETCHARINC(c, eptr) c = *eptr++;
+ #define GETCHARINCTEST(c, eptr) c = *eptr++;
+ #define GETCHARLEN(c, eptr, len) c = *eptr;
+ #define BACKCHAR(eptr)
+ #else   /* SUPPORT_UTF8 */
+ /* Get the next UTF-8 character, not advancing the pointer. This is called when
+ we know we are in UTF-8 mode. */
+ #define GETCHAR(c, eptr) \
+   c = *eptr; \
+   if ((c & 0xc0) == 0xc0) \
+     { \
+     int i; \
+     int a = utf8_table4[c & 0x3f];  /* Number of additional bytes */ \
+     int s = 6*a; \
+     c = (c & utf8_table3[a]) << s; \
+     for (i = 1; i <= a; i++) \
+       { \
+       s -= 6; \
+       c |= (eptr[i] & 0x3f) << s; \
+       } \
+     }
+ /* Get the next UTF-8 character, advancing the pointer. This is called when we
+ know we are in UTF-8 mode. */
+ #define GETCHARINC(c, eptr) \
+   c = *eptr++; \
+   if ((c & 0xc0) == 0xc0) \
+     { \
+     int a = utf8_table4[c & 0x3f];  /* Number of additional bytes */ \
+     int s = 6*a; \
+     c = (c & utf8_table3[a]) << s; \
+     while (a-- > 0) \
+       { \
+       s -= 6; \
+       c |= (*eptr++ & 0x3f) << s; \
+       } \
+     }
+ /* Get the next character, testing for UTF-8 mode, and advancing the pointer */
+ #define GETCHARINCTEST(c, eptr) \
+   c = *eptr++; \
+   if (md->utf8 && (c & 0xc0) == 0xc0) \
+     { \
+     int a = utf8_table4[c & 0x3f];  /* Number of additional bytes */ \
+     int s = 6*a; \
+     c = (c & utf8_table3[a]) << s; \
+     while (a-- > 0) \
+       { \
+       s -= 6; \
+       c |= (*eptr++ & 0x3f) << s; \
+       } \
+     }
+ /* Get the next UTF-8 character, not advancing the pointer, incrementing length
+ if there are extra bytes. This is called when we know we are in UTF-8 mode. */
+ #define GETCHARLEN(c, eptr, len) \
+   c = *eptr; \
+   if ((c & 0xc0) == 0xc0) \
+     { \
+     int i; \
+     int a = utf8_table4[c & 0x3f];  /* Number of additional bytes */ \
+     int s = 6*a; \
+     c = (c & utf8_table3[a]) << s; \
+     for (i = 1; i <= a; i++) \
+       { \
+       s -= 6; \
+       c |= (eptr[i] & 0x3f) << s; \
+       } \
+     len += a; \
+     }
+ /* If the pointer is not at the start of a character, move it back until
+ it is. Called only in UTF-8 mode. */
+ #define BACKCHAR(eptr) while((*eptr & 0xc0) == 0x80) eptr--;
+ #endif
-Line 176 
 tables. */
+Line 302 
 tables. */
+ #ifdef SUPPORT_UTF8
+ /*************************************************
+ *           Tables for UTF-8 support             *
+ *************************************************/
+ /* These are the breakpoints for different numbers of bytes in a UTF-8
+ character. */
+ static int utf8_table1[] = { 0x7f, 0x7ff, 0xffff, 0x1fffff, 0x3ffffff, 0x7fffffff};
+ /* These are the indicator bits and the mask for the data bits to set in the
+ first byte of a character, indexed by the number of additional bytes. */
+ static int utf8_table2[] = { 0,    0xc0, 0xe0, 0xf0, 0xf8, 0xfc};
+ static int utf8_table3[] = { 0xff, 0x1f, 0x0f, 0x07, 0x03, 0x01};
+ /* Table of the number of extra characters, indexed by the first character
+ masked with 0x3f. The highest number for a valid UTF-8 character is in fact
+x3d. */
+ static uschar utf8_table4[] = {
+,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
+,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5 };
+ /*************************************************
+ *       Convert character value to UTF-8         *
+ *************************************************/
+ /* This function takes an integer value in the range 0 - 0x7fffffff
+ and encodes it as a UTF-8 character in 0 to 6 bytes.
+ Arguments:
+   cvalue     the character value
+   buffer     pointer to buffer for result - at least 6 bytes long
+ Returns:     number of characters placed in the buffer
+ */
+ static int
+ ord2utf8(int cvalue, uschar *buffer)
+ {
+ register int i, j;
+ for (i = 0; i < sizeof(utf8_table1)/sizeof(int); i++)
+   if (cvalue <= utf8_table1[i]) break;
+ buffer += i;
+ for (j = i; j > 0; j--)
+  {
+  *buffer-- = 0x80 | (cvalue & 0x3f);
+  cvalue >>= 6;
+  }
+ *buffer = utf8_table2[i] | cvalue;
+ return i + 1;
+ }
+ #endif
+ /*************************************************
+ *         Print compiled regex                   *
+ *************************************************/
+ /* The code for doing this is held in a separate file that is also included in
+ pcretest.c. It defines a function called print_internals(). */
+ #ifdef DEBUG
+ #include "printint.c"
+ #endif
  /*************************************************
  *          Return version string                 *
  *************************************************/
-Line 206 
 Therefore, I haven't changed the API for
+Line 405 
 Therefore, I haven't changed the API for
  Arguments:
    external_re   points to compiled code
    optptr        where to pass back the options
-   first_char    where to pass back the first character,
+   first_byte    where to pass back the first character,
                  or -1 if multiline and all branches start ^,
                  or -2 otherwise
-Line 215 
 Returns:        number of capturing subp
+Line 414 
 Returns:        number of capturing subp
  */
  int
- pcre_info(const pcre *external_re, int *optptr, int *first_char)
+ pcre_info(const pcre *external_re, int *optptr, int *first_byte)
  {
  const real_pcre *re = (const real_pcre *)external_re;
  if (re == NULL) return PCRE_ERROR_NULL;
  if (re->magic_number != MAGIC_NUMBER) return PCRE_ERROR_BADMAGIC;
  if (optptr != NULL) *optptr = (int)(re->options & PUBLIC_OPTIONS);
- if (first_char != NULL)
+ if (first_byte != NULL)
-   *first_char = ((re->options & PCRE_FIRSTSET) != 0)? re->first_char :
+   *first_byte = ((re->options & PCRE_FIRSTSET) != 0)? re->first_byte :
       ((re->options & PCRE_STARTLINE) != 0)? -1 : -2;
  return re->top_bracket;
  }
-Line 238 
 that additional items can be added compa
+Line 437 
 that additional items can be added compa
  Arguments:
    external_re      points to compiled code
-   external_study   points to study data, or NULL
+   extra_data       points extra data, or NULL
    what             what information is required
    where            where to put the information
-Line 246 
 Returns:           0 if data returned, n
+Line 445 
 Returns:           0 if data returned, n
  */
  int
- pcre_fullinfo(const pcre *external_re, const pcre_extra *study_data, int what,
+ pcre_fullinfo(const pcre *external_re, const pcre_extra *extra_data, int what,
    void *where)
  {
  const real_pcre *re = (const real_pcre *)external_re;
- const real_pcre_extra *study = (const real_pcre_extra *)study_data;
+ const pcre_study_data *study = NULL;
  if (re == NULL || where == NULL) return PCRE_ERROR_NULL;
  if (re->magic_number != MAGIC_NUMBER) return PCRE_ERROR_BADMAGIC;
+ if (extra_data != NULL && (extra_data->flags & PCRE_EXTRA_STUDY_DATA) != 0)
+   study = extra_data->study_data;
  switch (what)
    {
    case PCRE_INFO_OPTIONS:
-Line 265 
 switch (what)
+Line 467 
 switch (what)
    *((size_t *)where) = re->size;
    break;
+   case PCRE_INFO_STUDYSIZE:
+   *((size_t *)where) = (study == NULL)? 0 : study->size;
+   break;
    case PCRE_INFO_CAPTURECOUNT:
    *((int *)where) = re->top_bracket;
    break;
-Line 273 
 switch (what)
+Line 479 
 switch (what)
    *((int *)where) = re->top_backref;
    break;
-   case PCRE_INFO_FIRSTCHAR:
+   case PCRE_INFO_FIRSTBYTE:
    *((int *)where) =
-     ((re->options & PCRE_FIRSTSET) != 0)? re->first_char :
+     ((re->options & PCRE_FIRSTSET) != 0)? re->first_byte :
      ((re->options & PCRE_STARTLINE) != 0)? -1 : -2;
    break;
-Line 287 
 switch (what)
+Line 493 
 switch (what)
    case PCRE_INFO_LASTLITERAL:
    *((int *)where) =
-     ((re->options & PCRE_REQCHSET) != 0)? re->req_char : -1;
+     ((re->options & PCRE_REQCHSET) != 0)? re->req_byte : -1;
+   break;
+   case PCRE_INFO_NAMEENTRYSIZE:
+   *((int *)where) = re->name_entry_size;
+   break;
+   case PCRE_INFO_NAMECOUNT:
+   *((int *)where) = re->name_count;
+   break;
+   case PCRE_INFO_NAMETABLE:
+   *((const uschar **)where) = (const uschar *)re + sizeof(real_pcre);
+   break;
+   default: return PCRE_ERROR_BADOPTION;
+   }
+ return 0;
+ }
+ /*************************************************
+ * Return info about what features are configured *
+ *************************************************/
+ /* This is function which has an extensible interface so that additional items
+ can be added compatibly.
+ Arguments:
+   what             what information is required
+   where            where to put the information
+ Returns:           0 if data returned, negative on error
+ */
+ int
+ pcre_config(int what, void *where)
+ {
+ switch (what)
+   {
+   case PCRE_CONFIG_UTF8:
+   #ifdef SUPPORT_UTF8
+   *((int *)where) = 1;
+   #else
+   *((int *)where) = 0;
+   #endif
+   break;
+   case PCRE_CONFIG_NEWLINE:
+   *((int *)where) = NEWLINE;
+   break;
+   case PCRE_CONFIG_LINK_SIZE:
+   *((int *)where) = LINK_SIZE;
+   break;
+   case PCRE_CONFIG_POSIX_MALLOC_THRESHOLD:
+   *((int *)where) = POSIX_MALLOC_THRESHOLD;
+   break;
+   case PCRE_CONFIG_MATCH_LIMIT:
+   *((unsigned int *)where) = MATCH_LIMIT;
    break;
    default: return PCRE_ERROR_BADOPTION;
-Line 334 
 while (length-- > 0)
+Line 603 
 while (length-- > 0)
  /* This function is called when a \ has been encountered. It either returns a
  positive value for a simple escape such as \n, or a negative value which
- encodes one of the more complicated things such as \d. On entry, ptr is
+ encodes one of the more complicated things such as \d. When UTF-8 is enabled,
- pointing at the \. On exit, it is on the final character of the escape
+ a positive value greater than 255 may be returned. On entry, ptr is pointing at
- sequence.
+ the \. On exit, it is on the final character of the escape sequence.
  Arguments:
    ptrptr     points to the pattern position pointer
-Line 358 
 check_escape(const uschar **ptrptr, cons
+Line 627 
 check_escape(const uschar **ptrptr, cons
  const uschar *ptr = *ptrptr;
  int c, i;
- c = *(++ptr) & 255;   /* Ensure > 0 on signed-char systems */
+ /* If backslash is at the end of the pattern, it's an error. */
+ c = *(++ptr);
  if (c == 0) *errorptr = ERR1;
  /* Digits or letters may have special meaning; all others are literals. */
-Line 377 
 else
+Line 648 
 else
    const uschar *oldptr;
    switch (c)
      {
+     /* A number of Perl escapes are not handled by PCRE. We give an explicit
+     error. */
+     case 'l':
+     case 'L':
+     case 'N':
+     case 'p':
+     case 'P':
+     case 'u':
+     case 'U':
+     case 'X':
+     *errorptr = ERR37;
+     break;
      /* The handling of escape sequences consisting of a string of digits
      starting with one that is not zero is not straightforward. By experiment,
      the way Perl works seems to be as follows:
-Line 418 
 else
+Line 703 
 else
        }
      /* \0 always starts an octal number, but we may drop through to here with a
-     larger first octal digit */
+     larger first octal digit. */
      case '0':
      c -= '0';
      while(i++ < 2 && (cd->ctypes[ptr[1]] & ctype_digit) != 0 &&
        ptr[1] != '8' && ptr[1] != '9')
          c = c * 8 + *(++ptr) - '0';
+     c &= 255;     /* Take least significant 8 bits */
      break;
-     /* Special escapes not starting with a digit are straightforward */
+     /* \x is complicated when UTF-8 is enabled. \x{ddd} is a character number
+     which can be greater than 0xff, but only if the ddd are hex digits. */
      case 'x':
+ #ifdef SUPPORT_UTF8
+     if (ptr[1] == '{' && (options & PCRE_UTF8) != 0)
+       {
+       const uschar *pt = ptr + 2;
+       register int count = 0;
+       c = 0;
+       while ((cd->ctypes[*pt] & ctype_xdigit) != 0)
+         {
+         count++;
+         c = c * 16 + cd->lcc[*pt] -
+           (((cd->ctypes[*pt] & ctype_digit) != 0)? '0' : 'W');
+         pt++;
+         }
+       if (*pt == '}')
+         {
+         if (c < 0 || count > 8) *errorptr = ERR34;
+         ptr = pt;
+         break;
+         }
+       /* If the sequence of hex digits does not end with '}', then we don't
+       recognize this construct; fall through to the normal \x handling. */
+       }
+ #endif
+     /* Read just a single hex char */
      c = 0;
      while (i++ < 2 && (cd->ctypes[ptr[1]] & ctype_xdigit) != 0)
        {
-Line 439 
 else
+Line 752 
 else
        }
      break;
+     /* Other special escapes not starting with a digit are straightforward */
      case 'c':
      c = *(++ptr);
      if (c == 0)
-Line 568 
 return p;
+Line 883 
 return p;
  /*************************************************
+ *      Find first significant op code            *
+ *************************************************/
+ /* This is called by several functions that scan a compiled expression looking
+ for a fixed first character, or an anchoring op code etc. It skips over things
+ that do not influence this. For some calls, a change of option is important.
+ Arguments:
+   code       pointer to the start of the group
+   options    pointer to external options
+   optbit     the option bit whose changing is significant, or
+                zero if none are
+ Returns:     pointer to the first significant opcode
+ */
+ static const uschar*
+ first_significant_code(const uschar *code, int *options, int optbit)
+ {
+ for (;;)
+   {
+   switch ((int)*code)
+     {
+     case OP_OPT:
+     if (optbit > 0 && ((int)code[1] & optbit) != (*options & optbit))
+       *options = (int)code[1];
+     code += 2;
+     break;
+     case OP_ASSERT_NOT:
+     case OP_ASSERTBACK:
+     case OP_ASSERTBACK_NOT:
+     do code += GET(code, 1); while (*code == OP_ALT);
+     /* Fall through */
+     case OP_CALLOUT:
+     case OP_CREF:
+     case OP_BRANUMBER:
+     case OP_WORD_BOUNDARY:
+     case OP_NOT_WORD_BOUNDARY:
+     code += OP_lengths[*code];
+     break;
+     default:
+     return code;
+     }
+   }
+ /* Control never reaches here */
+ }
+ /*************************************************
  *        Find the fixed length of a pattern      *
  *************************************************/
  /* Scan a pattern and compute the fixed length of subject that will match it,
  if the length is fixed. This is needed for dealing with backward assertions.
+ In UTF8 mode, the result is in characters rather than bytes.
  Arguments:
    code     points to the start of the pattern (the bracket)
+   options  the compiling options
- Returns:   the fixed length, or -1 if there is no fixed length
+ Returns:   the fixed length, or -1 if there is no fixed length,
+              or -2 if \C was encountered
  */
  static int
- find_fixedlength(uschar *code)
+ find_fixedlength(uschar *code, int options)
  {
  int length = -1;
  register int branchlength = 0;
- register uschar *cc = code + 3;
+ register uschar *cc = code + 1 + LINK_SIZE;
  /* Scan along the opcodes for this branch. If we get to the end of the
  branch, check the length against that of the other branches. */
-Line 602 
 for (;;)
+Line 974 
 for (;;)
      case OP_BRA:
      case OP_ONCE:
      case OP_COND:
-     d = find_fixedlength(cc);
+     d = find_fixedlength(cc, options);
-     if (d < 0) return -1;
+     if (d < 0) return d;
      branchlength += d;
-     do cc += (cc[1] << 8) + cc[2]; while (*cc == OP_ALT);
+     do cc += GET(cc, 1); while (*cc == OP_ALT);
-     cc += 3;
+     cc += 1 + LINK_SIZE;
      break;
      /* Reached end of a branch; if it's a ket it is the end of a nested
-Line 621 
 for (;;)
+Line 993 
 for (;;)
      if (length < 0) length = branchlength;
        else if (length != branchlength) return -1;
      if (*cc != OP_ALT) return length;
-     cc += 3;
+     cc += 1 + LINK_SIZE;
      branchlength = 0;
      break;
-Line 631 
 for (;;)
+Line 1003 
 for (;;)
      case OP_ASSERT_NOT:
      case OP_ASSERTBACK:
      case OP_ASSERTBACK_NOT:
-     do cc += (cc[1] << 8) + cc[2]; while (*cc == OP_ALT);
+     do cc += GET(cc, 1); while (*cc == OP_ALT);
-     cc += 3;
+     /* Fall through */
-     break;
      /* Skip over things that don't match chars */
      case OP_REVERSE:
-     cc++;
+     case OP_BRANUMBER:
-     /* Fall through */
      case OP_CREF:
      case OP_OPT:
-     cc++;
+     case OP_CALLOUT:
-     /* Fall through */
      case OP_SOD:
+     case OP_SOM:
      case OP_EOD:
      case OP_EODN:
      case OP_CIRC:
      case OP_DOLL:
      case OP_NOT_WORD_BOUNDARY:
      case OP_WORD_BOUNDARY:
-     cc++;
+     cc += OP_lengths[*cc];
      break;
-     /* Handle char strings */
+     /* Handle char strings. In UTF-8 mode we must count characters, not bytes.
+     This requires a scan of the string, unfortunately. We assume valid UTF-8
+     strings, so all we do is reduce the length by one for every byte whose bits
+     are 10xxxxxx. */
      case OP_CHARS:
      branchlength += *(++cc);
+ #ifdef SUPPORT_UTF8
+     if ((options & PCRE_UTF8) != 0)
+       for (d = 1; d <= *cc; d++)
+         if ((cc[d] & 0xc0) == 0x80) branchlength--;
+ #endif
      cc += *cc + 1;
      break;
-     /* Handle exact repetitions */
+     /* Handle exact repetitions. The count is already in characters, but we
+     need to skip over a multibyte character in UTF8 mode.  */
      case OP_EXACT:
+     branchlength += GET2(cc,1);
+     cc += 4;
+ #ifdef SUPPORT_UTF8
+     if ((options & PCRE_UTF8) != 0)
+       {
+       while((*cc & 0x80) == 0x80) cc++;
+       }
+ #endif
+     break;
      case OP_TYPEEXACT:
-     branchlength += (cc[1] << 8) + cc[2];
+     branchlength += GET2(cc,1);
      cc += 4;
      break;
-Line 684 
 for (;;)
+Line 1071 
 for (;;)
      cc++;
      break;
+     /* The single-byte matcher isn't allowed */
+     case OP_ANYBYTE:
+     return -2;
      /* Check a class for variable quantification */
+ #ifdef SUPPORT_UTF8
+     case OP_XCLASS:
+     cc += GET(cc, 1) - 33;
+     /* Fall through */
+ #endif
      case OP_CLASS:
-     cc += (*cc == OP_REF)? 2 : 33;
+     case OP_NCLASS:
+     cc += 33;
      switch (*cc)
        {
-Line 700 
 for (;;)
+Line 1098 
 for (;;)
        case OP_CRRANGE:
        case OP_CRMINRANGE:
-       if ((cc[1] << 8) + cc[2] != (cc[3] << 8) + cc[4]) return -1;
+       if (GET2(cc,1) != GET2(cc,3)) return -1;
-       branchlength += (cc[1] << 8) + cc[2];
+       branchlength += GET2(cc,1);
        cc += 5;
        break;
-Line 723 
 for (;;)
+Line 1121 
 for (;;)
  /*************************************************
- *           Check for POSIX class syntax         *
+ *    Scan compiled regex for numbered bracket    *
  *************************************************/
- /* This function is called when the sequence "[:" or "[." or "[=" is
+ /* This little function scans through a compiled pattern until it finds a
- encountered in a character class. It checks whether this is followed by an
+ capturing bracket with the given number.
- optional ^ and then a sequence of letters, terminated by a matching ":]" or
- ".]" or "=]".
- Argument:
+ Arguments:
-   ptr      pointer to the initial [
+   code        points to start of expression
-   endptr   where to return the end pointer
+   utf8        TRUE in UTF-8 mode
-   cd       pointer to compile data
+   number      the required bracket number
- Returns:   TRUE or FALSE
+ Returns:      pointer to the opcode for the bracket, or NULL if not found
  */
- static BOOL
+ static const uschar *
- check_posix_syntax(const uschar *ptr, const uschar **endptr, compile_data *cd)
+ find_bracket(const uschar *code, BOOL utf8, int number)
  {
- int terminator;          /* Don't combine these lines; the Solaris cc */
+ for (;;)
- terminator = *(++ptr);   /* compiler warns about "non-constant" initializer. */
- if (*(++ptr) == '^') ptr++;
- while ((cd->ctypes[*ptr] & ctype_letter) != 0) ptr++;
- if (*ptr == terminator && ptr[1] == ']')
    {
-   *endptr = ptr;
+   register int c = *code;
-   return TRUE;
+   if (c == OP_END) return NULL;
+   else if (c == OP_CHARS) code += code[1] + OP_lengths[c];
+   else if (c > OP_BRA)
+     {
+     int n = c - OP_BRA;
+     if (n > EXTRACT_BASIC_MAX) n = GET2(code, 2+LINK_SIZE);
+     if (n == number) return (uschar *)code;
+     code += OP_lengths[OP_BRA];
+     }
+   else
+     {
+     code += OP_lengths[c];
+     /* In UTF-8 mode, opcodes that are followed by a character may be followed
+     by a multi-byte character. The length in the table is a minimum, so we have
+     to scan along to skip the extra characters. All opcodes are less than 128,
+     so we can use relatively efficient code. */
+ #ifdef SUPPORT_UTF8
+     if (utf8) switch(c)
+       {
+       case OP_EXACT:
+       case OP_UPTO:
+       case OP_MINUPTO:
+       case OP_STAR:
+       case OP_MINSTAR:
+       case OP_PLUS:
+       case OP_MINPLUS:
+       case OP_QUERY:
+       case OP_MINQUERY:
+       while ((*code & 0xc0) == 0x80) code++;
+       break;
+       }
+ #endif
+     }
    }
- return FALSE;
  }
  /*************************************************
- *          Check POSIX class name                *
+ *    Scan compiled branch for non-emptiness      *
  *************************************************/
- /* This function is called to check the name given in a POSIX-style class entry
+ /* This function scans through a branch of a compiled pattern to see whether it
- such as [:alnum:].
+ can match the empty string or not. It is called only from could_be_empty()
+ below. Note that first_significant_code() skips over assertions. If we hit an
+ unclosed bracket, we return "empty" - this means we've struck an inner bracket
+ whose current branch will already have been scanned.
  Arguments:
-   ptr        points to the first letter
+   code        points to start of search
-   len        the length of the name
+   endcode     points to where to stop
+   utf8        TRUE if in UTF8 mode
- Returns:     a value representing the name, or -1 if unknown
+ Returns:      TRUE if what is matched could be empty
  */
- static int
+ static BOOL
- check_posix_name(const uschar *ptr, int len)
+ could_be_empty_branch(const uschar *code, const uschar *endcode, BOOL utf8)
  {
- register int yield = 0;
+ register int c;
- while (posix_name_lengths[yield] != 0)
+ for (code = first_significant_code(code + 1 + LINK_SIZE, NULL, 0);
+      code < endcode;
+      code = first_significant_code(code + OP_lengths[c], NULL, 0))
    {
-   if (len == posix_name_lengths[yield] &&
+   const uschar *ccode;
-     strncmp((const char *)ptr, posix_names[yield], len) == 0) return yield;
-   yield++;
-   }
- return -1;
- }
+   c = *code;
+   if (c >= OP_BRA)
+     {
+     BOOL empty_branch;
+     if (GET(code, 1) == 0) return TRUE;    /* Hit unclosed bracket */
+     /* Scan a closed bracket */
- /*************************************************
+     empty_branch = FALSE;
- *           Compile one branch                   *
+     do
- *************************************************/
+       {
+       if (!empty_branch && could_be_empty_branch(code, endcode, utf8))
+         empty_branch = TRUE;
+       code += GET(code, 1);
+       }
+     while (*code == OP_ALT);
+     if (!empty_branch) return FALSE;   /* All branches are non-empty */
+     code += 1 + LINK_SIZE;
+     c = *code;
+     }
- /* Scan the pattern, compiling it into the code vector.
+   else switch (c)
+     {
+     /* Check for quantifiers after a class */
+ #ifdef SUPPORT_UTF8
+     case OP_XCLASS:
+     ccode = code + GET(code, 1);
+     goto CHECK_CLASS_REPEAT;
+ #endif
+     case OP_CLASS:
+     case OP_NCLASS:
+     ccode = code + 33;
+ #ifdef SUPPORT_UTF8
+     CHECK_CLASS_REPEAT:
+ #endif
+     switch (*ccode)
+       {
+       case OP_CRSTAR:            /* These could be empty; continue */
+       case OP_CRMINSTAR:
+       case OP_CRQUERY:
+       case OP_CRMINQUERY:
+       break;
+       default:                   /* Non-repeat => class must match */
+       case OP_CRPLUS:            /* These repeats aren't empty */
+       case OP_CRMINPLUS:
+       return FALSE;
+       case OP_CRRANGE:
+       case OP_CRMINRANGE:
+       if (GET2(ccode, 1) > 0) return FALSE;  /* Minimum > 0 */
+       break;
+       }
+     break;
+     /* Opcodes that must match a character */
+     case OP_NOT_DIGIT:
+     case OP_DIGIT:
+     case OP_NOT_WHITESPACE:
+     case OP_WHITESPACE:
+     case OP_NOT_WORDCHAR:
+     case OP_WORDCHAR:
+     case OP_ANY:
+     case OP_ANYBYTE:
+     case OP_CHARS:
+     case OP_NOT:
+     case OP_PLUS:
+     case OP_MINPLUS:
+     case OP_EXACT:
+     case OP_NOTPLUS:
+     case OP_NOTMINPLUS:
+     case OP_NOTEXACT:
+     case OP_TYPEPLUS:
+     case OP_TYPEMINPLUS:
+     case OP_TYPEEXACT:
+     return FALSE;
+     /* End of branch */
+     case OP_KET:
+     case OP_KETRMAX:
+     case OP_KETRMIN:
+     case OP_ALT:
+     return TRUE;
+     /* In UTF-8 mode, STAR, MINSTAR, QUERY, MINQUERY, UPTO, and MINUPTO  may be
+     followed by a multibyte character */
+ #ifdef SUPPORT_UTF8
+     case OP_STAR:
+     case OP_MINSTAR:
+     case OP_QUERY:
+     case OP_MINQUERY:
+     case OP_UPTO:
+     case OP_MINUPTO:
+     if (utf8) while ((code[2] & 0xc0) == 0x80) code++;
+     break;
+ #endif
+     }
+   }
+ return TRUE;
+ }
+ /*************************************************
+ *    Scan compiled regex for non-emptiness       *
+ *************************************************/
+ /* This function is called to check for left recursive calls. We want to check
+ the current branch of the current pattern to see if it could match the empty
+ string. If it could, we must look outwards for branches at other levels,
+ stopping when we pass beyond the bracket which is the subject of the recursion.
  Arguments:
-   options      the option bits
+   code        points to start of the recursion
-   brackets     points to number of brackets used
+   endcode     points to where to stop (current RECURSE item)
-   code         points to the pointer to the current code point
+   bcptr       points to the chain of current (unclosed) branch starts
-   ptrptr       points to the current pattern pointer
+   utf8        TRUE if in UTF-8 mode
-   errorptr     points to pointer to error message
-   optchanged   set to the value of the last OP_OPT item compiled
+ Returns:      TRUE if what is matched could be empty
-   reqchar      set to the last literal character required, else -1
+ */
-   countlits    set to count of mandatory literal characters
-   cd           contains pointers to tables
+ static BOOL
+ could_be_empty(const uschar *code, const uschar *endcode, branch_chain *bcptr,
+   BOOL utf8)
+ {
+ while (bcptr != NULL && bcptr->current >= code)
+   {
+   if (!could_be_empty_branch(bcptr->current, endcode, utf8)) return FALSE;
+   bcptr = bcptr->outer;
+   }
+ return TRUE;
+ }
+ /*************************************************
+ *           Check for POSIX class syntax         *
+ *************************************************/
+ /* This function is called when the sequence "[:" or "[." or "[=" is
+ encountered in a character class. It checks whether this is followed by an
+ optional ^ and then a sequence of letters, terminated by a matching ":]" or
+ ".]" or "=]".
+ Argument:
+   ptr      pointer to the initial [
+   endptr   where to return the end pointer
+   cd       pointer to compile data
- Returns:       TRUE on success
+ Returns:   TRUE or FALSE
-                FALSE, with *errorptr set on error
  */
  static BOOL
- compile_branch(int options, int *brackets, uschar **codeptr,
+ check_posix_syntax(const uschar *ptr, const uschar **endptr, compile_data *cd)
-   const uschar **ptrptr, const char **errorptr, int *optchanged,
+ {
-   int *reqchar, int *countlits, compile_data *cd)
+ int terminator;          /* Don't combine these lines; the Solaris cc */
+ terminator = *(++ptr);   /* compiler warns about "non-constant" initializer. */
+ if (*(++ptr) == '^') ptr++;
+ while ((cd->ctypes[*ptr] & ctype_letter) != 0) ptr++;
+ if (*ptr == terminator && ptr[1] == ']')
+   {
+   *endptr = ptr;
+   return TRUE;
+   }
+ return FALSE;
+ }
+ /*************************************************
+ *          Check POSIX class name                *
+ *************************************************/
+ /* This function is called to check the name given in a POSIX-style class entry
+ such as [:alnum:].
+ Arguments:
+   ptr        points to the first letter
+   len        the length of the name
+ Returns:     a value representing the name, or -1 if unknown
+ */
+ static int
+ check_posix_name(const uschar *ptr, int len)
+ {
+ register int yield = 0;
+ while (posix_name_lengths[yield] != 0)
+   {
+   if (len == posix_name_lengths[yield] &&
+     strncmp((const char *)ptr, posix_names[yield], len) == 0) return yield;
+   yield++;
+   }
+ return -1;
+ }
+ /*************************************************
+ *           Compile one branch                   *
+ *************************************************/
+ /* Scan the pattern, compiling it into the code vector. If the options are
+ changed during the branch, the pointer is used to change the external options
+ bits.
+ Arguments:
+   optionsptr     pointer to the option bits
+   brackets       points to number of extracting brackets used
+   code           points to the pointer to the current code point
+   ptrptr         points to the current pattern pointer
+   errorptr       points to pointer to error message
+   firstbyteptr   set to initial literal character, or < 0 (REQ_UNSET, REQ_NONE)
+   reqbyteptr     set to the last literal character required, else < 0
+   bcptr          points to current branch chain
+   cd             contains pointers to tables etc.
+ Returns:         TRUE on success
+                  FALSE, with *errorptr set on error
+ */
+ static BOOL
+ compile_branch(int *optionsptr, int *brackets, uschar **codeptr,
+   const uschar **ptrptr, const char **errorptr, int *firstbyteptr,
+   int *reqbyteptr, branch_chain *bcptr, compile_data *cd)
  {
  int repeat_type, op_type;
- int repeat_min, repeat_max;
+ int repeat_min = 0, repeat_max = 0;      /* To please picky compilers */
- int bravalue, length;
+ int bravalue = 0;
+ int length;
  int greedy_default, greedy_non_default;
- int prevreqchar;
+ int firstbyte, reqbyte;
+ int zeroreqbyte, zerofirstbyte;
+ int req_caseopt;
  int condcount = 0;
- int subcountlits = 0;
+ int options = *optionsptr;
  register int c;
  register uschar *code = *codeptr;
  uschar *tempcode;
+ BOOL inescq = FALSE;
+ BOOL groupsetfirstbyte = FALSE;
  const uschar *ptr = *ptrptr;
  const uschar *tempptr;
  uschar *previous = NULL;
  uschar class[32];
+ #ifdef SUPPORT_UTF8
+ BOOL class_utf8;
+ BOOL utf8 = (options & PCRE_UTF8) != 0;
+ uschar *class_utf8data;
+ uschar utf8_char[6];
+ #else
+ BOOL utf8 = FALSE;
+ #endif
  /* Set up the default and non-default settings for greediness */
  greedy_default = ((options & PCRE_UNGREEDY) != 0);
  greedy_non_default = greedy_default ^ 1;
- /* Initialize no required char, and count of literals */
+ /* Initialize no first char, no required char. REQ_UNSET means "no char
+ matching encountered yet". It gets changed to REQ_NONE if we hit something that
+ matches a non-fixed char first char; reqbyte just remains unset if we never
+ find one.
+ When we hit a repeat whose minimum is zero, we may have to adjust these values
+ to take the zero repeat into account. This is implemented by setting them to
+ zerofirstbyte and zeroreqbyte when such a repeat is encountered. The individual
+ item types that can be repeated set these backoff variables appropriately. */
+ firstbyte = reqbyte = zerofirstbyte = zeroreqbyte = REQ_UNSET;
+ /* The variable req_caseopt contains either the REQ_CASELESS value or zero,
+ according to the current setting of the caseless flag. REQ_CASELESS is a bit
+ value > 255. It is added into the firstbyte or reqbyte variables to record the
+ case status of the value. */
- *reqchar = prevreqchar = -1;
+ req_caseopt = ((options & PCRE_CASELESS) != 0)? REQ_CASELESS : 0;
- *countlits = 0;
  /* Switch on next character until the end of the branch */
  for (;; ptr++)
    {
    BOOL negate_class;
+   BOOL possessive_quantifier;
    int class_charcount;
    int class_lastchar;
    int newoptions;
-   int condref;
+   int recno;
-   int subreqchar;
+   int skipbytes;
+   int subreqbyte;
+   int subfirstbyte;
    c = *ptr;
+   if (inescq && c != 0) goto NORMAL_CHAR;
    if ((options & PCRE_EXTENDED) != 0)
      {
      if ((cd->ctypes[c] & ctype_space) != 0) continue;
      if (c == '#')
        {
-       while ((c = *(++ptr)) != 0 && c != '\n');
+       /* The space before the ; is to avoid a warning on a silly compiler
-       continue;
+       on the Macintosh. */
+       while ((c = *(++ptr)) != 0 && c != NEWLINE) ;
+       if (c != 0) continue;   /* Else fall through to handle end of string */
        }
      }
-Line 867 
 for (;; ptr++)
+Line 1535 
 for (;; ptr++)
      case 0:
      case '|':
      case ')':
+     *firstbyteptr = firstbyte;
+     *reqbyteptr = reqbyte;
      *codeptr = code;
      *ptrptr = ptr;
      return TRUE;
-     /* Handle single-character metacharacters */
+     /* Handle single-character metacharacters. In multiline mode, ^ disables
+     the setting of any following char as a first character. */
      case '^':
+     if ((options & PCRE_MULTILINE) != 0)
+       {
+       if (firstbyte == REQ_UNSET) firstbyte = REQ_NONE;
+       }
      previous = NULL;
      *code++ = OP_CIRC;
      break;
-Line 883 
 for (;; ptr++)
+Line 1558 
 for (;; ptr++)
      *code++ = OP_DOLL;
      break;
+     /* There can never be a first char if '.' is first, whatever happens about
+     repeats. The value of reqbyte doesn't change either. */
      case '.':
+     if (firstbyte == REQ_UNSET) firstbyte = REQ_NONE;
+     zerofirstbyte = firstbyte;
+     zeroreqbyte = reqbyte;
      previous = code;
      *code++ = OP_ANY;
      break;
-     /* Character classes. These always build a 32-byte bitmap of the permitted
+     /* Character classes. If the included characters are all < 255 in value, we
-     characters, except in the special case where there is only one character.
+     build a 32-byte bitmap of the permitted characters, except in the special
-     For negated classes, we build the map as usual, then invert it at the end.
+     case where there is only one such character. For negated classes, we build
+     the map as usual, then invert it at the end. However, we use a different
+     opcode so that data characters > 255 can be handled correctly.
+     If the class contains characters outside the 0-255 range, a different
+     opcode is compiled. It may optionally have a bit map for characters < 256,
+     but those above are are explicitly listed afterwards. A flag byte tells
+     whether the bitmap is present, and whether this is a negated class or not.
      */
      case '[':
      previous = code;
-     *code++ = OP_CLASS;
+     /* PCRE supports POSIX class stuff inside a class. Perl gives an error if
+     they are encountered at the top level, so we'll do that too. */
+     if ((ptr[1] == ':' || ptr[1] == '.' || ptr[1] == '=') &&
+         check_posix_syntax(ptr, &tempptr, cd))
+       {
+       *errorptr = (ptr[1] == ':')? ERR13 : ERR31;
+       goto FAILED;
+       }
      /* If the first character is '^', set the negation flag and skip it. */
-Line 904 
 for (;; ptr++)
+Line 1601 
 for (;; ptr++)
        negate_class = TRUE;
        c = *(++ptr);
        }
-     else negate_class = FALSE;
+     else
+       {
+       negate_class = FALSE;
+       }
-     /* Keep a count of chars so that we can optimize the case of just a single
+     /* Keep a count of chars with values < 256 so that we can optimize the case
-     character. */
+     of just a single character (as long as it's < 256). For higher valued UTF-8
+     characters, we don't yet do any optimization. */
      class_charcount = 0;
      class_lastchar = -1;
+ #ifdef SUPPORT_UTF8
+     class_utf8 = FALSE;                       /* No chars >= 256 */
+     class_utf8data = code + LINK_SIZE + 34;   /* For UTF-8 items */
+ #endif
      /* Initialize the 32-char bit map to all zeros. We have to build the
      map in a temporary bit of store, in case the class contains only 1
-     character, because in that case the compiled code doesn't use the
+     character (< 256), because in that case the compiled code doesn't use the
      bit map. */
      memset(class, 0, 32 * sizeof(uschar));
      /* Process characters until ] is reached. By writing this as a "do" it
-     means that an initial ] is taken as a data character. */
+     means that an initial ] is taken as a data character. The first pass
+     through the regex checked the overall syntax, so we don't need to be very
+     strict here. At the start of the loop, c contains the first byte of the
+     character. */
      do
        {
-       if (c == 0)
+ #ifdef SUPPORT_UTF8
+       if (utf8 && c > 127) GETCHARLEN(c, ptr, ptr);
+ #endif
+       /* Inside \Q...\E everything is literal except \E */
+       if (inescq)
          {
-         *errorptr = ERR6;
+         if (c == '\\' && ptr[1] == 'E')
-         goto FAILED;
+           {
+           inescq = FALSE;
+           ptr++;
+           continue;
+           }
+         else goto LONE_SINGLE_CHARACTER;
          }
        /* Handle POSIX class names. Perl allows a negation extension of the
-       form [:^name]. A square bracket that doesn't match the syntax is
+       form [:^name:]. A square bracket that doesn't match the syntax is
        treated as a literal. We also recognize the POSIX constructions
        [.ch.] and [=ch=] ("collating elements") and fault them, as Perl
-.6 does. */
+.6 and 5.8 do. */
        if (c == '[' &&
            (ptr[1] == ':' || ptr[1] == '.' || ptr[1] == '=') &&
-Line 972 
 for (;; ptr++)
+Line 1692 
 for (;; ptr++)
            posix_class = 0;
          /* Or into the map we are building up to 3 of the static class
-         tables, or their negations. */
+         tables, or their negations. The [:blank:] class sets up the same
+         chars as the [:space:] class (all white space). We remove the vertical
+         white space chars afterwards. */
          posix_class *= 3;
          for (i = 0; i < 3; i++)
            {
+           BOOL isblank = strncmp(ptr, "blank", 5) == 0;
            int taboffset = posix_class_maps[posix_class + i];
            if (taboffset < 0) break;
            if (local_negate)
+             {
              for (c = 0; c < 32; c++) class[c] |= ~cbits[c+taboffset];
+             if (isblank) class[1] |= 0x3c;
+             }
            else
+             {
              for (c = 0; c < 32; c++) class[c] |= cbits[c+taboffset];
+             if (isblank) class[1] &= ~0x3c;
+             }
            }
          ptr = tempptr + 1;
          class_charcount = 10;  /* Set > 1; assumes more than 1 per class */
-         continue;
+         continue;    /* End of POSIX syntax handling */
          }
        /* Backslash may introduce a single character, or it may introduce one
-Line 996 
 for (;; ptr++)
+Line 1725 
 for (;; ptr++)
        Inside a class (and only there) it is treated as backspace. Elsewhere
        it marks a word boundary. Other escapes have preset maps ready to
        or into the one we are building. We assume they have more than one
-       character in them, so set class_count bigger than one. */
+       character in them, so set class_charcount bigger than one. */
        if (c == '\\')
          {
          c = check_escape(&ptr, errorptr, *brackets, options, TRUE, cd);
-         if (-c == ESC_b) c = '\b';
+         if (-c == ESC_b) c = '\b';  /* \b is backslash in a class */
+         if (-c == ESC_Q)            /* Handle start of quoted string */
+           {
+           if (ptr[1] == '\\' && ptr[2] == 'E')
+             {
+             ptr += 2; /* avoid empty string */
+             }
+           else inescq = TRUE;
+           continue;
+           }
          else if (c < 0)
            {
            register const uschar *cbits = cd->cbits;
-           class_charcount = 10;
+           class_charcount = 10;     /* Greater than 1 is what matters */
            switch (-c)
              {
              case ESC_d:
-Line 1026 
 for (;; ptr++)
+Line 1766 
 for (;; ptr++)
              case ESC_s:
              for (c = 0; c < 32; c++) class[c] |= cbits[c+cbit_space];
+             class[1] &= ~0x08;   /* Perl 5.004 onwards omits VT from \s */
              continue;
              case ESC_S:
              for (c = 0; c < 32; c++) class[c] |= ~cbits[c+cbit_space];
+             class[1] |= 0x08;    /* Perl 5.004 onwards omits VT from \s */
              continue;
+             /* Unrecognized escapes are faulted if PCRE is running in its
+             strict mode. By default, for compatibility with Perl, they are
+             treated as literals. */
              default:
-             *errorptr = ERR7;
+             if ((options & PCRE_EXTRA) != 0)
-             goto FAILED;
+               {
+               *errorptr = ERR7;
+               goto FAILED;
+               }
+             c = *ptr;    /* The final character */
              }
            }
-         /* Fall through if single character */
-         }
+         /* Fall through if we have a single character (c >= 0). This may be
+         > 256 in UTF-8 mode. */
+         }   /* End of backslash handling */
        /* A single character may be followed by '-' to form a range. However,
        Perl does not permit ']' to be the end of the range. A '-' character
-Line 1048 
 for (;; ptr++)
+Line 1801 
 for (;; ptr++)
          {
          int d;
          ptr += 2;
-         d = *ptr;
-         if (d == 0)
+ #ifdef SUPPORT_UTF8
-           {
+         if (utf8)
-           *errorptr = ERR6;
+           {                           /* Braces are required because the */
-           goto FAILED;
+           GETCHARLEN(d, ptr, ptr);    /* macro generates multiple statements */
            }
+         else
+ #endif
+         d = *ptr;
          /* The second part of a range can be a single-character escape, but
-         not any of the other escapes. */
+         not any of the other escapes. Perl 5.6 treats a hyphen as a literal
+         in such circumstances. */
          if (d == '\\')
            {
+           const uschar *oldptr = ptr;
            d = check_escape(&ptr, errorptr, *brackets, options, TRUE, cd);
+           /* \b is backslash; any other special means the '-' was literal */
            if (d < 0)
              {
              if (d == -ESC_b) d = '\b'; else
                {
-               *errorptr = ERR7;
+               ptr = oldptr - 2;
-               goto FAILED;
+               goto LONE_SINGLE_CHARACTER;  /* A few lines below */
                }
              }
            }
+         /* Check that the two values are in the correct order */
          if (d < c)
            {
            *errorptr = ERR8;
            goto FAILED;
            }
+         /* If d is greater than 255, we can't just use the bit map, so set up
+         for the UTF-8 supporting class type. If we are not caseless, we can
+         just set up a single range. If we are caseless, the characters < 256
+         are handled with a bitmap, in order to get the case-insensitive
+         handling. */
+ #ifdef SUPPORT_UTF8
+         if (d > 255)
+           {
+           class_utf8 = TRUE;
+           *class_utf8data++ = XCL_RANGE;
+           if ((options & PCRE_CASELESS) == 0)
+             {
+             class_utf8data += ord2utf8(c, class_utf8data);
+             class_utf8data += ord2utf8(d, class_utf8data);
+             continue;  /* Go get the next char in the class */
+             }
+           class_utf8data += ord2utf8(256, class_utf8data);
+           class_utf8data += ord2utf8(d, class_utf8data);
+           d = 255;
+           /* Fall through */
+           }
+ #endif
+         /* We use the bit map if the range is entirely < 255, or if part of it
+         is < 255 and matching is caseless. */
          for (; c <= d; c++)
            {
            class[c/8] |= (1 << (c&7));
-Line 1089 
 for (;; ptr++)
+Line 1877 
 for (;; ptr++)
            class_charcount++;                /* in case a one-char range */
            class_lastchar = c;
            }
          continue;   /* Go get the next char in the class */
          }
        /* Handle a lone single character - we can get here for a normal
        non-escape char, or after \ that introduces a single character. */
-       class [c/8] |= (1 << (c&7));
+       LONE_SINGLE_CHARACTER:
-       if ((options & PCRE_CASELESS) != 0)
+       /* Handle a multibyte character */
+ #ifdef SUPPORT_UTF8
+       if (utf8 && c > 255)
          {
-         c = cd->fcc[c];   /* flip case */
+         class_utf8 = TRUE;
-         class[c/8] |= (1 << (c&7));
+         *class_utf8data++ = XCL_SINGLE;
+         class_utf8data += ord2utf8(c, class_utf8data);
+         }
+       else
+ #endif
+       /* Handle a single-byte character */
+         {
+         class [c/8] |= (1 << (c&7));
+         if ((options & PCRE_CASELESS) != 0)
+           {
+           c = cd->fcc[c];   /* flip case */
+           class[c/8] |= (1 << (c&7));
+           }
+         class_charcount++;
+         class_lastchar = c;
          }
-       class_charcount++;
-       class_lastchar = c;
        }
      /* Loop until ']' reached; the check for end of string happens inside the
      loop. This "while" is the end of the "do" above. */
-     while ((c = *(++ptr)) != ']');
+     while ((c = *(++ptr)) != ']' || inescq);
-     /* If class_charcount is 1 and class_lastchar is not negative, we saw
+     /* If class_charcount is 1, we saw precisely one character with a value <
-     precisely one character. This doesn't need the whole 32-byte bit map.
+. In UTF-8 mode, we can optimize if there were no characters >= 256 and
-     We turn it into a 1-character OP_CHAR if it's positive, or OP_NOT if
+     the one character is < 128. In non-UTF-8 mode we can always optimize.
-     it's negative. */
+     The optimization throws away the bit map. We turn the item into a
+-character OP_CHARS if it's positive, or OP_NOT if it's negative. Note
+     that OP_NOT does not support multibyte characters. In the positive case, it
+     can cause firstbyte to be set. Otherwise, there can be no first char if
+     this item is first, whatever repeat count may follow. In the case of
+     reqbyte, save the previous value for reinstating. */
-     if (class_charcount == 1 && class_lastchar >= 0)
+ #ifdef SUPPORT_UTF8
+     if (!class_utf8 && class_charcount == 1 && class_lastchar < 128)
+ #else
+     if (class_charcount == 1)
+ #endif
        {
+       zeroreqbyte = reqbyte;
        if (negate_class)
          {
-         code[-1] = OP_NOT;
+         if (firstbyte == REQ_UNSET) firstbyte = REQ_NONE;
+         zerofirstbyte = firstbyte;
+         *code++ = OP_NOT;
          }
        else
          {
-         code[-1] = OP_CHARS;
+         if (firstbyte == REQ_UNSET)
+           {
+           zerofirstbyte = REQ_NONE;
+           firstbyte = class_lastchar | req_caseopt;
+           }
+         else
+           {
+           zerofirstbyte = firstbyte;
+           reqbyte = class_lastchar | req_caseopt;
+           }
+         *code++ = OP_CHARS;
          *code++ = 1;
          }
        *code++ = class_lastchar;
+       break;  /* End of class handling */
+       }       /* End of 1-byte optimization */
+     /* Otherwise, if this is the first thing in the branch, there can be no
+     first char setting, whatever the repeat count. Any reqbyte setting must
+     remain unchanged after any kind of repeat. */
+     if (firstbyte == REQ_UNSET) firstbyte = REQ_NONE;
+     zerofirstbyte = firstbyte;
+     zeroreqbyte = reqbyte;
+     /* If there are characters with values > 255, we have to compile an
+     extended class, with its own opcode. If there are no characters < 256,
+     we can omit the bitmap. */
+ #ifdef SUPPORT_UTF8
+     if (class_utf8)
+       {
+       *class_utf8data++ = XCL_END;    /* Marks the end of extra data */
+       *code++ = OP_XCLASS;
+       code += LINK_SIZE;
+       *code = negate_class? XCL_NOT : 0;
+       /* If the map is required, install it, and move on to the end of
+       the extra data */
+       if (class_charcount > 0)
+         {
+         *code++ |= XCL_MAP;
+         memcpy(code, class, 32);
+         code = class_utf8data;
+         }
+       /* If the map is not required, slide down the extra data. */
+       else
+         {
+         int len = class_utf8data - (code + 33);
+         memmove(code + 1, code + 33, len);
+         code += len + 1;
+         }
+       /* Now fill in the complete length of the item */
+       PUT(previous, 1, code - previous);
+       break;   /* End of class handling */
        }
+ #endif
-     /* Otherwise, negate the 32-byte map if necessary, and copy it into
+     /* If there are no characters > 255, negate the 32-byte map if necessary,
-     the code vector. */
+     and copy it into the code vector. If this is the first thing in the branch,
+     there can be no first char setting, whatever the repeat count. Any reqbyte
+     setting must remain unchanged after any kind of repeat. */
+     if (negate_class)
+       {
+       *code++ = OP_NCLASS;
+       for (c = 0; c < 32; c++) code[c] = ~class[c];
+       }
      else
        {
-       if (negate_class)
+       *code++ = OP_CLASS;
-         for (c = 0; c < 32; c++) code[c] = ~class[c];
+       memcpy(code, class, 32);
-       else
-         memcpy(code, class, 32);
-       code += 32;
        }
+     code += 32;
      break;
      /* Various kinds of repeat */
-Line 1171 
 for (;; ptr++)
+Line 2051 
 for (;; ptr++)
        goto FAILED;
        }
-     /* If the next character is '?' this is a minimizing repeat, by default,
+     if (repeat_min == 0)
-     but if PCRE_UNGREEDY is set, it works the other way round. Advance to the
+       {
-     next character. */
+       firstbyte = zerofirstbyte;   /* Adjust for zero repeat */
+       reqbyte = zeroreqbyte;       /* Ditto */
+       }
+     op_type = 0;                    /* Default single-char op codes */
+     possessive_quantifier = FALSE;  /* Default not possessive quantifier */
-     if (ptr[1] == '?')
+     /* Save start of previous item, in case we have to move it up to make space
-       { repeat_type = greedy_non_default; ptr++; }
+     for an inserted OP_ONCE for the additional '+' extension. */
+     tempcode = previous;
+     /* If the next character is '+', we have a possessive quantifier. This
+     implies greediness, whatever the setting of the PCRE_UNGREEDY option.
+     If the next character is '?' this is a minimizing repeat, by default,
+     but if PCRE_UNGREEDY is set, it works the other way round. We change the
+     repeat type to the non-default. */
+     if (ptr[1] == '+')
+       {
+       repeat_type = 0;                  /* Force greedy */
+       possessive_quantifier = TRUE;
+       ptr++;
+       }
+     else if (ptr[1] == '?')
+       {
+       repeat_type = greedy_non_default;
+       ptr++;
+       }
      else repeat_type = greedy_default;
+     /* If previous was a recursion, we need to wrap it inside brackets so that
+     it can be replicated if necessary. */
+     if (*previous == OP_RECURSE)
+       {
+       memmove(previous + 1 + LINK_SIZE, previous, 1 + LINK_SIZE);
+       code += 1 + LINK_SIZE;
+       *previous = OP_BRA;
+       PUT(previous, 1, code - previous);
+       *code = OP_KET;
+       PUT(code, 1, code - previous);
+       code += 1 + LINK_SIZE;
+       }
      /* If previous was a string of characters, chop off the last one and use it
      as the subject of the repeat. If there was only one character, we can
-     abolish the previous item altogether. A repeat with a zero minimum wipes
+     abolish the previous item altogether. If a one-char item has a minumum of
-     out any reqchar setting, backing up to the previous value. We must also
+     more than one, ensure that it is set in reqbyte - it might not be if a
-     adjust the countlits value. */
+     sequence such as x{3} is the first thing in a branch because the x will
+     have gone into firstbyte instead.  */
      if (*previous == OP_CHARS)
        {
-       int len = previous[1];
+       /* Deal with UTF-8 characters that take up more than one byte. It's
+       easier to write this out separately than try to macrify it. Use c to
-       if (repeat_min == 0) *reqchar = prevreqchar;
+       hold the length of the character in bytes, plus 0x80 to flag that it's a
-       *countlits += repeat_min - 1;
+       length rather than a small character. */
-       if (len == 1)
+ #ifdef SUPPORT_UTF8
-         {
+       if (utf8 && (code[-1] & 0x80) != 0)
-         c = previous[2];
+         {
-         code = previous;
+         uschar *lastchar = code - 1;
+         while((*lastchar & 0xc0) == 0x80) lastchar--;
+         c = code - lastchar;            /* Length of UTF-8 character */
+         memcpy(utf8_char, lastchar, c); /* Save the char */
+         if (lastchar == previous + 2)   /* There was only one character */
+           {
+           code = previous;              /* Abolish the previous item */
+           }
+         else
+           {
+           previous[1] -= c;             /* Adjust length of previous */
+           code = lastchar;              /* Lost char off the end */
+           tempcode = code;              /* Adjust position to be moved for '+' */
+           }
+         c |= 0x80;                      /* Flag c as a length */
          }
        else
+ #endif
+       /* Handle the case of a single byte - either with no UTF8 support, or
+       with UTF-8 disabled, or for a UTF-8 character < 128. */
          {
-         c = previous[len+1];
+         c = *(--code);
-         previous[1]--;
+         if (code == previous + 2)   /* There was only one character */
-         code--;
+           {
+           code = previous;              /* Abolish the previous item */
+           if (repeat_min > 1) reqbyte = c | req_caseopt;
+           }
+         else
+           {
+           previous[1]--;             /* adjust length */
+           tempcode = code;           /* Adjust position to be moved for '+' */
+           }
          }
-       op_type = 0;                 /* Use single-char op codes */
        goto OUTPUT_SINGLE_REPEAT;   /* Code shared with single character types */
        }
      /* If previous was a single negated character ([^a] or similar), we use
      one of the special opcodes, replacing it. The code is shared with single-
-     character repeats by adding a suitable offset into repeat_type. */
+     character repeats by setting opt_type to add a suitable offset into
+     repeat_type. OP_NOT is currently used only for single-byte chars. */
-     else if ((int)*previous == OP_NOT)
+     else if (*previous == OP_NOT)
        {
        op_type = OP_NOTSTAR - OP_STAR;  /* Use "not" opcodes */
        c = previous[1];
-Line 1221 
 for (;; ptr++)
+Line 2169 
 for (;; ptr++)
      /* If previous was a character type match (\d or similar), abolish it and
      create a suitable repeat item. The code is shared with single-character
-     repeats by adding a suitable offset into repeat_type. */
+     repeats by setting op_type to add a suitable offset into repeat_type. */
-     else if ((int)*previous < OP_EODN || *previous == OP_ANY)
+     else if (*previous < OP_EODN)
        {
        op_type = OP_TYPESTAR - OP_STAR;  /* Use type opcodes */
        c = *previous;
-Line 1250 
 for (;; ptr++)
+Line 2198 
 for (;; ptr++)
          else
            {
            *code++ = OP_UPTO + repeat_type;
-           *code++ = repeat_max >> 8;
+           PUT2INC(code, 0, repeat_max);
-           *code++ = (repeat_max & 255);
            }
          }
-Line 1268 
 for (;; ptr++)
+Line 2215 
 for (;; ptr++)
          if (repeat_min != 1)
            {
            *code++ = OP_EXACT + op_type;  /* NB EXACT doesn't have repeat_type */
-           *code++ = repeat_min >> 8;
+           PUT2INC(code, 0, repeat_min);
-           *code++ = (repeat_min & 255);
            }
          /* If the mininum is 1 and the previous item was a character string,
-Line 1277 
 for (;; ptr++)
+Line 2223 
 for (;; ptr++)
          length was 1, or add the character back onto the end of a longer
          string. For a character type nothing need be done; it will just get
          put back naturally. Note that the final character is always going to
-         get added below. */
+         get added below, so we leave code ready for its insertion. */
          else if (*previous == OP_CHARS)
            {
-           if (code == previous) code += 2; else previous[1]++;
+           if (code == previous) code += 2; else
+           /* In UTF-8 mode, a multibyte char has its length in c, with the 0x80
+           bit set as a flag. The length will always be between 2 and 6. */
+ #ifdef SUPPORT_UTF8
+           if (utf8 && c >= 128) previous[1] += c & 7; else
+ #endif
+           previous[1]++;
            }
          /*  For a single negated character we also have to put back the
-         item that got cancelled. */
+         item that got cancelled. At present this applies only to single byte
+         characters in any mode. */
          else if (*previous == OP_NOT) code++;
-         /* If the maximum is unlimited, insert an OP_STAR. */
+         /* If the maximum is unlimited, insert an OP_STAR. Before doing so,
+         we have to insert the character for the previous code. In UTF-8 mode,
+         long characters have their length in c, with the 0x80 bit as a flag. */
          if (repeat_max < 0)
            {
+ #ifdef SUPPORT_UTF8
+           if (utf8 && c >= 128)
+             {
+             memcpy(code, utf8_char, c & 7);
+             code += c & 7;
+             }
+           else
+ #endif
            *code++ = c;
            *code++ = OP_STAR + repeat_type;
            }
-         /* Else insert an UPTO if the max is greater than the min. */
+         /* Else insert an UPTO if the max is greater than the min, again
+         preceded by the character, for the previously inserted code. */
          else if (repeat_max != repeat_min)
            {
+ #ifdef SUPPORT_UTF8
+           if (utf8 && c >= 128)
+             {
+             memcpy(code, utf8_char, c & 7);
+             code += c & 7;
+             }
+           else
+ #endif
            *code++ = c;
            repeat_max -= repeat_min;
            *code++ = OP_UPTO + repeat_type;
-           *code++ = repeat_max >> 8;
+           PUT2INC(code, 0, repeat_max);
-           *code++ = (repeat_max & 255);
            }
          }
        /* The character or character type itself comes last in all cases. */
+ #ifdef SUPPORT_UTF8
+       if (utf8 && c >= 128)
+         {
+         memcpy(code, utf8_char, c & 7);
+         code += c & 7;
+         }
+       else
+ #endif
        *code++ = c;
        }
      /* If previous was a character class or a back reference, we put the repeat
      stuff after it, but just skip the item if the repeat was {0,0}. */
-     else if (*previous == OP_CLASS || *previous == OP_REF)
+     else if (*previous == OP_CLASS ||
+              *previous == OP_NCLASS ||
+ #ifdef SUPPORT_UTF8
+              *previous == OP_XCLASS ||
+ #endif
+              *previous == OP_REF)
        {
        if (repeat_max == 0)
          {
-Line 1333 
 for (;; ptr++)
+Line 2320 
 for (;; ptr++)
        else
          {
          *code++ = OP_CRRANGE + repeat_type;
-         *code++ = repeat_min >> 8;
+         PUT2INC(code, 0, repeat_min);
-         *code++ = repeat_min & 255;
          if (repeat_max == -1) repeat_max = 0;  /* 2-byte encoding for max */
-         *code++ = repeat_max >> 8;
+         PUT2INC(code, 0, repeat_max);
-         *code++ = repeat_max & 255;
          }
        }
      /* If previous was a bracket group, we may have to replicate it in certain
      cases. */
-     else if ((int)*previous >= OP_BRA || (int)*previous == OP_ONCE ||
+     else if (*previous >= OP_BRA || *previous == OP_ONCE ||
-              (int)*previous == OP_COND)
+              *previous == OP_COND)
        {
        register int i;
        int ketoffset = 0;
-Line 1361 
 for (;; ptr++)
+Line 2346 
 for (;; ptr++)
        if (repeat_max == -1)
          {
          register uschar *ket = previous;
-         do ket += (ket[1] << 8) + ket[2]; while (*ket != OP_KET);
+         do ket += GET(ket, 1); while (*ket != OP_KET);
          ketoffset = code - ket;
          }
-Line 1369 
 for (;; ptr++)
+Line 2354 
 for (;; ptr++)
        OP_BRAZERO in front of it, and because the group appears once in the
        data, whereas in other cases it appears the minimum number of times. For
        this reason, it is simplest to treat this case separately, as otherwise
-       the code gets far too mess. There are several special subcases when the
+       the code gets far too messy. There are several special subcases when the
        minimum is zero. */
        if (repeat_min == 0)
          {
-         /* If we set up a required char from the bracket, we must back off
-         to the previous value and reset the countlits value too. */
-         if (subcountlits > 0)
-           {
-           *reqchar = prevreqchar;
-           *countlits -= subcountlits;
-           }
          /* If the maximum is also zero, we just omit the group from the output
          altogether. */
-Line 1412 
 for (;; ptr++)
+Line 2388 
 for (;; ptr++)
          else
            {
            int offset;
-           memmove(previous+4, previous, len);
+           memmove(previous + 2 + LINK_SIZE, previous, len);
-           code += 4;
+           code += 2 + LINK_SIZE;
            *previous++ = OP_BRAZERO + repeat_type;
            *previous++ = OP_BRA;
-Line 1422 
 for (;; ptr++)
+Line 2398 
 for (;; ptr++)
            offset = (bralink == NULL)? 0 : previous - bralink;
            bralink = previous;
-           *previous++ = offset >> 8;
+           PUTINC(previous, 0, offset);
-           *previous++ = offset & 255;
            }
          repeat_max--;
-Line 1431 
 for (;; ptr++)
+Line 2406 
 for (;; ptr++)
        /* If the minimum is greater than zero, replicate the group as many
        times as necessary, and adjust the maximum to the number of subsequent
-       copies that we need. */
+       copies that we need. If we set a first char from the group, and didn't
+       set a required char, copy the latter from the former. */
        else
          {
-         for (i = 1; i < repeat_min; i++)
+         if (repeat_min > 1)
            {
-           memcpy(code, previous, len);
+           if (groupsetfirstbyte && reqbyte < 0) reqbyte = firstbyte;
-           code += len;
+           for (i = 1; i < repeat_min; i++)
+             {
+             memcpy(code, previous, len);
+             code += len;
+             }
            }
          if (repeat_max > 0) repeat_max -= repeat_min;
          }
-Line 1464 
 for (;; ptr++)
+Line 2444 
 for (;; ptr++)
              *code++ = OP_BRA;
              offset = (bralink == NULL)? 0 : code - bralink;
              bralink = code;
-             *code++ = offset >> 8;
+             PUTINC(code, 0, offset);
-             *code++ = offset & 255;
              }
            memcpy(code, previous, len);
-Line 1480 
 for (;; ptr++)
+Line 2459 
 for (;; ptr++)
            int oldlinkoffset;
            int offset = code - bralink + 1;
            uschar *bra = code - offset;
-           oldlinkoffset = (bra[1] << 8) + bra[2];
+           oldlinkoffset = GET(bra, 1);
            bralink = (oldlinkoffset == 0)? NULL : bralink - oldlinkoffset;
            *code++ = OP_KET;
-           *code++ = bra[1] = offset >> 8;
+           PUTINC(code, 0, offset);
-           *code++ = bra[2] = (offset & 255);
+           PUT(bra, 1, offset);
            }
          }
-Line 1504 
 for (;; ptr++)
+Line 2483 
 for (;; ptr++)
        goto FAILED;
        }
+     /* If the character following a repeat is '+', we wrap the entire repeated
+     item inside OP_ONCE brackets. This is just syntactic sugar, taken from
+     Sun's Java package. The repeated item starts at tempcode, not at previous,
+     which might be the first part of a string whose (former) last char we
+     repeated. However, we don't support '+' after a greediness '?'. */
+     if (possessive_quantifier)
+       {
+       int len = code - tempcode;
+       memmove(tempcode + 1+LINK_SIZE, tempcode, len);
+       code += 1 + LINK_SIZE;
+       len += 1 + LINK_SIZE;
+       tempcode[0] = OP_ONCE;
+       *code++ = OP_KET;
+       PUTINC(code, 0, len);
+       PUT(tempcode, 1, len);
+       }
      /* In all case we no longer have a previous item. */
      END_REPEAT:
-Line 1520 
 for (;; ptr++)
+Line 2517 
 for (;; ptr++)
      case '(':
      newoptions = options;
-     condref = -1;
+     skipbytes = 0;
      if (*(++ptr) == '?')
        {
-Line 1541 
 for (;; ptr++)
+Line 2538 
 for (;; ptr++)
          case '(':
          bravalue = OP_COND;       /* Conditional group */
-         if ((cd->ctypes[*(++ptr)] & ctype_digit) != 0)
+         /* Condition to test for recursion */
+         if (ptr[1] == 'R')
            {
-           condref = *ptr - '0';
+           code[1+LINK_SIZE] = OP_CREF;
+           PUT2(code, 2+LINK_SIZE, CREF_RECURSE);
+           skipbytes = 3;
+           ptr += 3;
+           }
+         /* Condition to test for a numbered subpattern match */
+         else if ((cd->ctypes[ptr[1]] & ctype_digit) != 0)
+           {
+           int condref = *(++ptr) - '0';
            while (*(++ptr) != ')') condref = condref*10 + *ptr - '0';
+           if (condref == 0)
+             {
+             *errorptr = ERR35;
+             goto FAILED;
+             }
            ptr++;
+           code[1+LINK_SIZE] = OP_CREF;
+           PUT2(code, 2+LINK_SIZE, condref);
+           skipbytes = 3;
            }
-         else ptr--;
+         /* For conditions that are assertions, we just fall through, having
+         set bravalue above. */
          break;
          case '=':                 /* Positive lookahead */
-Line 1572 
 for (;; ptr++)
+Line 2591 
 for (;; ptr++)
            bravalue = OP_ASSERTBACK_NOT;
            ptr++;
            break;
-           default:                /* Syntax error */
-           *errorptr = ERR24;
-           goto FAILED;
            }
          break;
-Line 1584 
 for (;; ptr++)
+Line 2599 
 for (;; ptr++)
          ptr++;
          break;
+         case 'C':                 /* Callout - may be followed by digits */
+         *code++ = OP_CALLOUT;
+           {
+           int n = 0;
+           while ((cd->ctypes[*(++ptr)] & ctype_digit) != 0)
+             n = n * 10 + *ptr - '0';
+           if (n > 255)
+             {
+             *errorptr = ERR38;
+             goto FAILED;
+             }
+           *code++ = n;
+           }
+         previous = NULL;
+         continue;
+         case 'P':                 /* Named subpattern handling */
+         if (*(++ptr) == '<')      /* Definition */
+           {
+           int i, namelen;
+           const uschar *name = ++ptr;
+           uschar *slot = cd->name_table;
+           while (*ptr++ != '>');
+           namelen = ptr - name - 1;
+           for (i = 0; i < cd->names_found; i++)
+             {
+             int c = memcmp(name, slot+2, namelen + 1);
+             if (c == 0)
+               {
+               *errorptr = ERR43;
+               goto FAILED;
+               }
+             if (c < 0)
+               {
+               memmove(slot + cd->name_entry_size, slot,
+                 (cd->names_found - i) * cd->name_entry_size);
+               break;
+               }
+             slot += cd->name_entry_size;
+             }
+           PUT2(slot, 0, *brackets + 1);
+           memcpy(slot + 2, name, namelen);
+           slot[2+namelen] = 0;
+           cd->names_found++;
+           goto NUMBERED_GROUP;
+           }
+         if (*ptr == '=' || *ptr == '>')  /* Reference or recursion */
+           {
+           int i, namelen;
+           int type = *ptr++;
+           const uschar *name = ptr;
+           uschar *slot = cd->name_table;
+           while (*ptr != ')') ptr++;
+           namelen = ptr - name;
+           for (i = 0; i < cd->names_found; i++)
+             {
+             if (strncmp(name, slot+2, namelen) == 0) break;
+             slot += cd->name_entry_size;
+             }
+           if (i >= cd->names_found)
+             {
+             *errorptr = ERR15;
+             goto FAILED;
+             }
+           recno = GET2(slot, 0);
+           if (type == '>') goto HANDLE_RECURSION;  /* A few lines below */
+           /* Back reference */
+           previous = code;
+           *code++ = OP_REF;
+           PUT2INC(code, 0, recno);
+           cd->backref_map |= (recno < 32)? (1 << recno) : 1;
+           if (recno > cd->top_backref) cd->top_backref = recno;
+           continue;
+           }
+         /* Should never happen */
+         break;
          case 'R':                 /* Pattern recursion */
-         *code++ = OP_RECURSE;
+         ptr++;                    /* Same as (?0)      */
-         ptr++;
+         /* Fall through */
+         /* Recursion or "subroutine" call */
+         case '0': case '1': case '2': case '3': case '4':
+         case '5': case '6': case '7': case '8': case '9':
+           {
+           const uschar *called;
+           recno = 0;
+           while ((cd->ctypes[*ptr] & ctype_digit) != 0)
+             recno = recno * 10 + *ptr++ - '0';
+           /* Come here from code above that handles a named recursion */
+           HANDLE_RECURSION:
+           previous = code;
+           /* Find the bracket that is being referenced. Temporarily end the
+           regex in case it doesn't exist. */
+           *code = OP_END;
+           called = (recno == 0)?
+             cd->start_code : find_bracket(cd->start_code, utf8, recno);
+           if (called == NULL)
+             {
+             *errorptr = ERR15;
+             goto FAILED;
+             }
+           /* If the subpattern is still open, this is a recursive call. We
+           check to see if this is a left recursion that could loop for ever,
+           and diagnose that case. */
+           if (GET(called, 1) == 0 && could_be_empty(called, code, bcptr, utf8))
+             {
+             *errorptr = ERR40;
+             goto FAILED;
+             }
+           /* Insert the recursion/subroutine item */
+           *code = OP_RECURSE;
+           PUT(code, 1, called - cd->start_code);
+           code += 1 + LINK_SIZE;
+           }
          continue;
+         /* Character after (? not specially recognized */
          default:                  /* Option setting */
          set = unset = 0;
          optset = &set;
-Line 1605 
 for (;; ptr++)
+Line 2757 
 for (;; ptr++)
              case 'x': *optset |= PCRE_EXTENDED; break;
              case 'U': *optset |= PCRE_UNGREEDY; break;
              case 'X': *optset |= PCRE_EXTRA; break;
-             default:
-             *errorptr = ERR12;
-             goto FAILED;
              }
            }
-Line 1617 
 for (;; ptr++)
+Line 2765 
 for (;; ptr++)
          newoptions = (options | set) & (~unset);
          /* If the options ended with ')' this is not the start of a nested
-         group with option changes, so the options change at this level. At top
+         group with option changes, so the options change at this level. Compile
-         level there is nothing else to be done (the options will in fact have
+         code to change the ims options if this setting actually changes any of
-         been set from the start of compiling as a result of the first pass) but
+         them. We also pass the new setting back so that it can be put at the
-         at an inner level we must compile code to change the ims options if
+         start of any following branches, and when this group ends (if we are in
-         necessary, and pass the new setting back so that it can be put at the
+         a group), a resetting item can be compiled.
-         start of any following branches, and when this group ends, a resetting
-         item can be compiled. */
+         Note that if this item is right at the start of the pattern, the
+         options will have been abstracted and made global, so there will be no
+         change to compile. */
          if (*ptr == ')')
            {
-           if ((options & PCRE_INGROUP) != 0 &&
+           if ((options & PCRE_IMS) != (newoptions & PCRE_IMS))
-               (options & PCRE_IMS) != (newoptions & PCRE_IMS))
              {
              *code++ = OP_OPT;
-             *code++ = *optchanged = newoptions & PCRE_IMS;
+             *code++ = newoptions & PCRE_IMS;
              }
-           options = newoptions;  /* Change options at this level */
+           /* Change options at this level, and pass them back for use
+           in subsequent branches. Reset the greedy defaults and the case
+           value for firstbyte and reqbyte. */
+           *optionsptr = options = newoptions;
+           greedy_default = ((newoptions & PCRE_UNGREEDY) != 0);
+           greedy_non_default = greedy_default ^ 1;
+           req_caseopt = ((options & PCRE_CASELESS) != 0)? REQ_CASELESS : 0;
            previous = NULL;       /* This item can't be repeated */
            continue;              /* It is complete */
            }
-Line 1648 
 for (;; ptr++)
+Line 2806 
 for (;; ptr++)
          }
        }
-     /* Else we have a referencing group; adjust the opcode. */
+     /* If PCRE_NO_AUTO_CAPTURE is set, all unadorned brackets become
+     non-capturing and behave like (?:...) brackets */
+     else if ((options & PCRE_NO_AUTO_CAPTURE) != 0)
+       {
+       bravalue = OP_BRA;
+       }
+     /* Else we have a referencing group; adjust the opcode. If the bracket
+     number is greater than EXTRACT_BASIC_MAX, we set the opcode one higher, and
+     arrange for the true number to follow later, in an OP_BRANUMBER item. */
      else
        {
-       if (++(*brackets) > EXTRACT_MAX)
+       NUMBERED_GROUP:
+       if (++(*brackets) > EXTRACT_BASIC_MAX)
          {
-         *errorptr = ERR13;
+         bravalue = OP_BRA + EXTRACT_BASIC_MAX + 1;
-         goto FAILED;
+         code[1+LINK_SIZE] = OP_BRANUMBER;
+         PUT2(code, 2+LINK_SIZE, *brackets);
+         skipbytes = 3;
          }
-       bravalue = OP_BRA + *brackets;
+       else bravalue = OP_BRA + *brackets;
        }
      /* Process nested bracketed re. Assertions may not be repeated, but other
-Line 1670 
 for (;; ptr++)
+Line 2841 
 for (;; ptr++)
      tempcode = code;
      if (!compile_regex(
-          options | PCRE_INGROUP,       /* Set for all nested groups */
+          newoptions,                   /* The complete new option state */
-          ((options & PCRE_IMS) != (newoptions & PCRE_IMS))?
+          options & PCRE_IMS,           /* The previous ims option state */
-            newoptions & PCRE_IMS : -1, /* Pass ims options if changed */
+          brackets,                     /* Extracting bracket count */
-          brackets,                     /* Bracket level */
           &tempcode,                    /* Where to put code (updated) */
           &ptr,                         /* Input pointer (updated) */
           errorptr,                     /* Where to put an error message */
           (bravalue == OP_ASSERTBACK ||
            bravalue == OP_ASSERTBACK_NOT), /* TRUE if back assert */
-          condref,                      /* Condition reference number */
+          skipbytes,                    /* Skip over OP_COND/OP_BRANUMBER */
-          &subreqchar,                  /* For possible last char */
+          &subfirstbyte,                /* For possible first char */
-          &subcountlits,                /* For literal count */
+          &subreqbyte,                  /* For possible last char */
+          bcptr,                        /* Current branch chain */
           cd))                          /* Tables block */
        goto FAILED;
-Line 1693 
 for (;; ptr++)
+Line 2864 
 for (;; ptr++)
      /* If this is a conditional bracket, check that there are no more than
      two branches in the group. */
-     if (bravalue == OP_COND)
+     else if (bravalue == OP_COND)
        {
        uschar *tc = code;
        condcount = 0;
        do {
           condcount++;
-          tc += (tc[1] << 8) | tc[2];
+          tc += GET(tc,1);
           }
        while (*tc != OP_KET);
-Line 1709 
 for (;; ptr++)
+Line 2880 
 for (;; ptr++)
          *errorptr = ERR27;
          goto FAILED;
          }
+       /* If there is just one branch, we must not make use of its firstbyte or
+       reqbyte, because this is equivalent to an empty second branch. */
+       if (condcount == 1) subfirstbyte = subreqbyte = REQ_NONE;
        }
-     /* Handle updating of the required character. If the subpattern didn't
+     /* Handle updating of the required and first characters. Update for normal
-     set one, leave it as it was. Otherwise, update it for normal brackets of
+     brackets of all kinds, and conditions with two branches (see code above).
-     all kinds, forward assertions, and conditions with two branches. Don't
+     If the bracket is followed by a quantifier with zero repeat, we have to
-     update the literal count for forward assertions, however. If the bracket
+     back off. Hence the definition of zeroreqbyte and zerofirstbyte outside the
-     is followed by a quantifier with zero repeat, we have to back off. Hence
+     main loop so that they can be accessed for the back off. */
-     the definition of prevreqchar and subcountlits outside the main loop so
-     that they can be accessed for the back off. */
+     zeroreqbyte = reqbyte;
+     zerofirstbyte = firstbyte;
-     if (subreqchar > 0 &&
+     groupsetfirstbyte = FALSE;
-          (bravalue >= OP_BRA || bravalue == OP_ONCE || bravalue == OP_ASSERT ||
-          (bravalue == OP_COND && condcount == 2)))
+     if (bravalue >= OP_BRA || bravalue == OP_ONCE || bravalue == OP_COND)
        {
-       prevreqchar = *reqchar;
+       /* If we have not yet set a firstbyte in this branch, take it from the
-       *reqchar = subreqchar;
+       subpattern, remembering that it was set here so that a repeat of more
-       if (bravalue != OP_ASSERT) *countlits += subcountlits;
+       than one can replicate it as reqbyte if necessary. If the subpattern has
+       no firstbyte, set "none" for the whole branch. In both cases, a zero
+       repeat forces firstbyte to "none". */
+       if (firstbyte == REQ_UNSET)
+         {
+         if (subfirstbyte >= 0)
+           {
+           firstbyte = subfirstbyte;
+           groupsetfirstbyte = TRUE;
+           }
+         else firstbyte = REQ_NONE;
+         zerofirstbyte = REQ_NONE;
+         }
+       /* If firstbyte was previously set, convert the subpattern's firstbyte
+       into reqbyte if there wasn't one. */
+       else if (subfirstbyte >= 0 && subreqbyte < 0) subreqbyte = subfirstbyte;
+       /* If the subpattern set a required char (or set a first char that isn't
+       really the first char - see above), set it. */
+       if (subreqbyte >= 0) reqbyte = subreqbyte;
        }
+     /* For a forward assertion, we take the reqbyte, if set. This can be
+     helpful if the pattern that follows the assertion doesn't set a different
+     char. For example, it's useful for /(?=abcde).+/. We can't set firstbyte
+     for an assertion, however because it leads to incorrect effect for patterns
+     such as /(?=a)a.+/ when the "real" "a" would then become a reqbyte instead
+     of a firstbyte. This is overcome by a scan at the end if there's no
+     firstbyte, looking for an asserted first char. */
+     else if (bravalue == OP_ASSERT && subreqbyte >= 0) reqbyte = subreqbyte;
      /* Now update the main code pointer to the end of the group. */
      code = tempcode;
-Line 1758 
 for (;; ptr++)
+Line 2967 
 for (;; ptr++)
      if (c < 0)
        {
+       if (-c == ESC_Q)            /* Handle start of quoted string */
+         {
+         if (ptr[1] == '\\' && ptr[2] == 'E') ptr += 2; /* avoid empty string */
+           else inescq = TRUE;
+         continue;
+         }
+       /* For metasequences that actually match a character, we disable the
+       setting of a first character if it hasn't already been set. */
+       if (firstbyte == REQ_UNSET && -c > ESC_b && -c < ESC_Z)
+         firstbyte = REQ_NONE;
+       /* Set values to reset to if this is followed by a zero repeat. */
+       zerofirstbyte = firstbyte;
+       zeroreqbyte = reqbyte;
+       /* Back references are handled specially */
        if (-c >= ESC_REF)
          {
+         int number = -c - ESC_REF;
          previous = code;
          *code++ = OP_REF;
-         *code++ = -c - ESC_REF;
+         PUT2INC(code, 0, number);
          }
        else
          {
-Line 1788 
 for (;; ptr++)
+Line 3018 
 for (;; ptr++)
      code += 2;
      length = 0;
      do
+       {
+       /* If in \Q...\E, check for the end; if not, we always have a literal */
+       if (inescq)
+         {
+         if (c == '\\' && ptr[1] == 'E')
+           {
+           inescq = FALSE;
+           ptr++;
+           }
+         else
+           {
+           *code++ = c;
+           length++;
+           }
+         continue;
+         }
+       /* Skip white space and comments for /x patterns */
+       if ((options & PCRE_EXTENDED) != 0)
+         {
+         if ((cd->ctypes[c] & ctype_space) != 0) continue;
+         if (c == '#')
+           {
+           /* The space before the ; is to avoid a warning on a silly compiler
+           on the Macintosh. */
+           while ((c = *(++ptr)) != 0 && c != NEWLINE) ;
+           if (c == 0) break;
+           continue;
+           }
+         }
+       /* Backslash may introduce a data char or a metacharacter. Escaped items
+       are checked for validity in the pre-compiling pass. Stop the string
+       before a metaitem. */
+       if (c == '\\')
+         {
+         tempptr = ptr;
+         c = check_escape(&ptr, errorptr, *brackets, options, FALSE, cd);
+         if (c < 0) { ptr = tempptr; break; }
+         /* If a character is > 127 in UTF-8 mode, we have to turn it into
+         two or more characters in the UTF-8 encoding. */
+ #ifdef SUPPORT_UTF8
+         if (utf8 && c > 127)
+           {
+           uschar buffer[8];
+           int len = ord2utf8(c, buffer);
+           for (c = 0; c < len; c++) *code++ = buffer[c];
+           length += len;
+           continue;
+           }
+ #endif
+         }
+       /* Ordinary character or single-char escape */
+       *code++ = c;
+       length++;
+       }
+     /* This "while" is the end of the "do" above. */
+     while (length < MAXLIT && (cd->ctypes[c = *(++ptr)] & ctype_meta) == 0);
+     /* Update the first and last requirements. These are always bytes, even in
+     UTF-8 mode. However, there is a special case to be considered when there
+     are only one or two characters. Because this gets messy in UTF-8 mode, the
+     code is kept separate. When we get here "length" contains the number of
+     bytes. */
+ #ifdef SUPPORT_UTF8
+     if (utf8 && length > 1)
        {
-       if ((options & PCRE_EXTENDED) != 0)
+       uschar *t = previous + 3;                      /* After this code, t */
+       while (t < code && (*t & 0xc0) == 0x80) t++;   /* follows the 1st char */
+       /* Handle the case when there is only one multibyte character. It must
+       have at least two bytes because of the "length > 1" test above. */
+       if (t == code)
          {
-         if ((cd->ctypes[c] & ctype_space) != 0) continue;
+         /* If no previous first byte, set it from this character, but revert to
-         if (c == '#')
+         none on a zero repeat. */
+         if (firstbyte == REQ_UNSET)
            {
-           while ((c = *(++ptr)) != 0 && c != '\n');
+           zerofirstbyte = REQ_NONE;
-           if (c == 0) break;
+           firstbyte = previous[2];
-           continue;
            }
+         /* Otherwise, leave the first byte value alone, and don't change it on
+         a zero repeat */
+         else zerofirstbyte = firstbyte;
+         /* In both cases, a zero repeat resets the previous required byte */
+         zeroreqbyte = reqbyte;
          }
-       /* Backslash may introduce a data char or a metacharacter. Escaped items
+       /* Handle the case when there is more than one character. These may be
-       are checked for validity in the pre-compiling pass. Stop the string
+       single-byte or multibyte characters */
-       before a metaitem. */
-       if (c == '\\')
+       else
          {
-         tempptr = ptr;
+         uschar *t = code - 1;               /* After this code, t is at the */
-         c = check_escape(&ptr, errorptr, *brackets, options, FALSE, cd);
+         while ((*t & 0xc0) == 0x80) t--;    /* start of the last character */
-         if (c < 0) { ptr = tempptr; break; }
+         /* If no previous first byte, set it from the first character, and
+         retain it on a zero repeat (of the last character). The required byte
+         is reset on a zero repeat, either to the byte before the last
+         character, unless this is the first byte of the string. In that case,
+         it reverts to its previous value. */
+         if (firstbyte == REQ_UNSET)
+           {
+           zerofirstbyte = firstbyte = previous[2] | req_caseopt;
+           zeroreqbyte = (t - 1 == previous + 2)? reqbyte : t[-1] | req_caseopt;
+           }
+         /* If there was a previous first byte, leave it alone, and don't change
+         it on a zero repeat. The required byte is reset on a zero repeat to the
+         byte before the last character. */
+         else
+           {
+           zerofirstbyte = firstbyte;
+           zeroreqbyte = t[-1] | req_caseopt;
+           }
          }
-       /* Ordinary character or single-char escape */
+       /* In all cases (we know length > 1), the new required byte is the last
+       byte of the string. */
-       *code++ = c;
+       reqbyte = code[-1] | req_caseopt;
-       length++;
        }
-     /* This "while" is the end of the "do" above. */
+     else   /* End of UTF-8 coding */
+ #endif
-     while (length < 255 && (cd->ctypes[c = *(++ptr)] & ctype_meta) == 0);
+     /* This is the code for non-UTF-8 operation, either without UTF-8 support,
+     or when UTF-8 is not enabled. */
+       {
+       /* firstbyte was not previously set; take it from this string */
+       if (firstbyte == REQ_UNSET)
+         {
+         if (length == 1)
+           {
+           zerofirstbyte = REQ_NONE;
+           firstbyte = previous[2] | req_caseopt;
+           zeroreqbyte = reqbyte;
+           }
+         else
+           {
+           zerofirstbyte = firstbyte = previous[2] | req_caseopt;
+           zeroreqbyte = (length > 2)? (code[-2] | req_caseopt) : reqbyte;
+           reqbyte = code[-1] | req_caseopt;
+           }
+         }
-     /* Update the last character and the count of literals */
+       /* firstbyte was previously set */
-     prevreqchar = (length > 1)? code[-2] : *reqchar;
+       else
-     *reqchar = code[-1];
+         {
-     *countlits += length;
+         zerofirstbyte = firstbyte;
+         zeroreqbyte = (length == 1)? reqbyte : code[-2] | req_caseopt;
+         reqbyte = code[-1] | req_caseopt;
+         }
+       }
-     /* Compute the length and set it in the data vector, and advance to
+     /* Set the length in the data vector, and advance to the next state. */
-     the next state. */
      previous[1] = length;
-     if (length < 255) ptr--;
+     if (length < MAXLIT) ptr--;
      break;
      }
    }                   /* end of big loop */
-Line 1862 
 following branch to ensure they get set
+Line 3228 
 following branch to ensure they get set
  the new options into every subsequent branch compile.
  Argument:
-   options     the option bits
+   options        option bits, including any changes for this subpattern
-   optchanged  new ims options to set as if (?ims) were at the start, or -1
+   oldims         previous settings of ims option bits
-                for no change
+   brackets       -> int containing the number of extracting brackets used
-   brackets    -> int containing the number of extracting brackets used
+   codeptr        -> the address of the current code pointer
-   codeptr     -> the address of the current code pointer
+   ptrptr         -> the address of the current pattern pointer
-   ptrptr      -> the address of the current pattern pointer
+   errorptr       -> pointer to error message
-   errorptr    -> pointer to error message
+   lookbehind     TRUE if this is a lookbehind assertion
-   lookbehind  TRUE if this is a lookbehind assertion
+   skipbytes      skip this many bytes at start (for OP_COND, OP_BRANUMBER)
-   condref     > 0 for OPT_CREF setting at start of conditional group
+   firstbyteptr   place to put the first required character, or a negative number
-   reqchar     -> place to put the last required character, or a negative number
+   reqbyteptr     place to put the last required character, or a negative number
-   countlits   -> place to put the shortest literal count of any branch
+   bcptr          pointer to the chain of currently open branches
-   cd          points to the data block with tables pointers
+   cd             points to the data block with tables pointers etc.
  Returns:      TRUE on success
  */
  static BOOL
- compile_regex(int options, int optchanged, int *brackets, uschar **codeptr,
+ compile_regex(int options, int oldims, int *brackets, uschar **codeptr,
-   const uschar **ptrptr, const char **errorptr, BOOL lookbehind, int condref,
+   const uschar **ptrptr, const char **errorptr, BOOL lookbehind, int skipbytes,
-   int *reqchar, int *countlits, compile_data *cd)
+   int *firstbyteptr, int *reqbyteptr, branch_chain *bcptr, compile_data *cd)
  {
  const uschar *ptr = *ptrptr;
  uschar *code = *codeptr;
  uschar *last_branch = code;
  uschar *start_bracket = code;
  uschar *reverse_count = NULL;
- int oldoptions = options & PCRE_IMS;
+ int firstbyte, reqbyte;
- int branchreqchar, branchcountlits;
+ int branchfirstbyte, branchreqbyte;
+ branch_chain bc;
- *reqchar = -1;
+ bc.outer = bcptr;
- *countlits = INT_MAX;
+ bc.current = code;
- code += 3;
- /* At the start of a reference-based conditional group, insert the reference
+ firstbyte = reqbyte = REQ_UNSET;
- number as an OP_CREF item. */
- if (condref > 0)
+ /* Offset is set zero to mark that this bracket is still open */
-   {
-   *code++ = OP_CREF;
+ PUT(code, 1, 0);
-   *code++ = condref;
+ code += 1 + LINK_SIZE + skipbytes;
-   }
  /* Loop for each alternative branch */
  for (;;)
    {
-   int length;
+   /* Handle a change of ims options at the start of the branch */
-   /* Handle change of options */
-   if (optchanged >= 0)
+   if ((options & PCRE_IMS) != oldims)
      {
      *code++ = OP_OPT;
-     *code++ = optchanged;
+     *code++ = options & PCRE_IMS;
-     options = (options & ~PCRE_IMS) | optchanged;
      }
    /* Set up dummy OP_REVERSE if lookbehind assertion */
-Line 1925 
 for (;;)
+Line 3286 
 for (;;)
      {
      *code++ = OP_REVERSE;
      reverse_count = code;
-     *code++ = 0;
+     PUTINC(code, 0, 0);
-     *code++ = 0;
      }
    /* Now compile the branch */
-   if (!compile_branch(options, brackets, &code, &ptr, errorptr, &optchanged,
+   if (!compile_branch(&options, brackets, &code, &ptr, errorptr,
-       &branchreqchar, &branchcountlits, cd))
+         &branchfirstbyte, &branchreqbyte, &bc, cd))
      {
      *ptrptr = ptr;
      return FALSE;
      }
-   /* Fill in the length of the last branch */
+   /* If this is the first branch, the firstbyte and reqbyte values for the
+   branch become the values for the regex. */
-   length = code - last_branch;
+   if (*last_branch != OP_ALT)
-   last_branch[1] = length >> 8;
+     {
-   last_branch[2] = length & 255;
+     firstbyte = branchfirstbyte;
+     reqbyte = branchreqbyte;
+     }
-   /* Save the last required character if all branches have the same; a current
+   /* If this is not the first branch, the first char and reqbyte have to
-   value of -1 means unset, while -2 means "previous branch had no last required
+   match the values from all the previous branches. */
-   char".  */
-   if (*reqchar != -2)
+   else
      {
-     if (branchreqchar >= 0)
+     /* If we previously had a firstbyte, but it doesn't match the new branch,
+     we have to abandon the firstbyte for the regex, but if there was previously
+     no reqbyte, it takes on the value of the old firstbyte. */
+     if (firstbyte >= 0 && firstbyte != branchfirstbyte)
        {
-       if (*reqchar == -1) *reqchar = branchreqchar;
+       if (reqbyte < 0) reqbyte = firstbyte;
-       else if (*reqchar != branchreqchar) *reqchar = -2;
+       firstbyte = REQ_NONE;
        }
-     else *reqchar = -2;
-     }
-   /* Keep the shortest literal count */
+     /* If we (now or from before) have no firstbyte, a firstbyte from the
+     branch becomes a reqbyte if there isn't a branch reqbyte. */
+     if (firstbyte < 0 && branchfirstbyte >= 0 && branchreqbyte < 0)
+         branchreqbyte = branchfirstbyte;
-   if (branchcountlits < *countlits) *countlits = branchcountlits;
+     /* Now ensure that the reqbytes match */
-   DPRINTF(("literal count = %d min=%d\n", branchcountlits, *countlits));
+     if (reqbyte != branchreqbyte) reqbyte = REQ_NONE;
+     }
    /* If lookbehind, check that this branch matches a fixed-length string,
    and put the length into the OP_REVERSE item. Temporarily mark the end of
-Line 1969 
 for (;;)
+Line 3339 
 for (;;)
    if (lookbehind)
      {
+     int length;
      *code = OP_END;
-     length = find_fixedlength(last_branch);
+     length = find_fixedlength(last_branch, options);
      DPRINTF(("fixed length = %d\n", length));
      if (length < 0)
        {
-       *errorptr = ERR25;
+       *errorptr = (length == -2)? ERR36 : ERR25;
        *ptrptr = ptr;
        return FALSE;
        }
-     reverse_count[0] = (length >> 8);
+     PUT(reverse_count, 0, length);
-     reverse_count[1] = length & 255;
      }
-   /* Reached end of expression, either ')' or end of pattern. Insert a
+   /* Reached end of expression, either ')' or end of pattern. Go back through
-   terminating ket and the length of the whole bracketed item, and return,
+   the alternative branches and reverse the chain of offsets, with the field in
-   leaving the pointer at the terminating char. If any of the ims options
+   the BRA item now becoming an offset to the first alternative. If there are
-   were changed inside the group, compile a resetting op-code following. */
+   no alternatives, it points to the end of the group. The length in the
+   terminating ket is always the length of the whole bracketed item. If any of
+   the ims options were changed inside the group, compile a resetting op-code
+   following, except at the very end of the pattern. Return leaving the pointer
+   at the terminating char. */
    if (*ptr != '|')
      {
-     length = code - start_bracket;
+     int length = code - last_branch;
-     *code++ = OP_KET;
+     do
-     *code++ = length >> 8;
-     *code++ = length & 255;
-     if (optchanged >= 0)
        {
-       *code++ = OP_OPT;
+       int prev_length = GET(last_branch, 1);
-       *code++ = oldoptions;
+       PUT(last_branch, 1, length);
+       length = prev_length;
+       last_branch -= length;
        }
-     *codeptr = code;
+     while (length > 0);
-     *ptrptr = ptr;
-     return TRUE;
-     }
-   /* Another branch follows; insert an "or" node and advance the pointer. */
-   *code = OP_ALT;
-   last_branch = code;
-   code += 3;
-   ptr++;
-   }
- /* Control never reaches here */
- }
- /*************************************************
- *      Find first significant op code            *
- *************************************************/
- /* This is called by several functions that scan a compiled expression looking
+     /* Fill in the ket */
- for a fixed first character, or an anchoring op code etc. It skips over things
- that do not influence this. For one application, a change of caseless option is
- important.
- Arguments:
+     *code = OP_KET;
-   code       pointer to the start of the group
+     PUT(code, 1, code - start_bracket);
-   options    pointer to external options
+     code += 1 + LINK_SIZE;
-   optbit     the option bit whose changing is significant, or
-              zero if none are
-   optstop    TRUE to return on option change, otherwise change the options
-                value and continue
- Returns:     pointer to the first significant opcode
+     /* Resetting option if needed */
- */
- static const uschar*
+     if ((options & PCRE_IMS) != oldims && *ptr == ')')
- first_significant_code(const uschar *code, int *options, int optbit,
-   BOOL optstop)
- {
- for (;;)
-   {
-   switch ((int)*code)
-     {
-     case OP_OPT:
-     if (optbit > 0 && ((int)code[1] & optbit) != (*options & optbit))
        {
-       if (optstop) return code;
+       *code++ = OP_OPT;
-       *options = (int)code[1];
+       *code++ = oldims;
        }
-     code += 2;
-     break;
-     case OP_CREF:
+     /* Set values to pass back */
-     code += 2;
-     break;
-     case OP_WORD_BOUNDARY:
+     *codeptr = code;
-     case OP_NOT_WORD_BOUNDARY:
+     *ptrptr = ptr;
-     code++;
+     *firstbyteptr = firstbyte;
-     break;
+     *reqbyteptr = reqbyte;
+     return TRUE;
+     }
-     case OP_ASSERT_NOT:
+   /* Another branch follows; insert an "or" node. Its length field points back
-     case OP_ASSERTBACK:
+   to the previous branch while the bracket remains open. At the end the chain
-     case OP_ASSERTBACK_NOT:
+   is reversed. It's done like this so that the start of the bracket has a
-     do code += (code[1] << 8) + code[2]; while (*code == OP_ALT);
+   zero offset until it is closed, making it possible to detect recursion. */
-     code += 3;
-     break;
-     default:
+   *code = OP_ALT;
-     return code;
+   PUT(code, 1, code - last_branch);
-     }
+   bc.current = last_branch = code;
+   code += 1 + LINK_SIZE;
+   ptr++;
    }
  /* Control never reaches here */
  }
-Line 2089 
 all of whose alternatives start with OP_
+Line 3423 
 all of whose alternatives start with OP_
  it's anchored. However, if this is a multiline pattern, then only OP_SOD
  counts, since OP_CIRC can match in the middle.
+ We can also consider a regex to be anchored if OP_SOM starts all its branches.
+ This is the code for \G, which means "match at start of match position, taking
+ into account the match offset".
  A branch is also implicitly anchored if it starts with .* and DOTALL is set,
  because that will try the rest of the pattern at all possible matching points,
- so there is no point trying them again.
+ so there is no point trying again.... er ....
+ .... except when the .* appears inside capturing parentheses, and there is a
+ subsequent back reference to those parentheses. We haven't enough information
+ to catch that case precisely.
+ At first, the best we could do was to detect when .* was in capturing brackets
+ and the highest back reference was greater than or equal to that level.
+ However, by keeping a bitmap of the first 31 back references, we can catch some
+ of the more common cases more precisely.
  Arguments:
-   code       points to start of expression (the bracket)
+   code           points to start of expression (the bracket)
-   options    points to the options setting
+   options        points to the options setting
+   bracket_map    a bitmap of which brackets we are inside while testing; this
+                   handles up to substring 31; after that we just have to take
+                   the less precise approach
+   backref_map    the back reference bitmap
  Returns:     TRUE or FALSE
  */
  static BOOL
- is_anchored(register const uschar *code, int *options)
+ is_anchored(register const uschar *code, int *options, unsigned int bracket_map,
+   unsigned int backref_map)
  {
  do {
-    const uschar *scode = first_significant_code(code + 3, options,
+    const uschar *scode =
-      PCRE_MULTILINE, FALSE);
+      first_significant_code(code + 1+LINK_SIZE, options, PCRE_MULTILINE);
     register int op = *scode;
-    if (op >= OP_BRA || op == OP_ASSERT || op == OP_ONCE || op == OP_COND)
-      { if (!is_anchored(scode, options)) return FALSE; }
+    /* Capturing brackets */
+    if (op > OP_BRA)
+      {
+      int new_map;
+      op -= OP_BRA;
+      if (op > EXTRACT_BASIC_MAX) op = GET2(scode, 2+LINK_SIZE);
+      new_map = bracket_map | ((op < 32)? (1 << op) : 1);
+      if (!is_anchored(scode, options, new_map, backref_map)) return FALSE;
+      }
+    /* Other brackets */
+    else if (op == OP_BRA || op == OP_ASSERT || op == OP_ONCE || op == OP_COND)
+      {
+      if (!is_anchored(scode, options, bracket_map, backref_map)) return FALSE;
+      }
+    /* .* is not anchored unless DOTALL is set and it isn't in brackets that
+    are or may be referenced. */
     else if ((op == OP_TYPESTAR || op == OP_TYPEMINSTAR) &&
              (*options & PCRE_DOTALL) != 0)
-      { if (scode[1] != OP_ANY) return FALSE; }
+      {
-    else if (op != OP_SOD &&
+      if (scode[1] != OP_ANY || (bracket_map & backref_map) != 0) return FALSE;
+      }
+    /* Check for explicit anchoring */
+    else if (op != OP_SOD && op != OP_SOM &&
             ((*options & PCRE_MULTILINE) != 0 || op != OP_CIRC))
       return FALSE;
-    code += (code[1] << 8) + code[2];
+    code += GET(code, 1);
     }
- while (*code == OP_ALT);
+ while (*code == OP_ALT);   /* Loop for each alternative */
  return TRUE;
  }
-Line 2130 
 return TRUE;
+Line 3507 
 return TRUE;
  /* This is called to find out if every branch starts with ^ or .* so that
  "first char" processing can be done to speed things up in multiline
  matching and for non-DOTALL patterns that start with .* (which must start at
- the beginning or after \n).
+ the beginning or after \n). As in the case of is_anchored() (see above), we
+ have to take account of back references to capturing brackets that contain .*
+ because in that case we can't make the assumption.
- Argument:  points to start of expression (the bracket)
+ Arguments:
- Returns:   TRUE or FALSE
+   code           points to start of expression (the bracket)
+   bracket_map    a bitmap of which brackets we are inside while testing; this
+                   handles up to substring 31; after that we just have to take
+                   the less precise approach
+   backref_map    the back reference bitmap
+ Returns:         TRUE or FALSE
  */
  static BOOL
- is_startline(const uschar *code)
+ is_startline(const uschar *code, unsigned int bracket_map,
+   unsigned int backref_map)
  {
  do {
-    const uschar *scode = first_significant_code(code + 3, NULL, 0, FALSE);
+    const uschar *scode = first_significant_code(code + 1+LINK_SIZE, NULL, 0);
     register int op = *scode;
-    if (op >= OP_BRA || op == OP_ASSERT || op == OP_ONCE || op == OP_COND)
-      { if (!is_startline(scode)) return FALSE; }
+    /* Capturing brackets */
+    if (op > OP_BRA)
+      {
+      int new_map;
+      op -= OP_BRA;
+      if (op > EXTRACT_BASIC_MAX) op = GET2(scode, 2+LINK_SIZE);
+      new_map = bracket_map | ((op < 32)? (1 << op) : 1);
+      if (!is_startline(scode, new_map, backref_map)) return FALSE;
+      }
+    /* Other brackets */
+    else if (op == OP_BRA || op == OP_ASSERT || op == OP_ONCE || op == OP_COND)
+      { if (!is_startline(scode, bracket_map, backref_map)) return FALSE; }
+    /* .* is not anchored unless DOTALL is set and it isn't in brackets that
+    may be referenced. */
     else if (op == OP_TYPESTAR || op == OP_TYPEMINSTAR)
-      { if (scode[1] != OP_ANY) return FALSE; }
+      {
+      if (scode[1] != OP_ANY || (bracket_map & backref_map) != 0) return FALSE;
+      }
+    /* Check for explicit circumflex */
     else if (op != OP_CIRC) return FALSE;
-    code += (code[1] << 8) + code[2];
+    code += GET(code, 1);
     }
- while (*code == OP_ALT);
+ while (*code == OP_ALT);  /* Loop for each alternative */
  return TRUE;
  }
  /*************************************************
- *          Check for fixed first char            *
+ *       Check for asserted fixed first char      *
  *************************************************/
- /* Try to find out if there is a fixed first character. This is called for
+ /* During compilation, the "first char" settings from forward assertions are
- unanchored expressions, as it speeds up their processing quite considerably.
+ discarded, because they can cause conflicts with actual literals that follow.
- Consider each alternative branch. If they all start with the same char, or with
+ However, if we end up without a first char setting for an unanchored pattern,
- a bracket all of whose alternatives start with the same char (recurse ad lib),
+ it is worth scanning the regex to see if there is an initial asserted first
- then we return that char, otherwise -1.
+ char. If all branches start with the same asserted char, or with a bracket all
+ of whose alternatives start with the same asserted char (recurse ad lib), then
+ we return that char, otherwise -1.
  Arguments:
    code       points to start of expression (the bracket)
    options    pointer to the options (used to check casing changes)
+   inassert   TRUE if in an assertion
  Returns:     -1 or the fixed first char
  */
  static int
- find_firstchar(const uschar *code, int *options)
+ find_firstassertedchar(const uschar *code, int *options, BOOL inassert)
  {
  register int c = -1;
  do {
     int d;
-    const uschar *scode = first_significant_code(code + 3, options,
+    const uschar *scode =
-      PCRE_CASELESS, TRUE);
+      first_significant_code(code + 1+LINK_SIZE, options, PCRE_CASELESS);
     register int op = *scode;
     if (op >= OP_BRA) op = OP_BRA;
-Line 2193 
 do {
+Line 3605 
 do {
       case OP_ASSERT:
       case OP_ONCE:
       case OP_COND:
-      if ((d = find_firstchar(scode, options)) < 0) return -1;
+      if ((d = find_firstassertedchar(scode, options, op == OP_ASSERT)) < 0)
+        return -1;
       if (c < 0) c = d; else if (c != d) return -1;
       break;
-Line 2205 
 do {
+Line 3618 
 do {
       case OP_PLUS:
       case OP_MINPLUS:
-      if (c < 0) c = scode[1]; else if (c != scode[1]) return -1;
+      if (!inassert) return -1;
+      if (c < 0)
+        {
+        c = scode[1];
+        if ((*options & PCRE_CASELESS) != 0) c |= REQ_CASELESS;
+        }
+      else if (c != scode[1]) return -1;
       break;
       }
-    code += (code[1] << 8) + code[2];
+    code += GET(code, 1);
     }
  while (*code == OP_ALT);
  return c;
-Line 2218 
 return c;
+Line 3637 
 return c;
  /*************************************************
  *        Compile a Regular Expression            *
  *************************************************/
-Line 2242 
 pcre_compile(const char *pattern, int op
+Line 3660 
 pcre_compile(const char *pattern, int op
    int *erroroffset, const unsigned char *tables)
  {
  real_pcre *re;
- int length = 3;      /* For initial BRA plus length */
+ int length = 1 + LINK_SIZE;      /* For initial BRA plus length */
  int runlength;
- int c, reqchar, countlits;
+ int c, firstbyte, reqbyte;
  int bracount = 0;
- int top_backref = 0;
  int branch_extra = 0;
  int branch_newextra;
+ int item_count = -1;
+ int name_count = 0;
+ int max_name_size = 0;
+ #ifdef SUPPORT_UTF8
+ int lastcharlength = 0;
+ BOOL utf8;
+ BOOL class_utf8;
+ #endif
+ BOOL inescq = FALSE;
  unsigned int brastackptr = 0;
  size_t size;
  uschar *code;
+ const uschar *codestart;
  const uschar *ptr;
  compile_data compile_block;
  int brastack[BRASTACK_SIZE];
  uschar bralenstack[BRASTACK_SIZE];
- #ifdef DEBUG
- uschar *code_base, *code_end;
- #endif
  /* We can't pass back an error message if errorptr is NULL; I guess the best we
  can do is just return NULL. */
-Line 2276 
 if (erroroffset == NULL)
+Line 3699 
 if (erroroffset == NULL)
    }
  *erroroffset = 0;
+ /* Can't support UTF8 unless PCRE has been compiled to include the code. */
+ #ifdef SUPPORT_UTF8
+ utf8 = (options & PCRE_UTF8) != 0;
+ #else
+ if ((options & PCRE_UTF8) != 0)
+   {
+   *errorptr = ERR32;
+   return NULL;
+   }
+ #endif
  if ((options & ~PUBLIC_OPTIONS) != 0)
    {
    *errorptr = ERR17;
-Line 2290 
 compile_block.fcc = tables + fcc_offset;
+Line 3725 
 compile_block.fcc = tables + fcc_offset;
  compile_block.cbits = tables + cbits_offset;
  compile_block.ctypes = tables + ctypes_offset;
+ /* Maximum back reference and backref bitmap. This is updated for numeric
+ references during the first pass, but for named references during the actual
+ compile pass. The bitmap records up to 31 back references to help in deciding
+ whether (.*) can be treated as anchored or not. */
+ compile_block.top_backref = 0;
+ compile_block.backref_map = 0;
  /* Reflect pattern for debugging output */
  DPRINTF(("------------------------------------------------------------------\n"));
-Line 2298 
 DPRINTF(("%s\n", pattern));
+Line 3741 
 DPRINTF(("%s\n", pattern));
  /* The first thing to do is to make a pass over the pattern to compute the
  amount of store required to hold the compiled code. This does not have to be
  perfect as long as errors are overestimates. At the same time we can detect any
- internal flag settings. Make an attempt to correct for any counted white space
+ flag settings right at the start, and extract them. Make an attempt to correct
- if an "extended" flag setting appears late in the pattern. We can't be so
+ for any counted white space if an "extended" flag setting appears late in the
- clever for #-comments. */
+ pattern. We can't be so clever for #-comments. */
  ptr = (const uschar *)(pattern - 1);
  while ((c = *(++ptr)) != 0)
    {
    int min, max;
-   int class_charcount;
+   int class_optcount;
+   int bracket_length;
+   int duplength;
+   /* If we are inside a \Q...\E sequence, all chars are literal */
+   if (inescq) goto NORMAL_CHAR;
+   /* Otherwise, first check for ignored whitespace and comments */
    if ((options & PCRE_EXTENDED) != 0)
      {
      if ((compile_block.ctypes[c] & ctype_space) != 0) continue;
      if (c == '#')
        {
-       while ((c = *(++ptr)) != 0 && c != '\n');
+       /* The space before the ; is to avoid a warning on a silly compiler
+       on the Macintosh. */
+       while ((c = *(++ptr)) != 0 && c != NEWLINE) ;
+       if (c == 0) break;
        continue;
        }
      }
+   item_count++;    /* Is zero for the first non-comment item */
    switch(c)
      {
      /* A backslashed item may be an escaped "normal" character or a
-Line 2337 
 while ((c = *(++ptr)) != 0)
+Line 3793 
 while ((c = *(++ptr)) != 0)
          goto NORMAL_CHAR;
          }
        }
+     /* If \Q, enter "literal" mode */
+     if (-c == ESC_Q)
+       {
+       inescq = TRUE;
+       continue;
+       }
+     /* Other escapes need one byte, and are of length one for repeats */
      length++;
+ #ifdef SUPPORT_UTF8
+     lastcharlength = 1;
+ #endif
-     /* A back reference needs an additional char, plus either one or 5
+     /* A back reference needs an additional 2 bytes, plus either one or 5
      bytes for a repeat. We also need to keep the value of the highest
      back reference. */
      if (c <= -ESC_REF)
        {
        int refnum = -c - ESC_REF;
-       if (refnum > top_backref) top_backref = refnum;
+       compile_block.backref_map |= (refnum < 32)? (1 << refnum) : 1;
-       length++;   /* For single back reference */
+       if (refnum > compile_block.top_backref)
+         compile_block.top_backref = refnum;
+       length += 2;   /* For single back reference */
        if (ptr[1] == '{' && is_counted_repeat(ptr+2, &compile_block))
          {
          ptr = read_repeat_counts(ptr+2, &min, &max, errorptr, &compile_block);
-Line 2361 
 while ((c = *(++ptr)) != 0)
+Line 3833 
 while ((c = *(++ptr)) != 0)
        }
      continue;
-     case '^':
+     case '^':     /* Single-byte metacharacters */
      case '.':
      case '$':
-     case '*':     /* These repeats won't be after brackets; */
-     case '+':     /* those are handled separately */
-     case '?':
      length++;
+ #ifdef SUPPORT_UTF8
+     lastcharlength = 1;
+ #endif
      continue;
-     /* This covers the cases of repeats after a single char, metachar, class,
+     case '*':            /* These repeats won't be after brackets; */
-     or back reference. */
+     case '+':            /* those are handled separately */
+     case '?':
+     length++;
+     goto POSESSIVE;      /* A few lines below */
+     /* This covers the cases of braced repeats after a single char, metachar,
+     class, or back reference. */
      case '{':
      if (!is_counted_repeat(ptr+1, &compile_block)) goto NORMAL_CHAR;
      ptr = read_repeat_counts(ptr+1, &min, &max, errorptr, &compile_block);
      if (*errorptr != NULL) goto PCRE_ERROR_RETURN;
+     /* These special cases just insert one extra opcode */
      if ((min == 0 && (max == 1 || max == -1)) ||
        (min == 1 && max == -1))
          length++;
+     /* These cases might insert additional copies of a preceding character. */
      else
        {
-       length--;   /* Uncount the original char or metachar */
+ #ifdef SUPPORT_UTF8
-       if (min == 1) length++; else if (min > 0) length += 4;
+       /* In UTF-8 mode, we should find the length in lastcharlength */
-       if (max > 0) length += 4; else length += 2;
+       if (utf8)
+         {
+         if (min != 1)
+           {
+           length -= lastcharlength;   /* Uncount the original char or metachar */
+           if (min > 0) length += 3 + lastcharlength;
+           }
+         length += lastcharlength + ((max > 0)? 3 : 1);
+         }
+       else
+ #endif
+       /* Not UTF-8 mode: all characters are one byte */
+         {
+         if (min != 1)
+           {
+           length--;   /* Uncount the original char or metachar */
+           if (min > 0) length += 4;
+           }
+         length += (max > 0)? 4 : 2;
+         }
+       }
+     if (ptr[1] == '?') ptr++;      /* Needs no extra length */
+     POSESSIVE:                     /* Test for possessive quantifier */
+     if (ptr[1] == '+')
+       {
+       ptr++;
+       length += 2 + 2*LINK_SIZE;   /* Allow for atomic brackets */
        }
-     if (ptr[1] == '?') ptr++;
      continue;
      /* An alternation contains an offset to the next branch or ket. If any ims
      options changed in the previous branch(es), and/or if we are in a
      lookbehind assertion, extra space will be needed at the start of the
      branch. This is handled by branch_extra. */
+     case '|':
+     length += 1 + LINK_SIZE + branch_extra;
+     continue;
+     /* A character class uses 33 characters provided that all the character
+     values are less than 256. Otherwise, it uses a bit map for low valued
+     characters, and individual items for others. Don't worry about character
+     types that aren't allowed in classes - they'll get picked up during the
+     compile. A character class that contains only one single-byte character
+     uses 2 or 3 bytes, depending on whether it is negated or not. Notice this
+     where we can. (In UTF-8 mode we can do this only for chars < 128.) */
+     case '[':
+     class_optcount = 0;
+ #ifdef SUPPORT_UTF8
+     class_utf8 = FALSE;
+ #endif
+     if (*(++ptr) == '^') ptr++;
+     /* Written as a "do" so that an initial ']' is taken as data */
+     if (*ptr != 0) do
+       {
+       /* Inside \Q...\E everything is literal except \E */
+       if (inescq)
+         {
+         if (*ptr != '\\' || ptr[1] != 'E') goto NON_SPECIAL_CHARACTER;
+         inescq = FALSE;
+         ptr += 1;
+         continue;
+         }
+       /* Outside \Q...\E, check for escapes */
+       if (*ptr == '\\')
+         {
+ #ifdef SUPPORT_UTF8
+         int prevchar = ptr[-1];
+ #endif
+         int ch = check_escape(&ptr, errorptr, bracount, options, TRUE,
+           &compile_block);
+         if (*errorptr != NULL) goto PCRE_ERROR_RETURN;
+         /* \b is backspace inside a class */
+         if (-ch == ESC_b) ch = '\b';
+         /* \Q enters quoting mode */
+         if (-ch == ESC_Q)
+           {
+           inescq = TRUE;
+           continue;
+           }
+         /* Handle escapes that turn into characters */
+         if (ch >= 0)
+           {
+ #ifdef SUPPORT_UTF8
+           if (utf8)
+             {
+             if (ch > 127) class_optcount = 10;  /* Ensure > 1 */
+             if (ch > 255)
+               {
+               uschar buffer[6];
+               if (!class_utf8)
+                 {
+                 class_utf8 = TRUE;
+                 length += LINK_SIZE + 1 + 1;
+                 }
+               length += 1 + ord2utf8(ch, buffer);
+               /* If this wide character is preceded by '-', add an extra 2 to
+               the length in case the previous character was < 128, because in
+               this case the whole range will be put into the list. */
+               if (prevchar == '-') length += 2;
+               }
+             }
+ #endif
+           class_optcount++;            /* for possible optimization */
+           }
+         else class_optcount = 10;      /* \d, \s etc; make sure > 1 */
+         }
+       /* Check the syntax for POSIX stuff. The bits we actually handle are
+       checked during the real compile phase. */
-     case '|':
+       else if (*ptr == '[' && check_posix_syntax(ptr, &ptr, &compile_block))
-     length += 3 + branch_extra;
+         {
-     continue;
+         ptr++;
+         class_optcount = 10;    /* Make sure > 1 */
+         }
-     /* A character class uses 33 characters. Don't worry about character types
+       /* Anything else just increments the possible optimization count. If
-     that aren't allowed in classes - they'll get picked up during the compile.
+       there are wide characters, we are going to have to use an XCLASS. */
-     A character class that contains only one character uses 2 or 3 bytes,
-     depending on whether it is negated or not. Notice this where we can. */
-     case '[':
+       else
-     class_charcount = 0;
-     if (*(++ptr) == '^') ptr++;
-     do
-       {
-       if (*ptr == '\\')
          {
-         int ch = check_escape(&ptr, errorptr, bracount, options, TRUE,
+         NON_SPECIAL_CHARACTER:
-           &compile_block);
+         class_optcount++;
-         if (*errorptr != NULL) goto PCRE_ERROR_RETURN;
-         if (-ch == ESC_b) class_charcount++; else class_charcount = 10;
+ #ifdef SUPPORT_UTF8
+         if (utf8)
+           {
+           int c;
+           int extra = 0;
+           GETCHARLEN(c, ptr, extra);
+           if (c > 127) class_optcount = 10;   /* No optimization possible */
+           if (c > 255)
+             {
+             if (!class_utf8)
+               {
+               class_utf8 = TRUE;
+               length += LINK_SIZE + 1 + 1;
+               }
+             length += 2 + extra;
+             /* If this wide character is preceded by '-', add an extra 2 to
+             the length in case the previous character was < 128, because in
+             this case the whole range will be put into the list. */
+             if (ptr[-1] == '-') length += 2;
+             /* Advance to the end of this character */
+             ptr += extra;
+             }
+           }
+ #endif
          }
-       else class_charcount++;
-       ptr++;
        }
-     while (*ptr != 0 && *ptr != ']');
+     while (*(++ptr) != 0 && (inescq || *ptr != ']')); /* Concludes "do" above */
+     if (*ptr == 0)                          /* Missing terminating ']' */
+       {