/[pcre]/code/trunk/pcre_dfa_exec.c

Diff of /code/trunk/pcre_dfa_exec.c

Parent Directory | Revision Log | View Patch Patch

-revision 77 by nigel,
Sat Feb 24 21:40:45 2007 UTC
+revision 459 by ph10,
Sun Oct  4 09:21:39 2009 UTC
 Line 3
  *************************************************/
  /* PCRE is a library of functions to support regular expressions whose syntax
- and semantics are as close as possible to those of the Perl 5 language.
+ and semantics are as close as possible to those of the Perl 5 language (but see
+ below for why this module is different).
                         Written by Philip Hazel
-            Copyright (c) 1997-2005 University of Cambridge
+            Copyright (c) 1997-2009 University of Cambridge
  -----------------------------------------------------------------------------
  Redistribution and use in source and binary forms, with or without
-Line 39 
 POSSIBILITY OF SUCH DAMAGE.
+Line 40 
 POSSIBILITY OF SUCH DAMAGE.
  /* This module contains the external function pcre_dfa_exec(), which is an
- alternative matching function that uses a DFA algorithm. This is NOT Perl-
+ alternative matching function that uses a sort of DFA algorithm (not a true
- compatible, but it has advantages in certain applications. */
+ FSM). This is NOT Perl- compatible, but it has advantages in certain
+ applications. */
+ /* NOTE ABOUT PERFORMANCE: A user of this function sent some code that improved
+ the performance of his patterns greatly. I could not use it as it stood, as it
+ was not thread safe, and made assumptions about pattern sizes. Also, it caused
+ test 7 to loop, and test 9 to crash with a segfault.
+ The issue is the check for duplicate states, which is done by a simple linear
+ search up the state list. (Grep for "duplicate" below to find the code.) For
+ many patterns, there will never be many states active at one time, so a simple
+ linear search is fine. In patterns that have many active states, it might be a
+ bottleneck. The suggested code used an indexing scheme to remember which states
+ had previously been used for each character, and avoided the linear search when
+ it knew there was no chance of a duplicate. This was implemented when adding
+ states to the state lists.
+ I wrote some thread-safe, not-limited code to try something similar at the time
+ of checking for duplicates (instead of when adding states), using index vectors
+ on the stack. It did give a 13% improvement with one specially constructed
+ pattern for certain subject strings, but on other strings and on many of the
+ simpler patterns in the test suite it did worse. The major problem, I think,
+ was the extra time to initialize the index. This had to be done for each call
+ of internal_dfa_exec(). (The supplied patch used a static vector, initialized
+ only once - I suspect this was the cause of the problems with the tests.)
+ Overall, I concluded that the gains in some cases did not outweigh the losses
+ in others, so I abandoned this code. */
+ #ifdef HAVE_CONFIG_H
+ #include "config.h"
+ #endif
+ #define NLBLOCK md             /* Block containing newline information */
+ #define PSSTART start_subject  /* Field containing processed string start */
+ #define PSEND   end_subject    /* Field containing processed string end */
  #include "pcre_internal.h"
-Line 51 
 compatible, but it has advantages in cer
+Line 89 
 compatible, but it has advantages in cer
  #define SP "                   "
  /*************************************************
  *      Code parameters and static tables         *
  *************************************************/
  /* These are offsets that are used to turn the OP_TYPESTAR and friends opcodes
- into others, under special conditions. A gap of 10 between the blocks should be
+ into others, under special conditions. A gap of 20 between the blocks should be
- enough. */
+ enough. The resulting opcodes don't have to be less than 256 because they are
+ never stored, so we push them well clear of the normal opcodes. */
- #define OP_PROP_EXTRA    (EXTRACT_BASIC_MAX+1)
- #define OP_EXTUNI_EXTRA  (EXTRACT_BASIC_MAX+11)
+ #define OP_PROP_EXTRA       300
+ #define OP_EXTUNI_EXTRA     320
+ #define OP_ANYNL_EXTRA      340
+ #define OP_HSPACE_EXTRA     360
+ #define OP_VSPACE_EXTRA     380
  /* This table identifies those opcodes that are followed immediately by a
  character that is to be tested in some way. This makes is possible to
  centralize the loading of these characters. In the case of Type * etc, the
  "character" is the opcode for \D, \d, \S, \s, \W, or \w, which will always be a
- small value. */
+ small value. ***NOTE*** If the start of this table is modified, the two tables
+ that follow must also be modified. */
- static uschar coptable[] = {
+ static const uschar coptable[] = {
 ,                             /* End                                    */
-, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* \A, \G, \B, \b, \D, \d, \S, \s, \W, \w */
+, 0, 0, 0, 0,                 /* \A, \G, \K, \B, \b                     */
-, 0,                          /* Any, Anybyte                           */
+, 0, 0, 0, 0, 0,              /* \D, \d, \S, \s, \W, \w                 */
+, 0, 0,                       /* Any, AllAny, Anybyte                   */
 , 0, 0,                       /* NOTPROP, PROP, EXTUNI                  */
+, 0, 0, 0, 0,                 /* \R, \H, \h, \V, \v                     */
 , 0, 0, 0, 0,                 /* \Z, \z, Opt, ^, $                      */
 ,                             /* Char                                   */
 ,                             /* Charnc                                 */
-Line 82 
 static uschar coptable[] = {
+Line 126 
 static uschar coptable[] = {
    /* Positive single-char repeats                                          */
 , 1, 1, 1, 1, 1,              /* *, *?, +, +?, ?, ??                    */
 , 3, 3,                       /* upto, minupto, exact                   */
+, 1, 1, 3,                    /* *+, ++, ?+, upto+                      */
    /* Negative single-char repeats - only for chars < 256                   */
 , 1, 1, 1, 1, 1,              /* NOT *, *?, +, +?, ?, ??                */
 , 3, 3,                       /* NOT upto, minupto, exact               */
+, 1, 1, 3,                    /* NOT *+, ++, ?+, updo+                  */
    /* Positive type repeats                                                 */
 , 1, 1, 1, 1, 1,              /* Type *, *?, +, +?, ?, ??               */
 , 3, 3,                       /* Type upto, minupto, exact              */
+, 1, 1, 3,                    /* Type *+, ++, ?+, upto+                 */
    /* Character class & ref repeats                                         */
 , 0, 0, 0, 0, 0,              /* *, *?, +, +?, ?, ??                    */
 , 0,                          /* CRRANGE, CRMINRANGE                    */
-Line 106 
 static uschar coptable[] = {
+Line 153 
 static uschar coptable[] = {
 ,                             /* Assert behind                          */
 ,                             /* Assert behind not                      */
 ,                             /* Reverse                                */
-,                             /* Once                                   */
+, 0, 0, 0,                    /* ONCE, BRA, CBRA, COND                  */
-,                             /* COND                                   */
+, 0, 0,                       /* SBRA, SCBRA, SCOND                     */
 ,                             /* CREF                                   */
+,                             /* RREF                                   */
+,                             /* DEF                                    */
 , 0,                          /* BRAZERO, BRAMINZERO                    */
-,                             /* BRANUMBER                              */
+, 0, 0, 0,                    /* PRUNE, SKIP, THEN, COMMIT              */
-                             /* BRA                                    */
+, 0, 0                        /* FAIL, ACCEPT, SKIPZERO                 */
  };
  /* These 2 tables allow for compact code for testing for \D, \d, \S, \s, \W,
  and \w */
- static uschar toptable1[] = {
+ static const uschar toptable1[] = {
-, 0, 0, 0, 0,
+, 0, 0, 0, 0, 0,
    ctype_digit, ctype_digit,
    ctype_space, ctype_space,
    ctype_word,  ctype_word,
-                              /* OP_ANY */
+, 0                            /* OP_ANY, OP_ALLANY */
  };
- static uschar toptable2[] = {
+ static const uschar toptable2[] = {
-, 0, 0, 0, 0,
+, 0, 0, 0, 0, 0,
    ctype_digit, 0,
    ctype_space, 0,
    ctype_word,  0,
-                              /* OP_ANY */
+, 1                            /* OP_ANY, OP_ALLANY */
  };
-Line 202 
 Arguments:
+Line 251 
 Arguments:
    rlevel            function call recursion level
    recursing         regex recursive call level
- Returns:            > 0 =>
+ Returns:            > 0 => number of match offset pairs placed in offsets
-                     = 0 =>
+                     = 0 => offsets overflowed; longest matches are present
                       -1 => failed to match
                     < -1 => some kind of unexpected problem
-Line 277 
 stateblock *next_active_state, *next_new
+Line 326 
 stateblock *next_active_state, *next_new
  const uschar *ctypes, *lcc, *fcc;
  const uschar *ptr;
- const uschar *end_code;
+ const uschar *end_code, *first_op;
  int active_count, new_count, match_count;
-Line 288 
 const uschar *start_subject = md->start_
+Line 337 
 const uschar *start_subject = md->start_
  const uschar *end_subject = md->end_subject;
  const uschar *start_code = md->start_code;
+ #ifdef SUPPORT_UTF8
  BOOL utf8 = (md->poptions & PCRE_UTF8) != 0;
+ #else
+ BOOL utf8 = FALSE;
+ #endif
  rlevel++;
  offsetcount &= (-2);
-Line 311 
 active_states = (stateblock *)(workspace
+Line 364 
 active_states = (stateblock *)(workspace
  next_new_state = new_states = active_states + wscount;
  new_count = 0;
+ first_op = this_start_code + 1 + LINK_SIZE +
+   ((*this_start_code == OP_CBRA || *this_start_code == OP_SCBRA)? 2:0);
  /* The first thing in any (sub) pattern is a bracket of some sort. Push all
  the alternative states onto the list, and find out where the end is. This
  makes is possible to use this function recursively, when we want to stop at a
-Line 320 
 If the first opcode in the first alterna
+Line 376 
 If the first opcode in the first alterna
  a backward assertion. In that case, we have to find out the maximum amount to
  move back, and set up each alternative appropriately. */
- if (this_start_code[1+LINK_SIZE] == OP_REVERSE)
+ if (*first_op == OP_REVERSE)
    {
    int max_back = 0;
    int gone_back;
-Line 361 
 if (this_start_code[1+LINK_SIZE] == OP_R
+Line 417 
 if (this_start_code[1+LINK_SIZE] == OP_R
        current_subject - start_subject : max_back;
      current_subject -= gone_back;
      }
+   /* Save the earliest consulted character */
+   if (current_subject < md->start_used_ptr)
+     md->start_used_ptr = current_subject;
    /* Now we can process the individual branches. */
-Line 402 
 else
+Line 463 
 else
    else
      {
+     int length = 1 + LINK_SIZE +
+       ((*this_start_code == OP_CBRA || *this_start_code == OP_SCBRA)? 2:0);
      do
        {
-       ADD_NEW(end_code - start_code + 1 + LINK_SIZE, 0);
+       ADD_NEW(end_code - start_code + length, 0);
        end_code += GET(end_code, 1);
+       length = 1 + LINK_SIZE;
        }
      while (*end_code == OP_ALT);
      }
-Line 421 
 ptr = current_subject;
+Line 485 
 ptr = current_subject;
  for (;;)
    {
    int i, j;
-   int c, d, clen, dlen;
+   int clen, dlen;
+   unsigned int c, d;
+   int forced_fail = 0;
+   int reached_end = 0;
    /* Make the new state list into the active state list and empty the
    new state list. */
-Line 457 
 for (;;)
+Line 524 
 for (;;)
    if (ptr < end_subject)
      {
-     clen = 1;
+     clen = 1;        /* Number of bytes in the character */
  #ifdef SUPPORT_UTF8
      if (utf8) { GETCHARLEN(c, ptr, clen); } else
  #endif  /* SUPPORT_UTF8 */
-Line 465 
 for (;;)
+Line 532 
 for (;;)
      }
    else
      {
-     clen = 0;    /* At end subject */
+     clen = 0;        /* This indicates the end of the subject */
-     c = -1;
+     c = NOTACHAR;    /* This value should never actually be used */
      }
    /* Scan up the active states and act on each one. The result of an action
-Line 479 
 for (;;)
+Line 546 
 for (;;)
      stateblock *current_state = active_states + i;
      const uschar *code;
      int state_offset = current_state->offset;
-     int count, codevalue;
+     int count, codevalue, rrc;
-     int chartype, othercase;
  #ifdef DEBUG
      printf ("%.*sProcessing state %d c=", rlevel*2-2, SP, state_offset);
-     if (c < 0) printf("-1\n");
+     if (clen == 0) printf("EOL\n");
        else if (c > 32 && c < 127) printf("'%c'\n", c);
          else printf("0x%02x\n", c);
  #endif
-Line 512 
 for (;;)
+Line 578 
 for (;;)
          }
        }
-     /* Check for a duplicate state with the same count, and skip if found. */
+     /* Check for a duplicate state with the same count, and skip if found.
+     See the note at the head of this module about the possibility of improving
+     performance here. */
      for (j = 0; j < i; j++)
        {
-Line 528 
 for (;;)
+Line 596 
 for (;;)
      code = start_code + state_offset;
      codevalue = *code;
-     if (codevalue >= OP_BRA) codevalue = OP_BRA; /* All brackets are equal */
      /* If this opcode is followed by an inline character, load it. It is
      tempting to test for the presence of a subject character here, but that
-Line 536 
 for (;;)
+Line 603 
 for (;;)
      permitted.
      We also use this mechanism for opcodes such as OP_TYPEPLUS that take an
-     argument that is not a data character - but is always one byte long.
+     argument that is not a data character - but is always one byte long. We
-     Unfortunately, we have to take special action to deal with  \P, \p, and
+     have to take special action to deal with  \P, \p, \H, \h, \V, \v and \X in
-     \X in this case. To keep the other cases fast, convert these ones to new
+     this case. To keep the other cases fast, convert these ones to new opcodes.
-     opcodes. */
+     */
      if (coptable[codevalue] > 0)
        {
-Line 550 
 for (;;)
+Line 617 
 for (;;)
        d = code[coptable[codevalue]];
        if (codevalue >= OP_TYPESTAR)
          {
-         if (d == OP_ANYBYTE) return PCRE_ERROR_DFA_UITEM;
+         switch(d)
-         if (d >= OP_NOTPROP)
+           {
-           codevalue += (d == OP_EXTUNI)? OP_EXTUNI_EXTRA : OP_PROP_EXTRA;
+           case OP_ANYBYTE: return PCRE_ERROR_DFA_UITEM;
+           case OP_NOTPROP:
+           case OP_PROP: codevalue += OP_PROP_EXTRA; break;
+           case OP_ANYNL: codevalue += OP_ANYNL_EXTRA; break;
+           case OP_EXTUNI: codevalue += OP_EXTUNI_EXTRA; break;
+           case OP_NOT_HSPACE:
+           case OP_HSPACE: codevalue += OP_HSPACE_EXTRA; break;
+           case OP_NOT_VSPACE:
+           case OP_VSPACE: codevalue += OP_VSPACE_EXTRA; break;
+           default: break;
+           }
          }
        }
      else
        {
        dlen = 0;         /* Not strictly necessary, but compilers moan */
-       d = -1;           /* if these variables are not set. */
+       d = NOTACHAR;     /* if these variables are not set. */
        }
-Line 570 
 for (;;)
+Line 647 
 for (;;)
  /* ========================================================================== */
        /* Reached a closing bracket. If not at the end of the pattern, carry
        on with the next opcode. Otherwise, unless we have an empty string and
-       PCRE_NOTEMPTY is set, save the match data, shifting up all previous
+       PCRE_NOTEMPTY is set, or PCRE_NOTEMPTY_ATSTART is set and we are at the
+       start of the subject, save the match data, shifting up all previous
        matches so we always have the longest first. */
        case OP_KET:
-Line 584 
 for (;;)
+Line 662 
 for (;;)
            ADD_ACTIVE(state_offset - GET(code, 1), 0);
            }
          }
-       else if (ptr > current_subject || (md->moptions & PCRE_NOTEMPTY) == 0)
+       else
          {
-         if (match_count < 0) match_count = (offsetcount >= 2)? 1 : 0;
+         reached_end++;    /* Count branches that reach the end */
-           else if (match_count > 0 && ++match_count * 2 >= offsetcount)
+         if (ptr > current_subject ||
-             match_count = 0;
+             ((md->moptions & PCRE_NOTEMPTY) == 0 &&
-         count = ((match_count == 0)? offsetcount : match_count * 2) - 2;
+               ((md->moptions & PCRE_NOTEMPTY_ATSTART) == 0 ||
-         if (count > 0) memmove(offsets + 2, offsets, count * sizeof(int));
+                 current_subject > start_subject + md->start_offset)))
-         if (offsetcount >= 2)
+           {
-           {
+           if (match_count < 0) match_count = (offsetcount >= 2)? 1 : 0;
-           offsets[0] = current_subject - start_subject;
+             else if (match_count > 0 && ++match_count * 2 >= offsetcount)
-           offsets[1] = ptr - start_subject;
+               match_count = 0;
-           DPRINTF(("%.*sSet matched string = \"%.*s\"\n", rlevel*2-2, SP,
+           count = ((match_count == 0)? offsetcount : match_count * 2) - 2;
-             offsets[1] - offsets[0], current_subject));
+           if (count > 0) memmove(offsets + 2, offsets, count * sizeof(int));
-           }
+           if (offsetcount >= 2)
-         if ((md->moptions & PCRE_DFA_SHORTEST) != 0)
+             {
-           {
+             offsets[0] = current_subject - start_subject;
-           DPRINTF(("%.*sEnd of internal_dfa_exec %d: returning %d\n"
+             offsets[1] = ptr - start_subject;
-             "%.*s---------------------\n\n", rlevel*2-2, SP, rlevel,
+             DPRINTF(("%.*sSet matched string = \"%.*s\"\n", rlevel*2-2, SP,
-             match_count, rlevel*2-2, SP));
+               offsets[1] - offsets[0], current_subject));
-           return match_count;
+             }
-           }
+           if ((md->moptions & PCRE_DFA_SHORTEST) != 0)
+             {
+             DPRINTF(("%.*sEnd of internal_dfa_exec %d: returning %d\n"
+               "%.*s---------------------\n\n", rlevel*2-2, SP, rlevel,
+               match_count, rlevel*2-2, SP));
+             return match_count;
+             }
+           }
          }
        break;
-Line 620 
 for (;;)
+Line 705 
 for (;;)
        /*-----------------------------------------------------------------*/
        case OP_BRA:
+       case OP_SBRA:
        do
          {
          ADD_ACTIVE(code - start_code + 1 + LINK_SIZE, 0);
-Line 629 
 for (;;)
+Line 715 
 for (;;)
        break;
        /*-----------------------------------------------------------------*/
+       case OP_CBRA:
+       case OP_SCBRA:
+       ADD_ACTIVE(code - start_code + 3 + LINK_SIZE,  0);
+       code += GET(code, 1);
+       while (*code == OP_ALT)
+         {
+         ADD_ACTIVE(code - start_code + 1 + LINK_SIZE,  0);
+         code += GET(code, 1);
+         }
+       break;
+       /*-----------------------------------------------------------------*/
        case OP_BRAZERO:
        case OP_BRAMINZERO:
        ADD_ACTIVE(state_offset + 1, 0);
-Line 638 
 for (;;)
+Line 736 
 for (;;)
        break;
        /*-----------------------------------------------------------------*/
-       case OP_BRANUMBER:
+       case OP_SKIPZERO:
-       ADD_ACTIVE(state_offset + 1 + LINK_SIZE, 0);
+       code += 1 + GET(code, 2);
+       while (*code == OP_ALT) code += GET(code, 1);
+       ADD_ACTIVE(code - start_code + 1 + LINK_SIZE, 0);
        break;
        /*-----------------------------------------------------------------*/
        case OP_CIRC:
        if ((ptr == start_subject && (md->moptions & PCRE_NOTBOL) == 0) ||
-           ((ims & PCRE_MULTILINE) != 0 && ptr[-1] == NEWLINE))
+           ((ims & PCRE_MULTILINE) != 0 &&
+             ptr != end_subject &&
+             WAS_NEWLINE(ptr)))
          { ADD_ACTIVE(state_offset + 1, 0); }
        break;
-Line 679 
 for (;;)
+Line 781 
 for (;;)
        /*-----------------------------------------------------------------*/
        case OP_ANY:
-       if (clen > 0 && (c != NEWLINE || (ims & PCRE_DOTALL) != 0))
+       if (clen > 0 && !IS_NEWLINE(ptr))
+         { ADD_NEW(state_offset + 1, 0); }
+       break;
+       /*-----------------------------------------------------------------*/
+       case OP_ALLANY:
+       if (clen > 0)
          { ADD_NEW(state_offset + 1, 0); }
        break;
        /*-----------------------------------------------------------------*/
        case OP_EODN:
-       if (clen == 0 || (c == NEWLINE && ptr + 1 == end_subject))
+       if (clen == 0 || (IS_NEWLINE(ptr) && ptr == end_subject - md->nllen))
          { ADD_ACTIVE(state_offset + 1, 0); }
        break;
-Line 693 
 for (;;)
+Line 801 
 for (;;)
        case OP_DOLL:
        if ((md->moptions & PCRE_NOTEOL) == 0)
          {
-         if (clen == 0 || (c == NEWLINE && (ptr + 1 == end_subject ||
+         if (clen == 0 ||
-                                 (ims & PCRE_MULTILINE) != 0)))
+             ((md->poptions & PCRE_DOLLAR_ENDONLY) == 0 && IS_NEWLINE(ptr) &&
+                ((ims & PCRE_MULTILINE) != 0 || ptr == end_subject - md->nllen)
+             ))
            { ADD_ACTIVE(state_offset + 1, 0); }
          }
-       else if (c == NEWLINE && (ims & PCRE_MULTILINE) != 0)
+       else if ((ims & PCRE_MULTILINE) != 0 && IS_NEWLINE(ptr))
          { ADD_ACTIVE(state_offset + 1, 0); }
        break;
-Line 729 
 for (;;)
+Line 839 
 for (;;)
          if (ptr > start_subject)
            {
            const uschar *temp = ptr - 1;
+           if (temp < md->start_used_ptr) md->start_used_ptr = temp;
  #ifdef SUPPORT_UTF8
            if (utf8) BACKCHAR(temp);
  #endif
-Line 737 
 for (;;)
+Line 848 
 for (;;)
            }
          else left_word = 0;
-         if (clen > 0) right_word = c < 256 && (ctypes[c] & ctype_word) != 0;
+         if (clen > 0)
-           else right_word = 0;
+           right_word = c < 256 && (ctypes[c] & ctype_word) != 0;
+         else              /* This is a fudge to ensure that if this is the */
+           {               /* last item in the pattern, we don't count it as */
+           reached_end--;  /* reached, thus disabling a partial match. */
+           right_word = 0;
+           }
          if ((left_word == right_word) == (codevalue == OP_NOT_WORD_BOUNDARY))
            { ADD_ACTIVE(state_offset + 1, 0); }
-Line 746 
 for (;;)
+Line 862 
 for (;;)
        break;
- #ifdef SUPPORT_UCP
        /*-----------------------------------------------------------------*/
        /* Check the next character by Unicode property. We will get here only
        if the support is in the binary; otherwise a compile-time error occurs.
        */
+ #ifdef SUPPORT_UCP
        case OP_PROP:
        case OP_NOTPROP:
        if (clen > 0)
          {
-         int rqdtype, category;
+         BOOL OK;
-         category = ucp_findchar(c, &chartype, &othercase);
+         const ucd_record * prop = GET_UCD(c);
-         rqdtype = code[1];
+         switch(code[1])
-         if (rqdtype >= 128)
            {
-           if ((rqdtype - 128 == category) == (codevalue == OP_PROP))
+           case PT_ANY:
-             { ADD_NEW(state_offset + 2, 0); }
+           OK = TRUE;
-           }
+           break;
-         else
-           {
+           case PT_LAMP:
-           if ((rqdtype == chartype) == (codevalue == OP_PROP))
+           OK = prop->chartype == ucp_Lu || prop->chartype == ucp_Ll || prop->chartype == ucp_Lt;
-             { ADD_NEW(state_offset + 2, 0); }
+           break;
+           case PT_GC:
+           OK = _pcre_ucp_gentype[prop->chartype] == code[2];
+           break;
+           case PT_PC:
+           OK = prop->chartype == code[2];
+           break;
+           case PT_SC:
+           OK = prop->script == code[2];
+           break;
+           /* Should never occur, but keep compilers from grumbling. */
+           default:
+           OK = codevalue != OP_PROP;
+           break;
            }
+         if (OK == (codevalue == OP_PROP)) { ADD_NEW(state_offset + 3, 0); }
          }
        break;
  #endif
-Line 779 
 for (;;)
+Line 913 
 for (;;)
  /* ========================================================================== */
        /* These opcodes likewise inspect the subject character, but have an
        argument that is not a data character. It is one of these opcodes:
-       OP_ANY, OP_DIGIT, OP_NOT_DIGIT, OP_WHITESPACE, OP_NOT_SPACE, OP_WORDCHAR,
+       OP_ANY, OP_ALLANY, OP_DIGIT, OP_NOT_DIGIT, OP_WHITESPACE, OP_NOT_SPACE,
-       OP_NOT_WORDCHAR. The value is loaded into d. */
+       OP_WORDCHAR, OP_NOT_WORDCHAR. The value is loaded into d. */
        case OP_TYPEPLUS:
        case OP_TYPEMINPLUS:
+       case OP_TYPEPOSPLUS:
        count = current_state->count;  /* Already matched */
        if (count > 0) { ADD_ACTIVE(state_offset + 2, 0); }
        if (clen > 0)
          {
          if ((c >= 256 && d != OP_DIGIT && d != OP_WHITESPACE && d != OP_WORDCHAR) ||
              (c < 256 &&
-               (d != OP_ANY || c != '\n' || (ims & PCRE_DOTALL) != 0) &&
+               (d != OP_ANY || !IS_NEWLINE(ptr)) &&
                ((ctypes[c] & toptable1[d]) ^ toptable2[d]) != 0))
            {
+           if (count > 0 && codevalue == OP_TYPEPOSPLUS)
+             {
+             active_count--;            /* Remove non-match possibility */
+             next_active_state--;
+             }
            count++;
            ADD_NEW(state_offset, count);
            }
-Line 802 
 for (;;)
+Line 942 
 for (;;)
        /*-----------------------------------------------------------------*/
        case OP_TYPEQUERY:
        case OP_TYPEMINQUERY:
+       case OP_TYPEPOSQUERY:
        ADD_ACTIVE(state_offset + 2, 0);
        if (clen > 0)
          {
          if ((c >= 256 && d != OP_DIGIT && d != OP_WHITESPACE && d != OP_WORDCHAR) ||
              (c < 256 &&
-               (d != OP_ANY || c != '\n' || (ims & PCRE_DOTALL) != 0) &&
+               (d != OP_ANY || !IS_NEWLINE(ptr)) &&
                ((ctypes[c] & toptable1[d]) ^ toptable2[d]) != 0))
            {
+           if (codevalue == OP_TYPEPOSQUERY)
+             {
+             active_count--;            /* Remove non-match possibility */
+             next_active_state--;
+             }
            ADD_NEW(state_offset + 2, 0);
            }
          }
-Line 818 
 for (;;)
+Line 964 
 for (;;)
        /*-----------------------------------------------------------------*/
        case OP_TYPESTAR:
        case OP_TYPEMINSTAR:
+       case OP_TYPEPOSSTAR:
        ADD_ACTIVE(state_offset + 2, 0);
        if (clen > 0)
          {
          if ((c >= 256 && d != OP_DIGIT && d != OP_WHITESPACE && d != OP_WORDCHAR) ||
              (c < 256 &&
-               (d != OP_ANY || c != '\n' || (ims & PCRE_DOTALL) != 0) &&
+               (d != OP_ANY || !IS_NEWLINE(ptr)) &&
                ((ctypes[c] & toptable1[d]) ^ toptable2[d]) != 0))
            {
+           if (codevalue == OP_TYPEPOSSTAR)
+             {
+             active_count--;            /* Remove non-match possibility */
+             next_active_state--;
+             }
            ADD_NEW(state_offset, 0);
            }
          }
-Line 833 
 for (;;)
+Line 985 
 for (;;)
        /*-----------------------------------------------------------------*/
        case OP_TYPEEXACT:
+       count = current_state->count;  /* Number already matched */
+       if (clen > 0)
+         {
+         if ((c >= 256 && d != OP_DIGIT && d != OP_WHITESPACE && d != OP_WORDCHAR) ||
+             (c < 256 &&
+               (d != OP_ANY || !IS_NEWLINE(ptr)) &&
+               ((ctypes[c] & toptable1[d]) ^ toptable2[d]) != 0))
+           {
+           if (++count >= GET2(code, 1))
+             { ADD_NEW(state_offset + 4, 0); }
+           else
+             { ADD_NEW(state_offset, count); }
+           }
+         }
+       break;
+       /*-----------------------------------------------------------------*/
        case OP_TYPEUPTO:
        case OP_TYPEMINUPTO:
-       if (codevalue != OP_TYPEEXACT)
+       case OP_TYPEPOSUPTO:
-         { ADD_ACTIVE(state_offset + 4, 0); }
+       ADD_ACTIVE(state_offset + 4, 0);
        count = current_state->count;  /* Number already matched */
        if (clen > 0)
          {
          if ((c >= 256 && d != OP_DIGIT && d != OP_WHITESPACE && d != OP_WORDCHAR) ||
              (c < 256 &&
-               (d != OP_ANY || c != '\n' || (ims & PCRE_DOTALL) != 0) &&
+               (d != OP_ANY || !IS_NEWLINE(ptr)) &&
                ((ctypes[c] & toptable1[d]) ^ toptable2[d]) != 0))
            {
+           if (codevalue == OP_TYPEPOSUPTO)
+             {
+             active_count--;           /* Remove non-match possibility */
+             next_active_state--;
+             }
            if (++count >= GET2(code, 1))
              { ADD_NEW(state_offset + 4, 0); }
            else
-Line 855 
 for (;;)
+Line 1029 
 for (;;)
  /* ========================================================================== */
        /* These are virtual opcodes that are used when something like
-       OP_TYPEPLUS has OP_PROP, OP_NOTPROP, or OP_EXTUNI as its argument. It
+       OP_TYPEPLUS has OP_PROP, OP_NOTPROP, OP_ANYNL, or OP_EXTUNI as its
-       keeps the code above fast for the other cases. The argument is in the
+       argument. It keeps the code above fast for the other cases. The argument
-       d variable. */
+       is in the d variable. */
+ #ifdef SUPPORT_UCP
        case OP_PROP_EXTRA + OP_TYPEPLUS:
        case OP_PROP_EXTRA + OP_TYPEMINPLUS:
+       case OP_PROP_EXTRA + OP_TYPEPOSPLUS:
        count = current_state->count;           /* Already matched */
-       if (count > 0) { ADD_ACTIVE(state_offset + 3, 0); }
+       if (count > 0) { ADD_ACTIVE(state_offset + 4, 0); }
        if (clen > 0)
          {
-         int category = ucp_findchar(c, &chartype, &othercase);
+         BOOL OK;
-         int rqdtype = code[2];
+         const ucd_record * prop = GET_UCD(c);
-         if ((d == OP_PROP) ==
+         switch(code[2])
-             (rqdtype == ((rqdtype >= 128)? (category + 128) : chartype)))
+           {
-           { count++; ADD_NEW(state_offset, count); }
+           case PT_ANY:
+           OK = TRUE;
+           break;
+           case PT_LAMP:
+           OK = prop->chartype == ucp_Lu || prop->chartype == ucp_Ll || prop->chartype == ucp_Lt;
+           break;
+           case PT_GC:
+           OK = _pcre_ucp_gentype[prop->chartype] == code[3];
+           break;
+           case PT_PC:
+           OK = prop->chartype == code[3];
+           break;
+           case PT_SC:
+           OK = prop->script == code[3];
+           break;
+           /* Should never occur, but keep compilers from grumbling. */
+           default:
+           OK = codevalue != OP_PROP;
+           break;
+           }
+         if (OK == (d == OP_PROP))
+           {
+           if (count > 0 && codevalue == OP_PROP_EXTRA + OP_TYPEPOSPLUS)
+             {
+             active_count--;           /* Remove non-match possibility */
+             next_active_state--;
+             }
+           count++;
+           ADD_NEW(state_offset, count);
+           }
          }
        break;
        /*-----------------------------------------------------------------*/
        case OP_EXTUNI_EXTRA + OP_TYPEPLUS:
        case OP_EXTUNI_EXTRA + OP_TYPEMINPLUS:
+       case OP_EXTUNI_EXTRA + OP_TYPEPOSPLUS:
        count = current_state->count;  /* Already matched */
        if (count > 0) { ADD_ACTIVE(state_offset + 2, 0); }
-       if (clen > 0 && ucp_findchar(c, &chartype, &othercase) != ucp_M)
+       if (clen > 0 && UCD_CATEGORY(c) != ucp_M)
          {
          const uschar *nptr = ptr + clen;
          int ncount = 0;
+         if (count > 0 && codevalue == OP_EXTUNI_EXTRA + OP_TYPEPOSPLUS)
+           {
+           active_count--;           /* Remove non-match possibility */
+           next_active_state--;
+           }
          while (nptr < end_subject)
            {
            int nd;
            int ndlen = 1;
            GETCHARLEN(nd, nptr, ndlen);
-           if (ucp_findchar(nd, &chartype, &othercase) != ucp_M) break;
+           if (UCD_CATEGORY(nd) != ucp_M) break;
            ncount++;
            nptr += ndlen;
            }
-Line 895 
 for (;;)
+Line 1113 
 for (;;)
          ADD_NEW_DATA(-state_offset, count, ncount);
          }
        break;
+ #endif
        /*-----------------------------------------------------------------*/
+       case OP_ANYNL_EXTRA + OP_TYPEPLUS:
+       case OP_ANYNL_EXTRA + OP_TYPEMINPLUS:
+       case OP_ANYNL_EXTRA + OP_TYPEPOSPLUS:
+       count = current_state->count;  /* Already matched */
+       if (count > 0) { ADD_ACTIVE(state_offset + 2, 0); }
+       if (clen > 0)
+         {
+         int ncount = 0;
+         switch (c)
+           {
+           case 0x000b:
+           case 0x000c:
+           case 0x0085:
+           case 0x2028:
+           case 0x2029:
+           if ((md->moptions & PCRE_BSR_ANYCRLF) != 0) break;
+           goto ANYNL01;
+           case 0x000d:
+           if (ptr + 1 < end_subject && ptr[1] == 0x0a) ncount = 1;
+           /* Fall through */
+           ANYNL01:
+           case 0x000a:
+           if (count > 0 && codevalue == OP_ANYNL_EXTRA + OP_TYPEPOSPLUS)
+             {
+             active_count--;           /* Remove non-match possibility */
+             next_active_state--;
+             }
+           count++;
+           ADD_NEW_DATA(-state_offset, count, ncount);
+           break;
+           default:
+           break;
+           }
+         }
+       break;
+       /*-----------------------------------------------------------------*/
+       case OP_VSPACE_EXTRA + OP_TYPEPLUS:
+       case OP_VSPACE_EXTRA + OP_TYPEMINPLUS:
+       case OP_VSPACE_EXTRA + OP_TYPEPOSPLUS:
+       count = current_state->count;  /* Already matched */
+       if (count > 0) { ADD_ACTIVE(state_offset + 2, 0); }
+       if (clen > 0)
+         {
+         BOOL OK;
+         switch (c)
+           {
+           case 0x000a:
+           case 0x000b:
+           case 0x000c:
+           case 0x000d:
+           case 0x0085:
+           case 0x2028:
+           case 0x2029:
+           OK = TRUE;
+           break;
+           default:
+           OK = FALSE;
+           break;
+           }
+         if (OK == (d == OP_VSPACE))
+           {
+           if (count > 0 && codevalue == OP_VSPACE_EXTRA + OP_TYPEPOSPLUS)
+             {
+             active_count--;           /* Remove non-match possibility */
+             next_active_state--;
+             }
+           count++;
+           ADD_NEW_DATA(-state_offset, count, 0);
+           }
+         }
+       break;
+       /*-----------------------------------------------------------------*/
+       case OP_HSPACE_EXTRA + OP_TYPEPLUS:
+       case OP_HSPACE_EXTRA + OP_TYPEMINPLUS:
+       case OP_HSPACE_EXTRA + OP_TYPEPOSPLUS:
+       count = current_state->count;  /* Already matched */
+       if (count > 0) { ADD_ACTIVE(state_offset + 2, 0); }
+       if (clen > 0)
+         {
+         BOOL OK;
+         switch (c)
+           {
+           case 0x09:      /* HT */
+           case 0x20:      /* SPACE */
+           case 0xa0:      /* NBSP */
+           case 0x1680:    /* OGHAM SPACE MARK */
+           case 0x180e:    /* MONGOLIAN VOWEL SEPARATOR */
+           case 0x2000:    /* EN QUAD */
+           case 0x2001:    /* EM QUAD */
+           case 0x2002:    /* EN SPACE */
+           case 0x2003:    /* EM SPACE */
+           case 0x2004:    /* THREE-PER-EM SPACE */
+           case 0x2005:    /* FOUR-PER-EM SPACE */
+           case 0x2006:    /* SIX-PER-EM SPACE */
+           case 0x2007:    /* FIGURE SPACE */
+           case 0x2008:    /* PUNCTUATION SPACE */
+           case 0x2009:    /* THIN SPACE */
+           case 0x200A:    /* HAIR SPACE */
+           case 0x202f:    /* NARROW NO-BREAK SPACE */
+           case 0x205f:    /* MEDIUM MATHEMATICAL SPACE */
+           case 0x3000:    /* IDEOGRAPHIC SPACE */
+           OK = TRUE;
+           break;
+           default:
+           OK = FALSE;
+           break;
+           }
+         if (OK == (d == OP_HSPACE))
+           {
+           if (count > 0 && codevalue == OP_HSPACE_EXTRA + OP_TYPEPOSPLUS)
+             {
+             active_count--;           /* Remove non-match possibility */
+             next_active_state--;
+             }
+           count++;
+           ADD_NEW_DATA(-state_offset, count, 0);
+           }
+         }
+       break;
+       /*-----------------------------------------------------------------*/
+ #ifdef SUPPORT_UCP
        case OP_PROP_EXTRA + OP_TYPEQUERY:
        case OP_PROP_EXTRA + OP_TYPEMINQUERY:
-       count = 3;
+       case OP_PROP_EXTRA + OP_TYPEPOSQUERY:
+       count = 4;
        goto QS1;
        case OP_PROP_EXTRA + OP_TYPESTAR:
        case OP_PROP_EXTRA + OP_TYPEMINSTAR:
+       case OP_PROP_EXTRA + OP_TYPEPOSSTAR:
        count = 0;
        QS1:
-       ADD_ACTIVE(state_offset + 3, 0);
+       ADD_ACTIVE(state_offset + 4, 0);
        if (clen > 0)
          {
-         int category = ucp_findchar(c, &chartype, &othercase);
+         BOOL OK;
-         int rqdtype = code[2];
+         const ucd_record * prop = GET_UCD(c);
-         if ((d == OP_PROP) ==
+         switch(code[2])
-             (rqdtype == ((rqdtype >= 128)? (category + 128) : chartype)))
+           {
-           { ADD_NEW(state_offset + count, 0); }
+           case PT_ANY:
+           OK = TRUE;
+           break;
+           case PT_LAMP:
+           OK = prop->chartype == ucp_Lu || prop->chartype == ucp_Ll || prop->chartype == ucp_Lt;
+           break;
+           case PT_GC:
+           OK = _pcre_ucp_gentype[prop->chartype] == code[3];
+           break;
+           case PT_PC:
+           OK = prop->chartype == code[3];
+           break;
+           case PT_SC:
+           OK = prop->script == code[3];
+           break;
+           /* Should never occur, but keep compilers from grumbling. */
+           default:
+           OK = codevalue != OP_PROP;
+           break;
+           }
+         if (OK == (d == OP_PROP))
+           {
+           if (codevalue == OP_PROP_EXTRA + OP_TYPEPOSSTAR ||
+               codevalue == OP_PROP_EXTRA + OP_TYPEPOSQUERY)
+             {
+             active_count--;           /* Remove non-match possibility */
+             next_active_state--;
+             }
+           ADD_NEW(state_offset + count, 0);
+           }
          }
        break;
        /*-----------------------------------------------------------------*/
        case OP_EXTUNI_EXTRA + OP_TYPEQUERY:
        case OP_EXTUNI_EXTRA + OP_TYPEMINQUERY:
+       case OP_EXTUNI_EXTRA + OP_TYPEPOSQUERY:
        count = 2;
        goto QS2;
        case OP_EXTUNI_EXTRA + OP_TYPESTAR:
        case OP_EXTUNI_EXTRA + OP_TYPEMINSTAR:
+       case OP_EXTUNI_EXTRA + OP_TYPEPOSSTAR:
        count = 0;
        QS2:
        ADD_ACTIVE(state_offset + 2, 0);
-       if (clen > 0 && ucp_findchar(c, &chartype, &othercase) != ucp_M)
+       if (clen > 0 && UCD_CATEGORY(c) != ucp_M)
          {
          const uschar *nptr = ptr + clen;
          int ncount = 0;
+         if (codevalue == OP_EXTUNI_EXTRA + OP_TYPEPOSSTAR ||
+             codevalue == OP_EXTUNI_EXTRA + OP_TYPEPOSQUERY)
+           {
+           active_count--;           /* Remove non-match possibility */
+           next_active_state--;
+           }
          while (nptr < end_subject)
            {
            int nd;
            int ndlen = 1;
            GETCHARLEN(nd, nptr, ndlen);
-           if (ucp_findchar(nd, &chartype, &othercase) != ucp_M) break;
+           if (UCD_CATEGORY(nd) != ucp_M) break;
            ncount++;
            nptr += ndlen;
            }
          ADD_NEW_DATA(-(state_offset + count), 0, ncount);
          }
        break;
+ #endif
+       /*-----------------------------------------------------------------*/
+       case OP_ANYNL_EXTRA + OP_TYPEQUERY:
+       case OP_ANYNL_EXTRA + OP_TYPEMINQUERY:
+       case OP_ANYNL_EXTRA + OP_TYPEPOSQUERY:
+       count = 2;
+       goto QS3;
+       case OP_ANYNL_EXTRA + OP_TYPESTAR:
+       case OP_ANYNL_EXTRA + OP_TYPEMINSTAR:
+       case OP_ANYNL_EXTRA + OP_TYPEPOSSTAR:
+       count = 0;
+       QS3:
+       ADD_ACTIVE(state_offset + 2, 0);
+       if (clen > 0)
+         {
+         int ncount = 0;
+         switch (c)
+           {
+           case 0x000b:
+           case 0x000c:
+           case 0x0085:
+           case 0x2028:
+           case 0x2029:
+           if ((md->moptions & PCRE_BSR_ANYCRLF) != 0) break;
+           goto ANYNL02;
+           case 0x000d:
+           if (ptr + 1 < end_subject && ptr[1] == 0x0a) ncount = 1;
+           /* Fall through */
+           ANYNL02:
+           case 0x000a:
+           if (codevalue == OP_ANYNL_EXTRA + OP_TYPEPOSSTAR ||
+               codevalue == OP_ANYNL_EXTRA + OP_TYPEPOSQUERY)
+             {
+             active_count--;           /* Remove non-match possibility */
+             next_active_state--;
+             }
+           ADD_NEW_DATA(-(state_offset + count), 0, ncount);
+           break;
+           default:
+           break;
+           }
+         }
+       break;
+       /*-----------------------------------------------------------------*/
+       case OP_VSPACE_EXTRA + OP_TYPEQUERY:
+       case OP_VSPACE_EXTRA + OP_TYPEMINQUERY:
+       case OP_VSPACE_EXTRA + OP_TYPEPOSQUERY:
+       count = 2;
+       goto QS4;
+       case OP_VSPACE_EXTRA + OP_TYPESTAR:
+       case OP_VSPACE_EXTRA + OP_TYPEMINSTAR:
+       case OP_VSPACE_EXTRA + OP_TYPEPOSSTAR:
+       count = 0;
+       QS4:
+       ADD_ACTIVE(state_offset + 2, 0);
+       if (clen > 0)
+         {
+         BOOL OK;
+         switch (c)
+           {
+           case 0x000a:
+           case 0x000b:
+           case 0x000c:
+           case 0x000d:
+           case 0x0085:
+           case 0x2028:
+           case 0x2029:
+           OK = TRUE;
+           break;
+           default:
+           OK = FALSE;
+           break;
+           }
+         if (OK == (d == OP_VSPACE))
+           {
+           if (codevalue == OP_VSPACE_EXTRA + OP_TYPEPOSSTAR ||
+               codevalue == OP_VSPACE_EXTRA + OP_TYPEPOSQUERY)
+             {
+             active_count--;           /* Remove non-match possibility */
+             next_active_state--;
+             }
+           ADD_NEW_DATA(-(state_offset + count), 0, 0);
+           }
+         }
+       break;
+       /*-----------------------------------------------------------------*/
+       case OP_HSPACE_EXTRA + OP_TYPEQUERY:
+       case OP_HSPACE_EXTRA + OP_TYPEMINQUERY:
+       case OP_HSPACE_EXTRA + OP_TYPEPOSQUERY:
+       count = 2;
+       goto QS5;
+       case OP_HSPACE_EXTRA + OP_TYPESTAR:
+       case OP_HSPACE_EXTRA + OP_TYPEMINSTAR:
+       case OP_HSPACE_EXTRA + OP_TYPEPOSSTAR:
+       count = 0;
+       QS5:
+       ADD_ACTIVE(state_offset + 2, 0);
+       if (clen > 0)
+         {
+         BOOL OK;
+         switch (c)
+           {
+           case 0x09:      /* HT */
+           case 0x20:      /* SPACE */
+           case 0xa0:      /* NBSP */
+           case 0x1680:    /* OGHAM SPACE MARK */
+           case 0x180e:    /* MONGOLIAN VOWEL SEPARATOR */
+           case 0x2000:    /* EN QUAD */
+           case 0x2001:    /* EM QUAD */
+           case 0x2002:    /* EN SPACE */
+           case 0x2003:    /* EM SPACE */
+           case 0x2004:    /* THREE-PER-EM SPACE */
+           case 0x2005:    /* FOUR-PER-EM SPACE */
+           case 0x2006:    /* SIX-PER-EM SPACE */
+           case 0x2007:    /* FIGURE SPACE */
+           case 0x2008:    /* PUNCTUATION SPACE */
+           case 0x2009:    /* THIN SPACE */
+           case 0x200A:    /* HAIR SPACE */
+           case 0x202f:    /* NARROW NO-BREAK SPACE */
+           case 0x205f:    /* MEDIUM MATHEMATICAL SPACE */
+           case 0x3000:    /* IDEOGRAPHIC SPACE */
+           OK = TRUE;
+           break;
+           default:
+           OK = FALSE;
+           break;
+           }
+         if (OK == (d == OP_HSPACE))
+           {
+           if (codevalue == OP_HSPACE_EXTRA + OP_TYPEPOSSTAR ||
+               codevalue == OP_HSPACE_EXTRA + OP_TYPEPOSQUERY)
+             {
+             active_count--;           /* Remove non-match possibility */
+             next_active_state--;
+             }
+           ADD_NEW_DATA(-(state_offset + count), 0, 0);
+           }
+         }
+       break;
        /*-----------------------------------------------------------------*/
+ #ifdef SUPPORT_UCP
        case OP_PROP_EXTRA + OP_TYPEEXACT:
        case OP_PROP_EXTRA + OP_TYPEUPTO:
        case OP_PROP_EXTRA + OP_TYPEMINUPTO:
+       case OP_PROP_EXTRA + OP_TYPEPOSUPTO:
        if (codevalue != OP_PROP_EXTRA + OP_TYPEEXACT)
-         { ADD_ACTIVE(state_offset + 5, 0); }
+         { ADD_ACTIVE(state_offset + 6, 0); }
        count = current_state->count;  /* Number already matched */
        if (clen > 0)
          {
-         int category = ucp_findchar(c, &chartype, &othercase);
+         BOOL OK;
-         int rqdtype = code[4];
+         const ucd_record * prop = GET_UCD(c);
-         if ((d == OP_PROP) ==
+         switch(code[4])
-             (rqdtype == ((rqdtype >= 128)? (category + 128) : chartype)))
            {
+           case PT_ANY:
+           OK = TRUE;
+           break;
+           case PT_LAMP:
+           OK = prop->chartype == ucp_Lu || prop->chartype == ucp_Ll || prop->chartype == ucp_Lt;
+           break;
+           case PT_GC:
+           OK = _pcre_ucp_gentype[prop->chartype] == code[5];
+           break;
+           case PT_PC:
+           OK = prop->chartype == code[5];
+           break;
+           case PT_SC:
+           OK = prop->script == code[5];
+           break;
+           /* Should never occur, but keep compilers from grumbling. */
+           default:
+           OK = codevalue != OP_PROP;
+           break;
+           }
+         if (OK == (d == OP_PROP))
+           {
+           if (codevalue == OP_PROP_EXTRA + OP_TYPEPOSUPTO)
+             {
+             active_count--;           /* Remove non-match possibility */
+             next_active_state--;
+             }
            if (++count >= GET2(code, 1))
-             { ADD_NEW(state_offset + 5, 0); }
+             { ADD_NEW(state_offset + 6, 0); }
            else
              { ADD_NEW(state_offset, count); }
            }
-Line 975 
 for (;;)
+Line 1560 
 for (;;)
        case OP_EXTUNI_EXTRA + OP_TYPEEXACT:
        case OP_EXTUNI_EXTRA + OP_TYPEUPTO:
        case OP_EXTUNI_EXTRA + OP_TYPEMINUPTO:
+       case OP_EXTUNI_EXTRA + OP_TYPEPOSUPTO:
        if (codevalue != OP_EXTUNI_EXTRA + OP_TYPEEXACT)
          { ADD_ACTIVE(state_offset + 4, 0); }
        count = current_state->count;  /* Number already matched */
-       if (clen > 0 && ucp_findchar(c, &chartype, &othercase) != ucp_M)
+       if (clen > 0 && UCD_CATEGORY(c) != ucp_M)
          {
          const uschar *nptr = ptr + clen;
          int ncount = 0;
+         if (codevalue == OP_EXTUNI_EXTRA + OP_TYPEPOSUPTO)
+           {
+           active_count--;           /* Remove non-match possibility */
+           next_active_state--;
+           }
          while (nptr < end_subject)
            {
            int nd;
            int ndlen = 1;
            GETCHARLEN(nd, nptr, ndlen);
-           if (ucp_findchar(nd, &chartype, &othercase) != ucp_M) break;
+           if (UCD_CATEGORY(nd) != ucp_M) break;
            ncount++;
            nptr += ndlen;
            }
-Line 997 
 for (;;)
+Line 1588 
 for (;;)
            { ADD_NEW_DATA(-state_offset, count, ncount); }
          }
        break;
+ #endif
+       /*-----------------------------------------------------------------*/
+       case OP_ANYNL_EXTRA + OP_TYPEEXACT:
+       case OP_ANYNL_EXTRA + OP_TYPEUPTO:
+       case OP_ANYNL_EXTRA + OP_TYPEMINUPTO:
+       case OP_ANYNL_EXTRA + OP_TYPEPOSUPTO:
+       if (codevalue != OP_ANYNL_EXTRA + OP_TYPEEXACT)
+         { ADD_ACTIVE(state_offset + 4, 0); }
+       count = current_state->count;  /* Number already matched */
+       if (clen > 0)
+         {
+         int ncount = 0;
+         switch (c)
+           {
+           case 0x000b:
+           case 0x000c:
+           case 0x0085:
+           case 0x2028:
+           case 0x2029:
+           if ((md->moptions & PCRE_BSR_ANYCRLF) != 0) break;
+           goto ANYNL03;
+           case 0x000d:
+           if (ptr + 1 < end_subject && ptr[1] == 0x0a) ncount = 1;
+           /* Fall through */
+           ANYNL03:
+           case 0x000a:
+           if (codevalue == OP_ANYNL_EXTRA + OP_TYPEPOSUPTO)
+             {
+             active_count--;           /* Remove non-match possibility */
+             next_active_state--;
+             }
+           if (++count >= GET2(code, 1))
+             { ADD_NEW_DATA(-(state_offset + 4), 0, ncount); }
+           else
+             { ADD_NEW_DATA(-state_offset, count, ncount); }
+           break;
+           default:
+           break;
+           }
+         }
+       break;
+       /*-----------------------------------------------------------------*/
+       case OP_VSPACE_EXTRA + OP_TYPEEXACT:
+       case OP_VSPACE_EXTRA + OP_TYPEUPTO:
+       case OP_VSPACE_EXTRA + OP_TYPEMINUPTO:
+       case OP_VSPACE_EXTRA + OP_TYPEPOSUPTO:
+       if (codevalue != OP_VSPACE_EXTRA + OP_TYPEEXACT)
+         { ADD_ACTIVE(state_offset + 4, 0); }
+       count = current_state->count;  /* Number already matched */
+       if (clen > 0)
+         {
+         BOOL OK;
+         switch (c)
+           {
+           case 0x000a:
+           case 0x000b:
+           case 0x000c:
+           case 0x000d:
+           case 0x0085:
+           case 0x2028:
+           case 0x2029:
+           OK = TRUE;
+           break;
+           default:
+           OK = FALSE;
+           }
+         if (OK == (d == OP_VSPACE))
+           {
+           if (codevalue == OP_VSPACE_EXTRA + OP_TYPEPOSUPTO)
+             {
+             active_count--;           /* Remove non-match possibility */
+             next_active_state--;
+             }
+           if (++count >= GET2(code, 1))
+             { ADD_NEW_DATA(-(state_offset + 4), 0, 0); }
+           else
+             { ADD_NEW_DATA(-state_offset, count, 0); }
+           }
+         }
+       break;
+       /*-----------------------------------------------------------------*/
+       case OP_HSPACE_EXTRA + OP_TYPEEXACT:
+       case OP_HSPACE_EXTRA + OP_TYPEUPTO:
+       case OP_HSPACE_EXTRA + OP_TYPEMINUPTO:
+       case OP_HSPACE_EXTRA + OP_TYPEPOSUPTO:
+       if (codevalue != OP_HSPACE_EXTRA + OP_TYPEEXACT)
+         { ADD_ACTIVE(state_offset + 4, 0); }
+       count = current_state->count;  /* Number already matched */
+       if (clen > 0)
+         {
+         BOOL OK;
+         switch (c)
+           {
+           case 0x09:      /* HT */
+           case 0x20:      /* SPACE */
+           case 0xa0:      /* NBSP */
+           case 0x1680:    /* OGHAM SPACE MARK */
+           case 0x180e:    /* MONGOLIAN VOWEL SEPARATOR */
+           case 0x2000:    /* EN QUAD */
+           case 0x2001:    /* EM QUAD */
+           case 0x2002:    /* EN SPACE */
+           case 0x2003:    /* EM SPACE */
+           case 0x2004:    /* THREE-PER-EM SPACE */
+           case 0x2005:    /* FOUR-PER-EM SPACE */
+           case 0x2006:    /* SIX-PER-EM SPACE */
+           case 0x2007:    /* FIGURE SPACE */
+           case 0x2008:    /* PUNCTUATION SPACE */
+           case 0x2009:    /* THIN SPACE */
+           case 0x200A:    /* HAIR SPACE */
+           case 0x202f:    /* NARROW NO-BREAK SPACE */
+           case 0x205f:    /* MEDIUM MATHEMATICAL SPACE */
+           case 0x3000:    /* IDEOGRAPHIC SPACE */
+           OK = TRUE;
+           break;
+           default:
+           OK = FALSE;
+           break;
+           }
+         if (OK == (d == OP_HSPACE))
+           {
+           if (codevalue == OP_HSPACE_EXTRA + OP_TYPEPOSUPTO)
+             {
+             active_count--;           /* Remove non-match possibility */
+             next_active_state--;
+             }
+           if (++count >= GET2(code, 1))
+             { ADD_NEW_DATA(-(state_offset + 4), 0, 0); }
+           else
+             { ADD_NEW_DATA(-state_offset, count, 0); }
+           }
+         }
+       break;
  /* ========================================================================== */
        /* These opcodes are followed by a character that is usually compared
-Line 1018 
 for (;;)
+Line 1751 
 for (;;)
          {
          if (c == d) { ADD_NEW(state_offset + dlen + 1, 0); } else
            {
+           unsigned int othercase;
            if (c < 128) othercase = fcc[c]; else
            /* If we have Unicode property support, we can use it to test the
-           other case of the character, if there is one. The result of
+           other case of the character. */
-           ucp_findchar() is < 0 if the char isn't found, and othercase is
-           returned as zero if there isn't another case. */
  #ifdef SUPPORT_UCP
-           if (ucp_findchar(c, &chartype, &othercase) < 0)
+           othercase = UCD_OTHERCASE(c);
+ #else
+           othercase = NOTACHAR;
  #endif
-             othercase = -1;
            if (d == othercase) { ADD_NEW(state_offset + dlen + 1, 0); }
            }
-Line 1050 
 for (;;)
+Line 1783 
 for (;;)
        to wait for them to pass before continuing. */
        case OP_EXTUNI:
-       if (clen > 0 && ucp_findchar(c, &chartype, &othercase) != ucp_M)
+       if (clen > 0 && UCD_CATEGORY(c) != ucp_M)
          {
          const uschar *nptr = ptr + clen;
          int ncount = 0;
-Line 1058 
 for (;;)
+Line 1791 
 for (;;)
            {
            int nclen = 1;
            GETCHARLEN(c, nptr, nclen);
-           if (ucp_findchar(c, &chartype, &othercase) != ucp_M) break;
+           if (UCD_CATEGORY(c) != ucp_M) break;
            ncount++;
            nptr += nclen;
            }
-Line 1068 
 for (;;)
+Line 1801 
 for (;;)
  #endif
        /*-----------------------------------------------------------------*/
+       /* This is a tricky like EXTUNI because it too can match more than one
+       character (when CR is followed by LF). In this case, set up a negative
+       state to wait for one character to pass before continuing. */
+       case OP_ANYNL:
+       if (clen > 0) switch(c)
+         {
+         case 0x000b:
+         case 0x000c:
+         case 0x0085:
+         case 0x2028:
+         case 0x2029:
+         if ((md->moptions & PCRE_BSR_ANYCRLF) != 0) break;
+         case 0x000a:
+         ADD_NEW(state_offset + 1, 0);
+         break;
+         case 0x000d:
+         if (ptr + 1 < end_subject && ptr[1] == 0x0a)
+           {
+           ADD_NEW_DATA(-(state_offset + 1), 0, 1);
+           }
+         else
+           {
+           ADD_NEW(state_offset + 1, 0);
+           }
+         break;
+         }
+       break;
+       /*-----------------------------------------------------------------*/
+       case OP_NOT_VSPACE:
+       if (clen > 0) switch(c)
+         {
+         case 0x000a:
+         case 0x000b:
+         case 0x000c:
+         case 0x000d:
+         case 0x0085:
+         case 0x2028:
+         case 0x2029:
+         break;
+         default:
+         ADD_NEW(state_offset + 1, 0);
+         break;
+         }
+       break;
+       /*-----------------------------------------------------------------*/
+       case OP_VSPACE:
+       if (clen > 0) switch(c)
+         {
+         case 0x000a:
+         case 0x000b:
+         case 0x000c:
+         case 0x000d:
+         case 0x0085:
+         case 0x2028:
+         case 0x2029:
+         ADD_NEW(state_offset + 1, 0);
+         break;
+         default: break;
+         }
+       break;
+       /*-----------------------------------------------------------------*/
+       case OP_NOT_HSPACE:
+       if (clen > 0) switch(c)
+         {
+         case 0x09:      /* HT */
+         case 0x20:      /* SPACE */
+         case 0xa0:      /* NBSP */
+         case 0x1680:    /* OGHAM SPACE MARK */
+         case 0x180e:    /* MONGOLIAN VOWEL SEPARATOR */
+         case 0x2000:    /* EN QUAD */
+         case 0x2001:    /* EM QUAD */
+         case 0x2002:    /* EN SPACE */
+         case 0x2003:    /* EM SPACE */
+         case 0x2004:    /* THREE-PER-EM SPACE */
+         case 0x2005:    /* FOUR-PER-EM SPACE */
+         case 0x2006:    /* SIX-PER-EM SPACE */
+         case 0x2007:    /* FIGURE SPACE */
+         case 0x2008:    /* PUNCTUATION SPACE */
+         case 0x2009:    /* THIN SPACE */
+         case 0x200A:    /* HAIR SPACE */
+         case 0x202f:    /* NARROW NO-BREAK SPACE */
+         case 0x205f:    /* MEDIUM MATHEMATICAL SPACE */
+         case 0x3000:    /* IDEOGRAPHIC SPACE */
+         break;
+         default:
+         ADD_NEW(state_offset + 1, 0);
+         break;
+         }
+       break;
+       /*-----------------------------------------------------------------*/
+       case OP_HSPACE:
+       if (clen > 0) switch(c)
+         {
+         case 0x09:      /* HT */
+         case 0x20:      /* SPACE */
+         case 0xa0:      /* NBSP */
+         case 0x1680:    /* OGHAM SPACE MARK */
+         case 0x180e:    /* MONGOLIAN VOWEL SEPARATOR */
+         case 0x2000:    /* EN QUAD */
+         case 0x2001:    /* EM QUAD */
+         case 0x2002:    /* EN SPACE */
+         case 0x2003:    /* EM SPACE */
+         case 0x2004:    /* THREE-PER-EM SPACE */
+         case 0x2005:    /* FOUR-PER-EM SPACE */
+         case 0x2006:    /* SIX-PER-EM SPACE */
+         case 0x2007:    /* FIGURE SPACE */
+         case 0x2008:    /* PUNCTUATION SPACE */
+         case 0x2009:    /* THIN SPACE */
+         case 0x200A:    /* HAIR SPACE */
+         case 0x202f:    /* NARROW NO-BREAK SPACE */
+         case 0x205f:    /* MEDIUM MATHEMATICAL SPACE */
+         case 0x3000:    /* IDEOGRAPHIC SPACE */
+         ADD_NEW(state_offset + 1, 0);
+         break;
+         }
+       break;
+       /*-----------------------------------------------------------------*/
        /* Match a negated single character. This is only used for one-byte
        characters, that is, we know that d < 256. The character we are
        checking (c) can be multibyte. */
-Line 1075 
 for (;;)
+Line 1936 
 for (;;)
        case OP_NOT:
        if (clen > 0)
          {
-         int otherd = ((ims & PCRE_CASELESS) != 0)? fcc[d] : d;
+         unsigned int otherd = ((ims & PCRE_CASELESS) != 0)? fcc[d] : d;
          if (c != d && c != otherd) { ADD_NEW(state_offset + dlen + 1, 0); }
          }
        break;
-Line 1083 
 for (;;)
+Line 1944 
 for (;;)
        /*-----------------------------------------------------------------*/
        case OP_PLUS:
        case OP_MINPLUS:
+       case OP_POSPLUS:
        case OP_NOTPLUS:
        case OP_NOTMINPLUS:
+       case OP_NOTPOSPLUS:
        count = current_state->count;  /* Already matched */
        if (count > 0) { ADD_ACTIVE(state_offset + dlen + 1, 0); }
        if (clen > 0)
          {
-         int otherd = -1;
+         unsigned int otherd = NOTACHAR;
          if ((ims & PCRE_CASELESS) != 0)
            {
  #ifdef SUPPORT_UTF8
-           if (utf8 && c >= 128)
+           if (utf8 && d >= 128)
              {
  #ifdef SUPPORT_UCP
-             if (ucp_findchar(d, &chartype, &otherd) < 0) otherd = -1;
+             otherd = UCD_OTHERCASE(d);
  #endif  /* SUPPORT_UCP */
              }
            else
-Line 1104 
 for (;;)
+Line 1967 
 for (;;)
            otherd = fcc[d];
            }
          if ((c == d || c == otherd) == (codevalue < OP_NOTSTAR))
-           { count++; ADD_NEW(state_offset, count); }
+           {
+           if (count > 0 &&
+               (codevalue == OP_POSPLUS || codevalue == OP_NOTPOSPLUS))
+             {
+             active_count--;             /* Remove non-match possibility */
+             next_active_state--;
+             }
+           count++;
+           ADD_NEW(state_offset, count);
+           }
          }
        break;
        /*-----------------------------------------------------------------*/
        case OP_QUERY:
        case OP_MINQUERY:
+       case OP_POSQUERY:
        case OP_NOTQUERY:
        case OP_NOTMINQUERY:
+       case OP_NOTPOSQUERY:
        ADD_ACTIVE(state_offset + dlen + 1, 0);
        if (clen > 0)
          {
-         int otherd = -1;
+         unsigned int otherd = NOTACHAR;
-         if ((ims && PCRE_CASELESS) != 0)
+         if ((ims & PCRE_CASELESS) != 0)
            {
  #ifdef SUPPORT_UTF8
-           if (utf8 && c >= 128)
+           if (utf8 && d >= 128)
              {
  #ifdef SUPPORT_UCP
-             if (ucp_findchar(c, &chartype, &otherd) < 0) otherd = -1;
+             otherd = UCD_OTHERCASE(d);
  #endif  /* SUPPORT_UCP */
              }
            else
-Line 1131 
 for (;;)
+Line 2005 
 for (;;)
            otherd = fcc[d];
            }
          if ((c == d || c == otherd) == (codevalue < OP_NOTSTAR))
-           { ADD_NEW(state_offset + dlen + 1, 0); }
+           {
+           if (codevalue == OP_POSQUERY || codevalue == OP_NOTPOSQUERY)
+             {
+             active_count--;            /* Remove non-match possibility */
+             next_active_state--;
+             }
+           ADD_NEW(state_offset + dlen + 1, 0);
+           }
          }
        break;
        /*-----------------------------------------------------------------*/
        case OP_STAR:
        case OP_MINSTAR:
+       case OP_POSSTAR:
        case OP_NOTSTAR:
        case OP_NOTMINSTAR:
+       case OP_NOTPOSSTAR:
        ADD_ACTIVE(state_offset + dlen + 1, 0);
        if (clen > 0)
          {
-         int otherd = -1;
+         unsigned int otherd = NOTACHAR;
-         if ((ims && PCRE_CASELESS) != 0)
+         if ((ims & PCRE_CASELESS) != 0)
            {
  #ifdef SUPPORT_UTF8
-           if (utf8 && c >= 128)
+           if (utf8 && d >= 128)
              {
  #ifdef SUPPORT_UCP
-             if (ucp_findchar(c, &chartype, &otherd) < 0) otherd = -1;
+             otherd = UCD_OTHERCASE(d);
  #endif  /* SUPPORT_UCP */
              }
            else
-Line 1158 
 for (;;)
+Line 2041 
 for (;;)
            otherd = fcc[d];
            }
          if ((c == d || c == otherd) == (codevalue < OP_NOTSTAR))
-           { ADD_NEW(state_offset, 0); }
+           {
+           if (codevalue == OP_POSSTAR || codevalue == OP_NOTPOSSTAR)
+             {
+             active_count--;            /* Remove non-match possibility */
+             next_active_state--;
+             }
+           ADD_NEW(state_offset, 0);
+           }
          }
        break;
        /*-----------------------------------------------------------------*/
        case OP_EXACT:
+       case OP_NOTEXACT:
+       count = current_state->count;  /* Number already matched */
+       if (clen > 0)
+         {
+         unsigned int otherd = NOTACHAR;
+         if ((ims & PCRE_CASELESS) != 0)
+           {
+ #ifdef SUPPORT_UTF8
+           if (utf8 && d >= 128)
+             {
+ #ifdef SUPPORT_UCP
+             otherd = UCD_OTHERCASE(d);
+ #endif  /* SUPPORT_UCP */
+             }
+           else
+ #endif  /* SUPPORT_UTF8 */
+           otherd = fcc[d];
+           }
+         if ((c == d || c == otherd) == (codevalue < OP_NOTSTAR))
+           {
+           if (++count >= GET2(code, 1))
+             { ADD_NEW(state_offset + dlen + 3, 0); }
+           else
+             { ADD_NEW(state_offset, count); }
+           }
+         }
+       break;
+       /*-----------------------------------------------------------------*/
        case OP_UPTO:
        case OP_MINUPTO:
-       case OP_NOTEXACT:
+       case OP_POSUPTO:
        case OP_NOTUPTO:
        case OP_NOTMINUPTO:
-       if (codevalue != OP_EXACT && codevalue != OP_NOTEXACT)
+       case OP_NOTPOSUPTO:
-         { ADD_ACTIVE(state_offset + dlen + 3, 0); }
+       ADD_ACTIVE(state_offset + dlen + 3, 0);
        count = current_state->count;  /* Number already matched */
        if (clen > 0)
          {
-         int otherd = -1;
+         unsigned int otherd = NOTACHAR;
          if ((ims & PCRE_CASELESS) != 0)
            {
  #ifdef SUPPORT_UTF8
-           if (utf8 && c >= 128)
+           if (utf8 && d >= 128)
              {
  #ifdef SUPPORT_UCP
-             if (ucp_findchar(d, &chartype, &otherd) < 0) otherd = -1;
+             otherd = UCD_OTHERCASE(d);
  #endif  /* SUPPORT_UCP */
              }
            else
-Line 1190 
 for (;;)
+Line 2109 
 for (;;)
            }
          if ((c == d || c == otherd) == (codevalue < OP_NOTSTAR))
            {
+           if (codevalue == OP_POSUPTO || codevalue == OP_NOTPOSUPTO)
+             {
+             active_count--;             /* Remove non-match possibility */
+             next_active_state--;
+             }
            if (++count >= GET2(code, 1))
              { ADD_NEW(state_offset + dlen + 3, 0); }
            else
-Line 1267 
 for (;;)
+Line 2191 
 for (;;)
              { ADD_ACTIVE(next_state_offset + 5, 0); }
            if (isinclass)
              {
-             if (++count >= GET2(ecode, 3))
+             int max = GET2(ecode, 3);
+             if (++count >= max && max != 0)   /* Max 0 => no limit */
                { ADD_NEW(next_state_offset + 5, 0); }
              else
                { ADD_NEW(state_offset, count); }
-Line 1283 
 for (;;)
+Line 2208 
 for (;;)
  /* ========================================================================== */
        /* These are the opcodes for fancy brackets of various kinds. We have
-       to use recursion in order to handle them. */
+       to use recursion in order to handle them. The "always failing" assertion
+       (?!) is optimised to OP_FAIL when compiling, so we have to support that,
+       though the other "backtracking verbs" are not supported. */
+       case OP_FAIL:
+       forced_fail++;    /* Count FAILs for multiple states */
+       break;
        case OP_ASSERT:
        case OP_ASSERT_NOT:
-Line 1317 
 for (;;)
+Line 2248 
 for (;;)
        /*-----------------------------------------------------------------*/
        case OP_COND:
+       case OP_SCOND:
          {
          int local_offsets[1000];
          int local_workspace[1000];
-         int condcode = code[LINK_SIZE+1];
+         int codelink = GET(code, 1);
+         int condcode;
+         /* Because of the way auto-callout works during compile, a callout item
+         is inserted between OP_COND and an assertion condition. This does not
+         happen for the other conditions. */
+         if (code[LINK_SIZE+1] == OP_CALLOUT)
+           {
+           rrc = 0;
+           if (pcre_callout != NULL)
+             {
+             pcre_callout_block cb;
+             cb.version          = 1;   /* Version 1 of the callout block */
+             cb.callout_number   = code[LINK_SIZE+2];
+             cb.offset_vector    = offsets;
+             cb.subject          = (PCRE_SPTR)start_subject;
+             cb.subject_length   = end_subject - start_subject;
+             cb.start_match      = current_subject - start_subject;
+             cb.current_position = ptr - start_subject;
+             cb.pattern_position = GET(code, LINK_SIZE + 3);
+             cb.next_item_length = GET(code, 3 + 2*LINK_SIZE);
+             cb.capture_top      = 1;
+             cb.capture_last     = -1;
+             cb.callout_data     = md->callout_data;
+             if ((rrc = (*pcre_callout)(&cb)) < 0) return rrc;   /* Abandon */
+             }
+           if (rrc > 0) break;                      /* Fail this thread */
+           code += _pcre_OP_lengths[OP_CALLOUT];    /* Skip callout data */
+           }
+         condcode = code[LINK_SIZE+1];
+         /* Back reference conditions are not supported */
+         if (condcode == OP_CREF || condcode == OP_NCREF)
+           return PCRE_ERROR_DFA_UCOND;
+         /* The DEFINE condition is always false */
-         /* The only supported version of OP_CREF is for the value 0xffff, which
+         if (condcode == OP_DEF)
-         means "test if in a recursion". */
+           { ADD_ACTIVE(state_offset + codelink + LINK_SIZE + 1, 0); }
-         if (condcode == OP_CREF)
+         /* The only supported version of OP_RREF is for the value RREF_ANY,
+         which means "test if in any recursion". We can't test for specifically
+         recursed groups. */
+         else if (condcode == OP_RREF || condcode == OP_NRREF)
            {
            int value = GET2(code, LINK_SIZE+2);
-           if (value != 0xffff) return PCRE_ERROR_DFA_UCOND;
+           if (value != RREF_ANY) return PCRE_ERROR_DFA_UCOND;
-           if (recursing > 0) { ADD_ACTIVE(state_offset + LINK_SIZE + 4, 0); }
+           if (recursing > 0)
-             else { ADD_ACTIVE(state_offset + GET(code, 1) + LINK_SIZE + 1, 0); }
+             { ADD_ACTIVE(state_offset + LINK_SIZE + 4, 0); }
+           else { ADD_ACTIVE(state_offset + codelink + LINK_SIZE + 1, 0); }
            }
          /* Otherwise, the condition is an assertion */
-Line 1360 
 for (;;)
+Line 2335 
 for (;;)
                  (condcode == OP_ASSERT || condcode == OP_ASSERTBACK))
              { ADD_ACTIVE(endasscode + LINK_SIZE + 1 - start_code, 0); }
            else
-             { ADD_ACTIVE(state_offset + GET(code, 1) + LINK_SIZE + 1, 0); }
+             { ADD_ACTIVE(state_offset + codelink + LINK_SIZE + 1, 0); }
            }
          }
        break;
-Line 1424 
 for (;;)
+Line 2399 
 for (;;)
        /*-----------------------------------------------------------------*/
        case OP_ONCE:
          {
-         const uschar *endcode;
          int local_offsets[2];
          int local_workspace[1000];
-Line 1446 
 for (;;)
+Line 2420 
 for (;;)
            const uschar *end_subpattern = code;
            int charcount = local_offsets[1] - local_offsets[0];
            int next_state_offset, repeat_state_offset;
-           BOOL is_repeated;
            do { end_subpattern += GET(end_subpattern, 1); }
              while (*end_subpattern == OP_ALT);
-Line 1514 
 for (;;)
+Line 2487 
 for (;;)
        /* Handle callouts */
        case OP_CALLOUT:
+       rrc = 0;
        if (pcre_callout != NULL)
          {
-         int rrc;
          pcre_callout_block cb;
          cb.version          = 1;   /* Version 1 of the callout block */
          cb.callout_number   = code[1];
          cb.offset_vector    = offsets;
-         cb.subject          = (char *)start_subject;
+         cb.subject          = (PCRE_SPTR)start_subject;
          cb.subject_length   = end_subject - start_subject;
          cb.start_match      = current_subject - start_subject;
          cb.current_position = ptr - start_subject;
-Line 1531 
 for (;;)
+Line 2504 
 for (;;)
          cb.capture_last     = -1;
          cb.callout_data     = md->callout_data;
          if ((rrc = (*pcre_callout)(&cb)) < 0) return rrc;   /* Abandon */
-         if (rrc == 0) { ADD_ACTIVE(state_offset + 2 + 2*LINK_SIZE, 0); }
          }
+       if (rrc == 0)
+         { ADD_ACTIVE(state_offset + _pcre_OP_lengths[OP_CALLOUT], 0); }
        break;
-Line 1548 
 for (;;)
+Line 2522 
 for (;;)
    /* We have finished the processing at the current subject character. If no
    new states have been set for the next character, we have found all the
    matches that we are going to find. If we are at the top level and partial
-   matching has been requested, check for appropriate conditions. */
+   matching has been requested, check for appropriate conditions. The "forced_
+   fail" variable counts the number of (*F) encountered for the character. If it
+   is equal to the original active_count (saved in workspace[1]) it means that
+   (*F) was found on every active state. In this case we don't want to give a
+   partial match. */
    if (new_count <= 0)
      {
-     if (match_count < 0 &&                     /* No matches found */
+     if (rlevel == 1 &&                               /* Top level, and */
-         rlevel == 1 &&                         /* Top level match function */
+         reached_end != workspace[1] &&               /* Not all reached end */
-         (md->moptions & PCRE_PARTIAL) != 0 &&  /* Want partial matching */
+         forced_fail != workspace[1] &&               /* Not all forced fail & */
-         ptr >= end_subject &&                  /* Reached end of subject */
+         (                                            /* either... */
-         ptr > current_subject)                 /* Matched non-empty string */
+         (md->moptions & PCRE_PARTIAL_HARD) != 0      /* Hard partial */
+         ||                                           /* or... */
+         ((md->moptions & PCRE_PARTIAL_SOFT) != 0 &&  /* Soft partial and */
+          match_count < 0)                            /* no matches */
+         ) &&                                         /* And... */
+         ptr >= end_subject &&                     /* Reached end of subject */
+         ptr > current_subject)                    /* Matched non-empty string */
        {
        if (offsetcount >= 2)
          {
-         offsets[0] = current_subject - start_subject;
+         offsets[0] = md->start_used_ptr - start_subject;
          offsets[1] = end_subject - start_subject;
          }
        match_count = PCRE_ERROR_PARTIAL;
-Line 1569 
 for (;;)
+Line 2553 
 for (;;)
      DPRINTF(("%.*sEnd of internal_dfa_exec %d: returning %d\n"
        "%.*s---------------------\n\n", rlevel*2-2, SP, rlevel, match_count,
        rlevel*2-2, SP));
-     return match_count;
+     break;        /* In effect, "return", but see the comment below */
      }
    /* One or more states are active for the next character. */
-Line 1577 
 for (;;)
+Line 2561 
 for (;;)
    ptr += clen;    /* Advance to next subject character */
    }               /* Loop to move along the subject string */
- /* Control never gets here, but we must keep the compiler happy. */
+ /* Control gets here from "break" a few lines above. We do it this way because
+ if we use "return" above, we have compiler trouble. Some compilers warn if
+ there's nothing here because they think the function doesn't return a value. On
+ the other hand, if we put a dummy statement here, some more clever compilers
+ complain that it can't be reached. Sigh. */
- DPRINTF(("%.*s+++ Unexpected end of internal_dfa_exec %d +++\n"
+ return match_count;
-   "%.*s---------------------\n\n", rlevel*2-2, SP, rlevel, rlevel*2-2, SP));
- return PCRE_ERROR_NOMATCH;
  }
-Line 1597 
 is not anchored.
+Line 2583 
 is not anchored.
  Arguments:
    argument_re     points to the compiled expression
-   extra_data      points to extra data or is NULL (not currently used)
+   extra_data      points to extra data or is NULL
    subject         points to the subject string
    length          length of subject string (may contain binary zeros)
    start_offset    where to start in the subject string
-Line 1613 
 Returns:          > 0 => number of match
+Line 2599 
 Returns:          > 0 => number of match
                   < -1 => some kind of unexpected problem
  */
- EXPORT int
+ PCRE_EXP_DEFN int PCRE_CALL_CONVENTION
  pcre_dfa_exec(const pcre *argument_re, const pcre_extra *extra_data,
    const char *subject, int length, int start_offset, int options, int *offsets,
    int offsetcount, int *workspace, int wscount)
  {
  real_pcre *re = (real_pcre *)argument_re;
  dfa_match_data match_block;
+ dfa_match_data *md = &match_block;
  BOOL utf8, anchored, startline, firstline;
  const uschar *current_subject, *end_subject, *lcc;
-Line 1634 
 BOOL req_byte_caseless = FALSE;
+Line 2621 
 BOOL req_byte_caseless = FALSE;
  int first_byte = -1;
  int req_byte = -1;
  int req_byte2 = -1;
+ int newline;
  /* Plausibility checks */
-Line 1648 
 flipping, so we scan the extra_data bloc
+Line 2636 
 flipping, so we scan the extra_data bloc
  match block, so we must initialize them beforehand. However, the other fields
  in the match block must not be set until after the byte flipping. */
- match_block.tables = re->tables;
+ md->tables = re->tables;
- match_block.callout_data = NULL;
+ md->callout_data = NULL;
  if (extra_data != NULL)
    {
-Line 1657 
 if (extra_data != NULL)
+Line 2645 
 if (extra_data != NULL)
    if ((flags & PCRE_EXTRA_STUDY_DATA) != 0)
      study = (const pcre_study_data *)extra_data->study_data;
    if ((flags & PCRE_EXTRA_MATCH_LIMIT) != 0) return PCRE_ERROR_DFA_UMLIMIT;
+   if ((flags & PCRE_EXTRA_MATCH_LIMIT_RECURSION) != 0)
+     return PCRE_ERROR_DFA_UMLIMIT;
    if ((flags & PCRE_EXTRA_CALLOUT_DATA) != 0)
-     match_block.callout_data = extra_data->callout_data;
+     md->callout_data = extra_data->callout_data;
    if ((flags & PCRE_EXTRA_TABLES) != 0)
-     match_block.tables = extra_data->tables;
+     md->tables = extra_data->tables;
    }
  /* Check that the first field in the block is the magic number. If it is not,
  test for a regex that was compiled on a host of opposite endianness. If this is
  the case, flipped values are put in internal_re and internal_study if there was
-Line 1681 
 current_subject = (const unsigned char *
+Line 2671 
 current_subject = (const unsigned char *
  end_subject = (const unsigned char *)subject + length;
  req_byte_ptr = current_subject - 1;
+ #ifdef SUPPORT_UTF8
  utf8 = (re->options & PCRE_UTF8) != 0;
- anchored = (options & PCRE_ANCHORED) != 0 || (re->options & PCRE_ANCHORED) != 0;
+ #else
+ utf8 = FALSE;
+ #endif
+ anchored = (options & (PCRE_ANCHORED|PCRE_DFA_RESTART)) != 0 ||
+   (re->options & PCRE_ANCHORED) != 0;
  /* The remaining fixed data for passing around. */
- match_block.start_code = (const uschar *)argument_re +
+ md->start_code = (const uschar *)argument_re +
      re->name_table_offset + re->name_count * re->name_entry_size;
- match_block.start_subject = (const unsigned char *)subject;
+ md->start_subject = (const unsigned char *)subject;
- match_block.end_subject = end_subject;
+ md->end_subject = end_subject;
- match_block.moptions = options;
+ md->start_offset = start_offset;
- match_block.poptions = re->options;
+ md->moptions = options;
+ md->poptions = re->options;
+ /* If the BSR option is not set at match time, copy what was set
+ at compile time. */
+ if ((md->moptions & (PCRE_BSR_ANYCRLF|PCRE_BSR_UNICODE)) == 0)
+   {
+   if ((re->options & (PCRE_BSR_ANYCRLF|PCRE_BSR_UNICODE)) != 0)
+     md->moptions |= re->options & (PCRE_BSR_ANYCRLF|PCRE_BSR_UNICODE);
+ #ifdef BSR_ANYCRLF
+   else md->moptions |= PCRE_BSR_ANYCRLF;
+ #endif
+   }
+ /* Handle different types of newline. The three bits give eight cases. If
+ nothing is set at run time, whatever was used at compile time applies. */
+ switch ((((options & PCRE_NEWLINE_BITS) == 0)? re->options : (pcre_uint32)options) &
+          PCRE_NEWLINE_BITS)
+   {
+   case 0: newline = NEWLINE; break;   /* Compile-time default */
+   case PCRE_NEWLINE_CR: newline = CHAR_CR; break;
+   case PCRE_NEWLINE_LF: newline = CHAR_NL; break;
+   case PCRE_NEWLINE_CR+
+        PCRE_NEWLINE_LF: newline = (CHAR_CR << 8) | CHAR_NL; break;
+   case PCRE_NEWLINE_ANY: newline = -1; break;
+   case PCRE_NEWLINE_ANYCRLF: newline = -2; break;
+   default: return PCRE_ERROR_BADNEWLINE;
+   }
+ if (newline == -2)
+   {
+   md->nltype = NLTYPE_ANYCRLF;
+   }
+ else if (newline < 0)
+   {
+   md->nltype = NLTYPE_ANY;
+   }
+ else
+   {
+   md->nltype = NLTYPE_FIXED;
+   if (newline > 255)
+     {
+     md->nllen = 2;
+     md->nl[0] = (newline >> 8) & 255;
+     md->nl[1] = newline & 255;
+     }
+   else
+     {
+     md->nllen = 1;
+     md->nl[0] = newline;
+     }
+   }
  /* Check a UTF-8 string if required. Unfortunately there's no way of passing
  back the character offset. */
-Line 1717 
 if (utf8 && (options & PCRE_NO_UTF8_CHEC
+Line 2766 
 if (utf8 && (options & PCRE_NO_UTF8_CHEC
  is a feature that makes it possible to save compiled regex and re-use them
  in other programs later. */
- if (match_block.tables == NULL) match_block.tables = _pcre_default_tables;
+ if (md->tables == NULL) md->tables = _pcre_default_tables;
  /* The lower casing table and the "must be at the start of a line" flag are
  used in a loop when finding where to start. */
- lcc = match_block.tables + lcc_offset;
+ lcc = md->tables + lcc_offset;
- startline = (re->options & PCRE_STARTLINE) != 0;
+ startline = (re->flags & PCRE_STARTLINE) != 0;
  firstline = (re->options & PCRE_FIRSTLINE) != 0;
  /* Set up the first character to match, if available. The first_byte value is
-Line 1734 
 studied, there may be a bitmap of possib
+Line 2783 
 studied, there may be a bitmap of possib
  if (!anchored)
    {
-   if ((re->options & PCRE_FIRSTSET) != 0)
+   if ((re->flags & PCRE_FIRSTSET) != 0)
      {
      first_byte = re->first_byte & 255;
      if ((first_byte_caseless = ((re->first_byte & REQ_CASELESS) != 0)) == TRUE)
-Line 1742 
 if (!anchored)
+Line 2791 
 if (!anchored)
      }
    else
      {
-     if (startline && study != NULL &&
+     if (!startline && study != NULL &&
-          (study->options & PCRE_STUDY_MAPPED) != 0)
+          (study->flags & PCRE_STUDY_MAPPED) != 0)
        start_bits = study->start_bits;
      }
    }
-Line 1751 
 if (!anchored)
+Line 2800 
 if (!anchored)
  /* For anchored or unanchored matches, there may be a "last known required
  character" set. */
- if ((re->options & PCRE_REQCHSET) != 0)
+ if ((re->flags & PCRE_REQCHSET) != 0)
    {
    req_byte = re->req_byte & 255;
    req_byte_caseless = (re->req_byte & REQ_CASELESS) != 0;
-   req_byte2 = (match_block.tables + fcc_offset)[req_byte];  /* case flipped */
+   req_byte2 = (md->tables + fcc_offset)[req_byte];  /* case flipped */
    }
  /* Call the main matching function, looping for a non-anchored regex after a
- failed match. Unless restarting, optimize by moving to the first match
+ failed match. If not restarting, perform certain optimizations at the start of
- character if possible, when not anchored. Then unless wanting a partial match,
+ a match. */
- check for a required later character. */
  for (;;)
    {
-Line 1771 
 for (;;)
+Line 2819 
 for (;;)
      {
      const uschar *save_end_subject = end_subject;
-     /* Advance to a unique first char if possible. If firstline is TRUE, the
+     /* If firstline is TRUE, the start of the match is constrained to the first
-     start of the match is constrained to the first line of a multiline string.
+     line of a multiline string. Implement this by temporarily adjusting
-     Implement this by temporarily adjusting end_subject so that we stop scanning
+     end_subject so that we stop scanning at a newline. If the match fails at
-     at a newline. If the match fails at the newline, later code breaks this loop.
+     the newline, later code breaks this loop. */
-     */
      if (firstline)
        {
-       const uschar *t = current_subject;
+       USPTR t = current_subject;
-       while (t < save_end_subject && *t != '\n') t++;
+ #ifdef SUPPORT_UTF8
-       end_subject = t;
+       if (utf8)
-       }
-     if (first_byte >= 0)
-       {
-       if (first_byte_caseless)
-         while (current_subject < end_subject &&
-                lcc[*current_subject] != first_byte)
-           current_subject++;
-       else
-         while (current_subject < end_subject && *current_subject != first_byte)
-           current_subject++;
-       }
-     /* Or to just after \n for a multiline match if possible */
-     else if (startline)
-       {
-       if (current_subject > match_block.start_subject + start_offset)
          {
-         while (current_subject < end_subject && current_subject[-1] != NEWLINE)
+         while (t < md->end_subject && !IS_NEWLINE(t))
-           current_subject++;
+           {
+           t++;
+           while (t < end_subject && (*t & 0xc0) == 0x80) t++;
+           }
          }
+       else
+ #endif
+       while (t < md->end_subject && !IS_NEWLINE(t)) t++;
+       end_subject = t;
        }
-     /* Or to a non-unique first char after study */
+     /* There are some optimizations that avoid running the match if a known
+     starting point is not found. However, there is an option that disables
+     these, for testing and for ensuring that all callouts do actually occur. */
-     else if (start_bits != NULL)
+     if ((options & PCRE_NO_START_OPTIMIZE) == 0)
        {
-       while (current_subject < end_subject)
+       /* Advance to a known first byte. */
+       if (first_byte >= 0)
          {
-         register unsigned int c = *current_subject;
+         if (first_byte_caseless)
-         if ((start_bits[c/8] & (1 << (c&7))) == 0) current_subject++;
+           while (current_subject < end_subject &&
-           else break;
+                  lcc[*current_subject] != first_byte)
+             current_subject++;
+         else
+           while (current_subject < end_subject &&
+                  *current_subject != first_byte)
+             current_subject++;
          }
-       }
-     /* Restore fudged end_subject */
-     end_subject = save_end_subject;
+       /* Or to just after a linebreak for a multiline match if possible */
-     }
-   /* If req_byte is set, we know that that character must appear in the subject
-   for the match to succeed. If the first character is set, req_byte must be
-   later in the subject; otherwise the test starts at the match point. This
-   optimization can save a huge amount of work in patterns with nested unlimited
-   repeats that aren't going to match. Writing separate code for cased/caseless
-   versions makes it go faster, as does using an autoincrement and backing off
-   on a match.
-   HOWEVER: when the subject string is very, very long, searching to its end can
-   take a long time, and give bad performance on quite ordinary patterns. This
-   showed up when somebody was matching /^C/ on a 32-megabyte string... so we
-   don't do this when the string is sufficiently long.
-   ALSO: this processing is disabled when partial matching is requested.
-   */
-   if (req_byte >= 0 &&
-       end_subject - current_subject < REQ_BYTE_MAX &&
-       (options & PCRE_PARTIAL) == 0)
-     {
-     register const uschar *p = current_subject + ((first_byte >= 0)? 1 : 0);
-     /* We don't need to repeat the search if we haven't yet reached the
+       else if (startline)
-     place we found it at last time. */
-     if (p > req_byte_ptr)
-       {
-       if (req_byte_caseless)
          {
-         while (p < end_subject)
+         if (current_subject > md->start_subject + start_offset)
            {
-           register int pp = *p++;
+ #ifdef SUPPORT_UTF8
-           if (pp == req_byte || pp == req_byte2) { p--; break; }
+           if (utf8)
+             {
+             while (current_subject < end_subject &&
+                    !WAS_NEWLINE(current_subject))
+               {
+               current_subject++;
+               while(current_subject < end_subject &&
+                     (*current_subject & 0xc0) == 0x80)
+                 current_subject++;
+               }
+             }
+           else
+ #endif
+           while (current_subject < end_subject && !WAS_NEWLINE(current_subject))
+             current_subject++;
+           /* If we have just passed a CR and the newline option is ANY or
+           ANYCRLF, and we are now at a LF, advance the match position by one
+           more character. */
+           if (current_subject[-1] == CHAR_CR &&
+                (md->nltype == NLTYPE_ANY || md->nltype == NLTYPE_ANYCRLF) &&
+                current_subject < end_subject &&
+                *current_subject == CHAR_NL)
+             current_subject++;
            }
          }
-       else
+       /* Or to a non-unique first char after study */
+       else if (start_bits != NULL)
          {
-         while (p < end_subject)
+         while (current_subject < end_subject)
            {
-           if (*p++ == req_byte) { p--; break; }
+           register unsigned int c = *current_subject;
+           if ((start_bits[c/8] & (1 << (c&7))) == 0) current_subject++;
+             else break;
            }
          }
+       }
-       /* If we can't find the required character, break the matching loop,
+     /* Restore fudged end_subject */
-       which will cause a return or PCRE_ERROR_NOMATCH. */
-       if (p >= end_subject) break;
-       /* If we have found the required character, save the point where we
+     end_subject = save_end_subject;
-       found it, so that we don't search again next time round the loop if
-       the start hasn't passed this character yet. */
-       req_byte_ptr = p;
+     /* The following two optimizations are disabled for partial matching or if
+     disabling is explicitly requested (and of course, by the test above, this
+     code is not obeyed when restarting after a partial match). */
+     if ((options & PCRE_NO_START_OPTIMIZE) == 0 &&
+         (options & (PCRE_PARTIAL_HARD|PCRE_PARTIAL_SOFT)) == 0)
+       {
+       /* If the pattern was studied, a minimum subject length may be set. This
+       is a lower bound; no actual string of that length may actually match the
+       pattern. Although the value is, strictly, in characters, we treat it as
+       bytes to avoid spending too much time in this optimization. */
+       if (study != NULL && (study->flags & PCRE_STUDY_MINLEN) != 0 &&
+           end_subject - current_subject < study->minlength)
+         return PCRE_ERROR_NOMATCH;
+       /* If req_byte is set, we know that that character must appear in the
+       subject for the match to succeed. If the first character is set, req_byte
+       must be later in the subject; otherwise the test starts at the match
+       point. This optimization can save a huge amount of work in patterns with
+       nested unlimited repeats that aren't going to match. Writing separate
+       code for cased/caseless versions makes it go faster, as does using an
+       autoincrement and backing off on a match.
+       HOWEVER: when the subject string is very, very long, searching to its end
+       can take a long time, and give bad performance on quite ordinary
+       patterns. This showed up when somebody was matching /^C/ on a 32-megabyte
+       string... so we don't do this when the string is sufficiently long. */
+       if (req_byte >= 0 && end_subject - current_subject < REQ_BYTE_MAX)
+         {
+         register const uschar *p = current_subject + ((first_byte >= 0)? 1 : 0);
+         /* We don't need to repeat the search if we haven't yet reached the
+         place we found it at last time. */
+         if (p > req_byte_ptr)
+           {
+           if (req_byte_caseless)
+             {
+             while (p < end_subject)
+               {
+               register int pp = *p++;
+               if (pp == req_byte || pp == req_byte2) { p--; break; }
+               }
+             }
+           else
+             {
+             while (p < end_subject)
+               {
+               if (*p++ == req_byte) { p--; break; }
+               }
+             }
+           /* If we can't find the required character, break the matching loop,
+           which will cause a return or PCRE_ERROR_NOMATCH. */
+           if (p >= end_subject) break;
+           /* If we have found the required character, save the point where we
+           found it, so that we don't search again next time round the loop if
+           the start hasn't passed this character yet. */
+           req_byte_ptr = p;
+           }
+         }
        }
-     }
+     }   /* End of optimizations that are done when not restarting */
    /* OK, now we can do the business */
+   md->start_used_ptr = current_subject;
    rc = internal_dfa_exec(
-     &match_block,                              /* fixed match data */
+     md,                                /* fixed match data */
-     match_block.start_code,                    /* this subexpression's code */
+     md->start_code,                    /* this subexpression's code */
-     current_subject,                           /* where we currently are */
+     current_subject,                   /* where we currently are */
-     start_offset,                              /* start offset in subject */
+     start_offset,                      /* start offset in subject */
-     offsets,                                   /* offset vector */
+     offsets,                           /* offset vector */
-     offsetcount,                               /* size of same */
+     offsetcount,                       /* size of same */
-     workspace,                                 /* workspace vector */
+     workspace,                         /* workspace vector */
-     wscount,                                   /* size of same */
+     wscount,                           /* size of same */
      re->options & (PCRE_CASELESS|PCRE_MULTILINE|PCRE_DOTALL), /* ims flags */
-,                                         /* function recurse level */
+,                                 /* function recurse level */
-);                                        /* regex recurse level */
+);                                /* regex recurse level */
    /* Anything other than "no match" means we are done, always; otherwise, carry
    on only if not anchored. */
-Line 1902 
 for (;;)
+Line 3008 
 for (;;)
    /* Advance to the next subject character unless we are at the end of a line
    and firstline is set. */
-   if (firstline && *current_subject == NEWLINE) break;
+   if (firstline && IS_NEWLINE(current_subject)) break;
    current_subject++;
- #ifdef SUPPORT_UTF8
    if (utf8)
      {
      while (current_subject < end_subject && (*current_subject & 0xc0) == 0x80)
        current_subject++;
      }
- #endif
    if (current_subject > end_subject) break;
-   }
+   /* If we have just passed a CR and we are now at a LF, and the pattern does
+   not contain any explicit matches for \r or \n, and the newline option is CRLF
+   or ANY or ANYCRLF, advance the match position by one more character. */
+   if (current_subject[-1] == CHAR_CR &&
+       current_subject < end_subject &&
+       *current_subject == CHAR_NL &&
+       (re->flags & PCRE_HASCRORLF) == 0 &&
+         (md->nltype == NLTYPE_ANY ||
+          md->nltype == NLTYPE_ANYCRLF ||
+          md->nllen == 2))
+     current_subject++;
+   }   /* "Bumpalong" loop */
  return PCRE_ERROR_NOMATCH;
  }

 Legend:



Removed from v.77
 


changed lines


 
Added in v.459
 Legend:



Removed from v.77
 


changed lines


 
Added in v.459
-Removed from v.77
+Added in v.459

webmaster@exim.org	ViewVC Help
Powered by ViewVC 1.1.12