/[pcre]/code/branches/pcre16/pcretest.c

Diff of /code/branches/pcre16/pcretest.c

Parent Directory | Revision Log | View Patch Patch

-code/trunk/pcretest.c
revision 227 by ph10,
Tue Aug 21 15:00:15 2007 UTC
+code/branches/pcre16/pcretest.c
revision 805 by ph10,
Wed Dec 14 16:49:20 2011 UTC
 Line 37 
 POSSIBILITY OF SUCH DAMAGE.
  #ifdef HAVE_CONFIG_H
- #include <config.h>
+ #include "config.h"
  #endif
  #include <ctype.h>
 Line 48 
 POSSIBILITY OF SUCH DAMAGE.
  #include <locale.h>
  #include <errno.h>
+ #ifdef SUPPORT_LIBREADLINE
+ #ifdef HAVE_UNISTD_H
+ #include <unistd.h>
+ #endif
+ #include <readline/readline.h>
+ #include <readline/history.h>
+ #endif
  /* A number of things vary for Windows builds. Originally, pcretest opened its
  input and output without "b"; then I was told that "b" was needed in some
-Line 63 
 input mode under Windows. */
+Line 71 
 input mode under Windows. */
  #define INPUT_MODE   "r"
  #define OUTPUT_MODE  "wb"
+ #ifndef isatty
+ #define isatty _isatty         /* This is what Windows calls them, I'm told, */
+ #endif                         /* though in some environments they seem to   */
+                                /* be already defined, hence the #ifndefs.    */
+ #ifndef fileno
+ #define fileno _fileno
+ #endif
+ /* A user sent this fix for Borland Builder 5 under Windows. */
+ #ifdef __BORLANDC__
+ #define _setmode(handle, mode) setmode(handle, mode)
+ #endif
+ /* Not Windows */
  #else
  #include <sys/time.h>          /* These two includes are needed */
  #include <sys/resource.h>      /* for setrlimit(). */
-Line 83 
 appropriately for an application, not fo
+Line 107 
 appropriately for an application, not fo
  #include "pcre.h"
  #include "pcre_internal.h"
- /* We need access to the data tables that PCRE uses. So as not to have to keep
+ /* The pcre_printint() function, which prints the internal form of a compiled
- two copies, we include the source file here, changing the names of the external
+ regex, is held in a separate file so that (a) it can be compiled in either
- symbols to prevent clashes. */
+-bit or 16-bit mode, and (b) it can be #included directly in pcre_compile.c
+ when that is compiled in debug mode. */
+ #ifdef SUPPORT_PCRE8
+ void pcre_printint(pcre *external_re, FILE *f, BOOL print_lengths);
+ #endif
+ #ifdef SUPPORT_PCRE16
+ void pcre16_printint(pcre *external_re, FILE *f, BOOL print_lengths);
+ #endif
+ /* We need access to some of the data tables that PCRE uses. So as not to have
+ to keep two copies, we include the source file here, changing the names of the
+ external symbols to prevent clashes. */
+ #define _pcre_ucp_gentype      ucp_gentype
+ #define _pcre_ucp_typerange    ucp_typerange
  #define _pcre_utf8_table1      utf8_table1
  #define _pcre_utf8_table1_size utf8_table1_size
  #define _pcre_utf8_table2      utf8_table2
-Line 94 
 symbols to prevent clashes. */
+Line 132 
 symbols to prevent clashes. */
  #define _pcre_utf8_table4      utf8_table4
  #define _pcre_utt              utt
  #define _pcre_utt_size         utt_size
+ #define _pcre_utt_names        utt_names
  #define _pcre_OP_lengths       OP_lengths
  #include "pcre_tables.c"
- /* We also need the pcre_printint() function for printing out compiled
+ /* The definition of the macro PRINTABLE, which determines whether to print an
- patterns. This function is in a separate file so that it can be included in
- pcre_compile.c when that module is compiled with debugging enabled.
- The definition of the macro PRINTABLE, which determines whether to print an
  output character as-is or as a hex value when showing compiled patterns, is
- contained in this file. We uses it here also, in cases when the locale has not
+ the same as in the printint.src file. We uses it here in cases when the locale
- been explicitly changed, so as to get consistent output from systems that
+ has not been explicitly changed, so as to get consistent output from systems
- differ in their output from isprint() even in the "C" locale. */
+ that differ in their output from isprint() even in the "C" locale. */
- #include "pcre_printint.src"
+ #ifdef EBCDIC
+ #define PRINTABLE(c) ((c) >= 64 && (c) < 255)
+ #else
+ #define PRINTABLE(c) ((c) >= 32 && (c) < 127)
+ #endif
  #define PRINTHEX(c) (locale_set? isprint(c) : PRINTABLE(c))
  /* It is possible to compile this test program without including support for
  testing the POSIX interface, though this is not available via the standard
  Makefile. */
-Line 162 
 static int locale_set = 0;
+Line 200 
 static int locale_set = 0;
  static int show_malloc;
  static int use_utf8;
  static size_t gotten_store;
+ static size_t first_gotten_store = 0;
+ static const unsigned char *last_callout_mark = NULL;
+ static int (*fullinfo)(const pcre *, const pcre_extra *, int, void *);
  /* The buffers grow automatically if very long input lines are encountered. */
  static int buffer_size = 50000;
- static uschar *buffer = NULL;
+ static pcre_uint8 *buffer = NULL;
- static uschar *dbuffer = NULL;
+ static pcre_uint8 *dbuffer = NULL;
- static uschar *pbuffer = NULL;
+ static pcre_uint8 *pbuffer = NULL;
+ #ifdef SUPPORT_PCRE16
+ static int buffer16_size = 0;
+ static pcre_uint16 *buffer16 = NULL;
+ #endif
+ /* Textual explanations for runtime error codes */
+ static const char *errtexts[] = {
+   NULL,  /* 0 is no error */
+   NULL,  /* NOMATCH is handled specially */
+   "NULL argument passed",
+   "bad option value",
+   "magic number missing",
+   "unknown opcode - pattern overwritten?",
+   "no more memory",
+   NULL,  /* never returned by pcre_exec() or pcre_dfa_exec() */
+   "match limit exceeded",
+   "callout error code",
+   NULL,  /* BADUTF8 is handled specially */
+   "bad UTF-8 offset",
+   NULL,  /* PARTIAL is handled specially */
+   "not used - internal error",
+   "internal error - pattern overwritten?",
+   "bad count value",
+   "item unsupported for DFA matching",
+   "backreference condition or recursion test not supported for DFA matching",
+   "match limit not supported for DFA matching",
+   "workspace size exceeded in DFA matching",
+   "too much recursion for DFA matching",
+   "recursion limit exceeded",
+   "not used - internal error",
+   "invalid combination of newline options",
+   "bad offset value",
+   NULL,  /* SHORTUTF8 is handled specially */
+   "nested recursion at the same subject position",
+   "JIT stack limit reached",
+   "pattern compiled in wrong mode (8-bit/16-bit error)"
+ };
+ /*************************************************
+ *         Alternate character tables             *
+ *************************************************/
+ /* By default, the "tables" pointer when calling PCRE is set to NULL, thereby
+ using the default tables of the library. However, the T option can be used to
+ select alternate sets of tables, for different kinds of testing. Note also that
+ the L (locale) option also adjusts the tables. */
+ /* This is the set of tables distributed as default with PCRE. It recognizes
+ only ASCII characters. */
+ static const unsigned char tables0[] = {
+ /* This table is a lower casing table. */
+,  1,  2,  3,  4,  5,  6,  7,
+,  9, 10, 11, 12, 13, 14, 15,
+, 17, 18, 19, 20, 21, 22, 23,
+, 25, 26, 27, 28, 29, 30, 31,
+, 33, 34, 35, 36, 37, 38, 39,
+, 41, 42, 43, 44, 45, 46, 47,
+, 49, 50, 51, 52, 53, 54, 55,
+, 57, 58, 59, 60, 61, 62, 63,
+, 97, 98, 99,100,101,102,103,
+,105,106,107,108,109,110,111,
+,113,114,115,116,117,118,119,
+,121,122, 91, 92, 93, 94, 95,
+, 97, 98, 99,100,101,102,103,
+,105,106,107,108,109,110,111,
+,113,114,115,116,117,118,119,
+,121,122,123,124,125,126,127,
+,129,130,131,132,133,134,135,
+,137,138,139,140,141,142,143,
+,145,146,147,148,149,150,151,
+,153,154,155,156,157,158,159,
+,161,162,163,164,165,166,167,
+,169,170,171,172,173,174,175,
+,177,178,179,180,181,182,183,
+,185,186,187,188,189,190,191,
+,193,194,195,196,197,198,199,
+,201,202,203,204,205,206,207,
+,209,210,211,212,213,214,215,
+,217,218,219,220,221,222,223,
+,225,226,227,228,229,230,231,
+,233,234,235,236,237,238,239,
+,241,242,243,244,245,246,247,
+,249,250,251,252,253,254,255,
+ /* This table is a case flipping table. */
+,  1,  2,  3,  4,  5,  6,  7,
+,  9, 10, 11, 12, 13, 14, 15,
+, 17, 18, 19, 20, 21, 22, 23,
+, 25, 26, 27, 28, 29, 30, 31,
+, 33, 34, 35, 36, 37, 38, 39,
+, 41, 42, 43, 44, 45, 46, 47,
+, 49, 50, 51, 52, 53, 54, 55,
+, 57, 58, 59, 60, 61, 62, 63,
+, 97, 98, 99,100,101,102,103,
+,105,106,107,108,109,110,111,
+,113,114,115,116,117,118,119,
+,121,122, 91, 92, 93, 94, 95,
+, 65, 66, 67, 68, 69, 70, 71,
+, 73, 74, 75, 76, 77, 78, 79,
+, 81, 82, 83, 84, 85, 86, 87,
+, 89, 90,123,124,125,126,127,
+,129,130,131,132,133,134,135,
+,137,138,139,140,141,142,143,
+,145,146,147,148,149,150,151,
+,153,154,155,156,157,158,159,
+,161,162,163,164,165,166,167,
+,169,170,171,172,173,174,175,
+,177,178,179,180,181,182,183,
+,185,186,187,188,189,190,191,
+,193,194,195,196,197,198,199,
+,201,202,203,204,205,206,207,
+,209,210,211,212,213,214,215,
+,217,218,219,220,221,222,223,
+,225,226,227,228,229,230,231,
+,233,234,235,236,237,238,239,
+,241,242,243,244,245,246,247,
+,249,250,251,252,253,254,255,
+ /* This table contains bit maps for various character classes. Each map is 32
+ bytes long and the bits run from the least significant end of each byte. The
+ classes that have their own maps are: space, xdigit, digit, upper, lower, word,
+ graph, print, punct, and cntrl. Other classes are built from combinations. */
+x00,0x3e,0x00,0x00,0x01,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03,
+x7e,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+xfe,0xff,0xff,0x07,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0xfe,0xff,0xff,0x07,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03,
+xfe,0xff,0xff,0x87,0xfe,0xff,0xff,0x07,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0xfe,0xff,0xff,0xff,
+xff,0xff,0xff,0xff,0xff,0xff,0xff,0x7f,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0xff,0xff,0xff,0xff,
+xff,0xff,0xff,0xff,0xff,0xff,0xff,0x7f,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0xfe,0xff,0x00,0xfc,
+x01,0x00,0x00,0xf8,0x01,0x00,0x00,0x78,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+xff,0xff,0xff,0xff,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x80,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+ /* This table identifies various classes of character by individual bits:
+x01   white space character
+x02   letter
+x04   decimal digit
+x08   hexadecimal digit
+x10   alphanumeric or '_'
+x80   regular expression metacharacter or binary zero
+ */
+x80,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /*   0-  7 */
+x00,0x01,0x01,0x00,0x01,0x01,0x00,0x00, /*   8- 15 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /*  16- 23 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /*  24- 31 */
+x01,0x00,0x00,0x00,0x80,0x00,0x00,0x00, /*    - '  */
+x80,0x80,0x80,0x80,0x00,0x00,0x80,0x00, /*  ( - /  */
+x1c,0x1c,0x1c,0x1c,0x1c,0x1c,0x1c,0x1c, /*  0 - 7  */
+x1c,0x1c,0x00,0x00,0x00,0x00,0x00,0x80, /*  8 - ?  */
+x00,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x12, /*  @ - G  */
+x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /*  H - O  */
+x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /*  P - W  */
+x12,0x12,0x12,0x80,0x80,0x00,0x80,0x10, /*  X - _  */
+x00,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x12, /*  ` - g  */
+x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /*  h - o  */
+x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /*  p - w  */
+x12,0x12,0x12,0x80,0x80,0x00,0x00,0x00, /*  x -127 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 128-135 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 136-143 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 144-151 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 152-159 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 160-167 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 168-175 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 176-183 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 184-191 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 192-199 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 200-207 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 208-215 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 216-223 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 224-231 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 232-239 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 240-247 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00};/* 248-255 */
+ /* This is a set of tables that came orginally from a Windows user. It seems to
+ be at least an approximation of ISO 8859. In particular, there are characters
+ greater than 128 that are marked as spaces, letters, etc. */
+ static const unsigned char tables1[] = {
+,1,2,3,4,5,6,7,
+,9,10,11,12,13,14,15,
+,17,18,19,20,21,22,23,
+,25,26,27,28,29,30,31,
+,33,34,35,36,37,38,39,
+,41,42,43,44,45,46,47,
+,49,50,51,52,53,54,55,
+,57,58,59,60,61,62,63,
+,97,98,99,100,101,102,103,
+,105,106,107,108,109,110,111,
+,113,114,115,116,117,118,119,
+,121,122,91,92,93,94,95,
+,97,98,99,100,101,102,103,
+,105,106,107,108,109,110,111,
+,113,114,115,116,117,118,119,
+,121,122,123,124,125,126,127,
+,129,130,131,132,133,134,135,
+,137,138,139,140,141,142,143,
+,145,146,147,148,149,150,151,
+,153,154,155,156,157,158,159,
+,161,162,163,164,165,166,167,
+,169,170,171,172,173,174,175,
+,177,178,179,180,181,182,183,
+,185,186,187,188,189,190,191,
+,225,226,227,228,229,230,231,
+,233,234,235,236,237,238,239,
+,241,242,243,244,245,246,215,
+,249,250,251,252,253,254,223,
+,225,226,227,228,229,230,231,
+,233,234,235,236,237,238,239,
+,241,242,243,244,245,246,247,
+,249,250,251,252,253,254,255,
+,1,2,3,4,5,6,7,
+,9,10,11,12,13,14,15,
+,17,18,19,20,21,22,23,
+,25,26,27,28,29,30,31,
+,33,34,35,36,37,38,39,
+,41,42,43,44,45,46,47,
+,49,50,51,52,53,54,55,
+,57,58,59,60,61,62,63,
+,97,98,99,100,101,102,103,
+,105,106,107,108,109,110,111,
+,113,114,115,116,117,118,119,
+,121,122,91,92,93,94,95,
+,65,66,67,68,69,70,71,
+,73,74,75,76,77,78,79,
+,81,82,83,84,85,86,87,
+,89,90,123,124,125,126,127,
+,129,130,131,132,133,134,135,
+,137,138,139,140,141,142,143,
+,145,146,147,148,149,150,151,
+,153,154,155,156,157,158,159,
+,161,162,163,164,165,166,167,
+,169,170,171,172,173,174,175,
+,177,178,179,180,181,182,183,
+,185,186,187,188,189,190,191,
+,225,226,227,228,229,230,231,
+,233,234,235,236,237,238,239,
+,241,242,243,244,245,246,215,
+,249,250,251,252,253,254,223,
+,193,194,195,196,197,198,199,
+,201,202,203,204,205,206,207,
+,209,210,211,212,213,214,247,
+,217,218,219,220,221,222,255,
+,62,0,0,1,0,0,0,
+,0,0,0,0,0,0,0,
+,0,0,0,1,0,0,0,
+,0,0,0,0,0,0,0,
+,0,0,0,0,0,255,3,
+,0,0,0,126,0,0,0,
+,0,0,0,0,0,0,0,
+,0,0,0,0,0,0,0,
+,0,0,0,0,0,255,3,
+,0,0,0,0,0,0,0,
+,0,0,0,0,0,12,2,
+,0,0,0,0,0,0,0,
+,0,0,0,0,0,0,0,
+,255,255,7,0,0,0,0,
+,0,0,0,0,0,0,0,
+,255,127,127,0,0,0,0,
+,0,0,0,0,0,0,0,
+,0,0,0,254,255,255,7,
+,0,0,0,0,4,32,4,
+,0,0,128,255,255,127,255,
+,0,0,0,0,0,255,3,
+,255,255,135,254,255,255,7,
+,0,0,0,0,4,44,6,
+,255,127,255,255,255,127,255,
+,0,0,0,254,255,255,255,
+,255,255,255,255,255,255,127,
+,0,0,0,254,255,255,255,
+,255,255,255,255,255,255,255,
+,2,0,0,255,255,255,255,
+,255,255,255,255,255,255,127,
+,0,0,0,255,255,255,255,
+,255,255,255,255,255,255,255,
+,0,0,0,254,255,0,252,
+,0,0,248,1,0,0,120,
+,0,0,0,254,255,255,255,
+,0,128,0,0,0,128,0,
+,255,255,255,0,0,0,0,
+,0,0,0,0,0,0,128,
+,255,255,255,0,0,0,0,
+,0,0,0,0,0,0,0,
+,0,0,0,0,0,0,0,
+,1,1,0,1,1,0,0,
+,0,0,0,0,0,0,0,
+,0,0,0,0,0,0,0,
+,0,0,0,128,0,0,0,
+,128,128,128,0,0,128,0,
+,28,28,28,28,28,28,28,
+,28,0,0,0,0,0,128,
+,26,26,26,26,26,26,18,
+,18,18,18,18,18,18,18,
+,18,18,18,18,18,18,18,
+,18,18,128,128,0,128,16,
+,26,26,26,26,26,26,18,
+,18,18,18,18,18,18,18,
+,18,18,18,18,18,18,18,
+,18,18,128,128,0,0,0,
+,0,0,0,0,1,0,0,
+,0,0,0,0,0,0,0,
+,0,0,0,0,0,0,0,
+,0,0,0,0,0,0,0,
+,0,0,0,0,0,0,0,
+,0,18,0,0,0,0,0,
+,0,20,20,0,18,0,0,
+,20,18,0,0,0,0,0,
+,18,18,18,18,18,18,18,
+,18,18,18,18,18,18,18,
+,18,18,18,18,18,18,0,
+,18,18,18,18,18,18,18,
+,18,18,18,18,18,18,18,
+,18,18,18,18,18,18,18,
+,18,18,18,18,18,18,0,
+,18,18,18,18,18,18,18
+ };
+ #ifndef HAVE_STRERROR
+ /*************************************************
+ *     Provide strerror() for non-ANSI libraries  *
+ *************************************************/
+ /* Some old-fashioned systems still around (e.g. SunOS4) don't have strerror()
+ in their libraries, but can provide the same facility by this simple
+ alternative function. */
+ extern int   sys_nerr;
+ extern char *sys_errlist[];
+ char *
+ strerror(int n)
+ {
+ if (n < 0 || n >= sys_nerr) return "unknown error number";
+ return sys_errlist[n];
+ }
+ #endif /* HAVE_STRERROR */
+ /*************************************************
+ *         JIT memory callback                    *
+ *************************************************/
+ static pcre_jit_stack* jit_callback(void *arg)
+ {
+ return (pcre_jit_stack *)arg;
+ }
+ #ifdef SUPPORT_PCRE16
+ /*************************************************
+ *         Convert a string to 16-bit             *
+ *************************************************/
+ /* The result is always left in buffer16. */
+ static int
+ to16(unsigned char *p, int utf)
+ {
+ pcre_uint16 *pp;
+ int len = (int)strlen((char *)p) + 1;
+ if (buffer16_size < 2*len)
+   {
+   if (buffer16 != NULL) free(buffer16);
+   buffer16_size = 2*len;
+   buffer16 = (pcre_uint16 *)malloc(buffer16_size);
+   if (buffer16 == NULL)
+     {
+     fprintf(stderr, "pcretest: malloc(%d) failed for buffer16\n", buffer16_size);
+     exit(1);
+     }
+   }
+ pp = buffer16;
+ if (!utf)
+   {
+   while (*p != 0) *pp++ = *p++;
+   *pp++ = 0;
+   }
+ else
+   {
+ fprintf(stderr, "pcretest: no support yet for UTF-16\n");
+ exit(1);
+   }
+ return pp - buffer16;
+ }
+ #endif
  /*************************************************
-Line 188 
 optimal way of handling this, but hey, t
+Line 670 
 optimal way of handling this, but hey, t
  Arguments:
    f            the file to read
    start        where in buffer to start (this *must* be within buffer)
+   prompt       for stdin or readline()
  Returns:       pointer to the start of new data
                 could be a copy of start, or could be moved
                 NULL if no data read and EOF reached
  */
- static uschar *
+ static pcre_uint8 *
- extend_inputline(FILE *f, uschar *start)
+ extend_inputline(FILE *f, pcre_uint8 *start, const char *prompt)
  {
- uschar *here = start;
+ pcre_uint8 *here = start;
  for (;;)
    {
-   int rlen = buffer_size - (here - buffer);
+   int rlen = (int)(buffer_size - (here - buffer));
    if (rlen > 1000)
      {
      int dlen;
-     if (fgets((char *)here, rlen,  f) == NULL)
-       return (here == start)? NULL : start;
+     /* If libreadline support is required, use readline() to read a line if the
+     input is a terminal. Note that readline() removes the trailing newline, so
+     we must put it back again, to be compatible with fgets(). */
+ #ifdef SUPPORT_LIBREADLINE
+     if (isatty(fileno(f)))
+       {
+       size_t len;
+       char *s = readline(prompt);
+       if (s == NULL) return (here == start)? NULL : start;
+       len = strlen(s);
+       if (len > 0) add_history(s);
+       if (len > rlen - 1) len = rlen - 1;
+       memcpy(here, s, len);
+       here[len] = '\n';
+       here[len+1] = 0;
+       free(s);
+       }
+     else
+ #endif
+     /* Read the next line by normal means, prompting if the file is stdin. */
+       {
+       if (f == stdin) printf("%s", prompt);
+       if (fgets((char *)here, rlen,  f) == NULL)
+         return (here == start)? NULL : start;
+       }
      dlen = (int)strlen((char *)here);
      if (dlen > 0 && here[dlen - 1] == '\n') return start;
      here += dlen;
-Line 216 
 for (;;)
+Line 727 
 for (;;)
    else
      {
      int new_buffer_size = 2*buffer_size;
-     uschar *new_buffer = (unsigned char *)malloc(new_buffer_size);
+     pcre_uint8 *new_buffer = (unsigned char *)malloc(new_buffer_size);
-     uschar *new_dbuffer = (unsigned char *)malloc(new_buffer_size);
+     pcre_uint8 *new_dbuffer = (unsigned char *)malloc(new_buffer_size);
-     uschar *new_pbuffer = (unsigned char *)malloc(new_buffer_size);
+     pcre_uint8 *new_pbuffer = (unsigned char *)malloc(new_buffer_size);
      if (new_buffer == NULL || new_dbuffer == NULL || new_pbuffer == NULL)
        {
-Line 360 
 Returns:     number of characters placed
+Line 871 
 Returns:     number of characters placed
  #if !defined NOUTF8
  static int
- ord2utf8(int cvalue, uschar *utf8bytes)
+ ord2utf8(int cvalue, pcre_uint8 *utf8bytes)
  {
  register int i, j;
  for (i = 0; i < utf8_table1_size; i++)
-Line 523 
 fprintf(outfile, "%.*s", (cb->next_item_
+Line 1034 
 fprintf(outfile, "%.*s", (cb->next_item_
  fprintf(outfile, "\n");
  first_callout = 0;
+ if (cb->mark != last_callout_mark)
+   {
+   fprintf(outfile, "Latest Mark: %s\n",
+     (cb->mark == NULL)? "<unset>" : (char *)(cb->mark));
+   last_callout_mark = cb->mark;
+   }
  if (cb->callout_data != NULL)
    {
    int callout_data = *((int *)(cb->callout_data));
-Line 542 
 return (cb->callout_number != callout_fa
+Line 1060 
 return (cb->callout_number != callout_fa
  *            Local malloc functions              *
  *************************************************/
- /* Alternative malloc function, to test functionality and show the size of the
+ /* Alternative malloc function, to test functionality and save the size of a
- compiled re. */
+ compiled re, which is the first store request that pcre_compile() makes. The
+ show_malloc variable is set only during matching. */
  static void *new_malloc(size_t size)
  {
  void *block = malloc(size);
  gotten_store = size;
+ if (first_gotten_store == 0) first_gotten_store = size;
  if (show_malloc)
    fprintf(outfile, "malloc       %3d %p\n", (int)size, block);
  return block;
-Line 561 
 if (show_malloc)
+Line 1081 
 if (show_malloc)
  free(block);
  }
  /* For recursion malloc/free, to test stacking calls */
  static void *stack_malloc(size_t size)
-Line 589 
 free(block);
+Line 1108 
 free(block);
  static void new_info(pcre *re, pcre_extra *study, int option, void *ptr)
  {
  int rc;
- if ((rc = pcre_fullinfo(re, study, option, ptr)) < 0)
+ if ((rc = (fullinfo)(re, study, option, ptr)) < 0)
    fprintf(outfile, "Error %d from pcre_fullinfo(%d)\n", rc, option);
  }
-Line 617 
 return ((value & 0x000000ff) << 24) |
+Line 1136 
 return ((value & 0x000000ff) << 24) |
  *************************************************/
  static int
- check_match_limit(pcre *re, pcre_extra *extra, uschar *bptr, int len,
+ check_match_limit(pcre *re, pcre_extra *extra, pcre_uint8 *bptr, int len,
    int start_offset, int options, int *use_offsets, int use_size_offsets,
    int flag, unsigned long int *limit, int errnumber, const char *msg)
  {
-Line 677 
 Returns:    < 0, = 0, or > 0, according
+Line 1196 
 Returns:    < 0, = 0, or > 0, according
  */
  static int
- strncmpic(uschar *s, uschar *t, int n)
+ strncmpic(pcre_uint8 *s, pcre_uint8 *t, int n)
  {
  while (n--)
    {
-Line 693 
 return 0;
+Line 1212 
 return 0;
  *         Check newline indicator                *
  *************************************************/
- /* This is used both at compile and run-time to check for <xxx> escapes, where
+ /* This is used both at compile and run-time to check for <xxx> escapes. Print
- xxx is LF, CR, CRLF, ANYCRLF, or ANY. Print a message and return 0 if there is
+ a message and return 0 if there is no match.
- no match.
  Arguments:
    p           points after the leading '<'
-Line 705 
 Returns:      appropriate PCRE_NEWLINE_x
+Line 1223 
 Returns:      appropriate PCRE_NEWLINE_x
  */
  static int
- check_newline(uschar *p, FILE *f)
+ check_newline(pcre_uint8 *p, FILE *f)
  {
- if (strncmpic(p, (uschar *)"cr>", 3) == 0) return PCRE_NEWLINE_CR;
+ if (strncmpic(p, (pcre_uint8 *)"cr>", 3) == 0) return PCRE_NEWLINE_CR;
- if (strncmpic(p, (uschar *)"lf>", 3) == 0) return PCRE_NEWLINE_LF;
+ if (strncmpic(p, (pcre_uint8 *)"lf>", 3) == 0) return PCRE_NEWLINE_LF;
- if (strncmpic(p, (uschar *)"crlf>", 5) == 0) return PCRE_NEWLINE_CRLF;
+ if (strncmpic(p, (pcre_uint8 *)"crlf>", 5) == 0) return PCRE_NEWLINE_CRLF;
- if (strncmpic(p, (uschar *)"anycrlf>", 8) == 0) return PCRE_NEWLINE_ANYCRLF;
+ if (strncmpic(p, (pcre_uint8 *)"anycrlf>", 8) == 0) return PCRE_NEWLINE_ANYCRLF;
- if (strncmpic(p, (uschar *)"any>", 4) == 0) return PCRE_NEWLINE_ANY;
+ if (strncmpic(p, (pcre_uint8 *)"any>", 4) == 0) return PCRE_NEWLINE_ANY;
+ if (strncmpic(p, (pcre_uint8 *)"bsr_anycrlf>", 12) == 0) return PCRE_BSR_ANYCRLF;
+ if (strncmpic(p, (pcre_uint8 *)"bsr_unicode>", 12) == 0) return PCRE_BSR_UNICODE;
  fprintf(f, "Unknown newline type at: <%s\n", p);
  return 0;
  }
-Line 725 
 return 0;
+Line 1245 
 return 0;
  static void
  usage(void)
  {
- printf("Usage:     pcretest [options] [<input> [<output>]]\n");
+ printf("Usage:     pcretest [options] [<input file> [<output file>]]\n\n");
+ printf("Input and output default to stdin and stdout.\n");
+ #ifdef SUPPORT_LIBREADLINE
+ printf("If input is a terminal, readline() is used to read from it.\n");
+ #else
+ printf("This version of pcretest is not linked with readline().\n");
+ #endif
+ printf("\nOptions:\n");
+ #ifdef SUPPORT_PCRE16
+ printf("  -16      use 16-bit interface\n");
+ #endif
  printf("  -b       show compiled code (bytecode)\n");
  printf("  -C       show PCRE compile-time options and exit\n");
  printf("  -d       debug: show compiled code and information (-b and -i)\n");
-Line 734 
 printf("  -dfa     force DFA matching fo
+Line 1264 
 printf("  -dfa     force DFA matching fo
  #endif
  printf("  -help    show usage information\n");
  printf("  -i       show information about compiled patterns\n"
+        "  -M       find MATCH_LIMIT minimum for each subject\n"
         "  -m       output memory used information\n"
         "  -o <n>   set size of offsets vector to <n>\n");
  #if !defined NOPOSIX
-Line 741 
 printf("  -p       use POSIX interface\n
+Line 1272 
 printf("  -p       use POSIX interface\n
  #endif
  printf("  -q       quiet: do not output PCRE version number at start\n");
  printf("  -S <n>   set stack size to <n> megabytes\n");
- printf("  -s       output store (memory) used information\n"
+ printf("  -s       force each pattern to be studied at basic level\n"
+        "  -s+      force each pattern to be studied, using JIT if available\n"
         "  -t       time compilation and execution\n");
  printf("  -t <n>   time compilation and execution, repeating <n> times\n");
  printf("  -tm      time execution (matching) only\n");
-Line 763 
 int main(int argc, char **argv)
+Line 1295 
 int main(int argc, char **argv)
  FILE *infile = stdin;
  int options = 0;
  int study_options = 0;
+ int default_find_match_limit = FALSE;
  int op = 1;
  int timeit = 0;
  int timeitm = 0;
  int showinfo = 0;
  int showstore = 0;
+ int force_study = -1;
+ int force_study_options = 0;
  int quiet = 0;
  int size_offsets = 45;
  int size_offsets_max;
-Line 778 
 int posix = 0;
+Line 1313 
 int posix = 0;
  int debug = 0;
  int done = 0;
  int all_use_dfa = 0;
+ int use_pcre16 = 0;
  int yield = 0;
  int stack_size;
+ pcre_jit_stack *jit_stack = NULL;
  /* These vectors store, end-to-end, a list of captured substring names. Assume
  that 1024 is plenty long enough for the few names we'll be testing. */
- uschar copynames[1024];
+ pcre_uchar copynames[1024];
- uschar getnames[1024];
+ pcre_uchar getnames[1024];
- uschar *copynamesptr;
- uschar *getnamesptr;
- /* Get buffers from malloc() so that Electric Fence will check their misuse
+ pcre_uchar *copynamesptr;
- when I am debugging. They grow automatically when very long lines are read. */
+ pcre_uchar *getnamesptr;
- buffer = (unsigned char *)malloc(buffer_size);
+ /* Get buffers from malloc() so that valgrind will check their misuse when
- dbuffer = (unsigned char *)malloc(buffer_size);
+ debugging. They grow automatically when very long lines are read. The 16-bit
- pbuffer = (unsigned char *)malloc(buffer_size);
+ buffer (buffer16) is obtained only if needed. */
+ buffer = (pcre_uint8 *)malloc(buffer_size);
+ dbuffer = (pcre_uint8 *)malloc(buffer_size);
+ pbuffer = (pcre_uint8 *)malloc(buffer_size);
  /* The outfile variable is static so that new_malloc can use it. */
-Line 816 
 while (argc > 1 && argv[op][0] == '-')
+Line 1355 
 while (argc > 1 && argv[op][0] == '-')
    {
    unsigned char *endptr;
-   if (strcmp(argv[op], "-s") == 0 || strcmp(argv[op], "-m") == 0)
+   if (strcmp(argv[op], "-16") == 0) use_pcre16 = 1;
-     showstore = 1;
+   else if (strcmp(argv[op], "-m") == 0) showstore = 1;
+   else if (strcmp(argv[op], "-s") == 0) force_study = 0;
+   else if (strcmp(argv[op], "-s+") == 0)
+     {
+     force_study = 1;
+     force_study_options = PCRE_STUDY_JIT_COMPILE;
+     }
    else if (strcmp(argv[op], "-q") == 0) quiet = 1;
    else if (strcmp(argv[op], "-b") == 0) debug = 1;
    else if (strcmp(argv[op], "-i") == 0) showinfo = 1;
    else if (strcmp(argv[op], "-d") == 0) showinfo = debug = 1;
+   else if (strcmp(argv[op], "-M") == 0) default_find_match_limit = TRUE;
  #if !defined NODFA
    else if (strcmp(argv[op], "-dfa") == 0) all_use_dfa = 1;
  #endif
-Line 850 
 while (argc > 1 && argv[op][0] == '-')
+Line 1396 
 while (argc > 1 && argv[op][0] == '-')
        ((stack_size = get_value((unsigned char *)argv[op+1], &endptr)),
          *endptr == 0))
      {
- #if defined(_WIN32) || defined(WIN32)
+ #if defined(_WIN32) || defined(WIN32) || defined(__minix)
      printf("PCRE: -S not supported on this OS\n");
      exit(1);
  #else
-Line 874 
 while (argc > 1 && argv[op][0] == '-')
+Line 1420 
 while (argc > 1 && argv[op][0] == '-')
    else if (strcmp(argv[op], "-C") == 0)
      {
      int rc;
+     unsigned long int lrc;
      printf("PCRE version %s\n", pcre_version());
      printf("Compiled with\n");
+ /* At least one of SUPPORT_PCRE8 and SUPPORT_PCRE16 will be set. */
+ #if defined SUPPORT_PCRE8 && defined SUPPORT_PCRE16
+     printf("  8-bit and 16-bit support\n");
      (void)pcre_config(PCRE_CONFIG_UTF8, &rc);
      printf("  %sUTF-8 support\n", rc? "" : "No ");
+     (void)pcre16_config(PCRE_CONFIG_UTF16, &rc);
+     printf("  %sUTF-16 support\n", rc? "" : "No ");
+ #elif defined SUPPORT_PCRE8
+     printf("  8-bit support only\n");
+     (void)pcre_config(PCRE_CONFIG_UTF8, &rc);
+     printf("  %sUTF-8 support\n", rc? "" : "No ");
+ #else
+     printf("  16-bit support only\n");
+     (void)pcre16_config(PCRE_CONFIG_UTF16, &rc);
+     printf("  %sUTF-16 support\n", rc? "" : "No ");
+ #endif
      (void)pcre_config(PCRE_CONFIG_UNICODE_PROPERTIES, &rc);
      printf("  %sUnicode properties support\n", rc? "" : "No ");
+     (void)pcre_config(PCRE_CONFIG_JIT, &rc);
+     if (rc)
+       printf("  Just-in-time compiler support\n");
+     else
+       printf("  No just-in-time compiler support\n");
      (void)pcre_config(PCRE_CONFIG_NEWLINE, &rc);
-     printf("  Newline sequence is %s\n", (rc == '\r')? "CR" :
+     /* Note that these values are always the ASCII values, even
-       (rc == '\n')? "LF" : (rc == ('\r'<<8 | '\n'))? "CRLF" :
+     in EBCDIC environments. CR is 13 and NL is 10. */
+     printf("  Newline sequence is %s\n", (rc == 13)? "CR" :
+       (rc == 10)? "LF" : (rc == (13<<8 | 10))? "CRLF" :
        (rc == -2)? "ANYCRLF" :
        (rc == -1)? "ANY" : "???");
+     (void)pcre_config(PCRE_CONFIG_BSR, &rc);
+     printf("  \\R matches %s\n", rc? "CR, LF, or CRLF only" :
+                                      "all Unicode newlines");
      (void)pcre_config(PCRE_CONFIG_LINK_SIZE, &rc);
      printf("  Internal link size = %d\n", rc);
      (void)pcre_config(PCRE_CONFIG_POSIX_MALLOC_THRESHOLD, &rc);
      printf("  POSIX malloc threshold = %d\n", rc);
-     (void)pcre_config(PCRE_CONFIG_MATCH_LIMIT, &rc);
+     (void)pcre_config(PCRE_CONFIG_MATCH_LIMIT, &lrc);
-     printf("  Default match limit = %d\n", rc);
+     printf("  Default match limit = %ld\n", lrc);
-     (void)pcre_config(PCRE_CONFIG_MATCH_LIMIT_RECURSION, &rc);
+     (void)pcre_config(PCRE_CONFIG_MATCH_LIMIT_RECURSION, &lrc);
-     printf("  Default recursion depth limit = %d\n", rc);
+     printf("  Default recursion depth limit = %ld\n", lrc);
      (void)pcre_config(PCRE_CONFIG_STACKRECURSE, &rc);
      printf("  Match recursion uses %s\n", rc? "stack" : "heap");
      goto EXIT;
-Line 914 
 while (argc > 1 && argv[op][0] == '-')
+Line 1488 
 while (argc > 1 && argv[op][0] == '-')
    argc--;
    }
+ /* Select which fullinfo function to use. */
+ fullinfo = use_pcre16? pcre16_fullinfo : pcre_fullinfo;
  /* Get the store for the offsets vector, and remember what it was */
  size_offsets_max = size_offsets;
-Line 952 
 if (argc > 2)
+Line 1530 
 if (argc > 2)
  /* Set alternative malloc function */
+ #ifdef SUPPORT_PCRE8
  pcre_malloc = new_malloc;
  pcre_free = new_free;
  pcre_stack_malloc = stack_malloc;
  pcre_stack_free = stack_free;
+ #endif
+ #ifdef SUPPORT_PCRE16
+ pcre16_malloc = new_malloc;
+ pcre16_free = new_free;
+ pcre16_stack_malloc = stack_malloc;
+ pcre16_stack_free = stack_free;
+ #endif
  /* Heading line unless quiet, then prompt for first regex if stdin */
-Line 974 
 while (!done)
+Line 1561 
 while (!done)
  #endif
    const char *error;
+   unsigned char *markptr;
    unsigned char *p, *pp, *ppp;
    unsigned char *to_file = NULL;
    const unsigned char *tables = NULL;
    unsigned long int true_size, true_study_size = 0;
    size_t size, regex_gotten_store;
+   int do_allcaps = 0;
+   int do_mark = 0;
    int do_study = 0;
+   int no_force_study = 0;
    int do_debug = debug;
    int do_G = 0;
    int do_g = 0;
    int do_showinfo = showinfo;
    int do_showrest = 0;
+   int do_showcaprest = 0;
    int do_flip = 0;
    int erroroffset, len, delimiter, poffset;
    use_utf8 = 0;
    debug_lengths = 1;
-   if (infile == stdin) printf("  re> ");
+   if (extend_inputline(infile, buffer, "  re> ") == NULL) break;
-   if (extend_inputline(infile, buffer) == NULL) break;
    if (infile != stdin) fprintf(outfile, "%s", (char *)buffer);
    fflush(outfile);
-Line 1005 
 while (!done)
+Line 1596 
 while (!done)
    if (*p == '<' && strchr((char *)(p+1), '<') == NULL)
      {
      unsigned long int magic, get_options;
-     uschar sbuf[8];
+     pcre_uint8 sbuf[8];
      FILE *f;
      p++;
-Line 1028 
 while (!done)
+Line 1619 
 while (!done)
        (sbuf[4] << 24) | (sbuf[5] << 16) | (sbuf[6] << 8) | sbuf[7];
      re = (real_pcre *)new_malloc(true_size);
-     regex_gotten_store = gotten_store;
+     regex_gotten_store = first_gotten_store;
      if (fread(re, 1, true_size, f) != true_size) goto FAIL_READ;
-Line 1047 
 while (!done)
+Line 1638 
 while (!done)
          }
        }
-     fprintf(outfile, "Compiled regex%s loaded from %s\n",
+     fprintf(outfile, "Compiled pattern%s loaded from %s\n",
        do_flip? " (byte-inverted)" : "", p);
      /* Need to know if UTF-8 for printing data strings */
-Line 1055 
 while (!done)
+Line 1646 
 while (!done)
      new_info(re, NULL, PCRE_INFO_OPTIONS, &get_options);
      use_utf8 = (get_options & PCRE_UTF8) != 0;
-     /* Now see if there is any following study data */
+     /* Now see if there is any following study data. */
      if (true_study_size != 0)
        {
-Line 1071 
 while (!done)
+Line 1662 
 while (!done)
          {
          FAIL_READ:
          fprintf(outfile, "Failed to read data from %s\n", p);
-         if (extra != NULL) new_free(extra);
+         if (extra != NULL) pcre_free_study(extra);
          if (re != NULL) new_free(re);
          fclose(f);
          continue;
-Line 1092 
 while (!done)
+Line 1683 
 while (!done)
    if (isalnum(delimiter) || delimiter == '\\')
      {
-     fprintf(outfile, "** Delimiter must not be alphameric or \\\n");
+     fprintf(outfile, "** Delimiter must not be alphanumeric or \\\n");
      goto SKIP_DATA;
      }
    pp = p;
-   poffset = p - buffer;
+   poffset = (int)(p - buffer);
    for(;;)
      {
-Line 1108 
 while (!done)
+Line 1699 
 while (!done)
        pp++;
        }
      if (*pp != 0) break;
-     if (infile == stdin) printf("    > ");
+     if ((pp = extend_inputline(infile, pp, "    > ")) == NULL)
-     if ((pp = extend_inputline(infile, pp)) == NULL)
        {
        fprintf(outfile, "** Unexpected EOF\n");
        done = 1;
-Line 1152 
 while (!done)
+Line 1742 
 while (!done)
        case 's': options |= PCRE_DOTALL; break;
        case 'x': options |= PCRE_EXTENDED; break;
-       case '+': do_showrest = 1; break;
+       case '+':
+       if (do_showrest) do_showcaprest = 1; else do_showrest = 1;
+       break;
+       case '=': do_allcaps = 1; break;
        case 'A': options |= PCRE_ANCHORED; break;
        case 'B': do_debug = 1; break;
        case 'C': options |= PCRE_AUTO_CALLOUT; break;
-Line 1162 
 while (!done)
+Line 1756 
 while (!done)
        case 'G': do_G = 1; break;
        case 'I': do_showinfo = 1; break;
        case 'J': options |= PCRE_DUPNAMES; break;
+       case 'K': do_mark = 1; break;
        case 'M': log_store = 1; break;
        case 'N': options |= PCRE_NO_AUTO_CAPTURE; break;
-Line 1169 
 while (!done)
+Line 1764 
 while (!done)
        case 'P': do_posix = 1; break;
  #endif
-       case 'S': do_study = 1; break;
+       case 'S':
+       if (do_study == 0)
+         {
+         do_study = 1;
+         if (*pp == '+')
+           {
+           study_options |= PCRE_STUDY_JIT_COMPILE;
+           pp++;
+           }
+         }
+       else
+         {
+         do_study = 0;
+         no_force_study = 1;
+         }
+       break;
        case 'U': options |= PCRE_UNGREEDY; break;
+       case 'W': options |= PCRE_UCP; break;
        case 'X': options |= PCRE_EXTRA; break;
+       case 'Y': options |= PCRE_NO_START_OPTIMISE; break;
        case 'Z': debug_lengths = 0; break;
        case '8': options |= PCRE_UTF8; use_utf8 = 1; break;
        case '?': options |= PCRE_NO_UTF8_CHECK; break;
+       case 'T':
+       switch (*pp++)
+         {
+         case '0': tables = tables0; break;
+         case '1': tables = tables1; break;
+         case '\r':
+         case '\n':
+         case ' ':
+         case 0:
+         fprintf(outfile, "** Missing table number after /T\n");
+         goto SKIP_DATA;
+         default:
+         fprintf(outfile, "** Bad table number \"%c\" after /T\n", pp[-1]);
+         goto SKIP_DATA;
+         }
+       break;
        case 'L':
        ppp = pp;
        /* The '\r' test here is so that it works on Windows. */
-Line 1201 
 while (!done)
+Line 1833 
 while (!done)
        case '<':
          {
-         int x = check_newline(pp, outfile);
+         if (strncmpic(pp, (pcre_uint8 *)"JS>", 3) == 0)
-         if (x == 0) goto SKIP_DATA;
+           {
-         options |= x;
+           options |= PCRE_JAVASCRIPT_COMPAT;
-         while (*pp++ != '>');
+           pp += 3;
+           }
+         else
+           {
+           int x = check_newline(pp, outfile);
+           if (x == 0) goto SKIP_DATA;
+           options |= x;
+           while (*pp++ != '>');
+           }
          }
        break;
-Line 1221 
 while (!done)
+Line 1861 
 while (!done)
    /* Handle compiling via the POSIX interface, which doesn't support the
    timing, showing, or debugging options, nor the ability to pass over
-   local character tables. */
+   local character tables. Neither does it have 16-bit support. */
  #if !defined NOPOSIX
    if (posix || do_posix)
-Line 1234 
 while (!done)
+Line 1874 
 while (!done)
      if ((options & PCRE_DOTALL) != 0) cflags |= REG_DOTALL;
      if ((options & PCRE_NO_AUTO_CAPTURE) != 0) cflags |= REG_NOSUB;
      if ((options & PCRE_UTF8) != 0) cflags |= REG_UTF8;
+     if ((options & PCRE_UCP) != 0) cflags |= REG_UCP;
+     if ((options & PCRE_UNGREEDY) != 0) cflags |= REG_UNGREEDY;
+     first_gotten_store = 0;
      rc = regcomp(&preg, (char *)p, cflags);
      /* Compilation failed; go back for another re, skipping to blank line
-Line 1254 
 while (!done)
+Line 1897 
 while (!done)
  #endif  /* !defined NOPOSIX */
      {
+     unsigned long int get_options;
+     /* In 16-bit mode, convert the input. The space needed for a non-UTF string
+     is exactly double the 8-bit size. For a UTF-8 string, the size needed for
+     UTF-16 is no more than double, because up to 0xffff uses no more than 3
+     bytes in UTF-8 but possibly 4 in UTF-16. Higher values use 4 bytes in UTF-8
+     and up to 4 bytes in UTF-16. */
+ #ifdef SUPPORT_PCRE16
+     if (use_pcre16) (void)to16(p, options & PCRE_UTF8);
+ #endif
+     /* Compile many times when timing */
      if (timeit > 0)
        {
        register int i;
-Line 1261 
 while (!done)
+Line 1918 
 while (!done)
        clock_t start_time = clock();
        for (i = 0; i < timeit; i++)
          {
-         re = pcre_compile((char *)p, options, &error, &erroroffset, tables);
+ #ifdef SUPPORT_PCRE16
+         if (use_pcre16)
+           re = pcre16_compile((PCRE_SPTR16)buffer16, options, &error, &erroroffset, tables);
+         else
+ #endif
+           re = pcre_compile((char *)p, options, &error, &erroroffset, tables);
          if (re != NULL) free(re);
          }
        time_taken = clock() - start_time;
-Line 1270 
 while (!done)
+Line 1932 
 while (!done)
            (double)CLOCKS_PER_SEC);
        }
-     re = pcre_compile((char *)p, options, &error, &erroroffset, tables);
+     first_gotten_store = 0;
+ #ifdef SUPPORT_PCRE16
+     if (use_pcre16)
+       re = pcre16_compile((PCRE_SPTR16)buffer16, options, &error, &erroroffset, tables);
+     else
+ #endif
+       re = pcre_compile((char *)p, options, &error, &erroroffset, tables);
      /* Compilation failed; go back for another re, skipping to blank line
      if non-interactive. */
-Line 1283 
 while (!done)
+Line 1952 
 while (!done)
          {
          for (;;)
            {
-           if (extend_inputline(infile, buffer) == NULL)
+           if (extend_inputline(infile, buffer, NULL) == NULL)
              {
              done = 1;
              goto CONTINUE;
-Line 1297 
 while (!done)
+Line 1966 
 while (!done)
        goto CONTINUE;
        }
-     /* Compilation succeeded; print data if required. There are now two
+     /* Compilation succeeded. It is now possible to set the UTF-8 option from
-     info-returning functions. The old one has a limited interface and
+     within the regex; check for this so that we know how to process the data
-     returns only limited data. Check that it agrees with the newer one. */
+     lines. */
-     if (log_store)
+     new_info(re, NULL, PCRE_INFO_OPTIONS, &get_options);
-       fprintf(outfile, "Memory allocation (code space): %d\n",
+     if ((get_options & PCRE_UTF8) != 0) use_utf8 = 1;
-         (int)(gotten_store -
-               sizeof(real_pcre) -
-               ((real_pcre *)re)->name_count * ((real_pcre *)re)->name_entry_size));
      /* Extract the size for possible writing before possibly flipping it,
      and remember the store that was got. */
      true_size = ((real_pcre *)re)->size;
-     regex_gotten_store = gotten_store;
+     regex_gotten_store = first_gotten_store;
-     /* If /S was present, study the regexp to generate additional info to
+     /* Output code size information if requested */
-     help with the matching. */
-     if (do_study)
+     if (log_store)
+       fprintf(outfile, "Memory allocation (code space): %d\n",
+         (int)(first_gotten_store -
+               sizeof(real_pcre) -
+               ((real_pcre *)re)->name_count * ((real_pcre *)re)->name_entry_size));
+     /* If -s or /S was present, study the regex to generate additional info to
+     help with the matching, unless the pattern has the SS option, which
+     suppresses the effect of /S (used for a few test patterns where studying is
+     never sensible). */
+     if (do_study || (force_study >= 0 && !no_force_study))
        {
        if (timeit > 0)
          {
-Line 1324 
 while (!done)
+Line 2000 
 while (!done)
          clock_t time_taken;
          clock_t start_time = clock();
          for (i = 0; i < timeit; i++)
-           extra = pcre_study(re, study_options, &error);
+           {
+           if (use_pcre16)
+             extra = pcre16_study(re, study_options | force_study_options, &error);
+           else
+             extra = pcre_study(re, study_options | force_study_options, &error);
+           }
          time_taken = clock() - start_time;
-         if (extra != NULL) free(extra);
+         if (extra != NULL) pcre_free_study(extra);
          fprintf(outfile, "  Study time %.4f milliseconds\n",
            (((double)time_taken * 1000.0) / (double)timeit) /
              (double)CLOCKS_PER_SEC);
          }
-       extra = pcre_study(re, study_options, &error);
+       if (use_pcre16)
+         extra = pcre16_study(re, study_options | force_study_options, &error);
+       else
+         extra = pcre_study(re, study_options | force_study_options, &error);
        if (error != NULL)
          fprintf(outfile, "Failed to study: %s\n", error);
        else if (extra != NULL)
+         {
          true_study_size = ((pcre_study_data *)(extra->study_data))->size;
+         if (log_store)
+           {
+           size_t jitsize;
+           new_info(re, extra, PCRE_INFO_JITSIZE, &jitsize);
+           if (jitsize != 0)
+             fprintf(outfile, "Memory allocation (JIT code): %d\n", jitsize);
+           }
+         }
+       }
+     /* If /K was present, we set up for handling MARK data. */
+     if (do_mark)
+       {
+       if (extra == NULL)
+         {
+         extra = (pcre_extra *)malloc(sizeof(pcre_extra));
+         extra->flags = 0;
+         }
+       extra->mark = &markptr;
+       extra->flags |= PCRE_EXTRA_MARK;
        }
      /* If the 'F' option was present, we flip the bytes of all the integer
-Line 1346 
 while (!done)
+Line 2052 
 while (!done)
      if (do_flip)
        {
        real_pcre *rre = (real_pcre *)re;
-       rre->magic_number = byteflip(rre->magic_number, sizeof(rre->magic_number));
+       rre->magic_number =
+         byteflip(rre->magic_number, sizeof(rre->magic_number));
        rre->size = byteflip(rre->size, sizeof(rre->size));
        rre->options = byteflip(rre->options, sizeof(rre->options));
-       rre->top_bracket = byteflip(rre->top_bracket, sizeof(rre->top_bracket));
+       rre->flags = (pcre_uint16)byteflip(rre->flags, sizeof(rre->flags));
-       rre->top_backref = byteflip(rre->top_backref, sizeof(rre->top_backref));
+       rre->top_bracket =
-       rre->first_byte = byteflip(rre->first_byte, sizeof(rre->first_byte));
+         (pcre_uint16)byteflip(rre->top_bracket, sizeof(rre->top_bracket));
-       rre->req_byte = byteflip(rre->req_byte, sizeof(rre->req_byte));
+       rre->top_backref =
-       rre->name_table_offset = byteflip(rre->name_table_offset,
+         (pcre_uint16)byteflip(rre->top_backref, sizeof(rre->top_backref));
+       rre->first_char =
+         (pcre_uint16)byteflip(rre->first_char, sizeof(rre->first_char));
+       rre->req_char =
+         (pcre_uint16)byteflip(rre->req_char, sizeof(rre->req_char));
+       rre->name_table_offset = (pcre_uint16)byteflip(rre->name_table_offset,
          sizeof(rre->name_table_offset));
-       rre->name_entry_size = byteflip(rre->name_entry_size,
+       rre->name_entry_size = (pcre_uint16)byteflip(rre->name_entry_size,
          sizeof(rre->name_entry_size));
-       rre->name_count = byteflip(rre->name_count, sizeof(rre->name_count));
+       rre->name_count = (pcre_uint16)byteflip(rre->name_count,
+         sizeof(rre->name_count));
        if (extra != NULL)
          {
          pcre_study_data *rsd = (pcre_study_data *)(extra->study_data);
          rsd->size = byteflip(rsd->size, sizeof(rsd->size));
-         rsd->options = byteflip(rsd->options, sizeof(rsd->options));
+         rsd->flags = byteflip(rsd->flags, sizeof(rsd->flags));
+         rsd->minlength = byteflip(rsd->minlength, sizeof(rsd->minlength));
          }
        }
-     /* Extract information from the compiled data if required */
+     /* Extract and display information from the compiled data if required. */
      SHOW_INFO:
      if (do_debug)
        {
        fprintf(outfile, "------------------------------------------------------------------\n");
-       pcre_printint(re, outfile, debug_lengths);
+       if (use_pcre16)
+         pcre16_printint(re, outfile, debug_lengths);
+       else
+         pcre_printint(re, outfile, debug_lengths);
        }
+     /* We already have the options in get_options (see above) */
      if (do_showinfo)
        {
-       unsigned long int get_options, all_options;
+       unsigned long int all_options;
  #if !defined NOINFOCHECK
        int old_first_char, old_options, old_count;
  #endif
        int count, backrefmax, first_char, need_char, okpartial, jchanged,
          hascrorlf;
        int nameentrysize, namecount;
-       const uschar *nametable;
+       const pcre_uchar *nametable;
-       new_info(re, NULL, PCRE_INFO_OPTIONS, &get_options);
        new_info(re, NULL, PCRE_INFO_SIZE, &size);
        new_info(re, NULL, PCRE_INFO_CAPTURECOUNT, &count);
        new_info(re, NULL, PCRE_INFO_BACKREFMAX, &backrefmax);
-Line 1401 
 while (!done)
+Line 2119 
 while (!done)
        new_info(re, NULL, PCRE_INFO_JCHANGED, &jchanged);
        new_info(re, NULL, PCRE_INFO_HASCRORLF, &hascrorlf);
+       /* The old, obsolete function pcre_info() works only in 8-bit mode. Check
+       that it gives the same results as the new function. */
  #if !defined NOINFOCHECK
-       old_count = pcre_info(re, &old_options, &old_first_char);
+       if (!use_pcre16)
-       if (count < 0) fprintf(outfile,
+         {
-         "Error %d from pcre_info()\n", count);
+         old_count = pcre_info(re, &old_options, &old_first_char);
-       else
+         if (count < 0) fprintf(outfile,
-         {
+           "Error %d from pcre_info()\n", count);
-         if (old_count != count) fprintf(outfile,
+         else
-           "Count disagreement: pcre_fullinfo=%d pcre_info=%d\n", count,
+           {
-             old_count);
+           if (old_count != count) fprintf(outfile,
+             "Count disagreement: pcre_fullinfo=%d pcre_info=%d\n", count,
-         if (old_first_char != first_char) fprintf(outfile,
+               old_count);
-           "First char disagreement: pcre_fullinfo=%d pcre_info=%d\n",
-             first_char, old_first_char);
+           if (old_first_char != first_char) fprintf(outfile,
+             "First char disagreement: pcre_fullinfo=%d pcre_info=%d\n",
-         if (old_options != (int)get_options) fprintf(outfile,
+               first_char, old_first_char);
-           "Options disagreement: pcre_fullinfo=%ld pcre_info=%d\n",
-             get_options, old_options);
+           if (old_options != (int)get_options) fprintf(outfile,
-         }
+             "Options disagreement: pcre_fullinfo=%ld pcre_info=%d\n",
+               get_options, old_options);
+           }
+         }
  #endif
        if (size != regex_gotten_store) fprintf(outfile,
-Line 1448 
 while (!done)
+Line 2172 
 while (!done)
        if (do_flip) all_options = byteflip(all_options, sizeof(all_options));
        if (get_options == 0) fprintf(outfile, "No options\n");
-         else fprintf(outfile, "Options:%s%s%s%s%s%s%s%s%s%s%s%s%s\n",
+         else fprintf(outfile, "Options:%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s\n",
            ((get_options & PCRE_ANCHORED) != 0)? " anchored" : "",
            ((get_options & PCRE_CASELESS) != 0)? " caseless" : "",
            ((get_options & PCRE_EXTENDED) != 0)? " extended" : "",
            ((get_options & PCRE_MULTILINE) != 0)? " multiline" : "",
            ((get_options & PCRE_FIRSTLINE) != 0)? " firstline" : "",
            ((get_options & PCRE_DOTALL) != 0)? " dotall" : "",
+           ((get_options & PCRE_BSR_ANYCRLF) != 0)? " bsr_anycrlf" : "",
+           ((get_options & PCRE_BSR_UNICODE) != 0)? " bsr_unicode" : "",
            ((get_options & PCRE_DOLLAR_ENDONLY) != 0)? " dollar_endonly" : "",
            ((get_options & PCRE_EXTRA) != 0)? " extra" : "",
            ((get_options & PCRE_UNGREEDY) != 0)? " ungreedy" : "",
            ((get_options & PCRE_NO_AUTO_CAPTURE) != 0)? " no_auto_capture" : "",
            ((get_options & PCRE_UTF8) != 0)? " utf8" : "",
+           ((get_options & PCRE_UCP) != 0)? " ucp" : "",
            ((get_options & PCRE_NO_UTF8_CHECK) != 0)? " no_utf8_check" : "",
+           ((get_options & PCRE_NO_START_OPTIMIZE) != 0)? " no_start_optimize" : "",
            ((get_options & PCRE_DUPNAMES) != 0)? " dupnames" : "");
        if (jchanged) fprintf(outfile, "Duplicate name status changes\n");
-Line 1501 
 while (!done)
+Line 2229 
 while (!done)
          }
        else
          {
-         int ch = first_char & 255;
+         const char *caseless =
-         const char *caseless = ((first_char & REQ_CASELESS) == 0)?
+           ((((real_pcre *)re)->flags & PCRE_FCH_CASELESS) == 0)?
            "" : " (caseless)";
-         if (PRINTHEX(ch))
-           fprintf(outfile, "First char = \'%c\'%s\n", ch, caseless);
+         if (PRINTHEX(first_char))
+           fprintf(outfile, "First char = \'%c\'%s\n", first_char, caseless);
          else
-           fprintf(outfile, "First char = %d%s\n", ch, caseless);
+           fprintf(outfile, "First char = %d%s\n", first_char, caseless);
          }
        if (need_char < 0)
-Line 1516 
 while (!done)
+Line 2245 
 while (!done)
          }
        else
          {
-         int ch = need_char & 255;
+         const char *caseless =
-         const char *caseless = ((need_char & REQ_CASELESS) == 0)?
+           ((((real_pcre *)re)->flags & PCRE_RCH_CASELESS) == 0)?
            "" : " (caseless)";
-         if (PRINTHEX(ch))
-           fprintf(outfile, "Need char = \'%c\'%s\n", ch, caseless);
+         if (PRINTHEX(need_char))
+           fprintf(outfile, "Need char = \'%c\'%s\n", need_char, caseless);
          else
-           fprintf(outfile, "Need char = %d%s\n", ch, caseless);
+           fprintf(outfile, "Need char = %d%s\n", need_char, caseless);
          }
        /* Don't output study size; at present it is in any case a fixed
        value, but it varies, depending on the computer architecture, and
        so messes up the test suite. (And with the /F option, it might be
-       flipped.) */
+       flipped.) If study was forced by an external -s, don't show this
+       information unless -i or -d was also present. This means that, except
+       when auto-callouts are involved, the output from runs with and without
+       -s should be identical. */
-       if (do_study)
+       if (do_study || (force_study >= 0 && showinfo && !no_force_study))
          {
          if (extra == NULL)
            fprintf(outfile, "Study returned NULL\n");
          else
            {
-           uschar *start_bits = NULL;
+           pcre_uint8 *start_bits = NULL;
-           new_info(re, extra, PCRE_INFO_FIRSTTABLE, &start_bits);
+           int minlength;
+           new_info(re, extra, PCRE_INFO_MINLENGTH, &minlength);
+           fprintf(outfile, "Subject length lower bound = %d\n", minlength);
+           new_info(re, extra, PCRE_INFO_FIRSTTABLE, &start_bits);
            if (start_bits == NULL)
-             fprintf(outfile, "No starting byte set\n");
+             fprintf(outfile, "No set of starting bytes\n");
            else
              {
              int i;
-Line 1570 
 while (!done)
+Line 2307 
 while (!done)
              fprintf(outfile, "\n");
              }
            }
+         /* Show this only if the JIT was set by /S, not by -s. */
+         if ((study_options & PCRE_STUDY_JIT_COMPILE) != 0)
+           {
+           int jit;
+           new_info(re, extra, PCRE_INFO_JIT, &jit);
+           if (jit)
+             fprintf(outfile, "JIT study was successful\n");
+           else
+ #ifdef SUPPORT_JIT
+             fprintf(outfile, "JIT study was not successful\n");
+ #else
+             fprintf(outfile, "JIT support is not available in this version of PCRE\n");
+ #endif
+           }
          }
        }
-Line 1586 
 while (!done)
+Line 2339 
 while (!done)
          }
        else
          {
-         uschar sbuf[8];
+         pcre_uint8 sbuf[8];
-         sbuf[0] = (true_size >> 24)  & 255;
+         sbuf[0] = (pcre_uint8)((true_size >> 24) & 255);
-         sbuf[1] = (true_size >> 16)  & 255;
+         sbuf[1] = (pcre_uint8)((true_size >> 16) & 255);
-         sbuf[2] = (true_size >>  8)  & 255;
+         sbuf[2] = (pcre_uint8)((true_size >>  8) & 255);
-         sbuf[3] = (true_size)  & 255;
+         sbuf[3] = (pcre_uint8)((true_size) & 255);
-         sbuf[4] = (true_study_size >> 24)  & 255;
+         sbuf[4] = (pcre_uint8)((true_study_size >> 24) & 255);
-         sbuf[5] = (true_study_size >> 16)  & 255;
+         sbuf[5] = (pcre_uint8)((true_study_size >> 16) & 255);
-         sbuf[6] = (true_study_size >>  8)  & 255;
+         sbuf[6] = (pcre_uint8)((true_study_size >>  8) & 255);
-         sbuf[7] = (true_study_size)  & 255;
+         sbuf[7] = (pcre_uint8)((true_study_size) & 255);
          if (fwrite(sbuf, 1, 8, f) < 8 ||
              fwrite(re, 1, true_size, f) < true_size)
-Line 1604 
 while (!done)
+Line 2357 
 while (!done)
            }
          else
            {
-           fprintf(outfile, "Compiled regex written to %s\n", to_file);
+           fprintf(outfile, "Compiled pattern written to %s\n", to_file);
+           /* If there is study data, write it. */
            if (extra != NULL)
              {
              if (fwrite(extra->study_data, 1, true_study_size, f) <
-Line 1614 
 while (!done)
+Line 2370 
 while (!done)
                  strerror(errno));
                }
              else fprintf(outfile, "Study data written to %s\n", to_file);
              }
            }
          fclose(f);
          }
        new_free(re);
-       if (extra != NULL) new_free(extra);
+       if (extra != NULL) pcre_free_study(extra);
-       if (tables != NULL) new_free((void *)tables);
+       if (locale_set)
+         {
+         new_free((void *)tables);
+         setlocale(LC_CTYPE, "C");
+         locale_set = 0;
+         }
        continue;  /* With next regex */
        }
      }        /* End of non-POSIX compile */
-Line 1631 
 while (!done)
+Line 2391 
 while (!done)
    for (;;)
      {
-     uschar *q;
+     pcre_uint8 *q;
-     uschar *bptr;
+     pcre_uint8 *bptr;
      int *use_offsets = offsets;
      int use_size_offsets = size_offsets;
      int callout_data = 0;
      int callout_data_set = 0;
      int count, c;
      int copystrings = 0;
-     int find_match_limit = 0;
+     int find_match_limit = default_find_match_limit;
      int getstrings = 0;
      int getlist = 0;
      int gmatched = 0;
      int start_offset = 0;
+     int start_offset_sign = 1;
      int g_notempty = 0;
      int use_dfa = 0;
-Line 1657 
 while (!done)
+Line 2418 
 while (!done)
      pcre_callout = callout;
      first_callout = 1;
+     last_callout_mark = NULL;
      callout_extra = 0;
      callout_count = 0;
      callout_fail_count = 999999;
-Line 1669 
 while (!done)
+Line 2431 
 while (!done)
      len = 0;
      for (;;)
        {
-       if (infile == stdin) printf("data> ");
+       if (extend_inputline(infile, buffer + len, "data> ") == NULL)
-       if (extend_inputline(infile, buffer + len) == NULL)
          {
-         if (len > 0) break;
+         if (len > 0)    /* Reached EOF without hitting a newline */
+           {
+           fprintf(outfile, "\n");
+           break;
+           }
          done = 1;
          goto CONTINUE;
          }
-Line 1732 
 while (!done)
+Line 2497 
 while (!done)
            {
            unsigned char *pt = p;
            c = 0;
-           while (isxdigit(*(++pt)))
-             c = c * 16 + tolower(*pt) - ((isdigit(*pt))? '0' : 'W');
+           /* We used to have "while (isxdigit(*(++pt)))" here, but it fails
+           when isxdigit() is a macro that refers to its argument more than
+           once. This is banned by the C Standard, but apparently happens in at
+           least one MacOS environment. */
+           for (pt++; isxdigit(*pt); pt++)
+             c = c * 16 + tolower(*pt) - ((isdigit(*pt))? '0' : 'a' - 10);
            if (*pt == '}')
              {
              unsigned char buff8[8];
              int ii, utn;
-             utn = ord2utf8(c, buff8);
+             if (use_utf8)
-             for (ii = 0; ii < utn - 1; ii++) *q++ = buff8[ii];
+               {
-             c = buff8[ii];   /* Last byte */
+               utn = ord2utf8(c, buff8);
+               for (ii = 0; ii < utn - 1; ii++) *q++ = buff8[ii];
+               c = buff8[ii];   /* Last byte */
+               }
+             else
+              {
+              if (c > 255)
+                fprintf(outfile, "** Character \\x{%x} is greater than 255 and "
+                  "UTF-8 mode is not enabled.\n"
+                  "** Truncation will probably give the wrong result.\n", c);
+              }
              p = pt + 1;
              break;
              }
-Line 1753 
 while (!done)
+Line 2534 
 while (!done)
          c = 0;
          while (i++ < 2 && isxdigit(*p))
            {
-           c = c * 16 + tolower(*p) - ((isdigit(*p))? '0' : 'W');
+           c = c * 16 + tolower(*p) - ((isdigit(*p))? '0' : 'a' - 10);
            p++;
            }
          break;
-Line 1763 
 while (!done)
+Line 2544 
 while (!done)
          continue;
          case '>':
+         if (*p == '-')
+           {
+           start_offset_sign = -1;
+           p++;
+           }
          while(isdigit(*p)) start_offset = start_offset * 10 + *p++ - '0';
+         start_offset *= start_offset_sign;
          continue;
          case 'A':  /* Option setting */
-Line 1782 
 while (!done)
+Line 2569 
 while (!done)
            }
          else if (isalnum(*p))
            {
-           uschar *npp = copynamesptr;
+           pcre_uchar *npp = copynamesptr;
            while (isalnum(*p)) *npp++ = *p++;
            *npp++ = 0;
            *npp = 0;
-Line 1836 
 while (!done)
+Line 2623 
 while (!done)
  #endif
            use_dfa = 1;
          continue;
+ #endif
+ #if !defined NODFA
          case 'F':
          options |= PCRE_DFA_SHORTEST;
          continue;
-Line 1850 
 while (!done)
+Line 2639 
 while (!done)
            }
          else if (isalnum(*p))
            {
-           uschar *npp = getnamesptr;
+           pcre_uchar *npp = getnamesptr;
            while (isalnum(*p)) *npp++ = *p++;
            *npp++ = 0;
            *npp = 0;
-Line 1861 
 while (!done)
+Line 2650 
 while (!done)
            }
          continue;
+         case 'J':
+         while(isdigit(*p)) n = n * 10 + *p++ - '0';
+         if (extra != NULL
+             && (extra->flags & PCRE_EXTRA_EXECUTABLE_JIT) != 0
+             && extra->executable_jit != NULL)
+           {
+           if (jit_stack != NULL) pcre_jit_stack_free(jit_stack);
+           jit_stack = pcre_jit_stack_alloc(1, n * 1024);
+           pcre_assign_jit_stack(extra, jit_callback, jit_stack);
+           }
+         continue;
          case 'L':
          getlist = 1;
          continue;
-Line 1870 
 while (!done)
+Line 2671 
 while (!done)
          continue;
          case 'N':
-         options |= PCRE_NOTEMPTY;
+         if ((options & PCRE_NOTEMPTY) != 0)
+           options = (options & ~PCRE_NOTEMPTY) | PCRE_NOTEMPTY_ATSTART;
+         else
+           options |= PCRE_NOTEMPTY;
          continue;
          case 'O':
-Line 1893 
 while (!done)
+Line 2697 
 while (!done)
          continue;
          case 'P':
-         options |= PCRE_PARTIAL;
+         options |= ((options & PCRE_PARTIAL_SOFT) == 0)?
+           PCRE_PARTIAL_SOFT : PCRE_PARTIAL_HARD;
          continue;
          case 'Q':
-Line 1928 
 while (!done)
+Line 2733 
 while (!done)
          show_malloc = 1;
          continue;
+         case 'Y':
+         options |= PCRE_NO_START_OPTIMIZE;
+         continue;
          case 'Z':
          options |= PCRE_NOTEOL;
          continue;
-Line 1948 
 while (!done)
+Line 2757 
 while (!done)
        *q++ = c;
        }
      *q = 0;
-     len = q - dbuffer;
+     len = (int)(q - dbuffer);
+     /* Move the data to the end of the buffer so that a read over the end of
+     the buffer will be seen by valgrind, even if it doesn't cause a crash. If
+     we are using the POSIX interface, we must include the terminating zero. */
+ #if !defined NOPOSIX
+     if (posix || do_posix)
+       {
+       memmove(bptr + buffer_size - len - 1, bptr, len + 1);
+       bptr += buffer_size - len - 1;
+       }
+     else
+ #endif
+       {
+       memmove(bptr + buffer_size - len, bptr, len);
+       bptr += buffer_size - len;
+       }
      if ((all_use_dfa || use_dfa) && find_match_limit)
        {
-Line 1969 
 while (!done)
+Line 2795 
 while (!done)
          pmatch = (regmatch_t *)malloc(sizeof(regmatch_t) * use_size_offsets);
        if ((options & PCRE_NOTBOL) != 0) eflags |= REG_NOTBOL;
        if ((options & PCRE_NOTEOL) != 0) eflags |= REG_NOTEOL;
+       if ((options & PCRE_NOTEMPTY) != 0) eflags |= REG_NOTEMPTY;
        rc = regexec(&preg, (const char *)bptr, use_size_offsets, pmatch, eflags);
-Line 1993 
 while (!done)
+Line 2820 
 while (!done)
              (void)pchars(dbuffer + pmatch[i].rm_so,
                pmatch[i].rm_eo - pmatch[i].rm_so, outfile);
              fprintf(outfile, "\n");
-             if (i == 0 && do_showrest)
+             if (do_showcaprest || (i == 0 && do_showrest))
                {
-               fprintf(outfile, " 0+ ");
+               fprintf(outfile, "%2d+ ", (int)i);
                (void)pchars(dbuffer + pmatch[i].rm_eo, len - pmatch[i].rm_eo,
                  outfile);
                fprintf(outfile, "\n");
-Line 2013 
 while (!done)
+Line 2840 
 while (!done)
      for (;; gmatched++)    /* Loop for /g or /G */
        {
+       markptr = NULL;
        if (timeitm > 0)
          {
          register int i;
          clock_t time_taken;
          clock_t start_time = clock();
+ #ifdef SUPPORT_PCRE16
+         if (use_pcre16) len = to16(bptr, options & PCRE_UTF8);
+ #endif
  #if !defined NODFA
          if (all_use_dfa || use_dfa)
            {
            int workspace[1000];
            for (i = 0; i < timeitm; i++)
-             count = pcre_dfa_exec(re, NULL, (char *)bptr, len, start_offset,
+             count = pcre_dfa_exec(re, extra, (char *)bptr, len, start_offset,
                options | g_notempty, use_offsets, use_size_offsets, workspace,
                sizeof(workspace)/sizeof(int));
            }
-Line 2043 
 while (!done)
+Line 2877 
 while (!done)
        /* If find_match_limit is set, we want to do repeated matches with
        varying limits in order to find the minimum value for the match limit and
-       for the recursion limit. */
+       for the recursion limit. The match limits are relevant only to the normal
+       running of pcre_exec(), so disable the JIT optimization. This makes it
+       possible to run the same set of tests with and without JIT externally
+       requested. */
        if (find_match_limit)
          {
-Line 2052 
 while (!done)
+Line 2889 
 while (!done)
            extra = (pcre_extra *)malloc(sizeof(pcre_extra));
            extra->flags = 0;
            }
+         else extra->flags &= ~PCRE_EXTRA_EXECUTABLE_JIT;
          (void)check_match_limit(re, extra, bptr, len, start_offset,
            options|g_notempty, use_offsets, use_size_offsets,
-Line 2087 
 while (!done)
+Line 2925 
 while (!done)
        else if (all_use_dfa || use_dfa)
          {
          int workspace[1000];
-         count = pcre_dfa_exec(re, NULL, (char *)bptr, len, start_offset,
+         count = pcre_dfa_exec(re, extra, (char *)bptr, len, start_offset,
            options | g_notempty, use_offsets, use_size_offsets, workspace,
            sizeof(workspace)/sizeof(int));
          if (count == 0)
-Line 2100 
 while (!done)
+Line 2938 
 while (!done)
        else
          {
-         count = pcre_exec(re, extra, (char *)bptr, len,
+         if (use_pcre16)
-           start_offset, options | g_notempty, use_offsets, use_size_offsets);
+           count = pcre16_exec(re, extra, (PCRE_SPTR16)buffer16, len,
+             start_offset, options | g_notempty, use_offsets, use_size_offsets);
+         else
+           count = pcre_exec(re, extra, (char *)bptr, len,
+             start_offset, options | g_notempty, use_offsets, use_size_offsets);
          if (count == 0)
            {
            fprintf(outfile, "Matched, but too many substrings\n");
-Line 2135 
 while (!done)
+Line 2977 
 while (!done)
              }
            }
+         /* do_allcaps requests showing of all captures in the pattern, to check
+         unset ones at the end. */
+         if (do_allcaps)
+           {
+           new_info(re, NULL, PCRE_INFO_CAPTURECOUNT, &count);
+           count++;   /* Allow for full match */
+           if (count * 2 > use_size_offsets) count = use_size_offsets/2;
+           }
+         /* Output the captured substrings */
          for (i = 0; i < count * 2; i += 2)
            {
            if (use_offsets[i] < 0)
+             {
+             if (use_offsets[i] != -1)
+               fprintf(outfile, "ERROR: bad negative value %d for offset %d\n",
+                 use_offsets[i], i);
+             if (use_offsets[i+1] != -1)
+               fprintf(outfile, "ERROR: bad negative value %d for offset %d\n",
+                 use_offsets[i+1], i+1);
              fprintf(outfile, "%2d: <unset>\n", i/2);
+             }
            else
              {
              fprintf(outfile, "%2d: ", i/2);
              (void)pchars(bptr + use_offsets[i],
                use_offsets[i+1] - use_offsets[i], outfile);
              fprintf(outfile, "\n");
-             if (i == 0)
+             if (do_showcaprest || (i == 0 && do_showrest))
                {
-               if (do_showrest)
+               fprintf(outfile, "%2d+ ", i/2);
-                 {
+               (void)pchars(bptr + use_offsets[i+1], len - use_offsets[i+1],
-                 fprintf(outfile, " 0+ ");
+                 outfile);
-                 (void)pchars(bptr + use_offsets[i+1], len - use_offsets[i+1],
+               fprintf(outfile, "\n");
-                   outfile);
-                 fprintf(outfile, "\n");
-                 }
                }
              }
            }
+         if (markptr != NULL) fprintf(outfile, "MK: %s\n", markptr);
          for (i = 0; i < 32; i++)
            {
            if ((copystrings & (1 << i)) != 0)
-Line 2231 
 while (!done)
+Line 3092 
 while (!done)
                fprintf(outfile, "%2dL %s\n", i, stringlist[i]);
              if (stringlist[i] != NULL)
                fprintf(outfile, "string list not terminated by NULL\n");
-             /* free((void *)stringlist); */
              pcre_free_substring_list(stringlist);
              }
            }
-Line 2241 
 while (!done)
+Line 3101 
 while (!done)
        else if (count == PCRE_ERROR_PARTIAL)
          {
-         fprintf(outfile, "Partial match");
+         if (markptr == NULL) fprintf(outfile, "Partial match");
- #if !defined NODFA
+           else fprintf(outfile, "Partial match, mark=%s", markptr);
-         if ((all_use_dfa || use_dfa) && use_size_offsets > 2)
+         if (use_size_offsets > 1)
-           fprintf(outfile, ": %.*s", use_offsets[1] - use_offsets[0],
+           {
-             bptr + use_offsets[0]);
+           fprintf(outfile, ": ");
- #endif
+           pchars(bptr + use_offsets[0], use_offsets[1] - use_offsets[0],
+             outfile);
+           }
          fprintf(outfile, "\n");
          break;  /* Out of the /g loop */
          }
-Line 2256 
 while (!done)
+Line 3118 
 while (!done)
        to advance the start offset, and continue. We won't be at the end of the
        string - that was checked before setting g_notempty.
-       Complication arises in the case when the newline option is "any" or
+       Complication arises in the case when the newline convention is "any",
-       "anycrlf". If the previous match was at the end of a line terminated by
+       "crlf", or "anycrlf". If the previous match was at the end of a line
-       CRLF, an advance of one character just passes the \r, whereas we should
+       terminated by CRLF, an advance of one character just passes the \r,
-       prefer the longer newline sequence, as does the code in pcre_exec().
+       whereas we should prefer the longer newline sequence, as does the code in
-       Fudge the offset value to achieve this.
+       pcre_exec(). Fudge the offset value to achieve this. We check for a
+       newline setting in the pattern; if none was set, use pcre_config() to
+       find the default.
        Otherwise, in the case of UTF-8 matching, the advance must be one
        character, not one byte. */
-Line 2276 
 while (!done)
+Line 3140 
 while (!done)
              {
              int d;
              (void)pcre_config(PCRE_CONFIG_NEWLINE, &d);
-             obits = (d == '\r')? PCRE_NEWLINE_CR :
+             /* Note that these values are always the ASCII ones, even in
-                     (d == '\n')? PCRE_NEWLINE_LF :
+             EBCDIC environments. CR = 13, NL = 10. */
-                     (d == ('\r'<<8 | '\n'))? PCRE_NEWLINE_CRLF :
+             obits = (d == 13)? PCRE_NEWLINE_CR :
+                     (d == 10)? PCRE_NEWLINE_LF :
+                     (d == (13<<8 | 10))? PCRE_NEWLINE_CRLF :
                      (d == -2)? PCRE_NEWLINE_ANYCRLF :
                      (d == -1)? PCRE_NEWLINE_ANY : 0;
              }
            if (((obits & PCRE_NEWLINE_BITS) == PCRE_NEWLINE_ANY ||
+                (obits & PCRE_NEWLINE_BITS) == PCRE_NEWLINE_CRLF ||
                 (obits & PCRE_NEWLINE_BITS) == PCRE_NEWLINE_ANYCRLF)
                &&
                start_offset < len - 1 &&
-Line 2293 
 while (!done)
+Line 3160 
 while (!done)
              {
              while (start_offset + onechar < len)
                {
-               int tb = bptr[start_offset+onechar];
+               if ((bptr[start_offset+onechar] & 0xc0) != 0x80) break;
-               if (tb <= 127) break;
+               onechar++;
-               tb &= 0xc0;
-               if (tb != 0 && tb != 0xc0) onechar++;
                }
              }
            use_offsets[1] = start_offset + onechar;
            }
          else
            {
-           if (count == PCRE_ERROR_NOMATCH)
+           switch(count)
              {
-             if (gmatched == 0) fprintf(outfile, "No match\n");
+             case PCRE_ERROR_NOMATCH:
+             if (gmatched == 0)
+               {
+               if (markptr == NULL) fprintf(outfile, "No match\n");
+                 else fprintf(outfile, "No match, mark = %s\n", markptr);
+               }
+             break;
+             case PCRE_ERROR_BADUTF8:
+             case PCRE_ERROR_SHORTUTF8:
+             fprintf(outfile, "Error %d (%s UTF-8 string)", count,
+               (count == PCRE_ERROR_BADUTF8)? "bad" : "short");
+             if (use_size_offsets >= 2)
+               fprintf(outfile, " offset=%d reason=%d", use_offsets[0],
+                 use_offsets[1]);
+             fprintf(outfile, "\n");
+             break;
+             default:
+             if (count < 0 && (-count) < sizeof(errtexts)/sizeof(const char *))
+               fprintf(outfile, "Error %d (%s)\n", count, errtexts[-count]);
+             else
+               fprintf(outfile, "Error %d (Unexpected value)\n", count);
+             break;
              }
-           else fprintf(outfile, "Error %d\n", count);
            break;  /* Out of the /g loop */
            }
          }
-Line 2317 
 while (!done)
+Line 3205 
 while (!done)
        if (!do_g && !do_G) break;
        /* If we have matched an empty string, first check to see if we are at
-       the end of the subject. If so, the /g loop is over. Otherwise, mimic
+       the end of the subject. If so, the /g loop is over. Otherwise, mimic what
-       what Perl's /g options does. This turns out to be rather cunning. First
+       Perl's /g options does. This turns out to be rather cunning. First we set
-       we set PCRE_NOTEMPTY and PCRE_ANCHORED and try the match again at the
+       PCRE_NOTEMPTY_ATSTART and PCRE_ANCHORED and try the match again at the
        same point. If this fails (picked up above) we advance to the next
        character. */
-Line 2328 
 while (!done)
+Line 3216 
 while (!done)
        if (use_offsets[0] == use_offsets[1])
          {
          if (use_offsets[0] == len) break;
-         g_notempty = PCRE_NOTEMPTY | PCRE_ANCHORED;
+         g_notempty = PCRE_NOTEMPTY_ATSTART | PCRE_ANCHORED;
          }
        /* For /g, update the start offset, leaving the rest alone */
-Line 2354 
 while (!done)
+Line 3242 
 while (!done)
  #endif
    if (re != NULL) new_free(re);
-   if (extra != NULL) new_free(extra);
+   if (extra != NULL) pcre_free_study(extra);
-   if (tables != NULL)
+   if (locale_set)
      {
      new_free((void *)tables);
      setlocale(LC_CTYPE, "C");
      locale_set = 0;
      }
+   if (jit_stack != NULL)
+     {
+     pcre_jit_stack_free(jit_stack);
+     jit_stack = NULL;
+     }
    }
  if (infile == stdin) fprintf(outfile, "\n");
-Line 2375 
 free(dbuffer);
+Line 3268 
 free(dbuffer);
  free(pbuffer);
  free(offsets);
+ #ifdef SUPPORT_PCRE16
+ if (buffer16 != NULL) free(buffer16);
+ #endif
  return yield;
  }

 Legend:



Removed from v.227
 


changed lines


 
Added in v.805
 Legend:



Removed from v.227
 


changed lines


 
Added in v.805
-Removed from v.227
+Added in v.805

webmaster@exim.org	ViewVC Help
Powered by ViewVC 1.1.12