/[pcre]/code/branches/pcre16/pcretest.c

Diff of /code/branches/pcre16/pcretest.c

Parent Directory | Revision Log | View Patch Patch

-code/trunk/pcretest.c
revision 389 by ph10,
Sun Mar 15 18:24:05 2009 UTC
+code/branches/pcre16/pcretest.c
revision 814 by ph10,
Wed Dec 21 12:05:24 2011 UTC
 Line 4
  /* This program was hacked up as a tester for PCRE. I really should have
  written it more tidily in the first place. Will I ever learn? It has grown and
- been extended and consequently is now rather, er, *very* untidy in places.
+ been extended and consequently is now rather, er, *very* untidy in places. The
+ addition of 16-bit support has made it even worse. :-(
  -----------------------------------------------------------------------------
  Redistribution and use in source and binary forms, with or without
-Line 71 
 input mode under Windows. */
+Line 72 
 input mode under Windows. */
  #define INPUT_MODE   "r"
  #define OUTPUT_MODE  "wb"
- #define isatty _isatty         /* This is what Windows calls them, I'm told */
+ #ifndef isatty
+ #define isatty _isatty         /* This is what Windows calls them, I'm told, */
+ #endif                         /* though in some environments they seem to   */
+                                /* be already defined, hence the #ifndefs.    */
+ #ifndef fileno
  #define fileno _fileno
+ #endif
+ /* A user sent this fix for Borland Builder 5 under Windows. */
+ #ifdef __BORLANDC__
+ #define _setmode(handle, mode) setmode(handle, mode)
+ #endif
+ /* Not Windows */
  #else
  #include <sys/time.h>          /* These two includes are needed */
-Line 94 
 appropriately for an application, not fo
+Line 108 
 appropriately for an application, not fo
  #include "pcre.h"
  #include "pcre_internal.h"
+ /* The pcre_printint() function, which prints the internal form of a compiled
+ regex, is held in a separate file so that (a) it can be compiled in either
+-bit or 16-bit mode, and (b) it can be #included directly in pcre_compile.c
+ when that is compiled in debug mode. */
+ #ifdef SUPPORT_PCRE8
+ void pcre_printint(pcre *external_re, FILE *f, BOOL print_lengths);
+ #endif
+ #ifdef SUPPORT_PCRE16
+ void pcre16_printint(pcre *external_re, FILE *f, BOOL print_lengths);
+ #endif
  /* We need access to some of the data tables that PCRE uses. So as not to have
  to keep two copies, we include the source file here, changing the names of the
  external symbols to prevent clashes. */
  #define _pcre_ucp_gentype      ucp_gentype
+ #define _pcre_ucp_typerange    ucp_typerange
  #define _pcre_utf8_table1      utf8_table1
  #define _pcre_utf8_table1_size utf8_table1_size
  #define _pcre_utf8_table2      utf8_table2
-Line 111 
 external symbols to prevent clashes. */
+Line 138 
 external symbols to prevent clashes. */
  #include "pcre_tables.c"
- /* We also need the pcre_printint() function for printing out compiled
+ /* The definition of the macro PRINTABLE, which determines whether to print an
- patterns. This function is in a separate file so that it can be included in
- pcre_compile.c when that module is compiled with debugging enabled.
- The definition of the macro PRINTABLE, which determines whether to print an
  output character as-is or as a hex value when showing compiled patterns, is
- contained in this file. We uses it here also, in cases when the locale has not
+ the same as in the printint.src file. We uses it here in cases when the locale
- been explicitly changed, so as to get consistent output from systems that
+ has not been explicitly changed, so as to get consistent output from systems
- differ in their output from isprint() even in the "C" locale. */
+ that differ in their output from isprint() even in the "C" locale. */
- #include "pcre_printint.src"
- #define PRINTHEX(c) (locale_set? isprint(c) : PRINTABLE(c))
+ #ifdef EBCDIC
+ #define PRINTABLE(c) ((c) >= 64 && (c) < 255)
+ #else
+ #define PRINTABLE(c) ((c) >= 32 && (c) < 127)
+ #endif
+ #define PRINTOK(c) (locale_set? isprint(c) : PRINTABLE(c))
  /* It is possible to compile this test program without including support for
  testing the POSIX interface, though this is not available via the standard
-Line 134 
 Makefile. */
+Line 160 
 Makefile. */
  #include "pcreposix.h"
  #endif
- /* It is also possible, for the benefit of the version currently imported into
+ /* It is also possible, originally for the benefit of a version that was
- Exim, to build pcretest without support for UTF8 (define NOUTF8), without the
+ imported into Exim, to build pcretest without support for UTF8 (define NOUTF8),
- interface to the DFA matcher (NODFA), and without the doublecheck of the old
+ without the interface to the DFA matcher (NODFA), and without the doublecheck
- "info" function (define NOINFOCHECK). In fact, we automatically cut out the
+ of the old "info" function (define NOINFOCHECK). In fact, we automatically cut
- UTF8 support if PCRE is built without it. */
+ out the UTF8 support if PCRE is built without it. */
  #ifndef SUPPORT_UTF8
  #ifndef NOUTF8
-Line 146 
 UTF8 support if PCRE is built without it
+Line 172 
 UTF8 support if PCRE is built without it
  #endif
  #endif
+ /* To make the code a bit tidier for 8-bit and 16-bit support, we define macros
+ for all the pcre[16]_xxx functions (except pcre16_fullinfo, which is called
+ only from one place and is handled differently). I couldn't dream up any way of
+ using a single macro to do this in a generic way, because of the many different
+ argument requirements. We know that at least one of SUPPORT_PCRE8 and
+ SUPPORT_PCRE16 must be set. First define macros for each individual mode; then
+ use these in the definitions of generic macros. */
+ #ifdef SUPPORT_PCRE8
+ #define PCHARS8(lv, p, len, f) \
+   lv = pchars((pcre_uint8 *)p, len, f)
+ #define PCHARSV8(p, len, f) \
+   (void)pchars((pcre_uint8 *)p, len, f)
+ #define PCRE_COMPILE8(re, pat, options, error, erroffset, tables) \
+   re = pcre_compile((char *)pat, options, error, erroffset, tables)
+ #define PCRE_DFA_EXEC8(count, re, extra, bptr, len, start_offset, options, \
+     offsets, size_offsets, workspace, size_workspace) \
+   count = pcre_dfa_exec(re, extra, (char *)bptr, len, start_offset, options, \
+     offsets, size_offsets, workspace, size_workspace)
+ #define PCRE_EXEC8(count, re, extra, bptr, len, start_offset, options, \
+     offsets, size_offsets) \
+   count = pcre_exec(re, extra, (char *)bptr, len, start_offset, options, \
+     offsets, size_offsets)
+ #define PCRE_FREE_STUDY8(extra) \
+   pcre_free_study(extra)
+ #define PCRE_PATTERN_TO_HOST_BYTE_ORDER8(re, extra, tables) \
+   pcre_pattern_to_host_byte_order(re, extra, tables)
+ #define PCRE_STUDY8(extra, re, options, error) \
+   extra = pcre_study(re, options, error)
+ #endif /* SUPPORT_PCRE8 */
+ #ifdef SUPPORT_PCRE16
+ #define PCHARS16(lv, p, len, f) \
+   lv = pchars16((PCRE_SPTR16)p, len, f)
+ #define PCHARSV16(p, len, f) \
+   (void)pchars16((PCRE_SPTR16)p, len, f)
+ #define PCRE_COMPILE16(re, pat, options, error, erroffset, tables) \
+   re = pcre16_compile((PCRE_SPTR16)pat, options, error, erroffset, tables)
+ #define PCRE_DFA_EXEC16(count, re, extra, bptr, len, start_offset, options, \
+     offsets, size_offsets, workspace, size_workspace) \
+   count = pcre16_dfa_exec(re, extra, (PCRE_SPTR16)bptr, len, start_offset, \
+     options, offsets, size_offsets, workspace, size_workspace)
+ #define PCRE_EXEC16(count, re, extra, bptr, len, start_offset, options, \
+     offsets, size_offsets) \
+   count = pcre16_exec(re, extra, (PCRE_SPTR16)bptr, len, start_offset, \
+     options, offsets, size_offsets)
+ #define PCRE_FREE_STUDY16(extra) \
+   pcre16_free_study(extra)
+ #define PCRE_PATTERN_TO_HOST_BYTE_ORDER16(re, extra, tables) \
+   pcre16_pattern_to_host_byte_order(re, extra, tables)
+ #define PCRE_STUDY16(extra, re, options, error) \
+   extra = pcre16_study(re, options, error)
+ #endif /* SUPPORT_PCRE16 */
+ /* ----- Both modes are supported; a runtime test is needed ----- */
+ #if defined SUPPORT_PCRE8 && defined SUPPORT_PCRE16
+ #define PCHARS(lv, p, len, f) \
+   if (use_pcre16) \
+     PCHARS16(lv, p, len, f); \
+   else \
+     PCHARS8(lv, p, len, f)
+ #define PCHARSV(p, len, f) \
+   if (use_pcre16) \
+     PCHARSV16(p, len, f); \
+   else \
+     PCHARSV8(p, len, f)
+ #define PCRE_COMPILE(re, pat, options, error, erroffset, tables) \
+   if (use_pcre16) \
+     PCRE_COMPILE16(re, pat, options, error, erroffset, tables); \
+   else \
+     PCRE_COMPILE8(re, pat, options, error, erroffset, tables)
+ #define PCRE_DFA_EXEC(count, re, extra, bptr, len, start_offset, options, \
+     offsets, size_offsets, workspace, size_workspace) \
+   if (use_pcre16) \
+     PCRE_DFA_EXEC16(count, re, extra, bptr, len, start_offset, options, \
+       offsets, size_offsets, workspace, size_workspace); \
+   else \
+     PCRE_DFA_EXEC8(count, re, extra, bptr, len, start_offset, options, \
+       offsets, size_offsets, workspace, size_workspace)
+ #define PCRE_EXEC(count, re, extra, bptr, len, start_offset, options, \
+     offsets, size_offsets) \
+   if (use_pcre16) \
+     PCRE_EXEC16(count, re, extra, bptr, len, start_offset, options, \
+       offsets, size_offsets); \
+   else \
+     PCRE_EXEC8(count, re, extra, bptr, len, start_offset, options, \
+       offsets, size_offsets)
+ #define PCRE_FREE_STUDY(extra) \
+   if (use_pcre16) \
+     PCRE_FREE_STUDY16(extra); \
+   else \
+     PCRE_FREE_STUDY8(extra)
+ #define PCRE_PATTERN_TO_HOST_BYTE_ORDER(re, extra, tables) \
+   if (use_pcre16) \
+     PCRE_PATTERN_TO_HOST_BYTE_ORDER16(re, extra, tables); \
+   else \
+     PCRE_PATTERN_TO_HOST_BYTE_ORDER8(re, extra, tables)
+ #define PCRE_STUDY(extra, re, options, error) \
+   if (use_pcre16) \
+     PCRE_STUDY16(extra, re, options, error); \
+   else \
+     PCRE_STUDY8(extra, re, options, error)
+ /* ----- Only 8-bit mode is supported ----- */
+ #elif defined SUPPORT_PCRE8
+ #define PCHARS           PCHARS8
+ #define PCHARSV          PCHARSV8
+ #define PCRE_COMPILE     PCRE_COMPILE8
+ #define PCRE_DFA_EXEC    PCRE_DFA_EXEC8
+ #define PCRE_EXEC        PCRE_EXEC8
+ #define PCRE_FREE_STUDY  PCRE_FREE_STUDY8
+ #define PCRE_PATTERN_TO_HOST_BYTE_ORDER PCRE_PATTERN_TO_HOST_BYTE_ORDER8
+ #define PCRE_STUDY       PCRE_STUDY8
+ /* ----- Only 16-bit mode is supported ----- */
+ #else
+ #define PCHARS           PCHARS16
+ #define PCHARSV          PCHARSV16
+ #define PCRE_COMPILE     PCRE_COMPILE16
+ #define PCRE_DFA_EXEC    PCRE_DFA_EXEC16
+ #define PCRE_EXEC        PCRE_EXEC16
+ #define PCRE_FREE_STUDY  PCRE_FREE_STUDY16
+ #define PCRE_PATTERN_TO_HOST_BYTE_ORDER PCRE_PATTERN_TO_HOST_BYTE_ORDER16
+ #define PCRE_STUDY       PCRE_STUDY16
+ #endif
+ /* ----- End of mode-specific function call macros ----- */
  /* Other parameters */
-Line 173 
 static int debug_lengths;
+Line 358 
 static int debug_lengths;
  static int first_callout;
  static int locale_set = 0;
  static int show_malloc;
- static int use_utf8;
+ static int use_utf;
  static size_t gotten_store;
+ static size_t first_gotten_store = 0;
+ static const unsigned char *last_callout_mark = NULL;
  /* The buffers grow automatically if very long input lines are encountered. */
  static int buffer_size = 50000;
- static uschar *buffer = NULL;
+ static pcre_uint8 *buffer = NULL;
- static uschar *dbuffer = NULL;
+ static pcre_uint8 *dbuffer = NULL;
- static uschar *pbuffer = NULL;
+ static pcre_uint8 *pbuffer = NULL;
+ /* Another buffer is needed translation to 16-bit character strings. It will
+ obtained and extended as required. */
+ #ifdef SUPPORT_PCRE16
+ static int buffer16_size = 0;
+ static pcre_uint16 *buffer16 = NULL;
+ /* We need the table of operator lengths that is used for 16-bit compiling, in
+ order to swap bytes in a pattern for saving/reloading testing. Luckily, the
+ data is defined as a macro. However, we must ensure that LINK_SIZE is adjusted
+ appropriately for the 16-bit world. Just as a safety check, make sure that
+ COMPILE_PCRE16 is *not* set. */
+ #ifdef COMPILE_PCRE16
+ #error COMPILE_PCRE16 must not be set when compiling pcretest.c
+ #endif
+ #if LINK_SIZE == 2
+ #undef LINK_SIZE
+ #define LINK_SIZE 1
+ #elif LINK_SIZE == 3 || LINK_SIZE == 4
+ #undef LINK_SIZE
+ #define LINK_SIZE 2
+ #else
+ #error LINK_SIZE must be either 2, 3, or 4
+ #endif
+ static const pcre_uint16 OP_lengths16[] = { OP_LENGTHS };
+ #endif  /* SUPPORT_PCRE16 */
+ /* If we have 8-bit support, default use_pcre16 to false; if there is also
+-bit support, it can be changed by an option. If there is no 8-bit support,
+ there must be 16-bit support, so default it to 1. */
+ #ifdef SUPPORT_PCRE8
+ static int use_pcre16 = 0;
+ #else
+ static int use_pcre16 = 1;
+ #endif
+ /* Textual explanations for runtime error codes */
+ static const char *errtexts[] = {
+   NULL,  /* 0 is no error */
+   NULL,  /* NOMATCH is handled specially */
+   "NULL argument passed",
+   "bad option value",
+   "magic number missing",
+   "unknown opcode - pattern overwritten?",
+   "no more memory",
+   NULL,  /* never returned by pcre_exec() or pcre_dfa_exec() */
+   "match limit exceeded",
+   "callout error code",
+   NULL,  /* BADUTF8 is handled specially */
+   "bad UTF-8 offset",
+   NULL,  /* PARTIAL is handled specially */
+   "not used - internal error",
+   "internal error - pattern overwritten?",
+   "bad count value",
+   "item unsupported for DFA matching",
+   "backreference condition or recursion test not supported for DFA matching",
+   "match limit not supported for DFA matching",
+   "workspace size exceeded in DFA matching",
+   "too much recursion for DFA matching",
+   "recursion limit exceeded",
+   "not used - internal error",
+   "invalid combination of newline options",
+   "bad offset value",
+   NULL,  /* SHORTUTF8 is handled specially */
+   "nested recursion at the same subject position",
+   "JIT stack limit reached",
+   "pattern compiled in wrong mode (8-bit/16-bit error)"
+ };
+ /*************************************************
+ *         Alternate character tables             *
+ *************************************************/
+ /* By default, the "tables" pointer when calling PCRE is set to NULL, thereby
+ using the default tables of the library. However, the T option can be used to
+ select alternate sets of tables, for different kinds of testing. Note also that
+ the L (locale) option also adjusts the tables. */
+ /* This is the set of tables distributed as default with PCRE. It recognizes
+ only ASCII characters. */
+ static const pcre_uint8 tables0[] = {
+ /* This table is a lower casing table. */
+,  1,  2,  3,  4,  5,  6,  7,
+,  9, 10, 11, 12, 13, 14, 15,
+, 17, 18, 19, 20, 21, 22, 23,
+, 25, 26, 27, 28, 29, 30, 31,
+, 33, 34, 35, 36, 37, 38, 39,
+, 41, 42, 43, 44, 45, 46, 47,
+, 49, 50, 51, 52, 53, 54, 55,
+, 57, 58, 59, 60, 61, 62, 63,
+, 97, 98, 99,100,101,102,103,
+,105,106,107,108,109,110,111,
+,113,114,115,116,117,118,119,
+,121,122, 91, 92, 93, 94, 95,
+, 97, 98, 99,100,101,102,103,
+,105,106,107,108,109,110,111,
+,113,114,115,116,117,118,119,
+,121,122,123,124,125,126,127,
+,129,130,131,132,133,134,135,
+,137,138,139,140,141,142,143,
+,145,146,147,148,149,150,151,
+,153,154,155,156,157,158,159,
+,161,162,163,164,165,166,167,
+,169,170,171,172,173,174,175,
+,177,178,179,180,181,182,183,
+,185,186,187,188,189,190,191,
+,193,194,195,196,197,198,199,
+,201,202,203,204,205,206,207,
+,209,210,211,212,213,214,215,
+,217,218,219,220,221,222,223,
+,225,226,227,228,229,230,231,
+,233,234,235,236,237,238,239,
+,241,242,243,244,245,246,247,
+,249,250,251,252,253,254,255,
+ /* This table is a case flipping table. */
+,  1,  2,  3,  4,  5,  6,  7,
+,  9, 10, 11, 12, 13, 14, 15,
+, 17, 18, 19, 20, 21, 22, 23,
+, 25, 26, 27, 28, 29, 30, 31,
+, 33, 34, 35, 36, 37, 38, 39,
+, 41, 42, 43, 44, 45, 46, 47,
+, 49, 50, 51, 52, 53, 54, 55,
+, 57, 58, 59, 60, 61, 62, 63,
+, 97, 98, 99,100,101,102,103,
+,105,106,107,108,109,110,111,
+,113,114,115,116,117,118,119,
+,121,122, 91, 92, 93, 94, 95,
+, 65, 66, 67, 68, 69, 70, 71,
+, 73, 74, 75, 76, 77, 78, 79,
+, 81, 82, 83, 84, 85, 86, 87,
+, 89, 90,123,124,125,126,127,
+,129,130,131,132,133,134,135,
+,137,138,139,140,141,142,143,
+,145,146,147,148,149,150,151,
+,153,154,155,156,157,158,159,
+,161,162,163,164,165,166,167,
+,169,170,171,172,173,174,175,
+,177,178,179,180,181,182,183,
+,185,186,187,188,189,190,191,
+,193,194,195,196,197,198,199,
+,201,202,203,204,205,206,207,
+,209,210,211,212,213,214,215,
+,217,218,219,220,221,222,223,
+,225,226,227,228,229,230,231,
+,233,234,235,236,237,238,239,
+,241,242,243,244,245,246,247,
+,249,250,251,252,253,254,255,
+ /* This table contains bit maps for various character classes. Each map is 32
+ bytes long and the bits run from the least significant end of each byte. The
+ classes that have their own maps are: space, xdigit, digit, upper, lower, word,
+ graph, print, punct, and cntrl. Other classes are built from combinations. */
+x00,0x3e,0x00,0x00,0x01,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03,
+x7e,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+xfe,0xff,0xff,0x07,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0xfe,0xff,0xff,0x07,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03,
+xfe,0xff,0xff,0x87,0xfe,0xff,0xff,0x07,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0xfe,0xff,0xff,0xff,
+xff,0xff,0xff,0xff,0xff,0xff,0xff,0x7f,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0xff,0xff,0xff,0xff,
+xff,0xff,0xff,0xff,0xff,0xff,0xff,0x7f,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0xfe,0xff,0x00,0xfc,
+x01,0x00,0x00,0xf8,0x01,0x00,0x00,0x78,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+xff,0xff,0xff,0xff,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x80,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+ /* This table identifies various classes of character by individual bits:
+x01   white space character
+x02   letter
+x04   decimal digit
+x08   hexadecimal digit
+x10   alphanumeric or '_'
+x80   regular expression metacharacter or binary zero
+ */
+x80,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /*   0-  7 */
+x00,0x01,0x01,0x00,0x01,0x01,0x00,0x00, /*   8- 15 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /*  16- 23 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /*  24- 31 */
+x01,0x00,0x00,0x00,0x80,0x00,0x00,0x00, /*    - '  */
+x80,0x80,0x80,0x80,0x00,0x00,0x80,0x00, /*  ( - /  */
+x1c,0x1c,0x1c,0x1c,0x1c,0x1c,0x1c,0x1c, /*  0 - 7  */
+x1c,0x1c,0x00,0x00,0x00,0x00,0x00,0x80, /*  8 - ?  */
+x00,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x12, /*  @ - G  */
+x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /*  H - O  */
+x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /*  P - W  */
+x12,0x12,0x12,0x80,0x80,0x00,0x80,0x10, /*  X - _  */
+x00,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x12, /*  ` - g  */
+x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /*  h - o  */
+x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /*  p - w  */
+x12,0x12,0x12,0x80,0x80,0x00,0x00,0x00, /*  x -127 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 128-135 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 136-143 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 144-151 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 152-159 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 160-167 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 168-175 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 176-183 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 184-191 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 192-199 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 200-207 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 208-215 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 216-223 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 224-231 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 232-239 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 240-247 */
+x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00};/* 248-255 */
+ /* This is a set of tables that came orginally from a Windows user. It seems to
+ be at least an approximation of ISO 8859. In particular, there are characters
+ greater than 128 that are marked as spaces, letters, etc. */
+ static const pcre_uint8 tables1[] = {
+,1,2,3,4,5,6,7,
+,9,10,11,12,13,14,15,
+,17,18,19,20,21,22,23,
+,25,26,27,28,29,30,31,
+,33,34,35,36,37,38,39,
+,41,42,43,44,45,46,47,
+,49,50,51,52,53,54,55,
+,57,58,59,60,61,62,63,
+,97,98,99,100,101,102,103,
+,105,106,107,108,109,110,111,
+,113,114,115,116,117,118,119,
+,121,122,91,92,93,94,95,
+,97,98,99,100,101,102,103,
+,105,106,107,108,109,110,111,
+,113,114,115,116,117,118,119,
+,121,122,123,124,125,126,127,
+,129,130,131,132,133,134,135,
+,137,138,139,140,141,142,143,
+,145,146,147,148,149,150,151,
+,153,154,155,156,157,158,159,
+,161,162,163,164,165,166,167,
+,169,170,171,172,173,174,175,
+,177,178,179,180,181,182,183,
+,185,186,187,188,189,190,191,
+,225,226,227,228,229,230,231,
+,233,234,235,236,237,238,239,
+,241,242,243,244,245,246,215,
+,249,250,251,252,253,254,223,
+,225,226,227,228,229,230,231,
+,233,234,235,236,237,238,239,
+,241,242,243,244,245,246,247,
+,249,250,251,252,253,254,255,
+,1,2,3,4,5,6,7,
+,9,10,11,12,13,14,15,
+,17,18,19,20,21,22,23,
+,25,26,27,28,29,30,31,
+,33,34,35,36,37,38,39,
+,41,42,43,44,45,46,47,
+,49,50,51,52,53,54,55,
+,57,58,59,60,61,62,63,
+,97,98,99,100,101,102,103,
+,105,106,107,108,109,110,111,
+,113,114,115,116,117,118,119,
+,121,122,91,92,93,94,95,
+,65,66,67,68,69,70,71,
+,73,74,75,76,77,78,79,
+,81,82,83,84,85,86,87,
+,89,90,123,124,125,126,127,
+,129,130,131,132,133,134,135,
+,137,138,139,140,141,142,143,
+,145,146,147,148,149,150,151,
+,153,154,155,156,157,158,159,
+,161,162,163,164,165,166,167,
+,169,170,171,172,173,174,175,
+,177,178,179,180,181,182,183,
+,185,186,187,188,189,190,191,
+,225,226,227,228,229,230,231,
+,233,234,235,236,237,238,239,
+,241,242,243,244,245,246,215,
+,249,250,251,252,253,254,223,
+,193,194,195,196,197,198,199,
+,201,202,203,204,205,206,207,
+,209,210,211,212,213,214,247,
+,217,218,219,220,221,222,255,
+,62,0,0,1,0,0,0,
+,0,0,0,0,0,0,0,
+,0,0,0,1,0,0,0,
+,0,0,0,0,0,0,0,
+,0,0,0,0,0,255,3,
+,0,0,0,126,0,0,0,
+,0,0,0,0,0,0,0,
+,0,0,0,0,0,0,0,
+,0,0,0,0,0,255,3,
+,0,0,0,0,0,0,0,
+,0,0,0,0,0,12,2,
+,0,0,0,0,0,0,0,
+,0,0,0,0,0,0,0,
+,255,255,7,0,0,0,0,
+,0,0,0,0,0,0,0,
+,255,127,127,0,0,0,0,
+,0,0,0,0,0,0,0,
+,0,0,0,254,255,255,7,
+,0,0,0,0,4,32,4,
+,0,0,128,255,255,127,255,
+,0,0,0,0,0,255,3,
+,255,255,135,254,255,255,7,
+,0,0,0,0,4,44,6,
+,255,127,255,255,255,127,255,
+,0,0,0,254,255,255,255,
+,255,255,255,255,255,255,127,
+,0,0,0,254,255,255,255,
+,255,255,255,255,255,255,255,
+,2,0,0,255,255,255,255,
+,255,255,255,255,255,255,127,
+,0,0,0,255,255,255,255,
+,255,255,255,255,255,255,255,
+,0,0,0,254,255,0,252,
+,0,0,248,1,0,0,120,
+,0,0,0,254,255,255,255,
+,0,128,0,0,0,128,0,
+,255,255,255,0,0,0,0,
+,0,0,0,0,0,0,128,
+,255,255,255,0,0,0,0,
+,0,0,0,0,0,0,0,
+,0,0,0,0,0,0,0,
+,1,1,0,1,1,0,0,
+,0,0,0,0,0,0,0,
+,0,0,0,0,0,0,0,
+,0,0,0,128,0,0,0,
+,128,128,128,0,0,128,0,
+,28,28,28,28,28,28,28,
+,28,0,0,0,0,0,128,
+,26,26,26,26,26,26,18,
+,18,18,18,18,18,18,18,
+,18,18,18,18,18,18,18,
+,18,18,128,128,0,128,16,
+,26,26,26,26,26,26,18,
+,18,18,18,18,18,18,18,
+,18,18,18,18,18,18,18,
+,18,18,128,128,0,0,0,
+,0,0,0,0,1,0,0,
+,0,0,0,0,0,0,0,
+,0,0,0,0,0,0,0,
+,0,0,0,0,0,0,0,
+,0,0,0,0,0,0,0,
+,0,18,0,0,0,0,0,
+,0,20,20,0,18,0,0,
+,20,18,0,0,0,0,0,
+,18,18,18,18,18,18,18,
+,18,18,18,18,18,18,18,
+,18,18,18,18,18,18,0,
+,18,18,18,18,18,18,18,
+,18,18,18,18,18,18,18,
+,18,18,18,18,18,18,18,
+,18,18,18,18,18,18,0,
+,18,18,18,18,18,18,18
+ };
+ #ifndef HAVE_STRERROR
+ /*************************************************
+ *     Provide strerror() for non-ANSI libraries  *
+ *************************************************/
+ /* Some old-fashioned systems still around (e.g. SunOS4) don't have strerror()
+ in their libraries, but can provide the same facility by this simple
+ alternative function. */
+ extern int   sys_nerr;
+ extern char *sys_errlist[];
+ char *
+ strerror(int n)
+ {
+ if (n < 0 || n >= sys_nerr) return "unknown error number";
+ return sys_errlist[n];
+ }
+ #endif /* HAVE_STRERROR */
+ /*************************************************
+ *         JIT memory callback                    *
+ *************************************************/
+ static pcre_jit_stack* jit_callback(void *arg)
+ {
+ return (pcre_jit_stack *)arg;
+ }
+ /*************************************************
+ *            Convert UTF-8 string to value       *
+ *************************************************/
+ /* This function takes one or more bytes that represents a UTF-8 character,
+ and returns the value of the character.
+ Argument:
+   utf8bytes   a pointer to the byte vector
+   vptr        a pointer to an int to receive the value
+ Returns:      >  0 => the number of bytes consumed
+               -6 to 0 => malformed UTF-8 character at offset = (-return)
+ */
+ #if !defined NOUTF8
+ static int
+ utf82ord(pcre_uint8 *utf8bytes, int *vptr)
+ {
+ int c = *utf8bytes++;
+ int d = c;
+ int i, j, s;
+ for (i = -1; i < 6; i++)               /* i is number of additional bytes */
+   {
+   if ((d & 0x80) == 0) break;
+   d <<= 1;
+   }
+ if (i == -1) { *vptr = c; return 1; }  /* ascii character */
+ if (i == 0 || i == 6) return 0;        /* invalid UTF-8 */
+ /* i now has a value in the range 1-5 */
+ s = 6*i;
+ d = (c & utf8_table3[i]) << s;
+ for (j = 0; j < i; j++)
+   {
+   c = *utf8bytes++;
+   if ((c & 0xc0) != 0x80) return -(j+1);
+   s -= 6;
+   d |= (c & 0x3f) << s;
+   }
+ /* Check that encoding was the correct unique one */
+ for (j = 0; j < utf8_table1_size; j++)
+   if (d <= utf8_table1[j]) break;
+ if (j != i) return -(i+1);
+ /* Valid value */
+ *vptr = d;
+ return i+1;
+ }
+ #endif
+ /*************************************************
+ *       Convert character value to UTF-8         *
+ *************************************************/
+ /* This function takes an integer value in the range 0 - 0x7fffffff
+ and encodes it as a UTF-8 character in 0 to 6 bytes.
+ Arguments:
+   cvalue     the character value
+   utf8bytes  pointer to buffer for result - at least 6 bytes long
+ Returns:     number of characters placed in the buffer
+ */
+ #if !defined NOUTF8
+ static int
+ ord2utf8(int cvalue, pcre_uint8 *utf8bytes)
+ {
+ register int i, j;
+ for (i = 0; i < utf8_table1_size; i++)
+   if (cvalue <= utf8_table1[i]) break;
+ utf8bytes += i;
+ for (j = i; j > 0; j--)
+  {
+  *utf8bytes-- = 0x80 | (cvalue & 0x3f);
+  cvalue >>= 6;
+  }
+ *utf8bytes = utf8_table2[i] | cvalue;
+ return i + 1;
+ }
+ #endif
+ #ifdef SUPPORT_PCRE16
+ /*************************************************
+ *         Convert a string to 16-bit             *
+ *************************************************/
+ /* In non-UTF mode, the space needed for a 16-bit string is exactly double the
+-bit size. For a UTF-8 string, the size needed for UTF-16 is no more than
+ double, because up to 0xffff uses no more than 3 bytes in UTF-8 but possibly 4
+ in UTF-16. Higher values use 4 bytes in UTF-8 and up to 4 bytes in UTF-16. The
+ result is always left in buffer16.
+ Arguments:
+   p          points to a byte string
+   utf        true if UTF-8 (to be converted to UTF-16)
+   len        number of bytes in the string (excluding trailing zero)
+ Returns:     number of 16-bit data items used (excluding trailing zero)
+              OR -1 if a UTF-8 string is malformed
+ */
+ static int
+ to16(pcre_uint8 *p, int utf, int len)
+ {
+ pcre_uint16 *pp;
+ if (buffer16_size < 2*len + 2)
+   {
+   if (buffer16 != NULL) free(buffer16);
+   buffer16_size = 2*len + 2;
+   buffer16 = (pcre_uint16 *)malloc(buffer16_size);
+   if (buffer16 == NULL)
+     {
+     fprintf(stderr, "pcretest: malloc(%d) failed for buffer16\n", buffer16_size);
+     exit(1);
+     }
+   }
+ pp = buffer16;
+ if (!utf)
+   {
+   while (len-- > 0) *pp++ = *p++;
+   }
+ else
+   {
+   int c;
+   while (len > 0)
+     {
+     int chlen = utf82ord(p, &c);
+     if (chlen <= 0) return -1;
+     p += chlen;
+     len -= chlen;
+     if (c < 0x10000) *pp++ = c; else
+       {
+       c -= 0x10000;
+       *pp++ = 0xD800 | (c >> 10);
+       *pp++ = 0xDC00 | (c & 0x3ff);
+       }
+     }
+   }
+ *pp = 0;
+ return pp - buffer16;
+ }
+ #endif
  /*************************************************
-Line 208 
 Returns:       pointer to the start of n
+Line 993 
 Returns:       pointer to the start of n
                 NULL if no data read and EOF reached
  */
- static uschar *
+ static pcre_uint8 *
- extend_inputline(FILE *f, uschar *start, const char *prompt)
+ extend_inputline(FILE *f, pcre_uint8 *start, const char *prompt)
  {
- uschar *here = start;
+ pcre_uint8 *here = start;
  for (;;)
    {
-   int rlen = buffer_size - (here - buffer);
+   int rlen = (int)(buffer_size - (here - buffer));
    if (rlen > 1000)
      {
-Line 245 
 for (;;)
+Line 1030 
 for (;;)
      /* Read the next line by normal means, prompting if the file is stdin. */
        {
-       if (f == stdin) printf(prompt);
+       if (f == stdin) printf("%s", prompt);
        if (fgets((char *)here, rlen,  f) == NULL)
          return (here == start)? NULL : start;
        }
-Line 258 
 for (;;)
+Line 1043 
 for (;;)
    else
      {
      int new_buffer_size = 2*buffer_size;
-     uschar *new_buffer = (unsigned char *)malloc(new_buffer_size);
+     pcre_uint8 *new_buffer = (pcre_uint8 *)malloc(new_buffer_size);
-     uschar *new_dbuffer = (unsigned char *)malloc(new_buffer_size);
+     pcre_uint8 *new_dbuffer = (pcre_uint8 *)malloc(new_buffer_size);
-     uschar *new_pbuffer = (unsigned char *)malloc(new_buffer_size);
+     pcre_uint8 *new_pbuffer = (pcre_uint8 *)malloc(new_buffer_size);
      if (new_buffer == NULL || new_dbuffer == NULL || new_pbuffer == NULL)
        {
-Line 291 
 return NULL;  /* Control never gets here
+Line 1076 
 return NULL;  /* Control never gets here
  /*************************************************
  *          Read number from string               *
  *************************************************/
-Line 311 
 Returns:        the unsigned long
+Line 1092 
 Returns:        the unsigned long
  */
  static int
- get_value(unsigned char *str, unsigned char **endptr)
+ get_value(pcre_uint8 *str, pcre_uint8 **endptr)
  {
  int result = 0;
  while(*str != 0 && isspace(*str)) str++;
-Line 322 
 return(result);
+Line 1103 
 return(result);
  /*************************************************
- *            Convert UTF-8 string to value       *
+ *             Print one character                *
  *************************************************/
- /* This function takes one or more bytes that represents a UTF-8 character,
+ /* Print a single character either literally, or as a hex escape. */
- and returns the value of the character.
- Argument:
-   utf8bytes   a pointer to the byte vector
-   vptr        a pointer to an int to receive the value
- Returns:      >  0 => the number of bytes consumed
-               -6 to 0 => malformed UTF-8 character at offset = (-return)
- */
- #if !defined NOUTF8
- static int
+ static int pchar(int c, FILE *f)
- utf82ord(unsigned char *utf8bytes, int *vptr)
  {
- int c = *utf8bytes++;
+ if (PRINTOK(c))
- int d = c;
- int i, j, s;
- for (i = -1; i < 6; i++)               /* i is number of additional bytes */
    {
-   if ((d & 0x80) == 0) break;
+   if (f != NULL) fprintf(f, "%c", c);
-   d <<= 1;
+   return 1;
    }
- if (i == -1) { *vptr = c; return 1; }  /* ascii character */
+ if (c < 0x100)
- if (i == 0 || i == 6) return 0;        /* invalid UTF-8 */
- /* i now has a value in the range 1-5 */
- s = 6*i;
- d = (c & utf8_table3[i]) << s;
- for (j = 0; j < i; j++)
    {
-   c = *utf8bytes++;
+   if (use_utf)
-   if ((c & 0xc0) != 0x80) return -(j+1);
+     {
-   s -= 6;
+     if (f != NULL) fprintf(f, "\\x{%02x}", c);
-   d |= (c & 0x3f) << s;
+     return 6;
+     }
+   else
+     {
+     if (f != NULL) fprintf(f, "\\x%02x", c);
+     return 4;
+     }
    }
- /* Check that encoding was the correct unique one */
+ if (f != NULL) fprintf(f, "\\x{%02x}", c);
+ return (c <= 0x000000ff)? 6 :
- for (j = 0; j < utf8_table1_size; j++)
+        (c <= 0x00000fff)? 7 :
-   if (d <= utf8_table1[j]) break;
+        (c <= 0x0000ffff)? 8 :
- if (j != i) return -(i+1);
+        (c <= 0x000fffff)? 9 : 10;
- /* Valid value */
- *vptr = d;
- return i+1;
  }
- #endif
+ #ifdef SUPPORT_PCRE8
  /*************************************************
- *       Convert character value to UTF-8         *
+ *         Print 8-bit character string           *
  *************************************************/
- /* This function takes an integer value in the range 0 - 0x7fffffff
+ /* Must handle UTF-8 strings in utf8 mode. Yields number of characters printed.
- and encodes it as a UTF-8 character in 0 to 6 bytes.
+ If handed a NULL file, just counts chars without printing. */
- Arguments:
+ static int pchars(pcre_uint8 *p, int length, FILE *f)
-   cvalue     the character value
+ {
-   utf8bytes  pointer to buffer for result - at least 6 bytes long
+ int c = 0;
+ int yield = 0;
- Returns:     number of characters placed in the buffer
- */
+ while (length-- > 0)
+   {
  #if !defined NOUTF8
+   if (use_utf)
+     {
+     int rc = utf82ord(p, &c);
+     if (rc > 0 && rc <= length + 1)   /* Mustn't run over the end */
+       {
+       length -= rc - 1;
+       p += rc;
+       yield += pchar(c, f);
+       continue;
+       }
+     }
+ #endif
+   c = *p++;
+   yield += pchar(c, f);
+   }
- static int
+ return yield;
- ord2utf8(int cvalue, uschar *utf8bytes)
+ }
- {
- register int i, j;
- for (i = 0; i < utf8_table1_size; i++)
-   if (cvalue <= utf8_table1[i]) break;
- utf8bytes += i;
- for (j = i; j > 0; j--)
-  {
-  *utf8bytes-- = 0x80 | (cvalue & 0x3f);
-  cvalue >>= 6;
-  }
- *utf8bytes = utf8_table2[i] | cvalue;
- return i + 1;
- }
  #endif
+ #ifdef SUPPORT_PCRE16
  /*************************************************
- *             Print character string             *
+ *           Print 16-bit character string        *
  *************************************************/
- /* Character string printing function. Must handle UTF-8 strings in utf8
+ /* Must handle UTF-16 strings in utf mode. Yields number of characters printed.
- mode. Yields number of characters printed. If handed a NULL file, just counts
+ If handed a NULL file, just counts chars without printing. */
- chars without printing. */
- static int pchars(unsigned char *p, int length, FILE *f)
+ static int pchars16(PCRE_SPTR16 p, int length, FILE *f)
  {
- int c = 0;
  int yield = 0;
  while (length-- > 0)
    {
+   int c = *p++ & 0xffff;
  #if !defined NOUTF8
-   if (use_utf8)
+   if (use_utf && c >= 0xD800 && c < 0xDC00 && length > 0)
      {
-     int rc = utf82ord(p, &c);
+     int d = *p & 0xffff;
+     if (d >= 0xDC00 && d < 0xDFFF)
-     if (rc > 0 && rc <= length + 1)   /* Mustn't run over the end */
        {
-       length -= rc - 1;
+       c = ((c & 0x3ff) << 10) + (d & 0x3ff) + 0x10000;
-       p += rc;
+       length--;
-       if (PRINTHEX(c))
+       p++;
-         {
-         if (f != NULL) fprintf(f, "%c", c);
-         yield++;
-         }
-       else
-         {
-         int n = 4;
-         if (f != NULL) fprintf(f, "\\x{%02x}", c);
-         yield += (n <= 0x000000ff)? 2 :
-                  (n <= 0x00000fff)? 3 :
-                  (n <= 0x0000ffff)? 4 :
-                  (n <= 0x000fffff)? 5 : 6;
-         }
-       continue;
        }
      }
  #endif
+   yield += pchar(c, f);
-    /* Not UTF-8, or malformed UTF-8  */
-   c = *p++;
-   if (PRINTHEX(c))
-     {
-     if (f != NULL) fprintf(f, "%c", c);
-     yield++;
-     }
-   else
-     {
-     if (f != NULL) fprintf(f, "\\x%02x", c);
-     yield += 4;
-     }
    }
  return yield;
  }
+ #endif
-Line 509 
 if (callout_extra)
+Line 1239 
 if (callout_extra)
      else
        {
        fprintf(f, "%2d: ", i/2);
-       (void)pchars((unsigned char *)cb->subject + cb->offset_vector[i],
+       PCHARSV(cb->subject + cb->offset_vector[i],
          cb->offset_vector[i+1] - cb->offset_vector[i], f);
        fprintf(f, "\n");
        }
-Line 522 
 printed lengths of the substrings. */
+Line 1252 
 printed lengths of the substrings. */
  if (f != NULL) fprintf(f, "--->");
- pre_start = pchars((unsigned char *)cb->subject, cb->start_match, f);
+ PCHARS(pre_start, cb->subject, cb->start_match, f);
- post_start = pchars((unsigned char *)(cb->subject + cb->start_match),
+ PCHARS(post_start, cb->subject + cb->start_match,
    cb->current_position - cb->start_match, f);
- subject_length = pchars((unsigned char *)cb->subject, cb->subject_length, NULL);
+ PCHARS(subject_length, cb->subject, cb->subject_length, NULL);
- (void)pchars((unsigned char *)(cb->subject + cb->current_position),
+ PCHARSV(cb->subject + cb->current_position,
    cb->subject_length - cb->current_position, f);
  if (f != NULL) fprintf(f, "\n");
-Line 565 
 fprintf(outfile, "%.*s", (cb->next_item_
+Line 1295 
 fprintf(outfile, "%.*s", (cb->next_item_
  fprintf(outfile, "\n");
  first_callout = 0;
+ if (cb->mark != last_callout_mark)
+   {
+   fprintf(outfile, "Latest Mark: %s\n",
+     (cb->mark == NULL)? "<unset>" : (char *)(cb->mark));
+   last_callout_mark = cb->mark;
+   }
  if (cb->callout_data != NULL)
    {
    int callout_data = *((int *)(cb->callout_data));
-Line 584 
 return (cb->callout_number != callout_fa
+Line 1321 
 return (cb->callout_number != callout_fa
  *            Local malloc functions              *
  *************************************************/
- /* Alternative malloc function, to test functionality and show the size of the
+ /* Alternative malloc function, to test functionality and save the size of a
- compiled re. */
+ compiled re, which is the first store request that pcre_compile() makes. The
+ show_malloc variable is set only during matching. */
  static void *new_malloc(size_t size)
  {
  void *block = malloc(size);
  gotten_store = size;
+ if (first_gotten_store == 0) first_gotten_store = size;
  if (show_malloc)
    fprintf(outfile, "malloc       %3d %p\n", (int)size, block);
  return block;
-Line 603 
 if (show_malloc)
+Line 1342 
 if (show_malloc)
  free(block);
  }
  /* For recursion malloc/free, to test stacking calls */
  static void *stack_malloc(size_t size)
-Line 626 
 free(block);
+Line 1364 
 free(block);
  *          Call pcre_fullinfo()                  *
  *************************************************/
- /* Get one piece of information from the pcre_fullinfo() function */
+ /* Get one piece of information from the pcre_fullinfo() function. When only
+ one of 8-bit or 16-bit is supported, use_pcre16 should always have the correct
+ value, but the code is defensive. */
  static void new_info(pcre *re, pcre_extra *study, int option, void *ptr)
  {
  int rc;
- if ((rc = pcre_fullinfo(re, study, option, ptr)) < 0)
-   fprintf(outfile, "Error %d from pcre_fullinfo(%d)\n", rc, option);
+ if (use_pcre16)
+ #ifdef SUPPORT_PCRE16
+   rc = pcre16_fullinfo(re, study, option, ptr);
+ #else
+   rc = PCRE_ERROR_BADMODE;
+ #endif
+ else
+ #ifdef SUPPORT_PCRE8
+   rc = pcre_fullinfo(re, study, option, ptr);
+ #else
+   rc = PCRE_ERROR_BADMODE;
+ #endif
+ if (rc < 0) fprintf(outfile, "Error %d from pcre%s_fullinfo(%d)\n", rc,
+   use_pcre16? "16" : "", option);
  }
  /*************************************************
- *         Byte flipping function                 *
+ *             Swap byte functions                *
  *************************************************/
- static unsigned long int
+ /* The following functions swap the bytes of a pcre_uint16
- byteflip(unsigned long int value, int n)
+ and pcre_uint32 value.
+ Arguments:
+   value        any number
+ Returns:       the byte swapped value
+ */
+ static pcre_uint32
+ swap_uint32(pcre_uint32 value)
  {
- if (n == 2) return ((value & 0x00ff) << 8) | ((value & 0xff00) >> 8);
  return ((value & 0x000000ff) << 24) |
         ((value & 0x0000ff00) <<  8) |
         ((value & 0x00ff0000) >>  8) |
-        ((value & 0xff000000) >> 24);
+        (value >> 24);
+ }
+ static pcre_uint16
+ swap_uint16(pcre_uint16 value)
+ {
+ return (value >> 8) | (value << 8);
  }
+ /*************************************************
+ *        Flip bytes in a compiled pattern        *
+ *************************************************/
+ /* This function is called if the 'F' option was present on a pattern that is
+ to be written to a file. We flip the bytes of all the integer fields in the
+ regex data block and the study block. In 16-bit mode this also flips relevant
+ bytes in the pattern itself. This is to make it possible to test PCRE's
+ ability to reload byte-flipped patterns, e.g. those compiled on a different
+ architecture. */
+ static void
+ regexflip(pcre *ere, pcre_extra *extra)
+ {
+ real_pcre *re = (real_pcre *)ere;
+ int op;
+ #ifdef SUPPORT_PCRE16
+ pcre_uint16 *ptr = (pcre_uint16 *)re + re->name_table_offset;
+ int length = re->name_count * re->name_entry_size;
+ #ifdef SUPPORT_UTF
+ BOOL utf = (re->options & PCRE_UTF16) != 0;
+ BOOL utf16_char = FALSE;
+ #endif /* SUPPORT_UTF */
+ #endif /* SUPPORT_PCRE16 */
+ /* Always flip the bytes in the main data block and study blocks. */
+ re->magic_number = REVERSED_MAGIC_NUMBER;
+ re->size = swap_uint32(re->size);
+ re->options = swap_uint32(re->options);
+ re->flags = swap_uint16(re->flags);
+ re->top_bracket = swap_uint16(re->top_bracket);
+ re->top_backref = swap_uint16(re->top_backref);
+ re->first_char = swap_uint16(re->first_char);
+ re->req_char = swap_uint16(re->req_char);
+ re->name_table_offset = swap_uint16(re->name_table_offset);
+ re->name_entry_size = swap_uint16(re->name_entry_size);
+ re->name_count = swap_uint16(re->name_count);
+ if (extra != NULL)
+   {
+   pcre_study_data *rsd = (pcre_study_data *)(extra->study_data);
+   rsd->size = swap_uint32(rsd->size);
+   rsd->flags = swap_uint32(rsd->flags);
+   rsd->minlength = swap_uint32(rsd->minlength);
+   }
+ /* In 8-bit mode, that is all we need to do. In 16-bit mode we must swap bytes
+ in the name table, if present, and then in the pattern itself. */
+ #ifdef SUPPORT_PCRE16
+ if (!use_pcre16) return;
+ while(TRUE)
+   {
+   /* Swap previous characters. */
+   while (length-- > 0)
+     {
+     *ptr = swap_uint16(*ptr);
+     ptr++;
+     }
+ #ifdef SUPPORT_UTF
+   if (utf16_char)
+     {
+     if ((ptr[-1] & 0xfc00) == 0xd800)
+       {
+       /* We know that there is only one extra character in UTF-16. */
+       *ptr = swap_uint16(*ptr);
+       ptr++;
+       }
+     }
+   utf16_char = FALSE;
+ #endif /* SUPPORT_UTF */
+   /* Get next opcode. */
+   length = 0;
+   op = *ptr;
+   *ptr++ = swap_uint16(op);
+   switch (op)
+     {
+     case OP_END:
+     return;
+     case OP_CHAR:
+     case OP_CHARI:
+     case OP_NOT:
+     case OP_NOTI:
+     case OP_STAR:
+     case OP_MINSTAR:
+     case OP_PLUS:
+     case OP_MINPLUS:
+     case OP_QUERY:
+     case OP_MINQUERY:
+     case OP_UPTO:
+     case OP_MINUPTO:
+     case OP_EXACT:
+     case OP_POSSTAR:
+     case OP_POSPLUS:
+     case OP_POSQUERY:
+     case OP_POSUPTO:
+     case OP_STARI:
+     case OP_MINSTARI:
+     case OP_PLUSI:
+     case OP_MINPLUSI:
+     case OP_QUERYI:
+     case OP_MINQUERYI:
+     case OP_UPTOI:
+     case OP_MINUPTOI:
+     case OP_EXACTI:
+     case OP_POSSTARI:
+     case OP_POSPLUSI:
+     case OP_POSQUERYI:
+     case OP_POSUPTOI:
+     case OP_NOTSTAR:
+     case OP_NOTMINSTAR:
+     case OP_NOTPLUS:
+     case OP_NOTMINPLUS:
+     case OP_NOTQUERY:
+     case OP_NOTMINQUERY:
+     case OP_NOTUPTO:
+     case OP_NOTMINUPTO:
+     case OP_NOTEXACT:
+     case OP_NOTPOSSTAR:
+     case OP_NOTPOSPLUS:
+     case OP_NOTPOSQUERY:
+     case OP_NOTPOSUPTO:
+     case OP_NOTSTARI:
+     case OP_NOTMINSTARI:
+     case OP_NOTPLUSI:
+     case OP_NOTMINPLUSI:
+     case OP_NOTQUERYI:
+     case OP_NOTMINQUERYI:
+     case OP_NOTUPTOI:
+     case OP_NOTMINUPTOI:
+     case OP_NOTEXACTI:
+     case OP_NOTPOSSTARI:
+     case OP_NOTPOSPLUSI:
+     case OP_NOTPOSQUERYI:
+     case OP_NOTPOSUPTOI:
+ #ifdef SUPPORT_UTF
+     if (utf) utf16_char = TRUE;
+ #endif
+     length = OP_lengths16[op] - 1;
+     break;
+     case OP_CLASS:
+     case OP_NCLASS:
+     /* Skip the character bit map. */
+     ptr += 32/sizeof(pcre_uint16);
+     length = 0;
+     break;
+     case OP_XCLASS:
+     /* Reverse the size of the XCLASS instance. */
+     ptr++;
+     *ptr = swap_uint16(*ptr);
+     if (LINK_SIZE > 1)
+       {
+       /* LINK_SIZE can be 1 or 2 in 16 bit mode. */
+       ptr++;
+       *ptr = swap_uint16(*ptr);
+       }
+     ptr++;
+     if (LINK_SIZE > 1)
+       length = ((ptr[-LINK_SIZE] << 16) | ptr[-LINK_SIZE + 1]) -
+         (1 + LINK_SIZE + 1);
+     else
+       length = ptr[-LINK_SIZE] - (1 + LINK_SIZE + 1);
+     op = *ptr;
+     *ptr = swap_uint16(op);
+     if ((op & XCL_MAP) != 0)
+       {
+       /* Skip the character bit map. */
+       ptr += 32/sizeof(pcre_uint16);
+       length -= 32/sizeof(pcre_uint16);
+       }
+     break;
+     default:
+     length = OP_lengths16[op] - 1;
+     break;
+     }
+   }
+ /* Control should never reach here in 16 bit mode. */
+ #endif /* SUPPORT_PCRE16 */
+ }
  /*************************************************
  *        Check match or recursion limit          *
  *************************************************/
  static int
- check_match_limit(pcre *re, pcre_extra *extra, uschar *bptr, int len,
+ check_match_limit(pcre *re, pcre_extra *extra, pcre_uint8 *bptr, int len,
    int start_offset, int options, int *use_offsets, int use_size_offsets,
    int flag, unsigned long int *limit, int errnumber, const char *msg)
  {
-Line 674 
 for (;;)
+Line 1635 
 for (;;)
    {
    *limit = mid;
-   count = pcre_exec(re, extra, (char *)bptr, len, start_offset, options,
+   PCRE_EXEC(count, re, extra, bptr, len, start_offset, options,
      use_offsets, use_size_offsets);
    if (count == errnumber)
-Line 719 
 Returns:    < 0, = 0, or > 0, according
+Line 1680 
 Returns:    < 0, = 0, or > 0, according
  */
  static int
- strncmpic(uschar *s, uschar *t, int n)
+ strncmpic(pcre_uint8 *s, pcre_uint8 *t, int n)
  {
  while (n--)
    {
-Line 735 
 return 0;
+Line 1696 
 return 0;
  *         Check newline indicator                *
  *************************************************/
- /* This is used both at compile and run-time to check for <xxx> escapes, where
+ /* This is used both at compile and run-time to check for <xxx> escapes. Print
- xxx is LF, CR, CRLF, ANYCRLF, or ANY. Print a message and return 0 if there is
+ a message and return 0 if there is no match.
- no match.
  Arguments:
    p           points after the leading '<'
-Line 747 
 Returns:      appropriate PCRE_NEWLINE_x
+Line 1707 
 Returns:      appropriate PCRE_NEWLINE_x
  */
  static int
- check_newline(uschar *p, FILE *f)
+ check_newline(pcre_uint8 *p, FILE *f)
  {
- if (strncmpic(p, (uschar *)"cr>", 3) == 0) return PCRE_NEWLINE_CR;
+ if (strncmpic(p, (pcre_uint8 *)"cr>", 3) == 0) return PCRE_NEWLINE_CR;
- if (strncmpic(p, (uschar *)"lf>", 3) == 0) return PCRE_NEWLINE_LF;
+ if (strncmpic(p, (pcre_uint8 *)"lf>", 3) == 0) return PCRE_NEWLINE_LF;
- if (strncmpic(p, (uschar *)"crlf>", 5) == 0) return PCRE_NEWLINE_CRLF;
+ if (strncmpic(p, (pcre_uint8 *)"crlf>", 5) == 0) return PCRE_NEWLINE_CRLF;
- if (strncmpic(p, (uschar *)"anycrlf>", 8) == 0) return PCRE_NEWLINE_ANYCRLF;
+ if (strncmpic(p, (pcre_uint8 *)"anycrlf>", 8) == 0) return PCRE_NEWLINE_ANYCRLF;
- if (strncmpic(p, (uschar *)"any>", 4) == 0) return PCRE_NEWLINE_ANY;
+ if (strncmpic(p, (pcre_uint8 *)"any>", 4) == 0) return PCRE_NEWLINE_ANY;
- if (strncmpic(p, (uschar *)"bsr_anycrlf>", 12) == 0) return PCRE_BSR_ANYCRLF;
+ if (strncmpic(p, (pcre_uint8 *)"bsr_anycrlf>", 12) == 0) return PCRE_BSR_ANYCRLF;
- if (strncmpic(p, (uschar *)"bsr_unicode>", 12) == 0) return PCRE_BSR_UNICODE;
+ if (strncmpic(p, (pcre_uint8 *)"bsr_unicode>", 12) == 0) return PCRE_BSR_UNICODE;
  fprintf(f, "Unknown newline type at: <%s\n", p);
  return 0;
  }
-Line 777 
 printf("If input is a terminal, readline
+Line 1737 
 printf("If input is a terminal, readline
  printf("This version of pcretest is not linked with readline().\n");
  #endif
  printf("\nOptions:\n");
+ #ifdef SUPPORT_PCRE16
+ printf("  -16      use 16-bit interface\n");
+ #endif
  printf("  -b       show compiled code (bytecode)\n");
  printf("  -C       show PCRE compile-time options and exit\n");
  printf("  -d       debug: show compiled code and information (-b and -i)\n");
-Line 793 
 printf("  -p       use POSIX interface\n
+Line 1756 
 printf("  -p       use POSIX interface\n
  #endif
  printf("  -q       quiet: do not output PCRE version number at start\n");
  printf("  -S <n>   set stack size to <n> megabytes\n");
- printf("  -s       output store (memory) used information\n"
+ printf("  -s       force each pattern to be studied at basic level\n"
+        "  -s+      force each pattern to be studied, using JIT if available\n"
         "  -t       time compilation and execution\n");
  printf("  -t <n>   time compilation and execution, repeating <n> times\n");
  printf("  -tm      time execution (matching) only\n");
-Line 821 
 int timeit = 0;
+Line 1785 
 int timeit = 0;
  int timeitm = 0;
  int showinfo = 0;
  int showstore = 0;
+ int force_study = -1;
+ int force_study_options = 0;
  int quiet = 0;
  int size_offsets = 45;
  int size_offsets_max;
-Line 834 
 int all_use_dfa = 0;
+Line 1800 
 int all_use_dfa = 0;
  int yield = 0;
  int stack_size;
+ pcre_jit_stack *jit_stack = NULL;
  /* These vectors store, end-to-end, a list of captured substring names. Assume
  that 1024 is plenty long enough for the few names we'll be testing. */
- uschar copynames[1024];
+ pcre_uchar copynames[1024];
- uschar getnames[1024];
+ pcre_uchar getnames[1024];
- uschar *copynamesptr;
+ pcre_uchar *copynamesptr;
- uschar *getnamesptr;
+ pcre_uchar *getnamesptr;
- /* Get buffers from malloc() so that Electric Fence will check their misuse
+ /* Get buffers from malloc() so that valgrind will check their misuse when
- when I am debugging. They grow automatically when very long lines are read. */
+ debugging. They grow automatically when very long lines are read. The 16-bit
+ buffer (buffer16) is obtained only if needed. */
- buffer = (unsigned char *)malloc(buffer_size);
- dbuffer = (unsigned char *)malloc(buffer_size);
+ buffer = (pcre_uint8 *)malloc(buffer_size);
- pbuffer = (unsigned char *)malloc(buffer_size);
+ dbuffer = (pcre_uint8 *)malloc(buffer_size);
+ pbuffer = (pcre_uint8 *)malloc(buffer_size);
  /* The outfile variable is static so that new_malloc can use it. */
-Line 867 
 _setmode( _fileno( stdout ), _O_BINARY )
+Line 1836 
 _setmode( _fileno( stdout ), _O_BINARY )
  while (argc > 1 && argv[op][0] == '-')
    {
-   unsigned char *endptr;
+   pcre_uint8 *endptr;
+   if (strcmp(argv[op], "-m") == 0) showstore = 1;
+   else if (strcmp(argv[op], "-s") == 0) force_study = 0;
+   else if (strcmp(argv[op], "-s+") == 0)
+     {
+     force_study = 1;
+     force_study_options = PCRE_STUDY_JIT_COMPILE;
+     }
+ #ifdef SUPPORT_PCRE16
+   else if (strcmp(argv[op], "-16") == 0) use_pcre16 = 1;
+ #endif
-   if (strcmp(argv[op], "-s") == 0 || strcmp(argv[op], "-m") == 0)
-     showstore = 1;
    else if (strcmp(argv[op], "-q") == 0) quiet = 1;
    else if (strcmp(argv[op], "-b") == 0) debug = 1;
    else if (strcmp(argv[op], "-i") == 0) showinfo = 1;
    else if (strcmp(argv[op], "-d") == 0) showinfo = debug = 1;
    else if (strcmp(argv[op], "-M") == 0) default_find_match_limit = TRUE;
  #if !defined NODFA
    else if (strcmp(argv[op], "-dfa") == 0) all_use_dfa = 1;
  #endif
    else if (strcmp(argv[op], "-o") == 0 && argc > 2 &&
-       ((size_offsets = get_value((unsigned char *)argv[op+1], &endptr)),
+       ((size_offsets = get_value((pcre_uint8 *)argv[op+1], &endptr)),
          *endptr == 0))
      {
      op++;
-Line 890 
 while (argc > 1 && argv[op][0] == '-')
+Line 1868 
 while (argc > 1 && argv[op][0] == '-')
      {
      int both = argv[op][2] == 0;
      int temp;
-     if (argc > 2 && (temp = get_value((unsigned char *)argv[op+1], &endptr),
+     if (argc > 2 && (temp = get_value((pcre_uint8 *)argv[op+1], &endptr),
                       *endptr == 0))
        {
        timeitm = temp;
-Line 901 
 while (argc > 1 && argv[op][0] == '-')
+Line 1879 
 while (argc > 1 && argv[op][0] == '-')
      if (both) timeit = timeitm;
      }
    else if (strcmp(argv[op], "-S") == 0 && argc > 2 &&
-       ((stack_size = get_value((unsigned char *)argv[op+1], &endptr)),
+       ((stack_size = get_value((pcre_uint8 *)argv[op+1], &endptr)),
          *endptr == 0))
      {
- #if defined(_WIN32) || defined(WIN32)
+ #if defined(_WIN32) || defined(WIN32) || defined(__minix)
      printf("PCRE: -S not supported on this OS\n");
      exit(1);
  #else
-Line 928 
 while (argc > 1 && argv[op][0] == '-')
+Line 1906 
 while (argc > 1 && argv[op][0] == '-')
    else if (strcmp(argv[op], "-C") == 0)
      {
      int rc;
      unsigned long int lrc;
      printf("PCRE version %s\n", pcre_version());
      printf("Compiled with\n");
+ /* At least one of SUPPORT_PCRE8 and SUPPORT_PCRE16 will be set. If both
+ are set, either both UTFs are supported or both are not supported. */
+ #if defined SUPPORT_PCRE8 && defined SUPPORT_PCRE16
+     printf("  8-bit and 16-bit support\n");
+     (void)pcre_config(PCRE_CONFIG_UTF8, &rc);
+     if (rc)
+       printf("  UTF-8 and UTF-16 support\n");
+     else
+       printf("  No UTF-8 or UTF-16 support\n");
+ #elif defined SUPPORT_PCRE8
+     printf("  8-bit support only\n");
      (void)pcre_config(PCRE_CONFIG_UTF8, &rc);
      printf("  %sUTF-8 support\n", rc? "" : "No ");
+ #else
+     printf("  16-bit support only\n");
+     (void)pcre16_config(PCRE_CONFIG_UTF16, &rc);
+     printf("  %sUTF-16 support\n", rc? "" : "No ");
+ #endif
      (void)pcre_config(PCRE_CONFIG_UNICODE_PROPERTIES, &rc);
      printf("  %sUnicode properties support\n", rc? "" : "No ");
+     (void)pcre_config(PCRE_CONFIG_JIT, &rc);
+     if (rc)
+       printf("  Just-in-time compiler support\n");
+     else
+       printf("  No just-in-time compiler support\n");
      (void)pcre_config(PCRE_CONFIG_NEWLINE, &rc);
-     printf("  Newline sequence is %s\n", (rc == '\r')? "CR" :
+     /* Note that these values are always the ASCII values, even
-       (rc == '\n')? "LF" : (rc == ('\r'<<8 | '\n'))? "CRLF" :
+     in EBCDIC environments. CR is 13 and NL is 10. */
+     printf("  Newline sequence is %s\n", (rc == 13)? "CR" :
+       (rc == 10)? "LF" : (rc == (13<<8 | 10))? "CRLF" :
        (rc == -2)? "ANYCRLF" :
        (rc == -1)? "ANY" : "???");
      (void)pcre_config(PCRE_CONFIG_BSR, &rc);
-Line 1010 
 if (argc > 2)
+Line 2014 
 if (argc > 2)
  /* Set alternative malloc function */
+ #ifdef SUPPORT_PCRE8
  pcre_malloc = new_malloc;
  pcre_free = new_free;
  pcre_stack_malloc = stack_malloc;
  pcre_stack_free = stack_free;
+ #endif
+ #ifdef SUPPORT_PCRE16
+ pcre16_malloc = new_malloc;
+ pcre16_free = new_free;
+ pcre16_stack_malloc = stack_malloc;
+ pcre16_stack_free = stack_free;
+ #endif
  /* Heading line unless quiet, then prompt for first regex if stdin */
-Line 1032 
 while (!done)
+Line 2045 
 while (!done)
  #endif
    const char *error;
-   unsigned char *p, *pp, *ppp;
+   pcre_uint8 *markptr;
-   unsigned char *to_file = NULL;
+   pcre_uint8 *p, *pp, *ppp;
-   const unsigned char *tables = NULL;
+   pcre_uint8 *to_file = NULL;
+   const pcre_uint8 *tables = NULL;
    unsigned long int true_size, true_study_size = 0;
    size_t size, regex_gotten_store;
+   int do_allcaps = 0;
+   int do_mark = 0;
    int do_study = 0;
+   int no_force_study = 0;
    int do_debug = debug;
    int do_G = 0;
    int do_g = 0;
    int do_showinfo = showinfo;
    int do_showrest = 0;
+   int do_showcaprest = 0;
    int do_flip = 0;
    int erroroffset, len, delimiter, poffset;
-   use_utf8 = 0;
+   use_utf = 0;
    debug_lengths = 1;
    if (extend_inputline(infile, buffer, "  re> ") == NULL) break;
-Line 1062 
 while (!done)
+Line 2080 
 while (!done)
    if (*p == '<' && strchr((char *)(p+1), '<') == NULL)
      {
      unsigned long int magic, get_options;
-     uschar sbuf[8];
+     pcre_uint8 sbuf[8];
      FILE *f;
      p++;
-Line 1085 
 while (!done)
+Line 2103 
 while (!done)
        (sbuf[4] << 24) | (sbuf[5] << 16) | (sbuf[6] << 8) | sbuf[7];
      re = (real_pcre *)new_malloc(true_size);
-     regex_gotten_store = gotten_store;
+     regex_gotten_store = first_gotten_store;
      if (fread(re, 1, true_size, f) != true_size) goto FAIL_READ;
      magic = ((real_pcre *)re)->magic_number;
      if (magic != MAGIC_NUMBER)
        {
-       if (byteflip(magic, sizeof(magic)) == MAGIC_NUMBER)
+       if (swap_uint32(magic) == MAGIC_NUMBER)
          {
          do_flip = 1;
          }
-Line 1104 
 while (!done)
+Line 2122 
 while (!done)
          }
        }
-     fprintf(outfile, "Compiled regex%s loaded from %s\n",
+     fprintf(outfile, "Compiled pattern%s loaded from %s\n",
        do_flip? " (byte-inverted)" : "", p);
-     /* Need to know if UTF-8 for printing data strings */
+     /* Now see if there is any following study data. */
-     new_info(re, NULL, PCRE_INFO_OPTIONS, &get_options);
-     use_utf8 = (get_options & PCRE_UTF8) != 0;
-     /* Now see if there is any following study data */
      if (true_study_size != 0)
        {
-Line 1128 
 while (!done)
+Line 2141 
 while (!done)
          {
          FAIL_READ:
          fprintf(outfile, "Failed to read data from %s\n", p);
-         if (extra != NULL) new_free(extra);
+         if (extra != NULL)
+           {
+           PCRE_FREE_STUDY(extra);
+           }
          if (re != NULL) new_free(re);
          fclose(f);
          continue;
-Line 1138 
 while (!done)
+Line 2154 
 while (!done)
        }
      else fprintf(outfile, "No study data\n");
+     /* Flip the necessary bytes. */
+     if (do_flip)
+       {
+       PCRE_PATTERN_TO_HOST_BYTE_ORDER(re, extra, NULL);
+       }
+     /* Need to know if UTF-8 for printing data strings */
+     new_info(re, NULL, PCRE_INFO_OPTIONS, &get_options);
+     use_utf = (get_options & PCRE_UTF8) != 0;
      fclose(f);
      goto SHOW_INFO;
      }
    /* In-line pattern (the usual case). Get the delimiter and seek the end of
-   the pattern; if is isn't complete, read more. */
+   the pattern; if it isn't complete, read more. */
    delimiter = *p++;
-Line 1154 
 while (!done)
+Line 2181 
 while (!done)
      }
    pp = p;
-   poffset = p - buffer;
+   poffset = (int)(p - buffer);
    for(;;)
      {
-Line 1208 
 while (!done)
+Line 2235 
 while (!done)
        case 's': options |= PCRE_DOTALL; break;
        case 'x': options |= PCRE_EXTENDED; break;
-       case '+': do_showrest = 1; break;
+       case '+':
+       if (do_showrest) do_showcaprest = 1; else do_showrest = 1;
+       break;
+       case '=': do_allcaps = 1; break;
        case 'A': options |= PCRE_ANCHORED; break;
        case 'B': do_debug = 1; break;
        case 'C': options |= PCRE_AUTO_CALLOUT; break;
-Line 1218 
 while (!done)
+Line 2249 
 while (!done)
        case 'G': do_G = 1; break;
        case 'I': do_showinfo = 1; break;
        case 'J': options |= PCRE_DUPNAMES; break;
+       case 'K': do_mark = 1; break;
        case 'M': log_store = 1; break;
        case 'N': options |= PCRE_NO_AUTO_CAPTURE; break;
-Line 1225 
 while (!done)
+Line 2257 
 while (!done)
        case 'P': do_posix = 1; break;
  #endif
-       case 'S': do_study = 1; break;
+       case 'S':
+       if (do_study == 0)
+         {
+         do_study = 1;
+         if (*pp == '+')
+           {
+           study_options |= PCRE_STUDY_JIT_COMPILE;
+           pp++;
+           }
+         }
+       else
+         {
+         do_study = 0;
+         no_force_study = 1;
+         }
+       break;
        case 'U': options |= PCRE_UNGREEDY; break;
+       case 'W': options |= PCRE_UCP; break;
        case 'X': options |= PCRE_EXTRA; break;
+       case 'Y': options |= PCRE_NO_START_OPTIMISE; break;
        case 'Z': debug_lengths = 0; break;
-       case '8': options |= PCRE_UTF8; use_utf8 = 1; break;
+       case '8': options |= PCRE_UTF8; use_utf = 1; break;
        case '?': options |= PCRE_NO_UTF8_CHECK; break;
+       case 'T':
+       switch (*pp++)
+         {
+         case '0': tables = tables0; break;
+         case '1': tables = tables1; break;
+         case '\r':
+         case '\n':
+         case ' ':
+         case 0:
+         fprintf(outfile, "** Missing table number after /T\n");
+         goto SKIP_DATA;
+         default:
+         fprintf(outfile, "** Bad table number \"%c\" after /T\n", pp[-1]);
+         goto SKIP_DATA;
+         }
+       break;
        case 'L':
        ppp = pp;
        /* The '\r' test here is so that it works on Windows. */
-Line 1257 
 while (!done)
+Line 2326 
 while (!done)
        case '<':
          {
-         if (strncmp((char *)pp, "JS>", 3) == 0)
+         if (strncmpic(pp, (pcre_uint8 *)"JS>", 3) == 0)
            {
            options |= PCRE_JAVASCRIPT_COMPAT;
            pp += 3;
-Line 1285 
 while (!done)
+Line 2354 
 while (!done)
    /* Handle compiling via the POSIX interface, which doesn't support the
    timing, showing, or debugging options, nor the ability to pass over
-   local character tables. */
+   local character tables. Neither does it have 16-bit support. */
  #if !defined NOPOSIX
    if (posix || do_posix)
-Line 1298 
 while (!done)
+Line 2367 
 while (!done)
      if ((options & PCRE_DOTALL) != 0) cflags |= REG_DOTALL;
      if ((options & PCRE_NO_AUTO_CAPTURE) != 0) cflags |= REG_NOSUB;
      if ((options & PCRE_UTF8) != 0) cflags |= REG_UTF8;
+     if ((options & PCRE_UCP) != 0) cflags |= REG_UCP;
+     if ((options & PCRE_UNGREEDY) != 0) cflags |= REG_UNGREEDY;
+     first_gotten_store = 0;
      rc = regcomp(&preg, (char *)p, cflags);
      /* Compilation failed; go back for another re, skipping to blank line
-Line 1318 
 while (!done)
+Line 2390 
 while (!done)
  #endif  /* !defined NOPOSIX */
      {
+     unsigned long int get_options;
+     /* In 16-bit mode, convert the input. */
+ #ifdef SUPPORT_PCRE16
+     if (use_pcre16)
+       {
+       if (to16(p, options & PCRE_UTF8, (int)strlen((char *)p)) < 0)
+         {
+         fprintf(outfile, "**Failed: invalid UTF-8 string cannot be "
+           "converted to UTF-16\n");
+         goto SKIP_DATA;
+         }
+       p = (pcre_uint8 *)buffer16;
+       }
+ #endif
+     /* Compile many times when timing */
      if (timeit > 0)
        {
        register int i;
-Line 1325 
 while (!done)
+Line 2416 
 while (!done)
        clock_t start_time = clock();
        for (i = 0; i < timeit; i++)
          {
-         re = pcre_compile((char *)p, options, &error, &erroroffset, tables);
+         PCRE_COMPILE(re, p, options, &error, &erroroffset, tables);
          if (re != NULL) free(re);
          }
        time_taken = clock() - start_time;
-Line 1334 
 while (!done)
+Line 2425 
 while (!done)
            (double)CLOCKS_PER_SEC);
        }
-     re = pcre_compile((char *)p, options, &error, &erroroffset, tables);
+     first_gotten_store = 0;
+     PCRE_COMPILE(re, p, options, &error, &erroroffset, tables);
      /* Compilation failed; go back for another re, skipping to blank line
      if non-interactive. */
-Line 1361 
 while (!done)
+Line 2453 
 while (!done)
        goto CONTINUE;
        }
-     /* Compilation succeeded; print data if required. There are now two
+     /* Compilation succeeded. It is now possible to set the UTF-8 option from
-     info-returning functions. The old one has a limited interface and
+     within the regex; check for this so that we know how to process the data
-     returns only limited data. Check that it agrees with the newer one. */
+     lines. */
-     if (log_store)
+     new_info(re, NULL, PCRE_INFO_OPTIONS, &get_options);
-       fprintf(outfile, "Memory allocation (code space): %d\n",
+     if ((get_options & PCRE_UTF8) != 0) use_utf = 1;
-         (int)(gotten_store -
-               sizeof(real_pcre) -
-               ((real_pcre *)re)->name_count * ((real_pcre *)re)->name_entry_size));
      /* Extract the size for possible writing before possibly flipping it,
      and remember the store that was got. */
      true_size = ((real_pcre *)re)->size;
-     regex_gotten_store = gotten_store;
+     regex_gotten_store = first_gotten_store;
+     /* Output code size information if requested */
+     if (log_store)
+       fprintf(outfile, "Memory allocation (code space): %d\n",
+         (int)(first_gotten_store -
+               sizeof(real_pcre) -
+               ((real_pcre *)re)->name_count * ((real_pcre *)re)->name_entry_size));
-     /* If /S was present, study the regexp to generate additional info to
+     /* If -s or /S was present, study the regex to generate additional info to
-     help with the matching. */
+     help with the matching, unless the pattern has the SS option, which
+     suppresses the effect of /S (used for a few test patterns where studying is
+     never sensible). */
-     if (do_study)
+     if (do_study || (force_study >= 0 && !no_force_study))
        {
        if (timeit > 0)
          {
-Line 1388 
 while (!done)
+Line 2487 
 while (!done)
          clock_t time_taken;
          clock_t start_time = clock();
          for (i = 0; i < timeit; i++)
-           extra = pcre_study(re, study_options, &error);
+           {
+           PCRE_STUDY(extra, re, study_options | force_study_options, &error);
+           }
          time_taken = clock() - start_time;
-         if (extra != NULL) free(extra);
+         if (extra != NULL)
+           {
+           PCRE_FREE_STUDY(extra);
+           }
          fprintf(outfile, "  Study time %.4f milliseconds\n",
            (((double)time_taken * 1000.0) / (double)timeit) /
              (double)CLOCKS_PER_SEC);
          }
-       extra = pcre_study(re, study_options, &error);
+       PCRE_STUDY(extra, re, study_options | force_study_options, &error);
        if (error != NULL)
          fprintf(outfile, "Failed to study: %s\n", error);
        else if (extra != NULL)
+         {
          true_study_size = ((pcre_study_data *)(extra->study_data))->size;
+         if (log_store)
+           {
+           size_t jitsize;
+           new_info(re, extra, PCRE_INFO_JITSIZE, &jitsize);
+           if (jitsize != 0)
+             fprintf(outfile, "Memory allocation (JIT code): %d\n", jitsize);
+           }
+         }
        }
-     /* If the 'F' option was present, we flip the bytes of all the integer
+     /* If /K was present, we set up for handling MARK data. */
-     fields in the regex data block and the study block. This is to make it
-     possible to test PCRE's handling of byte-flipped patterns, e.g. those
-     compiled on a different architecture. */
-     if (do_flip)
+     if (do_mark)
        {
-       real_pcre *rre = (real_pcre *)re;
+       if (extra == NULL)
-       rre->magic_number =
-         byteflip(rre->magic_number, sizeof(rre->magic_number));
-       rre->size = byteflip(rre->size, sizeof(rre->size));
-       rre->options = byteflip(rre->options, sizeof(rre->options));
-       rre->flags = (pcre_uint16)byteflip(rre->flags, sizeof(rre->flags));
-       rre->top_bracket =
-         (pcre_uint16)byteflip(rre->top_bracket, sizeof(rre->top_bracket));
-       rre->top_backref =
-         (pcre_uint16)byteflip(rre->top_backref, sizeof(rre->top_backref));
-       rre->first_byte =
-         (pcre_uint16)byteflip(rre->first_byte, sizeof(rre->first_byte));
-       rre->req_byte =
-         (pcre_uint16)byteflip(rre->req_byte, sizeof(rre->req_byte));
-       rre->name_table_offset = (pcre_uint16)byteflip(rre->name_table_offset,
-         sizeof(rre->name_table_offset));
-       rre->name_entry_size = (pcre_uint16)byteflip(rre->name_entry_size,
-         sizeof(rre->name_entry_size));
-       rre->name_count = (pcre_uint16)byteflip(rre->name_count,
-         sizeof(rre->name_count));
-       if (extra != NULL)
          {
-         pcre_study_data *rsd = (pcre_study_data *)(extra->study_data);
+         extra = (pcre_extra *)malloc(sizeof(pcre_extra));
-         rsd->size = byteflip(rsd->size, sizeof(rsd->size));
+         extra->flags = 0;
-         rsd->options = byteflip(rsd->options, sizeof(rsd->options));
          }
+       extra->mark = &markptr;
+       extra->flags |= PCRE_EXTRA_MARK;
        }
-     /* Extract information from the compiled data if required */
+     /* Extract and display information from the compiled data if required. */
      SHOW_INFO:
      if (do_debug)
        {
        fprintf(outfile, "------------------------------------------------------------------\n");
+ #if defined SUPPORT_PCRE8 && defined SUPPORT_PCRE16
+       if (use_pcre16)
+         pcre16_printint(re, outfile, debug_lengths);
+       else
+         pcre_printint(re, outfile, debug_lengths);
+ #elif defined SUPPORT_PCRE8
        pcre_printint(re, outfile, debug_lengths);
+ #else
+       pcre16_printint(re, outfile, debug_lengths);
+ #endif
        }
+     /* We already have the options in get_options (see above) */
      if (do_showinfo)
        {
-       unsigned long int get_options, all_options;
+       unsigned long int all_options;
  #if !defined NOINFOCHECK
        int old_first_char, old_options, old_count;
  #endif
        int count, backrefmax, first_char, need_char, okpartial, jchanged,
          hascrorlf;
        int nameentrysize, namecount;
-       const uschar *nametable;
+       const pcre_uchar *nametable;
-       new_info(re, NULL, PCRE_INFO_OPTIONS, &get_options);
        new_info(re, NULL, PCRE_INFO_SIZE, &size);
        new_info(re, NULL, PCRE_INFO_CAPTURECOUNT, &count);
        new_info(re, NULL, PCRE_INFO_BACKREFMAX, &backrefmax);
-Line 1472 
 while (!done)
+Line 2572 
 while (!done)
        new_info(re, NULL, PCRE_INFO_JCHANGED, &jchanged);
        new_info(re, NULL, PCRE_INFO_HASCRORLF, &hascrorlf);
+       /* The old, obsolete function pcre_info() works only in 8-bit mode. Check
+       that it gives the same results as the new function. */
  #if !defined NOINFOCHECK
-       old_count = pcre_info(re, &old_options, &old_first_char);
+       if (!use_pcre16)
-       if (count < 0) fprintf(outfile,
-         "Error %d from pcre_info()\n", count);
-       else
          {
-         if (old_count != count) fprintf(outfile,
+         old_count = pcre_info(re, &old_options, &old_first_char);
-           "Count disagreement: pcre_fullinfo=%d pcre_info=%d\n", count,
+         if (count < 0) fprintf(outfile,
-             old_count);
+           "Error %d from pcre_info()\n", count);
+         else
-         if (old_first_char != first_char) fprintf(outfile,
+           {
-           "First char disagreement: pcre_fullinfo=%d pcre_info=%d\n",
+           if (old_count != count) fprintf(outfile,
-             first_char, old_first_char);
+             "Count disagreement: pcre_fullinfo=%d pcre_info=%d\n", count,
+               old_count);
-         if (old_options != (int)get_options) fprintf(outfile,
-           "Options disagreement: pcre_fullinfo=%ld pcre_info=%d\n",
+           if (old_first_char != first_char) fprintf(outfile,
-             get_options, old_options);
+             "First char disagreement: pcre_fullinfo=%d pcre_info=%d\n",
+               first_char, old_first_char);
+           if (old_options != (int)get_options) fprintf(outfile,
+             "Options disagreement: pcre_fullinfo=%ld pcre_info=%d\n",
+               get_options, old_options);
+           }
          }
  #endif
-Line 1516 
 while (!done)
+Line 2622 
 while (!done)
        if (hascrorlf) fprintf(outfile, "Contains explicit CR or LF match\n");
        all_options = ((real_pcre *)re)->options;
-       if (do_flip) all_options = byteflip(all_options, sizeof(all_options));
+       if (do_flip) all_options = swap_uint32(all_options);
        if (get_options == 0) fprintf(outfile, "No options\n");
-         else fprintf(outfile, "Options:%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s\n",
+         else fprintf(outfile, "Options:%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s\n",
            ((get_options & PCRE_ANCHORED) != 0)? " anchored" : "",
            ((get_options & PCRE_CASELESS) != 0)? " caseless" : "",
            ((get_options & PCRE_EXTENDED) != 0)? " extended" : "",
-Line 1532 
 while (!done)
+Line 2638 
 while (!done)
            ((get_options & PCRE_EXTRA) != 0)? " extra" : "",
            ((get_options & PCRE_UNGREEDY) != 0)? " ungreedy" : "",
            ((get_options & PCRE_NO_AUTO_CAPTURE) != 0)? " no_auto_capture" : "",
-           ((get_options & PCRE_UTF8) != 0)? " utf8" : "",
+           ((get_options & PCRE_UTF8) != 0)? " utf" : "",
-           ((get_options & PCRE_NO_UTF8_CHECK) != 0)? " no_utf8_check" : "",
+           ((get_options & PCRE_UCP) != 0)? " ucp" : "",
+           ((get_options & PCRE_NO_UTF8_CHECK) != 0)? " no_utf_check" : "",
+           ((get_options & PCRE_NO_START_OPTIMIZE) != 0)? " no_start_optimize" : "",
            ((get_options & PCRE_DUPNAMES) != 0)? " dupnames" : "");
        if (jchanged) fprintf(outfile, "Duplicate name status changes\n");
-Line 1574 
 while (!done)
+Line 2682 
 while (!done)
          }
        else
          {
-         int ch = first_char & 255;
+         const char *caseless =
-         const char *caseless = ((first_char & REQ_CASELESS) == 0)?
+           ((((real_pcre *)re)->flags & PCRE_FCH_CASELESS) == 0)?
            "" : " (caseless)";
-         if (PRINTHEX(ch))
-           fprintf(outfile, "First char = \'%c\'%s\n", ch, caseless);
+         if (PRINTOK(first_char))
+           fprintf(outfile, "First char = \'%c\'%s\n", first_char, caseless);
          else
-           fprintf(outfile, "First char = %d%s\n", ch, caseless);
+           {
+           fprintf(outfile, "First char = ");
+           pchar(first_char, outfile);
+           fprintf(outfile, "%s\n", caseless);
+           }
          }
        if (need_char < 0)
-Line 1589 
 while (!done)
+Line 2702 
 while (!done)
          }
        else
          {
-         int ch = need_char & 255;
+         const char *caseless =
-         const char *caseless = ((need_char & REQ_CASELESS) == 0)?
+           ((((real_pcre *)re)->flags & PCRE_RCH_CASELESS) == 0)?
            "" : " (caseless)";
-         if (PRINTHEX(ch))
-           fprintf(outfile, "Need char = \'%c\'%s\n", ch, caseless);
+         if (PRINTOK(need_char))
+           fprintf(outfile, "Need char = \'%c\'%s\n", need_char, caseless);
          else
-           fprintf(outfile, "Need char = %d%s\n", ch, caseless);
+           {
+           fprintf(outfile, "Need char = ");
+           pchar(need_char, outfile);
+           fprintf(outfile, "%s\n", caseless);
+           }
          }
        /* Don't output study size; at present it is in any case a fixed
        value, but it varies, depending on the computer architecture, and
        so messes up the test suite. (And with the /F option, it might be
-       flipped.) */
+       flipped.) If study was forced by an external -s, don't show this
+       information unless -i or -d was also present. This means that, except
+       when auto-callouts are involved, the output from runs with and without
+       -s should be identical. */
-       if (do_study)
+       if (do_study || (force_study >= 0 && showinfo && !no_force_study))
          {
          if (extra == NULL)
            fprintf(outfile, "Study returned NULL\n");
          else
            {
-           uschar *start_bits = NULL;
+           pcre_uint8 *start_bits = NULL;
-           new_info(re, extra, PCRE_INFO_FIRSTTABLE, &start_bits);
+           int minlength;
+           new_info(re, extra, PCRE_INFO_MINLENGTH, &minlength);
+           fprintf(outfile, "Subject length lower bound = %d\n", minlength);
+           new_info(re, extra, PCRE_INFO_FIRSTTABLE, &start_bits);
            if (start_bits == NULL)
-             fprintf(outfile, "No starting byte set\n");
+             fprintf(outfile, "No set of starting bytes\n");
            else
              {
              int i;
-Line 1628 
 while (!done)
+Line 2753 
 while (!done)
                    fprintf(outfile, "\n  ");
                    c = 2;
                    }
-                 if (PRINTHEX(i) && i != ' ')
+                 if (PRINTOK(i) && i != ' ')
                    {
                    fprintf(outfile, "%c ", i);
                    c += 2;
-Line 1643 
 while (!done)
+Line 2768 
 while (!done)
              fprintf(outfile, "\n");
              }
            }
+         /* Show this only if the JIT was set by /S, not by -s. */
+         if ((study_options & PCRE_STUDY_JIT_COMPILE) != 0)
+           {
+           int jit;
+           new_info(re, extra, PCRE_INFO_JIT, &jit);
+           if (jit)
+             fprintf(outfile, "JIT study was successful\n");
+           else
+ #ifdef SUPPORT_JIT
+             fprintf(outfile, "JIT study was not successful\n");
+ #else
+             fprintf(outfile, "JIT support is not available in this version of PCRE\n");
+ #endif
+           }
          }
        }
-Line 1659 
 while (!done)
+Line 2800 
 while (!done)
          }
        else
          {
-         uschar sbuf[8];
+         pcre_uint8 sbuf[8];
-         sbuf[0] = (uschar)((true_size >> 24) & 255);
-         sbuf[1] = (uschar)((true_size >> 16) & 255);
+         if (do_flip) regexflip(re, extra);
-         sbuf[2] = (uschar)((true_size >>  8) & 255);
+         sbuf[0] = (pcre_uint8)((true_size >> 24) & 255);
-         sbuf[3] = (uschar)((true_size) & 255);
+         sbuf[1] = (pcre_uint8)((true_size >> 16) & 255);
+         sbuf[2] = (pcre_uint8)((true_size >>  8) & 255);
-         sbuf[4] = (uschar)((true_study_size >> 24) & 255);
+         sbuf[3] = (pcre_uint8)((true_size) & 255);
-         sbuf[5] = (uschar)((true_study_size >> 16) & 255);
+         sbuf[4] = (pcre_uint8)((true_study_size >> 24) & 255);
-         sbuf[6] = (uschar)((true_study_size >>  8) & 255);
+         sbuf[5] = (pcre_uint8)((true_study_size >> 16) & 255);
-         sbuf[7] = (uschar)((true_study_size) & 255);
+         sbuf[6] = (pcre_uint8)((true_study_size >>  8) & 255);
+         sbuf[7] = (pcre_uint8)((true_study_size) & 255);
          if (fwrite(sbuf, 1, 8, f) < 8 ||
              fwrite(re, 1, true_size, f) < true_size)
-Line 1677 
 while (!done)
+Line 2819 
 while (!done)
            }
          else
            {
-           fprintf(outfile, "Compiled regex written to %s\n", to_file);
+           fprintf(outfile, "Compiled pattern written to %s\n", to_file);
+           /* If there is study data, write it. */
            if (extra != NULL)
              {
              if (fwrite(extra->study_data, 1, true_study_size, f) <
-Line 1687 
 while (!done)
+Line 2832 
 while (!done)
                  strerror(errno));
                }
              else fprintf(outfile, "Study data written to %s\n", to_file);
              }
            }
          fclose(f);
          }
        new_free(re);
-       if (extra != NULL) new_free(extra);
+       if (extra != NULL)
-       if (tables != NULL) new_free((void *)tables);
+         {
+         PCRE_FREE_STUDY(extra);
+         }
+       if (locale_set)
+         {
+         new_free((void *)tables);
+         setlocale(LC_CTYPE, "C");
+         locale_set = 0;
+         }
        continue;  /* With next regex */
        }
      }        /* End of non-POSIX compile */
-Line 1704 
 while (!done)
+Line 2856 
 while (!done)
    for (;;)
      {
-     uschar *q;
+     pcre_uint8 *q;
-     uschar *bptr;
+     pcre_uint8 *bptr;
      int *use_offsets = offsets;
      int use_size_offsets = size_offsets;
      int callout_data = 0;
-Line 1717 
 while (!done)
+Line 2869 
 while (!done)
      int getlist = 0;
      int gmatched = 0;
      int start_offset = 0;
+     int start_offset_sign = 1;
      int g_notempty = 0;
      int use_dfa = 0;
-Line 1730 
 while (!done)
+Line 2883 
 while (!done)
      pcre_callout = callout;
      first_callout = 1;
+     last_callout_mark = NULL;
      callout_extra = 0;
      callout_count = 0;
      callout_fail_count = 999999;
-Line 1744 
 while (!done)
+Line 2898 
 while (!done)
        {
        if (extend_inputline(infile, buffer + len, "data> ") == NULL)
          {
-         if (len > 0) break;
+         if (len > 0)    /* Reached EOF without hitting a newline */
+           {
+           fprintf(outfile, "\n");
+           break;
+           }
          done = 1;
          goto CONTINUE;
          }
-Line 1784 
 while (!done)
+Line 2942 
 while (!done)
            c = c * 8 + *p++ - '0';
  #if !defined NOUTF8
-         if (use_utf8 && c > 255)
+         if (use_utf && c > 255)
            {
-           unsigned char buff8[8];
+           pcre_uint8 buff8[8];
            int ii, utn;
            utn = ord2utf8(c, buff8);
            for (ii = 0; ii < utn - 1; ii++) *q++ = buff8[ii];
-Line 1802 
 while (!done)
+Line 2960 
 while (!done)
  #if !defined NOUTF8
          if (*p == '{')
            {
-           unsigned char *pt = p;
+           pcre_uint8 *pt = p;
            c = 0;
-           while (isxdigit(*(++pt)))
-             c = c * 16 + tolower(*pt) - ((isdigit(*pt))? '0' : 'W');
+           /* We used to have "while (isxdigit(*(++pt)))" here, but it fails
+           when isxdigit() is a macro that refers to its argument more than
+           once. This is banned by the C Standard, but apparently happens in at
+           least one MacOS environment. */
+           for (pt++; isxdigit(*pt); pt++)
+             c = c * 16 + tolower(*pt) - ((isdigit(*pt))? '0' : 'a' - 10);
            if (*pt == '}')
              {
-             unsigned char buff8[8];
+             pcre_uint8 buff8[8];
              int ii, utn;
-             if (use_utf8)
+             if (use_utf)
                {
                utn = ord2utf8(c, buff8);
                for (ii = 0; ii < utn - 1; ii++) *q++ = buff8[ii];
-Line 1819 
 while (!done)
+Line 2983 
 while (!done)
              else
               {
               if (c > 255)
-                fprintf(outfile, "** Character \\x{%x} is greater than 255 and "
+                {
-                  "UTF-8 mode is not enabled.\n"
+                if (use_pcre16)
-                  "** Truncation will probably give the wrong result.\n", c);
+                  fprintf(outfile, "** Character \\x{%x} is greater than 255.\n"
+                    "** Because its input is first processed as 8-bit, pcretest "
+                    "does not\n** support such characters in 16-bit mode when "
+                    "UTF-16 is not set.\n", c);
+                else
+                  fprintf(outfile, "** Character \\x{%x} is greater than 255 "
+                    "and UTF-8 mode is not enabled.\n", c);
+                fprintf(outfile, "** Truncation will probably give the wrong "
+                  "result.\n");
+                }
               }
              p = pt + 1;
              break;
-Line 1835 
 while (!done)
+Line 3009 
 while (!done)
          c = 0;
          while (i++ < 2 && isxdigit(*p))
            {
-           c = c * 16 + tolower(*p) - ((isdigit(*p))? '0' : 'W');
+           c = c * 16 + tolower(*p) - ((isdigit(*p))? '0' : 'a' - 10);
            p++;
            }
          break;
-Line 1845 
 while (!done)
+Line 3019 
 while (!done)
          continue;
          case '>':
+         if (*p == '-')
+           {
+           start_offset_sign = -1;
+           p++;
+           }
          while(isdigit(*p)) start_offset = start_offset * 10 + *p++ - '0';
+         start_offset *= start_offset_sign;
          continue;
          case 'A':  /* Option setting */
-Line 1864 
 while (!done)
+Line 3044 
 while (!done)
            }
          else if (isalnum(*p))
            {
-           uschar *npp = copynamesptr;
+           pcre_uchar *npp = copynamesptr;
            while (isalnum(*p)) *npp++ = *p++;
            *npp++ = 0;
            *npp = 0;
-Line 1918 
 while (!done)
+Line 3098 
 while (!done)
  #endif
            use_dfa = 1;
          continue;
+ #endif
+ #if !defined NODFA
          case 'F':
          options |= PCRE_DFA_SHORTEST;
          continue;
-Line 1932 
 while (!done)
+Line 3114 
 while (!done)
            }
          else if (isalnum(*p))
            {
-           uschar *npp = getnamesptr;
+           pcre_uchar *npp = getnamesptr;
            while (isalnum(*p)) *npp++ = *p++;
            *npp++ = 0;
            *npp = 0;
-Line 1943 
 while (!done)
+Line 3125 
 while (!done)
            }
          continue;
+         case 'J':
+         while(isdigit(*p)) n = n * 10 + *p++ - '0';
+         if (extra != NULL
+             && (extra->flags & PCRE_EXTRA_EXECUTABLE_JIT) != 0
+             && extra->executable_jit != NULL)
+           {
+           if (jit_stack != NULL) pcre_jit_stack_free(jit_stack);
+           jit_stack = pcre_jit_stack_alloc(1, n * 1024);
+           pcre_assign_jit_stack(extra, jit_callback, jit_stack);
+           }
+         continue;
          case 'L':
          getlist = 1;
          continue;
-Line 1952 
 while (!done)
+Line 3146 
 while (!done)
          continue;
          case 'N':
-         options |= PCRE_NOTEMPTY;
+         if ((options & PCRE_NOTEMPTY) != 0)
+           options = (options & ~PCRE_NOTEMPTY) | PCRE_NOTEMPTY_ATSTART;
+         else
+           options |= PCRE_NOTEMPTY;
          continue;
          case 'O':
-Line 1975 
 while (!done)
+Line 3172 
 while (!done)
          continue;
          case 'P':
-         options |= PCRE_PARTIAL;
+         options |= ((options & PCRE_PARTIAL_SOFT) == 0)?
+           PCRE_PARTIAL_SOFT : PCRE_PARTIAL_HARD;
          continue;
          case 'Q':
-Line 2009 
 while (!done)
+Line 3207 
 while (!done)
          case 'S':
          show_malloc = 1;
          continue;
          case 'Y':
          options |= PCRE_NO_START_OPTIMIZE;
          continue;
          case 'Z':
          options |= PCRE_NOTEOL;
-Line 2034 
 while (!done)
+Line 3232 
 while (!done)
        *q++ = c;
        }
      *q = 0;
-     len = q - dbuffer;
+     len = (int)(q - dbuffer);
      /* Move the data to the end of the buffer so that a read over the end of
      the buffer will be seen by valgrind, even if it doesn't cause a crash. If
-Line 2072 
 while (!done)
+Line 3270 
 while (!done)
          pmatch = (regmatch_t *)malloc(sizeof(regmatch_t) * use_size_offsets);
        if ((options & PCRE_NOTBOL) != 0) eflags |= REG_NOTBOL;
        if ((options & PCRE_NOTEOL) != 0) eflags |= REG_NOTEOL;
        if ((options & PCRE_NOTEMPTY) != 0) eflags |= REG_NOTEMPTY;
        rc = regexec(&preg, (const char *)bptr, use_size_offsets, pmatch, eflags);
-Line 2094 
 while (!done)
+Line 3292 
 while (!done)
            if (pmatch[i].rm_so >= 0)
              {
              fprintf(outfile, "%2d: ", (int)i);
-             (void)pchars(dbuffer + pmatch[i].rm_so,
+             PCHARSV(dbuffer + pmatch[i].rm_so,
                pmatch[i].rm_eo - pmatch[i].rm_so, outfile);
              fprintf(outfile, "\n");
-             if (i == 0 && do_showrest)
+             if (do_showcaprest || (i == 0 && do_showrest))
                {
-               fprintf(outfile, " 0+ ");
+               fprintf(outfile, "%2d+ ", (int)i);
-               (void)pchars(dbuffer + pmatch[i].rm_eo, len - pmatch[i].rm_eo,
+               PCHARSV(dbuffer + pmatch[i].rm_eo, len - pmatch[i].rm_eo,
                  outfile);
                fprintf(outfile, "\n");
                }
-Line 2108 
 while (!done)
+Line 3306 
 while (!done)
            }
          }
        free(pmatch);
+       goto NEXT_DATA;
        }
+ #endif  /* !defined NOPOSIX */
      /* Handle matching via the native interface - repeats for /g and /G */
-     else
+ #ifdef SUPPORT_PCRE16
- #endif  /* !defined NOPOSIX */
+     if (use_pcre16)
+       {
+       len = to16(bptr, (((real_pcre *)re)->options) & PCRE_UTF8, len);
+       if (len < 0)
+         {
+         fprintf(outfile, "**Failed: invalid UTF-8 string cannot be "
+           "converted to UTF-16\n");
+         goto NEXT_DATA;
+         }
+       bptr = (pcre_uint8 *)buffer16;
+       }
+ #endif
      for (;; gmatched++)    /* Loop for /g or /G */
        {
+       markptr = NULL;
        if (timeitm > 0)
          {
          register int i;
-Line 2128 
 while (!done)
+Line 3342 
 while (!done)
            {
            int workspace[1000];
            for (i = 0; i < timeitm; i++)
-             count = pcre_dfa_exec(re, NULL, (char *)bptr, len, start_offset,
+             {
-               options | g_notempty, use_offsets, use_size_offsets, workspace,
+             PCRE_DFA_EXEC(count, re, extra, bptr, len, start_offset,
-               sizeof(workspace)/sizeof(int));
+               (options | g_notempty), use_offsets, use_size_offsets, workspace,
+               (sizeof(workspace)/sizeof(int)));
+             }
            }
          else
  #endif
          for (i = 0; i < timeitm; i++)
-           count = pcre_exec(re, extra, (char *)bptr, len,
+           {
-             start_offset, options | g_notempty, use_offsets, use_size_offsets);
+           PCRE_EXEC(count, re, extra, bptr, len, start_offset,
+             (options | g_notempty), use_offsets, use_size_offsets);
+           }
          time_taken = clock() - start_time;
          fprintf(outfile, "Execute time %.4f milliseconds\n",
            (((double)time_taken * 1000.0) / (double)timeitm) /
-Line 2147 
 while (!done)
+Line 3364 
 while (!done)
        /* If find_match_limit is set, we want to do repeated matches with
        varying limits in order to find the minimum value for the match limit and
-       for the recursion limit. */
+       for the recursion limit. The match limits are relevant only to the normal
+       running of pcre_exec(), so disable the JIT optimization. This makes it
+       possible to run the same set of tests with and without JIT externally
+       requested. */
        if (find_match_limit)
          {
-Line 2156 
 while (!done)
+Line 3376 
 while (!done)
            extra = (pcre_extra *)malloc(sizeof(pcre_extra));
            extra->flags = 0;
            }
+         else extra->flags &= ~PCRE_EXTRA_EXECUTABLE_JIT;
          (void)check_match_limit(re, extra, bptr, len, start_offset,
            options|g_notempty, use_offsets, use_size_offsets,
-Line 2179 
 while (!done)
+Line 3400 
 while (!done)
            }
          extra->flags |= PCRE_EXTRA_CALLOUT_DATA;
          extra->callout_data = &callout_data;
-         count = pcre_exec(re, extra, (char *)bptr, len, start_offset,
+         PCRE_EXEC(count, re, extra, bptr, len, start_offset,
            options | g_notempty, use_offsets, use_size_offsets);
          extra->flags &= ~PCRE_EXTRA_CALLOUT_DATA;
          }
-Line 2191 
 while (!done)
+Line 3412 
 while (!done)
        else if (all_use_dfa || use_dfa)
          {
          int workspace[1000];
-         count = pcre_dfa_exec(re, NULL, (char *)bptr, len, start_offset,
+         PCRE_DFA_EXEC(count, re, extra, bptr, len, start_offset,
-           options | g_notempty, use_offsets, use_size_offsets, workspace,
+           (options | g_notempty), use_offsets, use_size_offsets, workspace,
-           sizeof(workspace)/sizeof(int));
+           (sizeof(workspace)/sizeof(int)));
          if (count == 0)
            {
            fprintf(outfile, "Matched, but too many subsidiary matches\n");
-Line 2204 
 while (!done)
+Line 3425 
 while (!done)
        else
          {
-         count = pcre_exec(re, extra, (char *)bptr, len,
+         PCRE_EXEC(count, re, extra, bptr, len, start_offset,
-           start_offset, options | g_notempty, use_offsets, use_size_offsets);
+           options | g_notempty, use_offsets, use_size_offsets);
          if (count == 0)
            {
            fprintf(outfile, "Matched, but too many substrings\n");
-Line 2239 
 while (!done)
+Line 3460 
 while (!done)
              }
            }
+         /* do_allcaps requests showing of all captures in the pattern, to check
+         unset ones at the end. */
+         if (do_allcaps)
+           {
+           new_info(re, NULL, PCRE_INFO_CAPTURECOUNT, &count);
+           count++;   /* Allow for full match */
+           if (count * 2 > use_size_offsets) count = use_size_offsets/2;
+           }
+         /* Output the captured substrings */
          for (i = 0; i < count * 2; i += 2)
            {
            if (use_offsets[i] < 0)
+             {
+             if (use_offsets[i] != -1)
+               fprintf(outfile, "ERROR: bad negative value %d for offset %d\n",
+                 use_offsets[i], i);
+             if (use_offsets[i+1] != -1)
+               fprintf(outfile, "ERROR: bad negative value %d for offset %d\n",
+                 use_offsets[i+1], i+1);
              fprintf(outfile, "%2d: <unset>\n", i/2);
+             }
            else
              {
              fprintf(outfile, "%2d: ", i/2);
-             (void)pchars(bptr + use_offsets[i],
+             PCHARSV(bptr + use_offsets[i],
                use_offsets[i+1] - use_offsets[i], outfile);
              fprintf(outfile, "\n");
-             if (i == 0)
+             if (do_showcaprest || (i == 0 && do_showrest))
                {
-               if (do_showrest)
+               fprintf(outfile, "%2d+ ", i/2);
-                 {
+               PCHARSV(bptr + use_offsets[i+1], len - use_offsets[i+1],
-                 fprintf(outfile, " 0+ ");
+                 outfile);
-                 (void)pchars(bptr + use_offsets[i+1], len - use_offsets[i+1],
+               fprintf(outfile, "\n");
-                   outfile);
-                 fprintf(outfile, "\n");
-                 }
                }
              }
            }
+         if (markptr != NULL)
+           {
+           int mplen;
+           if (use_pcre16)
+             {
+             pcre_uint16 *mp = (pcre_uint16 *)markptr;
+             mplen = 0;
+             while (*mp++ != 0) mplen++;
+             }
+           else mplen = (int)strlen((char *)markptr);
+           fprintf(outfile, "MK: ");
+           PCHARSV(markptr, mplen, outfile);
+           fprintf(outfile, "\n");
+           }
          for (i = 0; i < 32; i++)
            {
            if ((copystrings & (1 << i)) != 0)
-Line 2335 
 while (!done)
+Line 3588 
 while (!done)
                fprintf(outfile, "%2dL %s\n", i, stringlist[i]);
              if (stringlist[i] != NULL)
                fprintf(outfile, "string list not terminated by NULL\n");
-             /* free((void *)stringlist); */
              pcre_free_substring_list(stringlist);
              }
            }
-Line 2345 
 while (!done)
+Line 3597 
 while (!done)
        else if (count == PCRE_ERROR_PARTIAL)
          {
-         fprintf(outfile, "Partial match");
+         if (markptr == NULL) fprintf(outfile, "Partial match");
- #if !defined NODFA
+           else fprintf(outfile, "Partial match, mark=%s", markptr);
-         if ((all_use_dfa || use_dfa) && use_size_offsets > 2)
+         if (use_size_offsets > 1)
-           fprintf(outfile, ": %.*s", use_offsets[1] - use_offsets[0],
+           {
-             bptr + use_offsets[0]);
+           fprintf(outfile, ": ");
- #endif
+           PCHARSV(bptr + use_offsets[0], use_offsets[1] - use_offsets[0],
+             outfile);
+           }
          fprintf(outfile, "\n");
          break;  /* Out of the /g loop */
          }
-Line 2360 
 while (!done)
+Line 3614 
 while (!done)
        to advance the start offset, and continue. We won't be at the end of the
        string - that was checked before setting g_notempty.
-       Complication arises in the case when the newline option is "any" or
+       Complication arises in the case when the newline convention is "any",
-       "anycrlf". If the previous match was at the end of a line terminated by
+       "crlf", or "anycrlf". If the previous match was at the end of a line
-       CRLF, an advance of one character just passes the \r, whereas we should
+       terminated by CRLF, an advance of one character just passes the \r,
-       prefer the longer newline sequence, as does the code in pcre_exec().
+       whereas we should prefer the longer newline sequence, as does the code in
-       Fudge the offset value to achieve this.
+       pcre_exec(). Fudge the offset value to achieve this. We check for a
+       newline setting in the pattern; if none was set, use pcre_config() to
+       find the default.
        Otherwise, in the case of UTF-8 matching, the advance must be one
        character, not one byte. */
-Line 2380 
 while (!done)
+Line 3636 
 while (!done)
              {
              int d;
              (void)pcre_config(PCRE_CONFIG_NEWLINE, &d);
-             obits = (d == '\r')? PCRE_NEWLINE_CR :
+             /* Note that these values are always the ASCII ones, even in
-                     (d == '\n')? PCRE_NEWLINE_LF :
+             EBCDIC environments. CR = 13, NL = 10. */
-                     (d == ('\r'<<8 | '\n'))? PCRE_NEWLINE_CRLF :
+             obits = (d == 13)? PCRE_NEWLINE_CR :
+                     (d == 10)? PCRE_NEWLINE_LF :
+                     (d == (13<<8 | 10))? PCRE_NEWLINE_CRLF :
                      (d == -2)? PCRE_NEWLINE_ANYCRLF :
                      (d == -1)? PCRE_NEWLINE_ANY : 0;
              }
            if (((obits & PCRE_NEWLINE_BITS) == PCRE_NEWLINE_ANY ||
+                (obits & PCRE_NEWLINE_BITS) == PCRE_NEWLINE_CRLF ||
                 (obits & PCRE_NEWLINE_BITS) == PCRE_NEWLINE_ANYCRLF)
                &&
                start_offset < len - 1 &&
                bptr[start_offset] == '\r' &&
                bptr[start_offset+1] == '\n')
              onechar++;
-           else if (use_utf8)
+           else if (use_utf)
              {
              while (start_offset + onechar < len)
                {
-               int tb = bptr[start_offset+onechar];
+               if ((bptr[start_offset+onechar] & 0xc0) != 0x80) break;
-               if (tb <= 127) break;
+               onechar++;
-               tb &= 0xc0;
-               if (tb != 0 && tb != 0xc0) onechar++;
                }
              }
            use_offsets[1] = start_offset + onechar;
            }
          else
            {
-           if (count == PCRE_ERROR_NOMATCH)
+           switch(count)
              {
-             if (gmatched == 0) fprintf(outfile, "No match\n");
+             case PCRE_ERROR_NOMATCH:
+             if (gmatched == 0)
+               {
+               if (markptr == NULL) fprintf(outfile, "No match\n");
+                 else fprintf(outfile, "No match, mark = %s\n", markptr);
+               }
+             break;
+             case PCRE_ERROR_BADUTF8:
+             case PCRE_ERROR_SHORTUTF8:
+             fprintf(outfile, "Error %d (%s UTF-8 string)", count,
+               (count == PCRE_ERROR_BADUTF8)? "bad" : "short");
+             if (use_size_offsets >= 2)
+               fprintf(outfile, " offset=%d reason=%d", use_offsets[0],
+                 use_offsets[1]);
+             fprintf(outfile, "\n");
+             break;
+             default:
+             if (count < 0 && (-count) < sizeof(errtexts)/sizeof(const char *))
+               fprintf(outfile, "Error %d (%s)\n", count, errtexts[-count]);
+             else
+               fprintf(outfile, "Error %d (Unexpected value)\n", count);
+             break;
              }
-           else fprintf(outfile, "Error %d\n", count);
            break;  /* Out of the /g loop */
            }
          }
-Line 2421 
 while (!done)
+Line 3701 
 while (!done)
        if (!do_g && !do_G) break;
        /* If we have matched an empty string, first check to see if we are at
-       the end of the subject. If so, the /g loop is over. Otherwise, mimic
+       the end of the subject. If so, the /g loop is over. Otherwise, mimic what
-       what Perl's /g options does. This turns out to be rather cunning. First
+       Perl's /g options does. This turns out to be rather cunning. First we set
-       we set PCRE_NOTEMPTY and PCRE_ANCHORED and try the match again at the
+       PCRE_NOTEMPTY_ATSTART and PCRE_ANCHORED and try the match again at the
        same point. If this fails (picked up above) we advance to the next
        character. */
-Line 2432 
 while (!done)
+Line 3712 
 while (!done)
        if (use_offsets[0] == use_offsets[1])
          {
          if (use_offsets[0] == len) break;
-         g_notempty = PCRE_NOTEMPTY | PCRE_ANCHORED;
+         g_notempty = PCRE_NOTEMPTY_ATSTART | PCRE_ANCHORED;
          }
        /* For /g, update the start offset, leaving the rest alone */
-Line 2458 
 while (!done)
+Line 3738 
 while (!done)
  #endif
    if (re != NULL) new_free(re);
-   if (extra != NULL) new_free(extra);
+   if (extra != NULL)
-   if (tables != NULL)
+     {
+     PCRE_FREE_STUDY(extra);
+     }
+   if (locale_set)
      {
      new_free((void *)tables);
      setlocale(LC_CTYPE, "C");
      locale_set = 0;
      }
+   if (jit_stack != NULL)
+     {
+     pcre_jit_stack_free(jit_stack);
+     jit_stack = NULL;
+     }
    }
  if (infile == stdin) fprintf(outfile, "\n");
-Line 2479 
 free(dbuffer);
+Line 3767 
 free(dbuffer);
  free(pbuffer);
  free(offsets);
+ #ifdef SUPPORT_PCRE16
+ if (buffer16 != NULL) free(buffer16);
+ #endif
  return yield;
  }

 Legend:



Removed from v.389
 


changed lines


 
Added in v.814
 Legend:



Removed from v.389
 


changed lines


 
Added in v.814
-Removed from v.389
+Added in v.814

webmaster@exim.org	ViewVC Help
Powered by ViewVC 1.1.12