/[pcre]/code/trunk/pcre_study.c

Diff of /code/trunk/pcre_study.c

Parent Directory | Revision Log | View Patch Patch

-revision 524 by ph10,
Mon May 24 17:06:28 2010 UTC
+revision 545 by ph10,
Wed Jun 16 10:51:15 2010 UTC
 Line 441 
 for (;;)
  *      Set a bit and maybe its alternate case    *
  *************************************************/
  /* Given a character, set its first byte's bit in the table, and also the
  corresponding bit for the other version of a letter if we are caseless. In
  UTF-8 mode, for characters greater than 127, we can only do the caseless thing
  when Unicode property support is available.
 Line 451 
 Arguments:
    p             points to the character
    caseless      the caseless flag
    cd            the block with char table pointers
    utf8          TRUE for UTF-8 mode
  Returns:        pointer after the character
  */
 Line 471 
 if (utf8 && c > 127)
  #ifdef SUPPORT_UCP
    if (caseless)
      {
      uschar buff[8];
      c = UCD_OTHERCASE(c);
      (void)_pcre_ord2utf8(c, buff);
      SET_BIT(buff[0]);
      }
  #endif
    return p;
    }
  #endif
  /* Not UTF-8 mode, or character is less than 127. */
 Line 490 
 return p + 1;
  /*************************************************
+ *     Set bits for a positive character type     *
+ *************************************************/
+ /* This function sets starting bits for a character type. In UTF-8 mode, we can
+ only do a direct setting for bytes less than 128, as otherwise there can be
+ confusion with bytes in the middle of UTF-8 characters. In a "traditional"
+ environment, the tables will only recognize ASCII characters anyway, but in at
+ least one Windows environment, some higher bytes bits were set in the tables.
+ So we deal with that case by considering the UTF-8 encoding.
+ Arguments:
+   start_bits     the starting bitmap
+   cbit type      the type of character wanted
+   table_limit    32 for non-UTF-8; 16 for UTF-8
+   cd             the block with char table pointers
+ Returns:         nothing
+ */
+ static void
+ set_type_bits(uschar *start_bits, int cbit_type, int table_limit,
+   compile_data *cd)
+ {
+ register int c;
+ for (c = 0; c < table_limit; c++) start_bits[c] |= cd->cbits[c+cbit_type];
+ if (table_limit == 32) return;
+ for (c = 128; c < 256; c++)
+   {
+   if ((cd->cbits[c/8] & (1 << (c&7))) != 0)
+     {
+     uschar buff[8];
+     (void)_pcre_ord2utf8(c, buff);
+     SET_BIT(buff[0]);
+     }
+   }
+ }
+ /*************************************************
+ *     Set bits for a negative character type     *
+ *************************************************/
+ /* This function sets starting bits for a negative character type such as \D.
+ In UTF-8 mode, we can only do a direct setting for bytes less than 128, as
+ otherwise there can be confusion with bytes in the middle of UTF-8 characters.
+ Unlike in the positive case, where we can set appropriate starting bits for
+ specific high-valued UTF-8 characters, in this case we have to set the bits for
+ all high-valued characters. The lowest is 0xc2, but we overkill by starting at
+xc0 (192) for simplicity.
+ Arguments:
+   start_bits     the starting bitmap
+   cbit type      the type of character wanted
+   table_limit    32 for non-UTF-8; 16 for UTF-8
+   cd             the block with char table pointers
+ Returns:         nothing
+ */
+ static void
+ set_nottype_bits(uschar *start_bits, int cbit_type, int table_limit,
+   compile_data *cd)
+ {
+ register int c;
+ for (c = 0; c < table_limit; c++) start_bits[c] |= ~cd->cbits[c+cbit_type];
+ if (table_limit != 32) for (c = 24; c < 32; c++) start_bits[c] = 0xff;
+ }
+ /*************************************************
  *          Create bitmap of starting bytes       *
  *************************************************/
-Line 519 
 set_start_bits(const uschar *code, uscha
+Line 590 
 set_start_bits(const uschar *code, uscha
  {
  register int c;
  int yield = SSB_DONE;
+ int table_limit = utf8? 16:32;
  #if 0
  /* ========================================================================= */
-Line 666 
 do
+Line 738 
 do
        (void)set_table_bit(start_bits, tcode + 1, caseless, cd, utf8);
        try_next = FALSE;
        break;
        /* Special spacing and line-terminating items. These recognize specific
        lists of characters. The difference between VSPACE and ANYNL is that the
        latter can match the two-character CRLF sequence, but that is not
        relevant for finding the first character, so their code here is
        identical. */
        case OP_HSPACE:
        SET_BIT(0x09);
        SET_BIT(0x20);
-       SET_BIT(0xA0);
        if (utf8)
          {
+         SET_BIT(0xC2);  /* For U+00A0 */
          SET_BIT(0xE1);  /* For U+1680, U+180E */
          SET_BIT(0xE2);  /* For U+2000 - U+200A, U+202F, U+205F */
          SET_BIT(0xE3);  /* For U+3000 */
          }
+       else SET_BIT(0xA0);
        try_next = FALSE;
        break;
        case OP_ANYNL:
        case OP_VSPACE:
        SET_BIT(0x0A);
        SET_BIT(0x0B);
        SET_BIT(0x0C);
        SET_BIT(0x0D);
-       SET_BIT(0x85);
+       if (utf8)
-       if (utf8) SET_BIT(0xE2);    /* For U+2028, U+2029 */
+         {
+         SET_BIT(0xC2);  /* For U+0085 */
+         SET_BIT(0xE2);  /* For U+2028, U+2029 */
+         }
+       else SET_BIT(0x85);
        try_next = FALSE;
        break;
        /* Single character types set the bits and stop. Note that if PCRE_UCP
        is set, we do not see these op codes because \d etc are converted to
-       properties. Therefore, these apply in the case when only ASCII characters
+       properties. Therefore, these apply in the case when only characters less
-       are recognized to match the types. */
+       than 256 are recognized to match the types. */
        case OP_NOT_DIGIT:
-       for (c = 0; c < 32; c++)
+       set_nottype_bits(start_bits, cbit_digit, table_limit, cd);
-         start_bits[c] |= ~cd->cbits[c+cbit_digit];
        try_next = FALSE;
        break;
        case OP_DIGIT:
-       for (c = 0; c < 32; c++)
+       set_type_bits(start_bits, cbit_digit, table_limit, cd);
-         start_bits[c] |= cd->cbits[c+cbit_digit];
        try_next = FALSE;
        break;
        /* The cbit_space table has vertical tab as whitespace; we have to
-       discard it. */
+       ensure it is set as not whitespace. */
        case OP_NOT_WHITESPACE:
-       for (c = 0; c < 32; c++)
+       set_nottype_bits(start_bits, cbit_space, table_limit, cd);
-         {
+       start_bits[1] |= 0x08;
-         int d = cd->cbits[c+cbit_space];
-         if (c == 1) d &= ~0x08;
-         start_bits[c] |= ~d;
-         }
        try_next = FALSE;
        break;
        /* The cbit_space table has vertical tab as whitespace; we have to
-       discard it. */
+       not set it from the table. */
        case OP_WHITESPACE:
-       for (c = 0; c < 32; c++)
+       c = start_bits[1];    /* Save in case it was already set */
-         {
+       set_type_bits(start_bits, cbit_space, table_limit, cd);
-         int d = cd->cbits[c+cbit_space];
+       start_bits[1] = (start_bits[1] & ~0x08) | c;
-         if (c == 1) d &= ~0x08;
-         start_bits[c] |= d;
-         }
        try_next = FALSE;
        break;
        case OP_NOT_WORDCHAR:
-       for (c = 0; c < 32; c++)
+       set_nottype_bits(start_bits, cbit_word, table_limit, cd);
-         start_bits[c] |= ~cd->cbits[c+cbit_word];
        try_next = FALSE;
        break;
        case OP_WORDCHAR:
-       for (c = 0; c < 32; c++)
+       set_type_bits(start_bits, cbit_word, table_limit, cd);
-         start_bits[c] |= cd->cbits[c+cbit_word];
        try_next = FALSE;
        break;
-Line 757 
 do
+Line 823 
 do
        case OP_TYPEPLUS:
        case OP_TYPEMINPLUS:
        case OP_TYPEPOSPLUS:
        tcode++;
        break;
-Line 785 
 do
+Line 851 
 do
          case OP_ANY:
          case OP_ALLANY:
          return SSB_FAIL;
          case OP_HSPACE:
          SET_BIT(0x09);
          SET_BIT(0x20);
-         SET_BIT(0xA0);
          if (utf8)
            {
+           SET_BIT(0xC2);  /* For U+00A0 */
            SET_BIT(0xE1);  /* For U+1680, U+180E */
            SET_BIT(0xE2);  /* For U+2000 - U+200A, U+202F, U+205F */
            SET_BIT(0xE3);  /* For U+3000 */
            }
-         break;
+         else SET_BIT(0xA0);
+         break;
-         case OP_ANYNL:
+         case OP_ANYNL:
          case OP_VSPACE:
          SET_BIT(0x0A);
          SET_BIT(0x0B);
          SET_BIT(0x0C);
          SET_BIT(0x0D);
-         SET_BIT(0x85);
+         if (utf8)
-         if (utf8) SET_BIT(0xE2);    /* For U+2028, U+2029 */
+           {
-         break;
+           SET_BIT(0xC2);  /* For U+0085 */
+           SET_BIT(0xE2);  /* For U+2028, U+2029 */
+           }
+         else SET_BIT(0x85);
+         break;
          case OP_NOT_DIGIT:
-         for (c = 0; c < 32; c++)
+         set_nottype_bits(start_bits, cbit_digit, table_limit, cd);
-           start_bits[c] |= ~cd->cbits[c+cbit_digit];
          break;
          case OP_DIGIT:
-         for (c = 0; c < 32; c++)
+         set_type_bits(start_bits, cbit_digit, table_limit, cd);
-           start_bits[c] |= cd->cbits[c+cbit_digit];
          break;
          /* The cbit_space table has vertical tab as whitespace; we have to
-         discard it. */
+         ensure it gets set as not whitespace. */
          case OP_NOT_WHITESPACE:
-         for (c = 0; c < 32; c++)
+         set_nottype_bits(start_bits, cbit_space, table_limit, cd);
-           {
+         start_bits[1] |= 0x08;
-           int d = cd->cbits[c+cbit_space];
-           if (c == 1) d &= ~0x08;
-           start_bits[c] |= ~d;
-           }
          break;
          /* The cbit_space table has vertical tab as whitespace; we have to
-         discard it. */
+         avoid setting it. */
          case OP_WHITESPACE:
-         for (c = 0; c < 32; c++)
+         c = start_bits[1];    /* Save in case it was already set */
-           {
+         set_type_bits(start_bits, cbit_space, table_limit, cd);
-           int d = cd->cbits[c+cbit_space];
+         start_bits[1] = (start_bits[1] & ~0x08) | c;
-           if (c == 1) d &= ~0x08;
-           start_bits[c] |= d;
-           }
          break;
          case OP_NOT_WORDCHAR:
-         for (c = 0; c < 32; c++)
+         set_nottype_bits(start_bits, cbit_word, table_limit, cd);
-           start_bits[c] |= ~cd->cbits[c+cbit_word];
          break;
          case OP_WORDCHAR:
-         for (c = 0; c < 32; c++)
+         set_type_bits(start_bits, cbit_word, table_limit, cd);
-           start_bits[c] |= cd->cbits[c+cbit_word];
          break;
          }

 Legend:



Removed from v.524
 


changed lines


 
Added in v.545
 Legend:



Removed from v.524
 


changed lines


 
Added in v.545
-Removed from v.524
+Added in v.545

webmaster@exim.org	ViewVC Help
Powered by ViewVC 1.1.12