code/trunk/perltest.pl

#! /usr/bin/env perl

# Program for testing regular expressions with perl to check that PCRE handles
# them the same. This version supports /8 for UTF-8 testing. However, it needs
# to have "use utf8" at the start for running the UTF-8 tests, but *not* for
# the other tests. The only way I've found for doing this is to cat this line
# in explicitly in the RunPerlTest script. I've also used this method to supply
# "require Encode" for the UTF-8 tests, so that the main test will still run
# where Encode is not installed.

# use locale;  # With this included, \x0b matches \s!

# Function for turning a string into a string of printing chars.

#use utf8;
#require Encode;

sub pchars {
my($t) = "";

if ($utf8)
  {
  @p = unpack('U*', $_[0]);
  foreach $c (@p)
    {
    if ($c >= 32 && $c < 127) { $t .= chr $c; }
      else { $t .= sprintf("\\x{%02x}", $c);
      }
    }
  }
else
  {
  foreach $c (split(//, $_[0]))
    {
    if (ord $c >= 32 && ord $c < 127) { $t .= $c; }
      else { $t .= sprintf("\\x%02x", ord $c); }
    }
  }

$t;
}


# Read lines from named file or stdin and write to named file or stdout; lines
# consist of a regular expression, in delimiters and optionally followed by
# options, followed by a set of test data, terminated by an empty line.

# Sort out the input and output files

if (@ARGV > 0)
  {
  open(INFILE, "<$ARGV[0]") || die "Failed to open $ARGV[0]\n";
  $infile = "INFILE";
  }
else { $infile = "STDIN"; }

if (@ARGV > 1)
  {
  open(OUTFILE, ">$ARGV[1]") || die "Failed to open $ARGV[1]\n";
  $outfile = "OUTFILE";
  }
else { $outfile = "STDOUT"; }

printf($outfile "Perl $] Regular Expressions\n\n");

# Main loop

NEXT_RE:
for (;;)
  {
  printf "  re> " if $infile eq "STDIN";
  last if ! ($_ = <$infile>);
  printf $outfile "$_" if $infile ne "STDIN";
  next if ($_ eq "");

  $pattern = $_;

  while ($pattern !~ /^\s*(.).*\1/s)
    {
    printf "    > " if $infile eq "STDIN";
    last if ! ($_ = <$infile>);
    printf $outfile "$_" if $infile ne "STDIN";
    $pattern .= $_;
    }

   chomp($pattern);
   $pattern =~ s/\s+$//;

  # The private /+ modifier means "print $' afterwards".

  $showrest = ($pattern =~ s/\+(?=[a-zA-Z]*$)//);

  # A doubled version is used by pcretest to print remainders after captures

  $pattern =~ s/\+(?=[a-zA-Z]*$)//;

  # Remove /8 from a UTF-8 pattern.

  $utf8 = $pattern =~ s/8(?=[a-zA-Z]*$)//;

  # Remove /J from a pattern with duplicate names.

  $pattern =~ s/J(?=[a-zA-Z]*$)//;

  # Remove /K from a pattern (asks pcretest to check MARK data) */

  $pattern =~ s/K(?=[a-zA-Z]*$)//;

  # Remove /W from a pattern (asks pcretest to set PCRE_UCP)

  $pattern =~ s/W(?=[a-zA-Z]*$)//;

  # Remove /S or /SS from a pattern (asks pcretest to study or not to study)

  $pattern =~ s/S(?=[a-zA-Z]*$)//g;

  # Remove /Y from a pattern (asks pcretest to disable PCRE optimization)

  $pattern =~ s/Y(?=[a-zA-Z]*$)//;

  # Check that the pattern is valid

  eval "\$_ =~ ${pattern}";
  if ($@)
    {
    printf $outfile "Error: $@";
    next NEXT_RE;
    }

  # If the /g modifier is present, we want to put a loop round the matching;
  # otherwise just a single "if".

  $cmd = ($pattern =~ /g[a-z]*$/)? "while" : "if";

  # If the pattern is actually the null string, Perl uses the most recently
  # executed (and successfully compiled) regex is used instead. This is a
  # nasty trap for the unwary! The PCRE test suite does contain null strings
  # in places - if they are allowed through here all sorts of weird and
  # unexpected effects happen. To avoid this, we replace such patterns with
  # a non-null pattern that has the same effect.

  $pattern = "/(?#)/$2" if ($pattern =~ /^(.)\1(.*)$/);

  # Read data lines and test them

  for (;;)
    {
    printf "data> " if $infile eq "STDIN";
    last NEXT_RE if ! ($_ = <$infile>);
    chomp;
    printf $outfile "$_\n" if $infile ne "STDIN";

    s/\s+$//;  # Remove trailing space
    s/^\s+//;  # Remove leading space
    s/\\Y//g;  # Remove \Y (pcretest flag to set PCRE_NO_START_OPTIMIZE)

    last if ($_ eq "");
    $x = eval "\"$_\"";   # To get escapes processed

    # Empty array for holding results, ensure $REGERROR and $REGMARK are
    # unset, then do the matching.

    @subs = ();

    $pushes = "push \@subs,\$&;" .
         "push \@subs,\$1;" .
         "push \@subs,\$2;" .
         "push \@subs,\$3;" .
         "push \@subs,\$4;" .
         "push \@subs,\$5;" .
         "push \@subs,\$6;" .
         "push \@subs,\$7;" .
         "push \@subs,\$8;" .
         "push \@subs,\$9;" .
         "push \@subs,\$10;" .
         "push \@subs,\$11;" .
         "push \@subs,\$12;" .
         "push \@subs,\$13;" .
         "push \@subs,\$14;" .
         "push \@subs,\$15;" .
         "push \@subs,\$16;" .
         "push \@subs,\$'; }";

    undef $REGERROR;
    undef $REGMARK;

    eval "${cmd} (\$x =~ ${pattern}) {" . $pushes;

    if ($@)
      {
      printf $outfile "Error: $@\n";
      next NEXT_RE;
      }
    elsif (scalar(@subs) == 0)
      {
      printf $outfile "No match";
      if (defined $REGERROR && $REGERROR != 1)
        { printf $outfile (", mark = %s", &pchars($REGERROR)); }
      printf $outfile "\n";
      }
    else
      {
      while (scalar(@subs) != 0)
        {
        printf $outfile (" 0: %s\n", &pchars($subs[0]));
        printf $outfile (" 0+ %s\n", &pchars($subs[17])) if $showrest;
        $last_printed = 0;
        for ($i = 1; $i <= 16; $i++)
          {
          if (defined $subs[$i])
            {
            while ($last_printed++ < $i-1)
              { printf $outfile ("%2d: <unset>\n", $last_printed); }
            printf $outfile ("%2d: %s\n", $i, &pchars($subs[$i]));
            $last_printed = $i;
            }
          }
        splice(@subs, 0, 18);
        }

      # It seems that $REGMARK is not marked as UTF-8 even when use utf8 is
      # set and the input pattern was a UTF-8 string. We can, however, force
      # it to be so marked.

      if (defined $REGMARK && $REGMARK != 1)
        {
        $xx = $REGMARK;
        $xx = Encode::decode_utf8($xx) if $utf8;
        printf $outfile ("MK: %s\n", &pchars($xx));
        }
      }
    }
  }

# printf $outfile "\n";

# End
1	ph10	97	#! /usr/bin/env perl
2	nigel	3
3			# Program for testing regular expressions with perl to check that PCRE handles
4	ph10	871	# them the same. This version supports /8 for UTF-8 testing. However, it needs
5			# to have "use utf8" at the start for running the UTF-8 tests, but not for
6			# the other tests. The only way I've found for doing this is to cat this line
7			# in explicitly in the RunPerlTest script. I've also used this method to supply
8			# "require Encode" for the UTF-8 tests, so that the main test will still run
9			# where Encode is not installed.
10	nigel	3
11	nigel	91	# use locale; # With this included, \x0b matches \s!
12	nigel	3
13	ph10	871	# Function for turning a string into a string of printing chars.
14	nigel	3
15	ph10	1046	#use utf8;
16	ph10	871	#require Encode;
17
18	nigel	3	sub pchars {
19			my($t) = "";
20
21	nigel	63	if ($utf8)
22	nigel	3	{
23	nigel	63	@p = unpack('U*', $_[0]);
24			foreach $c (@p)
25			{
26			if ($c >= 32 && $c < 127) { $t .= chr $c; }
27	ph10	903	else { $t .= sprintf("\\x{%02x}", $c);
28	ph10	871	}
29	nigel	63	}
30	nigel	3	}
31	nigel	63	else
32			{
33			foreach $c (split(//, $_[0]))
34			{
35			if (ord $c >= 32 && ord $c < 127) { $t .= $c; }
36			else { $t .= sprintf("\\x%02x", ord $c); }
37			}
38			}
39
40	nigel	3	$t;
41			}
42
43
44			# Read lines from named file or stdin and write to named file or stdout; lines
45			# consist of a regular expression, in delimiters and optionally followed by
46			# options, followed by a set of test data, terminated by an empty line.
47
48			# Sort out the input and output files
49
50			if (@ARGV > 0)
51			{
52			open(INFILE, "<$ARGV[0]") \|\| die "Failed to open $ARGV[0]\n";
53			$infile = "INFILE";
54			}
55			else { $infile = "STDIN"; }
56
57			if (@ARGV > 1)
58			{
59			open(OUTFILE, ">$ARGV[1]") \|\| die "Failed to open $ARGV[1]\n";
60			$outfile = "OUTFILE";
61			}
62			else { $outfile = "STDOUT"; }
63
64	nigel	23	printf($outfile "Perl $] Regular Expressions\n\n");
65	nigel	3
66			# Main loop
67
68			NEXT_RE:
69			for (;;)
70			{
71			printf " re> " if $infile eq "STDIN";
72			last if ! ($_ = <$infile>);
73			printf $outfile "$_" if $infile ne "STDIN";
74			next if ($_ eq "");
75
76			$pattern = $_;
77
78			while ($pattern !~ /^\s(.).\1/s)
79			{
80			printf " > " if $infile eq "STDIN";
81			last if ! ($_ = <$infile>);
82			printf $outfile "$_" if $infile ne "STDIN";
83			$pattern .= $_;
84			}
85
86			chomp($pattern);
87			$pattern =~ s/\s+$//;
88
89	nigel	63	# The private /+ modifier means "print $' afterwards".
90	nigel	41
91	ph10	510	$showrest = ($pattern =~ s/\+(?=[a-zA-Z]*$)//);
92	ph10	654
93	ph10	617	# A doubled version is used by pcretest to print remainders after captures
94	nigel	41
95	ph10	654	$pattern =~ s/\+(?=[a-zA-Z]*$)//;
96
97	nigel	91	# Remove /8 from a UTF-8 pattern.
98	nigel	63
99	ph10	510	$utf8 = $pattern =~ s/8(?=[a-zA-Z]*$)//;
100	ph10	461
101	ph10	459	# Remove /J from a pattern with duplicate names.
102	nigel	63
103	ph10	510	$pattern =~ s/J(?=[a-zA-Z]*$)//;
104	ph10	461
105	ph10	512	# Remove /K from a pattern (asks pcretest to check MARK data) */
106
107	ph10	510	$pattern =~ s/K(?=[a-zA-Z]*$)//;
108
109	ph10	518	# Remove /W from a pattern (asks pcretest to set PCRE_UCP)
110
111			$pattern =~ s/W(?=[a-zA-Z]*$)//;
112
113	ph10	612	# Remove /S or /SS from a pattern (asks pcretest to study or not to study)
114
115			$pattern =~ s/S(?=[a-zA-Z]*$)//g;
116
117	ph10	836	# Remove /Y from a pattern (asks pcretest to disable PCRE optimization)
118
119			$pattern =~ s/Y(?=[a-zA-Z]*$)//;
120
121	nigel	3	# Check that the pattern is valid
122
123	nigel	91	eval "\$_ =~ ${pattern}";
124	nigel	3	if ($@)
125			{
126	nigel	23	printf $outfile "Error: $@";
127	nigel	3	next NEXT_RE;
128			}
129
130	nigel	41	# If the /g modifier is present, we want to put a loop round the matching;
131			# otherwise just a single "if".
132
133			$cmd = ($pattern =~ /g[a-z]*$/)? "while" : "if";
134
135			# If the pattern is actually the null string, Perl uses the most recently
136			# executed (and successfully compiled) regex is used instead. This is a
137			# nasty trap for the unwary! The PCRE test suite does contain null strings
138			# in places - if they are allowed through here all sorts of weird and
139			# unexpected effects happen. To avoid this, we replace such patterns with
140			# a non-null pattern that has the same effect.
141
142			$pattern = "/(?#)/$2" if ($pattern =~ /^(.)\1(.*)$/);
143
144	nigel	3	# Read data lines and test them
145
146			for (;;)
147			{
148			printf "data> " if $infile eq "STDIN";
149			last NEXT_RE if ! ($_ = <$infile>);
150			chomp;
151			printf $outfile "$_\n" if $infile ne "STDIN";
152
153	ph10	510	s/\s+$//; # Remove trailing space
154			s/^\s+//; # Remove leading space
155			s/\\Y//g; # Remove \Y (pcretest flag to set PCRE_NO_START_OPTIMIZE)
156	nigel	3
157			last if ($_ eq "");
158	nigel	41	$x = eval "\"$_\""; # To get escapes processed
159	nigel	3
160	ph10	488	# Empty array for holding results, ensure $REGERROR and $REGMARK are
161			# unset, then do the matching.
162	nigel	3
163	nigel	41	@subs = ();
164
165	nigel	63	$pushes = "push \@subs,\$&;" .
166	nigel	41	"push \@subs,\$1;" .
167			"push \@subs,\$2;" .
168			"push \@subs,\$3;" .
169			"push \@subs,\$4;" .
170			"push \@subs,\$5;" .
171			"push \@subs,\$6;" .
172			"push \@subs,\$7;" .
173			"push \@subs,\$8;" .
174			"push \@subs,\$9;" .
175			"push \@subs,\$10;" .
176			"push \@subs,\$11;" .
177			"push \@subs,\$12;" .
178			"push \@subs,\$13;" .
179			"push \@subs,\$14;" .
180			"push \@subs,\$15;" .
181			"push \@subs,\$16;" .
182			"push \@subs,\$'; }";
183
184	ph10	488	undef $REGERROR;
185			undef $REGMARK;
186
187	nigel	91	eval "${cmd} (\$x =~ ${pattern}) {" . $pushes;
188	nigel	63
189	nigel	3	if ($@)
190			{
191			printf $outfile "Error: $@\n";
192			next NEXT_RE;
193			}
194	nigel	41	elsif (scalar(@subs) == 0)
195	nigel	3	{
196	ph10	488	printf $outfile "No match";
197			if (defined $REGERROR && $REGERROR != 1)
198	ph10	871	{ printf $outfile (", mark = %s", &pchars($REGERROR)); }
199	ph10	488	printf $outfile "\n";
200	nigel	3	}
201			else
202			{
203	nigel	41	while (scalar(@subs) != 0)
204	nigel	3	{
205	nigel	41	printf $outfile (" 0: %s\n", &pchars($subs[0]));
206			printf $outfile (" 0+ %s\n", &pchars($subs[17])) if $showrest;
207			$last_printed = 0;
208			for ($i = 1; $i <= 16; $i++)
209	nigel	3	{
210	nigel	41	if (defined $subs[$i])
211			{
212			while ($last_printed++ < $i-1)
213			{ printf $outfile ("%2d: <unset>\n", $last_printed); }
214			printf $outfile ("%2d: %s\n", $i, &pchars($subs[$i]));
215			$last_printed = $i;
216			}
217	nigel	3	}
218	nigel	41	splice(@subs, 0, 18);
219	nigel	3	}
220	ph10	903
221	ph10	871	# It seems that $REGMARK is not marked as UTF-8 even when use utf8 is
222			# set and the input pattern was a UTF-8 string. We can, however, force
223	ph10	903	# it to be so marked.
224
225	ph10	488	if (defined $REGMARK && $REGMARK != 1)
226	ph10	871	{
227	ph10	903	$xx = $REGMARK;
228			$xx = Encode::decode_utf8($xx) if $utf8;
229			printf $outfile ("MK: %s\n", &pchars($xx));
230	ph10	871	}
231	nigel	3	}
232			}
233			}
234
235	nigel	75	# printf $outfile "\n";
236	nigel	3
237			# End