#!/usr/bin/perl -w

eval 'exec /usr/bin/perl -w -S $0 ${1+"$@"}'
    if 0; # not running under some shell

# grepmail

$VERSION = '4.80';

# Grepmail searches a normal, gzip'd, tzip'd, or bzip2'd mailbox for a given
# regular expression and returns those emails that match the query. It also
# supports piped compressed or ascii input, and searches constrained by date
# and size. 

# Visit the grepmail project homepage at http://grepmail.sourceforge.net/
# There you can join the announcements mailing list to be notified of updates,
# grab the development environment via CVS, participate in chats and mailing
# lists, report bugs, submit patches, etc.

# Do a pod2text on this file to get full documentation, or pod2man to get
# man pages.

# Written by David Coppit (david@coppit.org, http://coppit.org/) with lots of
# debugging and patching by others -- see the CHANGES file for a complete
# list.

# This code is distributed under the GNU General Public License (GPL). See
# http://www.opensource.org/gpl-license.html and http://www.opensource.org/.

# Notes:
# It turns out that -h, -b, -d, -s and -v have some nasty feature interaction.
# Here's a table of how matching should occur for each combination of flags:
#
# % B, H, S, D,!V
#   Match if body, header, size, and date match
# %!B, H, S, D,!V
#   Match if header, size, and date match -- don't care about body
# % B,!H, S, D,!V
#   Match if body, size, and date match -- don't care about header
# % B, H,!S, D,!V
#   Match if body, header, and date match -- don't care about size
# % B, H, S,!D,!V
#   Match if body, header, and size match -- don't care about date
# %!B,!H, S, D,!V
#   Match if size and date and (body or header) matches
#*%!B, H,!S, D,!V
#   Match if header and date matches -- don't care about body or size
# %!B, H, S,!D,!V
#   Match if header and size matches -- don't care about body or date
# % B,!H,!S, D,!V
#   Match if body and date matches -- don't care about header or size
# % B,!H, S,!D,!V
#   Match if body and size matches -- don't care about header or date
# % B, H,!S,!D,!V
#   Match if body and header matches -- don't care about size or date
#*%!B,!H,!S, D,!V
#   Match if date and (body or header) matches -- don't care about size
# %!B,!H, S,!D,!V
#   Match if size and (body or header) matches -- don't care about date
#*%!B, H,!S,!D,!V
#   Match if header matches -- don't care about body, size, or date
#   B,!H,!S,!D,!V
#   Match if body matches -- don't care about header, size, or date
#*  B, H, S, D, V
#   Match if body, header, size, or date doesn't match
#* !B, H, S, D, V
#   Match if header, size, or date doesn't match -- don't care about body
#*  B,!H, S, D, V
#   Match if body, size, or date doesn't match -- don't care about header
#*  B, H,!S, D, V
#   Match if body, header, or date doesn't match -- don't care about size
#*  B, H, S,!D, V
#   Match if body, header, or size doesn't match -- don't care about date
#* !B,!H, S, D, V
#   Match if size or date doesn't match -- don't care about body or header
#*%!B, H,!S, D, V
#   Match if header or date doesn't match -- don't care about body or size
#* !B, H, S,!D, V
#   Match if header or size doesn't match -- don't care about body or date
#*  B,!H,!S, D, V
#   Match if body or date doesn't match -- don't care about header or size
#*  B,!H, S,!D, V
#   Match if body or size doesn't match -- don't care about header or date
#*  B, H,!S,!D, V
#   Match if body or header doesn't match -- don't care about size or date
#*%!B,!H,!S, D, V
#   Match if date or (body and header) don't match -- don't care about size
#* !B,!H, S,!D, V
#   Match if size or (body and header) don't match -- don't care about date
#*%!B, H,!S,!D, V
#   Match if header doesn't match -- don't care about body, size, or date
#   B,!H,!S,!D, V
#   Match if body doesn't match -- don't care about header, size, or date
#
#* Indicates early match candidate based on the header only
#% Indicates early nonmatch candidate based on the header only

require 5.00396;

use vars qw( %opts $pattern $commandLine $VERSION %message_ids_seen );

use Getopt::Std;

use strict;
use FileHandle;
use Carp;

#-------------------------------------------------------------------------------

package Mail::Folder::SlowReader;

require Exporter;

no strict;

@ISA = qw(Exporter DynaLoader);
@EXPORT = qw();
@EXPORT_OK = qw( reset_file read_email );

use strict;

my $LINE = 0;
my $FILE_HANDLE = undef;
my $START = 0;
my $END = 0;
my $READ_BUFFER = '';

my %HEADER_PATTERNS = (
'^TO:' =>
  '(^((Original-)?(Resent-)?(To|Cc|Bcc)|(X-Envelope|Apparently(-Resent)?)-To):)',
'^FROM_DAEMON:' =>
  '(^(Mailing-List:|Precedence:.*(junk|bulk|list)|To: Multiple recipients of |(((Resent-)?(From|Sender)|X-Envelope-From):|>?From )([^>]*[^(.%@a-z0-9])?(Post(ma?(st(e?r)?|n)|office)|(send)?Mail(er)?|daemon|m(mdf|ajordomo)|n?uucp|LIST(SERV|proc)|NETSERV|o(wner|ps)|r(e(quest|sponse)|oot)|b(ounce|bs\.smtp)|echo|mirror|s(erv(ices?|er)|mtp(error)?|ystem)|A(dmin(istrator)?|MMGR|utoanswer))(([^).!:a-z0-9][-_a-z0-9]*)?[%@>\t ][^<)]*(\(.*\).*)?)?))',
'^FROM_MAILER:' =>
  '(^(((Resent-)?(From|Sender)|X-Envelope-From):|>?From)([^>]*[^(.%@a-z0-9])?(Post(ma(st(er)?|n)|office)|(send)?Mail(er)?|daemon|mmdf|n?uucp|ops|r(esponse|oot)|(bbs\.)?smtp(error)?|s(erv(ices?|er)|ystem)|A(dmin(istrator)?|MMGR))(([^).!:a-z0-9][-_a-z0-9]*)?[%@>\t][^<)]*(\(.*\).*)?)?$([^>]|$))',
);

sub reset_file
{
  my $file_handle = shift;

  $FILE_HANDLE = $file_handle;
  $LINE = 1;
  $START = 0;
  $END = 0;
  $READ_BUFFER = '';
}

#-------------------------------------------------------------------------------

# Need this for a lookahead.
my $READ_CHUNK_SIZE = 20000;

sub read_email
{
  # Undefined read buffer means we hit eof on the last read.
  return 0 unless defined $READ_BUFFER;

  my $line = $LINE;

  $START = $END;

  # Look for the start of the next email
  LOOK_FOR_NEXT_HEADER:
  while ($READ_BUFFER =~ m/^From\s
    (
      # Skip names, months, days
      (?> [^:]+ ) 
      # Match time
      (?: :\d\d){1,2}
      # Match time zone (EST), hour shift (+0500), and-or year
      (?: \s+ (?: [A-Z]{2,3} | [+-]?\d{4} ) ){1,3}
    )$/xmg)
  {
    $END = pos($READ_BUFFER) - length($1) - 5;

    # Don't stop on email header for the first email in the buffer
    next unless $END;

    # Keep looking if the header we found is part of a "Begin Included
    # Message".
    my $end_of_string = substr($READ_BUFFER, $END-200, 200);
    next if $end_of_string =~
        /\n-----(?: Begin Included Message |Original Message)-----\n[^\n]*\n*$/i;

    # Found the next email!
    my $email = substr($READ_BUFFER, $START, $END-$START);
    $LINE += ($email =~ tr/\n//);
    
    return (1, $email, $line);
  }

  # Didn't find next email in current buffer. Most likely we need to read some
  # more of the mailbox. Shift the current email to the front of the buffer
  # unless we've already done so.
  $READ_BUFFER = substr($READ_BUFFER,$START) unless $START == 0;
  $START = 0;

  # Start looking at the end of the buffer, but back up some in case the edge
  # of the newly read buffer contains the start of a new header. I believe the
  # RFC says header lines can be at most 90 characters long.
  my $search_position = length($READ_BUFFER) - 90;
  $search_position = 0 if $search_position < 0;

  # Can't use sysread because it doesn't work with ungetc
  if ($READ_CHUNK_SIZE == 0)
  {
    local $/ = undef;

    if (eof $FILE_HANDLE)
    {
      my $email = $READ_BUFFER;
      undef $READ_BUFFER;
      return (1, $email, $line);
    }
    else
    {
      $READ_BUFFER = <$FILE_HANDLE>;
      pos($READ_BUFFER) = $search_position;
      goto LOOK_FOR_NEXT_HEADER;
    }
  }
  else
  {
    if (read($FILE_HANDLE, $READ_BUFFER, $READ_CHUNK_SIZE, length($READ_BUFFER)))
    {
      pos($READ_BUFFER) = $search_position;
      goto LOOK_FOR_NEXT_HEADER;
    }
    else
    {
      my $email = $READ_BUFFER;
      undef $READ_BUFFER;
      return (1, $email, $line);
    }
  }
}

sub Read_Chunk_Of_Body
{
  my $email = shift;

  local $/ = "\nFrom ";
  my $chunk = <$FILE_HANDLE>;
  local $/ = 'From ';
  chomp $chunk;

  $LINE += ($chunk =~ tr/\n//);

  $$email .= $chunk;
}

package main;

# Internal function return values.
my $PRINT    = 0;
my $DONE     = 1;
my $SKIP     = 2;
my $CONTINUE = 3;
my $NONE     = 4;
my $BEFORE   = 5;
my $AFTER    = 6;
my $ON       = 7;
my $NODATE   = 8;
my $BETWEEN  = 9;

#-------------------------------------------------------------------------------

# Outputs debug messages with the -D flag. Be sure to return 1 so code like
# 'dprint "blah\n" and exit' works.

sub dprint
{
  return 1 unless $opts{'D'};

  my $message = join '',@_;

  foreach my $line (split /\n/, $message)
  {
    print STDERR "DEBUG: $line\n";
  }

  return 1;
}

#-------------------------------------------------------------------------------

# Print a nice error message before exiting

sub reportAndExit
{
  my $message = shift;

  print STDERR "grepmail: $message.\n";
  exit 1;
}

#-------------------------------------------------------------------------------

# Filter signals to print error messages when CTRL-C is caught, a pipe is
# empty, a pipe is killed, etc.

my %signals_and_messages = (
  'PIPE' => 'Broken Pipe',
  'HUP' => 'Hangup',
  'INT' => 'Canceled',
  'QUIT' => 'Quit',
  'SEGV' => 'Segmentation violation',
  'TERM' => 'Terminated',
);

# We'll store a copy of the original signal handlers and call them when we're
# done. This helps when running under the debugger.
my %old_SIG = %SIG;

sub signalHandler
{
  my $signal = $_[0];

  $old_SIG{$signal}->(@_) if $old_SIG{$signal};

  reportAndExit($signals_and_messages{$signal});
}

# Delete the HUP signal for Windows, where it doesn't exist
delete $signals_and_messages{HUP} if $^O eq 'MSWin32';

# We have to localize %SIG to prevent odd bugs from cropping up (see
# changelog). Using an array slice on %SIG, I assign an array consisting of as
# many copies of \&signalHandler as there are keys in %signals_and_messages.
local @SIG{keys %signals_and_messages} =
  (\&signalHandler) x keys %signals_and_messages;

################################ MAIN PROGRAM #################################

# PROCESS ARGUMENTS
my ($opts_ref,@remaining_arguments) = Get_Options(@ARGV);
%opts = %$opts_ref;

# Initialize seen messages data structure to empty.
%message_ids_seen = ();

# Save the command line for later when we try to decompress standard input
{
  # Need to quote arguments with spaces
  my @args = @ARGV;
  grep { $_ = "'$_'" if index($_, ' ') != -1; $_ } @args;

  $commandLine = "$0 @args";
}

Print_Debug_Information($commandLine);

# LOAD THE MAIL::FOLDER::FASTREADER IF IT'S INSTALLED
if (!$opts{'Z'} && eval "require Mail::Folder::FastReader;")
{
  import Mail::Folder::FastReader 'reset_file', 'read_email';
  dprint "Using Mail::Folder::FastReader";
}
else
{
  import Mail::Folder::SlowReader 'reset_file', 'read_email';
  dprint "Not using Mail::Folder::FastReader";
}

sub ProcessDate($);
sub GetFiles(@);

# Check for -E flag incompatibilities.
if ($opts{'E'})
{
  # Have to do -Y before -h because the former implies the latter
  for my $option qw(e M S Y)
  {
    if ($opts{$option})
    {
      reportAndExit "-$option can not be used with -E";
    }
  }

  if ($opts{'i'})
  {
    reportAndExit "-i can not be used with -E. Use -E '\$email =~ /pattern/i' instead";
  }

  if ($opts{'b'})
  {
    reportAndExit "-b can not be used with -E. Use -E '\$email_body =~ /pattern/' instead";
  }

  if ($opts{'h'})
  {
    reportAndExit "-h can not be used with -E. Use -E '\$email_header =~ /pattern/' instead";
  }
}

# Make the pattern insensitive if we need to
$pattern = "(?i)$pattern" if ($opts{'i'});

my ($dateRestriction, $date1, $date2);

if (defined $opts{'d'})
{
  ($dateRestriction,$date1,$date2) = ProcessDate($opts{'d'});
}
else
{
  $dateRestriction = $NONE;
}

dprint "PATTERN: $pattern\n";
dprint "FILES: @remaining_arguments\n";
dprint "DATE RESTRICTION: $dateRestriction\n";
dprint "FIRST DATE: $date1\n" unless $dateRestriction == $NONE;
dprint "SECOND DATE: $date2\n" unless $dateRestriction == $NONE;
dprint "SIZE RESTRICTION: $opts{'s'}\n";

ValidatePattern($pattern);

my @files = GetFiles(@remaining_arguments);

# If the user provided input files...
if (@files)
{
  HandleInputFiles(@files);
}
# Using STDIN
else
{ 
  HandleStandardInput();
}

exit 0;

#-------------------------------------------------------------------------------

sub Get_Options
{
  local @ARGV = @_;

  # Print usage error if no arguments given
  print "No arguments given.\n\n" . usage() and exit(1) unless @ARGV;

  # Check for --help, the standard usage command
  print help() and exit(0) if grep { /^--help$/ } @ARGV;

  my @valid_options =
    qw( D e E F i q h b v l r R a M m s u n Z H S d V X Y );

  my %opts;

  # Initialize all options to zero.
  map { $opts{$_} = 0; } @valid_options;

  # And some to non-zero.
  $opts{'d'} = $opts{'V'} = undef;
  $opts{'X'} = '^-- $';

  # Ensure valid options
  getopt("eEdsXY",\%opts);

  # Here we have to deal with the possibility that the user specified the
  # search pattern without the -e flag.

  # getopts stops as soon as it sees a non-flag, so $ARGV[0] may contain the
  # pattern with more flags after it.
  unless ($opts{'e'} || $opts{'E'})
  {
    my $missing_flags = '';

    foreach my $flag (keys %opts)
    {
      $missing_flags .= $flag unless $opts{$flag};
    }

    $missing_flags = "[$missing_flags]";

    # If it looks like more flags are following, then grab the pattern and
    # process them.
    if ( $#ARGV > 0 && $ARGV[1] =~ /^-$missing_flags$/ )
    {
      $pattern = shift @ARGV;
      getopt("ds",\%opts);
    }
    # If we've seen a -d, -s, or -u flag, and it doesn't look like there are
    # flags following $ARGV[0], then look at the value in $ARGV[0]
    elsif ( ( defined $opts{'d'} || $opts{'s'} || $opts{'u'} ) &&
         ( $#ARGV <= 0 ||
           ( $#ARGV > 0 && $ARGV[1] !~ /^-$missing_flags$/ )
         )
       )
    {
      # If $ARGV[0] looks like a file we assume there was no pattern and
      # set a default pattern of "." to match everything.
      if ($#ARGV != -1 && -f $ARGV[0])
      {
        $pattern = '.';
      }
      # Otherwise we take the pattern and move on
      else
      {
        $pattern = shift @ARGV;
      }
    }
    # If we still don't have a pattern or any -d, -s, or -u flag, we assume
    # that $ARGV[0] is the pattern
    elsif (!defined $opts{'d'} && !$opts{'s'} && !$opts{'u'})
    {
      $pattern = shift @ARGV;
    }
  }

  if ($opts{'e'} || $opts{'E'})
  {
    print "You specified two search patterns.\n" and exit (1)
      if defined $pattern;

    if ($opts{'e'})
    {
      $pattern = $opts{'e'};
    }
    else
    {
      $pattern = $opts{'E'};
#
#      if ($pattern !~ /=~/)
#      {
#      }
    }
  }
  elsif (defined $opts{'V'})
  {
    # Print version and exit if we need to
    print "$VERSION\n";
    exit (0);
  }
  elsif (!defined $pattern)
  {
    # The only times you don't have to specify the pattern is when -d, -s, or -u
    # is being used. This should catch people who do "grepmail -h" thinking
    # it's help.
    print "Invalid arguments.\n\n" . usage() and exit (1)
      unless defined $opts{'d'} || $opts{'s'} || $opts{'u'};

    $pattern = '.';
  }

  if (defined $opts{'d'})
  {
    if (eval 'require Date::Parse;')
    {
      import Date::Parse;
    }
    else
    {
      print 'You specified -d, but do not have Date::Parse. ' .
            "Get it from CPAN.\n";
      exit (1);
    }
  }

  $opts{'h'} = 1 if $opts{'Y'};

  # Make sure no unknown flags were given
  foreach my $option (keys %opts)
  {
    unless (grep {/^$option$/} @valid_options)
    {
      print "Invalid argument: \"$option\".\n\n" . usage() and exit (1)
    }
  }

  return (\%opts, @ARGV);
}

#-------------------------------------------------------------------------------

sub Print_Debug_Information
{
  my $commandLine = shift;

  return unless $opts{'D'};

  dprint "Version: $VERSION";

  dprint "Command line was (special characters not escaped):";
  dprint "  $commandLine";

  if (defined $Date::Parse::VERSION)
  {
    dprint "Date::Parse VERSION: $Date::Parse::VERSION";
  }

  dprint "Options are:";
  foreach my $i (sort keys %opts)
  {
    if (defined $opts{$i})
    {
      dprint "  $i: $opts{$i}";
    }
    else
    {
      dprint "  $i: undef";
    }
  }

  dprint "INC is:";
  foreach my $i (@INC)
  {
    dprint "  $i";
  }
}

#-------------------------------------------------------------------------------

# Dies if the given pattern's syntax is invalid
sub ValidatePattern
{
  my $pattern = shift;

  local $@;

  if ($opts{'E'})
  {
    eval {if ($pattern) {}};
    reportAndExit "The match condition \"$pattern\" is invalid.\n" if $@;
  }
  else
  {
    eval {'string' =~ /$pattern/};
    reportAndExit "The pattern \"$pattern\" is invalid.\n" if $@;
  }
}

#-------------------------------------------------------------------------------

# Get a list of files, taking recursion into account if necessary.

sub GetFiles(@)
{
  my @files_and_directories = @_;

  # We just return what we were given unless we need to recurse subdirectories.
  return @files_and_directories unless $opts{'R'};

  my @files;

  foreach my $arg (@files_and_directories)
  {
    if (-f $arg)
    {
      push @files, $arg;
    }
    elsif( -d $arg)
    {
      dprint "Recursing directory $arg looking for files...";

      unless (eval "require File::Find;")
      {
        print "You specified -R, but do not have File::Find. ".
              "Get it from CPAN.\n";
        exit (1);
      }

      import File::Find;

      # Gets all plain files in directory and descendents. Puts them in @files
      $File::Find::name = '';
      find(sub {push @files,"$File::Find::name" if -f $_}, $arg);
    }
    else
    {
      # Ignore unknown file types
    }
  }

  return @files;
}

#-------------------------------------------------------------------------------

sub HandleInputFiles
{
  my @files = @_;

  # For each input file...
  foreach my $file (@files)
  {
    dprint '#'x70;
    dprint "Processing file $file";

    # First of all, silently ignore empty files...
    next if -z $file;

    # ...and also ignore directories.
    if (-d $file)
    {
      warn "** Skipping directory: '$file'\n" unless $opts{'q'};
      next;
    }

    $file = Search_Mailbox_Directories($file) unless -e $file;

    my $fileHandle = new FileHandle;
    my ($filter,$filterError);

    # If it's not a compressed file
    if ($file !~ /\.(gz|Z|bz2|tz)$/)
    {
      if (-B $file)
      {
        warn "** Skipping binary file: '$file'\n" unless $opts{'q'};
        next;
      }

      unless ($fileHandle->open($file))
      {
        warn "** Can't open $file: $!, skipping\n" unless $opts{'q'};
        next;
      }
    }
    # If it is a tzipped file
    elsif ($file =~ /\.tz$/)
    {
      dprint "Calling tzip to decompress file.";

      $filter = 'tzip';

      use vars qw(*OLDSTDERR);
      open OLDSTDERR,">&STDERR" or reportAndExit "Can't save STDERR: $!\n";
      open STDERR,">/dev/null"
        or reportAndExit "Can't redirect STDERR to /dev/null: $!\n";

      unless ($fileHandle->open("tzip -cd '$file'|"))
      {
        $filterError = $!;
      }

      open STDERR,">&OLDSTDERR" or reportAndExit "Can't restore STDERR: $!\n";
    }
    # If it is a gzipped file
    elsif ($file =~ /\.(gz|Z)$/)
    {
      dprint "Calling gunzip to decompress file.";

      $filter = 'gunzip';

      use vars qw(*OLDSTDERR);
      open OLDSTDERR,">&STDERR" or reportAndExit "Can't save STDERR: $!\n";
      open STDERR,">/dev/null"
        or reportAndExit "Can't redirect STDERR to /dev/null: $!\n";

      unless ($fileHandle->open("gzip -dc '$file'|"))
      {
        $filterError = $!;
      }

      open STDERR,">&OLDSTDERR" or reportAndExit "Can't restore STDERR: $!\n";
    }
    # If it is a bzipped file
    elsif ($file =~ /\.bz2$/)
    {
      dprint "Calling bzip2 to decompress file.";

      $filter = 'bzip2';

      use vars qw(*OLDSTDERR);
      open OLDSTDERR,">&STDERR" or reportAndExit "Can't save STDERR: $!\n";
      open STDERR,">/dev/null"
        or reportAndExit "Can't redirect STDERR to /dev/null: $!\n";

      unless ($fileHandle->open("bzip2 -dc '$file'|"))
      {
        $filterError = $!;
      }

      open STDERR,">&OLDSTDERR" or reportAndExit "Can't restore STDERR: $!\n";
    }

    if ($filterError)
    {
      warn "** Can't execute \"$filter\" for file \"$file\": $filterError, ".
           "skipping\n" unless $opts{'q'};
      next;
    }

    unless (DataOnFileHandle($fileHandle))
    {
      unless ($fileHandle->close())
      {
        warn "** Can't execute \"$filter\" for file \"$file\": ".
             "skipping\n" unless $opts{'q'};
      }
      next;
    }

    if (!$opts{'F'} && !IsMailbox($fileHandle))
    {
      warn "** Skipping non-mailbox ASCII file: '$file'\n" unless $opts{'q'};
      next;
    }

    ProcessMailFile($fileHandle,$file,$#files+1);

    $fileHandle->close();
  }
}

#-------------------------------------------------------------------------------

sub Search_Mailbox_Directories
{
  my $file = shift;

  return $file unless exists $ENV{MAIL};

  my $mail_folder = $ENV{MAIL};

  for ('', 'mail/', 'Mail/', 'Mailbox/'){
    my $path_and_file = "$mail_folder/${_}$file";
    return $path_and_file if -e $path_and_file;
  }

  return $file;
}

#-------------------------------------------------------------------------------

sub HandleStandardInput
{
  dprint "Handling STDIN";

  # We have to implement our own -B and -s, because STDIN gets eaten by them
  binmode STDIN;

  my ($testChars,$isEmpty,$isBinary);

  my $fileHandle = new FileHandle;
  $fileHandle->open('-');

  $isEmpty = 0;
  $isBinary = 0;

  my $readResult = read($fileHandle,$testChars,200);

  reportAndExit "Can't read from standard input" unless defined $readResult;

  $isEmpty = 1 if $readResult == 0;

  reportAndExit "No data on standard input" if $isEmpty;

  # Do -B on the data stream
  unless ($isEmpty)
  {
    my $data_length = length $testChars;
    my $bin_length = $testChars =~ tr/[\t\n\x20-\x7e]//c;
    my $non_bin_length = $data_length - $bin_length;
    $isBinary = ($non_bin_length / $data_length) > .70 ? 0 : 1;
  }

  PutBackString($fileHandle,$testChars);

  # If it looks binary and is non-empty, try to uncompress it. Here we're
  # calling another copy of grepmail through the open command.
  if ($isBinary)
  {
    my $filter;

    # This seems to work. I'm not sure what the "proper" way to distinguish
    # between gzip'd and bzip2'd and tzip'd files is.
    if (substr($testChars, 0, 2) eq 'TZ')
    {
      dprint 'Trying to decompress using tzip.';
      $filter = 'tzip -dc';
    }
    elsif (substr($testChars, 0, 2) eq 'BZ')
    {
      dprint 'Trying to decompress using bzip2.';
      $filter = 'bzip2 -d';
    }
    else
    {
      dprint 'Trying to decompress using gunzip2.';
      $filter = 'gzip -dc';
    }

    # Implicit fork
    my $decompressedFileHandle = new FileHandle;
    my $pid = $decompressedFileHandle->open('-|');

    unless (defined($pid))
    {
      warn "** Can't fork to decompress standard input, stopping\n"
        unless $opts{'q'};
      return;
    }

    # In child. Write to the parent, giving it all the data to decompress.
    # We have to do it this way because other methods (e.g. open2) require us
    # to feed the filter as we use the filtered data. This method allows us to
    # keep the remainder of the code the same for both compressed and
    # uncompressed input.
    unless ($pid)
    {
      open(FRONT_OF_PIPE, "|$filter 2>/dev/null")
        or warn "** Can't execute \"$filter\" on STDIN: $!\n"
          unless $opts{'q'};

      while (!eof $fileHandle)
      {
        my $temp = <$fileHandle>;
        print FRONT_OF_PIPE $temp;
      }

      $fileHandle->close()
        or warn "** Can't execute \"$filter\" on STDIN: $!\n"
          unless $opts{'q'};
      # We intentionally don't check for error here. This is because the
      # parent may have aborted, in which case we let it take care of
      # error messages. (e.g. Non-mailbox standard input.)
      close FRONT_OF_PIPE;

      return;
    }

    # In parent
    if (!$opts{'F'} && !IsMailbox($decompressedFileHandle))
    {
      warn "** Skipping non-mailbox standard input\n" unless $opts{'q'};

      $decompressedFileHandle->close()
        or warn "** Can't execute \"$filter\" on STDIN: $!\n"
          unless $opts{'q'};

      return;
    }

    ProcessMailFile($decompressedFileHandle,"Standard input",1);

    $decompressedFileHandle->close()
      or warn "** Can't execute \"$filter\" on STDIN: $!\n"
        unless $opts{'q'};
  }
  # Otherwise process it directly
  else
  {
    if (!$opts{'F'} && !IsMailbox($fileHandle))
    {
      warn "** Skipping non-mailbox standard input\n" unless $opts{'q'};
      return;
    }

    ProcessMailFile($fileHandle,"Standard input",1);

    $fileHandle->close()
      or warn "** Can't process STDIN: $!\n"
        unless $opts{'q'};
  }
}

#-------------------------------------------------------------------------------

# Checks to see if there is data on a filehandle, without reading that data.

sub DataOnFileHandle
{
  my $fileHandle = shift;

  my $buffer = <$fileHandle>;

  return 0 unless defined $buffer;

  PutBackString($fileHandle,$buffer);

  return $buffer ? 1 : 0;
}

#-------------------------------------------------------------------------------

# Puts a string back on a file handle

sub PutBackString
{
  my $fileHandle = shift;
  my $string = shift;

  for (my $char_position=length($string)-1;$char_position >=0; $char_position--)
  {
    $fileHandle->ungetc(ord(substr($string,$char_position,1)));
  }
}

#-------------------------------------------------------------------------------

# Detects whether an ASCII file is a mailbox, based on whether it has
# a line whose prefix is 'From' or 'X-From-Line:' or 'X-Draft-From:',
# and another line whose prefix is 'Received ', 'Date:', 'Subject:',
# 'X-Status:', 'Status:', or 'To:'.

sub IsMailbox
{
  my $fileHandle = shift @_;

  # Read whole paragraphs
  local $/ = "\n\n";

  # Read a paragraph to get the header.
  my $buffer = <$fileHandle>;
  PutBackString($fileHandle,$buffer);
  
  # X-From-Line is used by Gnus, and From is used by normal Unix
  # format. Newer versions of Gnus use X-Draft-From
#  if ($buffer =~ /^(X-Draft-From:|X-From-Line:|From)\s/im &&
#      $buffer =~ /^(Date|Subject|X-Status|Status|To):\s/im)
  if ($buffer =~ /^(X-Draft-From:|X-From-Line:|From:?)\s/im &&
      $buffer =~ /^(Date|To|Bcc):\s/im)
  {
    return 1;
  }
  else
  {
    return 0;
  }
}

#-------------------------------------------------------------------------------

sub Do_Simple_Pattern_Matching
{
  my $email = shift;
  my $email_header = shift;
  my $email_body = shift;
  my $fileHandle = shift;
  my $fileName = shift;
  my $number_files = shift;
  my $numberOfMatches = shift;
  my $line = shift;

  dprint "Checking for early match or abort based on header information.";

  my ($result,$matchesHeader) =
    AnalyzeHeader($$email_header,$fileHandle,$pattern);

  if ($result == $SKIP)
  {
    dprint "Doing an early abort based on header.";
    return ($CONTINUE,$numberOfMatches);
  }

  if ($result == $PRINT)
  {
    dprint "Doing an early printout based on header.";
    if ($opts{'l'})
    {
      print "$fileName\n";
  
      # We can return since we found at least one email that matches.
      return ($DONE,$numberOfMatches);
    }
    elsif ($opts{'r'})
    {
      $numberOfMatches++;
      return ($CONTINUE,$numberOfMatches);
    }
    else
    {
      PrintEmail($fileName,$$email_header,$$email_body,$number_files,$line)
        if $opts{'u'} && NotADuplicate($$email_header) || !$opts{'u'};

      return ($CONTINUE,$numberOfMatches);
    }
  }

  #----------------------------------------------------------------

  my $matchesBody = 0;

  my $signature_offset = undef;

  if ($opts{'S'} && $$email_body =~ m/($opts{'X'})/mg)
  {
    $signature_offset = pos($$email_body) - length($1);
    pos($$email_body) = 0;
  }

  # Ignore the MIME attachments if -M was specified
  if ($opts{'M'} &&
     ($$email_header =~ /^Content-Type:.*?boundary=(?:"([^"]*)"|([^\n]*))/ism))
  {
    my $boundary;
    $boundary = $1 if defined $1;
    $boundary = $2 if defined $2;

    dprint "Found attachments with boundary:\n  $boundary";

    my @attachment_positions;

    # Get each of the binary attachment beginnings and endings.
    while ($$email_body =~ m/\n((?:--)?\Q$boundary\E(?:--)?\n(?:(.*?)\n\n)?)/sg)
    {
      my $position = pos($$email_body) - length($1);
      my $header = $2;

      # Remember that the beginning of the next attachment is the
      # end of the previous.
      $attachment_positions[-1]{'end'} = $position
        if @attachment_positions;

      # If it's the beginning of a binary attachment, store the position
      if (defined $header && $header =~ /^Content-Type:\s+(?!text)/i)
      {
        $attachment_positions[$#attachment_positions+1]{'beginning'} =
          $position;
      }
    }

    pos($$email_body) = 0;

    # Now search the body, ignoring any matches in binary
    # attachments.
    # Avoid perl 5.6 bug which causes spurious warning even though
    # $pattern is defined.
    local $^W = 0 if $] >= 5.006 && $] < 5.8;
    SEARCH: while ($$email_body =~ m/($pattern)/omg)
    {
      my $position = pos($$email_body) - length($1);

      last SEARCH if $opts{'S'} &&
        defined $signature_offset && $position > $signature_offset;

      foreach my $attachment (@attachment_positions)
      {
        next SEARCH
          if ($position > $attachment->{'beginning'} &&
              $position < $attachment->{'end'});
      }

      $matchesBody = 1;
      last;
    }
  }
  else
  {
    # Avoid perl 5.6 bug which causes spurious warning even though
    # $pattern is defined.
    local $^W = 0 if $] >= 5.006 && $] < 5.8;
    if ($$email_body =~ m/($pattern)/omg)
    {
      my $position = pos($$email_body) - length($1);

      $matchesBody = 1 unless $opts{'S'} && 
        defined $signature_offset && $position > $signature_offset;
    }
  }

  #----------------------------------------------------------------

  my $matchesSize = length $$email < $opts{'s'};

  #----------------------------------------------------------------

  dprint "Checking for early match or abort based on header, body, " .
    "and size information.";

  my $isMatch = 1;
  
  $isMatch = 0 if  $opts{'s'} && !$matchesSize ||
    $opts{'b'} && !$matchesBody ||
    $opts{'h'} && !$matchesHeader ||
    !$opts{'b'} && !$opts{'h'} && !($matchesBody || $matchesHeader);

  if (!$isMatch && !$opts{'v'})
  {
    dprint "Doing an early abort based on header, body, and size.";
    return ($CONTINUE,$numberOfMatches);
  }
  elsif (!$isMatch && $opts{'v'})
  {
    dprint "Doing an early printout based on header, body, and size.";
  
    if ($opts{'l'})
    {
      print "$fileName\n";
      
      # We can return since we found at least one email that matches.
      return ($DONE,$numberOfMatches);
    }
    elsif ($opts{'r'})
    {
      $numberOfMatches++;
      return ($CONTINUE,$numberOfMatches);
    }
    else
    {
      ConvertEmailToMboxAndPrintIt($fileName,$$email_header,
        $$email_body,$number_files,$line)
          if $opts{'u'} && NotADuplicate($$email_header) || !$opts{'u'};

      return ($CONTINUE,$numberOfMatches);
    }
  }

  #----------------------------------------------------------------

  dprint "Checking date constraint.";

  $isMatch = 1;

  {
    my $matchesDate = CheckDate($email_header);
    $isMatch = 0 if defined $opts{'d'} && !$matchesDate;

    dprint "Email matches date constraint\n"
      if defined $opts{'d'} && $matchesDate;
    dprint "Email doesn't match date constraint\n"
      if defined $opts{'d'} && !$matchesDate;
  }

  $isMatch = !$isMatch if $opts{'v'};

  # If the match occurred in the right place...
  if ($isMatch)
  {
    dprint "Email matches all patterns and constraints.";

    if ($opts{'l'})
    {
      print "$fileName\n";

      # We can return since we found at least one email that matches.
      return ($DONE,$numberOfMatches);
    }
    elsif ($opts{'r'})
    {
      $numberOfMatches++;
    }
    else
    {
      ConvertEmailToMboxAndPrintIt($fileName,$$email_header,
        $$email_body,$number_files,$line)
          if $opts{'u'} && NotADuplicate($$email_header) || !$opts{'u'};
    }
  }
  else
  {
    dprint "Email did not match all patterns and constraints.";
  }

  return ($CONTINUE,$numberOfMatches);
}

#-------------------------------------------------------------------------------

sub Do_Complex_Pattern_Matching
{
  my $email = shift;
  my $email_header = shift;
  my $email_body = shift;
  my $fileHandle = shift;
  my $fileName = shift;
  my $number_files = shift;
  my $numberOfMatches = shift;
  my $line = shift;

  my $modified_pattern = $pattern;

  $modified_pattern =~ s/\$email_header\b/\$\$email_header/g;
  $modified_pattern =~ s/\$email_body\b/\$\$email_body/g;
  $modified_pattern =~ s/\$email\b/\$\$email/g;
  $modified_pattern =~ s#(=~\s*)/([^/]*)/#$1/$2/om#g;

  my $matchesEmail;
  eval " \$matchesEmail = $modified_pattern ? 1 : 0 ";

  #----------------------------------------------------------------

  my $matchesSize = length $$email < $opts{'s'};

  #----------------------------------------------------------------

  dprint "Checking for early match or abort based on header, body, " .
    "and size information.";

  my $isMatch = 1;
  
  $isMatch = 0 if  $opts{'s'} && !$matchesSize ||
    !$matchesEmail;

  if (!$isMatch && !$opts{'v'})
  {
    dprint "Doing an early abort based on header, body, and size.";
    return ($CONTINUE,$numberOfMatches);
  }
  elsif (!$isMatch && $opts{'v'})
  {
    dprint "Doing an early printout based on header, body, and size.";
  
    if ($opts{'l'})
    {
      print "$fileName\n";
      
      # We can return since we found at least one email that matches.
      return ($DONE,$numberOfMatches);
    }
    elsif ($opts{'r'})
    {
      $numberOfMatches++;
      return ($CONTINUE,$numberOfMatches);
    }
    else
    {
      ConvertEmailToMboxAndPrintIt($fileName,$$email_header,
        $$email_body,$number_files,$line)
          if $opts{'u'} && NotADuplicate($$email_header) || !$opts{'u'};

      return ($CONTINUE,$numberOfMatches);
    }
  }

  #----------------------------------------------------------------

  dprint "Checking date constraint.";

  $isMatch = 1;

  {
    my $matchesDate = CheckDate($email_header);
    $isMatch = 0 if defined $opts{'d'} && !$matchesDate;

    dprint "Email matches date constraint\n"
      if defined $opts{'d'} && $matchesDate;
    dprint "Email doesn't match date constraint\n"
      if defined $opts{'d'} && !$matchesDate;
  }

  $isMatch = !$isMatch if $opts{'v'};

  # If the match occurred in the right place...
  if ($isMatch)
  {
    dprint "Email matches all patterns and constraints.";

    if ($opts{'l'})
    {
      print "$fileName\n";

      # We can return since we found at least one email that matches.
      return ($DONE,$numberOfMatches);
    }
    elsif ($opts{'r'})
    {
      $numberOfMatches++;
    }
    else
    {
      ConvertEmailToMboxAndPrintIt($fileName,$$email_header,
        $$email_body,$number_files,$line)
          if $opts{'u'} && NotADuplicate($$email_header) || !$opts{'u'};
    }
  }
  else
  {
    dprint "Email did not match all patterns and constraints.";
  }

  return ($CONTINUE,$numberOfMatches);
}

#-------------------------------------------------------------------------------

# This algorithm is complicated by code to short-circuit some
# computations. For example, if the user specified -h but not -b, when
# we can analyze the header for a match and avoid needing to search
# the body, which may be much larger.

sub ProcessMailFile
{
  my $fileHandle = shift @_;
  my $fileName = shift @_;
  my $number_files = shift @_;

  my $numberOfMatches = 0;

  reset_file($fileHandle);

  # This is the main loop. It's executed once for each email
  while(1)
  {
    dprint "Reading email";

    my ($status,$email,$line) = read_email();
    last unless $status == 1;

    my ($email_header,$email_body) = split /\n\n/,$email,2;
    $email_header .= "\n\n";

    PrintEmailStatistics($email) if $opts{'D'};

    #----------------------------------------------------------------

    if ($opts{'E'})
    {
      my $result;
      ($result, $numberOfMatches) = 
        Do_Complex_Pattern_Matching(\$email, \$email_header, \$email_body,
          $fileHandle, $fileName, $number_files, $numberOfMatches, $line);

      return if $result == $DONE;
    }
    else
    {
      my $result;
      ($result, $numberOfMatches) = 
        Do_Simple_Pattern_Matching(\$email, \$email_header, \$email_body,
          $fileHandle, $fileName, $number_files, $numberOfMatches, $line);

      return if $result == $DONE;
    }

  }

  print "$fileName: $numberOfMatches\n" if ($opts{'r'});
}

#-------------------------------------------------------------------------------

# Checks that an email is not a duplicate of one already printed. This should
# only be called when $opts{'u'} is true. Also, as a side-effect, it updates
# the %message_ids_seen when it sees an email that hasn't been printed yet.

{
my $tried_to_load_digest_md5;

sub NotADuplicate
{
  my $email_header = shift;

  my ($message_id) = $email_header =~ /^Message-Id:\s*<([^>]+)>/mi;

  if (defined $message_id)
  {
    dprint "Checking uniqueness of message id: $message_id";
  }
  else
  {
    dprint "Email does not have a message id";

    # Try to load Digest::MD5 if we haven't already
    unless (defined $tried_to_load_digest_md5)
    {
      $tried_to_load_digest_md5 = 1;

      if (eval "require Digest::MD5")
      {
        dprint "Digest::MD5 VERSION: $Digest::MD5::VERSION";
        # To prevent warning about variable being used only once
        my $dummy = $Digest::MD5::VERSION;
      }
      else
      {
        dprint "Digest::MD5 could not be loaded";
      }
    }

    # Now create a message id
    if (defined $Digest::MD5::VERSION)
    {
      $message_id = Digest::MD5::md5_hex($email_header);
      dprint "Generated message id $message_id with Digest::MD5";
    }
    else
    {
      $message_id = $email_header;
      dprint "Using email header as message id.";
    }
  }


  my $result;

  if (exists $message_ids_seen{$message_id})
  {
    $result = 0;
    dprint "Found duplicate message";
  }
  else
  {
    $result = 1;
    dprint "Found non-duplicate message";
    $message_ids_seen{$message_id} = 1;
  }

  return $result;
}
}

#-------------------------------------------------------------------------------

# Get the email author from the header or email. Return undef if it can't be
# found

sub GetFromLine
{
  my $email = shift;

  # Remove the body.
  my $header = $email;
  $header =~ s/\n\n.*/\n/s;

  # Avoid perl 5.6 bug which causes spurious warning even though $header is
  # defined.
  local $^W = 0 if $] >= 5.006 && $] < 5.8;
  if ($header =~ /^(From:\s.*)$/im)
  {
    return $1;
  }
  elsif ($header =~ /^(From\s.*)$/im)
  {
    return $1;
  }
  else
  {
    return undef;
  }
}

#-------------------------------------------------------------------------------

# Get the email author from the header or email. Return undef if it can't be
# found

sub GetSubjectLine
{
  my $email = shift;

  # Remove the body.
  my $header = $email;
  $header =~ s/\n\n.*/\n/s;

  # Avoid perl 5.6 bug which causes spurious warning even though $header is
  # defined.
  local $^W = 0 if $] >= 5.006 && $] < 5.8;
  if ($header =~ /^(Subject:\s.*)$/im)
  {
    return $1;
  }
  else
  {
    return undef;
  }
}

#-------------------------------------------------------------------------------

# Get the email author from the header or email. Return undef if it can't be
# found

sub GetDateLine
{
  my $email = shift;

  # Remove the body.
  my $header = $email;
  $header =~ s/\n\n.*/\n/s;

  # Avoid perl 5.6 bug which causes spurious warning even though $pattern is
  # defined.
  local $^W = 0 if $] >= 5.006 && $] < 5.8;
  if ($header =~ /^(Date:\s.*)$/im)
  {
    return $1;
  }
  else
  {
    return undef;
  }
}

#-------------------------------------------------------------------------------

# Print the email author and subject.

sub PrintEmailStatistics
{
  my $email = shift;

  dprint '-'x70;
  dprint "Processing email:";

  my $author = GetFromLine($email);

  if (defined $author)
  {
    dprint "  $author";
  }
  else
  {
    dprint "  [No from line found]";
  }

  my $subject = GetSubjectLine($email);

  if (defined $subject)
  {
    dprint "  $subject";
  }
  else
  {
    dprint "  [No subject line found]";
  }

  my $date = GetDateLine($email);

  if (defined $date)
  {
    dprint "  $date";
  }
  else
  {
    dprint "  [No subject line found]";
  }

  dprint "  Size: " . length $email;
}

#-------------------------------------------------------------------------------

# Returns:
# A result:
# - $PRINT if the email is a match and we need to print it
# - $SKIP if we should skip the current email and go on to the next one
# - $CONTINUE if we need to keep processing the email.
# A boolean for whether the header matches the pattern.
# A boolean for whether the header has the correct date.

sub AnalyzeHeader($$)
{
  my $email_header = shift;
  my $fileHandle = shift;
  my $pattern = shift;

  # See if the header matches the pattern
  # Avoid perl 5.6 bug which causes spurious warning even though $pattern is
  # defined.
  local $^W = 0 if $] >= 5.006 && $] < 5.8;
  my $matchesHeader = Check_For_Header_Match($email_header,$pattern);

  # See if the email failed the size restriction. If it passes here, we still
  # don't know if the entire length of the email is too long.
  my $matchesSize = length $email_header < $opts{'s'};

  # At this point, we might know enough to print the email, or call for an
  # early abort. See the documentation at the top for information. These
  # conditions have been simplified to save space at the cost of clarity.
  # Also, all date related conditions have been taken out, and will be handled
  # after the pattern is searched for in the body of the email. This is
  # because searching for the pattern is a lot faster than handling the date,
  # and we don't have to handle the date if the pattern isn't found.
  
  # First handle the situations where a date constraint isn't a factor.
  return ($PRINT,1) if (
      (!$opts{'b'} &&  $opts{'h'} && !$opts{'s'} && !defined $opts{'d'} &&
       !$opts{'v'} && $matchesHeader) ||
      (                $opts{'h'} &&  $opts{'s'} && !defined $opts{'d'} &&
        $opts{'v'} && (!$matchesHeader || !$matchesSize)) ||
      (                !$opts{'h'} &&  $opts{'s'} && !defined $opts{'d'} &&
        $opts{'v'} && (!$matchesSize)) ||
      (                $opts{'h'} && !$opts{'s'} && !defined $opts{'d'} &&
        $opts{'v'} && (!$matchesHeader)) ||

      (                $opts{'h'} &&  $opts{'s'} &&  defined $opts{'d'} &&
        $opts{'v'} && (!$matchesHeader || !$matchesSize)) ||
      (               !$opts{'h'} &&  $opts{'s'} &&  defined $opts{'d'} &&
        $opts{'v'} && (!$matchesSize)) ||
      (                $opts{'h'} && !$opts{'s'} &&  defined $opts{'d'} &&
        $opts{'v'} && (!$matchesHeader))
    );

  return ($SKIP,0) if (
      (                $opts{'h'} &&  $opts{'s'} && !defined $opts{'d'} &&
       !$opts{'v'} && (!$matchesHeader || !$matchesSize)) ||
      (               !$opts{'h'} &&  $opts{'s'} && !defined $opts{'d'} &&
       !$opts{'v'} && (!$matchesSize)) ||
      (                $opts{'h'} && !$opts{'s'} && !defined $opts{'d'} &&
       !$opts{'v'} && (!$matchesHeader)) ||
      (!$opts{'b'} &&  $opts{'h'} && !$opts{'s'} && !defined $opts{'d'} &&
        $opts{'v'} && ($matchesHeader)) ||

      (                $opts{'h'} &&  $opts{'s'} &&  defined $opts{'d'} &&
       !$opts{'v'} && (!$matchesHeader || !$matchesSize)) ||
      (               !$opts{'h'} &&  $opts{'s'} &&  defined $opts{'d'} &&
       !$opts{'v'} && (!$matchesSize)) ||
      (                $opts{'h'} && !$opts{'s'} &&  defined $opts{'d'} &&
       !$opts{'v'} && (!$matchesHeader))
    );

  return ($CONTINUE,$matchesHeader);
}

#-------------------------------------------------------------------------------

my $header_pattern = undef;

sub Check_For_Header_Match
{
  my $email_header = shift;
  my $pattern = shift;

  return ($email_header =~ /$pattern/om) || 0 unless $opts{'Y'};

  dprint "Searching individual headers.";

  $email_header =~ s/\n(\s+)/$1/g;

  unless (defined $header_pattern)
  {
    $header_pattern = $opts{'Y'};

    for my $special_header_pattern (keys %HEADER_PATTERNS)
    {
      $header_pattern =~
        s/\Q$special_header_pattern\E/$HEADER_PATTERNS{$special_header_pattern}/g;
    }
  }

  for my $header (split(/\n/, $email_header))
  {
    if ($header =~ /$header_pattern/)
    {
      dprint "Header matched header pattern:\n  $header\n";
      return 1 if $header =~ /$pattern/om;
    }
  }
  
  return 0;
}

#-------------------------------------------------------------------------------

sub ConvertEmailToMboxAndPrintIt
{
  my $fileName = shift;
  my $header = shift;
  my $body = shift;
  my $number_files = shift;
  my $line_number = shift;

  ($header,$body) = ConvertEmailToMbox($header,$body);
  PrintEmail($fileName,$header,$body,$number_files,$line_number);
}

#-------------------------------------------------------------------------------

sub ConvertEmailToMbox
{
  my $header = shift;
  my $body = shift;

  dprint "Making email mbox format.";

  # Check for a Gnus email
  if ($header =~ /^X-From-Line:\s+/)
  {
    $header =~ s/^X-From-Line:\s+/From /;
  }

  return ($header,$body);
}

#-------------------------------------------------------------------------------

sub PrintEmail
{
  my $fileName = shift;
  my $header = shift;
  my $body = shift;
  my $number_files = shift;
  my $line_number = shift;

  dprint "Printing email.";

  if ($opts{'n'})
  {
    # Print line-by-line
    while ($header =~ /([^\n]*\n)/g)
    {
      my $line = $1;

      # Add the mailfolder to the headers if -m was given. Careful
      # about line numbers!
      if ($opts{'m'} && $line eq "\n")
      {
        print "$fileName:" if $number_files > 1;
        print "$line_number:X-Mailfolder: $fileName\n";
      }

      print "$fileName:" if $number_files > 1;
      print "$line_number:";
      print $line;
      $line_number++;
    }

    # Don't print the body if -H is specified
    return if $opts{'H'};

    while ($body =~ /([^\n]*\n)/g)
    {
      my $line = $1;

      print "$fileName:" if $number_files > 1;
      print "$line_number:$line";
      $line_number++;
    }
  }
  else
  {
    $header =~ s/\n\n/\nX-Mailfolder: $fileName\n\n/ if $opts{'m'};

    print $header;

    # Don't print the body if -H is specified
    return if $opts{'H'};

    # Print whatever body we've read already.
    print $body;
  }
}

#-------------------------------------------------------------------------------

# Checks to see if the date in the header matches the date specification. The
# date specification can be $NODATE, meaning that the email doesn't have
# a Date: line.

sub CheckDate($)
{
  my $header = ${shift @_};
  my ($emailDate, $isInDate);
  $emailDate = '';
  $isInDate = 0;

  return 1 unless defined $opts{'d'};

  # RFC 822 allows header lines to be continued on the next line, in
  # which case they must be preceded by whitespace. Let's remove the
  # continuations.
  $header =~ s/\n\s+(\S)/ $1/g;

  if ( ($opts{'a'} && ($header =~ /^Received:\s.*\;\s*(.*?)$/im)) ||
    $header =~ /^Date:\s*(.*)$/im ||
    $header =~ /^From \S+\s+(.*)$/im)
  {
    dprint "Date in email is: $1.";

    $emailDate = str2time($1);

    return IsInDate($emailDate,$dateRestriction,$date1,$date2)
      if defined $emailDate;
    
    warn "** Couldn't parse email date \"$1\". " .
      "Assuming it doesn't match the date constraint **\n";
    warn "  " . GetFromLine($header) . "\n"
        if defined GetFromLine($header);
    warn "  " . GetSubjectLine($header) . "\n"
        if defined GetSubjectLine($header);

    return 0;
  }
  # The email might not have a date. In this case, print out a warning.
  else
  {
    return 1 if $dateRestriction == $NODATE;

    warn "** Couldn't find a date. Assuming email doesn't match the " .
         "date constraint:\n";
    warn "  " . GetFromLine($header) . "\n"
      if defined GetFromLine($header);
    warn "  " . GetSubjectLine($header) . "\n"
      if defined GetSubjectLine($header);

    return 0;
  }
}

#-------------------------------------------------------------------------------

# This function tries to parse a date first with Date::Parse. If Date::Parse
# can't parse the date, then the function tries to use Date::Manip to parse
# it. Returns '' if the date can't be parsed.

{
my $loaded_date_manip = undef;

sub ParseDate
{
  my $date = shift;

  my $parsedDate;

  # First try to parse the date with Date::Parse;
  $parsedDate = str2time($date);
  return $parsedDate if defined $parsedDate;

  # Try to load Date::Manip if we haven't already
  unless (defined $loaded_date_manip)
  {
    if (eval "require Date::Manip")
    {
      $loaded_date_manip = 1;

      dprint "Date::Manip VERSION: $Date::Manip::VERSION";
      # To prevent warning about variable being used only once
      my $dummy = $Date::Manip::VERSION;
    }
    else
    {
      $loaded_date_manip = 0;
    }
  }

  return '' unless $loaded_date_manip;

  my $temp_date = Date::Manip::UnixDate(Date::Manip::ParseDate($date),'%s');

  return '' unless defined $temp_date;
  return $temp_date;
}
}

#-------------------------------------------------------------------------------

# Figure out what kind of date restriction they want, and what the dates in
# question are. An empty date string results in the type of date restriction
# being $NODATE.
sub ProcessDate($)
{
  my $datestring = shift;

  return ($NODATE,'','') if $datestring eq '';

  if ($datestring =~ /^before (.*)/i)
  {
    $dateRestriction = $BEFORE;
    $date1 = ParseDate($1);
    $date2 = '';

    reportAndExit "\"$1\" is not a valid date" unless $date1;
  }
  elsif ($datestring =~ /^(after|since)\s(.*)/i)
  {
    $dateRestriction = $AFTER;
    $date1 = ParseDate($2);
    $date2 = '';

    reportAndExit "\"$2\" is not a valid date" unless $date1;
  }
  elsif ($datestring =~ /^between (.+) and (.+)/i)
  {
    $dateRestriction = $BETWEEN;
    $date1 = ParseDate($1);
    $date2 = ParseDate($2);

    reportAndExit "\"$1\" is not a valid date" unless $date1;
    reportAndExit "\"$2\" is not a valid date" unless $date2;

    # Swap the dates if the user gave them backwards.
    if ($date1 > $date2)
    {
      my $temp;
      $temp = $date1;
      $date1 = $date2;
      $date2 = $temp;
    }

  }
  elsif (ParseDate($datestring) ne '')
  {
    $dateRestriction = $ON;
    $date1 = ParseDate($datestring);
    $date2 = '';
  }
  else
  {
    reportAndExit "\"$datestring\" is an invalid date specification. Use \"$0 -h\" for help";
  }

  return ($dateRestriction,$date1,$date2);
}

#-------------------------------------------------------------------------------

sub IsInDate($$$$)
{
  my ($emailDate,$dateRestriction,$date1,$date2);
  $emailDate = shift @_;
  $dateRestriction = shift @_;
  $date1 = shift @_;
  $date2 = shift @_;

  # Now we do the date checking.
  return 1 if $dateRestriction == $NONE;
  return $emailDate < $date1 if $dateRestriction == $BEFORE;
  return $emailDate > $date1 if $dateRestriction == $AFTER;

  # Since these values are in seconds, we have to make sure that $emailDate
  # is within 24 hours after $date1
  return $emailDate > $date1 && $emailDate-$date1 < 24*60*60 if $dateRestriction == $ON;
  return $emailDate > $date1 && $emailDate < $date2 if $dateRestriction == $BETWEEN;

  return 0;
}

#-------------------------------------------------------------------------------

sub usage
{
<<EOF;
usage: grepmail [--help] [-abBDFhHilmrRuv] [-s size] [-d "datespec"] [[-e] <expr>] <files...>
EOF
}

#-------------------------------------------------------------------------------

sub help
{
<<EOF;
grepmail $VERSION

usage:

  grepmail [--help] [-abBDhHilmrRuv] [-s size] [-d <date-specification>]
    [-X <signature-pattern>] [-Y <header-pattern>] <files...>

  grepmail [--help] [-abBDhHilmrRuv] [-s size] [-d <date-specification>]
    [-X <signature-pattern>] [-Y <header-pattern>] [[-e] <pattern>] <files...>

  grepmail [--help] [-abBDhHilmrRuv] [-s size] [-d <date-specification>]
    [-X <signature-pattern>] [-Y <header-pattern>] [-E <expr>] <files...>

At least one of -s, -d, -u, -e, and -E must be specified, and can appear in
any relative order following the other flags. The -e flag is optional if
pattern appears immediately before -s or -d. Files can be plain ASCII or ASCII
files compressed with gzip, tzip, or bzip2. -E allows for complex pattern
matches involving logical operators. If no file is provided, normal or
compressed ASCII input is taken from STDIN.

-a Use received date instead of sent date for -d matching
-b Search must match body
-d Specify a date range (see below)
-D Debug mode
-e Explicitly name pattern (when searching for strings beginning with "-")
-E Specify a complex search expression
-h Search must match header
-H Print headers but not bodies of matching emails
-i Ignore case in the search expression
-l Output the names of files having an email matching the expression
-M Do not search non-text mime attachments
-m Append "X-Mailfolder: <folder>" to all headers to indicate in which folder
   the match occurred
-n Print the line number info (and filename if necessary) for the emails
-q Quiet mode -- don't output warnings
-r Output the names of the files and the number of emails matching the
   expression
-R Recurse directories
-s Restrict results to emails less than a certain size (in bytes)
-S Ignore signatures
-u Ensure that no duplicate emails are output
-v Output emails that don't match the expression
-V Display the version number
-X Specify a regular expression for the signature separator
-Y Specify a header to search (implies -h)
-Z Not use Mail::Folder::FastReader even if it is on the system
--help Print a help message

Date specifications must be of the form of:
a date like "today", "1st thursday in June 1992" (requires Date::Manip),
  "05/18/93", "12:30 Dec 12th 1880", "8:00pm december tenth",
OR "before", "after", or "since", followed by a date as defined above,
OR "between <date> and <date>", where <date> is defined as above.
EOF
}

#-------------------------------------------------------------------------------

=head1 NAME

grepmail - search mailboxes for mail matching a regular expression

=head1 SYNOPSIS

  grepmail [--help] [-abBDhHilmrRuv] [-s size] [-d <date-specification>]
    [-X <signature-pattern>] [-Y <header-pattern>] <files...>

  grepmail [--help] [-abBDhHilmrRuv] [-s size] [-d <date-specification>]
    [-X <signature-pattern>] [-Y <header-pattern>] [[-e] <pattern>] <files...>

  grepmail [--help] [-abBDhHilmrRuv] [-s size] [-d <date-specification>]
    [-X <signature-pattern>] [-Y <header-pattern>] [-E <expr>] <files...>

=head1 DESCRIPTION

=over 2

I<grepmail> looks for mail messages containing a pattern, and prints the
resulting messages on standard out.

By default I<grepmail> looks in both header and body for the specified pattern.

When redirected to a file, the result is another mailbox, which can, in turn,
be handled by standard User Agents, such as I<elm>, or even used as input for
another instance of I<grepmail>.

At least one of B<-E>, B<-e>, B<-d>, B<-s>, or B<-u> must be specified. The
pattern is optional if B<-d>, B<-s>, and/or B<-u> is used. The B<-e> flag is
optional if there is no file whose name is the pattern. The B<-E> option can
be used to specify complex search expressions involving logical operators.
(See below.)

If a mailbox can not be found, grepmail searches the $home/mail, $home/Mail,
$home/Mailbox directories (or the directory specified by the MAIL environment
variable).

=back

=head1 OPTIONS AND ARGUMENTS

Many of the options and arguments are analogous to those of grep.

=over 2

=item B<pattern>

The pattern to search for in the mail message.  May be any Perl regular
expression, but should be quoted on the command line to protect against
globbing (shell expansion). To search for more than one pattern, use the form
"(pattern1|pattern2|...)".

=item B<mailbox>

Mailboxes must be traditional, UNIX C</bin/mail> mailbox format.  The
mailboxes may be compressed by gzip, tzip, or bzip2, in which case
gunzip, tzip, or bzip2 must be installed on the system.

If no mailbox is specified, takes input from stdin, which can be compressed or
not. grepmail's behavior is undefined when ASCII and binary data is piped
together as input.

=item B<-a>

Use arrival date instead of sent date.

=item B<-b>

Asserts that the pattern must match in the body of the email. 

=item B<-D>

Enable debug mode, which prints diagnostic messages.

=item B<-d>

Date specifications must be of the form of:
  - a date like "today", "yesterday", "5/18/93", "5 days ago", "5 weeks ago",
  - OR "before", "after", or "since", followed by a date as defined above,
  - OR "between <date> and <date>", where <date> is defined as above.

Simple date expressions will first be parsed by Date::Parse. If this fails,
grepmail will attempt to parse the date with Date::Manip, if the module is
installed on the system. Use an empty pattern (i.e. B<-d "">) to find emails
without a "Date: ..." line in the header.

=item B<-E>

Specify a complex search expression using logical operators. The current
syntax allows the user to specify search expressions using Perl syntax. Three
values can be used: $email (the entire email message), $email_header (just the
header), or $email_body (just the body). A search is specified in the form
"$email =~ /pattern/", and multiple searches can be combined using "&&" and
"||" for "and" and "or".

For example, the expression

  $email_header =~ /^From: .*\@coppit.org/ && $email =~ /grepmail/i

will find all emails which originate from coppit.org (you must escape the "@"
sign with a backslash), and which contain the keyword "grepmail" anywhere in
the message, in any capitalization.

B<-E> is incompatible with B<-b>, B<-h>, and B<-e>. B<-i>, B<-M>, B<-S>, and
B<-Y> have not yet been implemented.

NOTE: The syntax of search expressions may change in the future. In
particular, support for size, date, and other constraints may be added. The
syntax may also be simplified in order to make expression formation easier to
use (and perhaps at the expense of reduced functionality).

=item B<-e>

Explicitly specify the search pattern. This is useful for specifying patterns
that begin with "-", which would otherwise be interpreted as a flag.

=item B<-F>

Force processing of ASCII files which don't appear to be mailboxes.

=item B<-h>

Asserts that the pattern must match in the header of the email.

=item B<-H>

Print the header but not body of matching emails.

=item B<-i>

Make the search case-insensitive (by analogy to I<grep -i>).

=item B<-l>

Output the names of files having an email matching the expression, (by analogy
to I<grep -l>).

=item B<-M>

Causes grepmail to ignore non-text MIME attachments. This removes false
positives resulting from binaries encoded as ASCII attachments.

=item B<-m>

Append "X-Mailfolder: <folder>" to all email headers, indicating which folder
contained the matched email.

=item B<-n>

Prefix each line with line number information. If multiple files are
specified, the filename will precede the line number. NOTE: When used in
conjunction with B<-m>, the X-Mailfolder header has the same line number as
the next (blank) line.

=item B<-q>

Quiet mode. Suppress the output of warning messages about non-mailbox files,
directories, etc.

=item B<-r>

Generate a report of the names of the files containing emails matching the
expression, along with a count of the number of matching emails.

=item B<-R>

Causes grepmail to recurse any directories encountered.

=item B<-s>

Return emails smaller than the size (in bytes) specified with this flag.

=item B<-S>

Ignore signatures. The signature consists of everything after a line
consisting of "-- ".

=item B<-u>

Output only unique emails, by analogy to I<sort -u>. Grepmail determines email
uniqueness by the Message-ID header.

=item B<-v>

Invert the sense of the search, by analogy to I<grep -v>. This results in the
set of emails printed being the complement of those that would be printed
without the B<-v> switch.

=item B<-X>

Specify a regular expression for the signature separator. By default this
pattern is '^-- $'.

=item B<-Y>

Specify a pattern which indicates specific headers to be searched. The search
will automatically treat headers which span multiple lines as one long line.
This flag implies B<-h>.

In the style of procmail, special strings in the pattern will be expanded as
follows:

=over 2

If the regular expression contains "^TO:" it will be  substituted by
"^((Original-)?(Resent-)?(To|Cc|Bcc)|(X-Envelope|Apparently(-Resent)?)-To):",
which should match all headers with destination addresses.

If the regular expression contains "^FROM_DAEMON:" it  will
be substituted by "(^(Mailing-List:|Precedence:.*(junk|bulk|list)|To: Multiple recipients of |(((Resent-)?(From|Sender)|X-Envelope-From):|>?From )([^>]*[^(.%@a-z0-9])?(Post(ma?(st(e?r)?|n)|office)|(send)?Mail(er)?|daemon|m(mdf|ajordomo)|n?uucp|LIST(SERV|proc)|NETSERV|o(wner|ps)|r(e(quest|sponse)|oot)|b(ounce|bs\.smtp)|echo|mirror|s(erv(ices?|er)|mtp(error)?|ystem)|A(dmin(istrator)?|MMGR|utoanswer))(([^).!:a-z0-9][-_a-z0-9]*)?[%@>\t ][^<)]*(\(.*\).*)?)?",
which should catch mails coming from most daemons.

If  the regular expression contains "^FROM_MAILER:" it will be substituted by
"(^(((Resent-)?(From|Sender)|X-Envelope-From):|>?From)([^>]*[^(.%@a-z0-9])?(Post(ma(st(er)?|n)|office)|(send)?Mail(er)?|daemon|mmdf|n?uucp|ops|r(esponse|oot)|(bbs\.)?smtp(error)?|s(erv(ices?|er)|ystem)|A(dmin(istrator)?|MMGR))(([^).!:a-z0-9][-_a-z0-9]*)?[%@>\t][^<)]*(\(.*\).*)?)?$([^>]|$))" (a stripped down version of
"^FROM_DAEMON:"), which should catch mails coming from most mailer-daemons.

=back

=item B<-Z>

By default, grepmail uses the Mail::Folder::FastReader module if it is on the
system. This switch tells it to use the slower Perl implementation instead.

=item B<--help>

Print a help message summarizing the usage.

=back

=head1 EXAMPLES

Count the number of emails. ("." matches every email.)

  grepmail -r . sent-mail

Get all email larger than 2000 bytes about books

  grepmail books -s 2000 sent-mail

Get all email that you mailed yesterday

  grepmail -d yesterday sent-mail

Get all email that you mailed before the first thursday in June 1998 that
pertains to research (requires Date::Manip):

  grepmail research -d "before 1st thursday in June 1992" sent-mail

Get all email that you mailed before the first of June 1998 that
pertains to research:

  grepmail research -d "before 6/1/92" sent-mail

Get all email you received since 8/20/98 that wasn't about research or your
job, ignoring case:

  grepmail -iv "(research|job)" -d "since 8/20/98" saved-mail

Get all email about mime but not about Netscape. Constrain the search to match
the body, since most headers contain the text "mime":

  grepmail -b mime saved-mail | grepmail Netscape -v

Print a list of all mailboxes containing a message from Rodney. Constrain the
search to the headers, since quoted emails may match the pattern:

  grepmail -hl "^From.*Rodney" saved-mail*

Find all emails with the text "Pilot" in both the header and the body:

  grepmail -hb "Pilot" saved-mail*

Print a count of the number of messages about grepmail in all saved-mail
mailboxes:

  grepmail -br grepmail saved-mail*

Remove any duplicates from a mailbox:

  grepmail -u saved-mail

Convert a Gnus mailbox to mbox format:

  grepmail . gnus-mailbox-dir/* > mbox

Search for all emails to or from an address (taking into account wrapped
headers and different header names):

  grepmail -Y '(^TO:|^From:)' my@email.address saved-mail

Find all emails from postmasters:

  grepmail -Y '^FROM_MAILER:' . saved-mail

=head1 FILES

grepmail will I<not> create temporary files while decompressing compressed
archives. The last version to do this was 3.5. While the new design uses
more memory, the code is much simpler, and there is less chance that email
can be read by malicious third parties. Memory usage is determined by the size
of the largest email message in the mailbox.

=head1 ENVIRONMENT

The MAIL environment variable can be used to specify the default mail
directory. This directory will be searched if the specified mailbox can not be
found directly.

=head1 BUGS AND LIMITATIONS

=over 2

=item Test case 1 fails on some platforms

Bug not squashed yet. Any info would be appreciated.

=item File names that look like flags cause problems.

In some special circumstances, grepmail will be confused by files whose names
look like flags. In such cases, use the B<-e> flag to specify the search
pattern.

=back

=head1 AUTHOR

David Coppit, <david@coppit.org>, http://coppit.org/

=head1 SEE ALSO

elm(1), mail(1), grep(1), perl(1), printmail(1), Mail::Internet(3),
procmailrc(5). Crocker, D.  H., Standard for the Format of Arpa Internet Text
Messages, RFC822.

=cut
