#!/usr/bin/perl

# Written by Jason Rennie (jr6b@andrew.cmu.edu)

# Format of data files is as follows:
# line 1: target value names each separated by a space
# line 2: # of total words for all documents in each target value each
#         separated by a space
# line 3: number of documents for each category
# line 4: a vocab word, followed by pairs of category numbers and the word
#         frequency for that category

# When a vocab word is refiled it is assumed that this is the desired
#      target value (or folder, in this case).

# tv = target value

# This program is written in an attempt to emulate the MH refile command
#   while adding the benefit of allowing it to learn user preference with
#   a bayesian-style learning algorithm.  
# The only options which it currently emulates are '-src +folder' and '-file
#   file'.  It supports refiling multiple messages, but does not support
#   filing messages into multiple folders.

open(TMP, ">> /tmp/irefile.info")
    || warn "Could not open /tmp/irefile.info: $!\n";

print TMP "irefile called ";
 print TMP "(@ARGV)\n";

$home_dir = $ENV{"HOME"}."/";
 
print TMP "Home dir = $home_dir\n";

open(MH, "< $home_dir.mh_profile") 
    || die "No .mh_profile found: $!\n";

while (<MH>) {
    if (m/Path:\s*(\w+)/) {
	$mail_path = $1;
	$mail_path .= "/";
    }
    if (m/draft-folder:\s*(\w+)/) {
	$draft_folder = $1;
    }
    if (m/context:\s*(\S+)/i) {
	$context_file = $1;
    }
    if (m/Inbox:\s*(\S+)/i) {
	$inbox_folder = $1;
    }
}

if (!$context) {
    $context_file = $home_dir.$mail_path."context";
}
if (!$inbox_folder) {
    $inbox_folder = "inbox";
}

print TMP "Mail Path = $mail_path\n";
print TMP "Draft Fodler = $draft_folder\n";
print TMP "Context File = $context_file\n";
print TMP "Inbox Folder = $inbox_folder\n";

# gets command line arguments - prints them to temporary file

$i = 0;
while ($i <= $#ARGV) {

    print TMP $ARGV[$i]." ";

    if ($ARGV[$i] =~ m/\-src/) 
    {
	if ($ARGV[$i+1] =~ m/^\+(\S+)/) 
	{
	    $source_folder = $1;
	    print TMP "source folder = $source_folder\n";
	    $i++;
	} 
	else {
	    print TMP "Error: folder name lacks preceeding '+' or is";
	    print TMP " in wrong format.\n";
	    &print_usage;
	}
    } elsif ($ARGV[$i] =~ m/\-file/) 
    {
	$source_file = $ARGV[$i+1];
	print TMP "source file = $source_file\n";
	$i++;
    } 
    elsif ($ARGV[$i] =~ /^-help/ || $ARGV[$i] =~ /^-h/)
    {
	&print_usage;
    }
    elsif ($ARGV[$i] =~ /^\-draft/
	   || $ARGV[$i] =~ /^\-nolink/
	   || $ARGV[$i] =~ /^\-preserve/
	   || $ARGV[$i] =~ /^\-nopreserve/
	   || $ARGV[$i] =~ /^\-normmproc/) { }
    elsif ($ARGV[$i] =~ /^\-rmmproc/) { $i++; }
    elsif ($ARGV[$i] =~ /^\-link/)
    {
	if (-x "/usr/bin/mh/refile.bak")
	{
	    print TMP "Using refile.bak to refile message: refile.bak ", join(" ",@ARGV), "\n";
	    exec "/usr/bin/mh/refile.bak ".join(" ",@ARGV);
	}
    }
    elsif ($ARGV[$i] =~ m/^\+([^+]+)/)
    {
	$dest_folder = $1;
	print TMP "destination folder = $dest_folder\n";
    } 
    else 
    {
	$orig_message[$#orig_message+1] = $ARGV[$i];
	print TMP "message name = $ARGV[$i]\n";
    }
    $i++;
}   

# if the '-file' command line option was used, disregard any message names
#   included on the command line.  Warn the user if '-src' or message names
#   were included as command line options

if ($source_file)
{
    if (($msg_count = @orig_message) > 0) {
	print "Messages other than $source_file are being ignored.\n";
	print TMP "Messages other than $source_file are being ignored.\n";
    }
    if ($source_folder) {
	print "\'-src\' command line option is being ignored.\n";
	print TMP "\'-src\' command line option is being ignored.\n";
	$source_folder = "";
    }

    $#orig_message = -1;
    $orig_message[0] = $source_file;
}

# updates accuracies file ~/.idata_accuracy

$acc_file = $home_dir.".idata_accuracy";

$filters = 0;
$refiles = @orig_message;      # number of original messages

if (open(ACC, $acc_file))
{
    $line = <ACC>;
    $line =~ m/filters\s*=\s*(\d*)\s*refiles\s*=\s*(\d*)/;

    $filters += $1;
    $refiles += $2;

    if ($filters > 0) {
	$accuracy = int(($filters - $refiles)/$filters * 10000)/100;
    } else {
	$accuracy = 0;
    }

    print TMP "filters = $filters  refiles = $refiles\n";
    print TMP "Accuracy = $accuracy \%\n";

    close(ACC);
}

if (open(ACC, "> $acc_file"))
{
    # reset accuracies file if unusual situation has come up
    if ($refiles > $filters && $refiles > 30) {
	$refiles = 0;
	$filters = 0;
	$accuracy = 0;
    }

    print ACC "filters = $filters  refiles = $refiles\n";
    if ($accuracy) { print ACC "Accuracy = $accuracy \%\n"; }
    close(ACC);
} else {
    print TMP "Was not able to write accuracies file\n";
}


$tot_words = 0;

# target values and 'n' both start with index 1
$tv[0] = 0;

$data_file = $home_dir.".idata";

if (open(DATA, "< $data_file")) 
{
    # reads in category names
    $line = <DATA>;
    while ($line =~ s/(\S+)\s+//) 
    {
	$tv[$#tv+1] = $1;
    }

    # reads in total number of words for each category
    $line = <DATA>;
    $i = 1;
    while ($line =~ s/(\d+)\s+//) 
    {
	$n{$tv[$i++]} = $1;
    }
    if ($i <= $#tv) { &ouch("Bad data file format - line #2"); }

    # reads in number of documents for each category
    $line = <DATA>;
    $i = 1;
    while ($line =~ s/(\d+)\s+//)
    {
	$docs{$tv[$i++]} = $1;
	$tot_docs += $1;
    }
    if ($i <= $#tv) { &ouch("Bad data file format - line #3"); }

    # reads in actual word frequencies
    $word[0] = "";

    while (($line = <DATA>) && (!($line =~ m/^\s*$/)))  {
	$tot_words++;

	# reads word
	$line =~ s/^(\w+)\s+// || &ouch("Couldn't read vocab word\n");
	$word[$#word+1] = $1;
	$index{$1} = $#word;
	
	# reads age of word
	$line =~ s/^(\d+)\s+// || &ouch("Couldn't read age of word");
	$age{$word[$#word]} = $1;

	# reads in frequency per category   cat_num:freq
	while ($line =~ s/(\d+):(\d+)\s+//) 
	{
	    $nk{$word[$#word].",".$tv[$1]} = $2;
	    $sum_nk{$word[$#word]} += $2;
	}
	
	if (!($line =~ m/^\s*$/)) { &ouch("Vocab line # $tot_words has bad format"); }
    }
    close(DATA);
    print TMP "Read data file\n";
}

# Determine highest message number in destination folder

$high_msg_num = 0;

opendir(DEST, "$home_dir$mail_path$dest_folder/")
    || &ouch("Could not open destination folder: $!\n");

while ($fn = readdir(DEST))
{
    # looks at first block of digits
    $fn =~ s/^(\d+)\D+/\1/;

    if ($fn > $high_msg_num)
    {
	$high_msg_num = $fn;
    }
}

$high_msg_num++;

closedir(DEST);

# refiles messages one at a time

foreach $msg (@orig_message) 
{
    if ($source_folder ne "") 
    {
	print TMP "Opening $home_dir$mail_path${source_folder}/$msg\n";
	if (!open(MSG, "< $home_dir$mail_path$source_folder/$msg"))
	{
	    print TMP "Could not open $home_dir$mail_path$source_folder/$msg: $!\n";
	    next;
	}
    } 
    else 
    {
	print "Opening $msg\n";
	if (!open(MSG, "< $msg"))
	{
	    print TMP "Could not open $msg: $!\n";
	    next;
	}
    }

    @lines = <MSG>;

# make copy of message in new folder

    if (-x "/usr/bin/mh/refile.bak")
    {
	$use_refile_bak = 1;
	print TMP "Waiting to refile messages\n";

	# This method assumes that nothing happens to /usr/bin/mh/refile.bak
	#   during execution of the program
    }
    elsif (open(MSG, "> $home_dir$mail_path$dest_folder/$high_msg_num"))
    {
	system "/usr/bin/mh/rmm +$source_folder $msg";
	print TMP "/usr/bin/mh/rmm +$source_folder $msg\n";

	print TMP "Creating refiled message: $home_dir$mail_path$dest_folder/$high_msg_num\n";

	foreach $line (@lines)
	{
	    print MSG $line;
	}

	$high_msg_num++;
	close(MSG);
    }

    # Update the .idata file to reflect new knowldege

    print "Input file: \n";

    foreach $line (@lines) 
    {
	$weight = 1;
	if ($line =~ s/^subject://i) { $weight = 5; }
	if ($line =~ s/^cc://i) { $weight = 3; }
	if ($line =~ s/^to://i) { $weight = 3; }
	if ($line =~ s/^from://i) { $weight = 5; }

	while ($line =~ s/([A-Za-z]+)[^A-Za-z]+//) 
	{
	    $wrd = $1;
            # get rid of tiny words
	    if (length($wrd) > 2) 
	    {
		$wrd =~ tr/A-Z/a-z/;
		print $wrd." ";
	 	if ($index{$wrd} == 0) 
		{
		    $word[$#word+1] = $wrd;
		    $index{$wrd} = $#word;
		    $nk{$wrd.",".$dest_folder} += $weight;
		} else 
		{
		    $nk{$wrd.",".$dest_folder} += $weight;
		}
		$n{$dest_folder} += $weight;
	    }
	}
    }
    close(MSG);

    $docs{$dest_folder}++;
    foreach $w (@word) {
	$age{$w}++;
    }
}

# reopen the data file and write the learned information

print "Home dir = $home_dir\n";
print "Mail path = $mail_path\n";

open(DATA, "> $home_dir.idata")
    || &ouch("Couldn't open data file\n");

open(FOLDERS, "< $home_dir${mail_path}.folders")
    || &ouch("Couldn't find .folders\n");

# prints category names to data file (uses .folders file to determine folders)
$i = 0;
while ($folder = <FOLDERS>) 
{
    chop $folder;
    print DATA $folder." ";
    $f_tv[++$i] = $folder;
}

close(FOLDER);

print DATA "\n";

$tv[0] = "nil";
$n{"nil"} = 0;

# prints total word frequency values
for ($i=1; $i <= $#f_tv; $i++) 
{
    if ($n{$f_tv[$i]} > 0) {
	print DATA $n{$f_tv[$i]}." ";
    } else {
	print DATA "0 ";
    }
}
print DATA "\n";

# prints number of documents for each category
for ($i=1; $i <= $#f_tv; $i++) 
{
    if ($n{$f_tv[$i]} > 0) {
	print DATA $docs{$f_tv[$i]}." ";
    } else {
	print DATA "0 ";
    }
}
print DATA "\n";

for ($i=1; $i <= $#word; $i++) 
{
    # if the word is infrequent, discard it (don't print it to DATA)
    if (($age{$word[$i]} < 16) || ($sum_nk{$word[$i]} > 3))
    {
	print DATA $word[$i]." ".$age{$word[$i]}." ";

	for ($j=1; $j <= $#f_tv; $j++) {
	    if ($nk{$word[$i].",".$f_tv[$j]} > 0) {
		print DATA $j.":".$nk{$word[$i].",".$f_tv[$j]}." ";
	    }
	}
	print DATA "\n";
    }
}

close(DATA);

if ($use_refile_bak = 1) {
    # refile all the messages if we haven't done it yet
    print TMP "Using refile.bak to refile message: refile.bak ", join(" ",@ARGV), "\n";
    system "/usr/bin/mh/refile.bak ".join(" ",@ARGV);
}

print TMP "\n";
close(TMP);

exit 0;

################

sub ouch {
    local($string) = @_;
    print TMP "$string: \n";
    die "$string: $!\n";
}

sub print_usage {
    print "Usage: irefile [msgs] [-src +folder] [-file file] [MH options] +folder\n";
    print "       irefile -help\n\n";
    print " msgs         => text files to be refiled into +'folder'\n";
    print " -src +folder => looks for 'msgs' in specified folder\n";
    print " -file file   => disregards previous two options; uses 'file' as\n";
    print "                 source message\n";
    print " +folder      => folder to direct messages into\n";
    print " MH options   => all command line options will be passed to the\n";
    print "                 MH refile, so all MH refile options are valid\n\n";
    exit(0);
}
