#!/usr/bin/perl

# Format of data files is as follows:
# line 1: target value names each separated by a space
# line 2: # of total words for all documents in each target value each
#         separated by a space
# line 3: number of documents for each category
# line 4: a vocab word, followed by pairs of category numbers and the word
#         frequency for that category

# When a vocab word is refiled it is assumed that this is the desired
#      target value (or folder, in this case).

# tv = target value

# This program is written in an attempt to emulate the MH refile command
#   while adding the benefit of allowing it to learn user preference with
#   a bayesian-style learning algorithm.  
# The only options which it currently emulates are '-src +folder' and '-file
#   file'.  It supports refiling multiple messages, but does not support
#   filing messages into multiple folders.

open(TMP, ">> /tmp/irefile.info")
    || warn "Could not open /tmp/irefile.info: $!\n";

print TMP "irefile called ";
print TMP "(@ARGV)\n";

$home_dir = $ENV{"HOME"}."/";

print TMP "Home dir = $home_dir\n";

open(MH, "< $home_dir.mh_profile") 
    || die "No .mh_profile found: $!\n";

while (<MH>) {
    if (m/Path: (\w+)/) {
	$mail_path = $1;
	$mail_path .= "/";
    }
    if (m/draft-folder: (\w+)/) {
	$draft_folder = $1;
    }
}

print TMP "Mail Path = $mail_path\n";


# gets command line arguments - prints them to temporary file

$i = 0;
while ($i <= $#ARGV) {

    print TMP $ARGV[$i]." ";

    if ($ARGV[$i] =~ m/\-src/) 
    {
	if ($ARGV[$i+1] =~ m/^\+(\S+)/) 
	{
	    $source_folder = $1;
	    print TMP "source folder = $source_folder\n";
	    $i++;
	} 
	else {
	    print TMP "Error: folder name lacks preceeding '+' or is";
	    print TMP " in wrong format.\n";
	    &print_usage;
	}
    } elsif ($ARGV[$i] =~ m/\-file/) 
    {
	$source_file = $ARGV[$i+1];
	print TMP "source file = $source_file\n";
	$i++;
    } 
    elsif ($ARGV[$i] =~ m/^\+([^+]+)/)
    {
	$dest_folder = $1;
	print TMP "destination folder = $dest_folder\n";
    } 
    else 
    {
	$orig_message[$#orig_message+1] = $ARGV[$i];
	print TMP "message name = $ARGV[$i]\n";
    }
    $i++;
}   

# updates accuracies file ~/.idata_accuracy

$acc_file = $home_dir.".idata_accuracy";

$filters = 0;
$refiles = @orig_message;

if (open(ACC, $acc_file))
{
    $line = <ACC>;
    $line =~ m/filters\s*=\s*(\d*)\s*refiles\s*=\s*(\d*)/;

    $filters += $1;
    $refiles += $2;

    if ($filters > 0) {
	$accuracy = int(($filters - $refiles)/$filters * 10000)/100;
    } else {
	$accuracy = 0;
    }

    print TMP "filters = $filters  refiles = $refiles\n";
    print TMP "Accuracy = $accuracy \%\n";

    close(ACC);
}

if (open(ACC, "> $acc_file"))
{
    print ACC "filters = $filters  refiles = $refiles\n";
    if ($accuracy) { print ACC "Accuracy = $accuracy \%\n"; }
    close(ACC);
} else {
    print TMP "Was not able to write accuracies file\n";
}


$tot_words = 0;

# target values and 'n' both start with index 1
$tv[0] = 0;

$data_file = $home_dir.".idata";

if (open(DATA, "< $data_file")) 
{
    # reads in category names
    $line = <DATA>;
    while ($line =~ s/(\S+)\s+//) 
    {
	$tv[$#tv+1] = $1;
    }

    # reads in total number of words for each category
    $line = <DATA>;
    $i = 1;
    while ($line =~ s/(\d+)\s+//) 
    {
	$n{$tv[$i++]} = $1;
    }
    if ($i <= $#tv) { &ouch("Bad data file format - line #2"); }

    # reads in number of documents for each category
    $line = <DATA>;
    $i = 1;
    while ($line =~ s/(\d+)\s+//)
    {
	$docs{$tv[$i++]} = $1;
	$tot_docs += $1;
    }
    if ($i <= $#tv) { &ouch("Bad data file format - line #3"); }

    # reads in actual word frequencies
    $word[0] = "";

    while (($line = <DATA>) && (!($line =~ m/^\s*$/)))  {
	$tot_words++;

	# reads word
	$line =~ s/^(\w+)\s+// || &ouch("Couldn't read vocab word");
	$word[$#word+1] = $1;
	$index{$1} = $#word;
	
	# reads age of word
	$line =~ s/^(\d+)\s+// || &ouch("Couldn't read age of word");
	$age{$word[$#word]} = $1;

	# reads in frequency per category   cat_num:freq
	while ($line =~ s/(\d+):(\d+)\s+//) 
	{
	    $nk{$word[$#word].",".$tv[$1]} = $2;
	    $sum_nk{$word[$#word]} += $2;
	}
	
	if (!($line =~ m/^\s*$/)) { &ouch("Vocab line # $tot_words has bad format"); }
    }
    close(DATA);
    print TMP "Read data file\n";
}

# Determine highest message number in destination folder

$high_msg_num = 0;

opendir(DEST, "$home_dir$mail_path$dest_folder/")
    || &ouch("Could not open destination folder: $!\n");

while ($fn = readdir(DEST))
{
    # looks at first block of digits
    $fn =~ s/^(\d+)\D+/\1/;

    if ($fn > $high_msg_num)
    {
	$high_msg_num = $fn;
    }
}

$high_msg_num++;

closedir(DEST);

# refiles messages one at a time

foreach $msg (@orig_message) 
{
    if ($source_folder ne "") 
    {
	print TMP "Opening $home_dir$mail_path${source_folder}/$msg\n";
	if (!open(MSG, "< $home_dir$mail_path$source_folder/$msg"))
	{
	    print TMP "Could not open $home_dir$mail_path$source_folder/$msg: $!\n";
	    next;
	}
    } 
    else 
    {
	print "Opening $msg\n";
	if (!open(MSG, "< $msg"))
	{
	    print TMP "Could not open $msg: $!\n";
	    next;
	}
    }

    @lines = <MSG>;

# make copy of message in new folder

    if (-x "/usr/bin/mh/refile.bak")
    {
	print TMP "Using refile.bak to refile message: refile.bak ", join(" ",@ARGV), "\n";

	system "/usr/bin/mh/refile.bak ".join(" ",@ARGV);
    }
    elsif (open(MSG, "> $home_dir$mail_path$dest_folder/$high_msg_num"))
    {
	system "/usr/bin/mh/rmm +$source_folder $msg";
	print TMP "/usr/bin/mh/rmm +$source_folder $msg\n";

	print TMP "Creating refiled message: $home_dir$mail_path$dest_folder/$high_msg_num\n";

	foreach $line (@lines)
	{
	    print MSG $line;
	}

	$high_msg_num++;
	close(MSG);
    }

    # Update the .idata file to reflect new knowldege

    print "Input file: \n";

    foreach $line (@lines) 
    {
	while ($line =~ s/([A-Za-z]+)[^A-Za-z]+//) 
	{
	    $wrd = $1;
            # get rid of tiny words
	    if (length($wrd) > 2) 
	    {
		$wrd =~ tr/A-Z/a-z/;
		print $wrd." ";
	 	if ($index{$wrd} == 0) 
		{
		    $word[$#word+1] = $wrd;
		    $index{$wrd} = $#word;
		    $nk{$wrd.",".$dest_folder}++;
		} else 
		{
		    $nk{$wrd.",".$dest_folder}++;
		}
		$n{$dest_folder}++;
	    }
	}
    }
    close(MSG);

    $docs{$dest_folder}++;
}

# reopen the data file and write the learned information

print "Home dir = $home_dir\n";
print "Mail path = $mail_path\n";

open(DATA, "> $home_dir.idata") || die "Couldn't open data file\n";

open(FOLDERS, "< $home_dir${mail_path}.folders") || die "Couldn't find .folders\n";

# prints category names to data file (uses .folders file to determine folders)
$i = 0;
while ($folder = <FOLDERS>) 
{
    chop $folder;
    print DATA $folder." ";
    $f_tv[++$i] = $folder;
}

close(FOLDER);

print DATA "\n";

$tv[0] = "nil";
$n{"nil"} = 0;

# prints total word frequency values
for ($i=1; $i <= $#f_tv; $i++) 
{
    if ($n{$f_tv[$i]} > 0) {
	print DATA $n{$f_tv[$i]}." ";
    } else {
	print DATA "0 ";
    }
}
print DATA "\n";

# prints number of documents for each category
for ($i=1; $i <= $#f_tv; $i++) 
{
    if ($n{$f_tv[$i]} > 0) {
	print DATA $docs{$f_tv[$i]}." ";
    } else {
	print DATA "0 ";
    }
}
print DATA "\n";

for ($i=1; $i <= $#word; $i++) 
{
    $age{$word[$i]}++;

    # if the word is infrequent, discard it (don't print it to DATA)
    if (($age{$word[$i]} < 16) || ($sum_nk{$word[$i]} > 3))
    {
	print DATA $word[$i]." ".$age{$word[$i]}." ";

	for ($j=1; $j <= $#f_tv; $j++) {
	    if ($nk{$word[$i].",".$f_tv[$j]} > 0) {
		print DATA $j.":".$nk{$word[$i].",".$f_tv[$j]}." ";
	    }
	}
	print DATA "\n";
    }
}

close(DATA);

print TMP "\n";
close(TMP);

exit 0;

################

sub ouch {
    local($string) = @_;
    print TMP $string."\n";
    die "$string.: $!\n";
}
