#!/usr/bin/perl

use strict;

my $text = "C:/Users/Paul/Documents/Projects/Rutgers/Corpus/textonly/"; # directory in which txt input files live
my $xml = "C:/Users/Paul/Documents/Projects/Rutgers/Corpus/xml/"; # directory in which xml-formatted txt output files live
my $minsize = 300; # minimum number of words in the input file before it is processed

print "Input Directory: $text\n";
print "Output Directory: $xml\n";

my $xmled = my $short = my $exists = my $wrongtype = 0; # we'll use this to keep track of how many files we looked at

my $name; # this is the name of a file in the $text directory

while (defined($name = <$text*>)) {
	my $output = $name; # get a copy of the filename, so we can manipulate it
	$output =~ s/$text//; # remove the file path from the filename
	my $filename = $output; # snag the file name for later use
	if ($output !~ m/[.txt]$/) { 
		$wrongtype++;
		next; # if the file is not a txt file, skip it
	} 
	
	print "$output... "; # print a little information
	$output =~ s/\s+/_/g; # substitute underscores for (repeated) whitespace, if need be
	$output = $xml . $output; #  concatinate the filename onto the output directory

	if (-e $output)
	{ # if the output file already exists, don't bather reparsing the word doc
		print " output exists, skipping\n";
		$exists++;
		next;
	}
	
	open (INFILE, "$name"); # open up the input file
	my @input = <INFILE>; # slurp the input file into memory
	close (INFILE); # close out the input file
	
	my $fullinput = join(' ', @input); # link together all of the lines in the input into a single scalar
	my @words = split(/ /, $fullinput); # split input into an array of words we can then count
	my $wordcount = @words;
	
	if ($wordcount < $minsize)
	{ # if the file is shorter than our specified min length, skip it
		print " too short, only $wordcount words, skipping\n";
		$short++;
		next;
	}
	
	open (FH, ">$output"); # open	up the output file

	# write the XML block at the beginning of the file, then dump in the text and close off the XML
	print FH "<DOC>\n<DOCID>$filename</DOCID>\n<AUTHOR></AUTHOR>\n<TEXT>\n";
	foreach my $line (@input)
		{ print FH "$line\n"; }
	print FH "</TEXT>\n</DOC>\n";
	close (FH); # close the output file
	
	print " done.\n"; # print out a bit of status
	$xmled++; # increment file count
}

print "\n";
print $wrongtype + $exists + $short + $xmled . " files processed, total\n";
print "$wrongtype files were skipped as non-txt\n";
print "$exists files were skipped because they already exist\n";
print "$short files were skipped for being < $minsize words\n";
print "$xmled files were XML-ified\n";