#!/usr/local/bin/perl

#use warnings;
#use strict;

#
# sent_length.pl
# This program finds statistics for sentence length frequency
#

local $/=undef;
@Madison = (10, 14, 37 .. 48); 
@Hamilton =(1, 6 .. 9, 11, 12, 24, 26, 36, 59, 65, 66, 70 .. 72, 74, 77);

#@files = (@Madison, @Hamilton);
@files = @Madison;
#@files = @Hamilton;

for (@files) {
    open FILE, "../federalist_papers/federalist$_.txt" or die "Couldn't open file: $!";
    #open FILE, "test_str" or die "Couldn't open file: $!"; #test file
    $data = <FILE>;
    close FILE;

    split_sentences(); 
    
    print "Federalist Paper $_:\n";
    get_stats();
    print "\n****************************************\n";
    $j=0; 
    for(@u){@total[$j++] += $_;}
}


#print "\n\n", $total/scalar(@files), "\n";

#normalize frequency sum by total number of files
for(@total)
{    $_ /= scalar @files;}
#print_array(\@total);

#sum all frequencies (should == 1)
for(@total) {
    $sum += $_;
}
#print $sum, "\n";



#place marker between sentences
sub split_sentences {
    $abbrevs = '(?<! \W (?: Mr | Ms | Dr | No)) (?<! \W (?: Mrs | e.g | i.e)) (?<! \WPh.D)';  #not end of sentence
    #$abbrevs = '(?<! \WMr \W) (?<! Mrs | e.g | i.e)';  #not end of sentence
    $punc = '[\?!]+';                             #exclamation mark, or questions mark (and combos)
    $period = '\.';
    $sentence_marker = "\n";                      #marker between sentences

    $data =~ s/\s+/ /g;                           #collapse internal whitespace 
    
    $data =~ s/
	($abbrevs $period | $punc )               #punctuation
	\s+ (?=[A-Z])                             #whitespace capitalized latter
	/\1$sentence_marker/xgo;
    
    #    print $data, "\n";
    
    @sentences = split /$sentence_marker/, $data; #split on 'SENT_end' marker.   
}

#take sentence statistics
sub  get_stats {
    undef @u; undef @vec; #clear the arrays;
    
    my $num_sentences = scalar @sentences;
    # print $num_sentences, " sentences found\n";
    my $wds=0;

    #vec[i] = num sentences of lengh i
    for (@sentences){ 
	$_ =~ s/^\s+|\s+$//; #discard leading and trailing whitespace
	@words = split / /; 
	
	$sentence_length = scalar @words;
	$wds += $sentence_length;
	@vec[$sentence_length]++;
	#  print "@vec[$sentence_length] sentences of length $sentence_length \n";
	
	if ($max_length < $sentence_length) { $max_length = $sentence_length} 
    }

    $avg_length = $wds/ $num_sentences;
    $total += $avg_length;
    #print "total words: ", $wds, "\n";
    #print "avg sentence length: ", $avg_length, "\n"; #avg sentence length

    $i=0;
    for(@vec) { #normalize by setting @u = @vec/# sentences
	@u[$i++] = @vec[$i]/$num_sentences;
    }
    print_array(\@u);
}

sub print_array{
    my ($aref) = @_;
    $k=0;
    print "i\tu_i\n";
    for(@$aref) {
	if ($_) {print "$k\t$_\n";}
	$k++;
    }
}