#!/usr/local/bin/perl
#
# Driver program to test word tokenization routines

$total  = 0;
while (<>) {
    @words = &TokenizeWords($_);
    @words = IndexableWords(@words);
    foreach $word (@words) {
	$total++;
	$wcount{$word}++;
    }
}

$unique = 0;
foreach $word (sort keys(%wcount)) {
    printf "%7d %s\n", $wcount{$word}, $word;
    $unique++;
}

printf "Total  words: %d\n", $total;
printf "Unique words: %d\n", $unique;
###  END of Main Routine ###

#### Begin Subroutines ####

sub TokenizeWords {
    use strict;
    my(@words);
    @words = ("No", "tokenization", "code", "yet");
    return @words;
}  # End of TokenizeWords


sub IndexableWords {
    use strict;
    my(@words_out);

    @words_out = @_;
    return(@words_out);
}


__DATA__
the
to
a
of
and
in
that
for