#!/usr/bin/perl

use strict;
#use warnings;
use utf8;

my $word  = "__UNDEF__";
my $lemma = "__UNDEF__";
my $pos   = "__UNDEF__";
my $msd   = "__UNDEF__";
my $newCohort = 1;
my $sentCounter = "1";
my $lang = 'smj';
my $date = '2013-12-18';
my %domain_map = ();
my $inDir = $date.'/'.$lang.'/dis';
my $outDir = 'out_xml_'.$date.'_'.$lang;
my $corpus_name = $lang.'_corpus_'.$date;

system("mkdir -p $outDir");
opendir(DIR, $inDir) || die $!;

init_map();

while (my $inFile = readdir(DIR)) {
    # only files
    next unless (-f "$inDir/$inFile");
    # find files ending in .dis
    next unless ($inFile =~ m/^(.*)\.dis$/);
    my $outFile = $1;
    my ($tlang, $slang, $domain) = split(/_/, $outFile);
    $outFile = $outFile."\.xml";
    #print "$outFile: $tlang $slang and $domain\n";
    #system("head -3 $inDir/$inFile");

    open (OUTFILE, '>', "$outDir/$outFile");
    open (INFILE,"$inDir/$inFile") || die "Can't Open File: $inFile\n";

    # from cwb-encode -h 
    #  -U <str>  insert <str> for missing columns [default: "__UNDEF__"]

    print OUTFILE '<?xml version="1.0" encoding="UTF-8"?>'."\n";
    print OUTFILE '<text title="'.$corpus_name.'" lang="'.$tlang.'" orig_lang="'.$slang.'" gt_domain="'.$domain_map{$domain}.'">'."\n";
    print OUTFILE '  <sentence id="'.$sentCounter.'">'."\n";

    while (<INFILE>) {
        chomp;
        if (/^\"</) { #new lemma
            #print "new Lemma: $_\n";
            ($word) = $_ =~ /<(.+)>/;
	    $word =~ s/_/ /g;
            $newCohort = 1;
        } 
        else { #no new lemma
            if (/^\t+(\"[^\"]+\")(.*)$/) { #analysis
                $lemma = lemma_trim($1);

                ($pos,$msd) = get_msd(trim($2));
      
                $word = unhtmlize($word);
                $lemma = unhtmlize($lemma);
                $pos = unhtmlize($pos);

                if ($newCohort) {
                    print OUTFILE "    <w word=\"".$word."\" lemma=\"".$lemma."\" pos=\"".$pos."\" msd=\"".$msd."\"/>\n";
                }
                reset_var();
            } #analysis 
            else { #newline in the input => new sentence
                print OUTFILE '  </sentence>'."\n";
                $sentCounter++;
                if (! eof)  { print OUTFILE '  <sentence id="'.$sentCounter.'">'."\n" }
            } #newline
        } #else no new lemma
    } #while

    print OUTFILE '</text>';
    print "$inDir/$inFile done!\n";
    close INFILE;
    close OUTFILE;
}
closedir(DIR);
exit 0;


# subroutines
sub trim($);
sub ltrim($);
sub rtrim($);
sub lemma_trim($);
sub  unhtmlize($);
sub reset_vars();
sub get_msd();

sub get_msd()
{
  local($a) = @_;
#  print STDOUT $a."\n";
  #remove sem- and other tags
  $a =~ s/Use\/[^\s]+ //g;
  $a =~ s/Sem\/[^\s]+ //g;
#  $a =~ s/Use\/(Spell|Money|Semcon|Lang|Ani|Body|Build|Clth|Edu|Event|Fem|Food|Group|Hum|Mal|Measr|Obj|Org|Plant|Plc|Route|Sur|Time|Txt|Veh|Wpn|Wthr|Allegro|v1|v2|v3|v4|v5|v6|v7|v8|v9|None) //g;
#  $a =~ s/Sem\/(Spell|Money|Semcon|Lang|Ani|Body|Build|Clth|Edu|Event|Fem|Food|Group|Hum|Mal|Measr|Obj|Org|Plant|Plc|Route|Sur|Time|Txt|Veh|Wpn|Wthr|Allegro|v1|v2|v3|v4|v5|v6|v7|v8|v9|None) //g;

  #remove syntax labels
  $a =~ s/ <[^>]+>/ /g;
#  print STDOUT $a."\n";
#  print STDOUT "_______________\n";
  #split the relation and find selfID and parentID
#  my ($hd, $slf, $prt) = $a =~ m/^(.*)#(\d+)->(\d+)$/;
  #split the function
  my ($r, $fct) = split(/\@/, $a, 2);
  my $rest = trim($r);
#  my $sfct = trim($fct);

  my ($vPOS, $vMSD) = $rest =~m/^([^\s]+)(.*)$/;
  
  # process pseudo-pos
  #if ($vPOS =~ m/\*/)  {
  #  ($vPOS,$vMSD) = generate_correct_lemma($vPOS, $vMSD);
  #}

  $vMSD =~ s/^\s+//;
  $vMSD =~ s/\s+$//;
  $vMSD =~ s/ /\./g;
  $vMSD =~ s/\//_/g;

  if ($vMSD eq "" || $vMSD eq "\?")  {
    $vMSD = "__UNDEF__";
  }
  if ($vPOS eq "" || $vPOS eq "\?")  {
    $vPOS = "__UNDEF__";
  }

  #print STDERR "h ".$h."s ".$s."p ".$p."\n";
  return ($vPOS,$vMSD);
}

sub generate_correct_lemma()
{
  my ($ps,$md) = @_;
  
  #return ($xpos,$xmsd);
  return ("alpha","omega")
}


sub reset_var()
{
  $word  = "__UNDEF__";
  $pos   = "__UNDEF__";
  $lemma = "__UNDEF__";
  $newCohort = 0;
}


# remove quotation marks from the lemma, replace spaces by underscore and # by the empty string
sub unhtmlize($)
{
    my $string = shift;
    $string =~ s/&/&amp;/g;
    $string =~ s/</&lt;/g;
    $string =~ s/>/&gt;/g;
    $string =~ s/\"//g;
    return $string;
}


# Perl trim function to remove whitespace from the start and end of the string
sub dtrim($)
{
    my $string = shift;
    $string =~ s/^\$//;
    return $string;
}

# remove quotation marks from the lemma, replace spaces by underscore and # by the empty string
sub lemma_trim($)
{
    my $string = shift;
    $string =~ s/\"$//;
    $string =~ s/ /_/g;
    $string =~ s/#//g;
    return $string;
}

# Perl trim function to remove whitespace from the start and end of the string
sub trim($)
{
    my $string = shift;
    $string =~ s/^\s+//;
    $string =~ s/\s+$//;
    return $string;
}



sub init_map()
{

%domain_map = (
        "admin" => "administration",
        "bible" => "bible",
        "facta" => "facts",
        "ficti" => "fiction",
        "laws" => "law",
        "news" => "news"
       );
}
