#!/usr/bin/perl

use strict;
#use warnings;
use utf8;

my $word  = "__UNDEF__";
my $lemma = "__UNDEF__";
my $pos   = "__UNDEF__";
my $msd   = "__UNDEF__";
my $sID   = "__UNDEF__";
my $pID   = "__UNDEF__";
my $depRel    = "__UNDEF__";
my $analysis  = "__UNDEF__";
my $newCohort = 1;
my $sentCounter = "1";
my $lang = 'sme';
my $date = '2013-12-18';
my %dep_rel = ();
my %domain_map = ();
my $inDir = $date.'/'.$lang.'/dep';
my $outDir = 'out_xml_'.$date.'_'.$lang;
my $corpus_name = $lang.'_corpus_'.$date;

system("mkdir -p $outDir");
opendir(DIR, $inDir) || die $!;

init_dep();

while (my $inFile = readdir(DIR)) {
    # only files
    next unless (-f "$inDir/$inFile");
    # find files ending in .dep
    next unless ($inFile =~ m/^(.*)\.dep$/);
    my $outFile = $1;
    my ($tlang, $slang, $domain) = split(/_/, $outFile);
    $outFile = $outFile."\.xml";
    #print "$outFile: $tlang $slang and $domain\n";
    #system("head -3 $inDir/$inFile");

    open (OUTFILE, '>', "$outDir/$outFile");
    open (INFILE,"$inDir/$inFile") || die "Can't Open File: $inFile\n";

    # from cwb-encode -h 
    #  -U <str>  insert <str> for missing columns [default: "__UNDEF__"]

    print OUTFILE '<?xml version="1.0" encoding="UTF-8"?>'."\n";
    print OUTFILE '<text title="'.$corpus_name.'" lang="'.$tlang.'" orig_lang="'.$slang.'" gt_domain="'.$domain_map{$domain}.'">'."\n";
    print OUTFILE '  <sentence id="'.$sentCounter.'">'."\n";

    while (<INFILE>) {
        chomp;
        if (/^\"</) { #new lemma
            #print "new Lemma: $_\n";
            ($word) = $_ =~ /<(.+)>/;
	    $word =~ s/_/ /g;
            $newCohort = 1;
        } 
        else { #no new lemma
            if (/^\t+(\"[^\"]+\")(.*)$/) { #analysis
                $lemma = lemma_trim($1);
                $analysis = trim($2);

                ($pos,$msd,$sID,$pID,$depRel) = get_msd(trim($2));
      
                $depRel = $dep_rel{$depRel};
                if ($depRel eq "" || $depRel eq "\?")  {
                    $depRel = "_U_";
                }

                $word = unhtmlize($word);
                $lemma = unhtmlize($lemma);
                $pos = unhtmlize($pos);

                if ($newCohort) {
                    print OUTFILE "    <w word=\"".$word."\" lemma=\"".$lemma."\" pos=\"".$pos."\" msd=\"".$msd."\" sID=\"".$sID."\" depRel=\"".$depRel."\" pID=\"".$pID."\"/>\n";
                }
                reset_var();
            } #analysis 
            else { #newline in the input => new sentence
                print OUTFILE '  </sentence>'."\n";
                $sentCounter++;
                if (! eof)  { print OUTFILE '  <sentence id="'.$sentCounter.'">'."\n" }
            } #newline
        } #else no new lemma
    } #while

    print OUTFILE '</text>';
    print "$inDir/$inFile done!\n";
    close INFILE;
    close OUTFILE;
}
closedir(DIR);
exit 0;


# subroutines
sub trim($);
sub ltrim($);
sub rtrim($);
sub lemma_trim($);
sub  unhtmlize($);
sub reset_vars();
sub get_msd();

sub get_msd()
{
  local($a) = @_;
  #remove sem- and other tags
  $a =~ s/Use\/[^\s]+ //g;
  $a =~ s/Sem\/[^\s]+ //g;
  #remove syntax labels
  $a =~ s/ <[^>]+>/ /g;
  #split the relation and find selfID and parentID
  my ($hd, $slf, $prt) = $a =~ m/^(.*)#(\d+)->(\d+)$/;
  #split the function
  my ($r, $fct) = split(/\@/, $hd, 2);
  my $rest = trim($r);
  my $sfct = trim($fct);

  my ($vPOS, $vMSD) = $rest =~m/^([^\s]+)(.*)$/;
  
  # process pseudo-pos
  #if ($vPOS =~ m/\*/)  {
  #  ($vPOS,$vMSD) = generate_correct_lemma($vPOS, $vMSD);
  #}

  $vMSD =~ s/^\s+//;
  $vMSD =~ s/\s+$//;
  $vMSD =~ s/ /\./g;
  $vMSD =~ s/\//_/g;

  if ($sfct eq "" || $sfct eq "\?")  {
    $sfct = "__UNDEF__";
  }
  if ($prt eq "" || $prt eq "\?")  {
    $prt = "__UNDEF__";
  }
  if ($slf eq "" || $slf eq "\?")  {
    $slf = "__UNDEF__";
  }
  if ($vMSD eq "" || $vMSD eq "\?")  {
    $vMSD = "__UNDEF__";
  }
  if ($vPOS eq "" || $vPOS eq "\?")  {
    $vPOS = "__UNDEF__";
  }

  #print STDERR "h ".$h."s ".$s."p ".$p."\n";
  return ($vPOS,$vMSD,$slf,$prt,$sfct);
}

sub generate_correct_lemma()
{
  my ($ps,$md) = @_;
  
  #return ($xpos,$xmsd);
  return ("alpha","omega")
}


sub reset_var()
{
  $word  = "__UNDEF__";
  $pos   = "__UNDEF__";
  $lemma = "__UNDEF__";
  $sID   = "__UNDEF__";
  $pID   = "__UNDEF__";
  $depRel    = "__UNDEF__";
  $analysis  = "__UNDEF__";
  $newCohort = 0;
}


# remove quotation marks from the lemma, replace spaces by underscore and # by the empty string
sub unhtmlize($)
{
    my $string = shift;
    $string =~ s/&/&amp;/g;
    $string =~ s/</&lt;/g;
    $string =~ s/>/&gt;/g;
    $string =~ s/\"//g;
    return $string;
}


# Perl trim function to remove whitespace from the start and end of the string
sub dtrim($)
{
    my $string = shift;
    $string =~ s/^\$//;
    return $string;
}

# remove quotation marks from the lemma, replace spaces by underscore and # by the empty string
sub lemma_trim($)
{
    my $string = shift;
    $string =~ s/\"$//;
    $string =~ s/ /_/g;
    $string =~ s/#//g;
    return $string;
}

# Perl trim function to remove whitespace from the start and end of the string
sub trim($)
{
    my $string = shift;
    $string =~ s/^\s+//;
    $string =~ s/\s+$//;
    return $string;
}



sub init_dep()
{


%dep_rel = (
        "__UNDEF__" => "_U_",
        ">N" => "MOD",
        "<ADVL" => "ADV",
        "<ADVL-Acc" => "ADV-Acc",
        "<ADVL-Adv" => "ADV-Adv",
        "<ADVL-Com" => "ADV-Com",
        "<ADVL-Ess" => "ADV-Ess",
        "<ADVL-Gen" => "ADV-Gen",
        "<ADVL-Ill" => "ADV-Ill",
        "<ADVL-Loc" => "ADV-Loc",
        "<ADVL-Nom" => "ADV-Nom",
        "ADVL>" => "ADV",
        "ADVL>-Acc" => "ADV-Acc",
        "ADVL>-Adv" => "ADV-Adv",
        "ADVL>-Com" => "ADV-Com",
        "ADVL>-Ess" => "ADV-Ess",
        "ADVL>-Gen" => "ADV-Gen",
        "ADVL>-Ill" => "ADV-Ill",
        "ADVL>-Loc" => "ADV-Loc",
        "ADVL>-Nom" => "ADV-Nom",
        "ADVL>CS" => "ADV",
        "FS-ADVL>" => "ADV",
        "-FADVL>" => "ADV",
        "SUBJ>" => "SBJ",
        "-FADVL>" => "ADV",
        "-FADVL>-Adv"  => "ADV-Adv",
        "-FADVL>-Com"  => "ADV-Com",
        "-FADVL>-Gen"  => "ADV-Gen",
        "-FADVL>-Ill"  => "ADV-Ill",
        "-FADVL>-Loc"  => "ADV-Loc",
        "-FADVL>-Nom"  => "ADV-Nom",
        "-F<ADVL" => "ADV",
        "-F<ADVL-Acc"  => "ADV-Acc",
        "-F<ADVL-Adv"  => "ADV-Adv",
        "-F<ADVL-Com"  => "ADV-Com",
        "-F<ADVL-Gen"  => "ADV-Gen",
        "-F<ADVL-Ill"  => "ADV-Ill",
        "-F<ADVL-Loc"  => "ADV-Loc",
        "-F<ADVL-Nom"  => "ADV-Nom",
        "FMV" => "+MV",
        "CVP" => "CVP",
        "<OBJ" => "OBJ",
        "IMV" => "-MV",
        "CNP" => "CNP",
        "HNOUN" => "FRG",
        "<SPRED" => "SPR",
        "FAUX" => "+AV",
        "<SUBJ" => "SBJ",
        ">P" => "MOD",
        "FS-<ADVL" => "ADV",
        "FS-N<" => "+V",
        "FS-IMV" => "-MV",
        ">A" => "ADV",
        "OBJ>" => "OBJ",
        ">Num" => "MOD",
        "FS-OBJ" => "OBJ",
        "Num<" => "MOD",
        "FS-N<IMV" => "-MV",
        "-F<OBJ" => "OBJ",
        "PCLE" => "PCL",
        "HAB>" => "HAB",
        "P<" => "MOD",
        "FMVdic" => "+MV",
        ">ADVL" => "ADV",
        "IAUX" => "-AV",
        "N<" => "MOD",
        "SPRED>" => "SPR",
        "FS-VFIN<" => "ADV",
        "FS-IAUX" => "-AV",
        "COMP-CS<" => "CMP",
        ">CC" => "MOD",
        "-FOBJ>" => "OBJ",
        "<OPRED" => "OPR",
        "X" => "X",
        "APP-N<" => "APP",
        "ICL-SUBJ" => "SBJ",
        ">Pron" => "MOD",
        "-FSUBJ>" => "SBJ",
        "ICL-OBJ" => "OBJ",
        "FS-N<IAUX" => "-AV",
        "Pron<" => "MOD",
        "INTERJ" => "INT",
        "OPRED>" => "OPR",
        "A<" => "MOD",
        "S<" => "MOD",
        "<HAB" => "HAB",
        "VOC" => "VOC",
        "<PPRED" => "PPR",
        "ADVL<" => "MOD",
        "APP-ADVL<" => "APP",
        "APP-Pron<" => "APP",
        "FS-<SUBJ" => "SBJ",
        "IMVdic" => "-MV",
        "FS-P<" => "+MV",
        "+FMAINV" => "+MV"
       );

%domain_map = (
        "admin" => "administration",
        "bible" => "bible",
        "facta" => "facts",
        "ficti" => "fiction",
        "laws" => "law",
        "news" => "news"
       );
}
