#!/usr/bin/perl -s

use warnings;
use strict;

our $sparql;
$sparql = 0 unless $sparql;

use Text::WordGrams;
use Lingua::StopWords qw( getStopWords );
use DBI;
use DBD::SQLite;
use Conclave::Utils::ISplitter;
use Data::Dumper;

my $corpus = shift;
die unless $corpus;
my $pkgid;
if ($sparql) {
  $pkgid = shift;
  die unless $pkgid;
}

# compute word unigrams
my $data = word_grams_from_files({size=>1}, $corpus);
my $total = 0;
$total += $data->{$_} foreach (keys %$data);

my $stopwords = getStopWords('en');
foreach (keys %$data) {
  # remove stop words
  delete $data->{$_} and next if ($stopwords->{$_} or $stopwords->{lc $_});

  # remove non words
  delete $data->{$_} and next unless $_ =~ m/^[\w\d\-]+$/;
  delete $data->{$_} and next if $_ =~ m/^[\-\_]+$/;

  # remove length < 3
  delete $data->{$_} and next if length($_) < 3;

  # remove if freq <= 2
  delete $data->{$_} and next if $data->{$_} <= 2;
}

my $final = {};
my $lemmas = build_lemmas($data);

foreach (keys %$data) {
  my $lemma = $lemmas->{$_} or next;
  #my $lemma = lemma($_);
  $final->{$lemma}->{$_} = $data->{$_};
}

foreach my $k (keys %$final) {
  my $sum = 0;
  foreach (keys %{$final->{$k}}) {
    $sum += $final->{$k}->{$_};
  }
  if ($sum/$total<0.005) {
    delete $final->{$k};
  }
}

#print Dumper $final;
print<<"EOF";
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
PREFIX owl: <http://www.w3.org/2002/07/owl#>
PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
PREFIX problem: <http://conclave.di.uminho.pt/owl/$pkgid/problem#>
EOF

foreach (keys %$final) {
foreach my $op (qw/DELETE INSERT/) {
print<<"EOF";
$op DATA {
  GRAPH <http://conclave.di.uminho.pt/owl/$pkgid/problem> {
    problem:$_ rdf:type owl:Class ;
      rdfs:subClassOf problem:Concept .
  }
}
EOF
}
}

#foreach my $c (keys %$final) {
#  my @terms = keys %{ $final->{$c} };
#
#  print " <term concept=\"$c\">$_</term>\n" foreach @terms;
#}
#print "  }\n}\n";

sub build_lemmas {
  my $h = shift;
  my $lemmas = {};

  open F, '>', '/tmp/input';
  print F "$_\n", foreach (keys %$h);
  close F;

  my $res = `cat /tmp/input | python /Users/smash/LOCAL/Python/lemma2.py`;
  foreach (split /\n+/, $res) {
    my ($l, $r) = split /,/, $_;
    $lemmas->{$l} = $r;
  }

  return $lemmas;
}

sub lemma {
  my $word = shift;

  my $res = `python /Users/smash/LOCAL/Python/lemma.py $word`;
  $res =~ s/\s*$//;

  return $res;
}