#!/bin/bash

# remove non-whitespace characters labeled O and remove duplicate empty lines
cat data/paisaselection.test.iob | grep '^10 O\|^32 O\|I\|S\|T\|^$' | awk '/^$/{ if (! blank++) print; next } { blank=0; print }' > punkt.in

# detect sentence boundaries in each document, using Punkt
cat punkt.in | ./src/scripts/iobpunkt.py --force-newline italian > punkt.out 2> punkt.err

# evaluate (only scores and confusion for the S label is interesting, all other differences should be ignored)
paste -d ' ' punkt.in punkt.out | cut -f 1,2,4 -d ' ' | ./src/scripts/eval.py
