#!/usr/bin/env python

"""Converts tokenization in IOB format to tokenization in offset format."""

import codecs
import sys
import utils

snum = 0
tnum = None
offset = 0
token = None

def start_sentence():
    global snum, tnum
    snum += 1
    tnum = 0

def start_token():
    global tnum, token, offset, fr, to
    if tnum == 999:
        raise Exception('A maximum of 999 tokens per sentence is allowed.')
    tnum += 1
    token = ''
    fr = offset
    to = None

def continue_token(code):
    global token, to
    token += unichr(int(code))
    to = None

def finish_token():
    global tnum, fr, to, offset, snum, tnum, token
    if to == None:
        to = offset
    utils.write_utf8(sys.stdout).write('%d %d %d%03d %s\n' % (fr, to, snum, tnum, token))

for line in utils.read_utf8(sys.stdin):
    line = line.rstrip()
    if not line:
        continue
    (code, label) = line.split(' ')
    if code == '10': # LF
        code = '32' # SP
    if label == 'S':
        if token:
            finish_token()
        start_sentence()
        start_token()
        continue_token(code)
    elif label == 'T':
        if token:
            finish_token()
        if tnum == None:
            start_sentence()
        start_token()
        continue_token(code)
    elif label == 'I':
        continue_token(code)
    elif label == 'O':
        if snum > 0 and to == None:
            to = offset
    offset += 1

if token:
    finish_token()

