$dict_size = 80000;
open(fp,"./data/dict_train_enWiki.$dict_size.txt");
@stuff=<fp>;
chomp(@stuff);

open(fp2,">./data/dict_train_enWiki.$dict_size.format.txt");

%dict={};
for($i=0;$i<@stuff;$i++)
{
	@line=split(/\s+/,$stuff[$i]);
	if($line[0] eq "")
	{
		$dict{$line[2]}=$line[2];
		print fp2 "$line[2] $line[1]\n";
	}
	else
	{
		$dict{$line[1]}=$line[1];
		print fp2 "$line[1] $line[0]\n";
	}
}
close(fp);
close(fp2);


open(fp,"./data/enwik9.clean.train");
open(fp2,">./data/enwik9.clean.$dict_size.format.train");
while($tmp=readline(fp))
{
	chomp($tmp);
	@stuff=split(/ /,$tmp);
	if(exists($dict{$stuff[0]}))
	{
		print fp2 $stuff[0];
	}
	
	for($i=1;$i<@stuff;$i++)
	{
		if(exists($dict{$stuff[$i]}))
		{
			print fp2 " $stuff[$i]";
		}
		else
		{
			print fp2 ' <unk>';
		}
	}
	print fp2 "\n";
}
close(fp);
close(fp2);

open(fp,"./data/enwik9.clean.dev");
open(fp2,">./data/enwik9.clean.$dict_size.format.dev");
while($tmp=readline(fp))
{
	chomp($tmp);
	@stuff=split(/ /,$tmp);
	if(exists($dict{$stuff[0]}))
	{
		print fp2 $stuff[0];
	}
	
	for($i=1;$i<@stuff;$i++)
	{
		if(exists($dict{$stuff[$i]}))
		{
			print fp2 " $stuff[$i]";
		}
		else
		{
			print fp2 ' <unk>';
		}
	}
	print fp2 "\n";
}
close(fp);
close(fp2);


open(fp,"./data/enwik9.clean.test");
open(fp2,">./data/enwik9.clean.$dict_size.format.test");
while($tmp=readline(fp))
{
	chomp($tmp);
	@stuff=split(/ /,$tmp);
	if(exists($dict{$stuff[0]}))
	{
		print fp2 $stuff[0];
	}
	
	for($i=1;$i<@stuff;$i++)
	{
		if(exists($dict{$stuff[$i]}))
		{
			print fp2 " $stuff[$i]";
		}
		else
		{
			print fp2 ' <unk>';
		}
	}
	print fp2 "\n";
}
close(fp);
close(fp2);