if __name__ == "__main__":

	import os
	import pickle
	import codecs
	from scipy.sparse import csr_matrix
	from sklearn.feature_extraction.text import HashingVectorizer
	from sklearn.feature_extraction.text import CountVectorizer

	in_directory = os.path.join(".", "c2xg", "IN", "eng")
	out_directory = os.path.join(".", "c2xg", "OUT", "eng")
	
	function_words = ["i", "me", "my", "myself", "we", "our", "ours", "ourselves", "you", "your", "yours", "yourself", "yourselves", "he", "him", "his", "himself", "she", "her", "hers", "herself", "it", "its", "itself", "they", "them", "their", "theirs", "themselves", "what", "which", "who", "whom", "this", "that", "these", "those", "am", "is", "are", "was", "were", "be", "been", "being", "have", "has", "had", "having", "do", "does", "did", "doing", "a", "an", "the", "and", "but", "if", "or", "because", "as", "until", "while", "of", "at", "by", "for", "with", "about", "against", "between", "into", "through", "during", "before", "after", "above", "below", "to", "from", "up", "down", "in", "out", "on", "off", "over", "under", "again", "further", "then", "once", "here", "there", "when", "where", "why", "how", "all", "any", "both", "each", "few", "more", "most", "other", "some", "such", "no", "nor", "not", "only", "own", "same", "so", "than", "too", "very", "s", "t", "can", "will", "just", "don", "should", "now"]
	
	HV_uni = HashingVectorizer(input = "content", 
								encoding = "utf-8", 
								decode_error = "strict", 
								ngram_range = (1, 1), 
								analyzer = "word", 
								n_features = 30000, 
								binary = False, 
								norm = None, 
								alternate_sign = False, 
								non_negative = False
								)
								
	HV_bi = HashingVectorizer(input = "content", 
								encoding = "utf-8", 
								decode_error = "strict", 
								ngram_range = (2, 2), 
								analyzer = "word", 
								n_features = 30000, 
								binary = False, 
								norm = None, 
								alternate_sign = False, 
								non_negative = False
								)
								
	HV_tri = HashingVectorizer(input = "content", 
								encoding = "utf-8", 
								decode_error = "strict", 
								ngram_range = (3, 3), 
								analyzer = "word", 
								n_features = 30000, 
								binary = False, 
								norm = None, 
								alternate_sign = False, 
								non_negative = False
								)
								
	CV_func = CountVectorizer(input = "content", 
								encoding = "utf-8", 
								decode_error = "strict", 
								ngram_range = (1, 1), 
								analyzer = "word",
								binary = False,
								vocabulary = function_words
								)
	
	for filename in os.listdir(in_directory):
		
		print("\tStarting " + str(filename))
		original_filename = filename
		filename = os.path.join(in_directory, filename)
		
		vectors = HV_uni.transform(codecs.open(filename, "r", encoding = "utf-8"))
		print(vectors.shape)
		write_name = original_filename.replace(".txt","") + ".unigrams.p"
		
		with open(os.path.join(out_directory, write_name), "wb") as fp:
			pickle.dump(vectors, fp)
		
		vectors = HV_bi.transform(codecs.open(filename, "r", encoding = "utf-8"))
		print(vectors.shape)
		write_name = original_filename.replace(".txt","") + ".bigrams.p"
		
		with open(os.path.join(out_directory, write_name), "wb") as fp:
			pickle.dump(vectors, fp)
		
		vectors = HV_tri.transform(codecs.open(filename, "r", encoding = "utf-8"))
		print(vectors.shape)
		write_name = original_filename.replace(".txt","") + ".trigrams.p"
		
		with open(os.path.join(out_directory, write_name), "wb") as fp:
			pickle.dump(vectors, fp)
		
		vectors = CV_func.transform(codecs.open(filename, "r", encoding = "utf-8"))
		print(vectors.shape)
		write_name = original_filename.replace(".txt","") + ".function.p"
		
		with open(os.path.join(out_directory, write_name), "wb") as fp:
			pickle.dump(vectors, fp)