import json


out_fp = open('./webhose_news_raw.json', 'w')

global_article_id = 1
global_line_id = 0
with open('./webhose_data_entries_with_translations_blank_filled_copied.jsonl', 'r', encoding='utf8') as fp:
	for lidx, line in enumerate(fp):
		if lidx % 10000 == 0:
			print(lidx)
		in_entry = json.loads(line)
		for in_sent in in_entry['english_splitted_text']:
			out_entry = {'s': in_sent, 'date': in_entry['published'], 'articleId': global_article_id, 'lineId': global_line_id}
			out_line = json.dumps(out_entry)
			out_fp.write(out_line+'\n')

			global_line_id += 1
		global_article_id += 1


out_fp.close()