#!/usr/bin/env python3 -u
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
"""
Translate pre-processed data with a trained model.
"""

import logging
import json
import os
import sys
import torch

import numpy as np
from fairseq import checkpoint_utils, options, tasks, utils
from fairseq.logging import progress_bar
from fairseq.data import encoders

def main(args):
    assert args.path is not None, '--path required for saving embedding!'

    if args.results_path is not None:
        os.makedirs(args.results_path, exist_ok=True)
    print("args.results_path: ", args.results_path)
    return _main(args, sys.stdout)


def _main(args, output_file):
    logging.basicConfig(
        format='%(asctime)s | %(levelname)s | %(name)s | %(message)s',
        datefmt='%Y-%m-%d %H:%M:%S',
        level=logging.INFO,
        stream=output_file,
    )
    logger = logging.getLogger('fairseq_cli.extract_summarization')

    utils.import_user_module(args)

    if args.max_tokens is None and args.max_sentences is None:
        args.max_tokens = 12000
    logger.info(args)

    use_cuda = torch.cuda.is_available() and not args.cpu

    # Load dataset splits
    task = tasks.setup_task(args)
    task.load_dataset(args.gen_subset)

    # Set dictionaries
    try:
        src_dict = getattr(task, 'source_dictionary', None)
    except NotImplementedError:
        src_dict = None
    tgt_dict = task.target_dictionary

    # Load ensemble
    logger.info('loading model(s) from {}'.format(args.path))
    models, _model_args = checkpoint_utils.load_model_ensemble(
        utils.split_paths(args.path),
        arg_overrides=eval(args.model_overrides),
        task=task,
    )

    # Optimize ensemble for generation
    for model in models:
        model.make_generation_fast_(
            beamable_mm_beam_size=None if args.no_beamable_mm else args.beam,
            need_attn=args.print_alignment,
        )
        if args.fp16:
            model.half()
        if use_cuda:
            model.cuda()

    # Load alignment dictionary for unknown word replacement
    # (None if no unknown word replacement, empty if no path to align dictionary)
    align_dict = utils.load_align_dict(args.replace_unk)

    # Load dataset (possibly sharded)
    itr = task.get_batch_iterator(
        dataset=task.dataset(args.gen_subset),
        max_tokens=args.max_tokens,
        max_sentences=args.max_sentences,
        max_positions=utils.resolve_max_positions(
            task.max_positions(),
            *[model.max_positions() for model in models]
        ),
        ignore_invalid_inputs=args.skip_invalid_size_inputs_valid_test,
        required_batch_size_multiple=args.required_batch_size_multiple,
        num_shards=args.num_shards,
        shard_id=args.shard_id,
        num_workers=args.num_workers,
    ).next_epoch_itr(shuffle=False)
    progress = progress_bar.progress_bar(
        itr,
        log_format=args.log_format,
        log_interval=args.log_interval,
        default_log_format=('tqdm' if not args.no_progress_bar else 'none'),
    )

    # Handle tokenization and BPE
    tokenizer = encoders.build_tokenizer(args)
    bpe = encoders.build_bpe(args)

    def decode_fn(x):
        if bpe is not None:
            x = bpe.decode(x)
        if tokenizer is not None:
            x = tokenizer.decode(x)
        return x

    document_infos = []
    # period_symbol = tgt_dict.index(".")
    # logger.info('period_symbol: {}'.format(period_symbol))
    
    period_symbols = [tgt_dict.eos()]
    # for key in tgt_dict.indices:
    #     if "." in key:
    #         period_symbols.append(tgt_dict.index(key))

    topk = 50 # dump embedding of the first topk samples

    for sample in progress:
        sample = utils.move_to_cuda(sample) if use_cuda else sample
        if 'net_input' not in sample:
            continue
        
        should_be_dumped = (sample['id'] <= topk).int()
        should_be_dumped = torch.sum(should_be_dumped)

        if should_be_dumped == 0:
            continue

        # dump source embedding
        source_embeddings = task.dump_source_embedding(models, sample, _model_args)

        for i in range(len(sample['id'].tolist())):
            sent_infos = []
            sample_id = sample['id'][i]
            if sample_id > topk:
                continue
            padded_src_tokens = sample['net_input']['src_tokens'][i, :]
            emb = source_embeddings[i][padded_src_tokens.ne(tgt_dict.pad())]
            # Remove padding
            src_tokens = utils.strip_pad(
                padded_src_tokens, tgt_dict.pad()
            )

            token_positions = None
            for ps in period_symbols:
                tmp = (src_tokens == ps)
                if token_positions is None:
                    token_positions = ~tmp
                else:
                    token_positions = ~tmp * token_positions
            period_positions = ~token_positions
            sentence_starts = (period_positions[:-1] * ~period_positions[1:]).nonzero(as_tuple=False) + 1
            # print("sentence_starts.shape: ", sentence_starts.shape)
            zero_tensor = torch.zeros([1, 1], dtype=sentence_starts.dtype).to(sentence_starts)
            sentence_starts = torch.cat([zero_tensor, sentence_starts], dim=0)
            
            # Either retrieve the original sentences or regenerate them from tokens.
            src_sents = None
            if align_dict is not None:
                full_src_str = task.dataset(args.gen_subset).src.get_original_text(sample_id)
                src_sents = full_src_str.split('<q>')
            else:
                full_src_str = src_dict.string(src_tokens, args.remove_bpe)
                full_src_str = decode_fn(full_src_str)
            for j in range(len(sentence_starts)-1):
                start, end = sentence_starts[j], sentence_starts[j+1]
                sent_embedding = torch.mean(emb[start:end], dim=0)
                if align_dict is not None:
                    # retrieve the original sentences
                    src_str = src_sents[j]
                else:
                    # regenerate the original sent from tokens
                    sent_tokens = src_tokens[start:end]
                    if src_dict is not None:
                        src_str = src_dict.string(sent_tokens, args.remove_bpe)
                    else:
                        src_str = ""

                src_str = decode_fn(src_str)
                sent_info = {
                    "sent": src_str, 
                    "embedding": sent_embedding.cpu().detach().numpy().tolist(), 
                    "id": j
                }
                sent_infos.append(sent_info)

            # embeddings = [info['embedding'] for info in sent_infos]
            # doc_embedding = np.array(embeddings)
            # doc_embedding = np.mean(embeddings, axis=0).tolist()
            doc_embedding = torch.mean(emb, dim=0).cpu().detach().numpy.tolist()
            document_infos.append({
                'embedding': doc_embedding,
                'src_str': full_src_str,
                'doc_id': int(sample_id),
                'sent_infos': sent_infos
            })

    document_infos = sorted(document_infos, key=lambda x: x['doc_id'])
    sorted_sent_infos = []
    for document_info in document_infos:
        for sent_info in document_info['sent_infos']:
            sorted_sent_infos.append(sent_info)

    with open(os.path.join(args.results_path, "sent_embedding.jsonl"), 'w') as fout:
        for info in sorted_sent_infos:
            fout.write(json.dumps(info, ensure_ascii=False) + '\n')

    with open(os.path.join(args.results_path, "document_embedding.jsonl"), 'w') as fout:
        for doc_info in document_infos:
            del doc_info['sent_infos']
            fout.write(json.dumps(doc_info, ensure_ascii=False) + '\n')


def cli_main():
    parser = options.get_generation_parser()
    args = options.parse_args_and_arch(parser)
    main(args)

if __name__ == '__main__':
    cli_main()
