import itertools
import json

from dataclasses import dataclass

import numpy as np
import pandas as pd
import seqeval.metrics as seqeval_metrics
import torch

from scipy.stats import pearsonr
from scipy.stats import spearmanr
from sklearn.metrics import f1_score
from sklearn.metrics import matthews_corrcoef
from typing import Dict
from typing import List

import jiant.shared.model_resolution as model_resolution

import jiant.tasks.lib.bucc2018 as bucc2018_lib
import jiant.tasks.lib.mlqa as mlqa_lib
import jiant.tasks.lib.tatoeba as tatoeba_lib
import jiant.tasks.lib.templates.squad_style.core as squad_style
import jiant.tasks.lib.templates.squad_style.utils as squad_style_utils

import jiant.tasks.retrieval as tasks_retrieval

from jiant.tasks.lib.templates import mlm as mlm_template
from jiant.utils.python.datastructures import ExtendedDataClassMixin
from jiant.utils.python.io import read_json
from jiant.utils.string_comparing import exact_match_score
from jiant.utils.string_comparing import string_f1_score


@dataclass
class Metrics(ExtendedDataClassMixin):
    major: float
    minor: Dict


class BaseEvaluation:
    pass


class BaseAccumulator:
    def update(self, batch_logits, batch_loss, batch, batch_metadata):
        raise NotImplementedError()

    def get_guids(self):
        return None

    def get_accumulated(self):
        raise NotImplementedError()


class BaseEvaluationScheme:
    def get_accumulator(self) -> BaseAccumulator:
        raise NotImplementedError()

    def get_labels_from_cache_and_examples(self, task, cache, examples):
        # Depending on the task, labels may be more easily extracted from
        #   a cache or raw examples.
        # Provide the EvaluationScheme with either, but delegate to another function
        #   using only one.
        raise NotImplementedError()

    def get_preds_from_accumulator(self, task, accumulator):
        raise NotImplementedError()

    def compute_metrics_from_accumulator(
        self, task, accumulator: BaseAccumulator, tokenizer, labels
    ) -> Metrics:
        raise NotImplementedError()


class ConcatenateLogitsAccumulator(BaseAccumulator):
    def __init__(self):
        self.logits_list = []
        self.guid_list = []

    def update(self, batch_logits, batch_loss, batch, batch_metadata):
        self.logits_list.append(batch_logits)
        batch_guid = batch_metadata.get("guid")
        if batch_guid is not None:
            self.guid_list.append(batch_guid)

    def get_guids(self):
        if self.guid_list:
            return np.concatenate(self.guid_list)
        else:
            return None

    def get_accumulated(self):
        all_logits = np.concatenate(self.logits_list)
        return all_logits


class ConcatenateLossAccumulator(BaseAccumulator):
    def __init__(self):
        self.loss_list = []

    def update(self, batch_logits, batch_loss, batch, batch_metadata):
        self.loss_list.append(batch_loss)

    def get_accumulated(self):
        all_loss = np.array(self.loss_list)
        return all_loss


class ConcatenateStringListAccumulator(BaseAccumulator):
    def __init__(self):
        self.str_list = []

    def update(self, batch_logits, batch_loss, batch, batch_metadata):
        bs = len(batch_logits)
        span_pred = batch_logits.argmax(axis=1)
        pred_token_start, pred_token_end = span_pred[:, 0], span_pred[:, 1]
        pred_char_start = batch.token_idx_to_char_idx_start.cpu().numpy()[
            range(bs), pred_token_start
        ]
        pred_char_end = batch.token_idx_to_char_idx_end.cpu().numpy()[range(bs), pred_token_end]
        self.str_list.extend(
            [
                s[i1 : i2 + 1]
                for i1, i2, s in zip(pred_char_start, pred_char_end, batch.selection_str)
            ]
        )

    def get_accumulated(self):
        return self.str_list


class SpanPredictionF1andEMScheme(BaseEvaluationScheme):
    def get_accumulator(self):
        return ConcatenateStringListAccumulator()

    def get_labels_from_cache_and_examples(self, task, cache, examples):
        return [datum["data_row"].gt_span_str for datum in cache.iter_all()]

    def get_preds_from_accumulator(self, task, accumulator):
        return accumulator.get_accumulated()

    @classmethod
    def compute_metrics_from_preds_and_labels(cls, preds, labels):
        em = sum([exact_match_score(s1, s2) for s1, s2 in zip(preds, labels)]) / len(labels)
        f1 = sum([string_f1_score(s1, s2) for s1, s2 in zip(preds, labels)]) / len(labels)
        scores = {"f1": f1, "em": em, "avg": (f1 + em) / 2}
        return Metrics(major=scores["avg"], minor=scores)

    def compute_metrics_from_accumulator(
        self, task, accumulator: ConcatenateStringListAccumulator, tokenizer, labels: list
    ) -> Metrics:
        preds = self.get_preds_from_accumulator(task=task, accumulator=accumulator)
        return self.compute_metrics_from_preds_and_labels(preds=preds, labels=labels)


class RecordAccumulator(ConcatenateLogitsAccumulator):
    def __init__(self):
        super().__init__()
        self.entity_strs = []
        self.gold_label_list = []

    def update(self, batch_logits, batch_loss, batch, batch_metadata):
        super().update(batch_logits, batch_loss, batch, batch_metadata)
        self.entity_strs.extend(batch.entity_str)
        self.gold_label_list.extend(batch.label_set)

    def get_accumulated(self):
        return super().get_accumulated(), self.entity_strs

    def get_gold_label_list(self):
        return self.gold_label_list


class MLMPremaskedAccumulator(BaseAccumulator):
    def __init__(self):
        self.loss_list = []
        self.logits_list = []

    def update(self, batch_logits, batch_loss, batch, batch_metadata):
        batch_size = len(batch)
        # Select the tokens that we do MLM prediction on
        masked_tokens_selector = (
            batch.masked_lm_labels.cpu().numpy() != mlm_template.NON_MASKED_TOKEN_LABEL_ID
        )
        for i in range(batch_size):
            # noinspection PyUnresolvedReferences
            self.logits_list.append(batch_logits[i][masked_tokens_selector[i]])
        self.loss_list.append(batch_loss)

    def get_accumulated(self):
        return self.loss_list, self.logits_list


class TatoebaAccumulator(BaseAccumulator):
    def __init__(self):
        self.embeddings_list = []
        self.is_english_list = []

    def update(self, batch_logits, batch_loss, batch, batch_metadata):
        self.embeddings_list.append(batch_logits)
        self.is_english_list.append(batch.is_english.cpu().numpy())

    @classmethod
    def get_guids(cls):
        return None

    def get_accumulated(self):
        all_embeddings = np.concatenate(self.embeddings_list)
        is_english_arr = np.concatenate(self.is_english_list).astype(bool)
        return all_embeddings, is_english_arr


class Bucc2018Accumulator(BaseAccumulator):
    def __init__(self):
        self.embeddings_list = []
        self.is_english_list = []
        self.text_hash_list = []
        self.guid_list = []

    def update(self, batch_logits, batch_loss, batch, batch_metadata):
        self.embeddings_list.append(batch_logits)
        self.is_english_list.append(batch.is_english.cpu().numpy())
        self.text_hash_list += batch.text_hash
        self.guid_list += batch.guid

    @classmethod
    def get_guids(cls):
        return None

    def get_accumulated(self):
        return {
            "all_embeddings": np.concatenate(self.embeddings_list),
            "is_english_arr": np.concatenate(self.is_english_list).astype(bool),
            "text_hash_list": self.text_hash_list,
            "guid_list": self.guid_list,
        }


class BaseLogitsEvaluationScheme(BaseEvaluationScheme):
    def get_accumulator(self):
        return ConcatenateLogitsAccumulator()

    def get_labels_from_cache_and_examples(self, task, cache, examples):
        return get_label_ids_from_cache(cache=cache)

    def get_preds_from_accumulator(self, task, accumulator):
        raise NotImplementedError()

    def compute_metrics_from_accumulator(
        self, task, accumulator: ConcatenateLogitsAccumulator, tokenizer, labels: list
    ) -> Metrics:
        preds = self.get_preds_from_accumulator(task=task, accumulator=accumulator)
        return self.compute_metrics_from_preds_and_labels(preds=preds, labels=labels)

    def compute_metrics_from_preds_and_labels(self, preds, labels):
        raise NotImplementedError()


class SimpleAccuracyEvaluationScheme(BaseLogitsEvaluationScheme):
    @classmethod
    def get_preds_from_accumulator(cls, task, accumulator):
        logits = accumulator.get_accumulated()
        return np.argmax(logits, axis=1)

    @classmethod
    def compute_metrics_from_preds_and_labels(cls, preds, labels):
        # noinspection PyUnresolvedReferences
        acc = float((preds == labels).mean())
        return Metrics(major=acc, minor={"acc": acc})


class MCTACOEvaluationScheme(BaseLogitsEvaluationScheme):
    @classmethod
    def get_preds_from_accumulator(self, task, accumulator):
        logits = accumulator.get_accumulated()
        pred = np.argmax(logits, axis=1)
        guid = accumulator.get_guids()
        return guid, pred

    @classmethod
    def compute_metrics_from_accumulator(self, task, accumulator, tokenizer, labels) -> Metrics:
        guid, pred = self.get_preds_from_accumulator(task=task, accumulator=accumulator)
        em_ls = []
        f1_ls = []
        label_pred_by_question = {}

        for one_guid, one_pred, one_label in zip(guid, pred, labels):
            split, question_id, example_id = one_guid.split("-")
            if question_id not in label_pred_by_question:
                label_pred_by_question[question_id] = [], []
            label_pred_by_question[question_id][0].append(one_label)
            label_pred_by_question[question_id][1].append(one_pred)

        em_ls = [
            float(group_label == group_pred)
            for group_label, group_pred in label_pred_by_question.values()
        ]
        f1_ls = [
            f1_score(y_true=group_label, y_pred=group_pred)
            for group_label, group_pred in label_pred_by_question.values()
        ]

        em = sum(em_ls) / len(em_ls)
        f1 = sum(f1_ls) / len(f1_ls)
        minor = {
            "em": em,
            "f1": f1,
            "f1_em": (f1 + em) / 2,
        }
        metrics = Metrics(major=minor["f1_em"], minor=minor,)
        return metrics


class MultiLabelAccAndF1EvaluationScheme(BaseLogitsEvaluationScheme):
    def get_labels_from_cache_and_examples(self, task, cache, examples):
        return get_multi_label_ids_from_cache(cache=cache)

    def get_preds_from_accumulator(self, task, accumulator):
        logits = accumulator.get_accumulated()
        return (logits > 0.5).astype(int)

    @classmethod
    def compute_metrics_from_preds_and_labels(cls, preds, labels):
        # noinspection PyUnresolvedReferences
        acc = float((preds == labels).mean())
        labels = np.array(labels)
        minor = {
            "acc": acc,
            "f1_micro": f1_score(y_true=labels, y_pred=preds, average="micro"),
            "acc_and_f1_micro": (acc + f1_score(y_true=labels, y_pred=preds, average="micro")) / 2,
        }
        return Metrics(major=minor["acc_and_f1_micro"], minor=minor)


class AccAndF1EvaluationScheme(BaseLogitsEvaluationScheme):
    def get_preds_from_accumulator(self, task, accumulator):
        logits = accumulator.get_accumulated()
        return np.argmax(logits, axis=1)

    @classmethod
    def compute_metrics_from_preds_and_labels(cls, preds, labels):
        # noinspection PyUnresolvedReferences
        acc = float((preds == labels).mean())
        labels = np.array(labels)
        f1 = f1_score(y_true=labels, y_pred=preds)
        minor = {
            "acc": acc,
            "f1": f1,
            "acc_and_f1": (acc + f1) / 2,
        }
        return Metrics(major=minor["acc_and_f1"], minor=minor)


class MCCEvaluationScheme(BaseLogitsEvaluationScheme):
    def get_preds_from_accumulator(self, task, accumulator):
        logits = accumulator.get_accumulated()
        return np.argmax(logits, axis=1)

    @classmethod
    def compute_metrics_from_preds_and_labels(cls, preds, labels):
        mcc = matthews_corrcoef(labels, preds)
        return Metrics(major=mcc, minor={"mcc": mcc})


class PearsonAndSpearmanEvaluationScheme(BaseLogitsEvaluationScheme):
    def get_labels_from_cache_and_examples(self, task, cache, examples):
        return get_label_vals_from_cache(cache=cache)

    def get_preds_from_accumulator(self, task, accumulator):
        logits = accumulator.get_accumulated()
        return np.squeeze(logits, axis=-1)

    @classmethod
    def compute_metrics_from_preds_and_labels(cls, preds, labels):
        pearson_corr = float(pearsonr(preds, labels)[0])
        spearman_corr = float(spearmanr(preds, labels)[0])
        minor = {
            "pearson": pearson_corr,
            "spearmanr": spearman_corr,
            "corr": (pearson_corr + spearman_corr) / 2,
        }
        return Metrics(major=minor["corr"], minor=minor)


class MultipleChoiceAccuracyEvaluationScheme(BaseLogitsEvaluationScheme):
    def get_accumulator(self):
        return ConcatenateLogitsAccumulator()

    @classmethod
    def get_labels_from_examples(cls, task, examples):
        return get_multiple_choice_label_ids_from_examples(task=task, examples=examples)

    def get_labels_from_cache_and_examples(self, task, cache, examples):
        return get_multiple_choice_labels_from_cache(cache=cache)

    def get_preds_from_accumulator(self, task, accumulator):
        return SimpleAccuracyEvaluationScheme.get_preds_from_accumulator(
            task=task, accumulator=accumulator,
        )

    def compute_metrics_from_preds_and_labels(self, preds, labels):
        return SimpleAccuracyEvaluationScheme.compute_metrics_from_preds_and_labels(
            preds=preds, labels=labels
        )


class CommitmentBankEvaluationScheme(BaseLogitsEvaluationScheme):
    def get_preds_from_accumulator(self, task, accumulator):
        logits = accumulator.get_accumulated()
        return np.argmax(logits, axis=1)

    @classmethod
    def compute_metrics_from_preds_and_labels(cls, preds, labels):
        # noinspection PyUnresolvedReferences
        acc = float((preds == labels).mean())
        labels = np.array(labels)
        f11 = f1_score(y_true=labels == 0, y_pred=preds == 0)
        f12 = f1_score(y_true=labels == 1, y_pred=preds == 1)
        f13 = f1_score(y_true=labels == 2, y_pred=preds == 2)
        avg_f1 = mean(f11, f12, f13)
        return Metrics(
            major=mean(acc, avg_f1),
            minor={"acc": acc, "avg_f1": avg_f1, "f11": f11, "f12": f12, "f13": f13},
        )


class MultiRCEvaluationScheme(BaseEvaluationScheme):
    def get_accumulator(self):
        return ConcatenateLogitsAccumulator()

    @classmethod
    def get_labels_from_examples(cls, task, examples):
        label_values = get_label_ids(examples=examples, task=task)
        question_ids = np.array([example.question_id for example in examples])
        assert len(label_values) == len(question_ids)
        return [
            {"label_values": lab, "question_ids": qid}
            for lab, qid in zip(label_values, question_ids)
        ]

    @classmethod
    def get_labels_from_cache(cls, cache):
        label_values = []
        question_ids = []
        for datum in cache.iter_all():
            label_values.append(datum["data_row"].label_id)
            question_ids.append(datum["data_row"].question_id)
        label_values = np.array(label_values)
        question_ids = np.array(question_ids)
        assert len(label_values) == len(question_ids)
        return [
            {"label_values": lab, "question_ids": qid}
            for lab, qid in zip(label_values, question_ids)
        ]

    def get_labels_from_cache_and_examples(self, task, cache, examples):
        return self.get_labels_from_examples(task=task, examples=examples)

    def get_preds_from_accumulator(self, task, accumulator):
        logits = accumulator.get_accumulated()
        return np.argmax(logits, axis=-1)

    def compute_metrics_from_accumulator(
        self, task, accumulator: ConcatenateLogitsAccumulator, tokenizer, labels: list
    ) -> Metrics:
        preds = self.get_preds_from_accumulator(task=task, accumulator=accumulator)
        return self.compute_metrics_from_preds_and_labels(preds=preds, labels=labels,)

    @classmethod
    def compute_metrics_from_preds_and_labels(cls, preds, labels):
        df = pd.DataFrame(labels)
        assert "label_values" in df.columns
        assert "question_ids" in df.columns
        df["preds"] = preds
        # noinspection PyUnresolvedReferences
        exact_match = (
            df.groupby("question_ids")
            .apply(lambda _: (_["preds"] == _["label_values"]).all())
            .mean()
        )
        exact_match = float(exact_match)
        f1 = f1_score(y_true=df["label_values"], y_pred=df["preds"])
        return Metrics(major=mean(exact_match, f1), minor={"em": exact_match, "f1": f1},)


@dataclass
class RecordLabelData:
    passage_idx: int
    question_idx: int
    entity_str: str
    answers_dict: Dict[str, str]


class ReCordEvaluationScheme(BaseEvaluationScheme):
    def get_accumulator(self):
        return RecordAccumulator()

    @classmethod
    def get_labels_from_examples(cls, examples):
        return [
            RecordLabelData(
                passage_idx=example.passage_idx,
                question_idx=example.question_idx,
                entity_str=example.entity_str,
                answers_dict=example.answers_dict,
            )
            for example in examples
        ]

    @classmethod
    def get_labels_from_cache_and_examples(cls, task, cache, examples):
        return cls.get_labels_from_examples(examples=examples)

    @classmethod
    def get_preds_from_accumulator(cls, task, accumulator):
        logits, entity_strs = accumulator.get_accumulated()
        guid_list = accumulator.get_guids()

        question_ids = []
        for guid in guid_list:
            question_ids.append(guid.split("-")[2])

        # group logits by question id then reorder for submission
        # need question id, logit, and entity_str
        # for example, dict of question id to logit and entity_str
        max_logits = {}
        for logit, entity_str, question_id in zip(logits, entity_strs, question_ids):
            if (question_id not in max_logits) or (max_logits[question_id]["logit"][1] < logit[1]):
                max_logits[question_id] = {"logit": logit, "entity_str": entity_str}

        # Convert labels of max_logits to prediction format
        preds = []
        for question_idx, logit_entity in max_logits.items():
            preds.append({"idx": question_idx, "label": logit_entity["entity_str"]})

        return preds

    def compute_metrics_from_accumulator(
        self, task, accumulator: RecordAccumulator, tokenizer, labels: List
    ) -> Metrics:
        predictions_dict, metrics = self.compute_preds_and_metrics(task, accumulator)
        return metrics

    @classmethod
    def compute_preds_and_metrics(cls, task, accumulator):
        f1_ls = []
        em_ls = []
        predictions_dict = {}

        preds = cls.get_preds_from_accumulator(task, accumulator)
        guid_list = accumulator.get_guids()
        gold_label_list_of_sets = accumulator.get_gold_label_list()

        question_ids = []
        for guid in guid_list:
            question_ids.append(guid.split("-")[2])

        # Reduce list of gold label sets to a gold label set per question_id
        gold_labels = {}
        for question_id, gold_label_set in zip(question_ids, gold_label_list_of_sets):
            if question_id in gold_labels:
                assert gold_label_set == gold_labels[question_id]
            else:
                gold_labels[question_id] = gold_label_set

        for pred, gold_label_set in zip(preds, gold_labels.values()):
            pred_ans = pred["label"]

            # F1
            f1 = cls.metric_max_over_ground_truths(string_f1_score, pred_ans, gold_label_set)
            f1_ls.append(f1)

            # EM
            em = cls.metric_max_over_ground_truths(exact_match_score, pred_ans, gold_label_set)
            em_ls.append(em)

        em = sum(em_ls) / len(em_ls)
        f1 = sum(f1_ls) / len(f1_ls)
        minor = {
            "em": em,
            "f1": f1,
            "f1_em": (f1 + em) / 2,
        }
        metrics = Metrics(major=minor["f1_em"], minor=minor,)
        return predictions_dict, metrics

    @classmethod
    def metric_max_over_ground_truths(cls, metric_fn, prediction, ground_truths):
        """Compute max metric between prediction and each ground truth.
        From official ReCoRD eval script
        """
        scores_for_ground_truths = []
        for ground_truth in ground_truths:
            score = metric_fn(prediction, ground_truth)
            scores_for_ground_truths.append(score)
        return max(scores_for_ground_truths)


class CCGEvaluationScheme(BaseEvaluationScheme):
    def get_accumulator(self):
        return ConcatenateLogitsAccumulator()

    @classmethod
    def get_label_ids_from_cache(cls, cache):
        return [
            {"label_ids": datum["data_row"].label_ids, "label_mask": datum["data_row"].label_mask}
            for datum in cache.iter_all()
        ]

    @classmethod
    def get_labels_from_cache_and_examples(cls, task, cache, examples):
        return cls.get_label_ids_from_cache(cache=cache)

    def get_preds_from_accumulator(self, task, accumulator):
        logits = accumulator.get_accumulated()
        return np.argmax(logits, axis=-1)

    def compute_metrics_from_accumulator(
        self, task, accumulator: ConcatenateLogitsAccumulator, tokenizer, labels: list
    ) -> Metrics:
        preds = self.get_preds_from_accumulator(task=task, accumulator=accumulator)
        return self.compute_metrics_from_preds_and_labels(preds=preds, labels=labels,)

    @classmethod
    def compute_metrics_from_preds_and_labels(cls, preds, labels):
        label_ids = np.stack([row["label_ids"] for row in labels])
        label_mask = np.stack([row["label_mask"] for row in labels])

        # Account for smart-truncate
        assert (label_mask[:, preds.shape[-1] :] == 0).all()
        label_ids = label_ids[:, : preds.shape[-1]]
        label_mask = label_mask[:, : preds.shape[-1]]

        bool_mask = label_mask.reshape(-1).astype(bool)
        flat_preds = preds.reshape(-1)[bool_mask]
        flat_labels = label_ids.reshape(-1)[bool_mask]
        return cls.compute_metrics_from_flat_preds_and_labels(
            flat_preds=flat_preds, flat_labels=flat_labels,
        )

    @classmethod
    def compute_metrics_from_flat_preds_and_labels(cls, flat_preds, flat_labels):
        return SimpleAccuracyEvaluationScheme.compute_metrics_from_preds_and_labels(
            preds=flat_preds, labels=flat_labels,
        )


class F1TaggingEvaluationScheme(BaseEvaluationScheme):
    def get_accumulator(self):
        return ConcatenateLogitsAccumulator()

    @classmethod
    def get_labels_from_cache_and_examples(cls, task, cache, examples):
        labels = []
        for datum in cache.iter_all():
            label_mask = datum["data_row"].label_mask.astype(bool)
            pos_list = [
                task.ID_TO_LABEL[pos_id] for pos_id in datum["data_row"].label_ids[label_mask]
            ]
            label = {
                "pos_list": pos_list,
                "label_mask": label_mask,
            }
            labels.append(label)
            assert len(pos_list) == label_mask.sum()
        return labels

    def get_preds_from_accumulator(self, task, accumulator):
        logits = accumulator.get_accumulated()
        return np.argmax(logits, axis=-1)

    def compute_metrics_from_accumulator(
        self, task, accumulator: ConcatenateLogitsAccumulator, tokenizer, labels: list
    ) -> Metrics:
        preds = self.get_preds_from_accumulator(task=task, accumulator=accumulator)
        return self.compute_metrics_from_preds_and_labels(task=task, preds=preds, labels=labels,)

    @classmethod
    def compute_metrics_from_preds_and_labels(cls, task, preds, labels):
        label_mask = np.stack([row["label_mask"] for row in labels])

        # Account for smart-truncate
        assert (label_mask[:, preds.shape[-1] :] == 0).all()
        label_mask = label_mask[:, : preds.shape[-1]]

        labels_for_eval = [label["pos_list"] for label in labels]
        preds_for_eval = []
        assert len(labels) == preds.shape[0]
        for i in range(len(labels)):
            relevant_preds = preds[i][label_mask[i]]
            relevant_preds_pos = [task.ID_TO_LABEL[pos_id] for pos_id in relevant_preds]
            preds_for_eval.append(relevant_preds_pos)

        minor = {
            "precision": seqeval_metrics.precision_score(labels_for_eval, preds_for_eval),
            "recall": seqeval_metrics.recall_score(labels_for_eval, preds_for_eval),
            "f1": seqeval_metrics.f1_score(labels_for_eval, preds_for_eval),
        }
        return Metrics(major=minor["f1"], minor=minor,)


class SQuADEvaluationScheme(BaseEvaluationScheme):
    @classmethod
    def get_accumulator(cls) -> BaseAccumulator:
        return ConcatenateLogitsAccumulator()

    @classmethod
    def get_labels_from_cache(cls, cache):
        return [cls.get_label_from_data_row(datum["data_row"]) for datum in cache.iter_all()]

    @classmethod
    def get_labels_from_cache_and_examples(cls, task, cache, examples):
        return cls.get_labels_from_cache(cache=cache)

    def get_preds_from_accumulator(self, task, accumulator):
        raise NotImplementedError("Currently can't be done without access to dataset")

    def compute_metrics_from_accumulator(
        self, task, accumulator: BaseAccumulator, tokenizer, labels
    ) -> Metrics:
        logits = accumulator.get_accumulated()
        results, predictions = squad_style.compute_predictions_logits_v3(
            data_rows=labels,
            logits=logits,
            n_best_size=task.n_best_size,
            max_answer_length=task.max_answer_length,
            do_lower_case=model_resolution.resolve_is_lower_case(tokenizer),
            version_2_with_negative=task.version_2_with_negative,
            null_score_diff_threshold=task.null_score_diff_threshold,
            tokenizer=tokenizer,
        )
        if task.version_2_with_negative:
            # Return the score after the best thresholds for answering has been selected
            return Metrics(major=(results["best_f1"] + results["best_exact"]) / 2, minor=results)
        else:
            return Metrics(major=(results["f1"] + results["exact"]) / 2, minor=results)

    @classmethod
    def get_label_from_data_row(cls, data_row):
        return squad_style.PartialDataRow.from_data_row(data_row)


class XlingQAEvaluationScheme(BaseEvaluationScheme):
    @classmethod
    def get_accumulator(cls) -> BaseAccumulator:
        return ConcatenateLogitsAccumulator()

    @classmethod
    def get_labels_from_cache(cls, cache):
        return [cls.get_label_from_data_row(datum["data_row"]) for datum in cache.iter_all()]

    @classmethod
    def get_labels_from_cache_and_examples(cls, task, cache, examples):
        return cls.get_labels_from_cache(cache=cache)

    def get_preds_from_accumulator(self, task, accumulator):
        raise NotImplementedError("Currently can't be done without access to dataset")

    def compute_metrics_from_accumulator(
        self, task, accumulator: BaseAccumulator, tokenizer, labels
    ) -> Metrics:
        logits = accumulator.get_accumulated()
        assert isinstance(task, (tasks_retrieval.TyDiQATask, tasks_retrieval.XquadTask))
        lang = task.language
        results, predictions = squad_style.compute_predictions_logits_v3(
            data_rows=labels,
            logits=logits,
            n_best_size=task.n_best_size,
            max_answer_length=task.max_answer_length,
            do_lower_case=model_resolution.resolve_is_lower_case(tokenizer),
            version_2_with_negative=task.version_2_with_negative,
            null_score_diff_threshold=task.null_score_diff_threshold,
            skip_get_final_text=(lang == "zh"),
            tokenizer=tokenizer,
        )
        return Metrics(major=(results["f1"] + results["exact"]) / 2, minor=results,)

    @classmethod
    def get_label_from_data_row(cls, data_row):
        return squad_style.PartialDataRow.from_data_row(data_row)


class MLQAEvaluationScheme(SQuADEvaluationScheme):
    def get_preds_from_accumulator(self, task, accumulator):
        raise NotImplementedError("Too hard for now, too much handled in one giant lib")

    def compute_metrics_from_accumulator(
        self, task, accumulator: BaseAccumulator, tokenizer, labels
    ) -> Metrics:

        # Todo: Fix val labels cache
        # This is a quick hack
        logits = accumulator.get_accumulated()
        partial_examples = squad_style.data_rows_to_partial_examples(data_rows=labels)
        all_pred_results = squad_style.logits_to_pred_results_list(logits)
        assert task.context_language == task.question_language
        lang = task.context_language
        predictions = squad_style_utils.compute_predictions_logits_v2(
            partial_examples=partial_examples,
            all_results=all_pred_results,
            n_best_size=task.n_best_size,
            max_answer_length=task.max_answer_length,
            do_lower_case=model_resolution.resolve_is_lower_case(tokenizer),
            version_2_with_negative=task.version_2_with_negative,
            null_score_diff_threshold=task.null_score_diff_threshold,
            tokenizer=tokenizer,
            skip_get_final_text=(lang == "zh"),
            verbose=True,
        )
        dataset = read_json(task.val_path)["data"]
        results = mlqa_lib.evaluate(dataset=dataset, predictions=predictions, lang=lang,)
        return Metrics(major=(results["f1"] + results["exact_match"]) / 2, minor=results,)


class MLMEvaluationScheme(BaseEvaluationScheme):
    @classmethod
    def get_accumulator(cls) -> BaseAccumulator:
        return ConcatenateLossAccumulator()

    def get_labels_from_cache_and_examples(self, task, cache, examples):
        # This is a dummy function. There are no external labels.
        return [None]

    def get_preds_from_accumulator(self, task, accumulator):
        raise NotImplementedError("Not possible")

    def compute_metrics_from_accumulator(
        self, task, accumulator: BaseAccumulator, tokenizer, labels
    ) -> Metrics:
        loss_list = accumulator.get_accumulated()
        average_loss = mean(loss_list)
        perplexity = np.exp(average_loss)
        return Metrics(
            # Major = negative perplexity
            major=-perplexity,
            minor={"perplexity": perplexity},
        )


class MLMPremaskedEvaluationScheme(MLMEvaluationScheme):
    @classmethod
    def get_accumulator(cls) -> BaseAccumulator:
        return MLMPremaskedAccumulator()

    @classmethod
    def get_labels_from_cache_and_examples(cls, task, cache, examples):
        labels = []
        for datum in cache.iter_all():
            masked_lm_labels = datum["data_row"].masked_lm_labels
            labels.append(
                masked_lm_labels[masked_lm_labels != mlm_template.NON_MASKED_TOKEN_LABEL_ID]
            )
        return labels

    def get_preds_from_accumulator(self, task, accumulator):
        _, preds = accumulator.get_accumulated()
        return preds

    def compute_metrics_from_accumulator(
        self, task, accumulator: BaseAccumulator, tokenizer, labels
    ) -> Metrics:
        loss_list, _ = accumulator.get_accumulated()
        average_loss = mean(loss_list)
        perplexity = np.exp(average_loss)
        return Metrics(
            # Major = negative perplexity
            major=-perplexity,
            minor={"perplexity": perplexity},
        )


class TatoebaEvaluationScheme(BaseEvaluationScheme):
    def get_accumulator(self):
        return TatoebaAccumulator()

    def get_labels_from_cache_and_examples(self, task, cache, examples):
        return task.get_val_labels()

    def get_preds_from_accumulator(self, task, accumulator):
        all_embeddings, is_english_arr = accumulator.get_accumulated()
        other_lang_embeddings = all_embeddings[~is_english_arr]
        eng_embeddings = all_embeddings[is_english_arr]
        predictions = tatoeba_lib.similarity_search(
            x=other_lang_embeddings,
            y=eng_embeddings,
            dim=other_lang_embeddings.shape[-1],
            normalize=True,
        ).flatten()
        return predictions

    def compute_metrics_from_accumulator(
        self, task, accumulator: ConcatenateLogitsAccumulator, tokenizer, labels: list
    ) -> Metrics:
        preds = self.get_preds_from_accumulator(task=task, accumulator=accumulator)
        return self.compute_metrics_from_preds_and_labels(preds=preds, labels=labels,)

    @classmethod
    def compute_metrics_from_preds_and_labels(cls, preds, labels):
        # noinspection PyUnresolvedReferences
        acc = (preds == labels).mean()
        return Metrics(major=acc, minor={"acc": acc})


class Bucc2018EvaluationScheme(BaseEvaluationScheme):
    def get_accumulator(self):
        return Bucc2018Accumulator()

    def get_labels_from_cache_and_examples(self, task, cache, examples):
        return task.get_val_labels()

    def get_preds_from_accumulator(self, task, accumulator, threshold=0):
        accumulated = accumulator.get_accumulated()
        is_english_arr = accumulated["is_english_arr"]
        all_embeddings = accumulated["all_embeddings"]
        guids = accumulated["guid_list"]
        text_hash_list = accumulated["text_hash_list"]
        other_lang_embeddings = all_embeddings[~is_english_arr]
        eng_embeddings = all_embeddings[is_english_arr]
        english_guids = [x.split("-", 1)[1] for x in np.array(guids)[is_english_arr]]
        other_guids = [x.split("-", 1)[1] for x in np.array(guids)[~is_english_arr]]

        n = len(is_english_arr)
        src_inds, _ = bucc2018_lib.get_unique_lines(
            [text_hash_list[i] for i in np.arange(n) if not is_english_arr[i]]
        )
        trg_inds, _ = bucc2018_lib.get_unique_lines(
            [text_hash_list[i] for i in np.arange(n) if is_english_arr[i]]
        )
        src_ids_map = bucc2018_lib.create_ids_map(src_inds, other_guids)
        trg_ids_map = bucc2018_lib.create_ids_map(trg_inds, english_guids)

        result = bucc2018_lib.mine_bitext(
            x=other_lang_embeddings,
            y=eng_embeddings,
            src_inds=src_inds,
            trg_inds=trg_inds,
            threshold=threshold,
            use_gpu=torch.cuda.is_available(),
        )
        # Note: Setting thresholds only available in test script
        candidates2score = {}
        for score, src_idx, trg_idx in result:
            for src_key, trg_key in itertools.product(src_ids_map[src_idx], trg_ids_map[trg_idx]):
                candidates2score[src_key, trg_key] = score
        return candidates2score

    def compute_metrics_from_accumulator(
        self, task, accumulator: ConcatenateLogitsAccumulator, tokenizer, labels: list
    ) -> Metrics:
        preds = self.get_preds_from_accumulator(task=task, accumulator=accumulator)
        return self.compute_metrics_from_preds_and_labels(preds=preds, labels=labels,)

    @classmethod
    def compute_metrics_from_preds_and_labels(cls, preds, labels):
        labels = [tuple(x.split("\t")) for x in labels]
        result = bucc2018_lib.bucc_eval(preds, gold=labels, threshold=None)
        return Metrics(major=result["F1"], minor=result,)


def get_evaluation_scheme_for_task(task) -> BaseEvaluationScheme:
    # TODO: move logic to task?  (issue #1182)
    if isinstance(
        task,
        (
            tasks_retrieval.AdversarialNliTask,
            tasks_retrieval.AbductiveNliTask,
            tasks_retrieval.AcceptabilityDefinitenessTask,
            tasks_retrieval.AcceptabilityCoordTask,
            tasks_retrieval.AcceptabilityEOSTask,
            tasks_retrieval.AcceptabilityWHwordsTask,
            tasks_retrieval.BoolQTask,
            tasks_retrieval.CopaTask,
            tasks_retrieval.FeverNliTask,
            tasks_retrieval.MnliTask,
            tasks_retrieval.PawsXTask,
            tasks_retrieval.QnliTask,
            tasks_retrieval.RaceTask,
            tasks_retrieval.RteTask,
            tasks_retrieval.SciTailTask,
            tasks_retrieval.SentEvalBigramShiftTask,
            tasks_retrieval.SentEvalCoordinationInversionTask,
            tasks_retrieval.SentEvalObjNumberTask,
            tasks_retrieval.SentEvalOddManOutTask,
            tasks_retrieval.SentEvalPastPresentTask,
            tasks_retrieval.SentEvalSentenceLengthTask,
            tasks_retrieval.SentEvalSubjNumberTask,
            tasks_retrieval.SentEvalTopConstituentsTask,
            tasks_retrieval.SentEvalTreeDepthTask,
            tasks_retrieval.SentEvalWordContentTask,
            tasks_retrieval.SnliTask,
            tasks_retrieval.SstTask,
            tasks_retrieval.WiCTask,
            tasks_retrieval.WnliTask,
            tasks_retrieval.WSCTask,
            tasks_retrieval.XnliTask,
            tasks_retrieval.MCScriptTask,
            tasks_retrieval.ArctTask,
            tasks_retrieval.PiqaTask,
        ),
    ):
        return SimpleAccuracyEvaluationScheme()
    elif isinstance(task, tasks_retrieval.MCTACOTask):
        return MCTACOEvaluationScheme()
    elif isinstance(task, tasks_retrieval.CCGTask):
        return CCGEvaluationScheme()
    elif isinstance(task, tasks_retrieval.CommitmentBankTask):
        return CommitmentBankEvaluationScheme()
    elif isinstance(task, tasks_retrieval.ColaTask):
        return MCCEvaluationScheme()
    elif isinstance(
        task,
        (
            tasks_retrieval.ArcEasyTask,
            tasks_retrieval.ArcChallengeTask,
            tasks_retrieval.CommonsenseQATask,
            tasks_retrieval.CosmosQATask,
            tasks_retrieval.SWAGTask,
            tasks_retrieval.HellaSwagTask,
            tasks_retrieval.MutualTask,
            tasks_retrieval.MutualPlusTask,
            tasks_retrieval.QuailTask,
            tasks_retrieval.SocialIQATask,
            tasks_retrieval.WinograndeTask,
            tasks_retrieval.MCTestTask,
        ),
    ):
        return MultipleChoiceAccuracyEvaluationScheme()
    elif isinstance(task, (tasks_retrieval.MrpcTask, tasks_retrieval.QqpTask)):
        return AccAndF1EvaluationScheme()
    elif isinstance(
        task,
        (
            tasks_retrieval.Spr1Task,
            tasks_retrieval.Spr2Task,
            tasks_retrieval.SemevalTask,
            tasks_retrieval.SrlTask,
            tasks_retrieval.NerTask,
            tasks_retrieval.CorefTask,
            tasks_retrieval.DprTask,
            tasks_retrieval.DepTask,
            tasks_retrieval.PosTask,
            tasks_retrieval.NonterminalTask,
        ),
    ):
        return MultiLabelAccAndF1EvaluationScheme()
    elif isinstance(task, tasks_retrieval.ReCoRDTask):
        return ReCordEvaluationScheme()
    elif isinstance(
        task,
        (
            tasks_retrieval.SquadTask,
            tasks_retrieval.RopesTask,
            tasks_retrieval.QuorefTask,
            tasks_retrieval.NewsQATask,
            tasks_retrieval.MrqaNaturalQuestionsTask,
        ),
    ):
        return SQuADEvaluationScheme()
    elif isinstance(task, (tasks_retrieval.TyDiQATask, tasks_retrieval.XquadTask)):
        return XlingQAEvaluationScheme()
    elif isinstance(task, tasks_retrieval.MlqaTask):
        return MLQAEvaluationScheme()
    elif isinstance(task, tasks_retrieval.MultiRCTask):
        return MultiRCEvaluationScheme()
    elif isinstance(task, tasks_retrieval.StsbTask):
        return PearsonAndSpearmanEvaluationScheme()
    elif isinstance(task, tasks_retrieval.MLMSimpleTask):
        return MLMEvaluationScheme()
    elif isinstance(task, (tasks_retrieval.MLMPremaskedTask, tasks_retrieval.MLMPretokenizedTask)):
        return MLMPremaskedEvaluationScheme()
    elif isinstance(task, (tasks_retrieval.QAMRTask, tasks_retrieval.QASRLTask)):
        return SpanPredictionF1andEMScheme()
    elif isinstance(task, (tasks_retrieval.UdposTask, tasks_retrieval.PanxTask)):
        return F1TaggingEvaluationScheme()
    elif isinstance(task, tasks_retrieval.Bucc2018Task):
        return Bucc2018EvaluationScheme()
    elif isinstance(task, tasks_retrieval.TatoebaTask):
        return TatoebaEvaluationScheme()
    else:
        raise KeyError(task)


def get_label_ids(task, examples):
    return np.array([task.LABEL_TO_ID[example.label] for example in examples])


def get_label_ids_from_data_row(data_row):
    return data_row.label_ids


def get_multi_label_ids_from_cache(cache):
    return np.array(
        [get_label_ids_from_data_row(data_row=datum["data_row"]) for datum in cache.iter_all()]
    )


def get_label_id_from_data_row(data_row):
    return data_row.label_id


def get_label_ids_from_cache(cache):
    return np.array(
        [get_label_id_from_data_row(data_row=datum["data_row"]) for datum in cache.iter_all()]
    )


def get_label_vals_from_cache(cache):
    return np.array(
        [get_label_val_from_data_row(data_row=datum["data_row"]) for datum in cache.iter_all()]
    )


def get_label_val_from_data_row(data_row):
    return data_row.label


def get_multiple_choice_label_ids_from_examples(task, examples):
    return np.array([task.CHOICE_BIMAP.a[example.label] for example in examples])


def get_multiple_choice_label_id_from_data_row(data_row):
    return data_row.label_id


def get_multiple_choice_labels_from_cache(cache):
    return np.array(
        [
            get_multiple_choice_label_id_from_data_row(data_row=datum["data_row"])
            for datum in cache.iter_all()
        ]
    )


def mean(*args) -> float:
    return float(np.mean(args))


def write_metrics(results, output_path, verbose=True):
    results_to_write = {}
    if "loss" in results:
        results_to_write["loss"] = results["loss"]
    if "metrics" in results:
        results_to_write["metrics"] = results["metrics"].to_dict()
    assert results_to_write
    metrics_str = json.dumps(results_to_write, indent=2)
    if verbose:
        print(metrics_str)
    with open(output_path, "w") as f:
        f.write(metrics_str)
