@article{keung-etal-2020-unsupervised,
    title = "Unsupervised Bitext Mining and Translation via Self-Trained Contextual Embeddings",
    author = "Keung, Phillip  and
      Salazar, Julian  and
      Lu, Yichao  and
      Smith, Noah A.",
    editor = "Johnson, Mark  and
      Roark, Brian  and
      Nenkova, Ani",
    journal = "Transactions of the Association for Computational Linguistics",
    volume = "8",
    year = "2020",
    address = "Cambridge, MA",
    publisher = "MIT Press",
    url = "https://preview.aclanthology.org/jlcl-multiple-ingestion/2020.tacl-1.53/",
    doi = "10.1162/tacl_a_00348",
    pages = "828--841",
    abstract = "We describe an unsupervised method to create pseudo-parallel corpora for machine translation (MT) from unaligned text. We use multilingual BERT to create source and target sentence embeddings for nearest-neighbor search and adapt the model via self-training. We validate our technique by extracting parallel sentence pairs on the BUCC 2017 bitext mining task and observe up to a 24.5 point increase (absolute) in F1 scores over previous unsupervised methods. We then improve an XLM-based unsupervised neural MT system pre-trained on Wikipedia by supplementing it with pseudo-parallel text mined from the same corpus, boosting unsupervised translation performance by up to 3.5 BLEU on the WMT`14 French-English and WMT`16 German-English tasks and outperforming the previous state-of-the-art. Finally, we enrich the IWSLT`15 English-Vietnamese corpus with pseudo-parallel Wikipedia sentence pairs, yielding a 1.2 BLEU improvement on the low-resource MT task. We demonstrate that unsupervised bitext mining is an effective way of augmenting MT datasets and complements existing techniques like initializing with pre-trained contextual embeddings."
}