@inproceedings{wang-cho-2019-bert,
    title = "{BERT} has a Mouth, and It Must Speak: {BERT} as a {M}arkov Random Field Language Model",
    author = "Wang, Alex  and
      Cho, Kyunghyun",
    editor = "Bosselut, Antoine  and
      Celikyilmaz, Asli  and
      Ghazvininejad, Marjan  and
      Iyer, Srinivasan  and
      Khandelwal, Urvashi  and
      Rashkin, Hannah  and
      Wolf, Thomas",
    booktitle = "Proceedings of the Workshop on Methods for Optimizing and Evaluating Neural Language Generation",
    month = jun,
    year = "2019",
    address = "Minneapolis, Minnesota",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/jlcl-multiple-ingestion/W19-2304/",
    doi = "10.18653/v1/W19-2304",
    pages = "30--36",
    abstract = "We show that BERT (Devlin et al., 2018) is a Markov random field language model. This formulation gives way to a natural procedure to sample sentences from BERT. We generate from BERT and find that it can produce high quality, fluent generations. Compared to the generations of a traditional left-to-right language model, BERT generates sentences that are more diverse but of slightly worse quality."
}