@inproceedings{chen-etal-2025-toward,
    title = "Toward Traditional {C}hinese {M}odern{BERT}: A Preliminary Study",
    author = "Chen, Yi-En  and
      He, Qiao-Ying  and
      Chen, Kuan-Yu",
    editor = "Chang, Kai-Wei  and
      Lu, Ke-Han  and
      Yang, Chih-Kai  and
      Tam, Zhi-Rui  and
      Chang, Wen-Yu  and
      Wang, Chung-Che",
    booktitle = "Proceedings of the 37th Conference on Computational Linguistics and Speech Processing (ROCLING 2025)",
    month = nov,
    year = "2025",
    address = "National Taiwan University, Taipei City, Taiwan",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/dashboard/2025.rocling-main.16/",
    pages = "133--139",
    ISBN = "979-8-89176-379-1",
    abstract = "This study employs several state-of-the-art techniques, including RoPE and Flash Attention, and leverages large-scale Chinese web corpora and encyclopedic data to pre-train an encoder model specifically designed for long text in Traditional Chinese. We evaluate the model on tasks such as reading comprehension and text classification, and the results show that its overall performance lags behind existing Chinese benchmarks. Through pseudo-perplexity analysis, we infer that the pre-training phase did not sufficiently capture the data distribution, potentially due to factors such as hyperparameters, convergence, and data quality. Although the results are suboptimal, this study still offers valuable experimental insights and directions for improving Chinese language model development."
}