@inproceedings{pietruszka-etal-2026-models,
    title = "Can Models Help Us Create Better Models? Evaluating {LLM}s as Data Scientists",
    author = "Pietruszka, Micha{\l}  and
      Borchmann, {\L}ukasz  and
      J{\k{e}}drosz, Aleksander  and
      Morawiecki, Pawe{\l}",
    editor = "Demberg, Vera  and
      Inui, Kentaro  and
      Marquez, Llu{\'i}s",
    booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EACL} 2026",
    month = mar,
    year = "2026",
    address = "Rabat, Morocco",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/ingest-eacl/2026.findings-eacl.308/",
    pages = "5862--5886",
    ISBN = "979-8-89176-386-9",
    abstract = "We present FeatEng, a novel benchmark designed to evaluate the ability of large language models (LLMs) to perform feature engineering, a critical and knowledge-intensive task in data science. FeatEng assesses LLMs by their capacity to generate Python code that transforms raw tabular data into features that improve the performance of a downstream machine learning model. Our analysis of LLM outputs reveals that success on FeatEng often requires the application of significant world and domain knowledge, along with complex reasoning, to construct novel data representations. While focused on feature engineering, the benchmark probes a confluence of abilities indicative of an LLM{'}s broader potential for practical, data-centric problem-solving. We demonstrate that FeatEng offers a targeted and efficient approach to assess a specific but crucial aspect of LLM capabilities relevant to real-world data science applications."
}