% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Remove the "review" option to generate the final version.
\usepackage{ACL2023}
\usepackage{graphicx}
\usepackage{booktabs}
\usepackage{tabularx}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out.
% However, it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}

\usepackage{cleveref}
\usepackage{tikz}

\usepackage[normalem]{ulem} % To strike through cosine similarity words

% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\usepackage{hyperref}
\usepackage[hidecomments]{ufdatastudio}
\usepackage{multirow}


%\title{Enhancing indigenous Language Translation with Synthetic Data Augmentation}
%\title{Language-Specific Preprocessing and Synthetic Data Augmentation for Low-Resource indigenous Machine Translation}
\title{Retrieval-Augmented Long-Context Translation for Cultural Image Captioning: Gators submission for AmericasNLP 2026 shared task}

% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
%         Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
%         Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a seperate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \AND
%         Author 2 \\ Address line \\ ... \\ Address line \And
%         Author 3 \\ Address line \\ ... \\ Address line}

\setlength\titlebox{7cm}

\author{
  Aashish Dhawan \\
  University of Florida \\
  \texttt{aashish.dhawan@ufl.edu}
  \And
  Christopher Driggers-Ellis \\
  University of Florida \\
  \texttt{driggersellis.cw@ufl.edu}
  \And
  Dzmitry Kasinets \\
  University of Florida \\
  \texttt{dkasinets@ufl.edu}
  \AND
  % Sanya Chaturvedi \\
  % University of Florida \\
  % \texttt{sanyachaturvedi@ufl.edu}
  Christan Grant \\
  University of Florida \\
  \texttt{christan@ufl.edu}
  \And
  Daisy Wang \\
  University of Florida \\
  \texttt{daisyw@cise.ufl.edu}
}

\begin{document}
\maketitle

\newcommand{\grn}{Guaraní }
\newcommand{\bzd}{Bribri }
\newcommand{\yua}{Yucatec Maya }
\newcommand{\nlv}{Orizaba Nahuatl }
\newcommand{\hch}{Wixárika }

%\begin{abstract}
%We present the University of Florida Gators submission to the \grn track of the AmericasNLP 2026 shared task on cultural image captioning for Indigenous languages. Our system uses a two-stage pipeline: a vision model first generates a Spanish caption, and a retrieval-augmented large language model then produces the final \grn caption using Spanish–\grn exemplars. We compare this approach against an inherited mBART baseline and a range of prompting configurations, varying model family, retrieval size, and exemplar strategy. OpenAI-based systems provide only modest gains over the baseline, while Gemini 2.5 Flash yields much stronger improvements. We also find that using development references from the same evaluation pool as in-context exemplars can substantially overstate performance. To address this, we separate contaminated same-pool results from a cleaner cross-half split evaluation. Under this split protocol, our best Gemini 2.5 Flash system reaches 48.24 chrF++, compared with 32.14 without development exemplars and 22.40 for the baseline. These results show that long-context in-context translation is highly effective for low-resource Indigenous caption generation, but that careful evaluation is essential when development references are reused as prompts.
%\end{abstract}

\begin{abstract}
    We present the University of Florida Gators submission to the AmericasNLP 2026 shared task on cultural image captioning for Indigenous languages. Our two-stage pipeline generates a Spanish intermediate caption with Qwen2.5-VL, then produces the target-language caption using retrieval-augmented many-shot prompting with Gemini 2.5 Flash. We achieve  164.1\%, 131.7\%, and 122.6\% improvements over the shared task baseline for Bribri, Guaraní, and Orizaba Nahuatl captioning, respectively,  in our dev set evaluation and maintain >150\% improvements for the Bribri and Orizaba Nahuatl languages in the test set evaluation. We find retrieval is highly language-dependent, beneficial only for large, in-domain corpora, and that synthetic data augmentation accounts for around 28 chrF++ of the dev set Guaraní performance gain.
    Our submission is the overall winner of the shared task, placing second out of five finalist submissions in human evaluations of target-language captions. Code and prompts are available on GitHub.\footnote{\url{https://github.com/dhawan98/AmericasNLP2026-Gators-Submission}}
\end{abstract}
% Bribri percent improvements
% Greatest percent improvements over the baseline come in the Bribri submission, wherein prompt engineering accounts for morphological complexity.
% Cross-half split part
%We show that reusing development references as in-context exemplars inflates same-pool scores and provide cross-half split estimates for credible reporting. 

%Team Gators placed second in all five language tracks under human evaluation, winning the overall competition.
% compared with 31.76 without development exemplars and 20.82 for the baseline.

% \section{Introduction}
% The world faces a critical challenge in the alarming rate at which languages are disappearing, with indigenous languages being particularly vulnerable \cite{mager2023ethical}. This loss extends beyond mere words, encompassing the erosion of unique cultural heritage and invaluable traditional knowledge \cite{woodbury2012endangered}. Aymara and Guarani, while spoken by a significant number of people in the Americas, are not immune to these challenges. The intricate structures of these languages, coupled with a scarcity of readily available digital resources, pose considerable obstacles to the development of effective machine translation (MT) systems \cite{rodriguez2022challenges}. Projections indicate that a substantial proportion of the world's currently spoken languages could face severe endangerment or extinction by the close of this century, underscoring the urgency of research aimed at preserving and facilitating communication in these languages \cite{woodbury2012endangered}.
% Traditional MT methodologies, which heavily depend on the existence of large parallel corpora, often prove inadequate for low-resource languages like Aymara and Guarani. However, recent progress in neural machine translation (NMT), along with techniques such as data augmentation and cross-lingual transfer learning, offers promising avenues for addressing these limitations. This paper explores the potential of synthetic data augmentation, drawing inspiration from the successful MultiScript30k project and directly contributing to the aims of initiatives like AmericasNLP \cite{mager2023ethical}. AmericasNLP organizes shared tasks specifically designed to foster advancements in MT for the indigenous languages of the Americas \cite{ebrahimi2023findings}. Our investigation focuses on the hypothesis that augmenting existing datasets with parallel data generated synthetically can enhance the performance of the multilingual mBART model in translating between these indigenous languages and a higher-resource language, Spanish. The methodology employed involves fine-tuning the mBART model on both curated and synthetically augmented datasets, with performance evaluated using the widely recognized BLEU and chrF++ scores. Ultimately, this research aims to demonstrate the potential of synthetic data to improve translation quality in these challenging low-resource scenarios, thereby contributing to the broader efforts in language preservation and supporting the goals of initiatives such as AmericasNLP\cite{ebrahimi2023findings}.

\section{Introduction}\label{sec:introduction}
The AmericasNLP 2026~\cite{bui-etal-2026-findings} consists of generating culturally grounded captions for images in Indigenous languages of the Americas. We identify three major challenges. First, the target languages are low-resource. Second, the captions are culturally specific rather than generic visual descriptions. Third, the task requires not only lexical transfer, but also stylistic control.
Successful systems must produce short, natural captions that match the reference register expected by the organizers.

%Our work builds directly on Dhawan et al. \cite{dhawan-etal-2026-improving}, which showed that synthetic parallel data and language-specific preprocessing improve low-resource Indigenous machine translation, including Guaraní. 
%That earlier work targeted sentence-level translation and found that curated-plus-synthetic training improved Guaraní translation with an mBART-based system. 

While the cultural image captioning task is new to AmericasNLP 2026, we find that current vision-language models (VLMs) are not able to directly caption images in the target languages.
This is unsurprising given the limited training data available for the target languages.
We therefore formulate the problem as a compound of image captioning in a high-resource language followed by machine translation from that language into one of the shared task targets, which mirrors the shared task's baseline approach.

% TODO: use VLM over VLLM. VLM is correct. 
We first attempt to build on the work of \citet{dhawan-etal-2026-improving} by providing intermediate Spanish (Es) VLM captions to an mBART-based machine translation (MT) model.
\citet{dhawan-etal-2026-improving} show that synthetic parallel data and language-specific preprocessing improve low-resource Indigenous MT, including Es-\grn (Grn) translation. 
In the 2026 shared task, however, a standard neural machine translation pipeline proves insufficient. When we apply the existing Es-Grn translation model to Spanish intermediate captions produced from the dev images, the resulting captions are often fluent enough at the sentence level but do not match the target caption register. 
In other words, the performance bottleneck shifts from generic translation quality to domain adaptation and culturally grounded caption style.

To address this mismatch, we move from sequence-to-sequence translation toward retrieval-augmented in-context translation with Large Language Models (LLMs). 
The core idea is simple. 
Instead of relying on a single model fine-tuned on mixed-domain low-resource language corpora, we retrieve Es-low-resource (LoRes) examples that are similar to the current caption and provide them as in-context exemplars. 
This design lets the decoder adapt its lexical choices and stylistic register at inference time. We evaluate multiple OpenAI and Gemini models, vary the number of retrieved training examples and development exemplars, and test prompt variants and retrieval heuristics.

Our experiments lead to three main findings. 
First, among OpenAI models, many-shot direct translation outperforms both our mBART baseline and LLM post-correction, but gains are modest and highly sensitive to prompt composition. 
Second, Gemini 2.5 Flash ~\cite{comanici2025gemini25} is dramatically stronger in Es-LoRes in-context translation than the GPT-family models. 
Third, development references are useful as in-context exemplars for matching the expected caption register, but they must be interpreted carefully because same-pool development prompting can inflate development-set scores.

Our primary contributions are as follows.
We present a retrieval-augmented LLM caption translation pipeline for low-resource cultural image captioning. 
We document an extensive set of negative and positive ablations, including model substitutions, prompt revisions, and development-exemplar hyperparameter searches. 
Finally, we highlight evaluation design as a central methodological issue for dev-conditioned in-context learning in low-resource captioning.

\section{Background}\label{sec:background_related_work}
Machine translation for Indigenous languages of the Americas has advanced largely through the AmericasNLP shared tasks, which have established evaluation benchmarks and made parallel corpora available for many low-resource languages \cite{mager2021findings, ebrahimi-etal-2023-findings, ebrahimi2024findings}. Strong systems in these shared tasks have typically relied on multilingual pretrained models such as mBART, M2M-100, or NLLB-200, often combined with synthetic data generation, multilingual transfer, and additional corpus collection \cite{gow2023sheffield, tonja2023enhancing, costa2022no}. A recurring pattern in this literature is that performance improvements come not only from larger models, but also from better alignment between model capacity, augmentation strategy, and the target domain.

Character-based evaluation metrics are also especially relevant in this area. chrF and chrF++ are widely used for morphologically rich languages because they are more robust than BLEU to inflectional variation and spelling differences \cite{popovic-2015-chrf, popovic-2017-chrf}. The AmericasNLP 2026 organizers likewise use chrF++ as the first-stage ranking metric for the shared task.

Finally, our work relates to the broader use of in-context learning for low-resource generation. Instead of relying exclusively on fixed model parameters after fine-tuning, retrieval-augmented prompting can adapt the decoder to the current example at inference time. In our setting, this is especially attractive because the target outputs are short and stylistically constrained: parallel demonstrations can serve not only as semantic guides, but also as direct evidence of the desired caption register.

\section{Dataset and Methodology}\label{sec:datsets_methodology}

\subsection{Baseline System}

We compare our retrieval-augmented LLM pipeline against the Qwen3VL-8B \cite{qwen3technicalreport} Captioning and Sheffield 2023 \cite{gow2023sheffield} MT method, which is the stated baseline for the current shared task on image captioning. 
We apply this baseline to our submission results for dev set captioning in each of the applicable target languages and report percent improvement over the baseline scores.
%We apply this baseline to the same fixed Spanish captions, isolating the effect of the target-language generation stage.

For MT in \grn captioning, we also compare against the mBART-based Es-Grn model from \citet{dhawan-etal-2026-improving}, trained on curated and synthetic parallel data. This serves as the strongest conventional translation baseline in our pipeline and allows us to assess whether retrieval-augmented in-context generation improves over standard sequence-to-sequence translation.
We evaluate several retrieval-augmented GPT-4-family~\citep{openai2024gpt4technicalreport} model variants, summarized in Table~\ref{tab:mt_ablation}: GPT-4o-mini, GPT-4.1-mini, GPT-4.1, along with the competing Gemini 2.5 Flash. 

Across these models, we vary the number of retrieved training examples $r$, development exemplars $d$, and prompting strategy.

% here...
\subsection{Task Data}

%\input{tables/data_stats}
The official development set $\mathcal{D}$ (dev set) contains 50 examples, and the organizers release the data in JSONL format paired with images. The pilot set includes Spanish captions for reference, but the organizers explicitly note that these are pilot-only and will not be present in development or test.

Our final submission pipeline is two-stage. Stage 1 produces a Spanish caption from the image using Qwen 2.5B \cite{bai2025qwen25vl}. Stage 2 utilizes Gemini 2.5 Flash \cite{comanici2025gemini25} for Es-LoRes translation. 


% Make sure there is no extra space
% \subsection{Datasets by Language}

% The shared task covers five target languages: Guaraní, Yucatec Maya, Orizaba Nahuatl, Bribri, and Wixárika. 

% As shown in \Cref{tab:dev_results}, the available retrieval data $\mathcal{R}$ differs substantially across languages, which motivates treating retrieval size $r$ and the number of development exemplars $d$ as language-dependent hyperparameters.

% For Guaraní, we use a Spanish--\grn retrieval bank augmented with synthetic examples from MultiScript30k project \cite{driggersellis2025multiscript30kleveragingmultilingualembeddings}. 
% This makes \grn the only target language in our setup for which we include MultiScript30k synthetic exemplars.

% For Yucatec Maya, the development set is the only retrieval source in our setup. We do not use retrieved training pairs. 

% For the remaining languages, we use a combination of the available parallel data and development exemplars as retrieval sources.

% \newcommand{\grn}{Guaraní }
% \newcommand{\bzd}{Bribri }
% \newcommand{\yua}{Yucatec Maya }
% \newcommand{\nlv}{Orizaba Nahuatl }
% \newcommand{\hch}{Wixárika }
\subsection{Datasets by Language}

The shared task covers five target languages: Guaraní, Yucatec Maya, Orizaba Nahuatl, Bribri, and Wixárika. As shown in \Cref{tab:dev_results}, the available retrieval data $\mathcal{R}$ differs substantially across languages, which motivates treating the retrieval size $r$ and the number of development exemplars $d$ as inference-time hyperparameters that vary across target language submissions.

\paragraph{\grn} We use the largest retrieval bank in our setup. The retrieval Es-Grn bank of 53,183 pairs contains  AmericasNLP 2023 \cite{ebrahimi-etal-2023-findings} training data augmented with synthetic examples from the MultiScript30k project \cite{driggersellis2025multiscript30kleveragingmultilingualembeddings}. 
%This makes \grn the only target language for which we include MultiScript30k synthetic exemplars. 
The \grn retrieval dataset is relatively well aligned with the captioning task, as it contains culturally specific terms, proper nouns, and short descriptive examples useful for visual caption generation.

\paragraph{\yua} We do not have a comparable parallel training corpus for retrieval. The development set $\mathcal{D}$ is the only retrieval source, so we rely on dev exemplars and the pretrained knowledge of the LLM and we fix $r = 0$ in all experiments.

\paragraph{Other Target Languages} We use the available Es-LoRes parallel data as retrieval banks. Their retrieval banks vary in size and domain match: \nlv has 16,145 pairs, \bzd has 7,508 pairs, and \hch has 8,966 pairs. The \hch retrieval data is notably less caption-like, since much of the available corpus is narrative or literary rather than visual-description oriented. These differences motivate the language-specific hyperparameter choices reported in \Cref{tab:dev_results}.

% generalize
% Don't list languages, consolidate
% \subsection{Retrieval Bank}
% \paragraph{\grn} The translation stage uses a Spanish–Guaraní retrieval bank of 53,183 parallel pairs. This bank combines curated Es-Grn training data from AmericasNLP2023 \cite{ebrahimi-etal-2023-findings} with synthetic parallel data from MultiScript30k project \cite{driggersellis2025multiscript30kleveragingmultilingualembeddings}. The resource composition follows the general approach introduced in Dhawan et al. \cite{dhawan-etal-2026-improving}, where synthetic augmentation improved Guaraní MT and increased the number of usable training pairs substantially.

% \citet{dhawan-etal-2026-improving} study Indigenous machine translation with synthetic data augmentation and language-specific preprocessing. 
% They show that curated-plus-synthetic training improved Guaraní and Quechua translation and utilize an mBART MT model.
% The findings emphasize the importance of language-aware normalization for morphologically rich and orthographically variable settings. 
% For the \grn language, the current paper extends that line of work from text-to-text MT to cultural image captioning, where the main challenge is no longer generic translation alone but also stylistic adaptation to a narrow caption register.

% % For retrieval, we index the Spanish side of this parallel corpus with BM25.
% For retrieval, we index the Spanish side of this parallel corpus with BM25 \cite{robertson2009probabilistic}, a TF-IDF-style lexical retrieval method that ranks candidate examples by query-term overlap while accounting for term importance and document length normalization. At inference time, each Spanish caption is used as a query, and the top-$k$ retrieved Spanish–Guaraní pairs are inserted into the prompt as parallel demonstrations. In some configurations, we also add development exemplars chosen from the shared-task development set.

% TODO: Generalize? Make it a sub-section? 
% \paragraph{Prompt construction.}
% Let $r$ denote the number of BM25-retrieved examples from a language-specific retrieval source, and let $d$ denote the number of development exemplars drawn from the shared-task development set. 
% For a query caption $q$, the full prompt context is defined as
% \[
% P(q) = \mathcal{R}_r(q) \cup \mathcal{D}_d,
% \]
% where $\mathcal{R}_r(q)$ contains the top-$r$ retrieved parallel examples available for the target language, and $\mathcal{D}_d$ contains $d$ development reference pairs used as in-context exemplars. 
% The model then generates the target-language caption conditioned on $P(q)$.

% The source of $\mathcal{R}_r(q)$ differs by language: \grn uses a MultiScript30k-augmented retrieval bank, \yua uses no separate training retrieval bank in our setup ($r=0$), and \nlv, \bzd, and \hch use their available parallel data. 
% We sweep $r \in \{0, 20, 28, 40, 80\}$ and $d \in \{0, 4, 10, 20, 49\}$ where applicable; optimal values are language-dependent, as summarized in \Cref{tab:dev_results}.
% \paragraph{Prompt construction.}
% Let hyperparameter $r$ denote the number of BM25-retrieved Spanish--Guaraní pairs from the training bank, and let hyperparameter $d$ denote the number of development exemplars drawn from the shared-task development set. For a query caption $q$, the full prompt context is defined as
% \[
% P(q) = \mathcal{D}_d \cup \mathcal{R}_r(q),
% \]
% where $\mathcal{R}_r(q)$ is the set of top-$r$ Spanish--Guaraní pairs retrieved by BM25 over the Spanish side of the training corpus for query $q$, and $\mathcal{D}_d$ is a fixed set of $d$ development reference pairs. The model then generates the target caption conditioned on $P(q)$.

% We sweep $r \in \{0, 20, 28, 40, 80\}$ and $d \in \{0, 4, 10, 20, 49\}$ across model families; optimal values are language-dependent.

% OK. move up and generalize
% OK. only mention shefield system (baseline)
% OK. the llm comparisons can move down to ablation
% \subsection{Baseline and LLM Systems}

% \paragraph{Guarani.}  
% Our baseline is the Guaraní translation model inherited from \cite{dhawan-etal-2026-improving}, based on mBART and applied to the fixed Spanish intermediate captions. We treat this as the strongest conventional translation baseline available in our pipeline.

% We then test a family of retrieval-augmented LLM systems. The OpenAI experiments use GPT-4o-mini, GPT-4.1-mini, and GPT-4.1. For these models, we vary the number of retrieved training examples, the number of development exemplars, and the prompt formulation. We also test a post-correction approach that asks the LLM to revise the mBART output rather than translate directly.

% Finally, we port the same prompting framework to Gemini 2.5 Flash. This turns out to be the strongest model family in our experiments.

% \paragraph{Wixarika.}

% to remove?
% \subsection{Model Fine-tuning}
% We fine-tune the multilingual mBART model (\texttt{facebook/mbart-large-50}) using the HuggingFace Transformers library. mBART is pretrained using a denoising autoencoding objective across 50 languages, enabling effective cross-lingual transfer in low-resource translation scenarios \cite{tang2020multilingual}. We use the \texttt{MBart50Tokenizer} with Spanish (\texttt{es\_XX}) as the source language and the appropriate target language tags (\texttt{aym\_XX} for Aymara and \texttt{gn\_XX} for Guarani).

% To better support Guarani orthography, we extend the tokenizer vocabulary with Guarani-specific characters and frequent multi-character units, including nasal vowels and diacritics (\textit{\~{a}, \~{e}, \~{\i}, \~{o}, \~{u}}), the combining tilde, and common digraphs (\textit{ch, mb, ng}). After extending the tokenizer, the model’s embedding matrix is resized accordingly.


% For Aymara--Spanish translation, we train using a learning rate of $2\times10^{-5}$, batch size 16, gradient accumulation of 4, and 20 epochs. For Guarani--Spanish translation, we use a learning rate of $3\times10^{-5}$, batch size 8, gradient accumulation of 4, and 15 epochs. All models are optimized using AdamW with a cosine learning rate schedule and warm-up. Early stopping with a patience of three evaluation intervals based on validation BLEU is applied to prevent overfitting. These configurations are consistent with recent AmericasNLP submissions \cite{gow2023sheffield, tonja2023enhancing}.

% Combine with prompt construction
% Captioning prompt, translation prompt
% \subsection{Prompting Strategy}
% The prompt structure is deliberately simple. The system prompt tells the model to translate from Spanish into natural Guaraní/Jopara, match the style of the examples, stay concise, preserve culturally specific nouns where appropriate, and produce exactly one line. The user prompt then presents two sections: a block of optional development exemplars and a block of retrieved Spanish–Guaraní parallel examples, followed by the current Spanish caption.

% This design was chosen after several failed prompt-engineering attempts. More aggressive instructions intended to suppress generic lead-ins or force noun-first phrasing reduced performance rather than improving it. Our final prompts therefore keep the system instruction minimal and let the retrieved demonstrations carry most of the stylistic signal.

\subsection{Retrieval Bank and Prompt Construction}

For each target language, we construct a language-specific Es-LoRes retrieval bank $\mathcal{R}$ from the available parallel data described above. The Spanish side of each retrieval bank is indexed with BM25 \cite{robertson2009probabilistic}, a TF-IDF-style lexical retrieval method that ranks candidate examples by query-term overlap while accounting for term importance and document length normalization. At inference time, the Spanish caption generated in Stage~1 is used as a query $q$, and the top-$r$ retrieved pairs are selected as $\mathcal{R}_r(q) \subset \mathcal{R}$.

In addition to retrieved training pairs, some configurations include development exemplars. Let $\mathcal{D}$ denote the shared-task development set for a target language, and let $\mathcal{D}_d \subset \mathcal{D}$ denote the $d$ development examples included in the prompt. For a query caption $q$, the full prompt context is defined as
\[
P(q) = \mathcal{D}_d \cup \mathcal{R}_r(q),
\]
where $\mathcal{R}_r(q)$ provides retrieval-based semantic and lexical grounding, and $\mathcal{D}_d$ provides examples of the caption style expected by the task. The model then generates the target-language caption conditioned on $P(q)$.

We sweep $r$ and $d$ where applicable. The values selected for submissions are in  \Cref{tab:dev_results}, and detailed grid-search results appear in our appendix.

\subsection{Prompting Strategy}

The MT prompt structure is deliberately simple. The system prompt instructs the model to translate from Spanish into the target language, match the example style, stay concise, preserve culturally specific nouns when appropriate, and produce exactly one line. The user prompt contains two evidence blocks, development exemplars $\mathcal{D}_d$ and retrieved Spanish--target-language pairs $\mathcal{R}_r(q)$, followed by the current Spanish caption.

We use the same general prompt structure across target languages, adding language-specific modifications only when required. After several prompt-engineering attempts, we found that more aggressive instructions, such as suppressing generic lead-ins or forcing noun-first phrasing, reduced performance. The final prompts therefore keep the system instruction minimal and rely on in-context examples for lexical and stylistic guidance. We summarize the final and ablation prompt files in \Cref{tab:prompts} of the appendix.

% \subsection{Evaluation Design}
% add image for Guarani
% gators submission pipeline rename
% \subsection{System Description}
% Our submission system is a retrieval-augmented long-context translator embedded in a two-stage captioning pipeline.
% Given an image, Stage 1 generates a Spanish caption. We treat this stage as a separate visual captioning component and do not optimize it extensively in this paper. Stage 2 then receives the Spanish caption and performs Guaraní generation using an LLM conditioned on two forms of evidence: retrieved Spanish–Guaraní parallel pairs from the training bank and, when allowed by the configuration, a set of development exemplars.
% For each query caption, BM25 retrieval is performed over the Spanish side of the 53,183-pair bank. We tested several retrieval sizes. Among OpenAI models, retrieval size had a narrow optimum around 28 examples, with both smaller and larger values performing worse. Gemini was far more robust to larger context windows, and our strongest final submission recipe uses 80 retrieved pairs.
% Development exemplars serve a different role from retrieved training pairs. Retrieved pairs provide semantic and lexical grounding from the translation bank, whereas development exemplars provide direct evidence of the target caption register used by the task. This distinction is crucial. In the best-performing Gemini configurations, development exemplars contribute a large part of the gain, but they also create evaluation risk if drawn from the same pool as the examples being scored.
% We tested one heuristic reranker that attempted to improve prompt quality by preferring shorter, more caption-like retrieved pairs while penalizing place names and boilerplate. This consistently hurt performance, suggesting that simple hand-designed notions of “good caption examples” were inferior to straightforward BM25 relevance for this task.
% At submission time, our chosen inference recipe is Gemini 2.5 Flash with 80 retrieved pairs, 49 development exemplars, no reranking, and temperature 0.0. This is the strongest test-time configuration we found. For scientific reporting, however, we rely on the cleaner cross-half split results rather than the inflated same-pool full-dev numbers.

% new
% \subsection{System Description}
% Done.  Cut down, some of the redundant text to get to 9 pages
\subsection{University of Florida Gators Submission}

Our submission system features a retrieval-augmented long-context translator embedded in a two-stage image captioning pipeline. Figure~\ref{fig:pipeline} illustrates the full system, which is organized into five steps. Stage~1 corresponds to Step~1 in the diagram. A vision-language model generates one Spanish caption $q$ for each target image. We use either Qwen2.5-VL-72B-Instruct in 4-bit precision or Qwen3-VL-8B~\cite{bai2025qwen3vl} for this stage. The prompt is culturally aware; follows a noun-first style; and encourages concise descriptions of visible entities, objects, clothing, actions, and scene context. We treat this stage as fixed and do not optimize it extensively in this paper.

\begin{figure}[!htbp]
    \centering
    \includegraphics[width=\columnwidth]{figures/v12_pipeline.pdf}
    \caption{Overview of the proposed two-stage image captioning pipeline. Here, $\mathcal{R}$ denotes the full Es-LoRes retrieval bank, $\mathcal{R}_r(q) \subset \mathcal{R}$ denotes the $r$ nearest Es-LoRes training pairs retrieved for Spanish caption $q$, $\mathcal{D}$ denotes the full development set, and $\mathcal{D}_d \subset \mathcal{D}$ denotes the $d$ gold dev pairs used in the many-shot prompt.}
    \label{fig:pipeline}
\end{figure}

Stage~2 corresponds to Steps~2--5 in Figure~\ref{fig:pipeline}. It transforms the generated Spanish caption $q$ into the final target-language caption using retrieval-augmented many-shot MT. The $q$ is used as a BM25 query over the Spanish side of the available Es-LoRes retrieval bank $\mathcal{R}$. In Step~3, the retrieved subset $\mathcal{R}_r(q)$ and development subset $\mathcal{D}_d$ are assembled into a many-shot prompt with an instruction to translate from Spanish to the target language while matching the example style. Each prompt contains approximately 3K--5K tokens. The $r$ nearest Es-LoRes training pairs provide semantic and lexical grounding, while the $d$ development pairs provide direct evidence of the target caption register. In Step~4 of Figure~\ref{fig:pipeline}, Gemini 2.5 Flash performs the final target-language generation with temperature set to 0.0, thinking disabled, and a maximum output length of 120 tokens. We then strip prefixes in Step~5 and normalize whitespace to produce the final JSONL submission. The system is evaluated using chrF++, followed by human judgment for the top-ranked submissions.


\input{tables/dev_results}

\section{Results}\label{sec:results}
\input{tables/result_gua}
\Cref{tab:dev_results} gives our final submission's performance for each of the target languages in the shared task while \Cref{tab:mt_ablation} summarizes the main progression of Es-Grn dev set experiments from an mBART translation model to in-context MT with Gemini 2.5 Flash, which is competent in all target languages.
Several trends emerge immediately. First, direct many-shot translation is better than post-correction, confirming that it is more effective to generate the caption in one step than to repair the mBART output after the fact. Second, among the OpenAI models we tested, GPT-4o-mini remains the strongest, but the gains over the mBART baseline are modest. Third, Gemini 2.5 Flash yields much larger improvements, even before adding any development exemplars.

In our final dev set results, the \grn target achieves the highest absolute chrF++ score among the five target languages by at least 10 chrF++ and more than doubles the \bzd and \hch target performances.
For \grn we achieve a remarkable 131.7\% improvement over the baseline method \cite{gow2023sheffield}.
In \Cref{sec:ablations}, we investigate the effect that synthetic exemplars may have had on \grn target language performance versus the other languages for which no MultiScript30k data exists.
However, as the final column in \Cref{tab:dev_results} attests, each language target outperforms the baseline method whenever the baseline is available.
In particular, for the \bzd and \nlv targets, respectively, we achieve 164.1\% and 122.6\% improvements over the baseline.
As we highlight in \Cref{tab:dev_results}, our improvement for \bzd is the greatest relative improvements over the baseline method for any target language.

The test results are slightly different. \bzd is replaced by \nlv as most improved language. The languages achieve 155.2\% and 166.9\% improvement over the testing baseline, respectively \cite{bui-etal-2026-findings}.
Guaraní performance drops by more than half in the test evaluation and most of the dev set performance gain is not reproduced in the official evaluation as a result. \hch performance improvement falls by a similar amount.

\section{Ablations}\label{sec:ablations}

In addition to the results we list in the previous section, we provide a number of ablations to demonstrate the superiority of the final submission over numerous alternative approaches to various facets of the captioning pipeline. 
In the experiments reported here, we hold the Spanish captions fixed and optimize only the \grn translation stage. 
This lets us analyze ablation effects independent of the image captioner and makes the ablations directly comparable to our dev set results.
We focus on the dev set evaluation here because ablations are all performed on the dev set before the release of final shared task results.


\subsection{Machine Translation Architecture}

We allude in the first section to how the initial approach for the machine translation step in \grn utilizes an mBART based MT model \cite{dhawan-etal-2026-improving}.
\Cref{tab:mt_ablation} shows Guaraní captioning performance across several MT architectures, including mBART, GPT-4o, and Gemini 2.5 Flash.
Where appropriate, we adopt the retrieval-augmented approach in our final submission, and we vary the values of $r$ and $d$ within architectures.

The results show that the best configuration for \grn translation is the Gemini 2.5 Flash LLM prompted with $r = 80$ and $d = 49$ exemplars.
We traverse much of the $r,d$ search grid with OpenAI models. The best OpenAI setting used GPT-4o-mini with 28 retrieved examples and 4 development exemplars, reaching 23.60 chrF++. This performance is reflected in \Cref{tab:mt_ablation}. 
Removing or increasing development exemplars reduced performance. 
GPT-4.1-mini and GPT-4.1 were both competitive but did not surpass GPT-4o-mini.
This pattern suggests that, for the OpenAI models we tested, the in-context regime has a narrow optimum. Too little context leaves the model underconstrained; too much context appears to add noise or dilute the style signal.

For its superior performance over GPT models and its positive receptivity to context, we adopt Gemini 2.5 Flash for Es-LoRes MT in our final submission.


\subsection{Hyperparameter Search}

For each language, we sweep values of $r$ and $d$ that we list in \Cref{sec:datsets_methodology}.
\Cref{tab:dev_results} of final submission results shows performance at the submittal configuration featuring its $r,d$ pair.
To be thorough, \Cref{tab:grid_search} reports the results of a partial grid search for each target language in our appendix.

Though this ablation shows that the $r$ and $d$ exemplar counts from \Cref{tab:dev_results} are optimal within our search grid, there is no accounting for values outside of it.
Additionally, performance changes as one scans the grid indicate differing impacts of the $r$ and $d$ hyperparameters for different languages and data sources.
Thus, we stress in this ablation the importance of a thorough search for the optimal $r,d$ pair for any new target language or data configuration in retrieval-augmented Es-LoRes MT.

\subsection{Synthetic Exemplars} \label{subsec:synthetic_exemplars}

We notice in \Cref{sec:results} that our pipeline performs \grn image captioning much more effectively in absolute terms versus any of the other target languages, even though we achieve similar percent improvement over the applicable baseline for the \bzd and \nlv targets.
We also note that \grn is the only target language for which we include synthetic exemplars from MultiScript30k.
Testing for the effect of this synthetic data, we ablate the original \grn captioning submission by only using AmericasNLP 2023 \cite{ebrahimi-etal-2023-findings} training data for retrieval exemplars ($r$).
\Cref{tab:ms30k_ablation} compares performance with and without MultiScript30k \cite{driggersellis2025multiscript30kleveragingmultilingualembeddings} synthetic exemplars for three $r, d$ pairs and provides the greatest performance overall without synthetic exemplars.

The results are clear.
Controlling for our retrieval hyperparameters by fixing $r$ and $d$ to three pairs, we observe that the original configuration with both genuine AmericasNLP 2023 training data and synthetic MultiScript30k exemplars outperforms the ablation with AmericasNLP 2023 alone by more than 100\% relative improvement in chrF++ in each case.
Additionally, the best \grn captioning performance without synthetic exemplars is 55.9\% less than the best performance with them.
These results mirror the comparison of our submission's \grn performance to the other target languages in the shared task and attribute much of our improvement in \grn captioning to synthetic exemplars in the retrieval-augmented Es-Grn translation step.

% TODO Remove repeating name? 
\input{tables/ms30k_ablation}

\subsection{Alternative Prompting and Reranking}
We also test several prompting ablations that looked reasonable but are consistently negative in their impact on performance, with the exception of a specific strategy we give additional attention to in \Cref{subsec:morph_bzd}.
In this section, we quickly summarize other alternative prompts, and in \Cref{tab:prompts}, we produce all of the relevant prompts for completeness.

A prompt rewrite aimed at suppressing generic scene-introduction phrases reduces performance substantially, and a retrieval reranker that attempts to prefer short caption-like pairs also reduces performance, both for OpenAI and Gemini. 
Likewise, using 49 same-pool development exemplars with GPT-4o-mini degraded performance instead of improving it.
These results matter because they show that this task does not respond well to aggressive heuristic control. 
The most effective systems are built by keeping the instruction stable and varying only model choice and amount of context supplied.

For \hch in particular, we devise special prompts with cultural context in the form of a glossary.
These glossaries contain the names of common objects from the \hch culture and their definitions in Spanish, with the hope that this additional context will help the LLM MT architectures translate intermediate Spanish captions into the \hch target language.
We utilize two versions of the cultural glossary, but the results do not improve for either one.
We include the additional prompts in \Cref{tab:prompts} to document this alternative prompting.

\subsection{Morphological Considerations for \bzd} \label{subsec:morph_bzd}

We ablate our submission for the \bzd (Bzd) language by considering performance with and without morphological prompting and post-processing for the Es-Bzd MT.
\bzd has complex tonal marking in orthography (circumflexes, underlines, multiple diacritics) which makes the deduplication logic harder \cite{coto-solano-2021-explicit}.
The tokenizer splits differently on tonal characters.

In our final submission, we utilize a complex postprocessing and additional prompting for the Gemini MT model to improve performance. 
\Cref{tab:morph_ablation} shows the evolution of our strategies to account for the morphological complexity of the \bzd language.
First, careful examination of our pipeline's output reveals that \bzd captions are initially in NFC encoding, which combines base letter and diacritic encodings into one character.
This does not match the dev set examples, which use NFD encoding.
For instance, our model outputs \textit{ë} as a single unit, but the dev set's NFD encoding would separate the letter \textit{e} from the umlaut above it.
Because chrF++ is a character-level lexical metric, this mismatch depresses the score. We therefore perform NFD-Normalization (NFD-Norm.) to account for the difference.
Secondly, we believe the morphological considerations of Es-Bzd translation significant enough that they deserve special prompting.
To this end, we formulate a Morphological Prompt (Morph.) for \bzd that includes special instructions about Subject-Object-Verb (SOV) word order, tonal diacritics, common consonant clusters, verb-final clauses, and possessive noun prefixes.

As in previous ablations, the results in \Cref{tab:morph_ablation} largely speak for themselves. NFD-normalization overcomes the encoding mismatch, and morphological prompting provides additional improvements.

\input{tables/morph_ablation}


\section{Discussion}\label{sec:discussion}

We now proceed to a discussion of our method and the results achieved.
We analyze the available data and investigate the potential sources of performance improvement over the baseline method.

\subsection{In-Context Retrieval}
Though limited to the \grn target, we show in our ablation of MT architectures that long-context retrieval-based MT using state-of-the-art LLMs greatly improves image captioning performance at the MT stage versus the dedicated mBART model \cite{dhawan-etal-2026-improving}.
While we cannot say for certain whether this relationship holds for other language targets, from comparisons to baseline performances, it appears that additional context from real and synthetic exemplars significantly boosts LLM translation in Es-LoRes tasks. 

\subsection{Synthetic Exemplars}

For the \grn language, we include approximately 30k synthetic exemplars for in-context retrieval from the MultiScript30k \cite{driggersellis2025multiscript30kleveragingmultilingualembeddings} dataset.
We observe in Section 4 that Guaraní achieves substantially higher absolute dev chrF++ than the other target languages.

Remembering that synthetic exemplars apply only to \grn and its significantly positive effect on Es-Grn MT in related work \cite{dhawan-etal-2026-improving}, we ablate for the synthetic exemplars' effect on \grn captioning.

For \grn captioning without the MultiScript30k synthetic retrieval pairs, the results we elaborate in \Cref{subsec:synthetic_exemplars} and \Cref{tab:ms30k_ablation} show a comparison similar to \grn captioning versus the other target languages.
Results improve over 100\%, approximately 28 total chrF++ or more, for \grn captioning with the synthetic exemplars for the same $r,d$ pair.
% For further validation we seek the optimal r,d for \grn captioning without synthetic exemplars. The caparison persists although the gap narrows slightly.
These results also mirror observations from \cite{dhawan-etal-2026-improving} on the effect of synthetic exemplars on Es-Grn MT.
We therefore conclude that synthetic exemplars drive \grn performance gains and speculate that synthetic retrieval pairs may further improve captioning performance for other target languages.

\subsection{Morphology}
Despite favorable results in the \grn target, we acknowledge in \Cref{sec:results} that our greatest relative improvement is for the \bzd language.
The ablation in \Cref{subsec:morph_bzd} clarifies that much of this performance improvement stems from the morphological considerations we take for the \bzd language target via specialized prompting.
%Without the specialized prompt we achieve just 51.92\% improvement over the shared task's baseline method \cite{gow2023sheffield}.
We observe that explicitly prompting for Bribri's morphological features accounts for over 10\% of the performance gain in our final submission over the shared task's baseline \cite{gow2023sheffield}.
We conclude morphological prompting has potential for LoRes MT in morphologically complex languages.

\section{Future Work}\label{sec:future_work}
The most impactful next step is improving the visual captioning stage. 
Error analysis indicates that approximately 54\% of remaining Guaraní errors originate in the vision model rather than the translator. 
A stronger VLM will likely yield larger gains than further translation tuning. 
Beyond captioning, we may extend retrieval to incorporate visual similarity rather than Spanish text overlap alone, which would help when the intermediate caption is noisy or culturally ambiguous. 
The lowest-resource languages (\hch and \bzd languages) are bottle-necked by corpus domain mismatch rather than model choice. 
Even small caption-style parallel data corpora for these languages would likely produce larger gains than any prompting improvement.

\section{Conclusion}\label{sec:conclusion}
We present the University of Florida Gators team system for the AmericasNLP 2026 shared task on cultural image captioning for Indigenous languages \cite{bui-etal-2026-findings}. 
Our two-stage pipeline, VLM captioning in Spanish followed by retrieval-augmented many-shot translation with Gemini 2.5 Flash, substantially outperforms fine-tuned baseline models across all five target languages, culminating at 48.24 chrF++ for Guaraní in the submission.
We find that retrieval behavior is highly language-dependent. 
Large retrieval windows help Guaraní but hurt Yucatec Maya, where Gemini's pre-training knowledge is sufficient and BM25 retrieval adds noise. 
We also find that development exemplars are useful for matching caption register, but their use requires careful interpretation because they can inflate development-set scores when drawn from the same evaluation pool.
For the lowest-resource languages, the ceiling is domain mismatch, not model capacity.

%Finally, we hope our submission to the AmericasNLP 2026 shared task on cultural image captioning and our ablative analyses will guide future efforts in LoRes image captioning.

\section*{Limitations}\label{sec:limitations}
Our system is a cascade: errors in the Spanish captions propagate into translation with no recovery mechanism. Because most ablations hold the Spanish captions fixed and vary only the translation stage, they likely underestimate the contribution of the visual captioning model to final performance.

A second limitation is the use of development examples as in-context exemplars. These examples are useful for matching the expected caption register, especially when little caption-style target-language data is available, but they can also inflate development-set scores when exemplar selection and evaluation draw from the same small pool. We therefore use dev-assisted results primarily primarily model-selection and submission-configuration. A more rigorous and comprehensive evaluation would report held-out or cross-split dev results.

Finally, evaluation relies primarily on chrF++, which is useful for low-resource and morphologically rich languages but cannot fully capture fluency, cultural appropriateness, or naturalness for native speakers. Although the shared task includes human evaluation for finalist systems, our ablations include no native-speaker evaluation.

% \section{Acknowledgments}
\section*{Acknowledgments}
The authors gratefully acknowledge Arnold and Lisa Goldberg, whose financial support helped to make this work possible. We also  acknowledge the Gatorade, whose support provided the compute resources necessary for this research.

% Entries for the entire Anthology, followed by custom entries
\bibliography{citations}
\bibliographystyle{acl_natbib}

\newpage
\section*{Appendix} \label{sec:appendix}

Here, we provide additional data for validation of our method versus various ablations with particular focus on alternative prompts for various target languages and the $r,d$ grid search.
In the following sections, we give tables of alternative prompts and performance at different $r,d$ pairs for each target language.

\subsection*{Alternative Prompts} \label{subsec:alternative_prompts}

\Cref{tab:prompts} yields the prompts that we utilize in our ablations and the prompts present in
our final submission for each target language as indicated in the \textit{Standing} column. 
The \textit{Language(s)} column shows which languages the prompt applies to.

\subsection*{$r,d$ Hyperparameter Search}

We frequently refer to a search for optimal $r$ and $d$ retrieval hyperparameters in the main body of this paper but reserve detailed communication of the sweep for this appendix due to the number of configurations we consider.
For the Gemini 2.5 Flash MT architecture and for each target language, we partially sweep a grid consisting of $r,d$ pairs such that $r \in \{ 0, 10, 20, 40, 80 \}$ and $d \in \{0, 10, 20, 30, 40, 49\}$. Table~\ref{tab:grid_search} reports the chrF++ scores for the final captioning pipeline for each combination of $r$ and $d$ tested for each target language.
% For each target language, we provide chrF++ scores for the final captioning pipeline at each combination of $r$ and $d$.

\input{tables/prompts}
\input{tables/grids}

\end{document}
