% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.
 


\documentclass[11pt]{article}

% Remove the "review" option to generate the final version.
\usepackage[]{ACL2023}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

\usepackage{float}

% This is not strictly necessary, and may be commented out.
% However, it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}

\usepackage{pgfplots}
\pgfplotsset{compat=1.17}

\usepackage{tikz}
\usetikzlibrary{shapes.geometric, arrows.meta, positioning, calc}

\usepackage{amsmath}

\usepackage{amssymb}

\usepackage{booktabs}

\usepackage{rotating}
%for the appendix
%\usepackage{booktabs}

%\usepackage[flushleft]{threeparttable}

% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

%\title{HHU-SyLo at SemEval-2026 Task 11: Logic in the Loop -- Augmenting LLMs with Theorem Provers to Disentangle Belief from Reasoning}
\title{HHU-SyLo at SemEval-2026 Task 11: Logic in the Loop -- Hybridizing LLMs and Theorem Provers for Robust Formal Reasoning}
%\title{HHU-SyLo at SemEval-2026 Task 11: The Best of Both Worlds -- Hybridizing LLMs and Theorem Provers for Robust Formal Reasoning}

% Titel Vorschläge:
% -1. "Neural Nets vs. Symbolic Sets: A Triple-Threat Approach to Syllogistic Disentanglement"
%%% -2. "Prompting the Proof: Iterative Approaches to Disentangling Logic from Belief"
% -3. "Square of Opposition, Circle of Confusion: Managing Semantic Bias with Structural Prompts"
% -4. "All Models are Smart, but Some are More Logical: Disentangling Content and Formal Reasoning in LLMs"
% -5. "All Prompts are Equal, but Some Approaches are More Logical: Disentangling Contotent and Formal Reasoning via Symbolic Translation
%-6. "Every Dog is an Airplane? & other Sentences to make your brain hurt: a Triple threat approach to pass the pain down to LLMs.
%-7 "Some animals are Windows" How 3 different architectures can help your LLM distangle logical reasoning from semantic plausibility.
%-8.All Syllogisms are Logic: Logic a Syllogism: ∀x(Syllogism(x) -> logic(x)).
%9 The Answer is Hybrid: LLMs meets Theorem Prover.
%10 Disentanglig Logic from Belief bias LLMs and Theoreme proves come together in a battle with Syllogisms.

%11
% "Logic in the Loop: Augmenting LLMs with Theorem Provers to Disentangle Belief from Reasoning"

%12
% ?"Hybrid Reasoners: Integrating Automated Theorem Proving for Bias-Resistant Formal Logic"

%13
% "Scaling Logic, Not Bias: Automated Proofs as a Corrective for Neural Belief Effects"

%+14
% "The Best of Both Worlds: Hybridizing LLMs and Theorem Provers for Robust Formal Reasoning"

%15
% "Neural Prompting, Symbolic Proof: A Hybrid Approach to Disentangling Logic from Belief Bias"

% Xuan: 14  10 11
% Cherine: 14 11 15 

%11
% "Logic in the Loop: Augmenting LLMs with Theorem Provers to Disentangle Belief from Reasoning"
% "Logic in the Loop: Hybridizing LLMs and Theorem Provers for Robust Formal Reasoning"

%+14
% "The Best of Both Worlds: Hybridizing LLMs and Theorem Provers for Robust Formal Reasoning"



\author{
  Wiebke Petersen \quad Cherine Jaziri \quad Diem-Xuan Tran \\
  Heinrich-Heine-Universität Düsseldorf, Germany \\
  \texttt{\{wiebke.petersen, cherine.jaziri, diem.tran\}@hhu.de}
}


\usepackage{url}
\begin{document}
\maketitle





% wir verwenden:
% einfache Anführungszeichen für Sprachbeispiele wie `rotten apple'
% kursiv für Eigennamen von Modellen/Task
% in Table (Table, Figure, ... groß schreiben, wenn Nr. darauf folgt).
% US-English (analyze)
% Anführungszeichen immer nach Satzzeichen
% Doppelte Anführungszeichen für ganze Sätze

\begin{abstract}
%In this paper, we describe our contribution to Semeval 2026 Task 11, across the four subtasks, which focus on reasoning disentanglement. The task requires models to distinguish between the logical validity and the semantic conent of a syllogism.We evaluate three architectures: direct neural inference, translation to first‑order logic, and translation to syllogistic triples; the translation pipelines offload inference to symbolic solvers to reduce content bias. Hybrid solver‑anchored models improve logical fidelity, though prompt–example alignment remains a key limitation.

%We describe our system for SemEval‑2026 Task 11 on reasoning disentanglement, which separates syllogistic logical validity from semantic plausibility. We evaluate three architectures — direct neural inference, translation to first‑order logic, and translation to syllogistic triples — where the translation pipelines offload inference to symbolic solvers to mitigate content bias. Hybrid solver‑anchored models improve logical fidelity, though alignment between prompts and few‑shot examples remains a primary limitation.

We present our system for SemEval-2026 Task 11 on reasoning disentanglement, separating syllogistic validity from semantic plausibility. We compare direct neural inference against two neuro-symbolic pipelines: translation to first-order logic and to syllogistic triples. By offloading inference to symbolic theorem provers, these hybrid models effectively mitigate content bias and improve logical fidelity. 
\end{abstract}


\section{Introduction}

SemEval-2026 Task 11 \citep{valentino-etal-2026-semeval}
evaluates syllogistic reasoning by disentangling logical validity from semantic plausibility. LLMs frequently succumb to the content effect, where world knowledge biases them toward labeling plausible conclusions as valid. Subtask 1 focuses on English validity checking, while Subtask 2 adds noise through irrelevant premise identification. Subtasks 3 and 4 extend these challenges to a multilingual setting (11 languages), further combining validity classification with premise retrieval.

% This shared task \cite{valentino-etal-2026-semeval} evaluates the validity of syllogistic arguments by disentangling logical reasoning from semantic plausibility. %A syllogism is a form of logical reasoning where a conclusion is drawn from two or more given premises. In this shared task, 
% These arguments %are presented in a way that
% vary in both their formal logic %(valid vs. invalid) 
% and their semantic plausibility, %(plausible vs. implausible)
% %Plausibility is determined only by real-world knowledge rather than the formal logic of the argument.
% challenging LLMs that often struggle with the %so-called 
% content effect, where pre-trained world knowledge biases them to label plausible conclusions as logically valid.%, regardless of their structure. 

% %To address this challenge, the task is organized into four distinct subtask: 
% \textbf{Subtask 1} focuses on binary classification of English syllogisms. %, penalizing susceptibility to content bias (TCE). 
% \textbf{Subtask 2} introduces noise by requiring both validity prediction and the retrieval of specific relevant premises ($F1_{premises}$) in English. \textbf{Subtask 3} extends %the binary
% classification to a multilingual setting %to measure logical competence and cross-lingual stability
% across 11 languages. \textbf{Subtask 4} %represents the most complex scenario, 
% combines multilingual retrieval with formal reasoning in the presence of irrelevant premises.

%Our system evaluates three architectures: to achieve reasoning disentanglement. We compare a direct neural classification baseline and two translation-based pipelines: one that maps natural language to formal logical predicates %for automated theorem proving and another that transforms arguments into structured syllogistic triples to be verified against a symbolic knowledge base. %This allows us to contrast end-to-end model performance with hybrid approaches that offload logical validation to external formal solvers. 
%In order to solve this problem, we evaluated a direct neural baseline and two translation-based pipelines that map text to formal logical predicates or structured triples[cite: 14]. By offloading inference to symbolic solvers, we isolate formal logic from semantic bias.


%Our best model from subtask 1 achieved an accuracy of 95.29\%, ranking 19th among 45 participants. In subtask 2, our model achieved an accuracy of 95.79\% ranking 9th out of 16 participants. For subtask 3, it reached an accuracy of 88.02\% and ranked 12th out of 15. In subtask 4, the model obtained an accuracy of 77.08\%, placing 11th out of 15.\footnote{The code of our approach can be found at \url{https://github.com/WiebkePetersen/HHU-SyLo-SemEval2026-Task11}.} 


To address this, we compare a neural baseline against two translation pipelines mapping text to first order logic (FOL) or Aristotelian Syllogistic Triples (ASTs). By offloading inference to symbolic solvers, we aim at decoupling formal logic from semantic bias.
In Subtask 1, our best model reached 95.29\% accuracy, ranking 19th among 45. For Subtasks 2, 3, and 4, it achieved 95.79\% (9th/16), 88.02\% (12th/15), and 77.08\% (11th/15), respectively.\footnote{Our code is available at \url{https://github.com/WiebkePetersen/HHU-SyLo-SemEval2026-Task11}.} 

%\section{Background} 

% %\subsection{Datasets}
% The SemEval 2026 Task 11 dataset is designed to evaluate models in different linguistic contexts. For Subtasks 1 and 2, the organizers provided an English-language dataset. In contrast, Subtasks 3 and 4 extend the evaluation to a multilingual setting, encompassing a diverse range of languages.

% \begin{table}[h]
% \centering
% \footnotesize

% \label{tab:data_distribution}
% \begin{tabular}{@{}l|c|cccc|c@{}}
% \hline
% \textbf{Example Set ID} & \textbf{\#} & \textbf{IV} & \textbf{PV} & \textbf{II} & \textbf{PI} & \textbf{$\varnothing$} \\ \hline
% Subtask 1 (Train)       & 960         & 240          & 240          & 246          & 234          & 3.0 \\
% Subtask 1 (Test)        & 191         & 48          & 48          & 48          & 47          & 3.0 \\ \hline
% Subtask 2 (Test)        & 96          & 24          & 24          & 24          & 24          & 7.0 \\ \hline
% Subtask 3 (Test)        & 121         & 32          & 24          & 33          & 32          & 3.2 \\ \hline
% Subtask 4 (Test)        & 101         & 26          & 22          & 26          & 27          & 7.4 \\ \hline
% \textbf{Total (Test)}   & \textbf{1469}& \textbf{370}& \textbf{358}& \textbf{377}& \textbf{364}& \textbf{5.1} \\ \hline
% \end{tabular}
% \caption{Distribution of Dataset across Subtasks (IV: Implausible Valid, PV: Plausible Valid, II: Implausible Invalid, PI: Plausible Invalid)}
% \end{table}
 
% \subsection{Related Work}

\section{Related Work} 


%The interference of prior knowledge with logical validity, known as belief bias, is a well-documented phenomenon in human cognition \citep{evans1983conflict}. Prior work has shown that neural language models exhibit belief bias in syllogistic reasoning tasks, favoring semantically plausible conclusions over formally valid ones \citep{ando-etal-2023-evaluating}. While earlier studies were conducted on smaller-scale language models, similar tendencies have been observed in recent large language models\citep{dasgupta2024languagemodelshumanlikecontent, kim-etal-2025-reasoning}. A mechanistic interpretation of this failure reveals that specific reasoning circuits in models like Llama 3 and Qwen are often contaminated by attention heads dedicated to commonsense knowledge \citep{kim-etal-2025-reasoning}. To address this, our Translation-Based Inference via First-Order Logic (TBI-FOL) explicitly bypasses these contaminated internal circuits by offloading the logical deduction to the Otter theorem prover, ensuring that the validity check remains strictly formal and content-independent.

Belief bias, the cognitive tendency to favor semantic plausibility over logical validity, is a well-documented phenomenon in humans \citep{evans1983conflict} that persists in both early and contemporary large language models \citep{ando-etal-2023-evaluating, dasgupta2024languagemodelshumanlikecontent, kim-etal-2025-reasoning}. Mechanistic analyses of models like Llama 3 and Qwen reveal that internal reasoning circuits are often `contaminated' by attention heads dedicated to commonsense knowledge \citep{kim-etal-2025-reasoning}. To resolve this, our Translation-Based Inference via First-Order Logic (TBI-FOL) explicitly bypasses these biased circuits. By offloading logical deduction to the Otter theorem prover, we ensure that validity checks remain strictly formal and entirely independent of semantic content.

Recent research has explored various methods for modeling logical validity and syllogistic inference in natural language. \citet{poddar2025understandingsyllogisticreasoningllms} evaluated 14 different models and found that logical reasoning is not a uniform emergent property, with models often struggling with quantifier interpretation. Furthermore, \citet{hua-etal-2025-disentangling} observed that reasoning capabilities are often tied to specific linguistic domains rather than generalizable rules. To mitigate these effects, \citet{valentino2025mitigating} proposed fine-grained activation steering to decouple content from logic. While steering shows promise in internal model adjustments, \citet{zong-lin-2024-categorical} argue that the vast variety of syllogistic moods still poses a significant challenge for purely neural architectures. These observations are consistent with our finding that compact models like SmolLM reach peak performance only when utilized as specialized structural parsers within a symbolic framework.


\begin{figure*}[t]
    \centering
%    \input{chart4} 
\resizebox{.7\textwidth}{!}{
    \input{chart5}}
    \caption{Overview of the modular neuro-symbolic pipeline. 
    %The system supports multiple inference paths, transitioning from different input types (\textit{Raw}, \textit{Simplified}, or \textit{Translated}) to reasoning strategies: Direct Neural Inference (DNI), Translation-Based Inference via Aristotelian Triples (TBI-SYL), or First-Order Logic (TBI-FOL). 
    Arrows indicate the transition mechanism: purely LLM-based prompting (Blue Diamond) or two alternative transitions via either LLM or rule-based processing (Red and Blue Diamond). 
    %Symbolic reasoning is finalized via a Prolog Knowledge Base or the Otter Theorem Prover to generate the validity label and relevant premises.
    } 
    \label{fig:chart}
\end{figure*}


To achieve full disentanglement, neuro-symbolic pipelines have emerged as a robust alternative to purely neural methods. \citet{xu-etal-2024-faithful} introduced Symbolic Chain-of-Thought (SymbCoT), proving that translating context into symbolic formats significantly reduces hallucinations. This paradigm is supported by \citet{pan-etal-2023-logic} and the need for faithful reasoning grounded in formal rules \citep{xu-etal-2024-faithful}. Our contribution builds on these insights by integrating translations from relatively small LLMs with established symbolic solvers. In contrast to previous work, our pipeline leverages these compact models alongside Otter and Prolog to mitigate content bias. By strictly separating linguistic translation from formal inference, our system seeks to reduce the semantic interference identified in recent literature, providing a scalable solution for multilingual syllogistic reasoning.







\section{System Overview}
\label{sec:system_overview}

Our system implements a neuro-symbolic pipeline designed to decouple linguistic surface forms from formal logical reasoning. To address the challenge of content bias in syllogistic reasoning, we developed three distinct architectures for assessing argument validity (see Figure \ref{fig:chart}).

%Our neuro-symbolic pipeline decouples linguistic forms from formal reasoning through three distinct architectures for assessing argument validity (see1)


\paragraph{Direct Neural Inference (DNI):} A baseline approach where a Large Language Model (LLM) predicts binary validity directly from the natural language prompt without intermediate symbolic steps.

\paragraph{Translation-Based Inference via First-Order Logic (TBI-FOL):}  In this pipeline, natural language syllogisms are converted into First-Order Logic (FOL) formulas using two distinct translation paradigms: (1) \textit{Neural Translation (Neur-TBI-FOL)}, where an LLM is prompted to  generate FOL syntax, and (2) \textit{Rule-based Translation (Rule-TBI-FOL)}, where a deterministic mapping is performed via a specialized Python-based regex engine to extract logical structures from the text.

The actual inference is then performed by the automated theorem prover \textit{Otter}.\footnote{\url{https://www.cs.unm.edu/~mccune/otter/}} A syllogism is valid if Otter finds a contradiction between premises %$\mathcal{P} = \{p_1, \dots, p_n\}$ 
and the negated conclusion.
%$c$:
%    \begin{equation}
%        \mathcal{P} \cup \{\neg c\} \vdash \bot
%    \end{equation}

    For Subtasks 2 and 4, we perform proof analysis by mapping the clause indices used in Otter's refutation proof back to the original premises to identify the minimal set of relevant premises.    

\paragraph{Translation-Based Inference via Syllogistic Triples (TBI-SYL):} In this approach, the LLM maps the text to Aristotelian triples $(q, S, P)$, where $q \in \{a, e, i, o\}$ represents the categorical proposition type (e.g., \textit{``All S are P''} $\rightarrow$ \texttt{('a', S, P)}). Similar to TBI-FOL, this uses %this mapping is executed either through
neural generation or rule-based extraction. Formal validity is then determined by a Prolog-based Knowledge Base encoding the 256 classical syllogistic moods.

\smallskip
The complete modular architecture and the various experimental paths, i.e.\ from input selection to final inference, are visualized in Figure~\ref{fig:chart}. This design allows us to compare a wide spectrum of reasoning strategies anchored by two fundamental baselines: (I) A \textbf{purely neural path (DNI)}, where the model performs the entire task end-to-end, relying solely on its internal representations; and (II) a \textbf{purely symbolic path}, where our deterministic rule-based translation engine is applied directly to the raw text, bypassing LLM-mediated reasoning entirely.  Figure~\ref{fig:example} provides an illustrative example of the concrete outputs generated at each translation step.



\begin{figure}[t]
    \centering
    \small
    \input{exampleFig} % Hier liegt das TikZ/Tabelle für das Beispiel
    \caption{Illustrative example of inputs and outputs generated at each translation step.}
    \label{fig:example}
\end{figure}


The core motivation for employing external engines like \textit{Otter} and the \textit{Prolog KB} was the goal of ensuring that the final validity decision is strictly a function of the extracted logical structure. By offloading the inference to these formal solvers, we aimed to render the reasoning process immune to the semantic plausibility of the premises, in an effort to reduce the impact of content bias that often affects direct LLM predictions.

To further mitigate the impact of complex natural language, we implemented an optional \textit{Linguistic Simplification} step. Here, an LLM simplifies raw text into standardized propositions with normalized predicates (e.g., \textit{``Triangles are shapes with three sides''} $\rightarrow$ \texttt{all Triangle is Shape\_With\_Three\_Sides}). The rationale behind this was to decouple the identification of conceptual predicates from the construction of a clean logical formula, thereby reducing the cognitive load on the subsequent translator.

By employing the three distinct target representations (FOL, syllogistic triples, and simplified English) exemplified in Figure~\ref{fig:example}, we aimed to disentangle logical validity from semantic plausibility by varying how strongly surface linguistic cues are preserved. We expected that the strict formal syntax of FOL would minimize reliance on plausibility heuristics and push the model toward structurally valid reasoning, albeit at the cost of higher susceptibility to syntactic errors that cannot always be repaired by postprocessing. Syllogistic triples $(q, S, P)$ were intended to strike a middle ground: by breaking the natural ordering of quantifier, subject, and predicate, they partially disrupt surface-level plausibility and encourage abstraction to logical roles. As syllogistic triples rarely occur in the training data of LLMs, this representation was expected to further force the model to focus on structural regularities. In contrast, simplified English remained close to natural language and the LLM’s training distribution, preserving fluency and plausibility while only lightly constraining structure. Through this spectrum, we aimed to test whether increasing formal rigidity helps isolate logical reasoning from linguistic bias. As shown in Section~\ref{sec:results}, the FOL-based approach performed best overall, although the differences remained moderate.


To address the multilingual challenges (\textbf{Subtasks 3 and 4}), we compared two distinct strategies: (I) a direct application of the pipelines developed for Subtasks 1 and 2 to the native multilingual inputs, and (II) a two-stage approach where the syllogisms were first translated into English using Google Translator via \texttt{pypi deep-translator}\footnote{pypi deep-translator: \url{https://pypi.org/project/deep-translator/}} before applying our reasoning pipelines.
 %For multilingual subtasks (3 and 4), we compared a \textit{Translate-test} approach (machine translation into English) against \textit{Native-test} (direct processing in the original language). This evaluates whether the symbolic translation layer can effectively serve as a language-agnostic bridge for formal reasoning.

\section{Experimental Setup}
\label{sec:experimental_setup}

To evaluate the robustness of our neuro-symbolic pipelines against content bias, we conducted extensive experiments across different tasks, model sizes, and prompting configurations. 

\subsection{Prompting Strategies}
The neuro-symbolic pipeline (\textit{TBI-FOL} and \textit{TBI-SYL}) faces two primary challenges: ensuring \textit{terminological consistency} (mapping semantically equivalent concepts to identical predicates) and adhering to the \textit{restrictive syntax} of the Otter prover and Prolog KB. 

To address these, we developed specialized prompt categories for logic translation, syllogistic mapping, and linguistic simplification. These range from minimalistic, example-driven prompts to complex instructions with multilingual trigger word mappings for Subtasks 3 and 4. A baseline \textit{Direct Neural Inference} (DNI) prompt was also used for direct one-step binary validity judgments. 

The prompts are supported by curated few-shot example sets, which are balanced across belief-bias conditions (validity vs. plausibility) to guide the models through the logical transformations. Detailed descriptions of all prompt variants and the composition of the few-shot sets are provided in Appendix \ref{appendix:prompts}.



\subsection{Models and Parameters}
\label{sec:models_parameters}

To investigate scaling effects and reasoning capabilities within the category of compact Large Language Models (LLMs), we evaluated our pipeline across a diverse selection of models ranging from 135M to 4B parameters. The selection includes the \textbf{SmolLM} series, chosen as a fully open model family (providing transparency regarding training data mixtures and recipes) while being specifically optimized for reasoning tasks at a small scale, the \textbf{Qwen} family, which was selected for its established strength in formal logic and its native multilingual capabilities (crucial for Subtasks 3 and 4). Additionally, \textbf{Llama} and \textbf{Gemma} models serve as industry-standard benchmarks for instruction-following and state-of-the-art reasoning in the 2B to 4B parameter range.

To ensure deterministic and reproducible outputs, we primarily employed greedy decoding (temperature $0.0$), with minor model-specific adjustments to ensure generation stability where required. Detailed model identifiers and the complete hyperparameter configuration are provided in Appendix \ref{appendix:model_details}.

%To investigate scaling effects, we evaluated our pipeline across a diverse selection of compact LLMs (135M to 4B parameters). The selection includes the SmolLM series (open-weight, reasoning-optimized), the Qwen family (strong formal logic and native multilingual capabilities), and Llama and Gemma as industry benchmarks for instruction-following. To ensure reproducibility, we primarily employed greedy decoding (temp=0.0), with minor adjustments for stability in Qwen models. Detailed model identifiers and hyperparameter configurations are provided in Appendix \ref{appendix:model_details}.


\subsection{Data Processing and Symbolic Integration}
\label{sec:processing_translation}

To bridge the gap between natural language and formal logic, we implemented a processing pipeline that handles output extraction, deterministic rule-based translation, and the interface to symbolic solvers. Detailed extraction steps and prover configurations are provided in Appendix \ref{appendix:implementation}.

\paragraph{Normalization and Rule-based Translation}
LLM output undergoes aggressive artifact stripping and term normalization (e.g. article removal and singularization), to ensure terminological consistency across premises. For the non-neural path, a deterministic engine maps propositions to FOL via copula identification and hierarchical regex patterns. By offloading the formalization, we guarantee valid FOL syntax, effectively mitigating the risk of malformed outputs often produced by LLMs. This allows the system to remain robust whether the LLM is used only for linguistic simplification or is bypassed entirely.

\paragraph{Inference and Existential Import}
To align FOL with the existential import of classical syllogisms, the pipeline automatically injects existence conditions ($\exists x P(x)$) for all detected predicates. Formulas are further standardized to meet the strict syntactic requirements of the \textit{Otter} theorem prover, including alpha conversion, negation normalization and quantifier scoping. For Subtasks 2 and 4, we utilize the resolution proof history from the prover to identify the minimal sufficient set of premises.


\paragraph{Syllogistic Path (TBI-SYL)}
The triple-based pipeline maps propositions to $(q, S, P)$ structures, which are validated against a Prolog knowledge base containing the 24 valid Aristotelian moods. Unlike the FOL path, this solver accounts for existential import through its internal encoding.


\subsection{Evaluation Metrics}
\label{sec:metrics}

We evaluate our systems using the official metrics defined for the SemEval-2026 Shared Task 11 \cite{valentino-etal-2026-semeval}. The evaluation focuses on three dimensions: (1) \textbf{Overall Accuracy (Acc)} for validity prediction in Subtask 1, (2) the \textbf{Macro-Averaged F1-Score ($F1_{prem}$)} for premise identification in Subtask 2, and (3) the \textbf{Total Content Effect (TCE)}, which quantifies belief bias by measuring performance disparities both within the same validity label (plausible vs. implausible cases) and across validity labels. To determine the final ranking, all subtasks compute a \textbf{Combined Score} that balances performance with robustness to content bias. In classification settings (Subtasks 1 and 3), accuracy is penalized based on the observed Total Content Effect (TCE). In retrieval-augmented settings (Subtasks 2 and 4), the same penalty is applied to the average of accuracy and premise selection performance ($F1_{prem}$). This ensures that models are rewarded not only for correctness but also for consistency across plausible and implausible cases.

% A detailed mathematical derivation of all metrics is provided in Appendix \ref{app:metrics}.



\section{Results and Discussion}
\label{sec:results}


This section evaluates the performance of our modular architecture across all shared task challenges on the subtasks' test sets provided by the organizers of the shared task. Due to its role as the logical foundation for all subsequent reasoning steps, we first provide an extensive analysis of Subtask 1, followed by an evaluation of the system's performance in premise selection and multilingual settings. For a comprehensive overview of all tested experimental configurations see Appendix \ref{appendix:results}.

\subsection{Subtask 1: Logical Validity}
\label{sec:results_st1}

The primary objective of Subtask 1 is to establish a robust backbone for validity prediction while minimizing belief bias. We conducted 31 experimental configurations comparing architectures, model families, scaling effects, and prompting strategies. Our analysis identifies \textbf{Eng2FOL5} as the superior configuration, utilizing the \textit{SmolLM3-3B} model with an \textit{Ultrashort Prompt} (320 tokens) and eight varied examples. It serves as the benchmark for all subsequent comparisons.

In Table~\ref{tab:all_results_subtask1} in the appendix, we report the results for all 31 experimental configurations on Subtask 1. Beyond the aggregated metrics discussed in the main text, the table also includes standard classification outcomes (FP, FN, Errors), as well as a fine-grained breakdown across the four plausibility / validity conditions ($Acc_{PV}$, $Acc_{IV}$, $Acc_{PI}$, $Acc_{II}$), enabling a detailed analysis of belief bias effects. It further provides full configuration details of the experimental setup, including model choice, prompt design, input format, and properties of the few-shot examples.

\paragraph{Architecture Comparison and Baselines}

We compare purely neural (DNI) and symbolic (Rule) baselines against various hybrid \textit{Translation-Based Inference} (TBI) approaches. These hybrids differ in their translation mechanism (LLM vs. Rule), the input format (Raw vs. Simplified), and the logical target (FOL vs. SYL). All experiments reported in Table~\ref{tab:architecture_comp} were conducted using the \textit{SmolLM3-3B} model as the primary engine for both direct inference and translation tasks.


\begin{table}[ht]
\centering
\small
\setlength{\tabcolsep}{4pt}
\begin{tabular}{@{}p{4.3cm}ccc@{}}
\toprule
\textbf{Architecture} & \textbf{Acc}  & \textbf{TCE}  & \textbf{Score}  \\
\midrule
\textit{Baselines} \\
DNI (Direct Neural Inference)       &       56.54 &           24.51 &           13.34 \\

Rule (Rule-TBI-FOL/Syl)        &     49.74 &          -- &           -- \\
\midrule
\textit{Hybrid TBI} \\
\textit{from raw syllogism} \\
LLM-TBI-FOL (Eng2FOL) &   91.62 &            4.17 &           34.68 \\
\textbf{LLM-TBI-FOL  (Eng2FOL5)} & \textbf{95.29} & \textbf{2.15} & \textbf{44.37} \\
LLM-TBI-Syl \newline (Eng2SyllogismProlog) &     72.25 &           18.66 &           18.16 
\\
\midrule
\textit{from simplified syllogism} \\

Simp(1)-Rule-TBI-FOL  &     86.39 &           14.58 &           23.06 \\
Simp(1)-Rule-TBI-Syl  & 85.34 & 18.75 & 21.43 \\
Simp-LLM-TBI-FOL \newline (SimpEng2FOL1)        &     85.86 &           12.50 &           23.83 \\
Simp-LLM-TBI-Syl \newline (SimpEng2SyllogismProlog) & 76.96 & 14.49 & 20.58 \\
\bottomrule
\end{tabular}
\caption{Architecture Comparison: Performance of Neural, Symbolic, and Hybrid Paradigms using \textit{SmolLM3-3B}.}
\label{tab:architecture_comp}
\end{table}

Both baseline approaches performed poorly. The symbolic baseline (\textit{Rule}) achieved an accuracy of only $\approx 50\%$ and failed to find a single valid proof. This stems from the fact that the rule-based grammar was tailored for simplified English; when faced with raw natural language, the extraction of syntactically well-formed formulas fails in 121 out of 191 cases. Similarly, the \textit{Direct Neural Inference} (DNI) baseline, despite a strict prompt emphasizing logical validity over plausibility, achieved only 56\% accuracy, with 25 cases being excluded as errors due to non-compliant output formats. The detailed accuracy breakdown in Table~\ref{tab:all_results_subtask1} further reveals a pronounced plausibility bias in DNI: when distinguishing between plausibility (P vs. I) and validity (V vs. I) conditions, performance drops from $Acc_{PV}=79.17$ to $Acc_{IV}=41.67$, while a similar distortion is observed in the invalid cases ($Acc_{II}=58.33$ vs. $Acc_{PI}=46.81$).

These results highlight that both purely symbolic and purely neural baselines are strongly affected either by syntactic fragility or by content-driven bias. This suggests that separating linguistic normalization from logical inference may be crucial for robust reasoning. Consistent with this observation, the efficacy of the rule-based system increases dramatically when integrated into a hybrid pipeline. When applied to syllogisms previously simplified by \textit{SmolLM3-3B}, the rule-based translation (\textit{Simp-Rule-TBI}) reaches accuracies of $\approx 86\%$ for both FOL and SYL (Aristotelian Triples) targets.

When delegating the translation entirely to the LLM, a significant performance gap emerges between logical targets. \textit{LLM-TBI-FOL} consistently outperforms \textit{LLM-TBI-Syl}. This advantage likely results from the higher prevalence of FOL-like structures in the LLM's training data. Furthermore, FOL offers higher flexibility, as multiple logically equivalent translations (e.g., varying term orders or conjunctions) can be correctly resolved by the FOL theorem prover, whereas the triple-based SYL format requires an exact match of subject, predicate, and mood. 
This finding contradicts our initial assumption that disrupting the natural ordering of quantifier, subject, and predicate in syllogistic triples and introducing a less familiar format would help the model abstract away from plausibility biases. Instead, the results suggest that this additional abstraction burden may hinder reliable generation rather than improve logical focus.
Due to this robustness and the superior traceability of premises in FOL proofs (crucial for Subtask 2) we focused our subsequent optimization on FOL-based translation.

Against our intuition, the use of a two-stage LLM pipeline (1st simplification, 2nd translation into FOL) did not yield improvements. Comparing \textit{Eng2FOL} (raw) and \textit{SimpEng2Fol1} (simplified) using identical prompts, we observe an accuracy drop and a sharp increase in the \textit{Total Content Effect} (TCE) from 4.17 to 12.5. This suggests that simplification introduces more noise than it removes.  Crucially, the 3B-parameter models already produced surprisingly clean FOL syntax from raw text, making an intermediate stage redundant. The increase in TCE is primarily driven by a substantial drop in performance on plausible-valid cases ($Acc_{PV}$), which decreases from 89.58 to 70.83, as shown in Table~\ref{tab:all_results_subtask1}. Consequently, our best system, \textit{Eng2FOL5}, builds directly on the single-stage-translation \textit{Eng2FOL} approach using optimized prompts and few-shot examples.

\paragraph{Impact of Model Selection and Prompting Strategies}

Our evaluation across various LLM families and prompting configurations reveals that while smaller models like \textit{SmolLM3-3B} and \textit{Qwen3-4B} can achieve high accuracy ($>95\%$), the results remain notably unstable. Performance metrics  fluctuate significantly depending on the specific few-shot example selection and the alignment between prompt length and model architecture. We observe that no single prompting strategy yields a uniform advantage. Furthermore, a model's peak performance on one subtask or dataset does not consistently translate to others, highlighting a sensitivity to input distributions that complicates the search for a universal "winning" configuration. A comprehensive breakdown of model scaling effects, prompt variations, and full performance tables is provided in Appendix \ref{appendix:results}.



\subsection{Subtasks 2--4: Premise Identification and Multilingual Robustness}
\label{sec:results_st24}


For \textbf{Subtask 2}, premise identification is natively handled by our TBI approach, as the FOL theorem prover (Otter) generates a resolution proof that explicitly lists all utilized premises. The primary challenge lies in the consistent handling of existence clauses introduced during preprocessing. 

The best results for Subtask 2 were achieved by \textit{Eng2FOL5Qwen4B}, using the \textit{Qwen3-4B-Instruct} model with the same few-shot configuration as our winning Subtask 1 setup (Acc: 95.79\%, $F1_{prem}$: 92.37\%, TCE: 3.26, Score": 38.43). This performance underscores the high sensitivity of smaller LLMs to specific input distributions: while \textit{SmolLM3-3B} dominated Subtask 1, it only reached 86.98\% accuracy ($F1_{prem}$: 78.38\%) on the Subtask 2 test set using the identical configuration (\textit{Eng2FOL5}).

For the multilingual challenges (\textbf{Subtasks 3 and 4}), we systematically compared two strategies: (I) direct multilingual processing and (II) automatic translation into English prior to inference.

In \textbf{Subtask 3}, strategy (I) performed slightly better:  configuration \textit{QwenEng2FOL\-multilingual\-NewShort} reached 88.02\% accuracy, a TCE of 6.38, and an overall score of 29.36. By contrast, strategy (II), using the same underlying \textit{Qwen3-4B-Instruct} model (\textit{QwenEng2FOL6}), delivered comparable performance, with a higher raw accuracy of 90.63\% but also an increased TCE of 7.33. A closer examination at the level of individual languages reveals substantial variation in performance: for instance, accuracy ranges from 100\% for Dutch to 52\% for Swahili in the best-performing configuration. Importantly, these disparities shift depending on the LLM and strategy, indicating substantial model-dependent variation. See Appendix~\ref{appendix:multilingual} for details.


In \textbf{Subtask 4}, strategy (II), i.e.\ translation into English combined with the best-performing configuration from Subtask 1 (\textit{Eng2FOL5}), achieved the strongest results (Acc: 77.08\%, $F1_{prem}$: 64.15\%, TCE: 6.07, Score: 23.89). In contrast, direct multilingual processing performed markedly worse in this setting: the corresponding \textit{Qwen} configuration exhibited a substantially higher content effect (TCE: 14.71). Notably, in 20 out of 192 instances, the LLM failed to generate outputs that could be processed by the theorem prover, indicating robustness limitations in this approach.



\section{Conclusion}
We presented a modular hybrid neuro-symbolic architecture for reasoning disentanglement. By contrasting pure neural and pure symbolic approaches with hybrid ones, we demonstrated that relatively small specialized LLMs like \textit{SmolLM3-3B} can achieve high logical fidelity when the LLM's linguistic flexibility is anchored by the formal rigor of a symbolic solver.


Our analysis reveals that performance remains highly sensitive to input distributions; small variations in few-shot examples or prompt length can lead to significant fluctuations in logical stability. For the multilingual challenges, the results highlight a trade-off between native processing and machine translation. While automated translation provides a robust baseline, it introduces semantic noise that can skew the \textit{Total Content Effect} (TCE). 

Future work should address identified limitations, particularly the alignment between prompt instructions and few-shot examples. Refined rule-based post-processing and more rigorous prompt optimization could further stabilize the translation layer, ensuring that hybrid neuro-symbolic pipelines remain robust across diverse linguistic and logical contexts.


\section*{Acknowledgements}

This work was developed within the framework of a seminar taught by Wiebke Petersen at Heinrich Heine University Düsseldorf during the winter term 2025/2026, in which student groups independently conducted computational linguistics projects. We would like to thank the participants of the other project groups for their valuable feedback and constructive discussions throughout the course, which helped to refine and sharpen our ideas.

Additionally, we would like to thank the anonymous reviewers for their insightful comments, which helped improve the clarity and quality of this paper. We also gratefully acknowledge the organizers of the shared task for their considerable effort, responsiveness, and flexibility in supporting participants.


%This work was developed within the framework of a course \textit{Applying Computational Linguistics Methods in a Research Project} at the Heinrich Heine University Duesseldorf during the winter semester 2025/2026. We would like to  thank our seminar group for the collaborative environment and the technical tips which greatly contributed to the refinement of this work.


%Additionally, we would like to thank the anonymous reviewers for their valuable comments, which helped improve this work. We also extend our gratitude to the organizers of the shared task for their tremendous effort, their prompt responsiveness, and their flexibility in accommodating the participants needs.
%oder
%We are grateful to the anonymous reviewers for their constructive suggestions and to the shared task organizers for their tireless work and commitment to making this competition possible."

\bibliographystyle{acl_natbib}
\bibliography{anthology,custom}

\appendix


\section{Experimental setup details}

\subsection{Detailed Prompting Configurations}
\label{appendix:prompts}

This section provides the technical specifications of the prompts and few-shot examples used in our experiments. 

\subsubsection{Prompts}
All full prompt templates are available in our repository: \url{https://github.com/WiebkePetersen/HHU-SyLo-SemEval2026-Task11/tree/main/prompts}.

\begin{itemize} 
    \item \textbf{Logic Translation (TBI-FOL):} We compared three prompt variants: \texttt{Eng2FOL} (focusing on explicit syntactic rules, 1721 chars), \texttt{Eng2FOLimproved} (emphasizing semantic decoupling, abstract treatment of symbols, and consistent PascalCase naming, 1634 chars), and \texttt{Eng2FOLultrashort} (a minimalistic, example-driven approach, 320 chars).

    \textbf{Multilingual TBI-FOL:} Two dedicated prompts, \texttt{MultiLingFOL} ( 3033 chars) and \texttt{MultiLingEng2FOLultrashort} ( 502 chars), handled the multilingual scenarios (Subtasks 3 and 4). \texttt{MultiLingFOL} includes a step-by-step reasoning instruction (in English) and a multilingual trigger word mapping for 11 languages to ensure logical fidelity during the \textit{Native-test} condition.
    
    \item \textbf{Syllogistic Mapping (TBI-SYL):} The \texttt{SyllTrip} prompt (1254 chars) maps natural language to Aristotelian triples $(q, S, P)$, enforcing naming consistency and mapping quantifiers to categorical proposition types (\textit{a, e, i, o}).

    \item \textbf{Linguistic Simplification:} Two prompts, \texttt{LogSimp} ( 1572 chars) and \texttt{LogSimpShortPrompt} ( 1016 chars), were employed to evaluate the impact of text normalization. These prompts instruct the LLM to extract a standardized "Simplified Logic English" (e.g., \texttt{all [Subject] is [Predicate]}, see Figure \ref{fig:example}), enforcing singular forms  capitalization, atomization for multi-word terms, and the elimination of negative atoms (e.g., converting "is not a fish" to a "no" quantifier and not to "not\_fish").
\end{itemize}

\subsubsection*{Few-Shot Example Sets}
Table~\ref{tab:few_shot_configs} details the composition of the curated example sets used to guide the models. It provides details regarding the quantity, average length, and the distribution of validity and plausibility across the example sets.

\begin{table}[h]
\centering
\footnotesize
\begin{tabular}{@{}l|c|cccc|c@{}}
\hline
\textbf{Example Set ID} & \textbf{\#} & \textbf{IV} & \textbf{PV} & \textbf{II} & \textbf{PI} & \textbf{$\varnothing$} \\ \hline
LogSimpEx               & 5           & 1           & 4           & 0           & 0           & 3.4 \\ \hline
SyllTripEx              & 4           & 1           & 3           & 0           & 0           & 3.8 \\
SimpSyllTripEx          & 4           & 1           & 3           & 0           & 0           & 3.8 \\ \hline
Eng2FOLex               & 3           & 0           & 0           & 1           & 2           & 3.0 \\
Eng2FOLexImproved       & 5           & 0           & 0           & 2           & 3           & 3.2 \\
Eng2FOLexImprovedX      & 8           & 0           & 3           & 2           & 3           & 3.1 \\
Eng2FOLexImproved2      & 8           & 2           & 2           & 2           & 2           & 3.6 \\
Eng2FOLexVaried         & 8           & 0           & 5           & 1           & 2           & 3.0 \\
Eng2FOLexLong           & 4           & 0           & 2           & 2           & 0           & 7.5 \\ \hline
MultiLingexImproved     & 22          & 2           & 9           & 1           & 10          & 3.0 \\
MultiLingexShort        & 6           & 1           & 2           & 0           & 3           & 3.0 \\ \hline
\end{tabular}

\caption{Composition of few-shot example sets. \textbf{\#} denotes the total number of examples; \textbf{$\varnothing$} represents the average number of logical statements (premises and conclusion) per syllogism. Categories reflect belief-bias conditions: Implausible/Valid (IV), Plausible/Valid (PV), Implausible/Invalid (II), and Plausible/Invalid (PI).}\label{tab:few_shot_configs}
\end{table}


\subsection{Model Specifications and Hyperparameters}
\label{appendix:model_details}

The following specific LLM checkpoints were utilized in our experiments:

\begin{itemize}
    \item \textbf{SmolLM-Series:} \texttt{SmolLM2-135M-Instruct}, \texttt{SmolLM2-1.7B-Instruct}, and \texttt{SmolLM3-3B}.
    \item \textbf{Qwen-Series:} \texttt{Qwen3-0.6B}, \texttt{Qwen3-1.7B}, and \texttt{Qwen3-4B-Instruct-2507}.
    \item \textbf{Gemma-Series:} \texttt{google/gemma-2-2b-it} and \texttt{google/gemma-3-4b-it}.
    \item \textbf{Llama-Series:} \texttt{meta-llama/Llama-3.2-3B-Instruct}.
\end{itemize}

To ensure reproducibility, we set the temperature to $0.0$ for the majority of models. For the \textit{Qwen} models, a specific configuration was used to ensure generation stability: \texttt{do\_sample=True}, \texttt{temperature=0.1}, \texttt{top\_p=0.9}, and a \texttt{repetition\_penalty} of $1.1$. Inference was performed with batch sizes between 8 and 16, depending on the respective model size.\footnote{A comprehensive list of all hyperparameters and model-specific settings is provided in the \texttt{experiments.yaml} configuration file in our repository.}



\subsection{Implementation Details: Processing and Translation}
\label{appendix:implementation}

\subsubsection*{Extraction and Normalization}
Since LLMs often embed formal outputs within conversational text, we employ a multi-step extraction layer involving recursive JSON and literal parsing to isolate logical lists. Artifact stripping (e.g., removing trailing explanations) is applied to ensure clean inputs for the solvers. During normalization, phrases like \textit{"the birds"} and \textit{"a bird"} are both mapped to the unified predicate \texttt{Bird}. All terms are capitalized, singularized, and atomized.

\subsubsection*{Rule-based Translation Engine}
The deterministic engine first attempts a heuristic split based on copula identification (\textit{"is"}, \textit{"are not"}). It specifically handles complex linguistic cases like double negations (e.g., \textit{"no X is not Y"}). If a direct split fails, a hierarchical regex safety net identifies the categorical proposition type.

\subsubsection*{Symbolic Standardization}
Before inference, formulas undergo refinement for \textit{Otter} compatibility:
\begin{itemize}
    \item \textbf{Alpha-conversion:} Ensures unique variable naming across the proof.
    \item \textbf{Syntactic Cleaning:} Parentheses balancing and negation standardization.
    \item \textbf{Syllogistic Existential Closure Assumption:} Injection of existence conditions ($\exists x P(x)$) for all detected predicates.
    \item \textbf{Negated Conclusion:} The conclusion is identified and provided to the prover in its negated form to facilitate a proof by contradiction.
\end{itemize}
For successful refutations, Otter's proof history is parsed to map used clauses back to their original premises.



\section{Detailed Results: Models and Prompting}
\label{appendix:results}


Table \ref{tab:all_results_subtask1} shows the results of all our experiments run on the test data of Subtask 1.

\subsection{Model Scaling and Family Performance}
To evaluate the impact of model capacity under controlled conditions, we compared various LLMs using the prompting configuration (\textit{Ultrashort Prompt}, 8 examples) from our winning experiment (Eng2FOL5). As shown in Table \ref{tab:models}, \textit{SmolLM3-3B} achieves the highest combined score, marginally outperforming the \textit{Qwen} models due to its superior bias mitigation (lowest TCE). It also significantly outperforms both \textit{Llama} and \textit{Gemma} in this task. While we observe a general scaling trend where larger models yield better results, the \textit{Qwen3-1.7B} model represents a notable exception, demonstrating remarkably high accuracy for its size. Notably, \textit{Qwen3-4B} achieves perfect accuracy (100\%) in both implausible conditions (valid and invalid), indicating a particularly strong ability to focus on logical structure when surface plausibility cues are misleading, as detailed in Table~\ref{tab:all_results_subtask1}.


\begin{table}[ht]
\centering
%\scriptsize
\small
\setlength{\tabcolsep}{2pt}
\begin{tabular}{@{}lccc@{}}
\toprule
\textbf{Model (Experiment)} & \textbf{Comb} & \textbf{Acc}& \textbf{TCE} \\
\midrule
SmolLM3-3B (Eng2FOL5) & \textbf{44.37} & 95.29 & \textbf{2.15} \\
Qwen3-4B-Inst. (Eng2FOL5Qwen4B) & 36.23 & \textbf{96.34} & 4.26 \\
Qwen3-1.7B (Eng2FOL5Qwen1.7B) & 36.03 & 95.81 & 4.26 \\
Llama-3.2-3B-Inst. (Eng2FOL3) & 24.89 & 85.34 & 10.35 \\
Gemma-2-2B-it (Eng2FOL5Gemma2B) & 20.08 & 73.82 & 13.54 \\
SmolLM2-1.7B-Inst. (Eng2FOL5small) & 17.61 & 65.97 & 14.58 \\
Qwen3-0.6B (Eng2FOL5Qwen0.6B) & 17.53 & 65.45 & 14.38 \\
SmolLM2-135M-Inst. (Eng2FOL5tiny) & 11.70 & 50.26 & 26.04 \\
\bottomrule
\end{tabular}
\caption{Impact of model size and family on TBI-FOL performance using a fixed configuration (\texttt{Eng2FOLUltrashort} prompt,  \texttt{Eng2FOLexImprovedX} examples; details in Appendix \ref{appendix:prompts}).}
\label{tab:models}
\end{table}


While Table \ref{tab:models} compares LLMs under identical conditions, Table \ref{tab:best_results} reports the peak performance for each LLM across all tested prompt configurations. While most architectures benefit from the \textit{Ultrashort} setup, individual tuning can further reduce the TCE for specific families like \textit{Qwen} (e.g., \textit{QwenEng2FOL1}).

\begin{table}[ht]
\centering
\scriptsize
\setlength{\tabcolsep}{5pt}
\begin{tabular}{@{}lccc@{}}
\toprule
\textbf{Model (Best Experiment)} & \textbf{Comb}  & \textbf{Acc} & \textbf{TCE} \\
\midrule
SmolLM3-3B (Eng2FOL5) & \textbf{44.37} & 95.29 & \textbf{2.15} \\
Qwen3-4B-Instruct-2507 (QwenEng2FOL1) & 38.44 & 93.72 & 3.21 \\
Qwen3-1.7B (Eng2FOL5Qwen1.7B) & 36.03 & \textbf{95.81} & 4.26 \\
Llama-3.2-3B-Instruct (Eng2FOL3) & 24.89 & 85.34 & 10.35 \\
gemma-2-2b-it (Eng2FOL5Gemma2B) & 20.08 & 73.82 & 13.54 \\
SmolLM2-1.7B-Instruct (Eng2FOL5small) & 17.61 & 65.97 & 14.58 \\
Qwen3-0.6B (Eng2FOL5Qwen0.6B) & 17.53 & 65.45 & 14.38 \\
SmolLM2-135M-Instruct (Eng2FOL5tiny) & 11.70 & 50.26 & 26.04 \\
gemma-3-4b-it (Eng2FOL11) & 10.41 & 43.46 & 22.92 \\
\bottomrule
\end{tabular}
\caption{Peak performance per model family across all experimental configurations. Each entry represents the best-performing individual setup for that specific architecture.}
\label{tab:best_results}
\end{table}

\subsection{Prompting Strategies and Few-Shot Effects}

The transition between prompt strategies yields no uniform advantage across all architectures. For \textit{SmolLM3-3B}, we observe a slight preference for shorter prompts (see Appendix \ref{appendix:prompts} for prompt details): comparing \texttt{Eng2FOLUltrashort} (Eng2FOL5) to the longer \texttt{Eng2FOL} (Eng2FOL7), accuracy marginally increases from 95.29\% to 96.34\%, yet this is offset by the \textit{Total Content Effect} (TCE) doubling from 2.15 to 4.26 (see Table \ref{tab:all_results_subtask1} for full details). Conversely, \textit{Qwen} models exhibit an opposite tendency, showing improved robustness with longer, more descriptive prompts (e.g., \texttt{QwenEng2FOL4} vs. \texttt{QwenEng2FOL2}).

Variations in the few-shot example sets similarly provide an ambiguous picture (see Table \ref{tab:few_shot_configs} for overview of used few-shot example sets). Analysis of experiments \texttt{Eng2FOL5}, \texttt{6}, \texttt{8}, and \texttt{9}—which differ solely in example selection—indicates a subtle trend where a higher quantity of examples improves performance. However, explicitly balancing the plausibility-validity distribution within the examples did not yield the expected gains in bias mitigation.


\subsection{Detailed Multilingual Performance Results}



 \label{appendix:multilingual}

The evaluation of the multilingual subtasks revealed a significant variance in performance across different languages and model architectures. While Western European languages like Dutch (nl) consistently achieved high accuracy (up to 100\%), results for other languages were markedly inconsistent. Swahili ($sw$), for instance, proved highly sensitive to model choice, with accuracy fluctuating between 52.94\% (best Qwen-configuration) and a mere 23.53\% (best SmolLM-configuration). 

Table \ref{tab:languages} provides a detailed breakdown of the 11 languages, comparing the best-performing configurations for Qwen-4B, SmolLM-3B, and the English-translation pipeline. A notable discrepancy occurs in Chinese ($zh$-CN): the translation strategy dropped to 77.78\% accuracy, whereas direct multilingual processing via SmolLM and Qwen achieved 94.44\%. These results suggest divergent training data distributions; while SmolLM remains robust for European datasets, its performance drops sharply for languages like Swahili or Telugu.


\begin{table}[htb]

\centering

\small

%\setlength{\tabcolsep}{3pt}

%\begin{threeparttable}

\begin{tabular}{@{}l ccc @{}}

\toprule

%\textbf{Language} & \textbf{QwenEng2FOLmultilingualNewShort }& \textbf{QwenEng2FOL6}& \textbf{Eng2FOL5multilingual}\\

\textbf{Language} & \textbf{Qwen}& \textbf{transl.}& \textbf{SmolLM}\\


\midrule

sw (Swahili)    & 52.94 & 88.24 & 23.53 \\

pt (Portuguese) & 94.12 & 94.12 & 88.24 \\

zh-CN (Chinese) & 94.44 & 77.78 & 94.44 \\

fr (French)     & 94.44 & 100.0 & 100.0 \\

nl (Dutch)      & 100.0 & 100.0 & 94.12 \\

bn (Bengali)    & 76.47 & 76.47 & 64.71 \\

it (Italian)    & 83.33 & 94.44 & 94.44 \\

es (Spanish)    & 88.89 & 88.89 & 100.0 \\

te (Telugu)     & 88.24 & 94.12 & 47.06 \\

de (German)     & 100.0 & 94.44 & 72.22 \\

ru (Russian)    & 82.35 & 88.24 & 64.71 \\

\midrule

\textbf{Total Acc.} & \textbf{88.02} & \textbf{90.63} & \textbf{79.69} \\

\textbf{Errors}     & \textbf{4}       & \textbf{0}       & \textbf{9}       \\

\bottomrule

\end{tabular}

\caption{Accuracies for the best-performing configurations across multilingual and translation-based strategies, broken down by 11 languages: direct multilingual processing with Qwen-4B (\textit{QwenEng2FOLmultilingualNewShort}) and SmolLM-3B (\textit{Eng2FOL5multilingual}), as well as the English-translation pipeline (\textit{QwenEng2FOL6}) using Google Translate.}

\label{tab:languages}

%\end{threeparttable}

\end{table}


\begin{sidewaystable*}[p] % [p] platziert sie auf einer eigenen Seite
\setlength{\tabcolsep}{2pt}
    \centering
    \footnotesize
    \vspace*{\fill} % Zentriert die Tabelle vertikal auf der gedrehten Seite
    \input{results_subtask1}
%    \caption{Detailed Performance Metrics per Model and Configuration on test dataset for Subtask 1.}
\caption{Detailed performance metrics per model and configuration on the test set for Subtask 1. Reported columns include standard classification outcomes (FP, FN, Errors), overall performance (Acc), bias sensitivity (TCE), and the official Combined Score. Subgroup accuracies are shown for the four plausibility/validity conditions ($Acc_{PV}$, $Acc_{IV}$, $Acc_{PI}$, $Acc_{II}$). The remaining columns describe the experimental setup, including model, prompt configuration, input type, and properties of the few-shot examples used (ExID, number of examples, and average example length).}
\label{tab:all_results_subtask1}
    \vspace*{\fill}
\end{sidewaystable*}



\end{document}


\begin{itemize}
    \item hier kannst du alles hinschreiben
\end{itemize}








Prompts:





It is obvious that not all dogs lay eggs.

some Dog is not LayEgg 

exists x ( Dog(x) & -LayEgg(x) )

(o, Dog, LayEgg)