\documentclass[11pt]{article}

% Change "review" to "final" to generate the final (sometimes called camera-ready) version.
% Change to "preprint" to generate a non-anonymous version with page numbers.
\usepackage[final]{acl}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}

%Including images in your LaTeX document requires adding
%additional package(s)
\usepackage{graphicx}

% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{booktabs}
\usepackage{minted}

\usemintedstyle{friendly}
\setminted[json5]{
    breaklines=true,
    breakanywhere=true,
    breaksymbolleft=,
    fontsize=\footnotesize,
    numbersep=5pt
}

\title{FactUEP at SemEval-2026 Task 4: Structured Narrative Similarity Scoring with Aspect Decomposition and Weak-Signal Gating}

\author{
  Marcin Sawi\'nski \\
  \texttt{marcin.sawinski@ue.poznan.pl}\\
  Pozna\'n University of Economics and Business\\
  Pozna\'n, Poland
}



\begin{document}
\maketitle
\begin{abstract}
This paper presents approach to narrative similarity prediction for SemEval-2026 Task 4 Track A. We introduce an LLM-based system that operationalizes the three core dimensions—Abstract Theme, Course of Action, and Outcomes—via schema-constrained prompting to enforce structured outputs and alignment with the annotation protocol. The system proceeds in three stages: structured aspect decomposition and scoring, weak-signal gating for low-confidence cases, and a targeted LLM-based tiebreak. The final model achieved near-human performance and ranked second on the Track A leaderboard.

\end{abstract}
\section{Introduction}
Narrative similarity concerns the degree to which stories share abstract structural patterns, independent of surface details such as setting, named entities, or writing style. SemEval-2026 Task 4 formalizes this problem as a comparative judgment task, requiring systems to determine which of two candidate stories is narratively closer to an anchor based on three dimensions: Abstract Theme, Course of Action, and Outcomes. The task is inherently subjective and characterized by low inter-annotator agreement, making robust operationalization challenging. In this work, we investigate whether structured decomposition, constrained prompting, and confidence-aware revision can improve large language model performance on this setting. We propose a schema-driven, multi-stage architecture that explicitly models aspect-level signals and conditionally applies revision via weak-signal gating.

This study addresses the following research questions:

RQ1: Does narrative similarity prediction require large-scale LLMs, or can smaller models achieve competitive performance?

RQ2: Does extended reasoning via verbose outputs improve performance?

RQ3: Can system confidence be estimated reliably and used to conditionally steer response generation?

RQ4: Does a multi-step prompting pipeline, including previous answer review or self-critique, improve prediction accuracy?

RQ5: Does decontextualization of input texts (i.e., removal of surface-specific details) improve narrative similarity assessment?

\section{Background}
The task comprises two tracks. Track A requires selecting, for each triple (anchor, A, B), the candidate narratively closer to the anchor. Track B requires generating embeddings whose distances reflect these similarity judgments \cite{hatzel-etal-2026-semeval}.

Narrative similarity refers to abstract structural relatedness between stories, explicitly disregarding surface details. It is defined along three dimensions: Abstract Theme, Course of Action, and Outcomes.\footnote{\url{https://narrative-similarity-task.github.io/static/narrative-similarity_annotation-guidelines.pdf}}

The task is not grounded in a specific narrative theory and provides no explicit weighting of the three aspects, aiming instead to reflect intuitive human judgments in an inherently subjective setting. This subjectivity is reflected in low inter-annotator agreement: Krippendorff’s alpha was 0.33 for overall similarity and near chance for individual aspects (0.05, 0.07, 0.11 for theme, course of action, and outcomes).

Importantly, this definition differs from SemEval-2025 Task 10, which focused on ideological framing and public discourse rather than broader storytelling patterns \cite{piskorski-etal-2025-semeval}. The approach to 2025 task 10 subtask 2 Narrative Classification was dominated by fine-tuning smaller models like Llama3.2 \cite{singh-etal-2025-gatenlp} or Phi-4 and Qwen2.5 \cite{sun-etal-2025-pateam} and a multi-agent approach \cite{eljadiri-nurbakova-2025-team}.


\section{Dataset}
The core dataset consists of annotated triples of story summaries drawn from Wikipedia. The development set includes 200 labeled triples, the test set - 400 triples.To enable meaningful human annotation, the authors employed the narrative embedding model \emph{story-emb} \cite{hatzel-biemann-2024-story} to construct triples expected to share a certain degree of similarity.
The organizers released additional 1,900 synthetic training triples generated by LLMs, however, their utility for training was limited, as the manually annotated dataset was significantly more challenging. 


\section{Methodology}
\subsection{Model selection}
The first stage benchmarked a baseline prompt across multiple LLMs of varying parameter scales to assess the impact of model capacity on narrative similarity performance. Using identical prompting conditions, we compared different architectures and instruction-tuning regimes, namely two 8B models (Llama 3.1, DeepSeek-R1), GPT-OSS 20B and 120B and GPT-5.2

\subsection{Prompt and context engineering}
The second stage focused on designing multiple prompt variants that differed in the level of structural detail required and in the sequencing of intermediate reasoning steps. 

Prompts were designed to enforce structured, schema-compliant outputs in a predefined JSON format. Each field imposed explicit constraints on content and granularity e.g.theme descriptions were required to be high-level abstractions, course-of-action representations had to be expressed as ordered event sequences, and outcomes were restricted to final resolution states only, decision output constrained to a single binary choice. In some variants, intermediate aspect scores were required before aggregation, ensuring that similarity judgments were decomposed prior to final comparison.

This schema-constrained design reduced reasoning drift, limited stylistic confounds, and increased consistency across model outputs. 

The prompts were generated with GPT-5.2 under manual guidance on structure and were informed by the official annotation guidelines and aspect definitions of the task.
The variants included the following prompt configurations.

\textbf{Basic decomposition.}
We used prompt that enforced explicit decomposition of narrative similarity into three aspects before issuing a final judgment. This was implemented via schema-constrained response format with fixed fields per aspect. For both candidates, the model produced structured aspect-level comparisons followed by a final decision (see listing \ref{prompt:basic_decomposition}).

\textbf{Abstract.}
A set of four prompts designed to extract structured narrative representations of each aspect and the overall evaluation. The output format included list of abstract items present in the story, each item expressed in 2–5 words (see \ref{prompt:abstract}).

\textbf{Scoring.}
Prompts that combined structured aspect extraction with verbose justification and explicit numerical similarity scoring and three-class similarity classification (A, B, \texttt{NONE}). These variants required the model to compare aspect-level representations and assign similarity scores prior to producing a final decision (see \ref{prompt:scoring}).

\textbf{Answer Review.}
Prompts designed to re-evaluate prior predictions, particularly borderline cases. These prompts explicitly revisited earlier outputs to identify potential inconsistencies between aspect-level comparisons and the final verdict (see \ref{prompt:answer_review}).

\subsection{Weak-Signal Gating}

We introduced an additional processing stage to determine when answer revision was warranted. Replacing the final LLM verdict with hard or weighted soft voting over per-aspect decisions did not improve performance. Instead, we adopted a targeted strategy that preserved the original prediction in strong cases and applied revision only to low-confidence or internally inconsistent outputs. This was implemented via weak-gate conditions that selectively triggered an \emph{Answer Review} stage. We define a weak-signal regime based on aspect-level similarity margins and decision consistency.

Let $m_T$, $m_C$, and $m_O$ denote the per-aspect similarity margins between candidate A and B for theme, course of action, and outcomes, respectively:

\begin{equation}
m_T = \text{sim}_T(A) - \text{sim}_T(B),
\end{equation}
\begin{equation}
m_C = \text{sim}_C(A) - \text{sim}_C(B),
\end{equation}
\begin{equation}
m_O = \text{sim}_O(A) - \text{sim}_O(B).
\end{equation}

We compute a weighted aggregate similarity margin, with empirically determined weights $w_T$, $w_C$, and $w_O$ reflecting the relative importance of each aspect:

\begin{equation}
S_{\text{margin}} =
\frac{
w_T m_T + w_C m_C + w_O m_O
}{
w_T + w_C + w_O
}.
\end{equation}

In addition, we define the strongest per-aspect signal as:

\begin{equation}
M_{\max} = \max \left( |m_T|, |m_C|, |m_O| \right).
\end{equation}

We further count the number of aspect-level decisions labeled as \texttt{NONE}:

\begin{equation}
N_{\text{none}} = \sum_{k \in \{T,C,O\}} \mathbf{1}[\text{closer}_k = \texttt{NONE}].
\end{equation}

A prediction is classified as \emph{weak} if at least one of the following conditions holds:

\begin{align}
|S_{\text{margin}}| &< \tau_S \quad \text{(low global margin)} \\
M_{\max} &< \tau_A \quad \text{(no strong aspect signal)} \\
N_{\text{none}} &\ge k_{\text{none}} \quad \text{(insufficient aspect evidence)}.
\end{align}

Here, $\tau_S$ controls the minimum required aggregate margin,
$\tau_A$ enforces a minimum per-aspect confidence threshold,
and $k_{\text{none}}$ specifies how many undefined aspect decisions
trigger the gate (if the \texttt{NONE} rule is enabled).

If any of the above criteria are satisfied, the instance is routed to the \emph{Answer Review} stage. Otherwise, the original prediction is retained. All parameter values and aspect weights  were determined via experimental tuning on the development set.

\section{Results}
All models were evaluated on the development set using \emph{Basic decomposition} prompt template.The 8B models (Llama 3.1, DeepSeek-R1) achieved below 60\% accuracy, GPT-OSS models (20B, 120B) exceeded 65\%, and GPT-5.2 surpassed 70\% (see Table \ref{tab:model_scale}).
\begin{table}[t]
\centering
\caption{Baseline decomposition prompt performance across models on the development set.}
\label{tab:model_scale}
\begin{tabular}{lr}
\toprule
Model & Accuracy \\
\midrule
Llama 3.1 8B & 58.0 \\
DeepSeek R1 8B & 58.0 \\
GPT OSS 20B & 65.5 \\ 
GPT OSS 120B & 67.0 \\
GPT-5.2 & \textbf{73.5} \\
\bottomrule
\end{tabular}
\end{table}

Subsequent experiments focused primarily on GPT-5.2, given its superior baseline performance.

Among single-prompt configurations, the highest score (77.5\% accuracy) was achieved with the \emph{No Monologue} variant. This prompt employed an elaborate system message detailing the task definition and decomposition rules, while limiting the final output strictly to a single-character decision (A or B). Enforcing structured reasoning instructions and suppressing any free-form explanations yielded the strongest results (see Table \ref{tab:prompt_variants}).

The second-best single-prompt configuration, \emph{Verbose Scoring} (76.5\%), used very similar system message but required structured output consisting of per-aspect abstraction, numerical similarity scoring and free-form contributions and decision explanations in the output. Longer output reduced single-pass accuracy, but produced richer intermediate signals for post-processing.

Applying a second-stage \emph{Answer Review} prompt to outputs of the \emph{Verbose Scoring} increased overall accuracy to 78\%. This revision step corrected a number of initial errors, and also overturned some previously correct predictions. The aggregate effect remained positive, despite local instability.

Applying a second-stage \emph{Answer Review} only to examples selected by the \emph{Weak Gating} mechanism yielded further improvements, raising accuracy to 79.5\%. The gating configuration was determined via random search over the  parameter space. Although multiple parameter combinations produced comparable results, consistent patterns emerged across runs. For the final submission, we selected $\tau_S = 0.18$ and $\tau_A = 0.25$, with weights 0.4, 0.3 and 0.3 for Theme, Course of Action and Outputs with \texttt{NONE} rule disabled (see Figure \ref{fig:tau_heatmap}).

Under the selected configuration, 69 out of 200 instances (34.5\%) were classified as weak and routed to the \emph{Answer Review} stage. Within this subset, 14 prediction flips occurred, corresponding to a 20.3\% flip rate among weak cases (7\% overall). Importantly, these revisions led to substantial gains: accuracy on weak instances improved from 62.3\% to 71.0\% (+8.7\% points). Strong instances were left unchanged, preserving their baseline accuracy (83.97\%). Overall accuracy increased from 76.5\% to 79.5\% (+3.0\% points). These results indicate that the weak-gate mechanism effectively concentrates revision effort on low-confidence cases, improving performance without degrading already confident predictions.

We also explored prompt variants aimed at reducing verbosity in the \emph{Verbose Scoring} configuration while preserving intermediate signals for two-stage processing. These modifications were intended to limit reasoning drift without sacrificing aspect-level information. Specifically, the \emph{No Why} and \emph{No Contribution} variants removed explicit rationales from the output, while the \emph{Reordered} variant required the model to produce the final verdict before generating explanations.

Contrary to expectations, all such modifications degraded performance. This suggests that the explanatory components may play a stabilizing role in structured reasoning, and that altering the ordering or suppressing justificatory signals can disrupt the internal coherence of the model’s decision process.

We also tested a two-stage prompting setup in which the model first generated abstract representations of theme, course of action, and outcomes, and then assessed similarity based on them. This reduced accuracy by about five percentage points, likely due to information loss during abstraction.

On the test set ($n{=}400$), \emph{Verbose Scoring} reached 74.25\% accuracy, \emph{Answer Review} 75.25\%, and \emph{Weak Gating} 75.75\%---drops of 2.25, 2.75, and 3.75 percentage points relative to the development set. The largest decline for \emph{Weak Gating} suggests that its thresholds may have been overfitted to the development distribution, although the relative ordering of variants is preserved across both splits, indicating that the benefits of structured prompting and confidence-aware revision generalize beyond the tuning sample. To assess whether the 1.5-point gain of \emph{Weak Gating} over \emph{Verbose Scoring} is statistically meaningful, we examined the 20 discordant pairs (13 helpful vs.\ 7 harmful flips). McNemar's exact test yielded $p{=}0.26$, and a paired bootstrap over 10{,}000 resamples produced a 95\% CI of $[-0.75, +3.75]$ percentage points ($p_{\text{boot}}{=}0.20$). Neither test rejects the null, and the interval includes zero. The direction is consistent with the development set and with the underlying mechanism, but the test sample is too small to establish significance, so we interpret \emph{Weak Gating} as directionally useful but not statistically conclusive.

\begin{table}[t]
\centering
\caption{GPT-5.2 performance across prompt configurations on the development and test set.}
\label{tab:prompt_variants}
\begin{tabular}{lrr}
\toprule
& \multicolumn{2}{c}{\textit{Accuracy}} \\
Prompt Variant & development & test\\
\midrule
\multicolumn{3}{l}{\textit{Single-Prompt}} \\
No Monologue      & \textbf{77.50} & \\
Verbose Scoring   & 76.50 & 74.25\\
No why & 75.00 & \\
No contribution & 74.00 & \\
Reordered & 74.50 & \\
\midrule
\multicolumn{3}{l}{\textit{Two-Prompt}} \\
Answer Review     & 78.00 & 75.25 \\
Weak Gating     & \textbf{79.50} & \textbf{75.75} \\
\bottomrule
\end{tabular}
\end{table}

\begin{figure}[t]
\centering
\includegraphics[width=\linewidth]{tau_heatmap.pdf}
\caption{Accuracy heatmap over weak-gate thresholds $\tau_S$ and $\tau_A$.}
\label{fig:tau_heatmap}
\end{figure}
\section{Discussion}
\paragraph{RQ1: Does narrative similarity prediction require large-scale LLMs?}
Yes. Results indicate a clear model-scale and training regime dependency. Smaller 8B models operated only marginally above chance, while larger GPT-OSS models improved by  +7.0  / +9.0 points with little difference between 20B and 120B models. GPT-5.2 consistently surpassed 70\% across prompt variants, however, the gap between results suggests that architectural design and training regime, not parameter count alone, play a decisive role.

\paragraph{RQ2: Does extended reasoning via verbose outputs improve performance?}
No. Extended reasoning via verbose outputs did not improve single-pass accuracy. The \emph{Verbose Scoring} prompt (76.5\%) underperformed the more constrained \emph{No Monologue} variant (77.5\%), indicating that longer outputs can introduce reasoning drift. Thus, elaborate reasoning output does not automatically translate into better performance.

\paragraph{RQ3: Can system confidence be estimated and used to conditionally steer response generation?}
Yes. \emph{Weak Gating} effectively identified low-confidence cases (34.5\% of instances) and selectively applied revision. This improved accuracy on weak cases from 62.3\% to 71.0\% without degrading strong predictions, yielding a +3.0\% points (dev) and +1.5\% point (test) gain. These results demonstrate that margin-based confidence estimation can reliably guide conditional response refinement.

\paragraph{RQ4: Does a multi-step prompting pipeline, including previous answer review or self-critique, improve prediction accuracy?}
Yes. Multi-step prompting improves performance when applied selectively. Unconditional \emph{Answer Review} yielded modest gains (+1.5\% point dev, +1.0\% point test), while gated application achieved larger improvements (+3.0\% points dev, +1.5\% point test). However, decomposition-first pipelines that separated abstraction and judgment into separate steps reduced accuracy, indicating that multi-stage architectures are beneficial only when initial signals are preserved without excessive information loss.

\paragraph{RQ5: Does decontextualization of input texts (i.e., removal of surface-specific details) improve narrative similarity assessment?}
No. The \emph{Abstract} prompt, which used only generated decontextualized representations of each aspect without presenting the full original text of stories, underperformed the full text setup by about -5\% points. This suggests that surface details may provide important cues for similarity assessment, and that abstraction can lead to information loss detrimental to performance.

\section{Conclusions and Future Work}

Across experiments, three main findings emerge.

First, model scale and architecture matter. Smaller 8B models are not suited for the task. However, improvements were not strictly monotonic with parameter count, suggesting that alignment and training regime are at least as important as scale alone.

Second, explicit decomposition improves the score by better mimicking the annotator's thinking process and decisions.
Highly constrained prompts that suppressed free-form explanations achieved the best standalone performance, indicating that limiting reasoning drift can outweigh the benefits of verbose deliberation. At the same time, structured intermediate signals (e.g., aspect-level scores) proved valuable when used for post-hoc calibration.

Third, confidence-aware revision is effective. The proposed Weak-Signal Gating mechanism reliably identified low-margin or internally inconsistent predictions and selectively routed them to a second-stage review. The results demonstrate that margin-based confidence estimation can be operationalized in a principled and computationally lightweight manner.


\paragraph{Limitations.}
Despite a strong leaderboard position, important limitations remain. First, the evaluation sample is small: on the 400-example dataset, a one percentage point gain equals only four additional correct predictions. Although patterns were consistent across dev and test sets, improvements of 1–3 percentage points remain vulnerable to statistical variance and should be interpreted cautiously.

Second, inter-annotator agreement on individual narrative aspects is low, reflecting the intrinsic subjectivity of narrative similarity judgments. The task formulation leaves aspect weighting unspecified, introducing variability that models must implicitly resolve.

Third, our approach relies on high-capacity LLMs, and its effectiveness may not transfer to resource-constrained settings without additional distillation or compression.

\paragraph{Research directions.}
First, incorporating more formal narrative theory into dataset generation and system design may improve interpretability, reduce ambiguity, and increase stability.

Second, extending experiments to substantially larger annotated datasets would allow more reliable estimation of effect sizes and statistical significance. Larger samples would reduce sensitivity to sampling noise and enable finer-grained analysis of gating thresholds, flip behavior, and multi-stage prompting strategies.


% Custom bibliography entries only
\bibliography{custom,anthology-1,anthology-2}
% \bibliography{custom}

\appendix

\section{Prompt templates}
\label{sec:appendix}
\subsection{Basic decomposition}
\label{prompt:basic_decomposition}
\inputminted{json5}{prompts/basic_decomposition.json}

\subsection{Abstract}
\label{prompt:abstract}

\subsubsection{Theme abstraction}
\label{prompt:theme_abstraction}
\inputminted{json5}{prompts/theme_abstraction.json}

\subsubsection{Course of action abstraction}
\label{prompt:course_abstraction}
\inputminted{json5}{prompts/course_abstraction.json}

\subsubsection{Outcomes abstraction}
\label{prompt:outcomes_abstraction}
\inputminted{json5}{prompts/outcomes_abstraction.json}

\subsubsection{Score abstraction}
\label{prompt:score_abstraction}
\inputminted{json5}{prompts/score_abstraction.json}

\subsection{Scoring}
\label{prompt:scoring}

\subsubsection{No Monologue}
\label{prompt:no_monologue}
\inputminted{json5}{prompts/no_monologue.json}

\subsubsection{Verbose Scoring}
\label{prompt:verbose_scoring}
\inputminted{json5}{prompts/verbose_scoring.json}

\subsubsection{No why}
\label{prompt:no_why}
Note: Fields \texttt{messages} and \texttt{enums} are identical to the \emph{Verbose Scoring} prompt, and only the \texttt{response\_format} field controlling the output is constrained.
\inputminted{json5}{prompts/no_why.json}

\subsubsection{No contribution}
Note: Fields \texttt{messages} and \texttt{enums} are identical to the \emph{Verbose Scoring} prompt, and only the \texttt{response\_format} field controlling the output is constrained.
\label{prompt:no_contribution}
\inputminted{json5}{prompts/no_contribution.json}

\subsubsection{Reordered}
Note: Fields \texttt{messages} and \texttt{enums} are identical to the \emph{Verbose Scoring} prompt, and only the \texttt{response\_format} field controlling the output is constrained.
\label{prompt:reordered}
\inputminted{json5}{prompts/reordered.json}

\subsection{Answer Review}
\label{prompt:answer_review}
\inputminted{json5}{prompts/answer_review.json}

\end{document}