% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Remove the "review" option to generate the final version.
\usepackage[final]{acl}
% \usepackage[preprint]{acl}
%\usepackage{acl}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}

% OWN PACKAGES
\usepackage{amsthm}
\usepackage{amssymb}
\usepackage{booktabs}
\usepackage{algorithm}
\usepackage{algorithmic}
\usepackage{enumitem}
\usepackage{tabularray}  % dotted hline
\usepackage{subcaption}  % side-to-side figures

% Aliki
\usepackage[graphicx]{realboxes}

\newtheorem{example}{Example}
\newtheorem{theorem}{Theorem}

\usepackage{caption}
%\captionsetup{belowskip=0pt}
\setlength{\belowcaptionskip}{-10pt}

%\usepackage{emoji}
% Downloaded from https://openmoji.org
\usepackage{makecell}
\usepackage{subcaption}

\usepackage{cleveref}
\usepackage{rotating}
%\newcommand\tabrotate[1]{\begin{turn}{45}\rlap{#1}\end{turn}}
\newcommand\tabrotate[1]{\rotatebox[origin=c]{70}{#1}}

\usepackage{xcolor,colortbl}
\definecolor{Gray}{gray}{0.94}
\definecolor{Tan}{rgb}{0.8203,0.7031,0.5469}
\definecolor{LightCyan}{rgb}{0.88,1,1}
\definecolor{babyblue}{cmyk}{0.3725, 0.2314, 0, 0.1}
\definecolor{crimson}{rgb}{0.8627, 0.0784, 0.2353}
\definecolor{ube}{cmyk}{0.286, 0.301, 0, 0.219}
\definecolor{LightBlue}{RGB}{139, 193, 230} % DA Blues
\definecolor{LightGreen}{RGB}{150, 229, 138} % EA Greens
\definecolor{LightOrange}{RGB}{255, 178, 102} % TA Oranges
\definecolor{T01Red}{HTML}{BA502E}
\definecolor{T02Yellow}{HTML}{EEC564}
\definecolor{T03Brown}{HTML}{A18446}
\definecolor{T04Green}{HTML}{83DC4D}
\definecolor{T05Purple}{HTML}{847CF5}
\definecolor{T06Pink}{HTML}{F789B8}
\definecolor{T07Blue}{HTML}{A7F7E3}
\definecolor{T08Red}{HTML}{F57F62}
\definecolor{T09Blue}{HTML}{1C52AA}

\usepackage{soul}



% Code box
\usepackage{tcolorbox}
\usepackage[finalizecache=true,cachedir=minted-cache]{minted} 
%\usepackage[frozencache=true,cachedir=minted-cache]{minted} 
\tcbuselibrary{minted,breakable,xparse,skins,hooks}
\definecolor{bg}{gray}{0.95}
\DeclareTCBListing{mintedbox}{O{}m!O{}}{%
  breakable=true,
  listing engine=minted,
  listing only,
  minted language=#2,
  minted style=default,
  minted options={%
    linenos,
    gobble=0,
    breaklines=true,
    breakafter=,,
    fontsize=\small,
    numbersep=8pt,
    escapeinside=||,
    #1},
  boxsep=0pt,
  left skip=0pt,
  right skip=0pt,
  left=25pt,
  right=0pt,
  top=3pt,
  bottom=3pt,
  arc=5pt,
  leftrule=0pt,
  rightrule=0pt,
  bottomrule=2pt,
  toprule=2pt,
  colback=bg,
  colframe=orange!70,
  enhanced,
  overlay={%
    \begin{tcbclipinterior}
    \fill[orange!20!white] (frame.south west) rectangle ([xshift=20pt]frame.north west);
    \end{tcbclipinterior}},
  #3}

\newtcolorbox{myquotebox}{
    colback=blue!5!white, % Background color
    colframe=blue!75!white, % Frame color
    arc=4mm, % Roundness of the corners
    top=5mm, % Top margin
    bottom=5mm, % Bottom margin
    left=5mm, % Left margin
    right=5mm, % Right margin
    boxrule=1pt, % Border thickness
    fonttitle=\bfseries, % Title font style
    title= % Box title
}


% Comments from Milad
\newsavebox\bscombox
\newcommand{\bscom}[3][]{%
	% #1 Optional comment.
	% #2 Original text.
	% #3 Replacement text.
	\sbox{\bscombox}{\fontsize{8}{9}\selectfont#1#2#3}
	\noindent
	\st{#2}{\selectfont
		\color{blue}#3\ifx\\#1\\\else{\fontsize{8}{9}\selectfont\color{violet}[#1]}\fi
	}
}

\newcommand{\added}[1]{{#1}}

\newcommand{\fixme}[2][]{\todo[color=yellow,size=\scriptsize,fancyline,caption={},#1]{#2}} % to mark stuff that you know is missing or wrong when you write the text
\newcommand{\note}[4][]{\todo[author=#2,color=#3,size=\scriptsize,fancyline,caption={},#1]{#4}} % default note settings, used by macros below.
\newcommand{\nils}[1]{\textcolor{Tan}{Nils: #1}}
\newcommand{\aliki}[1]{\textcolor{crimson}{Aliki: #1}}
\newcommand{\milad}[1]{\textcolor{ube}{Milad: #1}}
\newcommand{\gray}[1]{\colorbox{gray!25}{TODO: #1}}


\newcommand{\narrowtextsc}[1]{\textls[-50]{\textsc{#1}}}
\newcommand{\lm}[1]{\texttt{#1}}
\newcommand{\sys}[1]{\narrowtextsc{#1}}
\newcommand{\data}[1]{\textsf{#1}}

\title{
    Human and LLM-based Assessment of Teaching Acts \\in Expert-led Explanatory Dialogues
}

\newcommand{\affilsup}[1]{\rlap{\textsuperscript{\normalfont#1}}}
\author{
    Aliki Anagnostopoulou\thanks{Work done while at DFKI.}
    \qquad 
    \textbf{Nils Feldhus}\affilsup{1,3,4}
    \qquad
    Yi-Sheng Hsu\affilsup{*,7}
    \\
    \textbf{Milad Alshomary}\affilsup{5}
    \qquad
    \textbf{Henning Wachsmuth}\affilsup{6}
    \qquad
    \textbf{Daniel Sonntag}\affilsup{1,2}
    \\
    \small{
        $^1$German Research Center for Artificial Intelligence (DFKI)
        \quad
        $^2$Applied Artificial Intelligence, Oldenburg University
    }\\
    \small{
        $^3$Technische Universit\"at Berlin
        \quad 
        $^4$BIFOLD – Berlin Institute for the Foundations of Learning and Data 
    }\\
    \small{
        $^5$Data Science Institute, Columbia University
        \quad
        $^6$Leibniz University Hannover, Institute of Artificial Intelligence
    }\\
    \small{
        $^7$Computer Science Institute, Ruhr West University of Applied Sciences
    }\\
    \small{Corresponding authors: 
    \texttt{feldhus@tu-berlin.de}
    \quad 
    \texttt{h.wachsmuth@ai.uni-hannover.de}
    }
}


\begin{document}

\maketitle
\begin{abstract}

Understanding the strategies that make expert-led explanations effective is a core challenge in didactics and a key goal for explainable AI. To study this computationally, we introduce \data{ReWIRED}, a large corpus of explanatory dialogues annotated by education experts with fine-grained, span-level teaching acts across five levels of explainee knowledge. We use this resource to assess the capabilities of modern language models, finding that while few-shot LLMs struggle to label these acts, fine-tuning is a highly effective methodology. Moving beyond structural annotation, we propose and validate a suite of didactic quality metrics. We demonstrate that a prompt-based evaluation using an LLM as a ``judge'' is required to capture how the functional quality of an explanation aligns with the learner's expertise -- a nuance missed by simpler static metrics. Together, our dataset, modeling insights, and evaluation framework provide a comprehensive methodology to bridge pedagogical principles with computational discourse analysis.
\end{abstract}

\section{Introduction}

\begin{figure}[ht]
    \centering
    \resizebox{\columnwidth}{!}{%
    \includegraphics[width=\textwidth]{figures/ReWIRED 2.0.png}
    }
    \caption{
    Our workflow: We begin by having education experts create span-level annotations of teaching acts in explanatory dialogues. We then experiment with various LLMs to automate this annotation. Finally, we conduct a qualitative evaluation, using both human experts and LLMs, to assess the quality of the explanations based on didactic principles.
    }
    \label{fig:example-dialogue}
\end{figure}


Effective teaching is a masterclass in communication, where an expert dynamically adapts their language and strategy to guide a learner toward understanding. This process unfolds as a complex, structured dialogue, yet the specific discourse mechanisms that make an explanation effective, especially when tailored to different audiences, are not well understood from a computational perspective. While insights from education and psychology define what constitutes good teaching \citep{miller-2019-explanation, kulgemeyer-2018-framework}, we lack the fine-grained datasets and evaluation frameworks needed to model these principles in natural language.

This paper addresses that gap through a multi-faceted approach, as illustrated in Figure~\ref{fig:example-dialogue}. First, we introduce \data{ReWIRED}, a new corpus resource that significantly extends the \data{WIRED} dataset \citep{wachsmuth-alshomary-2022-mama}. Our contribution lies in a new layer of \textbf{span-level annotations of teaching acts}, provided by education domain experts, across dialogues tailored to five distinct knowledge levels (from child to colleague). This provides an empirical foundation for studying pedagogical discourse structure (\S \ref{sec:rewired}).

Second, we explore the feasibility of automating the detection of these acts. We evaluate a range of language models and prompting techniques, revealing that while few-shot LLMs struggle with this nuanced task, models fine-tuned on our data—even smaller ones—can achieve near-perfect accuracy. This establishes a robust methodology for analyzing instructional dialogues at scale (\S \ref{sec:experiments}).

Finally, we move from structural annotation to quality assessment. We employ and extend \sys{IXQuisite}, a suite of metrics grounded in didactics, to evaluate explanation quality. We validate these metrics with our expert annotators and demonstrate that a prompt-based evaluation using LLMs as ``judges'' is significantly more effective at capturing the functional quality of instructional discourse than traditional static methods. This provides a new paradigm for evaluating pedagogically-aware systems (\S \ref{sec:ixquisite}).

Together, these contributions -- a richly annotated corpus, a validated modeling approach, and a nuanced evaluation framework -- provide a comprehensive methodology for bridging educational theory with computational discourse analysis, paving the way for AI systems that can generate more effective, human-like explanatory dialogues
\footnote{The dataset, code, and test suite are available at \\ \url{https://github.com/nfelnlp/InstruX}.}.


\section{Background and related work}
\label{sec:teaching_models}

\textbf{Instructional explanations} are intended to transfer knowledge by introducing a new cognitive framework for understanding a concept or performing a task, bridging the gap between a knowledgeable individual and someone lacking that understanding. In science education, such explanations are considered both a fundamental activity and a goal of scientific practice, aimed at systematically addressing ``how'' and ``why'' questions \cite{kulgemeyer-2018-framework}. The authors highlight the separation of two interpretations for the term \textit{explanation}: One is an explanation seen as activity, whose goal is to ``engender understanding'' between an explanation holder and an explainee; the other is a more philosophical understanding explanation, as that which connects \textit{explanans} and \textit{explanandum} \cite{DBLP:zhu-2023}. 
Although most studies concerning explainability have focused on the latter, 
we focus on its execution as a social, dialogical practice \citep{miller-2019-explanation}. In this view, the sequence of communicative acts, the choice of examples, and the adaptation to the learner are all crucial elements of the dialogue's discourse structure.


\textbf{Modeling Pedagogical Strategies with Annotation Schemata.}
To analyze this structure computationally, we draw from established \textit{teaching models} from education science \citep{oser-baeriswyl-2002-aera, krabbe-2015-lernprozessorientierte}. These models are not just abstract theories; they provide a blueprint for effective instructional sequences. For instance, a common pattern is to first assess prior knowledge, then introduce a concept, provide an example, and finally test for understanding. We operationalize these pedagogical principles as a set of nine span-level \textit{teaching acts} (Table \ref{tab:acts}). This approach treats teaching strategies as a form of domain-specific discourse annotation, allowing us to model the underlying functional structure of the dialogue beyond surface-level linguistics.



\textbf{Corpora for Educational Dialogue and Explanation Quality.}
Several corpora have paved the way for analyzing educational dialogues. Datasets like \data{CIMA} \citep{stasaski-2020-cima}, \data{TSCC-2} \citep{caines-2022-tscc-2}, and \data{NCTE} \citep{demszky-hill-2023-ncte} capture teacher-student interactions, but often focus on general dialogue moves rather than the specific pedagogical functions within an explanation. The work closest to ours is the \data{WIRED} corpus \citep{wachsmuth-alshomary-2022-mama} and its analysis by \citet{alshomary-etal-2024-modeling}, which includes annotations for high-level explanation and dialogue moves. Our work significantly extends this by: (1) doubling the dataset size; (2) providing more granular, \textbf{span-level} annotations of teaching acts rather than turn-level classifications; and (3) using \textbf{domain experts} in education for annotation, increasing the validity of the labels. This finer granularity is crucial for understanding how different teaching strategies are woven together within a single conversational turn.


Recent work has also leveraged LLMs in education, for tasks like assessing student answers \citep{carpenter-etal-2024-assessing} or cognitive engagement \citep{mclure-etal-2024-leveraging}, and in human-AI tutoring systems \citep{wang-etal-2024-tutor, jurenka-etal-2024-towards}. Evaluating the quality of these interactions remains a challenge. While some metrics focus on general dialogue quality \citep{mehri-eskenazi-2020-fed} or textual features \citep{mcnamara-2014-coh-metrix}, they often miss the pedagogical dimension. Inspired by the approach of \citet{rooein-etal-2024-beyond}, who use both static and LLM-prompted metrics for readability, we adopt and expand a suite of quality metrics to specifically assess instructional explanations, connecting discourse phenomena to didactic principles. This addresses the challenge noted by \citet{xu-etal-2024-promises} that LLMs excel at simple evaluation but struggle with complex teaching practices without proper guidance.


\begin{table}[t!]
    \centering
    \resizebox{\columnwidth}{!}{%
    \renewcommand*{\arraystretch}{0.8}
    %\rowcolors{1}{white}{gray!25}
    \begin{tabular}{|ll|}
        \bottomrule
        \cellcolor{LightOrange} \textbf{Teaching Act} & \cellcolor{LightOrange} \textbf{T. Mdl.} \\
        
        \toprule
        
        \textbf{T01}: \fcolorbox{T01Red}{white}{\textit{Assess Prior Knowledge}} 
            & CB, UT
            \\
        \small ~Checking what the student knows before starting a lesson & \\
        
        \rowcolor{gray!10} 
        \textbf{T02}: \fcolorbox{T02Yellow}{white}{\textit{Lesson Proposal}} 
            & UT
            \\
        \rowcolor{gray!10} 
        \small ~Proposing the steps that will be taken during the lesson & \\
        
        \textbf{T03}: \fcolorbox{T03Brown}{white}{\textit{Active Experience}} 
            & CB, UT
            \\
        \small ~Providing the student with puzzle/question to explore; & \\
        \small ~(Student:) Interacting with a mental concept & \\
        
        \rowcolor{gray!10} 
        \textbf{T04}: \fcolorbox{T04Green}{white}{\textit{Reflection}} 
            & PS
            \\
        \rowcolor{gray!10} 
        \small ~Finding gaps in knowledge or inconsistencies; & \\
        \rowcolor{gray!10}
        \small ~Asking questions about the experience or concept & \\
            
        \textbf{T05}: \fcolorbox{T05Purple}{white}{\textit{Knowledge Statement}} 
            & PS
            \\
        \small ~Stating the concept(s) being taught via rules or facts & \\
            
        \rowcolor{gray!10} 
        \textbf{T06}: \fcolorbox{T06Pink}{white}{\textit{Comparison}} 
            & UT
        \\
        \rowcolor{gray!10} 
        \small ~Considering similarities and differences between & \\
        \rowcolor{gray!10} 
        \small ~the main concept and other related topics or facts & \\
            
        \textbf{T07}: \fcolorbox{T07Blue}{white}{\textit{Generalization}} 
            & CB, PS
            \\
        \small ~Exploring how the concept applies to new scenarios, & \\
        \small ~experiences and situations outside of the lesson topic & \\

        \rowcolor{gray!10} 
        \textbf{T08}: \fcolorbox{T08Red}{white}{\textit{Test Understanding}} 
            & CB
            \\
        \rowcolor{gray!10} 
        \small ~Finding out if the concept previously established & \\
        \rowcolor{gray!10} 
        \small ~was received correctly and is properly understood & \\
            
        \textbf{T09}: \fcolorbox{T09Blue}{white}{\textit{Engagement Management}} 
            & 
            \\
        \small ~Maintaining the classroom context to facilitate effective & \\
        \small ~teaching, creating rapport between teacher and student & \\
            
        \bottomrule
    \end{tabular}
    }
    \caption{Teaching acts in the \data{ReWIRED} dataset (with descriptions and their connection to a teaching model from didactics: Teaching as problem solving (\textbf{PS}), teaching as concept building (\textbf{CB}) \cite{krabbe-2015-lernprozessorientierte}, and unified teaching choreographies (\textbf{UT}) \cite{oser-baeriswyl-2002-aera}. 
    }
    \label{tab:acts}
\end{table}


% ----------------


%-------------

\begin{table}[]
    \centering
    \resizebox{\columnwidth}{!}{%
    %\renewcommand*{\arraystretch}{0.875}
    \begin{tabular}{p{0.35cm}  p{3cm}  p{0.35cm}  p{3.5cm}}
    \toprule
    \textbf{\#} & \textbf{Topic} & \textbf{\#} & \textbf{Topic} \\ %& \textbf{\# turns (Lvl. 1-5)} \\

    \midrule
    1 & Music harmony %& Jacob Collier \\
        & \cellcolor{yellow!30} 14 
        & \cellcolor{yellow!30} Memory \\
    2 & Blockchain %& Bettina Warburg \\
        & \cellcolor{yellow!30} 15 
        & \cellcolor{yellow!30} Zero-knowledge proofs \\
    3 & Virtual reality %& John Carmack \\
        & \cellcolor{yellow!30} 16 
        & \cellcolor{yellow!30} Black holes \\
    4 & Connectome %& Bobby Kasthuri \\
        & \cellcolor{yellow!30} 17 
        & \cellcolor{yellow!30} Quantum computing \\
    5 & Black holes %& Varoujan Gorjian \\
        & \cellcolor{yellow!30} 18 
        & \cellcolor{yellow!30} Quantum sensing \\
    6 & Lasers %& Donna Strickland \\
        & \cellcolor{yellow!30} 19 
        & \cellcolor{yellow!30} Fractals \\
    7 & Sleep science %& Aric A. Prather \\
        & \cellcolor{yellow!30} 20 
        & \cellcolor{yellow!30} Internet \\
    8 & Dimensions %& Sean Carroll \\
        & \cellcolor{yellow!30} 21 
        & \cellcolor{yellow!30} Moravecs Paradox \\
    9 & Gravity %& Janna Levin \\
        & \cellcolor{yellow!30} 22 
        & \cellcolor{yellow!30} Infinity \\
    10 & Computer hacking %& Samy Kamkar
        & \cellcolor{yellow!30} 23 
        & \cellcolor{yellow!30} Algorithms \\
    11 & Nanotechnology %& George Tulevski
        & \cellcolor{yellow!30} 24 
        & \cellcolor{yellow!30} Nuclear fusion \\
    12 & Origami %& Robert J. Lang
        & \cellcolor{yellow!30} 25 
        & \cellcolor{yellow!30} Time \\
    13 & Machine learning %& Hilary Mason \\
        & \cellcolor{yellow!30} 26 
        & \cellcolor{yellow!30} Chess \\ 
    
    \bottomrule
    \end{tabular}
    }
    \caption{Topics in \data{ReWIRED}. 14-26 (yellow) are transcripts that were not part of the original \data{WIRED} dataset \cite{wachsmuth-alshomary-2022-mama}. The topic ``black holes'' is explained in two different videos, resulting in the duplicate (5, 16). Chess (26) applies distinctive knowledge levels (novice, intermediate, FIDE master, Grandmaster, and AI expert), as educational background doesn't imply a player's capability.
    }
    \label{tab:topics}
\end{table}
%-------------



\section{The \data{ReWIRED} dataset}
\label{sec:rewired}

To study instructional strategies in explanatory dialogues, we introduce \data{ReWIRED}, a new corpus resource featuring a novel layer of expert-provided, span-level annotations. We build upon and significantly extend an existing dataset of instructional dialogues, enriching it with annotations grounded in pedagogical theory to facilitate fine-grained discourse analysis.



\subsection{Source data: Explanation dialogues}

Our starting point is the \data{WIRED} corpus \citep{wachsmuth-alshomary-2022-mama}, which contains transcripts from the \textit{5-Levels} video series\footnote{\url{https://www.wired.com/video/series/5-levels}}. These videos provide a unique setting for discourse analysis, as they feature a domain expert explaining a complex STEM topic to five different explainees of progressively higher expertise: (1) a child, (2) a teenager, (3) an undergraduate, (4) a graduate student, and (5) a colleague (a fellow expert).

We expanded this resource by transcribing and incorporating 13 additional topics released after the original corpus' publication, effectively doubling the dataset size. \data{ReWIRED} now comprises 130 dialogues across the 26 topics shown in Table~\ref{tab:topics}. This expansion broadens the dataset's scope and enriches the variety of linguistic phenomena available for analysis.



\subsection{Annotation of Teaching Acts}
\label{sec:annotation}

The primary contribution of our work is a new layer of annotation. We argue that to model how instruction is delivered, we need annotations that are more granular than turn-level labels. Pedagogical strategies are often embedded within a single utterance or can overlap. Therefore, we adopt a \textbf{span-labeling} approach to precisely identify segments corresponding to nine distinct \textit{teaching acts}, as defined in Table~\ref{tab:acts}. This annotation scheme allows us to capture the fine-grained, and often nested, discourse structure of instructional explanations.




% ----------------

\begin{figure}[t]
    \centering
    
    \begin{subfigure}[b]{.95\columnwidth}
        \includegraphics[width=\textwidth]{figures/ta_agreement_matrix.png}
        \includegraphics[width=\columnwidth]{figures/kappa_per_label.png}
        \label{fig:agreement_ta}
    \end{subfigure}
    
    \caption{\data{ReWIRED} inter-annotator agreement for teaching acts on token level. For better visibility, we scale-adjust the colors by \texttt{np.log1p}$(\dots)^3$. Each cell shows the number of tokens for which annotators (dis)agreed on a label in a pairwise comparison. The bottom row with green and red highlights show the Fleiss' $\kappa$ per teaching act.
    }
    \label{fig:agreement}
\end{figure}

% ----------------




\paragraph{Annotation Process and Quality.}

To ensure the validity of our annotations, we recruited four annotators, all of whom hold a Master of Education degree or equivalent and have practical in-classroom teaching experience. Annotators were onboarded through a detailed process that included a written guide with definitions and examples for each act (see Appendix \ref{app:instructions} and \ref{app:act_examples}), and a screencast demonstrating the annotation tool (\textsc{Label Studio} \cite{Label-Studio}) and walking through ambiguous cases.

The full dataset was split, with each half annotated by two experts. The task proved to be challenging, reflecting the inherent subjectivity of interpreting pedagogical intent. This is visible in Figure \ref{fig:IAA_example}, which shows how two experts can reasonably apply different labels to the same text. The resulting inter-annotator agreement is Fleiss’ $\kappa = 0.44$. While this value indicates moderate agreement, it is not unexpected for a complex discourse annotation task and highlights that human label variation can itself be an informative signal about the ambiguity of the underlying phenomena \citep{plank-2022-problem}. To create a reliable gold standard, we introduced the pre-existing non-expert annotations from \citet{ours2024WiP} as a third opinion and consolidated all three label sets to adjudicate disagreements.

The final distribution of teaching acts across the five knowledge levels is shown in Figure~\ref{fig:teaching_act_dist}. This newly annotated corpus provides a unique resource for studying how discourse strategies in explanations are adapted to listeners with varying levels of prior knowledge.




\begin{figure}[t]
    \centering
    \small
    \begin{subfigure}[b]{\columnwidth}
        \centering
        \includegraphics[width=\columnwidth]{figures/example_T05.png}
    \end{subfigure}
    T05: Knowledge statement
    \break \break
    \begin{subfigure}[b]{\columnwidth}
        \centering
        \includegraphics[width=\columnwidth]{figures/example_T06_T07.png}
    \end{subfigure}
    T06: Comparison (pink) and T07: Generalization (azure)
    \caption{An example of a turn given labeled as different teaching acts by the two expert annotators.
    }
    \label{fig:IAA_example}
\end{figure}

%-------------
\begin{figure}[t!]
    \includegraphics[width=\columnwidth]{figures/dist_ta.png}
    \caption{Distribution of teaching acts in \data{ReWIRED} across the five knowledge levels.
    }
    \label{fig:teaching_act_dist}
\end{figure}
%-------------



% ----------------

\begin{table*}[t!]
    \centering
    %\renewcommand*{\arraystretch}{0.825}
    \resizebox{\textwidth}{!}{%
    \begin{tabular}{|r|rrrrrrrrr|r|r|}
        
        \toprule
        \cellcolor{LightOrange} \textbf{Teaching acts} & \textbf{T01} & \textbf{T02} & \textbf{T03} & \textbf{T04} & \textbf{T05} & \textbf{T06} & \textbf{T07} & \textbf{T08} & \textbf{T09} 
        %& \textbf{T10} 
            & \textbf{Macro-$F_1$} 
            & \textbf{Span Al.} \\

        %\# Tokens & 3957 & 1148 & 13420 & 6325 & 13717 & 4268 & 5023 & 2202 & 3483 & 0 & 53543 & \textbf{Alignm.} \\
        \midrule
        
        \lm{BERT FT}
            & \textbf{80.68} \% & \textbf{72.15} \% & \textbf{87.93} \% & \textbf{83.07} \% & \textbf{90.18} \% & \textbf{81.57} \% & \textbf{83.75} \% & \textbf{82.53} \% & \textbf{80.31} \% 
            %& –
            & \textbf{84.17} \%
            & –
            \\ \midrule

        % GPT
        \lm{GPT-4o} JSON
            & 35.69 \% &
            49.38 \% &
            39.80 \% &
            34.60 \% &
            66.36 \% &
            38.76 \% &
            39.34 \% &
            29.19 \% &
            42.72 \% &

            41.76 \% &
            36.75 \%
            \\

        \lm{GPT-4o} TANL
            & 66.69 \% &
            70.39 \% &
            63.61 \% &
            80.22 \% &
            84.91 \% &
            75.10 \% &
            75.29 \% &
            61.96 \% &
            70.26 \% &

            72.05 \% &
            68.21 \%
            \\

        \lm{GPT-4o} GoLLIE
            & 71.39 \% &
            67.26 \% &
            72.83 \% &
            78.99 \% &
            82.70 \% &
            79.11 \% &
            78.05 \% &
            71.66 \% &
            67.07 \% &
            74.34 \% &
            73.54 \%
            \\ \midrule

        % Gemini
        \textls[-50]{\lm{Gemini 1.5 F} TANL}
            & 53.39 \% 
            & 71.65 \% 
            & 77.76 \% 
            & 85.86 \% 
            & 86.13 \% 
            & 81.88 \% 
            & 83.73 \% 
            & 63.04 \% 
            & 74.83 \%
            & 75.36 \% 
            & 74.09 \%
            \\

        \textls[-50]{\lm{Gemini 1.5 F} GoLLIE}
            & 46.17 \% 
            & 45.95 \%
            & 59.33 \% 
            & 69.39 \% 
            & 72.82 \% 
            & 64.41 \%
            & 65.47 \%
            & 47.84 \%
            & 49.89 \%
            & 57.92 \%
            & 58.80 \%
            \\ \midrule

        \textls[-50]{\lm{Gemini 1.5 P} TANL}
            & 67.11 \%
            & 74.00 \%
            & 79.97 \%
            & 79.45 \%
            & 87.18 \%
            & 81.35 \%
            & 82.03 \%
            & 53.70 \%
            & 77.51 \%
            & 75.71 \%
            & 69.81 \%
            \\

        \textls[-50]{\lm{Gemini 1.5 P} GoLLIE}
            & 46.25 \%
            & 30.56 \%
            & 53.60 \%
            & 63.00 \%
            & 70.56 \%
            & 47.44 \%
            & 49.23 \%
            & 24.88 \%
            & 48.60 \%
            & 48.23 \%
            & 49.53 \%
            \\

        \midrule

        \lm{GPT-4o-mini FT} TANL
            & 93.64 \%
            & 97.98 \%
            & 95.23 \%
            & \textbf{99.30} \%
            & 98.90 \%
            & \textbf{99.03} \%
            & 98.64 \%
            & 97.00 \%
            & 97.28 \%
            & 97.44 \%
            & 94.63 \%
            \\

        \lm{GPT-4o-mini FT} GoLLIE
            & \textbf{98.54} \%
            & \textbf{98.57} \%
            & \textbf{99.11} \%
            & 98.87 \%
            & \textbf{99.56} \%
            & 98.14 \%
            & \textbf{100.0} \%
            & \textbf{99.67} \%
            & \textbf{98.91} \%
            & \textbf{99.04} \%
            & \textbf{95.49} \%
            \\
        
        \bottomrule
    \end{tabular}
    }
    \caption{Language models evaluated on the tasks of sequence-labeling teaching acts within dialogue turns from our \data{ReWIRED} dataset. Percentages under each of the acts show micro-$F_1$ scores in a 3-shot or fine-tuning (FT) setting. Span Alignment (last column) refers to how well the spans extracted by LLMs align with human-annotated spans.
    }
    \label{tab:seq_results}
\end{table*}


\section{Experiments: Sequence-labeling acts}
\label{sec:experiments}

Having established a richly annotated dataset, a critical next step is to assess the feasibility of automating the detection of teaching acts. Automating this process is a prerequisite for analyzing instructional discourse at scale or for developing real-time assistive technologies. We therefore conduct a series of experiments to evaluate how well modern language models can perform this complex, span-level sequence labeling task.

We frame the task as structured prediction on the \data{ReWIRED} dialogues. Our evaluation compares three distinct approaches: a fine-tuned baseline model, large language models (LLMs) in a few-shot setting, and a fine-tuned LLM.

\textbf{Models and Setups.}
As a strong baseline, we fine-tune \lm{BERT-base} \citep{devlin-etal-2019-bert} for token-level classification using 5-fold cross-validation, following the setup of \citet{wachsmuth-alshomary-2022-mama}. We then evaluate large proprietary LLMs—\lm{GPT-4o} \citep{gpt4} and two versions of \lm{Gemini 1.5} \citep{gemini}—using few-shot prompting. Finally, to directly compare the effect of fine-tuning on a modern architecture, we fine-tune \lm{GPT-4o-mini} using the same 5-fold cross-validation setup. Further details on model implementation are in Appendix~\ref{app:models}.

\textbf{Prompting for Structured Prediction.}
For the LLM experiments, we test three different prompting paradigms designed to elicit structured, span-level output:
\begin{itemize}
    \item \textbf{JSON}: Requesting a list of JSON objects, each containing a text span and its predicted label \citep{wu-2024-learning-to-extract-structured-entities}.
    \item \textbf{TANL}: An inline tagging format where predictions are structured as \texttt{[span | label]} directly in the text \citep{paolini-2021-structured-prediction-as-translation}.
    \item \textbf{GoLLIE}: Generating Python-like code where spans and labels are assigned to data structures, guided by a schema provided in the prompt \citep{sainz-2024-gollie}.
\end{itemize}
\lm{GPT-4o-mini} is fine-tuned with 5-fold cross-validation (same setup as \lm{BERT}, but with DPO, learning rate multiplier = $1.8$, epochs = $3$). 
Details and examples of these prompts are provided in Appendix~\ref{app:prompts}.



\subsection{Results and discussion}

\label{sec:results_rewired}




Our experimental results, presented in Table~\ref{tab:seq_results}, reveal several key insights into modeling domain-specific discourse acts.

\textbf{Few-shot LLMs struggle with structured output and complex acts.}
Without fine-tuning, LLMs find the task challenging. The \textbf{JSON} format proved particularly unreliable, frequently producing malformed output that complicated post-processing and led to poor performance. While providing few-shot examples improved output consistency, the overall results remained low. Switching to more constrained output formats like \textbf{TANL} and \textbf{GoLLIE} yielded substantial improvements, nearly doubling the Macro-$F_1$ for \lm{GPT-4o}. This highlights that for complex structured prediction, the choice of output format is critical. Even so, performance varied substantially across models and prompting styles, with TANL emerging as the best few-shot approach, but still lagging behind the exceptional performance of fine-tuning.

\textbf{Fine-tuning is essential for high performance.}
The fine-tuned \lm{BERT} baseline handily outperformed all few-shot LLM configurations across nearly every teaching act. This underscores the difficulty of the task and suggests that successfully capturing nuanced, domain-specific discourse phenomena requires task-specific adaptation.

This conclusion is further reinforced by our final experiment: the fine-tuned \lm{GPT-4o-mini} achieves near-perfect scores, with a Macro-$F_1$ of up to 99.04\% and a span alignment of 95.49\%. 
Rather than suggesting the task is trivial, this result demonstrates that \textbf{fine-tuning is the most effective and reliable paradigm for this task}. It shows that even a smaller, more efficient LLM, when properly adapted with in-domain data, can master the complexities of annotating pedagogical discourse. For practitioners seeking to automate the analysis of such dialogues, we strongly recommend fine-tuning over few-shot prompting.


\section{The IXQuisite test suite}
\label{sec:ixquisite}


%-------------

\begin{table*}[t!]
%\begin{sidewaystable}
\begin{center}
    \centering
    %\renewcommand*{\arraystretch}{0.85}
    \resizebox{\textwidth}{!}{%
    \begin{tabular}{p{0.75cm} p{3.25cm}  p{8.25cm}  p{5.5cm}  p{2.25cm}}
        \toprule
        \multicolumn{5}{c}{\sys{IXQuisite}: Function metrics} \\ 
        \midrule
        \textbf{Abbr.} & \textbf{Category} & \textbf{Description} & \textbf{Origin} & \textbf{Static metric} \\
        \cmidrule(r){1-1}\cmidrule(lr){2-2}\cmidrule(l){3-3}\cmidrule(l){4-4}\cmidrule(l){5-5}
        %\hline
        
        PK & Check for prior knowledge 
            & The teacher inquires the student about prior knowledge, background, or what their interests might be
            & \citet{kulgemeyer-schecker-2009-kommunikationskompetenz},  \citet{leinhardt-steele-2005-seeing-complexity}
            & T01 \\ % check semantic similarity between student's answer and future utterances in dialogue >> incorporation of interests 
            
        \midrule
        MI & Mindfulness of common misconceptions 
            & The teacher addresses common misconceptions 
            & \citet{wittwer-2010-effective},  \citet{andrews-2011-active-learning} 
            & T04 \\
        
        \midrule
        RE & Rule-example structure 
            & The teacher states the abstract form of the concept being taught. Then, the teacher gives some examples to assist in understanding
            & \citet{tomlinson-hunt-1971-differential} 
            & T05 $\rightarrow$ T03 \\

        \midrule
        ER & Example-rule structure
            & For procedural knowledge, the teacher first provides examples and then derives the general rule from them
            & \citet{champagne-1982-cognitive-research}
            & T03 $\rightarrow$ T05 \\
        
        \midrule
        EA & Example/Analogy connection 
            & The teacher explains how parts of the analogy/example relate to the concept being explored
            & \citet{ogborn-1996-explaining}, \citet{valle-callanan-2006-similarity}
            & T06 \\
        
        \midrule
        UN & \textls[-60]{Check for understanding}
            & The teacher tests the understanding of the student 
            & \citet{webb-1995-constructive} 
            & T08%; E01 
            \\
        
        \bottomrule
    \end{tabular}
    }
    \caption{Explanation and teaching acts-related measures in \sys{IXQuisite} for instructional explanation quality based on occurrences of classes from our annotation schema. The right arrow between two teaching acts in static metrics refers to passages where two different acts directly follow one another in this exact sequence.
    }
    \label{tab:testsuite}
\end{center}
\end{table*}

\begin{table*}
    \centering
    %\renewcommand*{\arraystretch}{0.85}
    \resizebox{\textwidth}{!}{%
    \begin{tabular}{p{0.75cm} p{2.25cm}  p{6.25cm}  p{3.5cm}  p{4.25cm} }
     \toprule
    \multicolumn{5}{c}{\sys{IXQuisite}: Form metrics} \\ 
    \midrule
    \small \textbf{Abbr.} & \textbf{Category} & \small \textbf{Description} & \small \textbf{Origin} & \small \textbf{Static metric} \\
    \cmidrule(r){1-1}\cmidrule(lr){2-2}\cmidrule(l){3-3}\cmidrule(l){4-4}\cmidrule(l){5-5}
    
    ME & \small Minimal explanations 
        & \small Low cognitive load, e.g. avoid redundancies (verbosity) such as introducing named entities 
        & \small \citet{black-1986-minimal-instruction} 
        & \small Frequency of named entities \\
    
    \midrule
    LC & \small Lexical complexity 
        & \small The level of difficulty associated with any given word form by a particular individual or group 
        & \small \citet{kim-2016-simplescience} 
        & \small Frequency of difficult words \\
    
    \midrule
    SD & \small Synonym density 
        & \small Children are proven better aligned with consistent terminology; experts allow more synonyms 
        & \small \citet{wittwer-ihme-2014-reading-skill} 
        & \small Frequency of synonyms for the $n$ terms most connected to the topic \\ 

    \midrule
    TM & \small Correlation to teaching model 
        & \small Correlation of teaching act order to prescribed teaching models %(problem solving, concept building, unified choreographies) 
        & \small \citet{oser-baeriswyl-2002-aera}, \citet{krabbe-2015-lernprozessorientierte}  
        & \small Edit distance between T01-T08 (asc.) and actual occurrences \\

    \midrule
    AD & \small Adaptation 
        & \small The teacher incorporates prior knowledge, misconceptions and interests and uses analogies 
        & \small \citet{wittwer-2010-effective} 
        & \small Inverse frequency of synonyms in the text \\ 

    \midrule
    RL & \small Readability level 
        & \small \textls[-40]{Indicator of how difficult a passage is to understand}
        & \small \citet{crossley-2017-predicting-text-comprehension} 
        & \small Flesch-Kincaid Grade level \\ 

    \midrule
    CO & \small Coherence 
        & \small How sentences relate to each other to create a logical and meaningful flow for the reader or listener
        & \small \citet{lehman-schraw-2002-effects}, \citet{duffy-1986-conceptualizing} 
        & \small Frequency of conjunctions and linking language \\ 
    
    
    \bottomrule
    \end{tabular}
    }
    \caption{Categories for instructional explanation quality and associated numerical measures in \sys{IXQuisite}. 
    }
    \label{tab:testsuite_numerical}
\end{table*}

%-------------

While our experiments show that teaching acts can be reliably annotated with fine-tuning, the presence of individual acts does not guarantee a high-quality explanation. A good instructional dialogue must orchestrate these acts into a coherent and effective structure. Evaluating this holistic quality is challenging for standard automated metrics, which often fail to capture the nuances of conversational flow and engagement \citep{DBLP:deriu-2021}.

To address this, we employ and extend \sys{IXQuisite}, a test suite of quality metrics for instructional explanations grounded in didactic research \citep{ours2024WiP}. The metrics are divided into two categories:
\begin{itemize}
    \item \textbf{Function metrics} assess the pedagogical structure of the dialogue. They are calculated based on the presence, frequency, or sequence of the teaching acts annotated in our dataset (e.g., measuring if a \textit{Rule} is followed by an \textit{Example}). These are detailed in Table~\ref{tab:testsuite}.
    \item \textbf{Form metrics} evaluate linguistic and stylistic features of the explanation that impact cognitive load and readability, such as lexical complexity or coherence. These are detailed in Table~\ref{tab:testsuite_numerical}.
\end{itemize}
We investigate this suite through three lenses: human validation, traditional static evaluation, and a novel prompt-based LLM evaluation.


\subsection{Human Validation of Metrics}

Before applying the metrics, we first sought to validate their relevance with our domain experts. As a follow-up task to the span annotation, we asked our four annotators to assess each of the 13 metrics for every dialogue, with reference to the descriptions provided in \autoref{tab:testsuite} and \autoref{tab:testsuite_numerical}. Using a 3-point Likert scale, they rated the \textbf{presence} of each metric and its \textbf{contribution} to the explanation's quality for the given knowledge level. % This step grounds our framework in the judgment of education professionals.
This step anchors our framework in the expertise and judgment of education professionals.

The results of the annotators' assessment of metric presence are shown in Figure~\ref{fig:annotators}, based on the normalized average of the ratings. The analysis reveals a strong alignment between the perceived presence of most function metrics and the explainee's knowledge level. For instance, \textit{Check for prior knowledge (PK)}, \textit{Rule-example (RE)}, and \textit{Example-rule (ER)} structures are rated as more present in dialogues with less expert explainees. In contrast, form-based metrics like \textit{Adaptation (AD)}, \textit{Readability (RL)}, and \textit{Coherence (CO)} are consistently rated as important across all levels, indicating that they serve as foundational elements of any strong explanation.


\begin{figure*}[t]
    
    \centering
    
    \begin{subfigure}{\linewidth}
        \includegraphics[width=\textwidth]{figures/ixquisite/presence_features.png}
        \caption{Annotators assessment on presence of each metric in \sys{IXQuisite} for in each level.}
        \label{fig:annotators}
    \end{subfigure}

    \vspace*{2em}
    
    \begin{subfigure}{\linewidth}
        \includegraphics[width=\textwidth]{figures/ixquisite/202502_box_static.png}
        \caption{\sys{IXQuisite} metrics: Static evaluation of our dataset.}
        \label{fig:static}
    \end{subfigure}
    
    \vspace*{2em}
    
    \begin{subfigure}{\linewidth}
        \includegraphics[width=\textwidth]{figures/ixquisite/202502_box_promptbased.png}
        \caption{\sys{IXQuisite} function-related metrics: prompt-based evaluation of the five levels in the dataset.}
        \label{fig:prompt}
    \end{subfigure}

    \vspace*{2em}
    
    \caption{\sys{IXQuisite} results.}
    \label{fig:ixquisite_results}
\end{figure*}






\subsection{Static vs. Prompt-based Evaluation}

We then evaluated the dialogues automatically using two different methods to see how well they could replicate the nuanced judgments of our human experts.

\textbf{Static Evaluation.} Our first approach uses "static" or rule-based calculations. For function metrics, this involves counting the tokens in the corresponding gold-standard teaching act spans (e.g., T01 for PK). For form metrics, we use standard linguistic feature calculations like the Flesch-Kincaid grade level for readability (RL). The results, shown in Figure~\ref{fig:static}, reveal a key limitation of this approach. While some form-based metrics (e.g., LC, SD, RL) show a clear trend across knowledge levels, the function-based metrics appear noisy and fail to show a consistent correlation. The static method seems too superficial to capture the functional quality of the instructional discourse.

\textbf{Prompt-based Evaluation.} To overcome these limitations, we developed a "prompt-based" evaluation framework inspired by \citet{rooein-etal-2024-beyond}. Instead of relying on simple counts, we leverage an LLM's reasoning capabilities. We prompted \lm{GPT-4o} with the full dialogue and asked it to rate each metric on a scale from 0 to 10 (e.g., "On a scale from 0 to 10, how well does the explainer check for understanding?").

The results, shown in Figure~\ref{fig:prompt} for the function metrics, are strikingly different from the static evaluation. The prompt-based scores align remarkably well with the human judgments from our validation step. There is a clear, graded relationship between the metric scores and the explainees' knowledge levels, especially for \textit{PK}, \textit{EA}, \textit{RE}, and \textit{ER}. This demonstrates that an LLM-based "judge" is far more capable of capturing the nuanced, functional aspects of instructional quality than simple static heuristics. For form-related metrics (Appendix \ref{app:ixquisite}), the prompt-based scores were high and stable across levels, confirming the human assessment that these are universally important. This suggests a hybrid approach for future work: static metrics may suffice for form, but evaluating the functional discourse structure of explanations requires the inferential power of prompt-based LLM evaluation.



\section{Discussion}
\label{sec:discussion}

Our findings offer several key implications for the fields of computational discourse analysis, educational technology, as well as NLP practices such as fine-tuning and automated evaluation.

% Our findings carry several important implications for computational discourse analysis, educational technology, and the practice of automated evaluation.


\paragraph{Implications for Discourse Analysis.}
Our work treats teaching as a complex, goal-oriented discourse phenomenon. By creating a fine-grained, span-level annotation scheme for pedagogical strategies, we provide a new lens for analyzing dialogue structure. The \textit{teaching acts} in \data{ReWIRED} can be viewed as a domain-specific set of discourse relations that govern how instructional conversations are built. Our dataset, with its unique five-level structure of explainee expertise, offers a controlled environment to study \textbf{audience adaptation} at a granular level. Future work can analyze the typical sequences and flows of these acts to uncover the ``discourse grammar'' of effective explanation.

\paragraph{Implications for Educational Technology and XAI.}
Our contributions provide a direct pathway toward more effective and pedagogically-aware AI systems, e.g.:
\begin{itemize}
    \item \textbf{AI Tutors:} An automated tutor could use our models to self-assess its own dialogue strategies in real-time \cite{wang-etal-2024-tutor}. If it produces too many `Knowledge Statement`s without a corresponding `Check for Understanding`, it could adapt its strategy to be more interactive. The \sys{IXQuisite} metrics could serve as a reward function for RL-based dialogue managers.
    \item \textbf{Tools for Human Educators:} Our framework could power tools that provide feedback to trainee teachers. By analyzing a transcript of a practice lesson, such a tool could highlight strengths (e.g., ``Great use of analogy here!'') or suggest improvements (e.g., ``Consider first checking for prior knowledge.'').
    \item \textbf{Advancing Explainable AI (XAI):} True XAI should go beyond presenting information to actively fostering human understanding. Our work offers a blueprint for pedagogically sound explanatory dialogue, shifting the focus from producing static explanations to enabling interactive and adaptive exchanges \cite{feldhus-2023-interrolang}.
\end{itemize}

\paragraph{Methodological Takeaways for NLP.}
Finally, our experiments offer two clear methodological lessons. First, for complex, domain-specific structured prediction tasks like identifying teaching acts, \textbf{in-domain fine-tuning is critical}. It vastly outperforms even the most capable few-shot LLMs, demonstrating that task-specific adaptation remains essential for high-fidelity discourse analysis. The exceptional performance can be explained with the fact that the ground truth is a consolidation from multiple annotators. The model is exposed to many examples of the already consolidated teaching acts, which is in contrast to how human annotators are typically introduced to labeling efforts, namely with explicit instructions and few-shot examples. This is reinforced by our observation that models exposed only to few-shot examples without fine-tuning performed substantially worse.

Second, our work combines the strengths of two approaches: from the \textbf{LLM-as-a-judge} paradigm and static metrics. 
Our analyses suggest that for evaluating nuanced pragmatic qualities of discourse, leveraging the contextual reasoning of LLMs is a more promising path forward than relying on surface-level heuristics.
However, it should be taken into consideration that, depending on the task, judge models' agreement with human annotators can vary across datasets and domains \cite{bavaresco-2025-llms-instead-of-human-judges}. In future work, applying the same principles across multiple LLMs may yield different outcomes.


\section{Conclusion}

In this paper, we introduced \data{ReWIRED}, a dataset of instructional dialogues significantly extending prior work with expert, span-level annotations of teaching acts. We demonstrated that while automatically labeling these acts is challenging for few-shot LLMs, fine-tuning achieves excellent performance with both smaller and larger models, establishing a reliable methodology for analyzing pedagogical discourse at scale. Furthermore, we proposed a framework for evaluating the quality of these explanations, showing that while static metrics are limited for certain dimensions, a prompt-based approach using LLMs as evaluators more effectively captures how instructional strategies are adapted to explainees' knowledge levels.

Our contributions provide a crucial bridge between pedagogical theory and computational discourse analysis. The dataset and validated evaluation suite offer a concrete methodology for building and assessing systems that engage in instructional dialogue. This paves the way for a new generation of applications, from more adaptive and effective automated tutors to AI-powered tools that provide feedback to human educators. Ultimately, by modeling the structure of effective teaching, our work helps advance the broader goal of creating AI systems that can not only explain, but explain well.



\section*{Limitations}

We acknowledge that, despite our annotators' high expertise in the field of education, some teaching acts seem not as easily distinguishable as the other act dimensions, resulting in a relatively low inter-annotator agreement. However, the single aggregation-based Fleiss' $\kappa$ score might be too superficial to capture the complexity behind. Ultimately, the annotation variations also convey the subjectivity of teaching-related explanations, following the idea that human label variation should be encouraged \cite{plank-2022-problem}. 

% portion of test suite relying on annotations
Further limitations include that a portion of the test suite relies on human annotation, which may introduce inconsistencies. Replicating or extending the test suite might be difficult without a reliable teaching act prediction model.
% multimodality
Also, the dataset we present is extracted from videos—audio and visual elements not present in the transcription. The efficacy of our approach may vary depending on the complexity and diversity of the multimodal inputs, if present.
% poor generalizability
Last but not least, the generalizability of our findings may be constrained by the narrow domain of dialogues examined, limiting extrapolation to broader conversational contexts.



\section*{Ethical statement}

We do not see immediate ethical concerns regarding research and development. The data included in the corpus are readily available from WIRED Web resources. Following the ACM Code of Ethics (1.2, 1.6), all participants consented to be recorded as far as perceivable from the WIRED web resources, which are free to use for research purposes. 
The four annotators in our study were recruited over online platforms (LinkedIn, university forum). The annotation of each dialogue took an annotator an average of 10 minutes; depending on their workload, the annotation duration was between 12 and 20 hours. In our view, the provided prediction models target dimensions of dialogue turns that are not prone to misuse for ethically doubtful applications.



\section*{Acknowledgements}
We thank Christoph Kulgemeyer and David Buschhüter for their input and advice on didactics, the annotators and data curators, including Kevin Kornetzki-Lucza, Iris Vassiliadi, Marie-Theres Ebner, Sören Wulff, Lavanya Govindaraju, Elif Kara, Christopher Ebert, and Emiliano Valdes Menchaca.
Finally, we thank Qianli Wang for providing preprocessing code at an early stage of the project and David Schlangen and Tatiana Anikina for their feedback on earlier drafts of the paper.
This work was funded by the German Federal Ministry of Research, Technology and Space (BMFTR) under grant numbers 03RU2U151C (news-polygraph), 01IW23002 (No-IDLE), and 01IW24006 (NoIDLEChatGPT), as well as by the Endowed Chair of Applied AI at the University of Oldenburg,
and partially funded by the Deutsche Forschungsgemeinschaft (DFG, German Research Foundation): TRR 318/1 2021 - 438445824.




\bibliography{custom}

%-------------------------------------------

\appendix
\section*{Appendix}



% ----------------
\begin{table*}[t!]
    \centering
    %\renewcommand*{\arraystretch}{0.85}
    \resizebox{\textwidth}{!}{%
    \begin{tabular}{p{3cm}  p{1.5cm}  p{6cm}  p{2.5cm} p{2.5cm} p{2cm} }
        \toprule
        \textbf{Model name} & \textbf{\#Params} & \textbf{URL} & \textbf{Training times} & \textbf{Inference times} \\
        \midrule
        \lm{BERT}
            & 110M
            & \footnotesize \url{https://huggingface.co/bert-base-uncased}
            & 13 hours
            & <1 hour
            \\
        \lm{GPT-4o-mini (fine-tuned)}
            & ?
            & \footnotesize \url{https://platform.openai.com/docs/guides/fine-tuning}
            & 6 hours
            & 6 hours
            \\
        \lm{GPT-4o}
            & ?
            & \footnotesize \url{https://platform.openai.com/docs/api-reference/chat}
            & n.a.
            & 9 hours
            \\
        \lm{Gemini 1.5}
            & ?
            & \footnotesize \url{https://ai.google.dev/gemini-api/docs}
            & n.a.
            & 11 hours
            \\
        \bottomrule
    \end{tabular}
    }
    \caption{
        Language models with parameter counts, training times, inference times, and API costs.
    }
    \label{tab:models}
\end{table*}
% ----------------



\section{Examples for acts}
\label{app:act_examples}

Figure~\ref{fig:teaching_act_examples} shows examples from \data{ReWIRED} for each of the acts as provided to the annotators.


\section{Label distributions}

Figure~\ref{fig:acts_per_turn} shows the number of distinct acts per dialogue turn as per annotated.





\section{Models}
\label{app:models}

Table~\ref{tab:models} lists how the models in \S \ref{sec:experiments} were employed.
We used the following GPUs: A100, RTXA6000, RTX3080.
For the BERT fine-tuning, we reinitialized the BERT model for token classification at the start of every fold ($k=5$) and used a batch size of 4, an AdamW optimizer with a learning rate of $5 * 10^{-6}$, epsilon of $1 * 10^{-8}$, and warmup.




\section{Prompt design}
\label{app:prompts}

Figure~\ref{app:prompt_clf} and Figure~\ref{app:prompt_sl} depict the prompts used with LLMs such as \lm{GPT-4o} to produce the predictions whose evaluation is shown in Table~\ref{tab:seq_results}.
For few-shot demonstrations, we first presented the three preceding turns of the same dialogue (or from the end of last dialogue if the turn in question is at the start of a dialogue) and their corresponding gold spans (in the format required by the respective prompting paradigm) just as we elicit it from the model in the zero-shot setup.
Figure~\ref{app:result_gollie} and Figure~\ref{app:result_tanl} show the results from GoLLIE and TANL prompts for \lm{Gemini 1.5 Pro} and \lm{GPT-4o}, respectively.




\section{Annotation instructions}
\label{app:instructions}

To annotators, we provided examples from Appendix~\ref{app:act_examples} as well as further delineations of the acts with examples and descriptions of how to differentiate between them. %(Appendix~\ref{app:schema}).
We also provided a screencast with instructions on how to use \sys{Label Studio} and walk-through examples for each act.
The introductory text shown to all annotators before watching the recording and accessing \sys{Label Studio} is the following (unformatted version):

\begin{myquotebox}
\small
Your objective is annotating linguistic information about the multi-layered objectives each person performs when communicating. The dataset is comprised of transcribed conversations in which an expert in a field explains some concept to multiple people at varying levels of education: child, teenager, undergraduate, graduate and expert.

Your task as an annotator will be, given a transcript of one of these conversations, to use a highlighting tool to mark which ``acts'' are present in different parts of the text. These acts highlight some unspoken objectives present in the text. For example, the text ``Do you understand that?'' could be said to have both an objective of asking a yes/no question and checking for understanding.

Some of these will be straightforward to label and say ``that is clearly the intention behind that sentence'', while some will be a bit more complicated. We often have many intentions behind what we say, and we account for that by letting you tag any segment of text with as many labels as you see fit, even none at all.

Your annotation task is about labeling the aforementioned objectives from the perspective of Teaching Acts, which focus on conversation mechanics in terms of lesson planning and didactics.
\end{myquotebox}



\begin{figure*}[ht]
    \centering
    
    \begin{subfigure}[b]{0.49\textwidth}
        \includegraphics[width=\textwidth]{figures/acts/T01_example.jpg}
        \caption{T01: Assess Prior Knowledge}
    \end{subfigure}
    \hfill
    \begin{subfigure}[b]{0.49\textwidth}
        \includegraphics[width=\textwidth]{figures/acts/T02_example.jpg}
        \caption{T02: Lesson Proposal}
    \end{subfigure}

    \par\noindent\rule{\textwidth}{0.4pt}

    \begin{subfigure}[b]{0.49\textwidth}
        \includegraphics[width=\textwidth]{figures/acts/T03_example.jpg}
        \caption{T03: Active Experience}
    \end{subfigure}
    \hfill
    \begin{subfigure}[b]{0.49\textwidth}
        \includegraphics[width=\textwidth]{figures/acts/T04_example.jpg}
        \caption{T04: Reflection}
    \end{subfigure}

    \par\noindent\rule{\textwidth}{0.4pt}

    \begin{subfigure}[b]{0.49\textwidth}
        \includegraphics[width=\textwidth]{figures/acts/T05_example.jpg}
        \caption{T05: Knowledge Statement}
    \end{subfigure}
    \hfill
    \begin{subfigure}[b]{0.49\textwidth}
        \includegraphics[width=\textwidth]{figures/acts/T06_example.jpg}
        \caption{T06: Comparison}
    \end{subfigure}

    \par\noindent\rule{\textwidth}{0.4pt}

    \begin{subfigure}[b]{0.49\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figures/example_T07.png}
        \caption{T07: Generalization}
    \end{subfigure}
    \hfill
    \begin{subfigure}[b]{0.49\textwidth}
        \centering
        \includegraphics[width=\textwidth]{figures/example_T05_T08.png}
        \caption{T08: Test Understanding (vermilion) and T05: Knowledge Statement (blue)}
    \end{subfigure}

    \par\noindent\rule{\textwidth}{0.4pt}

    \begin{subfigure}[b]{0.49\textwidth}
        \centering
        \includegraphics[width=.7\textwidth]{figures/acts/T09_example.jpg}
        \caption{T09: Engagement Management}
    \end{subfigure}

    \par\noindent\rule{\textwidth}{0.4pt}
    
    \caption{Examples for teaching acts T01-T09.
    }
    \label{fig:teaching_act_examples}
\end{figure*}

% --------------------------------------------------------



\clearpage


\begin{figure*}[ht]
    \centering
    
    \includegraphics[width=\textwidth]{figures/ixquisite/contribution_features.png} % Replace with your image file
    
    \caption{Annotators assessment on contribution of each metric present in \sys{IXQuisite} for each level.}
    \label{fig:contribution}
\end{figure*}


\begin{figure*}[ht]
    \centering
    \includegraphics[width=\textwidth]{figures/ixquisite/202502_box_promptbased_num.png} % Replace with your image file
    \caption{\sys{IXQuisite} form metrics: prompt-based evaluation of the five levels in the dataset.}
    \label{fig:prompt:num}
\end{figure*}


\begin{figure}[t]
    \centering
    
    \includegraphics[width=\columnwidth]{figures/unique_labels_per_turn.png}
    
    \caption{Number of unique teaching acts per turn in \data{ReWIRED}.
    The bar chart reveals that more than half of all dialogue turns in \data{ReWIRED} contain more than one distinct teaching act.
    }
    \label{fig:acts_per_turn}
\end{figure}




% --------------------------------------------------------




%\clearpage

\begin{figure*}[!ht]
\centering

\begin{mintedbox}{python}
# Example label mapping (dialogue acts)
ReWIRED_ta_str_2_int = {
    'T01 - Assess Prior Knowledge': 1,
    'T02 - Lesson Proposal': 2,
    'T03 - Active Experience': 3,
    'T04 - Reflection': 4,
    'T05 - Knowledge Statement': 5,
    'T06 - Comparison': 6,
    'T07 - Generalization': 7,
    'T08 - Test Understanding': 8,
    'T09 - Engagement Management': 9,
    'T10 - Other Act': 0
}
label_schema = ("The label schema consists of the following 10 classes:\n* " + "\n* ".join(list(ReWIRED_ta_str_2_int.keys())) + "\n")
\end{mintedbox}
\caption{Label schema.}
\label{app:prompt_clf}
\end{figure*}


\begin{figure*}[!ht]
\centering
\begin{mintedbox}{python}
system_prompt = (f"You are an expert annotator. ")
read_instruction = (f"Here is one turn from a dialogue between an explainer and a {student_role} on the topic of {topic}:\n{turn_text}\n")

task_instruction_JSON = ("Please extract the spans from the turn and assign a label to each of the spans. It is possible that the whole turn is just one span, because the act applies to its entirety. Please present your predictions in a JSON format like this: {\n\t{\n\t\t'Span': '...', \n\t\t'Predicted label': '...' \n\t},\n}\n")
task_instruction_TANL = ("Please annotate the spans in the turn by marking them inline using the format [ span | label ]. It is possible that the whole turn is just one span if the act applies to its entirety.")
task_instruction_GoLLIE = ("Task: Annotate the following text with {TASK_NAME[task]} labels.\n\n'docstring += 'Guidelines:\n'docstring += '- Identify spans in the text that correspond to the following acts.\n'docstring += '- The act classes are defined below.")

entire_input = system_prompt + read_instruction + label_schema + task_instruction
\end{mintedbox}
\caption{Simplified version of the Python code showing the \underline{span-labeling} task prompt for \data{ReWIRED}.}
\label{app:prompt_sl}
\end{figure*}

\begin{figure*}[t]
\centering
\begin{mintedbox}{python}
Text = "Explainer: \"So machine learning is a way that we teach computers to learn things about the world by looking at patterns and looking at examples of things. So can I show you an example of how a machine might learn something?\""

labels = [
    {'span': "So machine learning is a way that we teach computers to learn things about the world by looking at patterns and looking at examples of things.", 'label': 'T05___Knowledge_Statement'},
    {'span': "So can I show you an example of how a machine might learn something?", 'label': 'T02___Lesson_Proposal'},
]
\end{mintedbox}
\caption{Example for a result from a GoLLIE prompt with \lm{Gemini 1.5 Pro}.}
\label{app:result_gollie}
\end{figure*}

\begin{figure*}[t]
\centering
\begin{mintedbox}{python}
"Explainer: ""It's a lot of practice and analysis. [Really, an advanced chess player was not born an advanced chess player. They have probably hundreds, if not thousands of more games in their mind, in their past, in their history that they've analyzed, that they've studied. It's like any athlete, you know? | T07 - Generalization] [I put my weight on this foot, and so I wasn't able to hit the shot back that well. So the next time that that happens, I'm gonna be more prepared. | T06 - Comparison]"""
\end{mintedbox}
\caption{Example for a result from a TANL prompt with \lm{GPT-4o}.}
\label{app:result_tanl}
\end{figure*}


\section{IXQuisite: additional information} 
\label{app:ixquisite}

\subsection{Annotator's assessment of contribution of metrics in each level}

Besides validating the presence of each \sys{IXQuisite} metric in every dialogue, annotators were additionally asked to assess their importance/contribution, especially in regards to the level of knowledge of the explainee. Figure \ref{fig:contribution} shows the annotator's assessment of the importance/contribution of each metric at each level.


\subsection{Form metrics: prompt-based evaluation}

Figure~\ref{fig:prompt:num} presents the results of the prompt-based evaluation of the form metrics in the dataset. The results do not exhibit a clear correlation with the five levels, predominantly falling within the range of 0.8 to 0.9. This may be attributed to the formulation of the prompts.0.9. This might be related to the way the prompts were formulated.


\subsection{Prompt-based metric questions}

Table \ref{tab:promptbased} shows the metrics formulated as questions for prompt-based evaluation of the explanatory dialogues in the ReWIRED dataset according to the \sys{IXQuisite} test suite.

\begin{table}[t!]
    \centering
    \resizebox{\columnwidth}{!}{
    \begin{tabular}{p{1cm} p{8cm}}
    \toprule
    Abbr. & On a scale from 0 to 10... \\
    \midrule
    PK & ... how well does the explainer   inquire about prior knowledge? \\
    MI & ... how well does the explainer deal with common   misconceptions? \\
    RE & ... how well does the explainer state the abstract form of a   statement and then some example to assist understanding? \\
    ER & ... how well does the explainer provide examples prior to   deriving a rule? \\
    EA & ... how well does the explainer explain ... how parts of the   analogy/example relate to the concept being explored? \\
    UN & ... how well does the explainer check the understanding of the   student? \\
     &  \\
    ME & ... how appropriate is the cognitive load for the explainee's   level? \\
    LC & ... how appropriate is the lexical complexity for the   explainee's level? \\
    SD & ... how appropriate is the amount of synonyms and technical   language used for the explainee's level? \\
    AD & ... how well-adapted is the content of the dialogue to the   explainee? \\
    RG & ... how appropriate is the readability level for the   explainee's level? \\
    CO & ... how appropriate is the number of conjuction and   subordination for the explainee's level? \\
    TM & ... how coherent is the text for the explainee's level?" \\ \bottomrule
    \end{tabular}
    }
    \caption{\sys{IXQuisite} metrics formulated as questions for prompt-based dialogue evaluation.}
    \label{tab:promptbased}
\end{table}


\end{document}
