% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Remove the "review" option to generate the final version.
\usepackage[]{acl}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}

\usepackage{graphicx}
\usepackage{amsfonts}
\usepackage{amsmath}
\graphicspath{{images/}}

% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\title{GroundHog: Dialogue Generation using Multi-Grained Linguistic Input}

% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
%         Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
%         Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a seperate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \AND
%         Author 2 \\ Address line \\ ... \\ Address line \And
%         Author 3 \\ Address line \\ ... \\ Address line}

\author{Alexander Chernyavskiy$^{1}$, Lidiia Ostyakova$^{1,2}$, and Dmitry Ilvovsky$^{1}$\\
  $^{1}$ HSE University, Russia\\
  $^{2}$ Moscow Institute of Physics and Technology, Russia \\
  \texttt{alschernyavskiy@gmail.com, ostyakova.ln@gmail.com}\\ 
  \texttt{dilvovsky@hse.ru} \\ }

\begin{document}
\maketitle
\begin{abstract}
Recent language models have significantly boosted conversational AI by enabling fast and cost-effective response generation in dialogue systems. However, dialogue systems based on neural generative approaches often lack truthfulness, reliability, and the ability to analyze the dialogue flow needed for smooth and consistent conversations with users. To address these issues, we introduce GroundHog, a modified BART architecture, to capture long multi-grained inputs gathered from various factual and linguistic sources, such as Abstract Meaning Representation, discourse relations, sentiment, and grounding information. For experiments, we present an automatically collected dataset from Reddit that includes multi-party conversations devoted to movies and TV series. The evaluation encompasses both automatic evaluation metrics and human evaluation. The obtained results demonstrate that using several linguistic inputs has the potential to enhance dialogue consistency, meaningfulness, and overall generation quality, even for automatically annotated data. We also provide an analysis that highlights the importance of individual linguistic features in interpreting the observed enhancements.
\end{abstract}


\section{Introduction}\label{sec1}

Text generation methods, particularly for conversational systems, have become increasingly popular in recent years. The conversational systems play a crucial role in enhancing  the effectiveness of user-agent interactions \cite{DBLP:conf/aaai/YoungCCZBH18,DBLP:conf/cikm/GuLL19,DBLP:conf/emnlp/LeHSYBZY19}. Dialogue systems are used for human-machine conversations on various topics. Some systems are built as question-answering systems or personal assistants, focusing on specific domains or general inquiries.

Despite showing impressive response generation capabilities, language models, even ones like \mbox{GPT-4}, have shortcomings in terms of truthfulness \citep{openai2023gpt4}. Consequently, researchers are exploring  methods to combine generative and extractive approaches in order to make the responses of dialogue systems more logical and reliable. Here, the primary objective is to incorporate external knowledge, resources, or databases into the response generation process. The previous studies have demonstrated a substantial enhancement in the quality of generation by incorporating grounding, which improves the factual accuracy of the responses~\cite{feng2020doc2dial}. 
Grounding input is commonly integrated into 
dialogue generation models along with the context of a particular utterance~\cite{zhao2020knowledge} or a preceding part of the dialogue that represents the conversational history~\cite{rashkin2021increasing}.

Furthermore, previous works have explored leveraging grounding in combination with other features, including commonsense and named entities~\cite{varshney-etal-2022-commonsense, wu2022section}, dialogue acts~\cite{hedayatnia2020policy}, topic shifts~\cite{wu2021augmenting}, discourse annotation~\cite{khalid2020discourse}, to improve dialogue generation. Despite the fact that additional linguistic features are frequently used to improve the consistency of generated dialogues~\cite{DBLP:journals/corr/JiHE16,DBLP:journals/corr/abs-1907-09527}, previous studies focused on individual and superficial examination of linguistic features. In our research, we conducted a more comprehensive analysis, evaluating the relative significance of each of them and the overall contribution.

We primarily investigate the impact of various linguistic features on response generation in a multi-grained input framework. Specifically, we analyze the effects of semantic relations derived from Abstract Meaning Representation (AMR) \cite{DBLP:conf/acllaw/BanarescuBCGGHK13}, dialogue acts extracted from dialogue discourse trees \cite{DBLP:conf/iwcs/StoneSL13,zhang2017characterizing}, and utterance-based sentiment representation.

Experiments on response generation are generally conducted using open-source sequence-to-sequence models~\cite{JMLR:v21:20-074, rashkin2021increasing}. Among these models, the BART architecture \citep{DBLP:conf/acl/LewisLGGMLSZ20} has gained significant popularity due to its state-of-the-art performance in various text generation tasks. Due to its efficiency in processing linearized inputs, it is often utilized in graph2text tasks \citep{DBLP:journals/corr/abs-2007-08426}. Moreover, this capability can be further extended to analyze conversation graphs. However, the length of input texts can often present challenges for Transformer-based models. In this study, we introduce GroundHog, an approach that uses multiple input encoders to preserve input information effectively.

Our contributions can be summarized as follows:
\begin{itemize}
\item We present a novel dataset consisting of open-domain conversations for dialogue system training. This dataset is augmented with linguistic features and grounding, enhancing its potential for training high-quality models.
\item We propose the use of grounding and linguistic features for response generation in dialogue systems. An ablation study is conducted to analyze their individual contributions.
\item A modification to the BART architecture is suggested to effectively capture long multi-grained inputs.
\item We perform an analysis to interpret the improvements and discuss our findings.
\end{itemize}


% It has already been shown that the incorporation of external information, known as grounding, considerably boosts generation quality by improving the factual component of the responses. At the same time, models without an extractive part, such as ChatGPT\footnote{\url{https://openai.com/blog/chatgpt/}}, have shortcomings related to truthfulness. Generated texts inconsistent with external knowledge, common sense, and dialogue history are considered to be hallucinations~\cite{DBLP:journals/corr/abs-2202-03629}.

\section{Dataset}

The most popular datasets, including open-domain conversations grounded in Wikipedia information, are \textit{Wizard of Wikipedia}~\cite{dinan2018wizard} and \textit{CMU DoG} \cite{DBLP:conf/emnlp/ZhouPB18}. To narrow the scope of this study and facilitate the language model training, CMU DoG was used as a starting point. This dataset contains 4112 grounded conversations devoted to the discussion of Wikipedia articles about popular movies.  To extend the dataset, we collected Reddit\footnote{\url{https://www.reddit.com/}} conversations on the same topic in English. Specifically, we parsed conversations from the 25 most popular subreddits related to films, series, and TV shows. These subreddits provided discussions that were tied to specific topics or comments. Additionally, we gathered comments that mentioned key phrases such as ``movie'' and ``film''.

The dataset preprocessing stage involved removing images, extra symbols, and emojis, as these were not considered in our research. In total, our collected dataset consists of approximately 62,500 multi-party dialogues, with an average of 5 turns per conversation (see Figure\ref{dataset}).
The length of extracted Reddit conversations is significantly shorter compared to CMU DoG dialogues, which have an average of 21.43 turns per conversation. 

\begin{figure}[t]
\centering
\includegraphics[width=\columnwidth, clip]{images/png2pdf_hist.pdf}
\caption{Distribution of dialogue lengths in collected dataset}
\label{dataset}
\end{figure}

The dataset contains conversations collected along with linguistic annotations, grounding, and meta information related to each extracted dialogue. Specifically, automatically retrieved linguistic features for each turn in the dataset are presented in the following format:
\begin{itemize}
    \item discourse annotation is represented as identifiers of connected turns with a discourse class describing the relation between them;  
    \item sentiment class of a turn, accompanied by its probability;
    \item AMR graph is provided in simplified form for each turn.
\end{itemize}

The process of annotating data is described in detail in Section \ref{repre}. Our final dataset is publicly available at the link: \url{https://huggingface.co/datasets/alexchern5757/groundhog_reddit}.

It should be emphasized that all datasets containing open-domain dialogues share the same limitations related to grounding. Casual conversations are distinguished by the absence of rigid topic boundaries, stylistic ambiguity, and a strong reliance on context. Evaluative information in these dialogues is often presented as facts, which can result in inaccurate grounding extraction.     

\section{Methods}\label{sec2}

\subsection{Dialogue Features} \label{repre}

In order to generate coherent and truthful responses, we incorporate grounding and several linguistic features that describe the current dialogue state as model inputs.

\paragraph{Discourse}
Discourse can be represented in various ways, with one of the most widely used approaches being Rhetorical Structure Theory (RST) for plain texts \cite{Mann1988RhetoricalST}. RST employs elementary discourse units to analyze the structure of the text, whereas in dialogue analysis, trees are constructed over utterances. Dialogue discourse graphs, as introduced by \citet{DBLP:conf/iwcs/StoneSL13}, extend the concept of standard dialogue graphs by including discourse labels for each utterance indicating the specific function or pragmatic purpose of the utterance (e.g., Disagreement, Appreciation, Question). An example is provided in  Appendix \ref{ling_repr}.

The application of discourse annotation, combined with grounding techniques, has demonstrated the potential for generating media dialogues that are more consistent and truthful \cite{majumder2020interview,chernyavskiy-ilvovsky-2023-transformer}. This integration of linguistic features and grounding methods has shown promise in enhancing the quality of such tasks.

%Similar idea was used in the storytelling task, where discourse planning improves coherence and overall text generation quality \cite{DBLP:journals/corr/JiHE16,DBLP:journals/corr/abs-1907-09527}. 


In order to achieve automatic discourse annotation, we implemented and trained the parser model suggested by \citet{shi2019deep}. The training process was started from scratch and utilized the Coarse Discourse Sequence Corpus (CDSC) \cite{zhang2017characterizing}, which is the largest manually annotated dataset of discourse acts in online discussions. 

\paragraph{Abstract Meaning Representation (AMR)}
 Abstract Meaning Representation is based on directed acyclic graphs and provides a structured semantic representation of language, including semantic role annotations consisting of arguments and values~\cite{DBLP:conf/acllaw/BanarescuBCGGHK13}. Given that incorporating AMR graphs enhances task-oriented dialogue generation~\cite{yang2023improving} and the promising prospects of integrating AMR with pragmatic intents ~\cite{bonial2020dialogue}, we use these graphs as one of the linguistically motivated inputs in the experiments.

In our dataset, an AMR graph was generated for each sentence within an utterance, and then these subgraphs were combined into a single graph. To reduce the complexity of the representation, we truncated vertices at a depth beyond a specified constant. A more detailed description of the AMR graphs is provided in Appendix \ref{ling_repr}. We adopt a similar method to linearize AMR graphs, as proposed by \citet{DBLP:journals/corr/abs-2007-08426}.


\paragraph{Sentiment}

The sentiment labels assigned to each utterance in the dataset indicate the polarity of the sentiment expressed, using a 3-point scale: Positive, Negative, or Neutral. The RoBERTa model, which was trained on tweets, was utilized for the corresponding labeling task \cite{DBLP:conf/emnlp/BarbieriCAN20}. To incorporate information about sentiment, special tokens were integrated into linearized representations of the dialogues.

\paragraph{Grounding}


\begin{figure*}[t]
\centering
\includegraphics[width=\textwidth, clip]{images/linearization_sent.drawio.pdf}
\caption{Example of the discursively annotated conversation linearization process. Firstly, all nodes are ordered temporally, forming a chain. Then, it is transformed into text representation using special tokens to display meta information: $\langle u_i \rangle$ are used for utterance ids; $\langle s_i \rangle$ are tokens for speaker ids (are signified by colors); $\langle \text{to:}u_i \rangle$ are used for addressees; and $\langle R_{ij} \rangle$ are used for relations. Additionally, an $\langle \text{init} \rangle$ token is introduced due to the fact that the first replica does not have an addressee.}
\label{fig_lin}
\end{figure*}

Grounding is an important aspect of model input as it serves to mitigate the issues associated with hallucinations in language models. Generally, when the utterance does not pertain to an opinion, the main fact can be derived from the provided grounding. 

There are several approaches to fact-control realization for overcoming hallucinations within a dialogue system. One of them is the use of external memory, which was proposed in RETRO \cite{DBLP:conf/icml/BorgeaudMHCRM0L22} and KELM \cite{DBLP:journals/corr/abs-2109-04223} models when the relevant parts of the training texts are passed to the cross-attention mechanism at the stage of next response generation. An alternative method is to extract grounding text from external databases, for instance, by using web mining like in the Sparrow \cite{DBLP:journals/corr/abs-2209-14375} approach. LaMDA \cite{DBLP:journals/corr/abs-2201-08239} proposes an approach combining structured factual grounding from an external knowledge base (Google Search API) and dialogue context both in the training and inference stages.

In this paper, we focus on the Sparrow approach and explore the importance of using grounding for generating consistent open-domain dialogues. We use the MediaWiki API\footnote{\url{https://github.com/goldsmith/Wikipedia}} to conduct searches for two types of queries: movie titles and entire Reddit thread titles. A restriction was imposed to retrieve a maximum of five documents for each query. Subsequently, a summarized version of these documents was created, consisting of five sentences. These summaries were then combined into a single grounding text.

\subsection{Dialogue Linearization}

The linearization of dialogue graphs plays a crucial role in our approach. \citet{DBLP:conf/acl/HoyleMS21} demonstrated that Transformers exhibit invariance to the specific method employed for linearization. Therefore, we employ discourse and AMR graphs for dialogue modeling, followed by a thoughtful linearization process.

Our linearization procedure is implemented in the following way. Firstly, all utterances are arranged in chronological order to establish a linear sequence. Secondly, each utterance is linearized independently, taking into account its own characteristics as well as the attributes of the connecting edge to its addressee. To achieve this, each utterance is assigned a unique identifier, the current speaker is indicated, and the addressee statement to which the utterance responds is specified. Thirdly, the appropriate response strategy is determined, as indicated by a discourse relation and sentiment tokens. Finally, the text of the subsequent utterance is incorporated. We utilize special tokens to identify speakers, utterances, and addressees, namely $\{\langle s_i \rangle\}$, $\{\langle u_i \rangle\}$ and $\{\langle \text{to:}u_i \rangle\}$ respectively. As an example, a linearized $i$-th utterance written by the $j$-th speaker in response to the $k$-th utterance has the following form:
$\text{``}\langle s_j \rangle \,\,\, \langle u_i \rangle \,\,\, \langle \text{to:}u_k \rangle \,\,\, \langle \text{relation} \rangle \,\,\, \langle \text{sentim.} \rangle \,\,\, \text{text''}$.

We employ a separation token to combine individual utterances and create a full representation of the dialogue state. Figure \ref{fig_lin} provides an example of the conversation linearization procedure. By eliminating all text and sentiment tokens, the linearized representation can be conveniently converted into a raw linear discourse representation.



\subsection{GroundHog Model}

We suggest the GroundHog model as an effective neural approach for encoding diverse types of input information. It incorporates multiple Transformer-based encoders to capture multiple levels of granularity in the input data. Unlike previous approaches such as Longformers \cite{DBLP:journals/corr/abs-2004-05150}, our focus is on the attention mechanism within each input rather than utilizing global attention. In addition, we reduce the size of the attention matrices compared to Longformers.

The architecture is based on the customized BART, as illustrated in Figure \ref{fig_1}. Our approach involves the utilization of multiple texts as input, on which it does not formally impose restrictions. The first input text should contain the primary information, whereas the others should provide supplementary information. In our case, the inputs are the following: (1) a dialogue history that has been enriched with discourse and sentiment tokens; (2) a raw, linearized representation of a discourse dialogue graph; and (3) an addressee's utterance and a part of its AMR graph.

\begin{figure*}[t!]
\centering
\includegraphics[scale=0.63, clip]{images/multienc_bart.drawio_v2.drawio.pdf}
\caption{GroundHog architecture. The GroundHog architecture comprises individual BART encoders for each input text, which are subsequently aggregated and used as input for the BART decoder. To reduce the dimensionality of the inputs, a 1D convolutional layer is applied to all inputs except the main input. The shared embedding layer is denoted by the gear icon. In addition, intermediate tensor dimensions are indicated (batchsize is denoted as bs).}
\label{fig_1}
\end{figure*}


Each input is first processed through a common tokenizer and then encoded separately using its own BART encoder. In order to create a more universal approach, embeddings from \emph{all} inputs could be aggregated through convolution. However, this would substantially change the standard input format of the pre-trained BART decoder, making the training process more challenging without a large dataset for additional pre-training. Therefore, we divide the inputs into two categories: the main text and the supplementary texts.

The model does not modify the embedding of the main text before the decoder, and it retains the attention mask for this text. The other inputs are combined using concatenation and a convolutional layer. However, this approach may introduce some disruption to the token order, and consequently, the attention masks from the encoder for these inputs are not  utilized in the decoder input. In this research, we conducted experiments using different aggregation methods and determined that the one-dimensional convolutional layer yielded the most favorable results.

As in the base model, the language modeling head is utilized after the decoder. We use the same tokenizers and shared embedding layer for all encoders and the decoder. As is common in language modeling decoder-based approaches, we employ a standard cross-entropy loss.


\section{Experiments}

\subsection{Implementation Details}

We fine-tuned the base-sized BART (139M parameters) model and the GroundHog models based on it. We used various lengths for different inputs but the maximum was 1024 tokens. The models were trained on batches of size 2, with a learning rate of 2e-5, for 5 epochs. For all other hyper-parameters, we used the default values.

All parsers and datasets used have the open source MIT license.

Each model was trained on the GPU Tesla V100 32G for approximately 10 hours.

\subsection{Automatic Evaluation}

In order to conduct a more comprehensive analysis of the generation of complex responses, we divided the dataset into two subsets: dialogues with long last responses (consisting of at least two sentences) and dialogues with short responses.

We conducted experiments using both the BART and GroundHog models for the several configurations of the dataset used for fine-tuning:

\begin{itemize}
\setlength{\itemsep}{1pt}
\setlength{\parskip}{0pt}
\setlength{\parsep}{0pt}
\item In $\mathcal{B}$, we fine-tuned the base BART model using the concatenation of the dialogue history, thread title, and grounding as the input.
\item In $\mathcal{G}_1$, we trained the GroundHog model using the concatenation of the dialogue histories and thread titles.
\item In $\mathcal{G}_2$, we extended input from $\mathcal{G}_1$ by adding grounding.
\item In $\mathcal{G}_3$, we enriched the dialogue history from $\mathcal{G}_2$ by discourse linguistic tokens.
\item In $\mathcal{G}_4$, we added separate linguistic inputs associated with AMR: (1) AMR for the full dialogue history (concatenated representations of single utterances); (2) AMR for the addressee.
\item In $\mathcal{G}_5$, we extended the input from $\mathcal{G}_4$ by adding sentiment tokens.
\end{itemize}

\begin{table*}
\centering
\scalebox{0.9}{
\begin{tabular}{cl|ccccc}
\hline \textbf{Model} & \textbf{Setting} & \textbf{ROUGE-1} & \textbf{ROUGE-2} & \textbf{ROUGE-L} & \textbf{BLEU-1} & \textbf{BLEU-2} \\ \hline
BART & \(\mathcal{B}\) [history; title; grounding]  & 17.71 & 3.69 & 15.95 & 17.2 & 2.86 \\
  GroundHog & \(\mathcal{G}_1\) [history; title]  & 17.79 & 3.71 & 16.05 & 16.99 & 2.88 \\
 & \(\mathcal{G}_2\) [+grounding]  & 17.86 & 3.85 & 16.08 & \textbf{17.32} & 3.00 \\
   & \(\mathcal{G}_3\) [+discourse]  & 17.88 & 3.87 & 16.09 & 17.15 & 3.04  \\
 & \(\mathcal{G}_4\) [+AMR]  & 17.88 & 3.80 & 16.17 & 17.26 & 2.94 \\ 
 & \(\mathcal{G}_5\) [+sentiment]  & \textbf{17.91} & \textbf{3.93} & \textbf{16.19} & 17.25 & \textbf{3.09} \\
\hline
\end{tabular}
}
\caption{\label{table_res} Model performance on the test set (\textbf{long responses}) for different model input settings. $\mathcal{B}_i$ and $\mathcal{G}_i$ are related to the BART and GroundHog models trained using different combinations of inputs. Here, the standard deviation is less than 0.007 in all cases.}
\end{table*}


\begin{table*}
\centering
\scalebox{0.9}{
\begin{tabular}{cl|ccccc}
\hline \textbf{Model} & \textbf{Setting} & \textbf{ROUGE-1} & \textbf{ROUGE-2} & \textbf{ROUGE-L} & \textbf{BLEU-1} & \textbf{BLEU-2} \\ \hline
BART & \(\mathcal{B}\) [history; title; grounding]  & 9.86 & 2.10 & 8.98 & 9.18 & 1.65 \\
  GroundHog & \(\mathcal{G}_1\) [history; title]  & 9.66 & 1.88 & 8.77 & 8.90 & 1.40 \\
 & \(\mathcal{G}_2\) [+grounding]  & 9.85 & 2.16 & 8.98 & 9.20 & 1.70 \\
   & \(\mathcal{G}_3\) [+discourse]  & 10.11 & 2.37 & 9.21 & 9.46 & 1.90  \\
 & \(\mathcal{G}_4\) [+AMR]  & 9.82 & 2.06 & 8.93 & 9.12 & 1.56 \\ 
 & \(\mathcal{G}_5\) [+sentiment]  & \textbf{10.23} & \textbf{2.47} & \textbf{9.32} & \textbf{9.52} & \textbf{1.98} \\
\hline
\end{tabular}
}
\caption{\label{table_res_short} Model performance on the test set (\textbf{short responses}) for different model input settings.}
\end{table*}

In all cases where grounding was utilized, it was concatenated with the main text input. This was necessary to ensure that the attention mechanism adequately considered the specific components of grounding. Simultaneously, grounding was treated as a distinct input due to its voluminous nature, which may necessitate its truncation.

For automatic evaluation of generated responses, we calculated the ROUGE-based\footnote{\url{https://pypi.org/project/rouge/}} \citep{Lin2004ROUGEAP} and BLEU-based\footnote{\url{https://www.nltk.org/_modules/nltk/translate/bleu_score.html}} \citep{DBLP:conf/acl/PapineniRWZ02} scores using target texts cleared of special tokens (raw texts). The obtained scores (mean F1 over three runs) are presented in Table \ref{table_res} for the long texts.

The GroundHog model (\(\mathcal{G}_2\)) exhibited superior performance compared to BART across all metrics when provided with the same inputs. This suggests that longer inputs are more effectively processed when handled separately. However, it is important to note that one limitation of the GroundHog model is that its decoder requires substantial amounts of training data to learn effectively from scratch. With sufficient pretraning, these results can be improved. Also, triggered by this limitation, we conducted a grid search and determined that setting the embedding size after the 1D convolutional layer in GroundHog to 256 would prevent an unnecessary increase in the decoder's hidden state.

The results demonstrate that grounding has a positive impact on the ROUGE and BLEU scores. This can be attributed to the fact that the generated responses exhibit a higher level of accuracy in terms of factual information. However, the observed difference is not statistically significant. It can be attributed to the subjective nature of most of the generated responses, which often involve personal evaluations. Additionally, the quality of the extracted grounding is not optimal, as it is extracted based on the full dialogues.

It was observed that the inclusion of linguistic features in the model led to improved performance compared to a model trained without these features. Specifically, the model that utilized all linguistic inputs (\(\mathcal{G}_5\)) achieved the highest scores. The incremental addition of linguistic features resulted in a monotonic improvement in the ROUGE-L metric as well as an overall improvement in other scores.

Furthermore, the experiments indicated that the use of AMR may not be as promising as dialogue acts or sentiment. This finding can be attributed to the complexity of the AMR structure.

Results for the short target responses are presented in Table \ref{table_res_short}. The overall conclusions drawn from the analysis are consistent with the findings discussed earlier, with the exception of the monotonous growth of metrics. However, a noteworthy observation is that even for short responses, the GroundHog model exhibits superior performance compared to the base BART model.

\subsection{Human Evaluation}

\begin{figure*}[t]
\centering
\includegraphics[width=\textwidth, clip]{images/groundhog_interp.drawio.pdf}
\caption{Examples of response generation by the base BART model and the GroundHog model fine-tuned with linguistic inputs. Each color represents a different speaker. The task was to generate text in the last utterance.}
\label{fig_ex}
\end{figure*}

A human evaluation was conducted to enhance the assessment process and account for aspects that cannot be evaluated using automated metrics. The main objective of this evaluation was to compare the texts generated by the BART model ($\mathcal{B}$) with those of the GroundHog model employing linguistic features ($\mathcal{G}_5$). Experts were tasked with determining the preferable option for continuing the conversation, or whether the alternatives were equal. Also, each option was evaluated on a 3-point scale based on coherence (utterance-based), meaningfulness, and consistency (dialogue-based) criteria.

Dialogue consistency assessed the connection between the current utterance and the addressee, as well as the overall logical progression of the dialogue. Meaningfulness assessed the semantic load of the utterance within its general context. Utterance-based coherence was assessed by evaluating the internal coherence of the utterance.

To ensure reliability in the evaluation process, the three scales were rated on a scale ranging from 0 to 2 (0 for poor prediction, 2 for good prediction). To minimize any potential bias, the options for rating were presented in a random order.

Table \ref{table_hum} presents the evaluation results obtained from 250 randomly selected dialogues from the test dataset. The linguistic approach, as observed, generates responses that are preferred in a larger number of cases. Additionally, these responses are more coherent, suitable for continuing the conversation, and formulated with better semantic appropriateness. While the overall improvement is not sizeable, there is notable progress in the generation of consistent conversations.

\begin{table}
\center
\scalebox{0.9}{
\begin{tabular}{ccccc}
\hline  \textbf{} & \textbf{\# better} & \textbf{Coherence}  & \textbf{Meaning.} & \textbf{Consist.}\\ \hline
$\mathcal{B}$ & 79 & 1.48 & 1.27 & 1.38 \\
$\mathcal{G}_5$ & \textbf{101} & \textbf{1.53} & \textbf{1.38} & \textbf{1.40} \\
\hline
\end{tabular}
}
\caption{\label{table_hum} Human evaluation results on the random test subset of 250 dialogues.}
\end{table}

\section{Discussion}


\begin{table*}[t]
\center
\scalebox{0.88}{
\begin{tabular}{c|ccccccccc}
\hline Model  & answer & elaboration & agreement & other & disagreement & appreciation & question & negative & humor \\ \hline
$\mathcal{B}$ & 1491 & \textbf{1231} & 446 & 211 & 158 & 107 & 80 & 15 & 1 \\
$\mathcal{G}_5$ & 1508 & 1174 & \textbf{454} & 223 & 159 & 101 & \textbf{97} & \textbf{22} & 2 \\
\hline
\end{tabular}
}
\caption{Statistics of dialogue acts in texts generated by the base BART and GroundHog models.}
\label{fig_stats}
\end{table*}

In this section, our major objective is to gain a deeper understanding of the linguistic features that contribute to the improved quality of GroundHog. To this end, we conduct a comparative analysis of the texts generated by BART ($\mathcal{B}$) and the texts generated by GroundHog ($\mathcal{G}_5$).

Regarding the interpretation of grounding, its incorporation enhances the factual component of generation. However, the qualitative aspects of grounding in our dataset are not very robust, and it can be a direction for further research.


\paragraph{Sentiment} We started our investigation with the analysis of sentiment due to its ease of interpretation. To assess the sentiment in the generated texts, we utilized the same classifier that was applied to the training dataset. The results yielded an overall accuracy of 0.43 for the BART model and 0.44 for the GroundHog model, with no sizeable difference observed. It is worth noting that the majority of texts in the dataset were negative or neutral, as users generally tend to criticize films or actors. Specifically, there were 1525 negative utterances, 1374 neutral utterances, and 841 positive utterances within the test dataset. Consequently, the focus should be shifted to generating more accurate negative responses. In terms of these responses, the base BART model achieved an F1-macro score of 0.461, while the GroundHog model achieved an F1-macro score of 0.487. This improvement is particularly noteworthy as it leads to an overall enhancement in language modeling.

Figure \ref{fig_ex} (a) presents an illustrative MPC example. It is observed that all input utterances within the dialogue are negative in nature. Consequently, the subsequent utterance should also embody a negative sentiment, either by aligning with the general criticism of the film director or by critiquing the statements expressed by other participants. In this context, it can be inferred that GroundHog has produced an appropriate response. Conversely, the response generated by the base BART model is positive in sentiment and considered inappropriate.

\paragraph{Discourse} Dialogue acts contribute to dialogue-based consistency and, to some extent, utterance-based coherence. Since existing models do not explicitly generate dialogue acts, we utilized a trained discourse parser to label these acts for comparison with the original responses.
The GroundHog model had a higher accuracy score of 0.551 compared to 0.538 for the base model. The confusion matrices showed similar patterns, but there was a slight difference in the distribution of dialogue acts (see Table \ref{fig_stats}). Specifically, the base model exhibited a higher frequency of ``Elaboration'', while the GroundHog model generated less common relations such as ``Question'' and ``Agreement''. This indicates that the linguistic model's responses are more diverse without compromising their quality.

In the conversation depicted in Figure \ref{fig_ex} (b), it can be observed that the custom model response exhibits better consistency. The most correct target response should include the Answer or Agreement relations rather than Elaboration. Unlike the BART model, which lacks information about the previous response being a question, the GroundHog model incorporates this knowledge in order to generate a response that is discursively consistent. Moreover, GroundHog aims to incorporate the main AMR entities, such as the concept of ``controversial.''

\paragraph{AMR} Interpreting the impact of AMR representations is challenging due to their inherent complexity. Generally, AMR has a direct influence on the semantic aspect, specifically the representation of entities and their relations. In this regard, human evaluation has shown that the scores for the criterion of ``meaningfulness'' are higher for GroundHog texts compared to BART texts.

Figure \ref{fig_ex} (c) provides a concrete example illustrating a discussion where each participant expresses their opinion about some statement. Here, both generative models produced thematically correct answers. However, the GroundHog model used more appropriate words, resulting in a response that was more consistent with the dialogue history. We hypothesize that this can be attributed primarily to the AMR input. For the first utterance, the AMR representation is as follows:

 \emph{( miss :ARG0 ( article ) :ARG1 ( trend :ARG1 ( and ) :ARG1-of ( have-degree ) ) ) ... ( process :ARG0 ( artist ) :ARG1 ( trauma :poss a ) :instrument ( art :poss a ) )
 }

 Therefore, the main entities are ``article'', ``trend'', ``artists'', ``trauma'', and ``art''. The GroundHog model primarily relies on these words, whereas BART's response is primarily influenced by the word ``generation''. However, the frequent occurrence of ``generation'' does not capture the underlying meaning of the text.

\paragraph{General View} We have determined that linguistic features individually demonstrate utility and yield interpretive results. There is also the potential for uncovering valuable hidden insights through their combination. Nevertheless, our research represents a step towards achieving a coherent and meaningful generation.

It is worth considering that linguistic features can also be manually specified when the current context is insufficient for parsers to accurately perform their tasks. Such manual specifications can facilitate dialogue management.

\section{Conclusion and Future Work}
In this paper, we investigated the efficacy of incorporating grounding and multi-grained linguistic information for multi-party conversation generation. To address the challenge of handling lengthy input texts, we proposed the GroundHog model, which leverages both grounding and linguistic features.

For evaluation, we collected a novel Reddit-based dataset designed for training dialogue systems. This dataset was augmented with linguistic features, including semantic and discourse information, as well as sentiment. Experiments involving both automatic metrics and human evaluation have shown that generated texts using linguistic inputs were more preferable. In our supplementary analysis, we interpreted the obtained results.

Further research directions include the investigation of other linguistic inputs as well as other representations of inputs. Also, we plan to experiment with the recent LLMs to analyze their possibilities of leveraging linguistic features.


\section*{Limitations}

Our approach is not constrained by language and has the potential for universal application. At the same time, we introduce a novel Transformer architecture that ideally requires pre-training on a large dataset. Furthermore, the effectiveness of the methodology is constrained by the accuracy and reliability of the parsers used to extract linguistic features, as well as the performance of the grounding extraction model.


\section*{Ethics and Broader Impact}

The use of large Transformer models for training has been linked to contributing to climate change. However, it is important to highlight that our research did not involve training these models from scratch. Instead, we conducted a fine-tuning process on pre-existing models.

As is the case with any generative model, it is not possible to ensure flawless quality in the generated output. At the same time, we do not make our model publicly available. We mitigate the risks associated with generation by filtering the dataset and making business logic modifications.

The presented dataset was collected from Reddit for the purpose of scientific research and subsequent analysis. It may exhibit certain inherent biases due to its specific origin, and we suggest using it for scientific purposes only.

\section*{Acknowledgements}
The article was prepared within the framework of the HSE University Basic Research Program. It was also supported
in part through the computational resources of HPC
facilities at NRU HSE. 

The publication was also supported by the grant for research centers in the field of AI provided by the Analytical Center for the Government of the Russian Federation (ACRF) in accordance with the agreement on the provision of subsidies (identifier of the agreement 000000D730321P5Q0002) and the agreement with HSE University No. 70-2021-00139.


% Entries for the entire Anthology, followed by custom entries
\bibliography{anthology,custom}


\appendix

 \section{Linguistic Representations}
\label{ling_repr}

  \paragraph{Dialogue Acts}
Figure \ref{fig_di} illustrates an example of a multi-party conversation that has been annotated with dialogue acts. In this figure, each node in the graph represents an utterance in the conversation and includes attributes such as the speaker's identifier (represented by colors) and the utterance text. The edges connecting the nodes indicate the flow of conversation and include attributes such as the addressee, representing the recipient of the utterance, and the dialogue act, representing the specific function or purpose of the utterance.

\begin{figure}[t]
\centering
\includegraphics[width=\columnwidth, clip]{discodial2.drawio.pdf}
\caption{A manually annotated discourse tree for the multi-party dialogue. The color identifies the speaker, and edges indicate dialogue acts.}
\label{fig_di}
\end{figure}


 \paragraph{Abstract Meaning Representation} Figure \ref{fig_amr} illustrates the representation of an utterance and its linearization using Abstract Meaning Representation (AMR).  In this representation, words from the utterance are depicted as nodes in a graph, with edges representing the semantic relations between them. Higher-level vertices closer to the root of the graph capture the overall meaning, while lower-level vertices offer more specific details. In the given example, the core concept of contradiction is conveyed through the first two levels of the AMR graph. To enhance the efficiency of processing and reduce the length of the linearized representation, we only truncate the first levels of these graphs.



 \begin{figure}[t]
\centering
\includegraphics[width=\columnwidth, clip]{amr_rot.drawio.pdf}
\caption{AMR representation for a single utterance and its truncated (by the first two levels) linearized representation. Here, (1) is the input text, (2) is the corresponding AMR graph, and (3) is the truncated plain graph2text representation.}
\label{fig_amr}
\end{figure}



\end{document}
