%
% File emnlp2016.tex
%

\documentclass[11pt,letterpaper]{article}
\usepackage{emnlp2016}
\usepackage{times}
\usepackage{latexsym}
\usepackage{diagbox}
%\usepackage{ulem}
\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{graphicx}
\usepackage{array}
\usepackage{multirow}
\usepackage{url}
% Uncomment this line for the final submission:
\emnlpfinalcopy

%  Enter the EMNLP Paper ID here:
\def\emnlppaperid{***}

% To expand the titlebox for more authors, uncomment
% below and set accordingly.
% \addtolength\titlebox{.5in}    

\newcommand\BibTeX{B{\sc ib}\TeX}


\title{A Stacking Gated Neural Architecture\\ for Implicit Discourse Relation Classification}

\author{ Lianhui Qin$^{1,2}$, Zhisong Zhang$^{1,2}$, Hai Zhao$^{1,2,}$\thanks{
		\, Corresponding author. This paper was partially supported by Cai Yuanpei Program (CSC No. 201304490199 and No. 201304490171), National Natural Science Foundation of China (No. 61170114, No. 61672343 and No. 61272248), National Basic Research Program of China (No. 2013CB329401), Major Basic Research Program of Shanghai Science and Technology Committee (No. 15JC1400103), Art and Science Interdisciplinary Funds of Shanghai Jiao Tong University (No. 14JCRZ04), and Key Project of National Society Science Foundation of China (No. 15-ZDA041).
	} \\
	$^1$Department of Computer Science and Engineering,\\
	Shanghai Jiao Tong University, Shanghai, 200240, China\\
	$^2$Key Laboratory of Shanghai Education Commission for Intelligent Interaction\\
	and Cognitive Engineering, Shanghai Jiao Tong University, Shanghai, 200240, China\\
	\tt \{qinlianhui, zzs2011\}@sjtu.edu.cn,zhaohai@cs.sjtu.edu.cn
}

\date{}

\begin{document}

\maketitle

\begin{abstract}
Discourse parsing is considered as one of the most challenging natural language processing (NLP) tasks. Implicit discourse relation classification is the bottleneck for discourse parsing. Without the guide of explicit discourse connectives, the relation of sentence pairs are very hard to be inferred.  This paper proposes a stacking neural network model to solve the classification problem in which a convolutional neural network (CNN) is utilized for sentence modeling and a collaborative gated neural network (CGNN) is proposed for feature transformation. 
Our evaluation and comparisons show that the proposed model outperforms previous state-of-the-art systems.

\end{abstract}


\section{Introduction}
As a fundamental task in natural language processing (NLP), discourse parsing entails the discovery of the latent relational structure in multi-sentence level analysis. It is also central to many practical tasks such as question answering \cite{liakata2013discourse,jansen2014discourse}, machine translation 
\cite{meyer2012using,meyer2013machine} and automatic summarization \cite{murray2006incorporating,yoshida2014dependency}. Discourse parsing is also the shared task of CoNLL 2015 and 2016 \cite{K15-2001,K16-2000}, and many previous works previous on this task \cite{K16-2010,K16-2008,K15-2005,K16-2004}. In a discourse parser, implicit relation recognition has been the bottleneck due to lack of explicit connectives (like \textit{``because"} or \textit{``and"}) that can be strong indicators for the senses between adjacent clauses \cite{K16-2010,pitler-louis-nenkova:2009:ACLIJCNLP,lin2014pdtb}.  This work therefore focuses on implicit relation recognition that infers the senses of the discourse relations within adjacent sentence pairs.

Most previous works on PDTB implicit relation recognition only focus on one-versus-others binary classification problems of the top level four classes \cite{pitler-louis-nenkova:2009:ACLIJCNLP,zhou2010predicting,park2012improving,mckeown2013aggregated,rutherford-xue:2014:EACL,braud2015comparing}. Traditional classification methods directly rely on feature engineering, based on bag-of-words, production rules, and some linguistically-informed features\label{key} \cite{zhou2010predicting,rutherford-xue:2014:EACL}. However, discourse relations root in semantics, which may be hard to recover from surface level feature, thus these methods did not report satisfactory performance. Recently, neural network (NN) models have shown competitive or even better results than traditional linear models with hand-crafted sparse features \cite{wang2016converting,zhang2016learning,jia2014joint}. They have been proved to be effective for many tasks \cite{coling,wang2016learning,zhang2016probabilistic,wang2015word,wang2014neural,P16-1039}, also including discourse parsing. Ji and Eisenstein  \shortcite{TACL536} adopt recursive neural network and incorporate with entity-augmented distributed semantics. Zhang et al. \shortcite{zhang-EtAl:2015:EMNLP4} explore a shallow convolutional neural network and achieve competitive performance. Although simple neural network has been shown effective, the result has not been quite satisfactory which suggests that there is still space for improving.

The concerned task could be straightforwardly formalized as a sentence-pair classification problem, which needs inferring senses solely based on the two arguments without cues of connectives. Two problems should be carefully handled in this task: how to model sentences and how to capture the interactions between the two arguments.  The former could be addressed by Convolutional Neural Network (CNN) which has been proved effective for sentence modeling \cite{kalchbrenner-grefenstette-blunsom:2014:P14-1,kim:2014:EMNLP2014}, while the latter is the key problem, which might need deep semantic analysis for the interaction of two arguments. To solve the latter problem, we propose collaborative gated neural network (CGNN) which is partially inspired by Highway Network whose gate mechanism achieves success \cite{srivastava2015highway}. Our method will be evaluated on the benchmark dataset against state-of-the-art methods. 

The rest of the paper is organized as follows: Section 2 briefly describes our model, introducing the stacking architecture of CNN and CGNN, Section 3 shows the experiments and analysis, and Section 4 concludes this paper.

\begin{table*}
	\centering
	\begin{tabular}{l|ccccc}
		\hline
		&\textsc{Comp.} & \textsc{Cont.} & \textsc{Exp.} & \textsc{Temp.} & \textsc{Avg} \\ 
		\hline
		CNN Only &39.07&54.73&65.94&30.19&47.48\\
		CNN+MLP   &37.81&56.30&69.44&32.29&48.96\\ 
		CNN+LSTM  &39.15&53.44&68.85&29.79&47.81\\
		CNN+Highway   &37.72&56.35&68.94&30.56&48.39\\
		CNN+CGNN    &\textbf{41.55}&\textbf{57.32}&\textbf{71.50}&\textbf{35.43}&\textbf{51.45}\\
		\hline
		
	\end{tabular}
	\caption{\label{layer} $F_1$ scores (\%) with different models. }
\end{table*}

\section{Method}

The architecture of the model, as shown in Figure 1, is straightforward. It can be divided into three parts: 1) CNN for modeling arguments; 2) CGNN unit for feature transformation; 3) a conventional softmax layer for the final classification. CNN is used to obtain the vector representations for the sentences, CGNN further captures and transforms the features for the final classification. 



\subsection{Convolutional Neural Network}
As CNN has been broadly adopted for modeling sentences,  we will explain it in brevity.
 For two arguments, typical sentence modeling process will be applied: sentence embedding (including embeddings for words and part-of-speech (POS) tags) through projection layer, convolution operations (with multiple groups of filters) through the convolution layer, obtaining the sentence representation through one-max-pooling. The two arguments will get their sentence vectors independently without any interfering, and the convolution operation will be the same by sharing parameters. The final argument-pair representation will be the vector $\mathbf{v}$ which is concatenated from two sentence vectors and this vector will be used as the input of the CGNN unit.

\begin{figure}
	\centering
	\includegraphics[width=0.5\textwidth]{cgnn.pdf}
	\caption{Model Architecture}
	\label{fig:2gate}
\end{figure}

\subsection{Collaborative Gated Neural Network}
For implicit sense classification, the key  is how to effectively capture the interactions between the two arguments. The interactions could be word pairs, phrase pairs or even the latent meaning of the two full arguments.  Pitler et al. \shortcite{pitler-louis-nenkova:2009:ACLIJCNLP} has shown that word pair features are helpful. To model these interactions, we have to make a full use of the sentence vectors obtained from CNN. However, common neural hidden layers might be insufficient to deal with the challenge. We need to seek more powerful neural models, i.e.,  gated neural network.

In recent years, gated mechanism has gained popularity in neural models. Although it is first introduced in the cells of recurrent neural networks, like Long-Short Term Memory (LSTM) \cite{hochreiter1997long} and Gated Recurrent Unit (GRU) \cite{chung2014empirical}, traditional feed-forward neural models such as the Highway Network could also benefit from it \cite{srivastava2015highway}. The existing studies show that the gated mechanism in highway network serves not only a means for easier training, but also a tool to route information in a trained network.

Motivated by the idea of highway network, we propose a collaborative gated neural network (CGNN) for this task. The architecture of CGNN is illustrated in Figure \ref{fig:2gate}, and it contains a sequence of transformations. First, the inner-cell $\mathbf{\hat c}$ is obtained through linear transformation and non-linear activation on the input $\mathbf{v}$, and this process is exactly the operation of an ordinary neural layer.
\begin{align*}
\mathbf{\hat c} =  \tanh(\mathbf{W}^c \cdot \mathbf{v} + \mathbf{b}^c)
\end{align*}
Meanwhile, the two gates $\mathbf{g}_i$ and $\mathbf{g}_o$ are calculated independently because they are only influenced by the original input through different parameters:
\begin{align*}
&\mathbf{g}_i  =  \sigma(\mathbf{W}^i \cdot \mathbf{v} + \mathbf{b}^i) \\
&\mathbf{g}_o  =  \sigma(\mathbf{W}^o \cdot \mathbf{v} + \mathbf{b}^o) 
\end{align*}
where the $\sigma$ denotes sigmoid function which guarantees the values in the gates are in [0,1]. Two gated operations are applied sequentially, where a gated operation indicates the element-wise multiplication of an inner-cell and a gate. Between the two gated operations, a non-linear activation operation is applied. The procedure could be formulated as follows:
\begin{align*}
&\mathbf{c} = \mathbf{\hat c} \odot \mathbf{g}_i\\
&\mathbf{h} =  \tanh(\mathbf{c}) \odot \mathbf{g}_o
\end{align*}
where $\odot$ denotes element-wise multiplication, $\mathbf{c}$ is the second inner-cell and $\mathbf{h}$ is the output of CGNN unit. 

Although the two gates are generated independently, they will work collaboratively because they control the information flow of the inner-cells sequentially which resembles logical AND operation in a probabilistic version. In fact, the transformations after $\mathbf{\hat c}$ will concern only element-wise operations which might give finer controls for each dimension, and the information can only flow on the dimensions where both gates are ``open''. This procedure will help select the most crucial features. 

The gates in this model are mainly used for routing information from sentence-pairs vectors. When there is only one gate in our network, the model works similar to the highway network \cite{srivastava2015highway}.  

\subsection{Output and Training}
After the transformation of the CGNN unit, the transformed vector $\mathbf{h}$ will be sent to a conventional softmax for classification.

The training object $\textit{J}$ will be the cross-entropy error $\textit{E}$ with $\textit{L2}$ regularization:
\begin{align*}
&\displaystyle E(\hat y,y)=- \sum_{j}^l y_j  \times \log(Pr(\hat y_j))\\
&\displaystyle J(\theta )=\frac{1}{m}\sum_{k}^m E(\hat y^{(k)},y^{(k)}) + \frac{\lambda }{2}\left \| \theta  \right \|^{2}
\end{align*}
where $y_j$ is the gold label and $\hat y_j$ is the predicted one. We adopt the diagonal variant of AdaGrad \cite{duchi2011adaptive} for the optimization process.




\begin{table*}[t]
	\small
	\centering
	\begin{tabular}{l | c c   c c   c c   c c c c}
		\hline
		&\multicolumn{2}{c}{ \textsc{Comp.}}&\multicolumn{2}{c}{\textsc{Cont.}} &\multicolumn{2}{c}{\textsc{Exp.+}} & \multicolumn{2}{c}{\textsc{Temp.}}&\multicolumn{2}{c}{\textsc{Avg.}} \\ 
		& $F_1$ & Acc & $F_1$ & Acc & $F_1$ & Acc & $F_1$ & Acc &F1 &Acc\\
		\hline
		Pitler et al. \shortcite{pitler-louis-nenkova:2009:ACLIJCNLP}
		&21.96&56.59  
		&47.13&67.30
		&76.42&63.62
		&16.76&63.49
		&40.57&62.75\\
		Zhou et al. \shortcite{zhou2010predicting}
		&31.79&58.22
		&47.16&48.96
		&70.11&54.54
		&20.30&55.48
		&40.32&54.30\\
		P\&C \shortcite{park2012improving}
		&31.32&74.66
		&49.82&72.09
		&79.22&69.14
		&26.57&79.32
		&46.73&73.80\\
		M\&B \shortcite{mckeown2013aggregated}
		&25.40&63.36
		&46.94&68.09
		&75.87&62.84
		&20.23&68.35
		&42.11&65.66\\
		J\& \shortcite{TACL536}
		&35.93&70.27
		&52.78&76.95
		&80.02&69.80
		&27.63&87.11
		&49.09&76.03\\
		B\&D\shortcite{braud2015comparing}
		&36.36&-
		&55.76&-
		&61.76&-
		&29.30&-
		&45.80&-\\
		Chen et al. \shortcite{P16-1163}
		&40.17&-
		&54.76&-
		&80.62&-
		&31.32&-
		&51.72&-\\
		\hline
		Current
		&\textbf{41.55}&71.22
		&\textbf{57.32}&73.80
		&\textbf{80.96}&68.44
		&\textbf{35.43}&84.32
		&\textbf{53.82}&74.45\\
		\hline
	\end{tabular}
	\caption{\label{resulta} Comparisons of $F_1$ scores (\%) (symbol \textbf{+} means \textit{\textsc{Exp.}} with \textit{Entrel}). }
\end{table*}

\begin{table}[t]
	\small
	\centering
	\begin{tabular}{l| l  c c c }
		\hline
		& & P & R & $F_1$\\
		\hline
		\multirow{3}{*}{\textsc{Comp.}}
		&R\&Xue \shortcite{rutherford-xue:2014:EACL}&27.34&72.41&39.70\\
		&Zhang et al.\shortcite{zhang-EtAl:2015:EMNLP4}&22.00&67.76&33.22\\
		&Current&29.48&70.39&\textbf{41.55}\\
		\hline
		\multirow{3}{*}{\textsc{Cont.}}
		&R\&Xue \shortcite{rutherford-xue:2014:EACL}&44.52&69.96&54.42\\
		&Zhang et al.\shortcite{zhang-EtAl:2015:EMNLP4}&39.80&75.29&52.04\\
		&Current&50.69&65.95&\textbf{57.32}\\
		\hline
		\multirow{3}{*}{\textsc{Exp.}}
		&R\&Xue \shortcite{rutherford-xue:2014:EACL}&59.59&85.50&70.23\\
		&Zhang et al.\shortcite{zhang-EtAl:2015:EMNLP4}&56.29&91.11&69.59\\
		&Current&60.81&86.76&\textbf{71.50}\\
		\hline
		\multirow{3}{*}{\textsc{Temp.}}
		&R\&Xue \shortcite{rutherford-xue:2014:EACL}&18.52&63.64&28.69\\
		&Zhang et al.\shortcite{zhang-EtAl:2015:EMNLP4}&20.22&62.35&30.54\\
		&Current&26.63&52.94&\textbf{35.43}\\
		\hline
		\hline
		\multirow{3}{*}{\textsc{Avg.}}
		&R\&Xue \shortcite{rutherford-xue:2014:EACL}&37.49&72.88&48.26\\
		&Zhang et al.\shortcite{zhang-EtAl:2015:EMNLP4}&34.58&74.13&46.35\\
		&Current&41.90&69.01&\textbf{51.45}\\
		\hline
	\end{tabular}
	\caption{\label{resultb} Comparisons of $F_1$ scores (\%) (\textit{\textsc{Exp.}} without \textit{Entrel}).}
\end{table}


\section{Experiments}

\subsection{Setting} 
As for the benchmark dataset, Penn Discourse Treebank (PDTB)  \cite{prasad2008penn} corpus\footnote{\url{http://www.seas.upenn.edu/~pdtb/}}   is used for evaluation. In the PDTB, each discourse relation is annotated between two argument spans. 

To be consistent with the setups of  prior works, we formulate the implicit relation classification task as four one-versus-other binary classification problems only using the four top level classes: \textsc{Comparison (Comp.)}, \textsc{Contingency (Cont.)}, \textsc{Expansion (Exp.)} and \textsc{Temporal (Temp.)}. While different works include different relations of varying specificities, all of them include these four core relations \cite{pitler-louis-nenkova:2009:ACLIJCNLP}. 
Following dataset splitting convention of the previous works, we use sections 2-20 for training, sections 21-22 for testing and sections 0-1 for development set. The proposed model is possible to be extended for multi-class classification of discourse parsing, but for the comparisons with most of previous works, we will follow them and focus on the binary classification problems. 

For other hyper-parameters of the model and training process, we fix the lengths of both the input arguments to be 80, and apply truncating or zero-padding when necessary. The dimensions for word embeddings and POS embeddings are respectively 300 and 50, and the embedding layer adopts a dropout of 0.2. The word embeddings are initialized with pre-trained word vectors using \textit{word2vec} \footnote{\url{http://www.code.google.com/p/word2vec}} \cite{mikolov2013distributed} and other parameters are randomly initialized including POS embeddings. We set the starting learning rate to 0.001. For CNN model, we utilize three groups of filters with window widths of (2, 2, 2) and their filter numbers are all set to 1024. The hyper-parameters are the same for all models and we do not tune them individually. 



	

\subsection{Model Analysis}
For transformation of sentence vectors, a simple Multilayer Perceptron (MLP) layer could be a straightforward choice, while more complex neural modules, such as LSTM and highway  network, could also be considered. Our model utilizes a CGNN unit with refined gated mechanism for the transformation. Will the proposed CGNN really bring about further performance improvement? We now answer this question empirically.


As shown in Table \ref{layer}, CNN model usually performs well on its own. Utilizing an MLP layer or a Highway layer could improve the accuracies on \textsc{Contingency, Expansion, Temporary} except for \textsc{Comparison}. Though the primary motivation of Highway is to ease gradient-based training of highly deep networks through utilizing gated units, it works merely as an ordinary MLP in the proposed model, which explains the reason that it performs  like MLP.  Despite one of four classes, \textsc{Comparison}, not receiving performance improvement, introducing a non-linear transformation layer lets the classification benefit as a whole. ``CNN+LSTM'' denotes the method of using LSTM to read the convolution sequence (without pooling operation), and it even does not perform better than MLP.

The CGNN achieves the best performance on all classes including \textsc{Comparison}. It gains 3.97\% imrovement on average F1 score using CNN only model.
We assume that CGNN is well-suited to work with CNN, adaptively transforming and combining local features detected by the individual filters. 

\subsection{Results}
We show the main results in Tables \ref{resulta} and \ref{resultb}. The metrics include precision (P), recall (R), accuracy (Acc) and F1 score. Since not all of these metrics are reported in previous work, the comparisons are correspondingly in Table \ref{resulta} and \ref{resultb}.
Some previous work merges \textit{Entrel} with \textit{Expansion}, which is also explored in our study and noted as \textit{\textsc{Exp.+}}. 

We compare with best-performed or competitive models including both traditional linear methods and recent neural methods. For traditional methods: Pitler et al. \shortcite{pitler-louis-nenkova:2009:ACLIJCNLP} use several linguistically in- formed features, including polarity tags, Levin verb classes, length of verb phrases, modality, context, and lexical features; Zhou et al. \shortcite{zhou2010predicting} improve the performance through predicting connective words as features; Park and Cardie \shortcite{park2012improving} propose a locally-optimal feature set and further identify factors for feature extraction that can have a major impact performance, including stemming and lexicon look-up; Biran and McKeown \shortcite{mckeown2013aggregated} collect word pairs from arguments of explicit examples to help the  learning; Rutherford and Xue \shortcite{rutherford-xue:2014:EACL} employ Brown cluster pair and coreference patterns for performance enhancement. Several neural methods have also been included for comparison: Zhang et al. \shortcite{zhang-EtAl:2015:EMNLP4} propose a simplified neural network which has only three different pooling operations (max, min, average); Ji and Eisenstein \shortcite{TACL536} compute distributed semantics representation by composition up the syntactic parse tree through recursive neural network; Braud and Denis \shortcite{braud2015comparing} consider shallow lexical features and word embeddings. Chen et al. \shortcite{P16-1163} replace the original words by word embeddings to overcome the data sparsity problem and they also utilize gated relevance network to capture the semantic interaction between word pairs. The gated network is different from ours but also works well. 

Our model achieves F-measure improvements of 1.85\% on \textsc{Comparison}, 1.56\% on \textsc{Contingency}, 1.27\% on \textsc{Expansion}, 0.94\% on \textsc{Expansion+}, 4.89\% on \textsc{Temporal}, against the state-of-the-art of each class. We improve by 4.73\% on average F1 score when not including \textsc{EntRel} in \textsc{Expansion} as reported in Table \ref{resulta} and 3.19\% on average F1 score otherwise as reported in Table \ref{resultb}. The results show that our model achieves the best performance and especially makes the most remarkable progress on \textsc{Temporal}. 


\section{Conclusion}
In this paper, we propose a stacking gated neural architecture for implicit discourse relation classification. Our model includes convolution and collaborative gated neural network. The analysis and experiments show that CNN performs well on its own and combining CGNN provides further gains. Our evaluation on PTDB shows that the proposed model outperforms previous state-of-the-art systems.




\bibliography{emnlp2016}
\bibliographystyle{emnlp2016}

\end{document}
