\section{Preliminaries}

\begin{figure*}[t]
    \centering
    \includegraphics[width=1.0\textwidth]{pictures/overview.pdf}
    \caption{[a] Overview of our RL architecture. \textcircled{1} Environment states are processed as inputs for agent; \textcircled{2} agent yield modification upon $SRC$ in environment; \textcircled{3} determine survival and step reward of environment; \textcircled{4} determine degradation with victim NMT as episodic reward; \textcircled{5} update agent with total rewards. During a training episode, we loop \textcircled{1} to \textcircled{3} and accumulate step rewards until environment terminates. Dash line indicates execution at the end of an episode. [b] Architecture of discriminator. [c] Architecture of agent. } 
    \label{fig:model_overview}
\end{figure*}

\subsection{Neural Machine Translation}
The most popular architectures for neural machine translation are RNN-search \citep{Bahdanau2015Neural} and Transformer \citep{vaswani2017attention}.
They share the paradigm to learn the conditional probability $P(Y|X)$ of a target translation $Y=[y_1,y_2,...,y_m]$ given a source input $X=[x_1,x_2, ..., x_n]$. A typical NMT architecture consists of an encoder, a decoder and attention networks. The encoder encodes the source embedding $X_{emb}=[emb_1,emb_2,...emb_n]$ into hidden representation $H=[h_1,h_2,...,h_n]$.
% \begin{align}
%     H=f_{enc}(X_{emb}; \theta_{enc}) \nonumber
% \end{align}
% where $\theta_{enc}$ denotes the encoder parameter set, and $f_{enc}$ denotes the encoder network. 
Then a decoder $f_{dec}$ with attention network attentively accesses $H$ for an auto-regressive generation of each $y_i$ until the end of sequence symbol (EOS) is generated:
\begin{align}
    P(y_i|y_{<i},X)&=\text{softmax}(f_{dec}(y_{i-1}, s_t,c_t; \theta_{dec})) \label{eq:nmt_output}
\end{align}
where $c_t$ is the attentive result for current decoder state $s_t$ given $H$.

\subsection{Actor-Critic for Reinforcement Learning}
% \SJ{intuition to choose RL in this paper.}
Reinforcement learning \citep[RL]{sutton2018reinforcement} is a widely used machine learning technique following the paradigm of \textit{explore and exploit}, which is apt for unsupervised policy learning in many challenging tasks (e.g., games \citep{mnih2015human}). It is also used for direct optimization for non-differentiable learning objectives \citep{wu2018study,bahdanau2016actor} in NLP. 

Actor-critic \citep{konda2000actor} is one of the most popular RL architectures where the agent consists of a separate policy and value networks called actor and critic. 
They both take in environment state $s_t$ at each time step as input, while actor determines an action $a_t$ among possible action set $\mathcal{A}$ and critic yields value estimation $V_t(s_t)$ . 
% In return, the environment yields new state $s_{t+1}$ and a corresponding reward $r_t$. The process continues until the agent reaches the terminal state after which the process restarts.
In general, the agent is trained to maximize discounted rewards $R_t=\sum_{i=0}^\infty{\gamma^i r_{t+i}}$ for each state, where $\gamma\in(0,1]$ is the discount factor. Such goal can be further derived as individual losses applied to actor and critic.
Thus the actor policy loss $L^{\pi}$ on step $t$ is: 
\begin{align}
    L^{\pi}_t(\theta_{\pi})=\log P(a_t|s_t)A_t(s_t, a_t); a_t\in \mathcal{A}  \label{eq:actor_loss}
\end{align}
where $\theta_{\pi}$ denotes actor parameters, $A_t(s_t, a_t)$ denotes general advantage function \citep{schulman2015high} on state $s_t$ for action $a_t$ given by $\sum^{k-1}_{i=0} \gamma^ir_{t+i}+\gamma^kV(s_{t+k})-V(s_t) $, which can be further derived as:
% gae function
\begin{align}
    A_t(s_t,a_t) = \gamma A_{t+1}(s_{t+1}, a_{t+1})+ r_t \nonumber \\ 
    +\gamma V_{t+1}(s_{t+1})-V_t(s_t)
    \label{eq:gae}
\end{align}
Meanwhile, critic learns to estimate $R_t$ via minimizing a temporal difference loss $L^{v}$ on each step $t$:
\begin{align}
    L^{v}_t(\theta_v)=\frac{1}{2}(r_t+\gamma R_{t+1}- V_t(s_t))^2 \label{eq:critic_loss}
\end{align}
where $\theta_v$ denotes critic parameter. 

Usually, the training is regularized by maximizing policy entropy  $H^{\pi}$  to avoid exploration failure before exploiting optimum policy \citep{ziebart2010modeling}. Thus the total loss becomes:
% training trg for AC 
\begin{align}
    L(\theta)=\sum_t(\alpha L^v_t - L^{\pi}_t - \beta H^{\pi}(\cdot|s_t))
    \label{eq:overall_loss}
\end{align}
where $\alpha$ and $\beta$ are hyperparameters for value loss and entropy coefficients. 

\subsection{adversarial examples in NLP}
% adversarial examples are special bad cases generated by \textbf{minor} perturbation on normal cases.
A general adversarial example generation can be described as the learning process to find a perturbation $\delta$ on input $X$ that maximize system degradation $L_{adv}$ within a certain constraint $C(\delta)$:
\begin{align}
    \underset{\delta}{\mathrm{argmax}} L_{adv}(X+\delta)-\lambda C(\delta)  \label{eq:adversarial_loss_continuous}
\end{align}
where $\lambda$ denotes the constraint coefficient, $ L_{adv}$ is determined by the goal of the attack. 
% With the perturbed representation, tokens with the nearest embedding are sampled as the results.
However, currently effective adversarial generation for NLP is to search by maximizing a surrogate gradient-based loss:
\begin{align}
    \underset{1 \leq i \leq n, x'\in vocab}{\mathrm{argmax}} L_{adv}(x_0, x_1,...x_i'...x_n)
    \label{eq:adversarial_loss_discrete}
\end{align}
where $L_{adv}$ is a differentiable function indicating the adversarial object. Due to its formidable search space, this paradigm simply perturbs on a small ratio of token positions and greedy search by brute force among candidates. 
Note that adversarial example generation is fundamentally different from noised hidden representation in adversarial training \citep{cheng2019robust, sano2019effective}, which is not to be concerned in this work.
