\section{Experimental Setup}
\label{exp}
\subsection{Datasets}
We remove stopwords and punctuations and choose the first 120 words per document. The word embeddings are 300 dimensional and initialized randomly. Kernel size for textCNN is 2,3,4 and 5, and the number of filters is 1024. For each CPC subclass, we use the descriptions of its own and of all its child labels. We train the model using Adam optimizer.

\textbf{Datasets}: Out of the USPTO patent set, We randomly sample 6.7 million abstracts as the patent training set and 60k as the testing set. For the paper testing set, the gold-standard is hard to obtain. We discover that some papers cited by patents are assigned CPC labels by European Patent Office, we collect those from the website \footnote{https://www.epo.org/index.html} and derive 4956 testing instances for paper-to-CPC classification. The datasets are described in table \ref{table:data_stats}.

\begin{table}[h]
\resizebox{0.45\textwidth}{!}{%
\begin{tabular}{lcc}
\hline
                    & \textbf{\# instances} & \textbf{\# avg label per example} \\ \hline
training set        & 6.7 million              & 1.47                              \\
patents testing set & 60k                      & 1.45                              \\
papers testing set  & 4956                     & 1.30                              \\ \hline
\end{tabular}%
}
\caption{Dataset Description}
\label{table:data_stats}
\end{table}
\vspace{-0.3cm}
\textbf{Evaluation metrics}: As each patent/paper has one or more CPC labels, we measure our model from both the classification and the ranking perspectives with 3 metrics: 
    (1) \textit{example-based precision/recall}: the average precision/recall per instance. We measure precision and recall on the top1, top3 predictions and  precision on all predictions with the probability score $\geq 0.5$.
    (2) \textit{macro precision/recall}: the average precision/recall per class.  
    (3) \textit{mean average precision (MAP)}: a ranking-based metric that measures whether the right labels are placed before the wrong ones.

% For a detailed explanation of the metrics, we refer the readers to a comprehensive overview the multi-label classification by Zhang and Zhou \shortcite{zhang:overview}

\begin{table*}[t]
\centering
\resizebox{0.85\textwidth}{!}{
\begin{tabular}{lllllllll}
\hline
\textbf{Model} & \textbf{p@1}   & \textbf{r@1}   & \textbf{p@3}   & \textbf{r@3}   & \textbf{\begin{tabular}[c]{@{}l@{}}precision\\ (prob\textgreater{}0.5)\end{tabular}} & \textbf{\begin{tabular}[c]{@{}l@{}}Macro\\ p@1\end{tabular}} & \textbf{\begin{tabular}[c]{@{}l@{}}Macro\\ r@1\end{tabular}} & \textbf{MAP}   \\ \hline
textCNN        & 75.51          & 55.24          & 40.64          & 78.92          & 62.16                                                                                 & 63.77                                                        & 32.15                                                        & 75.68          \\
attention-textCNN       & 70.63          & 51.49          & 38.11          & 74.17          & 51.69                                                                                 & 56.98                                                        & 26.15                                                        & 72.46          \\ \hline
WnD              & \textbf{77.11} & \textbf{56.42} & \textbf{41.18} & \textbf{80.12} & \textbf{64.01}                                                                        & \textbf{67.81}                                               & \textbf{34.18}                                               & \textbf{76.60} \\ \hline
\end{tabular}
}
\caption{Test results on patents. For attention-CNN we used the implementation of attention mechanism from \cite{wang2018joint}}
\label{table:exp_patents}
\end{table*}

\begin{table*}[h]
\centering
\resizebox{0.85\textwidth}{!}{
\begin{tabular}{lllllllll}
\hline
\textbf{Model}         & \textbf{p@1}   & \textbf{r@1}   & \textbf{p@3}   & \textbf{r@3}   & \textbf{\begin{tabular}[c]{@{}l@{}}precision \\ (prob \textgreater 0.5)\end{tabular}} & \textbf{\begin{tabular}[c]{@{}l@{}}Macro \\ p@1\end{tabular}} & \textbf{\begin{tabular}[c]{@{}l@{}}Macro \\ r@1\end{tabular}} & \textbf{MAP}   \\ \hline
textCNN                & 63.18          & 53.93          & 33.76          & 81.91          & 48.00                                                                                 & 16.07                                                         & 11.19                                                         & 71.76          \\
WnD                      & 66.16          & 56.73          & 34.15          & 82.86          & 50.09                                                                                 & 16.77                                                         & 11.74                                                         & 75.04          \\
KNN+reranking          & 63.99          & 54.98          & 32.85          & 79.18          & NA                                                                                    & 14.70                                                         & 10.60                                                         & 72.37          \\
Google Patent* &\textbf{69.59}&\textbf{59.95}&NA&NA&NA&14.28&10.25&NA \\
 \hline
WnD   + transfer         & 68.33          & 58.68          & 35.25          & 85.10          & \textbf{51.26}                                                                        &17.23                                                & 11.45                                                & 77.15          \\
WnD  +transfer+Prototype & 68.94 & 59.12 & \textbf{35.60} & \textbf{85.87} & 51.11                                                                                 & \textbf{17.63}                                                         & \textbf{12.39}                                                         & \textbf{77.84} \\
WnD  +transfer+labelExpand  & 68.69          & 58.96          & 35.31          & 85.22          & 50.37                                                                                 & 17.16                                                         & 11.71                                                         & 77.53          \\ \hline
\end{tabular}}
\caption{Test results on papers. textCNN and WnD are models trained on patents directly apply to papers. WnD+transfer refers to WnD fine tuned on auto-labelled papers.*The training set and granularity of Google Patent model may be different from other models. We put it here for the convenience to compare and discuss}
% The precision score for prob score $\geq 0.5$ is not available for KNN+reranking because it's not a probabilistic model}
\label{table:exp:papers}
\end{table*}

\subsection{Classify Patents to CPC}
\label{exp:patents}
We compare our WnD classifier on patents with two baselines: traditional textCNN and attention-textCNN. By comparing WnD with  textCNN, we want to know whether the label-text feature can complement semantic information for classification; by comparing with attention-textCNN, we want to compare our label integration method with other label embedding-based methods. For the attention model, we borrowed the idea of  label-embedding attentive model  \cite{wang2018joint}. The attention is a $T$-dimensional vector where $T$ is the text length. It calculates the importance of each word to the classification task.


The WnD achieves significant gains with label information (see table \ref{table:exp_patents}). It suggests that the complementary effects of string-level relatedness between label and texts indeed benefits the final classification decision. Our model also outperforms attention-textCNN. Although label embeddings are helpful for small label sets (around 10 labels) \cite{wang2018joint,honglun:2018}, it is less effective on hundreds of labels. We suspect the reason is that the attention is not discriminative between classes. When the label set is large, many non-stop words may be important for classification. But their weights should vary for different classes, which can hardly be captured by the attention vector. Also, attention-textCNN has much more parameters than textCNN and tends to overfit on the training data.


The best reported numbers on patent-to-IPC in subclass level achieves  0.74 on precision \cite{Verberne2011}. Although can not be directly compared, our large-enough testing set makes it confident that we are comparable with the state-of-the-art system while being more scalable. 

\subsection{Classify Papers to CPC}
\label{exp:papers}
There is not enough labelled data for the paper-to-CPC task. We can directly apply the model trained on patents to papers, but the performance will degrade significantly due to domain difference. For example, the word \textit{camera} in papers is commonly referred to as \textit{photo capturing apparatus} in patents. To deal with the domain-adaptation issue, we propose two approaches:

\textbf{distant supervision}: We auto-label papers using patent-paper citation. For each paper, we label it with the CPCs of the patents that cite it. We assume that papers cited by a patent should be relevant to the given patent in terms of background and technology domain. We get a total 1.7 million papers with abstract that are cited by the patents in the USPTO patent set, and each paper gets on average 2.8 labels. On these auto-labelled papers we then  fine tune the model originally trained on patents.

\textbf{domain-adapted features}: The WnD structure enables us to incorporate domain-adapted features to the Wide part. Here we proposed two ways to add such features: 
\begin{itemize}
    \item \textit{prototyping}: We pick the top 20 terms from the papers for each of the K classes according to the tf-idf score. Those representative terms are used as the label descriptions for papers;
    \item \textit{label expansion}: We train word embeddings using skip-gram \cite{Mikolov2013} on papers and expand the original label descriptions with the 10 nearest words in the embedding space according to their cosine distance. 
\end{itemize}


We compare our paper-to-CPC model with the best-performing KNN+reranking model \cite{Xiao2008} introduced in section \ref{patent cls}. We also want to compare our model with large-scale classification systems used in industry. In order to do that, we crawl from Google Patent the machine-classified CPC labels of scholar papers for our testing set. Google classifies papers on the finest level, but we use only the subclass part of the first label for comparison.
 
 The comparison results are shown in table \ref{table:exp:papers}. WnD benefits from both transfer learning and domain-adapted features. Since the prototypes are automatically collected, it is possible to apply WnD to classification tasks where detailed label descriptions are not available.
 
 Google Patent scores better on precision/recall@1, but performs less well on macro precision/recall. Since Google classifies the papers on a finer grained level, the classifier may receive more information during training, thus performing better on coarser grained levels. To investigate the effects of classification granularity on performance, we mapped the WnD subclass predictions to class level (128 label) and trained another WnD on class level. The p@1 and r@1 are $86.94\%$ and $79.15\%$ for WnD subclass-to-class and $82.43\%$ and $74.72\%$ for WnD class. The gap indicates the possible positive effects of fine-granularity training. 
 
%  \begin{table}[]
% \centering
% \resizebox{0.3\textwidth}{!}{%
% \begin{tabular}{lll}
% \hline
% \textbf{Model} & \textbf{p@1} & \textbf{r@1} \\ \hline
% Google Patent  & 84.46        & 76.14        \\
% WnD\_subclass  & 86.94        & 79.15        \\
% WnD\_class     & 82.43        & 74.72        \\ \hline
% \end{tabular}%
% }
% \caption{Test on class (the second) level. WnD_subclass maps the subclass predictions to class level.}
% \label{tab:google}
% \vspace{-0.3}
% \end{table}
 
 
 
 




% \subsection{Comparison with Google Patent}

% It's important to test our model with industrial application already served online. For this purpose, Google Patent also indexed scholar articles with CPCs. It is reasonable to compare the WnD   classifier with the current application performance in industry. However, Google patents trained their classifiers on subgroup level, a much more finer-grained level. If we only evaluate on the subclass level, the comparison would be unfair because classifiers trained on fine-grained labels tend to perform better on coarse-grained labels \cite{finegrained}. As a result, we train the WnD   classifier on the subgroup level in section G on 2 million patents, and aggregate the subgroup predictions to subclass level. The result is presented in table \ref{tab:google_res}.

% \begin{table}[h]
% \centering
% \resizebox{0.45\textwidth}{!}{%
% \begin{tabular}{lllll}
% \hline
% \textbf{Model} & \textbf{p@1} & \textbf{r@1} & \textbf{\begin{tabular}[c]{@{}l@{}}Macro \\ p@1\end{tabular}} & \textbf{\begin{tabular}[c]{@{}l@{}}Macro \\ r@1\end{tabular}} \\ \hline
% Google Patent  & 68.62        & 64.57        & 29.29                                                         & 20.68                                                         \\
% WnD  \_aggregate & 72.87        & 68.34        & 31.95                                                         & 26.23                                                         \\ \hline
% \end{tabular}%
% }
% \caption{Testing results on 2.2k papers under section G on subclass level}
% \label{tab:google_res}
% \end{table}

% Under section G, the WnD   classifier trained on finer grainularity outperforms the Google Patent results. However, the training process is extremely slow due to the exploding number of parameters. There are 37k subclass labels under section G, and the parameters for the last layer increase exponentially. It's our future work to enhance the scalability of the model to classify on extremely large label sets.

