---
title: "Data preparation for merging sbn_combined.csv and reading data from Frank et al. (2013)"
author: ""
---

We also include other information useful for psycholinguists - namely, word frequency databases from van Heuven et al. (2014).

```{r setup, include=FALSE}
knitr::opts_chunk$set(echo = TRUE)
```

```{r}
library(tidyverse)
```

Load data.

```{r}

# freq taken from:
# van Heuven, W. J. B., Mandera, P., Keuleers, E., & Brysbaert, M. (2014). SUBTLEX-UK: A new and improved word frequency database for British English. The Quarterly Journal of Experimental Psychology, 67(6), 1176–1190. https://doi.org/10.1080/17470218.2013.850521
# see here: https://osf.io/zq49t (the original file is in xlsx, we save it as csv)

freq <- read_table("subtlex_uk.csv")
spr <- read_table("frank_etal/selfpacedreading.RT.txt")
et <- read_table("frank_etal/eyetracking.RT.txt")

print(freq, n=10)
print(spr, n=10)
print(et, n=10)

spr$exp <- "spr"
et$exp <- "et"
spr$RTfirstfix <- NA
spr$RTfirstpass <- NA
spr$RTrightbound <- NA
spr$RTgopast <- NA
et$RT <- NA

```

How many sentences did people read in et (eye tracking) and spr (self-paced reading)?

```{r}
print(et %>% group_by(subj_nr) %>% summarise(n_sent = length(levels(as.factor(sent_nr)))), n=20)
print(et %>% group_by(subj_nr) %>% summarise(n_sent = length(levels(as.factor(sent_nr)))) %>% ungroup() %>% summarise(mean(n_sent)), n=20)
print(spr %>% group_by(subj_nr) %>% summarise(n_sent = length(levels(as.factor(sent_nr))), mean(n_sent)), n=20)
print(spr %>% group_by(subj_nr) %>% summarise(n_sent = length(levels(as.factor(sent_nr)))) %>% ungroup() %>% summarise(mean(n_sent)), n=20)
```

How many participants read each sentence in et and spr?

```{r}

print(et %>% group_by(sent_nr) %>% summarise(n_subj = length(levels(as.factor(subj_nr)))), n=Inf)
print(et %>% group_by(sent_nr) %>% summarise(n_subj = length(levels(as.factor(subj_nr)))) %>% ungroup() %>% summarise(mean(n_subj)), n=Inf)
print(spr %>% group_by(sent_nr) %>% summarise(n_subj = length(levels(as.factor(subj_nr)))), n=Inf)
print(spr %>% group_by(sent_nr) %>% summarise(n_subj = length(levels(as.factor(subj_nr)))) %>% ungroup() %>% summarise(mean(n_subj)), n=Inf)
```

Preparation for merging with frequency measures and with PMB.

```{r}
et <- et %>% mutate(subj_nr=paste("ET", as.character(subj_nr), sep="_"))
reading <- rbind(spr, et)
# remove punctuation and do lower case for merging purposes
reading$wordstripped <- gsub("[[:punct:]]", "", reading$word)
reading$Spelling <- tolower(reading$wordstripped)
reading <- left_join(reading, freq, by="Spelling")

reading <- reading %>% select(-c("Spelling"))

reading$logfreqscaled <- scale(log(reading$FreqCount))[,1]

```

Load pmb.

```{r}

pmb <- read_tsv("data/merged/sbn_combined.tsv")
print(pmb, n=10, width=Inf)
tail(pmb, n=20)

```

# Merging

We now merge the two datasets.

```{r}

# the word info in pmb is stored can consist of up to several words; split those
pmb <- pmb %>% group_by(Word) %>% mutate(Word1 = strsplit(Word, "\\s+")[[1]][2], , Word2 = strsplit(Word, "\\s+")[[1]][3], Word3 = strsplit(Word, "\\s+")[[1]][4], Word4 = strsplit(Word, "\\s+")[[1]][5], Word5 = strsplit(Word, "\\s+")[[1]][6])

# store character position info, matching pattern [n-n] in CharactersOrder column
pmb <- pmb %>% mutate(
	CharactersOrder = case_when(
		grepl("^\\[\\d+-\\d+\\]$", Word1) ~ Word1,
		grepl("^\\[\\d+-\\d+\\]$", Word2) ~ Word2,
		grepl("^\\[\\d+-\\d+\\]$", Word3) ~ Word3,
		grepl("^\\[\\d+-\\d+\\]$", Word4) ~ Word4,
		grepl("^\\[\\d+-\\d+\\]$", Word5) ~ Word5,
		TRUE ~ NA_character_
	),
	Word1 = ifelse(grepl("^\\[\\d+-\\d+\\]$", Word1), NA, Word1),
	Word2 = ifelse(grepl("^\\[\\d+-\\d+\\]$", Word2), NA, Word2),
	Word3 = ifelse(grepl("^\\[\\d+-\\d+\\]$", Word3), NA, Word3),
	Word4 = ifelse(grepl("^\\[\\d+-\\d+\\]$", Word4), NA, Word4),
	Word5 = ifelse(grepl("^\\[\\d+-\\d+\\]$", Word5), NA, Word5)
)

# some words are the core of the meaning; these are marked as not dashed in the Word column; we choose those words as CoreWord
pmb <- pmb %>% mutate(CoreWord =
	case_when(
	!grepl("-", Word1) ~ Word1,
	!grepl("-", Word2) ~ Word2,
	!grepl("-", Word3) ~ Word3,
	!grepl("-", Word4) ~ Word4,
	!grepl("-", Word5) ~ Word5,
	T ~ NA))

print(pmb, width=Inf)

# uniquepos ensures that we merge the right token (in case more than one token appears in the same sentence, they are ordered).
reading <- reading %>% group_by(subj_nr, sent_nr, wordstripped) %>% mutate(uniquepos=1:length(wordstripped)) %>% ungroup()

print(filter(reading, sent_nr==100 & exp=="et"), n=10, width=Inf) # the appears 2x in the sentence, is labeled 1 and 2 by uniquepos; note that this is even though the subject did not attend those words in ET (RTs are 0)

# We currently merge by CoreWord; another option is to merge by Word1, Word2, or Word3
pmb <- pmb %>% group_by(sent_nr, CoreWord) %>% mutate(uniquepos=1:length(CoreWord)) %>% ungroup()
print(filter(pmb, sent_nr==100))

pmb$wordstripped <- pmb$CoreWord

merged <- inner_join(pmb, reading, by=c("sent_nr", "wordstripped", "uniquepos"))

# If you use Word1, Word2 and Word3 together, then uncomment this part

#pmb <- pmb %>% group_by(sent_nr, Word1) %>% mutate(uniquepos1=1:length(Word1)) %>% ungroup()
#pmb <- pmb %>% group_by(sent_nr, Word2) %>% mutate(uniquepos2=1:length(Word2)) %>% ungroup()
#pmb <- pmb %>% group_by(sent_nr, Word3) %>% mutate(uniquepos3=1:length(Word2)) %>% ungroup()

#pmb$wordstripped <- pmb$Word1

# uniquepos1 is used also for match
#pmb$uniquepos <- pmb$uniquepos1

#part1 <- inner_join(pmb, reading, by=c("sent_nr", "wordstripped", "uniquepos"))

#part1$basedonword <- 1

#print(part1, width=Inf)

#pmb$wordstripped <- pmb$Word2

# uniquepos2 is used also for match
#pmb$uniquepos <- pmb$uniquepos2

#part2 <- inner_join(pmb, reading, by=c("sent_nr", "wordstripped", "uniquepos"))

#part2$basedonword <- 2

#pmb$wordstripped <- pmb$Word3

# uniquepos3 is used also for match
#pmb$uniquepos <- pmb$uniquepos3

#part3 <- inner_join(pmb, reading, by=c("sent_nr", "wordstripped", "uniquepos"))

#part3$basedonword <- 3

#merged <- rbind(part1, part2, part3)

print(merged, width=Inf)
# check that words match and are at the right word_pos
print(subset(merged, sent_nr==100) %>% group_by(word) %>% summarise(first(CoreWord), first(wordstripped), first(word_pos)), width=Inf)

# remove auxiliary columns

merged <- merged %>% select(-c(uniquepos, wordstripped))

# reorder data by sent_nr and word_pos
merged <- merged %>% arrange(exp, subj_nr, sent_nr, word_pos)
```

We store the results.

```{r}

write_tsv(merged, "UCL_corpus_with_PMB_and_frequencies.tsv")

```

We store the same results, but without the frequency information.

```{r}

merged_small <- merged %>% select(-c(FreqCount:ncol(merged)))
write_tsv(merged_small, "UCL_corpus_with_PMB_without_frequencies.tsv")

```
