RC subextraction in English: Set-up notebook for Experiment 6

library(ggplot2)
library(dplyr)

Attaching package: ‘dplyr’

The following objects are masked from ‘package:stats’:

    filter, lag

The following objects are masked from ‘package:base’:

    intersect, setdiff, setequal, union

library(tidyr)
library(magrittr)

Attaching package: ‘magrittr’

The following object is masked from ‘package:tidyr’:

    extract

library(anytime)
library(stringr)

# Define function for generating latin square given the # of items and # of conditions
latin.square <- function(n.items, n.conditions){
  # Define the height and width of the table in which the latin square will be stored
  ls_table <- matrix(nrow = n.items, ncol = n.conditions)
  # Make a vector to store the last cell's value
  last_cell <- vector(length = 0)
  # For each row associated w/ each item number, ...
  for (row in 1:n.items){
    # If "last" hasn't received a value yet, ...
    if(length(last_cell) == 0){
      # Then value the cells in the first row from 1 to n.conditions, and...
      ls_table[row,] <- 1:n.conditions
      # Assign the value of the last cell in the first row to "last".
      last_cell <- ls_table[row, n.conditions]
    } else{
      # If "last" has a value, assign to row the sequence from last value of last row, ending w/ n.conditions
      ls_table[row, 1:length(last_cell:n.conditions)] <- last_cell:n.conditions
      # If x is anything other than 1, ...
      if(last_cell != 1){
        # Then starting w/ the cell after the cell in row whose value matches n.conditions,
        # assign values to the remaining cells starting w/ 1 and going to the number before, ...
        ls_table[row,(length(last_cell:n.conditions)+1):n.conditions] <- 1:(n.conditions - length(last_cell:n.conditions))
        # and assign the value of the last cell in the first row to "last".
        last_cell <- ls_table[row, n.conditions]
      } else{
        # Otherwise (if last_cell is 1), just assign the value of the last cell in the first row to "last" (without )
        last_cell <- ls_table[row, n.conditions]
      }
    }
  }
  # Convert the matrix into a dataframe
  ls_table <- as.data.frame(ls_table)
  # Give the conditions letter labels (as column names)
  colnames(ls_table) <- letters[1:n.conditions]
  # Give the items numeric labels (as row names)
  row.names(ls_table) <- 1:n.items
  return(ls_table)
}

# Read in "results" file from IBEX (in csv format) for AJT controller
# Supply column names
read.csv("Results/results_2020-01-20(1).csv",
         header = FALSE,
         col.names = c("epoch_time",
                       "MD5",
                       "controller",
                       "IBEX_ID",
                       "element",
                       "type",
                       "item",
                       "question",
                       "rating",
                       "correct",
                       "rt")) -> ibex_import

# Replace %2C w/ commas and é w/ é
ibex_import[,"question"] %<>% str_replace("%2C", ",")
ibex_import[,"question"] %<>% str_replace("%0A", ";")
ibex_import[,"question"] %<>% str_replace("é", "é")

# Associate all sentences w/ IBEX_ID, remove duplicate rows
ibex_import %>%
  subset(controller == "AcceptabilityJudgment" & question != "NULL",
         select = c(IBEX_ID,
                    question)) %>%
  distinct() %>%
  arrange(IBEX_ID) -> ID_key

# Get ratings data
ibex_import %>%
  subset(controller == "AcceptabilityJudgment" & question == "NULL",
         select = c(epoch_time,
                    IBEX_ID,
                    type,
                    rating,
                    rt)) -> ratings_data_incl

# Add trial order information (trial order at this stage is still preserved in the order)
n <- length(unique(ratings_data_incl$epoch_time))
ratings_data_incl$trial <- rep(1:138, n)

# Get form data (intro and debrief)
ibex_import %>%
  subset(controller == "Form",
         select = c(epoch_time,
                    question,
                    rating)) %>%
  droplevels -> participant_data

# Change column names
colnames(participant_data) <- c("epoch_time", "question", "response")

# Change participant data to long format (one row per submission/subject)
participant_data %>%
  group_by(epoch_time) %>%
  summarize(prolific_id = str_sub(response[question == "prolificid"],
                                  -4, -1),
            age = response[question == "age"],
            native_eng = response[question == "languageEnglish"],
            other_langs = response[question == "otherLanguages"],
            ling_major = response[question == "linguistics"],
            consent = response[question == "consent"],
            what_about = response[question == "debrief1Response"],
            strategy = response[question == "debrief2Response"],
            patterns_noticed = response[question == "debrief3Response"],
            cp_vs_vt = response[question == "debrief4Response"],
            cp_vs_ee = response[question == "debrief5Response"],
            vt_vs_ee = response[question == "debrief6Response"],
            difficulty = response[question == "difficulty"],
            form_rts = sum(as.numeric(as.character(response[question == "_REACTION_TIME_"]))),
            seps = 500*(138-1)) -> participant_data

# Make epoch_time numeric (agnostic of which column it's in)
ratings_data_incl[,which(colnames(ratings_data_incl) == "epoch_time")] %>%
  as.character %>%
  as.numeric -> ratings_data_incl[,which(colnames(ratings_data_incl) == "epoch_time")]

# Import items, fillers, practice for both experiments
read.csv("Items/CSVs/exp6_items.csv") -> items
  items$frequency <- NULL
  items$context <- NULL
  items$sentence_w_gaps <- NULL
  items$comments <- NULL
read.csv("Items/CSVs/exp6_fillers.csv") -> fillers
read.csv("Items/CSVs/exp6_practice.csv") -> practice
read.csv("Items/CSVs/exp6_burn-in_items.csv") -> burn_in_items
  burn_in_items$frequency <- NULL
  burn_in_items$context <- NULL
  burn_in_items$sentence_w_gaps <- NULL
read.csv("Items/CSVs/exp6_burn-in_fillers.csv") -> burn_in_fillers
  burn_in_fillers$mean.z.rating. <- NULL
read.csv("Items/CSVs/exp6_attention.csv") -> attention_checks
  
# Associate conditions w/ their IBEX ID
merge(items,
      ID_key,
      by.x = "sentence",
      by.y = "question") -> items

merge(fillers,
      ID_key,
      by.x = "sentence",
      by.y = "question") -> fillers

merge(practice,
      ID_key,
      by.x = "sentence",
      by.y = "question") -> practice

merge(burn_in_items,
      ID_key,
      by.x = "sentence",
      by.y = "question") -> burn_in_items

merge(burn_in_fillers,
      ID_key,
      by.x = "sentence",
      by.y = "question") -> burn_in_fillers

merge(attention_checks,
      ID_key,
      by.x = "sentence",
      by.y = "question") -> attention_checks

# Generate latin square for experiment & convert to long format
lsq <- latin.square(n.items = 36, n.conditions = 6)
lsq <- cbind(item = as.numeric(row.names(lsq)), lsq)
lsq %>%
  gather(condition, list, a:f, factor_key = TRUE) -> lsq
lsq %>%
  arrange(item) -> lsq

# Add list number that conditions appeared in to exp items
items %>%
  merge(.,
        lsq,
        by = c("item", "condition")) -> items

# Create master list of items, fillers, and practice
## Make copies of dfs, and add/remove columns to make columns consistent
items -> items_m
  items_m$requested_rating <- NA
fillers -> fillers_m
  fillers_m$s_type <- NULL
  fillers_m$gramm_s_type <- NULL
  fillers_m$only <- NULL
  fillers_m$original_source <- NULL
  fillers_m$last_used <- NULL
  fillers_m$X <- NULL
  fillers_m$requested_rating <- NA
  fillers_m$condition <- NA
  fillers_m$length <- NA
  fillers_m$structure <- NA
  fillers_m$env <- NA
  fillers_m$list <- NA
  fillers_m$item <- NA
  fillers_m$verb <- NA
burn_in_items -> burn_in_items_m
  burn_in_items_m$requested_rating <- NA
  burn_in_items_m$list <- NA
burn_in_fillers -> burn_in_fillers_m
  burn_in_fillers_m$orig_ID <- NULL
  burn_in_fillers_m$orig_sentence <- NULL
  burn_in_fillers_m$requested_rating <- NA
  burn_in_fillers_m$condition <- NA
  burn_in_fillers_m$length <- NA
  burn_in_fillers_m$structure <- NA
  burn_in_fillers_m$env <- NA
  burn_in_fillers_m$list <- NA
  burn_in_fillers_m$item <- NA
  burn_in_fillers_m$verb <- NA
practice -> practice_m
  colnames(practice_m)[3] <- "gramm"
  practice_m$requested_rating <- NA
  practice_m$condition <- NA
  practice_m$length <- NA
  practice_m$structure <- NA
  practice_m$env <- NA
  practice_m$list <- NA
  practice_m$item <- NA
  practice_m$verb <- NA
attention_checks -> attention_checks_m
  attention_checks_m$in_IBEX <- NULL
  attention_checks_m$condition <- NA
  attention_checks_m$length <- NA
  attention_checks_m$structure <- NA
  attention_checks_m$env <- NA
  attention_checks_m$list <- NA
  attention_checks_m$item <- NA
  attention_checks_m$gramm <- NA
  attention_checks_m$verb <- NA

# Combine items, fillers, practice items, burn-in items, burn-in fillers, and attention checks
master <- rbind(items_m, fillers_m, burn_in_items_m, burn_in_fillers_m, practice_m, attention_checks_m)

# Match master to ratings data
ratings_data_incl %>%
  merge(.,
        master,
        by = "IBEX_ID") -> ratings_data_incl

# Associate epoch_time w/ subject number and native speaker status
ratings_data_incl %>%
  merge(.,
        participant_data[,1:2],
        by = "epoch_time") -> ratings_data_incl

# Save ratings data as csv
write.csv(ratings_data_incl, file = "exp6_ratings.csv")

# Find the list each subject was given
ratings_data_incl %>%
  subset(type == "experimental",
         select = c(prolific_id,
                    list)) %>%
  droplevels %>%
  group_by(prolific_id) %>%
  summarize(one_list = length(unique(list)) == 1,
            list = if(one_list == TRUE){
              min(list)
              } else {
                NA
              }) -> list_key

# Join list_key w/ participant_data
participant_data %>%
  merge(.,
        list_key,
        by = "prolific_id") -> participant_data

# Add sentence RT sums to participant data
ratings_data_incl %>%
  group_by(prolific_id) %>%
  summarize(stc_rts = sum(rt)) -> participant_rts

# Add RTs to participant_data & sum it up
merge(participant_data,
      participant_rts,
      by = "prolific_id") -> participant_data

apply(participant_data, 1,
      function(x){
        sum(as.numeric(as.character(x["form_rts"])),
            as.numeric(as.character(x["seps"])),
            as.numeric(as.character(x["stc_rts"]))) -> time
        return(time)
      }) -> participant_data$tot_time_ms

participant_data$tot_time_s <- participant_data$tot_time_ms/1000
participant_data$tot_time_m <- participant_data$tot_time_s/60

# Remove excluded participants from participant_data (SEE EXCLUDED PARTICIPANTS IN LIST BELOW)
participant_data %>%
  subset(prolific_id %in% excluded) -> participant_data_rm

participant_data %>%
  subset(!(prolific_id %in% excluded)) -> participant_data

# Determine how balanced the lists are. For this experiment, there are 6 lists.
participant_data %>%
  group_by(list) %>%
  summarize(counter = first(list) - 1,
            n = n()) -> list_dist

1 Exclusion criteria

A participant will be excluded if any of the following conditions are met: 1. They fail at least one of the six attention checks. 2. The proportion of response times lower than one second is greater than or equal to 0.10. 3. Their mean ratings for fillers expected to be ungrammatical and fillers expected to be grammatical are either inverted or are too close. Too close is defined on normalized ratings, where a difference that is more than two standard deviations below the mean difference is too close.

1.1 Attention checks

ratings_data_incl %>%
  subset(type == "attention") %>%
  group_by(epoch_time,
           prolific_id) %>%
  summarize(first = rating[ID == "a01"],
            second = rating[ID == "a02"],
            third = rating[ID == "a03"],
            fourth = rating[ID == "a04"],
            fifth = rating[ID == "a05"],
            sixth = rating[ID == "a06"])

1.2 Response times

# Generate RT density plots for each participant
for (n in 1:length(unique(ratings_data_incl$prolific_id))) {
  participant <- unique(ratings_data_incl$prolific_id)[n]
  ratings_data_incl %>%
    subset(prolific_id == participant) %>%
    group_by(IBEX_ID) %>%
    summarize(rt_s = rt/1000) %>%
    ggplot(aes(x = rt_s)) +
    geom_density() +
    geom_vline(aes(xintercept = mean(rt_s))) +
    xlim(xmin = 0,
         xmax = 10) +
    labs(title = paste("Prolific ID", participant, "response times"),
         x = "Response time (seconds)",
         y = "Density") -> plot
  print(plot)
}

# Get RT stats
ratings_data_incl %>%
  group_by(epoch_time,
           prolific_id) %>%
  summarize(mean_rt = mean(rt),
            p_below_1000 = length(rt[rt < 1000])/length(rt),
            min_rt = min(rt),
            max_rt = max(rt),
            median_rt = median(rt),
            n = n(),
            sd = sd(rt),
            se = sd/sqrt(n)) -> rt_stats
print(rt_stats)

# Print participants for whom 25% or more of their RTs are less than a second.
rt_stats %>%
  subset(p_below_1000 >= 0.10,
         select = c(epoch_time, prolific_id))

1.3 Filler ratings

# Select filler & experimental sentences and transform ratings to z-scores (grouping by participant)
ratings_data_incl %>%
  subset(type == "experimental" | type == "filler") %>%
  group_by(prolific_id) %>%
  mutate(z_rating = scale(as.numeric(as.character(rating)))) %>%
  ungroup() %>%
  # Select only the fillers
  subset(type == "filler") %>%
  # Group by participant and grammaticality & summarize ratings
  group_by(prolific_id,
           gramm) %>%
  summarize(mean_z_rating = mean(z_rating)) -> filler_z_means
print(filler_z_means)

# Get difference in mean z_ratings per participant and join w/ filler_z_means
filler_z_means %>%
  group_by(prolific_id) %>%
  summarize(diff = mean_z_rating[gramm == "1"] - mean_z_rating[gramm == "0"]) %>%
  merge(filler_z_means, .) -> filler_z_means

# Show a list of participants whose difference is over two SDs away from the mean
#filler_z_means %>%
#  subset(diff < mean_diff-(2*sd_diff))

filler_z_means %>%
  ggplot(aes(x = reorder(prolific_id, diff),
             y = mean_z_rating,
             color = as.factor(gramm))) +
  geom_point() +
  labs(x = "Prolific ID (first 4)",
       y = "Mean z-score-transformed rating") +
  scale_color_discrete("Grammaticality") +
  theme(axis.text.x = element_text(angle = 45, vjust = 0.95, hjust = 0.95)) -> filler_ratings_plot
print(filler_ratings_plot)

1.4 Make list of excluded participants

excluded <- NULL
excluded <- c(excluded, "134d") # Failed one attention check
excluded <- c(excluded, "d6ad") # Mean gramm and ungramm filler ratings too close; most RTs below 2.5s
excluded <- c(excluded, "d797") # Mean gramm and ungramm filler ratings too close