02_webscraperei.Rmd

---
title: "Webscraperei"
output: html_notebook
---
  
# Idea
  
## Load Packages
  
```{r}
pacman::p_load(tidyverse, wdman, jsonlite, rvest, xml2, RSelenium, here, crayon, here, magrittr)
```


```{r}
library(RSelenium)

port <- sample(4000L:5000L, 1)
rD <- rsDriver(verbose = FALSE, port = port)
rD

remDr <- rD$client


url <- "https://www.idea.int/advanced-search?th=Direct%20Democracy%20Database"
remDr$navigate(url)

checkboxes1 <- c("t46", "t47", "t48", "t49", "t50")

for (jj in checkboxes1) {
  webElem <- remDr$findElement(using = 'xpath', 
                               value = paste0(
                     "//input[@value='",jj ,"'][@type='checkbox']"
                               )
  )
  
  webElem$clickElement()
}

checkboxes2 <- c("africa", "americas", "asia", "europe", "oceania")

for (jj in checkboxes2) {
  webElem <- remDr$findElement(using = 'xpath', 
                               value = paste0(
                                 "//input[@value='",jj ,"'][@type='checkbox']"
                               )
  )
  
  webElem$clickElement()  
  
}



webElem <- remDr$findElement(using = 'css', 
                             value ="#submit-assisted-search")

webElem$clickElement()
```

## Webscraping

```{r}
library(rvest)

idea_html <- read_html("data/idea.html")

# labels <- idea_html %>% 
#   rvest::html_nodes(".country , .answer-value") %>% 
#   rvest::html_text()
# 
# codes <- idea_html %>% 
#   rvest::html_nodes(".country , .answer-value") %>% 
#   rvest::html_attr("data-nid")
# 
# idea_html %>%
#   rvest::html_nodes(".fixed-header div") %>%
#   rvest::html_text()
# 
# data.frame(labels, codes)
# 9108 / 198

cntry <- idea_html %>% 
  rvest::html_nodes(".country") %>% 
  rvest::html_text() %>% 
  rep(each=46)

codes <- idea_html %>% 
  rvest::html_nodes(".answer-value") %>% 
  rvest::html_attr("data-nid") %>% 
  as.character()

labels <- readxl::read_xls("data/idea.xls") %>% 
  names() %>% 
  .[-1] %>% 
  rep(198)

val_labs <- idea_html %>% 
  rvest::html_nodes(".answer-value") %>% 
  rvest::html_text() 

scrape_dat <- data.frame(labels, val_labs, cntry, codes, stringsAsFactors = F) %>% 
  mutate(val_labs = ifelse(val_labs == "", NA, val_labs)) %>% 
  mutate(urls = paste0("https://www.idea.int/node/", codes))



```

## Web Crawling

### Extract Dates

```{r}

years <- 1776:2018 %>% 
  as.character() %>% 
  paste0(collapse = "|")

# texts <- read_html(scrape_dat$urls[1]) %>% 
#   html_nodes("#block-system-main") %>% 
#   html_text()# 

# test <- scrape_dat %>% 
#   filter(codes %in% c("200518", "226549", "229780", "0")) %>% 
#   .[1:4,] %>% 
#   mutate(urls = ifelse(codes == "0", NA, urls))

scrape_dat %<>% 
  mutate(urls = ifelse(codes == "0", NA, urls))

source("helper_function.R")



length(lastyear)

start <- 1
end <- 9108


year_dat <- scraper_of_the_year(scrape_dat, start, end) 

scraperei <- cbind(scrape_dat, year_dat)

save(scraperei, file = "data/scraperei.Rdata")
# 
# str_extract_all("hello this is one date 1999, this wouldn't be a date 3456 but this is one 1890", years)



```


## Take a look


```{r}
load("data/scraperei.Rdata")

scraperei %>% 
  filter(nyears == 2 & firstyear == lastyear) %>% 
  arrange(desc(nyears))

scraperei %>% 
  filter(nyears == 1) %>% 
  arrange(desc(nyears))

scraperei %>% 
  drop_na(firstyear)
```

# Sudd Data

## Load Packages

```{r}
pacman::p_load(tidyverse, magrittr, sjmisc, stringr)

```

## Webscraping

```{r}
library(rvest)


ref_name <- read_html("data/sudd.html") %>% 
  html_nodes(".leer a") %>% 
  html_text()

urls <- read_html("data/sudd.html") %>% 
  html_nodes(".leer a") %>% 
  html_attr("href")

dates <- read_html("data/sudd.html") %>% 
  html_nodes(".leer:nth-child(3)") %>% 
  html_text()

cntry <- read_html("data/sudd.html") %>% 
  html_nodes(".leer:nth-child(2)") %>% 
  html_text()

scrape_sudd <- data.frame(cntry, dates, ref_name, urls, stringsAsFactors = F)

read_html("data/sudd.html") %>% 
  html_nodes("td") %>% 
  html_table()

source("helper_function.R")

tablestyle <- read_html("https://www.sudd.ch/event.php?lang=de&id=nl012018") %>% 
  html_nodes("table") %>% 
  html_table() %>% 
  .[[1]] %>% 
  dplyr::mutate_all(fix_the_string)

rownames(tablestyle) <- tablestyle$X1

tablestyle %>% 
  dplyr::select(-X1, -X3) %>% 
  t() %>% 
  as_tibble() %>% 
  janitor::clean_names()

```

## Webcrawling

```{r}


set.seed(2018)

# test <- scrape_sudd %>% 
#   sample_n(200)

start <- 1
end <- 2813
source("helper_function.R")
sudd_dat <- scraper_of_the_dats(scrape_sudd, start, end) %>%
  bind_rows()

head(names(sudd_dat), 40)

save(sudd_dat, file = "data/sudd_dat.Rdata")

# sudd_dat[,1:25]
# 
# selector <- function(variables) {
#  ss <- data.frame(table(variables))
# 
#  ss$Freq[2]
# }
# 
# load("data/sudd_dat.Rdata")
# 
# detecter <- sudd_dat %>%
#   mutate_all(is.na) %>%
#   mutate_all(selector) %>%
#   .[1,] %>%
#   t() %>%
#   as.data.frame()
# 
# vars_ex <- detecter %>%
#   mutate(var_names = rownames(detecter)) %>%
#   filter(V1 < 1000) %>%
#   select(var_names)
# 
# sudd_dat <- sudd_dat %>%
#   select(vars_ex$var_names)

sudd_dat %<>% 
  select(1:30)
```

#### Take a look

```{r}
load("data/sudd_dat.Rdata")

# years <- 1776:2018 %>% 
#   as.character() %>% 
#   paste0(collapse = "|")
# 
# sudd_dat %<>% 
#   mutate(date = str_extract(datum, years)) %>% 
#   group_by(gebiet, date) %>% 
#   tally() %>% 
#   mutate(date = as.numeric(date)) 
# 
# 
# sudd_dat %>% 
#   ggplot(aes(x = date, n, colour = gebiet)) +
#   geom_line() +
#   guides(colour = F)
# 
# sudd_dat %>% 
#   group_by(gebiet) %>% 
#   mutate(cumn = cumsum(n)) %>% 
#   filter(gebiet != "Schweiz") %>% 
#   ggplot(aes(x = date, cumn, colour = gebiet)) +
#   geom_line() +
#   guides(colour = F)

sudd_dat %<>% 
  mutate(results = case_when(
    stringr::str_detect(ergebnis, "verworfen") ~ "unsuccessful",
    stringr::str_detect(ergebnis, "keine Option angenommen") ~ "unsuccessful",
    stringr::str_detect(ergebnis, "angenommen") ~ "successful",   
    stringr::str_detect(ergebnis, "keine Mehrheit") ~ "unsuccessful",
    stringr::str_detect(ergebnis, "kassiert") ~ "unsuccessful",
    TRUE ~ NA_character_
  )) %>% 
  mutate(success = ifelse(results == "successful", 1, 0))

#dput(unique(sudd_dat$results))

table(sudd_dat$success)

sudd_dat %<>% 
  mutate(participation = stringr::str_replace_all(stimmbeteiligung, "'", "")) %>% 
  mutate(participation = ifelse(participation == "---", NA_character_, participation) %>% 
           as.numeric()) %>% 
  mutate(total_peeps = stringr::str_replace_all(stimmberechtigte, "'", "")) %>% 
  mutate(total_peeps = ifelse(total_peeps == "---", NA_character_, total_peeps) %>% 
           as.numeric()) %>% 
  mutate(ratio = participation/total_peeps) %>% 
  filter(gebiet != "Maghrebinien")


fit1 <- glm(success ~ ratio, data = sudd_dat)

sjPlot::plot_model(fit1, type = "eff", terms = "ratio")

sjPlot::sjp.glm(fit1, type = "dots")

sudd_dat %>% 
  cbind(data.frame(fitted = predict(fit1, sudd_dat))) %>% 
  filter(fitted > 1)

max(sudd_dat$fitted, na.rm = T)

sudd_dat %>% 
   ggplot(aes(ratio, fitted)) +
   geom_smooth()

sudd_dat %>% 
  filter(ratio < 0.2)

sudd_dat %<>% 
#  select(stellung) %>% 
  # BY WHOM?!
  mutate(entity = case_when(
    stellung == "unabhaengiger Staat" ~ "independent",
    TRUE ~ "dependent"
  )) %>%   
  mutate(bottom_up = case_when(
    str_detect(abstimmungstyp, "durch Volk") ~ "1",
    str_detect(abstimmungstyp, "durch Parlament") ~ "0",
    str_detect(abstimmungstyp, "durch Staatsrat") ~ "0",
    str_detect(abstimmungstyp, "durch Parlamentsminderheit") ~ "0",
    str_detect(abstimmungstyp, "durch Regionalraete") ~ "0",
    str_detect(abstimmungstyp, "durch Senatsminderheit") ~ "0",
    str_detect(abstimmungstyp, "durch Regierung des Mutterlands") ~ "0",
    str_detect(abstimmungstyp, "durch Verfassungsrat") ~ "0",
    str_detect(abstimmungstyp, "durch Gemeinden") ~ "0",
    str_detect(abstimmungstyp, "durch Gemeinderaete") ~ "0",
    str_detect(abstimmungstyp, "durch Praesident") ~ "0",
    str_detect(abstimmungstyp, "durch Regierung") ~ "0",
    str_detect(abstimmungstyp, "durch Exekutive") ~ "0",
    str_detect(abstimmungstyp, "durch Emir") ~ "0",
    str_detect(abstimmungstyp, "durch C\034bergangsparlament") ~ "0",    
    str_detect(abstimmungstyp, "durch Gouverneur") ~ "0",
    str_detect(abstimmungstyp, "durch Frankreich/BRD") ~ "0",
    str_detect(abstimmungstyp, "durch Generalgouverneur") ~ "0",
    str_detect(abstimmungstyp, "durch Kaiser") ~ "0",
    str_detect(abstimmungstyp, "durch KC6nig") ~ "0",
    str_detect(abstimmungstyp, "durch ZentralbehC6rden") ~ "0",
    str_detect(abstimmungstyp, "durch Administrator") ~ "0",
    str_detect(abstimmungstyp, "durch Fuerst") ~ "0",
    str_detect(abstimmungstyp, "durch KolonialbehC6rden") ~ "0",
    str_detect(abstimmungstyp, "durch Militaerrat") ~ "0",
    str_detect(abstimmungstyp, "durch Lokalregierung") ~ "0",
    str_detect(abstimmungstyp, "durch Militaerregierung") ~ "0",
    str_detect(abstimmungstyp, "durch Regenz") ~ "0",
    str_detect(abstimmungstyp, "durch Regent") ~ "0",
    str_detect(abstimmungstyp, "durch Schah") ~ "0",
    str_detect(abstimmungstyp, "durch Sultan") ~ "0",
    str_detect(abstimmungstyp, "durch US-Regierung") ~ "0",
    str_detect(abstimmungstyp, "durch UNO") ~ "0",
    str_detect(abstimmungstyp, "durch WahlbehC6rde") ~ "0",
    str_detect(abstimmungstyp, "durch Konsul") ~ "0",
    str_detect(abstimmungstyp, "durch Ministerium") ~ "0",
    str_detect(abstimmungstyp, "durch Inselrat") ~ "0",
    str_detect(abstimmungstyp, "durch Notablenversammlung") ~ "0",
    str_detect(abstimmungstyp, "durch Staatspraesident") ~ "0",
    str_detect(abstimmungstyp, "durch Stadtregierung") ~ "0",
    str_detect(abstimmungstyp, "durch VC6lkerbund") ~ "0",
    str_detect(abstimmungstyp, "durch Verwaltung") ~ "0",
    str_detect(abstimmungstyp, "durch Ministerpraesident") ~ "0",
    str_detect(abstimmungstyp, "durch Automatisch") ~ "0",
    str_detect(abstimmungstyp, "durch Tagsatzung") ~ "0",
    str_detect(abstimmungstyp, "durch Sejm") ~ "0",
    str_detect(abstimmungstyp, "durch Caudillo") ~ "0",
    TRUE ~ NA_character_
  )) 





sudd_dat %<>% 
  # What is being decided?
  mutate(level = ifelse(str_detect(abstimmungstyp, "Stufe:"), 
                        str_extract(abstimmungstyp, "Stufe:.*$") %>% 
                          stringr::str_replace(., "Stufe: ", ""), NA)) #%>% #there is a few NA that need to be fixed
  # select(level) %>% 
  # table()

sudd_dat %<>% 
    # Bindingness?
  mutate(bindingness = case_when(
    str_detect(abstimmungstyp, "nicht bindend") ~ "non-binding",
    str_detect(abstimmungstyp, "je nach Stimmverhalten bindend") ~ "sometimes binding",    
    str_detect(abstimmungstyp, "bindend") ~ "binding",
    str_detect(abstimmungstyp, "ad hoc") ~ "ad hoc",
    TRUE ~ NA_character_))


sudd_dat %<>% 
    # Type?
  mutate(type = case_when(
    str_detect(abstimmungstyp, "Inoffizielle Abstimmung") ~ "inofficial",
    str_detect(abstimmungstyp, "Plebiszit") ~ "plebsicite",    
    str_detect(abstimmungstyp, "Obligatorisch") ~ "obligatory",
    str_detect(abstimmungstyp, "Fakultativ") ~ "non-obligatory",
    str_detect(abstimmungstyp, "Initiative") ~ "initiative",
    TRUE ~ abstimmungstyp)) 

sudd_dat %>%
  select(entity, mehrheiten, bottom_up, abstimmungstyp, level, bindingness, type) %>%
  filter(entity == "independent") %>%
  select(type, bindingness) %>%
  table()

sudd_dat %>%
  filter(entity == "independent") %>%
  filter(bindingness == "ad hoc") 



```


#### Test it

```{r}


tablestyle <- read_html("https://www.sudd.ch/event.php?lang=de&id=bz012012") %>% 
  html_nodes("table") %>% 
  html_table(fill = TRUE) %>% 
  .[[1]] %>% 
  mutate(X1 = fix_the_string(X1)) %>% 
  mutate_cond(condition = X1 %nin% c("Bemerkungen", "Quellen"), 
              X2 = fix_the_string(X2)) %>% 
  mutate_cond(condition = X1 %nin% c("Bemerkungen", "Quellen"), 
              X3 = fix_the_string(X3))
# 
# stringi::stri_enc_isascii(tablestyle$X1)
# is.character(tablestyle$X1)
# 
# suppressMessages(duplicates <- tablestyle %>% 
#   janitor::get_dupes(X1))
library(sjmisc)


tablestyle
rownames(tablestyle) <- tablestyle$X1 

final_tab <- tablestyle %>% 
  dplyr::select(-X1, -X3) %>% 
  t() %>% 
  as_tibble() %>% 
  janitor::clean_names()

stringi::stri_enc_isascii(tablestyle$X2)

fix_the_string <- function(string) {
  string <- tablestyle$X2
  detect <- stringi::stri_enc_mark(string) 
  utf8_det <- ifelse(detect == "UTF-8", T, F)
  ASCII_det <- ifelse(detect == "ASCII", T, F)
  
  if (ASCII_det)
    
    case_when(
      ifelse(detect == "ASCII", T, F) ~   ,
    
    TRUE ~ string[utf8_det]
    )
  
 string[ASCII_det] %>%   
    iconv(from = "ASCII", to = "utf-8") %>% 
    stringr::str_remove_all("b\024\027b\024\001 ") %>% 
    stringr::str_replace_all("C\004", "ae") %>% 
    stringr::str_replace_all("C\\$", "ae") %>% 
    stringr::str_replace_all("C<", "ue") %>% 
    stringr::str_replace_all("b\006\022", "-")
    
  string[utf8_det] %>%   
    iconv(from = "utf-8", to = "ASCII") %>% 
    stringr::str_remove_all("b\024\027b\024\001 ") %>% 
    stringr::str_replace_all("C\004", "ae") %>% 
    stringr::str_replace_all("C\\$", "ae") %>% 
    stringr::str_replace_all("C<", "ue") %>% 
    stringr::str_replace_all("b\006\022", "-")  
  
  if (any(stringi::stri_enc_mark(tablestyle$X2))
  tablestyle$X2 %>%   
    iconv(from = "ASCII", to = "utf-8") %>% 
    stringr::str_remove_all("b\024\027b\024\001 ") %>% 
    stringr::str_replace_all("C\004", "ae") %>% 
    stringr::str_replace_all("C\\$", "ae") %>% 
    stringr::str_replace_all("C<", "ue") %>% 
    stringr::str_replace_all("b\006\022", "-")     
}
```


# ace data

```{r}

urls <- c("http://aceproject.org/epic-en/CDTable?question=DD003&view=country&set_language=en",
  "http://aceproject.org/epic-en/CDTable?question=DD004&view=country&set_language=en",
  "http://aceproject.org/epic-en/CDTable?question=DD005&view=country&set_language=en",
  "http://aceproject.org/epic-en/CDTable?question=DD006&view=country&set_language=en",
  "http://aceproject.org/epic-en/CDTable?question=DD007&view=country&set_language=en",
  "http://aceproject.org/epic-en/CDTable?question=DD129&view=country&set_language=en",
  "http://aceproject.org/epic-en/CDTable?question=DD130&view=country&set_language=en",
  "http://aceproject.org/epic-en/CDTable?question=DD131&view=country&set_language=en")

ace_html <- read_html("http://aceproject.org/epic-en/CDTable?question=DD004&view=country&set_language=en")
var_name <- ace_html %>% 
  rvest::html_nodes("h3") %>% 
  html_text()

ace_data <- ace_html %>% 
  rvest::html_nodes("table .CDlisting") %>% 
  html_table()

ace_data <- ace_data[[1]]

names(ace_data) <- c("cntry", var_name, "comments", "verified")


# ace_dat <- ace_data %>% 
#   janitor::clean_names() %>% 
#   select(cntry, are_there_any_legal_provisions_for_mandatory_referendums_at_the_national_level)

cbind(ace_dat, ace_data %>% 
  janitor::clean_names() %>% 
  select(are_there_any_legal_provisions_for_optional_referendums_at_the_national_level))

```


# direct democracy navigator

```{r}
library(rvest)
navigator_html <- read_html("http://www.direct-democracy-navigator.org/countries?view=table")

navigator_dat <- navigator_html %>% 
  rvest::html_nodes("table") %>% 
  html_table()

navigator_dat <- navigator_dat[[1]]

navigator_dat %<>% filter(Name != "Saint Lucia (Saint Lucia)")

rif_navigator <- navigator_dat %>% 
  mutate(cntry = countrycode::countrycode(Name, "country.name", "country.name")) %>% 
  mutate(cntry = ifelse(is.na(cntry), "Micronesia (Federated States of)", cntry)) %>% 
  select(cntry, local, regional, national)

save(rif_navigator, file = "data/rif_navigator.Rdata")
```