diff --git a/.Rhistory b/.Rhistory new file mode 100644 index 0000000..555ce24 --- /dev/null +++ b/.Rhistory @@ -0,0 +1,238 @@ +install.packages("jsonlite") +install.packages(c("BH", "dplyr", "Lahman", "magrittr", "maps", "sp", "xtable")) +setwd("/home/max/Dokumente/Uni/Aktuell/BA-Arbeit/R-Code") +setwd("/home/max/Dokumente/Uni/Aktuell/BA-Arbeit/R-Code") +setwd("~/") +setwd("~/Dokumente/Uni/Aktuell/BA-Arbeit/R-Code") +ls() +# Set curl handle for friendly scraping +handle <- getCurlHandle(httpheader = list(from = "max.mehl@uni.kn", +'user-agent' = str_c(R.version$version.string) +) +) +source("functions.R") +handle <- getCurlHandle(httpheader = list(from = "max.mehl@uni.kn", +'user-agent' = str_c(R.version$version.string) +) +) +require(stringr) +require(RCurl) +handle <- getCurlHandle(httpheader = list(from = "max.mehl@uni.kn", +'user-agent' = str_c(R.version$version.string) +) +) +acc_url <- "http://www.bundestwitter.de/api/politiker" +source("functions.R") +require(jsonlite) +fromJSON(acc) +fromJSON(acc_url) +require(httr) +install.packages("httr") +require(httr) +fromJSON(acc_url) +df <- fromJSON(acc_url) +View(df) +require(jsonlite) +require(stringr) +require(RCurl) +require(httr) +source("functions.R") +setwd("~/Dokumente/Uni/Aktuell/BA-Arbeit/R-Code") +# Set curl handle for friendly scraping +handle <- getCurlHandle(httpheader = list(from = "max.mehl@uni.kn", +'user-agent' = str_c(R.version$version.string) +) +) +acc_url <- "http://www.bundestwitter.de/api/politiker" +df <- fromJSON(acc_url) +setwd("~/Dokumente/Uni/Aktuell/BA-Arbeit/uni-ba-issuecomp") +View(df) +acc_df <- fromJSON(acc_url) +rm df +rm(df) +View(acc_df) +install.packages("twitteR") +require(twitteR) +userTimeline(user = "peteraltmaier", n = "20") +read("twitter-api-keys.txt") +read.csv(file = "twitter-api-keys.txt", header = TRUE, sep = ";") +read.csv(file = "twitter-api-keys.txt", header = TRUE, sep = ";") +read.csv(file = "twitter-api-keys.txt", header = TRUE, sep = ";") +oauth <- read.csv(file = "twitter-api-keys.txt", header = TRUE, sep = ";") +View(oauth) +getTwitterOAuth(oauth$ConsumerKey, oauth$ConsumerSecret) +readLines("twitter-api-consumerkey.txt") +oauthKey <- readLines("twitter-api-consumerkey.txt") +oauthSec <- readLines("twitter-api-consumersecret.txt") +rm(oauth) +getTwitterOAuth(oauthKey, oauthSec) +getTwitterOAuth(oauthKey, oauthSec) +reqURL <- "https://api.twitter.com/oauth/request_token" +accessURL <- "https://api.twitter.com/oauth/access_token" +authURL <- "https://api.twitter.com/oauth/authorize" +consumerKey <- readLines("twitter-api-consumerkey.txt") +consumerSecret <- readLines("twitter-api-consumersecret.txt") +twitCred <- OAuthFactory$new(consumerKey=consumerKey, +consumerSecret=consumerSecret, +requestURL=reqURL, +accessURL=accessURL, +authURL=authURL) +twitCred$handshake() +rm(list=ls()) +require(jsonlite) +require(stringr) +require(RCurl) +require(httr) +require(twitteR) +source("functions.R") +setwd("~/Dokumente/Uni/Aktuell/BA-Arbeit/uni-ba-issuecomp") +# Set curl handle for friendly scraping +handle <- getCurlHandle(httpheader = list(from = "max.mehl@uni.kn", +'user-agent' = str_c(R.version$version.string) +) +) +acc_url <- "http://www.bundestwitter.de/api/politiker" +acc_df <- fromJSON(acc_url) +# -------------- +twAPI_reqURL <- "https://api.twitter.com/oauth/request_token" +twAPI_accessURL <- "https://api.twitter.com/oauth/access_token" +twAPI_authURL <- "https://api.twitter.com/oauth/authorize" +twAPI_consumerKey <- readLines("twitter-api-consumerkey.txt") +twAPI_consumerSecret <- readLines("twitter-api-consumersecret.txt") +twAPI_twitCred <- OAuthFactory$new(consumerKey=consumerKey, +consumerSecret=consumerSecret, +requestURL=reqURL, +accessURL=accessURL, +authURL=authURL) +twAPI_twitCred$handshake() +registerTwitterOAuth(twAPI_twitCred) +twAPI_reqURL <- "https://api.twitter.com/oauth/request_token" +twAPI_accessURL <- "https://api.twitter.com/oauth/access_token" +twAPI_authURL <- "https://api.twitter.com/oauth/authorize" +twAPI_consumerKey <- readLines("twitter-api-consumerkey.txt") +twAPI_consumerSecret <- readLines("twitter-api-consumersecret.txt") +twAPI_twitCred <- OAuthFactory$new(consumerKey=twAPI_consumerKey, +consumerSecret=twAPI_consumerSecret, +requestURL=twAPI_reqURL, +accessURL=twAPI_accessURL, +authURL=twAPI_authURL) +twAPI_twitCred$handshake() +twAPI_twitCred$handshake() +registerTwitterOAuth(twAPI_twitCred) +registerTwitterOAuth(twAPI_twitCred) +userTimeline(user = "peteraltmaier", n = "20") +userTimeline(user = "peteraltmaier", n = "20", includeRts = TRUE) +showStatus("530033823577550848") +getUser("peteraltmaier") +getUser("mxmehl") +lookupUsers("mxmehl") +userTimeline(user = "peteraltmaier", n = "20", includeRts = TRUE) +wd +userTimeline(user = "peteraltmaier", n = "20", includeRts = TRUE) +userTimeline(user = "peteraltmaier", n = "20", includeRts = TRUE) +lookupUsers("mxmehl") +lookupUsers("mxmehl") +install.packages("devtools") +require(devtools) +devtools::install_github("joyofdata/RTwitterAPI") +require(RTwitterAPI) +arams <- c( +"oauth_consumer_key" = "c9Ob2fWNSONMC0mA2JlNaeRke", +"oauth_nonce" = NA, +"oauth_signature_method" = "HMAC-SHA1", +"oauth_timestamp" = NA, +"oauth_token" = "1007025684-RFxCDFc4OPkt02bASmdci00TB4jgaPjfqxLRT58", +"oauth_version" = "1.0", +"consumer_secret" = "cZ3Il2hmbLgK0Lc57mj5kUvymjVdsmZKYwKOGHR3NhCpvWgEOI", +"oauth_token_secret" = "rvfv8MgexFKTqrPNSoGrdrZVNhV4fTJb2Bgz249nbvKNg" +); +rm(arams) +params <- c( +"oauth_consumer_key" = "c9Ob2fWNSONMC0mA2JlNaeRke", +"oauth_nonce" = NA, +"oauth_signature_method" = "HMAC-SHA1", +"oauth_timestamp" = NA, +"oauth_token" = "1007025684-RFxCDFc4OPkt02bASmdci00TB4jgaPjfqxLRT58", +"oauth_version" = "1.0", +"consumer_secret" = "cZ3Il2hmbLgK0Lc57mj5kUvymjVdsmZKYwKOGHR3NhCpvWgEOI", +"oauth_token_secret" = "rvfv8MgexFKTqrPNSoGrdrZVNhV4fTJb2Bgz249nbvKNg" +); +rm(twAPI_*) +rm(twAPI_) +rm(twAPI_accessURL) +params <- c( +"oauth_consumer_key" = "c9Ob2fWNSONMC0mA2JlNaeRke", +"oauth_nonce" = NA, +"oauth_signature_method" = "HMAC-SHA1", +"oauth_timestamp" = NA, +"oauth_token" = "1007025684-RFxCDFc4OPkt02bASmdci00TB4jgaPjfqxLRT58", +"oauth_version" = "1.0", +"consumer_secret" = "cZ3Il2hmbLgK0Lc57mj5kUvymjVdsmZKYwKOGHR3NhCpvWgEOI", +"oauth_token_secret" = "rvfv8MgexFKTqrPNSoGrdrZVNhV4fTJb2Bgz249nbvKNg" +); +url <- "https://api.twitter.com/1.1/friends/ids.json"; +query <- c(cursor=-1, screen_name="hrw", count=10); +result <- RTwitterAPI::twitter_api_call(url, query, params) +result +url <- "https://api.twitter.com/1.1/statuses/user_timeline.json"; +query <- c(screen_name="peteraltmaier", exclude_replies="true", include_rts="true"); +result <- RTwitterAPI::twitter_api_call(url, query, params) +result +query <- c(screen_name="peteraltmaier", exclude_replies="true", include_rts="true", count="5"); +result <- fromJSON(RTwitterAPI::twitter_api_call(url, query, params)) +result +query <- c(screen_name="peteraltmaier", exclude_replies="true", include_rts=1, count="5"); +result <- fromJSON(RTwitterAPI::twitter_api_call(url, query, params)) +query <- c(screen_name="peteraltmaier", exclude_replies="true", include_rts=1, count="1"); +result <- fromJSON(RTwitterAPI::twitter_api_call(url, query, params)) +query <- c(screen_name="peteraltmaier", exclude_replies="true", include_rts=1, count="20"); +result <- fromJSON(RTwitterAPI::twitter_api_call(url, query, params)) +query <- c(screen_name="peteraltmaier", exclude_replies="true", include_rts=1, count="1"); +result <- fromJSON(RTwitterAPI::twitter_api_call(url, query, params)) +result +result[[1]] +result[[1]]$created_at +result[[1]]$id_str +result[[1]]$text +require(jsonlite) +require(stringr) +require(RCurl) +require(httr) +require(twitteR) +require(devtools) +require(RTwitterAPI) +source("functions.R") +setwd("~/Dokumente/Uni/Aktuell/BA-Arbeit/uni-ba-issuecomp") +# Set curl handle for friendly scraping +handle <- getCurlHandle(httpheader = list(from = "max.mehl@uni.kn", +'user-agent' = str_c(R.version$version.string) +) +) +acc_url <- "http://www.bundestwitter.de/api/politiker" +acc_df <- fromJSON(acc_url) +require(jsonlite) +require(stringr) +require(RCurl) +require(httr) +require(twitteR) +require(devtools) +require(RTwitterAPI) +source("functions.R") +setwd("~/Dokumente/Uni/Aktuell/BA-Arbeit/uni-ba-issuecomp") +# Set curl handle for friendly scraping +handle <- getCurlHandle(httpheader = list(from = "max.mehl@uni.kn", +'user-agent' = str_c(R.version$version.string) +) +) +acc_url <- "http://www.bundestwitter.de/api/politiker" +acc_df <- fromJSON(acc_url) +acc_df <- fromJSON(acc_url) +acc_url <- "http://www.bundestwitter.de/api/politiker" +acc_df <- fromJSON(acc_url) +acc_url <- "http://www.bundestwitter.de/api/politiker" +acc_df <- fromJSON(acc_url) +acc_jsonreadLines("politiker.txt") +acc_json <- readLines("politiker.txt") +acc_df <- fromJSON(acc_json) +acc_df$name +acc_df[[1]]$name diff --git a/extract-twitter-accounts.R b/extract-twitter-accounts.R index 90067f4..d7eb83e 100644 --- a/extract-twitter-accounts.R +++ b/extract-twitter-accounts.R @@ -36,15 +36,11 @@ params <- c( ); url <- "https://api.twitter.com/1.1/statuses/user_timeline.json"; -query <- c(screen_name="peteraltmaier", include_rts=1, count="100", exclude_replies="true", trim_user="true", include_entities="false"); +query <- c(screen_name="mxmehl", include_rts=1, count="100", exclude_replies="true", trim_user="true", include_entities="false"); result <- twitter_api_call(url, query, params) -# Replace characters messing up JSON validation (\ and \n) -result <- str_replace(string = result, pattern = perl('\\\\(?![tn"])'), replacement = " ") -result <- str_replace(string = result, pattern = "\n", replacement = " ") - -result <- fromJSON(result) +result <- fromJSON(correctJSON(result)) # --------------- diff --git a/functions.R b/functions.R index aec40ad..62bdf5c 100644 --- a/functions.R +++ b/functions.R @@ -1,2 +1,9 @@ require(stringr) -require(RCurl) + +# Replace characters messing up JSON validation (\,\n,^) +correctJSON <- function(string) { + string <- str_replace_all(string, pattern = perl('\\\\(?![tn"])'), replacement = " ") + string <- str_replace_all(string, pattern = fixed("\n"), replacement = " ") + string <- str_replace_all(string, pattern = fixed("^"), replacement = " ") + return(string) +} \ No newline at end of file