more error handling but it seems to run
This commit is contained in:
+58
-34
@@ -1,3 +1,6 @@
|
||||
# PREPARATIONS ------------------------------------------------------------
|
||||
|
||||
|
||||
require(jsonlite)
|
||||
require(stringr)
|
||||
require(RCurl)
|
||||
@@ -15,14 +18,16 @@ source("functions.R")
|
||||
# )
|
||||
|
||||
acc_url <- "http://www.bundestwitter.de/api/politiker"
|
||||
#acc_json <- readLines("politiker.txt")
|
||||
acc_df <- fromJSON(acc_url)
|
||||
|
||||
|
||||
# ---------------
|
||||
# http://www.joyofdata.de/blog/twitters-rest-api-v1-1-with-r-for-linux-and-windows/
|
||||
# devtools::install_github("joyofdata/RTwitterAPI")
|
||||
|
||||
|
||||
# COLLECT ALL TWEETS ------------------------------------------------------
|
||||
|
||||
# http://www.joyofdata.de/blog/twitters-rest-api-v1-1-with-r-for-linux-and-windows/
|
||||
# --> devtools::install_github("joyofdata/RTwitterAPI")
|
||||
# https://dev.twitter.com/rest/reference/get/statuses/user_timeline
|
||||
api_params <- c(
|
||||
"oauth_consumer_key" = "c9Ob2fWNSONMC0mA2JlNaeRke",
|
||||
"oauth_nonce" = NA,
|
||||
@@ -37,8 +42,7 @@ api_params <- c(
|
||||
api_url <- "https://api.twitter.com/1.1/statuses/user_timeline.json";
|
||||
max_count <- "200"
|
||||
keep <- c("created_at", "id_str", "text", "retweet_count")
|
||||
rm(tweets_full)
|
||||
|
||||
tweets_full <- data.frame(user=character(), name=character(), created_at=character(), id_str=character(), text=character(), retweet_count=character())
|
||||
for(a in 1:nrow(acc_df)) {
|
||||
user <- as.character(acc_df$screenname[a])
|
||||
name <- as.character(acc_df$name[a])
|
||||
@@ -51,65 +55,85 @@ for(a in 1:nrow(acc_df)) {
|
||||
count=max_count,
|
||||
max_id=max_id);
|
||||
|
||||
# If a tweets_full DB already exists (after the first loop this should be the case)
|
||||
if(exists("tweets_full")) {
|
||||
current <- twitter_api_call(api_url, query, api_params)
|
||||
tweets_temp <- fromJSON(correctJSON(current))
|
||||
tweets_temp <- tweets_temp[keep]
|
||||
tweets_temp <- cbind(user=user, name=name, tweets_temp)
|
||||
tweets_full <- insertRow(tweets_full, tweets_temp)
|
||||
# At first, work with an temporary tweet-DB
|
||||
current <- twitter_api_call(api_url, query, api_params)
|
||||
tweets_temp <- fromJSON(correctJSON(current))
|
||||
|
||||
# Check if API output contains error fields
|
||||
status <- "error" %in% names(tweets_temp)
|
||||
if(status) {
|
||||
cat("[WARNING] Error in API request:", tweets_temp$error[1],"\n")
|
||||
rm(tweets_temp)
|
||||
}
|
||||
# First loop
|
||||
else {
|
||||
current <- twitter_api_call(api_url, query, api_params)
|
||||
tweets_full <- fromJSON(correctJSON(current))
|
||||
tweets_full <- tweets_full[keep]
|
||||
tweets_full <- cbind(user=user, name=name, tweets_full)
|
||||
break
|
||||
}
|
||||
|
||||
# Delete unnecessary columns and add username and real name to dataframe
|
||||
tweets_temp <- tweets_temp[keep]
|
||||
tweets_temp <- cbind(user=user, name=name, tweets_temp)
|
||||
|
||||
# Now sleep 3 second to dodge 300queries/15min limit
|
||||
stat_tweet <- nrow(tweets_full)
|
||||
cat("[",a,"/",nrow(acc_df),"] ", sep = "")
|
||||
cat("User: ",user," in loop: ",loop,". \n", sep = "")
|
||||
Sys.sleep(2)
|
||||
|
||||
# Is the last tweet in tweets_full from 2013?
|
||||
status <- str_detect(tweets_full$created_at[nrow(tweets_full)], "2014$")
|
||||
if(tweets_full$id_str[nrow(tweets_full)] == tweets_temp$id_str[nrow(tweets_temp)] && nrow(tweets_full) > 0) {
|
||||
cat("[INFO] Last tweet of temp is last tweet of full. Abort loop and begin with next user.\n")
|
||||
break
|
||||
}
|
||||
|
||||
# Is the last tweet in tweets_temp from 2013?
|
||||
status <- str_detect(tweets_temp$created_at[nrow(tweets_temp)], "2014$")
|
||||
# Last loop is reached. Now clear the data frame
|
||||
if (!status) { # Starting when tweet not from 2014
|
||||
|
||||
# Delete all tweets from 2013
|
||||
# Delete all tweets other than from 2014
|
||||
old <- 0
|
||||
for(r in 1:nrow(tweets_full)) {
|
||||
status <- str_detect(tweets_full$created_at[r], "2014$")
|
||||
if(is.na(status)) { status <- FALSE }
|
||||
for(r in 1:nrow(tweets_temp)) {
|
||||
status <- str_detect(tweets_temp$created_at[r], "2014$")
|
||||
if(is.na(status)) {
|
||||
#status <- FALSE
|
||||
cat("[INFO] NA-Status in Tweet", r)
|
||||
}
|
||||
if(!status) { # Starting when tweet not from 2014
|
||||
old <- old + 1
|
||||
}
|
||||
}
|
||||
if(old > 0) {
|
||||
old <- old - 1
|
||||
tweets_full <- head(tweets_full, -old)
|
||||
|
||||
# If even the first entry isn't from 2014, we have to set "old" manually because of a bug
|
||||
status <- str_detect(tweets_temp$created_at[1], "2014$")
|
||||
if(!status) {
|
||||
old <- nrow(tweets_temp)
|
||||
cat("[INFO] Timeline enhält keinen einzigen aus 2014\n")
|
||||
}
|
||||
|
||||
# delete all lines which are older than 2014
|
||||
tweets_temp <- head(tweets_temp, -old)
|
||||
}
|
||||
rm(old)
|
||||
|
||||
tweets_full <- insertRow(tweets_full, tweets_temp)
|
||||
rm(tweets_temp)
|
||||
break # End loop because 2013 is reached
|
||||
}
|
||||
|
||||
# The last tweet is still from 2014, so we need another loop
|
||||
else {
|
||||
# Setting max_id to gather next 200 tweets
|
||||
max_id <- tweets_full$id_str[nrow(tweets_full)]
|
||||
max_id <- tweets_temp$id_str[nrow(tweets_temp)]
|
||||
loop <- loop + 1 # just for stats
|
||||
|
||||
tweets_full <- insertRow(tweets_full, tweets_temp)
|
||||
rm(tweets_temp)
|
||||
}
|
||||
}
|
||||
} # /repeat
|
||||
|
||||
stat_tweet <- nrow(tweets_full)
|
||||
cat("User:",user,"finished after",loop,"loops. Total Tweets now:",nrow(tweets_full),"\n")
|
||||
write.csv(tweets_full, "tweets_full.csv")
|
||||
|
||||
# Every tweet from 2014 from user[r] is downloaded. Now next user in for-loop
|
||||
cat("User:",user,"finished after",loop,"loops. Total Tweets now:",nrow(tweets_full),"\n")
|
||||
}
|
||||
|
||||
|
||||
# ---------------
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user