# PREPARATIONS ------------------------------------------------------------ require(jsonlite) require(stringr) require(RCurl) require(devtools) require(RTwitterAPI) setwd("~/Dokumente/Uni/Aktuell/BA-Arbeit/uni-ba-issuecomp") source("functions.R") # # Set curl handle for friendly scraping # handle <- getCurlHandle(httpheader = list(from = "max.mehl@uni.kn", # 'user-agent' = str_c(R.version$version.string) # ) # ) acc_url <- "http://www.bundestwitter.de/api/politiker" acc_df <- fromJSON(acc_url) # COLLECT ALL TWEETS ------------------------------------------------------ # http://www.joyofdata.de/blog/twitters-rest-api-v1-1-with-r-for-linux-and-windows/ # --> devtools::install_github("joyofdata/RTwitterAPI") # https://dev.twitter.com/rest/reference/get/statuses/user_timeline api_params <- c( "oauth_consumer_key" = "c9Ob2fWNSONMC0mA2JlNaeRke", "oauth_nonce" = NA, "oauth_signature_method" = "HMAC-SHA1", "oauth_timestamp" = NA, "oauth_token" = "1007025684-RFxCDFc4OPkt02bASmdci00TB4jgaPjfqxLRT58", "oauth_version" = "1.0", "consumer_secret" = "cZ3Il2hmbLgK0Lc57mj5kUvymjVdsmZKYwKOGHR3NhCpvWgEOI", "oauth_token_secret" = "rvfv8MgexFKTqrPNSoGrdrZVNhV4fTJb2Bgz249nbvKNg" ) #api_url2 <- "https://api.twitter.com/1.1/statuses/show.json" #id2="498492933922754560" # 499533113676931073(\" ), 325320073906622464(\\>), 498492933922754560(\"W) #query2 <- c(id=id2, trim_user="true", include_entities="false") #current2 <- twitter_api_call(api_url2, query2, api_params) api_url <- "https://api.twitter.com/1.1/statuses/user_timeline.json"; max_count <- "200" keep <- c("created_at", "id_str", "text", "retweet_count") tweets_full <- data.frame(user=character(), name=character(), created_at=character(), id_str=character(), text=character(), retweet_count=character()) tweets_complete <- tweets_full for(a in 1:nrow(acc_df)) { user <- as.character(acc_df$screenname[a]) name <- as.character(acc_df$name[a]) max_id <- "999999999999999999" loop <- 1 error <- 0 repeat { # Define specific search query query <- c(include_rts=1, exclude_replies="true", trim_user="true", include_entities="false", screen_name=user, count=max_count, max_id=max_id); # At first, work with an temporary tweet-DB current <- twitter_api_call(api_url, query, api_params) tweets_temp <- fromJSON(correctJSON(current)) ## START ERROR HANDLING ## # Check for empty API returns status <- length(tweets_temp) if(status == 0) { if(error > 2) { cat("[WARNING] 3x empty API result. Aborting now.\n") break } cat("[WARNING] Empty API result. Trying again.\n") rm(tweets_temp) error <- error + 1 Sys.sleep(3) next } # Check if API output contains error fields status <- "error" %in% names(tweets_temp) if(status) { cat("[WARNING] Error in API request:", tweets_temp$error[1],"\n") rm(tweets_temp) break } # Check for other errors, mostly rate limits status <- "errors" %in% names(tweets_temp) if(status) { cat("[WARNING] Error in API request:", tweets_temp$errors[1,1],"\n") status <- tweets_temp$errors[1,2] # "Rate limit exceeded" if(status == 88) { rate_api_url <- "https://api.twitter.com/1.1/application/rate_limit_status.json" rate_query <-c (resources="statuses") resettime <- fromJSON(twitter_api_call(rate_api_url, rate_query, api_params)) resettime <- resettime$resources$statuses$`/statuses/user_timeline`$reset curtime <- as.numeric(as.POSIXct(Sys.time())) wait <- round(resettime - curtime + 10) cat("[INFO] Rate limit is exceeded. Now waiting",wait,"seconds.\n") Sys.sleep(wait) } # "Sorry, that page does not exist" if(status == 34) { if(error > 2) { cat("[WARNING] 3x Not existing page. Aborting now.\n") break } error <- error + 1 } rm(tweets_temp) Sys.sleep(3) next } ## END ERROR HANDLING ## # Delete unnecessary columns and add username and real name to dataframe tweets_temp <- tweets_temp[keep] tweets_temp <- cbind(user=user, name=name, tweets_temp) # Now sleep 3 second to dodge 300queries/15min limit cat("[",a,"/",nrow(acc_df),"] ", sep = "") cat("User: ",user," in loop: ",loop,". \n", sep = "") Sys.sleep(2) if(tweets_full$id_str[nrow(tweets_full)] == tweets_temp$id_str[nrow(tweets_temp)] && nrow(tweets_full) > 0) { cat("[INFO] Last tweet of temp is last tweet of full. Abort loop and begin with next user.\n") break } ## Last loop is reached. Now clear the data frame # Is the last tweet in tweets_temp from 2013? status <- str_detect(tweets_temp$created_at[nrow(tweets_temp)], "2014$") if (!status) { # Starting when tweet not from 2014 # Delete all tweets other than from 2014 old <- 0 for(r in 1:nrow(tweets_temp)) { status <- str_detect(tweets_temp$created_at[r], "2014$") if(is.na(status)) { #status <- FALSE cat("[INFO] NA-Status in Tweet", r) } if(!status) { # Starting when tweet not from 2014 old <- old + 1 } } if(old > 0) { old <- old - 1 # If even the first entry isn't from 2014, we have to set "old" manually because of a bug status <- str_detect(tweets_temp$created_at[1], "2014$") if(!status) { old <- nrow(tweets_temp) cat("[INFO] Timeline enhält keinen einzigen aus 2014\n") } # delete all lines which are older than 2014 tweets_temp <- head(tweets_temp, -old) } rm(old) tweets_full <- insertRow(tweets_full, tweets_temp) rm(tweets_temp) break # End loop because 2013 is reached } # The last tweet is still from 2014, so we need another loop else { # Setting max_id to gather next 200 tweets max_id <- tweets_temp$id_str[nrow(tweets_temp)] loop <- loop + 1 # just for stats tweets_full <- insertRow(tweets_full, tweets_temp) rm(tweets_temp) } } # /repeat tweets_complete <- insertRow(tweets_complete, tweets_full) tweets_full <- head(tweets_full, -nrow(tweets_full)) # Empty tweets_full cat("User:",user,"finished after",loop,"loops. Total Tweets now:",nrow(tweets_complete),"\n") write.csv(tweets_complete, "tweets_complete.csv") # Every tweet from 2014 from user[r] is downloaded. Now next user in for-loop }