# PREPARATIONS ------------------------------------------------------------ require(jsonlite) require(stringr) require(devtools) require(RTwitterAPI) setwd("~/Dokumente/Uni/Aktuell/BA-Arbeit/uni-ba-issuecomp") source("issuecomp-functions.R") #acc_url <- "http://www.bundestwitter.de/api/politiker" #acc_df <- fromJSON(acc_url) acc_df <- read.csv("MdB-twitter.csv") delrow <- NULL for(r in 1:nrow(acc_df)) { acc <- as.character(acc_df$twitter_acc[r]) if(!nzchar(acc)) { delrow <- c(delrow, r) } } acc_df <- acc_df[-delrow, ] rm(delrow, r, acc) acc_df$row.names <- NULL row.names(acc_df) <- NULL # COLLECT ALL TWEETS ------------------------------------------------------ # http://www.joyofdata.de/blog/twitters-rest-api-v1-1-with-r-for-linux-and-windows/ # --> devtools::install_github("joyofdata/RTwitterAPI") # https://dev.twitter.com/rest/reference/get/statuses/user_timeline api_params <- c( "oauth_consumer_key" = readLines("twitter-api-credentials.txt")[2], "oauth_nonce" = NA, "oauth_signature_method" = "HMAC-SHA1", "oauth_timestamp" = NA, "oauth_token" = readLines("twitter-api-credentials.txt")[4], "oauth_version" = "1.0", "consumer_secret" = readLines("twitter-api-credentials.txt")[3], "oauth_token_secret" = readLines("twitter-api-credentials.txt")[5] ) #api_url2 <- "https://api.twitter.com/1.1/statuses/show.json" #id2="498492933922754560" # 499533113676931073(\" ), 325320073906622464(\\>), 498492933922754560(\"W) #query2 <- c(id=id2, trim_user="true", include_entities="false") #current2 <- twitter_api_call(api_url2, query2, api_params) api_url <- "https://api.twitter.com/1.1/statuses/user_timeline.json"; max_count <- "200" keep <- c("created_at", "id_str", "text", "retweet_count") # tweets_complete: All tweets # tweets_full: All tweets of current user # tweets_temp: The current max 200 tweets of current user tweets_full <- data.frame(user=character(), name=character(), created_at=character(), id_str=character(), text=character(), retweet_count=character()) tweets_complete <- tweets_full for(a in 1:nrow(acc_df)) { user <- as.character(acc_df$twitter_acc[a]) name <- as.character(acc_df$name[a]) max_id <- "999999999999999999" loop <- 1 error <- 0 repeat { # Define specific search query query <- c(include_rts=1, exclude_replies="true", trim_user="true", include_entities="false", screen_name=user, count=max_count, max_id=max_id); # At first, work with an temporary tweet-DB current <- twitter_api_call(api_url, query, api_params) rm(tweets_temp) tweets_temp <- fromJSON(correctJSON(current)) ## START ERROR HANDLING ## # Empty API output status <- errorEmptyAPI(tweets_temp) if(status == 1) { Sys.sleep(3);error <- error + 1;next} if(status == 2) {break} # Contains "error" column status <- errorErrorColumn(tweets_temp) if(status == 1) { Sys.sleep(3);error <- error + 1;next} if(status == 2) {break} # Check if error code exists code <- errorCheckCode(tweets_temp) # 0 if no error if(code == 34) { # page does not exist status <- errorCode34() if(status == 1) { Sys.sleep(3);error <- error + 1;next} if(status == 2) {break} } if(code == 88) { # rate limit exceeded wait <- errorCode88() Sys.sleep(wait) next } ## END ERROR HANDLING ## # Delete unnecessary columns and add username and real name to dataframe tweets_temp <- tweets_temp[keep] tweets_temp <- cbind(user=user, name=name, tweets_temp) # Now sleep 3 second to dodge 300queries/15min limit cat("[",a,"/",nrow(acc_df),"] ", sep = "") cat("User: ",user," in loop: ",loop,". \n", sep = "") Sys.sleep(2) if(tweets_full$id_str[nrow(tweets_full)] == tweets_temp$id_str[nrow(tweets_temp)] && nrow(tweets_full) > 0) { cat("[INFO] Last tweet of temp is last tweet of full. Abort loop and begin with next user.\n") break } ## CHECK if we need another loop # Extract year of last tweet in tweets_temp year_last <- as.numeric(str_extract(tweets_temp$created_at[nrow(tweets_temp)], "\\d{4}$")) status <- year_last < 2014 # Is last tweet earlier than 2014? So break the loop if (status) { # Is even the first tweet older than 2014? year_first <- as.numeric(str_extract(tweets_temp$created_at[1], "\\d{4}$")) status <- year_first < 2014 if(status) { cat("[INFO] Timeline enhält keinen einzigen aus 2014\n") } tweets_full <- insertRow(tweets_full, tweets_temp) break # End loop because 2013 is reached } # The last tweet is newer or equal 2014, so we need another loop else { # Setting max_id to gather next 200 tweets max_id <- tweets_temp$id_str[nrow(tweets_temp)] loop <- loop + 1 # just for stats tweets_full <- insertRow(tweets_full, tweets_temp) } } # /repeat tweets_complete <- insertRow(tweets_complete, tweets_full) tweets_full <- head(tweets_full, -nrow(tweets_full)) # Empty tweets_full cat("User:",user,"finished after",loop,"loops. Total Tweets now:",nrow(tweets_complete),"\n") write.csv(tweets_complete, "tweets_complete.csv") # Every tweet from 2014 or newer from user[a] is downloaded. Now next user in for-loop } rm(a, code, current, error, loop, max_id, max_count, year_first, year_last, name, query, status, user, wait, tweets_full, tweets_temp) # CLEAR DATAFRAME --------------------------------------------------------- save(tweets_complete, file="tweets_complete.RData") # Remove duplicates tweets <- tweets_complete[!duplicated(tweets_complete), ] tweets <- na.omit(tweets) rm(tweets_complete) # Format dates in data frame Sys.setlocale("LC_TIME", "C") tweets$created_at <- as.POSIXct(tweets$created_at, format = "%a %b %d %H:%M:%S %z %Y") tweets <- tweets[order(tweets$created_at), ] # Finally delete every tweet not from 2014 (2013 or 2015) delrow <- NULL for(r in 1:nrow(tweets)) { if(format(tweets$created_at[r], "%Y") != "2014") { delrow <- c(delrow, r) } curtext <- as.character(tweets$text[r]) curtext <- str_replace_all(curtext, "$", " ") curtext <- str_replace_all(curtext, "http://.+?\\s", "URL ") tweets$text[r] <- curtext } tweets <- tweets[-delrow, ] rm(delrow, r) # Convert dates to omit (unnecessary) time tweets$created_at <- format(tweets$created_at, "%Y-%m-%d") save(tweets, file="tweets_untagged.RData")