uni-ba-socialagenda/extract-twitter-accounts.R

# PREPARATIONS ------------------------------------------------------------


require(jsonlite)
require(stringr)
require(RCurl)
require(devtools)
require(RTwitterAPI)

setwd("~/Dokumente/Uni/Aktuell/BA-Arbeit/uni-ba-issuecomp")

source("functions.R")

# # Set curl handle for friendly scraping
# handle <- getCurlHandle(httpheader = list(from         = "max.mehl@uni.kn",
#                                           'user-agent' = str_c(R.version$version.string)
# )
# )

acc_url <- "http://www.bundestwitter.de/api/politiker"
acc_df <- fromJSON(acc_url)




# COLLECT ALL TWEETS ------------------------------------------------------

# http://www.joyofdata.de/blog/twitters-rest-api-v1-1-with-r-for-linux-and-windows/
#   --> devtools::install_github("joyofdata/RTwitterAPI")
# https://dev.twitter.com/rest/reference/get/statuses/user_timeline
api_params <- c(
  "oauth_consumer_key"     = "c9Ob2fWNSONMC0mA2JlNaeRke", 
  "oauth_nonce"            = NA,
  "oauth_signature_method" = "HMAC-SHA1",
  "oauth_timestamp"        = NA,
  "oauth_token"            = "1007025684-RFxCDFc4OPkt02bASmdci00TB4jgaPjfqxLRT58",
  "oauth_version"          = "1.0",
  "consumer_secret"        = "cZ3Il2hmbLgK0Lc57mj5kUvymjVdsmZKYwKOGHR3NhCpvWgEOI",
  "oauth_token_secret"     = "rvfv8MgexFKTqrPNSoGrdrZVNhV4fTJb2Bgz249nbvKNg"
)

#api_url2 <- "https://api.twitter.com/1.1/statuses/show.json"
#id2="498492933922754560" # 499533113676931073(\" ), 325320073906622464(\\>), 498492933922754560(\"W)
#query2 <- c(id=id2, trim_user="true", include_entities="false")
#current2 <- twitter_api_call(api_url2, query2, api_params)



api_url <- "https://api.twitter.com/1.1/statuses/user_timeline.json";
max_count <- "200"
keep <- c("created_at", "id_str", "text", "retweet_count")
tweets_full <- data.frame(user=character(), name=character(), created_at=character(), id_str=character(), text=character(), retweet_count=character())
tweets_complete <- tweets_full
for(a in 1:nrow(acc_df)) {
  user <- as.character(acc_df$screenname[a])
  name <- as.character(acc_df$name[a])
  max_id <- "999999999999999999"
  loop <- 1
  error <- 0
  repeat {
    # Define specific search query
    query <- c(include_rts=1, exclude_replies="true", trim_user="true", include_entities="false", 
               screen_name=user, 
               count=max_count, 
               max_id=max_id);
    
    # At first, work with an temporary tweet-DB
    current <- twitter_api_call(api_url, query, api_params)
    tweets_temp <- fromJSON(correctJSON(current))
    
    ## START ERROR HANDLING ##
    
    # Check for empty API returns
    status <- length(tweets_temp)
    if(status == 0) {
      if(error > 2) {
        cat("[WARNING] 3x empty API result. Aborting now.\n")
        break
      }
      cat("[WARNING] Empty API result. Trying again.\n")
      rm(tweets_temp)
      error <- error + 1
      Sys.sleep(3)
      next
    }
    
    # Check if API output contains error fields
    status <- "error" %in% names(tweets_temp)
    if(status) {
      cat("[WARNING] Error in API request:", tweets_temp$error[1],"\n")
      rm(tweets_temp)
      break
    }
    
    # Check for other errors, mostly rate limits
    status <- "errors" %in% names(tweets_temp)
    if(status) {
      cat("[WARNING] Error in API request:", tweets_temp$errors[1,1],"\n")
      status <- tweets_temp$errors[1,2]
      
      # "Rate limit exceeded"
      if(status == 88) {
        rate_api_url <- "https://api.twitter.com/1.1/application/rate_limit_status.json"
        rate_query <-c (resources="statuses")
        resettime <- fromJSON(twitter_api_call(rate_api_url, rate_query, api_params))
        resettime <- resettime$resources$statuses$`/statuses/user_timeline`$reset
        curtime <- as.numeric(as.POSIXct(Sys.time()))
        wait <- round(resettime - curtime + 10)
        cat("[INFO] Rate limit is exceeded. Now waiting",wait,"seconds.\n")
        Sys.sleep(wait)
      }
      
      # "Sorry, that page does not exist"
      if(status == 34) {
        if(error > 2) {
          cat("[WARNING] 3x Not existing page. Aborting now.\n")
          break
        }
        error <- error + 1
      }
      
      rm(tweets_temp)
      Sys.sleep(3)
      next
    }
    ## END ERROR HANDLING ##
    
    
    # Delete unnecessary columns and add username and real name to dataframe
    tweets_temp <- tweets_temp[keep]
    tweets_temp <- cbind(user=user, name=name, tweets_temp)
    
    # Now sleep 3 second to dodge 300queries/15min limit
    cat("[",a,"/",nrow(acc_df),"] ", sep = "")
    cat("User: ",user," in loop: ",loop,". \n", sep = "")
    Sys.sleep(2)
    
    if(tweets_full$id_str[nrow(tweets_full)] == tweets_temp$id_str[nrow(tweets_temp)] && nrow(tweets_full) > 0) {
      cat("[INFO] Last tweet of temp is last tweet of full. Abort loop and begin with next user.\n")
      break
    }
   
    ## Last loop is reached. Now clear the data frame
    # Is the last tweet in tweets_temp from 2013?
    status <- str_detect(tweets_temp$created_at[nrow(tweets_temp)], "2014$")
    if (!status) { # Starting when tweet not from 2014
      
      # Delete all tweets other than from 2014
      old <- 0
      for(r in 1:nrow(tweets_temp)) {
        status <- str_detect(tweets_temp$created_at[r], "2014$")
        if(is.na(status)) { 
          #status <- FALSE
          cat("[INFO] NA-Status in Tweet", r)
        }
        if(!status) { # Starting when tweet not from 2014
          old <- old + 1
        }
      }
      if(old > 0) {
        old <- old - 1
        
        # If even the first entry isn't from 2014, we have to set "old" manually because of a bug
        status <- str_detect(tweets_temp$created_at[1], "2014$")
        if(!status) {
          old <- nrow(tweets_temp)
          cat("[INFO] Timeline enhält keinen einzigen aus 2014\n")
        }
        
        # delete all lines which are older than 2014
        tweets_temp <- head(tweets_temp, -old)
      }
      rm(old)
      
      tweets_full <- insertRow(tweets_full, tweets_temp)
      rm(tweets_temp)
      break  # End loop because 2013 is reached
    }
    
    # The last tweet is still from 2014, so we need another loop
    else {
      # Setting max_id to gather next 200 tweets
      max_id <- tweets_temp$id_str[nrow(tweets_temp)]
      loop <- loop + 1  # just for stats
      
      tweets_full <- insertRow(tweets_full, tweets_temp)
      rm(tweets_temp)
    }
  } # /repeat
  
  tweets_complete <- insertRow(tweets_complete, tweets_full)
  tweets_full <- head(tweets_full, -nrow(tweets_full)) # Empty tweets_full
  cat("User:",user,"finished after",loop,"loops. Total Tweets now:",nrow(tweets_complete),"\n")
  write.csv(tweets_complete, "tweets_complete.csv")

  
  # Every tweet from 2014 from user[r] is downloaded. Now next user in for-loop
}