uni-ba-socialagenda/extract-twitter-accounts.R

require(jsonlite)
require(stringr)
require(RCurl)
require(devtools)
require(RTwitterAPI)

setwd("~/Dokumente/Uni/Aktuell/BA-Arbeit/uni-ba-issuecomp")

source("functions.R")

# Set curl handle for friendly scraping
handle <- getCurlHandle(httpheader = list(from         = "max.mehl@uni.kn",
                                          'user-agent' = str_c(R.version$version.string)
)
)

acc_url <- "http://www.bundestwitter.de/api/politiker"
#acc_json <- readLines("politiker.txt")
acc_df <- fromJSON(acc_url)


# ---------------
# http://www.joyofdata.de/blog/twitters-rest-api-v1-1-with-r-for-linux-and-windows/
# devtools::install_github("joyofdata/RTwitterAPI")

api_params <- c(
  "oauth_consumer_key"     = "c9Ob2fWNSONMC0mA2JlNaeRke", 
  "oauth_nonce"            = NA,
  "oauth_signature_method" = "HMAC-SHA1",
  "oauth_timestamp"        = NA,
  "oauth_token"            = "1007025684-RFxCDFc4OPkt02bASmdci00TB4jgaPjfqxLRT58",
  "oauth_version"          = "1.0",
  "consumer_secret"        = "cZ3Il2hmbLgK0Lc57mj5kUvymjVdsmZKYwKOGHR3NhCpvWgEOI",
  "oauth_token_secret"     = "rvfv8MgexFKTqrPNSoGrdrZVNhV4fTJb2Bgz249nbvKNg"
)

api_url   <- "https://api.twitter.com/1.1/statuses/user_timeline.json";
user <- "GregorGysi"
max_count <- "200"
max_id <- "999999999999999999"
loop <- 1
keep <- c("created_at", "id_str", "text", "retweet_count")
rm(tweets_full)
last_id <- NULL
repeat {
  # Define specific search query
  query <- c(include_rts=1, exclude_replies="true", trim_user="true", include_entities="false", 
             screen_name=user, 
             count=max_count, 
             max_id=max_id);
  
  # If a tweets_full DB already exists (after the first loop this should be the case)
  if(exists("tweets_full")) {
    current <- twitter_api_call(api_url, query, api_params)
    tweets_temp <- fromJSON(correctJSON(current))
    tweets_temp <- tweets_temp[keep]
    tweets_full <- insertRow(tweets_full, tweets_temp)
    rm(tweets_temp)
  }
  # First loop
  else {
    current <- twitter_api_call(api_url, query, api_params)
    tweets_full <- fromJSON(correctJSON(current))
    tweets_full <- tweets_full[keep]
  }
  
  # Now sleep 3 second to dodge 300queries/15min limit
  cat("User:",user,"in loop:",loop,"- now waiting 3 secs...\n")
  Sys.sleep(3)
  
  # Is the last tweet in tweets_full from 2013?
  status <- str_detect(tweets_full$created_at[nrow(tweets_full)], "2013$")
  # Last loop is reached. Now clear the data frame
  if (status) {
    
    # Delete all tweets from 2013
    old <- 0
    for(r in 1:nrow(tweets_full)) {
      status <- str_detect(tweets_full$created_at[r], "2013$")
      if(is.na(status)) { status <- FALSE }
      if(status) {
        old <- old + 1
      }
    }
    if(old > 0) {
      old <- old - 1
      tweets_full <- head(tweets_full, -old)
    }
    rm(old)
    
    break  # End loop because 2013 is reached
  }
  
  # The last tweet is still from 2014, so we need another loop
  else {
    # Setting max_id to gather next 200 tweets
    max_id <- tweets_full$id_str[nrow(tweets_full)]
    loop <- loop + 1  # just for stats
  }
}

# ---------------