current status

extract-twitter-accounts.R

@@ -39,15 +39,17 @@ api_params <- c(
   "oauth_token_secret"     = "rvfv8MgexFKTqrPNSoGrdrZVNhV4fTJb2Bgz249nbvKNg"
 )
 
-api_url   <- "https://api.twitter.com/1.1/statuses/user_timeline.json";
+
+api_url <- "https://api.twitter.com/1.1/statuses/user_timeline.json";
 max_count <- "200"
 keep <- c("created_at", "id_str", "text", "retweet_count")
 tweets_full <- data.frame(user=character(), name=character(), created_at=character(), id_str=character(), text=character(), retweet_count=character())
-for(a in 1:nrow(acc_df)) {
+for(a in 157:nrow(acc_df)) {
   user <- as.character(acc_df$screenname[a])
   name <- as.character(acc_df$name[a])
   max_id <- "999999999999999999"
   loop <- 1
+  error <- 0
   repeat {
     # Define specific search query
     query <- c(include_rts=1, exclude_replies="true", trim_user="true", include_entities="false", 
@@ -59,11 +61,19 @@ for(a in 1:nrow(acc_df)) {
     current <- twitter_api_call(api_url, query, api_params)
     tweets_temp <- fromJSON(correctJSON(current))
     
+    ## STAT ERROR HANDLING ##
+    
+    # Check for empty API returns
     status <- length(tweets_temp)
     if(status == 0) {
+      if(error > 2) {
+        cat("[WARNING] 3x empty API result. Aborting now.\n")
+        break
+      }
       cat("[WARNING] Empty API result. Trying again.\n")
       rm(tweets_temp)
-      Sys.sleep(2)
+      error <- error + 1
+      Sys.sleep(3)
       next
     }
     
@@ -75,6 +85,30 @@ for(a in 1:nrow(acc_df)) {
       break
     }
     
+    # Check for other errors, mostly rate limits
+    status <- "errors" %in% names(tweets_temp)
+    if(status) {
+      cat("[WARNING] Error in API request:", tweets_temp$errors[1,1],"\n")
+      
+      # Rate limit exceeded?
+      status <- tweets_temp$errors[1,2]
+      if(status == 88) {
+        rate_api_url <- "https://api.twitter.com/1.1/application/rate_limit_status.json"
+        rate_query <-c (resources="statuses")
+        resettime <- fromJSON(twitter_api_call(rate_api_url, rate_query, api_params))
+        resettime <- resettime$resources$statuses$`/statuses/user_timeline`$reset
+        curtime <- as.numeric(as.POSIXct(Sys.time()))
+        wait <- round(resettime - curtime + 10)
+        cat("[INFO] Rate limit is exceeded. Now waiting",wait,"seconds.\n")
+        Sys.sleep(wait)
+      }   
+      rm(tweets_temp)
+      Sys.sleep(3)
+      next
+    }
+    ## END ERROR HANDLING ##
+    
+    
     # Delete unnecessary columns and add username and real name to dataframe
     tweets_temp <- tweets_temp[keep]
     tweets_temp <- cbind(user=user, name=name, tweets_temp)