working userloop

extract-twitter-accounts.R

@@ -8,11 +8,11 @@ setwd("~/Dokumente/Uni/Aktuell/BA-Arbeit/uni-ba-issuecomp")
 
 source("functions.R")
 
-# Set curl handle for friendly scraping
+# # Set curl handle for friendly scraping
-handle <- getCurlHandle(httpheader = list(from         = "max.mehl@uni.kn",
+# handle <- getCurlHandle(httpheader = list(from         = "max.mehl@uni.kn",
-                                          'user-agent' = str_c(R.version$version.string)
+#                                           'user-agent' = str_c(R.version$version.string)
-)
+# )
-)
+# )
 
 acc_url <- "http://www.bundestwitter.de/api/politiker"
 #acc_json <- readLines("politiker.txt")
@@ -35,70 +35,79 @@ api_params <- c(
 )
 
 api_url   <- "https://api.twitter.com/1.1/statuses/user_timeline.json";
-user <- "GregorGysi"
 max_count <- "200"
-max_id <- "999999999999999999"
-loop <- 1
 keep <- c("created_at", "id_str", "text", "retweet_count")
 rm(tweets_full)
-last_id <- NULL
+
-repeat {
+for(a in 1:nrow(acc_df)) {
-  # Define specific search query
+  user <- as.character(acc_df$screenname[a])
-  query <- c(include_rts=1, exclude_replies="true", trim_user="true", include_entities="false", 
+  name <- as.character(acc_df$name[a])
-             screen_name=user, 
+  max_id <- "999999999999999999"
-             count=max_count, 
+  loop <- 1
-             max_id=max_id);
+  repeat {
-  
+    # Define specific search query
-  # If a tweets_full DB already exists (after the first loop this should be the case)
+    query <- c(include_rts=1, exclude_replies="true", trim_user="true", include_entities="false", 
-  if(exists("tweets_full")) {
+               screen_name=user, 
-    current <- twitter_api_call(api_url, query, api_params)
+               count=max_count, 
-    tweets_temp <- fromJSON(correctJSON(current))
+               max_id=max_id);
-    tweets_temp <- tweets_temp[keep]
-    tweets_full <- insertRow(tweets_full, tweets_temp)
-    rm(tweets_temp)
-  }
-  # First loop
-  else {
-    current <- twitter_api_call(api_url, query, api_params)
-    tweets_full <- fromJSON(correctJSON(current))
-    tweets_full <- tweets_full[keep]
-  }
-  
-  # Now sleep 3 second to dodge 300queries/15min limit
-  cat("User:",user,"in loop:",loop,"- now waiting 3 secs...\n")
-  Sys.sleep(3)
-  
-  # Is the last tweet in tweets_full from 2013?
-  status <- str_detect(tweets_full$created_at[nrow(tweets_full)], "2013$")
-  # Last loop is reached. Now clear the data frame
-  if (status) {
     
-    # Delete all tweets from 2013
+    # If a tweets_full DB already exists (after the first loop this should be the case)
-    old <- 0
+    if(exists("tweets_full")) {
-    for(r in 1:nrow(tweets_full)) {
+      current <- twitter_api_call(api_url, query, api_params)
-      status <- str_detect(tweets_full$created_at[r], "2013$")
+      tweets_temp <- fromJSON(correctJSON(current))
-      if(is.na(status)) { status <- FALSE }
+      tweets_temp <- tweets_temp[keep]
-      if(status) {
+      tweets_temp <- cbind(user=user, name=name, tweets_temp)
-        old <- old + 1
+      tweets_full <- insertRow(tweets_full, tweets_temp)
+      rm(tweets_temp)
+    }
+    # First loop
+    else {
+      current <- twitter_api_call(api_url, query, api_params)
+      tweets_full <- fromJSON(correctJSON(current))
+      tweets_full <- tweets_full[keep]
+      tweets_full <- cbind(user=user, name=name, tweets_full)
+    }
+    
+    # Now sleep 3 second to dodge 300queries/15min limit
+    cat("User:",user,"in loop:",loop,"- now waiting 2 secs...\n")
+    Sys.sleep(2)
+    
+    # Is the last tweet in tweets_full from 2013?
+    status <- str_detect(tweets_full$created_at[nrow(tweets_full)], "2013$")
+    # Last loop is reached. Now clear the data frame
+    if (status) {
+      
+      # Delete all tweets from 2013
+      old <- 0
+      for(r in 1:nrow(tweets_full)) {
+        status <- str_detect(tweets_full$created_at[r], "2013$")
+        if(is.na(status)) { status <- FALSE }
+        if(status) {
+          old <- old + 1
+        }
       }
+      if(old > 0) {
+        old <- old - 1
+        tweets_full <- head(tweets_full, -old)
+      }
+      rm(old)
+      
+      break  # End loop because 2013 is reached
     }
-    if(old > 0) {
-      old <- old - 1
-      tweets_full <- head(tweets_full, -old)
-    }
-    rm(old)
     
-    break  # End loop because 2013 is reached
+    # The last tweet is still from 2014, so we need another loop
+    else {
+      # Setting max_id to gather next 200 tweets
+      max_id <- tweets_full$id_str[nrow(tweets_full)]
+      loop <- loop + 1  # just for stats
+    }
   }
   
-  # The last tweet is still from 2014, so we need another loop
+  # Every tweet from 2014 from user[r] is downloaded. Now next user in for-loop
-  else {
+  cat("User:",user,"finished after",loop,"loops\n")
-    # Setting max_id to gather next 200 tweets
-    max_id <- tweets_full$id_str[nrow(tweets_full)]
-    loop <- loop + 1  # just for stats
-  }
 }
 
+
 # ---------------
 
 

functions.R

@@ -10,15 +10,6 @@ correctJSON <- function(string) {
   return(string)
 }
 
-correctJSON2 <- function(string) {
-  #string <- sub(x=string, pattern = perl('\\\\(?![tn"])'), replacement = " ")
-  string <- gsub(x=string, pattern = "\n", replacement = " ")
-  string <- gsub(x=string, pattern = "\r", replacement = " ")
-  string <- gsub(x=string, pattern = "\\^", replacement = " ")
-  #\xed\xa0\xbd\xed\xb1\x8d\xed\xa0\xbd\xed\xb8\x8e\
-  return(string)
-}
-
 insertRow <- function(existingDF, newrow, r) {
   r <- as.numeric(nrow(existingDF)) + 1
   existingDF <- rbind(existingDF,newrow)