more api stuff

extract-twitter-accounts.R

@@ -1,11 +1,12 @@
 require(jsonlite)
 require(stringr)
 require(RCurl)
-require(httr)
+require(devtools)
+require(RTwitterAPI)
 
 source("functions.R")
 
-setwd("~/Dokumente/Uni/Aktuell/BA-Arbeit/R-Code")
+setwd("~/Dokumente/Uni/Aktuell/BA-Arbeit/uni-ba-issuecomp")
 
 # Set curl handle for friendly scraping
 handle <- getCurlHandle(httpheader = list(from         = "max.mehl@uni.kn",
@@ -14,5 +15,27 @@ handle <- getCurlHandle(httpheader = list(from         = "max.mehl@uni.kn",
 )
 
 acc_url <- "http://www.bundestwitter.de/api/politiker"
+acc_json <- readLines("politiker.txt")
+acc_df <- fromJSON(acc_json)
 
-df <- fromJSON(acc_url)
+# --------------
+
+
+# http://www.joyofdata.de/blog/twitters-rest-api-v1-1-with-r-for-linux-and-windows/
+# devtools::install_github("joyofdata/RTwitterAPI")
+
+params <- c(
+  "oauth_consumer_key"     = "c9Ob2fWNSONMC0mA2JlNaeRke", 
+  "oauth_nonce"            = NA,
+  "oauth_signature_method" = "HMAC-SHA1",
+  "oauth_timestamp"        = NA,
+  "oauth_token"            = "1007025684-RFxCDFc4OPkt02bASmdci00TB4jgaPjfqxLRT58",
+  "oauth_version"          = "1.0",
+  "consumer_secret"        = "cZ3Il2hmbLgK0Lc57mj5kUvymjVdsmZKYwKOGHR3NhCpvWgEOI",
+  "oauth_token_secret"     = "rvfv8MgexFKTqrPNSoGrdrZVNhV4fTJb2Bgz249nbvKNg"
+);
+
+url   <- "https://api.twitter.com/1.1/statuses/user_timeline.json";
+query <- c(screen_name="peteraltmaier", exclude_replies="true", include_rts=1, count="1");
+
+result <- fromJSON(twitter_api_call(url, query, params))

twitter-API/oauth1_signature.R

@@ -0,0 +1,69 @@
+# calculates the signature for the request as described in the docs:
+# https://dev.twitter.com/docs/auth/creating-signature
+
+#' Generates OAuth1 signature needed to authorize API request
+#'
+#' @param method f.x. "HMAC-SHA1"
+#' @param url f.x. "https://api.twitter.com/1.1/friends/ids.json"
+#' @param api f.x. c(cursor=-1, screen_name="hrw", count=10)
+#' @param params named vector needed for generating oauth1 signature
+#' @return signature string
+oauth1_signature <- function(method, url, api, params, test=FALSE) {
+  library(RCurl);
+  library(digest);
+  library(base64enc);
+  
+  # http://oauth.net/core/1.0/#encoding_parameters
+  
+  if(curlEscape(".") == ".") {
+    # that's how we want it!
+  } else if(curlEscape(".") == "%2E") {
+    curlEscape <- function(str) {
+      esc <- RCurl::curlEscape(str)
+      esc <- gsub("%2E",".",esc)
+      esc <- gsub("%2D","-",esc)
+      esc <- gsub("%5F","_",esc)
+      esc <- gsub("%7E","~",esc)
+    }
+  } else {
+    stop("curlEscape('.') is supposed to be either '.' or '%2E'")
+  }
+    
+  secrets <- params[c("consumer_secret","oauth_token_secret")];
+  
+  # exclude *_secret
+  params <- params[! names(params) %in% c("consumer_secret","oauth_token_secret")]
+  
+  params <- c(api, params);
+  params_sorted <- params[sort(names(params))];
+  params_escaped <- sapply(params_sorted, curlEscape);
+  pstr <- paste(paste(names(params_escaped),"=",params_escaped,sep=""),collapse="&");
+  
+  # it is important to use curlEscape instead of URLencode because the latter encodes
+  # using lower case letters but we need upper case letters. Otherwise the resulting
+  # signature will be different from what Twitter/OAuth expects/calculates.
+  final <- sprintf("%s&%s&%s",toupper(method), curlEscape(url), curlEscape(pstr));
+  
+  sig <- sprintf("%s&%s",curlEscape(secrets["consumer_secret"]),curlEscape(secrets["oauth_token_secret"]));
+  
+  hmac <- hmac(sig,final,algo="sha1",raw=TRUE)
+  signature <- base64encode(hmac);
+  signature_escaped <- curlEscape(signature);
+  
+  if(test) {
+    return(
+      list(
+        params_escaped = params_escaped,
+        pstr = pstr,
+        final = final,
+        sig = sig,
+        hmac = hmac,
+        signature = signature,
+        signature_escaped = signature_escaped
+      )
+    )
+  }
+  
+  return(signature_escaped);
+}
+

twitter-API/twitter_api_call.R

@@ -0,0 +1,73 @@
+#' GET requests Twitter API
+#'
+#' @param url f.x. "https://api.twitter.com/1.1/friends/ids.json"
+#' @param api f.x. c(cursor=-1, screen_name="hrw", count=10)
+#' @param params named vector needed for generating oauth1 signature
+#' @param print_result TRUE if you want to have resulting JSON printed out
+#' @param use_cygwin TRUE if you would like to resort to Cygwin/Curl workaround
+#' @param cygwin_bash f.x. "c:\\cygwin64\\bin\\bash.exe"
+#' @return JSON
+#' @export
+twitter_api_call <- function(
+    url, api, params, print_result=FALSE, 
+    use_cygwin=FALSE, cygwin_bash="c:\\cygwin64\\bin\\bash.exe", print_cmd=FALSE,
+    test=FALSE) {
+  library(jsonlite);
+  library(RCurl);
+  
+  if(is.na(params["oauth_timestamp"])) {
+    params["oauth_timestamp"] <- as.character(as.integer(Sys.time()));
+  }
+  
+  if(is.na(params["oauth_nonce"])) {
+    params["oauth_nonce"] <- sprintf("%d%s",as.integer(Sys.time()),paste(floor(runif(6)*10^6),collapse=""));
+  }
+  
+  if(test) {
+    test_data <- oauth1_signature(method = "GET", url, api, params, test=TRUE);
+    params["oauth_signature"] <- test_data[["signature_escaped"]]
+  } else {
+    params["oauth_signature"] <- oauth1_signature(method = "GET", url, api, params, test=FALSE);
+  }
+  
+  httpheader <- c(
+    "Authorization" = sprintf(paste(c(
+      'OAuth oauth_consumer_key="%s", oauth_nonce="%s", oauth_signature="%s", ',
+      'oauth_signature_method="%s", oauth_timestamp="%s", oauth_token="%s", oauth_version="1.0"'),
+      collapse=""),
+      params["oauth_consumer_key"], params["oauth_nonce"], params["oauth_signature"],
+      params["oauth_signature_method"], params["oauth_timestamp"], params["oauth_token"], params["oauth_version"])
+  );
+  
+  q <- paste(paste(names(api),api,sep="="), collapse="&");
+  urlq <- paste(url,q,sep="?");
+  
+  if(!test) {
+    if(!use_cygwin) {
+      result <- getURL(urlq, httpheader=httpheader);
+    } else {
+      httpheader_escaped <- sprintf("Authorization: %s",gsub('"','\"',httpheader["Authorization"]))
+      cmd <- sprintf("%s -c \"/usr/bin/curl --silent --get '%s' --data '%s' --header '%s'\"", cygwin_bash, url, q, httpheader_escaped)
+      if(print_cmd) {
+        cat(cmd)
+      }
+      result <- system(cmd, intern=TRUE)
+    }
+  } else {
+    result <- "{}"
+  }
+  
+  if(print_result) {
+    cat(prettify(result));
+  }
+  
+  if(test) {
+    test_data[["httpheader"]] <- httpheader
+    test_data[["q"]] <- q
+    test_data[["urlq"]] <- urlq
+    
+    return(test_data)
+  }
+  
+  return(result);
+}

twitter-api-consumerkey.txt

@@ -0,0 +1 @@
+c9Ob2fWNSONMC0mA2JlNaeRke

twitter-api-consumersecret.txt

@@ -0,0 +1 @@
+cZ3Il2hmbLgK0Lc57mj5kUvymjVdsmZKYwKOGHR3NhCpvWgEOI