Browse Source

more api stuff

mxmehl 5 years ago
parent
commit
7723ceca8e

+ 26
- 3
extract-twitter-accounts.R View File

@@ -1,11 +1,12 @@
1 1
 require(jsonlite)
2 2
 require(stringr)
3 3
 require(RCurl)
4
-require(httr)
4
+require(devtools)
5
+require(RTwitterAPI)
5 6
 
6 7
 source("functions.R")
7 8
 
8
-setwd("~/Dokumente/Uni/Aktuell/BA-Arbeit/R-Code")
9
+setwd("~/Dokumente/Uni/Aktuell/BA-Arbeit/uni-ba-issuecomp")
9 10
 
10 11
 # Set curl handle for friendly scraping
11 12
 handle <- getCurlHandle(httpheader = list(from         = "max.mehl@uni.kn",
@@ -14,5 +15,27 @@ handle <- getCurlHandle(httpheader = list(from         = "max.mehl@uni.kn",
14 15
 )
15 16
 
16 17
 acc_url <- "http://www.bundestwitter.de/api/politiker"
18
+acc_json <- readLines("politiker.txt")
19
+acc_df <- fromJSON(acc_json)
17 20
 
18
-df <- fromJSON(acc_url)
21
+# --------------
22
+
23
+
24
+# http://www.joyofdata.de/blog/twitters-rest-api-v1-1-with-r-for-linux-and-windows/
25
+# devtools::install_github("joyofdata/RTwitterAPI")
26
+
27
+params <- c(
28
+  "oauth_consumer_key"     = "c9Ob2fWNSONMC0mA2JlNaeRke", 
29
+  "oauth_nonce"            = NA,
30
+  "oauth_signature_method" = "HMAC-SHA1",
31
+  "oauth_timestamp"        = NA,
32
+  "oauth_token"            = "1007025684-RFxCDFc4OPkt02bASmdci00TB4jgaPjfqxLRT58",
33
+  "oauth_version"          = "1.0",
34
+  "consumer_secret"        = "cZ3Il2hmbLgK0Lc57mj5kUvymjVdsmZKYwKOGHR3NhCpvWgEOI",
35
+  "oauth_token_secret"     = "rvfv8MgexFKTqrPNSoGrdrZVNhV4fTJb2Bgz249nbvKNg"
36
+);
37
+
38
+url   <- "https://api.twitter.com/1.1/statuses/user_timeline.json";
39
+query <- c(screen_name="peteraltmaier", exclude_replies="true", include_rts=1, count="1");
40
+
41
+result <- fromJSON(twitter_api_call(url, query, params))

+ 1
- 0
politiker.txt
File diff suppressed because it is too large
View File


+ 69
- 0
twitter-API/oauth1_signature.R View File

@@ -0,0 +1,69 @@
1
+# calculates the signature for the request as described in the docs:
2
+# https://dev.twitter.com/docs/auth/creating-signature
3
+
4
+#' Generates OAuth1 signature needed to authorize API request
5
+#'
6
+#' @param method f.x. "HMAC-SHA1"
7
+#' @param url f.x. "https://api.twitter.com/1.1/friends/ids.json"
8
+#' @param api f.x. c(cursor=-1, screen_name="hrw", count=10)
9
+#' @param params named vector needed for generating oauth1 signature
10
+#' @return signature string
11
+oauth1_signature <- function(method, url, api, params, test=FALSE) {
12
+  library(RCurl);
13
+  library(digest);
14
+  library(base64enc);
15
+  
16
+  # http://oauth.net/core/1.0/#encoding_parameters
17
+  
18
+  if(curlEscape(".") == ".") {
19
+    # that's how we want it!
20
+  } else if(curlEscape(".") == "%2E") {
21
+    curlEscape <- function(str) {
22
+      esc <- RCurl::curlEscape(str)
23
+      esc <- gsub("%2E",".",esc)
24
+      esc <- gsub("%2D","-",esc)
25
+      esc <- gsub("%5F","_",esc)
26
+      esc <- gsub("%7E","~",esc)
27
+    }
28
+  } else {
29
+    stop("curlEscape('.') is supposed to be either '.' or '%2E'")
30
+  }
31
+    
32
+  secrets <- params[c("consumer_secret","oauth_token_secret")];
33
+  
34
+  # exclude *_secret
35
+  params <- params[! names(params) %in% c("consumer_secret","oauth_token_secret")]
36
+  
37
+  params <- c(api, params);
38
+  params_sorted <- params[sort(names(params))];
39
+  params_escaped <- sapply(params_sorted, curlEscape);
40
+  pstr <- paste(paste(names(params_escaped),"=",params_escaped,sep=""),collapse="&");
41
+  
42
+  # it is important to use curlEscape instead of URLencode because the latter encodes
43
+  # using lower case letters but we need upper case letters. Otherwise the resulting
44
+  # signature will be different from what Twitter/OAuth expects/calculates.
45
+  final <- sprintf("%s&%s&%s",toupper(method), curlEscape(url), curlEscape(pstr));
46
+  
47
+  sig <- sprintf("%s&%s",curlEscape(secrets["consumer_secret"]),curlEscape(secrets["oauth_token_secret"]));
48
+  
49
+  hmac <- hmac(sig,final,algo="sha1",raw=TRUE)
50
+  signature <- base64encode(hmac);
51
+  signature_escaped <- curlEscape(signature);
52
+  
53
+  if(test) {
54
+    return(
55
+      list(
56
+        params_escaped = params_escaped,
57
+        pstr = pstr,
58
+        final = final,
59
+        sig = sig,
60
+        hmac = hmac,
61
+        signature = signature,
62
+        signature_escaped = signature_escaped
63
+      )
64
+    )
65
+  }
66
+  
67
+  return(signature_escaped);
68
+}
69
+

+ 73
- 0
twitter-API/twitter_api_call.R View File

@@ -0,0 +1,73 @@
1
+#' GET requests Twitter API
2
+#'
3
+#' @param url f.x. "https://api.twitter.com/1.1/friends/ids.json"
4
+#' @param api f.x. c(cursor=-1, screen_name="hrw", count=10)
5
+#' @param params named vector needed for generating oauth1 signature
6
+#' @param print_result TRUE if you want to have resulting JSON printed out
7
+#' @param use_cygwin TRUE if you would like to resort to Cygwin/Curl workaround
8
+#' @param cygwin_bash f.x. "c:\\cygwin64\\bin\\bash.exe"
9
+#' @return JSON
10
+#' @export
11
+twitter_api_call <- function(
12
+    url, api, params, print_result=FALSE, 
13
+    use_cygwin=FALSE, cygwin_bash="c:\\cygwin64\\bin\\bash.exe", print_cmd=FALSE,
14
+    test=FALSE) {
15
+  library(jsonlite);
16
+  library(RCurl);
17
+  
18
+  if(is.na(params["oauth_timestamp"])) {
19
+    params["oauth_timestamp"] <- as.character(as.integer(Sys.time()));
20
+  }
21
+  
22
+  if(is.na(params["oauth_nonce"])) {
23
+    params["oauth_nonce"] <- sprintf("%d%s",as.integer(Sys.time()),paste(floor(runif(6)*10^6),collapse=""));
24
+  }
25
+  
26
+  if(test) {
27
+    test_data <- oauth1_signature(method = "GET", url, api, params, test=TRUE);
28
+    params["oauth_signature"] <- test_data[["signature_escaped"]]
29
+  } else {
30
+    params["oauth_signature"] <- oauth1_signature(method = "GET", url, api, params, test=FALSE);
31
+  }
32
+  
33
+  httpheader <- c(
34
+    "Authorization" = sprintf(paste(c(
35
+      'OAuth oauth_consumer_key="%s", oauth_nonce="%s", oauth_signature="%s", ',
36
+      'oauth_signature_method="%s", oauth_timestamp="%s", oauth_token="%s", oauth_version="1.0"'),
37
+      collapse=""),
38
+      params["oauth_consumer_key"], params["oauth_nonce"], params["oauth_signature"],
39
+      params["oauth_signature_method"], params["oauth_timestamp"], params["oauth_token"], params["oauth_version"])
40
+  );
41
+  
42
+  q <- paste(paste(names(api),api,sep="="), collapse="&");
43
+  urlq <- paste(url,q,sep="?");
44
+  
45
+  if(!test) {
46
+    if(!use_cygwin) {
47
+      result <- getURL(urlq, httpheader=httpheader);
48
+    } else {
49
+      httpheader_escaped <- sprintf("Authorization: %s",gsub('"','\"',httpheader["Authorization"]))
50
+      cmd <- sprintf("%s -c \"/usr/bin/curl --silent --get '%s' --data '%s' --header '%s'\"", cygwin_bash, url, q, httpheader_escaped)
51
+      if(print_cmd) {
52
+        cat(cmd)
53
+      }
54
+      result <- system(cmd, intern=TRUE)
55
+    }
56
+  } else {
57
+    result <- "{}"
58
+  }
59
+  
60
+  if(print_result) {
61
+    cat(prettify(result));
62
+  }
63
+  
64
+  if(test) {
65
+    test_data[["httpheader"]] <- httpheader
66
+    test_data[["q"]] <- q
67
+    test_data[["urlq"]] <- urlq
68
+    
69
+    return(test_data)
70
+  }
71
+  
72
+  return(result);
73
+}

+ 1
- 0
twitter-api-consumerkey.txt View File

@@ -0,0 +1 @@
1
+c9Ob2fWNSONMC0mA2JlNaeRke

+ 1
- 0
twitter-api-consumersecret.txt View File

@@ -0,0 +1 @@
1
+cZ3Il2hmbLgK0Lc57mj5kUvymjVdsmZKYwKOGHR3NhCpvWgEOI

Loading…
Cancel
Save