JSON fixed, now large numbers to do
This commit is contained in:
@@ -1,17 +1,12 @@
|
||||
status <- str_detect(tweets_full$created_at[r], "2013$")
|
||||
if(is.na(status)) { status <- FALSE }
|
||||
if(status) {
|
||||
tweets_full <- tweets_full[-r,]
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
else {
|
||||
max_id <- as.character(as.numeric(tweets_full$id_str[nrow(tweets_full)]) - 1)
|
||||
loop <- loop + 1
|
||||
}
|
||||
}
|
||||
View(tweets_full)
|
||||
source("functions.R")
|
||||
setwd("~/Dokumente/Uni/Aktuell/BA-Arbeit/uni-ba-issuecomp")
|
||||
require(jsonlite)
|
||||
require(stringr)
|
||||
require(RCurl)
|
||||
require(devtools)
|
||||
require(RTwitterAPI)
|
||||
source("functions.R")
|
||||
setwd("~/Dokumente/Uni/Aktuell/BA-Arbeit/uni-ba-issuecomp")
|
||||
api_params <- c(
|
||||
"oauth_consumer_key" = "c9Ob2fWNSONMC0mA2JlNaeRke",
|
||||
"oauth_nonce" = NA,
|
||||
@@ -23,86 +18,7 @@ api_params <- c(
|
||||
"oauth_token_secret" = "rvfv8MgexFKTqrPNSoGrdrZVNhV4fTJb2Bgz249nbvKNg"
|
||||
)
|
||||
api_url <- "https://api.twitter.com/1.1/statuses/user_timeline.json";
|
||||
user <- "peteraltmaier"
|
||||
max_count <- "10"
|
||||
max_id <- "454359322768990208"
|
||||
loop <- 1
|
||||
keep <- c("created_at", "id_str", "text", "retweet_count")
|
||||
rm(tweets_full, tweets_temp)
|
||||
repeat {
|
||||
query <- c(include_rts=1, exclude_replies="true", trim_user="true", include_entities="false",
|
||||
screen_name=user,
|
||||
count=max_count,
|
||||
max_id=max_id);
|
||||
if(exists("tweets_full")) {
|
||||
tweets_temp <- fromJSON(correctJSON(twitter_api_call(api_url, query, api_params)))
|
||||
tweets_temp <- tweets_temp[keep]
|
||||
tweets_full <- insertRow(tweets_full, tweets_temp)
|
||||
}
|
||||
else {
|
||||
tweets_full <- fromJSON(correctJSON(twitter_api_call(api_url, query, api_params)))
|
||||
tweets_full <- tweets_full[keep]
|
||||
}
|
||||
status <- str_detect(tweets_full$created_at[nrow(tweets_full)], "2013$")
|
||||
if (status) {
|
||||
rm(tweets_temp)
|
||||
for(r in 1:nrow(tweets_full)) {
|
||||
status <- str_detect(tweets_full$created_at[r], "2013$")
|
||||
if(is.na(status)) { status <- FALSE }
|
||||
if(status) {
|
||||
tweets_full <- tweets_full[-r,]
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
else {
|
||||
max_id <- as.character(as.numeric(tweets_full$id_str[nrow(tweets_full)]) - 1)
|
||||
loop <- loop + 1
|
||||
}
|
||||
}
|
||||
View(tweets_full)
|
||||
api_url <- "https://api.twitter.com/1.1/statuses/user_timeline.json";
|
||||
user <- "peteraltmaier"
|
||||
max_count <- "200"
|
||||
max_id <- "999999999999999999"
|
||||
loop <- 1
|
||||
keep <- c("created_at", "id_str", "text", "retweet_count")
|
||||
rm(tweets_full, tweets_temp)
|
||||
repeat {
|
||||
query <- c(include_rts=1, exclude_replies="true", trim_user="true", include_entities="false",
|
||||
screen_name=user,
|
||||
count=max_count,
|
||||
max_id=max_id);
|
||||
if(exists("tweets_full")) {
|
||||
tweets_temp <- fromJSON(correctJSON(twitter_api_call(api_url, query, api_params)))
|
||||
tweets_temp <- tweets_temp[keep]
|
||||
tweets_full <- insertRow(tweets_full, tweets_temp)
|
||||
}
|
||||
else {
|
||||
tweets_full <- fromJSON(correctJSON(twitter_api_call(api_url, query, api_params)))
|
||||
tweets_full <- tweets_full[keep]
|
||||
}
|
||||
status <- str_detect(tweets_full$created_at[nrow(tweets_full)], "2013$")
|
||||
if (status) {
|
||||
rm(tweets_temp)
|
||||
for(r in 1:nrow(tweets_full)) {
|
||||
status <- str_detect(tweets_full$created_at[r], "2013$")
|
||||
if(is.na(status)) { status <- FALSE }
|
||||
if(status) {
|
||||
tweets_full <- tweets_full[-r,]
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
else {
|
||||
max_id <- as.character(as.numeric(tweets_full$id_str[nrow(tweets_full)]) - 1)
|
||||
loop <- loop + 1
|
||||
}
|
||||
}
|
||||
tweets_temp <- fromJSON(correctJSON(twitter_api_call(api_url, query, api_params)))
|
||||
current <- twitter_api_call(api_url, query, api_params)
|
||||
correctJSON(current)
|
||||
api_url <- "https://api.twitter.com/1.1/statuses/user_timeline.json";
|
||||
# api_url <- "https://api.twitter.com/1.1/statuses/show.json";
|
||||
user <- "peteraltmaier"
|
||||
max_count <- "200"
|
||||
max_id <- "999999999999999999"
|
||||
@@ -114,9 +30,11 @@ query <- c(include_rts=1, exclude_replies="true", trim_user="true", include_enti
|
||||
screen_name=user,
|
||||
count=max_count,
|
||||
max_id=max_id);
|
||||
# query <- c(trim_user="true", include_entities="false",
|
||||
# id="431858659656990721");
|
||||
if(exists("tweets_full")) {
|
||||
current <- twitter_api_call(api_url, query, api_params)
|
||||
tweets_temp <- fromJSON(correctJSON(current))
|
||||
tweets_temp <- fromJSON(current)
|
||||
tweets_temp <- tweets_temp[keep]
|
||||
tweets_full <- insertRow(tweets_full, tweets_temp)
|
||||
}
|
||||
@@ -142,371 +60,13 @@ max_id <- as.character(as.numeric(tweets_full$id_str[nrow(tweets_full)]) - 1)
|
||||
loop <- loop + 1
|
||||
}
|
||||
}
|
||||
current <- twitter_api_call(api_url, query, api_params)
|
||||
tweets_temp <- fromJSON(current)
|
||||
tweets_temp <- fromJSON(correctJSON(current))
|
||||
current
|
||||
string <- current
|
||||
string <- str_replace_all(string, pattern = perl('\\\\(?![tn"])'), replacement = " ")
|
||||
string <- str_replace_all(string, pattern = fixed("\n"), replacement = " ")
|
||||
str_replace(string, pattern = fixed("\n"), replacement = "X-X-X-X")
|
||||
str_replace_all(string, pattern = fixed("\n"), replacement = "X-X-X-X")
|
||||
current
|
||||
rm(tweets_full, api_params, api_url, keep, loop, max_count, max_id, status, user, insertRow, correctJSON2)
|
||||
query
|
||||
api_url <- "https://api.twitter.com/1.1/statuses/user_timeline.json";
|
||||
user <- "peteraltmaier"
|
||||
max_count <- "10"
|
||||
max_id <- "999999999999999999"
|
||||
loop <- 1
|
||||
keep <- c("created_at", "id_str", "text", "retweet_count")
|
||||
rm(tweets_full, tweets_temp)
|
||||
repeat {
|
||||
query <- c(include_rts=1, exclude_replies="true", trim_user="true", include_entities="false",
|
||||
screen_name=user,
|
||||
count=max_count,
|
||||
max_id=max_id);
|
||||
if(exists("tweets_full")) {
|
||||
current <- twitter_api_call(api_url, query, api_params)
|
||||
tweets_temp <- fromJSON(correctJSON(current))
|
||||
tweets_temp <- tweets_temp[keep]
|
||||
tweets_full <- insertRow(tweets_full, tweets_temp)
|
||||
}
|
||||
else {
|
||||
current <- twitter_api_call(api_url, query, api_params)
|
||||
tweets_full <- fromJSON(correctJSON(current))
|
||||
tweets_full <- tweets_full[keep]
|
||||
}
|
||||
status <- str_detect(tweets_full$created_at[nrow(tweets_full)], "2013$")
|
||||
if (status) {
|
||||
rm(tweets_temp)
|
||||
for(r in 1:nrow(tweets_full)) {
|
||||
status <- str_detect(tweets_full$created_at[r], "2013$")
|
||||
if(is.na(status)) { status <- FALSE }
|
||||
if(status) {
|
||||
tweets_full <- tweets_full[-r,]
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
else {
|
||||
max_id <- as.character(as.numeric(tweets_full$id_str[nrow(tweets_full)]) - 1)
|
||||
loop <- loop + 1
|
||||
}
|
||||
}
|
||||
View(tweets_full)
|
||||
View(tweets_temp)
|
||||
api_url <- "https://api.twitter.com/1.1/statuses/user_timeline.json";
|
||||
user <- "peteraltmaier"
|
||||
max_count <- "20"
|
||||
max_id <- "999999999999999999"
|
||||
loop <- 1
|
||||
keep <- c("created_at", "id_str", "text", "retweet_count")
|
||||
rm(tweets_full, tweets_temp)
|
||||
repeat {
|
||||
query <- c(include_rts=1, exclude_replies="true", trim_user="true", include_entities="false",
|
||||
screen_name=user,
|
||||
count=max_count,
|
||||
max_id=max_id);
|
||||
if(exists("tweets_full")) {
|
||||
current <- twitter_api_call(api_url, query, api_params)
|
||||
tweets_temp <- fromJSON(correctJSON(current))
|
||||
tweets_temp <- tweets_temp[keep]
|
||||
tweets_full <- insertRow(tweets_full, tweets_temp)
|
||||
}
|
||||
else {
|
||||
current <- twitter_api_call(api_url, query, api_params)
|
||||
tweets_full <- fromJSON(correctJSON(current))
|
||||
tweets_full <- tweets_full[keep]
|
||||
}
|
||||
status <- str_detect(tweets_full$created_at[nrow(tweets_full)], "2013$")
|
||||
if (status) {
|
||||
rm(tweets_temp)
|
||||
for(r in 1:nrow(tweets_full)) {
|
||||
status <- str_detect(tweets_full$created_at[r], "2013$")
|
||||
if(is.na(status)) { status <- FALSE }
|
||||
if(status) {
|
||||
tweets_full <- tweets_full[-r,]
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
else {
|
||||
max_id <- as.character(as.numeric(tweets_full$id_str[nrow(tweets_full)]) - 1)
|
||||
loop <- loop + 1
|
||||
}
|
||||
}
|
||||
View(tweets_full)
|
||||
api_url <- "https://api.twitter.com/1.1/statuses/user_timeline.json";
|
||||
user <- "peteraltmaier"
|
||||
max_count <- "10"
|
||||
max_id <- "999999999999999999"
|
||||
loop <- 1
|
||||
keep <- c("created_at", "id_str", "text", "retweet_count")
|
||||
rm(tweets_full, tweets_temp)
|
||||
repeat {
|
||||
query <- c(include_rts=1, exclude_replies="false", trim_user="true", include_entities="false",
|
||||
screen_name=user,
|
||||
count=max_count,
|
||||
max_id=max_id);
|
||||
if(exists("tweets_full")) {
|
||||
current <- twitter_api_call(api_url, query, api_params)
|
||||
tweets_temp <- fromJSON(correctJSON(current))
|
||||
tweets_temp <- tweets_temp[keep]
|
||||
tweets_full <- insertRow(tweets_full, tweets_temp)
|
||||
}
|
||||
else {
|
||||
current <- twitter_api_call(api_url, query, api_params)
|
||||
tweets_full <- fromJSON(correctJSON(current))
|
||||
tweets_full <- tweets_full[keep]
|
||||
}
|
||||
status <- str_detect(tweets_full$created_at[nrow(tweets_full)], "2013$")
|
||||
if (status) {
|
||||
rm(tweets_temp)
|
||||
for(r in 1:nrow(tweets_full)) {
|
||||
status <- str_detect(tweets_full$created_at[r], "2013$")
|
||||
if(is.na(status)) { status <- FALSE }
|
||||
if(status) {
|
||||
tweets_full <- tweets_full[-r,]
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
else {
|
||||
max_id <- as.character(as.numeric(tweets_full$id_str[nrow(tweets_full)]) - 1)
|
||||
loop <- loop + 1
|
||||
}
|
||||
}
|
||||
View(tweets_full)
|
||||
api_params <- c(
|
||||
"oauth_consumer_key" = "c9Ob2fWNSONMC0mA2JlNaeRke",
|
||||
"oauth_nonce" = NA,
|
||||
"oauth_signature_method" = "HMAC-SHA1",
|
||||
"oauth_timestamp" = NA,
|
||||
"oauth_token" = "1007025684-RFxCDFc4OPkt02bASmdci00TB4jgaPjfqxLRT58",
|
||||
"oauth_version" = "1.0",
|
||||
"consumer_secret" = "cZ3Il2hmbLgK0Lc57mj5kUvymjVdsmZKYwKOGHR3NhCpvWgEOI",
|
||||
"oauth_token_secret" = "rvfv8MgexFKTqrPNSoGrdrZVNhV4fTJb2Bgz249nbvKNg"
|
||||
)
|
||||
api_url <- "https://api.twitter.com/1.1/statuses/user_timeline.json";
|
||||
user <- "peteraltmaier"
|
||||
max_count <- "10"
|
||||
max_id <- "999999999999999999"
|
||||
loop <- 1
|
||||
keep <- c("created_at", "id_str", "text", "retweet_count")
|
||||
rm(tweets_full, tweets_temp)
|
||||
repeat {
|
||||
query <- c(include_rts=1, exclude_replies="false", trim_user="true", include_entities="false",
|
||||
screen_name=user,
|
||||
count=max_count,
|
||||
max_id=max_id);
|
||||
if(exists("tweets_full")) {
|
||||
current <- twitter_api_call(api_url, query, api_params)
|
||||
tweets_temp <- fromJSON(correctJSON(current))
|
||||
tweets_temp <- tweets_temp[keep]
|
||||
tweets_full <- insertRow(tweets_full, tweets_temp)
|
||||
}
|
||||
else {
|
||||
current <- twitter_api_call(api_url, query, api_params)
|
||||
tweets_full <- fromJSON(correctJSON(current))
|
||||
tweets_full <- tweets_full[keep]
|
||||
}
|
||||
status <- str_detect(tweets_full$created_at[nrow(tweets_full)], "2013$")
|
||||
if (status) {
|
||||
rm(tweets_temp)
|
||||
for(r in 1:nrow(tweets_full)) {
|
||||
status <- str_detect(tweets_full$created_at[r], "2013$")
|
||||
if(is.na(status)) { status <- FALSE }
|
||||
if(status) {
|
||||
tweets_full <- tweets_full[-r,]
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
else {
|
||||
max_id <- as.character(as.numeric(tweets_full$id_str[nrow(tweets_full)]) - 1)
|
||||
loop <- loop + 1
|
||||
}
|
||||
}
|
||||
current
|
||||
correctJSON(current)
|
||||
validate(current)
|
||||
string <- current
|
||||
str_replace_all(string, pattern = fixed("\n"), replacement = " ")
|
||||
current
|
||||
fromJSON(current)
|
||||
str_replace_all(string, pattern = fixed("\n"), replacement = " ")
|
||||
str_replace_all(string, pattern = fixed("ewifjweaoegj"), replacement = " ")
|
||||
str_detect(string, pattern = fixed("ewifjweaoegj"))
|
||||
str_detect(string, pattern = fixed("ewifjweaoegj"))
|
||||
str_detect(string, pattern = fixed("wfwe"))
|
||||
str_detect(string, pattern = fixed("peter"))
|
||||
str_detect(string, pattern = "kefme")
|
||||
str_detect(string, pattern = "\n")
|
||||
str_detect(string, pattern = "\\n")
|
||||
string <- str_replace_all(string, pattern = fixed("\n"), replacement = " ")
|
||||
string <- "bla bla \nwoqdwqowd"
|
||||
string
|
||||
str_replace_all(string, pattern = fixed("\n"), replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "\n", replacement = " ")
|
||||
string
|
||||
correctJSON <- function(string) {
|
||||
string <- str_replace_all(string, pattern = perl('\\\\(?![tn"])'), replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "\n", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "\r", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = fixed("^"), replacement = " ")
|
||||
return(string)
|
||||
}
|
||||
api_url <- "https://api.twitter.com/1.1/statuses/user_timeline.json";
|
||||
user <- "peteraltmaier"
|
||||
max_count <- "200"
|
||||
max_id <- "999999999999999999"
|
||||
loop <- 1
|
||||
keep <- c("created_at", "id_str", "text", "retweet_count")
|
||||
rm(tweets_full, tweets_temp)
|
||||
repeat {
|
||||
query <- c(include_rts=1, exclude_replies="true", trim_user="true", include_entities="false",
|
||||
screen_name=user,
|
||||
count=max_count,
|
||||
max_id=max_id);
|
||||
if(exists("tweets_full")) {
|
||||
current <- twitter_api_call(api_url, query, api_params)
|
||||
tweets_temp <- fromJSON(correctJSON(current))
|
||||
tweets_temp <- tweets_temp[keep]
|
||||
tweets_full <- insertRow(tweets_full, tweets_temp)
|
||||
}
|
||||
else {
|
||||
current <- twitter_api_call(api_url, query, api_params)
|
||||
tweets_full <- fromJSON(correctJSON(current))
|
||||
tweets_full <- tweets_full[keep]
|
||||
}
|
||||
status <- str_detect(tweets_full$created_at[nrow(tweets_full)], "2013$")
|
||||
if (status) {
|
||||
rm(tweets_temp)
|
||||
for(r in 1:nrow(tweets_full)) {
|
||||
status <- str_detect(tweets_full$created_at[r], "2013$")
|
||||
if(is.na(status)) { status <- FALSE }
|
||||
if(status) {
|
||||
tweets_full <- tweets_full[-r,]
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
else {
|
||||
max_id <- as.character(as.numeric(tweets_full$id_str[nrow(tweets_full)]) - 1)
|
||||
loop <- loop + 1
|
||||
}
|
||||
}
|
||||
current
|
||||
tweets_temp <- fromJSON(correctJSON(current))
|
||||
correctJSON(current)
|
||||
string <- current
|
||||
str_replace_all(string, pattern = perl('\\\\(?![tn"])'), replacement = " ")
|
||||
str_replace_all(string, pattern = "\n", replacement = " ")
|
||||
str_replace_all(string, pattern = "\r", replacement = " ")
|
||||
str_replace_all(string, pattern = fixed("^"), replacement = " ")
|
||||
str_replace_all(string, pattern = "^", replacement = " ")
|
||||
correctJSON <- function(string) {
|
||||
string <- str_replace_all(string, pattern = perl('\\\\(?![tn"])'), replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "\n", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "\r", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "^", replacement = " ")
|
||||
return(string)
|
||||
}
|
||||
api_url <- "https://api.twitter.com/1.1/statuses/user_timeline.json";
|
||||
user <- "peteraltmaier"
|
||||
max_count <- "200"
|
||||
max_id <- "999999999999999999"
|
||||
loop <- 1
|
||||
keep <- c("created_at", "id_str", "text", "retweet_count")
|
||||
rm(tweets_full, tweets_temp)
|
||||
repeat {
|
||||
query <- c(include_rts=1, exclude_replies="true", trim_user="true", include_entities="false",
|
||||
screen_name=user,
|
||||
count=max_count,
|
||||
max_id=max_id);
|
||||
if(exists("tweets_full")) {
|
||||
current <- twitter_api_call(api_url, query, api_params)
|
||||
tweets_temp <- fromJSON(correctJSON(current))
|
||||
tweets_temp <- tweets_temp[keep]
|
||||
tweets_full <- insertRow(tweets_full, tweets_temp)
|
||||
}
|
||||
else {
|
||||
current <- twitter_api_call(api_url, query, api_params)
|
||||
tweets_full <- fromJSON(correctJSON(current))
|
||||
tweets_full <- tweets_full[keep]
|
||||
}
|
||||
status <- str_detect(tweets_full$created_at[nrow(tweets_full)], "2013$")
|
||||
if (status) {
|
||||
rm(tweets_temp)
|
||||
for(r in 1:nrow(tweets_full)) {
|
||||
status <- str_detect(tweets_full$created_at[r], "2013$")
|
||||
if(is.na(status)) { status <- FALSE }
|
||||
if(status) {
|
||||
tweets_full <- tweets_full[-r,]
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
else {
|
||||
max_id <- as.character(as.numeric(tweets_full$id_str[nrow(tweets_full)]) - 1)
|
||||
loop <- loop + 1
|
||||
}
|
||||
}
|
||||
current
|
||||
str_replace_all(string, pattern = "^", replacement = " ")
|
||||
current
|
||||
correctJSON <- function(string) {
|
||||
string <- str_replace_all(string, pattern = perl('\\\\(?![tn"])'), replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "\n", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "\r", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "^", replacement = " ")
|
||||
return(string)
|
||||
}
|
||||
tweets_temp <- fromJSON(correctJSON(current))
|
||||
fromJSON(current)
|
||||
validate(correctJSON(current))
|
||||
current
|
||||
str_extract_all(string, pattern = "\n")
|
||||
str_extract_all(string, pattern = "\xed")
|
||||
str_extract_all(string, pattern = "\[:alnum:]{3}")
|
||||
str_extract_all(string, pattern = "\[[:alnum:]]{3}")
|
||||
str_extract_all(string, pattern = "\\[[:alnum:]]{3}")
|
||||
str_extract_all(string, pattern = "\([[:alnum:]]{3})")
|
||||
str_extract_all(string, pattern = "\\([[:alnum:]]{3})")
|
||||
str_extract_all(string, pattern = "([[:alnum:]]{3})")
|
||||
str_extract_all(string, pattern = "\x..")
|
||||
str_extract_all(string, pattern = "\\x..")
|
||||
str_extract_all(string, pattern = "\x..")
|
||||
str_extract_all(string, pattern = "\xed")
|
||||
str_extract_all(string, pattern = "\x")
|
||||
str_extract_all(string, pattern = "\x+")
|
||||
str_extract_all(string, pattern = "\x.+")
|
||||
str_extract_all(string, pattern = "\x")
|
||||
str_extract_all(string, pattern = "\xhh")
|
||||
str_extract_all(string, pattern = "\xxx")
|
||||
str_extract_all(string, pattern = "[[:xdigit:]]")
|
||||
str_extract_all(string, pattern = "(\xhh)")
|
||||
str_extract_all(string, pattern = "[\xhh]")
|
||||
str_extract_all(string, pattern = "\xHH")
|
||||
str_extract_all(string, pattern = "\xa0")
|
||||
str_extract_all(string, pattern = "\xba?")
|
||||
str_extract_all(string, pattern = "(\xba)+?")
|
||||
str_extract_all(string, pattern = "(\xba)?")
|
||||
str_extract_all(string, pattern = "(\xba)")
|
||||
str_extract_all(string, pattern = "\xba")
|
||||
str_extract_all(string, pattern = "\xba")
|
||||
str_extract_all(string, pattern = "\xed")
|
||||
str_extract_all(string, pattern = "(\xed)")
|
||||
str_extract_all(string, pattern = "(\xed)?")
|
||||
str_extract_all(string, pattern = "(\xed)+")
|
||||
str_extract_all(string, pattern = "(\xed)+?")
|
||||
str_extract_all(string, pattern = "\xed")
|
||||
write(string, "temp.txt")
|
||||
Encoding("temp.txt")
|
||||
readLines("temp.txt")
|
||||
validate(string)
|
||||
string <- readLines("temp.txt")
|
||||
validate(string)
|
||||
string <- str_replace_all(string, pattern = perl('\\\\(?![tn"])'), replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "\n", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "\r", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "^", replacement = " ")
|
||||
validate(string)
|
||||
fromJSON("temp.txt")
|
||||
current
|
||||
fromJSON(current)
|
||||
save.image("~/Dokumente/Uni/Aktuell/BA-Arbeit/uni-ba-issuecomp/RData.RData")
|
||||
load("/home/max/Dokumente/Uni/Aktuell/BA-Arbeit/uni-ba-issuecomp/RData.RData")
|
||||
|
||||
BIN
Binary file not shown.
File diff suppressed because one or more lines are too long
+26
-12
@@ -4,10 +4,10 @@ require(RCurl)
|
||||
require(devtools)
|
||||
require(RTwitterAPI)
|
||||
|
||||
source("functions.R")
|
||||
|
||||
setwd("~/Dokumente/Uni/Aktuell/BA-Arbeit/uni-ba-issuecomp")
|
||||
|
||||
source("functions.R")
|
||||
|
||||
# Set curl handle for friendly scraping
|
||||
handle <- getCurlHandle(httpheader = list(from = "max.mehl@uni.kn",
|
||||
'user-agent' = str_c(R.version$version.string)
|
||||
@@ -47,46 +47,60 @@ api_params <- c(
|
||||
)
|
||||
|
||||
api_url <- "https://api.twitter.com/1.1/statuses/user_timeline.json";
|
||||
# api_url <- "https://api.twitter.com/1.1/statuses/show.json";
|
||||
user <- "peteraltmaier"
|
||||
user <- "GregorGysi"
|
||||
max_count <- "200"
|
||||
max_id <- "999999999999999999"
|
||||
loop <- 1
|
||||
keep <- c("created_at", "id_str", "text", "retweet_count")
|
||||
rm(tweets_full, tweets_temp)
|
||||
rm(tweets_full)
|
||||
repeat {
|
||||
# Define specific search query
|
||||
query <- c(include_rts=1, exclude_replies="true", trim_user="true", include_entities="false",
|
||||
screen_name=user,
|
||||
count=max_count,
|
||||
max_id=max_id);
|
||||
# query <- c(trim_user="true", include_entities="false",
|
||||
# id="431858659656990721");
|
||||
|
||||
# If a tweets_full DB already exists (after the first loop this should be the case)
|
||||
if(exists("tweets_full")) {
|
||||
current <- twitter_api_call(api_url, query, api_params)
|
||||
tweets_temp <- fromJSON(current)
|
||||
tweets_temp <- fromJSON(correctJSON(current))
|
||||
tweets_temp <- tweets_temp[keep]
|
||||
tweets_full <- insertRow(tweets_full, tweets_temp)
|
||||
rm(tweets_temp)
|
||||
}
|
||||
# First loop
|
||||
else {
|
||||
current <- twitter_api_call(api_url, query, api_params)
|
||||
tweets_full <- fromJSON(correctJSON(current))
|
||||
tweets_full <- tweets_full[keep]
|
||||
}
|
||||
|
||||
# Is the last tweet in tweets_full from 2013?
|
||||
status <- str_detect(tweets_full$created_at[nrow(tweets_full)], "2013$")
|
||||
# Last loop is reached. Now clear the data frame
|
||||
if (status) {
|
||||
rm(tweets_temp)
|
||||
|
||||
# Delete all tweets from 2013
|
||||
old <- 0
|
||||
for(r in 1:nrow(tweets_full)) {
|
||||
status <- str_detect(tweets_full$created_at[r], "2013$")
|
||||
if(is.na(status)) { status <- FALSE }
|
||||
if(status) {
|
||||
tweets_full <- tweets_full[-r,]
|
||||
old <- old + 1
|
||||
}
|
||||
}
|
||||
break
|
||||
if(old > 0) {
|
||||
old <- old - 1
|
||||
tweets_full <- head(tweets_full, -old)
|
||||
}
|
||||
rm(old)
|
||||
|
||||
break # End loop because 2013 is reached
|
||||
}
|
||||
|
||||
# The last tweet is still from 2014, so we need another loop
|
||||
else {
|
||||
max_id <- as.character(as.numeric(tweets_full$id_str[nrow(tweets_full)]) - 1)
|
||||
max_id <- as.character(as.numeric(tweets_full$id_str[nrow(tweets_full)]) - 2)
|
||||
loop <- loop + 1
|
||||
}
|
||||
}
|
||||
|
||||
+4
-3
@@ -2,10 +2,11 @@ require(stringr)
|
||||
|
||||
# Replace characters messing up JSON validation (\,\n,^)
|
||||
correctJSON <- function(string) {
|
||||
# string <- str_replace_all(string, pattern = "\n", replacement = " ")
|
||||
# string <- str_replace_all(string, pattern = "\r", replacement = " ")
|
||||
# string <- str_replace_all(string, pattern = "\\^", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "[^[:print:]]", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = perl('\\\\(?![tn"])'), replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "\n", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "\r", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "\\^", replacement = " ")
|
||||
return(string)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
[{"created_at":"Fri Feb 07 18:35:02 +0000 2014","id":431858659656990721,"id_str":"431858659656990721","text":"RT @FHubersr: @peteraltmaier //die Schwarz-Grünen werden zeigen, daß sich Ökologie und Ökonomie vertragen und kein Widerspruch sind.Mutig v…","source":"<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a>","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":378693834,"id_str":"378693834"},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweeted_status":{"created_at":"Fri Feb 07 18:32:30 +0000 2014","id":431858022366064640,"id_str":"431858022366064640","text":"@peteraltmaier //die Schwarz-Grünen werden zeigen, daß sich Ökologie und Ökonomie vertragen und kein Widerspruch sind.Mutig von bd. Seiten������������","source":"<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a>","truncated":false,"in_reply_to_status_id":431845492579123201,"in_reply_to_status_id_str":"431845492579123201","in_reply_to_user_id":378693834,"in_reply_to_user_id_str":"378693834","in_reply_to_screen_name":"peteraltmaier","user":{"id":2172292811,"id_str":"2172292811"},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":3,"favorite_count":4,"favorited":false,"retweeted":false,"lang":"de"},"retweet_count":3,"favorite_count":0,"favorited":false,"retweeted":false,"lang":"de"}]
|
||||
Reference in New Issue
Block a user