JSON fixed, now large numbers to do
This commit is contained in:
@@ -1,17 +1,12 @@
|
|||||||
status <- str_detect(tweets_full$created_at[r], "2013$")
|
source("functions.R")
|
||||||
if(is.na(status)) { status <- FALSE }
|
setwd("~/Dokumente/Uni/Aktuell/BA-Arbeit/uni-ba-issuecomp")
|
||||||
if(status) {
|
require(jsonlite)
|
||||||
tweets_full <- tweets_full[-r,]
|
require(stringr)
|
||||||
}
|
require(RCurl)
|
||||||
}
|
require(devtools)
|
||||||
break
|
require(RTwitterAPI)
|
||||||
}
|
source("functions.R")
|
||||||
else {
|
setwd("~/Dokumente/Uni/Aktuell/BA-Arbeit/uni-ba-issuecomp")
|
||||||
max_id <- as.character(as.numeric(tweets_full$id_str[nrow(tweets_full)]) - 1)
|
|
||||||
loop <- loop + 1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
View(tweets_full)
|
|
||||||
api_params <- c(
|
api_params <- c(
|
||||||
"oauth_consumer_key" = "c9Ob2fWNSONMC0mA2JlNaeRke",
|
"oauth_consumer_key" = "c9Ob2fWNSONMC0mA2JlNaeRke",
|
||||||
"oauth_nonce" = NA,
|
"oauth_nonce" = NA,
|
||||||
@@ -23,86 +18,7 @@ api_params <- c(
|
|||||||
"oauth_token_secret" = "rvfv8MgexFKTqrPNSoGrdrZVNhV4fTJb2Bgz249nbvKNg"
|
"oauth_token_secret" = "rvfv8MgexFKTqrPNSoGrdrZVNhV4fTJb2Bgz249nbvKNg"
|
||||||
)
|
)
|
||||||
api_url <- "https://api.twitter.com/1.1/statuses/user_timeline.json";
|
api_url <- "https://api.twitter.com/1.1/statuses/user_timeline.json";
|
||||||
user <- "peteraltmaier"
|
# api_url <- "https://api.twitter.com/1.1/statuses/show.json";
|
||||||
max_count <- "10"
|
|
||||||
max_id <- "454359322768990208"
|
|
||||||
loop <- 1
|
|
||||||
keep <- c("created_at", "id_str", "text", "retweet_count")
|
|
||||||
rm(tweets_full, tweets_temp)
|
|
||||||
repeat {
|
|
||||||
query <- c(include_rts=1, exclude_replies="true", trim_user="true", include_entities="false",
|
|
||||||
screen_name=user,
|
|
||||||
count=max_count,
|
|
||||||
max_id=max_id);
|
|
||||||
if(exists("tweets_full")) {
|
|
||||||
tweets_temp <- fromJSON(correctJSON(twitter_api_call(api_url, query, api_params)))
|
|
||||||
tweets_temp <- tweets_temp[keep]
|
|
||||||
tweets_full <- insertRow(tweets_full, tweets_temp)
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
tweets_full <- fromJSON(correctJSON(twitter_api_call(api_url, query, api_params)))
|
|
||||||
tweets_full <- tweets_full[keep]
|
|
||||||
}
|
|
||||||
status <- str_detect(tweets_full$created_at[nrow(tweets_full)], "2013$")
|
|
||||||
if (status) {
|
|
||||||
rm(tweets_temp)
|
|
||||||
for(r in 1:nrow(tweets_full)) {
|
|
||||||
status <- str_detect(tweets_full$created_at[r], "2013$")
|
|
||||||
if(is.na(status)) { status <- FALSE }
|
|
||||||
if(status) {
|
|
||||||
tweets_full <- tweets_full[-r,]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
break
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
max_id <- as.character(as.numeric(tweets_full$id_str[nrow(tweets_full)]) - 1)
|
|
||||||
loop <- loop + 1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
View(tweets_full)
|
|
||||||
api_url <- "https://api.twitter.com/1.1/statuses/user_timeline.json";
|
|
||||||
user <- "peteraltmaier"
|
|
||||||
max_count <- "200"
|
|
||||||
max_id <- "999999999999999999"
|
|
||||||
loop <- 1
|
|
||||||
keep <- c("created_at", "id_str", "text", "retweet_count")
|
|
||||||
rm(tweets_full, tweets_temp)
|
|
||||||
repeat {
|
|
||||||
query <- c(include_rts=1, exclude_replies="true", trim_user="true", include_entities="false",
|
|
||||||
screen_name=user,
|
|
||||||
count=max_count,
|
|
||||||
max_id=max_id);
|
|
||||||
if(exists("tweets_full")) {
|
|
||||||
tweets_temp <- fromJSON(correctJSON(twitter_api_call(api_url, query, api_params)))
|
|
||||||
tweets_temp <- tweets_temp[keep]
|
|
||||||
tweets_full <- insertRow(tweets_full, tweets_temp)
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
tweets_full <- fromJSON(correctJSON(twitter_api_call(api_url, query, api_params)))
|
|
||||||
tweets_full <- tweets_full[keep]
|
|
||||||
}
|
|
||||||
status <- str_detect(tweets_full$created_at[nrow(tweets_full)], "2013$")
|
|
||||||
if (status) {
|
|
||||||
rm(tweets_temp)
|
|
||||||
for(r in 1:nrow(tweets_full)) {
|
|
||||||
status <- str_detect(tweets_full$created_at[r], "2013$")
|
|
||||||
if(is.na(status)) { status <- FALSE }
|
|
||||||
if(status) {
|
|
||||||
tweets_full <- tweets_full[-r,]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
break
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
max_id <- as.character(as.numeric(tweets_full$id_str[nrow(tweets_full)]) - 1)
|
|
||||||
loop <- loop + 1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
tweets_temp <- fromJSON(correctJSON(twitter_api_call(api_url, query, api_params)))
|
|
||||||
current <- twitter_api_call(api_url, query, api_params)
|
|
||||||
correctJSON(current)
|
|
||||||
api_url <- "https://api.twitter.com/1.1/statuses/user_timeline.json";
|
|
||||||
user <- "peteraltmaier"
|
user <- "peteraltmaier"
|
||||||
max_count <- "200"
|
max_count <- "200"
|
||||||
max_id <- "999999999999999999"
|
max_id <- "999999999999999999"
|
||||||
@@ -114,9 +30,11 @@ query <- c(include_rts=1, exclude_replies="true", trim_user="true", include_enti
|
|||||||
screen_name=user,
|
screen_name=user,
|
||||||
count=max_count,
|
count=max_count,
|
||||||
max_id=max_id);
|
max_id=max_id);
|
||||||
|
# query <- c(trim_user="true", include_entities="false",
|
||||||
|
# id="431858659656990721");
|
||||||
if(exists("tweets_full")) {
|
if(exists("tweets_full")) {
|
||||||
current <- twitter_api_call(api_url, query, api_params)
|
current <- twitter_api_call(api_url, query, api_params)
|
||||||
tweets_temp <- fromJSON(correctJSON(current))
|
tweets_temp <- fromJSON(current)
|
||||||
tweets_temp <- tweets_temp[keep]
|
tweets_temp <- tweets_temp[keep]
|
||||||
tweets_full <- insertRow(tweets_full, tweets_temp)
|
tweets_full <- insertRow(tweets_full, tweets_temp)
|
||||||
}
|
}
|
||||||
@@ -142,371 +60,13 @@ max_id <- as.character(as.numeric(tweets_full$id_str[nrow(tweets_full)]) - 1)
|
|||||||
loop <- loop + 1
|
loop <- loop + 1
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
current <- twitter_api_call(api_url, query, api_params)
|
||||||
|
tweets_temp <- fromJSON(current)
|
||||||
|
tweets_temp <- fromJSON(correctJSON(current))
|
||||||
current
|
current
|
||||||
string <- current
|
current
|
||||||
string <- str_replace_all(string, pattern = perl('\\\\(?![tn"])'), replacement = " ")
|
rm(tweets_full, api_params, api_url, keep, loop, max_count, max_id, status, user, insertRow, correctJSON2)
|
||||||
string <- str_replace_all(string, pattern = fixed("\n"), replacement = " ")
|
query
|
||||||
str_replace(string, pattern = fixed("\n"), replacement = "X-X-X-X")
|
|
||||||
str_replace_all(string, pattern = fixed("\n"), replacement = "X-X-X-X")
|
|
||||||
api_url <- "https://api.twitter.com/1.1/statuses/user_timeline.json";
|
api_url <- "https://api.twitter.com/1.1/statuses/user_timeline.json";
|
||||||
user <- "peteraltmaier"
|
save.image("~/Dokumente/Uni/Aktuell/BA-Arbeit/uni-ba-issuecomp/RData.RData")
|
||||||
max_count <- "10"
|
load("/home/max/Dokumente/Uni/Aktuell/BA-Arbeit/uni-ba-issuecomp/RData.RData")
|
||||||
max_id <- "999999999999999999"
|
|
||||||
loop <- 1
|
|
||||||
keep <- c("created_at", "id_str", "text", "retweet_count")
|
|
||||||
rm(tweets_full, tweets_temp)
|
|
||||||
repeat {
|
|
||||||
query <- c(include_rts=1, exclude_replies="true", trim_user="true", include_entities="false",
|
|
||||||
screen_name=user,
|
|
||||||
count=max_count,
|
|
||||||
max_id=max_id);
|
|
||||||
if(exists("tweets_full")) {
|
|
||||||
current <- twitter_api_call(api_url, query, api_params)
|
|
||||||
tweets_temp <- fromJSON(correctJSON(current))
|
|
||||||
tweets_temp <- tweets_temp[keep]
|
|
||||||
tweets_full <- insertRow(tweets_full, tweets_temp)
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
current <- twitter_api_call(api_url, query, api_params)
|
|
||||||
tweets_full <- fromJSON(correctJSON(current))
|
|
||||||
tweets_full <- tweets_full[keep]
|
|
||||||
}
|
|
||||||
status <- str_detect(tweets_full$created_at[nrow(tweets_full)], "2013$")
|
|
||||||
if (status) {
|
|
||||||
rm(tweets_temp)
|
|
||||||
for(r in 1:nrow(tweets_full)) {
|
|
||||||
status <- str_detect(tweets_full$created_at[r], "2013$")
|
|
||||||
if(is.na(status)) { status <- FALSE }
|
|
||||||
if(status) {
|
|
||||||
tweets_full <- tweets_full[-r,]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
break
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
max_id <- as.character(as.numeric(tweets_full$id_str[nrow(tweets_full)]) - 1)
|
|
||||||
loop <- loop + 1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
View(tweets_full)
|
|
||||||
View(tweets_temp)
|
|
||||||
api_url <- "https://api.twitter.com/1.1/statuses/user_timeline.json";
|
|
||||||
user <- "peteraltmaier"
|
|
||||||
max_count <- "20"
|
|
||||||
max_id <- "999999999999999999"
|
|
||||||
loop <- 1
|
|
||||||
keep <- c("created_at", "id_str", "text", "retweet_count")
|
|
||||||
rm(tweets_full, tweets_temp)
|
|
||||||
repeat {
|
|
||||||
query <- c(include_rts=1, exclude_replies="true", trim_user="true", include_entities="false",
|
|
||||||
screen_name=user,
|
|
||||||
count=max_count,
|
|
||||||
max_id=max_id);
|
|
||||||
if(exists("tweets_full")) {
|
|
||||||
current <- twitter_api_call(api_url, query, api_params)
|
|
||||||
tweets_temp <- fromJSON(correctJSON(current))
|
|
||||||
tweets_temp <- tweets_temp[keep]
|
|
||||||
tweets_full <- insertRow(tweets_full, tweets_temp)
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
current <- twitter_api_call(api_url, query, api_params)
|
|
||||||
tweets_full <- fromJSON(correctJSON(current))
|
|
||||||
tweets_full <- tweets_full[keep]
|
|
||||||
}
|
|
||||||
status <- str_detect(tweets_full$created_at[nrow(tweets_full)], "2013$")
|
|
||||||
if (status) {
|
|
||||||
rm(tweets_temp)
|
|
||||||
for(r in 1:nrow(tweets_full)) {
|
|
||||||
status <- str_detect(tweets_full$created_at[r], "2013$")
|
|
||||||
if(is.na(status)) { status <- FALSE }
|
|
||||||
if(status) {
|
|
||||||
tweets_full <- tweets_full[-r,]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
break
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
max_id <- as.character(as.numeric(tweets_full$id_str[nrow(tweets_full)]) - 1)
|
|
||||||
loop <- loop + 1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
View(tweets_full)
|
|
||||||
api_url <- "https://api.twitter.com/1.1/statuses/user_timeline.json";
|
|
||||||
user <- "peteraltmaier"
|
|
||||||
max_count <- "10"
|
|
||||||
max_id <- "999999999999999999"
|
|
||||||
loop <- 1
|
|
||||||
keep <- c("created_at", "id_str", "text", "retweet_count")
|
|
||||||
rm(tweets_full, tweets_temp)
|
|
||||||
repeat {
|
|
||||||
query <- c(include_rts=1, exclude_replies="false", trim_user="true", include_entities="false",
|
|
||||||
screen_name=user,
|
|
||||||
count=max_count,
|
|
||||||
max_id=max_id);
|
|
||||||
if(exists("tweets_full")) {
|
|
||||||
current <- twitter_api_call(api_url, query, api_params)
|
|
||||||
tweets_temp <- fromJSON(correctJSON(current))
|
|
||||||
tweets_temp <- tweets_temp[keep]
|
|
||||||
tweets_full <- insertRow(tweets_full, tweets_temp)
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
current <- twitter_api_call(api_url, query, api_params)
|
|
||||||
tweets_full <- fromJSON(correctJSON(current))
|
|
||||||
tweets_full <- tweets_full[keep]
|
|
||||||
}
|
|
||||||
status <- str_detect(tweets_full$created_at[nrow(tweets_full)], "2013$")
|
|
||||||
if (status) {
|
|
||||||
rm(tweets_temp)
|
|
||||||
for(r in 1:nrow(tweets_full)) {
|
|
||||||
status <- str_detect(tweets_full$created_at[r], "2013$")
|
|
||||||
if(is.na(status)) { status <- FALSE }
|
|
||||||
if(status) {
|
|
||||||
tweets_full <- tweets_full[-r,]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
break
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
max_id <- as.character(as.numeric(tweets_full$id_str[nrow(tweets_full)]) - 1)
|
|
||||||
loop <- loop + 1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
View(tweets_full)
|
|
||||||
api_params <- c(
|
|
||||||
"oauth_consumer_key" = "c9Ob2fWNSONMC0mA2JlNaeRke",
|
|
||||||
"oauth_nonce" = NA,
|
|
||||||
"oauth_signature_method" = "HMAC-SHA1",
|
|
||||||
"oauth_timestamp" = NA,
|
|
||||||
"oauth_token" = "1007025684-RFxCDFc4OPkt02bASmdci00TB4jgaPjfqxLRT58",
|
|
||||||
"oauth_version" = "1.0",
|
|
||||||
"consumer_secret" = "cZ3Il2hmbLgK0Lc57mj5kUvymjVdsmZKYwKOGHR3NhCpvWgEOI",
|
|
||||||
"oauth_token_secret" = "rvfv8MgexFKTqrPNSoGrdrZVNhV4fTJb2Bgz249nbvKNg"
|
|
||||||
)
|
|
||||||
api_url <- "https://api.twitter.com/1.1/statuses/user_timeline.json";
|
|
||||||
user <- "peteraltmaier"
|
|
||||||
max_count <- "10"
|
|
||||||
max_id <- "999999999999999999"
|
|
||||||
loop <- 1
|
|
||||||
keep <- c("created_at", "id_str", "text", "retweet_count")
|
|
||||||
rm(tweets_full, tweets_temp)
|
|
||||||
repeat {
|
|
||||||
query <- c(include_rts=1, exclude_replies="false", trim_user="true", include_entities="false",
|
|
||||||
screen_name=user,
|
|
||||||
count=max_count,
|
|
||||||
max_id=max_id);
|
|
||||||
if(exists("tweets_full")) {
|
|
||||||
current <- twitter_api_call(api_url, query, api_params)
|
|
||||||
tweets_temp <- fromJSON(correctJSON(current))
|
|
||||||
tweets_temp <- tweets_temp[keep]
|
|
||||||
tweets_full <- insertRow(tweets_full, tweets_temp)
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
current <- twitter_api_call(api_url, query, api_params)
|
|
||||||
tweets_full <- fromJSON(correctJSON(current))
|
|
||||||
tweets_full <- tweets_full[keep]
|
|
||||||
}
|
|
||||||
status <- str_detect(tweets_full$created_at[nrow(tweets_full)], "2013$")
|
|
||||||
if (status) {
|
|
||||||
rm(tweets_temp)
|
|
||||||
for(r in 1:nrow(tweets_full)) {
|
|
||||||
status <- str_detect(tweets_full$created_at[r], "2013$")
|
|
||||||
if(is.na(status)) { status <- FALSE }
|
|
||||||
if(status) {
|
|
||||||
tweets_full <- tweets_full[-r,]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
break
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
max_id <- as.character(as.numeric(tweets_full$id_str[nrow(tweets_full)]) - 1)
|
|
||||||
loop <- loop + 1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
current
|
|
||||||
correctJSON(current)
|
|
||||||
validate(current)
|
|
||||||
string <- current
|
|
||||||
str_replace_all(string, pattern = fixed("\n"), replacement = " ")
|
|
||||||
current
|
|
||||||
fromJSON(current)
|
|
||||||
str_replace_all(string, pattern = fixed("\n"), replacement = " ")
|
|
||||||
str_replace_all(string, pattern = fixed("ewifjweaoegj"), replacement = " ")
|
|
||||||
str_detect(string, pattern = fixed("ewifjweaoegj"))
|
|
||||||
str_detect(string, pattern = fixed("ewifjweaoegj"))
|
|
||||||
str_detect(string, pattern = fixed("wfwe"))
|
|
||||||
str_detect(string, pattern = fixed("peter"))
|
|
||||||
str_detect(string, pattern = "kefme")
|
|
||||||
str_detect(string, pattern = "\n")
|
|
||||||
str_detect(string, pattern = "\\n")
|
|
||||||
string <- str_replace_all(string, pattern = fixed("\n"), replacement = " ")
|
|
||||||
string <- "bla bla \nwoqdwqowd"
|
|
||||||
string
|
|
||||||
str_replace_all(string, pattern = fixed("\n"), replacement = " ")
|
|
||||||
string <- str_replace_all(string, pattern = "\n", replacement = " ")
|
|
||||||
string
|
|
||||||
correctJSON <- function(string) {
|
|
||||||
string <- str_replace_all(string, pattern = perl('\\\\(?![tn"])'), replacement = " ")
|
|
||||||
string <- str_replace_all(string, pattern = "\n", replacement = " ")
|
|
||||||
string <- str_replace_all(string, pattern = "\r", replacement = " ")
|
|
||||||
string <- str_replace_all(string, pattern = fixed("^"), replacement = " ")
|
|
||||||
return(string)
|
|
||||||
}
|
|
||||||
api_url <- "https://api.twitter.com/1.1/statuses/user_timeline.json";
|
|
||||||
user <- "peteraltmaier"
|
|
||||||
max_count <- "200"
|
|
||||||
max_id <- "999999999999999999"
|
|
||||||
loop <- 1
|
|
||||||
keep <- c("created_at", "id_str", "text", "retweet_count")
|
|
||||||
rm(tweets_full, tweets_temp)
|
|
||||||
repeat {
|
|
||||||
query <- c(include_rts=1, exclude_replies="true", trim_user="true", include_entities="false",
|
|
||||||
screen_name=user,
|
|
||||||
count=max_count,
|
|
||||||
max_id=max_id);
|
|
||||||
if(exists("tweets_full")) {
|
|
||||||
current <- twitter_api_call(api_url, query, api_params)
|
|
||||||
tweets_temp <- fromJSON(correctJSON(current))
|
|
||||||
tweets_temp <- tweets_temp[keep]
|
|
||||||
tweets_full <- insertRow(tweets_full, tweets_temp)
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
current <- twitter_api_call(api_url, query, api_params)
|
|
||||||
tweets_full <- fromJSON(correctJSON(current))
|
|
||||||
tweets_full <- tweets_full[keep]
|
|
||||||
}
|
|
||||||
status <- str_detect(tweets_full$created_at[nrow(tweets_full)], "2013$")
|
|
||||||
if (status) {
|
|
||||||
rm(tweets_temp)
|
|
||||||
for(r in 1:nrow(tweets_full)) {
|
|
||||||
status <- str_detect(tweets_full$created_at[r], "2013$")
|
|
||||||
if(is.na(status)) { status <- FALSE }
|
|
||||||
if(status) {
|
|
||||||
tweets_full <- tweets_full[-r,]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
break
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
max_id <- as.character(as.numeric(tweets_full$id_str[nrow(tweets_full)]) - 1)
|
|
||||||
loop <- loop + 1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
current
|
|
||||||
tweets_temp <- fromJSON(correctJSON(current))
|
|
||||||
correctJSON(current)
|
|
||||||
string <- current
|
|
||||||
str_replace_all(string, pattern = perl('\\\\(?![tn"])'), replacement = " ")
|
|
||||||
str_replace_all(string, pattern = "\n", replacement = " ")
|
|
||||||
str_replace_all(string, pattern = "\r", replacement = " ")
|
|
||||||
str_replace_all(string, pattern = fixed("^"), replacement = " ")
|
|
||||||
str_replace_all(string, pattern = "^", replacement = " ")
|
|
||||||
correctJSON <- function(string) {
|
|
||||||
string <- str_replace_all(string, pattern = perl('\\\\(?![tn"])'), replacement = " ")
|
|
||||||
string <- str_replace_all(string, pattern = "\n", replacement = " ")
|
|
||||||
string <- str_replace_all(string, pattern = "\r", replacement = " ")
|
|
||||||
string <- str_replace_all(string, pattern = "^", replacement = " ")
|
|
||||||
return(string)
|
|
||||||
}
|
|
||||||
api_url <- "https://api.twitter.com/1.1/statuses/user_timeline.json";
|
|
||||||
user <- "peteraltmaier"
|
|
||||||
max_count <- "200"
|
|
||||||
max_id <- "999999999999999999"
|
|
||||||
loop <- 1
|
|
||||||
keep <- c("created_at", "id_str", "text", "retweet_count")
|
|
||||||
rm(tweets_full, tweets_temp)
|
|
||||||
repeat {
|
|
||||||
query <- c(include_rts=1, exclude_replies="true", trim_user="true", include_entities="false",
|
|
||||||
screen_name=user,
|
|
||||||
count=max_count,
|
|
||||||
max_id=max_id);
|
|
||||||
if(exists("tweets_full")) {
|
|
||||||
current <- twitter_api_call(api_url, query, api_params)
|
|
||||||
tweets_temp <- fromJSON(correctJSON(current))
|
|
||||||
tweets_temp <- tweets_temp[keep]
|
|
||||||
tweets_full <- insertRow(tweets_full, tweets_temp)
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
current <- twitter_api_call(api_url, query, api_params)
|
|
||||||
tweets_full <- fromJSON(correctJSON(current))
|
|
||||||
tweets_full <- tweets_full[keep]
|
|
||||||
}
|
|
||||||
status <- str_detect(tweets_full$created_at[nrow(tweets_full)], "2013$")
|
|
||||||
if (status) {
|
|
||||||
rm(tweets_temp)
|
|
||||||
for(r in 1:nrow(tweets_full)) {
|
|
||||||
status <- str_detect(tweets_full$created_at[r], "2013$")
|
|
||||||
if(is.na(status)) { status <- FALSE }
|
|
||||||
if(status) {
|
|
||||||
tweets_full <- tweets_full[-r,]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
break
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
max_id <- as.character(as.numeric(tweets_full$id_str[nrow(tweets_full)]) - 1)
|
|
||||||
loop <- loop + 1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
current
|
|
||||||
str_replace_all(string, pattern = "^", replacement = " ")
|
|
||||||
current
|
|
||||||
correctJSON <- function(string) {
|
|
||||||
string <- str_replace_all(string, pattern = perl('\\\\(?![tn"])'), replacement = " ")
|
|
||||||
string <- str_replace_all(string, pattern = "\n", replacement = " ")
|
|
||||||
string <- str_replace_all(string, pattern = "\r", replacement = " ")
|
|
||||||
string <- str_replace_all(string, pattern = "^", replacement = " ")
|
|
||||||
return(string)
|
|
||||||
}
|
|
||||||
tweets_temp <- fromJSON(correctJSON(current))
|
|
||||||
fromJSON(current)
|
|
||||||
validate(correctJSON(current))
|
|
||||||
current
|
|
||||||
str_extract_all(string, pattern = "\n")
|
|
||||||
str_extract_all(string, pattern = "\xed")
|
|
||||||
str_extract_all(string, pattern = "\[:alnum:]{3}")
|
|
||||||
str_extract_all(string, pattern = "\[[:alnum:]]{3}")
|
|
||||||
str_extract_all(string, pattern = "\\[[:alnum:]]{3}")
|
|
||||||
str_extract_all(string, pattern = "\([[:alnum:]]{3})")
|
|
||||||
str_extract_all(string, pattern = "\\([[:alnum:]]{3})")
|
|
||||||
str_extract_all(string, pattern = "([[:alnum:]]{3})")
|
|
||||||
str_extract_all(string, pattern = "\x..")
|
|
||||||
str_extract_all(string, pattern = "\\x..")
|
|
||||||
str_extract_all(string, pattern = "\x..")
|
|
||||||
str_extract_all(string, pattern = "\xed")
|
|
||||||
str_extract_all(string, pattern = "\x")
|
|
||||||
str_extract_all(string, pattern = "\x+")
|
|
||||||
str_extract_all(string, pattern = "\x.+")
|
|
||||||
str_extract_all(string, pattern = "\x")
|
|
||||||
str_extract_all(string, pattern = "\xhh")
|
|
||||||
str_extract_all(string, pattern = "\xxx")
|
|
||||||
str_extract_all(string, pattern = "[[:xdigit:]]")
|
|
||||||
str_extract_all(string, pattern = "(\xhh)")
|
|
||||||
str_extract_all(string, pattern = "[\xhh]")
|
|
||||||
str_extract_all(string, pattern = "\xHH")
|
|
||||||
str_extract_all(string, pattern = "\xa0")
|
|
||||||
str_extract_all(string, pattern = "\xba?")
|
|
||||||
str_extract_all(string, pattern = "(\xba)+?")
|
|
||||||
str_extract_all(string, pattern = "(\xba)?")
|
|
||||||
str_extract_all(string, pattern = "(\xba)")
|
|
||||||
str_extract_all(string, pattern = "\xba")
|
|
||||||
str_extract_all(string, pattern = "\xba")
|
|
||||||
str_extract_all(string, pattern = "\xed")
|
|
||||||
str_extract_all(string, pattern = "(\xed)")
|
|
||||||
str_extract_all(string, pattern = "(\xed)?")
|
|
||||||
str_extract_all(string, pattern = "(\xed)+")
|
|
||||||
str_extract_all(string, pattern = "(\xed)+?")
|
|
||||||
str_extract_all(string, pattern = "\xed")
|
|
||||||
write(string, "temp.txt")
|
|
||||||
Encoding("temp.txt")
|
|
||||||
readLines("temp.txt")
|
|
||||||
validate(string)
|
|
||||||
string <- readLines("temp.txt")
|
|
||||||
validate(string)
|
|
||||||
string <- str_replace_all(string, pattern = perl('\\\\(?![tn"])'), replacement = " ")
|
|
||||||
string <- str_replace_all(string, pattern = "\n", replacement = " ")
|
|
||||||
string <- str_replace_all(string, pattern = "\r", replacement = " ")
|
|
||||||
string <- str_replace_all(string, pattern = "^", replacement = " ")
|
|
||||||
validate(string)
|
|
||||||
fromJSON("temp.txt")
|
|
||||||
current
|
|
||||||
fromJSON(current)
|
|
||||||
|
|||||||
BIN
Binary file not shown.
File diff suppressed because one or more lines are too long
+26
-12
@@ -4,10 +4,10 @@ require(RCurl)
|
|||||||
require(devtools)
|
require(devtools)
|
||||||
require(RTwitterAPI)
|
require(RTwitterAPI)
|
||||||
|
|
||||||
source("functions.R")
|
|
||||||
|
|
||||||
setwd("~/Dokumente/Uni/Aktuell/BA-Arbeit/uni-ba-issuecomp")
|
setwd("~/Dokumente/Uni/Aktuell/BA-Arbeit/uni-ba-issuecomp")
|
||||||
|
|
||||||
|
source("functions.R")
|
||||||
|
|
||||||
# Set curl handle for friendly scraping
|
# Set curl handle for friendly scraping
|
||||||
handle <- getCurlHandle(httpheader = list(from = "max.mehl@uni.kn",
|
handle <- getCurlHandle(httpheader = list(from = "max.mehl@uni.kn",
|
||||||
'user-agent' = str_c(R.version$version.string)
|
'user-agent' = str_c(R.version$version.string)
|
||||||
@@ -47,46 +47,60 @@ api_params <- c(
|
|||||||
)
|
)
|
||||||
|
|
||||||
api_url <- "https://api.twitter.com/1.1/statuses/user_timeline.json";
|
api_url <- "https://api.twitter.com/1.1/statuses/user_timeline.json";
|
||||||
# api_url <- "https://api.twitter.com/1.1/statuses/show.json";
|
user <- "GregorGysi"
|
||||||
user <- "peteraltmaier"
|
|
||||||
max_count <- "200"
|
max_count <- "200"
|
||||||
max_id <- "999999999999999999"
|
max_id <- "999999999999999999"
|
||||||
loop <- 1
|
loop <- 1
|
||||||
keep <- c("created_at", "id_str", "text", "retweet_count")
|
keep <- c("created_at", "id_str", "text", "retweet_count")
|
||||||
rm(tweets_full, tweets_temp)
|
rm(tweets_full)
|
||||||
repeat {
|
repeat {
|
||||||
|
# Define specific search query
|
||||||
query <- c(include_rts=1, exclude_replies="true", trim_user="true", include_entities="false",
|
query <- c(include_rts=1, exclude_replies="true", trim_user="true", include_entities="false",
|
||||||
screen_name=user,
|
screen_name=user,
|
||||||
count=max_count,
|
count=max_count,
|
||||||
max_id=max_id);
|
max_id=max_id);
|
||||||
# query <- c(trim_user="true", include_entities="false",
|
|
||||||
# id="431858659656990721");
|
# If a tweets_full DB already exists (after the first loop this should be the case)
|
||||||
if(exists("tweets_full")) {
|
if(exists("tweets_full")) {
|
||||||
current <- twitter_api_call(api_url, query, api_params)
|
current <- twitter_api_call(api_url, query, api_params)
|
||||||
tweets_temp <- fromJSON(current)
|
tweets_temp <- fromJSON(correctJSON(current))
|
||||||
tweets_temp <- tweets_temp[keep]
|
tweets_temp <- tweets_temp[keep]
|
||||||
tweets_full <- insertRow(tweets_full, tweets_temp)
|
tweets_full <- insertRow(tweets_full, tweets_temp)
|
||||||
|
rm(tweets_temp)
|
||||||
}
|
}
|
||||||
|
# First loop
|
||||||
else {
|
else {
|
||||||
current <- twitter_api_call(api_url, query, api_params)
|
current <- twitter_api_call(api_url, query, api_params)
|
||||||
tweets_full <- fromJSON(correctJSON(current))
|
tweets_full <- fromJSON(correctJSON(current))
|
||||||
tweets_full <- tweets_full[keep]
|
tweets_full <- tweets_full[keep]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Is the last tweet in tweets_full from 2013?
|
||||||
status <- str_detect(tweets_full$created_at[nrow(tweets_full)], "2013$")
|
status <- str_detect(tweets_full$created_at[nrow(tweets_full)], "2013$")
|
||||||
|
# Last loop is reached. Now clear the data frame
|
||||||
if (status) {
|
if (status) {
|
||||||
rm(tweets_temp)
|
|
||||||
|
# Delete all tweets from 2013
|
||||||
|
old <- 0
|
||||||
for(r in 1:nrow(tweets_full)) {
|
for(r in 1:nrow(tweets_full)) {
|
||||||
status <- str_detect(tweets_full$created_at[r], "2013$")
|
status <- str_detect(tweets_full$created_at[r], "2013$")
|
||||||
if(is.na(status)) { status <- FALSE }
|
if(is.na(status)) { status <- FALSE }
|
||||||
if(status) {
|
if(status) {
|
||||||
tweets_full <- tweets_full[-r,]
|
old <- old + 1
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
break
|
if(old > 0) {
|
||||||
|
old <- old - 1
|
||||||
|
tweets_full <- head(tweets_full, -old)
|
||||||
}
|
}
|
||||||
|
rm(old)
|
||||||
|
|
||||||
|
break # End loop because 2013 is reached
|
||||||
|
}
|
||||||
|
|
||||||
|
# The last tweet is still from 2014, so we need another loop
|
||||||
else {
|
else {
|
||||||
max_id <- as.character(as.numeric(tweets_full$id_str[nrow(tweets_full)]) - 1)
|
max_id <- as.character(as.numeric(tweets_full$id_str[nrow(tweets_full)]) - 2)
|
||||||
loop <- loop + 1
|
loop <- loop + 1
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+4
-3
@@ -2,10 +2,11 @@ require(stringr)
|
|||||||
|
|
||||||
# Replace characters messing up JSON validation (\,\n,^)
|
# Replace characters messing up JSON validation (\,\n,^)
|
||||||
correctJSON <- function(string) {
|
correctJSON <- function(string) {
|
||||||
|
# string <- str_replace_all(string, pattern = "\n", replacement = " ")
|
||||||
|
# string <- str_replace_all(string, pattern = "\r", replacement = " ")
|
||||||
|
# string <- str_replace_all(string, pattern = "\\^", replacement = " ")
|
||||||
|
string <- str_replace_all(string, pattern = "[^[:print:]]", replacement = " ")
|
||||||
string <- str_replace_all(string, pattern = perl('\\\\(?![tn"])'), replacement = " ")
|
string <- str_replace_all(string, pattern = perl('\\\\(?![tn"])'), replacement = " ")
|
||||||
string <- str_replace_all(string, pattern = "\n", replacement = " ")
|
|
||||||
string <- str_replace_all(string, pattern = "\r", replacement = " ")
|
|
||||||
string <- str_replace_all(string, pattern = "\\^", replacement = " ")
|
|
||||||
return(string)
|
return(string)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1 @@
|
|||||||
|
[{"created_at":"Fri Feb 07 18:35:02 +0000 2014","id":431858659656990721,"id_str":"431858659656990721","text":"RT @FHubersr: @peteraltmaier //die Schwarz-Grünen werden zeigen, daß sich Ökologie und Ökonomie vertragen und kein Widerspruch sind.Mutig v…","source":"<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a>","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":378693834,"id_str":"378693834"},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweeted_status":{"created_at":"Fri Feb 07 18:32:30 +0000 2014","id":431858022366064640,"id_str":"431858022366064640","text":"@peteraltmaier //die Schwarz-Grünen werden zeigen, daß sich Ökologie und Ökonomie vertragen und kein Widerspruch sind.Mutig von bd. Seiten������������","source":"<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a>","truncated":false,"in_reply_to_status_id":431845492579123201,"in_reply_to_status_id_str":"431845492579123201","in_reply_to_user_id":378693834,"in_reply_to_user_id_str":"378693834","in_reply_to_screen_name":"peteraltmaier","user":{"id":2172292811,"id_str":"2172292811"},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":3,"favorite_count":4,"favorited":false,"retweeted":false,"lang":"de"},"retweet_count":3,"favorite_count":0,"favorited":false,"retweeted":false,"lang":"de"}]
|
||||||
Reference in New Issue
Block a user