user <- as.character(acc_df$screenname[a])
name <- as.character(acc_df$name[a])
max_id <- "999999999999999999"
loop <- 1
repeat {
# Define specific search query
query <- c(include_rts=1, exclude_replies="true", trim_user="true", include_entities="false",
screen_name=user,
count=max_count,
max_id=max_id);
# At first, work with an temporary tweet-DB
current <- twitter_api_call(api_url, query, api_params)
tweets_temp <- fromJSON(correctJSON(current))
## STAT ERROR HANDLING ##
# Check for empty API returns
status <- length(tweets_temp)
if(status == 0) {
cat("[WARNING] Empty API result. Trying again.\n")
rm(tweets_temp)
Sys.sleep(3)
next
}
# Check if API output contains error fields
status <- "error" %in% names(tweets_temp)
if(status) {
cat("[WARNING] Error in API request:", tweets_temp$error[1],"\n")
rm(tweets_temp)
break
}
# Check for other errors, mostly rate limits
status <- "errors" %in% names(tweets_temp)
if(status) {
cat("[WARNING] Error in API request:", tweets_temp$errors[1,1],"\n")
# Rate limit exceeded?
status <- tweets_temp$errors[1,2]
if(status == 88) {
rate_api_url <- "https://api.twitter.com/1.1/application/rate_limit_status.json"
rate_query <-c (resources="statuses")
resettime <- fromJSON(twitter_api_call(rate_api_url, rate_query, api_params))
resettime <- resettime$resources$statuses$`/statuses/user_timeline`$reset
curtime <- as.numeric(as.POSIXct(Sys.time()))
wait <- round(resettime - curtime + 10)
cat("[INFO] Rate limit is exceeded. Now waiting",wait,"seconds.\n")
Sys.sleep(wait)
}
rm(tweets_temp)
Sys.sleep(3)
next
}
## END ERROR HANDLING ##
# Delete unnecessary columns and add username and real name to dataframe
tweets_temp <- tweets_temp[keep]
tweets_temp <- cbind(user=user, name=name, tweets_temp)
# Now sleep 3 second to dodge 300queries/15min limit
cat("[",a,"/",nrow(acc_df),"] ", sep = "")
cat("User: ",user," in loop: ",loop,". \n", sep = "")
Sys.sleep(2)
if(tweets_full$id_str[nrow(tweets_full)] == tweets_temp$id_str[nrow(tweets_temp)] && nrow(tweets_full) > 0) {
cat("[INFO] Last tweet of temp is last tweet of full. Abort loop and begin with next user.\n")
break
}
# Is the last tweet in tweets_temp from 2013?
status <- str_detect(tweets_temp$created_at[nrow(tweets_temp)], "2014$")
# Last loop is reached. Now clear the data frame
if (!status) { # Starting when tweet not from 2014
# Delete all tweets other than from 2014
old <- 0
for(r in 1:nrow(tweets_temp)) {
status <- str_detect(tweets_temp$created_at[r], "2014$")
if(is.na(status)) {
#status <- FALSE
cat("[INFO] NA-Status in Tweet", r)
}
if(!status) { # Starting when tweet not from 2014
old <- old + 1
}
}
if(old > 0) {
old <- old - 1
# If even the first entry isn't from 2014, we have to set "old" manually because of a bug
status <- str_detect(tweets_temp$created_at[1], "2014$")
if(!status) {
old <- nrow(tweets_temp)
cat("[INFO] Timeline enhält keinen einzigen aus 2014\n")
}
# delete all lines which are older than 2014
tweets_temp <- head(tweets_temp, -old)
}
rm(old)
tweets_full <- insertRow(tweets_full, tweets_temp)
rm(tweets_temp)
break  # End loop because 2013 is reached
}
# The last tweet is still from 2014, so we need another loop
else {
# Setting max_id to gather next 200 tweets
max_id <- tweets_temp$id_str[nrow(tweets_temp)]
loop <- loop + 1  # just for stats
tweets_full <- insertRow(tweets_full, tweets_temp)
rm(tweets_temp)
}
} # /repeat
stat_tweet <- nrow(tweets_full)
cat("User:",user,"finished after",loop,"loops. Total Tweets now:",nrow(tweets_full),"\n")
write.csv(tweets_full, "tweets_full.csv")
# Every tweet from 2014 from user[r] is downloaded. Now next user in for-loop
}
status
content
current
query
tweets_full$text[59203]
tweets_full$text[59202]
tweets_full$text[59203]
tweets_full$user[59203]
tweets_full$user[59202]
for(a in 157:nrow(acc_df)) {
user <- as.character(acc_df$screenname[a])
name <- as.character(acc_df$name[a])
max_id <- "999999999999999999"
loop <- 1
error <- 0
repeat {
# Define specific search query
query <- c(include_rts=1, exclude_replies="true", trim_user="true", include_entities="false",
screen_name=user,
count=max_count,
max_id=max_id);
# At first, work with an temporary tweet-DB
current <- twitter_api_call(api_url, query, api_params)
tweets_temp <- fromJSON(correctJSON(current))
## STAT ERROR HANDLING ##
# Check for empty API returns
status <- length(tweets_temp)
if(status == 0) {
if(error > 3) {
cat("[WARNING] 3x empty API result. Aborting now.\n")
}
cat("[WARNING] Empty API result. Trying again.\n")
rm(tweets_temp)
error <- error + 1
Sys.sleep(3)
next
}
# Check if API output contains error fields
status <- "error" %in% names(tweets_temp)
if(status) {
cat("[WARNING] Error in API request:", tweets_temp$error[1],"\n")
rm(tweets_temp)
break
}
# Check for other errors, mostly rate limits
status <- "errors" %in% names(tweets_temp)
if(status) {
cat("[WARNING] Error in API request:", tweets_temp$errors[1,1],"\n")
# Rate limit exceeded?
status <- tweets_temp$errors[1,2]
if(status == 88) {
rate_api_url <- "https://api.twitter.com/1.1/application/rate_limit_status.json"
rate_query <-c (resources="statuses")
resettime <- fromJSON(twitter_api_call(rate_api_url, rate_query, api_params))
resettime <- resettime$resources$statuses$`/statuses/user_timeline`$reset
curtime <- as.numeric(as.POSIXct(Sys.time()))
wait <- round(resettime - curtime + 10)
cat("[INFO] Rate limit is exceeded. Now waiting",wait,"seconds.\n")
Sys.sleep(wait)
}
rm(tweets_temp)
Sys.sleep(3)
next
}
## END ERROR HANDLING ##
# Delete unnecessary columns and add username and real name to dataframe
tweets_temp <- tweets_temp[keep]
tweets_temp <- cbind(user=user, name=name, tweets_temp)
# Now sleep 3 second to dodge 300queries/15min limit
cat("[",a,"/",nrow(acc_df),"] ", sep = "")
cat("User: ",user," in loop: ",loop,". \n", sep = "")
Sys.sleep(2)
if(tweets_full$id_str[nrow(tweets_full)] == tweets_temp$id_str[nrow(tweets_temp)] && nrow(tweets_full) > 0) {
cat("[INFO] Last tweet of temp is last tweet of full. Abort loop and begin with next user.\n")
break
}
# Is the last tweet in tweets_temp from 2013?
status <- str_detect(tweets_temp$created_at[nrow(tweets_temp)], "2014$")
# Last loop is reached. Now clear the data frame
if (!status) { # Starting when tweet not from 2014
# Delete all tweets other than from 2014
old <- 0
for(r in 1:nrow(tweets_temp)) {
status <- str_detect(tweets_temp$created_at[r], "2014$")
if(is.na(status)) {
#status <- FALSE
cat("[INFO] NA-Status in Tweet", r)
}
if(!status) { # Starting when tweet not from 2014
old <- old + 1
}
}
if(old > 0) {
old <- old - 1
# If even the first entry isn't from 2014, we have to set "old" manually because of a bug
status <- str_detect(tweets_temp$created_at[1], "2014$")
if(!status) {
old <- nrow(tweets_temp)
cat("[INFO] Timeline enhält keinen einzigen aus 2014\n")
}
# delete all lines which are older than 2014
tweets_temp <- head(tweets_temp, -old)
}
rm(old)
tweets_full <- insertRow(tweets_full, tweets_temp)
rm(tweets_temp)
break  # End loop because 2013 is reached
}
# The last tweet is still from 2014, so we need another loop
else {
# Setting max_id to gather next 200 tweets
max_id <- tweets_temp$id_str[nrow(tweets_temp)]
loop <- loop + 1  # just for stats
tweets_full <- insertRow(tweets_full, tweets_temp)
rm(tweets_temp)
}
} # /repeat
stat_tweet <- nrow(tweets_full)
cat("User:",user,"finished after",loop,"loops. Total Tweets now:",nrow(tweets_full),"\n")
write.csv(tweets_full, "tweets_full.csv")
# Every tweet from 2014 from user[r] is downloaded. Now next user in for-loop
}
for(a in 157:nrow(acc_df)) {
user <- as.character(acc_df$screenname[a])
name <- as.character(acc_df$name[a])
max_id <- "999999999999999999"
loop <- 1
error <- 0
repeat {
# Define specific search query
query <- c(include_rts=1, exclude_replies="true", trim_user="true", include_entities="false",
screen_name=user,
count=max_count,
max_id=max_id);
# At first, work with an temporary tweet-DB
current <- twitter_api_call(api_url, query, api_params)
tweets_temp <- fromJSON(correctJSON(current))
## STAT ERROR HANDLING ##
# Check for empty API returns
status <- length(tweets_temp)
if(status == 0) {
if(error > 2) {
cat("[WARNING] 3x empty API result. Aborting now.\n")
break
}
cat("[WARNING] Empty API result. Trying again.\n")
rm(tweets_temp)
error <- error + 1
Sys.sleep(3)
next
}
# Check if API output contains error fields
status <- "error" %in% names(tweets_temp)
if(status) {
cat("[WARNING] Error in API request:", tweets_temp$error[1],"\n")
rm(tweets_temp)
break
}
# Check for other errors, mostly rate limits
status <- "errors" %in% names(tweets_temp)
if(status) {
cat("[WARNING] Error in API request:", tweets_temp$errors[1,1],"\n")
# Rate limit exceeded?
status <- tweets_temp$errors[1,2]
if(status == 88) {
rate_api_url <- "https://api.twitter.com/1.1/application/rate_limit_status.json"
rate_query <-c (resources="statuses")
resettime <- fromJSON(twitter_api_call(rate_api_url, rate_query, api_params))
resettime <- resettime$resources$statuses$`/statuses/user_timeline`$reset
curtime <- as.numeric(as.POSIXct(Sys.time()))
wait <- round(resettime - curtime + 10)
cat("[INFO] Rate limit is exceeded. Now waiting",wait,"seconds.\n")
Sys.sleep(wait)
}
rm(tweets_temp)
Sys.sleep(3)
next
}
## END ERROR HANDLING ##
# Delete unnecessary columns and add username and real name to dataframe
tweets_temp <- tweets_temp[keep]
tweets_temp <- cbind(user=user, name=name, tweets_temp)
# Now sleep 3 second to dodge 300queries/15min limit
cat("[",a,"/",nrow(acc_df),"] ", sep = "")
cat("User: ",user," in loop: ",loop,". \n", sep = "")
Sys.sleep(2)
if(tweets_full$id_str[nrow(tweets_full)] == tweets_temp$id_str[nrow(tweets_temp)] && nrow(tweets_full) > 0) {
cat("[INFO] Last tweet of temp is last tweet of full. Abort loop and begin with next user.\n")
break
}
# Is the last tweet in tweets_temp from 2013?
status <- str_detect(tweets_temp$created_at[nrow(tweets_temp)], "2014$")
# Last loop is reached. Now clear the data frame
if (!status) { # Starting when tweet not from 2014
# Delete all tweets other than from 2014
old <- 0
for(r in 1:nrow(tweets_temp)) {
status <- str_detect(tweets_temp$created_at[r], "2014$")
if(is.na(status)) {
#status <- FALSE
cat("[INFO] NA-Status in Tweet", r)
}
if(!status) { # Starting when tweet not from 2014
old <- old + 1
}
}
if(old > 0) {
old <- old - 1
# If even the first entry isn't from 2014, we have to set "old" manually because of a bug
status <- str_detect(tweets_temp$created_at[1], "2014$")
if(!status) {
old <- nrow(tweets_temp)
cat("[INFO] Timeline enhält keinen einzigen aus 2014\n")
}
# delete all lines which are older than 2014
tweets_temp <- head(tweets_temp, -old)
}
rm(old)
tweets_full <- insertRow(tweets_full, tweets_temp)
rm(tweets_temp)
break  # End loop because 2013 is reached
}
# The last tweet is still from 2014, so we need another loop
else {
# Setting max_id to gather next 200 tweets
max_id <- tweets_temp$id_str[nrow(tweets_temp)]
loop <- loop + 1  # just for stats
tweets_full <- insertRow(tweets_full, tweets_temp)
rm(tweets_temp)
}
} # /repeat
stat_tweet <- nrow(tweets_full)
cat("User:",user,"finished after",loop,"loops. Total Tweets now:",nrow(tweets_full),"\n")
write.csv(tweets_full, "tweets_full.csv")
# Every tweet from 2014 from user[r] is downloaded. Now next user in for-loop
}
current
correctJSON(current)
correctJSON <- function(string) {
#   string <- str_replace_all(string, pattern = "\n", replacement = " ")
#   string <- str_replace_all(string, pattern = "\r", replacement = " ")
#   string <- str_replace_all(string, pattern = "\\^", replacement = " ")
string <- str_replace_all(string, pattern = "[^[:print:]]", replacement = " ")
string <- str_replace_all(string, pattern = "&..;", replacement = " ")
string <- str_replace_all(string, pattern = perl('\\\\(?![tn"])'), replacement = " ")
return(string)
}
correctJSON(current)
a
tweets_temp <- fromJSON(correctJSON(current))
current
correctJSON <- function(string) {
#   string <- str_replace_all(string, pattern = "\n", replacement = " ")
#   string <- str_replace_all(string, pattern = "\r", replacement = " ")
#   string <- str_replace_all(string, pattern = "\\^", replacement = " ")
string <- str_replace_all(string, pattern = "[^[:print:]]", replacement = " ")
string <- str_replace_all(string, pattern = "&..;", replacement = " ")
string <- str_replace_all(string, pattern = perl('\\\\(?![tn"])'), replacement = " ")
string <- str_replace_all(string, pattern = "\\\\\\\\\\", replacement = "\\\\\\")
return(string)
}
tweets_temp <- fromJSON(correctJSON(current))
correctJSON <- function(string) {
#   string <- str_replace_all(string, pattern = "\n", replacement = " ")
#   string <- str_replace_all(string, pattern = "\r", replacement = " ")
#   string <- str_replace_all(string, pattern = "\\^", replacement = " ")
string <- str_replace_all(string, pattern = "[^[:print:]]", replacement = " ")
string <- str_replace_all(string, pattern = "&..;", replacement = " ")
string <- str_replace_all(string, pattern = perl('\\\\(?![tn"])'), replacement = " ")
string <- str_replace_all(string, pattern = perl('\\\\\\(?![tn"])'), replacement = " ")
return(string)
}
tweets_temp <- fromJSON(correctJSON(current))
correctJSON <- function(string) {
#   string <- str_replace_all(string, pattern = "\n", replacement = " ")
#   string <- str_replace_all(string, pattern = "\r", replacement = " ")
#   string <- str_replace_all(string, pattern = "\\^", replacement = " ")
string <- str_replace_all(string, pattern = "[^[:print:]]", replacement = " ")
string <- str_replace_all(string, pattern = "&..;", replacement = " ")
string <- str_replace_all(string, pattern = perl('\\\\(?![tn"])'), replacement = " ")
string <- str_replace_all(string, pattern = perl('\\\\\\\\(?![tn"])'), replacement = " ")
return(string)
}
tweets_temp <- fromJSON(correctJSON(current))
correctJSON <- function(string) {
#   string <- str_replace_all(string, pattern = "\n", replacement = " ")
#   string <- str_replace_all(string, pattern = "\r", replacement = " ")
#   string <- str_replace_all(string, pattern = "\\^", replacement = " ")
string <- str_replace_all(string, pattern = "[^[:print:]]", replacement = " ")
string <- str_replace_all(string, pattern = "&..;", replacement = " ")
string <- str_replace_all(string, pattern = perl('\\\\(?![tn"])'), replacement = " ")
string <- str_replace_all(string, pattern = perl('\\\\\\\\\\'), replacement = " ")
return(string)
}
tweets_temp <- fromJSON(correctJSON(current))
correctJSON <- function(string) {
#   string <- str_replace_all(string, pattern = "\n", replacement = " ")
#   string <- str_replace_all(string, pattern = "\r", replacement = " ")
#   string <- str_replace_all(string, pattern = "\\^", replacement = " ")
string <- str_replace_all(string, pattern = "[^[:print:]]", replacement = " ")
string <- str_replace_all(string, pattern = "&..;", replacement = " ")
string <- str_replace_all(string, pattern = perl('\\\\(?![tn"])'), replacement = " ")
string <- str_replace_all(string, pattern = "[\]{5}", replacement = " ")
return(string)
}
correctJSON <- function(string) {
#   string <- str_replace_all(string, pattern = "\n", replacement = " ")
#   string <- str_replace_all(string, pattern = "\r", replacement = " ")
#   string <- str_replace_all(string, pattern = "\\^", replacement = " ")
string <- str_replace_all(string, pattern = "[^[:print:]]", replacement = " ")
string <- str_replace_all(string, pattern = "&..;", replacement = " ")
string <- str_replace_all(string, pattern = perl('\\\\(?![tn"])'), replacement = " ")
string <- str_replace_all(string, pattern = "[\\]{5}", replacement = " ")
return(string)
}
tweets_temp <- fromJSON(correctJSON(current))
correctJSON <- function(string) {
#   string <- str_replace_all(string, pattern = "\n", replacement = " ")
#   string <- str_replace_all(string, pattern = "\r", replacement = " ")
#   string <- str_replace_all(string, pattern = "\\^", replacement = " ")
string <- str_replace_all(string, pattern = "[^[:print:]]", replacement = " ")
string <- str_replace_all(string, pattern = "&..;", replacement = " ")
string <- str_replace_all(string, pattern = perl('\\\\(?![tn"])'), replacement = " ")
string <- str_replace_all(string, pattern = "[\\]{6}", replacement = " ")
return(string)
}
tweets_temp <- fromJSON(correctJSON(current))
correctJSON <- function(string) {
#   string <- str_replace_all(string, pattern = "\n", replacement = " ")
#   string <- str_replace_all(string, pattern = "\r", replacement = " ")
#   string <- str_replace_all(string, pattern = "\\^", replacement = " ")
string <- str_replace_all(string, pattern = "[^[:print:]]", replacement = " ")
string <- str_replace_all(string, pattern = "&..;", replacement = " ")
string <- str_replace_all(string, pattern = perl('\\\\(?![tn"])'), replacement = " ")
string <- str_replace_all(string, pattern = "\\{6}", replacement = " ")
return(string)
}
tweets_temp <- fromJSON(correctJSON(current))
correctJSON <- function(string) {
#   string <- str_replace_all(string, pattern = "\n", replacement = " ")
#   string <- str_replace_all(string, pattern = "\r", replacement = " ")
#   string <- str_replace_all(string, pattern = "\\^", replacement = " ")
string <- str_replace_all(string, pattern = "[^[:print:]]", replacement = " ")
string <- str_replace_all(string, pattern = "&..;", replacement = " ")
string <- str_replace_all(string, pattern = perl('\\\\(?![tn"])'), replacement = " ")
string <- str_replace_all(string, pattern = "\\{5}", replacement = " ")
return(string)
}
insertRow <- function(existingDF, newrow, r) {
r <- as.numeric(nrow(existingDF)) + 1
existingDF <- rbind(existingDF,newrow)
existingDF <- existingDF[order(c(1:(nrow(existingDF)-1),r-0.5)),]
row.names(existingDF) <- 1:nrow(existingDF)
return(existingDF)
}
tweets_temp <- fromJSON(correctJSON(current))
str_detect(current, pattern = "\\")
str_detect(current, pattern = '\\')
str_detect(current, pattern = "\\\")
str_detect(current, pattern = "[\\]")
str_detect(current, pattern = "[\\\\]")
str_detect(current, pattern = "[\\\\\\]")
str_detect(current, pattern = "[\\\\\\\\]")
str_detect(current, pattern = "[\\\\\\\\\\\\]")
str_detect(current, pattern = "[\\\\\\\\\\\\\\]")
str_detect(current, pattern = "[\\]{1}")
str_detect(current, pattern = "[\\]{7}")
str_detect(current, pattern = "[\\]{2}")
str_detect(current, pattern = "[\\]{3}")
str_detect(current, pattern = "[\\]{4}")
str_detect(current, pattern = "[\\]{5}")
current
str_detect(current, pattern = "[\\]{5}")
str_detect(current, pattern = "[\]{5}")
str_detect(current, pattern = "[\\]{2}")
grep("[\\]", current)
grep("[\\]{2}", current)
grep("[\\]{3}", current)
grep("[\\]{4}", current)
grep("[\\]{5}", current)
grep("[\\]{1}", current)
str_detect(current, pattern = fixed("\\\\\"))
str_detect(current, pattern = fixed("\\\\\\\\\\\"))
str_detect(current, pattern = fixed("\\\\\\\\\\"))
str_detect(current, pattern = fixed("\\\\\\\\"))
str_detect(current, pattern = fixed("\\\\\\"))
str_detect(current, pattern = fixed("\\\\\"))
str_detect(current, pattern = fixed("\\\\"))
str_detect(current, pattern = fixed("\\"))
nchar("\\")
nchar("\\\\")
nchar("\\\\\\")
nchar("\\\\\\\\")
nchar("\\\\\\\\\\")
print("\\\\\\\\\\")
print("\\\\\")
print("\\\\\\")
print("\\\\")
print("\\\\\")
write(current, "current.txt")
current2 <- readLines("current.txt")
current
current2
rm(current2)
a