working
This commit is contained in:
@@ -1,7 +1,18 @@
|
||||
cat("[INFO] Last tweet of temp is last tweet of full. Abort loop and begin with next user.\n")
|
||||
#break
|
||||
}
|
||||
tweets_full$id_str[nrow(tweets_full)]
|
||||
tweets_temp$id_str[nrow(tweets_temp)]
|
||||
tweets_complete$user[20674]
|
||||
a
|
||||
current
|
||||
tweets_full <- data.frame(user=character(), name=character(), created_at=character(), id_str=character(), text=character(), retweet_count=character())
|
||||
for(a in 1:nrow(acc_df)) {
|
||||
user <- as.character(acc_df$screenname[a])
|
||||
name <- as.character(acc_df$name[a])
|
||||
max_id <- "999999999999999999"
|
||||
loop <- 1
|
||||
error <- 0
|
||||
repeat {
|
||||
# Define specific search query
|
||||
query <- c(include_rts=1, exclude_replies="true", trim_user="true", include_entities="false",
|
||||
@@ -10,41 +21,27 @@ count=max_count,
|
||||
max_id=max_id);
|
||||
# At first, work with an temporary tweet-DB
|
||||
current <- twitter_api_call(api_url, query, api_params)
|
||||
rm(tweets_temp)
|
||||
tweets_temp <- fromJSON(correctJSON(current))
|
||||
## STAT ERROR HANDLING ##
|
||||
# Check for empty API returns
|
||||
status <- length(tweets_temp)
|
||||
if(status == 0) {
|
||||
cat("[WARNING] Empty API result. Trying again.\n")
|
||||
rm(tweets_temp)
|
||||
Sys.sleep(3)
|
||||
next
|
||||
## START ERROR HANDLING ##
|
||||
# Empty API output
|
||||
status <- errorEmptyAPI(tweets_temp)
|
||||
if(status == 1) { Sys.sleep(3);error <- error + 1;next}
|
||||
if(status == 2) {break}
|
||||
# Contains "error" column
|
||||
status <- errorEmptyAPI(tweets_temp)
|
||||
if(status == 1) { Sys.sleep(3);error <- error + 1;next}
|
||||
if(status == 2) {break}
|
||||
# Check if error code exists
|
||||
code <- errorCheckCode(tweets_temp) # 0 if no error
|
||||
if(code == 34) { # page does not exist
|
||||
status <- errorCode34
|
||||
if(status == 1) { Sys.sleep(3);error <- error + 1;next}
|
||||
if(status == 2) {break}
|
||||
}
|
||||
# Check if API output contains error fields
|
||||
status <- "error" %in% names(tweets_temp)
|
||||
if(status) {
|
||||
cat("[WARNING] Error in API request:", tweets_temp$error[1],"\n")
|
||||
rm(tweets_temp)
|
||||
break
|
||||
}
|
||||
# Check for other errors, mostly rate limits
|
||||
status <- "errors" %in% names(tweets_temp)
|
||||
if(status) {
|
||||
cat("[WARNING] Error in API request:", tweets_temp$errors[1,1],"\n")
|
||||
# Rate limit exceeded?
|
||||
status <- tweets_temp$errors[1,2]
|
||||
if(status == 88) {
|
||||
rate_api_url <- "https://api.twitter.com/1.1/application/rate_limit_status.json"
|
||||
rate_query <-c (resources="statuses")
|
||||
resettime <- fromJSON(twitter_api_call(rate_api_url, rate_query, api_params))
|
||||
resettime <- resettime$resources$statuses$`/statuses/user_timeline`$reset
|
||||
curtime <- as.numeric(as.POSIXct(Sys.time()))
|
||||
wait <- round(resettime - curtime + 10)
|
||||
cat("[INFO] Rate limit is exceeded. Now waiting",wait,"seconds.\n")
|
||||
if(code == 88) { # rate limit exceeded
|
||||
wait <- errorCode88()
|
||||
Sys.sleep(wait)
|
||||
}
|
||||
rm(tweets_temp)
|
||||
Sys.sleep(3)
|
||||
next
|
||||
}
|
||||
## END ERROR HANDLING ##
|
||||
@@ -57,11 +54,11 @@ cat("User: ",user," in loop: ",loop,". \n", sep = "")
|
||||
Sys.sleep(2)
|
||||
if(tweets_full$id_str[nrow(tweets_full)] == tweets_temp$id_str[nrow(tweets_temp)] && nrow(tweets_full) > 0) {
|
||||
cat("[INFO] Last tweet of temp is last tweet of full. Abort loop and begin with next user.\n")
|
||||
break
|
||||
#break
|
||||
}
|
||||
## Last loop is reached. Now clear the data frame
|
||||
# Is the last tweet in tweets_temp from 2013?
|
||||
status <- str_detect(tweets_temp$created_at[nrow(tweets_temp)], "2014$")
|
||||
# Last loop is reached. Now clear the data frame
|
||||
if (!status) { # Starting when tweet not from 2014
|
||||
# Delete all tweets other than from 2014
|
||||
old <- 0
|
||||
@@ -88,7 +85,7 @@ tweets_temp <- head(tweets_temp, -old)
|
||||
}
|
||||
rm(old)
|
||||
tweets_full <- insertRow(tweets_full, tweets_temp)
|
||||
rm(tweets_temp)
|
||||
#rm(tweets_temp)
|
||||
break # End loop because 2013 is reached
|
||||
}
|
||||
# The last tweet is still from 2014, so we need another loop
|
||||
@@ -97,24 +94,224 @@ else {
|
||||
max_id <- tweets_temp$id_str[nrow(tweets_temp)]
|
||||
loop <- loop + 1 # just for stats
|
||||
tweets_full <- insertRow(tweets_full, tweets_temp)
|
||||
rm(tweets_temp)
|
||||
#rm(tweets_temp)
|
||||
}
|
||||
} # /repeat
|
||||
stat_tweet <- nrow(tweets_full)
|
||||
cat("User:",user,"finished after",loop,"loops. Total Tweets now:",nrow(tweets_full),"\n")
|
||||
write.csv(tweets_full, "tweets_full.csv")
|
||||
tweets_complete <- insertRow(tweets_complete, tweets_full)
|
||||
tweets_full <- head(tweets_full, -nrow(tweets_full)) # Empty tweets_full
|
||||
cat("User:",user,"finished after",loop,"loops. Total Tweets now:",nrow(tweets_complete),"\n")
|
||||
write.csv(tweets_complete, "tweets_complete.csv")
|
||||
# Every tweet from 2014 from user[r] is downloaded. Now next user in for-loop
|
||||
}
|
||||
tweets_full <- data.frame(user=character(), name=character(), created_at=character(), id_str=character(), text=character(), retweet_count=character())
|
||||
for(a in 66:nrow(acc_df)) {
|
||||
user <- as.character(acc_df$screenname[a])
|
||||
name <- as.character(acc_df$name[a])
|
||||
max_id <- "999999999999999999"
|
||||
loop <- 1
|
||||
error <- 0
|
||||
repeat {
|
||||
# Define specific search query
|
||||
query <- c(include_rts=1, exclude_replies="true", trim_user="true", include_entities="false",
|
||||
screen_name=user,
|
||||
count=max_count,
|
||||
max_id=max_id);
|
||||
# At first, work with an temporary tweet-DB
|
||||
current <- twitter_api_call(api_url, query, api_params)
|
||||
rm(tweets_temp)
|
||||
tweets_temp <- fromJSON(correctJSON(current))
|
||||
## START ERROR HANDLING ##
|
||||
# Empty API output
|
||||
status <- errorEmptyAPI(tweets_temp)
|
||||
if(status == 1) { Sys.sleep(3);error <- error + 1;next}
|
||||
if(status == 2) {break}
|
||||
# Contains "error" column
|
||||
status <- errorEmptyAPI(tweets_temp)
|
||||
if(status == 1) { Sys.sleep(3);error <- error + 1;next}
|
||||
if(status == 2) {break}
|
||||
# Check if error code exists
|
||||
code <- errorCheckCode(tweets_temp) # 0 if no error
|
||||
if(code == 34) { # page does not exist
|
||||
status <- errorCode34
|
||||
if(status == 1) { Sys.sleep(3);error <- error + 1;next}
|
||||
if(status == 2) {break}
|
||||
}
|
||||
if(code == 88) { # rate limit exceeded
|
||||
wait <- errorCode88()
|
||||
Sys.sleep(wait)
|
||||
next
|
||||
}
|
||||
## END ERROR HANDLING ##
|
||||
# Delete unnecessary columns and add username and real name to dataframe
|
||||
tweets_temp <- tweets_temp[keep]
|
||||
tweets_temp <- cbind(user=user, name=name, tweets_temp)
|
||||
# Now sleep 3 second to dodge 300queries/15min limit
|
||||
cat("[",a,"/",nrow(acc_df),"] ", sep = "")
|
||||
cat("User: ",user," in loop: ",loop,". \n", sep = "")
|
||||
Sys.sleep(2)
|
||||
if(tweets_full$id_str[nrow(tweets_full)] == tweets_temp$id_str[nrow(tweets_temp)] && nrow(tweets_full) > 0) {
|
||||
cat("[INFO] Last tweet of temp is last tweet of full. Abort loop and begin with next user.\n")
|
||||
#break
|
||||
}
|
||||
## Last loop is reached. Now clear the data frame
|
||||
# Is the last tweet in tweets_temp from 2013?
|
||||
status <- str_detect(tweets_temp$created_at[nrow(tweets_temp)], "2014$")
|
||||
if (!status) { # Starting when tweet not from 2014
|
||||
# Delete all tweets other than from 2014
|
||||
old <- 0
|
||||
for(r in 1:nrow(tweets_temp)) {
|
||||
status <- str_detect(tweets_temp$created_at[r], "2014$")
|
||||
if(is.na(status)) {
|
||||
#status <- FALSE
|
||||
cat("[INFO] NA-Status in Tweet", r)
|
||||
}
|
||||
if(!status) { # Starting when tweet not from 2014
|
||||
old <- old + 1
|
||||
}
|
||||
}
|
||||
if(old > 0) {
|
||||
old <- old - 1
|
||||
# If even the first entry isn't from 2014, we have to set "old" manually because of a bug
|
||||
status <- str_detect(tweets_temp$created_at[1], "2014$")
|
||||
if(!status) {
|
||||
old <- nrow(tweets_temp)
|
||||
cat("[INFO] Timeline enhält keinen einzigen aus 2014\n")
|
||||
}
|
||||
# delete all lines which are older than 2014
|
||||
tweets_temp <- head(tweets_temp, -old)
|
||||
}
|
||||
rm(old)
|
||||
tweets_full <- insertRow(tweets_full, tweets_temp)
|
||||
#rm(tweets_temp)
|
||||
break # End loop because 2013 is reached
|
||||
}
|
||||
# The last tweet is still from 2014, so we need another loop
|
||||
else {
|
||||
# Setting max_id to gather next 200 tweets
|
||||
max_id <- tweets_temp$id_str[nrow(tweets_temp)]
|
||||
loop <- loop + 1 # just for stats
|
||||
tweets_full <- insertRow(tweets_full, tweets_temp)
|
||||
#rm(tweets_temp)
|
||||
}
|
||||
} # /repeat
|
||||
tweets_complete <- insertRow(tweets_complete, tweets_full)
|
||||
tweets_full <- head(tweets_full, -nrow(tweets_full)) # Empty tweets_full
|
||||
cat("User:",user,"finished after",loop,"loops. Total Tweets now:",nrow(tweets_complete),"\n")
|
||||
write.csv(tweets_complete, "tweets_complete.csv")
|
||||
# Every tweet from 2014 from user[r] is downloaded. Now next user in for-loop
|
||||
}
|
||||
a
|
||||
tweets_complete$user[22982]
|
||||
tweets_full <- data.frame(user=character(), name=character(), created_at=character(), id_str=character(), text=character(), retweet_count=character())
|
||||
for(a in 68:nrow(acc_df)) {
|
||||
user <- as.character(acc_df$screenname[a])
|
||||
name <- as.character(acc_df$name[a])
|
||||
max_id <- "999999999999999999"
|
||||
loop <- 1
|
||||
error <- 0
|
||||
repeat {
|
||||
# Define specific search query
|
||||
query <- c(include_rts=1, exclude_replies="true", trim_user="true", include_entities="false",
|
||||
screen_name=user,
|
||||
count=max_count,
|
||||
max_id=max_id);
|
||||
# At first, work with an temporary tweet-DB
|
||||
current <- twitter_api_call(api_url, query, api_params)
|
||||
rm(tweets_temp)
|
||||
tweets_temp <- fromJSON(correctJSON(current))
|
||||
## START ERROR HANDLING ##
|
||||
# Empty API output
|
||||
status <- errorEmptyAPI(tweets_temp)
|
||||
if(status == 1) { Sys.sleep(3);error <- error + 1;next}
|
||||
if(status == 2) {break}
|
||||
# Contains "error" column
|
||||
status <- errorEmptyAPI(tweets_temp)
|
||||
if(status == 1) { Sys.sleep(3);error <- error + 1;next}
|
||||
if(status == 2) {break}
|
||||
# Check if error code exists
|
||||
code <- errorCheckCode(tweets_temp) # 0 if no error
|
||||
if(code == 34) { # page does not exist
|
||||
status <- errorCode34
|
||||
if(status == 1) { Sys.sleep(3);error <- error + 1;next}
|
||||
if(status == 2) {break}
|
||||
}
|
||||
if(code == 88) { # rate limit exceeded
|
||||
wait <- errorCode88()
|
||||
Sys.sleep(wait)
|
||||
next
|
||||
}
|
||||
## END ERROR HANDLING ##
|
||||
# Delete unnecessary columns and add username and real name to dataframe
|
||||
tweets_temp <- tweets_temp[keep]
|
||||
tweets_temp <- cbind(user=user, name=name, tweets_temp)
|
||||
# Now sleep 3 second to dodge 300queries/15min limit
|
||||
cat("[",a,"/",nrow(acc_df),"] ", sep = "")
|
||||
cat("User: ",user," in loop: ",loop,". \n", sep = "")
|
||||
Sys.sleep(2)
|
||||
if(tweets_full$id_str[nrow(tweets_full)] == tweets_temp$id_str[nrow(tweets_temp)] && nrow(tweets_full) > 0) {
|
||||
cat("[INFO] Last tweet of temp is last tweet of full. Abort loop and begin with next user.\n")
|
||||
break
|
||||
}
|
||||
## Last loop is reached. Now clear the data frame
|
||||
# Is the last tweet in tweets_temp from 2013?
|
||||
status <- str_detect(tweets_temp$created_at[nrow(tweets_temp)], "2014$")
|
||||
if (!status) { # Starting when tweet not from 2014
|
||||
# Delete all tweets other than from 2014
|
||||
old <- 0
|
||||
for(r in 1:nrow(tweets_temp)) {
|
||||
status <- str_detect(tweets_temp$created_at[r], "2014$")
|
||||
if(is.na(status)) {
|
||||
#status <- FALSE
|
||||
cat("[INFO] NA-Status in Tweet", r)
|
||||
}
|
||||
if(!status) { # Starting when tweet not from 2014
|
||||
old <- old + 1
|
||||
}
|
||||
}
|
||||
if(old > 0) {
|
||||
old <- old - 1
|
||||
# If even the first entry isn't from 2014, we have to set "old" manually because of a bug
|
||||
status <- str_detect(tweets_temp$created_at[1], "2014$")
|
||||
if(!status) {
|
||||
old <- nrow(tweets_temp)
|
||||
cat("[INFO] Timeline enhält keinen einzigen aus 2014\n")
|
||||
}
|
||||
# delete all lines which are older than 2014
|
||||
tweets_temp <- head(tweets_temp, -old)
|
||||
}
|
||||
rm(old)
|
||||
tweets_full <- insertRow(tweets_full, tweets_temp)
|
||||
#rm(tweets_temp)
|
||||
break # End loop because 2013 is reached
|
||||
}
|
||||
# The last tweet is still from 2014, so we need another loop
|
||||
else {
|
||||
# Setting max_id to gather next 200 tweets
|
||||
max_id <- tweets_temp$id_str[nrow(tweets_temp)]
|
||||
loop <- loop + 1 # just for stats
|
||||
tweets_full <- insertRow(tweets_full, tweets_temp)
|
||||
#rm(tweets_temp)
|
||||
}
|
||||
} # /repeat
|
||||
tweets_complete <- insertRow(tweets_complete, tweets_full)
|
||||
tweets_full <- head(tweets_full, -nrow(tweets_full)) # Empty tweets_full
|
||||
cat("User:",user,"finished after",loop,"loops. Total Tweets now:",nrow(tweets_complete),"\n")
|
||||
write.csv(tweets_complete, "tweets_complete.csv")
|
||||
# Every tweet from 2014 from user[r] is downloaded. Now next user in for-loop
|
||||
}
|
||||
status
|
||||
content
|
||||
current
|
||||
query
|
||||
tweets_full$text[59203]
|
||||
tweets_full$text[59202]
|
||||
tweets_full$text[59203]
|
||||
tweets_full$user[59203]
|
||||
tweets_full$user[59202]
|
||||
for(a in 157:nrow(acc_df)) {
|
||||
tweets_temp
|
||||
status
|
||||
fromJSON(current)
|
||||
tweets_temp <- fromJSON(correctJSON(current))
|
||||
tweets_temp
|
||||
status <- errorErrorColumn(tweets_temp)
|
||||
a
|
||||
View(acc_df)
|
||||
tweets_complete$user[32539]
|
||||
tweets_full <- data.frame(user=character(), name=character(), created_at=character(), id_str=character(), text=character(), retweet_count=character())
|
||||
for(a in 94:nrow(acc_df)) {
|
||||
user <- as.character(acc_df$screenname[a])
|
||||
name <- as.character(acc_df$name[a])
|
||||
max_id <- "999999999999999999"
|
||||
@@ -128,45 +325,27 @@ count=max_count,
|
||||
max_id=max_id);
|
||||
# At first, work with an temporary tweet-DB
|
||||
current <- twitter_api_call(api_url, query, api_params)
|
||||
rm(tweets_temp)
|
||||
tweets_temp <- fromJSON(correctJSON(current))
|
||||
## STAT ERROR HANDLING ##
|
||||
# Check for empty API returns
|
||||
status <- length(tweets_temp)
|
||||
if(status == 0) {
|
||||
if(error > 3) {
|
||||
cat("[WARNING] 3x empty API result. Aborting now.\n")
|
||||
## START ERROR HANDLING ##
|
||||
# Empty API output
|
||||
status <- errorEmptyAPI(tweets_temp)
|
||||
if(status == 1) { Sys.sleep(3);error <- error + 1;next}
|
||||
if(status == 2) {break}
|
||||
# Contains "error" column
|
||||
status <- errorErrorColumn(tweets_temp)
|
||||
if(status == 1) { Sys.sleep(3);error <- error + 1;next}
|
||||
if(status == 2) {break}
|
||||
# Check if error code exists
|
||||
code <- errorCheckCode(tweets_temp) # 0 if no error
|
||||
if(code == 34) { # page does not exist
|
||||
status <- errorCode34
|
||||
if(status == 1) { Sys.sleep(3);error <- error + 1;next}
|
||||
if(status == 2) {break}
|
||||
}
|
||||
cat("[WARNING] Empty API result. Trying again.\n")
|
||||
rm(tweets_temp)
|
||||
error <- error + 1
|
||||
Sys.sleep(3)
|
||||
next
|
||||
}
|
||||
# Check if API output contains error fields
|
||||
status <- "error" %in% names(tweets_temp)
|
||||
if(status) {
|
||||
cat("[WARNING] Error in API request:", tweets_temp$error[1],"\n")
|
||||
rm(tweets_temp)
|
||||
break
|
||||
}
|
||||
# Check for other errors, mostly rate limits
|
||||
status <- "errors" %in% names(tweets_temp)
|
||||
if(status) {
|
||||
cat("[WARNING] Error in API request:", tweets_temp$errors[1,1],"\n")
|
||||
# Rate limit exceeded?
|
||||
status <- tweets_temp$errors[1,2]
|
||||
if(status == 88) {
|
||||
rate_api_url <- "https://api.twitter.com/1.1/application/rate_limit_status.json"
|
||||
rate_query <-c (resources="statuses")
|
||||
resettime <- fromJSON(twitter_api_call(rate_api_url, rate_query, api_params))
|
||||
resettime <- resettime$resources$statuses$`/statuses/user_timeline`$reset
|
||||
curtime <- as.numeric(as.POSIXct(Sys.time()))
|
||||
wait <- round(resettime - curtime + 10)
|
||||
cat("[INFO] Rate limit is exceeded. Now waiting",wait,"seconds.\n")
|
||||
if(code == 88) { # rate limit exceeded
|
||||
wait <- errorCode88()
|
||||
Sys.sleep(wait)
|
||||
}
|
||||
rm(tweets_temp)
|
||||
Sys.sleep(3)
|
||||
next
|
||||
}
|
||||
## END ERROR HANDLING ##
|
||||
@@ -181,9 +360,9 @@ if(tweets_full$id_str[nrow(tweets_full)] == tweets_temp$id_str[nrow(tweets_temp)
|
||||
cat("[INFO] Last tweet of temp is last tweet of full. Abort loop and begin with next user.\n")
|
||||
break
|
||||
}
|
||||
## Last loop is reached. Now clear the data frame
|
||||
# Is the last tweet in tweets_temp from 2013?
|
||||
status <- str_detect(tweets_temp$created_at[nrow(tweets_temp)], "2014$")
|
||||
# Last loop is reached. Now clear the data frame
|
||||
if (!status) { # Starting when tweet not from 2014
|
||||
# Delete all tweets other than from 2014
|
||||
old <- 0
|
||||
@@ -210,7 +389,7 @@ tweets_temp <- head(tweets_temp, -old)
|
||||
}
|
||||
rm(old)
|
||||
tweets_full <- insertRow(tweets_full, tweets_temp)
|
||||
rm(tweets_temp)
|
||||
#rm(tweets_temp)
|
||||
break # End loop because 2013 is reached
|
||||
}
|
||||
# The last tweet is still from 2014, so we need another loop
|
||||
@@ -219,15 +398,19 @@ else {
|
||||
max_id <- tweets_temp$id_str[nrow(tweets_temp)]
|
||||
loop <- loop + 1 # just for stats
|
||||
tweets_full <- insertRow(tweets_full, tweets_temp)
|
||||
rm(tweets_temp)
|
||||
#rm(tweets_temp)
|
||||
}
|
||||
} # /repeat
|
||||
stat_tweet <- nrow(tweets_full)
|
||||
cat("User:",user,"finished after",loop,"loops. Total Tweets now:",nrow(tweets_full),"\n")
|
||||
write.csv(tweets_full, "tweets_full.csv")
|
||||
tweets_complete <- insertRow(tweets_complete, tweets_full)
|
||||
tweets_full <- head(tweets_full, -nrow(tweets_full)) # Empty tweets_full
|
||||
cat("User:",user,"finished after",loop,"loops. Total Tweets now:",nrow(tweets_complete),"\n")
|
||||
write.csv(tweets_complete, "tweets_complete.csv")
|
||||
# Every tweet from 2014 from user[r] is downloaded. Now next user in for-loop
|
||||
}
|
||||
for(a in 157:nrow(acc_df)) {
|
||||
status
|
||||
tweets_full <- data.frame(user=character(), name=character(), created_at=character(), id_str=character(), text=character(), retweet_count=character())
|
||||
a
|
||||
for(a in 346:nrow(acc_df)) {
|
||||
user <- as.character(acc_df$screenname[a])
|
||||
name <- as.character(acc_df$name[a])
|
||||
max_id <- "999999999999999999"
|
||||
@@ -241,46 +424,27 @@ count=max_count,
|
||||
max_id=max_id);
|
||||
# At first, work with an temporary tweet-DB
|
||||
current <- twitter_api_call(api_url, query, api_params)
|
||||
rm(tweets_temp)
|
||||
tweets_temp <- fromJSON(correctJSON(current))
|
||||
## STAT ERROR HANDLING ##
|
||||
# Check for empty API returns
|
||||
status <- length(tweets_temp)
|
||||
if(status == 0) {
|
||||
if(error > 2) {
|
||||
cat("[WARNING] 3x empty API result. Aborting now.\n")
|
||||
break
|
||||
## START ERROR HANDLING ##
|
||||
# Empty API output
|
||||
status <- errorEmptyAPI(tweets_temp)
|
||||
if(status == 1) { Sys.sleep(3);error <- error + 1;next}
|
||||
if(status == 2) {break}
|
||||
# Contains "error" column
|
||||
status <- errorErrorColumn(tweets_temp)
|
||||
if(status == 1) { Sys.sleep(3);error <- error + 1;next}
|
||||
if(status == 2) {break}
|
||||
# Check if error code exists
|
||||
code <- errorCheckCode(tweets_temp) # 0 if no error
|
||||
if(code == 34) { # page does not exist
|
||||
status <- errorCode34()
|
||||
if(status == 1) { Sys.sleep(3);error <- error + 1;next}
|
||||
if(status == 2) {break}
|
||||
}
|
||||
cat("[WARNING] Empty API result. Trying again.\n")
|
||||
rm(tweets_temp)
|
||||
error <- error + 1
|
||||
Sys.sleep(3)
|
||||
next
|
||||
}
|
||||
# Check if API output contains error fields
|
||||
status <- "error" %in% names(tweets_temp)
|
||||
if(status) {
|
||||
cat("[WARNING] Error in API request:", tweets_temp$error[1],"\n")
|
||||
rm(tweets_temp)
|
||||
break
|
||||
}
|
||||
# Check for other errors, mostly rate limits
|
||||
status <- "errors" %in% names(tweets_temp)
|
||||
if(status) {
|
||||
cat("[WARNING] Error in API request:", tweets_temp$errors[1,1],"\n")
|
||||
# Rate limit exceeded?
|
||||
status <- tweets_temp$errors[1,2]
|
||||
if(status == 88) {
|
||||
rate_api_url <- "https://api.twitter.com/1.1/application/rate_limit_status.json"
|
||||
rate_query <-c (resources="statuses")
|
||||
resettime <- fromJSON(twitter_api_call(rate_api_url, rate_query, api_params))
|
||||
resettime <- resettime$resources$statuses$`/statuses/user_timeline`$reset
|
||||
curtime <- as.numeric(as.POSIXct(Sys.time()))
|
||||
wait <- round(resettime - curtime + 10)
|
||||
cat("[INFO] Rate limit is exceeded. Now waiting",wait,"seconds.\n")
|
||||
if(code == 88) { # rate limit exceeded
|
||||
wait <- errorCode88()
|
||||
Sys.sleep(wait)
|
||||
}
|
||||
rm(tweets_temp)
|
||||
Sys.sleep(3)
|
||||
next
|
||||
}
|
||||
## END ERROR HANDLING ##
|
||||
@@ -295,9 +459,9 @@ if(tweets_full$id_str[nrow(tweets_full)] == tweets_temp$id_str[nrow(tweets_temp)
|
||||
cat("[INFO] Last tweet of temp is last tweet of full. Abort loop and begin with next user.\n")
|
||||
break
|
||||
}
|
||||
## Last loop is reached. Now clear the data frame
|
||||
# Is the last tweet in tweets_temp from 2013?
|
||||
status <- str_detect(tweets_temp$created_at[nrow(tweets_temp)], "2014$")
|
||||
# Last loop is reached. Now clear the data frame
|
||||
if (!status) { # Starting when tweet not from 2014
|
||||
# Delete all tweets other than from 2014
|
||||
old <- 0
|
||||
@@ -324,7 +488,7 @@ tweets_temp <- head(tweets_temp, -old)
|
||||
}
|
||||
rm(old)
|
||||
tweets_full <- insertRow(tweets_full, tweets_temp)
|
||||
rm(tweets_temp)
|
||||
#rm(tweets_temp)
|
||||
break # End loop because 2013 is reached
|
||||
}
|
||||
# The last tweet is still from 2014, so we need another loop
|
||||
@@ -333,180 +497,16 @@ else {
|
||||
max_id <- tweets_temp$id_str[nrow(tweets_temp)]
|
||||
loop <- loop + 1 # just for stats
|
||||
tweets_full <- insertRow(tweets_full, tweets_temp)
|
||||
rm(tweets_temp)
|
||||
#rm(tweets_temp)
|
||||
}
|
||||
} # /repeat
|
||||
stat_tweet <- nrow(tweets_full)
|
||||
cat("User:",user,"finished after",loop,"loops. Total Tweets now:",nrow(tweets_full),"\n")
|
||||
write.csv(tweets_full, "tweets_full.csv")
|
||||
# Every tweet from 2014 from user[r] is downloaded. Now next user in for-loop
|
||||
tweets_complete <- insertRow(tweets_complete, tweets_full)
|
||||
tweets_full <- head(tweets_full, -nrow(tweets_full)) # Empty tweets_full
|
||||
cat("User:",user,"finished after",loop,"loops. Total Tweets now:",nrow(tweets_complete),"\n")
|
||||
write.csv(tweets_complete, "tweets_complete.csv")
|
||||
# Every tweet from 2014 from user[a] is downloaded. Now next user in for-loop
|
||||
}
|
||||
current
|
||||
correctJSON(current)
|
||||
correctJSON <- function(string) {
|
||||
# string <- str_replace_all(string, pattern = "\n", replacement = " ")
|
||||
# string <- str_replace_all(string, pattern = "\r", replacement = " ")
|
||||
# string <- str_replace_all(string, pattern = "\\^", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "[^[:print:]]", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "&..;", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = perl('\\\\(?![tn"])'), replacement = " ")
|
||||
return(string)
|
||||
}
|
||||
correctJSON(current)
|
||||
a
|
||||
tweets_temp <- fromJSON(correctJSON(current))
|
||||
current
|
||||
correctJSON <- function(string) {
|
||||
# string <- str_replace_all(string, pattern = "\n", replacement = " ")
|
||||
# string <- str_replace_all(string, pattern = "\r", replacement = " ")
|
||||
# string <- str_replace_all(string, pattern = "\\^", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "[^[:print:]]", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "&..;", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = perl('\\\\(?![tn"])'), replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "\\\\\\\\\\", replacement = "\\\\\\")
|
||||
return(string)
|
||||
}
|
||||
tweets_temp <- fromJSON(correctJSON(current))
|
||||
correctJSON <- function(string) {
|
||||
# string <- str_replace_all(string, pattern = "\n", replacement = " ")
|
||||
# string <- str_replace_all(string, pattern = "\r", replacement = " ")
|
||||
# string <- str_replace_all(string, pattern = "\\^", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "[^[:print:]]", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "&..;", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = perl('\\\\(?![tn"])'), replacement = " ")
|
||||
string <- str_replace_all(string, pattern = perl('\\\\\\(?![tn"])'), replacement = " ")
|
||||
return(string)
|
||||
}
|
||||
tweets_temp <- fromJSON(correctJSON(current))
|
||||
correctJSON <- function(string) {
|
||||
# string <- str_replace_all(string, pattern = "\n", replacement = " ")
|
||||
# string <- str_replace_all(string, pattern = "\r", replacement = " ")
|
||||
# string <- str_replace_all(string, pattern = "\\^", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "[^[:print:]]", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "&..;", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = perl('\\\\(?![tn"])'), replacement = " ")
|
||||
string <- str_replace_all(string, pattern = perl('\\\\\\\\(?![tn"])'), replacement = " ")
|
||||
return(string)
|
||||
}
|
||||
tweets_temp <- fromJSON(correctJSON(current))
|
||||
correctJSON <- function(string) {
|
||||
# string <- str_replace_all(string, pattern = "\n", replacement = " ")
|
||||
# string <- str_replace_all(string, pattern = "\r", replacement = " ")
|
||||
# string <- str_replace_all(string, pattern = "\\^", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "[^[:print:]]", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "&..;", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = perl('\\\\(?![tn"])'), replacement = " ")
|
||||
string <- str_replace_all(string, pattern = perl('\\\\\\\\\\'), replacement = " ")
|
||||
return(string)
|
||||
}
|
||||
tweets_temp <- fromJSON(correctJSON(current))
|
||||
correctJSON <- function(string) {
|
||||
# string <- str_replace_all(string, pattern = "\n", replacement = " ")
|
||||
# string <- str_replace_all(string, pattern = "\r", replacement = " ")
|
||||
# string <- str_replace_all(string, pattern = "\\^", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "[^[:print:]]", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "&..;", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = perl('\\\\(?![tn"])'), replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "[\]{5}", replacement = " ")
|
||||
return(string)
|
||||
}
|
||||
correctJSON <- function(string) {
|
||||
# string <- str_replace_all(string, pattern = "\n", replacement = " ")
|
||||
# string <- str_replace_all(string, pattern = "\r", replacement = " ")
|
||||
# string <- str_replace_all(string, pattern = "\\^", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "[^[:print:]]", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "&..;", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = perl('\\\\(?![tn"])'), replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "[\\]{5}", replacement = " ")
|
||||
return(string)
|
||||
}
|
||||
tweets_temp <- fromJSON(correctJSON(current))
|
||||
correctJSON <- function(string) {
|
||||
# string <- str_replace_all(string, pattern = "\n", replacement = " ")
|
||||
# string <- str_replace_all(string, pattern = "\r", replacement = " ")
|
||||
# string <- str_replace_all(string, pattern = "\\^", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "[^[:print:]]", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "&..;", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = perl('\\\\(?![tn"])'), replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "[\\]{6}", replacement = " ")
|
||||
return(string)
|
||||
}
|
||||
tweets_temp <- fromJSON(correctJSON(current))
|
||||
correctJSON <- function(string) {
|
||||
# string <- str_replace_all(string, pattern = "\n", replacement = " ")
|
||||
# string <- str_replace_all(string, pattern = "\r", replacement = " ")
|
||||
# string <- str_replace_all(string, pattern = "\\^", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "[^[:print:]]", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "&..;", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = perl('\\\\(?![tn"])'), replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "\\{6}", replacement = " ")
|
||||
return(string)
|
||||
}
|
||||
tweets_temp <- fromJSON(correctJSON(current))
|
||||
correctJSON <- function(string) {
|
||||
# string <- str_replace_all(string, pattern = "\n", replacement = " ")
|
||||
# string <- str_replace_all(string, pattern = "\r", replacement = " ")
|
||||
# string <- str_replace_all(string, pattern = "\\^", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "[^[:print:]]", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "&..;", replacement = " ")
|
||||
string <- str_replace_all(string, pattern = perl('\\\\(?![tn"])'), replacement = " ")
|
||||
string <- str_replace_all(string, pattern = "\\{5}", replacement = " ")
|
||||
return(string)
|
||||
}
|
||||
insertRow <- function(existingDF, newrow, r) {
|
||||
r <- as.numeric(nrow(existingDF)) + 1
|
||||
existingDF <- rbind(existingDF,newrow)
|
||||
existingDF <- existingDF[order(c(1:(nrow(existingDF)-1),r-0.5)),]
|
||||
row.names(existingDF) <- 1:nrow(existingDF)
|
||||
return(existingDF)
|
||||
}
|
||||
tweets_temp <- fromJSON(correctJSON(current))
|
||||
str_detect(current, pattern = "\\")
|
||||
str_detect(current, pattern = '\\')
|
||||
str_detect(current, pattern = "\\\")
|
||||
str_detect(current, pattern = "[\\]")
|
||||
str_detect(current, pattern = "[\\\\]")
|
||||
str_detect(current, pattern = "[\\\\\\]")
|
||||
str_detect(current, pattern = "[\\\\\\\\]")
|
||||
str_detect(current, pattern = "[\\\\\\\\\\\\]")
|
||||
str_detect(current, pattern = "[\\\\\\\\\\\\\\]")
|
||||
str_detect(current, pattern = "[\\]{1}")
|
||||
str_detect(current, pattern = "[\\]{7}")
|
||||
str_detect(current, pattern = "[\\]{2}")
|
||||
str_detect(current, pattern = "[\\]{3}")
|
||||
str_detect(current, pattern = "[\\]{4}")
|
||||
str_detect(current, pattern = "[\\]{5}")
|
||||
current
|
||||
str_detect(current, pattern = "[\\]{5}")
|
||||
str_detect(current, pattern = "[\]{5}")
|
||||
str_detect(current, pattern = "[\\]{2}")
|
||||
grep("[\\]", current)
|
||||
grep("[\\]{2}", current)
|
||||
grep("[\\]{3}", current)
|
||||
grep("[\\]{4}", current)
|
||||
grep("[\\]{5}", current)
|
||||
grep("[\\]{1}", current)
|
||||
str_detect(current, pattern = fixed("\\\\\"))
|
||||
str_detect(current, pattern = fixed("\\\\\\\\\\\"))
|
||||
str_detect(current, pattern = fixed("\\\\\\\\\\"))
|
||||
str_detect(current, pattern = fixed("\\\\\\\\"))
|
||||
str_detect(current, pattern = fixed("\\\\\\"))
|
||||
str_detect(current, pattern = fixed("\\\\\"))
|
||||
str_detect(current, pattern = fixed("\\\\"))
|
||||
str_detect(current, pattern = fixed("\\"))
|
||||
nchar("\\")
|
||||
nchar("\\\\")
|
||||
nchar("\\\\\\")
|
||||
nchar("\\\\\\\\")
|
||||
nchar("\\\\\\\\\\")
|
||||
print("\\\\\\\\\\")
|
||||
print("\\\\\")
|
||||
print("\\\\\\")
|
||||
print("\\\\")
|
||||
print("\\\\\")
|
||||
write(current, "current.txt")
|
||||
current2 <- readLines("current.txt")
|
||||
current
|
||||
current2
|
||||
rm(current2)
|
||||
a
|
||||
save(tweets_complete, file="tweets_complete.RData")
|
||||
tweets_complete$id_str[146982]
|
||||
class(tweets_complete$id_str[146982])
|
||||
tweets_complete$id_str[1]
|
||||
|
||||
Reference in New Issue
Block a user