c_errcode <- as.character(c_errors$code[r])
c_errissue <- as.character(c_errors$issue[r])
c_errtags <- as.character(c_errors$tags[r])
c_errtext <- as.character(c_errors$text[r])
c_errid <- as.character(c_errors$str_id[r])
cat("===============\n\n[TWEET]: ",c_errtext,"\n[ISSUES]: ", c_errissue, " (", c_errtags, ")\n", sep="")
source("issuecomp-codingsample-function2.R")
}
c_tmp <- read.csv("issuecomp-codingsample-error1.csv", header = F, colClasses="character")
View(c_tmp)
View(c_errors)
View(c_tmp)
names(c_tmp) <- c("str_id", "all", "wrong", "tags", "text")
View(c_tmp)
c_tmp[, c("wrong", "tagged", "all", "text")]
View(c_tmp)
names(c_tmp) <- c("str_id", "all", "wrong", "tagged", "text")
c_tmp[, c("wrong", "tagged", "all", "text")]
c_error1 <- c_tmp[, c("wrong", "tagged", "all", "text")]
View(c_error1)
c_tmp <- read.csv("issuecomp-codingsample-error2.csv", header = F, colClasses="character")
View(c_tmp)
c_tmp <- read.csv("issuecomp-codingsample-error2.csv", header = F, colClasses="character")
names(c_tmp) <- c("str_id", "all", "missing", "tagged", "text")
c_error1 <- c_tmp[, c("missing", "tagged", "all", "text")]
c_error2 <- c_tmp[, c("missing", "tagged", "all", "text")]
View(c_error2)
c_error2 <- c_tmp[, c("missing", "text", "tagged", "all")]
View(c_error2)
View(c_error1)
View(c_error2)
c_tmp <- read.csv("issuecomp-codingsample-correct.csv", header = F, colClasses="character")
View(c_tmp)
names(c_tmp) <- c("str_id", "status", "issue", "tags", "text")
View(c_tmp)
c_currect <- c_tmp
c_correct <- c_tmp
rm(c_currect)
View(c_correct)
source("issuecomp-codingsample-function.R")
rm(c_err, c_result, c_samid, c_samno,c_samtags,c_samissue,c_samtext,c_yn)
c_errors <- read.csv("issuecomp-codingsample-error.csv", header = F, sep=",", colClasses="character")
c_errors <- read.csv("issuecomp-codingsample-error.csv", header = F, sep=",", colClasses="character")
c_errors <- read.csv("issuecomp-codingsample-error.csv", header = F, sep=",", colClasses="character")
c_errors <- read.csv("issuecomp-codingsample-error.csv", header = F, sep=",", colClasses="character")
c_errors <- read.csv("issuecomp-codingsample-error.csv", header = F, sep=",", colClasses="character", quote = "")
View(c_errors)
c_errors <- read.csv("issuecomp-codingsample-error.csv", header = F, sep=",", colClasses="character")
test <- "Zitat "total dämlich!""
tweets$id_str == "523512815425175552"
tweets[tweets$id_str == "523512815425175552"]
tweets[tweets$id_str == "523512815425175552", ]
tweets[tweets$id_str == "523512815425175552", "text"]
test <- tweets[tweets$id_str == "523512815425175552", "text"]
test
test <- c_tweets[ctweets$id_str == "523512815425175552", "text"]
test <- c_tweets[c_tweets$id_str == "523512815425175552", "text"]
test
str_replace(test, "\\"", ")
str_replace(test, "\\"", "")
str_replace(test, "\"", "")
str_detect(test, "\"")
test <- as.character(c_tweets[c_tweets$id_str == "523512815425175552", "text"])
test
c_tweets <- read.csv("tweets.csv", colClasses="character")
for(r in 1:nrow(c_tweets)) {
curtext <- as.character(c_tweets$text[r])
if(str_detect(curtext, "\"") {
c_tweets$text[r] <- str_replace(curtext, "\"", "")
}
}
for(r in 1:nrow(c_tweets)) {
curtext <- as.character(c_tweets$text[r])
if(str_detect(curtext, "\"") {
c_tweets$text[r] <- str_replace(curtext, "\"", "")
} else {}
}
for(r in 1:nrow(c_tweets)) {
curtext <- as.character(c_tweets$text[r])
if(str_detect(curtext, "\"") {
c_tweets$text[r] <- str_replace(curtext, "\"", "")
} else {
}
}
for(r in 1:nrow(c_tweets)) {
curtext <- as.character(c_tweets$text[r])
if(str_detect(curtext, "\"")) {
c_tweets$text[r] <- str_replace(curtext, "\"", "")
}
}
test <- as.character(c_tweets[c_tweets$id_str == "523512815425175552", "text"])
test
View(c_tweets)
c_errors <- read.csv("issuecomp-codingsample-error.csv", header = F, sep=",", colClasses="character")
View(c_errors)
names(c_errors) <- c("str_id", "code", "issue", "tags", "text")
View(c_errors)
for(r in 1:nrow(c_errors)) {
c_errcode <- as.character(c_errors$code[r])
c_errissue <- as.character(c_errors$issue[r])
c_errtags <- as.character(c_errors$tags[r])
c_errtext <- as.character(c_errors$text[r])
c_errid <- as.character(c_errors$str_id[r])
cat("===============\n\n[TWEET]: ",c_errtext,"\n[ISSUES]: ", c_errissue, " (", c_errtags, ")\n", sep="")
source("issuecomp-codingsample-function2.R")
}
issueheads
for(r in 1:nrow(c_errors)) {
c_errcode <- as.character(c_errors$code[r])
c_errissue <- as.character(c_errors$issue[r])
c_errtags <- as.character(c_errors$tags[r])
c_errtext <- as.character(c_errors$text[r])
c_errid <- as.character(c_errors$str_id[r])
cat("===============\n\n[TWEET]: ",c_errtext,"\n[ISSUES]: ", c_errissue, " (", c_errtags, ")\n", sep="")
source("issuecomp-codingsample-function2.R")
}
# All tweets with WRONG ISSUES
c_tmp <- read.csv("issuecomp-codingsample-error1.csv", header = F, colClasses="character")
names(c_tmp) <- c("str_id", "all", "wrong", "tagged", "text")
c_error1 <- c_tmp[, c("wrong", "tagged", "all", "text")]
# All tweets with MISSING ISSUES
c_tmp <- read.csv("issuecomp-codingsample-error2.csv", header = F, colClasses="character")
names(c_tmp) <- c("str_id", "all", "missing", "tagged", "text")
c_error2 <- c_tmp[, c("missing", "text", "tagged", "all")]
# All CORRECT tweets
c_tmp <- read.csv("issuecomp-codingsample-correct.csv", header = F, colClasses="character")
names(c_tmp) <- c("str_id", "status", "issue", "tags", "text")
c_correct <- c_tmp
View(c_error1)
View(c_error2)
View(c_error1)
View(c_correct)
test <- VAR(issues_i[,2:22], p=1, type="none", exogen = issues_s[,2:3])
plot(irf(test, impulse = names(issues_s[2:11]), response = names(issues_i[2:22])))
test <- VAR(issues[,2:32], p=1, type="none")
plot(irf(test, impulse = names(issues_s[2:11]), response = names(issues_i[2:22])))
VARselect(issues[,2:32], lag.max=8, type="none")
VARselect(issues[,2:32], lag.max=8, type="both")
VARselect(issues[,2:32], lag.max=30, type="both")
VARselect(issues[,2:32], lag.max=15, type="both")
for(i in 1:20) { cat(i,"\n") Sys.sleep(10)}
for(i in 1:20) { cat(i,"\n")Sys.sleep(10)}
for(i in 1:20) { cat(i,"\n")Sys.sleep(10)}
for(i in 1:20) { cat(i,"\n")
Sys.sleep(10)}
list.dirs()
list.files()
rm(results)
setwd("matched-ids/")
results_files <- list.files()
results_files
results_files <- "all.csv"
for(r in 1:length(results_files)) {
if(r == 1) {
results <- read.csv(results_files[r], sep=";", colClasses=c("character", "character", "character", "character"), header=F)
} else {
results_temp <- read.csv(results_files[r], sep=";", colClasses=c("character", "character", "character", "character"), header=F)
results <- insertRow(results, results_temp)
}
}
rm(r, results_temp, results_files)
results <- results[!duplicated(results), ]
names(results) <- c("date", "id_str", "issue", "tags")
results <- results[order(results$id_str), ]
row.names(results) <- NULL
results[23381,]
results[53381,]
results[43253,]
for(r in 53371:nrow(results)) {
curdate <- as.character(results$date[r])
curid <- as.character(results$id_str[r])
curissue <- as.character(results$issue[r])
curtag <- as.character(results$tags[r])
cat("Sorting match", r, "of 53383 \n")
# Update issue counter (date and issue)
issues[issues[, "date"] == curdate, curissue] <- issues[issues[, "date"] == curdate, curissue] + 1
# Update tweet dataframe (id, issue and tags)
oldissue <- tweets[tweets[, "id_str"] == curid, "issue"]
tweets[tweets[, "id_str"] == curid, "issue"] <- str_c(oldissue, curissue, ",")
oldtag <- tweets[tweets[, "id_str"] == curid, "tags"]
tweets[tweets[, "id_str"] == curid, "tags"] <- str_c(oldtag, curtag, ",")
}
issues[issueheads] <- 0
View(issues)
for(r in 1:nrow(results)) {
curdate <- as.character(results$date[r])
curid <- as.character(results$id_str[r])
curissue <- as.character(results$issue[r])
curtag <- as.character(results$tags[r])
cat("Sorting match", r, "of 53383 \n")
# Update issue counter (date and issue)
issues[issues[, "date"] == curdate, curissue] <- issues[issues[, "date"] == curdate, curissue] + 1
# Update tweet dataframe (id, issue and tags)
oldissue <- tweets[tweets[, "id_str"] == curid, "issue"]
tweets[tweets[, "id_str"] == curid, "issue"] <- str_c(oldissue, curissue, ",")
oldtag <- tweets[tweets[, "id_str"] == curid, "tags"]
tweets[tweets[, "id_str"] == curid, "tags"] <- str_c(oldtag, curtag, ",")
}
require(lubridate)
require(XML)
require(ggplot2)
require(reshape2)
require(stringr)
require(foreach)
require(doParallel)
for(r in 1:nrow(results)) {
curdate <- as.character(results$date[r])
curid <- as.character(results$id_str[r])
curissue <- as.character(results$issue[r])
curtag <- as.character(results$tags[r])
cat("Sorting match", r, "of 53383 \n")
# Update issue counter (date and issue)
issues[issues[, "date"] == curdate, curissue] <- issues[issues[, "date"] == curdate, curissue] + 1
# Update tweet dataframe (id, issue and tags)
oldissue <- tweets[tweets[, "id_str"] == curid, "issue"]
tweets[tweets[, "id_str"] == curid, "issue"] <- str_c(oldissue, curissue, ",")
oldtag <- tweets[tweets[, "id_str"] == curid, "tags"]
tweets[tweets[, "id_str"] == curid, "tags"] <- str_c(oldtag, curtag, ",")
}
results[119,]
results[120,]
load(file = "tweets_untagged.RData")
setwd("~/Dokumente/Uni/Aktuell/BA-Arbeit/uni-ba-issuecomp")
results_files <- "matched-ids/all.csv"
load(file = "tweets_untagged.RData")
View(issues)
issues <- data.frame(date = drange)
issuelist <- readLines("issues.xml")
issuelist <- str_replace_all(string = issuelist, pattern = ".*<!-- .+ -->", "")
issuelist <- xmlToList(issuelist)
issueheads <- names(issuelist)
issues[issueheads] <- 0
tweets$issue <- ""
tweets$tags <- ""
View(results)
rm(r, results_temp, results_files)
results <- results[!duplicated(results), ]
names(results) <- c("date", "id_str", "issue", "tags")
results <- results[order(results$id_str), ]
row.names(results) <- NULL
for(r in 1:nrow(results)) {
curdate <- as.character(results$date[r])
curid <- as.character(results$id_str[r])
curissue <- as.character(results$issue[r])
curtag <- as.character(results$tags[r])
cat("Sorting match", r, "of 53383 \n")
# Update issue counter (date and issue)
issues[issues[, "date"] == curdate, curissue] <- issues[issues[, "date"] == curdate, curissue] + 1
# Update tweet dataframe (id, issue and tags)
oldissue <- tweets[tweets[, "id_str"] == curid, "issue"]
tweets[tweets[, "id_str"] == curid, "issue"] <- str_c(oldissue, curissue, ",")
oldtag <- tweets[tweets[, "id_str"] == curid, "tags"]
tweets[tweets[, "id_str"] == curid, "tags"] <- str_c(oldtag, curtag, ",")
}
curdate
curissue
issues[issues[, "date"] == curdate, curissue]
issueheads
issuelist <- readLines("issues-v2.xml")
issues <- data.frame(date = drange)
issuelist <- readLines("issues-v2.xml")
issuelist <- str_replace_all(string = issuelist, pattern = ".*<!-- .+ -->", "")
issuelist <- xmlToList(issuelist)
issueheads <- names(issuelist)
issues[issueheads] <- 0
tweets$issue <- ""
tweets$tags <- ""
for(r in 1:nrow(results)) {
curdate <- as.character(results$date[r])
curid <- as.character(results$id_str[r])
curissue <- as.character(results$issue[r])
curtag <- as.character(results$tags[r])
cat("Sorting match", r, "of 53383 \n")
# Update issue counter (date and issue)
issues[issues[, "date"] == curdate, curissue] <- issues[issues[, "date"] == curdate, curissue] + 1
# Update tweet dataframe (id, issue and tags)
oldissue <- tweets[tweets[, "id_str"] == curid, "issue"]
tweets[tweets[, "id_str"] == curid, "issue"] <- str_c(oldissue, curissue, ",")
oldtag <- tweets[tweets[, "id_str"] == curid, "tags"]
tweets[tweets[, "id_str"] == curid, "tags"] <- str_c(oldtag, curtag, ",")
}
results[33170,]
results[33171,]
results$date[33170]
results$date[33170] <- "2014-08-21"
for(r in 33170:nrow(results)) {
curdate <- as.character(results$date[r])
curid <- as.character(results$id_str[r])
curissue <- as.character(results$issue[r])
curtag <- as.character(results$tags[r])
cat("Sorting match", r, "of 53383 \n")
# Update issue counter (date and issue)
issues[issues[, "date"] == curdate, curissue] <- issues[issues[, "date"] == curdate, curissue] + 1
# Update tweet dataframe (id, issue and tags)
oldissue <- tweets[tweets[, "id_str"] == curid, "issue"]
tweets[tweets[, "id_str"] == curid, "issue"] <- str_c(oldissue, curissue, ",")
oldtag <- tweets[tweets[, "id_str"] == curid, "tags"]
tweets[tweets[, "id_str"] == curid, "tags"] <- str_c(oldtag, curtag, ",")
}
save(tweets, file="tweets_tagged.RData")
write.csv(tweets, file="tweets.csv")
save(issues, file="issues.RData")
require(stringr)
require(reshape2)
require(ggplot2)
require(vars)
drop_s <- which(str_detect(names(issues), "^s"))
drop_i <- which(str_detect(names(issues), "^i"))
issues_i <- issues[,-drop_s]
issues_s <- issues[,-drop_i]
issues_i$total <- rowSums(issues_i[2:ncol(issues_i)])
issues_i$entropy <- 0
for(r in 1:nrow(issues_i)) {
curtotal <- as.numeric(issues_i$total[r])
curp <- 0
for(c in 2:ncol(issues_i)) {
curcount <- as.numeric(issues_i[r,c])
curp[c] <- curcount / curtotal
}
curp <- curp [2:length(curp)-2]
curdrop <- which(curp==0)
curp <- curp[-curdrop]
issues_i$entropy[r] <- sum(-1 * curp * log(curp))
}
issues_s$total <- rowSums(issues_s[2:ncol(issues_s)])
issues_s$entropy <- 0
for(r in 1:nrow(issues_s)) {
curtotal <- as.numeric(issues_s$total[r])
curp <- 0
for(c in 2:ncol(issues_s)) {
curcount <- as.numeric(issues_s[r,c])
curp[c] <- curcount / curtotal
}
curp <- curp [2:length(curp)-2]
curdrop <- which(curp==0)
curp <- curp[-curdrop]
issues_s$entropy[r] <- sum(-1 * curp * log(curp))
}
stats_total <- data.frame(date=drange)
stats_total$tpd <- 0
stats_total$ipd <- issues_i$total
stats_total$spd <- issues_s$total
# Total number of tweets per day over time
for(r in 1:length(drange)) {
stats_total$tpd[r] <- length(tweets[tweets[, "created_at"] == drange[r], "id_str"])
}
g1 <- ggplot(data = stats_melt, aes(x=date,y=value,colour=variable, group=variable)) +
geom_line()+
geom_smooth(size=1,formula = y ~ x, method="loess", se=FALSE, color=1)
g1
stats_entropy <- data.frame(date=drange)
stats_entropy$entropy <- issues_i$entropy
stats_entropy <- melt(stats_entropy, id="date")
g1 <- ggplot(data = stats_entropy, aes(x=date,y=value,colour=variable, group=variable)) +
geom_line() +
geom_smooth(size=1,formula = y ~ x, method="loess", se=FALSE, color=1)
g1
test <- VAR(issues[,2:32], p=1, type="none")
View(issues_i)
View(issues_s)
View(issues)
test <- VAR(issues[,2:44], p=1, type="none")
VAR(issues_s[,2:23], p=1, type=c("const", "trend", "both", "none"), season=NULL, exogen = issues_i[2:22])
plot(irf(test, impulse = names(issues_s[2:23]), response = names(issues_i[2:22])))
rm(c_correct, c_curissue, c_errcode, c_errid, c_errissue, c_error1, c_error2, c_errors)
rm(c_issues, c_issuelist, c_issueheads)
rm(c_errtags, c_errtext, c_result, c_tag, c_tmp, c_tweets)
require(stringr)
require(XML)
readYN <- function(question) {
n <- readline(prompt=question)
n <- as.character(n)
return(n)
}
checkIssue <- function(string, issuelist) {
status <- any(str_detect(string, issuelist))
return(status)
}
checkAllIssues <- function(string, issuelist) {
status <- NULL
for(i in 1:length(string)) {
if(checkIssue(string[i], issuelist)) {
status[i] <- TRUE
}
else {
cat("Issue",string[i],"does not exist. Please try again.\n")
status[i] <- FALSE
}
}
return(status)
}
View(tweets)
c_tweets <- read.csv("tweets.csv", colClasses="character")
for(r in 1:nrow(c_tweets)) {
curtext <- as.character(c_tweets$text[r])
if(str_detect(curtext, "\"")) {
c_tweets$text[r] <- str_replace(curtext, "\"", "")
}
}
c_tweets$X <- NULL
c_issues <- data.frame(date = drange)
c_issuelist <- xmlToList("issues-v2.xml")
c_issueheads <- names(issuelist)
c_issues[issueheads] <- 0
source("issuecomp-codingsample-function.R")
require(stringr)
curtext
curtext <- str_replace_all(curtext, "#", "")
curtext <- str_replace_all(curtext, "-", " ")
curtext
curtext
str_replace_all(curtext, "[^[:alnum:]]", "")
str_replace_all(curtext, "[^[:alnum:]\s]", "")
str_replace_all(curtext, "[^[:alnum:]\\s]", "")
str_replace_all(curtext, "[^[:alnum:]^\\s]", "")
str_replace_all(curtext, "[^[:alnum:]^\\S]", "")
str_replace_all(curtext, "[^[:alnum:]][^\\s]", "")
str_replace_all(curtext, "[^[:alnum:]][^\\S]", "")
str_replace_all(curtext, "[^[:alnum:]][^[:blank]]", "")
str_replace_all(curtext, "[^[:alnum:]][^[:blank:]]", "")
str_replace_all(curtext, "[^[:alnum:]]", "")
str_replace_all(curtext, "\\W", "")
str_replace_all(curtext, "[\\W|\\S]", "")
str_replace_all(curtext, "(\\W|\\S)", "")
str_replace_all(curtext, "\\W|\\S", "")
str_replace_all(curtext, "\\W", "")
str_replace_all(curtext, "[\\W\\S]", "")
str_replace_all(curtext, "[\\S\\W]", "")
str_replace_all(curtext, "[\\s\\W]", "")
str_replace_all(curtext, "[\\W\\s]", "")
str_replace_all(curtext, "[\\W\s]", "")
str_replace_all(curtext, "[\\Ws]", "")
str_replace_all(curtext, "[\\W]", "")
str_replace_all(curtext, "\\W", "")
str_replace_all(curtext, "\\W|\\S", "")
str_replace_all(curtext, "\\W|\\s", "")
str_replace_all(curtext, "[^[:alnum:]]", "")
str_replace_all(curtext, "[^[:alnum:] ]", "")
str_replace_all(curtext, "[^[:alnum:]\\s]", "")
str_replace_all(curtext, "[^[:alnum:] ]", "")
curtext
curtext <- "liebe @cdu, wir finden #Steuer gut, aber die KFZ-Steuer nicht!"
curtext <- str_replace_all(curtext, "-", " ")
curtext <- str_replace_all(curtext, "[^[:alnum:] ]", "")
curtext
curtext <- "liebe @cdu, wir finden #Steuer gut, aber die KFZ--Steuer nicht!"
curtext <- str_replace_all(curtext, "-", " ")
curtext <- str_replace_all(curtext, "[^[:alnum:] ]", "")
curtext
str_replace_all(curtext, "  ", " ")
smartPatternMatch
require(vars)
require(stringr)
adf1 <- summary(ur.df(issues))
issues
summary(issues)
summary(issues[2:44])
summary(issues[2:44], digits = 2)
adf1 <- summary(ur.df(issues[, 2:44]), type ="trend", lags=1)
data("Canda")
data("Canada")
class(Canada)
class(issues)
view(Canada)
View(Canada)
as.ts(issues)
issues_ts <- as.ts(issues)
class(issues_ts)
View(issues_ts)
View(issues)
adf1 <- summary(ur.df(issues_ts[, 2:44]), type ="trend", lags=1)
adf1 <- summary(ur.df(issues_ts[, 2]), type ="trend", lags=1)
adf1 <- summary(ur.df(issues_ts[, 2], type ="trend", lags=1))
adf1 <- summary(ur.df(issues_ts[, 2:44], type ="trend", lags=1))
adf1 <- summary(ur.df(issues_ts[, 2], type ="trend", lags=1))
adf1
adf1 <- summary(ur.df(issues_ts[, 3], type ="trend", lags=1))
adf1
adf1 <- summary(ur.df(issues_ts[, 2], type ="none", lags=1))
adf1
adf1 <- summary(ur.df(issues_ts[, 2], type ="trend", lags=1))
adf1
summary(ur.df(issues_ts[, 2], type ="none", lags=1))
VARselect(issues_ts[2:44], lag.max = 8, type = "both")
VARselect(issues_ts[1:44], lag.max = 8, type = "both")
VARselect(issues[1:44], lag.max = 8, type = "both")
VARselect(issues[2:44], lag.max = 8, type = "both")
VARselect(issues_ts[2:44], lag.max = 8, type = "both")
VARselect(issues[2:44], lag.max = 8, type = "none")
VARselect(issues[2:44], lag.max = 8, type = "trend")
VARselect(issues[2:44], lag.max = 8, type = "const")
VARselect(issues[2:44], lag.max = 8, type = "both")
test <- VAR(issues[,2:44], p=1, type="both")
# VAR(issues_s[,2:23], p=1, type=c("const", "trend", "both", "none"), season=NULL, exogen = issues_i[2:22])
plot(irf(test, impulse = names(issues_s[2:23]), response = names(issues_i[2:22])))
summary(ur.df(issues_ts[, 2], type ="both", lags=1))
summary(ur.df(issues_ts[, 2], type ="none", lags=1))
test <- VAR(issues_ts[,2:44], p=1, type="both")
plot(irf(test, impulse = names(issues_s[2:23]), response = names(issues_i[2:22])))
acc_df <- read.csv("MdB-twitter.csv")
delrow <- NULL
for(r in 1:nrow(acc_df)) {
acc <- as.character(acc_df$twitter_acc[r])
if(!nzchar(acc)) {
delrow <- c(delrow, r)
}
}
acc_df <- acc_df[-delrow, ]
rm(delrow, r, acc)
acc_df$row.names <- NULL
row.names(acc_df) <- NULL
View(acc_df)