curdistance <- 1 } # Match current tweet with tag. If >= 5 letters allow 1 changed letter, if >=8 letters allow also 1 (Levenshtein distance) tags_found <- NULL # Match the tweet with each variation of tagexpand for(e in 1:length(curtag)) { tags_found[e] <- smartPatternMatch(curtext, curtag[e], curdistance, curacro) } tags_found <- any(tags_found) tags_found curtag curtext curdistance test <- VAR(issues[,2:32], p=3, type=c("const", "trend", "both", "none"), season=NULL, exogen = NULL, lag.max = NULL, ic = c("AIC", "HQ", "SC", "FPE")) test test <- VAR(issues[,2:32], p=1, type="none") capture.output(print(summary(test), prmsd=TRUE, digits=1), file="out.txt") View(issues) test <- VAR(issues_i[,2:22], p=1, type="none", exogen = issues_s[,2]) test <- VAR(issues_i[,2:22], p=1, type="none", exogen = issues_s[,2:3]) capture.output(print(summary(test), prmsd=TRUE, digits=1), file="out.txt") irf(test) test <- VAR(issues_s[,2:11], p=1, type="none") irf(test) plot(irf(test)) test <- VAR(issues[,2:32], p=1, type="none") plot(irf(test, impulse = names(issues_s[2:11]), response = names(issues_i[2:22]))) plot(irf(test, impulse = names(issues_s[2:11]), response = names(issues_i[2:22]), n.ahead = 5)) require(stringr) require(XML) readYN <- function(question) { n <- readline(prompt=question) n <- as.character(n) return(n) } checkIssue <- function(string, issuelist) { status <- any(str_detect(string, issuelist)) return(status) } checkAllIssues <- function(string, issuelist) { status <- NULL for(i in 1:length(string)) { if(checkIssue(string[i], issuelist)) { status[i] <- TRUE } else { cat("Issue",string[i],"does not exist. Please try again.\n") status[i] <- FALSE } } return(status) } View(tweets) write.csv(tweets, file="tweets.csv") c_tweets <- read.csv("tweets.csv", colClasses="character") View(c_tweets) c_tweets$X <- NULL c_issues <- data.frame(date = drange) c_issuelist <- xmlToList("issues.xml") c_issueheads <- names(issuelist) c_issues[issueheads] <- 0 source("issuecomp-codingsample-function.R") rm(c_err, c_result, c_samid, c_samno,c_samtags,c_samissue,c_samtext,c_yn) c_errors <- read.csv("issuecomp-codingsample-error.csv", header = F, sep=",", colClasses="character") View(c_errors) names(c_errors) <- c("str_id", "code", "issue", "tags", "text") View(c_errors) for(r in 1:nrow(c_errors)) { c_errcode <- as.character(c_errors$code[r]) c_errissue <- as.character(c_errors$issue[r]) c_errtags <- as.character(c_errors$tags[r]) c_errtext <- as.character(c_errors$text[r]) c_errid <- as.character(c_errors$str_id[r]) cat("===============\n\n[TWEET]: ",c_errtext,"\n[ISSUES]: ", c_errtags, "\n", sep="") source("issuecomp-codingsample-function2.R") } c_errors <- read.csv("issuecomp-codingsample-error.csv", header = F, sep=",", colClasses="character") names(c_errors) <- c("str_id", "code", "issue", "tags", "text") for(r in 1:nrow(c_errors)) { c_errcode <- as.character(c_errors$code[r]) c_errissue <- as.character(c_errors$issue[r]) c_errtags <- as.character(c_errors$tags[r]) c_errtext <- as.character(c_errors$text[r]) c_errid <- as.character(c_errors$str_id[r]) cat("===============\n\n[TWEET]: ",c_errtext,"\n[ISSUES]: ", c_errtags, "\n", sep="") source("issuecomp-codingsample-function2.R") } for(r in 1:nrow(c_errors)) { c_errcode <- as.character(c_errors$code[r]) c_errissue <- as.character(c_errors$issue[r]) c_errtags <- as.character(c_errors$tags[r]) c_errtext <- as.character(c_errors$text[r]) c_errid <- as.character(c_errors$str_id[r]) cat("===============\n\n[TWEET]: ",c_errtext,"\n[ISSUES]: ", c_errissue, " (", c_errtags, ")\n", sep="") source("issuecomp-codingsample-function2.R") } for(r in 1:nrow(c_errors)) { c_errcode <- as.character(c_errors$code[r]) c_errissue <- as.character(c_errors$issue[r]) c_errtags <- as.character(c_errors$tags[r]) c_errtext <- as.character(c_errors$text[r]) c_errid <- as.character(c_errors$str_id[r]) cat("===============\n\n[TWEET]: ",c_errtext,"\n[ISSUES]: ", c_errissue, " (", c_errtags, ")\n", sep="") source("issuecomp-codingsample-function2.R") } tagexpand source("issuecomp-codingsample-function.R") source("issuecomp-codingsample-function.R") source("issuecomp-codingsample-function.R") c_errors <- read.csv("issuecomp-codingsample-error.csv", header = F, sep=",", colClasses="character") names(c_errors) <- c("str_id", "code", "issue", "tags", "text") for(r in 1:nrow(c_errors)) { c_errcode <- as.character(c_errors$code[r]) c_errissue <- as.character(c_errors$issue[r]) c_errtags <- as.character(c_errors$tags[r]) c_errtext <- as.character(c_errors$text[r]) c_errid <- as.character(c_errors$str_id[r]) cat("===============\n\n[TWEET]: ",c_errtext,"\n[ISSUES]: ", c_errissue, " (", c_errtags, ")\n", sep="") source("issuecomp-codingsample-function2.R") } for(r in 1:nrow(c_errors)) { c_errcode <- as.character(c_errors$code[r]) c_errissue <- as.character(c_errors$issue[r]) c_errtags <- as.character(c_errors$tags[r]) c_errtext <- as.character(c_errors$text[r]) c_errid <- as.character(c_errors$str_id[r]) cat("===============\n\n[TWEET]: ",c_errtext,"\n[ISSUES]: ", c_errissue, " (", c_errtags, ")\n", sep="") source("issuecomp-codingsample-function2.R") } for(r in 1:nrow(c_errors)) { c_errcode <- as.character(c_errors$code[r]) c_errissue <- as.character(c_errors$issue[r]) c_errtags <- as.character(c_errors$tags[r]) c_errtext <- as.character(c_errors$text[r]) c_errid <- as.character(c_errors$str_id[r]) cat("===============\n\n[TWEET]: ",c_errtext,"\n[ISSUES]: ", c_errissue, " (", c_errtags, ")\n", sep="") source("issuecomp-codingsample-function2.R") } for(r in 1:nrow(c_errors)) { c_errcode <- as.character(c_errors$code[r]) c_errissue <- as.character(c_errors$issue[r]) c_errtags <- as.character(c_errors$tags[r]) c_errtext <- as.character(c_errors$text[r]) c_errid <- as.character(c_errors$str_id[r]) cat("===============\n\n[TWEET]: ",c_errtext,"\n[ISSUES]: ", c_errissue, " (", c_errtags, ")\n", sep="") source("issuecomp-codingsample-function2.R") } for(r in 1:nrow(c_errors)) { c_errcode <- as.character(c_errors$code[r]) c_errissue <- as.character(c_errors$issue[r]) c_errtags <- as.character(c_errors$tags[r]) c_errtext <- as.character(c_errors$text[r]) c_errid <- as.character(c_errors$str_id[r]) cat("===============\n\n[TWEET]: ",c_errtext,"\n[ISSUES]: ", c_errissue, " (", c_errtags, ")\n", sep="") source("issuecomp-codingsample-function2.R") } c_tmp <- read.csv("issuecomp-codingsample-error1.csv", header = F, colClasses="character") View(c_tmp) View(c_errors) View(c_tmp) names(c_tmp) <- c("str_id", "all", "wrong", "tags", "text") View(c_tmp) c_tmp[, c("wrong", "tagged", "all", "text")] View(c_tmp) names(c_tmp) <- c("str_id", "all", "wrong", "tagged", "text") c_tmp[, c("wrong", "tagged", "all", "text")] c_error1 <- c_tmp[, c("wrong", "tagged", "all", "text")] View(c_error1) c_tmp <- read.csv("issuecomp-codingsample-error2.csv", header = F, colClasses="character") View(c_tmp) c_tmp <- read.csv("issuecomp-codingsample-error2.csv", header = F, colClasses="character") names(c_tmp) <- c("str_id", "all", "missing", "tagged", "text") c_error1 <- c_tmp[, c("missing", "tagged", "all", "text")] c_error2 <- c_tmp[, c("missing", "tagged", "all", "text")] View(c_error2) c_error2 <- c_tmp[, c("missing", "text", "tagged", "all")] View(c_error2) View(c_error1) View(c_error2) c_tmp <- read.csv("issuecomp-codingsample-correct.csv", header = F, colClasses="character") View(c_tmp) names(c_tmp) <- c("str_id", "status", "issue", "tags", "text") View(c_tmp) c_currect <- c_tmp c_correct <- c_tmp rm(c_currect) View(c_correct) source("issuecomp-codingsample-function.R") rm(c_err, c_result, c_samid, c_samno,c_samtags,c_samissue,c_samtext,c_yn) c_errors <- read.csv("issuecomp-codingsample-error.csv", header = F, sep=",", colClasses="character") c_errors <- read.csv("issuecomp-codingsample-error.csv", header = F, sep=",", colClasses="character") c_errors <- read.csv("issuecomp-codingsample-error.csv", header = F, sep=",", colClasses="character") c_errors <- read.csv("issuecomp-codingsample-error.csv", header = F, sep=",", colClasses="character") c_errors <- read.csv("issuecomp-codingsample-error.csv", header = F, sep=",", colClasses="character", quote = "") View(c_errors) c_errors <- read.csv("issuecomp-codingsample-error.csv", header = F, sep=",", colClasses="character") test <- "Zitat "total dämlich!"" tweets$id_str == "523512815425175552" tweets[tweets$id_str == "523512815425175552"] tweets[tweets$id_str == "523512815425175552", ] tweets[tweets$id_str == "523512815425175552", "text"] test <- tweets[tweets$id_str == "523512815425175552", "text"] test test <- c_tweets[ctweets$id_str == "523512815425175552", "text"] test <- c_tweets[c_tweets$id_str == "523512815425175552", "text"] test str_replace(test, "\\"", ") str_replace(test, "\\"", "") str_replace(test, "\"", "") str_detect(test, "\"") test <- as.character(c_tweets[c_tweets$id_str == "523512815425175552", "text"]) test c_tweets <- read.csv("tweets.csv", colClasses="character") for(r in 1:nrow(c_tweets)) { curtext <- as.character(c_tweets$text[r]) if(str_detect(curtext, "\"") { c_tweets$text[r] <- str_replace(curtext, "\"", "") } } for(r in 1:nrow(c_tweets)) { curtext <- as.character(c_tweets$text[r]) if(str_detect(curtext, "\"") { c_tweets$text[r] <- str_replace(curtext, "\"", "") } else {} } for(r in 1:nrow(c_tweets)) { curtext <- as.character(c_tweets$text[r]) if(str_detect(curtext, "\"") { c_tweets$text[r] <- str_replace(curtext, "\"", "") } else { } } for(r in 1:nrow(c_tweets)) { curtext <- as.character(c_tweets$text[r]) if(str_detect(curtext, "\"")) { c_tweets$text[r] <- str_replace(curtext, "\"", "") } } test <- as.character(c_tweets[c_tweets$id_str == "523512815425175552", "text"]) test View(c_tweets) c_errors <- read.csv("issuecomp-codingsample-error.csv", header = F, sep=",", colClasses="character") View(c_errors) names(c_errors) <- c("str_id", "code", "issue", "tags", "text") View(c_errors) for(r in 1:nrow(c_errors)) { c_errcode <- as.character(c_errors$code[r]) c_errissue <- as.character(c_errors$issue[r]) c_errtags <- as.character(c_errors$tags[r]) c_errtext <- as.character(c_errors$text[r]) c_errid <- as.character(c_errors$str_id[r]) cat("===============\n\n[TWEET]: ",c_errtext,"\n[ISSUES]: ", c_errissue, " (", c_errtags, ")\n", sep="") source("issuecomp-codingsample-function2.R") } issueheads for(r in 1:nrow(c_errors)) { c_errcode <- as.character(c_errors$code[r]) c_errissue <- as.character(c_errors$issue[r]) c_errtags <- as.character(c_errors$tags[r]) c_errtext <- as.character(c_errors$text[r]) c_errid <- as.character(c_errors$str_id[r]) cat("===============\n\n[TWEET]: ",c_errtext,"\n[ISSUES]: ", c_errissue, " (", c_errtags, ")\n", sep="") source("issuecomp-codingsample-function2.R") } # All tweets with WRONG ISSUES c_tmp <- read.csv("issuecomp-codingsample-error1.csv", header = F, colClasses="character") names(c_tmp) <- c("str_id", "all", "wrong", "tagged", "text") c_error1 <- c_tmp[, c("wrong", "tagged", "all", "text")] # All tweets with MISSING ISSUES c_tmp <- read.csv("issuecomp-codingsample-error2.csv", header = F, colClasses="character") names(c_tmp) <- c("str_id", "all", "missing", "tagged", "text") c_error2 <- c_tmp[, c("missing", "text", "tagged", "all")] # All CORRECT tweets c_tmp <- read.csv("issuecomp-codingsample-correct.csv", header = F, colClasses="character") names(c_tmp) <- c("str_id", "status", "issue", "tags", "text") c_correct <- c_tmp View(c_error1) View(c_error2) View(c_error1) View(c_correct) test <- VAR(issues_i[,2:22], p=1, type="none", exogen = issues_s[,2:3]) plot(irf(test, impulse = names(issues_s[2:11]), response = names(issues_i[2:22]))) test <- VAR(issues[,2:32], p=1, type="none") plot(irf(test, impulse = names(issues_s[2:11]), response = names(issues_i[2:22]))) VARselect(issues[,2:32], lag.max=8, type="none") VARselect(issues[,2:32], lag.max=8, type="both") VARselect(issues[,2:32], lag.max=30, type="both") VARselect(issues[,2:32], lag.max=15, type="both") for(i in 1:20) { cat(i,"\n") Sys.sleep(10)} for(i in 1:20) { cat(i,"\n")Sys.sleep(10)} for(i in 1:20) { cat(i,"\n")Sys.sleep(10)} for(i in 1:20) { cat(i,"\n") Sys.sleep(10)} list.dirs() list.files() rm(results) setwd("matched-ids/") results_files <- list.files() results_files results_files <- "all.csv" for(r in 1:length(results_files)) { if(r == 1) { results <- read.csv(results_files[r], sep=";", colClasses=c("character", "character", "character", "character"), header=F) } else { results_temp <- read.csv(results_files[r], sep=";", colClasses=c("character", "character", "character", "character"), header=F) results <- insertRow(results, results_temp) } } rm(r, results_temp, results_files) results <- results[!duplicated(results), ] names(results) <- c("date", "id_str", "issue", "tags") results <- results[order(results$id_str), ] row.names(results) <- NULL results[23381,] results[53381,] results[43253,] for(r in 53371:nrow(results)) { curdate <- as.character(results$date[r]) curid <- as.character(results$id_str[r]) curissue <- as.character(results$issue[r]) curtag <- as.character(results$tags[r]) cat("Sorting match", r, "of 53383 \n") # Update issue counter (date and issue) issues[issues[, "date"] == curdate, curissue] <- issues[issues[, "date"] == curdate, curissue] + 1 # Update tweet dataframe (id, issue and tags) oldissue <- tweets[tweets[, "id_str"] == curid, "issue"] tweets[tweets[, "id_str"] == curid, "issue"] <- str_c(oldissue, curissue, ",") oldtag <- tweets[tweets[, "id_str"] == curid, "tags"] tweets[tweets[, "id_str"] == curid, "tags"] <- str_c(oldtag, curtag, ",") } issues[issueheads] <- 0 View(issues) for(r in 1:nrow(results)) { curdate <- as.character(results$date[r]) curid <- as.character(results$id_str[r]) curissue <- as.character(results$issue[r]) curtag <- as.character(results$tags[r]) cat("Sorting match", r, "of 53383 \n") # Update issue counter (date and issue) issues[issues[, "date"] == curdate, curissue] <- issues[issues[, "date"] == curdate, curissue] + 1 # Update tweet dataframe (id, issue and tags) oldissue <- tweets[tweets[, "id_str"] == curid, "issue"] tweets[tweets[, "id_str"] == curid, "issue"] <- str_c(oldissue, curissue, ",") oldtag <- tweets[tweets[, "id_str"] == curid, "tags"] tweets[tweets[, "id_str"] == curid, "tags"] <- str_c(oldtag, curtag, ",") } require(lubridate) require(XML) require(ggplot2) require(reshape2) require(stringr) require(foreach) require(doParallel) for(r in 1:nrow(results)) { curdate <- as.character(results$date[r]) curid <- as.character(results$id_str[r]) curissue <- as.character(results$issue[r]) curtag <- as.character(results$tags[r]) cat("Sorting match", r, "of 53383 \n") # Update issue counter (date and issue) issues[issues[, "date"] == curdate, curissue] <- issues[issues[, "date"] == curdate, curissue] + 1 # Update tweet dataframe (id, issue and tags) oldissue <- tweets[tweets[, "id_str"] == curid, "issue"] tweets[tweets[, "id_str"] == curid, "issue"] <- str_c(oldissue, curissue, ",") oldtag <- tweets[tweets[, "id_str"] == curid, "tags"] tweets[tweets[, "id_str"] == curid, "tags"] <- str_c(oldtag, curtag, ",") } results[119,] results[120,] load(file = "tweets_untagged.RData") setwd("~/Dokumente/Uni/Aktuell/BA-Arbeit/uni-ba-issuecomp") results_files <- "matched-ids/all.csv" load(file = "tweets_untagged.RData") View(issues) issues <- data.frame(date = drange) issuelist <- readLines("issues.xml") issuelist <- str_replace_all(string = issuelist, pattern = ".*", "") issuelist <- xmlToList(issuelist) issueheads <- names(issuelist) issues[issueheads] <- 0 tweets$issue <- "" tweets$tags <- "" View(results) rm(r, results_temp, results_files) results <- results[!duplicated(results), ] names(results) <- c("date", "id_str", "issue", "tags") results <- results[order(results$id_str), ] row.names(results) <- NULL for(r in 1:nrow(results)) { curdate <- as.character(results$date[r]) curid <- as.character(results$id_str[r]) curissue <- as.character(results$issue[r]) curtag <- as.character(results$tags[r]) cat("Sorting match", r, "of 53383 \n") # Update issue counter (date and issue) issues[issues[, "date"] == curdate, curissue] <- issues[issues[, "date"] == curdate, curissue] + 1 # Update tweet dataframe (id, issue and tags) oldissue <- tweets[tweets[, "id_str"] == curid, "issue"] tweets[tweets[, "id_str"] == curid, "issue"] <- str_c(oldissue, curissue, ",") oldtag <- tweets[tweets[, "id_str"] == curid, "tags"] tweets[tweets[, "id_str"] == curid, "tags"] <- str_c(oldtag, curtag, ",") } curdate curissue issues[issues[, "date"] == curdate, curissue] issueheads issuelist <- readLines("issues-v2.xml") issues <- data.frame(date = drange) issuelist <- readLines("issues-v2.xml") issuelist <- str_replace_all(string = issuelist, pattern = ".*", "") issuelist <- xmlToList(issuelist) issueheads <- names(issuelist) issues[issueheads] <- 0 tweets$issue <- "" tweets$tags <- "" for(r in 1:nrow(results)) { curdate <- as.character(results$date[r]) curid <- as.character(results$id_str[r]) curissue <- as.character(results$issue[r]) curtag <- as.character(results$tags[r]) cat("Sorting match", r, "of 53383 \n") # Update issue counter (date and issue) issues[issues[, "date"] == curdate, curissue] <- issues[issues[, "date"] == curdate, curissue] + 1 # Update tweet dataframe (id, issue and tags) oldissue <- tweets[tweets[, "id_str"] == curid, "issue"] tweets[tweets[, "id_str"] == curid, "issue"] <- str_c(oldissue, curissue, ",") oldtag <- tweets[tweets[, "id_str"] == curid, "tags"] tweets[tweets[, "id_str"] == curid, "tags"] <- str_c(oldtag, curtag, ",") } results[33170,] results[33171,] results$date[33170] results$date[33170] <- "2014-08-21" for(r in 33170:nrow(results)) { curdate <- as.character(results$date[r]) curid <- as.character(results$id_str[r]) curissue <- as.character(results$issue[r]) curtag <- as.character(results$tags[r]) cat("Sorting match", r, "of 53383 \n") # Update issue counter (date and issue) issues[issues[, "date"] == curdate, curissue] <- issues[issues[, "date"] == curdate, curissue] + 1 # Update tweet dataframe (id, issue and tags) oldissue <- tweets[tweets[, "id_str"] == curid, "issue"] tweets[tweets[, "id_str"] == curid, "issue"] <- str_c(oldissue, curissue, ",") oldtag <- tweets[tweets[, "id_str"] == curid, "tags"] tweets[tweets[, "id_str"] == curid, "tags"] <- str_c(oldtag, curtag, ",") } save(tweets, file="tweets_tagged.RData") write.csv(tweets, file="tweets.csv") save(issues, file="issues.RData") require(stringr) require(reshape2) require(ggplot2) require(vars) drop_s <- which(str_detect(names(issues), "^s")) drop_i <- which(str_detect(names(issues), "^i")) issues_i <- issues[,-drop_s] issues_s <- issues[,-drop_i] issues_i$total <- rowSums(issues_i[2:ncol(issues_i)]) issues_i$entropy <- 0 for(r in 1:nrow(issues_i)) { curtotal <- as.numeric(issues_i$total[r]) curp <- 0 for(c in 2:ncol(issues_i)) { curcount <- as.numeric(issues_i[r,c]) curp[c] <- curcount / curtotal } curp <- curp [2:length(curp)-2] curdrop <- which(curp==0) curp <- curp[-curdrop] issues_i$entropy[r] <- sum(-1 * curp * log(curp)) } issues_s$total <- rowSums(issues_s[2:ncol(issues_s)]) issues_s$entropy <- 0 for(r in 1:nrow(issues_s)) { curtotal <- as.numeric(issues_s$total[r]) curp <- 0 for(c in 2:ncol(issues_s)) { curcount <- as.numeric(issues_s[r,c]) curp[c] <- curcount / curtotal } curp <- curp [2:length(curp)-2] curdrop <- which(curp==0) curp <- curp[-curdrop] issues_s$entropy[r] <- sum(-1 * curp * log(curp)) } stats_total <- data.frame(date=drange) stats_total$tpd <- 0 stats_total$ipd <- issues_i$total stats_total$spd <- issues_s$total # Total number of tweets per day over time for(r in 1:length(drange)) { stats_total$tpd[r] <- length(tweets[tweets[, "created_at"] == drange[r], "id_str"]) } g1 <- ggplot(data = stats_melt, aes(x=date,y=value,colour=variable, group=variable)) + geom_line()+ geom_smooth(size=1,formula = y ~ x, method="loess", se=FALSE, color=1) g1 stats_entropy <- data.frame(date=drange) stats_entropy$entropy <- issues_i$entropy stats_entropy <- melt(stats_entropy, id="date") g1 <- ggplot(data = stats_entropy, aes(x=date,y=value,colour=variable, group=variable)) + geom_line() + geom_smooth(size=1,formula = y ~ x, method="loess", se=FALSE, color=1) g1 test <- VAR(issues[,2:32], p=1, type="none") View(issues_i) View(issues_s) View(issues) test <- VAR(issues[,2:44], p=1, type="none") VAR(issues_s[,2:23], p=1, type=c("const", "trend", "both", "none"), season=NULL, exogen = issues_i[2:22]) plot(irf(test, impulse = names(issues_s[2:23]), response = names(issues_i[2:22])))