diff --git a/.Rhistory b/.Rhistory
index a3941ed..1707561 100644
--- a/.Rhistory
+++ b/.Rhistory
@@ -1,224 +1,3 @@
-c_errcode <- as.character(c_errors$code[r])
-c_errissue <- as.character(c_errors$issue[r])
-c_errtags <- as.character(c_errors$tags[r])
-c_errtext <- as.character(c_errors$text[r])
-c_errid <- as.character(c_errors$str_id[r])
-cat("===============\n\n[TWEET]: ",c_errtext,"\n[ISSUES]: ", c_errissue, " (", c_errtags, ")\n", sep="")
-source("issuecomp-codingsample-function2.R")
-}
-c_tmp <- read.csv("issuecomp-codingsample-error1.csv", header = F, colClasses="character")
-View(c_tmp)
-View(c_errors)
-View(c_tmp)
-names(c_tmp) <- c("str_id", "all", "wrong", "tags", "text")
-View(c_tmp)
-c_tmp[, c("wrong", "tagged", "all", "text")]
-View(c_tmp)
-names(c_tmp) <- c("str_id", "all", "wrong", "tagged", "text")
-c_tmp[, c("wrong", "tagged", "all", "text")]
-c_error1 <- c_tmp[, c("wrong", "tagged", "all", "text")]
-View(c_error1)
-c_tmp <- read.csv("issuecomp-codingsample-error2.csv", header = F, colClasses="character")
-View(c_tmp)
-c_tmp <- read.csv("issuecomp-codingsample-error2.csv", header = F, colClasses="character")
-names(c_tmp) <- c("str_id", "all", "missing", "tagged", "text")
-c_error1 <- c_tmp[, c("missing", "tagged", "all", "text")]
-c_error2 <- c_tmp[, c("missing", "tagged", "all", "text")]
-View(c_error2)
-c_error2 <- c_tmp[, c("missing", "text", "tagged", "all")]
-View(c_error2)
-View(c_error1)
-View(c_error2)
-c_tmp <- read.csv("issuecomp-codingsample-correct.csv", header = F, colClasses="character")
-View(c_tmp)
-names(c_tmp) <- c("str_id", "status", "issue", "tags", "text")
-View(c_tmp)
-c_currect <- c_tmp
-c_correct <- c_tmp
-rm(c_currect)
-View(c_correct)
-source("issuecomp-codingsample-function.R")
-rm(c_err, c_result, c_samid, c_samno,c_samtags,c_samissue,c_samtext,c_yn)
-c_errors <- read.csv("issuecomp-codingsample-error.csv", header = F, sep=",", colClasses="character")
-c_errors <- read.csv("issuecomp-codingsample-error.csv", header = F, sep=",", colClasses="character")
-c_errors <- read.csv("issuecomp-codingsample-error.csv", header = F, sep=",", colClasses="character")
-c_errors <- read.csv("issuecomp-codingsample-error.csv", header = F, sep=",", colClasses="character")
-c_errors <- read.csv("issuecomp-codingsample-error.csv", header = F, sep=",", colClasses="character", quote = "")
-View(c_errors)
-c_errors <- read.csv("issuecomp-codingsample-error.csv", header = F, sep=",", colClasses="character")
-test <- "Zitat "total dämlich!""
-tweets$id_str == "523512815425175552"
-tweets[tweets$id_str == "523512815425175552"]
-tweets[tweets$id_str == "523512815425175552", ]
-tweets[tweets$id_str == "523512815425175552", "text"]
-test <- tweets[tweets$id_str == "523512815425175552", "text"]
-test
-test <- c_tweets[ctweets$id_str == "523512815425175552", "text"]
-test <- c_tweets[c_tweets$id_str == "523512815425175552", "text"]
-test
-str_replace(test, "\\"", ")
-str_replace(test, "\\"", "")
-str_replace(test, "\"", "")
-str_detect(test, "\"")
-test <- as.character(c_tweets[c_tweets$id_str == "523512815425175552", "text"])
-test
-c_tweets <- read.csv("tweets.csv", colClasses="character")
-for(r in 1:nrow(c_tweets)) {
-curtext <- as.character(c_tweets$text[r])
-if(str_detect(curtext, "\"") {
-c_tweets$text[r] <- str_replace(curtext, "\"", "")
-}
-}
-for(r in 1:nrow(c_tweets)) {
-curtext <- as.character(c_tweets$text[r])
-if(str_detect(curtext, "\"") {
-c_tweets$text[r] <- str_replace(curtext, "\"", "")
-} else {}
-}
-for(r in 1:nrow(c_tweets)) {
-curtext <- as.character(c_tweets$text[r])
-if(str_detect(curtext, "\"") {
-c_tweets$text[r] <- str_replace(curtext, "\"", "")
-} else {
-}
-}
-for(r in 1:nrow(c_tweets)) {
-curtext <- as.character(c_tweets$text[r])
-if(str_detect(curtext, "\"")) {
-c_tweets$text[r] <- str_replace(curtext, "\"", "")
-}
-}
-test <- as.character(c_tweets[c_tweets$id_str == "523512815425175552", "text"])
-test
-View(c_tweets)
-c_errors <- read.csv("issuecomp-codingsample-error.csv", header = F, sep=",", colClasses="character")
-View(c_errors)
-names(c_errors) <- c("str_id", "code", "issue", "tags", "text")
-View(c_errors)
-for(r in 1:nrow(c_errors)) {
-c_errcode <- as.character(c_errors$code[r])
-c_errissue <- as.character(c_errors$issue[r])
-c_errtags <- as.character(c_errors$tags[r])
-c_errtext <- as.character(c_errors$text[r])
-c_errid <- as.character(c_errors$str_id[r])
-cat("===============\n\n[TWEET]: ",c_errtext,"\n[ISSUES]: ", c_errissue, " (", c_errtags, ")\n", sep="")
-source("issuecomp-codingsample-function2.R")
-}
-issueheads
-for(r in 1:nrow(c_errors)) {
-c_errcode <- as.character(c_errors$code[r])
-c_errissue <- as.character(c_errors$issue[r])
-c_errtags <- as.character(c_errors$tags[r])
-c_errtext <- as.character(c_errors$text[r])
-c_errid <- as.character(c_errors$str_id[r])
-cat("===============\n\n[TWEET]: ",c_errtext,"\n[ISSUES]: ", c_errissue, " (", c_errtags, ")\n", sep="")
-source("issuecomp-codingsample-function2.R")
-}
-# All tweets with WRONG ISSUES
-c_tmp <- read.csv("issuecomp-codingsample-error1.csv", header = F, colClasses="character")
-names(c_tmp) <- c("str_id", "all", "wrong", "tagged", "text")
-c_error1 <- c_tmp[, c("wrong", "tagged", "all", "text")]
-# All tweets with MISSING ISSUES
-c_tmp <- read.csv("issuecomp-codingsample-error2.csv", header = F, colClasses="character")
-names(c_tmp) <- c("str_id", "all", "missing", "tagged", "text")
-c_error2 <- c_tmp[, c("missing", "text", "tagged", "all")]
-# All CORRECT tweets
-c_tmp <- read.csv("issuecomp-codingsample-correct.csv", header = F, colClasses="character")
-names(c_tmp) <- c("str_id", "status", "issue", "tags", "text")
-c_correct <- c_tmp
-View(c_error1)
-View(c_error2)
-View(c_error1)
-View(c_correct)
-test <- VAR(issues_i[,2:22], p=1, type="none", exogen = issues_s[,2:3])
-plot(irf(test, impulse = names(issues_s[2:11]), response = names(issues_i[2:22])))
-test <- VAR(issues[,2:32], p=1, type="none")
-plot(irf(test, impulse = names(issues_s[2:11]), response = names(issues_i[2:22])))
-VARselect(issues[,2:32], lag.max=8, type="none")
-VARselect(issues[,2:32], lag.max=8, type="both")
-VARselect(issues[,2:32], lag.max=30, type="both")
-VARselect(issues[,2:32], lag.max=15, type="both")
-for(i in 1:20) { cat(i,"\n") Sys.sleep(10)}
-for(i in 1:20) { cat(i,"\n")Sys.sleep(10)}
-for(i in 1:20) { cat(i,"\n")Sys.sleep(10)}
-for(i in 1:20) { cat(i,"\n")
-Sys.sleep(10)}
-list.dirs()
-list.files()
-rm(results)
-setwd("matched-ids/")
-results_files <- list.files()
-results_files
-results_files <- "all.csv"
-for(r in 1:length(results_files)) {
-if(r == 1) {
-results <- read.csv(results_files[r], sep=";", colClasses=c("character", "character", "character", "character"), header=F)
-} else {
-results_temp <- read.csv(results_files[r], sep=";", colClasses=c("character", "character", "character", "character"), header=F)
-results <- insertRow(results, results_temp)
-}
-}
-rm(r, results_temp, results_files)
-results <- results[!duplicated(results), ]
-names(results) <- c("date", "id_str", "issue", "tags")
-results <- results[order(results$id_str), ]
-row.names(results) <- NULL
-results[23381,]
-results[53381,]
-results[43253,]
-for(r in 53371:nrow(results)) {
-curdate <- as.character(results$date[r])
-curid <- as.character(results$id_str[r])
-curissue <- as.character(results$issue[r])
-curtag <- as.character(results$tags[r])
-cat("Sorting match", r, "of 53383 \n")
-# Update issue counter (date and issue)
-issues[issues[, "date"] == curdate, curissue] <- issues[issues[, "date"] == curdate, curissue] + 1
-# Update tweet dataframe (id, issue and tags)
-oldissue <- tweets[tweets[, "id_str"] == curid, "issue"]
-tweets[tweets[, "id_str"] == curid, "issue"] <- str_c(oldissue, curissue, ",")
-oldtag <- tweets[tweets[, "id_str"] == curid, "tags"]
-tweets[tweets[, "id_str"] == curid, "tags"] <- str_c(oldtag, curtag, ",")
-}
-issues[issueheads] <- 0
-View(issues)
-for(r in 1:nrow(results)) {
-curdate <- as.character(results$date[r])
-curid <- as.character(results$id_str[r])
-curissue <- as.character(results$issue[r])
-curtag <- as.character(results$tags[r])
-cat("Sorting match", r, "of 53383 \n")
-# Update issue counter (date and issue)
-issues[issues[, "date"] == curdate, curissue] <- issues[issues[, "date"] == curdate, curissue] + 1
-# Update tweet dataframe (id, issue and tags)
-oldissue <- tweets[tweets[, "id_str"] == curid, "issue"]
-tweets[tweets[, "id_str"] == curid, "issue"] <- str_c(oldissue, curissue, ",")
-oldtag <- tweets[tweets[, "id_str"] == curid, "tags"]
-tweets[tweets[, "id_str"] == curid, "tags"] <- str_c(oldtag, curtag, ",")
-}
-require(lubridate)
-require(XML)
-require(ggplot2)
-require(reshape2)
-require(stringr)
-require(foreach)
-require(doParallel)
-for(r in 1:nrow(results)) {
-curdate <- as.character(results$date[r])
-curid <- as.character(results$id_str[r])
-curissue <- as.character(results$issue[r])
-curtag <- as.character(results$tags[r])
-cat("Sorting match", r, "of 53383 \n")
-# Update issue counter (date and issue)
-issues[issues[, "date"] == curdate, curissue] <- issues[issues[, "date"] == curdate, curissue] + 1
-# Update tweet dataframe (id, issue and tags)
-oldissue <- tweets[tweets[, "id_str"] == curid, "issue"]
-tweets[tweets[, "id_str"] == curid, "issue"] <- str_c(oldissue, curissue, ",")
-oldtag <- tweets[tweets[, "id_str"] == curid, "tags"]
-tweets[tweets[, "id_str"] == curid, "tags"] <- str_c(oldtag, curtag, ",")
-}
-results[119,]
-results[120,]
 load(file = "tweets_untagged.RData")
 setwd("~/Dokumente/Uni/Aktuell/BA-Arbeit/uni-ba-issuecomp")
 results_files <- "matched-ids/all.csv"
@@ -510,3 +289,224 @@ rm(delrow, r, acc)
 acc_df$row.names <- NULL
 row.names(acc_df) <- NULL
 View(acc_df)
+View(c_tweets)
+issueheads
+length(issueheads)
+issuelist
+length(issuelist)
+length(issuelist[*])
+length(issuelist[[*]])
+length(issuelist[1:43])
+length(issuelist[1)
+length(issuelist[1])
+length(issuelist[2])
+length(issuelist[[1]])
+length(issuelist[[2]])
+length(issuelist[[70]])
+length(issuelist[[43]])
+length(issuelist[[44]])
+length(issuelist[[1:43]])
+length(issuelist[[1-43]])
+length(issuelist[[2]])
+test <- 0
+num <- 0
+for(i in 1:length(issuelist)) {
+j <- length(issuelist[[i]])
+num <- num + j
+rm(j)
+}
+num
+drop_s <- which(str_detect(names(issues), "^s"))
+drop_i <- which(str_detect(names(issues), "^i"))
+issues_i <- issues[,-drop_s]
+issues_s <- issues[,-drop_i]
+require(stringr)
+drop_s <- which(str_detect(names(issues), "^s"))
+drop_i <- which(str_detect(names(issues), "^i"))
+issues_i <- issues[,-drop_s]
+issues_s <- issues[,-drop_i]
+issues_i$total <- rowSums(issues_i[2:ncol(issues_i)])
+issues_i$entropy <- 0
+for(r in 1:nrow(issues_i)) {
+curtotal <- as.numeric(issues_i$total[r])
+curp <- 0
+for(c in 2:ncol(issues_i)) {
+curcount <- as.numeric(issues_i[r,c])
+curp[c] <- curcount / curtotal
+}
+curp <- curp [2:length(curp)-2]
+curdrop <- which(curp==0)
+curp <- curp[-curdrop]
+issues_i$entropy[r] <- sum(-1 * curp * log(curp))
+}
+issues_s$total <- rowSums(issues_s[2:ncol(issues_s)])
+issues_s$entropy <- 0
+for(r in 1:nrow(issues_s)) {
+curtotal <- as.numeric(issues_s$total[r])
+curp <- 0
+for(c in 2:ncol(issues_s)) {
+curcount <- as.numeric(issues_s[r,c])
+curp[c] <- curcount / curtotal
+}
+curp <- curp [2:length(curp)-2]
+curdrop <- which(curp==0)
+curp <- curp[-curdrop]
+issues_s$entropy[r] <- sum(-1 * curp * log(curp))
+}
+stats_total <- data.frame(date=drange)
+stats_total$tpd <- 0
+stats_total$ipd <- issues_i$total
+stats_total$spd <- issues_s$total
+# Total number of tweets per day over time
+for(r in 1:length(drange)) {
+stats_total$tpd[r] <- length(tweets[tweets[, "created_at"] == drange[r], "id_str"])
+}
+stats_melt <- melt(stats_total, id="date")
+g1 <- ggplot(data = stats_melt, aes(x=date,y=value,colour=variable, group=variable)) +
+geom_line()+
+geom_smooth(size=1,formula = y ~ x, method="loess", se=FALSE, color=1)
+g1
+require(ggplot2)
+stats_melt <- melt(stats_total, id="date")
+g1 <- ggplot(data = stats_melt, aes(x=date,y=value,colour=variable, group=variable)) +
+geom_line()+
+geom_smooth(size=1,formula = y ~ x, method="loess", se=FALSE, color=1)
+g1
+g1 <- ggplot(data = stats_melt, aes(x=date,y=value,colour=variable, group=variable)) +
+geom_line()+
+geom_smooth(size=1,formula = y ~ x, method="lm", se=FALSE, color=1)
+g1
+g1 <- ggplot(data = stats_melt, aes(x=date,y=value,colour=variable, group=variable)) +
+geom_line()+
+geom_smooth(size=1,formula = y ~ x, method="loess", se=FALSE, color=1)
+g1
+# Visuals for entropy in time series
+stats_entropy <- data.frame(date=drange)
+stats_entropy$entropy <- issues_i$entropy
+stats_entropy <- melt(stats_entropy, id="date")
+require(reshape2)
+stats_melt <- melt(stats_total, id="date")
+g1 <- ggplot(data = stats_melt, aes(x=date,y=value,colour=variable, group=variable)) +
+geom_line()+
+geom_smooth(size=1,formula = y ~ x, method="loess", se=FALSE, color=1)
+g1
+stats_entropy <- data.frame(date=drange)
+stats_entropy$entropy <- issues_i$entropy
+stats_entropy <- melt(stats_entropy, id="date")
+g1 <- ggplot(data = stats_entropy, aes(x=date,y=value,colour=variable, group=variable)) +
+geom_line() +
+geom_smooth(size=1,formula = y ~ x, method="loess", se=FALSE, color=1)
+g1
+g1 <- ggplot(data = stats_entropy, aes(x=date,y=value,colour=variable, group=variable)) +
+geom_line() +
+geom_smooth(size=1,formula = y ~ x, method="lm", se=FALSE, color=1)
+g1
+vIssues <- VAR(issues_ts[,2:44], p=1, type="both")
+require(vars)
+VARselect(issues_ts, lag.max = 8, type = "both")
+vIssues <- VAR(issues_ts[,2:44], p=1, type="both")
+VARselect(issues_ts, lag.max = 16, type = "both")
+VARselect(issues_ts, lag.max = 4, type = "both")
+VARselect(issues_ts, lag.max = 5, type = "both")
+VARselect(issues_ts, lag.max = 6, type = "both")
+VARselect(issues_ts, lag.max = 5, type = "both")
+names(issues_ts)
+issues_ts
+issues_ts[2:44]
+issues_ts <- as.ts(issues[,2:44])
+issues_ts[1:1]
+issues_ts[,1]
+issues_ts[1,1]
+issues_ts2,1]
+issues_ts[2,1]
+issues_ts <- as.ts(issues[,2:44])
+VARselect(issues_ts, lag.max = 5, type = "both")
+VARselect(issues_ts, lag.max = 8, type = "both")
+VARselect(issues_ts, lag.max = 7, type = "both")
+VARselect(issues_ts, lag.max = 5, type = "both")
+vIssues <- VAR(issues_ts[,2:44], p=5, type="both")
+vIssues <- VAR(issues_ts, p=5, type="both")
+plot(irf(vIssues, impulse = names(issues_s[2:23]), response = names(issues_i[2:22])))
+require(stringr)
+require(XML)
+c_errors <- read.csv("issuecomp-codingsample-error.csv", header = F, sep=",", colClasses="character")
+names(c_errors) <- c("str_id", "code", "issue", "tags", "text")
+for(r in 1:nrow(c_errors)) {
+c_errcode <- as.character(c_errors$code[r])
+c_errissue <- as.character(c_errors$issue[r])
+c_errtags <- as.character(c_errors$tags[r])
+c_errtext <- as.character(c_errors$text[r])
+c_errid <- as.character(c_errors$str_id[r])
+cat("===============\n\n[TWEET]: ",c_errtext,"\n[ISSUES]: ", c_errissue, " (", c_errtags, ")\n", sep="")
+source("issuecomp-codingsample-function2.R")
+}
+for(r in 1:nrow(c_errors)) {
+c_errcode <- as.character(c_errors$code[r])
+c_errissue <- as.character(c_errors$issue[r])
+c_errtags <- as.character(c_errors$tags[r])
+c_errtext <- as.character(c_errors$text[r])
+c_errid <- as.character(c_errors$str_id[r])
+cat("===============\n\n[TWEET]: ",c_errtext,"\n[ISSUES]: ", c_errissue, " (", c_errtags, ")\n", sep="")
+source("issuecomp-codingsample-function2.R")
+}
+issueheads
+for(i in 1:length(issueheads)) {paste(issueheads[i])}
+for(i in 1:length(issueheads)) {cat(issueheads[i], "\n")}
+for(r in 1:nrow(c_errors)) {
+c_errcode <- as.character(c_errors$code[r])
+c_errissue <- as.character(c_errors$issue[r])
+c_errtags <- as.character(c_errors$tags[r])
+c_errtext <- as.character(c_errors$text[r])
+c_errid <- as.character(c_errors$str_id[r])
+cat("===============\n\n[TWEET]: ",c_errtext,"\n[ISSUES]: ", c_errissue, " (", c_errtags, ")\n", sep="")
+source("issuecomp-codingsample-function2.R")
+}
+for(r in 1:nrow(c_errors)) {
+c_errcode <- as.character(c_errors$code[r])
+c_errissue <- as.character(c_errors$issue[r])
+c_errtags <- as.character(c_errors$tags[r])
+c_errtext <- as.character(c_errors$text[r])
+c_errid <- as.character(c_errors$str_id[r])
+cat("===============\n\n[TWEET]: ",c_errtext,"\n[ISSUES]: ", c_errissue, " (", c_errtags, ")\n", sep="")
+source("issuecomp-codingsample-function2.R")
+}
+c_errors <- read.csv("issuecomp-codingsample-error.csv", header = F, sep=",", colClasses="character")
+names(c_errors) <- c("str_id", "code", "issue", "tags", "text")
+for(r in 1:nrow(c_errors)) {
+c_errcode <- as.character(c_errors$code[r])
+c_errissue <- as.character(c_errors$issue[r])
+c_errtags <- as.character(c_errors$tags[r])
+c_errtext <- as.character(c_errors$text[r])
+c_errid <- as.character(c_errors$str_id[r])
+cat("===============\n\n[TWEET]: ",c_errtext,"\n[ISSUES]: ", c_errissue, " (", c_errtags, ")\n", sep="")
+source("issuecomp-codingsample-function2.R")
+}
+c_tmp <- read.csv("issuecomp-codingsample-error1.csv", header = F, colClasses="character")
+names(c_tmp) <- c("str_id", "all", "wrong", "tagged", "text")
+c_error1 <- c_tmp[, c("wrong", "tagged", "all", "text")]
+c_tmp <- read.csv("issuecomp-codingsample-error2.csv", header = F, colClasses="character")
+names(c_tmp) <- c("str_id", "all", "missing", "tagged", "text")
+c_error2 <- c_tmp[, c("missing", "text", "tagged", "all")]
+View(c_error2)
+summary(ur.df(issues_ts[, 2], type ="none", lags=1))
+require(vars)
+summary(ur.df(issues_ts[, 2], type ="none", lags=1))
+stability(vIssues)
+stability(vIssues[2:])
+stability(vIssues[2:44])
+plot(stability(vIssues))
+class(vIssues)
+summary(vIssues)
+plot(stability(vIssues[2]))
+plot(stability(vIssues), nc=2)
+plot(stability(vIssues), h=0.15)
+stability(vIssues)
+efp(formula = formula, data = data, type = type, h = h, dynamic = dynamic,
+rescale = rescale)
+plot(stability(vIssues), h=0.15)
+plot(stability(vIssues, h=0.15))
+plot(stability(vIssues, h=0.15, rescale = TRUE))
+plot(stability(vIssues, h=0.15, rescale = TRUE), nc=2)
+par("mar")
+par(mar=c(1,1,1,1))
+plot(stability(vIssues, h=0.15, rescale = TRUE), nc=2)
diff --git a/issuecomp-2-analysis.R b/issuecomp-2-analysis.R
index e79b864..92c228a 100644
--- a/issuecomp-2-analysis.R
+++ b/issuecomp-2-analysis.R
@@ -147,9 +147,9 @@ stopCluster(cl)
 # IMPORT RESULTS ----------------------------------------------------------
 
 # Import all files which have been generated at the categorisation run above.
-#setwd("matched-ids/")
-#results_files <- list.files()
-results_files <- "matched-ids/all.csv"
+
+results_files <- list.files("matched-ids/", full.names = T)
+
 for(r in 1:length(results_files)) {
   if(r == 1) {
     results <- read.csv(results_files[r], sep=";", colClasses=c("character", "character", "character", "character"), header=F)
@@ -172,13 +172,14 @@ row.names(results) <- NULL
 # Reset issues counter
 #issues[issueheads] <- 0
 
-for(r in 33170:nrow(results)) {
+nrow_results <- nrow(results)
+for(r in 1:nrow_results) {
   curdate <- as.character(results$date[r])
   curid <- as.character(results$id_str[r])
   curissue <- as.character(results$issue[r])
   curtag <- as.character(results$tags[r])
   
-  cat("Sorting match", r, "of 53383 \n")
+  cat("Sorting match", r, "of", nrow_results, "\n")
   
   # Update issue counter (date and issue)
   issues[issues[, "date"] == curdate, curissue] <- issues[issues[, "date"] == curdate, curissue] + 1
diff --git a/issuecomp-3-calc.R b/issuecomp-3-calc.R
index e450b18..55b4cea 100644
--- a/issuecomp-3-calc.R
+++ b/issuecomp-3-calc.R
@@ -83,10 +83,11 @@ g1
 # test <- VAR(issues_s[,2:11], p=1, type="none")
 # VAR(issues_s[,2:23], p=1, type=c("const", "trend", "both", "none"), season=NULL, exogen = issues_i[2:22])
 
-issues_ts <- as.ts(issues)
-vIssues <- VAR(issues_ts[,2:44], p=1, type="both")
+issues_ts <- as.ts(issues[,2:44])
+VARselect(issues_ts, lag.max = 5, type = "both")
+vIssues <- VAR(issues_ts, p=5, type="both")
 
-plot(irf(test, impulse = names(issues_s[2:23]), response = names(issues_i[2:22])))
+plot(irf(vIssues, impulse = names(issues_s[2:23]), response = names(issues_i[2:22])))
 
 capture.output(print(summary(test), prmsd=TRUE, digits=1), file="out.txt")
 
diff --git a/issues-expand.xml b/issues-expand.xml
index 35f8ab8..8b13789 100644
--- a/issues-expand.xml
+++ b/issues-expand.xml
@@ -1,11 +1 @@
-<s.ukraine>
-  <tag>#Janukowitsch</tag>
-</s.ukraine>
 
-<i2.civil>
-  <tag>Foltermethode</tag>
-</i2.civil>
-
-<i19.ib>
-  --<tag>Afghanistan</tag>
-</i19.ib>
diff --git a/issues-v3.xml b/issues-v3.xml
index 63f8b76..b09a62d 100644
--- a/issues-v3.xml
+++ b/issues-v3.xml
@@ -46,6 +46,17 @@
 </i1.macro>
 
 <i2.civil>
+  <tag>Foltermethode</tag>
+  <tag>Immigrant</tag>
+  <tag>Menschengerichtshof</tag>
+  <tag>Einwander</tag>
+  <tag>Faschismus</tag>
+  <tag>Faschist</tag>
+  <tag>Antisemitismus</tag>
+  <tag>Antisemitist</tag>
+  <tag>Nationalismus</tag>
+  <tag>Nationalist</tag>
+  <tag>Flüchtlingspolitik</tag>
   <tag>Migration</tag>
   <tag>Migrant</tag>
   <tag>Flüchtlingsstrom</tag>
@@ -67,6 +78,7 @@
   <tag>Rassismus</tag>
   <tag>rassistisch</tag>
   <tag>Rechtsextremismus</tag>
+  <tag>Nazi</tag>
   <tag>Nazis</tag>
 
   <tag>Wahlrecht</tag>
@@ -397,6 +409,11 @@
 </i6.edu>
 
 <i7.env>
+  <tag>Energiestrategie</tag>
+  <tag>#EEG</tag>
+  <tag>bundnaturschutz</tag>
+  <tag>BUND</tag>
+  <tag>Klimakonferenz</tag>
   <tag>Energiewende</tag>
   <tag>Klimaschutz</tag>
   <tag>Klimagipfel</tag>
@@ -607,6 +624,8 @@
 </i8.energy>
 
 <i10.trans>
+  <tag>Deutsche Bahn</tag>
+  <tag>#GDL</tag>
   <tag>LKWs</tag> 
   <tag>PKWs</tag>
   
@@ -741,6 +760,12 @@
 </i10.trans>
 
 <i12.law>
+  <tag>Euthanasie</tag>
+  <tag>Familienarbeitszeit</tag>
+  <tag>Waffenarsenal</tag>
+  <tag>NSU</tag>
+  <tag>Crystal Meth</tag>
+  <tag>Ecstacy</tag>
   <tag>Vorratsdatenspeicherung</tag>
   <tag>VDS</tag>
   <tag>Cybercrime</tag>
@@ -904,15 +929,14 @@
   <tag>Erdbeben</tag> 
   <tag>Frühwarnsystem</tag> 
   <!-- 1227 -->
-  <tag>Terrorismus</tag> 
   <tag>Personalausweis</tag> 
   <tag>Ausweis</tag> 
-  <tag>Terrorist</tag> 
   <!-- 1299 -->
   <tag>Opferentschädigung</tag> 
 </i12.law>
 
 <i13.social>
+  <tag>Pflegezeit</tag>
   <!-- 1300 --> 
   <tag>Sozialpolitik</tag>
   <tag>Pflegeversicherung</tag>
@@ -982,6 +1006,8 @@
 </i13.social>
 
 <i14.house>
+  <tag>preiswert Wohnen</tag>
+  <tag>preiswertes Wohnen</tag>
   <!-- 1400 --> 
   <tag>Wohnungswesen</tag>
   <tag>Raumungordnung</tag>
@@ -1157,6 +1183,7 @@
   <tag>Reiseversicherung</tag>
   <!-- 1525 --> 
   <tag>Verbaucherschutz</tag>
+  <tag>Verbraucherinteressen</tag>
   <tag>Verbaucherbetrug</tag>
   <tag>Werbebetrug</tag>
   <tag>Verbraucherschutzministerium</tag>
@@ -1189,6 +1216,8 @@
 
 <i16.defense>
   <tag>Auschwitz</tag>
+  <tag>2 Weltkrieg</tag>
+  <tag>zweiter Weltkrieg</tag>
 
   <tag>Rüstungsbudget</tag>
   <tag>Rüstungsausgaben</tag>
@@ -1352,6 +1381,7 @@
 
 <i17.science>
   <tag>Sicherheitslücke</tag>
+  <tag>Internetsteuer</tag>
 
   <!-- 1700 --> 
   <tag>Weltraumforschung</tag>
@@ -1494,9 +1524,12 @@
 </i18.trade>
 
 <i19.ib>
+  <tag>#EU</tag>
+  <tag>Ungarn</tag>
   <tag>Außenpolitik</tag>
   <tag>außenpolitisch</tag>
   <tag>menschenrechtsbetont</tag>
+  <tag>Türkei</tag>
   
   <!-- 1900 --> 
   <tag>internationale Beziehungen</tag>
@@ -1619,7 +1652,6 @@
   <tag>Japan</tag>
   <tag>Südostasien</tag>
   <tag>Indien</tag>
-  <tag>Afghanistan</tag>
   <tag>China</tag>
   <tag>chinesisch</tag>
   <tag>Taiwan</tag>
@@ -1644,7 +1676,6 @@
   <tag>religiöse Verfolgung</tag>
   <tag>Verbrechen gegen die Menschheit</tag>
   <tag>Verbrechen gegen die Menschlichkeit</tag>
-  <tag>Folter</tag>
   <tag>Kindersoldat</tag>
   <tag>Menschenrechtskonvention</tag>
   <!-- 1926 --> 
@@ -1824,6 +1855,8 @@
 </s.nsa>
 
 <s.is>
+  <tag>Islamischer Staat</tag>
+  <tag>Islamischen Staates</tag>
   <tag>ISIS</tag>
   <tag>IS</tag>
   <tag>al Baghdadi</tag>
@@ -1858,6 +1891,8 @@
   <tag>Donetsk</tag>
   <tag>Donezk</tag>
   <tag>Euromaidan</tag>
+  <tag>#Maidan</tag>
+  <tag>#Janukowitsch</tag>
 </s.ukraine>
 
 <s.hk>
@@ -1891,6 +1926,7 @@
 </s.ferguson>
 
 <s.boko>
+  <tag>200 entführte Mädchen</tag>
   <tag>Boko Haram</tag>
 </s.boko>
 
@@ -1970,6 +2006,8 @@
   <tag>Brasilien</tag>
   <tag>#WorldCup</tag>
   <tag>#WM2014</tag>
+  <tag>#GERALG</tag>
+  <tag>#ALGGER</tag>
 </s.wm>
 
 <s.sotschi>
diff --git a/issues.RData b/issues.RData
index 51f2d93..f1fa548 100644
Binary files a/issues.RData and b/issues.RData differ
diff --git a/matched-ids-v3.zip b/matched-ids-v3.zip
new file mode 100644
index 0000000..ff0230a
Binary files /dev/null and b/matched-ids-v3.zip differ
diff --git a/tweets_tagged.RData b/tweets_tagged.RData
index 47c5d28..c63c8d5 100644
Binary files a/tweets_tagged.RData and b/tweets_tagged.RData differ