2015-02-24 22:03:47 +01:00
|
|
|
require(stringr)
|
|
|
|
require(reshape2)
|
|
|
|
require(ggplot2)
|
|
|
|
require(vars)
|
|
|
|
|
|
|
|
# Create dataframes with only non-sensational (i) and sensational (s) issue columns
|
|
|
|
drop_s <- which(str_detect(names(issues), "^s"))
|
|
|
|
drop_i <- which(str_detect(names(issues), "^i"))
|
|
|
|
issues_i <- issues[,-drop_s]
|
2015-07-07 15:34:44 +02:00
|
|
|
issues <- issues[,-drop_i]
|
2015-02-24 22:03:47 +01:00
|
|
|
|
|
|
|
# #
|
|
|
|
# ENTROPY
|
|
|
|
# #
|
|
|
|
# Entropy non-sensational issues
|
|
|
|
issues_i$total <- rowSums(issues_i[2:ncol(issues_i)])
|
|
|
|
issues_i$entropy <- 0
|
|
|
|
for(r in 1:nrow(issues_i)) {
|
|
|
|
curtotal <- as.numeric(issues_i$total[r])
|
|
|
|
curp <- 0
|
|
|
|
for(c in 2:ncol(issues_i)) {
|
|
|
|
curcount <- as.numeric(issues_i[r,c])
|
|
|
|
curp[c] <- curcount / curtotal
|
|
|
|
}
|
|
|
|
curp <- curp [2:length(curp)-2]
|
|
|
|
curdrop <- which(curp==0)
|
|
|
|
curp <- curp[-curdrop]
|
|
|
|
issues_i$entropy[r] <- sum(-1 * curp * log(curp))
|
|
|
|
}
|
|
|
|
|
|
|
|
# Entropy sensational issues
|
2015-07-07 15:34:44 +02:00
|
|
|
issues$total <- rowSums(issues[2:ncol(issues)])
|
|
|
|
issues$entropy <- 0
|
|
|
|
for(r in 1:nrow(issues)) {
|
|
|
|
curtotal <- as.numeric(issues$total[r])
|
|
|
|
curp <- 0
|
|
|
|
for(c in 2:ncol(issues)) {
|
|
|
|
curcount <- as.numeric(issues[r,c])
|
|
|
|
curp[c] <- curcount / curtotal
|
|
|
|
}
|
|
|
|
curp <- curp [2:length(curp)-2]
|
|
|
|
curdrop <- which(curp==0)
|
|
|
|
curp <- curp[-curdrop]
|
|
|
|
issues$entropy[r] <- sum(-1 * curp * log(curp))
|
|
|
|
}
|
2015-02-24 22:03:47 +01:00
|
|
|
|
2015-07-07 15:34:44 +02:00
|
|
|
# Entropy overall
|
|
|
|
issues$total <- rowSums(issues[2:ncol(issues)])
|
|
|
|
issues$entropy <- 0
|
|
|
|
for(r in 1:nrow(issues)) {
|
|
|
|
curtotal <- as.numeric(issues$total[r])
|
2015-02-24 22:03:47 +01:00
|
|
|
curp <- 0
|
2015-07-07 15:34:44 +02:00
|
|
|
for(c in 2:ncol(issues)) {
|
|
|
|
curcount <- as.numeric(issues[r,c])
|
2015-02-24 22:03:47 +01:00
|
|
|
curp[c] <- curcount / curtotal
|
|
|
|
}
|
|
|
|
curp <- curp [2:length(curp)-2]
|
|
|
|
curdrop <- which(curp==0)
|
|
|
|
curp <- curp[-curdrop]
|
2015-07-07 15:34:44 +02:00
|
|
|
issues$entropy[r] <- sum(-1 * curp * log(curp))
|
2015-02-24 22:03:47 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2015-07-07 15:34:44 +02:00
|
|
|
|
|
|
|
# Compare total tweets vs. total sensational & total unsensational
|
2015-02-24 22:03:47 +01:00
|
|
|
stats_total <- data.frame(date=drange)
|
|
|
|
stats_total$tpd <- 0
|
|
|
|
stats_total$ipd <- issues_i$total
|
2015-07-07 15:34:44 +02:00
|
|
|
stats_total$spd <- issues$total
|
2015-02-24 22:03:47 +01:00
|
|
|
# Total number of tweets per day over time
|
|
|
|
for(r in 1:length(drange)) {
|
|
|
|
stats_total$tpd[r] <- length(tweets[tweets[, "created_at"] == drange[r], "id_str"])
|
|
|
|
}
|
|
|
|
|
2015-07-07 15:34:44 +02:00
|
|
|
# VISUALS: Tweets per day vs. sensational vs. general findings
|
2015-02-24 22:03:47 +01:00
|
|
|
stats_melt <- melt(stats_total, id="date")
|
2015-07-07 15:34:44 +02:00
|
|
|
g_perday <- ggplot(data = stats_melt, aes(x=date,y=value,colour=variable, group=variable)) +
|
2015-02-24 22:03:47 +01:00
|
|
|
geom_line()+
|
2015-07-07 15:34:44 +02:00
|
|
|
geom_smooth(size=1,formula = y ~ x, method="loess", se=FALSE, color=1) +
|
|
|
|
xlab("Zeitraum") + ylab("Tweets pro Tag") +
|
|
|
|
scale_colour_discrete(name = "Tweets", labels = c("Gesamt", "Allgemein", "Sensation")) +
|
|
|
|
theme(legend.title = element_text(size=14)) +
|
|
|
|
theme(legend.text = element_text(size=12)) +
|
|
|
|
theme(axis.title = element_text(size = 14))
|
|
|
|
g_perday
|
2015-02-24 22:03:47 +01:00
|
|
|
|
|
|
|
# Visuals for entropy in time series
|
|
|
|
stats_entropy <- data.frame(date=drange)
|
2015-07-07 15:34:44 +02:00
|
|
|
stats_entropy$entropy <- issues$entropy
|
2015-02-24 22:03:47 +01:00
|
|
|
|
|
|
|
stats_entropy <- melt(stats_entropy, id="date")
|
|
|
|
|
2015-07-07 15:34:44 +02:00
|
|
|
g_entrop <- ggplot(data = stats_entropy, aes(x=date,y=value,colour=variable, group=variable)) +
|
2015-02-24 22:03:47 +01:00
|
|
|
geom_line() +
|
2015-07-07 15:34:44 +02:00
|
|
|
geom_smooth(size=1,formula = y ~ x, method="loess", se=FALSE, color=1) +
|
|
|
|
xlab("Zeitraum") + ylab("Entropie") +
|
|
|
|
scale_colour_discrete(name = "", labels = "Entropie") +
|
|
|
|
theme(legend.title = element_text(size=14)) +
|
|
|
|
theme(legend.text = element_text(size=12)) +
|
|
|
|
theme(axis.title = element_text(size = 14))
|
|
|
|
g_entrop
|
2015-02-24 22:03:47 +01:00
|
|
|
|
2015-02-26 01:27:40 +01:00
|
|
|
|
|
|
|
|
|
|
|
# VAR ---------------------------------------------------------------------
|
|
|
|
|
2015-03-06 12:40:22 +01:00
|
|
|
# test <- VAR(issues[,2:32], p=1, type=c("const", "trend", "both", "none"), season=NULL, exogen = NULL, lag.max = NULL, ic = c("AIC", "HQ", "SC", "FPE"))
|
2015-07-07 15:34:44 +02:00
|
|
|
# test <- VAR(issues_i[,2:22], p=1, type="none", exogen = issues[,2:3])
|
|
|
|
# test <- VAR(issues[,2:11], p=1, type="none")
|
|
|
|
# VAR(issues[,2:23], p=1, type=c("const", "trend", "both", "none"), season=NULL, exogen = issues_i[2:22])
|
2015-02-26 01:27:40 +01:00
|
|
|
|
2015-03-15 10:26:07 +01:00
|
|
|
issues_ts <- as.ts(issues[,2:44])
|
2015-07-07 15:34:44 +02:00
|
|
|
|
|
|
|
# Tests
|
2015-03-15 10:26:07 +01:00
|
|
|
VARselect(issues_ts, lag.max = 5, type = "both")
|
2015-07-07 15:34:44 +02:00
|
|
|
i <- 0
|
|
|
|
i <- i + 1
|
|
|
|
ur.df(issues_ts[, i], type ="none", lags=1)
|
|
|
|
summary(issues[2:44])
|
2015-03-13 15:11:49 +01:00
|
|
|
|
2015-03-04 11:36:31 +01:00
|
|
|
|
2015-07-07 15:34:44 +02:00
|
|
|
# VAR and IRF
|
|
|
|
vIssues <- VAR(issues_ts, p=1, type="both")
|
|
|
|
vIRF <- irf(vIssues, impulse = names(issues[2:23]), response = names(issues_i[2:22]))
|
|
|
|
plot(vIRF)
|
2015-02-26 01:27:40 +01:00
|
|
|
|
2015-07-07 15:34:44 +02:00
|
|
|
# capture.output(print(summary(test), prmsd=TRUE, digits=1), file="out.txt")
|
2015-02-26 01:27:40 +01:00
|
|
|
|
2015-02-24 22:03:47 +01:00
|
|
|
# SOME TESTS --------------------------------------------------------------
|
|
|
|
|
|
|
|
stats <- data.frame(date=drange)
|
|
|
|
stats$tpd <- 0
|
|
|
|
|
|
|
|
# Total number of tweets per day over time
|
|
|
|
for(r in 1:length(drange)) {
|
|
|
|
stats$tpd[r] <- length(tweets[tweets[, "created_at"] == drange[r], "id_str"])
|
|
|
|
}
|
|
|
|
|
|
|
|
stats_melt <- melt(stats, id="date")
|
|
|
|
g1 <- ggplot(data = stats_melt, aes(x=date,y=value,colour=variable, group=variable)) +
|
|
|
|
geom_line() +
|
|
|
|
geom_smooth(size=1,formula = y ~ x, method="loess", se=FALSE, color=1)
|
|
|
|
g1
|
|
|
|
|
|
|
|
rm(g1, r)
|
|
|
|
|
|
|
|
|
|
|
|
# Show party percentage of twitter users
|
2015-07-07 15:34:44 +02:00
|
|
|
acc_parties <- data.frame(party = c("cducsu", "spd", "gruene", "linke"))
|
|
|
|
acc_parties$btw13 <- c(49.3, 30.6, 10.0, 10.1) # seats of party / 631 seats
|
2015-02-24 22:03:47 +01:00
|
|
|
acc_parties$twitter <- 0
|
|
|
|
for(p in 1:nrow(acc_parties)) {
|
|
|
|
acc_parties$twitter[p] <- round(nrow(acc_df[acc_df$party == as.character(acc_parties$party[p]), ]) / 280 * 100)
|
|
|
|
}
|
2015-07-07 15:34:44 +02:00
|
|
|
pie(acc_parties$btw13, col=c("black", "red", "green", "purple"),
|
|
|
|
labels = c("CDU/CSU (49,3%)", "SPD (30,6%)", "Bündnis 90/Grüne(10,0%)", "Die LINKE (10,1%)"),
|
|
|
|
clockwise = T)
|
|
|
|
pie(acc_parties$twitter, col=c("black", "red", "green", "purple"),
|
|
|
|
labels = c("CDU/CSU (36%)", "SPD (30%)", "Bündnis 90/Grüne(19%)", "Die LINKE (15%)"),
|
|
|
|
clockwise = T)
|
2015-02-24 22:03:47 +01:00
|
|
|
|
|
|
|
rm(acc_parties, p)
|
|
|
|
|
|
|
|
|
2015-03-13 15:11:49 +01:00
|
|
|
# Count all tags
|
|
|
|
num <- 0
|
|
|
|
for(i in 1:length(issuelist)) {
|
|
|
|
j <- length(issuelist[[i]])
|
|
|
|
num <- num + j
|
|
|
|
rm(j)
|
|
|
|
}
|
|
|
|
num
|
|
|
|
|
|
|
|
|
2015-02-24 22:03:47 +01:00
|
|
|
# VISUALS -----------------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
|
|
# Level: days
|
|
|
|
issues_melt <- melt(issues,id="date")
|
|
|
|
ggplot(issues_melt,aes(x=date,y=value,colour=variable,group=variable)) + geom_line(size=1)
|
|
|
|
ggplot(issues_melt,aes(x=date,y=value,colour=variable,group=variable)) + geom_smooth(size=1,method="loess",formula = y ~ x, se=FALSE)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# POSSIBLY USEFUL CODE ----------------------------------------------------
|
|
|
|
|
2015-07-07 15:34:44 +02:00
|
|
|
# List all issues in one row
|
|
|
|
for(i in 1:length(issueheads)) {
|
|
|
|
cat(issueheads[i], "\n")
|
|
|
|
}
|
|
|
|
|
2015-02-24 22:03:47 +01:00
|
|
|
# Limits of list
|
|
|
|
length(issuelist)
|
|
|
|
length(issuelist[[2]])
|
|
|
|
|
|
|
|
# Select all tweets from current day in drange
|
|
|
|
tweets_curday <- tweets[tweets[, "created_at"] == drange[5], ]
|
|
|
|
# Is column a issue counting column?
|
|
|
|
str_detect(names(issues[2]), "^issue")
|