Browse Source

splitting files, adding entropy test

mxmehl 5 years ago
parent
commit
a32447a27b
4 changed files with 144 additions and 68 deletions
  1. 0
    0
      issuecomp-1-scraping.R
  2. 14
    68
      issuecomp-2-analysis.R
  3. 130
    0
      issuecomp-3-calc.R
  4. BIN
      issues.RData

issuecomp-scraping.R → issuecomp-1-scraping.R View File


issuecomp-analysis.R → issuecomp-2-analysis.R View File

@@ -55,7 +55,7 @@ foreach(d = 1:nrow(issues), .packages = c("stringr"), .combine=rbind) %dopar% {
55 55
   tweets_curday <- tweets[tweets[, "created_at"] == curdate, ]
56 56
   
57 57
   for(t in 1:nrow(tweets_curday)){
58
-#     cat(paste("Starting tweet", t, "of",as.character(curdate),"\n"), file="issuecomp-analysis.log", append=TRUE)
58
+    #     cat(paste("Starting tweet", t, "of",as.character(curdate),"\n"), file="issuecomp-analysis.log", append=TRUE)
59 59
     # Select tweet's text, make it lowercase and remove hashtag indicators (#)
60 60
     curtext <- as.character(tweets_curday$text[t])
61 61
     curtext <- str_replace_all(curtext, "#", "")
@@ -67,7 +67,7 @@ foreach(d = 1:nrow(issues), .packages = c("stringr"), .combine=rbind) %dopar% {
67 67
       curissue <- issueheads[i]
68 68
       curtags <- as.character(issuelist[[curissue]])  
69 69
       curfile <- str_c(id_folder,"/",curissue,".csv")
70
-
70
+      
71 71
       # Now test all tags of a single issue
72 72
       for(s in 1:length(curtags)) {
73 73
         curtag <- curtags[s]
@@ -93,7 +93,7 @@ foreach(d = 1:nrow(issues), .packages = c("stringr"), .combine=rbind) %dopar% {
93 93
         } else {
94 94
           curdistance <- 1
95 95
         }
96
-                 
96
+        
97 97
         # Match current tweet with tag. If >= 5 letters allow 1 changed letter, if >=8 letters allow also 1 (Levenshtein distance)
98 98
         tags_found <- NULL
99 99
         # Match the tweet with each variation of tagexpand
@@ -104,19 +104,19 @@ foreach(d = 1:nrow(issues), .packages = c("stringr"), .combine=rbind) %dopar% {
104 104
         curtag <- curtag[1]
105 105
         
106 106
         if(tags_found == TRUE) {
107
-#           # Raise number of findings on this day for this issue by 1
108
-#           issues[d,curissue] <- issues[d,curissue] + 1
109
-#           
110
-#           # Add issue and first matched tag of tweet to tweets-DF
111
-#           oldissue <- tweets[tweets[, "id_str"] == curid, "issue"]
112
-#           tweets[tweets[, "id_str"] == curid, "issue"] <- str_c(oldissue, curissue, ";")
113
-#           oldtag <- tweets[tweets[, "id_str"] == curid, "tags"]
114
-#           tweets[tweets[, "id_str"] == curid, "tags"] <- str_c(oldtag, curtag, ";")
107
+          #           # Raise number of findings on this day for this issue by 1
108
+          #           issues[d,curissue] <- issues[d,curissue] + 1
109
+          #           
110
+          #           # Add issue and first matched tag of tweet to tweets-DF
111
+          #           oldissue <- tweets[tweets[, "id_str"] == curid, "issue"]
112
+          #           tweets[tweets[, "id_str"] == curid, "issue"] <- str_c(oldissue, curissue, ";")
113
+          #           oldtag <- tweets[tweets[, "id_str"] == curid, "tags"]
114
+          #           tweets[tweets[, "id_str"] == curid, "tags"] <- str_c(oldtag, curtag, ";")
115 115
           
116 116
           # Add information to file for function viewPatternMatching
117 117
           write(str_c(curdate,";\"",curid,"\";",curissue,";",curtag), curfile, append = TRUE)
118
-#           cat(paste("Match!\n"), file="issuecomp-analysis.log", append=TRUE)
119
-#           data.frame(date=curdate, issue=curissue)
118
+          #           cat(paste("Match!\n"), file="issuecomp-analysis.log", append=TRUE)
119
+          #           data.frame(date=curdate, issue=curissue)
120 120
           break   # next issue, no more tags from same issue
121 121
         }
122 122
         else {
@@ -182,60 +182,6 @@ for(r in 1:nrow(results)) {
182 182
 # SAVING ------------------------------------------------------------------
183 183
 
184 184
 save(tweets, file="tweets_tagged.RData")
185
+save(issues, file="issues.RData")
185 186
 
186 187
 
187
-
188
-# SOME TESTS --------------------------------------------------------------
189
-
190
-stats <- data.frame(date=drange)
191
-stats$tpd <- 0
192
-
193
-# Total number of tweets per day over time
194
-for(r in 1:length(drange)) {
195
-  stats$tpd[r] <- length(tweets[tweets[, "created_at"] == drange[r], "id_str"])
196
-}
197
-
198
-stats_melt <- melt(stats, id="date")
199
-g1 <- ggplot(data = stats_melt, aes(x=date,y=value,colour=variable, group=variable)) + 
200
-  geom_line() + 
201
-  geom_smooth(size=1,formula = y ~ x, method="loess", se=FALSE, color=1)
202
-g1
203
-
204
-rm(g1, r)
205
-
206
-
207
-# Show party percentage of twitter users
208
-acc_parties <- data.frame(party = c("cducsu", "spd", "linke", "gruene"))
209
-acc_parties$btw13 <- c(49.3, 30.6, 10.1, 10.0) # seats of party / 631 seats
210
-acc_parties$twitter <- 0
211
-for(p in 1:nrow(acc_parties)) {
212
-  acc_parties$twitter[p] <- round(nrow(acc_df[acc_df$party == as.character(acc_parties$party[p]), ]) / 280 * 100)
213
-}
214
-pie(acc_parties$btw13, col=c("black", "red", "purple", "green"), labels = c("CDU/CSU", "SPD", "Die LINKE", "Bündnis 90/Grüne"), clockwise = T,
215
-    main = "Seats of parties in the parliament")
216
-pie(acc_parties$twitter, col=c("black", "red", "purple", "green"), labels = c("CDU/CSU", "SPD", "Die LINKE", "Bündnis 90/Grüne"), clockwise = T,
217
-    main = "Percentage of parties' MdBs of all Twitter accounts")
218
-
219
-rm(acc_parties, p)
220
-
221
-
222
-# VISUALS -----------------------------------------------------------------
223
-
224
-
225
-# Level: days
226
-issues_melt <- melt(issues,id="date")
227
-ggplot(issues_melt,aes(x=date,y=value,colour=variable,group=variable)) + geom_line(size=1)
228
-ggplot(issues_melt,aes(x=date,y=value,colour=variable,group=variable)) + geom_smooth(size=1,method="loess",formula = y ~ x, se=FALSE)
229
-
230
-
231
-
232
-# POSSIBLY USEFUL CODE ----------------------------------------------------
233
-
234
-# Limits of list
235
-length(issuelist)
236
-length(issuelist[[2]])
237
-
238
-# Select all tweets from current day in drange
239
-tweets_curday <- tweets[tweets[, "created_at"] == drange[5], ]
240
-# Is column a issue counting column?
241
-str_detect(names(issues[2]), "^issue")

+ 130
- 0
issuecomp-3-calc.R View File

@@ -0,0 +1,130 @@
1
+require(stringr)
2
+require(reshape2)
3
+require(ggplot2)
4
+require(vars)
5
+
6
+# Create dataframes with only non-sensational (i) and sensational (s) issue columns
7
+drop_s <- which(str_detect(names(issues), "^s"))
8
+drop_i <- which(str_detect(names(issues), "^i"))
9
+issues_i <- issues[,-drop_s]
10
+issues_s <- issues[,-drop_i]
11
+
12
+# #
13
+# ENTROPY
14
+# #
15
+# Entropy non-sensational issues
16
+issues_i$total <- rowSums(issues_i[2:ncol(issues_i)])
17
+issues_i$entropy <- 0
18
+
19
+for(r in 1:nrow(issues_i)) {
20
+  curtotal <- as.numeric(issues_i$total[r])
21
+  curp <- 0
22
+  for(c in 2:ncol(issues_i)) {
23
+    curcount <- as.numeric(issues_i[r,c])
24
+    curp[c] <- curcount / curtotal
25
+  }
26
+  curp <- curp [2:length(curp)-2]
27
+  curdrop <- which(curp==0)
28
+  curp <- curp[-curdrop]
29
+  issues_i$entropy[r] <- sum(-1 * curp * log(curp))
30
+}
31
+
32
+# Entropy sensational issues
33
+issues_s$total <- rowSums(issues_s[2:ncol(issues_s)])
34
+issues_s$entropy <- 0
35
+
36
+for(r in 1:nrow(issues_s)) {
37
+  curtotal <- as.numeric(issues_s$total[r])
38
+  curp <- 0
39
+  for(c in 2:ncol(issues_s)) {
40
+    curcount <- as.numeric(issues_s[r,c])
41
+    curp[c] <- curcount / curtotal
42
+  }
43
+  curp <- curp [2:length(curp)-2]
44
+  curdrop <- which(curp==0)
45
+  curp <- curp[-curdrop]
46
+  issues_s$entropy[r] <- sum(-1 * curp * log(curp))
47
+}
48
+
49
+
50
+# Compare total tweets vs. total issue findings
51
+stats_total <- data.frame(date=drange)
52
+stats_total$tpd <- 0
53
+stats_total$ipd <- issues_i$total
54
+stats_total$spd <- issues_s$total
55
+# Total number of tweets per day over time
56
+for(r in 1:length(drange)) {
57
+  stats_total$tpd[r] <- length(tweets[tweets[, "created_at"] == drange[r], "id_str"])
58
+}
59
+
60
+stats_melt <- melt(stats_total, id="date")
61
+g1 <- ggplot(data = stats_melt, aes(x=date,y=value,colour=variable, group=variable)) + 
62
+  geom_line()+ 
63
+  geom_smooth(size=1,formula = y ~ x, method="loess", se=FALSE, color=1)
64
+g1
65
+
66
+# Visuals for entropy in time series
67
+stats_entropy <- data.frame(date=drange)
68
+stats_entropy$entropy <- issues_i$entropy
69
+
70
+stats_entropy <- melt(stats_entropy, id="date")
71
+
72
+g1 <- ggplot(data = stats_entropy, aes(x=date,y=value,colour=variable, group=variable)) + 
73
+  geom_line() + 
74
+  geom_smooth(size=1,formula = y ~ x, method="loess", se=FALSE, color=1)
75
+g1
76
+
77
+# SOME TESTS --------------------------------------------------------------
78
+
79
+stats <- data.frame(date=drange)
80
+stats$tpd <- 0
81
+
82
+# Total number of tweets per day over time
83
+for(r in 1:length(drange)) {
84
+  stats$tpd[r] <- length(tweets[tweets[, "created_at"] == drange[r], "id_str"])
85
+}
86
+
87
+stats_melt <- melt(stats, id="date")
88
+g1 <- ggplot(data = stats_melt, aes(x=date,y=value,colour=variable, group=variable)) + 
89
+  geom_line() + 
90
+  geom_smooth(size=1,formula = y ~ x, method="loess", se=FALSE, color=1)
91
+g1
92
+
93
+rm(g1, r)
94
+
95
+
96
+# Show party percentage of twitter users
97
+acc_parties <- data.frame(party = c("cducsu", "spd", "linke", "gruene"))
98
+acc_parties$btw13 <- c(49.3, 30.6, 10.1, 10.0) # seats of party / 631 seats
99
+acc_parties$twitter <- 0
100
+for(p in 1:nrow(acc_parties)) {
101
+  acc_parties$twitter[p] <- round(nrow(acc_df[acc_df$party == as.character(acc_parties$party[p]), ]) / 280 * 100)
102
+}
103
+pie(acc_parties$btw13, col=c("black", "red", "purple", "green"), labels = c("CDU/CSU", "SPD", "Die LINKE", "Bündnis 90/Grüne"), clockwise = T,
104
+    main = "Seats of parties in the parliament")
105
+pie(acc_parties$twitter, col=c("black", "red", "purple", "green"), labels = c("CDU/CSU", "SPD", "Die LINKE", "Bündnis 90/Grüne"), clockwise = T,
106
+    main = "Percentage of parties' MdBs of all Twitter accounts")
107
+
108
+rm(acc_parties, p)
109
+
110
+
111
+# VISUALS -----------------------------------------------------------------
112
+
113
+
114
+# Level: days
115
+issues_melt <- melt(issues,id="date")
116
+ggplot(issues_melt,aes(x=date,y=value,colour=variable,group=variable)) + geom_line(size=1)
117
+ggplot(issues_melt,aes(x=date,y=value,colour=variable,group=variable)) + geom_smooth(size=1,method="loess",formula = y ~ x, se=FALSE)
118
+
119
+
120
+
121
+# POSSIBLY USEFUL CODE ----------------------------------------------------
122
+
123
+# Limits of list
124
+length(issuelist)
125
+length(issuelist[[2]])
126
+
127
+# Select all tweets from current day in drange
128
+tweets_curday <- tweets[tweets[, "created_at"] == drange[5], ]
129
+# Is column a issue counting column?
130
+str_detect(names(issues[2]), "^issue")

BIN
issues.RData View File


Loading…
Cancel
Save