243 lines
8.7 KiB
R
243 lines
8.7 KiB
R
require(lubridate)
|
|
require(XML)
|
|
require(ggplot2)
|
|
require(reshape2)
|
|
require(stringr)
|
|
library(foreach)
|
|
library(doParallel)
|
|
source("issuecomp-functions.R")
|
|
getwd()
|
|
setwd("~/Dokumente/Uni/Aktuell/BA-Arbeit/uni-ba-issuecomp")
|
|
getwd()
|
|
list.files()
|
|
list.files("matched-ids/")
|
|
load(file = "tweets_untagged.RData")
|
|
issues <- data.frame(date = drange)
|
|
# Create date range
|
|
date_start <- as.Date("2014-01-01")
|
|
date_end <- as.Date("2014-12-31")
|
|
drange <- as.integer(date_end - date_start)
|
|
drange <- date_start + days(0:drange)
|
|
issues <- data.frame(date = drange)
|
|
issuelist <- readLines("issues.xml")
|
|
issuelist <- str_replace_all(string = issuelist, pattern = ".*<!-- .+ -->", "")
|
|
issuelist <- xmlToList(issuelist)
|
|
issueheads <- names(issuelist)
|
|
issues[issueheads] <- 0
|
|
tweets$issue <- ""
|
|
tweets$tags <- ""
|
|
View(issues)
|
|
list.files("matched-ids/")
|
|
results <- list.files("matched-ids/")
|
|
results
|
|
read.csv("matched-ids/i10.trans.csv")
|
|
read.csv("matched-ids/i10.trans.csv", sep=";")
|
|
read.csv("matched-ids/i10.trans.csv", sep=";", stringsAsFactors=F)
|
|
read.csv("matched-ids/i10.trans.csv", sep=";", stringsAsFactors=T)
|
|
reesult_files <- read.csv("matched-ids/i10.trans.csv", sep=";", stringsAsFactors=F)
|
|
View(reesult_files)
|
|
result_files <- read.csv("matched-ids/i10.trans.csv", sep=";", colClasses=c("date", "character", "character", "character"))
|
|
result_files <- read.csv("matched-ids/i10.trans.csv", sep=";", colClasses=c("character", "character", "character", "character"))
|
|
rm(reesult_files)
|
|
View(result_files)
|
|
nrow(result_files)
|
|
result_files <- result_files(!duplicated(result_files))
|
|
result_files <- result_files[!duplicated(result_files)]
|
|
result_files <- result_files[!duplicated(result_files), ]
|
|
nrow(result_files)
|
|
result_files <- read.csv("matched-ids/i10.trans.csv", sep=";", colClasses=c("character", "character", "character", "character"), header=F)
|
|
View(result_files)
|
|
read.results
|
|
results
|
|
setwd("matched-ids/")
|
|
list.files("")
|
|
getwd()
|
|
list.files()
|
|
results <- list.files()
|
|
results
|
|
results_cat <- read.csv(results, sep=";", colClasses=c("character", "character", "character", "character"), header=F)
|
|
results_cat <- read.csv(results[1], sep=";", colClasses=c("character", "character", "character", "character"), header=F)
|
|
results_cat
|
|
View(results_cat)
|
|
source("issuecomp-functions.R")
|
|
setwd("~/Dokumente/Uni/Aktuell/BA-Arbeit/uni-ba-issuecomp")
|
|
source("issuecomp-functions.R")
|
|
insertRow
|
|
results_temp <- read.csv(results[2], sep=";", colClasses=c("character", "character", "character", "character"), header=F)
|
|
setwd("matched-ids/")
|
|
results_temp <- read.csv(results[2], sep=";", colClasses=c("character", "character", "character", "character"), header=F)
|
|
rm(result_files)
|
|
insertRow(existingDF = results_cat, results_temp)
|
|
rm(results_cat)
|
|
for(r in 1:length(results)) {
|
|
if(r == 1) {
|
|
results_cat <- read.csv(results[r], sep=";", colClasses=c("character", "character", "character", "character"), header=F)
|
|
} else {
|
|
results_temp <- read.csv(results[r], sep=";", colClasses=c("character", "character", "character", "character"), header=F)
|
|
insertRow(results_cat, results_temp)
|
|
}
|
|
}
|
|
for(r in 1:length(results)) {
|
|
if(r == 1) {
|
|
results_cat <- read.csv(results[r], sep=";", colClasses=c("character", "character", "character", "character"), header=F)
|
|
} else {
|
|
results_temp <- read.csv(results[r], sep=";", colClasses=c("character", "character", "character", "character"), header=F)
|
|
results_cat insertRow(results_cat, results_temp)
|
|
}
|
|
}
|
|
for(r in 1:length(results)) {
|
|
if(r == 1) {
|
|
results_cat <- read.csv(results[r], sep=";", colClasses=c("character", "character", "character", "character"), header=F)
|
|
} else {
|
|
results_temp <- read.csv(results[r], sep=";", colClasses=c("character", "character", "character", "character"), header=F)
|
|
results_cat <- insertRow(results_cat, results_temp)
|
|
}
|
|
}
|
|
View(results_cat)
|
|
results_cat[20000]
|
|
results_cat[20000, ]
|
|
rm(r, results_temp)
|
|
results_cat <- results_cat[!duplicated(results_cat), ]
|
|
View(results_cat)
|
|
rm(results, results_cat)
|
|
results_files <- list.files()
|
|
for(r in 1:length(results)) {
|
|
if(r == 1) {
|
|
results <- read.csv(results_files[r], sep=";", colClasses=c("character", "character", "character", "character"), header=F)
|
|
} else {
|
|
results_temp <- read.csv(results_files[r], sep=";", colClasses=c("character", "character", "character", "character"), header=F)
|
|
results <- insertRow(results_cat, results_temp)
|
|
}
|
|
}
|
|
rm(r, results_temp)
|
|
results <- results[!duplicated(results), ]
|
|
results_files <- list.files()
|
|
for(r in 1:length(results_files)) {
|
|
if(r == 1) {
|
|
results <- read.csv(results_files[r], sep=";", colClasses=c("character", "character", "character", "character"), header=F)
|
|
} else {
|
|
results_temp <- read.csv(results_files[r], sep=";", colClasses=c("character", "character", "character", "character"), header=F)
|
|
results <- insertRow(results, results_temp)
|
|
}
|
|
}
|
|
rm(r, results_temp)
|
|
results <- results[!duplicated(results), ]
|
|
View(results)
|
|
View(issues)
|
|
row.names(results) <- NULL
|
|
View(results)
|
|
rownames(results)
|
|
row.names(results)
|
|
names(results)
|
|
View(tweets)
|
|
View(tweets)
|
|
names(results) <- c("date", "id_str", "issue", "tags")
|
|
View(results)
|
|
results_test <- results[order(results$id_str)]
|
|
results_test <- results[order(results$id_str), ]
|
|
View(results_test)
|
|
results_files <- list.files()
|
|
for(r in 1:length(results_files)) {
|
|
if(r == 1) {
|
|
results <- read.csv(results_files[r], sep=";", colClasses=c("character", "character", "character", "character"), header=F)
|
|
} else {
|
|
results_temp <- read.csv(results_files[r], sep=";", colClasses=c("character", "character", "character", "character"), header=F)
|
|
results <- insertRow(results, results_temp)
|
|
}
|
|
}
|
|
rm(r, results_temp)
|
|
rm(r, results_temp, results_files)
|
|
results <- results[!duplicated(results), ]
|
|
names(results)
|
|
names(results) <- c("date", "id_str", "issue", "tags")
|
|
View(results)
|
|
results_test <- results[order(results$id_str), ]
|
|
row.names(results) <- NULL
|
|
results <- results[order(results$id_str), ]
|
|
row.names(results) <- NULL
|
|
View(results)
|
|
rm(results_test)
|
|
View(issues)
|
|
as.character(results$date[2])
|
|
class(results$date)
|
|
class(issues$date)
|
|
View(issues)
|
|
as.character(issues$date[2])
|
|
issues$date[2]
|
|
issuelist <- readLines("issues.xml")
|
|
issuelist <- str_replace_all(string = issuelist, pattern = ".*<!-- .+ -->", "")
|
|
issuelist <- xmlToList(issuelist)
|
|
issueheads <- names(issuelist)
|
|
require(lubridate)
|
|
require(XML)
|
|
require(ggplot2)
|
|
require(reshape2)
|
|
require(stringr)
|
|
library(foreach)
|
|
library(doParallel)
|
|
issuelist <- readLines("issues.xml")
|
|
issuelist <- str_replace_all(string = issuelist, pattern = ".*<!-- .+ -->", "")
|
|
issuelist <- xmlToList(issuelist)
|
|
issueheads <- names(issuelist)
|
|
setwd("~/Dokumente/Uni/Aktuell/BA-Arbeit/uni-ba-issuecomp")
|
|
issuelist <- readLines("issues.xml")
|
|
issuelist <- str_replace_all(string = issuelist, pattern = ".*<!-- .+ -->", "")
|
|
issuelist <- xmlToList(issuelist)
|
|
issueheads <- names(issuelist)
|
|
issues[issueheads] <- 0
|
|
curdate <- as.character(results$date[3])
|
|
curissue <- as.character(results$issue[3])
|
|
curdate
|
|
curissue
|
|
issues[curdate, curissue] <- issues[curdate, curissue] + 1
|
|
View(issues)
|
|
issues <- data.frame(date = drange)
|
|
issues[issueheads] <- 0
|
|
View(issues)
|
|
issues[issues[, "date"] == curdate, curissue] <- issues[issues[, "date"] == curdate, curissue] + 1
|
|
View(issues)
|
|
for(r in 1:nrow(results)) {
|
|
curdate <- as.character(results$date[r])
|
|
curissue <- as.character(results$issue[r])
|
|
issues[issues[, "date"] == curdate, curissue] <- issues[issues[, "date"] == curdate, curissue] + 1
|
|
}
|
|
View(issues)
|
|
issues[issueheads] <- 0
|
|
View(issues)
|
|
for(r in 1:nrow(results)) {
|
|
curdate <- as.character(results$date[r])
|
|
curid <- as.character(results$id_str[r])
|
|
curissue <- as.character(results$issue[r])
|
|
curtag <- as.character(results$tags[r])
|
|
# Update issue counter (date and issue)
|
|
issues[issues[, "date"] == curdate, curissue] <- issues[issues[, "date"] == curdate, curissue] + 1
|
|
# Update tweet dataframe (id, issue and tags)
|
|
oldissue <- tweets[tweets[, "id_str"] == curid, "issue"]
|
|
tweets[tweets[, "id_str"] == curid, "issue"] <- str_c(oldissue, curissue, ",")
|
|
oldtag <- tweets[tweets[, "id_str"] == curid, "tags"]
|
|
tweets[tweets[, "id_str"] == curid, "tags"] <- str_c(oldtag, curtag, ",")
|
|
}
|
|
View(tweets)
|
|
tweets$issue <- ""
|
|
tweets$tags <- ""
|
|
View(tweets)
|
|
issues[issueheads] <- 0
|
|
for(r in 1:nrow(results)) {
|
|
curdate <- as.character(results$date[r])
|
|
curid <- as.character(results$id_str[r])
|
|
curissue <- as.character(results$issue[r])
|
|
curtag <- as.character(results$tags[r])
|
|
cat("Sorting match", r, "from", nrow(results), "\n")
|
|
# Update issue counter (date and issue)
|
|
issues[issues[, "date"] == curdate, curissue] <- issues[issues[, "date"] == curdate, curissue] + 1
|
|
# Update tweet dataframe (id, issue and tags)
|
|
oldissue <- tweets[tweets[, "id_str"] == curid, "issue"]
|
|
tweets[tweets[, "id_str"] == curid, "issue"] <- str_c(oldissue, curissue, ",")
|
|
oldtag <- tweets[tweets[, "id_str"] == curid, "tags"]
|
|
tweets[tweets[, "id_str"] == curid, "tags"] <- str_c(oldtag, curtag, ",")
|
|
}
|
|
View(issues)
|
|
View(tweets)
|
|
View(tweets)
|
|
save(tweets, file="tweets_tagged.RData")
|