-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathTextAnalysis(Clinical Trials, CART).R
68 lines (68 loc) · 2.33 KB
/
TextAnalysis(Clinical Trials, CART).R
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
rm(list=ls())
trails=read.csv("C:/Users/Zahid/Downloads/clinical_trial.csv", stringsAsFactors=FALSE)
str(trails)
which.max(nchar(trails$abstract)) #abstract with maximum number of characters
nchar(trails$abstract[664]) #number of characters
nrow(nchar(trails$abstract)==0)
table(nchar(trails$abstract)==0)
which.min(nchar(trails$title))
nchar(trails$title[1258])
trails$title[1258]
library(SnowballC)
library(tm)
corpusAbstract=Corpus(VectorSource(trails$abstract))
corpusTitle=Corpus(VectorSource(trails$title))
corpusTitle=tm_map(corpusTitle, tolower)
corpusAbstract=tm_map(corpusAbstract, tolower)
corpusAbstract=tm_map(corpusAbstract, removePunctuation)
corpusTitle=tm_map(corpusTitle, removePunctuation)
corpusAbstract=tm_map(corpusAbstract,removeWords, stopwords('english'))
corpusTitle=tm_map(corpusTitle, removeWords, stopwords('english'))
corpusAbstract=tm_map(corpusAbstract, stemDocument)
corpusTitle=tm_map(corpusTitle, stemDocument)
dtmTitle=DocumentTermMatrix(corpusTitle)
dtmAbstract=DocumentTermMatrix(corpusAbstract)
inspect(dtmAbstract[1:5,5:15])
findFreqTerms(dtmAbstract, lowfreq=20)
dtmTitle=removeSparseTerms(dtmTitle, 0.95)
dtmAbstract=removeSparseTerms(dtmAbstract, 0.95)
ncol(dtmTitle)
ncol(dtmAbstract)
which.max(colSums(as.matrix(dtmAbstract))) #word use frequently in all the abstract
colnames(dtmTitle) = paste0("T", colnames(dtmTitle))
colnames(dtmAbstract) = paste0("A", colnames(dtmAbstract))
dtm=cbind(as.data.frame(as.matrix(dtmTitle)), as.data.frame(as.matrix(dtmAbstract)))
dtm$trial=trails$trial
str(dtm)
set.seed(144)
library(caTools)
str(dtm)
split=sample.split(dtm$trial, SplitRatio=0.7)
train=subset(dtm, split==TRUE)
test=subset(dtm, split==FALSE)
table(train$trial)
730/(730+572)
library(rpart)
library(rpart.plot)
trialCART= rpart(trial~., data=train, method="class")
prp(trialCART)
pred=predict(trialCART)
str(pred)
pred=pred[,2]
max(pred)
pred1=predict(trialCART, newdata=test)
str(pred1)
pred1=pred1[,2]
max(pred1)
table(train$trial, pred>=0.5)
(631+441)/nrow(train)
441/(441+131) #sensitive second row
631/(631+99) #specitivity first row
testPred=predict(trialCART, newdata=test,method="class" )
testPred=testPred[,2]
table(test$trial, testPred>=0.5)
(261+162)/nrow(test)
library(ROCR)
predROCR=prediction(testPred,test$trial)
ROCRperf=performance(predROCR,'tpr','fpr')
attributes(performance(predROCR, 'auc'))$y.values[[1]]