Transaction Weight Calibration Visualized
Combined Data Effort Values
#The previous dataset
outputDir = "./res/test3" ###don't write to existing folders!!!
print(outputDir)## [1] "./res/test3"dataset <- read.csv("dsets/combined_data_set_6_26.csv")
modelData <- selectData(dataset)#load the comparative models. Each model for comparison should include two functions for evaluateion: #1. m_fit(params, trainSet) # params: a list of hyper parameters # trainSet: the training dataset #2. m_predict(model, testSet) # model: the trained model # testSet: the testing Set
After create the two functions, register your model into the “models” list, as shown beblow, with a list of hyper-parameters indexed with the model name.
#initialize models for training, testing, and evaluation. The models are put into a list with model names referencing a list of hyper-parameters, which will be passed to the model training function.
models = list()
#transaction_models <- trainsaction_based_model(modelData)
#models = append(models, transaction_models)
#initialize the size metric based models
size_models <- size_metric_models(modelData)
#register the list of the size metric based models.
models = append(models, size_models)
models$cosmic = size_models$cosmic
models$fp = size_models$fp
models$step_lnr = stepwise_linear_model(modelData)
models$neuralnet = neuralnet_model(modelData)
models$reg_tree= regression_tree_model(modelData)
#models$ensemble_trees = ensemble_trees_model(modelData)
#load the machine learning based models
#to create a model, following 3 steps:
# 1.create the model training function by rewriting this following template function
# m_fit.MODEL_NAME <- function(MODEL_NAME,dataset){}
# 2.create the prediction function by rewriting this following template function
# m_predict.MODEL_NAME <- function(MODEL_NAME, testData){}
# 3.add your model name into the "models"(above) variable for referencing
# models.MODEL_NAME <- list(hyper-params...)
# examples can be found in the size_metric_based_models.R or transaction_based_model.RBenchmark the candidate models: SWTIII, UCP, COCOMO, a-priori COCOMO, using cross-validation and bootstrapping
#setup benchmark parameters for different experiments
config = list()
config$goodness_fit_metrics <- c("R2", "f_test")
config$cv_accuracy_metrics <- c('mmre','pred15','pred25','pred50', "mdmre", "mae", "predRange50")
config$cv_nfold = 4
config$bs_accuracy_metrics <- c('mmre','pred15','pred25','pred50', "mdmre", "mae", "predRange50")
config$bs_niters = 10
config$bs_confidence_level = 0.84
benchmarkResults <- modelBenchmark(models, modelData, config)## [1] "ucp"
## [1] "ucp fit"
## [1] "ucp predict"
## [1] "fp"
## [1] "cosmic"
## [1] "ksloc"
## [1] "step_lnr"
## avoid inifite at cor= 0.4[1] "neuralnet"
## # weights: 9
## initial value 3556453557.798720
## iter 10 value 1600140846.520458
## iter 20 value 1493239733.310095
## iter 30 value 1487159968.125692
## iter 30 value 1487159967.866226
## iter 30 value 1487159954.450651
## final value 1487159954.450651
## converged
## [1] "reg_tree"
## [1] "iter:1"
## [1] "ucp"
## [1] "ucp fit"
## [1] "ucp predict"
## [1] "fp"
## [1] "cosmic"
## [1] "ksloc"
## [1] "step_lnr"
## avoid inifite at cor= 0.3[1] "neuralnet"
## # weights: 9
## initial value 2193763184.441940
## iter 10 value 864962651.891244
## iter 20 value 814723825.965126
## iter 30 value 807315524.959258
## final value 807315465.091329
## converged
## [1] "reg_tree"
## [1] "CP VALUE is:"
## [1] 0.01241864
## [1] "iter:2"
## [1] "ucp"
## [1] "ucp fit"
## [1] "ucp predict"
## [1] "fp"
## [1] "cosmic"
## [1] "ksloc"
## [1] "step_lnr"
## avoid inifite at cor= 0.4[1] "neuralnet"
## # weights: 9
## initial value 2798205648.551866
## iter 10 value 1106077353.571909
## iter 20 value 1105796044.646456
## iter 30 value 1072894590.330063
## iter 40 value 895245303.599061
## iter 50 value 892428212.662630
## iter 60 value 865419447.910536
## iter 70 value 863667772.400653
## iter 80 value 820227692.521132
## iter 90 value 816885718.663373
## iter 100 value 816460061.871477
## iter 110 value 816200580.727463
## iter 120 value 816066074.139800
## final value 816051851.911456
## converged
## [1] "reg_tree"
## [1] "CP VALUE is:"
## [1] 0.03272636
## [1] "iter:3"
## [1] "ucp"
## [1] "ucp fit"
## [1] "ucp predict"
## [1] "fp"
## [1] "cosmic"
## [1] "ksloc"
## [1] "step_lnr"
## avoid inifite at cor= 0.4[1] "neuralnet"
## # weights: 9
## initial value 2823803764.120466
## iter 10 value 1390946001.779317
## iter 20 value 1360067511.545309
## iter 30 value 1356837607.386359
## iter 40 value 1111532427.285001
## iter 50 value 1104297248.779110
## iter 60 value 1104290581.719967
## iter 60 value 1104290578.029642
## final value 1104290543.285472
## converged
## [1] "reg_tree"
## [1] "CP VALUE is:"
## [1] 0.06193489
## [1] "iter:4"
## [1] "ucp"
## [1] "ucp fit"
## [1] "ucp predict"
## [1] "fp"
## [1] "cosmic"
## [1] "ksloc"
## [1] "step_lnr"
## avoid inifite at cor= 0.4[1] "neuralnet"
## # weights: 9
## initial value 2852926005.642265
## iter 10 value 974072496.902858
## iter 20 value 962158861.705153
## iter 30 value 872867435.017993
## final value 789839113.080508
## converged
## [1] "reg_tree"
## [1] "CP VALUE is:"
## [1] 0.02595591
## [1] "iter: 1"
## [1] "ucp fit"
## [1] "ucp predict"
## avoid inifite at cor= 0.4# weights: 9
## initial value 1491562191.477533
## iter 10 value 508857038.023115
## iter 20 value 429479942.587954
## iter 30 value 422809325.738342
## iter 40 value 422543220.979145
## iter 50 value 412873378.612046
## final value 411785612.263989
## converged
## [1] "iter: 2"
## [1] "ucp fit"
## [1] "ucp predict"
## avoid inifite at cor= 0.4# weights: 9
## initial value 3058410244.278642
## iter 10 value 1480173451.948404
## iter 20 value 1007149586.344433
## iter 30 value 1003319055.609428
## iter 40 value 1002067408.061959
## iter 50 value 1001908965.214846
## iter 60 value 1001659105.110202
## iter 70 value 1001569560.989594
## final value 1001567688.933150
## converged
## [1] "CP VALUE is:"
## [1] 0.03276197
## [1] "iter: 3"
## [1] "ucp fit"
## [1] "ucp predict"
## avoid inifite at cor= 0.4# weights: 9
## initial value 3196010902.170886
## iter 10 value 1223496135.082328
## iter 20 value 1117296221.784943
## iter 30 value 1111729906.994099
## iter 40 value 1111548092.859767
## iter 50 value 876838437.268594
## iter 60 value 870179065.335262
## iter 70 value 860154997.544593
## iter 80 value 647615305.166509
## iter 90 value 603349919.850962
## iter 100 value 581492655.811909
## iter 110 value 511109984.203271
## iter 120 value 505390157.084100
## iter 130 value 494762935.558488
## iter 140 value 453826080.684968
## iter 150 value 447811664.772335
## iter 160 value 447545104.912907
## iter 170 value 447432653.269616
## final value 447432304.697574
## converged
## [1] "iter: 4"
## [1] "ucp fit"
## [1] "ucp predict"
## avoid inifite at cor= 0.4# weights: 9
## initial value 3226507162.583477
## iter 10 value 1151971341.776443
## iter 20 value 1151664281.511261
## iter 30 value 1151659437.324433
## iter 40 value 754077023.957653
## iter 50 value 743817009.831632
## iter 60 value 743795671.680283
## iter 70 value 743791645.391946
## iter 70 value 743791645.383224
## iter 70 value 743791645.382432
## final value 743791645.382432
## converged
## [1] "CP VALUE is:"
## [1] 0.03154248
## [1] "iter: 5"
## [1] "ucp fit"
## [1] "ucp predict"
## avoid inifite at cor= 0.4# weights: 9
## initial value 2991883923.153482
## iter 10 value 1392642923.242226
## iter 20 value 1391544299.018072
## iter 30 value 1205511986.035864
## iter 40 value 1202022533.506682
## iter 50 value 1098881058.635096
## iter 60 value 1098855904.886920
## iter 70 value 1055201667.014186
## final value 1054345997.007671
## converged
## [1] "iter: 6"
## [1] "ucp fit"
## [1] "ucp predict"
## avoid inifite at cor= 0.4# weights: 9
## initial value 2917316950.707774
## iter 10 value 1229016347.329754
## iter 20 value 1067667516.220008
## iter 30 value 952463265.450521
## iter 40 value 945474653.298618
## iter 50 value 945457779.753562
## final value 945457672.874308
## converged
## [1] "CP VALUE is:"
## [1] 0.0303313
## [1] "iter: 7"
## [1] "ucp fit"
## [1] "ucp predict"
## avoid inifite at cor= 0.4# weights: 9
## initial value 1937349883.559948
## iter 10 value 755122550.990362
## iter 20 value 752943931.531872
## iter 30 value 667224113.961327
## iter 40 value 478980119.122350
## iter 50 value 478804722.544702
## final value 478804512.410221
## converged
## [1] "CP VALUE is:"
## [1] 0.0229896
## [1] "iter: 8"
## [1] "ucp fit"
## [1] "ucp predict"
## avoid inifite at cor= 0.3# weights: 9
## initial value 2782470436.619922
## iter 10 value 1206348688.317254
## iter 20 value 1095662887.927838
## iter 30 value 1095505631.734425
## final value 1015392493.669246
## converged
## [1] "CP VALUE is:"
## [1] 0.02956246
## [1] "iter: 9"
## [1] "ucp fit"
## [1] "ucp predict"
## avoid inifite at cor= 0.4# weights: 9
## initial value 2789242818.516318
## iter 10 value 1387718432.199795
## iter 20 value 1367373261.711841
## iter 30 value 1206387617.347235
## final value 1161363546.408724
## converged
## [1] "iter: 10"
## [1] "ucp fit"
## [1] "ucp predict"
## avoid inifite at cor= 0.3# weights: 9
## initial value 3288729967.092281
## iter 10 value 1236886581.289634
## iter 20 value 1234234386.046739
## iter 30 value 1234098086.538860
## iter 40 value 1042391762.734748
## iter 50 value 1024863255.137386
## iter 60 value 1024778252.286547
## iter 70 value 738621247.727437
## iter 80 value 610371823.410463
## iter 90 value 609208876.264849
## iter 100 value 608478248.916309
## iter 110 value 579801368.192989
## iter 120 value 577147741.474657
## iter 130 value 577111743.801635
## iter 140 value 575023214.638897
## final value 574992312.215778
## converged
## [1] "CP VALUE is:"
## [1] 0.03070291# select the models that will be shown in the following procedure to draw the graphs
model_names <- c("reg_tree","ucp", "fp", "cosmic", "ksloc", "step_lnr", "neuralnet")
model_texts <- c("REG_TREE","UCP", "IFPUG", "COSMIC", "KSLOC", "STEP_LNR", "NEURALNET")
model_estimator <- c("LSR", "LSR", "LSR", "LSR", "LSR", "LSR", "LSR")
model_mapping <- data.frame(model_names=model_names, model_texts=model_texts, model_estimator=model_estimator)
# select the accuracy measures that will be shown in the following procedure to draw the graphs
accuracy_metrics <- c('mmre', 'pred25', 'mdmre', 'mae')
accuracy_metric_texts <- c('MMRE', 'PRED25', 'MDMRE', 'MAE')
accuracy_mapping <- data.frame(accuracy_metrics = accuracy_metrics, accuracy_metric_texts=accuracy_metric_texts)
#read the model evaluation results
goodness_fit_metrics = benchmarkResults$goodness_fit_metrics
fitResults <- benchmarkResults$fitResults
cvResults <- benchmarkResults$cvResults
foldResults <- cvResults$foldResults
bsRet <- benchmarkResults$bsResults
bsEstimations <- bsRet[['bsEstimations']]
iterResults <- bsRet[['iterResults']]
#create some combinations of the model labels, which will be used to show the results.
model_labels <- c()
for(i in 1:length(model_names)){
for(j in 1:length(accuracy_metrics)){
model_labels = c(model_labels, model_names[i])
}
}
print(model_labels)## [1] "reg_tree" "reg_tree" "reg_tree" "reg_tree" "ucp"
## [6] "ucp" "ucp" "ucp" "fp" "fp"
## [11] "fp" "fp" "cosmic" "cosmic" "cosmic"
## [16] "cosmic" "ksloc" "ksloc" "ksloc" "ksloc"
## [21] "step_lnr" "step_lnr" "step_lnr" "step_lnr" "neuralnet"
## [26] "neuralnet" "neuralnet" "neuralnet"#create some combinations of the accuracy metric labels, which will be used to show the results.
metric_labels <- c()
for(i in 1:length(model_names)){
for(j in 1:length(accuracy_metrics)){
metric_labels = c(metric_labels, accuracy_metrics[j])
}
}
accuracy_labels <- paste(model_labels, metric_labels, sep="_")
model_mapping_2 <- data.frame(model_labels = model_labels, metric_labels=metric_labels, accuracy_labels = accuracy_labels)
print(accuracy_labels)## [1] "reg_tree_mmre" "reg_tree_pred25" "reg_tree_mdmre"
## [4] "reg_tree_mae" "ucp_mmre" "ucp_pred25"
## [7] "ucp_mdmre" "ucp_mae" "fp_mmre"
## [10] "fp_pred25" "fp_mdmre" "fp_mae"
## [13] "cosmic_mmre" "cosmic_pred25" "cosmic_mdmre"
## [16] "cosmic_mae" "ksloc_mmre" "ksloc_pred25"
## [19] "ksloc_mdmre" "ksloc_mae" "step_lnr_mmre"
## [22] "step_lnr_pred25" "step_lnr_mdmre" "step_lnr_mae"
## [25] "neuralnet_mmre" "neuralnet_pred25" "neuralnet_mdmre"
## [28] "neuralnet_mae"goodnessRankResults = data.frame(model_labels = model_names)
goodnessRankResults <- merge(goodnessRankResults, model_mapping, by.x = "model_labels", by.y="model_names", all=FALSE)
print(goodnessRankResults)## model_labels model_texts model_estimator
## 1 cosmic COSMIC LSR
## 2 fp IFPUG LSR
## 3 ksloc KSLOC LSR
## 4 neuralnet NEURALNET LSR
## 5 reg_tree REG_TREE LSR
## 6 step_lnr STEP_LNR LSR
## 7 ucp UCP LSRfor (i in 1:length(goodness_fit_metrics)){
g = goodness_fit_metrics[i]
selectData = data.frame(matrix(ncol=0, nrow=length(model_names)))
selectData$model_labels <- model_names
rownames(selectData) = selectData$model_labels
selectData[, g] <- c()
for(j in 1:length(model_names)){
m = fitResults[[model_names[j]]]
m_name = model_names[j]
selectData[m_name, g] <- m[[g]]
}
selectData[, g] = round(selectData[, g],2)
goodnessRankResults <- merge(goodnessRankResults, selectData, by = "model_labels", all=FALSE)
}
# show R^2 in a table
goodnessRankResults["Rank"] <- rank(-goodnessRankResults[,"R2"], ties.method = "min")
goodnessRankResults <- goodnessRankResults[order(goodnessRankResults$Rank),]
print(goodnessRankResults)## model_labels model_texts model_estimator R2 f_test Rank
## 5 reg_tree REG_TREE LSR 0.94 0.00 1
## 6 step_lnr STEP_LNR LSR 0.71 0.00 2
## 2 fp IFPUG LSR 0.45 0.10 3
## 4 neuralnet NEURALNET LSR 0.25 0.41 4
## 1 cosmic COSMIC LSR 0.13 0.69 5
## 3 ksloc KSLOC LSR 0.00 0.99 6
## 7 ucp UCP LSR 0.00 1.00 6# write the results to the output dir
write.csv(goodnessRankResults, paste(outputDir, "goodness_rank_results.csv", sep="/"))avgPreds <- cvResults[["avgPreds"]]
avgPreds <- data.frame(avgPreds)
avgPreds <- avgPreds[,names(avgPreds) %in% c(model_names, "Pred")]
names(avgPreds) <- model_mapping$model_texts[match(names(avgPreds), model_mapping$model_names)]
names(avgPreds)[1] = c("Pred")
# draw curves from pred(0.01) - pred(0.50)
meltAvgPreds = melt(avgPreds, id.vars="Pred")
colnames(meltAvgPreds) <- c("Pred", "Method", "Value")
avg_pred_plot = ggplot(meltAvgPreds) + theme_bw() +
geom_point(aes(x=Pred, y=Value, group=Method,color=Method,shape=Method),size=1.5) +
scale_shape_manual(values=seq(from=1, to = length(model_names), by =1))+
stat_smooth(aes(x=Pred, y=Value, group=Method,color=Method), method = lm, formula = y ~ poly(x, 10), se = FALSE)+ xlab("Relative Deviation (%)") +
ylab("Percentage of Estimates <= x%")+ theme(legend.position="bottom")
ggsave(paste(outputDir, "avg_pred_plot.png", sep="/"), width=5, height=3.5)
#rank the cv results of different metric
cvAccuracyResults <- data.frame(cvResults$accuracyResults)
cvAccuracyResults$label <- rownames(cvAccuracyResults)
cvAccuracyResults <- cvAccuracyResults[cvAccuracyResults$label %in% accuracy_labels,]
cvAccuracyResults <- merge(cvAccuracyResults, model_mapping_2, by.x = "label", by.y="accuracy_labels", all=FALSE)
cvAccuracyResults$label <- NULL
#calculate ranking results
cvRankResults <- data.frame(model_names)
names(cvRankResults)<-c("model_labels")
for (i in 1:length(accuracy_metrics)){
g = accuracy_metrics[i]
selectedData <- cvAccuracyResults[cvAccuracyResults$metric_labels == g,]
selectedData <- selectedData[,-3]#delete the metric_labels
colnames(selectedData)<-c(g, "model_labels")
if(g == "mmre" || g == "mdmre" || g == "mae"){
selectedData[paste("rank", i, sep = "")] <- rank(selectedData[,1], ties.method = "min")
}else{
selectedData[paste("rank", i, sep = "")] <- rank(-selectedData[,1], ties.method = "min")
}
cvRankResults <- merge(cvRankResults, selectedData, by = "model_labels", all=FALSE)
}
#make a total rank(rank*) base on the ranks
rank_sum <- vector(mode = "integer",length = length(model_names))
for (i in 1:length(model_names)){
selectedData <- cvRankResults[i,]
for(j in 1:length(accuracy_metrics)){
rank_sum[i] <- rank_sum[i] + selectedData[,2*j+1]
}
}
rank_sum <- rank(rank_sum, ties.method = "min")
print(rank_sum)## [1] 2 2 7 4 1 4 6cvRankResults["rank*"] <- rank_sum
rownames(cvRankResults) <- model_mapping$model_texts[match(cvRankResults$model_labels, model_mapping$model_names)]
cvRankResults$model_labels = NULL
cvRankResults <- cvRankResults[order(cvRankResults$'rank*'),]
print(round(cvRankResults,2))## mmre rank1 pred25 rank2 mdmre rank3 mae rank4 rank*
## REG_TREE 0.84 1 0.46 1 0.63 1 10785.31 1 1
## COSMIC 1.95 3 0.16 2 1.27 3 21114.11 3 2
## IFPUG 1.49 2 0.09 5 1.02 2 19997.84 2 2
## NEURALNET 2.67 5 0.12 3 1.82 5 27803.00 5 4
## STEP_LNR 2.45 4 0.09 4 1.79 4 29338.81 6 4
## UCP 2.98 6 0.03 6 2.37 6 26290.00 4 6
## KSLOC 3.75 7 0.00 7 3.08 7 31740.35 7 7write.csv(round(cvRankResults,2), paste(outputDir, "cv_rank_results.csv", sep="/"))
# draw histogram based on ranking
library(ggplot2)
p <- list()
for(i in 1:length(accuracy_metrics)){
g = paste("rank", i, sep = "")
selectedData <- cvRankResults[names(cvRankResults) == g]
names(selectedData) <- c("rank");
p[[i]] <- ggplot(selectedData, aes(x=rownames(cvRankResults), y=rank, fill=rownames(cvRankResults))) +
geom_bar(stat="identity", colour = "black") +
#scale_y_discrete(expand = c(0, 0)) +
guides(fill = guide_legend(title = "MODEL", nrow = 2)) +
geom_text(aes(label = rank, vjust = -0.3, hjust = 0.5)) +
#ggtitle(accuracy_metrics[i]) +
labs(caption=toupper(accuracy_metrics[i])) +
theme(plot.caption = element_text(hjust=0.5, vjust = 2.5, size=rel(1)),
#axis.line=element_blank(),
axis.title=element_blank(), axis.text=element_blank(),axis.ticks=element_blank(), panel.background = element_blank())
}
prow <- plot_grid( p[[1]] + theme(legend.position="none"),
p[[2]] + theme(legend.position="none"),
p[[3]] + theme(legend.position="none"),
p[[4]] + theme(legend.position="none"),
align = 'vh',
hjust = 0,
nrow = 1
)
legend_b <- get_legend(p[[1]] + theme(legend.position="bottom", legend.justification="center"))
title <- ggdraw() + draw_label("Ranking Result for Cross Validation", fontface='bold')
p_cvRank <- plot_grid(title, prow, legend_b, ncol = 1, rel_heights = c(.2 , 1, .1))
p_cvRank
# draw overall ranking histogram
selectedData <- cvRankResults[names(cvRankResults) == "rank*"]
names(selectedData) <- c("rank");
p_cvAllRank <- ggplot(selectedData, aes(x=rownames(selectedData), y=rank, fill=rownames(selectedData))) +
geom_bar(stat="identity", colour = "black", width = 0.7) +
#scale_y_discrete(expand = c(0, 0)) +
guides(fill = guide_legend(title = "MODEL", nrow = 2)) +
geom_text(aes(label = rank, vjust = -0.4, hjust = 0.5)) +
ggtitle("Overall Ranking Result for Cross Validation") +
#labs(caption=toupper("Total Rank")) +
theme(plot.caption = element_text(hjust=0.5, size=rel(0.5)), legend.position = "bottom",legend.text=element_text(size=10),
plot.title = element_text(hjust = 0.5),
#axis.line=element_blank(),
axis.title=element_blank(), axis.text=element_blank(),axis.ticks=element_blank(), panel.background = element_blank())
p_cvAllRank
df <- data.frame(t(bsEstimations))
df$label <- rownames(df)
df <- df[df$label %in% accuracy_labels,]
df <- merge(df, model_mapping_2, by.x = "label", by.y="accuracy_labels", all=FALSE)
# read the pairs of models that are overlapped for 84% confidence intervals
nonOverlappingPairs <- data.frame(matrix(ncol = 8, nrow = 0))
overlappingPairAttrs <- c("model1", "model2", "metric", "direction", "mean1", "mean2", "84% CI1", "84% CI2")
colnames(nonOverlappingPairs) <- overlappingPairAttrs
for (i in 1:length(accuracy_metrics)){
g = metric_labels[i]
selectedData <- df[df$metric_labels == g,]
for (j in 1:(nrow(selectedData)-1)){
for (k in (j+1):nrow(selectedData)){
if(selectedData[j,]$lower>selectedData[k,]$upper | selectedData[j,]$upper<selectedData[k,]$lower){
#selectedData[j,] and selectedData[k,] non-overlap
direction = "="
if(selectedData[j,]$mean > selectedData[k,]$mean){
direction = "+"
}
else if(selectedData[j,]$mean < selectedData[k,]$mean){
direction = "-"
}
if(selectedData[j,]$metric %in% c("mae", "mdmre", "mmre")){
if(direction == "+"){
direction = "-"
}
else if(direction == "-"){
direction = "+"
}
}
nonOverlap <- data.frame(
selectedData[j,]$model_labels,
selectedData[k,]$model_labels,
g,
direction,
round(selectedData[j,]$mean, 3),
round(selectedData[k,]$mean, 3),
paste0("[", as.character(round(selectedData[j,]$lower, 3)), ", ", as.character(round(selectedData[j,]$upper, 3)), "]"),
paste0("[", as.character(round(selectedData[k,]$lower, 3)), ", ", as.character(round(selectedData[k,]$upper, 3)), "]")
)
colnames(nonOverlap) <- overlappingPairAttrs
nonOverlappingPairs <- rbind(nonOverlappingPairs, nonOverlap)
}
}
}
}
print(nonOverlappingPairs)## model1 model2 metric direction mean1 mean2
## 1 cosmic ksloc pred25 + 0.214 0.014
## 2 cosmic reg_tree pred25 - 0.214 0.500
## 3 fp reg_tree pred25 - 0.114 0.500
## 4 ksloc neuralnet pred25 - 0.014 0.286
## 5 ksloc reg_tree pred25 - 0.014 0.500
## 6 reg_tree step_lnr pred25 + 0.500 0.129
## 7 reg_tree ucp pred25 + 0.500 0.029
## 8 cosmic reg_tree mae - 19032.826 8357.436
## 9 ksloc reg_tree mae - 23355.684 8357.436
## 10 reg_tree step_lnr mae + 8357.436 22670.877
## 11 reg_tree ucp mae + 8357.436 22431.653
## 84% CI1 84% CI2
## 1 [0.103, 0.326] [0, 0.04]
## 2 [0.103, 0.326] [0.349, 0.714]
## 3 [0, 0.286] [0.349, 0.714]
## 4 [0, 0.04] [0.103, 0.509]
## 5 [0, 0.04] [0.349, 0.714]
## 6 [0.349, 0.714] [0, 0.286]
## 7 [0.349, 0.714] [0, 0.143]
## 8 [14569.645, 26358.305] [4940.607, 13127.974]
## 9 [17726.378, 31463.514] [4940.607, 13127.974]
## 10 [4940.607, 13127.974] [18258.167, 27150.284]
## 11 [4940.607, 13127.974] [16948.92, 30254.267]write.csv(nonOverlappingPairs, paste(outputDir, 'nonOverlappingPairs.csv', sep="/"))
# apply filters on the overlapping results.
filteredNonOverlappingPairs = nonOverlappingPairs[which(nonOverlappingPairs$metric %in% accuracy_metrics),]
filteredNonOverlappingPairs = nonOverlappingPairs[which((nonOverlappingPairs$model1 %in% c("tm3", "tm2", "tm1") | nonOverlappingPairs$model2 %in% c("tm3", "tm2", "tm1"))) ,]
filteredNonOverlappingPairsIndices <- paste(filteredNonOverlappingPairs$model1, filteredNonOverlappingPairs$model2, filteredNonOverlappingPairs$metric, sep="-")
write.csv(filteredNonOverlappingPairs, paste(outputDir, 'filteredNonOverlappingPairs.csv', sep="/"))
print(filteredNonOverlappingPairs)## [1] model1 model2 metric direction mean1 mean2 84% CI1
## [8] 84% CI2
## <0 rows> (or 0-length row.names)print(filteredNonOverlappingPairsIndices)## character(0)confidence_interval_graph_plots = list()
for (i in 1:length(accuracy_metrics)){
g = accuracy_metrics[i]
g_label <- toupper(g)
selectedData <- df[df$metric_labels == g,]
confidence_interval_graph_plots[[i]] <- ggplot(selectedData, aes(x = label, y = mean)) +
geom_errorbar(aes(ymin=lower, ymax=upper), colour="black", width=.1) +
geom_point(size=2, shape=21, fill="black") + # 21 is filled circle
xlab('') +
ylab(g_label) +
scale_x_discrete(breaks=selectedData$label, labels=as.vector(
model_mapping$model_texts[match(selectedData$model_labels, model_mapping$model_names)])) +
#ggtitle(paste(g_label, "- 84% Confidence Intervals", setp=""))+
theme_bw()+
theme(axis.text.x = element_text(angle = 90, hjust = 1, face = "bold"))
print(confidence_interval_graph_plots[[i]])
}
prow <- plot_grid( confidence_interval_graph_plots[[1]] + theme(legend.position="none"),
confidence_interval_graph_plots[[2]] + theme(legend.position="none"),
confidence_interval_graph_plots[[3]] + theme(legend.position="none"),
confidence_interval_graph_plots[[4]] + theme(legend.position="none"),
align = 'vh',
hjust = 0,
nrow = 2
)
#title <- ggdraw() + draw_label("Ranking Result for Cross Validation", fontface='bold')
confidence_interval_graphs <- plot_grid(prow, ncol = 1, rel_heights = c(.2, 1, .1), label_size=5)
print(confidence_interval_graphs)
ggsave(paste(outputDir, "confidence_interval_graphs.png", sep="/"), width=8, height=6)
# draw a partially ordered graph based on non-overlapping pairs
# Using the "sig_bs" results, create a graph to represent the direct graph for the models.
library(igraph)##
## Attaching package: 'igraph'## The following objects are masked from 'package:purrr':
##
## compose, simplify## The following object is masked from 'package:tidyr':
##
## crossing## The following object is masked from 'package:tibble':
##
## as_data_frame## The following object is masked from 'package:tree':
##
## tree## The following objects are masked from 'package:dplyr':
##
## as_data_frame, groups, union## The following objects are masked from 'package:stats':
##
## decompose, spectrum## The following object is masked from 'package:base':
##
## unionfor(metric_i in 1:length(accuracy_metrics)){
selectedData <- nonOverlappingPairs[nonOverlappingPairs$metric == metric_labels[metric_i],]
m <- matrix(0, nrow = length(model_names), ncol = length(model_names), byrow = FALSE)
colnames(m) <- model_names
rownames(m) <- model_names
if(nrow(selectedData) > 0){
for(i in 1:nrow(selectedData)){
if(selectedData$direction[i] == "+"){
m[as.character(selectedData$model1[i]), as.character(selectedData$model2[i])] = 1
}else if(selectedData$direction[i] == "-"){
m[as.character(selectedData$model2[i]), as.character(selectedData$model1[i])] = 1
}
}
}
edge_val <- c()
for(i in 1:length(model_names)){
for(j in 1:length(model_names)){
if(m[i,j] != 0){
edge_val <- c(edge_val, m[i,j])
for(k in 1:length(model_names)){
if(m[j,k] != 0)
m[i,k] = 0
}
}
}
}
#plot the directed graph
model_mean <- matrix(0, nrow = length(model_names), byrow = FALSE)
rownames(model_mean) <- model_names
colnames(model_mean) <- "mean"
if(nrow(selectedData) > 0){
for(i in 1:nrow(selectedData)){
model_mean[which(rownames(model_mean) == selectedData[i,]$model1)] = round(selectedData[i,]$mean1, 3)
model_mean[which(rownames(model_mean) == selectedData[i,]$model2)] = round(selectedData[i,]$mean2, 3)
}
}
net=graph.adjacency(m,mode="directed",weighted=TRUE,diag=FALSE)
lo <- layout.fruchterman.reingold(net, niter = 1000)
plot(net, vertex.label=paste(model_mapping$model_texts[match(V(net)$name, model_mapping$model_names)], model_mean[which(rownames(model_mean) == V(net)$name)], sep = " : "), layout = lo, vertex.size = 5, vertex.frame.color = NULL,
vertex.label.dist = 1, vertex.label.cex = 0.7, vertex.label.color="black",
edge.color="black", edge.arrow.size=0.5, edge.width = 0.5, edge.label.cex=0.7)
title(main = accuracy_mapping[accuracy_mapping$accuracy_metrics == metric_labels[metric_i], "accuracy_metric_texts"])
}
sig_bs <- familywiseHypoTest(iterationResults=iterResults, accuracy_metrics, model_names, "boot")## [1] "number of hypothesis:"
## [1] 84
## [1] "mmre"
## [1] 1 5 9 13 17 21 25
## [1] "reg_tree vs ucp"
## [1] "bootstrap hypothesis test"
## [1] 0.9035818
## [1] 3.219033
## [1] "reg_tree vs fp"
## [1] "bootstrap hypothesis test"
## [1] 0.9035818
## [1] 1.182202
## [1] "reg_tree vs cosmic"
## [1] "bootstrap hypothesis test"
## [1] 0.9035818
## [1] 2.27918
## [1] "reg_tree vs ksloc"
## [1] "bootstrap hypothesis test"
## [1] 0.9035818
## [1] 3.505693
## [1] "reg_tree vs step_lnr"
## [1] "bootstrap hypothesis test"
## [1] 0.9035818
## [1] 2.628055
## [1] "reg_tree vs neuralnet"
## [1] "bootstrap hypothesis test"
## [1] 0.9035818
## [1] 1.579081
## [1] "ucp vs fp"
## [1] "bootstrap hypothesis test"
## [1] 3.219033
## [1] 1.182202
## [1] "ucp vs cosmic"
## [1] "bootstrap hypothesis test"
## [1] 3.219033
## [1] 2.27918
## [1] "ucp vs ksloc"
## [1] "bootstrap hypothesis test"
## [1] 3.219033
## [1] 3.505693
## [1] "ucp vs step_lnr"
## [1] "bootstrap hypothesis test"
## [1] 3.219033
## [1] 2.628055
## [1] "ucp vs neuralnet"
## [1] "bootstrap hypothesis test"
## [1] 3.219033
## [1] 1.579081
## [1] "fp vs cosmic"
## [1] "bootstrap hypothesis test"
## [1] 1.182202
## [1] 2.27918
## [1] "fp vs ksloc"
## [1] "bootstrap hypothesis test"
## [1] 1.182202
## [1] 3.505693
## [1] "fp vs step_lnr"
## [1] "bootstrap hypothesis test"
## [1] 1.182202
## [1] 2.628055
## [1] "fp vs neuralnet"
## [1] "bootstrap hypothesis test"
## [1] 1.182202
## [1] 1.579081
## [1] "cosmic vs ksloc"
## [1] "bootstrap hypothesis test"
## [1] 2.27918
## [1] 3.505693
## [1] "cosmic vs step_lnr"
## [1] "bootstrap hypothesis test"
## [1] 2.27918
## [1] 2.628055
## [1] "cosmic vs neuralnet"
## [1] "bootstrap hypothesis test"
## [1] 2.27918
## [1] 1.579081
## [1] "ksloc vs step_lnr"
## [1] "bootstrap hypothesis test"
## [1] 3.505693
## [1] 2.628055
## [1] "ksloc vs neuralnet"
## [1] "bootstrap hypothesis test"
## [1] 3.505693
## [1] 1.579081
## [1] "step_lnr vs neuralnet"
## [1] "bootstrap hypothesis test"
## [1] 2.628055
## [1] 1.579081
## [1] "pred25"
## [1] 2 6 10 14 18 22 26
## [1] "reg_tree vs ucp"
## [1] "bootstrap hypothesis test"
## [1] 0.5
## [1] 0.02857143
## [1] "reg_tree vs fp"
## [1] "bootstrap hypothesis test"
## [1] 0.5
## [1] 0.1142857
## [1] "reg_tree vs cosmic"
## [1] "bootstrap hypothesis test"
## [1] 0.5
## [1] 0.2142857
## [1] "reg_tree vs ksloc"
## [1] "bootstrap hypothesis test"
## [1] 0.5
## [1] 0.01428571
## [1] "reg_tree vs step_lnr"
## [1] "bootstrap hypothesis test"
## [1] 0.5
## [1] 0.1285714
## [1] "reg_tree vs neuralnet"
## [1] "bootstrap hypothesis test"
## [1] 0.5
## [1] 0.2857143
## [1] "ucp vs fp"
## [1] "bootstrap hypothesis test"
## [1] 0.02857143
## [1] 0.1142857
## [1] "ucp vs cosmic"
## [1] "bootstrap hypothesis test"
## [1] 0.02857143
## [1] 0.2142857
## [1] "ucp vs ksloc"
## [1] "bootstrap hypothesis test"
## [1] 0.02857143
## [1] 0.01428571
## [1] "ucp vs step_lnr"
## [1] "bootstrap hypothesis test"
## [1] 0.02857143
## [1] 0.1285714
## [1] "ucp vs neuralnet"
## [1] "bootstrap hypothesis test"
## [1] 0.02857143
## [1] 0.2857143
## [1] "fp vs cosmic"
## [1] "bootstrap hypothesis test"
## [1] 0.1142857
## [1] 0.2142857
## [1] "fp vs ksloc"
## [1] "bootstrap hypothesis test"
## [1] 0.1142857
## [1] 0.01428571
## [1] "fp vs step_lnr"
## [1] "bootstrap hypothesis test"
## [1] 0.1142857
## [1] 0.1285714
## [1] "fp vs neuralnet"
## [1] "bootstrap hypothesis test"
## [1] 0.1142857
## [1] 0.2857143
## [1] "cosmic vs ksloc"
## [1] "bootstrap hypothesis test"
## [1] 0.2142857
## [1] 0.01428571
## [1] "cosmic vs step_lnr"
## [1] "bootstrap hypothesis test"
## [1] 0.2142857
## [1] 0.1285714
## [1] "cosmic vs neuralnet"
## [1] "bootstrap hypothesis test"
## [1] 0.2142857
## [1] 0.2857143
## [1] "ksloc vs step_lnr"
## [1] "bootstrap hypothesis test"
## [1] 0.01428571
## [1] 0.1285714
## [1] "ksloc vs neuralnet"
## [1] "bootstrap hypothesis test"
## [1] 0.01428571
## [1] 0.2857143
## [1] "step_lnr vs neuralnet"
## [1] "bootstrap hypothesis test"
## [1] 0.1285714
## [1] 0.2857143
## [1] "mdmre"
## [1] 3 7 11 15 19 23 27
## [1] "reg_tree vs ucp"
## [1] "bootstrap hypothesis test"
## [1] 0.3754988
## [1] 1.644638
## [1] "reg_tree vs fp"
## [1] "bootstrap hypothesis test"
## [1] 0.3754988
## [1] 0.753444
## [1] "reg_tree vs cosmic"
## [1] "bootstrap hypothesis test"
## [1] 0.3754988
## [1] 1.299454
## [1] "reg_tree vs ksloc"
## [1] "bootstrap hypothesis test"
## [1] 0.3754988
## [1] 1.687268
## [1] "reg_tree vs step_lnr"
## [1] "bootstrap hypothesis test"
## [1] 0.3754988
## [1] 1.454499
## [1] "reg_tree vs neuralnet"
## [1] "bootstrap hypothesis test"
## [1] 0.3754988
## [1] 1.070883
## [1] "ucp vs fp"
## [1] "bootstrap hypothesis test"
## [1] 1.644638
## [1] 0.753444
## [1] "ucp vs cosmic"
## [1] "bootstrap hypothesis test"
## [1] 1.644638
## [1] 1.299454
## [1] "ucp vs ksloc"
## [1] "bootstrap hypothesis test"
## [1] 1.644638
## [1] 1.687268
## [1] "ucp vs step_lnr"
## [1] "bootstrap hypothesis test"
## [1] 1.644638
## [1] 1.454499
## [1] "ucp vs neuralnet"
## [1] "bootstrap hypothesis test"
## [1] 1.644638
## [1] 1.070883
## [1] "fp vs cosmic"
## [1] "bootstrap hypothesis test"
## [1] 0.753444
## [1] 1.299454
## [1] "fp vs ksloc"
## [1] "bootstrap hypothesis test"
## [1] 0.753444
## [1] 1.687268
## [1] "fp vs step_lnr"
## [1] "bootstrap hypothesis test"
## [1] 0.753444
## [1] 1.454499
## [1] "fp vs neuralnet"
## [1] "bootstrap hypothesis test"
## [1] 0.753444
## [1] 1.070883
## [1] "cosmic vs ksloc"
## [1] "bootstrap hypothesis test"
## [1] 1.299454
## [1] 1.687268
## [1] "cosmic vs step_lnr"
## [1] "bootstrap hypothesis test"
## [1] 1.299454
## [1] 1.454499
## [1] "cosmic vs neuralnet"
## [1] "bootstrap hypothesis test"
## [1] 1.299454
## [1] 1.070883
## [1] "ksloc vs step_lnr"
## [1] "bootstrap hypothesis test"
## [1] 1.687268
## [1] 1.454499
## [1] "ksloc vs neuralnet"
## [1] "bootstrap hypothesis test"
## [1] 1.687268
## [1] 1.070883
## [1] "step_lnr vs neuralnet"
## [1] "bootstrap hypothesis test"
## [1] 1.454499
## [1] 1.070883
## [1] "mae"
## [1] 4 8 12 16 20 24 28
## [1] "reg_tree vs ucp"
## [1] "bootstrap hypothesis test"
## [1] 8357.436
## [1] 22431.65
## [1] "reg_tree vs fp"
## [1] "bootstrap hypothesis test"
## [1] 8357.436
## [1] 15734.3
## [1] "reg_tree vs cosmic"
## [1] "bootstrap hypothesis test"
## [1] 8357.436
## [1] 19032.83
## [1] "reg_tree vs ksloc"
## [1] "bootstrap hypothesis test"
## [1] 8357.436
## [1] 23355.68
## [1] "reg_tree vs step_lnr"
## [1] "bootstrap hypothesis test"
## [1] 8357.436
## [1] 22670.88
## [1] "reg_tree vs neuralnet"
## [1] "bootstrap hypothesis test"
## [1] 8357.436
## [1] 17900.05
## [1] "ucp vs fp"
## [1] "bootstrap hypothesis test"
## [1] 22431.65
## [1] 15734.3
## [1] "ucp vs cosmic"
## [1] "bootstrap hypothesis test"
## [1] 22431.65
## [1] 19032.83
## [1] "ucp vs ksloc"
## [1] "bootstrap hypothesis test"
## [1] 22431.65
## [1] 23355.68
## [1] "ucp vs step_lnr"
## [1] "bootstrap hypothesis test"
## [1] 22431.65
## [1] 22670.88
## [1] "ucp vs neuralnet"
## [1] "bootstrap hypothesis test"
## [1] 22431.65
## [1] 17900.05
## [1] "fp vs cosmic"
## [1] "bootstrap hypothesis test"
## [1] 15734.3
## [1] 19032.83
## [1] "fp vs ksloc"
## [1] "bootstrap hypothesis test"
## [1] 15734.3
## [1] 23355.68
## [1] "fp vs step_lnr"
## [1] "bootstrap hypothesis test"
## [1] 15734.3
## [1] 22670.88
## [1] "fp vs neuralnet"
## [1] "bootstrap hypothesis test"
## [1] 15734.3
## [1] 17900.05
## [1] "cosmic vs ksloc"
## [1] "bootstrap hypothesis test"
## [1] 19032.83
## [1] 23355.68
## [1] "cosmic vs step_lnr"
## [1] "bootstrap hypothesis test"
## [1] 19032.83
## [1] 22670.88
## [1] "cosmic vs neuralnet"
## [1] "bootstrap hypothesis test"
## [1] 19032.83
## [1] 17900.05
## [1] "ksloc vs step_lnr"
## [1] "bootstrap hypothesis test"
## [1] 23355.68
## [1] 22670.88
## [1] "ksloc vs neuralnet"
## [1] "bootstrap hypothesis test"
## [1] 23355.68
## [1] 17900.05
## [1] "step_lnr vs neuralnet"
## [1] "bootstrap hypothesis test"
## [1] 22670.88
## [1] 17900.05sig_bs = sig_bs[which(sig_bs$model1 %in% model_names | sig_bs$model2 %in% model_names),]
round_df <- function(df, digits) {
nums <- vapply(df, is.numeric, FUN.VALUE = logical(1))
df[,nums] <- round(df[,nums], digits = digits)
(df)
}
write.csv(sig_bs,paste(outputDir, 'sig_bs.csv', sep="/"))
#print(sig_bs$p_value)
#print only pair comparisons that are related to transaction models
sig_bs_f = sig_bs[which(sig_bs$p_value < 0.05),]
sig_bs_f = sig_bs_f[which(sig_bs_f$metric %in% accuracy_metrics),]
sig_bs_f = sig_bs_f[order(sig_bs_f$p_value),]
sig_bs_f = round_df(sig_bs_f, 2)
#print(sig_bs_f$p_value)
#order models
sig_bs_f <- as.data.frame(t(apply(sig_bs_f, 1, function(x) {
#print(x)
if(x['direction'] == "-"){
model_2 = x['model2']
x['model2'] = x['model1']
x['model1'] = model_2
mean_2 = x['model2_mean']
x['model2_mean'] = x['model1_mean']
x['model1_mean'] = mean_2
x['direction'] = "+"
x['cohen_d'] = -as.numeric(x['cohen_d'])
}
x
})))
sig_bs_f$model1 = model_mapping$model_texts[match(sig_bs_f$model1, model_mapping$model_names)]
sig_bs_f$model2 = model_mapping$model_texts[match(sig_bs_f$model2, model_mapping$model_names)]
print(sig_bs_f$cohen_d)## 29 64 66 67 68 69 76
## 1.94 -2.78 -2.21 -2.95 -3.55 -1.79 -1.41
## Levels: -1.41 -1.79 -2.21 -2.78 -2.95 -3.55 1.94write.csv(sig_bs_f,paste(outputDir, 'sig_bs_f.csv', sep="/"))
sig_bs_f_1 = sig_bs_f[which(sig_bs_f$bonferroni_p_value < 0.05),]## Warning in Ops.factor(sig_bs_f$bonferroni_p_value, 0.05): '<' not
## meaningful for factorssig_bs_f_1 = sig_bs_f_1[which(sig_bs_f_1$metric %in% accuracy_metrics),]
sig_bs_f_1 = sig_bs_f_1[order(sig_bs_f_1$bonferroni_p_value),]
sig_bs_f_1 = round_df(sig_bs_f_1, 2)
write.csv(sig_bs_f_1,paste(outputDir, 'sig_bs_bc.csv', sep="/"))
#identify the parirs that are not identified from the 84% overlaps.
sig_bs_tm = sig_bs_f[which(sig_bs_f$model1 %in% c("SWTI", "SWTII", "SWTIII") | sig_bs_f$model2 %in% c("SWTI", "SWTII", "SWTIII")),]
sig_bs_f_indices <- paste(sig_bs_tm$model1, sig_bs_tm$model2, sig_bs_tm$metric, sep="-")
print(sig_bs_f_indices)## character(0)sig_bs_tm$index <- sig_bs_f_indices
`%ni%` <- Negate(`%in%`)
filtered_sig_bs_tm = sig_bs_tm[which(sig_bs_tm$index %ni% filteredNonOverlappingPairsIndices), ]
write.csv(filtered_sig_bs_tm,paste(outputDir, 'filtered_sig_bs_tm.csv', sep="/"))
# Using the "sig_bs" results, create a graph to represent the direct graph for the models.
sig_rank_graph_plots = list()
library(igraph)
for(metric_i in 1:length(accuracy_metrics)){
#print(accuracy_metrics[metric_i])
#if(accuracy_metrics[metric_i] == "predRange"){
# next
#}
#print(accuracy_metrics)
selectedData <- sig_bs[sig_bs$metric == metric_labels[metric_i],]
m <- matrix(0, nrow = length(model_names), ncol = length(model_names), byrow = FALSE)
#colnames(m) <- names(models)
#rownames(m) <- names(models)
colnames(m) <- model_names
rownames(m) <- model_names
for(i in 1:nrow(selectedData)){
if(selectedData$p_value[i] < 0.05){
if(selectedData$direction[i] == "+"){
m[as.character(selectedData$model1[i]), as.character(selectedData$model2[i])] = round(selectedData$p_value[i], 3)
}else if(selectedData$direction[i] == "-"){
m[as.character(selectedData$model2[i]), as.character(selectedData$model1[i])] = round(selectedData$p_value[i], 3)
}
}
}
edge_val <- c()
# if A -> B -> C, remove edge A -> C
for(i in 1:length(model_names)){
for(j in 1:length(model_names)){
if(m[i,j] != 0){
edge_val <- c(edge_val, m[i,j])
for(k in 1:length(model_names)){
if(m[j,k] != 0)
m[i,k] = 0
}
}
}
}
#plot the directed graph
model_mean <- matrix(0, nrow = length(model_names), byrow = FALSE)
rownames(model_mean) <- model_names
colnames(model_mean) <- "mean"
for(i in 1:nrow(selectedData)){
model_mean[which(rownames(model_mean) == selectedData[i,]$model1)] = round(selectedData[i,]$model1_mean, 3)
model_mean[which(rownames(model_mean) == selectedData[i,]$model2)] = round(selectedData[i,]$model2_mean, 3)
}
net=graph.adjacency(m,mode="directed",weighted=TRUE,diag=FALSE)
lo <- layout.fruchterman.reingold(net, niter = 1000)
print(metric_i)
sig_rank_graph_plot = as.ggplot(
function() {
plot(net, vertex.label=paste(model_mapping$model_texts[match(V(net)$name, model_mapping$model_names)], model_mean[which(rownames(model_mean) == V(net)$name)], sep = " : "), layout = lo, vertex.size = 5, vertex.frame.color = NULL,
vertex.label.dist = 1, vertex.label.cex = 0.7, vertex.label.color="black",
edge.color="black", edge.arrow.size=0.5, edge.width = 0.5, edge.label = edge_val, edge.label.cex=0.7)
title(main = accuracy_mapping[accuracy_mapping$accuracy_metrics == metric_labels[metric_i], "accuracy_metric_texts"])
}
)
print(sig_rank_graph_plot)
sig_rank_graph_plots[[metric_i]] = sig_rank_graph_plot
#plot.igraph(net,vertex.label=paste(V(net)$name, model_mean[which(rownames(model_mean) == V(net)$name)], sep = " : "), layout=layout.fruchterman.reingold(net)*30.0, vertex.color="white", vertex.label.color="black", vertex.size=3, edge.color="black", vertex.label.dist = 0.5, edge.label = edge_val, edge.width=3, edge.arrow.size=0.5, edge.arrow.width=1.2)
}## [1] 1## [1] 2
## [1] 3
## [1] 4
prow <- plot_grid( sig_rank_graph_plots[[1]] + theme(legend.position="none"),
sig_rank_graph_plots[[2]] + theme(legend.position="none"),
sig_rank_graph_plots[[3]] + theme(legend.position="none"),
sig_rank_graph_plots[[4]] + theme(legend.position="none"),
align = 'vh',
hjust = 0,
nrow = 2
)
#title <- ggdraw() + draw_label("Ranking Result for Cross Validation", fontface='bold')*
sig_rank_graphs <- plot_grid(prow, ncol = 1, rel_heights = c(.2, 1, .1), label_size=5)
print(sig_rank_graphs)
ggsave(paste(outputDir, "sig_rank_graphs.png", sep="/"), width=6, height=10)
# filter for