# # Multiple training routines EPage 82 # Problem EPage 161 # # Written by: # -- # John L. Weatherwax 2009-04-21 # # email: wax@alum.mit.edu # # Please send comments and especially bug reports to the # above email address. # #----- save_plots = F library(caret) library(AppliedPredictiveModeling) set.seed(0) data(permeability) # Part (b): # zero_cols = nearZeroVar( fingerprints ) print( sprintf("Found %d zero variance columns from %d",length(zero_cols), dim(fingerprints)[2] ) ) fingerprints = fingerprints[,-zero_cols] # drop these zero variance columns # Split this data into training and testing sets: # training = createDataPartition( permeability, p=0.8 ) fingerprints_training = fingerprints[training\$Resample1,] permeability_training = permeability[training\$Resample1] fingerprints_testing = fingerprints[-training\$Resample1,] permeability_testing = permeability[-training\$Resample1] # Build various nonlinear models and then compare performance: # # Note we use the default "trainControl" bootstrap evaluations for each of the models below: # preProc_Arguments = c("center","scale") # A K-NN model: # set.seed(0) knnModel = train(x=fingerprints_training, y=permeability_training, method="knn", preProc=preProc_Arguments, tuneLength=10) # predict on training/testing sets knnPred = predict(knnModel, newdata=fingerprints_training) knnPR = postResample(pred=knnPred, obs=permeability_training) rmses_training = c(knnPR[1]) r2s_training = c(knnPR[2]) methods = c("KNN") knnPred = predict(knnModel, newdata=fingerprints_testing) knnPR = postResample(pred=knnPred, obs=permeability_testing) rmses_testing = c(knnPR[1]) r2s_testing = c(knnPR[2]) # A Neural Network model: # nnGrid = expand.grid( .decay=c(0,0.01,0.1), .size=1:10, .bag=FALSE ) set.seed(0) nnetModel = train(x=fingerprints_training, y=permeability_training, method="nnet", preProc=preProc_Arguments, linout=TRUE,trace=FALSE,MaxNWts=10 * (ncol(fingerprints_training)+1) + 10 + 1, maxit=500) nnetPred = predict(nnetModel, newdata=fingerprints_training) nnetPR = postResample(pred=nnetPred, obs=permeability_training) rmses_training = c(rmses_training,nnetPR[1]) r2s_training = c(r2s_training,nnetPR[2]) methods = c(methods,"NN") nnetPred = predict(nnetModel, newdata=fingerprints_testing) nnetPR = postResample(pred=nnetPred, obs=permeability_testing) rmses_testing = c(rmses_testing,nnetPR[1]) r2s_testing = c(r2s_testing,nnetPR[2]) # Averaged Neural Network models: # set.seed(0) avNNetModel = train(x=fingerprints_training, y=permeability_training, method="avNNet", preProc=preProc_Arguments, linout=TRUE,trace=FALSE,MaxNWts=10 * (ncol(fingerprints_training)+1) + 10 + 1, maxit=500) avNNetPred = predict(avNNetModel, newdata=fingerprints_training) avNNetPR = postResample(pred=avNNetPred, obs=permeability_training) rmses_training = c(rmses_training,avNNetPR[1]) r2s_training = c(r2s_training,avNNetPR[2]) methods = c(methods,"AvgNN") avNNetPred = predict(avNNetModel, newdata=fingerprints_testing) avNNetPR = postResample(pred=avNNetPred, obs=permeability_testing) rmses_testing = c(rmses_testing,avNNetPR[1]) r2s_testing = c(r2s_testing,avNNetPR[2]) # MARS model: # marsGrid = expand.grid(.degree=1:2, .nprune=2:38) set.seed(0) marsModel = train(x=fingerprints_training, y=permeability_training, method="earth", preProc=preProc_Arguments, tuneGrid=marsGrid) marsPred = predict(marsModel, newdata=fingerprints_training) marsPR = postResample(pred=marsPred, obs=permeability_training) rmses_training = c(rmses_training,marsPR[1]) r2s_training = c(r2s_training,marsPR[2]) methods = c(methods,"MARS") marsPred = predict(marsModel, newdata=fingerprints_testing) marsPR = postResample(pred=marsPred, obs=permeability_testing) rmses_testing = c(rmses_testing,marsPR[1]) r2s_testing = c(r2s_testing,marsPR[2]) # Lets see what variables are most important in the MARS model: varImp(marsModel) # A Support Vector Machine (SVM): # set.seed(0) svmModel = train(x=fingerprints_training, y=permeability_training, method="svmRadial", preProc=preProc_Arguments, tuneLength=20) svmPred = predict(svmModel, newdata=fingerprints_training) svmPR = postResample(pred=svmPred, obs=permeability_training) rmses_training = c(rmses_training,svmPR[1]) r2s_training = c(r2s_training,svmPR[2]) methods = c(methods,"SVM") svmPred = predict(svmModel, newdata=fingerprints_testing) svmPR = postResample(pred=svmPred, obs=permeability_testing) rmses_testing = c(rmses_testing,svmPR[1]) r2s_testing = c(r2s_testing,svmPR[2]) # Package the results up: # res_training = data.frame( rmse=rmses_training, r2=r2s_training ) rownames(res_training) = methods training_order = order( -res_training\$rmse ) res_training = res_training[ training_order, ] # Order the dataframe so that the best results are at the bottom: print( "Final Training Results" ) print( res_training ) res_testing = data.frame( rmse=rmses_testing, r2=r2s_testing ) rownames(res_testing) = methods res_testing = res_testing[ training_order, ] # Order the dataframe so that the best results for the training set are at the bottom: print( "Final Testing Results" ) print( res_testing ) # EPage 82 resamp = resamples( list(svm=svmModel,knn=knnModel,nnet=nnetModel,avnnet=avNNetModel,mars=marsModel) ) print( summary(resamp) ) if( save_plots ){ postscript("../../WriteUp/Graphics/Chapter7/chap_7_prob_4_resamp_dotplot.eps", onefile=FALSE, horizontal=FALSE) } dotplot( resamp, metric="RMSE" ) if( save_plots ){ dev.off() } print( summary(diff(resamp)) )