MetaModeling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "┌ Info: Precompiling CombineML [f3117721-c1d3-5ff8-9230-e4dd1549a252]\n",
      "└ @ Base loading.jl:1192\n",
      "┌ Info: Running `conda install -y scikit-learn` in root environment\n",
      "└ @ Conda /Users/ppalmes/.julia/packages/Conda/CpuvI/src/Conda.jl:112\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting package metadata: ...working... done\n",
      "Solving environment: ...working... done\n",
      "\n",
      "# All requested packages already installed.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "┌ Warning: RCall.jl: Loading required package: lattice\n",
      "│ Loading required package: ggplot2\n",
      "└ @ RCall /Users/ppalmes/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: Warning: package ‘e1071’ was built under R version 3.5.2\n",
      "└ @ RCall /Users/ppalmes/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: Loading required package: splines\n",
      "│ Loading required package: foreach\n",
      "│ Loaded gam 1.16\n",
      "│ \n",
      "└ @ RCall /Users/ppalmes/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: randomForest 4.6-14\n",
      "│ Type rfNews() to see new features/changes/bug fixes.\n",
      "│ \n",
      "│ Attaching package: ‘randomForest’\n",
      "│ \n",
      "│ The following object is masked from ‘package:ggplot2’:\n",
      "│ \n",
      "│     margin\n",
      "│ \n",
      "└ @ RCall /Users/ppalmes/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: \n",
      "│ Attaching package: ‘kernlab’\n",
      "│ \n",
      "│ The following object is masked from ‘package:ggplot2’:\n",
      "│ \n",
      "│     alpha\n",
      "│ \n",
      "└ @ RCall /Users/ppalmes/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: \n",
      "│ Attaching package: ‘pls’\n",
      "│ \n",
      "│ The following object is masked from ‘package:caret’:\n",
      "│ \n",
      "│     R2\n",
      "│ \n",
      "│ The following object is masked from ‘package:stats’:\n",
      "│ \n",
      "│     loadings\n",
      "│ \n",
      "└ @ RCall /Users/ppalmes/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: Warning: package ‘xgboost’ was built under R version 3.5.2\n",
      "└ @ RCall /Users/ppalmes/.julia/packages/RCall/ffM0W/src/io.jl:113\n"
     ]
    }
   ],
   "source": [
    "import CombineML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "try\n",
    "  import RDatasets\n",
    "catch\n",
    "  using Pkg\n",
    "  Pkg.add(\"RDatasets\")\n",
    "  import RDatasets\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8-element Array{Int64,1}:\n",
       " 2\n",
       " 3\n",
       " 4\n",
       " 5\n",
       " 6\n",
       " 7\n",
       " 8\n",
       " 9"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "using Distributed\n",
    "addprocs()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[ Info: Running `conda install -y scikit-learn` in root environment\n",
      "[ Info: Running `conda install -y scikit-learn` in root environment\n",
      "[ Info: Running `conda install -y scikit-learn` in root environment\n",
      "[ Info: Running `conda install -y scikit-learn` in root environment\n",
      "[ Info: Running `conda install -y scikit-learn` in root environment\n",
      "[ Info: Running `conda install -y scikit-learn` in root environment\n",
      "[ Info: Running `conda install -y scikit-learn` in root environment\n",
      "[ Info: Running `conda install -y scikit-learn` in root environment\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      From worker 9:\tCollecting package metadata: ...working... done\n",
      "      From worker 3:\tCollecting package metadata: ...working... done\n",
      "      From worker 7:\tCollecting package metadata: ...working... done\n",
      "      From worker 2:\tCollecting package metadata: ...working... done\n",
      "      From worker 8:\tCollecting package metadata: ...working... done\n",
      "      From worker 6:\tCollecting package metadata: ...working... done\n",
      "      From worker 4:\tCollecting package metadata: ...working... done\n",
      "      From worker 5:\tCollecting package metadata: ...working... done\n",
      "      From worker 3:\tSolving environment: ...working... done\n",
      "      From worker 9:\tSolving environment: ...working... done\n",
      "      From worker 2:\tSolving environment: ...working... done\n",
      "      From worker 4:\tSolving environment: ...working... done\n",
      "      From worker 7:\tSolving environment: ...working... done\n",
      "      From worker 5:\tSolving environment: ...working... done\n",
      "      From worker 8:\tSolving environment: ...working... done\n",
      "      From worker 3:\t\n",
      "      From worker 3:\t# All requested packages already installed.\n",
      "      From worker 3:\t\n",
      "      From worker 6:\tSolving environment: ...working... done\n",
      "      From worker 9:\t\n",
      "      From worker 9:\t# All requested packages already installed.\n",
      "      From worker 9:\t\n",
      "      From worker 2:\t\n",
      "      From worker 2:\t# All requested packages already installed.\n",
      "      From worker 2:\t\n",
      "      From worker 4:\t\n",
      "      From worker 4:\t# All requested packages already installed.\n",
      "      From worker 4:\t\n",
      "      From worker 7:\t\n",
      "      From worker 7:\t# All requested packages already installed.\n",
      "      From worker 7:\t\n",
      "      From worker 5:\t\n",
      "      From worker 5:\t# All requested packages already installed.\n",
      "      From worker 5:\t\n",
      "      From worker 8:\t\n",
      "      From worker 8:\t# All requested packages already installed.\n",
      "      From worker 8:\t\n",
      "      From worker 6:\t\n",
      "      From worker 6:\t# All requested packages already installed.\n",
      "      From worker 6:\t\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "┌ Warning: RCall.jl: Loading required package: lattice\n",
      "│ Loading required package: ggplot2\n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: Loading required package: lattice\n",
      "│ Loading required package: ggplot2\n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: Loading required package: lattice\n",
      "│ Loading required package: ggplot2\n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: Loading required package: lattice\n",
      "│ Loading required package: ggplot2\n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: Loading required package: lattice\n",
      "│ Loading required package: ggplot2\n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: Loading required package: lattice\n",
      "│ Loading required package: ggplot2\n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: Loading required package: lattice\n",
      "│ Loading required package: ggplot2\n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: Loading required package: lattice\n",
      "│ Loading required package: ggplot2\n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: Warning: package ‘e1071’ was built under R version 3.5.2\n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: Loading required package: splines\n",
      "│ Loading required package: foreach\n",
      "│ Loaded gam 1.16\n",
      "│ \n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: randomForest 4.6-14\n",
      "│ Type rfNews() to see new features/changes/bug fixes.\n",
      "│ \n",
      "│ Attaching package: ‘randomForest’\n",
      "│ \n",
      "│ The following object is masked from ‘package:ggplot2’:\n",
      "│ \n",
      "│     margin\n",
      "│ \n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: Warning: package ‘e1071’ was built under R version 3.5.2\n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: Loading required package: splines\n",
      "│ Loading required package: foreach\n",
      "│ Loaded gam 1.16\n",
      "│ \n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: randomForest 4.6-14\n",
      "│ Type rfNews() to see new features/changes/bug fixes.\n",
      "│ \n",
      "│ Attaching package: ‘randomForest’\n",
      "│ \n",
      "│ The following object is masked from ‘package:ggplot2’:\n",
      "│ \n",
      "│     margin\n",
      "│ \n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: Warning: package ‘e1071’ was built under R version 3.5.2\n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: Warning: package ‘e1071’ was built under R version 3.5.2\n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: Loading required package: splines\n",
      "│ Loading required package: foreach\n",
      "│ Loaded gam 1.16\n",
      "│ \n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: randomForest 4.6-14\n",
      "│ Type rfNews() to see new features/changes/bug fixes.\n",
      "│ \n",
      "│ Attaching package: ‘randomForest’\n",
      "│ \n",
      "│ The following object is masked from ‘package:ggplot2’:\n",
      "│ \n",
      "│     margin\n",
      "│ \n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: Warning: package ‘e1071’ was built under R version 3.5.2\n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: Loading required package: splines\n",
      "│ Loading required package: foreach\n",
      "│ Loaded gam 1.16\n",
      "│ \n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: Warning: package ‘e1071’ was built under R version 3.5.2\n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: randomForest 4.6-14\n",
      "│ Type rfNews() to see new features/changes/bug fixes.\n",
      "│ \n",
      "│ Attaching package: ‘randomForest’\n",
      "│ \n",
      "│ The following object is masked from ‘package:ggplot2’:\n",
      "│ \n",
      "│     margin\n",
      "│ \n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: Loading required package: splines\n",
      "│ Loading required package: foreach\n",
      "│ Loaded gam 1.16\n",
      "│ \n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: randomForest 4.6-14\n",
      "│ Type rfNews() to see new features/changes/bug fixes.\n",
      "│ \n",
      "│ Attaching package: ‘randomForest’\n",
      "│ \n",
      "│ The following object is masked from ‘package:ggplot2’:\n",
      "│ \n",
      "│     margin\n",
      "│ \n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: Loading required package: splines\n",
      "│ Loading required package: foreach\n",
      "│ Loaded gam 1.16\n",
      "│ \n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: randomForest 4.6-14\n",
      "│ Type rfNews() to see new features/changes/bug fixes.\n",
      "│ \n",
      "│ Attaching package: ‘randomForest’\n",
      "│ \n",
      "│ The following object is masked from ‘package:ggplot2’:\n",
      "│ \n",
      "│     margin\n",
      "│ \n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: \n",
      "│ Attaching package: ‘kernlab’\n",
      "│ \n",
      "│ The following object is masked from ‘package:ggplot2’:\n",
      "│ \n",
      "│     alpha\n",
      "│ \n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: \n",
      "│ Attaching package: ‘pls’\n",
      "│ \n",
      "│ The following object is masked from ‘package:caret’:\n",
      "│ \n",
      "│     R2\n",
      "│ \n",
      "│ The following object is masked from ‘package:stats’:\n",
      "│ \n",
      "│     loadings\n",
      "│ \n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: Warning: package ‘xgboost’ was built under R version 3.5.2\n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: Warning: package ‘e1071’ was built under R version 3.5.2\n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: Loading required package: splines\n",
      "│ Loading required package: foreach\n",
      "│ Loaded gam 1.16\n",
      "│ \n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: randomForest 4.6-14\n",
      "│ Type rfNews() to see new features/changes/bug fixes.\n",
      "│ \n",
      "│ Attaching package: ‘randomForest’\n",
      "│ \n",
      "│ The following object is masked from ‘package:ggplot2’:\n",
      "│ \n",
      "│     margin\n",
      "│ \n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: \n",
      "│ Attaching package: ‘kernlab’\n",
      "│ \n",
      "│ The following object is masked from ‘package:ggplot2’:\n",
      "│ \n",
      "│     alpha\n",
      "│ \n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: \n",
      "│ Attaching package: ‘pls’\n",
      "│ \n",
      "│ The following object is masked from ‘package:caret’:\n",
      "│ \n",
      "│     R2\n",
      "│ \n",
      "│ The following object is masked from ‘package:stats’:\n",
      "│ \n",
      "│     loadings\n",
      "│ \n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: Warning: package ‘e1071’ was built under R version 3.5.2\n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: Warning: package ‘xgboost’ was built under R version 3.5.2\n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: \n",
      "│ Attaching package: ‘kernlab’\n",
      "│ \n",
      "│ The following object is masked from ‘package:ggplot2’:\n",
      "│ \n",
      "│     alpha\n",
      "│ \n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: Loading required package: splines\n",
      "│ Loading required package: foreach\n",
      "│ Loaded gam 1.16\n",
      "│ \n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: \n",
      "│ Attaching package: ‘kernlab’\n",
      "│ \n",
      "│ The following object is masked from ‘package:ggplot2’:\n",
      "│ \n",
      "│     alpha\n",
      "│ \n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: randomForest 4.6-14\n",
      "│ Type rfNews() to see new features/changes/bug fixes.\n",
      "│ \n",
      "│ Attaching package: ‘randomForest’\n",
      "│ \n",
      "│ The following object is masked from ‘package:ggplot2’:\n",
      "│ \n",
      "│     margin\n",
      "│ \n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: \n",
      "│ Attaching package: ‘pls’\n",
      "│ \n",
      "│ The following object is masked from ‘package:caret’:\n",
      "│ \n",
      "│     R2\n",
      "│ \n",
      "│ The following object is masked from ‘package:stats’:\n",
      "│ \n",
      "│     loadings\n",
      "│ \n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: \n",
      "│ Attaching package: ‘kernlab’\n",
      "│ \n",
      "│ The following object is masked from ‘package:ggplot2’:\n",
      "│ \n",
      "│     alpha\n",
      "│ \n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: \n",
      "│ Attaching package: ‘pls’\n",
      "│ \n",
      "│ The following object is masked from ‘package:caret’:\n",
      "│ \n",
      "│     R2\n",
      "│ \n",
      "│ The following object is masked from ‘package:stats’:\n",
      "│ \n",
      "│     loadings\n",
      "│ \n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: Warning: package ‘xgboost’ was built under R version 3.5.2\n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: \n",
      "│ Attaching package: ‘kernlab’\n",
      "│ \n",
      "│ The following object is masked from ‘package:ggplot2’:\n",
      "│ \n",
      "│     alpha\n",
      "│ \n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: \n",
      "│ Attaching package: ‘pls’\n",
      "│ \n",
      "│ The following object is masked from ‘package:caret’:\n",
      "│ \n",
      "│     R2\n",
      "│ \n",
      "│ The following object is masked from ‘package:stats’:\n",
      "│ \n",
      "│     loadings\n",
      "│ \n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: Warning: package ‘xgboost’ was built under R version 3.5.2\n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "┌ Warning: RCall.jl: \n",
      "│ Attaching package: ‘pls’\n",
      "│ \n",
      "│ The following object is masked from ‘package:caret’:\n",
      "│ \n",
      "│     R2\n",
      "│ \n",
      "│ The following object is masked from ‘package:stats’:\n",
      "│ \n",
      "│     loadings\n",
      "│ \n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: Warning: package ‘xgboost’ was built under R version 3.5.2\n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: Warning: package ‘xgboost’ was built under R version 3.5.2\n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: \n",
      "│ Attaching package: ‘kernlab’\n",
      "│ \n",
      "│ The following object is masked from ‘package:ggplot2’:\n",
      "│ \n",
      "│     alpha\n",
      "│ \n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: \n",
      "│ Attaching package: ‘pls’\n",
      "│ \n",
      "│ The following object is masked from ‘package:caret’:\n",
      "│ \n",
      "│     R2\n",
      "│ \n",
      "│ The following object is masked from ‘package:stats’:\n",
      "│ \n",
      "│     loadings\n",
      "│ \n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: Warning: package ‘xgboost’ was built under R version 3.5.2\n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: \n",
      "│ Attaching package: ‘kernlab’\n",
      "│ \n",
      "│ The following object is masked from ‘package:ggplot2’:\n",
      "│ \n",
      "│     alpha\n",
      "│ \n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: \n",
      "│ Attaching package: ‘pls’\n",
      "│ \n",
      "│ The following object is masked from ‘package:caret’:\n",
      "│ \n",
      "│     R2\n",
      "│ \n",
      "│ The following object is masked from ‘package:stats’:\n",
      "│ \n",
      "│     loadings\n",
      "│ \n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n",
      "┌ Warning: RCall.jl: Warning: package ‘xgboost’ was built under R version 3.5.2\n",
      "└ @ RCall ~/.julia/packages/RCall/ffM0W/src/io.jl:113\n"
     ]
    }
   ],
   "source": [
    "@everywhere using Random\n",
    "@everywhere using Statistics\n",
    "@everywhere import CombineML\n",
    "@everywhere import CombineML.Util\n",
    "@everywhere import CombineML.Transformers\n",
    "@everywhere import RDatasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "@everywhere CU=CombineML.Util\n",
    "@everywhere CT=CombineML.Transformers\n",
    "@everywhere RD=RDatasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "93.33333333333333"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@everywhere function predict(learner)\n",
    "    dataset = RD.dataset(\"datasets\", \"iris\")\n",
    "    features = convert(Matrix,dataset[:, 1:(end-1)])\n",
    "    labels = convert(Array,dataset[:, end])\n",
    "    # Split into training and test sets\n",
    "    (train_ind, test_ind) = CU.holdout(size(features, 1), 0.3)\n",
    "    # Create pipeline\n",
    "    pipeline = CT.Pipeline(\n",
    "      Dict(\n",
    "        :transformers => [\n",
    "          CT.OneHotEncoder(), # Encodes nominal features into numeric\n",
    "          CT.Imputer(), # Imputes NA values\n",
    "          CT.StandardScaler(), # Standardizes features\n",
    "          learner # Predicts labels on instances\n",
    "        ]\n",
    "      )\n",
    "    )\n",
    "    # Train\n",
    "    CT.fit!(pipeline, features[train_ind, :], labels[train_ind]);\n",
    "    # Predict\n",
    "    predictions = CT.transform!(pipeline, features[test_ind, :]);\n",
    "    # Assess predictions\n",
    "    result = CU.score(:accuracy, labels[test_ind], predictions)\n",
    "    return result\n",
    "end\n",
    "# Learner with default settings\n",
    "learner = CT.PrunedTree()\n",
    "predict(learner)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "95.55555555555556"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner = CT.PrunedTree(\n",
    "  Dict(\n",
    "    :output => :class,\n",
    "    :impl_options => Dict(:purity_threshold => 1.0)\n",
    "  )\n",
    ")\n",
    "predict(learner)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "97.77777777777777"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner = CT.VoteEnsemble(\n",
    "  Dict(\n",
    "    :output => :class,\n",
    "    :learners => [CT.PrunedTree(), CT.DecisionStumpAdaboost(), CT.RandomForest()]\n",
    "  )\n",
    ")\n",
    "predict(learner)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "95.55555555555556"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner = CT.BestLearner(\n",
    "  Dict(\n",
    "    :output => :class,\n",
    "    #:partition_generator => (instances, labels) -> CU.kfold(size(instances, 1), 5),\n",
    "    #:selection_function => (learner_partition_scores) -> findmax(mean(learner_partition_scores, 2))[2],\n",
    "    :score_type => Real,\n",
    "    :learners => [CT.PrunedTree(), CT.DecisionStumpAdaboost(), CT.RandomForest()],\n",
    "    :learner_options_grid => nothing\n",
    "  )\n",
    ")\n",
    "predict(learner)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "95.55555555555556"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@everywhere rf = CT.RandomForest(\n",
    "  Dict(\n",
    "    :output => :class,\n",
    "    :impl_options => Dict(\n",
    "      :num_trees => 100,\n",
    "      :partial_sampling => 0.7\n",
    "    )\n",
    "  )\n",
    ")\n",
    "predict(rf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "86.66666666666667"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@everywhere ada = CT.DecisionStumpAdaboost(\n",
    "  Dict(\n",
    "    :output => :class,\n",
    "    :impl_options => Dict(:num_iterations => 7)\n",
    "  )\n",
    ")\n",
    "predict(ada)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "97.77777777777777"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if CombineML.System.LIB_SKL_AVAILABLE\n",
    "  @everywhere sk_learner = CT.SKLLearner(\n",
    "   Dict(\n",
    "        :output => :class,\n",
    "        #:learner => \"BaggingClassifier\",\n",
    "        #:learner => \"ExtraTreesClassifier\",\n",
    "        :learner => \"GradientBoostingClassifier\",\n",
    "        :impl_options => Dict()\n",
    "       )\n",
    "  )\n",
    "else\n",
    "  @everywhere sk_learner = CT.Baseline()\n",
    "end\n",
    "predict(sk_learner)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "97.77777777777777"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if CombineML.System.LIB_CRT_AVAILABLE\n",
    "  @everywhere crt_learner = CT.CRTLearner(\n",
    "    Dict(\n",
    "      :output => :class,\n",
    "      #:learner => \"rf\",\n",
    "      :learner => \"svmLinear2\",\n",
    "      #:learner => \"rpart\",\n",
    "      :impl_options => Dict()\n",
    "    )\n",
    "  )\n",
    "else\n",
    "  @everywhere crt_learner = CT.Baseline()\n",
    "end\n",
    "predict(crt_learner)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "93.33333333333333"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@everywhere stacklearner = CT.StackEnsemble(Dict(\n",
    "    :output => :class,\n",
    "    :learners => [CT.PrunedTree(), CT.RandomForest(),CT.DecisionStumpAdaboost(),rf,crt_learner,sk_learner],\n",
    "    :stacker =>\n",
    "       CT.RandomForest(\n",
    "          Dict(\n",
    "             :output => :class,\n",
    "             :impl_options => Dict(\n",
    "               :num_trees => 20,\n",
    "               :partial_sampling => 0.7\n",
    "             )\n",
    "          )\n",
    "       ),\n",
    "    #:stacker_training_proportion => 0.3,\n",
    "    :keep_original_features => true\n",
    "))\n",
    "predict(stacklearner)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "93.33333333333333"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@everywhere bestlearner = CT.BestLearner(\n",
    "  Dict(\n",
    "    :learners => [stacklearner,CT.PrunedTree(), CT.RandomForest(),CT.DecisionStumpAdaboost(),rf,sk_learner,crt_learner],\n",
    "    :output => :class,\n",
    "    :score_type => Real,\n",
    "    :learner_options_grid => nothing\n",
    "  )\n",
    ")\n",
    "predict(bestlearner)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "95.55555555555556"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@everywhere votelearner = CT.VoteEnsemble(\n",
    "  Dict(\n",
    "    :learners => [stacklearner,CT.PrunedTree(), CT.RandomForest(),CT.DecisionStumpAdaboost(),rf,crt_learner,sk_learner],\n",
    "    :output => :class,\n",
    "  )\n",
    ")\n",
    "predict(votelearner)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "97.77777777777777"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@everywhere stackstacklearner = CT.StackEnsemble(\n",
    "  Dict(\n",
    "    :learners => [stacklearner,votelearner,CT.PrunedTree(), CT.RandomForest(),CT.DecisionStumpAdaboost(),rf,crt_learner,sk_learner],\n",
    "    #:keep_original_features => false,\n",
    "    #:stacker_training_proportion => 0.3,\n",
    "    :stacker =>\n",
    "       CT.RandomForest(\n",
    "          Dict(\n",
    "             :output => :class,\n",
    "             :impl_options => Dict(\n",
    "               :num_trees => 20,\n",
    "               :partial_sampling => 0.7\n",
    "             )\n",
    "          )\n",
    "       )\n",
    "  )\n",
    ")\n",
    "predict(stackstacklearner)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      From worker 3:\tbaseline => 29.0\n",
      "      From worker 3:\tbaseline => 29.0\n",
      "      From worker 2:\tbaseline => 27.0\n",
      "      From worker 2:\tbaseline => 27.0\n",
      "      From worker 4:\tscikitlearner => 98.0\n",
      "      From worker 8:\tjuliaprunetree => 93.0\n",
      "      From worker 9:\tbaseline => 29.0\n",
      "      From worker 7:\tbaseline => 22.0\n",
      "      From worker 8:\tjuliaprunetree => 96.0\n",
      "      From worker 2:\tjuliaprunetree => 93.0\n",
      "      From worker 2:\tjuliaprunetree => 100.0\n",
      "      From worker 3:\tjuliaprunetree => 96.0\n",
      "      From worker 9:\tjuliaprunetree => 96.0\n",
      "      From worker 7:\tjuliaprunetree => 96.0\n",
      "      From worker 8:\tbaseline => 27.0\n",
      "      From worker 5:\tbaseline => 31.0\n",
      "      From worker 5:\tjuliaprunetree => 93.0\n",
      "      From worker 4:\tscikitlearner => 93.0\n",
      "      From worker 4:\tbaseline => 31.0\n",
      "      From worker 4:\tjuliaprunetree => 96.0\n",
      "      From worker 6:\tjuliaadaboost => 93.0\n",
      "      From worker 7:\tscikitlearner => 91.0\n",
      "      From worker 2:\tscikitlearner => 98.0\n",
      "      From worker 3:\tscikitlearner => 91.0\n",
      "      From worker 9:\tscikitlearner => 93.0\n",
      "      From worker 2:\tscikitlearner => 98.0\n",
      "      From worker 5:\tscikitlearner => 98.0\n",
      "      From worker 8:\tscikitlearner => 98.0\n",
      "      From worker 6:\tjuliaadaboost => 98.0\n",
      "      From worker 6:\tscikitlearner => 96.0\n",
      "      From worker 6:\tbaseline => 24.0\n",
      "      From worker 6:\tjuliaprunetree => 93.0\n",
      "      From worker 2:\tjuliaadaboost => 93.0\n",
      "      From worker 2:\tjuliaadaboost => 91.0\n",
      "      From worker 2:\tjuliarandomforest => 96.0\n",
      "      From worker 5:\tjuliaadaboost => 89.0\n",
      "      From worker 5:\tjuliarandomforest => 93.0\n",
      "      From worker 7:\tjuliaadaboost => 93.0\n",
      "      From worker 7:\tjuliarandomforest => 91.0\n",
      "      From worker 9:\tjuliaadaboost => 93.0\n",
      "      From worker 6:\tjuliarandomforest => 100.0\n",
      "      From worker 9:\tjuliarandomforest => 87.0\n",
      "      From worker 3:\tjuliaadaboost => 98.0\n",
      "      From worker 3:\tjuliarandomforest => 93.0\n",
      "      From worker 4:\tjuliaadaboost => 96.0\n",
      "      From worker 4:\tjuliarandomforest => 91.0\n",
      "      From worker 8:\tjuliaadaboost => 96.0\n",
      "      From worker 8:\tjuliarandomforest => 93.0\n",
      "      From worker 2:\tjuliarandomforest => 98.0\n",
      "      From worker 2:\tstacklearner => 93.0\n",
      "      From worker 7:\tjuliarandomforest => 98.0\n",
      "      From worker 4:\tstacklearner => 98.0\n",
      "      From worker 3:\tstacklearner => 89.0\n",
      "      From worker 7:\tstacklearner => 98.0\n",
      "      From worker 5:\tstacklearner => 93.0\n",
      "      From worker 9:\tstacklearner => 98.0\n",
      "      From worker 8:\tstacklearner => 98.0\n",
      "      From worker 7:\tvotelearner => 93.0\n",
      "      From worker 9:\tvotelearner => 91.0\n",
      "      From worker 6:\tstacklearner => 98.0\n",
      "      From worker 3:\tvotelearner => 96.0\n",
      "      From worker 3:\tstacklearner => 93.0\n",
      "      From worker 4:\tvotelearner => 96.0\n",
      "      From worker 5:\tvotelearner => 100.0\n",
      "      From worker 8:\tvotelearner => 96.0\n",
      "      From worker 2:\tstacklearner => 93.0\n",
      "      From worker 2:\tvotelearner => 96.0\n",
      "      From worker 2:\tcaretlearner => 89.0\n",
      "      From worker 2:\tcaretlearner => 100.0\n",
      "      From worker 8:\tcaretlearner => 93.0\n",
      "      From worker 5:\tcaretlearner => 91.0\n",
      "      From worker 7:\tcaretlearner => 96.0\n",
      "      From worker 3:\tcaretlearner => 96.0\n",
      "      From worker 3:\tcaretlearner => 91.0\n",
      "      From worker 4:\tcaretlearner => 98.0\n",
      "      From worker 9:\tcaretlearner => 93.0\n",
      "      From worker 6:\tvotelearner => 98.0\n",
      "      From worker 6:\tcaretlearner => 96.0\n",
      "      From worker 4:\tstackstacklearner => 93.0\n",
      "      From worker 5:\tstackstacklearner => 100.0\n",
      "      From worker 8:\tstackstacklearner => 93.0\n",
      "      From worker 9:\tvotelearner => 93.0\n",
      "      From worker 3:\tstackstacklearner => 93.0\n",
      "      From worker 7:\tstackstacklearner => 98.0\n",
      "      From worker 9:\tstackstacklearner => 93.0\n",
      "      From worker 2:\tvotelearner => 98.0\n",
      "      From worker 6:\tstackstacklearner => 89.0\n",
      "      From worker 2:\tstackstacklearner => 98.0\n",
      "      From worker 5:\tstackstacklearner => 98.0\n",
      "      From worker 2:\tstackstacklearner => 98.0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>model</th><th>mean_acc</th><th>std_acc</th><th>trials</th></tr><tr><th></th><th>Any</th><th>Any</th><th>Any</th><th>Any</th></tr></thead><tbody><p>9 rows × 4 columns</p><tr><th>1</th><td>votelearner</td><td>96.0</td><td>3.0</td><td>10</td></tr><tr><th>2</th><td>stacklearner</td><td>95.0</td><td>3.0</td><td>10</td></tr><tr><th>3</th><td>scikitlearner</td><td>95.0</td><td>3.0</td><td>10</td></tr><tr><th>4</th><td>stackstacklearner</td><td>95.0</td><td>3.0</td><td>10</td></tr><tr><th>5</th><td>juliaprunetree</td><td>95.0</td><td>2.0</td><td>10</td></tr><tr><th>6</th><td>caretlearner</td><td>94.0</td><td>3.0</td><td>10</td></tr><tr><th>7</th><td>juliaadaboost</td><td>94.0</td><td>3.0</td><td>10</td></tr><tr><th>8</th><td>juliarandomforest</td><td>94.0</td><td>4.0</td><td>10</td></tr><tr><th>9</th><td>baseline</td><td>28.0</td><td>3.0</td><td>10</td></tr></tbody></table>"
      ],
      "text/latex": [
       "\\begin{tabular}{r|cccc}\n",
       "\t& model & mean\\_acc & std\\_acc & trials\\\\\n",
       "\t\\hline\n",
       "\t& Any & Any & Any & Any\\\\\n",
       "\t\\hline\n",
       "\t1 & votelearner & 96.0 & 3.0 & 10 \\\\\n",
       "\t2 & stacklearner & 95.0 & 3.0 & 10 \\\\\n",
       "\t3 & scikitlearner & 95.0 & 3.0 & 10 \\\\\n",
       "\t4 & stackstacklearner & 95.0 & 3.0 & 10 \\\\\n",
       "\t5 & juliaprunetree & 95.0 & 2.0 & 10 \\\\\n",
       "\t6 & caretlearner & 94.0 & 3.0 & 10 \\\\\n",
       "\t7 & juliaadaboost & 94.0 & 3.0 & 10 \\\\\n",
       "\t8 & juliarandomforest & 94.0 & 4.0 & 10 \\\\\n",
       "\t9 & baseline & 28.0 & 3.0 & 10 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/plain": [
       "9×4 DataFrame\n",
       "│ Row │ model             │ mean_acc │ std_acc │ trials │\n",
       "│     │ \u001b[90mAny\u001b[39m               │ \u001b[90mAny\u001b[39m      │ \u001b[90mAny\u001b[39m     │ \u001b[90mAny\u001b[39m    │\n",
       "├─────┼───────────────────┼──────────┼─────────┼────────┤\n",
       "│ 1   │ votelearner       │ 96.0     │ 3.0     │ 10     │\n",
       "│ 2   │ stacklearner      │ 95.0     │ 3.0     │ 10     │\n",
       "│ 3   │ scikitlearner     │ 95.0     │ 3.0     │ 10     │\n",
       "│ 4   │ stackstacklearner │ 95.0     │ 3.0     │ 10     │\n",
       "│ 5   │ juliaprunetree    │ 95.0     │ 2.0     │ 10     │\n",
       "│ 6   │ caretlearner      │ 94.0     │ 3.0     │ 10     │\n",
       "│ 7   │ juliaadaboost     │ 94.0     │ 3.0     │ 10     │\n",
       "│ 8   │ juliarandomforest │ 94.0     │ 4.0     │ 10     │\n",
       "│ 9   │ baseline          │ 28.0     │ 3.0     │ 10     │"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "using DataFrames\n",
    "function main(trials)\n",
    "    learners=Dict(:votelearner=>votelearner,:stacklearner=>stacklearner,\n",
    "                  :stackstacklearner=>stackstacklearner,\n",
    "                  :caretlearner=>crt_learner,\n",
    "                  :scikitlearner=>sk_learner,\n",
    "                  :juliaprunetree=>CT.PrunedTree(),\n",
    "                  :juliaadaboost=>CT.DecisionStumpAdaboost(),\n",
    "                  :baseline=>CT.Baseline(),\n",
    "                  :juliarandomforest=>CT.RandomForest())\n",
    "    models=collect(keys(learners))\n",
    "    ctable=@distributed (vcat) for model in models\n",
    "        acc=@distributed (vcat) for i=1:trials\n",
    "            res=predict(learners[model])\n",
    "            println(model,\" => \",round(res))\n",
    "            res\n",
    "        end\n",
    "        [model round(mean(acc)) round(std(acc)) length(acc)]\n",
    "    end\n",
    "    sorted=sort(DataFrame(ctable),:x2,rev=true)\n",
    "    rename!(sorted,Dict(:x1=>:model,:x2=>:mean_acc,:x3=>:std_acc,:x4=>:trials))\n",
    "    return sorted\n",
    "end\n",
    "res=main(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 1.0.3",
   "language": "julia",
   "name": "julia-1.0"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.0.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}