update to 1.0.0

Dockerfile

@@ -1,77 +1,83 @@
 # kaggle/julia dockerfile
 
-FROM ubuntu:16.04
-
+FROM julia:1.0.0
 
 ADD package_installs.jl /tmp/package_installs.jl
+ADD test_build.jl /tmp/test_build.jl
+
+RUN apt-get update && \
+    apt-get install -y build-essential gettext git hdf5-tools libcairo2 libpango1.0-0 python3 python3-dev python3-pip
+
+# Pycall
+ENV PYTHON /usr/bin/python3
 
-RUN  apt-get update && \
-     apt-get install git software-properties-common curl wget libcairo2 libpango1.0-0 -y && \
-     add-apt-repository ppa:staticfloat/julia-deps -y && \
-     apt-get update -y && \
-     apt-get install -y libpcre3-dev build-essential && \
-     apt-get install -y gettext hdf5-tools && \
-     apt-get install -y gfortran python && \
-     apt-get install -y m4 cmake libssl-dev && \
-     cd /usr/local/src && git clone https://github.com/JuliaLang/julia.git && \
-     cd julia && git checkout v0.6.2 && \
-     # Use generic instruction set; see https://github.com/JuliaLang/julia/pull/6220
-     #   and https://groups.google.com/forum/#!topic/julia-dev/Eqp0GhZWxME
-     echo "JULIA_CPU_TARGET=core2" > Make.user && \
-     echo "OPENBLAS_TARGET_ARCH=NEHALEM" > Make.user && \
-     make -j 4 julia-deps && make -j 4 && make install && \
-     ln -s /usr/local/src/julia/julia /usr/local/bin/julia
+# Conda
+ENV CONDA_JL_VERSION 3
 
-ENV JULIA_PKGDIR /root/.julia/v0.6
+ENV JULIA_PKGDIR /root/.julia/
 
 RUN julia /tmp/package_installs.jl
 
 # IJulia
-RUN   apt-get update && apt-get install -y python3-pip python3-dev && pip3 install jupyter && \
-        julia -e "Pkg.add(\"Nettle\")" && \
-        julia -e "Pkg.add(\"IJulia\")" && \
-        julia -e "Pkg.build(\"IJulia\")" && \
-# Make sure Jupyter won't try to migrate old settings
-        mkdir -p /root/.jupyter/kernels && \
-        cp -r /root/.local/share/jupyter/kernels/julia-0.6 /root/.jupyter/kernels && \
-        touch /root/.jupyter/jupyter_nbconvert_config.py && touch /root/.jupyter/migrated && \
-        julia -e "Base.compilecache(\"IJulia\")" && \
-        julia -e "Base.compilecache(\"ZMQ\")" && \
-        julia -e "Base.compilecache(\"Nettle\")"
+RUN pip3 install jupyter && \
+    julia -e "using Pkg;Pkg.add(\"Nettle\")" && \
+    julia -e "using Pkg;Pkg.add(\"IJulia\")" && \
+    # Make sure Jupyter won't try to migrate old settings
+    mkdir -p /root/.jupyter/kernels && \
+    cp -r /root/.local/share/jupyter/kernels/julia-1.0 /root/.jupyter/kernels && \
+    touch /root/.jupyter/jupyter_nbconvert_config.py && touch /root/.jupyter/migrated
+
+# TensorFlow
+RUN pip3 install tensorflow && \
+    julia -e "using Pkg;Pkg.add(\"TensorFlow\")"
+
+RUN julia -e "using Pkg;Pkg.status()"
+
+RUN julia /tmp/test_build.jl
 
-RUN julia -e "Base.compilecache(\"BinDeps\")" && \
-    julia -e "Base.compilecache(\"Cairo\")" && \
-    julia -e "Base.compilecache(\"Calculus\")" && \
-    julia -e "Base.compilecache(\"Clustering\")" && \
-    julia -e "Base.compilecache(\"Compose\")" && \
-    julia -e "Base.compilecache(\"DataArrays\")" && \
-    julia -e "Base.compilecache(\"DataFrames\")" && \
-    julia -e "Base.compilecache(\"DataFramesMeta\")" && \
-    julia -e "Base.compilecache(\"Dates\")" && \
-    julia -e "Base.compilecache(\"DecisionTree\")" && \
-    julia -e "Base.compilecache(\"Distributions\")" && \
-    julia -e "Base.compilecache(\"Distances\")" && \
-    julia -e "Base.compilecache(\"GLM\")" && \
-    julia -e "Base.compilecache(\"HDF5\")" && \
-    julia -e "Base.compilecache(\"HypothesisTests\")" && \
-    julia -e "Base.compilecache(\"JSON\")" && \
-    julia -e "Base.compilecache(\"KernelDensity\")" && \
-    julia -e "Base.compilecache(\"Loess\")" && \
-    #julia -e "Base.compilecache(\"Lora\")" && \
-    julia -e "Base.compilecache(\"MLBase\")" && \
-    julia -e "Base.compilecache(\"MultivariateStats\")" && \
-    julia -e "Base.compilecache(\"NMF\")" && \
-    julia -e "Base.compilecache(\"Optim\")" && \
-    julia -e "Base.compilecache(\"PDMats\")" && \
-    julia -e "Base.compilecache(\"RDatasets\")" && \
-    julia -e "Base.compilecache(\"SQLite\")" && \
-    julia -e "Base.compilecache(\"StatsBase\")" && \
-    julia -e "Base.compilecache(\"TextAnalysis\")" && \
-    julia -e "Base.compilecache(\"TimeSeries\")" && \
-    julia -e "Base.compilecache(\"ZipFile\")" && \
-    julia -e "Base.compilecache(\"Gadfly\")" && \
-    julia -e "Base.compilecache(\"MLBase\")" && \
-    julia -e "Base.compilecache(\"Clustering\")" && \
+RUN julia -e "Base.compilecache(Base.identify_package(\"BinDeps\"))" && \
+    julia -e "Base.compilecache(Base.identify_package(\"Cairo\"))" && \
+    julia -e "Base.compilecache(Base.identify_package(\"Calculus\"))" && \
+    julia -e "Base.compilecache(Base.identify_package(\"Clustering\"))" && \
+    # https://github.com/GiovineItalia/Compose.jl/pull/282
+    # julia -e "Base.compilecache(Base.identify_package(\"Compose\"))" && \
+    # https://github.com/JuliaStats/DataArrays.jl/pull/304
+    # julia -e "Base.compilecache(Base.identify_package(\"DataArrays\"))" && \
+    julia -e "Base.compilecache(Base.identify_package(\"DataFrames\"))" && \
+    julia -e "Base.compilecache(Base.identify_package(\"DataFramesMeta\"))" && \
+    julia -e "Base.compilecache(Base.identify_package(\"DecisionTree\"))" && \
+    julia -e "Base.compilecache(Base.identify_package(\"Distributions\"))" && \
+    julia -e "Base.compilecache(Base.identify_package(\"Distances\"))" && \
+    # https://github.com/GiovineItalia/Gadfly.jl/issues/1158
+    # julia -e "Base.compilecache(Base.identify_package(\"Gadfly\"))" && \
+    # https://github.com/JuliaStats/GLM.jl/pull/242
+    # julia -e "Base.compilecache(Base.identify_package(\"GLM\"))" && \
+    julia -e "Base.compilecache(Base.identify_package(\"HDF5\"))" && \
+    julia -e "Base.compilecache(Base.identify_package(\"HypothesisTests\"))" && \
+    julia -e "Base.compilecache(Base.identify_package(\"IJulia\"))" && \
+    julia -e "Base.compilecache(Base.identify_package(\"JSON\"))" && \
+    julia -e "Base.compilecache(Base.identify_package(\"KernelDensity\"))" && \
+    julia -e "Base.compilecache(Base.identify_package(\"Loess\"))" && \
+    julia -e "Base.compilecache(Base.identify_package(\"MLBase\"))" && \
+    # https://github.com/JuliaStats/MultivariateStats.jl/issues/69 wait for new release
+    # julia -e "Base.compilecache(Base.identify_package(\"MultivariateStats\"))" && \
+    julia -e "Base.compilecache(Base.identify_package(\"Nettle\"))" && \
+    # https://github.com/JuliaStats/NMF.jl/pull/24
+    # julia -e "Base.compilecache(Base.identify_package(\"NMF\"))" && \
+    julia -e "Base.compilecache(Base.identify_package(\"Optim\"))" && \
+    julia -e "Base.compilecache(Base.identify_package(\"PDMats\"))" && \
+    julia -e "Base.compilecache(Base.identify_package(\"RDatasets\"))" && \
+    julia -e "Base.compilecache(Base.identify_package(\"SQLite\"))" && \
+    julia -e "Base.compilecache(Base.identify_package(\"StatsBase\"))" && \
+    julia -e "Base.compilecache(Base.identify_package(\"TensorFlow\"))" && \
+    # https://github.com/JuliaText/TextAnalysis.jl/releases
+    # julia -e "Base.compilecache(Base.identify_package(\"TextAnalysis\"))" && \
+    # https://github.com/JuliaStats/TimeSeries.jl/pull/370
+    # julia -e "Base.compilecache(Base.identify_package(\"TimeSeries\"))" && \
+    # https://github.com/TuringLang/Turing.jl/pull/469 wait for new release
+    # julia -e "Base.compilecache(Base.identify_package(\"Turing\"))" && \
+    julia -e "Base.compilecache(Base.identify_package(\"ZipFile\"))" && \
+    julia -e "Base.compilecache(Base.identify_package(\"ZMQ\"))" && \
     julia -e "using IJulia"
 
 CMD ["julia"]

build

@@ -6,7 +6,7 @@ set -e
 if [[ "$1" ==  "--use-cache" ]]; then
     docker build --rm -t kaggle/julia-build .
 else
-    docker pull ubuntu:16.04
+    docker pull julia:1.0.0
     docker build --rm --no-cache -t kaggle/julia-build .
 fi
 

package_installs.jl

@@ -1,15 +1,18 @@
+using Pkg
+Pkg.update()
+
 metadata_packages = [
     "BinDeps",
     "Bootstrap",
     "Cairo",
     "Calculus",
     "Clustering",
+    "Compose",
+    "Conda",
     "CSV",
     "DataArrays",
     "DataFrames",
     "DataFramesMeta",
-    "DataStreams",
-    "Dates",
     "DecisionTree",
     "Distributions",
     "Distances",
@@ -28,6 +31,7 @@ metadata_packages = [
     "ManifoldLearning",
     "MLBase",
     "MultivariateStats",
+    "Loess",
     "NMF",
     "OnlineStats",
     "Optim",
@@ -39,30 +43,13 @@ metadata_packages = [
     "RDatasets",
     "SQLite",
     "StatsBase",
-    "TensorFlow",
     "TextAnalysis",
     "TSne",
-    "Turing",
+    # "Turing",
     "TimeSeries",
     "Query",
-    "ZipFile"]
-
-
-Pkg.init()
-Pkg.update()
-
-for package=metadata_packages
-    Pkg.add(package)
-end
-
-# need to build XGBoost version for it to work
-Pkg.clone("https://github.com/antinucleon/XGBoost.jl.git")
-Pkg.build("XGBoost")
-
-Pkg.clone("https://github.com/benhamner/MachineLearning.jl")
-Pkg.pin("MachineLearning")
-
-Pkg.clone("https://github.com/Allardvm/LightGBM.jl.git")
-ENV["LIGHTGBM_PATH"] = "../LightGBM"
+    # "XGBoost",
+    "ZipFile",
+    "ZMQ"]
 
-Pkg.resolve()
+Pkg.add(metadata_packages)

test_build.jl

@@ -1,8 +1,8 @@
 println("Starting...")
 using DataFrames
-using Gadfly
-using TextAnalysis
+# using Gadfly
+# using TextAnalysis
 using IJulia
 using SQLite
-using XGBoost
+# using XGBoost
 println("Ok!")