-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathDockerfile
More file actions
145 lines (111 loc) · 4.74 KB
/
Dockerfile
File metadata and controls
145 lines (111 loc) · 4.74 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
FROM debian:jessie
MAINTAINER bbp_user <bluebrainproject@4quant.com>
ENV DEBIAN_FRONTEND noninteractive
ENV CONDA_DIR /opt/conda
# Core installs
RUN apt-get update && \
apt-get install -y git vim wget build-essential python-dev ca-certificates bzip2 libsm6 && \
apt-get clean
# Install conda
RUN echo 'export PATH=$CONDA_DIR/bin:$PATH' > /etc/profile.d/conda.sh && \
wget --quiet https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
/bin/bash /Miniconda3-latest-Linux-x86_64.sh -b -p $CONDA_DIR && \
rm Miniconda3-latest-Linux-x86_64.sh && \
$CONDA_DIR/bin/conda install --yes conda==4.2.9
# Create a user
RUN useradd -m -s /bin/bash bbp_user
RUN chown -R bbp_user:bbp_user $CONDA_DIR
# Open port
EXPOSE 8888
# Env vars
USER bbp_user
ENV HOME /home/bbp_user
ENV SHELL /bin/bash
ENV USER bbp_user
ENV PATH $CONDA_DIR/bin:$PATH
WORKDIR $HOME
# Setup ipython
RUN conda install --yes ipython-notebook terminado && conda clean -yt
RUN ipython profile create
# Switch to root for permissions
USER root
# Java setup
RUN apt-get install -y default-jre
# Spark setup
## Spark dependencies
ENV APACHE_SPARK_VERSION 2.0.1
ENV PYJ_VERSION py4j-0.10.1-src.zip
RUN apt-get -y update && \
apt-get install -y --no-install-recommends openjdk-7-jre-headless && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
RUN cd /tmp && \
wget -q http://d3kbcqa49mib13.cloudfront.net/spark-${APACHE_SPARK_VERSION}-bin-hadoop2.6.tgz && \
tar xzf spark-${APACHE_SPARK_VERSION}-bin-hadoop2.6.tgz -C /usr/local && \
rm spark-${APACHE_SPARK_VERSION}-bin-hadoop2.6.tgz
RUN cd /usr/local && ln -s spark-${APACHE_SPARK_VERSION}-bin-hadoop2.6 spark
# Spark and Mesos config
ENV SPARK_HOME /usr/local/spark
ENV PATH $PATH:$SPARK_HOME/bin
RUN sed 's/log4j.rootCategory=INFO/log4j.rootCategory=ERROR/g' $SPARK_HOME/conf/log4j.properties.template > $SPARK_HOME/conf/log4j.properties
ENV _JAVA_OPTIONS "-Xms512m -Xmx4g"
ENV PYTHONPATH $SPARK_HOME/python:$SPARK_HOME/python/lib/$PYJ_LIB_VERSION
ENV SPARK_OPTS --driver-java-options=-Xms1024M --driver-java-options=-Xmx4096M --driver-java-options=-Dlog4j.logLevel=info
# Install useful Python packages
# RUN apt-get install -y libxrender1 && apt-get clean
RUN apt-get -y update && \
apt-get install -y libxrender1 fonts-dejavu && \
apt-get clean
#RUN conda create --yes -q -n python3.5-env python=3.5 nose numpy pandas scikit-learn scikit-image matplotlib scipy seaborn sympy cython patsy statsmodels cloudpickle numba bokeh pillow ipython jsonschema boto
RUN conda install --yes nose numpy pandas scikit-learn scikit-image matplotlib scipy seaborn sympy cython patsy statsmodels cloudpickle numba bokeh pillow ipython jsonschema boto
ENV PATH $CONDA_DIR/bin:$PATH
RUN conda install --yes numpy pandas scikit-learn scikit-image matplotlib scipy seaborn sympy cython patsy statsmodels cloudpickle numba bokeh pillow && conda clean -yt
RUN /bin/bash -c "pip install mistune"
# Bolt setup
# use the latest from the master branch
RUN git clone https://github.com/bolt-project/bolt
RUN /bin/bash -c "pip install -r bolt/requirements.txt"
ENV BOLT_ROOT $HOME/bolt
ENV PATH $PATH:$BOLT_ROOT/bin
ENV PYTHONPATH $PYTHONPATH:$BOLT_ROOT
USER bbp_user
# Install Python 3 Tensorflow 0.10.0
RUN conda install --quiet --yes --channel https://conda.anaconda.org/conda-forge 'tensorflow=0.10.0'
# Keras
RUN conda install --channel https://conda.anaconda.org/KEHANG --quiet --yes 'keras=1.0.8'
# Simple ITK
RUN conda install --channel https://conda.anaconda.org/SimpleITK --quiet --yes 'SimpleITK=0.10.0'
# GDAL
# RUN conda install --channel https://conda.anaconda.org/conda-forge --quiet --yes 'gdal=2.1.2'
# RUN pip install -i https://pypi.anaconda.org/pypi/simple 'pygdal==1.11.0.0'
# Install Jupyter notebook as bbp_user
RUN conda install --quiet --yes \
'notebook=4.2*' \
&& conda clean -tipsy
# Install JupyterHub to get the jupyterhub-singleuser startup script
RUN pip --no-cache-dir install 'jupyterhub==0.5'
# setup shared folders
USER root
# test data
ADD data $HOME/data
#spark extensions / packages
ADD jars $HOME/spark_jars
#Add any prebuilt python packages in the eggs folder
ADD eggs $HOME/py_eggs
# Setup for standard runtime
# Add the notebooks directory
ADD notebooks $HOME/notebooks
# Set permissions on the notebooks
RUN chown -R bbp_user:bbp_user $HOME/notebooks
RUN chown -R bbp_user:bbp_user $HOME/data
RUN chown -R bbp_user:bbp_user $HOME/spark_jars
RUN chown -R bbp_user:bbp_user $HOME/py_eggs
#TODO FIX RUN easy_install $HOME/py_eggs/*.egg
# Switch back to non-root user
USER bbp_user
WORKDIR $HOME/notebooks
# Setup Spark + IPython env vars
ENV PYSPARK_PYTHON=/opt/conda/bin/python
ENV PYSPARK_DRIVER_PYTHON=/opt/conda/bin/jupyter
ENV PYSPARK_DRIVER_PYTHON_OPTS "notebook --ip=0.0.0.0"
CMD /bin/bash -c pyspark