quantconnect--lean/DockerfileLeanFoundationARM

# This is a version of DockerfileLeanFoundation for ARM
# Some packages from the AMD image are excluded because they are not available on ARM or take too long to build

# Use base system for cleaning up wayward processes
FROM phusion/baseimage:focal-1.0.0

MAINTAINER QuantConnect <contact@quantconnect.com>

# Use baseimage-docker's init system.
CMD ["/sbin/my_init"]

# Have to add env TZ=UTC. See https://github.com/dotnet/coreclr/issues/602
RUN env TZ=UTC

# Install OS Packages:
# Misc tools for running Python.NET and IB inside a headless container.
RUN add-apt-repository ppa:ubuntu-toolchain-r/test && apt-get update
RUN add-apt-repository ppa:apt-fast/stable && apt-get update && apt-get -y install apt-fast
RUN apt-get update && apt-fast install -y git libgtk2.0.0 cmake bzip2 curl unzip wget python3-pip python-opengl zlib1g-dev \
    xvfb libxrender1 libxtst6 libxi6 libglib2.0-dev libopenmpi-dev libstdc++6 openmpi-bin \
    r-base pandoc libcurl4-openssl-dev \
    openjdk-8-jdk openjdk-8-jre bbe \
    && apt-fast clean && apt-get remove -y apt-fast && apt-get clean && apt-get autoclean && apt-get autoremove --purge -y \
    && rm -rf /var/lib/apt/lists/*

# Install IB Gateway: Installs to /root/ibgateway
# We update the install script so it doesn't use the bundled JVM
# The bundled JVM doesn't work on ARM64, so we update it to use the JVM installed in the previous command
RUN mkdir -p /root/ibgateway && \
    wget https://cdn.quantconnect.com/interactive/ibgateway-latest-standalone-linux-x64.v10.12.2d.sh && \
    java_patch_version=$(java -version 2>&1 | head -n 1 | cut -d'_' -f2 | cut -d'"' -f1) && \
    bbe -e 's|# INSTALL4J_JAVA_HOME_OVERRIDE=|INSTALL4J_JAVA_HOME_OVERRIDE="/usr/lib/jvm/java-1.8.0-openjdk-arm64"|' -e "s|-lt \"152\"|-lt \"$java_patch_version\"|" -e "s|-gt \"152\"|-gt \"$java_patch_version\"|" ibgateway-latest-standalone-linux-x64.v10.12.2d.sh > ibgateway-stable-standalone-linux-custom-jvm.sh && \
    chmod 777 ibgateway-stable-standalone-linux-custom-jvm.sh && \
    ./ibgateway-stable-standalone-linux-custom-jvm.sh -q -dir /root/ibgateway && \
    rm ibgateway-latest-standalone-linux-x64.v10.12.2d.sh ibgateway-stable-standalone-linux-custom-jvm.sh

# Install dotnet 5 sdk & runtime
# The .deb packages don't support ARM, the install script does
ENV PATH="/root/.dotnet:${PATH}"
RUN wget https://dot.net/v1/dotnet-install.sh && \
    chmod 777 dotnet-install.sh && \
    ./dotnet-install.sh -c 5.0 && \
    rm dotnet-install.sh
ENV DOTNET_ROOT="/root/.dotnet"

# Set PythonDLL variable for PythonNet
ENV PYTHONNET_PYDLL="/opt/miniforge3/lib/libpython3.6m.so"

# Install miniforge
# miniconda doesn't support ARM, so we use miniforge instead
ENV PATH="/opt/miniforge3/bin:${PATH}"
RUN wget -O Miniforge3.sh https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh && \
    bash Miniforge3.sh -b -p /opt/miniforge3 && \
    rm Miniforge3.sh && \
    ln -s /opt/miniforge3/lib/libpython3.6m.so /usr/lib/libpython3.6m.so && \
    conda config --append channels https://repo.anaconda.com/pkgs/main && \
    conda install -y conda=4.10.3 && \
    pip install --upgrade --no-cache-dir pip==21.2.2 && \
    conda install -y python=3.6.7 && conda install -y pip=21.2.2 && conda clean -y --all

# Avoid pip install read timeouts
ENV PIP_DEFAULT_TIMEOUT=120

# Conda install required so that the packages are not
# overwritten and have their version invalidated by
# subsequent calls to conda install
RUN conda install -y                \
    cython=0.29.17                  \
    pandas=0.25.3                   \
    numpy=1.18.1                    \
    wrapt=1.12.1                    \
    jupyterlab=2.1.2                \
    matplotlib=3.2.1                \
    scipy=1.4.1                     \
    && conda clean -y --all

# The list of packages in this image is shorter than the list in the AMD images
# This list only includes packages that can be installed within 2 minutes on ARM
RUN pip install --no-cache-dir      \
    beautifulsoup4==4.9.0           \
    dill==0.3.1.1                   \
    jsonschema==3.2.0               \
    msgpack==1.0.0                  \
    setuptools-git==1.2             \
    xarray==0.15.1                  \
    plotly==4.7.1                   \
    docutils==0.14                  \
    lightgbm==2.3.0                 \
    nltk==3.4.5                     \
    graphviz==0.8.4                 \
    cmdstanpy==0.4                  \
    featuretools==0.14.0            \
    PuLP==1.6.8                     \
    rauth==0.7.3                    \
    scikit-multiflow==0.4.1         \
    scikit-optimize==0.7.4          \
    Theano==1.0.4                   \
    tweepy==3.8.0                   \
    PyWavelets==1.1.1               \
    nvidia-ml-py3==7.352.0          \
    copulalib==1.1.0                \
    copulas==0.3.3                  \
    cufflinks==0.17.3               \
    gym==0.17.2                     \
    ipywidgets==7.5.1               \
    deap==1.3.1                     \
    pykalman==0.9.5                 \
    pyro-ppl==1.3.1                 \
    sklearn-json==0.1.0             \
    statistics==1.0.3.5             \
    dtw-python==1.0.5               \
    mxnet==1.6                      \
    gluonts==0.4.3                  \
    gplearn==0.4.1                  \
    jax==0.1.68                     \
    pennylane==0.9.0                \
    neural-tangents==0.2.1          \
    mplfinance==0.12.4a0            \
    hmmlearn==0.2.3                 \
    ppscore==0.0.2                  \
    ta==0.5.25                      \
    seaborn==0.11.0                 \
    optuna==2.3.0                   \
    findiff==0.8.5                  \
    hyperopt==0.2.5                 \
    bayesian-optimization==1.2.0    \
    rpy2==3.3.6                     \
    sdeint==0.2.1                   \
    pandas_market_calendars==1.7    \
    ruptures==1.1.3                 \
    simpy==4.0.1                    \
    scikit-learn-extra==0.2.0

# feature_selector has overly strict dependency version ranges
# We already installed close-enough versions of all of its dependencies above
# All features in the usage notebook in https://github.com/Jie-Yuan/FeatureSelector work
RUN pip install --no-cache-dir --no-dependencies feature_selector==1.0.0

RUN wget -O mlfinlab.zip https://cdn.quantconnect.com/mlfinlab/mlfinlab-0.9.3.zip && \
    unzip -q mlfinlab.zip && \
    mkdir -p /opt/miniconda3/lib/python3.6/site-packages/ && \
    mv mlfinlab /opt/miniconda3/lib/python3.6/site-packages/ && rm mlfinlab.zip

RUN conda install -y -c conda-forge \
    openmpi=4.0.3 && conda clean -y --all

RUN python -m nltk.downloader -d /usr/share/nltk_data punkt && \
    python -m nltk.downloader -d /usr/share/nltk_data vader_lexicon && \
    python -m nltk.downloader -d /usr/share/nltk_data stopwords

# Update ODO
RUN wget https://cdn.quantconnect.com/odo/odo-master-9fce669.zip && \
    unzip -q odo-master-9fce669.zip && cd odo-master && \
    pip install . && cd .. && rm -rf odo-master && rm odo-master-9fce669.zip

# Install DX Analytics
RUN wget https://cdn.quantconnect.com/dx/dx-master-4c47c25.zip && \
    unzip -q dx-master-4c47c25.zip && cd dx-master && \
    pip install . && cd .. && rm -rf dx-master && rm dx-master-4c47c25.zip

# Install Auto-KS
RUN wget https://cdn.quantconnect.com/auto_ks/auto_ks-master-b39e8f3.zip && \
    unzip -q auto_ks-master-b39e8f3.zip && cd auto_ks-master && \
    pip install . && cd .. && rm -rf auto_ks-master && rm auto_ks-master-b39e8f3.zip

# Install py-earth
RUN wget https://cdn.quantconnect.com/py-earth/py-earth-0.1.0.zip && \
    unzip -q py-earth-0.1.0.zip && cd py-earth-0.1.0 && \
    python setup.py install && cd .. && rm -rf py-earth-0.1.0 && rm py-earth-0.1.0.zip

# Install fastText
RUN wget https://cdn.quantconnect.com/fastText/fastText-0.9.2.zip && \
    unzip -q fastText-0.9.2.zip && cd fastText-0.9.2 && \
    pip install . && cd .. && rm -rf fastText-0.9.2 && rm fastText-0.9.2.zip

# Install Tigramite
RUN wget https://cdn.quantconnect.com/tigramite/tigramite-4.1.zip && \
    unzip -q tigramite-4.1.zip && cd tigramite-4.1 && \
    python setup.py install && cd .. && rm -rf tigramite-4.1 && rm tigramite-4.1.zip

# Install H2O: https://www.h2o.ai/download/
RUN wget https://cdn.quantconnect.com/h2o/h2o-3.34.0.7.zip && \
    unzip -q h2o-3.34.0.7.zip && \
    pip install h2o-3.34.0.7-py2.py3-none-any.whl && \
    rm h2o-3.34.0.7.zip h2o-3.34.0.7-py2.py3-none-any.whl

# Remove black-listed packages
RUN pip uninstall -y s3transfer

# List all packages
RUN conda list