#
#   LEAN Foundation Docker Container 20201214
#   Cross platform deployment for multiple brokerages
#   Intended to be used in conjunction with Dockerfile. This is just the foundation common OS+Dependencies required.
#

# Use base system for cleaning up wayward processes
FROM phusion/baseimage:focal-1.0.0

MAINTAINER QuantConnect <contact@quantconnect.com>

# Use baseimage-docker's init system.
CMD ["/sbin/my_init"]

# Have to add env TZ=UTC. See https://github.com/dotnet/coreclr/issues/602
RUN env TZ=UTC

# Install OS Packages:
# Misc tools for running Python.NET and IB inside a headless container.
RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
   && echo deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/ >> /etc/apt/sources.list \
   && apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 \
   && add-apt-repository ppa:apt-fast/stable && apt-get update && apt-get -y install apt-fast \
   && apt-fast install -y git bzip2 curl unzip wget python3-pip python-opengl zlib1g-dev \
   xvfb libxrender1 libxtst6 libxi6 libglib2.0-dev libopenmpi-dev libstdc++6 openmpi-bin \
   pandoc libcurl4-openssl-dev libgtk2.0.0 r-base \
   && apt-fast clean && apt-get remove -y apt-fast && apt-get clean && apt-get autoclean && apt-get autoremove --purge -y \
   && rm -rf /var/lib/apt/lists/*

# Install IB Gateway: Installs to /root/ibgateway
RUN mkdir -p /root/ibgateway && \
    wget https://cdn.quantconnect.com/interactive/ibgateway-latest-standalone-linux-x64.v985.1j.sh && \
    chmod 777 ibgateway-latest-standalone-linux-x64.v985.1j.sh && \
    ./ibgateway-latest-standalone-linux-x64.v985.1j.sh -q -dir /root/ibgateway && \
    rm ibgateway-latest-standalone-linux-x64.v985.1j.sh

# Install dotnet 5 sdk & runtime
RUN wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb && \
    dpkg -i packages-microsoft-prod.deb && \
    apt-get update; \
    apt-get install -y apt-transport-https && \
    apt-get update && \
    apt-get install -y dotnet-sdk-5.0 && \
    rm packages-microsoft-prod.deb && \
    apt-get clean && apt-get autoclean && apt-get autoremove --purge -y && rm -rf /var/lib/apt/lists/*

# Set PythonDLL variable for PythonNet
ENV PYTHONNET_PYDLL="/opt/miniconda3/lib/libpython3.6m.so"

# Install miniconda
ENV CONDA="Miniconda3-4.5.12-Linux-x86_64.sh"
ENV PATH="/opt/miniconda3/bin:${PATH}"
RUN wget https://cdn.quantconnect.com/miniconda/${CONDA} && \
    bash ${CONDA} -b -p /opt/miniconda3 && rm -rf ${CONDA} && \
    ln -s /opt/miniconda3/lib/libpython3.6m.so /usr/lib/libpython3.6m.so && \
    conda update -y conda pip && \
    pip install --upgrade --no-cache-dir pip && \
    conda install -y python=3.6.8 && conda clean -y --all

# Avoid pip install read timeouts
ENV PIP_DEFAULT_TIMEOUT=120

# Conda install required so that the packages are not
# overwritten and have their version invalidated by
# subsequent calls to conda install
RUN conda install -y                                 \
    cython=0.29.17                                   \
    pandas=0.25.3                                    \
    numpy=1.18.1                                     \
    && conda install -y -c conda-forge fbprophet=0.6 \
    && conda clean -y --all

# Install all packages
RUN pip install --no-cache-dir      \
    wrapt==1.12.1                   \
    astropy==4.0.1.post1            \
    beautifulsoup4==4.9.0           \
    dill==0.3.1.1                   \
    jsonschema==3.2.0               \
    lxml==4.5.0                     \
    msgpack==1.0.0                  \
    numba==0.46                     \
    setuptools-git==1.2             \
    xarray==0.15.1                  \
    plotly==4.7.1                   \
    jupyterlab==2.1.2               \
    tensorflow==1.15.2              \
    docutils==0.14                  \
    cvxopt==1.2.0                   \
    gensim==3.8.0                   \
    Keras==2.3.1                    \
    lightgbm==2.3.0                 \
    mpi4py==3.0.3                   \
    nltk==3.4.5                     \
    pomegranate==0.11.1             \
    graphviz==0.8.4                 \
    cmdstanpy==0.4                  \
    copulae==0.3.1                  \
    featuretools==0.14.0            \
    PuLP==1.6.8                     \
    pymc3==3.8                      \
    rauth==0.7.3                    \
    scikit-learn==0.23.2            \
    scikit-multiflow==0.4.1         \
    scikit-optimize==0.7.4          \
    Theano==1.0.4                   \
    tsfresh==0.15.1                 \
    tslearn==0.3.1                  \
    tweepy==3.8.0                   \
    PyWavelets==1.1.1               \
    umap-learn==0.4.3               \
    nvidia-ml-py3==7.352.0          \
    fastai==1.0.61                  \
    arch==4.14                      \
    copulalib==1.1.0                \
    copulas==0.3.3                  \
    creme==0.5.1                    \
    cufflinks==0.17.3               \
    gym==0.17.2                     \
    ipywidgets==7.5.1               \
    deap==1.3.1                     \
    cvxpy==1.1.15                   \
    pykalman==0.9.5                 \
    pyportfolioopt==1.2.2           \
    pyramid-arima==0.9.0            \
    pyro-ppl==1.3.1                 \
    riskparityportfolio==0.2        \
    sklearn-json==0.1.0             \
    stable-baselines==2.10.0        \
    statistics==1.0.3.5             \
    statsmodels==0.11.1             \
    tensorforce==0.5.5              \
    QuantLib-Python==1.18           \
    xgboost==1.1.0                  \
    dtw-python==1.0.5               \
    cntk==2.7                       \
    mxnet==1.6                      \
    gluonts==0.4.3                  \
    gplearn==0.4.1                  \
    jax==0.1.68                     \
    jaxlib==0.1.69                  \
    keras-rl==0.4.2                 \
    pennylane==0.9.0                \
    neural-tangents==0.2.1          \
    mplfinance==0.12.4a0            \
    hmmlearn==0.2.3                 \
    catboost==0.23.2                \
    fastai2==0.0.17                 \
    ppscore==0.0.2                  \
    scikit-tda==0.0.3               \
    ta==0.5.25                      \
    seaborn==0.11.0                 \
    pyflux==0.4.15                  \
    optuna==2.3.0                   \
    findiff==0.8.5                  \
    sktime==0.3.0                   \
    sktime-dl==0.1.0                \
    hyperopt==0.2.5                 \
    bayesian-optimization==1.2.0    \
    rpy2==3.3.6                     \
    pingouin==0.3.8                 \
    quantecon==0.4.8                \
    matplotlib==3.2.1               \
    sdeint==0.2.1                   \
    pandas_market_calendars==1.7    \
    dgl==0.6.1                      \
    ruptures==1.1.3                 \
    simpy==4.0.1                    \
    scikit-learn-extra==0.2.0       \
    ray==1.5.1

# feature_selector has overly strict dependency version ranges
# We already installed close-enough versions of all of its dependencies above
# All features in the usage notebook in https://github.com/Jie-Yuan/FeatureSelector work
RUN pip install --no-cache-dir --no-dependencies feature_selector==1.0.0

# Notes about pip install:
# sktime==0.3.1 is max version we can use without causing backwards incompatible changes to pandas (>= 1.0.0)
# PyS3DE==1.0.5 not installable

RUN wget -O mlfinlab.zip https://cdn.quantconnect.com/mlfinlab/mlfinlab-0.9.3.zip && \
    unzip -q mlfinlab.zip && \
    mkdir -p /opt/miniconda3/lib/python3.6/site-packages/ && \
    mv mlfinlab /opt/miniconda3/lib/python3.6/site-packages/ && rm mlfinlab.zip

RUN conda install -y -c conda-forge \
    openmpi=4.0.3                   \
    && conda clean -y --all

# Install non-math packages
RUN conda install -y                \
    blaze=0.11.3                    \
    tensorflow-base=1.15.0          \
    && conda clean -y --all

# Install math/ML from pytorch
RUN conda install -y -c pytorch     \
    pytorch=1.5.0                   \
    torchvision=0.6.0               \
    && conda clean -y --all

# Install PyTorch Geometric
RUN TORCH=$(python -c "import torch; print(torch.__version__)") && \
    CUDA=$(python -c "import torch; print('cu' + torch.version.cuda.replace('.', ''))") && \
    pip install --no-cache-dir -f https://pytorch-geometric.com/whl/torch-${TORCH}+${CUDA}.html \
    torch-scatter==2.0.5 torch-sparse==0.6.7 torch-cluster==1.5.7 torch-spline-conv==1.2.0 torch-geometric==1.7.0

RUN python -m nltk.downloader -d /usr/share/nltk_data punkt && \
    python -m nltk.downloader -d /usr/share/nltk_data vader_lexicon && \
    python -m nltk.downloader -d /usr/share/nltk_data stopwords

# Update ODO
RUN conda remove --force-remove -y odo && conda clean -y --all
RUN wget https://cdn.quantconnect.com/odo/odo-master-9fce669.zip && \
    unzip -q odo-master-9fce669.zip && cd odo-master && \
    pip install . && cd .. && rm -rf odo-master && rm odo-master-9fce669.zip

# Install DX Analytics
RUN wget https://cdn.quantconnect.com/dx/dx-master-4c47c25.zip && \
    unzip -q dx-master-4c47c25.zip && cd dx-master && \
    pip install . && cd .. && rm -rf dx-master && rm dx-master-4c47c25.zip

# Install Auto-KS
RUN wget https://cdn.quantconnect.com/auto_ks/auto_ks-master-b39e8f3.zip && \
    unzip -q auto_ks-master-b39e8f3.zip && cd auto_ks-master && \
    pip install . && cd .. && rm -rf auto_ks-master && rm auto_ks-master-b39e8f3.zip

# Install Pyrb
RUN wget https://cdn.quantconnect.com/pyrb/pyrb-master-d02b56a.zip && \
    unzip -q pyrb-master-d02b56a.zip && cd pyrb-master && \
    pip install . && cd .. && rm -rf pyrb-master && rm pyrb-master-d02b56a.zip

# Install SSM
RUN wget https://cdn.quantconnect.com/ssm/ssm-9fd66aed.zip && \
    unzip -q ssm-9fd66aed.zip && cd ssm && \
    pip install . && cd .. && rm -rf ssm && rm ssm-9fd66aed.zip

# Install TA-lib for python
RUN wget https://cdn.quantconnect.com/ta-lib/ta-lib-0.4.0-src.tar.gz && \
    tar -zxvf ta-lib-0.4.0-src.tar.gz && cd ta-lib && \
    ./configure --prefix=/usr && make && make install && \
    wget https://cdn.quantconnect.com/ta-lib/TA_Lib-0.4.18.zip && \
    unzip -q TA_Lib-0.4.18.zip && cd ta-lib-TA_Lib-0.4.18 && \
    python setup.py install && cd ../.. && rm -rf ta-lib && rm ta-lib-0.4.0-src.tar.gz

# Install py-earth
RUN wget https://cdn.quantconnect.com/py-earth/py-earth-0.1.0.zip && \
    unzip -q py-earth-0.1.0.zip && cd py-earth-0.1.0 && \
    python setup.py install && cd .. && rm -rf py-earth-0.1.0 && rm py-earth-0.1.0.zip

# Install fastText
RUN wget https://cdn.quantconnect.com/fastText/fastText-0.9.2.zip && \
    unzip -q fastText-0.9.2.zip && cd fastText-0.9.2 && \
    pip install . && cd .. && rm -rf fastText-0.9.2 && rm fastText-0.9.2.zip

# Install Tigramite
RUN wget https://cdn.quantconnect.com/tigramite/tigramite-4.1.zip && \
    unzip -q tigramite-4.1.zip && cd tigramite-4.1 && \
    python setup.py install && cd .. && rm -rf tigramite-4.1 && rm tigramite-4.1.zip

# Install H2O: https://www.h2o.ai/download/
RUN wget https://cdn.quantconnect.com/h2o/h2o-3.30.0.3.zip && \
    unzip -q h2o-3.30.0.3.zip && \
    pip install h2o-3.30.0.3/python/h2o-3.30.0.3-py2.py3-none-any.whl && \
    rm -rf h2o-3.30.0.3 && rm h2o-3.30.0.3.zip

# Remove black-listed packages
RUN pip uninstall -y s3transfer

# List all packages
RUN conda list