Bootstrap: docker
From: nvidia/cuda:12.3.1-devel-ubuntu20.04

%labels
    Requires CUDA driver 470.57+.

%environment
    export LANG=C

%runscript
    nvidia-smi
    ./run.sh

%post
    . /.singularity.d/env/10-docker*.sh

    apt-get update
    apt-get install -y pciutils
    DEBIAN_FRONTEND='noninteractive' apt-get -y -o Dpkg::Options::='--force-confdef' -o Dpkg::Options::='--force-confold' install build-essential cuda-compat-12-3 libibverbs-dev ibverbs-utils gcc wget git libcap2-bin
    apt-get -y autoremove; apt-get -y clean

    cd /tmp

    # install slurm
    : ${SLURM_VERSION:=17-02-11-1}
    wget https://github.com/SchedMD/slurm/archive/slurm-${SLURM_VERSION}.tar.gz
    tar -xf slurm-${SLURM_VERSION}.tar.gz
        cd slurm-slurm-${SLURM_VERSION}
        ./configure --prefix=/usr/ --sysconfdir=/etc/slurm --localstatedir=/var --disable-debug
        make -C contribs/pmi2 -j$(nproc) install
    cd ..
    rm -rf slurm-*

    # install julia
    cd ~
    wget https://julialang-s3.julialang.org/bin/linux/x64/1.9/julia-1.9.4-linux-x86_64.tar.gz
    tar zxvf julia-1.9.4-linux-x86_64.tar.gz
    mv julia-1.9.4/ /opt/julia-1.9.4
    #mkdir /usr/local/bin
    ln -s /opt/julia-1.9.4/bin/julia /usr/local/bin/julia

    #Add nvidia driver paths to the environment variables
    echo "\n #Nvidia driver paths \n" >> /environment
    echo 'export PATH="/nvbin:$PATH"' >> /environment
    echo 'export LD_LIBRARY_PATH="/nvlib:$LD_LIBRARY_PATH"' >> /environment

    #Add CUDA paths
    echo "\n #Cuda paths \n" >> /environment
    echo 'export CPATH="/usr/local/cuda/include:$CPATH"' >> /environment
    echo 'export PATH="/usr/local/cuda/bin:$PATH"' >> /environment
    echo 'export LD_LIBRARY_PATH="/usr/local/cuda/lib64:$LD_LIBRARY_PATH"' >> /environment
    echo 'export CUDA_HOME="/usr/local/cuda"' >> /environment

    # install likwid
    VERSION=5.3.0
    wget http://ftp.fau.de/pub/likwid/likwid-$VERSION.tar.gz
    tar -xaf likwid-$VERSION.tar.gz
    cd likwid-$VERSION
    # accessdaemon doesn't work because of permissions
    sed -i 's/ACCESSMODE = accessdaemon/ACCESSMODE = perf_event/g' config.mk
    make -j4
    make -j4 install 
    echo 'export LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH"' >> /environment