Skip to content
Snippets Groups Projects
Verified Commit d95f1fe1 authored by Olivier Benz's avatar Olivier Benz
Browse files

Install R packages in parallel

parent f74958f5
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
geospatial/.gitlab-ci.yml
+ 9
− 5
View file @ d95f1fe1
......@@ -35,7 +35,7 @@
variables:
DOCKER_DRIVER: overlay2
DOCKER_TLS_CERTDIR: "/certs"
R_VERSION_UPDATE: "false"
R_VERSION_UPDATE: "true"
before_script:
- *before-script-build
......@@ -46,7 +46,8 @@ build:latest-linux-amd64:
- os:linux
script:
- >
docker build --pull -t $CI_APP_REPO:$CI_APP_TAG-linux-amd64
docker build --pull --build-arg NCPUS=$((`nproc`/2+1))
-t $CI_APP_REPO:$CI_APP_TAG-linux-amd64
-t $CI_APP_REPO:$VERSION_MAJ_MIN_PAT-linux-amd64
-f geospatial/$CI_APP_TAG.Dockerfile .
- |
......@@ -68,7 +69,8 @@ build:latest-linux-arm64v8:
- os:linux
script:
- >
docker build --pull -t $CI_APP_REPO:$CI_APP_TAG-linux-arm64v8
docker build --pull --build-arg NCPUS=$((`nproc`/2+1))
-t $CI_APP_REPO:$CI_APP_TAG-linux-arm64v8
-t $CI_APP_REPO:$VERSION_MAJ_MIN_PAT-linux-arm64v8
-f geospatial/$CI_APP_TAG.Dockerfile .
- |
......@@ -90,7 +92,8 @@ build:version-linux-amd64:
- os:linux
script:
- >
docker build --pull -t $CI_APP_REPO:$CI_APP_TAG-linux-amd64
docker build --pull --build-arg NCPUS=$((`nproc`/2+1))
-t $CI_APP_REPO:$CI_APP_TAG-linux-amd64
-f geospatial/$CI_APP_TAG.Dockerfile .
- |
docker build -t $CI_APP_REPO:$CI_APP_TAG-root-linux-amd64 -<<EOF
......@@ -112,7 +115,8 @@ build:version-linux-arm64v8:
- os:linux
script:
- >
docker build --pull -t $CI_APP_REPO:$CI_APP_TAG-linux-arm64v8
docker build --pull --build-arg NCPUS=$((`nproc`/2+1))
-t $CI_APP_REPO:$CI_APP_TAG-linux-arm64v8
-f geospatial/$CI_APP_TAG.Dockerfile .
- |
docker build -t $CI_APP_REPO:$CI_APP_TAG-root-linux-arm64v8 -<<EOF
......
geospatial/latest.Dockerfile
+ 4
− 2
View file @ d95f1fe1
FROM registry.gitlab.b-data.ch/jupyterlab/r/verse:4.1.2
ARG NCPUS=1
ARG DEBIAN_FRONTEND=noninteractive
USER root
......@@ -29,7 +31,7 @@ RUN apt-get update \
sqlite3 \
tk-dev \
#unixodbc-dev
&& install2.r --error --skipinstalled \
&& install2.r --error --skipinstalled -n $NCPUS \
#RColorBrewer \
RandomFields \
RNetCDF \
......@@ -60,7 +62,7 @@ RUN apt-get update \
geoR \
geosphere \
## from bioconductor
&& R -e "BiocManager::install('rhdf5', update=FALSE, ask=FALSE)" \
&& R -e "BiocManager::install('rhdf5', update=FALSE, ask=FALSE, Ncpus = Sys.getenv('NCPUS'))" \
## Clean up
&& rm -rf /tmp/* \
&& rm -rf /var/lib/apt/lists/*
......
r-ver/.gitlab-ci.yml
+ 8
− 4
View file @ d95f1fe1
......@@ -45,7 +45,8 @@ build:latest-linux-amd64:
- os:linux
script:
- >
cd r-ver && docker build --pull -t $CI_APP_REPO:$CI_APP_TAG-linux-amd64
cd r-ver && docker build --pull --build-arg NCPUS=$((`nproc`/2+1))
-t $CI_APP_REPO:$CI_APP_TAG-linux-amd64
-t $CI_APP_REPO:$VERSION_MAJ_MIN_PAT-linux-amd64
-f $CI_APP_TAG.Dockerfile .
- |
......@@ -67,7 +68,8 @@ build:latest-linux-arm64v8:
- os:linux
script:
- >
cd r-ver && docker build --pull -t $CI_APP_REPO:$CI_APP_TAG-linux-arm64v8
cd r-ver && docker build --pull --build-arg NCPUS=$((`nproc`/2+1))
-t $CI_APP_REPO:$CI_APP_TAG-linux-arm64v8
-t $CI_APP_REPO:$VERSION_MAJ_MIN_PAT-linux-arm64v8
-f $CI_APP_TAG.Dockerfile .
- |
......@@ -89,7 +91,8 @@ build:version-linux-amd64:
- os:linux
script:
- >
cd r-ver && docker build --pull -t $CI_APP_REPO:$CI_APP_TAG-linux-amd64
cd r-ver && docker build --pull --build-arg NCPUS=$((`nproc`/2+1))
-t $CI_APP_REPO:$CI_APP_TAG-linux-amd64
-f $CI_APP_TAG.Dockerfile .
- |
docker build -t $CI_APP_REPO:$CI_APP_TAG-root-linux-amd64 -<<EOF
......@@ -111,7 +114,8 @@ build:version-linux-arm64v8:
- os:linux
script:
- >
cd r-ver && docker build --pull -t $CI_APP_REPO:$CI_APP_TAG-linux-arm64v8
cd r-ver && docker build --pull --build-arg NCPUS=$((`nproc`/2+1))
-t $CI_APP_REPO:$CI_APP_TAG-linux-arm64v8
-f $CI_APP_TAG.Dockerfile .
- |
docker build -t $CI_APP_REPO:$CI_APP_TAG-root-linux-arm64v8 -<<EOF
......
r-ver/latest.Dockerfile
+ 3
− 1
View file @ d95f1fe1
......@@ -10,6 +10,8 @@ LABEL org.opencontainers.image.licenses="MIT" \
org.opencontainers.image.vendor="b-data GmbH" \
org.opencontainers.image.authors="Olivier Benz <olivier.benz@b-data.ch>"
ARG NCPUS=1
ARG DEBIAN_FRONTEND=noninteractive
ARG NB_USER=jovyan
......@@ -203,7 +205,7 @@ RUN export CODE_BUILTIN_EXTENSIONS_DIR=/opt/code-server/vendor/modules/code-oss-
/root/.vscode-remote
## Install the R kernel for JupyterLab
RUN install2.r --error --deps TRUE --skipinstalled \
RUN install2.r --error --deps TRUE --skipinstalled -n $NCPUS \
IRkernel \
languageserver \
httpgd \
......
tidyverse/.gitlab-ci.yml
+ 9
− 5
View file @ d95f1fe1
......@@ -35,7 +35,7 @@
variables:
DOCKER_DRIVER: overlay2
DOCKER_TLS_CERTDIR: "/certs"
R_VERSION_UPDATE: "false"
R_VERSION_UPDATE: "true"
before_script:
- *before-script-build
......@@ -46,7 +46,8 @@ build:latest-linux-amd64:
- os:linux
script:
- >
docker build --pull -t $CI_APP_REPO:$CI_APP_TAG-linux-amd64
docker build --pull --build-arg NCPUS=$((`nproc`/2+1))
-t $CI_APP_REPO:$CI_APP_TAG-linux-amd64
-t $CI_APP_REPO:$VERSION_MAJ_MIN_PAT-linux-amd64
-f tidyverse/$CI_APP_TAG.Dockerfile .
- |
......@@ -68,7 +69,8 @@ build:latest-linux-arm64v8:
- os:linux
script:
- >
docker build --pull -t $CI_APP_REPO:$CI_APP_TAG-linux-arm64v8
docker build --pull --build-arg NCPUS=$((`nproc`/2+1))
-t $CI_APP_REPO:$CI_APP_TAG-linux-arm64v8
-t $CI_APP_REPO:$VERSION_MAJ_MIN_PAT-linux-arm64v8
-f tidyverse/$CI_APP_TAG.Dockerfile .
- |
......@@ -90,7 +92,8 @@ build:version-linux-amd64:
- os:linux
script:
- >
docker build --pull -t $CI_APP_REPO:$CI_APP_TAG-linux-amd64
docker build --pull --build-arg NCPUS=$((`nproc`/2+1))
-t $CI_APP_REPO:$CI_APP_TAG-linux-amd64
-f tidyverse/$CI_APP_TAG.Dockerfile .
- |
docker build -t $CI_APP_REPO:$CI_APP_TAG-root-linux-amd64 -<<EOF
......@@ -112,7 +115,8 @@ build:version-linux-arm64v8:
- os:linux
script:
- >
docker build --pull -t $CI_APP_REPO:$CI_APP_TAG-linux-arm64v8
docker build --pull --build-arg NCPUS=$((`nproc`/2+1))
-t $CI_APP_REPO:$CI_APP_TAG-linux-arm64v8
-f tidyverse/$CI_APP_TAG.Dockerfile .
- |
docker build -t $CI_APP_REPO:$CI_APP_TAG-root-linux-arm64v8 -<<EOF
......
tidyverse/latest.Dockerfile
+ 6
− 4
View file @ d95f1fe1
FROM registry.gitlab.b-data.ch/jupyterlab/r/r-ver:4.1.2
ARG NCPUS=1
ARG DEBIAN_FRONTEND=noninteractive
USER root
......@@ -20,15 +22,15 @@ RUN apt-get update \
libtiff-dev \
libxtst6 \
unixodbc-dev \
&& install2.r --error --skipinstalled BiocManager \
&& install2.r --error --deps TRUE --skipinstalled \
&& install2.r --error --skipinstalled -n $NCPUS BiocManager \
&& install2.r --error --deps TRUE --skipinstalled -n $NCPUS \
tidyverse \
dplyr \
devtools \
formatR \
## dplyr database backends
&& Rscript -e "devtools::install_version('duckdb', version = '0.3.1')" \
&& install2.r --error --skipinstalled \
&& Rscript -e "devtools::install_version('duckdb', version = '0.3.1', Ncpus = Sys.getenv('NCPUS'))" \
&& install2.r --error --skipinstalled -n $NCPUS \
arrow \
fst \
## Clean up
......
verse/.gitlab-ci.yml
+ 9
− 5
View file @ d95f1fe1
......@@ -35,7 +35,7 @@
variables:
DOCKER_DRIVER: overlay2
DOCKER_TLS_CERTDIR: "/certs"
R_VERSION_UPDATE: "false"
R_VERSION_UPDATE: "true"
before_script:
- *before-script-build
......@@ -46,7 +46,8 @@ build:latest-linux-amd64:
- os:linux
script:
- >
docker build --pull -t $CI_APP_REPO:$CI_APP_TAG-linux-amd64
docker build --pull --build-arg NCPUS=$((`nproc`/2+1))
-t $CI_APP_REPO:$CI_APP_TAG-linux-amd64
-t $CI_APP_REPO:$VERSION_MAJ_MIN_PAT-linux-amd64
-f verse/$CI_APP_TAG.Dockerfile .
- |
......@@ -68,7 +69,8 @@ build:latest-linux-arm64v8:
- os:linux
script:
- >
docker build --pull -t $CI_APP_REPO:$CI_APP_TAG-linux-arm64v8
docker build --pull --build-arg NCPUS=$((`nproc`/2+1))
-t $CI_APP_REPO:$CI_APP_TAG-linux-arm64v8
-t $CI_APP_REPO:$VERSION_MAJ_MIN_PAT-linux-arm64v8
-f verse/$CI_APP_TAG.Dockerfile .
- |
......@@ -90,7 +92,8 @@ build:version-linux-amd64:
- os:linux
script:
- >
docker build --pull -t $CI_APP_REPO:$CI_APP_TAG-linux-amd64
docker build --pull --build-arg NCPUS=$((`nproc`/2+1))
-t $CI_APP_REPO:$CI_APP_TAG-linux-amd64
-f verse/$CI_APP_TAG.Dockerfile .
- |
docker build -t $CI_APP_REPO:$CI_APP_TAG-root-linux-amd64 -<<EOF
......@@ -112,7 +115,8 @@ build:version-linux-arm64v8:
- os:linux
script:
- >
docker build --pull -t $CI_APP_REPO:$CI_APP_TAG-linux-arm64v8
docker build --pull --build-arg NCPUS=$((`nproc`/2+1))
-t $CI_APP_REPO:$CI_APP_TAG-linux-arm64v8
-f verse/$CI_APP_TAG.Dockerfile .
- |
docker build -t $CI_APP_REPO:$CI_APP_TAG-root-linux-arm64v8 -<<EOF
......
verse/latest.Dockerfile
+ 5
− 3
View file @ d95f1fe1
FROM registry.gitlab.b-data.ch/jupyterlab/r/tidyverse:4.1.2
ARG NCPUS=1
ARG DEBIAN_FRONTEND=noninteractive
ARG CTAN_REPO=${CTAN_REPO:-https://mirror.ctan.org/systems/texlive/tlnet}
......@@ -43,7 +45,7 @@ RUN export CODE_BUILTIN_EXTENSIONS_DIR=/opt/code-server/vendor/modules/code-oss-
qpdf \
texinfo \
## Install R package redland
&& install2.r --error --skipinstalled redland \
&& install2.r --error --skipinstalled -n $NCPUS redland \
## Explicitly install runtime library sub-deps of librdf0-dev
&& apt-get install -y \
libcurl4-openssl-dev \
......@@ -75,9 +77,9 @@ RUN export CODE_BUILTIN_EXTENSIONS_DIR=/opt/code-server/vendor/modules/code-oss-
&& chmod -R g+w /opt/TinyTeX \
&& chmod -R g+wx /opt/TinyTeX/bin \
&& echo "PATH=${PATH}" >> /usr/local/lib/R/etc/Renviron.site \
&& install2.r --error --skipinstalled PKI \
&& install2.r --error --skipinstalled -n $NCPUS PKI \
## And some nice R packages for publishing-related stuff
&& install2.r --error --deps TRUE --skipinstalled \
&& install2.r --error --deps TRUE --skipinstalled -n $NCPUS \
blogdown \
bookdown \
distill \
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment