From faeac15f34ce4a29d234fad67c061fc19d8a88fa Mon Sep 17 00:00:00 2001 From: Ryan J Field Date: Wed, 31 Aug 2022 14:24:16 +0100 Subject: [PATCH 01/15] aarch64 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 007aa6a..50e876f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM jupyter/datascience-notebook:julia-1.7.3 +FROM jupyter/datascience-notebook:aarch64-julia-1.7.3 LABEL org.opencontainers.image.source="https://github.com/FAIRDataPipeline/RSECon22" USER root From db06ce03379f6c342d6b983b017ff20232b3f6dd Mon Sep 17 00:00:00 2001 From: Ryan J Field Date: Wed, 31 Aug 2022 14:36:28 +0100 Subject: [PATCH 02/15] Use buildx --- .github/workflows/docker-publish.yaml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-publish.yaml b/.github/workflows/docker-publish.yaml index 958745f..e05c6cd 100644 --- a/.github/workflows/docker-publish.yaml +++ b/.github/workflows/docker-publish.yaml @@ -6,8 +6,13 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: setup-docker - uses: docker-practice/actions-setup-docker@1.0.11 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + - name: Available platforms + run: echo ${{ steps.buildx.outputs.platforms }} - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: @@ -16,5 +21,5 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and Publish Docker Image run: | - docker build -t ghcr.io/fairdatapipeline/rsecon:latest . - docker push ghcr.io/fairdatapipeline/rsecon:latest + buildx build -t ghcr.io/fairdatapipeline/aarm64-rsecon:latest . + docker push ghcr.io/fairdatapipeline/aarm64-rsecon:latest From fb00e2160e68b743b2ed841ccc6e75b3692e1321 Mon Sep 17 00:00:00 2001 From: Ryan J Field Date: Wed, 31 Aug 2022 14:38:57 +0100 Subject: [PATCH 03/15] add install step --- .github/workflows/docker-publish.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yaml b/.github/workflows/docker-publish.yaml index e05c6cd..f11cb1b 100644 --- a/.github/workflows/docker-publish.yaml +++ b/.github/workflows/docker-publish.yaml @@ -10,6 +10,8 @@ jobs: uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx id: buildx + with: + install: true uses: docker/setup-buildx-action@v2 - name: Available platforms run: echo ${{ steps.buildx.outputs.platforms }} @@ -21,5 +23,5 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and Publish Docker Image run: | - buildx build -t ghcr.io/fairdatapipeline/aarm64-rsecon:latest . + docker build -t ghcr.io/fairdatapipeline/aarm64-rsecon:latest . docker push ghcr.io/fairdatapipeline/aarm64-rsecon:latest From e106189b8d78364d48d4c3c20b4017284c7ef195 Mon Sep 17 00:00:00 2001 From: Ryan J Field Date: Wed, 31 Aug 2022 15:07:49 +0100 Subject: [PATCH 04/15] Use poetry lock to speed up dependencies --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 50e876f..531b524 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,7 +34,7 @@ RUN rm -r gradle-7.5 && \ # Python Dependencies WORKDIR ${USER_HOME}/temp RUN wget https://raw.githubusercontent.com/FAIRDataPipeline/FAIR-CLI/develop/pyproject.toml && \ - wget https://raw.githubusercontent.com/FAIRDataPipeline/data-registry/main/local-requirements.txt + wget https://raw.githubusercontent.com/FAIRDataPipeline/FAIR-CLI/develop/poetry.lock RUN mamba install --quiet --yes 'poetry' && \ mamba clean --all -f -y RUN poetry config virtualenvs.create false \ From 8c6bf96988c4e891a7bdffe776b7818a1880098a Mon Sep 17 00:00:00 2001 From: Ryan J Field Date: Wed, 31 Aug 2022 15:30:31 +0100 Subject: [PATCH 05/15] Use pip for dependencies --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 531b524..79beba6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,12 +33,12 @@ RUN rm -r gradle-7.5 && \ # Python Dependencies WORKDIR ${USER_HOME}/temp -RUN wget https://raw.githubusercontent.com/FAIRDataPipeline/FAIR-CLI/develop/pyproject.toml && \ - wget https://raw.githubusercontent.com/FAIRDataPipeline/FAIR-CLI/develop/poetry.lock +RUN wget https://raw.githubusercontent.com/FAIRDataPipeline/FAIR-CLI/develop/pyproject.toml RUN mamba install --quiet --yes 'poetry' && \ mamba clean --all -f -y RUN poetry config virtualenvs.create false \ - && poetry install --no-root --no-interaction --no-ansi + && poetry export -f requirements.txt > requirements.txt && \ + /opt/conda/bin/python -m pip install -r requirements.txt # Clone Repos and allow ambigous permissions WORKDIR ${USER_HOME} From a65c0087b857c0b1dc1439665b7b47270e1470b4 Mon Sep 17 00:00:00 2001 From: Ryan J Field Date: Wed, 31 Aug 2022 15:53:15 +0100 Subject: [PATCH 06/15] FIx permission for cache --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 79beba6..f3067f8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,7 +38,7 @@ RUN mamba install --quiet --yes 'poetry' && \ mamba clean --all -f -y RUN poetry config virtualenvs.create false \ && poetry export -f requirements.txt > requirements.txt && \ - /opt/conda/bin/python -m pip install -r requirements.txt + sudo -H /opt/conda/bin/python -m pip install -r requirements.txt # Clone Repos and allow ambigous permissions WORKDIR ${USER_HOME} From 2e3d6bfaae33af9fcc13f0d0f35204a88c783234 Mon Sep 17 00:00:00 2001 From: Ryan J Field Date: Wed, 31 Aug 2022 16:20:31 +0100 Subject: [PATCH 07/15] Use poetry --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index f3067f8..caf7d07 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,8 +37,7 @@ RUN wget https://raw.githubusercontent.com/FAIRDataPipeline/FAIR-CLI/develop/pyp RUN mamba install --quiet --yes 'poetry' && \ mamba clean --all -f -y RUN poetry config virtualenvs.create false \ - && poetry export -f requirements.txt > requirements.txt && \ - sudo -H /opt/conda/bin/python -m pip install -r requirements.txt + && poetry install --no-root --no-interaction --no-ansi # Clone Repos and allow ambigous permissions WORKDIR ${USER_HOME} From c5d4627f2a7373d7fc5362ed603019eda36882a7 Mon Sep 17 00:00:00 2001 From: Ryan J Field Date: Wed, 31 Aug 2022 17:37:10 +0100 Subject: [PATCH 08/15] Use updated deps --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index caf7d07..31ef931 100644 --- a/Dockerfile +++ b/Dockerfile @@ -59,7 +59,8 @@ RUN cmake --build build -j4 #Julia Simple Model WORKDIR "${USER_HOME}/DataPipeline.jl" -RUN julia -e 'using Pkg; Pkg.instantiate()' && \ +RUN git checkout updated-deps && \ + julia -e 'using Pkg; Pkg.instantiate()' && \ julia --project=examples/fdp -e 'using Pkg; Pkg.instantiate()' # Java Simple Model From cc1b8d610c6b608a370f025492ed77b542be9169 Mon Sep 17 00:00:00 2001 From: Ryan J Field Date: Wed, 31 Aug 2022 19:24:41 +0100 Subject: [PATCH 09/15] Use julia 1.8 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 31ef931..fbdbab0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM jupyter/datascience-notebook:aarch64-julia-1.7.3 +FROM jupyter/datascience-notebook:aarch64-julia-1.8.0 LABEL org.opencontainers.image.source="https://github.com/FAIRDataPipeline/RSECon22" USER root From a2420b4b5887f6e10e625c9896eba363bc38d697 Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Wed, 31 Aug 2022 20:40:38 +0100 Subject: [PATCH 10/15] Update Dockerfile Try without conda r packages --- Dockerfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index fbdbab0..f816077 100644 --- a/Dockerfile +++ b/Dockerfile @@ -75,10 +75,6 @@ RUN conda config --add channels pcgr && \ 'poppler' \ 'librsvg' \ 'glib' \ - 'bioconductor-rhdf5' \ - 'r-rsvg' \ - 'r-magick' \ - 'r-pdftools' \ && \ mamba clean --all -f -y RUN R -e 'cat(withr::with_libpaths(new="/opt/conda/lib/R/library", devtools::install_local() ) )' From 84185891921d5e4c8b0ab698fa9cecb65fcf13d2 Mon Sep 17 00:00:00 2001 From: Ryan Field Date: Fri, 2 Sep 2022 11:39:34 +0100 Subject: [PATCH 11/15] Fix Image --- .github/workflows/docker-publish.yaml | 6 +- Dockerfile | 27 +++- Notebooks/1_registry_cli_install.ipynb | 26 +++- Notebooks/2e_r.ipynb | 171 +++++++++++++++++++++++-- 4 files changed, 210 insertions(+), 20 deletions(-) diff --git a/.github/workflows/docker-publish.yaml b/.github/workflows/docker-publish.yaml index f11cb1b..2e5f23b 100644 --- a/.github/workflows/docker-publish.yaml +++ b/.github/workflows/docker-publish.yaml @@ -7,6 +7,8 @@ jobs: steps: - uses: actions/checkout@v2 - name: Set up QEMU + with: + platforms: arm64 uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx id: buildx @@ -23,5 +25,5 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and Publish Docker Image run: | - docker build -t ghcr.io/fairdatapipeline/aarm64-rsecon:latest . - docker push ghcr.io/fairdatapipeline/aarm64-rsecon:latest + docker build -t ghcr.io/fairdatapipeline/rsecon:aarm64 . + docker push ghcr.io/fairdatapipeline/rsecon:aarm64 diff --git a/Dockerfile b/Dockerfile index f816077..83c5c12 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,8 @@ RUN apt update && \ libjsoncpp-dev \ curl \ libcurl4-openssl-dev \ - libyaml-cpp-dev + libyaml-cpp-dev \ + imagemagick # Java RUN wget https://services.gradle.org/distributions/gradle-7.5-bin.zip && \ @@ -49,8 +50,7 @@ RUN git clone https://github.com/FAIRDataPipeline/cppSimpleModel.git && \ git config --global --add safe.directory ${USER_HOME}/cppSimpleModel && \ git config --global --add safe.directory ${USER_HOME}/DataPipeline.jl && \ git config --global --add safe.directory ${USER_HOME}/javaSimpleModel && \ - git config --global --add safe.directory ${USER_HOME}/rSimpleModel && \ - rm -r temp + git config --global --add safe.directory ${USER_HOME}/rSimpleModel # CPP Simple Model WORKDIR ${USER_HOME}/cppSimpleModel @@ -68,21 +68,36 @@ WORKDIR "${USER_HOME}/javaSimpleModel" RUN gradle build # R Simple Model -WORKDIR ${USER_HOME}/rSimpleModel +WORKDIR ${USER_HOME}/temp RUN conda config --add channels pcgr && \ + conda config --add channels anaconda && \ conda config --add channels bioconda && \ mamba install --quiet --yes \ + 'pkg-config' \ + 'libcurl' \ 'poppler' \ 'librsvg' \ 'glib' \ + 'libgit2' \ && \ mamba clean --all -f -y -RUN R -e 'cat(withr::with_libpaths(new="/opt/conda/lib/R/library", devtools::install_local() ) )' +RUN wget https://imagemagick.org/archive/ImageMagick.tar.gz && \ + tar xvzf ImageMagick.tar.gz && \ + cd ImageMagick-* && \ + ./configure --prefix=/opt/conda && \ + make && \ + make install +WORKDIR ${USER_HOME}/rSimpleModel +RUN echo 'options(stringsAsFactors = FALSE)' >> /opt/conda/lib/R/etc/Rprofile.site && \ + R -e 'install.packages("magick", lib ="/opt/conda/lib/R/library", repos="https://cloud.r-project.org/", configure.vars="INCLUDE_DIR=/opt/conda/include/ImageMagick-7")' && \ + R -e 'cat(withr::with_libpaths(new="/opt/conda/lib/R/library", devtools::install_local("/home/jovyan/rDataPipeline") ) )' && \ + R -e 'cat(withr::with_libpaths(new="/opt/conda/lib/R/library", devtools::install_local() ) )' WORKDIR ${USER_HOME} COPY ./Notebooks . # Permissiona -RUN fix-permissions "${JULIA_PKGDIR}" && \ +RUN rm -r temp && \ + fix-permissions "${JULIA_PKGDIR}" && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" diff --git a/Notebooks/1_registry_cli_install.ipynb b/Notebooks/1_registry_cli_install.ipynb index b5f6ee2..c77392c 100644 --- a/Notebooks/1_registry_cli_install.ipynb +++ b/Notebooks/1_registry_cli_install.ipynb @@ -17,21 +17,39 @@ "metadata": {}, "outputs": [], "source": [ + "# Install the Local Registry\n", "!fair registry install" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Start the local registry binding to all addresses (0.0.0.0)\n", + "!fair registry start --address 0.0.0.0" + ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3.9.12 64-bit", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", "name": "python", - "version": "3.9.12" + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.6" }, - "orig_nbformat": 4, "vscode": { "interpreter": { "hash": "b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e" @@ -39,5 +57,5 @@ } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 } diff --git a/Notebooks/2e_r.ipynb b/Notebooks/2e_r.ipynb index 2afe4d1..9ced0a4 100644 --- a/Notebooks/2e_r.ipynb +++ b/Notebooks/2e_r.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "caf1d8b8-3d2e-4646-b64a-fa3f42b955d6", "metadata": {}, "outputs": [], @@ -13,10 +13,29 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "e995a0c1-c702-4908-9e7d-ea9d75bc2bcd", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "'Initialised empty fair repository in /home/jovyan/rSimpleModel/.fair'" + ], + "text/latex": [ + "'Initialised empty fair repository in /home/jovyan/rSimpleModel/.fair'" + ], + "text/markdown": [ + "'Initialised empty fair repository in /home/jovyan/rSimpleModel/.fair'" + ], + "text/plain": [ + "[1] \"Initialised empty fair repository in /home/jovyan/rSimpleModel/.fair\"" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "# Initialise a 'fair' repository for local use without user prompts\n", "system(\"fair init --ci --local\", intern = T)" @@ -24,10 +43,29 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "33b85e52-4d9f-4c02-8319-a6610e5a36e5", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "'No items to retrieve from remote \\'origin\\'.'" + ], + "text/latex": [ + "'No items to retrieve from remote \\textbackslash{}'origin\\textbackslash{}'.'" + ], + "text/markdown": [ + "'No items to retrieve from remote \\'origin\\'.'" + ], + "text/plain": [ + "[1] \"No items to retrieve from remote 'origin'.\"" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "# Register the input data product from the Config Yaml\n", "system(\"fair pull --local inst/extdata/SEIRSconfig.yaml\", intern = T)" @@ -35,10 +73,127 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "44f20635-b8db-4181-a50a-192cf97eef04", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "R version 3.6.3 (2020-02-29) -- \"Holding the Windsock\"\n", + "Copyright (C) 2020 The R Foundation for Statistical Computing\n", + "Platform: aarch64-conda-linux-gnu (64-bit)\n", + "\n", + "R is free software and comes with ABSOLUTELY NO WARRANTY.\n", + "You are welcome to redistribute it under certain conditions.\n", + "Type 'license()' or 'licence()' for distribution details.\n", + "\n", + " Natural language support but running in an English locale\n", + "\n", + "R is a collaborative project with many contributors.\n", + "Type 'contributors()' for more information and\n", + "'citation()' on how to cite R or R packages in publications.\n", + "\n", + "Type 'demo()' for some demos, 'help()' for on-line help, or\n", + "'help.start()' for an HTML browser interface to help.\n", + "Type 'q()' to quit R.\n", + "\n", + "> library(rSimpleModel)\n", + "> library(rDataPipeline)\n", + "> library(ggplot2)\n", + "> library(dplyr)\n", + "\n", + "Attaching package: ‘dplyr’\n", + "\n", + "The following objects are masked from ‘package:stats’:\n", + "\n", + " filter, lag\n", + "\n", + "The following objects are masked from ‘package:base’:\n", + "\n", + " intersect, setdiff, setequal, union\n", + "\n", + "> library(magrittr)\n", + "> \n", + "> # Initialise code run\n", + "> conf.dir <- Sys.getenv(\"FDP_CONFIG_DIR\")\n", + "> config <- file.path(conf.dir, \"config.yaml\")\n", + "> script <- file.path(conf.dir, \"script.sh\")\n", + "> if(.Platform$OS.type != \"unix\") {\n", + "+ script <- file.path(conf.dir, \"script.bat\")\n", + "+ }\n", + "> handle <- initialise(config, script)\n", + "ℹ Reading 'config.yaml' from data store\n", + "✔ Writing '/home/jovyan/rSimpleModel/data_store/data_store/jobs/2022-09-02_10_18_23_671269/config.yaml' to local registry\n", + "✔ Writing '/home/jovyan/rSimpleModel/data_store/data_store/jobs/2022-09-02_10_18_23_671269/script.sh' to local registry\n", + "✔ Writing 'FAIRDataPipeline/rSimpleModel.git' to local registry\n", + "✔ Writing new code_run to local registry\n", + "> \n", + "> # Read model parameters\n", + "> params <- handle %>% link_read(\"SEIRS_model/parameters\") %>% read.csv\n", + "ℹ Locating 'SEIRS_model/parameters'\n", + "> a <- params %>% filter(param == \"alpha\") %$% value\n", + "> b <- params %>% filter(param == \"beta\") %$% value\n", + "> ig <- params %>% filter(param == \"inv_gamma\") %$% value\n", + "> io <- params %>% filter(param == \"inv_omega\") %$% value\n", + "> im <- params %>% filter(param == \"inv_mu\") %$% value\n", + "> is <- params %>% filter(param == \"inv_sigma\") %$% value\n", + "> \n", + "> # Set initial state\n", + "> initial.state <- data.frame(S = 0.999, E = 0.001, I = 0, R = 0)\n", + "> \n", + "> # Run the model\n", + "> results <- SEIRS_model(initial.state, timesteps = 1000, years = 5,\n", + "+ alpha = a, beta = b, inv_gamma = ig,\n", + "+ inv_omega = io, inv_mu = im, inv_sigma = is)\n", + "> g <- plot_SEIRS(results)\n", + "> \n", + "> # Save outputs to data store\n", + "> results %>% write.csv(link_write(handle, \"model_output\"), row.names = FALSE)\n", + "> \n", + "> handle %>% link_write(\"figure\") %>% ggsave(g, width=20, height=10, units=\"cm\")\n", + "> \n", + "> # Register code run in local registry\n", + "> finalise(handle)\n", + "✔ Writing 'SEIRS_model/results/model_output/R' to local registry\n", + "✔ Writing 'SEIRS_model/results/figure/R' to local registry\n", + "-> PATCH /api/code_run/1/ HTTP/1.1\n", + "-> Host: 127.0.0.1:8000\n", + "-> User-Agent: libcurl/7.83.1 r-curl/4.3.2 httr/1.4.4\n", + "-> Accept-Encoding: deflate, gzip\n", + "-> Accept: application/json, text/xml, application/xml, */*\n", + "-> Content-Type: application/json\n", + "-> Authorization: token e3b2f5dfd878c96e61645c7e533f9a133e3cedf3\n", + "-> Content-Length: 198\n", + "-> \n", + ">> {\n", + ">> \"inputs\": [\n", + ">> \"http://127.0.0.1:8000/api/object_component/1/\"\n", + ">> ],\n", + ">> \"outputs\": [\n", + ">> \"http://127.0.0.1:8000/api/object_component/5/\",\n", + ">> \"http://127.0.0.1:8000/api/object_component/6/\"\n", + ">> ]\n", + ">> }\n", + "\n", + "<- HTTP/1.1 200 OK\n", + "<- Date: Fri, 02 Sep 2022 10:18:27 GMT\n", + "<- Server: WSGIServer/0.2 CPython/3.10.6\n", + "<- Content-Type: application/json\n", + "<- Vary: Accept, Cookie\n", + "<- Allow: GET, PUT, PATCH, DELETE, HEAD, OPTIONS\n", + "<- X-Frame-Options: DENY\n", + "<- Content-Length: 580\n", + "<- X-Content-Type-Options: nosniff\n", + "<- Referrer-Policy: same-origin\n", + "<- \n", + "No encoding supplied: defaulting to UTF-8.\n", + "> \n" + ] + } + ], "source": [ "# Run the R Simple Model as define in the Config Yaml\n", "cat(system(\"fair run --local --debug inst/extdata/SEIRSconfig.yaml\", intern = T), sep=\"\\n\")" @@ -57,7 +212,7 @@ "mimetype": "text/x-r-source", "name": "R", "pygments_lexer": "r", - "version": "4.1.3" + "version": "3.6.3" } }, "nbformat": 4, From f0047c143359ecb0722f04b860a81c5fcb1193e6 Mon Sep 17 00:00:00 2001 From: Ryan Field Date: Sat, 3 Sep 2022 19:21:23 +0100 Subject: [PATCH 12/15] Update to ro_crate --- Dockerfile | 11 +- Notebooks/1_registry_cli_install.ipynb | 4 +- Notebooks/2e_r.ipynb | 169 +------------------------ 3 files changed, 19 insertions(+), 165 deletions(-) diff --git a/Dockerfile b/Dockerfile index 83c5c12..9d083d8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,6 +44,7 @@ RUN poetry config virtualenvs.create false \ WORKDIR ${USER_HOME} RUN git clone https://github.com/FAIRDataPipeline/cppSimpleModel.git && \ git clone https://github.com/FAIRDataPipeline/DataPipeline.jl.git && \ + git clone https://github.com/FAIRDataPipeline/javaDataPipeline.git && \ git clone https://github.com/FAIRDataPipeline/javaSimpleModel.git && \ git clone https://github.com/FAIRDataPipeline/rSimpleModel.git && \ git clone https://github.com/FAIRDataPipeline/rDataPipeline.git && \ @@ -63,9 +64,17 @@ RUN git checkout updated-deps && \ julia -e 'using Pkg; Pkg.instantiate()' && \ julia --project=examples/fdp -e 'using Pkg; Pkg.instantiate()' +# Java Data Pipeline +WORKDIR "${USER_HOME}/javaDataPipeline" +RUN git checkout ro-crate-fix && \ + gradle clean && \ + gradle build + # Java Simple Model WORKDIR "${USER_HOME}/javaSimpleModel" -RUN gradle build +RUN git checkout local_deps && \ + gradle clean && \ + gradle build # R Simple Model WORKDIR ${USER_HOME}/temp diff --git a/Notebooks/1_registry_cli_install.ipynb b/Notebooks/1_registry_cli_install.ipynb index c77392c..1c46721 100644 --- a/Notebooks/1_registry_cli_install.ipynb +++ b/Notebooks/1_registry_cli_install.ipynb @@ -17,8 +17,8 @@ "metadata": {}, "outputs": [], "source": [ - "# Install the Local Registry\n", - "!fair registry install" + "# Install the Local Registry using the ro_crate enabled version\n", + "!fair registry install --version origin/ro_crate" ] }, { diff --git a/Notebooks/2e_r.ipynb b/Notebooks/2e_r.ipynb index 9ced0a4..ad1542d 100644 --- a/Notebooks/2e_r.ipynb +++ b/Notebooks/2e_r.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "caf1d8b8-3d2e-4646-b64a-fa3f42b955d6", "metadata": {}, "outputs": [], @@ -13,29 +13,10 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "e995a0c1-c702-4908-9e7d-ea9d75bc2bcd", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "'Initialised empty fair repository in /home/jovyan/rSimpleModel/.fair'" - ], - "text/latex": [ - "'Initialised empty fair repository in /home/jovyan/rSimpleModel/.fair'" - ], - "text/markdown": [ - "'Initialised empty fair repository in /home/jovyan/rSimpleModel/.fair'" - ], - "text/plain": [ - "[1] \"Initialised empty fair repository in /home/jovyan/rSimpleModel/.fair\"" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "# Initialise a 'fair' repository for local use without user prompts\n", "system(\"fair init --ci --local\", intern = T)" @@ -43,29 +24,10 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "33b85e52-4d9f-4c02-8319-a6610e5a36e5", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "'No items to retrieve from remote \\'origin\\'.'" - ], - "text/latex": [ - "'No items to retrieve from remote \\textbackslash{}'origin\\textbackslash{}'.'" - ], - "text/markdown": [ - "'No items to retrieve from remote \\'origin\\'.'" - ], - "text/plain": [ - "[1] \"No items to retrieve from remote 'origin'.\"" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "# Register the input data product from the Config Yaml\n", "system(\"fair pull --local inst/extdata/SEIRSconfig.yaml\", intern = T)" @@ -73,127 +35,10 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "44f20635-b8db-4181-a50a-192cf97eef04", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "R version 3.6.3 (2020-02-29) -- \"Holding the Windsock\"\n", - "Copyright (C) 2020 The R Foundation for Statistical Computing\n", - "Platform: aarch64-conda-linux-gnu (64-bit)\n", - "\n", - "R is free software and comes with ABSOLUTELY NO WARRANTY.\n", - "You are welcome to redistribute it under certain conditions.\n", - "Type 'license()' or 'licence()' for distribution details.\n", - "\n", - " Natural language support but running in an English locale\n", - "\n", - "R is a collaborative project with many contributors.\n", - "Type 'contributors()' for more information and\n", - "'citation()' on how to cite R or R packages in publications.\n", - "\n", - "Type 'demo()' for some demos, 'help()' for on-line help, or\n", - "'help.start()' for an HTML browser interface to help.\n", - "Type 'q()' to quit R.\n", - "\n", - "> library(rSimpleModel)\n", - "> library(rDataPipeline)\n", - "> library(ggplot2)\n", - "> library(dplyr)\n", - "\n", - "Attaching package: ‘dplyr’\n", - "\n", - "The following objects are masked from ‘package:stats’:\n", - "\n", - " filter, lag\n", - "\n", - "The following objects are masked from ‘package:base’:\n", - "\n", - " intersect, setdiff, setequal, union\n", - "\n", - "> library(magrittr)\n", - "> \n", - "> # Initialise code run\n", - "> conf.dir <- Sys.getenv(\"FDP_CONFIG_DIR\")\n", - "> config <- file.path(conf.dir, \"config.yaml\")\n", - "> script <- file.path(conf.dir, \"script.sh\")\n", - "> if(.Platform$OS.type != \"unix\") {\n", - "+ script <- file.path(conf.dir, \"script.bat\")\n", - "+ }\n", - "> handle <- initialise(config, script)\n", - "ℹ Reading 'config.yaml' from data store\n", - "✔ Writing '/home/jovyan/rSimpleModel/data_store/data_store/jobs/2022-09-02_10_18_23_671269/config.yaml' to local registry\n", - "✔ Writing '/home/jovyan/rSimpleModel/data_store/data_store/jobs/2022-09-02_10_18_23_671269/script.sh' to local registry\n", - "✔ Writing 'FAIRDataPipeline/rSimpleModel.git' to local registry\n", - "✔ Writing new code_run to local registry\n", - "> \n", - "> # Read model parameters\n", - "> params <- handle %>% link_read(\"SEIRS_model/parameters\") %>% read.csv\n", - "ℹ Locating 'SEIRS_model/parameters'\n", - "> a <- params %>% filter(param == \"alpha\") %$% value\n", - "> b <- params %>% filter(param == \"beta\") %$% value\n", - "> ig <- params %>% filter(param == \"inv_gamma\") %$% value\n", - "> io <- params %>% filter(param == \"inv_omega\") %$% value\n", - "> im <- params %>% filter(param == \"inv_mu\") %$% value\n", - "> is <- params %>% filter(param == \"inv_sigma\") %$% value\n", - "> \n", - "> # Set initial state\n", - "> initial.state <- data.frame(S = 0.999, E = 0.001, I = 0, R = 0)\n", - "> \n", - "> # Run the model\n", - "> results <- SEIRS_model(initial.state, timesteps = 1000, years = 5,\n", - "+ alpha = a, beta = b, inv_gamma = ig,\n", - "+ inv_omega = io, inv_mu = im, inv_sigma = is)\n", - "> g <- plot_SEIRS(results)\n", - "> \n", - "> # Save outputs to data store\n", - "> results %>% write.csv(link_write(handle, \"model_output\"), row.names = FALSE)\n", - "> \n", - "> handle %>% link_write(\"figure\") %>% ggsave(g, width=20, height=10, units=\"cm\")\n", - "> \n", - "> # Register code run in local registry\n", - "> finalise(handle)\n", - "✔ Writing 'SEIRS_model/results/model_output/R' to local registry\n", - "✔ Writing 'SEIRS_model/results/figure/R' to local registry\n", - "-> PATCH /api/code_run/1/ HTTP/1.1\n", - "-> Host: 127.0.0.1:8000\n", - "-> User-Agent: libcurl/7.83.1 r-curl/4.3.2 httr/1.4.4\n", - "-> Accept-Encoding: deflate, gzip\n", - "-> Accept: application/json, text/xml, application/xml, */*\n", - "-> Content-Type: application/json\n", - "-> Authorization: token e3b2f5dfd878c96e61645c7e533f9a133e3cedf3\n", - "-> Content-Length: 198\n", - "-> \n", - ">> {\n", - ">> \"inputs\": [\n", - ">> \"http://127.0.0.1:8000/api/object_component/1/\"\n", - ">> ],\n", - ">> \"outputs\": [\n", - ">> \"http://127.0.0.1:8000/api/object_component/5/\",\n", - ">> \"http://127.0.0.1:8000/api/object_component/6/\"\n", - ">> ]\n", - ">> }\n", - "\n", - "<- HTTP/1.1 200 OK\n", - "<- Date: Fri, 02 Sep 2022 10:18:27 GMT\n", - "<- Server: WSGIServer/0.2 CPython/3.10.6\n", - "<- Content-Type: application/json\n", - "<- Vary: Accept, Cookie\n", - "<- Allow: GET, PUT, PATCH, DELETE, HEAD, OPTIONS\n", - "<- X-Frame-Options: DENY\n", - "<- Content-Length: 580\n", - "<- X-Content-Type-Options: nosniff\n", - "<- Referrer-Policy: same-origin\n", - "<- \n", - "No encoding supplied: defaulting to UTF-8.\n", - "> \n" - ] - } - ], + "outputs": [], "source": [ "# Run the R Simple Model as define in the Config Yaml\n", "cat(system(\"fair run --local --debug inst/extdata/SEIRSconfig.yaml\", intern = T), sep=\"\\n\")" From bf477234cadf59f6238f8a074a7c88e35be85c64 Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Sun, 4 Sep 2022 20:41:20 +0100 Subject: [PATCH 13/15] Minor Updates --- README.md | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 7668baf..69905ee 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,35 @@ -# RSECon22 Walktrhough: A FAIR Data Pipeline: provenance-driven data management for traceable scientific workflows +# RSECon22 Walkthrough: A FAIR Data Pipeline: provenance-driven data management for traceable scientific workflows -This repo contains the material for the RSECon22 walkthrough titled: A FAIR Data Pipeline: provenance-driven data management for traceable scientific workflows. The walkthrough uses a docker containter an Jupyter Labs (formally notebook) to run through an example usage of the FAIR DataPipeline. +This repo contains the material for the RSECon22 walkthrough titled: *"A FAIR Data Pipeline: provenance-driven data management for traceable scientific workflows"*. The Walkthrough uses a docker containter an Jupyter Labs (formally notebook) to run through an example usage of the FAIR DataPipeline. ## Prerequisites The only prerequisite is an installation of Docker, Docker is available free from [docker.com](https://www.docker.com/). ## Running the Docker Container -The docker container is available on the GitHub Registry and can be pulled using the following command: +The docker container is available on the GitHub Registry and can be pulled using the following command(s): +### AMD64 ``` docker pull ghcr.io/fairdatapipeline/rsecon:latest ``` +### ARM64 +``` +docker pull ghcr.io/fairdatapipeline/rsecon:aarm64 +``` + The container can then be run using the following command: ``` -docker run -p 8000:8000 -p 8888:8888 +docker run -p 8000:8000 -p 8888:8888 ghcr.io/fairdatapipeline/rsecon:latest +``` + +OR + +``` +docker run -p 8000:8000 -p 8888:8888 ghcr.io/fairdatapipeline/rsecon:aarm64 ``` + Once the container has started, there will be an address to access the Jupyter Lab within the console, this address with include a token for authentication to the Jupyter Labs page. The link will take the form of: `http://127.0.0.1:8888/lab?token=`. This address can then be accessed through your web browser to give you access to the Jupyter Lab installation. @@ -60,11 +73,7 @@ The SEIRS models can be compared and graphed using the following notebook. Code block to `run` a comparison of the simple models producing a graph. ## Explore the registry -The local registry can be explored by running it and navigating to the web interface. +The local registry can be explored by running it and navigating to the web interface at: `127.0.0.1:8000` ### 4_start_registry Notebook to start and stop registry - - - - From 0da2f0f63d43cc8428bdda96240058eb153d442a Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Sun, 4 Sep 2022 21:29:15 +0100 Subject: [PATCH 14/15] Update README.md --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 69905ee..db0c93c 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ # RSECon22 Walkthrough: A FAIR Data Pipeline: provenance-driven data management for traceable scientific workflows -This repo contains the material for the RSECon22 walkthrough titled: *"A FAIR Data Pipeline: provenance-driven data management for traceable scientific workflows"*. The Walkthrough uses a docker containter an Jupyter Labs (formally notebook) to run through an example usage of the FAIR DataPipeline. +This repo contains the material for the RSECon22 walkthrough titled: [*"A FAIR Data Pipeline: provenance-driven data management for traceable scientific workflows"*](https://virtual.oxfordabstracts.com/#/event/public/3101/submission/113). The walkthrough uses a docker container and Jupyter Labs (formally, a notebook) to run through an example usage of the [FAIR Data Pipeline](https://www.fairdatapipeline.org/). ## Prerequisites -The only prerequisite is an installation of Docker, Docker is available free from [docker.com](https://www.docker.com/). +The only prerequisite is an installation of Docker, which is available free from [docker.com](https://www.docker.com/). ## Running the Docker Container -The docker container is available on the GitHub Registry and can be pulled using the following command(s): +The docker container is available on the GitHub Package Registry and can be pulled using the following command(s): ### AMD64 ``` @@ -30,7 +30,7 @@ OR docker run -p 8000:8000 -p 8888:8888 ghcr.io/fairdatapipeline/rsecon:aarm64 ``` -Once the container has started, there will be an address to access the Jupyter Lab within the console, this address with include a token for authentication to the Jupyter Labs page. The link will take the form of: `http://127.0.0.1:8888/lab?token=`. +Once the container has started, there will be an address to access the Jupyter Lab within the console. This address will include a token for authentication to the Jupyter Labs page. The link will take the form of: `http://127.0.0.1:8888/lab?token=`. This address can then be accessed through your web browser to give you access to the Jupyter Lab installation. @@ -47,7 +47,7 @@ The docker container contains 8 Jupyter Notebooks detailed below. This notebook contains codeblocks to install the FAIR Command Line Interface (CLI) and the FAIR Local Registry. ## Simple Models -The notebooks contain code blocks to run the SEIRS model example in different languages, they contain code to `register` inputs and `run` the models. +The notebooks contain code blocks to run the SEIRS model example in different languages: they contain code to `register` inputs and `run` the models. All the models use the same input and therefor the `pull` code block only needs to run from one of the files. From 271452c16630d275e8846804c4a4f424aef1c395 Mon Sep 17 00:00:00 2001 From: Ryan Field Date: Mon, 5 Sep 2022 09:26:22 +0100 Subject: [PATCH 15/15] final touches --- .github/workflows/docker-publish.yaml | 2 +- Dockerfile | 21 ++++++++++++--------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.github/workflows/docker-publish.yaml b/.github/workflows/docker-publish.yaml index 2e5f23b..3e2d3f5 100644 --- a/.github/workflows/docker-publish.yaml +++ b/.github/workflows/docker-publish.yaml @@ -1,5 +1,5 @@ name: Publish Docker Container -on: [push] +on: #[push] jobs: Build: name: Create and Push Docker Container diff --git a/Dockerfile b/Dockerfile index 9d083d8..46f4fb1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,20 +26,21 @@ RUN apt update && \ # Java RUN wget https://services.gradle.org/distributions/gradle-7.5-bin.zip && \ - unzip gradle-*.zip -RUN cp -pr gradle-*/* /usr/local -RUN rm -r gradle-7.5 && \ + unzip gradle-*.zip && \ + cp -pr gradle-*/* /usr/local && \ + rm -r gradle-7.5 && \ rm gradle-7.5-bin.zip && \ mkdir temp # Python Dependencies WORKDIR ${USER_HOME}/temp -RUN wget https://raw.githubusercontent.com/FAIRDataPipeline/FAIR-CLI/develop/pyproject.toml -RUN mamba install --quiet --yes 'poetry' && \ +RUN wget https://raw.githubusercontent.com/FAIRDataPipeline/FAIR-CLI/develop/pyproject.toml && \ + mamba install --quiet --yes 'poetry' && \ mamba clean --all -f -y RUN poetry config virtualenvs.create false \ && poetry install --no-root --no-interaction --no-ansi + # Clone Repos and allow ambigous permissions WORKDIR ${USER_HOME} RUN git clone https://github.com/FAIRDataPipeline/cppSimpleModel.git && \ @@ -51,18 +52,20 @@ RUN git clone https://github.com/FAIRDataPipeline/cppSimpleModel.git && \ git config --global --add safe.directory ${USER_HOME}/cppSimpleModel && \ git config --global --add safe.directory ${USER_HOME}/DataPipeline.jl && \ git config --global --add safe.directory ${USER_HOME}/javaSimpleModel && \ - git config --global --add safe.directory ${USER_HOME}/rSimpleModel + git config --global --add safe.directory ${USER_HOME}/rSimpleModel && \ + git config --global --add safe.directory ${USER_HOME}/DataPipeline.jl # CPP Simple Model WORKDIR ${USER_HOME}/cppSimpleModel -RUN cmake -Bbuild -RUN cmake --build build -j4 +RUN cmake -Bbuild && \ + cmake --build build -j4 #Julia Simple Model WORKDIR "${USER_HOME}/DataPipeline.jl" RUN git checkout updated-deps && \ julia -e 'using Pkg; Pkg.instantiate()' && \ - julia --project=examples/fdp -e 'using Pkg; Pkg.instantiate()' + julia --project=examples/fdp -e 'using Pkg; Pkg.instantiate()' && \ + julia --project=examples/fdp -e 'using Pkg; Pkg.precompile()' # Java Data Pipeline WORKDIR "${USER_HOME}/javaDataPipeline"