diff --git a/.github/workflows/build-env.yml b/.github/workflows/build-env.yml new file mode 100644 index 00000000..35396f6c --- /dev/null +++ b/.github/workflows/build-env.yml @@ -0,0 +1,51 @@ +name: Build Golden Base Image + +on: + push: + branches: + - 'v*-dev' + workflow_dispatch: + inputs: + release_tag: + description: 'Image tag (e.g., v7.0.0-dev)' + required: true + default: 'v7.0.0-dev' + danger_version: + description: 'dsDangerClient branch/tag' + required: true + default: 'v6.3.4-dev' + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + driver: docker-container + + - name: Build and Push + uses: docker/build-push-action@v5 + with: + context: . + push: true + cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/ds-base-env:build-cache + cache-to: type=registry,ref=ghcr.io/${{ github.repository_owner }}/ds-base-env:build-cache,mode=max + build-args: | + DANGER_VERSION=${{ github.event.inputs.danger_version || 'v6.3.4-dev' }} + tags: | + ghcr.io/${{ github.repository_owner }}/ds-base-env:${{ github.event.inputs.release_tag || github.ref_name }} diff --git a/.github/workflows/dsBaseClient_test_suite.yaml b/.github/workflows/dsBaseClient_test_suite.yaml index b8a6f3cc..0e35ba1f 100644 --- a/.github/workflows/dsBaseClient_test_suite.yaml +++ b/.github/workflows/dsBaseClient_test_suite.yaml @@ -3,9 +3,9 @@ # Adapted from `armadillo_azure-pipelines.yml` by Roberto Villegas-Diaz # # Inside the root directory $(Pipeline.Workspace) will be a file tree like: -# /dsBaseClient <- Checked out version of datashield/dsBaseClient -# /dsBaseClient/logs <- Where results of tests and logs are collated -# /testStatus <- Checked out version of datashield/testStatus +# /dsBaseClient <- Checked out version of datashield/dsBaseClient +# /dsBaseClient/logs <- Where results of tests and logs are collated +# /testStatus <- Checked out version of datashield/testStatus # # As of Sept. 2025 this takes ~ 95 mins to run. ################################################################################ @@ -24,11 +24,12 @@ jobs: permissions: contents: read - # These should all be constant, except TEST_FILTER. This can be used to test - # subsets of test files in the testthat directory. Options are like: - # '*' <- Run all tests. - # 'asNumericDS*' <- Run all asNumericDS tests, i.e. all the arg, etc. tests. - # '*_smk_*' <- Run all the smoke tests for all functions. + # This logic automatically selects the image tag + container: + image: >- + ghcr.io/${{ github.repository_owner }}/ds-base-env:${{ env.IMAGE_TAG }} + options: --user root -v /var/run/docker.sock:/var/run/docker.sock + env: TEST_FILTER: '_-|datachk-|smk-|arg-|disc-|perf-|smk_expt-|expt-|math-' _r_check_system_clock_: 0 @@ -36,6 +37,7 @@ jobs: PROJECT_NAME: dsBaseClient BRANCH_NAME: ${{ github.head_ref || github.ref_name }} REPO_OWNER: ${{ github.repository_owner }} + IMAGE_TAG: ${{ github.event.pull_request.base.ref || github.ref_name }} R_KEEP_PKG_SOURCE: yes GITHUB_TOKEN: ${{ github.token || 'placeholder-token' }} @@ -55,57 +57,8 @@ jobs: persist-credentials: false token: ${{ env.GITHUB_TOKEN }} - - name: Uninstall default MySQL - run: | - curl https://bazel.build/bazel-release.pub.gpg | sudo apt-key add - - sudo service mysql stop || true - sudo apt-get update - sudo apt-get remove --purge mysql-client mysql-server mysql-common -y - sudo apt-get autoremove -y - sudo apt-get autoclean -y - sudo rm -rf /var/lib/mysql/ - - - uses: r-lib/actions/setup-pandoc@v2 - - - uses: r-lib/actions/setup-r@v2 - with: - r-version: release - http-user-agent: release - use-public-rspm: true - - - name: Install R and dependencies - run: | - sudo apt-get install --no-install-recommends software-properties-common dirmngr -y - wget -qO- https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | sudo tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc - sudo add-apt-repository "deb https://cloud.r-project.org/bin/linux/ubuntu $(lsb_release -cs)-cran40/" - sudo apt-get update -qq - sudo apt-get upgrade -y - sudo apt-get install -qq libxml2-dev libcurl4-openssl-dev libssl-dev libgsl-dev libgit2-dev r-base -y - sudo apt-get install -qq libharfbuzz-dev libfribidi-dev libmagick++-dev xml-twig-tools -y - sudo R -q -e "install.packages(c('devtools','covr','fields','meta','metafor','ggplot2','gridExtra','data.table','DSI','DSOpal','DSLite','MolgenisAuth','MolgenisArmadillo','DSMolgenisArmadillo','DescTools','e1071'), repos='https://cloud.r-project.org')" - sudo R -q -e "devtools::install_github(repo='datashield/dsDangerClient', ref=Sys.getenv('BRANCH_NAME'))" - - - uses: r-lib/actions/setup-r-dependencies@v2 - with: - dependencies: 'c("Imports")' - extra-packages: | - any::rcmdcheck - cran::devtools - cran::git2r - cran::RCurl - cran::readr - cran::magrittr - cran::xml2 - cran::purrr - cran::dplyr - cran::stringr - cran::tidyr - cran::quarto - cran::knitr - cran::kableExtra - cran::rmarkdown - cran::downlit - needs: check + - name: Install PR version of dsBaseClient + run: R CMD INSTALL dsBaseClient - name: Check manual updated run: | @@ -169,7 +122,6 @@ jobs: ), "coveragelist.csv" )' - mv coveragelist.csv logs/ mv test_* logs/ working-directory: dsBaseClient @@ -180,7 +132,6 @@ jobs: echo "Number of testsuites with issues: $issue_count" sed 's/failures="0" errors="0"//' test_results.xml | grep errors= > issues.log || true cat issues.log || true - # continue with workflow even when some tests fail exit 0 working-directory: dsBaseClient/logs @@ -193,52 +144,14 @@ jobs: - name: Parse results from testthat and covr run: | - Rscript --verbose --vanilla ../testStatus/source/parse_test_report.R logs/ + Rscript --verbose --vanilla ../testStatus/source/parse_test_report.R logs/ logs/ https://github.com/datashield/${{ env.PROJECT_NAME }}/blob/${{ env.BRANCH_NAME }} '([^:]+)' '(?<=::)[^:]+(?=::)' working-directory: dsBaseClient + env: + PROJECT_NAME: ${{ env.PROJECT_NAME }} + BRANCH_NAME: ${{ env.BRANCH_NAME }} - name: Render report run: | cd testStatus - mkdir -p new/logs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/ mkdir -p new/docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/ - mkdir -p new/docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/latest/ - - # Copy logs to new logs directory location - cp -rv ../dsBaseClient/logs/* new/logs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/ - cp -rv ../dsBaseClient/logs/${{ env.WORKFLOW_ID }}.txt new/logs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/ - - R -e 'input_dir <- file.path("../new/logs", Sys.getenv("PROJECT_NAME"), Sys.getenv("BRANCH_NAME"), Sys.getenv("WORKFLOW_ID")); quarto::quarto_render("source/test_report.qmd", execute_params = list(input_dir = input_dir))' - mv source/test_report.html new/docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/index.html - cp -r new/docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/* new/docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/latest - - env: - PROJECT_NAME: ${{ env.PROJECT_NAME }} - BRANCH_NAME: ${{ env.BRANCH_NAME }} - WORKFLOW_ID: ${{ env.WORKFLOW_ID }} - - - name: Upload test logs - uses: actions/upload-artifact@v4 - with: - name: dsbaseclient-logs - path: testStatus/new - - - name: Dump environment info - run: | - echo -e "\n#############################" - echo -e "ls /: ######################" - ls -al . - echo -e "\n#############################" - echo -e "lscpu: ######################" - lscpu - echo -e "\n#############################" - echo -e "memory: #####################" - free -m - echo -e "\n#############################" - echo -e "env: ########################" - env - echo -e "\n#############################" - echo -e "R sessionInfo(): ############" - R -e 'sessionInfo()' - sudo apt install tree -y - tree . diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..b5522c6d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM rocker/r-ver:4.3.0 + +# 1. System Libraries +RUN apt-get update && apt-get install -y --no-install-recommends \ + sudo software-properties-common dirmngr wget curl \ + libxml2-dev libcurl4-openssl-dev libssl-dev libgsl-dev libgit2-dev \ + libharfbuzz-dev libfribidi-dev libmagick++-dev xml-twig-tools \ + docker.io docker-compose-v2 \ + && rm -rf /var/lib/apt/lists/* + +# 2. R Packages (Stable dependencies) +RUN R -e "options(repos = c(RSPM = 'https://packagemanager.posit.co/cran/__linux__/jammy/latest', CRAN = 'https://cloud.r-project.org')); \ + install.packages(c('devtools','remotes','covr','fields','meta','metafor','ggplot2','gridExtra','data.table','DSI','DSOpal','DSLite','MolgenisAuth','MolgenisArmadillo','DSMolgenisArmadillo','DescTools','e1071', \ + 'rcmdcheck','git2r','RCurl','readr','magrittr','xml2','purrr','dplyr','stringr','tidyr','knitr','kableExtra','rmarkdown','downlit','lme4','httr','spelling','tibble','testthat'))" + +# 3. dsDangerClient (Pass the version as a build argument) +ARG DANGER_VERSION=master +RUN R -e "remotes::install_github('datashield/dsDangerClient', ref='${DANGER_VERSION}')" diff --git a/armadillo_azure-pipelines.yml b/armadillo_azure-pipelines.yml index 563fe858..751e7076 100644 --- a/armadillo_azure-pipelines.yml +++ b/armadillo_azure-pipelines.yml @@ -34,6 +34,7 @@ variables: branchName: $(Build.SourceBranchName) test_filter: '*' _r_check_system_clock_: 0 + R_LIBS_USER: $(Pipeline.Workspace)/R/library ######################################################################################### @@ -124,6 +125,16 @@ jobs: ##################################################################################### # Install R and all the dependencies dsBaseClient requires. # If previous steps have failed then don't run. + - task: Cache@2 + inputs: + key: 'r-pkgs | "$(Agent.OS)" | $(Build.SourceBranchName) | v1' + restoreKeys: | + r-pkgs | "$(Agent.OS)" | $(Build.SourceBranchName) + r-pkgs | "$(Agent.OS)" + path: $(R_LIBS_USER) + displayName: 'Cache R packages' + condition: succeeded() + - bash: | sudo apt-get install --no-install-recommends software-properties-common dirmngr wget -qO- https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | sudo tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc @@ -133,13 +144,10 @@ jobs: sudo apt-get install -qq libxml2-dev libcurl4-openssl-dev libssl-dev libgsl-dev libgit2-dev r-base -y sudo apt-get install -qq libharfbuzz-dev libfribidi-dev libmagick++-dev libudunits2-dev -y - sudo R -q -e "install.packages(c('devtools','covr'), dependencies=TRUE, repos='https://cloud.r-project.org')" - sudo R -q -e "install.packages(c('fields','meta','metafor','ggplot2','gridExtra','data.table'), dependencies=TRUE, repos='https://cloud.r-project.org')" - sudo R -q -e "install.packages(c('DSI','DSOpal','DSLite'), dependencies=TRUE, repos='https://cloud.r-project.org')" - sudo R -q -e "install.packages(c('MolgenisAuth', 'MolgenisArmadillo', 'DSMolgenisArmadillo'), dependencies=TRUE, repos='https://cloud.r-project.org')" - sudo R -q -e "install.packages(c('DescTools','e1071'), dependencies=TRUE, repos='https://cloud.r-project.org')" + mkdir -p "$R_LIBS_USER" + R -q -e "pkgs <- c('devtools','covr','fields','meta','metafor','ggplot2','gridExtra','data.table','DSI','DSOpal','DSLite','MolgenisAuth','MolgenisArmadillo','DSMolgenisArmadillo','DescTools','e1071'); inst <- rownames(installed.packages()); to_install <- setdiff(pkgs, inst); if (length(to_install)) install.packages(to_install, dependencies=TRUE, repos='https://cloud.r-project.org')" - sudo R -q -e "library('devtools'); devtools::install_github(repo='datashield/dsDangerClient', ref='v6.3.4-dev', dependencies = TRUE)" + R -q -e "library('devtools'); devtools::install_github(repo='datashield/dsDangerClient', ref='v6.3.4-dev', dependencies=TRUE, upgrade='never')" # XML grep for coverage report merging sudo apt-get install -qq xml-twig-tools -y diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 01ac339f..25ec730f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -32,6 +32,7 @@ variables: branchName: $(Build.SourceBranchName) test_filter: '*' _r_check_system_clock_: 0 + R_LIBS_USER: $(Pipeline.Workspace)/R/library ######################################################################################### @@ -105,6 +106,16 @@ jobs: ##################################################################################### # Install R and all the dependencies dsBaseClient requires. # If previous steps have failed then don't run. + - task: Cache@2 + inputs: + key: 'r-pkgs | "$(Agent.OS)" | $(Build.SourceBranchName) | v1' + restoreKeys: | + r-pkgs | "$(Agent.OS)" | $(Build.SourceBranchName) + r-pkgs | "$(Agent.OS)" + path: $(R_LIBS_USER) + displayName: 'Cache R packages' + condition: succeeded() + - bash: | sudo apt-get install --no-install-recommends software-properties-common dirmngr wget -qO- https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | sudo tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc @@ -114,14 +125,10 @@ jobs: sudo apt-get install -qq libxml2-dev libcurl4-openssl-dev libssl-dev libgsl-dev libgit2-dev r-base -y sudo apt-get install -qq libharfbuzz-dev libfribidi-dev libmagick++-dev libudunits2-dev -y - sudo R -q -e "install.packages(c('curl','httr'), dependencies=TRUE, repos='https://cloud.r-project.org')" - sudo R -q -e "install.packages(c('devtools','covr'), dependencies=TRUE, repos='https://cloud.r-project.org')" - sudo R -q -e "install.packages(c('fields','meta','metafor','ggplot2','gridExtra','data.table'), dependencies=TRUE, repos='https://cloud.r-project.org')" - sudo R -q -e "install.packages(c('DSI','DSOpal','DSLite'), dependencies=TRUE, repos='https://cloud.r-project.org')" - sudo R -q -e "install.packages(c('MolgenisAuth', 'MolgenisArmadillo', 'DSMolgenisArmadillo'), dependencies=TRUE, repos='https://cloud.r-project.org')" - sudo R -q -e "install.packages(c('DescTools','e1071'), dependencies=TRUE, repos='https://cloud.r-project.org')" - - sudo R -q -e "library('devtools'); devtools::install_github(repo='datashield/dsDangerClient', ref='6.3.4', dependencies = TRUE)" + mkdir -p "$R_LIBS_USER" + R -q -e "pkgs <- c('curl','httr','devtools','covr','fields','meta','metafor','ggplot2','gridExtra','data.table','DSI','DSOpal','DSLite','MolgenisAuth','MolgenisArmadillo','DSMolgenisArmadillo','DescTools','e1071'); inst <- rownames(installed.packages()); to_install <- setdiff(pkgs, inst); if (length(to_install)) install.packages(to_install, dependencies=TRUE, repos='https://cloud.r-project.org')" + + R -q -e "library('devtools'); devtools::install_github(repo='datashield/dsDangerClient', ref='6.3.4', dependencies=TRUE, upgrade='never')" # XML grep for coverage report merging sudo apt-get install -qq xml-twig-tools -y diff --git a/opal_azure-pipelines.yml b/opal_azure-pipelines.yml index 438e63ba..343c7040 100644 --- a/opal_azure-pipelines.yml +++ b/opal_azure-pipelines.yml @@ -34,6 +34,7 @@ variables: branchName: $(Build.SourceBranchName) test_filter: '*' _r_check_system_clock_: 0 + R_LIBS_USER: $(Pipeline.Workspace)/R/library ######################################################################################### @@ -124,6 +125,16 @@ jobs: ##################################################################################### # Install R and all the dependencies dsBaseClient requires. # If previous steps have failed then don't run. + - task: Cache@2 + inputs: + key: 'r-pkgs | "$(Agent.OS)" | $(Build.SourceBranchName) | v1' + restoreKeys: | + r-pkgs | "$(Agent.OS)" | $(Build.SourceBranchName) + r-pkgs | "$(Agent.OS)" + path: $(R_LIBS_USER) + displayName: 'Cache R packages' + condition: succeeded() + - bash: | sudo apt-get install --no-install-recommends software-properties-common dirmngr wget -qO- https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | sudo tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc @@ -133,14 +144,10 @@ jobs: sudo apt-get install -qq libxml2-dev libcurl4-openssl-dev libssl-dev libgsl-dev libgit2-dev r-base -y sudo apt-get install -qq libharfbuzz-dev libfribidi-dev libmagick++-dev libudunits2-dev -y - sudo R -q -e "install.packages(c('curl','httr'), dependencies=TRUE, repos='https://cloud.r-project.org')" - sudo R -q -e "install.packages(c('devtools','covr'), dependencies=TRUE, repos='https://cloud.r-project.org')" - sudo R -q -e "install.packages(c('fields','meta','metafor','ggplot2','gridExtra','data.table'), dependencies=TRUE, repos='https://cloud.r-project.org')" - sudo R -q -e "install.packages(c('DSI','DSOpal','DSLite'), dependencies=TRUE, repos='https://cloud.r-project.org')" - sudo R -q -e "install.packages(c('MolgenisAuth', 'MolgenisArmadillo', 'DSMolgenisArmadillo'), dependencies=TRUE, repos='https://cloud.r-project.org')" - sudo R -q -e "install.packages(c('DescTools','e1071'), dependencies=TRUE, repos='https://cloud.r-project.org')" - - sudo R -q -e "library('devtools'); devtools::install_github(repo='datashield/dsDangerClient', ref='6.3.4', dependencies = TRUE)" + mkdir -p "$R_LIBS_USER" + R -q -e "pkgs <- c('curl','httr','devtools','covr','fields','meta','metafor','ggplot2','gridExtra','data.table','DSI','DSOpal','DSLite','MolgenisAuth','MolgenisArmadillo','DSMolgenisArmadillo','DescTools','e1071'); inst <- rownames(installed.packages()); to_install <- setdiff(pkgs, inst); if (length(to_install)) install.packages(to_install, dependencies=TRUE, repos='https://cloud.r-project.org')" + + R -q -e "library('devtools'); devtools::install_github(repo='datashield/dsDangerClient', ref='6.3.4', dependencies=TRUE, upgrade='never')" # XML grep for coverage report merging sudo apt-get install -qq xml-twig-tools -y