From 500adb31a4016956cfcb661312521fb9c778c043 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 9 Nov 2026 23:01:40 +0000 Subject: [PATCH 1/3] Harvest source tarballs and linux binaries from R Universe get_candidates() takes type = "binary" (default) or "source". Source candidates need a passing source build, a passing linux x86_64 R CMD check, and, with commit = TRUE, the same commit as BBS. Binary candidates now only use binary jobs, so linux no longer mixes in source and bioc-checks jobs. get_jobs() links source jobs to their tarball and linux binaries to R Universe's bin/linux/- layout instead of src/contrib. get_repository_path() puts sources in src/contrib and linux binaries in bin/linux/-/contrib/. harvest.R accepts source and requires an arch for linux. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017odhKfCYGBNQ8MYTwajjmc --- R/utils.R | 256 ++++++++++++++++++++++++------- inst/scripts/harvest.R | 69 ++++++--- man/get_candidates.Rd | 39 +++-- man/get_jobs.Rd | 3 +- man/get_repository_path.Rd | 17 +- man/get_subpath.Rd | 6 +- man/remove_old_binaries.Rd | 12 +- man/uni_repo_url.Rd | 24 ++- tests/testthat/test-candidates.R | 69 +++++++++ tests/testthat/test-utils.R | 16 +- 10 files changed, 407 insertions(+), 104 deletions(-) create mode 100644 tests/testthat/test-candidates.R diff --git a/R/utils.R b/R/utils.R index 3c4cb9e..314f5b9 100644 --- a/R/utils.R +++ b/R/utils.R @@ -43,17 +43,22 @@ uni_pkg_file <- function(pkg, os, version) { #' @param arch character x86_64 or arm64 #' @param r_xy R X.Y character #' @param cran_subpath (default FALSE) uses CRAN style subpaths +#' @param distro character Linux distribution codename, e.g. "resolute", used +#' for linux #' #' @returns character #' #' @examples #' get_subpath("macosx", "arm64", "4.6") #' get_subpath(c("macosx", "windows"), c("arm64", "x86_64"), c("4.6", "4.5")) +#' get_subpath("linux", "x86_64", "4.6", distro = "resolute") #' #' @export -get_subpath <- function(os, arch, r_xy, cran_subpath = FALSE) { +get_subpath <- function(os, arch, r_xy, cran_subpath = FALSE, + distro = NA_character_) { is_mac <- !is.na(os) & stringr::str_detect(os, "mac") is_win <- !is.na(os) & stringr::str_detect(os, "win") + is_linux <- !is.na(os) & stringr::str_detect(os, "linux") if (any(is_mac & as.double(r_xy) < 4.6 & arch == "arm64", na.rm = TRUE)) stop("R version must be greater than or equal to 4.6 if arch is arm64") @@ -64,6 +69,7 @@ get_subpath <- function(os, arch, r_xy, cran_subpath = FALSE) { is_win & (as.double(r_xy) < 4.6 | cran_subpath) ~ "", is_win & arch == "arm64" ~ "clang-arm64", is_win ~ "gcc-x86_64", + is_linux & !is.na(distro) & !is.na(arch) ~ paste(distro, arch, sep = "-"), .default = "" ) } @@ -78,9 +84,13 @@ get_subpath <- function(os, arch, r_xy, cran_subpath = FALSE) { #' #' @param repo_root character #' @param r_version character -#' @param os character +#' @param os character, ignored when `type` is "source" #' @param arch character x86_64 or arm64 #' @param cran_subpath (default FALSE) uses CRAN style subpaths +#' @param type character "binary" (default) or "source". Source packages are +#' in `src/contrib`. +#' @param distro character Linux distribution codename, e.g. "resolute", +#' required for linux binaries #' #' @returns character #' @@ -89,24 +99,37 @@ get_subpath <- function(os, arch, r_xy, cran_subpath = FALSE) { #' get_repository_path(repo_root, "4.6.0", "windows", "arm64") #' get_repository_path(repo_root, c("4.6.0", "4.6.0"), c("windows", "macosx"), #' c("arm64", "arm64")) +#' get_repository_path(repo_root, "4.6.0", "linux", "x86_64", +#' distro = "resolute") +#' get_repository_path(repo_root, "4.6.0", type = "source") #' #' @export -get_repository_path <- function(repo_root, r_version, os, arch = NA_character_, - cran_subpath = FALSE) { +get_repository_path <- function(repo_root, r_version, os = NA_character_, + arch = NA_character_, cran_subpath = FALSE, + type = c("binary", "source"), + distro = NA_character_) { + type <- match.arg(type) + if (type == "source") + return(file.path(repo_root, "src/contrib")) + is_mac <- !is.na(os) & stringr::str_detect(os, "mac") is_win <- !is.na(os) & stringr::str_detect(os, "win") + is_linux <- !is.na(os) & stringr::str_detect(os, "linux") - if (any((is_mac | is_win) & (is.na(arch) | + if (any((is_mac | is_win | is_linux) & (is.na(arch) | !stringr::str_detect(arch, "^(x86_64|arm64)$")))) stop("arch must be x86_64 or arm64") + if (any(is_linux & is.na(distro))) + stop("distro is required for linux binaries") r_xy <- r_xy_ver(r_version) - subpath <- get_subpath(os, arch, r_xy, cran_subpath) + subpath <- get_subpath(os, arch, r_xy, cran_subpath, distro) repo_path <- dplyr::case_when( is.na(os) ~ NA_character_, is_mac ~ file.path("bin", "macosx", subpath, "contrib", r_xy), is_win ~ file.path("bin", "windows", subpath, "contrib", r_xy), - .default = "src/contrib" + is_linux ~ file.path("bin", "linux", subpath, "contrib", r_xy), + .default = NA_character_ ) repo_path <- gsub("//+", "/", repo_path) @@ -118,20 +141,57 @@ get_repository_path <- function(repo_root, r_version, os, arch = NA_character_, #' Get universe URL for an os and R version #' +#' @details R Universe puts linux binaries in +#' `bin/linux/-//src/contrib`. +#' #' @param uni character universe name #' @param r_version character -#' @param os character +#' @param os character, ignored when `type` is "source" #' @param arch character x86_64 or arm64 +#' @param type character "binary" (default) or "source" +#' @param distro character Linux distribution codename, e.g. "resolute", +#' needed for linux binaries #' -#' @returns character +#' @returns character, `NA` for linux binaries without `distro` #' #' @examples #' uni_repo_url("bioc", "4.5.3", "windows", "x86_64") +#' uni_repo_url("bioc", "4.6.1", "linux", "x86_64", distro = "resolute") +#' uni_repo_url("bioc", "4.6.1", type = "source") #' #' @export -uni_repo_url <- function(uni, r_version, os, arch = NA_character_) { +uni_repo_url <- function(uni, r_version, os = NA_character_, + arch = NA_character_, type = c("binary", "source"), + distro = NA_character_) { + type <- match.arg(type) universe <- paste0("https://", uni, ".r-universe.dev") - get_repository_path(universe, r_version, os, arch, cran_subpath = FALSE) + if (type == "source") + return(get_repository_path(universe, r_version, type = "source")) + + n <- max(length(universe), length(r_version), length(os), length(arch), + length(distro)) + universe <- rep_len(universe, n) + r_version <- rep_len(r_version, n) + os <- rep_len(os, n) + arch <- rep_len(arch, n) + distro <- rep_len(distro, n) + + is_linux <- !is.na(os) & stringr::str_detect(os, "linux") + url <- rep(NA_character_, n) + if (any(!is_linux)) + url[!is_linux] <- get_repository_path(universe[!is_linux], + r_version[!is_linux], + os[!is_linux], arch[!is_linux]) + # Without a distro there is no linux binary to point to + is_linux <- is_linux & !is.na(distro) + if (any(is_linux)) { + url[is_linux] <- file.path(universe[is_linux], "bin", "linux", + paste(distro[is_linux], arch[is_linux], + sep = "-"), + r_xy_ver(r_version[is_linux]), "src", + "contrib") + } + url } #' Get OS abbreviation @@ -352,7 +412,8 @@ get_uni_df <- function(raw_df) { #' Get jobs for an R Universe build #' -#' @description Adds artifact and job_url +#' @description Adds job_url and artifact, the download link of the binary or +#' source package a job built, `NA` if the job failed #' #' @param universe_df data.frame output of get_uni_df #' @param bu result of uni_for_bioc @@ -369,6 +430,12 @@ get_uni_df <- function(raw_df) { get_jobs <- function(universe_df, r_version, universe) { + if (!"binaries_distro" %in% names(universe_df)) + universe_df$binaries_distro <- NA_character_ + failed <- c("FAIL", "ERROR", "CANCELLED") + in_repo <- function(url, file) + ifelse(is.na(url), NA_character_, file.path(url, file)) + # Packages without an explicit binaries_arch have universal binaries binary_rows <- universe_df |> dplyr::filter( @@ -376,22 +443,31 @@ get_jobs <- function(universe_df, r_version, universe) { job_r_xy == r_xy_ver(r_version)) |> dplyr::mutate( artifact = ifelse( - binaries_check %in% c("FAIL", "ERROR", "CANCELLED") | - jobs_check %in% c("FAIL", "ERROR", "CANCELLED") | + binaries_check %in% failed | jobs_check %in% failed | binaries_status == "failure", NA_character_, - file.path(uni_repo_url(universe, binary_r_xy, - get_binary_os(binary_os), - arch = get_arch(binaries_arch, - job_arch)), - uni_pkg_file(Package, binary_os, - binaries_version)))) - - nonbinary_rows <- universe_df |> - dplyr::filter(job_type != "binary") |> + in_repo(uni_repo_url(universe, binary_r_xy, + get_binary_os(binary_os), + arch = get_arch(binaries_arch, job_arch), + distro = binaries_distro), + uni_pkg_file(Package, binary_os, binaries_version)))) + + # Source packages are built by the source job + source_rows <- universe_df |> + dplyr::filter(job_type == "source") |> + dplyr::mutate( + artifact = ifelse( + jobs_check %in% failed | is.na(Version), + NA_character_, + in_repo(uni_repo_url(universe, r_version, type = "source"), + uni_pkg_file(Package, "source", Version)))) + + other_rows <- universe_df |> + dplyr::filter(!job_type %in% c("binary", "source")) |> dplyr::mutate(artifact = NA_character_) - updated_universe_df <- dplyr::bind_rows(binary_rows, nonbinary_rows) |> + updated_universe_df <- dplyr::bind_rows(binary_rows, source_rows, + other_rows) |> dplyr::mutate(job_url = ifelse(is.na(job_id), NA_character_, file.path(`_buildurl`, "job", job_id))) |> dplyr::arrange(Package) @@ -505,34 +581,54 @@ is_unsupported_platform <- function(unsupported_str, os, arch) { #' Get candidate packages in R Universe based on criteria #' -#' @description Filters packages by same commit in BBS, vignettes passing, or -#' check status values. +#' @description Filters binary or source packages that R Universe built by +#' check results and, optionally, the same commit in BBS. +#' +#' Binary candidates are binaries for `os` and `arch` whose check passed, +#' optionally with a passing source build (`vignettes`) and the same commit as +#' BBS (`commit`). Source candidates have a passing source build and a passing +#' Linux x86_64 `R CMD check`, optionally with the same commit as BBS +#' (`commit`). #' #' @param bu result of uni_for_bioc -#' @param os character -#' @param arch character (default NA_character_) x86_64 or arm64 -#' @param vignettes logical (default TRUE) Check status of source / vignettes +#' @param os character, for binaries +#' @param arch character (default NA_character_) x86_64 or arm64, for binaries +#' @param type character "binary" (default) or "source" +#' @param vignettes logical (default TRUE) Check status of source / vignettes, +#' for binaries #' @param commit logical (default FALSE) Check commit hash in RU and BBS? #' @param unsupported_platforms logical (default TRUE) Check for unsupported -#' platforms +#' platforms, for binaries +#' @param check character results that pass +#' @param jobs data.frame from `get_jobs()`. If `NULL` (default), downloaded. #' -#' @returns data.frame of filtered candidate packages +#' @returns data.frame of filtered candidate packages, with `artifact` the +#' download link #' #' @examples +#' \dontrun{ #' bu <- uni_for_bioc("devel") -#' candidates <- get_candidates(bu, "windows", commit = TRUE) +#' candidates <- get_candidates(bu, "windows", "x86_64", commit = TRUE) +#' sources <- get_candidates(bu, type = "source", commit = TRUE) +#' } #' #' @export -get_candidates <- function(bu, os, arch = NA_character_, vignettes = TRUE, +get_candidates <- function(bu, os = NA_character_, arch = NA_character_, + type = c("binary", "source"), vignettes = TRUE, commit = FALSE, unsupported_platforms = TRUE, - check = c("NOTE", "WARNING", "OK")) { + check = c("NOTE", "WARNING", "OK"), jobs = NULL) { + type <- match.arg(type) + if (is.null(jobs)) { + raw_universe_df <- get_raw_uni_df(bu$universe) + universe_df <- get_uni_df(raw_universe_df) + jobs <- get_jobs(universe_df, bu$r_version, bu$universe) + } + if (type == "source") + return(.source_candidates(bu, jobs, commit, check)) os <- get_binary_os(os) - raw_universe_df <- get_raw_uni_df(bu$universe) - universe_df <- get_uni_df(raw_universe_df) - jobs <- get_jobs(universe_df, bu$r_version, bu$universe) candidates <- jobs |> - dplyr::filter(job_os == os) + dplyr::filter(job_type == "binary", job_os == os) if (!is.na(arch)) candidates <- candidates |> @@ -552,15 +648,7 @@ get_candidates <- function(bu, os, arch = NA_character_, vignettes = TRUE, } if (commit) { - bbs_info <- BiocPkgTools::biocBuildReport(version = bu$bioc_version, - pkgType = "software") - bbs_builds <- bbs_info |> - dplyr::rename(Package = pkg, Version = version, - BBS_commit = git_last_commit) |> - dplyr::select(Package, Version, BBS_commit, Deprecated) |> - dplyr::distinct() - - candidates <- merge(candidates, bbs_builds, + candidates <- merge(candidates, .bbs_commits(bu), by = c("Package", "Version")) candidates <- candidates |> @@ -573,20 +661,65 @@ get_candidates <- function(bu, os, arch = NA_character_, vignettes = TRUE, if (unsupported_platforms) { candidates <- candidates |> dplyr::filter( - job_type == "source" | - !purrr::pmap_lgl( - list(`Config/Bioconductor/UnsupportedPlatforms`, - job_os, - job_arch), - is_unsupported_platform - ) - ) + !purrr::pmap_lgl( + list(`Config/Bioconductor/UnsupportedPlatforms`, + job_os, + job_arch), + is_unsupported_platform)) } candidates |> dplyr::arrange(Package) } +#' Commits BBS built for software packages +#' +#' @param bu result of uni_for_bioc +#' +#' @returns data.frame with columns Package, Version, BBS_commit, Deprecated +#' +#' @noRd +.bbs_commits <- function(bu) { + BiocPkgTools::biocBuildReport(version = bu$bioc_version, + pkgType = "software") |> + dplyr::rename(Package = pkg, Version = version, + BBS_commit = git_last_commit) |> + dplyr::select(Package, Version, BBS_commit, Deprecated) |> + dplyr::distinct() +} + +#' Source packages that pass, for `get_candidates()` +#' +#' @param bu result of uni_for_bioc +#' @param jobs data.frame from `get_jobs()` +#' @param commit logical whether to require the same commit as BBS +#' @param check character results that pass +#' +#' @returns data.frame +#' +#' @noRd +.source_candidates <- function(bu, jobs, commit, check) { + linux_checks <- jobs |> + dplyr::filter(job_type == "binary", job_os == "linux") |> + filter_by_arch("linux", "x86_64") |> + dplyr::distinct(Package, Version, .keep_all = TRUE) |> + dplyr::select(Package, Version, linux_check = jobs_check) + + candidates <- jobs |> + dplyr::filter(job_type == "source", jobs_check %in% check, + !is.na(artifact)) |> + dplyr::inner_join(linux_checks, by = c("Package", "Version")) |> + dplyr::filter(linux_check %in% check) + + if (commit) + candidates <- candidates |> + dplyr::inner_join(.bbs_commits(bu), by = c("Package", "Version")) |> + dplyr::filter(BBS_commit == substr(RemoteSha, 1, 7)) + + candidates |> + dplyr::arrange(Package) +} + #' Get information about R Universe building a Bioconductor version #' #' @param branch character Bioconductor branch: "devel" or "release" @@ -625,6 +758,10 @@ uni_for_bioc <- function(branch) { #' @param r_version R x.y.z version #' @param os name, full or abbreviation #' @param arch x86_64 or arm64 +#' @param type character "binary" (default) or "source", to remove old source +#' packages from `src/contrib` +#' @param distro character Linux distribution codename, required for linux +#' binaries #' @param test logical (default TRUE) don't remove, only print packages marked #' for removal #' @@ -638,10 +775,13 @@ uni_for_bioc <- function(branch) { #' } #' #' @export -remove_old_binaries <- function(repo_root, r_version, os, arch = NA_character_, - test = TRUE) { +remove_old_binaries <- function(repo_root, r_version, os = NA_character_, + arch = NA_character_, test = TRUE, + type = c("binary", "source"), + distro = NA_character_) { binaries_path <- get_repository_path(repo_root, r_version, os, arch, - cran_subpath = TRUE) + cran_subpath = TRUE, type = type, + distro = distro) files <- list.files(binaries_path, pattern=".*._[0-9]+\\.[0-9]+\\.[0-9]+\\..*") binaries <- data.frame(file = files, latest = NA_character_) diff --git a/inst/scripts/harvest.R b/inst/scripts/harvest.R index 9c5f01e..eba726c 100644 --- a/inst/scripts/harvest.R +++ b/inst/scripts/harvest.R @@ -5,10 +5,19 @@ library(biocUniTools) # Usage: Rscript harvest.R [TEST_REMOVAL] [BIOC_VERSION] [OS] [ARCH] +# Rscript harvest.R [TEST_REMOVAL] [BIOC_VERSION] source # Example: Rscript harvest.R TRUE devel macosx arm64 # Example: Rscript harvest.R FALSE 3.23 windows arm64 - -msg <- "Usage: Rscript harvest.R [TEST_REMOVAL] [BIOC_VERSION] [OS] [ARCH]" +# Example: Rscript harvest.R FALSE 3.23 linux x86_64 +# Example: Rscript harvest.R FALSE devel source +# +# Binaries must pass R CMD check and the source build, and have the same +# commit as BBS. Source packages must pass the source build and R CMD check on +# Linux x86_64, and have the same commit as BBS. + +msg <- paste("Usage: Rscript harvest.R [TEST_REMOVAL] [BIOC_VERSION] [OS] [ARCH]", + "or Rscript harvest.R [TEST_REMOVAL] [BIOC_VERSION] source", + sep = "\n") args <- commandArgs(trailingOnly=TRUE) if (length(args) < 3) @@ -24,22 +33,20 @@ bu <- tryCatch( }) BIOC_VERSION <- bu$bioc_version -stopifnot(args[3] %in% c("windows", "linux", "macosx")) -OS <- args[3] - -if (OS %in% c("macosx", "windows") && length(args) != 4) { - stop(msg, call.=FALSE) -} else if (OS == "macosx" && length(args) == 4) { - stopifnot(args[4] %in% c("x86_64", "arm64")) - ARCH <- args[4] - LOG_FILE_BASE <- paste("harvest", OS, ARCH, sep = "-") -} else if (OS == "windows" && length(args) == 4) { +stopifnot(args[3] %in% c("windows", "linux", "macosx", "source")) +if (args[3] == "source") { + TYPE <- "source" + OS <- NA_character_ + ARCH <- NA_character_ + LOG_FILE_BASE <- "harvest-source" +} else { + TYPE <- "binary" + OS <- args[3] + if (length(args) != 4) + stop(msg, call. = FALSE) stopifnot(args[4] %in% c("x86_64", "arm64")) ARCH <- args[4] LOG_FILE_BASE <- paste("harvest", OS, ARCH, sep = "-") -} else { - ARCH <- NA - LOG_FILE_BASE <- paste("harvest", OS, sep = "-") } REPO_ROOT <- file.path("/home/biocpush/PACKAGES", BIOC_VERSION, "bioc") @@ -53,14 +60,29 @@ options(max.print = 3000L) logger::log_appender(logger::appender_file(LOG_PATH)) logger::log_info("{Sys.time()} Start") +candidates <- get_candidates(bu, os = OS, arch = ARCH, type = TYPE, + commit = TRUE) + +# Linux binaries are built for one distribution, which is part of the path +DISTRO <- NA_character_ +if (TYPE == "binary" && OS == "linux") { + DISTRO <- unique(stats::na.omit(candidates$binaries_distro)) + if (length(DISTRO) != 1) + stop("Expected linux binaries for one distribution, found: ", + paste(DISTRO, collapse = ", ")) +} + repo_path <- get_repository_path(REPO_ROOT, bu$r_version, OS, ARCH, - cran_subpath = OS == "windows") -candidates <- get_candidates(bu, os = OS, arch = ARCH, commit = TRUE) + cran_subpath = OS %in% "windows", + type = TYPE, distro = DISTRO) +dir.create(repo_path, showWarnings = FALSE, recursive = TRUE) # Remove any candidates from the list that are currently in the repository binaries <- list.files(repo_path, pattern = ".*._[0-9]+\\.[0-9]+\\.[0-9]+\\..*") candidates <- candidates |> - dplyr::mutate(File = uni_pkg_file(Package, OS, Version)) |> + dplyr::mutate(File = uni_pkg_file(Package, + if (TYPE == "source") "source" else OS, + Version)) |> dplyr::filter(!File %in% binaries) # Check for version downgrades against what is currently in the repo @@ -101,12 +123,12 @@ if (nrow(candidates) >= 1) { # Log successful downloads successful <- downloaded[downloaded$success, ] if (nrow(successful) >= 1) - logger::log_info("Downloaded {nrow(successful)} binaries: {successful$destfile}") + logger::log_info("Downloaded {nrow(successful)} {TYPE} packages: {successful$destfile}") # Log failed downloads failed <- downloaded[!downloaded$success, ] if (nrow(failed) >= 1) { - logger::log_warn("Failed to download {nrow(failed)} binaries:") + logger::log_warn("Failed to download {nrow(failed)} {TYPE} packages:") for (i in seq_len(nrow(failed))) { logger::log_warn(" {failed$url[i]} - status: {failed$status_code[i]}, error: {failed$error[i]}") } @@ -119,16 +141,17 @@ if (nrow(candidates) >= 1) { } } } else { - logger::log_info("No new binaries available.") + logger::log_info("No new {TYPE} packages available.") } prefix <- ifelse(TEST_REMOVAL, "[TEST] ", "") removed <- remove_old_binaries(REPO_ROOT, bu$r_version, OS, ARCH, - test = TEST_REMOVAL) + test = TEST_REMOVAL, type = TYPE, + distro = DISTRO) if (length(removed) >= 1) { logger::log_info("{prefix}Removed {removed}") } else { - logger::log_info("{prefix}No binaries to remove.") + logger::log_info("{prefix}No {TYPE} packages to remove.") } logger::log_info("{Sys.time()} End") diff --git a/man/get_candidates.Rd b/man/get_candidates.Rd index d383d80..280bbd5 100644 --- a/man/get_candidates.Rd +++ b/man/get_candidates.Rd @@ -6,37 +6,56 @@ \usage{ get_candidates( bu, - os, + os = NA_character_, arch = NA_character_, + type = c("binary", "source"), vignettes = TRUE, commit = FALSE, unsupported_platforms = TRUE, - check = c("NOTE", "WARNING", "OK") + check = c("NOTE", "WARNING", "OK"), + jobs = NULL ) } \arguments{ \item{bu}{result of uni_for_bioc} -\item{os}{character} +\item{os}{character, for binaries} -\item{arch}{character (default NA_character_) x86_64 or arm64} +\item{arch}{character (default NA_character_) x86_64 or arm64, for binaries} -\item{vignettes}{logical (default TRUE) Check status of source / vignettes} +\item{type}{character "binary" (default) or "source"} + +\item{vignettes}{logical (default TRUE) Check status of source / vignettes, +for binaries} \item{commit}{logical (default FALSE) Check commit hash in RU and BBS?} \item{unsupported_platforms}{logical (default TRUE) Check for unsupported -platforms} +platforms, for binaries} + +\item{check}{character results that pass} + +\item{jobs}{data.frame from \code{get_jobs()}. If \code{NULL} (default), downloaded.} } \value{ -data.frame of filtered candidate packages +data.frame of filtered candidate packages, with \code{artifact} the +download link } \description{ -Filters packages by same commit in BBS, vignettes passing, or -check status values. +Filters binary or source packages that R Universe built by +check results and, optionally, the same commit in BBS. + +Binary candidates are binaries for \code{os} and \code{arch} whose check passed, +optionally with a passing source build (\code{vignettes}) and the same commit as +BBS (\code{commit}). Source candidates have a passing source build and a passing +Linux x86_64 \verb{R CMD check}, optionally with the same commit as BBS +(\code{commit}). } \examples{ +\dontrun{ bu <- uni_for_bioc("devel") -candidates <- get_candidates(bu, "windows", commit = TRUE) +candidates <- get_candidates(bu, "windows", "x86_64", commit = TRUE) +sources <- get_candidates(bu, type = "source", commit = TRUE) +} } diff --git a/man/get_jobs.Rd b/man/get_jobs.Rd index 3576445..ab7216a 100644 --- a/man/get_jobs.Rd +++ b/man/get_jobs.Rd @@ -15,7 +15,8 @@ get_jobs(universe_df, r_version, universe) data.frame packages in a universe } \description{ -Adds artifact and job_url +Adds job_url and artifact, the download link of the binary or +source package a job built, \code{NA} if the job failed } \examples{ bu <- uni_for_bioc("devel") diff --git a/man/get_repository_path.Rd b/man/get_repository_path.Rd index 9713e17..415ad58 100644 --- a/man/get_repository_path.Rd +++ b/man/get_repository_path.Rd @@ -7,9 +7,11 @@ get_repository_path( repo_root, r_version, - os, + os = NA_character_, arch = NA_character_, - cran_subpath = FALSE + cran_subpath = FALSE, + type = c("binary", "source"), + distro = NA_character_ ) } \arguments{ @@ -17,11 +19,17 @@ get_repository_path( \item{r_version}{character} -\item{os}{character} +\item{os}{character, ignored when \code{type} is "source"} \item{arch}{character x86_64 or arm64} \item{cran_subpath}{(default FALSE) uses CRAN style subpaths} + +\item{type}{character "binary" (default) or "source". Source packages are +in \code{src/contrib}.} + +\item{distro}{character Linux distribution codename, e.g. "resolute", +required for linux binaries} } \value{ character @@ -41,5 +49,8 @@ repo_root <- "/home/biocpush/PACKAGES/3.22/bioc" get_repository_path(repo_root, "4.6.0", "windows", "arm64") get_repository_path(repo_root, c("4.6.0", "4.6.0"), c("windows", "macosx"), c("arm64", "arm64")) +get_repository_path(repo_root, "4.6.0", "linux", "x86_64", + distro = "resolute") +get_repository_path(repo_root, "4.6.0", type = "source") } diff --git a/man/get_subpath.Rd b/man/get_subpath.Rd index 93da492..afddd00 100644 --- a/man/get_subpath.Rd +++ b/man/get_subpath.Rd @@ -4,7 +4,7 @@ \alias{get_subpath} \title{Get repo subpath by R version} \usage{ -get_subpath(os, arch, r_xy, cran_subpath = FALSE) +get_subpath(os, arch, r_xy, cran_subpath = FALSE, distro = NA_character_) } \arguments{ \item{os}{character OS name, e.g. "macosx", "windows", "linux"} @@ -14,6 +14,9 @@ get_subpath(os, arch, r_xy, cran_subpath = FALSE) \item{r_xy}{R X.Y character} \item{cran_subpath}{(default FALSE) uses CRAN style subpaths} + +\item{distro}{character Linux distribution codename, e.g. "resolute", used +for linux} } \value{ character @@ -24,5 +27,6 @@ Get repo subpath by R version \examples{ get_subpath("macosx", "arm64", "4.6") get_subpath(c("macosx", "windows"), c("arm64", "x86_64"), c("4.6", "4.5")) +get_subpath("linux", "x86_64", "4.6", distro = "resolute") } diff --git a/man/remove_old_binaries.Rd b/man/remove_old_binaries.Rd index 704d9fe..37b0471 100644 --- a/man/remove_old_binaries.Rd +++ b/man/remove_old_binaries.Rd @@ -7,9 +7,11 @@ remove_old_binaries( repo_root, r_version, - os, + os = NA_character_, arch = NA_character_, - test = TRUE + test = TRUE, + type = c("binary", "source"), + distro = NA_character_ ) } \arguments{ @@ -24,6 +26,12 @@ remove_old_binaries( \item{test}{logical (default TRUE) don't remove, only print packages marked for removal} + +\item{type}{character "binary" (default) or "source", to remove old source +packages from \code{src/contrib}} + +\item{distro}{character Linux distribution codename, required for linux +binaries} } \value{ vector of the full path of binaries removed diff --git a/man/uni_repo_url.Rd b/man/uni_repo_url.Rd index 82b20b1..4e29678 100644 --- a/man/uni_repo_url.Rd +++ b/man/uni_repo_url.Rd @@ -4,24 +4,42 @@ \alias{uni_repo_url} \title{Get universe URL for an os and R version} \usage{ -uni_repo_url(uni, r_version, os, arch = NA_character_) +uni_repo_url( + uni, + r_version, + os = NA_character_, + arch = NA_character_, + type = c("binary", "source"), + distro = NA_character_ +) } \arguments{ \item{uni}{character universe name} \item{r_version}{character} -\item{os}{character} +\item{os}{character, ignored when \code{type} is "source"} \item{arch}{character x86_64 or arm64} + +\item{type}{character "binary" (default) or "source"} + +\item{distro}{character Linux distribution codename, e.g. "resolute", +needed for linux binaries} } \value{ -character +character, \code{NA} for linux binaries without \code{distro} } \description{ Get universe URL for an os and R version } +\details{ +R Universe puts linux binaries in +\verb{bin/linux/-//src/contrib}. +} \examples{ uni_repo_url("bioc", "4.5.3", "windows", "x86_64") +uni_repo_url("bioc", "4.6.1", "linux", "x86_64", distro = "resolute") +uni_repo_url("bioc", "4.6.1", type = "source") } diff --git a/tests/testthat/test-candidates.R b/tests/testthat/test-candidates.R new file mode 100644 index 0000000..717cfec --- /dev/null +++ b/tests/testthat/test-candidates.R @@ -0,0 +1,69 @@ +test_that("uni_repo_url uses R Universe's linux and source layouts", { + expect_equal(uni_repo_url("bioc", "4.6.1", "linux", "x86_64", + distro = "resolute"), + "https://bioc.r-universe.dev/bin/linux/resolute-x86_64/4.6/src/contrib") + expect_equal(uni_repo_url("bioc", "4.6.1", type = "source"), + "https://bioc.r-universe.dev/src/contrib") + expect_equal(uni_repo_url("bioc", "4.6.1", "linux", "x86_64"), + NA_character_) + expect_equal(uni_repo_url("bioc", "4.6.1", "windows", "x86_64"), + "https://bioc.r-universe.dev/bin/windows/gcc-x86_64/contrib/4.6") +}) + +# Two packages: "a" passes everywhere, "b" fails the linux check +uni_df <- data.frame( + Package = rep(c("a", "b"), each = 3), + Version = rep(c("1.0.0", "2.0.0"), each = 3), + RemoteSha = rep(c("aaaaaaa1", "bbbbbbb1"), each = 3), + job_type = rep(c("source", "binary", "binary"), 2), + job_os = rep(c("linux", "linux", "win"), 2), + job_arch = rep(c(NA, "x86_64", "x86_64"), 2), + job_r_xy = "4.6", + jobs_check = c("OK", "NOTE", "OK", "OK", "ERROR", "OK"), + binary_os = rep(c(NA, "linux", "win"), 2), + binary_r_xy = rep(c(NA, "4.6", "4.6"), 2), + binaries_version = rep(c(NA, "1.0.0", NA, "2.0.0"), c(1, 2, 1, 2)), + binaries_arch = NA_character_, + binaries_distro = rep(c(NA, "resolute", NA), 2), + binaries_check = rep(c(NA, "OK", "OK"), 2), + binaries_status = rep(c(NA, "success", "success"), 2), + job_id = 1:6, + `_buildurl` = "https://github.com/r-universe/bioc/actions/runs/1", + `Config/Bioconductor/UnsupportedPlatforms` = NA_character_, + check.names = FALSE) +jobs <- get_jobs(uni_df, "4.6.1", "bioc") + +test_that("get_jobs links source and linux binary packages", { + src <- jobs[jobs$job_type == "source" & jobs$Package == "a", ] + expect_equal(src$artifact, + "https://bioc.r-universe.dev/src/contrib/a_1.0.0.tar.gz") + lin <- jobs[jobs$job_os == "linux" & jobs$job_type == "binary" & + jobs$Package == "a", ] + expect_equal(lin$artifact, + paste0("https://bioc.r-universe.dev/bin/linux/", + "resolute-x86_64/4.6/src/contrib/a_1.0.0.tar.gz")) +}) + +test_that("get_candidates keeps only binaries for type binary", { + x <- get_candidates(list(), "linux", "x86_64", vignettes = FALSE, + jobs = jobs) + expect_true(all(x$job_type == "binary")) + expect_equal(x$Package, c("a", "b")) + # "b" failed its check, so there is nothing to download + expect_true(is.na(x$artifact[x$Package == "b"])) +}) + +test_that("get_candidates source needs the source build and linux check", { + x <- get_candidates(list(), type = "source", jobs = jobs) + expect_equal(x$Package, "a") + expect_equal(x$artifact, + "https://bioc.r-universe.dev/src/contrib/a_1.0.0.tar.gz") +}) + +test_that("get_candidates source can require the BBS commit", { + local_mocked_bindings(.bbs_commits = function(bu) + data.frame(Package = c("a", "b"), Version = c("1.0.0", "2.0.0"), + BBS_commit = c("zzzzzzz", "bbbbbbb"), Deprecated = FALSE)) + x <- get_candidates(list(), type = "source", commit = TRUE, jobs = jobs) + expect_equal(nrow(x), 0) +}) diff --git a/tests/testthat/test-utils.R b/tests/testthat/test-utils.R index 4a0bafa..43d73a0 100644 --- a/tests/testthat/test-utils.R +++ b/tests/testthat/test-utils.R @@ -51,6 +51,8 @@ test_that("get_subpath returns correct paths", { expect_equal(get_subpath("windows", "arm64", "4.6"), "clang-arm64") expect_equal(get_subpath("windows", "arm64", "4.6", TRUE), "") expect_equal(get_subpath("linux", NA_character_, "4.6"), "") + expect_equal(get_subpath("linux", "arm64", "4.6", distro = "resolute"), + "resolute-arm64") }) test_that("get_repository_path creates good paths", { @@ -58,8 +60,16 @@ test_that("get_repository_path creates good paths", { "arch must be x86_64 or arm64") expect_error(get_repository_path("reporoot", "4.6", "windows"), "arch must be x86_64 or arm64") - expect_equal(get_repository_path("reporoot", "4.5", "linux"), + expect_equal(get_repository_path("reporoot", "4.5", type = "source"), "reporoot/src/contrib") + expect_equal(get_repository_path("reporoot", "4.6", "linux", "x86_64", + distro = "resolute"), + "reporoot/bin/linux/resolute-x86_64/contrib/4.6") + expect_error(get_repository_path("reporoot", "4.6", "linux", "x86_64"), + "distro is required") + expect_error(get_repository_path("reporoot", "4.6", "linux", + distro = "resolute"), + "arch must be x86_64 or arm64") expect_equal(get_repository_path("reporoot", "4.6", "macos", "x86_64"), "reporoot/bin/macosx/big-sur-x86_64/contrib/4.6") expect_equal(get_repository_path("reporoot", "4.6", "macos", "x86_64", TRUE), @@ -263,7 +273,7 @@ test_that("uni_for_bioc gets correct universe", { test_that("remove_old_binaries only removes older binaries", { repo <- tempdir() - repo_path <- get_repository_path(repo, "4.6.0", "linux") + repo_path <- get_repository_path(repo, "4.6.0", type = "source") pkgs <- c("mytestpkg_1.2.3.tar.gz", "mytestpkg_1.3.0.tar.gz", @@ -277,7 +287,7 @@ test_that("remove_old_binaries only removes older binaries", { file.create(file.path(repo_path, pkg)) removed <- expect_no_warning( - remove_old_binaries(repo, "4.6.0", "linux", test = TRUE) + remove_old_binaries(repo, "4.6.0", test = TRUE, type = "source") ) # older versions should be marked for removal From d5c948148f36598c2f995370c3e5f8f797e62802 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 9 Nov 2026 23:09:18 +0000 Subject: [PATCH 2/3] Fix filter_by_arch() without arch, and test the new linux path filter_by_arch() failed for windows and macosx when arch is NA instead of keeping every row. The repository path test now expects linux binaries in bin/linux/-. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017odhKfCYGBNQ8MYTwajjmc --- R/utils.R | 2 ++ tests/testthat/test-utils.R | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/R/utils.R b/R/utils.R index 314f5b9..4cf84d6 100644 --- a/R/utils.R +++ b/R/utils.R @@ -500,6 +500,8 @@ get_jobs <- function(universe_df, r_version, universe) { filter_by_arch <- function(df, os, arch = "x86_64") { if (os == "linux" && is.na(arch)) arch <- "x86_64" + if (is.na(arch)) + return(df) if (os == "mac" && arch == "x86_64") { dplyr::filter(df, diff --git a/tests/testthat/test-utils.R b/tests/testthat/test-utils.R index 43d73a0..678b32f 100644 --- a/tests/testthat/test-utils.R +++ b/tests/testthat/test-utils.R @@ -121,12 +121,13 @@ test_that("get_repository_path returns the correct paths", { repo_root = "reporoot", r_version = c("4.6.0", "4.6.0", "4.5.0"), os = c("macosx", "windows", "linux"), - arch = c("arm64", "x86_64", NA_character_) + arch = c("arm64", "x86_64", "x86_64"), + distro = c(NA, NA, "resolute") ) expect_equal(result, c( "reporoot/bin/macosx/sonoma-arm64/contrib/4.6", "reporoot/bin/windows/gcc-x86_64/contrib/4.6", - "reporoot/src/contrib" + "reporoot/bin/linux/resolute-x86_64/contrib/4.5" )) }) From ed1ad9415a56445c742eb283bff9bf6d123e924d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 9 Nov 2026 23:58:30 +0000 Subject: [PATCH 3/3] Build R Universe URLs from the pattern in uni_repo_url() Choose the URL pattern per row with case_when() instead of splitting rows into linux and other platforms, which needed inputs recycled to one length and silently recycled mismatched lengths. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017odhKfCYGBNQ8MYTwajjmc --- R/utils.R | 38 +++++++++++++------------------------- 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/R/utils.R b/R/utils.R index 4cf84d6..6ece544 100644 --- a/R/utils.R +++ b/R/utils.R @@ -166,32 +166,20 @@ uni_repo_url <- function(uni, r_version, os = NA_character_, type <- match.arg(type) universe <- paste0("https://", uni, ".r-universe.dev") if (type == "source") - return(get_repository_path(universe, r_version, type = "source")) + return(file.path(universe, "src", "contrib")) - n <- max(length(universe), length(r_version), length(os), length(arch), - length(distro)) - universe <- rep_len(universe, n) - r_version <- rep_len(r_version, n) - os <- rep_len(os, n) - arch <- rep_len(arch, n) - distro <- rep_len(distro, n) - - is_linux <- !is.na(os) & stringr::str_detect(os, "linux") - url <- rep(NA_character_, n) - if (any(!is_linux)) - url[!is_linux] <- get_repository_path(universe[!is_linux], - r_version[!is_linux], - os[!is_linux], arch[!is_linux]) - # Without a distro there is no linux binary to point to - is_linux <- is_linux & !is.na(distro) - if (any(is_linux)) { - url[is_linux] <- file.path(universe[is_linux], "bin", "linux", - paste(distro[is_linux], arch[is_linux], - sep = "-"), - r_xy_ver(r_version[is_linux]), "src", - "contrib") - } - url + is_linux <- !is.na(os) & grepl("linux", os) + dplyr::case_when( + # R Universe: bin/linux/-//src/contrib + is_linux & !is.na(distro) ~ file.path(universe, "bin", "linux", + paste(distro, arch, sep = "-"), + r_xy_ver(r_version), "src", + "contrib"), + # A linux job without a binary has nothing to download + is_linux ~ NA_character_, + # Windows and macOS use the same layout as Bioconductor + .default = get_repository_path(universe, r_version, + ifelse(is_linux, NA, os), arch)) } #' Get OS abbreviation