diff --git a/DESCRIPTION b/DESCRIPTION index c25de42..1dc39a4 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -9,7 +9,6 @@ Description: A package of tools for working Bioconductor universes. License: Artistic-2.0 Depends: R (>= 4.6.0) Imports: - BiocPkgTools, dplyr, htmltools, httr2, @@ -21,7 +20,6 @@ Imports: Suggests: curl, knitr, - kableExtra, logger, quarto, reactable, diff --git a/NAMESPACE b/NAMESPACE index 9e47f66..baf343f 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -30,6 +30,10 @@ export(is_unsupported_platform) export(manifest_url) export(normalize_result) export(parse_uni_log) +export(propagation_status) +export(propagation_summary_table) +export(propagation_table) +export(propagation_targets) export(r_xy_ver) export(remove_old_binaries) export(summary_table) diff --git a/R/compare.R b/R/compare.R index 11d4855..4c2380b 100644 --- a/R/compare.R +++ b/R/compare.R @@ -143,7 +143,8 @@ compare_summary <- function(cmp) { #' @param file character path to a YAML file with `branches`, `types`, and #' `data_dir`. Optional: `platform` (BBS), `os` and `arch` (R Universe), #' `errors` (default `true`), and `max_log_downloads`, the number of R -#' Universe logs to download per run across branches (default 400). +#' Universe logs to download per run across branches (default 400), and +#' `propagation` (default `true`). #' #' @returns list #' @@ -151,7 +152,8 @@ compare_summary <- function(cmp) { .read_compare_config <- function(file) { config <- yaml::read_yaml(file) defaults <- list(platform = "x86_64-linux-gnu", os = "linux", - arch = "x86_64", errors = TRUE, max_log_downloads = 400) + arch = "x86_64", errors = TRUE, max_log_downloads = 400, + propagation = TRUE) utils::modifyList(defaults, config) } @@ -161,8 +163,9 @@ compare_summary <- function(cmp) { #' Saves one `compare--.rds` per branch and type and a #' `summary.rds` in `data_dir`. If the configuration has `errors: true`, also #' saves `errors-.rds` from `uni_errors()`, with R Universe logs cached -#' in `data_dir/logs`. Reports read these files, so the data is downloaded -#' once. +#' in `data_dir/logs`. If it has `propagation: true`, also saves +#' `propagation-.rds` from `propagation_status()`. Reports read these +#' files, so the data is downloaded once. #' #' @param config character path to a YAML configuration. See #' `reports/config.yml` on the `reports` branch of the biocUniTools @@ -199,6 +202,20 @@ write_comparisons <- function(config) { summaries[[paste(branch, type)]] <- compare_summary(cmp) cmps[[type]] <- cmp } + if (isTRUE(config$propagation)) { + # BBS commits are in the comparisons, so they are not read again + bbs <- do.call(rbind, lapply(cmps, function(cmp) { + cmp <- cmp[!is.na(cmp$bbs_node), ] + data.frame(Package = cmp$Package, Version = cmp$bbs_version, + BBS_commit = substr(cmp$bbs_commit, 1, 7), + Deprecated = cmp$bbs_deprecated) + })) + status <- propagation_status(branch, types = config$types, + jobs = jobs, bbs = bbs) + attr(status, "created") <- Sys.time() + saveRDS(status, file.path(data_dir, + sprintf("propagation-%s.rds", branch))) + } if (isTRUE(config$errors)) { errors <- uni_errors( compare_failing(cmps), cache_dir = log_dir, diff --git a/R/propagation.R b/R/propagation.R new file mode 100644 index 0000000..3dc47e8 --- /dev/null +++ b/R/propagation.R @@ -0,0 +1,208 @@ +#' Targets packages propagate to +#' +#' @description The source package and the binaries Bioconductor distributes. +#' Bioconductor only distributes binaries of software packages, so `types` +#' lists the package types each target applies to. +#' +#' @returns data.frame with columns id, label, type, os, arch, and types (a +#' list column) +#' +#' @examples +#' propagation_targets() +#' +#' @export +propagation_targets <- function() { + data.frame( + id = c("source", "windows_x86_64", "macosx_arm64", "macosx_x86_64"), + label = c("Source", "Windows x86_64", "macOS arm64", "macOS x86_64"), + type = c("source", "binary", "binary", "binary"), + os = c(NA, "windows", "macosx", "macosx"), + arch = c(NA, "x86_64", "arm64", "x86_64"), + types = I(c(list(bioc_pkg_types()$type), rep(list("software"), 3)))) +} + +#' Which Bioconductor packages propagate from R Universe +#' +#' @description +#' Uses `get_candidates()`, requiring the same commit as BBS, for each of +#' `targets` and every package of `types` in the branch's manifest. A package +#' propagates to a target if it is one of the target's candidates. +#' +#' @param branch character "devel" or "release" +#' @param types character package types, from `bioc_pkg_types()$type` +#' @param jobs data.frame from `uni_jobs()`. If `NULL` (default), downloaded. +#' @param bbs data.frame of BBS commits with columns Package, Version, +#' BBS_commit, and Deprecated. If `NULL` (default), read with +#' `bbs_status()`. +#' @param targets data.frame from `propagation_targets()` +#' +#' @returns data.frame with one row per package and columns Package, type, +#' Version, deprecated, uni_commit and bbs_commit (the commits R Universe +#' and BBS built), and, for each target id, a logical column whether it +#' propagates, `_artifact`, the file to download if it does, and +#' `_job_url`, the target's R Universe job if it does not. Targets +#' that do not apply to a package's type are `NA`. +#' +#' @examples +#' \dontrun{ +#' propagation_status("devel", types = "books") +#' } +#' +#' @export +propagation_status <- function(branch, types = bioc_pkg_types()$type, + jobs = NULL, bbs = NULL, + targets = propagation_targets()) { + bu <- uni_for_bioc(branch) + if (is.null(jobs)) + jobs <- uni_jobs(branch) + if (is.null(bbs)) + bbs <- .bbs_commits(bu) + + pkgs <- do.call(rbind, lapply(types, function(type) + data.frame(Package = bioc_manifest(branch, type), type = type))) + pkgs$Version <- jobs$Version[match(pkgs$Package, jobs$Package)] + pkgs$deprecated <- bbs$Deprecated[match(pkgs$Package, bbs$Package)] %in% + TRUE + pkgs$uni_commit <- substr(jobs$RemoteSha[match(pkgs$Package, jobs$Package)], + 1, 7) + pkgs$bbs_commit <- bbs$BBS_commit[match(pkgs$Package, bbs$Package)] + + for (i in seq_len(nrow(targets))) { + id <- targets$id[i] + type <- targets$type[i] + os <- targets$os[i] + arch <- targets$arch[i] + candidates <- get_candidates(bu, os = os, arch = arch, type = type, + commit = TRUE, jobs = jobs, bbs = bbs) + + # The target's own job, linked when a package does not propagate + target_jobs <- if (type == "source") { + jobs[jobs$job_type == "source", ] + } else { + binary_os <- get_binary_os(os) + filter_by_arch(jobs[jobs$job_type == "binary" & + jobs$job_os == binary_os, ], + binary_os, arch) + } + + applies <- pkgs$type %in% targets$types[[i]] + yes <- pkgs$Package %in% candidates$Package + pkgs[[id]] <- ifelse(applies, yes, NA) + pkgs[[paste0(id, "_artifact")]] <- ifelse( + applies & yes, + candidates$artifact[match(pkgs$Package, candidates$Package)], + NA_character_) + pkgs[[paste0(id, "_job_url")]] <- ifelse( + applies & !yes, + target_jobs$job_url[match(pkgs$Package, target_jobs$Package)], + NA_character_) + } + pkgs <- pkgs[order(pkgs$Package), , drop = FALSE] + rownames(pkgs) <- NULL + structure(pkgs, targets = targets, branch = branch) +} + +#' Number of packages that propagate to each target +#' +#' @param status data.frame from `propagation_status()` +#' +#' @returns htmltools table with a row per target +#' +#' @examples +#' \dontrun{ +#' propagation_summary_table(propagation_status("devel", types = "books")) +#' } +#' +#' @export +propagation_summary_table <- function(status) { + targets <- attr(status, "targets") + fmt <- function(x) format(x, big.mark = ",") + rows <- lapply(seq_len(nrow(targets)), function(i) { + x <- status[[targets$id[i]]] + htmltools::tags$tr( + htmltools::tags$td(targets$label[i]), + htmltools::tags$td(class = "text-end", fmt(sum(x, na.rm = TRUE))), + htmltools::tags$td(class = "text-end", + fmt(sum(!x, na.rm = TRUE)))) + }) + htmltools::tags$table( + class = "table table-sm propagation-summary", + htmltools::tags$thead(htmltools::tags$tr( + htmltools::tags$th("Target"), + htmltools::tags$th(class = "text-end", "Propagate"), + htmltools::tags$th(class = "text-end", "Don't propagate"))), + htmltools::tags$tbody(rows)) +} + +#' Interactive table of which packages propagate +#' +#' @param status data.frame from `propagation_status()` +#' +#' @returns reactable htmlwidget +#' +#' @examples +#' \dontrun{ +#' propagation_table(propagation_status("devel", types = "books")) +#' } +#' +#' @export +propagation_table <- function(status) { + if (!requireNamespace("reactable", quietly = TRUE)) + stop("Install 'reactable' to use propagation_table()") + targets <- attr(status, "targets") + types <- bioc_pkg_types() + branch <- attr(status, "branch") + df <- data.frame( + Package = sprintf('%s', + mapply(.landing_url, status$Package, status$type, + branch), + status$Package), + type = types$label[match(status$type, types$type)], + Version = status$Version, + uni_commit = status$uni_commit, + bbs_commit = status$bbs_commit, + deprecated = ifelse(status$deprecated, + paste0('', + 'Deprecated'), + "")) + pill <- function(level, text) + sprintf(paste0('%2$s'), level, text) + link <- function(url, html) + ifelse(is.na(url), html, + sprintf('%s', url, html)) + # "Yes" linked to the file, or "No" linked to the target's job + answer <- function(yes, artifact, job_url) + dplyr::case_when( + is.na(yes) ~ '—', + yes ~ link(artifact, pill("success", "Yes")), + .default = link(job_url, pill("danger", "No"))) + columns <- list( + Package = reactable::colDef(name = "Package", html = TRUE, + minWidth = 160), + type = reactable::colDef(name = "Type", minWidth = 120), + Version = reactable::colDef(name = "Version", minWidth = 90), + uni_commit = reactable::colDef(name = "R Universe", minWidth = 100), + bbs_commit = reactable::colDef(name = "BBS", minWidth = 90), + deprecated = reactable::colDef(name = "Deprecated", html = TRUE, + minWidth = 110)) + for (i in seq_len(nrow(targets))) { + id <- targets$id[i] + df[[id]] <- answer(status[[id]], status[[paste0(id, "_artifact")]], + status[[paste0(id, "_job_url")]]) + columns[[id]] <- reactable::colDef(name = targets$label[i], + html = TRUE, minWidth = 130) + } + reactable::reactable( + df, columns = columns, + columnGroups = list( + reactable::colGroup("Commit", c("uni_commit", "bbs_commit")), + reactable::colGroup("Propagates to", targets$id)), + searchable = TRUE, filterable = TRUE, compact = TRUE, striped = TRUE, + highlight = TRUE, showPageSizeOptions = TRUE, defaultPageSize = 50, + pageSizeOptions = c(25, 50, 100, 500), + class = "compare-table") +} diff --git a/R/utils.R b/R/utils.R index 6ece544..af119f7 100644 --- a/R/utils.R +++ b/R/utils.R @@ -430,7 +430,7 @@ get_jobs <- function(universe_df, r_version, universe) { job_type == "binary", job_r_xy == r_xy_ver(r_version)) |> dplyr::mutate( - artifact = ifelse( + artifact = as.character(ifelse( binaries_check %in% failed | jobs_check %in% failed | binaries_status == "failure", NA_character_, @@ -438,17 +438,17 @@ get_jobs <- function(universe_df, r_version, universe) { get_binary_os(binary_os), arch = get_arch(binaries_arch, job_arch), distro = binaries_distro), - uni_pkg_file(Package, binary_os, binaries_version)))) + uni_pkg_file(Package, binary_os, binaries_version))))) # Source packages are built by the source job source_rows <- universe_df |> dplyr::filter(job_type == "source") |> dplyr::mutate( - artifact = ifelse( + artifact = as.character(ifelse( jobs_check %in% failed | is.na(Version), NA_character_, in_repo(uni_repo_url(universe, r_version, type = "source"), - uni_pkg_file(Package, "source", Version)))) + uni_pkg_file(Package, "source", Version))))) other_rows <- universe_df |> dplyr::filter(!job_type %in% c("binary", "source")) |> @@ -468,8 +468,14 @@ get_jobs <- function(universe_df, r_version, universe) { #' function filters specifically for data observed in R Universe, which could #' change in the future. #' -#' * mac universal binaries: job_arch == 'arm64' & !is.na(binaries_arch) -#' * linux/win default binaries: job_arch == 'x86_64' & !is.na(binaries_arch) +#' * mac universal binaries: job_arch == 'arm64' & is.na(binaries_arch), for +#' packages without a mac x86_64 job +#' * linux/win universal binaries: job_arch == 'x86_64' & is.na(binaries_arch), +#' for packages without an arm64 job +#' +#' Packages that need compilation have a job for each architecture, so a row +#' without binaries_arch from them is a job that built no binary, not a +#' universal binary, and is not used for the other architecture. #' #' @param df data.frame from get_jobs() #' @param os character @@ -491,9 +497,17 @@ filter_by_arch <- function(df, os, arch = "x86_64") { if (is.na(arch)) return(df) + # Packages that need compilation have a job for each architecture; only + # packages without one for `job_arch` can have a universal binary + has_job <- function(job_arch) + paste(df$Package, df$job_os) %in% + paste(df$Package, df$job_os)[df$job_arch %in% job_arch] + if (os == "mac" && arch == "x86_64") { dplyr::filter(df, - (binaries_arch == "x86_64") | (job_arch == "arm64" & is.na(binaries_arch))) + (binaries_arch == "x86_64") | + (job_arch == "arm64" & is.na(binaries_arch) & + !has_job("x86_64"))) } else if (os == "mac" && arch == "arm64") { dplyr::filter(df, (binaries_arch == "aarch64") | (job_arch == "arm64" & is.na(binaries_arch))) @@ -502,7 +516,9 @@ filter_by_arch <- function(df, os, arch = "x86_64") { (binaries_arch == "x86_64") | (job_arch == "x86_64" & is.na(binaries_arch))) } else if (os %in% c("linux", "win") && arch == "arm64") { dplyr::filter(df, - (binaries_arch == "aarch64") | (job_arch == "x86_64" & is.na(binaries_arch))) + (binaries_arch == "aarch64") | + (job_arch == "x86_64" & is.na(binaries_arch) & + !has_job("arm64"))) } else { df } @@ -571,18 +587,29 @@ is_unsupported_platform <- function(unsupported_str, os, arch) { #' Get candidate packages in R Universe based on criteria #' -#' @description Filters binary or source packages that R Universe built by -#' check results and, optionally, the same commit in BBS. +#' @description +#' Finds the packages R Universe built that can propagate to Bioconductor. All +#' propagation criteria are in this function; each argument turns one on or +#' off. +#' +#' Binary candidates are binaries for `os` and `arch` that passed +#' `R CMD check` and built, and pass: +#' * `vignettes`: the source build (`R CMD build`, which builds vignettes) +#' passed +#' * `commit`: R Universe built the same commit as BBS +#' * `unsupported_platforms`: the package does not declare the platform +#' unsupported #' -#' Binary candidates are binaries for `os` and `arch` whose check passed, -#' optionally with a passing source build (`vignettes`) and the same commit as -#' BBS (`commit`). Source candidates have a passing source build and a passing -#' Linux x86_64 `R CMD check`, optionally with the same commit as BBS -#' (`commit`). +#' Source candidates are source packages whose source build passed and whose +#' `R CMD check` on Linux x86_64 passed, and, with `commit`, R Universe built +#' the same commit as BBS. +#' +#' `check` is the results that pass. #' #' @param bu result of uni_for_bioc #' @param os character, for binaries -#' @param arch character (default NA_character_) x86_64 or arm64, for binaries +#' @param arch character x86_64 or arm64, required for windows and macosx +#' binaries #' @param type character "binary" (default) or "source" #' @param vignettes logical (default TRUE) Check status of source / vignettes, #' for binaries @@ -591,8 +618,11 @@ is_unsupported_platform <- function(unsupported_str, os, arch) { #' platforms, for binaries #' @param check character results that pass #' @param jobs data.frame from `get_jobs()`. If `NULL` (default), downloaded. +#' @param bbs data.frame of BBS commits with columns Package, Version, and +#' BBS_commit, used when `commit` is `TRUE`. If `NULL` (default), read with +#' `bbs_status()`. #' -#' @returns data.frame of filtered candidate packages, with `artifact` the +#' @returns data.frame of the jobs that propagate, with `artifact` the #' download link #' #' @examples @@ -606,49 +636,73 @@ is_unsupported_platform <- function(unsupported_str, os, arch) { get_candidates <- function(bu, os = NA_character_, arch = NA_character_, type = c("binary", "source"), vignettes = TRUE, commit = FALSE, unsupported_platforms = TRUE, - check = c("NOTE", "WARNING", "OK"), jobs = NULL) { + check = c("NOTE", "WARNING", "OK"), jobs = NULL, + bbs = NULL) { type <- match.arg(type) + if (type == "binary" && get_binary_os(os) %in% c("mac", "win") && + is.na(arch)) + stop("arch must be x86_64 or arm64 for ", os, " binaries") if (is.null(jobs)) { raw_universe_df <- get_raw_uni_df(bu$universe) universe_df <- get_uni_df(raw_universe_df) jobs <- get_jobs(universe_df, bu$r_version, bu$universe) } - if (type == "source") - return(.source_candidates(bu, jobs, commit, check)) + key <- function(df) paste(df$Package, df$Version) + source_jobs <- jobs |> + dplyr::filter(job_type == "source") - os <- get_binary_os(os) - candidates <- jobs |> - dplyr::filter(job_type == "binary", job_os == os) + if (type == "source") { + candidates <- source_jobs - if (!is.na(arch)) + # The source build passed, so there is a source package candidates <- candidates |> - filter_by_arch(os, arch) + dplyr::filter(jobs_check %in% check, !is.na(artifact)) + + # R CMD check passed on Linux x86_64 + linux_jobs <- jobs |> + dplyr::filter(job_type == "binary", job_os == "linux") |> + filter_by_arch("linux", "x86_64") + linux_check <- linux_jobs$jobs_check[match(key(candidates), + key(linux_jobs))] + candidates <- candidates[linux_check %in% check, ] + } else { + os <- get_binary_os(os) + candidates <- jobs |> + dplyr::filter(job_type == "binary", job_os == os) - if (vignettes) { - vignette_jobs <- jobs |> - dplyr::filter(job_type == "source") |> - dplyr::select(Package, Version, jobs_check) |> - dplyr::rename(vignettes_check = jobs_check) + if (!is.na(arch)) + candidates <- candidates |> + filter_by_arch(os, arch) - candidates <- merge(candidates, vignette_jobs, - by = c("Package", "Version")) + # A Mac binary without compiled code works on both architectures; + # link arm64 to the arm64 folder, where R Universe also serves it + if (os == "mac" && arch %in% "arm64") + candidates$artifact <- .mac_arm64_artifact(candidates) + # The binary passed R CMD check and built candidates <- candidates |> - dplyr::filter(vignettes_check %in% check) + dplyr::filter(binaries_check %in% check, + jobs_check %in% check, + binaries_status == "success") + + if (vignettes) { + vignettes_check <- source_jobs$jobs_check[match(key(candidates), + key(source_jobs))] + candidates <- candidates[vignettes_check %in% check, ] + } } if (commit) { - candidates <- merge(candidates, .bbs_commits(bu), - by = c("Package", "Version")) + if (is.null(bbs)) + bbs <- .bbs_commits(bu) - candidates <- candidates |> - dplyr::filter(binaries_check %in% check, - jobs_check %in% check, - binaries_status == "success") |> - dplyr::filter(BBS_commit == substr(RemoteSha, 1, 7)) + # R Universe built the same commit as BBS; packages not in BBS fail + bbs_commit <- bbs$BBS_commit[match(key(candidates), key(bbs))] + same_commit <- substr(candidates$RemoteSha, 1, 7) == bbs_commit + candidates <- candidates[same_commit %in% TRUE, ] } - if (unsupported_platforms) { + if (unsupported_platforms && type == "binary") { candidates <- candidates |> dplyr::filter( !purrr::pmap_lgl( @@ -662,52 +716,44 @@ get_candidates <- function(bu, os = NA_character_, arch = NA_character_, dplyr::arrange(Package) } -#' Commits BBS built for software packages +#' Link a Mac binary built for both architectures to the arm64 folder #' -#' @param bu result of uni_for_bioc +#' @description A binary without compiled code works on both Mac +#' architectures. `get_jobs()` links it to the x86_64 (big-sur) folder, and R +#' Universe serves the same file from the arm64 (sonoma) folder, which is +#' where arm64 binaries are expected. #' -#' @returns data.frame with columns Package, Version, BBS_commit, Deprecated +#' @param rows data.frame of mac binary jobs from `get_jobs()` +#' +#' @returns character artifacts #' #' @noRd -.bbs_commits <- function(bu) { - BiocPkgTools::biocBuildReport(version = bu$bioc_version, - pkgType = "software") |> - dplyr::rename(Package = pkg, Version = version, - BBS_commit = git_last_commit) |> - dplyr::select(Package, Version, BBS_commit, Deprecated) |> - dplyr::distinct() +.mac_arm64_artifact <- function(rows) { + r_xy <- rows$binary_r_xy + x86 <- paste0("/bin/macosx/", get_subpath("mac", "x86_64", r_xy), "/") + arm <- paste0("/bin/macosx/", get_subpath("mac", "arm64", r_xy), "/") + universal <- is.na(rows$binaries_arch) & !is.na(rows$artifact) & + !is.na(r_xy) + out <- rows$artifact + out[universal] <- vapply(which(universal), function(i) + sub(x86[i], arm[i], out[i], fixed = TRUE), character(1)) + out } -#' Source packages that pass, for `get_candidates()` +#' Commits BBS built, for every package type #' #' @param bu result of uni_for_bioc -#' @param jobs data.frame from `get_jobs()` -#' @param commit logical whether to require the same commit as BBS -#' @param check character results that pass #' -#' @returns data.frame +#' @returns data.frame with columns Package, Version, BBS_commit, Deprecated #' #' @noRd -.source_candidates <- function(bu, jobs, commit, check) { - linux_checks <- jobs |> - dplyr::filter(job_type == "binary", job_os == "linux") |> - filter_by_arch("linux", "x86_64") |> - dplyr::distinct(Package, Version, .keep_all = TRUE) |> - dplyr::select(Package, Version, linux_check = jobs_check) - - candidates <- jobs |> - dplyr::filter(job_type == "source", jobs_check %in% check, - !is.na(artifact)) |> - dplyr::inner_join(linux_checks, by = c("Package", "Version")) |> - dplyr::filter(linux_check %in% check) - - if (commit) - candidates <- candidates |> - dplyr::inner_join(.bbs_commits(bu), by = c("Package", "Version")) |> - dplyr::filter(BBS_commit == substr(RemoteSha, 1, 7)) - - candidates |> - dplyr::arrange(Package) +.bbs_commits <- function(bu) { + do.call(rbind, lapply(bioc_pkg_types()$type, function(type) { + bbs <- bbs_status(bu$bioc_branch, type) + data.frame(Package = bbs$Package, Version = bbs$Version, + BBS_commit = substr(bbs$commit, 1, 7), + Deprecated = bbs$deprecated) + })) } #' Get information about R Universe building a Bioconductor version diff --git a/man/filter_by_arch.Rd b/man/filter_by_arch.Rd index 5013465..2a40ec7 100644 --- a/man/filter_by_arch.Rd +++ b/man/filter_by_arch.Rd @@ -21,9 +21,15 @@ Some builds can have arch-specific and universal binaries. This function filters specifically for data observed in R Universe, which could change in the future. \itemize{ -\item mac universal binaries: job_arch == 'arm64' & !is.na(binaries_arch) -\item linux/win default binaries: job_arch == 'x86_64' & !is.na(binaries_arch) +\item mac universal binaries: job_arch == 'arm64' & is.na(binaries_arch), for +packages without a mac x86_64 job +\item linux/win universal binaries: job_arch == 'x86_64' & is.na(binaries_arch), +for packages without an arm64 job } + +Packages that need compilation have a job for each architecture, so a row +without binaries_arch from them is a job that built no binary, not a +universal binary, and is not used for the other architecture. } \examples{ bu <- uni_for_bioc("devel") diff --git a/man/get_candidates.Rd b/man/get_candidates.Rd index 280bbd5..477c659 100644 --- a/man/get_candidates.Rd +++ b/man/get_candidates.Rd @@ -13,7 +13,8 @@ get_candidates( commit = FALSE, unsupported_platforms = TRUE, check = c("NOTE", "WARNING", "OK"), - jobs = NULL + jobs = NULL, + bbs = NULL ) } \arguments{ @@ -21,7 +22,8 @@ get_candidates( \item{os}{character, for binaries} -\item{arch}{character (default NA_character_) x86_64 or arm64, for binaries} +\item{arch}{character x86_64 or arm64, required for windows and macosx +binaries} \item{type}{character "binary" (default) or "source"} @@ -36,20 +38,35 @@ platforms, for binaries} \item{check}{character results that pass} \item{jobs}{data.frame from \code{get_jobs()}. If \code{NULL} (default), downloaded.} + +\item{bbs}{data.frame of BBS commits with columns Package, Version, and +BBS_commit, used when \code{commit} is \code{TRUE}. If \code{NULL} (default), read with +\code{bbs_status()}.} } \value{ -data.frame of filtered candidate packages, with \code{artifact} the +data.frame of the jobs that propagate, with \code{artifact} the download link } \description{ -Filters binary or source packages that R Universe built by -check results and, optionally, the same commit in BBS. - -Binary candidates are binaries for \code{os} and \code{arch} whose check passed, -optionally with a passing source build (\code{vignettes}) and the same commit as -BBS (\code{commit}). Source candidates have a passing source build and a passing -Linux x86_64 \verb{R CMD check}, optionally with the same commit as BBS -(\code{commit}). +Finds the packages R Universe built that can propagate to Bioconductor. All +propagation criteria are in this function; each argument turns one on or +off. + +Binary candidates are binaries for \code{os} and \code{arch} that passed +\verb{R CMD check} and built, and pass: +\itemize{ +\item \code{vignettes}: the source build (\verb{R CMD build}, which builds vignettes) +passed +\item \code{commit}: R Universe built the same commit as BBS +\item \code{unsupported_platforms}: the package does not declare the platform +unsupported +} + +Source candidates are source packages whose source build passed and whose +\verb{R CMD check} on Linux x86_64 passed, and, with \code{commit}, R Universe built +the same commit as BBS. + +\code{check} is the results that pass. } \examples{ \dontrun{ diff --git a/man/propagation_status.Rd b/man/propagation_status.Rd new file mode 100644 index 0000000..70d451f --- /dev/null +++ b/man/propagation_status.Rd @@ -0,0 +1,46 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/propagation.R +\name{propagation_status} +\alias{propagation_status} +\title{Which Bioconductor packages propagate from R Universe} +\usage{ +propagation_status( + branch, + types = bioc_pkg_types()$type, + jobs = NULL, + bbs = NULL, + targets = propagation_targets() +) +} +\arguments{ +\item{branch}{character "devel" or "release"} + +\item{types}{character package types, from \code{bioc_pkg_types()$type}} + +\item{jobs}{data.frame from \code{uni_jobs()}. If \code{NULL} (default), downloaded.} + +\item{bbs}{data.frame of BBS commits with columns Package, Version, +BBS_commit, and Deprecated. If \code{NULL} (default), read with +\code{bbs_status()}.} + +\item{targets}{data.frame from \code{propagation_targets()}} +} +\value{ +data.frame with one row per package and columns Package, type, +Version, deprecated, uni_commit and bbs_commit (the commits R Universe +and BBS built), and, for each target id, a logical column whether it +propagates, \verb{_artifact}, the file to download if it does, and +\verb{_job_url}, the target's R Universe job if it does not. Targets +that do not apply to a package's type are \code{NA}. +} +\description{ +Uses \code{get_candidates()}, requiring the same commit as BBS, for each of +\code{targets} and every package of \code{types} in the branch's manifest. A package +propagates to a target if it is one of the target's candidates. +} +\examples{ +\dontrun{ +propagation_status("devel", types = "books") +} + +} diff --git a/man/propagation_summary_table.Rd b/man/propagation_summary_table.Rd new file mode 100644 index 0000000..2929a95 --- /dev/null +++ b/man/propagation_summary_table.Rd @@ -0,0 +1,23 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/propagation.R +\name{propagation_summary_table} +\alias{propagation_summary_table} +\title{Number of packages that propagate to each target} +\usage{ +propagation_summary_table(status) +} +\arguments{ +\item{status}{data.frame from \code{propagation_status()}} +} +\value{ +htmltools table with a row per target +} +\description{ +Number of packages that propagate to each target +} +\examples{ +\dontrun{ +propagation_summary_table(propagation_status("devel", types = "books")) +} + +} diff --git a/man/propagation_table.Rd b/man/propagation_table.Rd new file mode 100644 index 0000000..437633c --- /dev/null +++ b/man/propagation_table.Rd @@ -0,0 +1,23 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/propagation.R +\name{propagation_table} +\alias{propagation_table} +\title{Interactive table of which packages propagate} +\usage{ +propagation_table(status) +} +\arguments{ +\item{status}{data.frame from \code{propagation_status()}} +} +\value{ +reactable htmlwidget +} +\description{ +Interactive table of which packages propagate +} +\examples{ +\dontrun{ +propagation_table(propagation_status("devel", types = "books")) +} + +} diff --git a/man/propagation_targets.Rd b/man/propagation_targets.Rd new file mode 100644 index 0000000..211706a --- /dev/null +++ b/man/propagation_targets.Rd @@ -0,0 +1,21 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/propagation.R +\name{propagation_targets} +\alias{propagation_targets} +\title{Targets packages propagate to} +\usage{ +propagation_targets() +} +\value{ +data.frame with columns id, label, type, os, arch, and types (a +list column) +} +\description{ +The source package and the binaries Bioconductor distributes. +Bioconductor only distributes binaries of software packages, so \code{types} +lists the package types each target applies to. +} +\examples{ +propagation_targets() + +} diff --git a/man/write_comparisons.Rd b/man/write_comparisons.Rd index b5904a8..ef6e313 100644 --- a/man/write_comparisons.Rd +++ b/man/write_comparisons.Rd @@ -18,8 +18,9 @@ data.frame summary, invisibly Saves one \verb{compare--.rds} per branch and type and a \code{summary.rds} in \code{data_dir}. If the configuration has \code{errors: true}, also saves \verb{errors-.rds} from \code{uni_errors()}, with R Universe logs cached -in \code{data_dir/logs}. Reports read these files, so the data is downloaded -once. +in \code{data_dir/logs}. If it has \code{propagation: true}, also saves +\verb{propagation-.rds} from \code{propagation_status()}. Reports read these +files, so the data is downloaded once. } \examples{ \dontrun{ diff --git a/tests/testthat/test-candidates.R b/tests/testthat/test-candidates.R index 717cfec..3ba74cf 100644 --- a/tests/testthat/test-candidates.R +++ b/tests/testthat/test-candidates.R @@ -48,9 +48,8 @@ test_that("get_candidates keeps only binaries for type binary", { x <- get_candidates(list(), "linux", "x86_64", vignettes = FALSE, jobs = jobs) expect_true(all(x$job_type == "binary")) - expect_equal(x$Package, c("a", "b")) - # "b" failed its check, so there is nothing to download - expect_true(is.na(x$artifact[x$Package == "b"])) + # "b" failed its linux check, so it is not a candidate + expect_equal(x$Package, "a") }) test_that("get_candidates source needs the source build and linux check", { @@ -67,3 +66,25 @@ test_that("get_candidates source can require the BBS commit", { x <- get_candidates(list(), type = "source", commit = TRUE, jobs = jobs) expect_equal(nrow(x), 0) }) + +test_that("mac binaries for both architectures use each target's folder", { + mac <- data.frame( + Package = "m", Version = "1.0.0", RemoteSha = "mmmmmmm1", + job_type = "binary", job_os = "mac", job_arch = "arm64", + job_r_xy = "4.6", jobs_check = "OK", binary_os = "mac", + binary_r_xy = "4.6", binaries_version = "1.0.0", + binaries_arch = NA_character_, binaries_distro = NA_character_, + binaries_check = "OK", binaries_status = "success", job_id = 9L, + `_buildurl` = "https://github.com/r-universe/bioc/actions/runs/1", + `Config/Bioconductor/UnsupportedPlatforms` = NA_character_, + check.names = FALSE) + mac_jobs <- get_jobs(mac, "4.6.1", "bioc") + arm <- get_candidates(list(), "macosx", "arm64", vignettes = FALSE, + jobs = mac_jobs) + x86 <- get_candidates(list(), "macosx", "x86_64", vignettes = FALSE, + jobs = mac_jobs) + expect_equal(arm$artifact, paste0("https://bioc.r-universe.dev/bin/macosx/", + "sonoma-arm64/contrib/4.6/m_1.0.0.tgz")) + expect_equal(x86$artifact, paste0("https://bioc.r-universe.dev/bin/macosx/", + "big-sur-x86_64/contrib/4.6/m_1.0.0.tgz")) +}) diff --git a/tests/testthat/test-criteria.R b/tests/testthat/test-criteria.R new file mode 100644 index 0000000..2f7f8ce --- /dev/null +++ b/tests/testthat/test-criteria.R @@ -0,0 +1,103 @@ +# One package per criterion, each failing only that criterion, plus one +# that passes everything. Jobs are built from a small universe table. +make_jobs <- function() { + pkgs <- c("pass", "no_source", "no_check", "no_commit", "unsupported", + "note_only") + one <- function(pkg, source_check, win_check, unsupported) { + data.frame( + Package = pkg, Version = "1.0.0", + RemoteSha = paste0(substr(pkg, 1, 1), "aaaaaa1"), + job_type = c("source", "binary", "binary"), + job_os = c("linux", "linux", "win"), + job_arch = c(NA, "x86_64", "x86_64"), job_r_xy = "4.6", + jobs_check = c(source_check, "OK", win_check), + binary_os = c(NA, "linux", "win"), + binary_r_xy = c(NA, "4.6", "4.6"), + binaries_version = c(NA, "1.0.0", "1.0.0"), + binaries_arch = NA_character_, + binaries_distro = c(NA, "resolute", NA), + binaries_check = c(NA, "OK", win_check), + binaries_status = c(NA, "success", "success"), + job_id = seq_len(3) + 10 * match(pkg, pkgs), + `_buildurl` = "https://github.com/r-universe/bioc/actions/runs/1", + `Config/Bioconductor/UnsupportedPlatforms` = unsupported, + check.names = FALSE) + } + uni_df <- rbind(one("pass", "OK", "OK", NA), + one("no_source", "ERROR", "OK", NA), + one("no_check", "OK", "ERROR", NA), + one("no_commit", "OK", "OK", NA), + one("unsupported", "OK", "OK", "windows"), + one("note_only", "NOTE", "OK", NA)) + get_jobs(uni_df, "4.6.1", "bioc") +} +jobs <- make_jobs() +# BBS built the same commits, except for no_commit +bbs <- data.frame(Package = unique(jobs$Package), Version = "1.0.0") +bbs$BBS_commit <- paste0(substr(bbs$Package, 1, 1), "aaaaaa") +bbs$BBS_commit[bbs$Package == "no_commit"] <- "zzzzzzz" + +win <- function(...) + sort(get_candidates(list(), "windows", "x86_64", jobs = jobs, bbs = bbs, + ...)$Package) + +test_that("binary candidates pass every criterion that is on", { + expect_equal(win(commit = TRUE), c("note_only", "pass")) +}) + +test_that("vignettes turns only the source build criterion on and off", { + expect_false("no_source" %in% win(commit = TRUE)) + expect_true("no_source" %in% win(commit = TRUE, vignettes = FALSE)) +}) + +test_that("commit turns the commit criterion on and off", { + expect_false("no_commit" %in% win(commit = TRUE)) + expect_true("no_commit" %in% win(commit = FALSE)) +}) + +test_that("the binary check applies whether or not commit is on", { + expect_false("no_check" %in% win(commit = TRUE)) + expect_false("no_check" %in% win(commit = FALSE)) +}) + +test_that("arch is required for windows and macosx binaries", { + expect_error(get_candidates(list(), "windows", jobs = jobs), + "arch must be x86_64 or arm64") + expect_error(get_candidates(list(), "macosx", jobs = jobs), + "arch must be x86_64 or arm64") +}) + +test_that("unsupported_platforms turns only the platform criterion on and off", { + expect_false("unsupported" %in% win(commit = TRUE)) + expect_true("unsupported" %in% + win(commit = TRUE, unsupported_platforms = FALSE)) +}) + +test_that("check sets which results pass", { + expect_false("note_only" %in% win(commit = TRUE, check = "OK")) +}) + +src <- function(...) + sort(get_candidates(list(), type = "source", jobs = jobs, bbs = bbs, + ...)$Package) + +test_that("source candidates need the source build, linux check, and commit", { + # Every package passed its linux check; no_source failed its source build + expect_equal(src(commit = TRUE), + c("no_check", "note_only", "pass", "unsupported")) + expect_true("no_commit" %in% src(commit = FALSE)) + expect_false("no_source" %in% src(commit = FALSE)) + expect_false("note_only" %in% src(check = "OK")) +}) + +test_that("source candidates link to the source package", { + x <- get_candidates(list(), type = "source", jobs = jobs) + expect_equal(x$artifact[x$Package == "pass"], + "https://bioc.r-universe.dev/src/contrib/pass_1.0.0.tar.gz") +}) + +test_that("get_candidates returns only the jobs' columns", { + x <- get_candidates(list(), "windows", "x86_64", commit = TRUE, + jobs = jobs, bbs = bbs) + expect_true(all(names(x) %in% names(jobs))) +}) diff --git a/tests/testthat/test-propagation.R b/tests/testthat/test-propagation.R new file mode 100644 index 0000000..52b57eb --- /dev/null +++ b/tests/testthat/test-propagation.R @@ -0,0 +1,53 @@ +# "a" passes everywhere; "b" fails the linux check; "d" is a data package +uni_df <- data.frame( + Package = c(rep(c("a", "b"), each = 3), "d"), + Version = c(rep(c("1.0.0", "2.0.0"), each = 3), "3.0.0"), + RemoteSha = c(rep(c("aaaaaaa1", "bbbbbbb1"), each = 3), "ddddddd1"), + job_type = c(rep(c("source", "binary", "binary"), 2), "source"), + job_os = c(rep(c("linux", "linux", "win"), 2), "linux"), + job_arch = c(rep(c(NA, "x86_64", "x86_64"), 2), NA), + job_r_xy = "4.6", + jobs_check = c("OK", "NOTE", "OK", "OK", "ERROR", "OK", "OK"), + binary_os = c(rep(c(NA, "linux", "win"), 2), NA), + binary_r_xy = c(rep(c(NA, "4.6", "4.6"), 2), NA), + binaries_version = c(NA, "1.0.0", "1.0.0", NA, "2.0.0", "2.0.0", NA), + binaries_arch = NA_character_, + binaries_distro = c(rep(c(NA, "resolute", NA), 2), NA), + binaries_check = c(rep(c(NA, "OK", "OK"), 2), NA), + binaries_status = c(rep(c(NA, "success", "success"), 2), NA), + job_id = 1:7, + `_buildurl` = "https://github.com/r-universe/bioc/actions/runs/1", + `Config/Bioconductor/UnsupportedPlatforms` = NA_character_, + check.names = FALSE) +jobs <- get_jobs(uni_df, "4.6.1", "bioc") +bbs <- data.frame(Package = c("a", "b", "d"), + Version = c("1.0.0", "2.0.0", "3.0.0"), + BBS_commit = c("aaaaaaa", "bbbbbbb", "zzzzzzz"), + Deprecated = c(FALSE, TRUE, FALSE)) + +test_that("propagation_status marks what get_candidates() selects", { + local_mocked_bindings( + uni_for_bioc = function(branch) list(bioc_branch = branch), + bioc_manifest = function(branch, type) + switch(type, software = c("a", "b"), "data-experiment" = "d")) + targets <- propagation_targets()[1:2, ] + st <- propagation_status("devel", types = c("software", "data-experiment"), + jobs = jobs, bbs = bbs, targets = targets) + expect_equal(st$Package, c("a", "b", "d")) + # "b" fails its linux check; "d" has a different commit than BBS + expect_equal(st$source, c(TRUE, FALSE, FALSE)) + # binaries only apply to software + expect_equal(st$windows_x86_64, c(TRUE, TRUE, NA)) + expect_equal(st$deprecated, c(FALSE, TRUE, FALSE)) + expect_equal(st$uni_commit, c("aaaaaaa", "bbbbbbb", "ddddddd")) + expect_equal(st$bbs_commit, c("aaaaaaa", "bbbbbbb", "zzzzzzz")) + # "Yes" links to the file; "No" links to the target's own job + expect_equal(st$source_artifact, + c("https://bioc.r-universe.dev/src/contrib/a_1.0.0.tar.gz", + NA, NA)) + expect_equal(st$source_job_url, + c(NA, "https://github.com/r-universe/bioc/actions/runs/1/job/4", + "https://github.com/r-universe/bioc/actions/runs/1/job/7")) + skip_if_not_installed("reactable") + expect_s3_class(propagation_table(st), "reactable") +}) diff --git a/tests/testthat/test-utils.R b/tests/testthat/test-utils.R index 678b32f..5f90575 100644 --- a/tests/testthat/test-utils.R +++ b/tests/testthat/test-utils.R @@ -231,32 +231,36 @@ test_that("get_candidates gets R Universe data", { test_that("get_candidates filters on check status", { # without commit filter, check status is not filtered - candidates <- get_candidates(devel, "windows") + candidates <- get_candidates(devel, "windows", "x86_64") expect_gt(nrow(candidates), 0) # with commit filter, check status is filtered - candidates_commit <- get_candidates(devel, "windows", commit = TRUE) + candidates_commit <- get_candidates(devel, "windows", "x86_64", commit = TRUE) expect_true(all(candidates_commit$binaries_check %in% c("NOTE", "WARNING", "OK"))) expect_true(all(candidates_commit$jobs_check %in% c("NOTE", "WARNING", "OK"))) expect_true(all(candidates_commit$binaries_status == "success")) # restrict to OK only - ok_only <- get_candidates(devel, "windows", commit = TRUE, check = "OK") + ok_only <- get_candidates(devel, "windows", "x86_64", commit = TRUE, check = "OK") expect_true(all(ok_only$binaries_check == "OK")) expect_true(all(ok_only$jobs_check == "OK")) }) test_that("get_candidates commit filter works", { - without_commit <- get_candidates(devel, "windows", commit = FALSE) - with_commit <- get_candidates(devel, "windows", commit = TRUE) + without_commit <- get_candidates(devel, "windows", "x86_64", commit = FALSE) + with_commit <- get_candidates(devel, "windows", "x86_64", commit = TRUE) # commit filter should only reduce or maintain row count expect_lte(nrow(with_commit), nrow(without_commit)) - # commit-filtered rows should all have matching commits - if (nrow(with_commit) > 0) - expect_true(all(with_commit$BBS_commit == - substr(with_commit$RemoteSha, 1, 7))) + # commit-filtered rows should all have the commit BBS built + if (nrow(with_commit) > 0) { + bbs <- biocUniTools:::.bbs_commits(devel) + bbs_commit <- bbs$BBS_commit[match( + paste(with_commit$Package, with_commit$Version), + paste(bbs$Package, bbs$Version))] + expect_true(all(bbs_commit == substr(with_commit$RemoteSha, 1, 7))) + } }) test_that("get_candidates for sonoma includes arm64 and universal binaries", { @@ -434,24 +438,58 @@ test_that("filter_by_arch passes all rows for windows", { expect_equal(nrow(result), 2) }) +# Source build results from R Universe, to check candidates against +release_jobs <- get_jobs(get_uni_df(get_raw_uni_df(release$universe)), + release$r_version, release$universe) +source_check <- function(candidates) { + source_jobs <- release_jobs[release_jobs$job_type == "source", ] + source_jobs$jobs_check[match(candidates$Package, source_jobs$Package)] +} + test_that("get_candidates vignette filter works", { # vignettes = TRUE (default) should filter on source check - with_vignettes <- get_candidates(release, "windows", vignettes = TRUE) - without_vignettes <- get_candidates(release, "windows", vignettes = FALSE) - + with_vignettes <- get_candidates(release, "windows", "x86_64", vignettes = TRUE, + jobs = release_jobs) + without_vignettes <- get_candidates(release, "windows", "x86_64", vignettes = FALSE, + jobs = release_jobs) + # vignette filter should only reduce or maintain row count expect_lte(nrow(with_vignettes), nrow(without_vignettes)) - - # all rows should have passing vignette check - expect_true(all(with_vignettes$vignettes_check %in% c("NOTE", "WARNING", "OK"))) + + # all candidates should have a passing source build + expect_true(all(source_check(with_vignettes) %in% + c("NOTE", "WARNING", "OK"))) }) test_that("get_candidates vignette check respects check argument", { - ok_only <- get_candidates(release, "windows", vignettes = TRUE, check = "OK") - expect_true(all(ok_only$vignettes_check == "OK")) + ok_only <- get_candidates(release, "windows", "x86_64", vignettes = TRUE, + check = "OK", jobs = release_jobs) + expect_true(all(source_check(ok_only) == "OK")) }) -test_that("get_candidates without vignettes has no Vignettes Check column", { - without_vignettes <- get_candidates(release, "windows", vignettes = FALSE) - expect_false("Vignettes Check" %in% names(without_vignettes)) + +test_that("filter_by_arch uses universal binaries only for packages without the other arch's job", { + # "pure" has one universal binary from its arm64 job; "compiled" has jobs + # for both architectures and its arm64 job built no binary + df <- data.frame( + Package = c("pure", "compiled", "compiled"), + job_os = "mac", + job_arch = c("arm64", "arm64", "x86_64"), + binaries_arch = c(NA, NA, "x86_64")) + result <- filter_by_arch(df, "mac", "x86_64") + expect_equal(result$Package, c("pure", "compiled")) + expect_equal(result$job_arch, c("arm64", "x86_64")) + # the compiled package's failed arm64 job is still its own arm64 row + expect_equal(nrow(filter_by_arch(df, "mac", "arm64")), 2) +}) + +test_that("filter_by_arch windows arm64 uses x86_64 universal binaries only for packages without an arm64 job", { + df <- data.frame( + Package = c("pure", "compiled", "compiled"), + job_os = "win", + job_arch = c("x86_64", "x86_64", "arm64"), + binaries_arch = c(NA, NA, "aarch64")) + result <- filter_by_arch(df, "win", "arm64") + expect_equal(result$Package, c("pure", "compiled")) + expect_equal(result$job_arch, c("x86_64", "arm64")) }) diff --git a/vignettes/index.qmd b/vignettes/index.qmd deleted file mode 100644 index 3b0cb08..0000000 --- a/vignettes/index.qmd +++ /dev/null @@ -1,286 +0,0 @@ ---- -title: "Bioconductor in R Universe: Linux with Criteria on Commits and Vignettes" -format: - html: - code-fold: true - toc: false -sidebar: false -editor: visual -engine: quarto -vignette: > - %\VignetteIndexEntry{Bioconductor in R Universe: Linux with Criteria on Commits and Vignettes} - %\VignetteEngine{quarto::html} - %\VignetteEncoding{UTF-8} ---- - -```{r install_universe, echo = FALSE, message = FALSE} -library(BiocPkgTools) -library(dplyr) -library(biocUniTools) -library(httr2) -library(reactable) -library(purrr) -``` - -```{r get_buildurl, echo = FALSE} -# When a package fails early in the build, we don't get a _buildurl but we can -# construct it -get_buildurl <- function(pkg, uni) { - u <-paste0("https://", uni, ".r-universe.dev/api/packages/", pkg) - buildurl <- tryCatch({ - pkglist <- httr2::request(u) |> - httr2::req_timeout(20) |> - httr2::req_perform() |> - httr2::resp_body_json() - pkglist$`_buildurl` - - }, error = function(e) { - if (is.null(e$resp)) return(NA_character_) - body <- httr2::resp_body_string(e$resp) - buildurl <- stringr::str_extract(body, - paste0("https://github.com/r-universe/", - uni, "/actions/runs/[0-9]+")) - buildurl - }) - buildurl -} -``` - -```{r propagation-criteria, echo = FALSE} -propagate <- function(commit_match, deprecated, job_check, binary_check, - binary_status, vignettes_check) { - !is.na(commit_match) & commit_match & - !is.na(deprecated) & !deprecated & - !is.na(job_check) & (!job_check %in% c("CANCELLED", "ERROR", "FAIL")) & - !is.na(binary_check) & (!binary_check %in% c("CANCELLED", "ERROR", "FAIL")) & - !is.na(binary_status) & (binary_status == "success") & - !is.na(vignettes_check) & (!vignettes_check %in% c("CANCELLED", "ERROR", "FAIL")) -} -``` - -The following tables attempt to describe the potential propagation status of a package in R universe. Propagation is based on - -- the same commit in both build systems -- the job building in R Universe -- the binary building in R Universe -- the vignettes building in R Universe -- package is not deprecated - -Possible failure states are `CANCELLED`, `ERROR`, or `FAIL`. - -## R Universe and Bioconductor devel - -https://bioc.r-universe.dev - -```{r devel-settings, echo = FALSE, message = FALSE} -bu <- uni_for_bioc("devel") -os <- "linux" -nde <- "nebbiolo2" -raw_universe_df <- biocUniTools::get_raw_uni_df(bu$universe) -universe_df <- biocUniTools::get_uni_df(raw_universe_df) -ru_builds <- biocUniTools::get_jobs(universe_df, bu$r_version, bu$universe) -bbs_pkgs <- BiocPkgTools::biocBuildReport(version = bu$bioc_version, - pkgType = "software") -``` - -::: column-screen-inset -```{r devel-data, echo = FALSE} -bbs_builds <- bbs_pkgs |> - dplyr::filter(node == nde, stage == "checksrc") |> - dplyr::select(pkg, version, Deprecated, git_last_commit, result) |> - dplyr::rename(Package = pkg, Version = version, - `BBS Commit` = git_last_commit, Check = result) |> - dplyr::mutate(`BBS Commit` = substr(`BBS Commit`, 1, 7)) |> - dplyr::distinct() - -filtered_ru_builds <- ru_builds |> - dplyr::filter(job_os == os) |> - biocUniTools::filter_by_arch(os, arch = "x86_64") -vignettes_ru_builds <- ru_builds |> - dplyr::filter(job_type == "source") |> - dplyr::select(Package, Version, jobs_check, `_buildurl`) |> - dplyr::rename(`Vignettes Check` = jobs_check, - vignettes_buildurl = `_buildurl`) - -builds <- merge(bbs_builds, filtered_ru_builds, by = c("Package", "Version"), - all.x = TRUE) -builds <- merge(builds, vignettes_ru_builds, by = c("Package", "Version"), - all.x = TRUE) - -# Get _buildurl if not available -needs_url <- builds |> - dplyr::filter(!Deprecated & is.na(`_buildurl`)) |> - dplyr::distinct(Package, .keep_all = TRUE) - -needs_url$`_buildurl` <- purrr::map_chr(needs_url$Package, get_buildurl, bu$universe) - -builds <- builds |> - dplyr::rows_update(needs_url |> dplyr::select(Package, `_buildurl`), - by = "Package", - unmatched = "ignore") - -condensed_builds <- builds |> - dplyr::rename(`Job Check` = jobs_check, `Binary Check` = binaries_check, - `Build Url` = `_buildurl`, `RU Commit` = RemoteSha, - `Job Url` = job_url, `Binary Status` = binaries_status) |> - dplyr::mutate(`RU Commit` = substr(`RU Commit`, 1, 7)) |> - dplyr::mutate(`Commit Match` = `BBS Commit` == `RU Commit`, - Propagate = propagate(`Commit Match`, Deprecated, - `Job Check`, `Binary Check`, - `Binary Status`, `Vignettes Check`)) |> - dplyr::select(Package, Version, `BBS Commit`, Deprecated, `Commit Match`, - Check, `Job Check`, `Binary Check`, `Binary Status`, - `Job Url`, `Build Url`, `Vignettes Check`, vignettes_buildurl, - Propagate) - -condensed_builds_display <- condensed_builds |> - dplyr::mutate( - Check = ifelse(!is.na(Check), - paste0('', Check, ''), - as.character(Check)), - `Job Check` = ifelse(!is.na(`Job Url`) & !is.na(`Job Check`), - paste0('', `Job Check`, ''), - as.character(`Job Check`)), - `Binary Check` = ifelse(!is.na(`Build Url`) & !is.na(`Binary Check`), - paste0('', `Binary Check`, - ''), - as.character(`Binary Check`)), - `Vignettes Check` = ifelse(!is.na(vignettes_buildurl) & !is.na(`Vignettes Check`), - paste0('', - `Vignettes Check`, ''), - as.character(`Vignettes Check`)), - Propagate = ifelse(Propagate, - '✅', - '\U0001F6AB') - ) |> - dplyr::select(-`Job Url`, -`Build Url`, -vignettes_buildurl) - -reactable::reactable(condensed_builds_display, searchable = TRUE, #wrap = FALSE, - showPageSizeOptions = TRUE, compact = TRUE, - columns = list( - Check = reactable::colDef(html = TRUE), - `Job Check` = reactable::colDef(html = TRUE), - `Binary Check` = reactable::colDef(html = TRUE), - `Vignettes Check` = reactable::colDef(html = TRUE), - Propagate = reactable::colDef(html = TRUE) - - ), - defaultColDef = reactable::colDef( - minWidth = 150, - style = list(whiteSpace = "nowrap"), - headerStyle = list(whiteSpace = "nowrap") - ) -) -``` -::: - -## R Universe and Bioconductor release - -https://bioc-release.r-universe.dev - -```{r release-settings, echo = FALSE, message = FALSE} -bu <- uni_for_bioc("release") -nde <- "nebbiolo1" -raw_universe_df <- biocUniTools::get_raw_uni_df(bu$universe) -universe_df <- biocUniTools::get_uni_df(raw_universe_df) -ru_builds <- biocUniTools::get_jobs(universe_df, bu$r_version, bu$universe) -bbs_pkgs <- BiocPkgTools::biocBuildReport(version = bu$bioc_version, - pkgType = "software") -``` - -::: column-screen-inset -```{r release-data, echo = FALSE} -bbs_builds <- bbs_pkgs |> - dplyr::filter(node == nde, stage == "checksrc") |> - dplyr::select(pkg, version, Deprecated, git_last_commit, result) |> - dplyr::rename(Package = pkg, Version = version, - `BBS Commit` = git_last_commit, Check = result) |> - dplyr::mutate(`BBS Commit` = substr(`BBS Commit`, 1, 7)) |> - dplyr::distinct() - -filtered_ru_builds <- ru_builds |> - dplyr::filter(job_os == os) |> - biocUniTools::filter_by_arch(os, arch = "x86_64") -vignettes_ru_builds <- ru_builds |> - dplyr::filter(job_type == "source") |> - dplyr::select(Package, Version, jobs_check, `_buildurl`) |> - dplyr::rename(`Vignettes Check` = jobs_check, - vignettes_buildurl = `_buildurl`) - -builds <- merge(bbs_builds, filtered_ru_builds, by = c("Package", "Version"), - all.x = TRUE) -builds <- merge(builds, vignettes_ru_builds, by = c("Package", "Version"), - all.x = TRUE) - -# Get _buildurl if not available -needs_url <- builds |> - dplyr::filter(!Deprecated & is.na(`_buildurl`)) |> - dplyr::distinct(Package, .keep_all = TRUE) - -needs_url$`_buildurl` <- purrr::map_chr(needs_url$Package, get_buildurl, bu$universe) - -builds <- builds |> - dplyr::rows_update(needs_url |> dplyr::select(Package, `_buildurl`), - by = "Package", - unmatched = "ignore") - -condensed_builds <- builds |> - dplyr::rename(`Job Check` = jobs_check, `Binary Check` = binaries_check, - `Build Url` = `_buildurl`, `RU Commit` = RemoteSha, - `Job Url` = job_url, `Binary Status` = binaries_status) |> - dplyr::mutate(`RU Commit` = substr(`RU Commit`, 1, 7)) |> - dplyr::mutate(`Commit Match` = `BBS Commit` == `RU Commit`, - Propagate = propagate(`Commit Match`, Deprecated, - `Job Check`, `Binary Check`, - `Binary Status`, `Vignettes Check`)) |> - dplyr::select(Package, Version, `BBS Commit`, Deprecated, `Commit Match`, - Check, `Job Check`, `Binary Check`, `Binary Status`, - `Job Url`, `Build Url`, `Vignettes Check`, vignettes_buildurl, - Propagate) - -condensed_builds_display <- condensed_builds |> - dplyr::mutate( - Check = ifelse(!is.na(Check), - paste0('', Check, ''), - as.character(Check)), - `Job Check` = ifelse(!is.na(`Job Url`) & !is.na(`Job Check`), - paste0('', `Job Check`, ''), - as.character(`Job Check`)), - `Binary Check` = ifelse(!is.na(`Build Url`) & !is.na(`Binary Check`), - paste0('', `Binary Check`, - ''), - as.character(`Binary Check`)), - `Vignettes Check` = ifelse(!is.na(vignettes_buildurl) & !is.na(`Vignettes Check`), - paste0('', - `Vignettes Check`, ''), - as.character(`Vignettes Check`)), - Propagate = ifelse(Propagate, - '✅', - '\U0001F6AB') - ) |> - dplyr::select(-`Job Url`, -`Build Url`, -vignettes_buildurl) - -reactable::reactable(condensed_builds_display, searchable = TRUE, #wrap = FALSE, - showPageSizeOptions = TRUE, compact = TRUE, - columns = list( - Check = reactable::colDef(html = TRUE), - `Job Check` = reactable::colDef(html = TRUE), - `Binary Check` = reactable::colDef(html = TRUE), - `Vignettes Check` = reactable::colDef(html = TRUE), - Propagate = reactable::colDef(html = TRUE) - - ), - defaultColDef = reactable::colDef( - minWidth = 150, - style = list(whiteSpace = "nowrap"), - headerStyle = list(whiteSpace = "nowrap") - ) -) -``` -::: - -```{r sessionInfo} -sessionInfo() -```