Skip to content

Commit eb6b27b

Browse files
richfitzrichfitz
authored
Merge pull request #42 from vimc/vimc-7724
vimc-7724: update to allow in-memory db
2 parents fe2af64 + a09429e commit eb6b27b

9 files changed

Lines changed: 118 additions & 129 deletions

‎.Rbuildignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,3 +8,4 @@
88
^TODO.md$
99
^.github$
1010
^vignettes_src$
11+
^\.github$

‎.github/workflows/R-CMD-check.yaml‎

Lines changed: 23 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,13 @@
1-
# For help debugging build failures open an issue on the RStudio community with the 'github-actions' tag.
2-
# https://community.rstudio.com/new-topic?category=Package%20development&tags=github-actions
1+
# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples
2+
# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help
33
on:
44
push:
5-
branches:
6-
- main
7-
- master
5+
branches: [main, master]
86
pull_request:
9-
branches:
10-
- main
11-
- master
127

13-
name: R-CMD-check
8+
name: R-CMD-check.yaml
9+
10+
permissions: read-all
1411

1512
jobs:
1613
R-CMD-check:
@@ -22,69 +19,33 @@ jobs:
2219
fail-fast: false
2320
matrix:
2421
config:
22+
- {os: macos-latest, r: 'release'}
2523
- {os: windows-latest, r: 'release'}
26-
- {os: macOS-latest, r: 'release'}
27-
- {os: ubuntu-20.04, r: 'release', rspm: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest"}
28-
- {os: ubuntu-20.04, r: 'devel', rspm: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest"}
24+
- {os: ubuntu-latest, r: 'devel', http-user-agent: 'release'}
25+
- {os: ubuntu-latest, r: 'release'}
26+
- {os: ubuntu-latest, r: 'oldrel-1'}
2927

3028
env:
31-
R_REMOTES_NO_ERRORS_FROM_WARNINGS: true
32-
RSPM: ${{ matrix.config.rspm }}
3329
GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
30+
R_KEEP_PKG_SOURCE: yes
3431

3532
steps:
36-
- uses: actions/checkout@v2
33+
- uses: actions/checkout@v4
34+
35+
- uses: r-lib/actions/setup-pandoc@v2
3736

38-
- uses: r-lib/actions/setup-r@v1
37+
- uses: r-lib/actions/setup-r@v2
3938
with:
4039
r-version: ${{ matrix.config.r }}
40+
http-user-agent: ${{ matrix.config.http-user-agent }}
41+
use-public-rspm: true
4142

42-
- uses: r-lib/actions/setup-pandoc@v1
43-
44-
- name: Query dependencies
45-
run: |
46-
install.packages('remotes')
47-
saveRDS(remotes::dev_package_deps(dependencies = TRUE), ".github/depends.Rds", version = 2)
48-
writeLines(sprintf("R-%i.%i", getRversion()$major, getRversion()$minor), ".github/R-version")
49-
shell: Rscript {0}
50-
51-
- name: Cache R packages
52-
if: runner.os != 'Windows'
53-
uses: actions/cache@v2
43+
- uses: r-lib/actions/setup-r-dependencies@v2
5444
with:
55-
path: ${{ env.R_LIBS_USER }}
56-
key: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-${{ hashFiles('.github/depends.Rds') }}
57-
restore-keys: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-
58-
59-
- name: Install system dependencies
60-
if: runner.os == 'Linux'
61-
run: |
62-
while read -r cmd
63-
do
64-
eval sudo $cmd
65-
done < <(Rscript -e 'writeLines(remotes::system_requirements("ubuntu", "20.04"))')
66-
67-
- name: Install dependencies
68-
run: |
69-
remotes::install_deps(dependencies = TRUE)
70-
remotes::install_cran("rcmdcheck")
71-
shell: Rscript {0}
72-
73-
- name: Start test DB
74-
if: runner.os == 'Linux'
75-
run: ./scripts/test_db_start.sh
76-
77-
- name: Check
78-
env:
79-
_R_CHECK_CRAN_INCOMING_REMOTE_: false
80-
run: |
81-
options(crayon.enabled = TRUE)
82-
rcmdcheck::rcmdcheck(args = c("--no-manual", "--as-cran"), error_on = "warning", check_dir = "check")
83-
shell: Rscript {0}
45+
extra-packages: any::rcmdcheck
46+
needs: check
8447

85-
- name: Upload check results
86-
if: failure()
87-
uses: actions/upload-artifact@main
48+
- uses: r-lib/actions/check-r-package@v2
8849
with:
89-
name: ${{ runner.os }}-r${{ matrix.config.r }}-results
90-
path: check
50+
upload-snapshots: true
51+
build_args: 'c("--no-manual","--compact-vignettes=gs+qpdf")'

‎.github/workflows/pkgdown.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ jobs:
1414
steps:
1515
- uses: actions/checkout@v2
1616

17-
- uses: r-lib/actions/setup-r@v1
17+
- uses: r-lib/actions/setup-r@v2
1818

1919
- uses: r-lib/actions/setup-pandoc@v1
2020

@@ -26,7 +26,7 @@ jobs:
2626
shell: Rscript {0}
2727

2828
- name: Cache R packages
29-
uses: actions/cache@v2
29+
uses: actions/cache@v4
3030
with:
3131
path: ${{ env.R_LIBS_USER }}
3232
key: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-${{ hashFiles('.github/depends.Rds') }}

‎.github/workflows/test-coverage.yaml‎

Lines changed: 43 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -1,62 +1,62 @@
1+
# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples
2+
# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help
13
on:
24
push:
3-
branches:
4-
- main
5-
- master
5+
branches: [main, master]
66
pull_request:
7-
branches:
8-
- main
9-
- master
107

11-
name: test-coverage
8+
name: test-coverage.yaml
9+
10+
permissions: read-all
1211

1312
jobs:
1413
test-coverage:
15-
runs-on: ubuntu-20.04
14+
runs-on: ubuntu-latest
1615
env:
1716
GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
17+
1818
steps:
19-
- uses: actions/checkout@v2
19+
- uses: actions/checkout@v4
2020

21-
- uses: r-lib/actions/setup-r@v1
21+
- uses: r-lib/actions/setup-r@v2
22+
with:
23+
use-public-rspm: true
2224

23-
- uses: r-lib/actions/setup-pandoc@v1
25+
- uses: r-lib/actions/setup-r-dependencies@v2
26+
with:
27+
extra-packages: any::covr, any::xml2
28+
needs: coverage
2429

25-
- name: Query dependencies
30+
- name: Test coverage
2631
run: |
27-
install.packages('remotes')
28-
saveRDS(remotes::dev_package_deps(dependencies = TRUE), ".github/depends.Rds", version = 2)
29-
writeLines(sprintf("R-%i.%i", getRversion()$major, getRversion()$minor), ".github/R-version")
32+
cov <- covr::package_coverage(
33+
quiet = FALSE,
34+
clean = FALSE,
35+
install_path = file.path(normalizePath(Sys.getenv("RUNNER_TEMP"), winslash = "/"), "package")
36+
)
37+
print(cov)
38+
covr::to_cobertura(cov)
3039
shell: Rscript {0}
3140

32-
- name: Cache R packages
33-
uses: actions/cache@v2
41+
- uses: codecov/codecov-action@v5
3442
with:
35-
path: ${{ env.R_LIBS_USER }}
36-
key: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-${{ hashFiles('.github/depends.Rds') }}
37-
restore-keys: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-
38-
39-
- name: Install system dependencies
40-
if: runner.os == 'Linux'
41-
run: |
42-
while read -r cmd
43-
do
44-
eval sudo $cmd
45-
done < <(Rscript -e 'writeLines(remotes::system_requirements("ubuntu", "20.04"))')
46-
sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
47-
48-
- name: Install dependencies
43+
# Fail if error if not on PR, or if on PR and token is given
44+
fail_ci_if_error: ${{ github.event_name != 'pull_request' || secrets.CODECOV_TOKEN }}
45+
files: ./cobertura.xml
46+
plugins: noop
47+
disable_search: true
48+
token: ${{ secrets.CODECOV_TOKEN }}
49+
50+
- name: Show testthat output
51+
if: always()
4952
run: |
50-
install.packages(c("remotes"))
51-
remotes::install_deps(dependencies = TRUE)
52-
remotes::install_cran("covr")
53-
shell: Rscript {0}
53+
## --------------------------------------------------------------------
54+
find '${{ runner.temp }}/package' -name 'testthat.Rout*' -exec cat '{}' \; || true
55+
shell: bash
5456

55-
- name: Start test DB
56-
if: runner.os == 'Linux'
57-
run: ./scripts/test_db_start.sh
58-
59-
- name: Test coverage
60-
run: |
61-
covr::codecov()
62-
shell: Rscript {0}
57+
- name: Upload test results
58+
if: failure()
59+
uses: actions/upload-artifact@v4
60+
with:
61+
name: coverage-test-failures
62+
path: ${{ runner.temp }}/package

‎R/commonly_used_db_data.R‎

Lines changed: 31 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ get_touchstone_id <- function(con, touchstone) {
5959
##' @param demographic_source Demographic_source.code
6060
##' @param coverage_scenario_type Coverage scenario type. This is particularly useful for a coverage touchstone. Montagu coverage.name follows <disease>:<vaccine>,<activity_type>,<gavi_support_level>:<coverage_scenario_type>
6161
##' It is NULL by default. When it is not null, only pulls coverage.name that contains specified pattern.
62+
##' @param db_memory if not null, coverage is pulled from in-memory db; this is used from 202507july onwards, as we are moving away from db imports
6263
##' @export
6364
extract_vaccination_history <- function(con, touchstone_cov = "201710gavi", touchstone_pop = NULL,
6465
year_min = 2000, year_max = 2100,
@@ -70,7 +71,8 @@ extract_vaccination_history <- function(con, touchstone_cov = "201710gavi", touc
7071
external_population_estimates = NULL,
7172
full_description = FALSE,
7273
demographic_source = NULL,
73-
coverage_scenario_type = NULL) {
74+
coverage_scenario_type = NULL,
75+
db_memory = NULL) {
7476

7577
## validate demography parameter
7678
## when touchstone_pop is null, touchstone_cov is used for touchstone_pop
@@ -84,25 +86,32 @@ extract_vaccination_history <- function(con, touchstone_cov = "201710gavi", touc
8486

8587
### This function converts input coverage data to be dis-aggregated by gender and age
8688
### i.e. input data by country, year and age
87-
89+
### determine where to pull coverage
90+
if(!is.null(db_memory)){
91+
con_cov <- db_memory
92+
} else {
93+
con_cov <- con
94+
}
8895
## 1. touchstone specification
8996
## which coverage touchstone to use - given touchstone name, use the latest version touchstone
9097
if (grepl("-", touchstone_cov)) {
91-
tmp <- DBI::dbGetQuery(con, "SELECT * FROM touchstone WHERE id = $1", touchstone_cov)
98+
tmp <- DBI::dbGetQuery(con_cov, "SELECT * FROM touchstone WHERE id = $1", touchstone_cov)
9299
if (nrow(tmp) == 1L) {
93100
message("User defined touchstone version is used.")
94101
} else {
95102
stop("User defined touchstone does not exist.")
96103
}
97104
} else {
98-
touchstone_cov <- get_touchstone(con, touchstone_cov)
105+
touchstone_cov <- get_touchstone(con_cov, touchstone_cov)
99106
}
100107

101108
## which demographic touchstone to use
102-
if (is.null(touchstone_pop)) {
109+
if(!is.null(touchstone_pop)){
110+
touchstone_pop <- get_touchstone(con, touchstone_pop)
111+
} else if(is.null(touchstone_pop) & is.null(db_memory)){
103112
touchstone_pop <- touchstone_cov
104113
} else {
105-
touchstone_pop <- get_touchstone(con, touchstone_pop)
114+
message("touchstone_pop is rulled out")
106115
}
107116

108117
message("Converting input coverage data......")
@@ -124,7 +133,8 @@ extract_vaccination_history <- function(con, touchstone_cov = "201710gavi", touc
124133
## select minimal needed coverage data from the db
125134
disease_vaccine_delivery <- read_csv(system_file("disease_vaccine_delivery.csv"))
126135

127-
cov_sets <- DBI::dbGetQuery(con, paste(sprintf("SELECT DISTINCT scenario_type, scenario_description, disease, coverage_set.id AS coverage_set, vaccine, activity_type, gavi_support_level
136+
if(all(c("scenario_description", "scenario_type", "scenario", "scenario_coverage_set") %in% DBI::dbListTables(con_cov))){
137+
cov_sets <- DBI::dbGetQuery(con_cov, paste(sprintf("SELECT DISTINCT scenario_type, scenario_description, disease, coverage_set.id AS coverage_set, vaccine, activity_type, gavi_support_level
128138
FROM scenario
129139
JOIN scenario_coverage_set
130140
ON scenario_coverage_set.scenario = scenario.id
@@ -136,14 +146,18 @@ extract_vaccination_history <- function(con, touchstone_cov = "201710gavi", touc
136146
AND scenario_type IN %s
137147
AND gavi_support_level IN %s
138148
AND vaccine NOT IN %s",
139-
sql_in(scenario_type),
140-
sql_in(gavi_support_levels),
141-
sql_in(vaccine_to_ignore)),
142-
"AND scenario_description NOT LIKE '%LiST%'"),
143-
touchstone_cov)
149+
sql_in(scenario_type),
150+
sql_in(gavi_support_levels),
151+
sql_in(vaccine_to_ignore)),
152+
"AND scenario_description NOT LIKE '%LiST%'"),
153+
touchstone_cov)
154+
} else {
155+
cov_sets <- data.frame(NULL)
156+
}
157+
144158
if (nrow(cov_sets) == 0L) {
145159
# this is not a model run touchstone, need to extract coverage set directly
146-
cov_sets2 <- DBI::dbGetQuery(con, "SELECT name, coverage_set.id AS coverage_set, vaccine, activity_type, gavi_support_level
160+
cov_sets2 <- DBI::dbGetQuery(con_cov, "SELECT name, coverage_set.id AS coverage_set, vaccine, activity_type, gavi_support_level
147161
FROM coverage_set
148162
WHERE touchstone = $1
149163
AND gavi_support_level != 'none'", touchstone_cov)
@@ -166,16 +180,16 @@ extract_vaccination_history <- function(con, touchstone_cov = "201710gavi", touc
166180
"",
167181
sprintf("AND country IN %s", sql_in(countries_to_extract, text_item = TRUE)))
168182

169-
cov <- DBI::dbGetQuery(con, sprintf("SELECT coverage_set, country, year, age_from, age_to, gender.name AS gender, gavi_support, target, coverage
183+
cov <- DBI::dbGetQuery(con_cov, sprintf("SELECT coverage_set, country, year, age_from, age_to, gender.name AS gender, gavi_support, target, coverage
170184
FROM coverage
171185
JOIN gender ON gender.id = gender
172186
WHERE coverage_set IN %s
173187
AND coverage > 0
174188
AND year IN %s
175189
%s",
176-
sql_in(unique(cov_sets$coverage_set), text_item = FALSE),
177-
sql_in(year_min:year_max, text_item = FALSE),
178-
country_))
190+
sql_in(unique(cov_sets$coverage_set), text_item = FALSE),
191+
sql_in(year_min:year_max, text_item = FALSE),
192+
country_))
179193

180194
message("Extracted raw coverage data...")
181195
## transform coverage data

‎README.Rmd‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22

33
<!-- badges: start -->
44
[![Project Status: Concept – Minimal or no implementation has been done yet, or the repository is only intended to be a limited example, demo, or proof-of-concept.](https://www.repostatus.org/badges/latest/concept.svg)](https://www.repostatus.org/#concept)
5-
[![Build Status](https://travis-ci.com/vimc/vimpact.svg?branch=master)](https://travis-ci.com/vimc/vimpact)
6-
[![codecov.io](https://codecov.io/github/vimc/vimpact/coverage.svg?branch=master)](https://codecov.io/github/vimc/vimpact?branch=master)
5+
[![R-CMD-check](https://github.com/vimc/vimpact/actions/workflows/R-CMD-check.yaml/badge.svg)](https://github.com/vimc/vimpact/actions/workflows/R-CMD-check.yaml)
6+
[![Codecov test coverage](https://codecov.io/gh/vimc/vimpact/graph/badge.svg)](https://app.codecov.io/gh/vimc/vimpact)
77
<!-- badges: end -->
88

99
Support code for impact calculations.

‎README.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22

33
<!-- badges: start -->
44
[![Project Status: Concept – Minimal or no implementation has been done yet, or the repository is only intended to be a limited example, demo, or proof-of-concept.](https://www.repostatus.org/badges/latest/concept.svg)](https://www.repostatus.org/#concept)
5-
[![Build Status](https://travis-ci.com/vimc/vimpact.svg?branch=master)](https://travis-ci.com/vimc/vimpact)
6-
[![codecov.io](https://codecov.io/github/vimc/vimpact/coverage.svg?branch=master)](https://codecov.io/github/vimc/vimpact?branch=master)
5+
[![R-CMD-check](https://github.com/vimc/vimpact/actions/workflows/R-CMD-check.yaml/badge.svg)](https://github.com/vimc/vimpact/actions/workflows/R-CMD-check.yaml)
6+
[![Codecov test coverage](https://codecov.io/gh/vimc/vimpact/graph/badge.svg)](https://app.codecov.io/gh/vimc/vimpact)
77
<!-- badges: end -->
88

99
Support code for impact calculations.

‎inst/disease_vaccine_delivery.csv‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
disease,vaccine,activity_type
22
Cholera,Cholera,campaign
3+
Cholera,OCV1,campaign
4+
Cholera,OCV2,campaign
35
DTP,DTP3,routine
46
HepB,HepB,routine
57
HepB,HepB_BD,routine
@@ -16,6 +18,8 @@ Measles,Measles,campaign
1618
Measles,Measles,routine-intensified
1719
MenA,MenA,campaign
1820
MenA,MenA,routine
21+
MenA,MenACWYX,campaign
22+
MenA,MenACWYX,routine
1923
MenA,MenA,routine-intensified
2024
PCV,PCV3,routine
2125
Rota,Rota,routine
@@ -28,3 +32,9 @@ Typhoid,Typhoid,routine
2832
YF,YF,campaign
2933
YF,YF,routine
3034
YF,YF,routine-intensified
35+
Malaria,R3,routine
36+
Malaria,R4,routine
37+
Malaria,RTS3,routine
38+
Malaria,RTS4,routine
39+
COVID,COVID,campaign
40+
COVID,COVID,routine

0 commit comments

Comments
 (0)