title: “Downloading All Data Dictionaries”
output: rmarkdown::html_vignette
vignette: >
%
%
%

Get all Data Set Slugs

library(nsrr)
df = nsrr_datasets()
head(df)
#>                                                          name    slug
#> 1 Alcohol, Sleep, and Circadian Rhythms in Young Humans Study   sandd
#> 2                    Apnea, Bariatric surgery, and CPAP study     abc
#> 3            Apnea Positive Pressure Long-term Efficacy Study  apples
#> 4   Assessing Nocturnal Sleep/Wake Effects on Risk of Suicide answers
#> 5                        Best Apnea Interventions in Research bestair
#> 6                          C57BL6/J Mouse Morphine Withdrawal    cmmw
#>                      created_at                    updated_at
#> 1 2024-06-12T13:38:54.813-04:00 2025-01-16T15:15:37.749-05:00
#> 2 2014-07-11T16:32:56.676-04:00 2022-02-11T16:10:12.956-05:00
#> 3 2023-02-16T16:30:57.702-05:00 2024-02-15T12:16:51.125-05:00
#> 4 2022-10-13T11:22:40.317-04:00 2024-03-19T11:54:36.975-04:00
#> 5 2013-10-31T11:46:58.631-04:00 2022-10-11T14:17:22.755-04:00
#> 6 2024-01-24T11:28:41.967-05:00 2024-06-03T12:57:15.404-04:00
#>                     path                        files page
#> 1   /datasets/sandd.json   /datasets/sandd/files.json    1
#> 2     /datasets/abc.json     /datasets/abc/files.json    1
#> 3  /datasets/apples.json  /datasets/apples/files.json    1
#> 4 /datasets/answers.json /datasets/answers/files.json    1
#> 5 /datasets/bestair.json /datasets/bestair/files.json    1
#> 6    /datasets/cmmw.json    /datasets/cmmw/files.json    1

Here we will loop through each data set to get a list of the files in the datasets/ path:

datasets = df$slug
L = vector(mode = "list", length = length(datasets))
names(L) = datasets
for (dataset in datasets) {
  a = nsrr_dataset_files(dataset = dataset, path = "datasets")
  L[[dataset]] = a
  print(dataset)
}
#> [1] "sandd"
#> [1] "abc"
#> [1] "apples"
#> [1] "answers"
#> [1] "bestair"
#> [1] "cmmw"
#> [1] "chat"
#> [1] "ccshs"
#> [1] "cfs"
#> [1] "coxfell2020"
#> [1] "nop-nhp"
#> [1] "fdcsr"
#> [1] "heartbeat"
#> [1] "hchs"
#> [1] "homepap"
#> [1] "haassa"
#> [1] "learn"
#> [1] "msp"
#> [1] "mnc"
#> [1] "mros"
#> [1] "mesa"
#> [1] "nchsdb"
#> [1] "nfs"
#> [1] "numom2b"
#> [1] "oya"
#> [1] "pats"
#> [1] "lhcamkiia"
#> [1] "sleepaccel"
#> [1] "shank3zz"
#> [1] "apoe"
#> [1] "shine"
#> [1] "shhs"
#> [1] "stages"
#> [1] "sof"
#> [1] "urbanpoor"
#> [1] "wumod"
#> [1] "wsc"
head(L[[1]])
#>   dataset                                          full_path    folder
#> 1   sandd                                   datasets/archive datasets/
#> 2   sandd                              datasets/CHANGELOG.md datasets/
#> 3   sandd   datasets/sandd-data-dictionary-0.1.0-domains.csv datasets/
#> 4   sandd     datasets/sandd-data-dictionary-0.1.0-forms.csv datasets/
#> 5   sandd datasets/sandd-data-dictionary-0.1.0-variables.csv datasets/
#> 6   sandd                   datasets/sandd-dataset-0.1.0.csv datasets/
#>                                   file_name is_file file_size
#> 1                                   archive   FALSE         0
#> 2                              CHANGELOG.md    TRUE       224
#> 3   sandd-data-dictionary-0.1.0-domains.csv    TRUE     14741
#> 4     sandd-data-dictionary-0.1.0-forms.csv    TRUE        33
#> 5 sandd-data-dictionary-0.1.0-variables.csv    TRUE    202017
#> 6                   sandd-dataset-0.1.0.csv    TRUE    866891
#>                  file_checksum_md5 archived
#> 1                             <NA>    FALSE
#> 2 c7275872f12f1c716c56dcaeb4276ea1    FALSE
#> 3 b77066755566bd91bf722e9ccc0aef09    FALSE
#> 4 d8f70b400480b548e21038a737d12691    FALSE
#> 5 7d8482391758f3a74f955f519b660e28    FALSE
#> 6 d85d1f2d50eb9c146bf523e189e30e34    FALSE

Let’s subset the data on only the files we want to find (data dictionaries):

files = lapply(L, function(x) {
  if (length(x) == 0) {
    return(NULL)
  }
  x[ grepl("dictionary", tolower(x$file_name)),]
})
files = do.call("rbind", files)
rownames(files) = NULL
head(files)
#>   dataset                                          full_path    folder
#> 1   sandd   datasets/sandd-data-dictionary-0.1.0-domains.csv datasets/
#> 2   sandd     datasets/sandd-data-dictionary-0.1.0-forms.csv datasets/
#> 3   sandd datasets/sandd-data-dictionary-0.1.0-variables.csv datasets/
#> 4     abc     datasets/abc-data-dictionary-0.4.0-domains.csv datasets/
#> 5     abc       datasets/abc-data-dictionary-0.4.0-forms.csv datasets/
#> 6     abc   datasets/abc-data-dictionary-0.4.0-variables.csv datasets/
#>                                   file_name is_file file_size
#> 1   sandd-data-dictionary-0.1.0-domains.csv    TRUE     14741
#> 2     sandd-data-dictionary-0.1.0-forms.csv    TRUE        33
#> 3 sandd-data-dictionary-0.1.0-variables.csv    TRUE    202017
#> 4     abc-data-dictionary-0.4.0-domains.csv    TRUE      1808
#> 5       abc-data-dictionary-0.4.0-forms.csv    TRUE        33
#> 6   abc-data-dictionary-0.4.0-variables.csv    TRUE     36715
#>                  file_checksum_md5 archived
#> 1 b77066755566bd91bf722e9ccc0aef09    FALSE
#> 2 d8f70b400480b548e21038a737d12691    FALSE
#> 3 7d8482391758f3a74f955f519b660e28    FALSE
#> 4 cc631ae308709d7bd739d1b484f426b5    FALSE
#> 5 d8f70b400480b548e21038a737d12691    FALSE
#> 6 5780016d1172f44ae650a43692442764    FALSE
vars = files[ grepl("variables", tolower(files$file_name)),]
head(vars)
#>    dataset                                            full_path    folder
#> 3    sandd   datasets/sandd-data-dictionary-0.1.0-variables.csv datasets/
#> 6      abc     datasets/abc-data-dictionary-0.4.0-variables.csv datasets/
#> 9   apples  datasets/apples-data-dictionary-0.1.0-variables.csv datasets/
#> 12 answers datasets/answers-data-dictionary-0.2.0-variables.csv datasets/
#> 15 bestair datasets/bestair-data-dictionary-0.6.1-variables.csv datasets/
#> 18    chat   datasets/chat-data-dictionary-0.14.0-variables.csv datasets/
#>                                      file_name is_file file_size
#> 3    sandd-data-dictionary-0.1.0-variables.csv    TRUE    202017
#> 6      abc-data-dictionary-0.4.0-variables.csv    TRUE     36715
#> 9   apples-data-dictionary-0.1.0-variables.csv    TRUE     78547
#> 12 answers-data-dictionary-0.2.0-variables.csv    TRUE    150330
#> 15 bestair-data-dictionary-0.6.1-variables.csv    TRUE     60616
#> 18   chat-data-dictionary-0.14.0-variables.csv    TRUE    791331
#>                   file_checksum_md5 archived
#> 3  7d8482391758f3a74f955f519b660e28    FALSE
#> 6  5780016d1172f44ae650a43692442764    FALSE
#> 9  64d9b9823eae7e34190324d0874f7113    FALSE
#> 12 f6e5cbff1163c69d298f3f725b3f09a7    FALSE
#> 15 5cc3e107feeafe1dac2b39be89d3267c    FALSE
#> 18 9cf88bab37550445fb3604b7efe71595    FALSE

Here we will download one of the data dictionaries:

i = 3
# for (i in seq(nrow(vars))) {
  idf = vars[i,]
  out = nsrr::nsrr_download_file(
    dataset = idf$dataset, 
    path = idf$full_path,
    check_md5 = FALSE
  )
  if (requireNamespace("readr", quietly = TRUE)) {
    var_df = readr::read_csv(out$outfile)
  } else {
    var_df = utils::read.csv(out$outfile, as.is = TRUE)
  }
#> Rows: 353 Columns: 11
#> ── Column specification ────────────────────────────────────────────────────────
#> Delimiter: ","
#> chr (8): folder, id, display_name, description, type, units, domain, labels
#> lgl (3): calculation, commonly_used, forms
#> 
#> ℹ Use `spec()` to retrieve the full column specification for this data.
#> ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
  print(head(var_df))
#> # A tibble: 6 × 11
#>   folder    id    display_name description type  units domain labels calculation
#>   <chr>     <chr> <chr>        <chr>       <chr> <chr> <chr>  <chr>  <lgl>      
#> 1 Administ… appl… APPLES Part… NA          iden… NA    NA     ID Ta… NA         
#> 2 Administ… days… Days Since … NA          nume… days  NA     Adher… NA         
#> 3 Administ… excl… Exclusion, … APPLES Sub… choi… NA    exclu… SISS   NA         
#> 4 Administ… excl… Exclusion, … APPLES Sub… text  NA    NA     SISS   NA         
#> 5 Administ… file… APPLES File… Used for l… iden… NA    NA     NA     NA         
#> 6 Administ… nsrr… NSRR subjec… This subje… iden… NA    NA     NA     NA         
#> # ℹ 2 more variables: commonly_used <lgl>, forms <lgl>
# }

Run the outer loop and bind the data together using a similar list of data.frames as above to get all the variables from all data sets.