vignettes/dictionary.Rmd
dictionary.Rmd| title: “Downloading All Data Dictionaries” |
| output: rmarkdown::html_vignette |
| vignette: > |
| % |
| % |
| % |
library(nsrr)
df = nsrr_datasets()
head(df)
#> name slug
#> 1 Alcohol, Sleep, and Circadian Rhythms in Young Humans Study sandd
#> 2 Apnea, Bariatric surgery, and CPAP study abc
#> 3 Apnea Positive Pressure Long-term Efficacy Study apples
#> 4 Assessing Nocturnal Sleep/Wake Effects on Risk of Suicide answers
#> 5 Best Apnea Interventions in Research bestair
#> 6 C57BL6/J Mouse Morphine Withdrawal cmmw
#> created_at updated_at
#> 1 2024-06-12T13:38:54.813-04:00 2025-01-16T15:15:37.749-05:00
#> 2 2014-07-11T16:32:56.676-04:00 2022-02-11T16:10:12.956-05:00
#> 3 2023-02-16T16:30:57.702-05:00 2024-02-15T12:16:51.125-05:00
#> 4 2022-10-13T11:22:40.317-04:00 2024-03-19T11:54:36.975-04:00
#> 5 2013-10-31T11:46:58.631-04:00 2022-10-11T14:17:22.755-04:00
#> 6 2024-01-24T11:28:41.967-05:00 2024-06-03T12:57:15.404-04:00
#> path files page
#> 1 /datasets/sandd.json /datasets/sandd/files.json 1
#> 2 /datasets/abc.json /datasets/abc/files.json 1
#> 3 /datasets/apples.json /datasets/apples/files.json 1
#> 4 /datasets/answers.json /datasets/answers/files.json 1
#> 5 /datasets/bestair.json /datasets/bestair/files.json 1
#> 6 /datasets/cmmw.json /datasets/cmmw/files.json 1Here we will loop through each data set to get a list of the files in
the datasets/ path:
datasets = df$slug
L = vector(mode = "list", length = length(datasets))
names(L) = datasets
for (dataset in datasets) {
a = nsrr_dataset_files(dataset = dataset, path = "datasets")
L[[dataset]] = a
print(dataset)
}
#> [1] "sandd"
#> [1] "abc"
#> [1] "apples"
#> [1] "answers"
#> [1] "bestair"
#> [1] "cmmw"
#> [1] "chat"
#> [1] "ccshs"
#> [1] "cfs"
#> [1] "coxfell2020"
#> [1] "nop-nhp"
#> [1] "fdcsr"
#> [1] "heartbeat"
#> [1] "hchs"
#> [1] "homepap"
#> [1] "haassa"
#> [1] "learn"
#> [1] "msp"
#> [1] "mnc"
#> [1] "mros"
#> [1] "mesa"
#> [1] "nchsdb"
#> [1] "nfs"
#> [1] "numom2b"
#> [1] "oya"
#> [1] "pats"
#> [1] "lhcamkiia"
#> [1] "sleepaccel"
#> [1] "shank3zz"
#> [1] "apoe"
#> [1] "shine"
#> [1] "shhs"
#> [1] "stages"
#> [1] "sof"
#> [1] "urbanpoor"
#> [1] "wumod"
#> [1] "wsc"
head(L[[1]])
#> dataset full_path folder
#> 1 sandd datasets/archive datasets/
#> 2 sandd datasets/CHANGELOG.md datasets/
#> 3 sandd datasets/sandd-data-dictionary-0.1.0-domains.csv datasets/
#> 4 sandd datasets/sandd-data-dictionary-0.1.0-forms.csv datasets/
#> 5 sandd datasets/sandd-data-dictionary-0.1.0-variables.csv datasets/
#> 6 sandd datasets/sandd-dataset-0.1.0.csv datasets/
#> file_name is_file file_size
#> 1 archive FALSE 0
#> 2 CHANGELOG.md TRUE 224
#> 3 sandd-data-dictionary-0.1.0-domains.csv TRUE 14741
#> 4 sandd-data-dictionary-0.1.0-forms.csv TRUE 33
#> 5 sandd-data-dictionary-0.1.0-variables.csv TRUE 202017
#> 6 sandd-dataset-0.1.0.csv TRUE 866891
#> file_checksum_md5 archived
#> 1 <NA> FALSE
#> 2 c7275872f12f1c716c56dcaeb4276ea1 FALSE
#> 3 b77066755566bd91bf722e9ccc0aef09 FALSE
#> 4 d8f70b400480b548e21038a737d12691 FALSE
#> 5 7d8482391758f3a74f955f519b660e28 FALSE
#> 6 d85d1f2d50eb9c146bf523e189e30e34 FALSELet’s subset the data on only the files we want to find (data dictionaries):
files = lapply(L, function(x) {
if (length(x) == 0) {
return(NULL)
}
x[ grepl("dictionary", tolower(x$file_name)),]
})
files = do.call("rbind", files)
rownames(files) = NULL
head(files)
#> dataset full_path folder
#> 1 sandd datasets/sandd-data-dictionary-0.1.0-domains.csv datasets/
#> 2 sandd datasets/sandd-data-dictionary-0.1.0-forms.csv datasets/
#> 3 sandd datasets/sandd-data-dictionary-0.1.0-variables.csv datasets/
#> 4 abc datasets/abc-data-dictionary-0.4.0-domains.csv datasets/
#> 5 abc datasets/abc-data-dictionary-0.4.0-forms.csv datasets/
#> 6 abc datasets/abc-data-dictionary-0.4.0-variables.csv datasets/
#> file_name is_file file_size
#> 1 sandd-data-dictionary-0.1.0-domains.csv TRUE 14741
#> 2 sandd-data-dictionary-0.1.0-forms.csv TRUE 33
#> 3 sandd-data-dictionary-0.1.0-variables.csv TRUE 202017
#> 4 abc-data-dictionary-0.4.0-domains.csv TRUE 1808
#> 5 abc-data-dictionary-0.4.0-forms.csv TRUE 33
#> 6 abc-data-dictionary-0.4.0-variables.csv TRUE 36715
#> file_checksum_md5 archived
#> 1 b77066755566bd91bf722e9ccc0aef09 FALSE
#> 2 d8f70b400480b548e21038a737d12691 FALSE
#> 3 7d8482391758f3a74f955f519b660e28 FALSE
#> 4 cc631ae308709d7bd739d1b484f426b5 FALSE
#> 5 d8f70b400480b548e21038a737d12691 FALSE
#> 6 5780016d1172f44ae650a43692442764 FALSE
vars = files[ grepl("variables", tolower(files$file_name)),]
head(vars)
#> dataset full_path folder
#> 3 sandd datasets/sandd-data-dictionary-0.1.0-variables.csv datasets/
#> 6 abc datasets/abc-data-dictionary-0.4.0-variables.csv datasets/
#> 9 apples datasets/apples-data-dictionary-0.1.0-variables.csv datasets/
#> 12 answers datasets/answers-data-dictionary-0.2.0-variables.csv datasets/
#> 15 bestair datasets/bestair-data-dictionary-0.6.1-variables.csv datasets/
#> 18 chat datasets/chat-data-dictionary-0.14.0-variables.csv datasets/
#> file_name is_file file_size
#> 3 sandd-data-dictionary-0.1.0-variables.csv TRUE 202017
#> 6 abc-data-dictionary-0.4.0-variables.csv TRUE 36715
#> 9 apples-data-dictionary-0.1.0-variables.csv TRUE 78547
#> 12 answers-data-dictionary-0.2.0-variables.csv TRUE 150330
#> 15 bestair-data-dictionary-0.6.1-variables.csv TRUE 60616
#> 18 chat-data-dictionary-0.14.0-variables.csv TRUE 791331
#> file_checksum_md5 archived
#> 3 7d8482391758f3a74f955f519b660e28 FALSE
#> 6 5780016d1172f44ae650a43692442764 FALSE
#> 9 64d9b9823eae7e34190324d0874f7113 FALSE
#> 12 f6e5cbff1163c69d298f3f725b3f09a7 FALSE
#> 15 5cc3e107feeafe1dac2b39be89d3267c FALSE
#> 18 9cf88bab37550445fb3604b7efe71595 FALSEHere we will download one of the data dictionaries:
i = 3
# for (i in seq(nrow(vars))) {
idf = vars[i,]
out = nsrr::nsrr_download_file(
dataset = idf$dataset,
path = idf$full_path,
check_md5 = FALSE
)
if (requireNamespace("readr", quietly = TRUE)) {
var_df = readr::read_csv(out$outfile)
} else {
var_df = utils::read.csv(out$outfile, as.is = TRUE)
}
#> Rows: 353 Columns: 11
#> ── Column specification ────────────────────────────────────────────────────────
#> Delimiter: ","
#> chr (8): folder, id, display_name, description, type, units, domain, labels
#> lgl (3): calculation, commonly_used, forms
#>
#> ℹ Use `spec()` to retrieve the full column specification for this data.
#> ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
print(head(var_df))
#> # A tibble: 6 × 11
#> folder id display_name description type units domain labels calculation
#> <chr> <chr> <chr> <chr> <chr> <chr> <chr> <chr> <lgl>
#> 1 Administ… appl… APPLES Part… NA iden… NA NA ID Ta… NA
#> 2 Administ… days… Days Since … NA nume… days NA Adher… NA
#> 3 Administ… excl… Exclusion, … APPLES Sub… choi… NA exclu… SISS NA
#> 4 Administ… excl… Exclusion, … APPLES Sub… text NA NA SISS NA
#> 5 Administ… file… APPLES File… Used for l… iden… NA NA NA NA
#> 6 Administ… nsrr… NSRR subjec… This subje… iden… NA NA NA NA
#> # ℹ 2 more variables: commonly_used <lgl>, forms <lgl>
# }Run the outer loop and bind the data together using a similar list of
data.frames as above to get all the variables from all data
sets.