Pre-processing Delayed Data
Dawid Kałędkowski
15.05.2022
preprocessing-delayed-data.RmdWhen creating apps which do not use DDL, once the
datasets are created there is often some pre-processing required before
initializing the teal app. Similarly, in the case of delayed data
additional code instructions to pre-process data can be added to
DDL objects which will be run after the data is loaded,
which may happen after the launching of the shiny app or when the
pull() method is called.
-
mutate_dataset: Individual datasets can be processed using themutate_datasetfunction. For reproducibility to be maintained withmutate_dataset, all pre-processing code should modify one dataset at a time.
library(scda)##
library(teal.data)## Loading required package: shiny
library(magrittr)
adsl_cf <- callable_function(function() synthetic_cdisc_data("latest")$adsl)
adsl <- cdisc_dataset_connector(
dataname = "ADSL",
pull_callable = adsl_cf,
keys = get_cdisc_keys("ADSL")
) %>%
mutate_dataset("ADSL$SEX <- as.factor(ADSL$SEX)")
adae_cf <- callable_function(function() synthetic_cdisc_data("latest")$adae)
adae <- cdisc_dataset_connector(
dataname = "ADAE",
pull_callable = adae_cf,
keys = get_cdisc_keys("ADAE")
) %>%
mutate_dataset("ADAE$X <- rep(ADSL$SEX[1])", vars = list(ADSL = adsl))
adsl$pull() %>%
get_raw_data() %>%
head(n = 3)## STUDYID USUBJID SUBJID SITEID AGE AGEU SEX
## 1 AB12345 AB12345-CHN-3-id-128 id-128 CHN-3 32 YEARS M
## 2 AB12345 AB12345-CHN-15-id-262 id-262 CHN-15 35 YEARS M
## 3 AB12345 AB12345-RUS-3-id-378 id-378 RUS-3 30 YEARS F
## RACE ETHNIC COUNTRY DTHFL INVID
## 1 ASIAN NOT HISPANIC OR LATINO CHN N INV ID CHN-3
## 2 BLACK OR AFRICAN AMERICAN NOT HISPANIC OR LATINO CHN N INV ID CHN-15
## 3 ASIAN NOT HISPANIC OR LATINO RUS N INV ID RUS-3
## INVNAM ARM ARMCD ACTARM ACTARMCD TRT01P
## 1 Dr. CHN-3 Doe A: Drug X ARM A A: Drug X ARM A A: Drug X
## 2 Dr. CHN-15 Doe C: Combination ARM C C: Combination ARM C C: Combination
## 3 Dr. RUS-3 Doe C: Combination ARM C C: Combination ARM C C: Combination
## TRT01A REGION1 STRATA1 STRATA2 BMRKR1 BMRKR2 ITTFL SAFFL BMEASIFL
## 1 A: Drug X Asia C S2 14.424934 MEDIUM Y Y Y
## 2 C: Combination Asia C S1 4.055463 LOW Y Y N
## 3 C: Combination Eurasia A S1 2.803240 HIGH Y Y Y
## BEP01FL RANDDT TRTSDTM TRTEDTM EOSSTT
## 1 Y 2019-02-22 2019-02-24 11:09:18 2021-02-23 22:47:42 COMPLETED
## 2 N 2019-02-26 2019-02-26 09:05:00 2021-02-25 20:43:24 COMPLETED
## 3 N 2019-02-24 2019-02-28 03:19:08 2021-02-27 14:57:32 COMPLETED
## EOTSTT EOSDT EOSDY DCSREAS DTHDT DTHCAUS DTHCAT LDDTHELD LDDTHGR1
## 1 COMPLETED 2021-02-23 731 <NA> <NA> <NA> <NA> NA <NA>
## 2 COMPLETED 2021-02-25 731 <NA> <NA> <NA> <NA> NA <NA>
## 3 COMPLETED 2021-02-27 731 <NA> <NA> <NA> <NA> NA <NA>
## LSTALVDT DTHADY study_duration_secs
## 1 2021-03-05 NA 63113904
## 2 2021-03-15 NA 63113904
## 3 2021-03-15 NA 63113904
adae$pull() %>%
get_raw_data() %>%
head(n = 3)## STUDYID USUBJID SUBJID SITEID AGE AGEU SEX RACE
## 1 AB12345 AB12345-BRA-1-id-134 id-134 BRA-1 47 YEARS M WHITE
## 2 AB12345 AB12345-BRA-1-id-134 id-134 BRA-1 47 YEARS M WHITE
## 3 AB12345 AB12345-BRA-1-id-134 id-134 BRA-1 47 YEARS M WHITE
## ETHNIC COUNTRY DTHFL INVID INVNAM ARM
## 1 NOT HISPANIC OR LATINO BRA N INV ID BRA-1 Dr. BRA-1 Doe A: Drug X
## 2 NOT HISPANIC OR LATINO BRA N INV ID BRA-1 Dr. BRA-1 Doe A: Drug X
## 3 NOT HISPANIC OR LATINO BRA N INV ID BRA-1 Dr. BRA-1 Doe A: Drug X
## ARMCD ACTARM ACTARMCD TRT01P TRT01A REGION1 STRATA1 STRATA2
## 1 ARM A A: Drug X ARM A A: Drug X A: Drug X South America B S2
## 2 ARM A A: Drug X ARM A A: Drug X A: Drug X South America B S2
## 3 ARM A A: Drug X ARM A A: Drug X A: Drug X South America B S2
## BMRKR1 BMRKR2 ITTFL SAFFL BMEASIFL BEP01FL RANDDT TRTSDTM
## 1 6.462991 LOW Y Y Y N 2020-11-03 2020-11-04 03:50:33
## 2 6.462991 LOW Y Y Y N 2020-11-03 2020-11-04 03:50:33
## 3 6.462991 LOW Y Y Y N 2020-11-03 2020-11-04 03:50:33
## TRTEDTM EOSSTT EOTSTT EOSDT EOSDY DCSREAS DTHDT
## 1 2022-11-04 15:28:57 COMPLETED COMPLETED 2022-11-04 731 <NA> <NA>
## 2 2022-11-04 15:28:57 COMPLETED COMPLETED 2022-11-04 731 <NA> <NA>
## 3 2022-11-04 15:28:57 COMPLETED COMPLETED 2022-11-04 731 <NA> <NA>
## DTHCAUS DTHCAT LDDTHELD LDDTHGR1 LSTALVDT DTHADY study_duration_secs ASEQ
## 1 <NA> <NA> NA <NA> 2022-11-15 NA 63113904 1
## 2 <NA> <NA> NA <NA> 2022-11-15 NA 63113904 2
## 3 <NA> <NA> NA <NA> 2022-11-15 NA 63113904 3
## AESEQ AETERM AELLT AEDECOD AEHLT AEHLGT
## 1 1 trm B.2.1.2.1 llt B.2.1.2.1 dcd B.2.1.2.1 hlt B.2.1.2 hlgt B.2.1
## 2 2 trm D.1.1.4.2 llt D.1.1.4.2 dcd D.1.1.4.2 hlt D.1.1.4 hlgt D.1.1
## 3 3 trm A.1.1.1.2 llt A.1.1.1.2 dcd A.1.1.1.2 hlt A.1.1.1 hlgt A.1.1
## AEBODSYS AESOC AESEV AESER AEACN AEREL
## 1 cl B.2 cl B MODERATE N DOSE NOT CHANGED N
## 2 cl D.1 cl D MODERATE N DOSE NOT CHANGED N
## 3 cl A.1 cl A MODERATE Y DOSE NOT CHANGED N
## AEOUT AESDTH TRTEMFL AECONTRT ASTDTM
## 1 RECOVERED/RESOLVED N Y N 2021-07-13
## 2 RECOVERING/RESOLVING N Y N 2021-09-04
## 3 RECOVERED/RESOLVED WITH SEQUELAE N Y Y 2022-03-15
## AENDTM ASTDY AENDY AETOXGR SMQ01NAM SMQ02NAM SMQ01SC SMQ02SC CQ01NAM
## 1 2022-04-05 251 517 3 <NA> <NA> <NA> <NA> <NA>
## 2 2022-05-16 304 558 3 <NA> <NA> <NA> <NA> <NA>
## 3 2022-10-29 496 724 2 <NA> <NA> <NA> <NA> <NA>
## ANL01FL AERELNST AEACNOTH X
## 1 Y OTHER PROCEDURE/SURGERY M
## 2 Y DISEASE UNDER STUDY MEDICATION M
## 3 Y DISEASE UNDER STUDY NONE M-
mutate_data: Collections of datasets should only be processed using themutate_datafunction:
cdisc_data(adsl, adae, check = TRUE) %>%
mutate_data("ADAE$x <- ADSL$SUBJID[1]")The code is processed in the order the datasets are pulled so if
there are dependencies between datasets it matters the order in which
pre-processing code is added to the CDISCTealData object
just as order matters when the arguments are inputted to the
cdisc_data function to create the
CDISCTealData object.
Finally, the code argument directly in
teal_data and cdisc_data call does not need to
be used for DDL because data loaded with DDL
are reproducible by design. Because of this, it is recommended to set
argument check = TRUE inside cdisc_data
function when creating apps with DDL.
Processing dependencies
It may be required to generate a delayed data object that is dependent on some other delayed object or some constant value.
For this, when creating your delayed data object it’s possible to
supply the additional variables that are to be accessed during the data
loading (pull) using additional arguments through ...:
get_code(adsl)## [1] "ADSL <- (function() synthetic_cdisc_data(\"latest\")$adsl)()\nADSL$SEX <- as.factor(ADSL$SEX)"
pull_fun_adae <- callable_function(
function() {
synthetic_cdisc_data("latest")$adae
}
)
adae <- dataset_connector(
dataname = "ADAE",
pull_callable = pull_fun_adae,
keys = get_cdisc_keys("ADAE")
)
get_code(adae)## [1] "ADAE <- (function() {\n synthetic_cdisc_data(\"latest\")$adae\n})()"It’s also possible to supply these additional variables after
creating your object using the mutate_dataset function.
last_run <- Sys.Date() # constant value stored as a variable in the current session
adsl_cf <- callable_function(function() synthetic_cdisc_data("latest")$adsl)
adsl <- cdisc_dataset_connector(
dataname = "ADSL",
pull_callable = adsl_cf,
keys = get_cdisc_keys("ADSL")
) %>%
mutate_dataset("ADSL$last_run <- last_run", vars = list(last_run = last_run))
cat(get_code(adsl))## ADSL <- (function() synthetic_cdisc_data("latest")$adsl)()
## last_run <- structure(19165, class = "Date")
## ADSL$last_run <- last_run
# compared to evaluating the variable at the time of loading
adsl_cf <- callable_function(function() synthetic_cdisc_data("latest")$adsl)
adsl <- cdisc_dataset_connector(
dataname = "ADSL",
pull_callable = adsl_cf,
keys = get_cdisc_keys("ADSL")
) %>%
mutate_dataset("last_run <- Sys.Date()\nADSL$last_run <- last_run")
adsl %>%
get_code() %>%
cat()## ADSL <- (function() synthetic_cdisc_data("latest")$adsl)()
## last_run <- Sys.Date()
## ADSL$last_run <- last_runThis is also required when creating the object depends on another delayed data object:
adsl <- synthetic_cdisc_data("latest")$adsl
adae_cf <- callable_function(function() synthetic_cdisc_data("latest")$adae)
adae <- cdisc_dataset_connector(
dataname = "ADAE",
pull_callable = adae_cf,
keys = get_cdisc_keys("ADAE")
) %>%
mutate_dataset("ADAE$n <- nrow(ADSL)")
cat(get_code(adae)) # the code returned by `adae` is not sufficient to reproduce `adae`## ADAE <- (function() synthetic_cdisc_data("latest")$adae)()
## ADAE$n <- nrow(ADSL)
adsl_cf <- callable_function(function() synthetic_cdisc_data("latest")$adsl)
adsl <- cdisc_dataset_connector(
dataname = "ADSL",
pull_callable = adsl_cf,
keys = get_cdisc_keys("ADSL")
)
adae_cf <- callable_function(function() synthetic_cdisc_data("latest")$adae)
adae <- cdisc_dataset_connector(
dataname = "ADAE",
pull_callable = adae_cf,
keys = get_cdisc_keys("ADAE")
) %>%
mutate_dataset("ADAE$n <- nrow(ADSL)", vars = list(ADSL = adsl))
cat(get_code(adae)) # this code can be run independently## ADAE <- (function() synthetic_cdisc_data("latest")$adae)()
## ADSL <- (function() synthetic_cdisc_data("latest")$adsl)()
## ADAE$n <- nrow(ADSL)Related to this idea, it is possible to provide the code on a
Data level. However, this will always return all the code
used to generate all the datasets in the object:
adsl_adae <- cdisc_data(
adsl,
adae
) %>% mutate_data("ADAE$avg_age <- mean(ADAE$AGE)")
# the output for all 3 are the same
adsl_adae %>%
get_code() %>%
cat()## ADSL <- (function() synthetic_cdisc_data("latest")$adsl)()
## ADAE <- (function() synthetic_cdisc_data("latest")$adae)()
## ADAE$n <- nrow(ADSL)
## ADAE$avg_age <- mean(ADAE$AGE)## ADSL <- (function() synthetic_cdisc_data("latest")$adsl)()
## ADAE <- (function() synthetic_cdisc_data("latest")$adae)()
## ADAE$n <- nrow(ADSL)
## ADAE$avg_age <- mean(ADAE$AGE)## ADSL <- (function() synthetic_cdisc_data("latest")$adsl)()
## ADAE <- (function() synthetic_cdisc_data("latest")$adae)()
## ADAE$n <- nrow(ADSL)
## ADAE$avg_age <- mean(ADAE$AGE)The better approach would be to supply the code on a
Dataset level. This ensures that the code accessed on a
dataset level only contains the snippets that pertains to itself:
adsl_adae <- cdisc_data(
adsl,
adae %>% mutate_dataset("ADAE$avg_age <- mean(ADAE$AGE)")
)
adsl_adae %>%
get_code() %>%
cat()## ADSL <- (function() synthetic_cdisc_data("latest")$adsl)()
## ADAE <- (function() synthetic_cdisc_data("latest")$adae)()
## ADAE$n <- nrow(ADSL)
## ADAE$avg_age <- mean(ADAE$AGE)## ADSL <- (function() synthetic_cdisc_data("latest")$adsl)()
## ADAE <- (function() synthetic_cdisc_data("latest")$adae)()
## ADAE$n <- nrow(ADSL)
## ADAE$avg_age <- mean(ADAE$AGE)## ADSL <- (function() synthetic_cdisc_data("latest")$adsl)()Related to this idea, the delayed data object needs to be supplied
with the code needed to reproduce the data. This can be provided at the
Dataset level or the Data level.
Below is a comparison of these two approaches:
adsl <- synthetic_cdisc_data("latest")$adsl
cdisc_dataset("ADSL", adsl) %>% get_code() # no reproducible code## [1] ""
# provide the code to reproduce the data:
cdisc_dataset("ADSL", adsl,
code = "ADSL <- synthetic_cdisc_data(\"latest\")$adsl"
) %>%
get_code()## [1] "ADSL <- synthetic_cdisc_data(\"latest\")$adsl"
# it's possible to supply the code at the `Data` level:
adae <- synthetic_cdisc_data("latest")$adae
adsl_adae <- cdisc_data(
cdisc_dataset("ADSL", adsl),
cdisc_dataset("ADAE", adae),
code = "ADSL <- synthetic_cdisc_data(\"latest\")$adsl\nADAE <- synthetic_cdisc_data(\"latest\")$adae"
)
adsl_adae %>%
get_code() %>%
cat()## ADSL <- synthetic_cdisc_data("latest")$adsl
## ADAE <- synthetic_cdisc_data("latest")$adae
# but it's not possible then to access the code at a `Dataset` level:
adsl_adae %>%
get_code("ADSL") %>%
cat()## ADSL <- synthetic_cdisc_data("latest")$adsl
## ADAE <- synthetic_cdisc_data("latest")$adae
# this can be avoided by storing the code like so:
adsl_adae <- cdisc_data(
cdisc_dataset("ADSL", adsl, code = "ADSL <- synthetic_cdisc_data(\"latest\")$adsl"),
cdisc_dataset("ADAE", adae, code = "ADAE <- synthetic_cdisc_data(\"latest\")$adsl")
)
adsl_adae %>%
get_code("ADSL") %>%
cat()## ADSL <- synthetic_cdisc_data("latest")$adsl