From bec13e74ad32988b33bd15e31024875a4d29afa8 Mon Sep 17 00:00:00 2001 From: Philipp Bach Date: Mon, 27 Mar 2023 15:52:13 +0200 Subject: [PATCH 01/12] change directory name to "Code" instead of "R" --- .github/workflows/runcodechunks_vignette.yml | 2 +- .github/workflows/runreplication.yml | 12 ++++++------ .github/workflows/runreplication_sim_inf.yml | 2 +- {R => Code}/doubleml_codechunks.R | 0 {R => Code}/examples_failure_n_500_p_20.R | 0 {R => Code}/merge_plots.R | 0 {R => Code}/sim_iivm.R | 0 {R => Code}/sim_irm.R | 0 {R => Code}/sim_plivX.R | 0 {R => Code}/sim_plr.R | 0 {R => Code}/sim_siminf.R | 0 11 files changed, 8 insertions(+), 8 deletions(-) rename {R => Code}/doubleml_codechunks.R (100%) rename {R => Code}/examples_failure_n_500_p_20.R (100%) rename {R => Code}/merge_plots.R (100%) rename {R => Code}/sim_iivm.R (100%) rename {R => Code}/sim_irm.R (100%) rename {R => Code}/sim_plivX.R (100%) rename {R => Code}/sim_plr.R (100%) rename {R => Code}/sim_siminf.R (100%) diff --git a/.github/workflows/runcodechunks_vignette.yml b/.github/workflows/runcodechunks_vignette.yml index d8ec16d..19d9442 100644 --- a/.github/workflows/runcodechunks_vignette.yml +++ b/.github/workflows/runcodechunks_vignette.yml @@ -58,7 +58,7 @@ jobs: - name: Run codechunks in vignette run: | - source("R/doubleml_codechunks.R") + source("Code/doubleml_codechunks.R") shell: Rscript {0} - name: Session info diff --git a/.github/workflows/runreplication.yml b/.github/workflows/runreplication.yml index 63722f9..d98122a 100644 --- a/.github/workflows/runreplication.yml +++ b/.github/workflows/runreplication.yml @@ -58,27 +58,27 @@ jobs: - name: Introduction and Cross Fit run: | - source("R/examples_failure_n_500_p_20.R") + source("Code/examples_failure_n_500_p_20.R") shell: Rscript {0} - name: Run PLR run: | - source("R/sim_plr.R") + source("Code/sim_plr.R") shell: Rscript {0} - name: Run PLIV run: | - source("R/sim_plivX.R") + source("Code/sim_plivX.R") shell: Rscript {0} - name: Run IRM run: | - source("R/sim_irm.R") + source("Code/sim_irm.R") shell: Rscript {0} - name: Run IIVM run: | - source("R/sim_iivm.R") + source("Code/sim_iivm.R") shell: Rscript {0} - uses: actions/upload-artifact@v3 @@ -93,7 +93,7 @@ jobs: - name: Merge results run: | - source("R/merge_plots.R") + source("Code/merge_plots.R") shell: Rscript {0} - uses: actions/upload-artifact@v3 diff --git a/.github/workflows/runreplication_sim_inf.yml b/.github/workflows/runreplication_sim_inf.yml index 3f986fe..b4370c4 100644 --- a/.github/workflows/runreplication_sim_inf.yml +++ b/.github/workflows/runreplication_sim_inf.yml @@ -58,7 +58,7 @@ jobs: - name: Run SimInf run: | - source("R/sim_siminf.R") + source("Code/sim_siminf.R") shell: Rscript {0} - uses: actions/upload-artifact@v3 diff --git a/R/doubleml_codechunks.R b/Code/doubleml_codechunks.R similarity index 100% rename from R/doubleml_codechunks.R rename to Code/doubleml_codechunks.R diff --git a/R/examples_failure_n_500_p_20.R b/Code/examples_failure_n_500_p_20.R similarity index 100% rename from R/examples_failure_n_500_p_20.R rename to Code/examples_failure_n_500_p_20.R diff --git a/R/merge_plots.R b/Code/merge_plots.R similarity index 100% rename from R/merge_plots.R rename to Code/merge_plots.R diff --git a/R/sim_iivm.R b/Code/sim_iivm.R similarity index 100% rename from R/sim_iivm.R rename to Code/sim_iivm.R diff --git a/R/sim_irm.R b/Code/sim_irm.R similarity index 100% rename from R/sim_irm.R rename to Code/sim_irm.R diff --git a/R/sim_plivX.R b/Code/sim_plivX.R similarity index 100% rename from R/sim_plivX.R rename to Code/sim_plivX.R diff --git a/R/sim_plr.R b/Code/sim_plr.R similarity index 100% rename from R/sim_plr.R rename to Code/sim_plr.R diff --git a/R/sim_siminf.R b/Code/sim_siminf.R similarity index 100% rename from R/sim_siminf.R rename to Code/sim_siminf.R From a7168e7587d6d6ede9cb9f895a5f883ddeddd071 Mon Sep 17 00:00:00 2001 From: Philipp Bach Date: Mon, 27 Mar 2023 16:00:57 +0200 Subject: [PATCH 02/12] save figures to "Figures/" directory --- .github/workflows/runreplication.yml | 12 +++++++++--- Code/examples_failure_n_500_p_20.R | 20 ++++++++++---------- Code/merge_plots.R | 4 ++-- Code/sim_iivm.R | 3 ++- Code/sim_irm.R | 3 ++- Code/sim_plivX.R | 3 ++- Code/sim_plr.R | 3 ++- 7 files changed, 29 insertions(+), 19 deletions(-) diff --git a/.github/workflows/runreplication.yml b/.github/workflows/runreplication.yml index d98122a..13eeb90 100644 --- a/.github/workflows/runreplication.yml +++ b/.github/workflows/runreplication.yml @@ -84,12 +84,16 @@ jobs: - uses: actions/upload-artifact@v3 with: name: sim-results - path: ./simresults/ + path: | + ./simresults/ + ./Figures/ - uses: actions/download-artifact@v3 with: name: sim-results - path: ./simresults/ + path: | + ./simresults/ + ./Figures/ - name: Merge results run: | @@ -99,7 +103,9 @@ jobs: - uses: actions/upload-artifact@v3 with: name: sim-results - path: ./simresults/ + path: | + ./simresults/ + ./Figures/ - name: Session info run: | diff --git a/Code/examples_failure_n_500_p_20.R b/Code/examples_failure_n_500_p_20.R index 10ea0a9..9132f0b 100644 --- a/Code/examples_failure_n_500_p_20.R +++ b/Code/examples_failure_n_500_p_20.R @@ -11,7 +11,7 @@ lgr::get_logger("mlr3")$set_threshold("warn") # Create a new directory plus subdirectories dir.create("simresults") dir.create("simresults/examples_fail") - +dir.create("Figures") set.seed(1234) n_rep = 1000 @@ -72,7 +72,7 @@ g_nonorth_resc = ggplot(df_nonorth_resc, aes(x = theta_nonorth)) + g_nonorth_resc -ggsave(filename = paste0("simresults/examples_fail/nonorth_resc_n", n_obs, "_p", n_vars, ".pdf"), +ggsave(filename = paste0("Figures/nonorth_resc_n", n_obs, "_p", n_vars, ".pdf"), plot = g_nonorth_resc) ################ @@ -106,7 +106,7 @@ g_orth_nosplit_resc = ggplot(df_orth_nosplit_resc, aes(x = theta_orth_nosplit)) g_orth_nosplit_resc -ggsave(filename = paste0("simresults/examples_fail/orth_nosplit_resc_n", n_obs, "_p", n_vars, ".pdf"), +ggsave(filename = paste0("Figures/orth_nosplit_resc_n", n_obs, "_p", n_vars, ".pdf"), plot = g_orth_nosplit_resc) ########### @@ -140,7 +140,7 @@ g_dml = ggplot(df_dml, aes(x = theta_dml)) + g_dml -ggsave(filename = paste0("simresults/examples_fail/double_ml_n", n_obs, "_p", n_vars, ".pdf"), +ggsave(filename = paste0("Figures/double_ml_n", n_obs, "_p", n_vars, ".pdf"), plot = g_dml) @@ -155,7 +155,7 @@ g_dml_resc = ggplot(df_dml_resc, aes(x = theta_dml)) + g_dml_resc -ggsave(filename = paste0("simresults/examples_fail/double_ml_resc_n", n_obs, "_p", n_vars, ".pdf"), +ggsave(filename = paste0("Figures/double_ml_resc_n", n_obs, "_p", n_vars, ".pdf"), plot = g_dml_resc) @@ -170,20 +170,20 @@ g_dml_orth_resc = ggplot(df_dml_resc, aes(x = theta_dml)) + g_dml_orth_resc -ggsave(filename = paste0("simresults/examples_fail/double_ml_orth_resc_n", n_obs, "_p", n_vars, ".pdf"), +ggsave(filename = paste0("Figures/double_ml_orth_resc_n", n_obs, "_p", n_vars, ".pdf"), plot = g_dml_resc) # Combined plots g_comb1_resc = plot_grid(g_nonorth_resc, g_dml_orth_resc) -ggsave(filename = paste0("simresults/examples_fail/nonorth_doubleml_n", n_obs, "_p", n_vars, "_resc.pdf"), +ggsave(filename = paste0("Figures/nonorth_doubleml_n", n_obs, "_p", n_vars, "_resc.pdf"), plot = g_comb1_resc, width=8, height=3, dpi=150) g_comb2_resc = plot_grid(g_orth_nosplit_resc, g_dml_resc) -ggsave(filename = paste0("simresults/examples_fail/nosplit_doubleml_n", n_obs, "_p", n_vars, "_resc.pdf"), +ggsave(filename = paste0("Figures/nosplit_doubleml_n", n_obs, "_p", n_vars, "_resc.pdf"), plot = g_comb2_resc, width=8, height=3, dpi=150) @@ -215,7 +215,7 @@ g_dml_split_no_cross = ggplot(df_dml_split_no_cross, aes(x = theta_dml_split_no_ g_dml_split_no_cross -ggsave(filename = paste0("simresults/examples_fail/double_ml_resc_no_cross_n", n_obs, "_p", n_vars, ".pdf"), +ggsave(filename = paste0("Figures/double_ml_resc_no_cross_n", n_obs, "_p", n_vars, ".pdf"), plot = g_dml_split_no_cross) @@ -230,5 +230,5 @@ g_dml g_comb3 = plot_grid(g_dml_split_no_cross, g_dml) -ggsave(filename = paste0("simresults/examples_fail/nocrossfit_doubleml_n", n_obs, "_p", n_vars, ".pdf"), +ggsave(filename = paste0("Figures/nocrossfit_doubleml_n", n_obs, "_p", n_vars, ".pdf"), plot = g_comb3, width=8, height=3, dpi=150) diff --git a/Code/merge_plots.R b/Code/merge_plots.R index 8d7fffe..5b3c7f1 100644 --- a/Code/merge_plots.R +++ b/Code/merge_plots.R @@ -10,7 +10,7 @@ library(data.table) library(DoubleML) # Create new subdirectory -dir.create("simresults/merge") +dir.create("Figures") #load raw results and generate plots # PLR @@ -155,5 +155,5 @@ library(cowplot) g_grid = plot_grid(g_plr, g_pliv, g_irm, g_iivm, nrow = 2) g_grid -ggsave(filename = "simresults/merge/simulations_doubleml_key_models.pdf", +ggsave(filename = "Figures/simulations_doubleml_key_models.pdf", plot = g_grid, width=5, height=5, dpi=150) diff --git a/Code/sim_iivm.R b/Code/sim_iivm.R index 60dfcae..9b5255b 100644 --- a/Code/sim_iivm.R +++ b/Code/sim_iivm.R @@ -12,6 +12,7 @@ library(doRNG) # Create a new directory plus subdirectories dir.create("simresults") dir.create("simresults/iivm") +dir.create("Figures") date = Sys.Date() learner_name = "regr.cv_glmnet" @@ -92,7 +93,7 @@ g_iivm = ggplot(df_iivm, aes(x = coef_resc)) + g_iivm -ggsave(filename = paste0("simresults/iivm/densplot_IIVM_", n_obs, "_", dim_x, "_", +ggsave(filename = paste0("Figures/densplot_IIVM_", n_obs, "_", dim_x, "_", "_", dml_procedure, "_", n_folds, "_", n_rep_folds, "_", learner_name, "_", R, "_", alpha, ".pdf"), diff --git a/Code/sim_irm.R b/Code/sim_irm.R index 5652a98..3bf08e1 100644 --- a/Code/sim_irm.R +++ b/Code/sim_irm.R @@ -12,6 +12,7 @@ library(doRNG) # Create a new directory plus subdirectories dir.create("simresults") dir.create("simresults/irm") +dir.create("Figures") date = Sys.Date() learner_name = "regr.cv_glmnet" @@ -92,7 +93,7 @@ g_est = ggplot(df, aes(x = coef_resc)) + g_est -ggsave(filename = paste0("simresults/irm/densplot_IRM_", n_obs, "_", dim_x, "_", +ggsave(filename = paste0("Figures/densplot_IRM_", n_obs, "_", dim_x, "_", "_", dml_procedure, "_", n_folds, "_", n_rep_folds, "_", learner_name, "_", R, "_", alpha, ".pdf"), diff --git a/Code/sim_plivX.R b/Code/sim_plivX.R index 1835240..ae8632c 100644 --- a/Code/sim_plivX.R +++ b/Code/sim_plivX.R @@ -12,6 +12,7 @@ library(doRNG) # Create a new directory plus subdirectories dir.create("simresults") dir.create("simresults/pliv") +dir.create("Figures") date = Sys.Date() learner_name = "regr.cv_glmnet" @@ -89,7 +90,7 @@ g_est = ggplot(df, aes(x = coef_resc)) + g_est -ggsave(filename = paste0("simresults/pliv/densplot_PLIV_", n_obs, "_", dim_x, "_", +ggsave(filename = paste0("Figures/densplot_PLIV_", n_obs, "_", dim_x, "_", "_", dml_procedure, "_", n_folds, "_", n_rep_folds, "_", learner_name, "_", R, "_", alpha, ".pdf"), diff --git a/Code/sim_plr.R b/Code/sim_plr.R index 2d99d61..c8f3922 100644 --- a/Code/sim_plr.R +++ b/Code/sim_plr.R @@ -12,6 +12,7 @@ library(doRNG) # Create a new directory plus subdirectories dir.create("simresults") dir.create("simresults/plr") +dir.create("Figures") date = Sys.Date() learner_name = "regr.ranger" @@ -95,7 +96,7 @@ g_est = ggplot(df, aes(x = coef_resc)) + g_est -ggsave(filename = paste0("simresults/plr/densplot_PLR_", n_obs, "_", dim_x, "_", +ggsave(filename = paste0("Figures/densplot_PLR_", n_obs, "_", dim_x, "_", "_", dml_procedure, "_", n_folds, "_", n_rep_folds, "_", learner_name, "_", R, "_", alpha, ".pdf"), From a6928e158d34e11cbe7b3fd88c4a1f3f4032ae51 Mon Sep 17 00:00:00 2001 From: Philipp Bach Date: Mon, 27 Mar 2023 16:26:22 +0200 Subject: [PATCH 03/12] upd readme; fix incorrect citation, add S. Klaassen as co-author --- README.Rmd | 11 ++++++----- README.md | 40 +++++++++++++++++++++------------------- 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/README.Rmd b/README.Rmd index c116d97..5212b9f 100644 --- a/README.Rmd +++ b/README.Rmd @@ -1,11 +1,11 @@ --- -title: "Replication of Simulations in Bach et al. (2020) - DoubleML - An Object-Oriented Implementation of Double Machine Learning in R" +title: "Replication of Simulations in Bach et al. (forthcoming) - DoubleML - An Object-Oriented Implementation of Double Machine Learning in R" author: "Philipp Bach" date: "`r format(Sys.Date(), format='%b %d, %Y')`" output: github_document --- -In this repository, we list and automatically run the simulation examples presented in [Bach et al. (2020)](https://arxiv.org/abs/2103.09603) +In this repository, we list and automatically run the simulation examples presented in [Bach et al. (forthcoming)](https://arxiv.org/abs/2103.09603) ## Files for Replication of Figures and Results in DoubleML Package Vignette @@ -23,6 +23,7 @@ The examples and results from the paper *DoubleML - An Object-Oriented Implement + Simultaneous inference: `sim_siminf.R` 3. Code chunks: The reproducible code contained in the code chunks is available via + `doubleml_codechunks.R` + + *Note*: The code in `doubleml_codechunks.R` has been automatically produced from the manuscript (`.Rmd`) using All data sets and DGPs used in the paper can be replicated via corresponding functions as provided in the `DoubleML` package. The required function calls are contained in the code chunks of the paper. The API documentation is available via https://docs.doubleml.org/r/stable/reference/index.html. @@ -34,12 +35,12 @@ In case you have any questions, do not hesitate to contact philipp.bach@uni-hamb If you use the DoubleML package a citation is highly appreciated: -Bach, P., Chernozhukov, V., Kurz, M. S., and Spindler, M. (2021), DoubleML - An Object-Oriented Implementation of Double Machine Learning in Python, arXiv:2104.03220. +Bach, P., Chernozhukov, V., Kurz, M. S., Spindler, M., and Klaassen, S. (2021), DoubleML - An Object-Oriented Implementation of Double Machine Learning in R, arXiv:2103.09603. ``` @misc{DoubleML2021R, title={{DoubleML} -- {A}n Object-Oriented Implementation of Double Machine Learning in {R}}, - author={P. Bach and V. Chernozhukov and M. S. Kurz and M. Spindler}, + author={P. Bach and V. Chernozhukov and M. S. Kurz and M. Spindler and S. Klaassen}, year={2021}, eprint={2103.09603}, archivePrefix={arXiv}, @@ -61,4 +62,4 @@ We would like to thank Simon Couch for providing [a detailed blogpost on how to ## References -Bach, P., Chernozhukov, V., Kurz, M. S., and Spindler, M. (2021), DoubleML - An Object-Oriented Implementation of Double Machine Learning in Python, arXiv:2104.03220. +Bach, P., Chernozhukov, V., Kurz, M. S., Spindler, M. and Klaassen, S. (2021), DoubleML - An Object-Oriented Implementation of Double Machine Learning in R, arXiv:2104.03220. diff --git a/README.md b/README.md index 006f748..fe18b85 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ -Replication of Simulations in Bach et al. (2020) - DoubleML - An +Replication of Simulations in Bach et al. (forthcoming) - DoubleML - An Object-Oriented Implementation of Double Machine Learning in R ================ Philipp Bach -Aug 02, 2021 +Mrz 27, 2023 In this repository, we list and automatically run the simulation examples presented in [Bach et -al. (2020)](https://arxiv.org/abs/2103.09603) +al. (forthcoming)](https://arxiv.org/abs/2103.09603) ## Files for Replication of Figures and Results in DoubleML Package Vignette @@ -16,19 +16,21 @@ the R files listed in the following: 1. Section 4: *Basic idea and key ingredients of double machine learning* - Code for replication of simulation examples - - `examples_failure_n_500_p_20.R` + - `examples_failure_n_500_p_20.R` 2. Section 7.8: *A short simulation study* - Code for replication of simulation results - - Cross-fitting: `examples_failure_n_500_p_20.R` - - PLR: `sim_plr.R` - - PLIV: `sim_plivX.R` - - IRM: `sim_irm.R` - - IIVM: `sim_IIVM.R` - - Merging plots: `merge_plots.R` - - Simultaneous inference: `sim_siminf.R` + - Cross-fitting: `examples_failure_n_500_p_20.R` + - PLR: `sim_plr.R` + - PLIV: `sim_plivX.R` + - IRM: `sim_irm.R` + - IIVM: `sim_IIVM.R` + - Merging plots: `merge_plots.R` + - Simultaneous inference: `sim_siminf.R` 3. Code chunks: The reproducible code contained in the code chunks is available via - - `doubleml_codechunks.R` + - `doubleml_codechunks.R` + - *Note*: The code in `doubleml_codechunks.R` has been automatically + produced from the manuscript (`.Rmd`) using All data sets and DGPs used in the paper can be replicated via corresponding functions as provided in the `DoubleML` package. The @@ -48,13 +50,13 @@ In case you have any questions, do not hesitate to contact If you use the DoubleML package a citation is highly appreciated: -Bach, P., Chernozhukov, V., Kurz, M. S., and Spindler, M. (2021), -DoubleML - An Object-Oriented Implementation of Double Machine Learning -in Python, arXiv:2104.03220. +Bach, P., Chernozhukov, V., Kurz, M. S., Spindler, M., and Klaassen, S. +(2021), DoubleML - An Object-Oriented Implementation of Double Machine +Learning in R, arXiv:2103.09603. @misc{DoubleML2021R, title={{DoubleML} -- {A}n Object-Oriented Implementation of Double Machine Learning in {R}}, - author={P. Bach and V. Chernozhukov and M. S. Kurz and M. Spindler}, + author={P. Bach and V. Chernozhukov and M. S. Kurz and M. Spindler and S. Klaassen}, year={2021}, eprint={2103.09603}, archivePrefix={arXiv}, @@ -77,6 +79,6 @@ replication of the simulation results. ## References -Bach, P., Chernozhukov, V., Kurz, M. S., and Spindler, M. (2021), -DoubleML - An Object-Oriented Implementation of Double Machine Learning -in Python, arXiv:2104.03220. +Bach, P., Chernozhukov, V., Kurz, M. S., Spindler, M. and Klaassen, S. +(2021), DoubleML - An Object-Oriented Implementation of Double Machine +Learning in R, arXiv:2104.03220. From 217eae51e542161bc511090e2dedbbf3c14eb50a Mon Sep 17 00:00:00 2001 From: Philipp Bach Date: Mon, 27 Mar 2023 16:52:35 +0200 Subject: [PATCH 04/12] include note for knitr::purl() and reference to Xi et al. --- README.Rmd | 5 ++++- README.md | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/README.Rmd b/README.Rmd index 5212b9f..d8bf84a 100644 --- a/README.Rmd +++ b/README.Rmd @@ -23,7 +23,7 @@ The examples and results from the paper *DoubleML - An Object-Oriented Implement + Simultaneous inference: `sim_siminf.R` 3. Code chunks: The reproducible code contained in the code chunks is available via + `doubleml_codechunks.R` - + *Note*: The code in `doubleml_codechunks.R` has been automatically produced from the manuscript (`.Rmd`) using + + *Note*: The code in `doubleml_codechunks.R` has been automatically produced from the manuscript (`.Rmd`) using `knitr::purl()`, see also Chapter 3.4 of *Xi et al. (2020)* All data sets and DGPs used in the paper can be replicated via corresponding functions as provided in the `DoubleML` package. The required function calls are contained in the code chunks of the paper. The API documentation is available via https://docs.doubleml.org/r/stable/reference/index.html. @@ -63,3 +63,6 @@ We would like to thank Simon Couch for providing [a detailed blogpost on how to ## References Bach, P., Chernozhukov, V., Kurz, M. S., Spindler, M. and Klaassen, S. (2021), DoubleML - An Object-Oriented Implementation of Double Machine Learning in R, arXiv:2104.03220. + +Xie, Y., Dervieux, C., & Riederer, E. (2020). R markdown cookbook. CRC Press. + diff --git a/README.md b/README.md index fe18b85..6f167d4 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,8 @@ the R files listed in the following: available via - `doubleml_codechunks.R` - *Note*: The code in `doubleml_codechunks.R` has been automatically - produced from the manuscript (`.Rmd`) using + produced from the manuscript (`.Rmd`) using `knitr::purl()`, see + also Chapter 3.4 of *Xi et al. (2020)* All data sets and DGPs used in the paper can be replicated via corresponding functions as provided in the `DoubleML` package. The @@ -82,3 +83,6 @@ replication of the simulation results. Bach, P., Chernozhukov, V., Kurz, M. S., Spindler, M. and Klaassen, S. (2021), DoubleML - An Object-Oriented Implementation of Double Machine Learning in R, arXiv:2104.03220. + +Xie, Y., Dervieux, C., & Riederer, E. (2020). R markdown cookbook. CRC +Press. From 3010f35ea78cdd563e14923c3aee96ca4c2ebdd7 Mon Sep 17 00:00:00 2001 From: Philipp Bach Date: Tue, 28 Mar 2023 09:33:18 +0200 Subject: [PATCH 05/12] separate job for failure examples --- .github/workflows/runreplication_failure.yml | 76 ++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 .github/workflows/runreplication_failure.yml diff --git a/.github/workflows/runreplication_failure.yml b/.github/workflows/runreplication_failure.yml new file mode 100644 index 0000000..5a95e05 --- /dev/null +++ b/.github/workflows/runreplication_failure.yml @@ -0,0 +1,76 @@ +# Workflow based on https://blog--simonpcouch.netlify.app/blog/r-github-actions-commit/ +name: Replication + +on: + schedule: + - cron: "0 9 1 * *" + workflow_dispatch: + +jobs: + replicate-simulation: + runs-on: ${{ matrix.config.os }} + + name: ${{ matrix.config.os }} (R ${{ matrix.config.r }}; mlr3 ${{ matrix.config.mlr3 }}; DoubleML ${{ matrix.config.DoubleML }}) + + strategy: + fail-fast: false + matrix: + config: + - {os: macos-11, r: 'release', mlr3: 'release', DoubleML: 'release', rspm: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest"} + + env: + R_REMOTES_NO_ERRORS_FROM_WARNINGS: true + RSPM: ${{ matrix.config.rspm }} + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 2 + + - uses: r-lib/actions/setup-r@v2 + with: + r-version: ${{ matrix.config.r }} + + - uses: r-lib/actions/setup-pandoc@v2 + + - name: Query dependencies + run: | + install.packages('remotes') + install.packages('sessioninfo') + saveRDS(remotes::dev_package_deps(dependencies = TRUE), ".github/depends.Rds", version = 2) + writeLines(sprintf("R-%i.%i", getRversion()$major, getRversion()$minor), ".github/R-version") + shell: Rscript {0} + + - name: Cache R packages + if: runner.os != 'Windows' + uses: actions/cache@v3 + with: + path: ${{ env.R_LIBS_USER }} + key: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-${{ matrix.config.mlr3 }}-1-${{ hashFiles('.github/depends.Rds') }} + restore-keys: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-${{ matrix.config.mlr3 }}-1- + + - name: Install dependencies + run: | + remotes::install_deps(dependencies = TRUE) + remotes::install_cran("rcmdcheck") + shell: Rscript {0} + + - name: Introduction and Cross Fit + run: | + source("Code/examples_failure_n_500_p_20.R") + shell: Rscript {0} + + - uses: actions/upload-artifact@v3 + with: + name: sim-results + path: | + ./simresults/ + ./Figures/ + + - name: Session info + run: | + options(width = 100) + pkgs <- installed.packages()[, "Package"] + sessioninfo::session_info(pkgs, include_base = TRUE) + shell: Rscript {0} From 55c06a85bcd8a29a587e88f48bac5d857d178014 Mon Sep 17 00:00:00 2001 From: Philipp Bach Date: Tue, 28 Mar 2023 09:34:25 +0200 Subject: [PATCH 06/12] rename --- ...plication_failure.yml => runreplication_key_ingredients.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{runreplication_failure.yml => runreplication_key_ingredients.yml} (98%) diff --git a/.github/workflows/runreplication_failure.yml b/.github/workflows/runreplication_key_ingredients.yml similarity index 98% rename from .github/workflows/runreplication_failure.yml rename to .github/workflows/runreplication_key_ingredients.yml index 5a95e05..9c08618 100644 --- a/.github/workflows/runreplication_failure.yml +++ b/.github/workflows/runreplication_key_ingredients.yml @@ -1,5 +1,5 @@ # Workflow based on https://blog--simonpcouch.netlify.app/blog/r-github-actions-commit/ -name: Replication +name: Replication Key Ingredients on: schedule: From a30135494bb656abed89bf2d501fe296c6933fec Mon Sep 17 00:00:00 2001 From: Philipp Bach Date: Tue, 28 Mar 2023 09:44:20 +0200 Subject: [PATCH 07/12] upd workflow --- .github/workflows/runreplication.yml | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/.github/workflows/runreplication.yml b/.github/workflows/runreplication.yml index 13eeb90..e878d8e 100644 --- a/.github/workflows/runreplication.yml +++ b/.github/workflows/runreplication.yml @@ -1,5 +1,5 @@ # Workflow based on https://blog--simonpcouch.netlify.app/blog/r-github-actions-commit/ -name: Replication +name: Replication of Simulation Study on: schedule: @@ -56,11 +56,6 @@ jobs: remotes::install_cran("rcmdcheck") shell: Rscript {0} - - name: Introduction and Cross Fit - run: | - source("Code/examples_failure_n_500_p_20.R") - shell: Rscript {0} - - name: Run PLR run: | source("Code/sim_plr.R") @@ -84,16 +79,12 @@ jobs: - uses: actions/upload-artifact@v3 with: name: sim-results - path: | - ./simresults/ - ./Figures/ + path: ./simresults/ - uses: actions/download-artifact@v3 with: name: sim-results - path: | - ./simresults/ - ./Figures/ + path: ./simresults/ - name: Merge results run: | From 5041c4d21062f0e45b3e01be1da0d19215f4817b Mon Sep 17 00:00:00 2001 From: Philipp Bach Date: Tue, 28 Mar 2023 09:54:04 +0200 Subject: [PATCH 08/12] include option to (not) save figures from single simulations --- Code/sim_iivm.R | 19 +++++++++++++------ Code/sim_irm.R | 20 +++++++++++++------- Code/sim_plivX.R | 21 +++++++++++++-------- Code/sim_plr.R | 18 ++++++++++++------ 4 files changed, 51 insertions(+), 27 deletions(-) diff --git a/Code/sim_iivm.R b/Code/sim_iivm.R index 9b5255b..cac47c7 100644 --- a/Code/sim_iivm.R +++ b/Code/sim_iivm.R @@ -12,7 +12,12 @@ library(doRNG) # Create a new directory plus subdirectories dir.create("simresults") dir.create("simresults/iivm") -dir.create("Figures") + +save_plot = FALSE + +if (save_plot) { + dir.create("Figures") +} date = Sys.Date() learner_name = "regr.cv_glmnet" @@ -93,11 +98,13 @@ g_iivm = ggplot(df_iivm, aes(x = coef_resc)) + g_iivm -ggsave(filename = paste0("Figures/densplot_IIVM_", n_obs, "_", dim_x, "_", - "_", dml_procedure, "_", n_folds, - "_", n_rep_folds, "_", learner_name, "_", R, - "_", alpha, ".pdf"), - plot = g_iivm) +if (save_plot) { + ggsave(filename = paste0("Figures/densplot_IIVM_", n_obs, "_", dim_x, "_", + "_", dml_procedure, "_", n_folds, + "_", n_rep_folds, "_", learner_name, "_", R, + "_", alpha, ".pdf"), + plot = g_iivm) +} results_summary = data.table( "learner" = learner_name, diff --git a/Code/sim_irm.R b/Code/sim_irm.R index 3bf08e1..990f737 100644 --- a/Code/sim_irm.R +++ b/Code/sim_irm.R @@ -12,7 +12,12 @@ library(doRNG) # Create a new directory plus subdirectories dir.create("simresults") dir.create("simresults/irm") -dir.create("Figures") + +save_plot = FALSE + +if (save_plot) { + dir.create("Figures") +} date = Sys.Date() learner_name = "regr.cv_glmnet" @@ -93,12 +98,13 @@ g_est = ggplot(df, aes(x = coef_resc)) + g_est -ggsave(filename = paste0("Figures/densplot_IRM_", n_obs, "_", dim_x, "_", - "_", dml_procedure, "_", n_folds, - "_", n_rep_folds, "_", learner_name, "_", R, - "_", alpha, ".pdf"), - plot = g_est) - +if (save_plot) { + ggsave(filename = paste0("Figures/densplot_IRM_", n_obs, "_", dim_x, "_", + "_", dml_procedure, "_", n_folds, + "_", n_rep_folds, "_", learner_name, "_", R, + "_", alpha, ".pdf"), + plot = g_est) +} results_summary = data.table( "learner" = learner_name, "n_folds" = n_folds, diff --git a/Code/sim_plivX.R b/Code/sim_plivX.R index ae8632c..312e6b0 100644 --- a/Code/sim_plivX.R +++ b/Code/sim_plivX.R @@ -12,7 +12,12 @@ library(doRNG) # Create a new directory plus subdirectories dir.create("simresults") dir.create("simresults/pliv") -dir.create("Figures") + +save_plot = FALSE + +if (save_plot) { + dir.create("Figures") +} date = Sys.Date() learner_name = "regr.cv_glmnet" @@ -89,13 +94,13 @@ g_est = ggplot(df, aes(x = coef_resc)) + g_est - -ggsave(filename = paste0("Figures/densplot_PLIV_", n_obs, "_", dim_x, "_", - "_", dml_procedure, "_", n_folds, - "_", n_rep_folds, "_", learner_name, "_", R, - "_", alpha, ".pdf"), - plot = g_est) - +if (save_plot) { + ggsave(filename = paste0("Figures/densplot_PLIV_", n_obs, "_", dim_x, "_", + "_", dml_procedure, "_", n_folds, + "_", n_rep_folds, "_", learner_name, "_", R, + "_", alpha, ".pdf"), + plot = g_est) +} results_summary = data.table( "learner" = learner_name, diff --git a/Code/sim_plr.R b/Code/sim_plr.R index c8f3922..efbf399 100644 --- a/Code/sim_plr.R +++ b/Code/sim_plr.R @@ -12,7 +12,10 @@ library(doRNG) # Create a new directory plus subdirectories dir.create("simresults") dir.create("simresults/plr") -dir.create("Figures") + +if (save_plot) { + dir.create("Figures") +} date = Sys.Date() learner_name = "regr.ranger" @@ -96,11 +99,14 @@ g_est = ggplot(df, aes(x = coef_resc)) + g_est -ggsave(filename = paste0("Figures/densplot_PLR_", n_obs, "_", dim_x, "_", - "_", dml_procedure, "_", n_folds, - "_", n_rep_folds, "_", learner_name, "_", R, - "_", alpha, ".pdf"), - plot = g_est) +if (save_plot) { + + ggsave(filename = paste0("Figures/densplot_PLR_", n_obs, "_", dim_x, "_", + "_", dml_procedure, "_", n_folds, + "_", n_rep_folds, "_", learner_name, "_", R, + "_", alpha, ".pdf"), + plot = g_est) +} results_summary = data.table( "learner" = learner_name, From 1c300bd708a2562ea31c5da992751c0de65511c2 Mon Sep 17 00:00:00 2001 From: Philipp Bach Date: Tue, 28 Mar 2023 10:19:44 +0200 Subject: [PATCH 09/12] set save_plot to FALSE --- Code/sim_plr.R | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Code/sim_plr.R b/Code/sim_plr.R index efbf399..d058445 100644 --- a/Code/sim_plr.R +++ b/Code/sim_plr.R @@ -13,6 +13,8 @@ library(doRNG) dir.create("simresults") dir.create("simresults/plr") +save_plot = FALSE + if (save_plot) { dir.create("Figures") } From 14fff233bd093f6e5790e882a6154d30b79096ac Mon Sep 17 00:00:00 2001 From: PhilippBach Date: Wed, 5 Jun 2024 13:32:16 +0200 Subject: [PATCH 10/12] update to paradox breaking changes --- Code/doubleml_codechunks.R | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Code/doubleml_codechunks.R b/Code/doubleml_codechunks.R index c669193..3a3998b 100644 --- a/Code/doubleml_codechunks.R +++ b/Code/doubleml_codechunks.R @@ -244,11 +244,8 @@ doubleml_plr = DoubleMLPLR$new(doubleml_data, ml_l, ml_m) ## ---- eval = TRUE----------------------------------------------------------------------------------------- par_grids = list( - "ml_l" = ParamSet$new(list( - ParamDbl$new("lambda", lower = 0.05, upper = 0.1))), - "ml_m" = ParamSet$new(list( - ParamDbl$new("lambda", lower = 0.05, upper = 0.1)))) - + "ml_l" = ps(lambda = p_dbl(lower = 0.05, upper = 0.1)), + "ml_m" = ps(lambda = p_dbl(lower = 0.05, upper = 0.1))) ## ---- eval = TRUE----------------------------------------------------------------------------------------- # Provide tune settings From 28eb2db32ba17dfc16bb8a30774369b0c85b8c06 Mon Sep 17 00:00:00 2001 From: PhilippBach Date: Wed, 5 Jun 2024 13:36:49 +0200 Subject: [PATCH 11/12] increment GH actions --- .github/workflows/runcodechunks_vignette.yml | 4 ++-- .github/workflows/runreplication.yml | 10 +++++----- .github/workflows/runreplication_key_ingredients.yml | 6 +++--- .github/workflows/runreplication_sim_inf.yml | 6 +++--- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/runcodechunks_vignette.yml b/.github/workflows/runcodechunks_vignette.yml index 19d9442..5efff12 100644 --- a/.github/workflows/runcodechunks_vignette.yml +++ b/.github/workflows/runcodechunks_vignette.yml @@ -24,7 +24,7 @@ jobs: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 2 @@ -44,7 +44,7 @@ jobs: - name: Cache R packages if: runner.os != 'Windows' - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ env.R_LIBS_USER }} key: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-${{ matrix.config.mlr3 }}-1-${{ hashFiles('.github/depends.Rds') }} diff --git a/.github/workflows/runreplication.yml b/.github/workflows/runreplication.yml index e878d8e..b1425a5 100644 --- a/.github/workflows/runreplication.yml +++ b/.github/workflows/runreplication.yml @@ -24,7 +24,7 @@ jobs: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 2 @@ -44,7 +44,7 @@ jobs: - name: Cache R packages if: runner.os != 'Windows' - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ env.R_LIBS_USER }} key: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-${{ matrix.config.mlr3 }}-1-${{ hashFiles('.github/depends.Rds') }} @@ -76,12 +76,12 @@ jobs: source("Code/sim_iivm.R") shell: Rscript {0} - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: sim-results path: ./simresults/ - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: sim-results path: ./simresults/ @@ -91,7 +91,7 @@ jobs: source("Code/merge_plots.R") shell: Rscript {0} - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: sim-results path: | diff --git a/.github/workflows/runreplication_key_ingredients.yml b/.github/workflows/runreplication_key_ingredients.yml index 9c08618..46c186f 100644 --- a/.github/workflows/runreplication_key_ingredients.yml +++ b/.github/workflows/runreplication_key_ingredients.yml @@ -24,7 +24,7 @@ jobs: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 2 @@ -44,7 +44,7 @@ jobs: - name: Cache R packages if: runner.os != 'Windows' - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ env.R_LIBS_USER }} key: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-${{ matrix.config.mlr3 }}-1-${{ hashFiles('.github/depends.Rds') }} @@ -61,7 +61,7 @@ jobs: source("Code/examples_failure_n_500_p_20.R") shell: Rscript {0} - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: sim-results path: | diff --git a/.github/workflows/runreplication_sim_inf.yml b/.github/workflows/runreplication_sim_inf.yml index b4370c4..da229b3 100644 --- a/.github/workflows/runreplication_sim_inf.yml +++ b/.github/workflows/runreplication_sim_inf.yml @@ -24,7 +24,7 @@ jobs: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 2 @@ -44,7 +44,7 @@ jobs: - name: Cache R packages if: runner.os != 'Windows' - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ env.R_LIBS_USER }} key: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-${{ matrix.config.mlr3 }}-1-${{ hashFiles('.github/depends.Rds') }} @@ -61,7 +61,7 @@ jobs: source("Code/sim_siminf.R") shell: Rscript {0} - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: sim-siminf-results path: ./simresults/ From 245bc0c23183d8b91b43c8cbce7eed31a502839a Mon Sep 17 00:00:00 2001 From: PhilippBach Date: Wed, 5 Jun 2024 14:57:45 +0200 Subject: [PATCH 12/12] rename artifact for final results --- .github/workflows/runreplication_sim_inf.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runreplication_sim_inf.yml b/.github/workflows/runreplication_sim_inf.yml index da229b3..7277901 100644 --- a/.github/workflows/runreplication_sim_inf.yml +++ b/.github/workflows/runreplication_sim_inf.yml @@ -63,7 +63,7 @@ jobs: - uses: actions/upload-artifact@v4 with: - name: sim-siminf-results + name: sim-siminf-results-final path: ./simresults/ - name: Session info