Skip to content

New estimates (data and do files updates; no code updates) #210

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
pbronka wants to merge 1 commit into from
Closed

New estimates (data and do files updates; no code updates) #210

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file added input/InitialPopulations/SimPaths - lag strucure - UK.xlsx
View file
Open in desktop
Binary file not shown.
30 changes: 14 additions & 16 deletions input/InitialPopulations/compile/00_master.do
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
* DATA: UKHLS EUL version - UKDA-6614-stata [to wave n]
* WAS EUL version - UKDA-7215-stata [to wave 7]
* AUTHORS: Daria Popova, Justin van de Ven
* LAST UPDATE: 30 Apr 2025
* LAST UPDATE: 1 July 2025 DP
***************************************************************************************

***************************************************************************************
Expand All @@ -33,15 +33,14 @@ set matsize 1000

/**************************************************************************************
* DEFINE DIRECTORIES
*************************************************************************************/
**************************************************************************************/

* Working directory
*global dir_work "C:\MyFiles\99 DEV ENV\JAS-MINE\data work\initial_populations"
global dir_work "C:\Users\Patryk\Documents\SP_prep_pop"
//global dir_work "C:\MyFiles\99 DEV ENV\JAS-MINE\data work\initial_populations"
global dir_work "D:\Dasha\ESSEX\ESPON 2024\UK\initial_populations"

* Directory which contains do files
*global dir_do "${dir_work}/do"
global dir_do "C:\Users\Patryk\git\SimPathsFork\input\InitialPopulations\compile"
global dir_do "${dir_work}/do"

* Directory which contains data files
global dir_data "${dir_work}/data"
Expand All @@ -50,22 +49,20 @@ global dir_data "${dir_work}/data"
global dir_log "${dir_work}/log"

* Directory which contains UKHLS data
*global dir_ukhls_data "J:\01 DATA\UK\ukhls\wave14\stata\stata13_se\ukhls"
//global dir_ukhls_data "D:\Dasha\UK-original-data\USoc\UKDA-6614-stata\stata\stata13_se\ukhls"
global dir_ukhls_data "C:\Users\Patryk\Documents\SP_prep_pop\ukhls\UKDA-6614-stata\stata\stata13_se\ukhls"
//global dir_ukhls_data "J:\01 DATA\UK\ukhls\wave13\stata\stata13_se\ukhls"
global dir_ukhls_data "D:\Dasha\UK-original-data\USoc\UKDA-6614-stata\stata\stata13_se\ukhls"

* Directory which contains WAS data
*global dir_was_data "J:\01 DATA\UK\was\wave7\stata\stata13_se"
//global dir_was_data "D:\Dasha\UK-original-data\WAS\UKDA-7215-stata\stata\stata13_se"
global dir_was_data "C:\Users\Patryk\Documents\WAS\UKDA-7215-stata\stata\stata13_se"
//global dir_was_data "J:\01 DATA\UK\was\wave7\stata\stata13_se"
global dir_was_data "D:\Dasha\UK-original-data\WAS\UKDA-7215-stata\stata\stata13_se"

* Directory which contains original initial popultions
global dir_ipop_orig "${dir_work}/original_initial_populations"


/**************************************************************************************
* DEFINE OTHER GLOBAL VARIABLES
*************************************************************************************/
**************************************************************************************/
* Define age to become responsible as defined in the simulation
global age_become_responsible 18

Expand Down Expand Up @@ -100,7 +97,8 @@ global wealthEndYear = 2019

/**************************************************************************************
* ROUTE TO WORKER FILES
*************************************************************************************/
**************************************************************************************/

* Prepare simulated and observed data
do "${dir_do}/01_prepare_UKHLS_pooled_data.do"
* Process UKHLS data
Expand All @@ -112,12 +110,12 @@ do "${dir_do}/04_social_care_provided.do"
do "${dir_do}/05_create_benefit_units.do"
* reweight data and slice into yearly segments
do "${dir_do}/06_reweight_and_slice.do"
/* impute wealth data for selected years
* impute wealth data for selected years
do "${dir_do}/07_was_wealth_data.do"
forvalues year = $wealthStartYear / $wealthEndYear {
global yearWealth = `year'
do "${dir_do}/08_wealth_to_ukhls.do"
}*/
}
do "${dir_do}/09_finalise_input_data.do"
do "${dir_do}/10_check_yearly_data.do"

Expand Down
103 changes: 39 additions & 64 deletions input/InitialPopulations/compile/01_prepare_UKHLS_pooled_data.do
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
* COUNTRY: UK
* DATA: UKHLS EUL version - UKDA-6614-stata [to wave n]
* AUTHORS: Daria Popova, Justin van de Ven
* LAST UPDATE: 14 Jan 2025 DP
* LAST UPDATE: 30 June 2025 DP
* NOTE: Called from 00_master.do - see master file for further details
***************************************************************************************

Expand All @@ -21,7 +21,7 @@ log using "${dir_log}/01_prepare_UKHLS_pooled_data.log", replace

/**************************************************************************************
* Select and merge UKHLS data
*************************************************************************************/
**************************************************************************************/

*add variables from the all persons (Household grid) dataset
foreach w of global UKHLSwaves {
Expand All @@ -31,7 +31,7 @@ foreach w of global UKHLSwaves {

if (`waveno'<13) {
use pidp `w'_ivfho `w'_ivfio `w'_hhorig `w'_buno_dv `w'_dvage `w'_sex `w'_depchl `w'_hidp `w'_pno `w'_pns1pid `w'_pns2pid `w'_month `w'_intdaty_dv ///
`w'_mnspid `w'_fnspid `w'_ppid `w'_ppno `w'_sppid `w'_sex_dv `w'_mastat_dv `w'_gor_dv `w'_age_dv /* `w'_hgbioad1 `w'_hgbioad2 */ ///
`w'_mnspid `w'_fnspid `w'_ppid `w'_ppno `w'_sppid `w'_sex_dv `w'_mastat_dv `w'_gor_dv `w'_age_dv /* `w'_hgbioad1 `w'_hgbioad2 */ ///
`w'_intdatd_dv `w'_intdatm_dv `w'_intdaty_dv `w'_ethn_dv using `w'_indall.dta, clear
}
else {
Expand Down Expand Up @@ -62,33 +62,29 @@ foreach w of global UKHLSwaves {
`w'_fimnmisc_dv `w'_fimnprben_dv `w'_fimninvnet_dv `w'_fimnsben_dv `w'_fimnlabgrs_dv `w'_fimnpen_dv `w'_jbstat `w'_hiqual_dv `w'_jbhrs ///
`w'_j2hrs `w'_jshrs /*`w'_scsfl*/ `w'_scghq1_dv `w'_scghq2_dv `w'_jbsic07_cc `w'_bendis* `w'_scghq1_dv `w'_scghq2_dv ///
/*`w'_indinus_lw `w'_indscus_lw `w'_indpxub_xw `w'_indpxui_xw `w'_relup `w'_currpart* `w'_lmcbm* `w'_lmcby4* */ `w'_sf12mcs_dv `w'_sf12pcs_dv ///
`w'_sclfsato `w'_finnow ///
using `w'_indresp.dta, clear
`w'_bendis* using `w'_indresp.dta, clear
}
else if (`waveno'<6) {
use pidp `w'_hidp `w'_pno `w'_buno_dv `w'_jbhrs `w'_jbot `w'_jshrs `w'_scghq1_dv `w'_scghq2_dv `w'_fimngrs_dv `w'_fimnnet_dv `w'_fimnlabnet_dv ///
`w'_fimnmisc_dv `w'_fimnprben_dv `w'_fimninvnet_dv `w'_fimnsben_dv `w'_fimnlabgrs_dv `w'_fimnpen_dv `w'_jbstat `w'_hiqual_dv `w'_jbhrs ///
`w'_j2hrs `w'_jshrs `w'_scsf1 `w'_scghq1_dv `w'_scghq2_dv `w'_jbsic07_cc `w'_bendis* `w'_scghq1_dv `w'_scghq2_dv ///
`w'_indinus_lw `w'_indscus_lw `w'_indpxub_xw /*`w'_indpxui_xw*/ `w'_relup `w'_currpart* `w'_lmcbm* `w'_lmcby4* `w'_sf12mcs_dv `w'_sf12pcs_dv ///
`w'_sclfsato `w'_finnow ///
using `w'_indresp.dta, clear
`w'_bendis* using `w'_indresp.dta, clear
}
else if (`waveno'<13) {
use pidp `w'_hidp `w'_pno `w'_buno_dv `w'_jbhrs `w'_jbot `w'_jshrs `w'_scghq1_dv `w'_scghq2_dv `w'_fimngrs_dv `w'_fimnnet_dv `w'_fimnlabnet_dv ///
`w'_fimnmisc_dv `w'_fimnprben_dv `w'_fimninvnet_dv `w'_fimnsben_dv `w'_fimnlabgrs_dv `w'_fimnpen_dv `w'_jbstat `w'_hiqual_dv `w'_jbhrs ///
`w'_j2hrs `w'_jshrs `w'_scsf1 `w'_scghq1_dv `w'_scghq2_dv `w'_jbsic07_cc `w'_bendis* `w'_scghq1_dv `w'_scghq2_dv ///
`w'_indinus_lw `w'_indscus_lw /*`w'_indpxub_xw*/ `w'_indpxui_xw `w'_relup `w'_currpart* `w'_lmcbm* `w'_lmcby4* `w'_sf12mcs_dv `w'_sf12pcs_dv ///
`w'_sclfsato `w'_finnow ///
using `w'_indresp.dta, clear
`w'_bendis* using `w'_indresp.dta, clear
}

else if (`waveno'==13) {
use pidp `w'_hidp `w'_pno /*`w'_buno_dv*/ `w'_jbhrs `w'_jbot `w'_jshrs `w'_scghq1_dv `w'_scghq2_dv `w'_fimngrs_dv `w'_fimnnet_dv `w'_fimnlabnet_dv ///
`w'_fimnmisc_dv `w'_fimnprben_dv `w'_fimninvnet_dv `w'_fimnsben_dv `w'_fimnlabgrs_dv `w'_fimnpen_dv `w'_jbstat `w'_hiqual_dv `w'_jbhrs ///
/*`w'_j2hrs*/ `w'_jshrs `w'_scsf1 `w'_scghq1_dv `w'_scghq2_dv `w'_jbsic07_cc `w'_bendis* `w'_scghq1_dv `w'_scghq2_dv ///
`w'_indinus_lw `w'_indscus_lw /*`w'_indpxub_xw `w'_indpxui_xw*/ `w'_relup `w'_currpart* `w'_lmcbm* `w'_lmcby4* `w'_indpxui_xw `w'_sf12mcs_dv `w'_sf12pcs_dv ///
`w'_sclfsato `w'_finnow ///
using `w'_indresp.dta, clear
`w'_indinus_lw `w'_indscus_lw /*`w'_indpxub_xw*/ `w'_indpxui_xw `w'_relup `w'_currpart* `w'_lmcbm* `w'_lmcby4* `w'_indpxui_xw `w'_sf12mcs_dv `w'_sf12pcs_dv ///
`w'_bendis* using `w'_indresp.dta, clear
gen m_j2hrs=-9 /*m_j2hrs not available in wave 13*/
}

Expand All @@ -97,7 +93,7 @@ foreach w of global UKHLSwaves {
`w'_fimnmisc_dv `w'_fimnprben_dv `w'_fimninvnet_dv `w'_fimnsben_dv `w'_fimnlabgrs_dv `w'_fimnpen_dv `w'_jbstat `w'_hiqual_dv `w'_jbhrs ///
/*`w'_j2hrs*/ `w'_jshrs `w'_scsf1 `w'_scghq1_dv `w'_scghq2_dv `w'_jbsic07_cc `w'_bendis* `w'_scghq1_dv `w'_scghq2_dv ///
`w'_indinus_lw `w'_indscus_lw /*`w'_indpxub_xw `w'_indpxui_xw*/ `w'_indpxg2_xw `w'_relup `w'_currpart* `w'_lmcbm* `w'_lmcby4* `w'_sf12mcs_dv `w'_sf12pcs_dv ///
using `w'_indresp.dta, clear
`w'_bendis* using `w'_indresp.dta, clear
gen m_j2hrs=-9 /*m_j2hrs not available in wave 14*/
}

Expand All @@ -117,16 +113,16 @@ foreach w of global UKHLSwaves {
local waveno=strpos("abcdefghijklmnopqrstuvwxyz","`w'")

if (`waveno'==1) {
use `w'_hidp `w'_fihhmnnet1_dv `w'_fihhmngrs1_dv `w'_fihhmnsben_dv `w'_nch02_dv /*`w'_hhdenub_xw `w'_hhdenui_xw*/ `w'_hsownd using `w'_hhresp.dta, clear
use `w'_hidp `w'_fihhmnnet1_dv `w'_fihhmngrs1_dv `w'_nch02_dv /*`w'_hhdenub_xw `w'_hhdenui_xw*/ `w'_hsownd using `w'_hhresp.dta, clear
}
else if (`waveno'<6) {
use `w'_hidp `w'_fihhmnnet1_dv `w'_fihhmngrs1_dv `w'_fihhmnsben_dv `w'_nch02_dv `w'_hhdenub_xw /*`w'_hhdenui_xw*/ `w'_hsownd using `w'_hhresp.dta, clear
use `w'_hidp `w'_fihhmnnet1_dv `w'_fihhmngrs1_dv `w'_nch02_dv `w'_hhdenub_xw /*`w'_hhdenui_xw*/ `w'_hsownd using `w'_hhresp.dta, clear
}
else if (`waveno'<14) {
use `w'_hidp `w'_fihhmnnet1_dv `w'_fihhmngrs1_dv `w'_fihhmnsben_dv `w'_nch02_dv /*`w'_hhdenub_xw*/ `w'_hhdenui_xw `w'_hsownd using `w'_hhresp.dta, clear
use `w'_hidp `w'_fihhmnnet1_dv `w'_fihhmngrs1_dv `w'_nch02_dv /*`w'_hhdenub_xw*/ `w'_hhdenui_xw `w'_hsownd using `w'_hhresp.dta, clear
}
else if (`waveno'==14) {
use `w'_hidp `w'_fihhmnnet1_dv `w'_fihhmngrs1_dv `w'_fihhmnsben_dv `w'_nch02_dv /*`w'_hhdenub_xw `w'_hhdenui_xw*/ `w'_hhdeng2_xw `w'_hsownd using `w'_hhresp.dta, clear
use `w'_hidp `w'_fihhmnnet1_dv `w'_fihhmngrs1_dv `w'_nch02_dv /*`w'_hhdenub_xw `w'_hhdenui_xw*/ `w'_hhdeng2_xw `w'_hsownd using `w'_hhresp.dta, clear
}

gen swv = `waveno'
Expand All @@ -140,7 +136,7 @@ foreach w of global UKHLSwaves {

/**************************************************************************************
* Prepare and merge income variables:
*************************************************************************************/
**************************************************************************************/
foreach w of global UKHLSwaves {

// find the wave number
Expand All @@ -167,29 +163,31 @@ gen inc_tu = frmnthimp_dv if ficode == 25 //Trade Union / Friendly Society Payme
gen inc_ma = frmnthimp_dv if ficode == 26 //Maintenance or Alimony
gen inc_fm = frmnthimp_dv if ficode == 27 //payments from a family member not living here
gen inc_oth = frmnthimp_dv if ficode == 38 //any other regular payment (not asked in Wave 1)
keep swv pidp hidp inc_pp inc_tu inc_ma inc_fm inc_oth
drop if missing(inc_pp) & missing(inc_tu) & missing(inc_ma) & missing(inc_fm) & missing(inc_oth)
collapse (sum) inc_pp inc_tu inc_ma inc_fm inc_oth, by(swv pidp hidp)
/*
8 Severe Disablement Allowance
9 Industrial Injury Disablement Allowance
10 Disability Living Allowance
11 Attendance Allowance
12 Carer's Allowance (formerly Invalid Care Allowance)
13 War Disablement Pension
14 Incapacity Benefit
33 Employment and Support Allowance
34 Return to Work Credit
35 Sickness and Accident Insurance
37 Other Disability Related Benefit or Payment
41 Personal Independence Payments
43 Child Disability Payment
44 Adult Disability Payment
45 Pension Age Disability Payment
*/
gen inc_disab = frmnthimp_dv if (ficode>=8 & ficode<=14) | ficode==33 | ficode==34 | ficode==35 | ficode==37 | ficode==41 | ficode==43 | ficode==44 | ficode==45

keep swv pidp hidp inc_pp inc_tu inc_ma inc_fm inc_oth inc_disab
drop if missing(inc_pp) & missing(inc_tu) & missing(inc_ma) & missing(inc_fm) & missing(inc_oth) & missing(inc_disab)
collapse (sum) inc_pp inc_tu inc_ma inc_fm inc_oth inc_disab, by(swv pidp hidp)
save "$dir_data\tmp_income", replace
restore

/******************************Benefits receipt *****************************/

preserve
* Generate UC benefit marker
gen benefits_uc=(ficode==40)
label var benefits_uc "Universal Credit indicator"


keep hidp pidp swv benefits_uc
collapse (max) benefits_uc, by(hidp swv)
compress

save "$dir_data/tmp_ucrcpt", replace
restore



//merge variables from the youth dataset 9-18 years old *
foreach w of global UKHLSwaves {

Expand All @@ -208,34 +206,13 @@ foreach w of global UKHLSwaves {
}
}

//merge variables from the youth dataset 9-18 years old *

foreach w of global UKHLSwaves {

// find the wave number
local waveno=strpos("abcdefghijklmnopqrstuvwxyz","`w'")

if (`waveno'>7 | mod(`waveno',2)==0) {
use pidp `w'_hidp `w'_ypsrhlth using `w'_youth.dta, clear

gen swv = `waveno'
rename `w'_* *
if (`waveno'>2) {
append using "$dir_data\add_vars_ukhls_youth.dta"
}
save "$dir_data\add_vars_ukhls_youth.dta", replace
}
}



/**************************************************************************************
* merge all datasets together
*************************************************************************************/
**************************************************************************************/
use "$dir_data\add_vars_ukhls.dta", clear
merge 1:1 pidp hidp swv using "$dir_data\add_vars_ukhls_indresp.dta", keep(1 3) nogen
merge m:1 hidp swv using "$dir_data\add_vars_ukhls_hhresp.dta", keep(1 3) nogen
merge m:1 hidp swv using "$dir_data\tmp_ucrcpt", keep(1 3) nogen
merge 1:1 pidp hidp swv using "$dir_data\tmp_income", keep(1 3) nogen
merge 1:1 pidp hidp swv using "$dir_data\add_vars_ukhls_youth.dta", keep(1 3) nogen

Expand All @@ -252,14 +229,14 @@ replace month = 1 if month == -10 // month not available for IEMB (Ethnic Minori

/**************************************************************************************
* save output
*************************************************************************************/
**************************************************************************************/
save "$dir_data\ukhls_pooled_all_obs_01.dta", replace
cap log close


/**************************************************************************************
* clean-up and exit
*************************************************************************************/
**************************************************************************************/
#delimit ;
local files_to_drop
add_vars_ukhls.dta
Expand All @@ -276,5 +253,3 @@ foreach file of local files_to_drop {
}




Loading
Loading