From 35c7e26022a9750a68055f7ca1797344ce90fe13 Mon Sep 17 00:00:00 2001 From: Henry Doupe Date: Mon, 23 Oct 2017 15:04:41 -0400 Subject: [PATCH 01/10] Reinstate OG-USA --- webapp/apps/dynamic/compute.py | 20 +++--------- webapp/apps/dynamic/helpers.py | 2 +- webapp/apps/dynamic/views.py | 56 +++++++++++++--------------------- 3 files changed, 26 insertions(+), 52 deletions(-) diff --git a/webapp/apps/dynamic/compute.py b/webapp/apps/dynamic/compute.py index cb394685..36f891fb 100644 --- a/webapp/apps/dynamic/compute.py +++ b/webapp/apps/dynamic/compute.py @@ -33,25 +33,13 @@ def remote_register_job(self, theurl, data, timeout=TIMEOUT_IN_SECONDS): response = requests.post(theurl, data=data, timeout=timeout) return response - def submit_json_ogusa_calculation(self, ogusa_mods, first_budget_year, - microsim_data, pack_up_user_mods): - return self.submit_ogusa_calculation(ogusa_mods, first_budget_year, - microsim_data, pack_up_user_mods=False) - - def submit_ogusa_calculation(self, ogusa_mods, first_budget_year, microsim_data, - pack_up_user_mods=True): - + def submit_ogusa_calculation(self, ogusa_mods, first_budget_year, microsim_data): print "mods is ", ogusa_mods ogusa_params = filter_ogusa_only(ogusa_mods) data = {} - if pack_up_user_mods: - microsim_params = package_up_vars(microsim_data, first_budget_year) - microsim_params = {first_budget_year:microsim_params} - print "microsim data is", microsim_params - else: - data['taxio_format'] = True - data['first_budget_year'] = first_budget_year - microsim_params = microsim_data + data['taxio_format'] = True + data['first_budget_year'] = first_budget_year + microsim_params = microsim_data print "submit dynamic work" diff --git a/webapp/apps/dynamic/helpers.py b/webapp/apps/dynamic/helpers.py index 7a9f9d85..a9a2b15d 100644 --- a/webapp/apps/dynamic/helpers.py +++ b/webapp/apps/dynamic/helpers.py @@ -209,7 +209,7 @@ def filter_ogusa_only(user_values): print "Removing ", k, v del user_values[k] else: - user_values[k] = float(v) + user_values[k] = float(v[0]) if isinstance(v, list) else float(v) return user_values diff --git a/webapp/apps/dynamic/views.py b/webapp/apps/dynamic/views.py index 0c41f9c4..30644365 100644 --- a/webapp/apps/dynamic/views.py +++ b/webapp/apps/dynamic/views.py @@ -89,17 +89,15 @@ def dynamic_input(request, pk): if dyn_mod_form.is_valid(): model = dyn_mod_form.save() - #Can't proceed if there is no email address + # Can't proceed if there is no email address if not (request.user.is_authenticated() or model.user_email): - msg = 'Dynamic simulation must have an email address to send notification to!' - return HttpResponse(msg, status=403) - - curr_dict = dict(model.__dict__) - for key, value in curr_dict.items(): - print "got this ", key, value + msg = ("Dynamic simulation must have an email " + "address to send notification to!") + return HttpResponse(msg, status=403) # get macrosim data from form - worker_data = {k:v for k, v in curr_dict.items() if v not in (u'', None, [])} + curr_dict = dict(model.__dict__) + worker_data = parse_fields(curr_dict) #get microsim data outputsurl = OutputUrl.objects.get(pk=pk) @@ -108,35 +106,23 @@ def dynamic_input(request, pk): submitted_ids = None if not taxbrain_model.json_text: - taxbrain_dict = dict(taxbrain_model.__dict__) - growth_fixup(taxbrain_dict) - for key, value in taxbrain_dict.items(): - if type(value) == type(unicode()): - try: - taxbrain_dict[key] = [float(x) for x in value.split(',') if x] - except ValueError: - taxbrain_dict[key] = [make_bool(x) for x in value.split(',') if x] - else: - print "missing this: ", key - - - microsim_data = {k:v for k, v in taxbrain_dict.items() if not (v == [] or v == None)} - - #Don't need to pass around the microsim results - if 'tax_result' in microsim_data: - del microsim_data['tax_result'] - - benefit_switch_fixup(request.REQUEST, microsim_data, taxbrain_model) + (reform_dict, _, _, _, + errors_warnings) = get_reform_from_gui( + request, + taxbrain_model=taxbrain_model, + behavior_model=None + ) - # start calc job - submitted_ids, guids = dynamic_compute.submit_ogusa_calculation(worker_data, int(start_year), microsim_data) else: - microsim_data = {"reform": taxbrain_model.json_text.reform_text, "assumptions": taxbrain_model.json_text.assumption_text} - # start calc job - submitted_ids, guids = dynamic_compute.submit_json_ogusa_calculation(worker_data, - int(start_year), - microsim_data, - pack_up_user_mods=False) + reform_dict = json.loads(taxbrain_model.json_text.reform_text) + print(reform_dict) + print(worker_data) + submitted_ids, guids = dynamic_compute.submit_ogusa_calculation( + worker_data, + int(start_year), + reform_dict + ) + # TODO: use OutputUrl class if submitted_ids: model.job_ids = denormalize(submitted_ids) From a29a2c2ed28cf64d01e150138d3c1977073cae32 Mon Sep 17 00:00:00 2001 From: Henry Doupe Date: Mon, 23 Oct 2017 15:22:20 -0400 Subject: [PATCH 02/10] Move all parameter processing into dynamic/views.py --- webapp/apps/dynamic/compute.py | 15 ++------------- webapp/apps/dynamic/views.py | 19 +++++++++++++------ 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/webapp/apps/dynamic/compute.py b/webapp/apps/dynamic/compute.py index 36f891fb..1789b4db 100644 --- a/webapp/apps/dynamic/compute.py +++ b/webapp/apps/dynamic/compute.py @@ -7,7 +7,6 @@ import taxcalc from ..taxbrain.compute import DropqCompute, MockCompute from .models import OGUSAWorkerNodesCounter -from .helpers import filter_ogusa_only from ..constants import START_YEAR dqversion_info = taxcalc._version.get_versions() @@ -33,15 +32,8 @@ def remote_register_job(self, theurl, data, timeout=TIMEOUT_IN_SECONDS): response = requests.post(theurl, data=data, timeout=timeout) return response - def submit_ogusa_calculation(self, ogusa_mods, first_budget_year, microsim_data): - print "mods is ", ogusa_mods - ogusa_params = filter_ogusa_only(ogusa_mods) - data = {} - data['taxio_format'] = True - data['first_budget_year'] = first_budget_year - microsim_params = microsim_data - - + def submit_ogusa_calculation(self, data): + print "mods is ", data print "submit dynamic work" hostnames = OGUSA_WORKERS @@ -50,9 +42,6 @@ def submit_ogusa_calculation(self, ogusa_mods, first_budget_year, microsim_data) 'callback': "http://{}/dynamic/dynamic_finished".format(CALLBACK_HOSTNAME), } - data['ogusa_params'] = json.dumps(ogusa_params) - data['user_mods'] = json.dumps(microsim_params) - data['first_year'] = first_budget_year job_ids = [] guids = [] diff --git a/webapp/apps/dynamic/views.py b/webapp/apps/dynamic/views.py index 30644365..3b3436f3 100644 --- a/webapp/apps/dynamic/views.py +++ b/webapp/apps/dynamic/views.py @@ -45,7 +45,8 @@ failure_text, normalize, denormalize, strip_empty_lists, cc_text_finished, cc_text_failure, dynamic_params_from_model, send_cc_email, default_behavior_parameters, - elast_results_to_tables, default_elasticity_parameters) + elast_results_to_tables, default_elasticity_parameters, + filter_ogusa_only) from .compute import DynamicCompute @@ -115,12 +116,18 @@ def dynamic_input(request, pk): else: reform_dict = json.loads(taxbrain_model.json_text.reform_text) - print(reform_dict) - print(worker_data) + + ogusa_params = filter_ogusa_only(worker_data) + data = { + 'taxio_format': True, + 'first_budget_year': int(start_year), + 'ogusa_params': json.dumps(ogusa_params), + 'user_mods': json.dumps(reform_dict), + 'first_year': int(start_year) + } + submitted_ids, guids = dynamic_compute.submit_ogusa_calculation( - worker_data, - int(start_year), - reform_dict + data ) # TODO: use OutputUrl class From 8294ab52d22ccfa9c28e633a8dd38dc06fb7e280 Mon Sep 17 00:00:00 2001 From: Henry Doupe Date: Mon, 23 Oct 2017 15:52:27 -0400 Subject: [PATCH 03/10] Remove striped contruction bars and enable link --- templates/dynamic/dynamic_input_form.html | 5 ----- templates/dynamic/landing.html | 9 +-------- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/templates/dynamic/dynamic_input_form.html b/templates/dynamic/dynamic_input_form.html index 3ebc8905..b83688ad 100644 --- a/templates/dynamic/dynamic_input_form.html +++ b/templates/dynamic/dynamic_input_form.html @@ -61,11 +61,6 @@

Tax-Calculator Code Build

-
-
-
-
-
diff --git a/templates/dynamic/landing.html b/templates/dynamic/landing.html index 168ce4de..12f4ca44 100644 --- a/templates/dynamic/landing.html +++ b/templates/dynamic/landing.html @@ -72,21 +72,14 @@

Choose a Dynamic Simulation Type

{% flatblock "dynamic_behavioral_blurb" %} - {% if include_ogusa %} - Overlapping Generations Simulation + Overlapping Generations Simulation -
-
-
-
-
{% flatblock "dynamic_olg_blurb" %} - {% endif %} Macro Elasticities Simulation From 7b6758a0aa25f66344d0ac3b1199260f9293f749 Mon Sep 17 00:00:00 2001 From: Henry Doupe Date: Wed, 25 Oct 2017 11:46:33 -0400 Subject: [PATCH 04/10] Accommodate new reform format --- deploy/taxbrain_server/run_ogusa.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/deploy/taxbrain_server/run_ogusa.py b/deploy/taxbrain_server/run_ogusa.py index e673dde3..756a5d48 100644 --- a/deploy/taxbrain_server/run_ogusa.py +++ b/deploy/taxbrain_server/run_ogusa.py @@ -19,12 +19,12 @@ def run_micro_macro(reform, user_params, guid): REFORM_DIR = "./OUTPUT_REFORM_" + guid BASELINE_DIR = "./OUTPUT_BASELINE_" + guid - # Add start year from reform to user parameters - if isinstance(reform, tuple): - start_year = sorted(reform[0].keys())[0] - else: - start_year = sorted(reform.keys())[0] + # if isinstance(reform, tuple): + # start_year = sorted(reform[0].keys())[0] + # else: + assert isinstance(reform, dict) + start_year = min(reform["policy"].keys(), key=int) user_params['start_year'] = start_year with open("log_{}.log".format(guid), 'w') as f: From bf73682f6633b2c929a4f589150e08ec41bcc253 Mon Sep 17 00:00:00 2001 From: Henry Doupe Date: Wed, 25 Oct 2017 14:39:20 -0400 Subject: [PATCH 05/10] Update run_ogusa.py example --- deploy/taxbrain_server/run_ogusa.py | 32 ++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/deploy/taxbrain_server/run_ogusa.py b/deploy/taxbrain_server/run_ogusa.py index 756a5d48..b490eb99 100644 --- a/deploy/taxbrain_server/run_ogusa.py +++ b/deploy/taxbrain_server/run_ogusa.py @@ -75,13 +75,25 @@ def run_micro_macro(reform, user_params, guid): if __name__ == "__main__": reform = { - 2017: { - '_II_rt1': [.09], - '_II_rt2': [.135], - '_II_rt3': [.225], - '_II_rt4': [.252], - '_II_rt5': [.297], - '_II_rt6': [.315], - '_II_rt7': [0.3564], - }, } - run_micro_macro(reform=reform, user_params={'frisch': 0.44}, guid='abc') + u'growdiff_response': {}, + u'consumption': {}, + u'growdiff_baseline': {}, + u'behavior': {}, + u'policy': { + 2017: { + u'_II_no_em_nu18': [False], + u'_NIIT_PT_taxed': [False], + u'_FICA_ss_trt': [0.1], + u'_ID_BenefitCap_Switch': [[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]], + u'_ALD_InvInc_ec_base_RyanBrady': [False], + u'_EITC_indiv': [False], + u'_ID_BenefitSurtax_Switch': [[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]], + u'_CTC_new_refund_limited': [False], + u'_CG_nodiff': [False] + } + }, + u'gdp_elasticity': {} + } + user_params = {u'g_y_annual': 0.04, u'frisch': 0.3} + + run_micro_macro(reform=reform, user_params=user_params, guid='abc') From 491ab014bc89dcc47f53b4080bdb4ae282f058d7 Mon Sep 17 00:00:00 2001 From: Henry Doupe Date: Wed, 25 Oct 2017 18:09:36 -0400 Subject: [PATCH 06/10] Remove duplicate start_year argument and use start_year argument --- deploy/taxbrain_server/celery_tasks.py | 2 +- deploy/taxbrain_server/flask_server.py | 8 +++++++- deploy/taxbrain_server/run_ogusa.py | 11 ++--------- webapp/apps/dynamic/views.py | 3 +-- 4 files changed, 11 insertions(+), 13 deletions(-) diff --git a/deploy/taxbrain_server/celery_tasks.py b/deploy/taxbrain_server/celery_tasks.py index 5477ebb2..98119323 100644 --- a/deploy/taxbrain_server/celery_tasks.py +++ b/deploy/taxbrain_server/celery_tasks.py @@ -147,7 +147,7 @@ def elasticity_gdp_task_async(year, user_mods, first_budget_year, elast_params): @celery_app.task -def ogusa_async(user_mods, ogusa_params, guid): +def ogusa_async(start_year, user_mods, ogusa_params, guid): print("user mods: ", user_mods) user_mods = convert_int_key(user_mods) user_reform = {'policy': user_mods} diff --git a/deploy/taxbrain_server/flask_server.py b/deploy/taxbrain_server/flask_server.py index ee9344b4..bef8a647 100644 --- a/deploy/taxbrain_server/flask_server.py +++ b/deploy/taxbrain_server/flask_server.py @@ -179,6 +179,7 @@ def example(): @app.route('/ogusa_start_job', methods=['POST']) def ogusa_start_job(): + start_year int(request.form["start_year"]) user_mods = json.loads(request.form['user_mods']) ogusa_params = json.loads(request.form['ogusa_params']) @@ -186,7 +187,12 @@ def ogusa_start_job(): with ticket_lock_context(): guid = uuid.uuid1().hex - job = ogusa_async.delay(user_mods=user_mods, ogusa_params=ogusa_params, guid=guid) + job = ogusa_async.delay( + start_year=start_year, + user_mods=user_mods, + ogusa_params=ogusa_params, + guid=guid + ) RUNNING_JOBS[job.id] = job print('job_id', job) print('GUID IS ', guid) diff --git a/deploy/taxbrain_server/run_ogusa.py b/deploy/taxbrain_server/run_ogusa.py index b490eb99..ad39b7a2 100644 --- a/deploy/taxbrain_server/run_ogusa.py +++ b/deploy/taxbrain_server/run_ogusa.py @@ -13,19 +13,12 @@ from ogusa.scripts.execute import runner -def run_micro_macro(reform, user_params, guid): +def run_micro_macro(start_year, reform, user_params, guid): start_time = time.time() REFORM_DIR = "./OUTPUT_REFORM_" + guid BASELINE_DIR = "./OUTPUT_BASELINE_" + guid - # Add start year from reform to user parameters - # if isinstance(reform, tuple): - # start_year = sorted(reform[0].keys())[0] - # else: - assert isinstance(reform, dict) - start_year = min(reform["policy"].keys(), key=int) - user_params['start_year'] = start_year with open("log_{}.log".format(guid), 'w') as f: f.write("guid: {}\n".format(guid)) @@ -91,7 +84,7 @@ def run_micro_macro(reform, user_params, guid): u'_CTC_new_refund_limited': [False], u'_CG_nodiff': [False] } - }, + }, u'gdp_elasticity': {} } user_params = {u'g_y_annual': 0.04, u'frisch': 0.3} diff --git a/webapp/apps/dynamic/views.py b/webapp/apps/dynamic/views.py index 3b3436f3..73fec1a3 100644 --- a/webapp/apps/dynamic/views.py +++ b/webapp/apps/dynamic/views.py @@ -120,10 +120,9 @@ def dynamic_input(request, pk): ogusa_params = filter_ogusa_only(worker_data) data = { 'taxio_format': True, - 'first_budget_year': int(start_year), 'ogusa_params': json.dumps(ogusa_params), 'user_mods': json.dumps(reform_dict), - 'first_year': int(start_year) + 'start_year': int(start_year) } submitted_ids, guids = dynamic_compute.submit_ogusa_calculation( From 6e95e84dccb798e07bf2b7ce83525ad553e5eb05 Mon Sep 17 00:00:00 2001 From: Henry Doupe Date: Wed, 25 Oct 2017 18:15:17 -0400 Subject: [PATCH 07/10] Add comments to dynamic/views.py --- webapp/apps/dynamic/views.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/webapp/apps/dynamic/views.py b/webapp/apps/dynamic/views.py index 73fec1a3..b1cff254 100644 --- a/webapp/apps/dynamic/views.py +++ b/webapp/apps/dynamic/views.py @@ -106,6 +106,7 @@ def dynamic_input(request, pk): taxbrain_model = outputsurl.unique_inputs submitted_ids = None + # necessary for simulations before PR 641 if not taxbrain_model.json_text: (reform_dict, _, _, _, errors_warnings) = get_reform_from_gui( @@ -117,6 +118,7 @@ def dynamic_input(request, pk): else: reform_dict = json.loads(taxbrain_model.json_text.reform_text) + # package up variables ogusa_params = filter_ogusa_only(worker_data) data = { 'taxio_format': True, @@ -124,7 +126,7 @@ def dynamic_input(request, pk): 'user_mods': json.dumps(reform_dict), 'start_year': int(start_year) } - + # start model run submitted_ids, guids = dynamic_compute.submit_ogusa_calculation( data ) From eedc230481754fe770e83972f2d08c05670f609d Mon Sep 17 00:00:00 2001 From: Henry Doupe Date: Thu, 26 Oct 2017 13:12:20 -0400 Subject: [PATCH 08/10] Fix syntax error --- deploy/taxbrain_server/flask_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/taxbrain_server/flask_server.py b/deploy/taxbrain_server/flask_server.py index bef8a647..2b0a9cba 100644 --- a/deploy/taxbrain_server/flask_server.py +++ b/deploy/taxbrain_server/flask_server.py @@ -179,7 +179,7 @@ def example(): @app.route('/ogusa_start_job', methods=['POST']) def ogusa_start_job(): - start_year int(request.form["start_year"]) + start_year = int(request.form["start_year"]) user_mods = json.loads(request.form['user_mods']) ogusa_params = json.loads(request.form['ogusa_params']) From 8ed6ab90ef0fcd5c8e6ced948887391faae6ca44 Mon Sep 17 00:00:00 2001 From: Henry Doupe Date: Thu, 26 Oct 2017 13:23:28 -0400 Subject: [PATCH 09/10] Clean up backend changes --- deploy/taxbrain_server/celery_tasks.py | 3 ++- deploy/taxbrain_server/run_ogusa.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/deploy/taxbrain_server/celery_tasks.py b/deploy/taxbrain_server/celery_tasks.py index 98119323..beb95073 100644 --- a/deploy/taxbrain_server/celery_tasks.py +++ b/deploy/taxbrain_server/celery_tasks.py @@ -154,7 +154,8 @@ def ogusa_async(start_year, user_mods, ogusa_params, guid): for key in EXPECTED_KEYS: if key not in user_reform: user_reform[key] = {} - diff_data = run_ogusa.run_micro_macro(reform=user_reform, + diff_data = run_ogusa.run_micro_macro(start_year=start_year, + reform=user_reform, user_params=ogusa_params, guid=guid) diff --git a/deploy/taxbrain_server/run_ogusa.py b/deploy/taxbrain_server/run_ogusa.py index ad39b7a2..d70cc84d 100644 --- a/deploy/taxbrain_server/run_ogusa.py +++ b/deploy/taxbrain_server/run_ogusa.py @@ -30,6 +30,8 @@ def run_micro_macro(start_year, reform, user_params, guid): Run baseline ------------------------------------------------------------------------ ''' + + user_params["start_year"] = start_year output_base = BASELINE_DIR kwargs={'output_base':output_base, 'baseline_dir':BASELINE_DIR, 'test':False, 'time_path':True, 'baseline':True, From 26695734b292ededc82235dd6b3a5d93f14d4eb1 Mon Sep 17 00:00:00 2001 From: Henry Doupe Date: Thu, 26 Oct 2017 15:07:39 -0400 Subject: [PATCH 10/10] Update run_ogusa.py example for new arg --- deploy/taxbrain_server/run_ogusa.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/taxbrain_server/run_ogusa.py b/deploy/taxbrain_server/run_ogusa.py index d70cc84d..cc59952c 100644 --- a/deploy/taxbrain_server/run_ogusa.py +++ b/deploy/taxbrain_server/run_ogusa.py @@ -91,4 +91,4 @@ def run_micro_macro(start_year, reform, user_params, guid): } user_params = {u'g_y_annual': 0.04, u'frisch': 0.3} - run_micro_macro(reform=reform, user_params=user_params, guid='abc') + run_micro_macro(start_year=2017, reform=reform, user_params=user_params, guid='abc')