diff --git a/regr_smlp/data/smlp_toy_num_resp_noknobs.csv b/regr_smlp/data/smlp_toy_num_resp_noknobs.csv new file mode 100644 index 00000000..7a4c8240 --- /dev/null +++ b/regr_smlp/data/smlp_toy_num_resp_noknobs.csv @@ -0,0 +1,12 @@ +categ,y1,y2,x0,x1,x2 +c14,5,9,10.0,2.0,3 +c15,9,9,12.0,,4 +c1,5,9,,3.0,4 +c9,5,5,11.0,2.0,6 +c5,9,5,10.0,2.0,8 +c10,9,9,9.0,4.0,7 +c13,5,5,9.0,3.0,6 +c4,5,5,10.0,3.0,4 +c15,9,9,11.0,4.0,4 +c11,5,5,12.0,2.0,7 +c19,9,5,10.0,3.0,7 diff --git a/regr_smlp/data/smlp_toy_num_resp_noknobs_pred_labeled.csv b/regr_smlp/data/smlp_toy_num_resp_noknobs_pred_labeled.csv new file mode 100644 index 00000000..2b0ea3c0 --- /dev/null +++ b/regr_smlp/data/smlp_toy_num_resp_noknobs_pred_labeled.csv @@ -0,0 +1,10 @@ +categ,y1,y2,x0,x1,x2 +c0,5,9,10.0,2.0,3 +c12,9,9,12.0,,4 +c2,5,9,,3.0,4 +c17,5,5,11.0,2.0,6 +c18,9,5,10.0,2.0,8 +c8,9,9,9.0,4.0,7 +c7,5,5,9.0,3.0,6 +c3,5,5,10.0,3.0,4 +c12,9,9,11.0,4.0,4 diff --git a/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_data_bounds.json b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_data_bounds.json new file mode 100644 index 00000000..510fd80e --- /dev/null +++ b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_data_bounds.json @@ -0,0 +1,18 @@ +{ + "x1": { + "min": 2.0, + "max": 4.0 + }, + "x2": { + "min": 3.0, + "max": 8.0 + }, + "y1": { + "min": 5.0, + "max": 9.0 + }, + "y2": { + "min": 5.0, + "max": 9.0 + } +} \ No newline at end of file diff --git a/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_dt_sklearn_model_complete.pkl b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_dt_sklearn_model_complete.pkl new file mode 100644 index 00000000..9af00b41 Binary files /dev/null and b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_dt_sklearn_model_complete.pkl differ diff --git a/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_dt_sklearn_y1_tree_rules.txt b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_dt_sklearn_y1_tree_rules.txt new file mode 100644 index 00000000..9f2dbad2 --- /dev/null +++ b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_dt_sklearn_y1_tree_rules.txt @@ -0,0 +1,10 @@ +#Forest semantics: majority vote +#Number of trees: 1 + +#TREE 0 +if (x2 > 0.7000000178813934) and (x1 > 0.25) then (y1 = 1.0) | based on 2 samples +if (x2 <= 0.7000000178813934) and (x2 > 0.4000000134110451) then (y1 = 0.0) | based on 2 samples +if (x2 > 0.7000000178813934) and (x1 <= 0.25) and (x2 > 0.9000000059604645) then (y1 = 1.0) | based on 1 samples +if (x2 > 0.7000000178813934) and (x1 <= 0.25) and (x2 <= 0.9000000059604645) then (y1 = 0.0) | based on 1 samples +if (x2 <= 0.7000000178813934) and (x2 <= 0.4000000134110451) and (x2 > 0.10000000149011612) then (y1 = 1.0) | based on 1 samples +if (x2 <= 0.7000000178813934) and (x2 <= 0.4000000134110451) and (x2 <= 0.10000000149011612) then (y1 = 0.0) | based on 1 samples diff --git a/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_dt_sklearn_y2_tree_rules.txt b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_dt_sklearn_y2_tree_rules.txt new file mode 100644 index 00000000..5015336b --- /dev/null +++ b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_dt_sklearn_y2_tree_rules.txt @@ -0,0 +1,7 @@ +#Forest semantics: majority vote +#Number of trees: 1 + +#TREE 0 +if (x2 > 0.4000000134110451) and (x1 <= 0.75) then (y2 = 0.0) | based on 5 samples +if (x2 <= 0.4000000134110451) then (y2 = 1.0) | based on 2 samples +if (x2 > 0.4000000134110451) and (x1 > 0.75) then (y2 = 1.0) | based on 1 samples diff --git a/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_features_scaler.pkl b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_features_scaler.pkl new file mode 100644 index 00000000..62adbe18 Binary files /dev/null and b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_features_scaler.pkl differ diff --git a/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_model_features_dict.json b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_model_features_dict.json new file mode 100644 index 00000000..85782d17 --- /dev/null +++ b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_model_features_dict.json @@ -0,0 +1,10 @@ +{ + "y1": [ + "x1", + "x2" + ], + "y2": [ + "x1", + "x2" + ] +} \ No newline at end of file diff --git a/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_model_levels_dict.json b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_model_levels_dict.json new file mode 100644 index 00000000..9e26dfee --- /dev/null +++ b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_model_levels_dict.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_rerun_model_config.json b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_rerun_model_config.json new file mode 100644 index 00000000..1313617f --- /dev/null +++ b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_rerun_model_config.json @@ -0,0 +1,172 @@ +{ + "alpha": null, + "analytics_mode": "verify", + "approximate_fractions": true, + "assertions_expressions": "(y2**3+x2)/2<6;y1>=9;y2<0", + "assertions_names": "asrt1,asrt2,asrt3", + "beta": null, + "center_offset": "0", + "compress_rules": false, + "continuous_correlation_estimators": [ + "pearson", + "spearman" + ], + "correlations_and_mutual_information": true, + "data_scaler": "min_max", + "delta_absolute": 0.0, + "delta_relative": 0.01, + "discretization_algo": "uniform", + "discretization_bins": 10, + "discretization_labels": true, + "discretization_type": "category", + "discretize_numeric_features": null, + "doe_algo": null, + "doe_box_behnken_centers": 1, + "doe_central_composite_alpha": "o", + "doe_central_composite_center": "2,2", + "doe_central_composite_face": "ccf", + "doe_design_resolution": null, + "doe_factor_level_ranges": null, + "doe_num_samples": null, + "doe_prob_distribution": "Normal", + "doe_spec_file": null, + "dt_sklearn_ccp_alpha": 0.0, + "dt_sklearn_criterion": "squared_error", + "dt_sklearn_max_depth": 15, + "dt_sklearn_max_features": null, + "dt_sklearn_max_leaf_nodes": null, + "dt_sklearn_min_impurity_decrease": 0.0, + "dt_sklearn_min_samples_leaf": 1, + "dt_sklearn_min_samples_split": 2, + "dt_sklearn_min_weight_fraction_leaf": 0.0, + "dt_sklearn_random_state": null, + "dt_sklearn_splitter": "best", + "epsilon": 0.05, + "et_sklearn_bootstrap": true, + "et_sklearn_ccp_alpha": 0.0, + "et_sklearn_criterion": "squared_error", + "et_sklearn_max_depth": null, + "et_sklearn_max_features": 1.0, + "et_sklearn_max_leaf_nodes": null, + "et_sklearn_max_samples": null, + "et_sklearn_min_impurity_decrease": 0.0, + "et_sklearn_min_samples_leaf": 1, + "et_sklearn_min_samples_split": 2, + "et_sklearn_min_weight_fraction_leaf": 0.0, + "et_sklearn_n_estimators": 100, + "et_sklearn_random_state": null, + "et_sklearn_verbose": 0, + "et_sklearn_warm_start": false, + "eta": null, + "features": "x0,x1,x2", + "fraction_precision": 64, + "impute_responses": false, + "interactive_plots": false, + "keep_features": [], + "labeled_data": null, + "lemma_precision": 0, + "load_configuration": null, + "log_files_prefix": null, + "log_level": "info", + "log_mode": "w", + "log_time": false, + "model": "dt_sklearn", + "model_caret_cross_validation": true, + "model_caret_return_train_score": false, + "model_caret_verbose": true, + "model_name": null, + "model_per_response": true, + "mrmr_feat_count_for_correlation": 15, + "mrmr_feat_count_for_prediction": 2, + "mutual_information_method": "normalized", + "negative_value": 0, + "new_data": null, + "nn_keras_batch_size": 200, + "nn_keras_batches_grid": null, + "nn_keras_epochs": 2000, + "nn_keras_hid_activation": "relu", + "nn_keras_layers": "2,1", + "nn_keras_layers_grid": null, + "nn_keras_learning_rate": 0.001, + "nn_keras_learning_rates_grid": null, + "nn_keras_loss_function": "mse", + "nn_keras_loss_functions_grid": null, + "nn_keras_metrics": [ + "mse" + ], + "nn_keras_optimizer": "adam", + "nn_keras_out_activation": "linear", + "nn_keras_sequential_api": true, + "nn_keras_tuner_algo": null, + "nn_keras_weights_precision": null, + "nnet_encoding": "nested", + "objectives_expressions": null, + "objectives_names": "None", + "optimization_strategy": "eager", + "optimize_pareto": true, + "output_directory": "./", + "poly_sklearn_copy_X": true, + "poly_sklearn_degree": 2, + "poly_sklearn_fit_intercept": true, + "poly_sklearn_n_jobs": null, + "poly_sklearn_positive": false, + "positive_value": 1, + "prediction_plots": false, + "psg_max_dimension": 3, + "psg_quality_target": "Lift", + "psg_top_ranked": 15, + "query_expressions": null, + "query_names": "None", + "radius_absolute": null, + "radius_relative": null, + "response": "y1,y2", + "response_map": null, + "response_plots": false, + "response_to_bool": null, + "rf_sklearn_bootstrap": true, + "rf_sklearn_ccp_alpha": 0.0, + "rf_sklearn_criterion": "squared_error", + "rf_sklearn_max_depth": null, + "rf_sklearn_max_features": 1.0, + "rf_sklearn_max_leaf_nodes": null, + "rf_sklearn_max_samples": null, + "rf_sklearn_min_impurity_decrease": 0.0, + "rf_sklearn_min_samples_leaf": 1, + "rf_sklearn_min_samples_split": 2, + "rf_sklearn_min_weight_fraction_leaf": 0.0, + "rf_sklearn_n_estimators": 100, + "rf_sklearn_random_state": null, + "rf_sklearn_verbose": 0, + "rf_sklearn_warm_start": false, + "sample_weights_coef": 0, + "sample_weights_exponent": 0, + "sample_weights_intercept": 0, + "save_configuration": false, + "save_model": "false", + "save_model_rerun_configuration": true, + "scale_features": true, + "scale_objectives": true, + "scale_responses": true, + "seed": 10, + "setup_caret_data_split_shuffle": true, + "setup_caret_fold": 0, + "setup_caret_session_id": null, + "setup_caret_verbose": true, + "simplify_terms": false, + "solver": "z3", + "solver_logic": "ALL", + "solver_path": null, + "spec": "../specs/smlp_toy_num_resp_noknobs_verify.spec", + "split_test": 0.2, + "trace_anonymize": true, + "trace_precision": 3, + "trace_runtime": 0, + "train_first_n": 0, + "train_random_n": 0, + "train_uniform_n": 0, + "tree_encoding": "nested", + "tuner_caret_search_algorithm": "random", + "tuner_caret_tuner_verbose": true, + "use_model": "true", + "vacuity_check": true +} \ No newline at end of file diff --git a/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_responses_scaler.pkl b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_responses_scaler.pkl new file mode 100644 index 00000000..f2576be2 Binary files /dev/null and b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_responses_scaler.pkl differ diff --git a/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled.txt b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled.txt new file mode 100644 index 00000000..5a2a920f --- /dev/null +++ b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled.txt @@ -0,0 +1,392 @@ + +smlp_logger - INFO - Model exploration specification: +{'version': '1.1', 'spec': [{'label': 'y1', 'type': 'response', 'range': 'float'}, {'label': 'y2', 'type': 'response', 'range': 'float'}, {'label': 'x0', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x1', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x2', 'type': 'input', 'range': 'float', 'bounds': [3, 7]}], 'alpha': 'x1==1 or x1==4 or x1==7'} + +smlp_logger - INFO - Executing run_smlp.py script: Start + +smlp_logger - INFO - Running SMLP in mode "verify": Start + +smlp_logger - INFO - Computed spec global constraint expressions: + +smlp_logger - INFO - Global alpha : x1==1 or x1==4 or x1==7 + +smlp_logger - INFO - Global beta : None + +smlp_logger - INFO - Radii theta : {} + +smlp_logger - INFO - Delta const : {'delta_abs': 0.0, 'delta_rel': 0.01} + +smlp_logger - INFO - Assertion asrt1: (y2**3+x2)/2<6 + +smlp_logger - INFO - Assertion asrt2: y1>=9 + +smlp_logger - INFO - Assertion asrt3: y2<0 + +smlp_logger - INFO - PREPARE DATA FOR MODELING + +smlp_logger - INFO - Preparing training data for modeling: start + +smlp_logger - INFO - loading training data + +smlp_logger - INFO - data summary + y1 y2 x0 x1 x2 +count 11.000000 11.000000 10.000000 10.000000 11.000000 +mean 6.818182 6.818182 10.400000 2.800000 5.454545 +std 2.088932 2.088932 1.074968 0.788811 1.694912 +min 5.000000 5.000000 9.000000 2.000000 3.000000 +25% 5.000000 5.000000 10.000000 2.000000 4.000000 +50% 5.000000 5.000000 10.000000 3.000000 6.000000 +75% 9.000000 9.000000 11.000000 3.000000 7.000000 +max 9.000000 9.000000 12.000000 4.000000 8.000000 + +smlp_logger - INFO - training data + categ y1 y2 x0 x1 x2 +0 c14 5 9 10.0 2.0 3 +1 c15 9 9 12.0 NaN 4 +2 c1 5 9 NaN 3.0 4 +3 c9 5 5 11.0 2.0 6 +4 c5 9 5 10.0 2.0 8 +5 c10 9 9 9.0 4.0 7 +6 c13 5 5 9.0 3.0 6 +7 c4 5 5 10.0 3.0 4 +8 c15 9 9 11.0 4.0 4 +9 c11 5 5 12.0 2.0 7 +10 c19 9 5 10.0 3.0 7 + +smlp_logger - INFO - training data after imputing missing values + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - training data after processing responses + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - MRMR feature selection for response y1 : start + +smlp_logger - INFO - MRMR selected feature scores (in the ranked order) for response y1 : + Feature Score +0 x1 1.115702 +1 x2 0.944056 + +smlp_logger - INFO - MRMR feature selection for response y1 : end + +smlp_logger - INFO - MRMR feature selection for response y2 : start + +smlp_logger - INFO - MRMR selected feature scores (in the ranked order) for response y2 : + Feature Score +1 x2 4.950294 +0 x1 1.115702 + +smlp_logger - INFO - MRMR feature selection for response y2 : end + +smlp_logger - INFO - training data after encoding levels of categorical features with integers + x1 x2 y1 y2 +0 2.0 3 5 9 +1 2.0 4 9 9 +2 3.0 4 5 9 +3 2.0 6 5 5 +4 2.0 8 9 5 +5 4.0 7 9 9 +6 3.0 6 5 5 +7 3.0 4 5 5 +8 4.0 4 9 9 +9 2.0 7 5 5 +10 3.0 7 9 5 + +smlp_logger - INFO - training data after scaling (normalizing) features and responses + x1 x2 y1 y2 +0 0.0 0.0 0.0 1.0 +1 0.0 0.2 1.0 1.0 +2 0.5 0.2 0.0 1.0 +3 0.0 0.6 0.0 0.0 +4 0.0 1.0 1.0 0.0 +5 1.0 0.8 1.0 1.0 +6 0.5 0.6 0.0 0.0 +7 0.5 0.2 0.0 0.0 +8 1.0 0.2 1.0 1.0 +9 0.0 0.8 0.0 0.0 +10 0.5 0.8 1.0 0.0 + +smlp_logger - INFO - Sampling from training data: start + +smlp_logger - INFO - Sampling from training data: end + +smlp_logger - INFO - X_train after sampling: (8, 2) + +smlp_logger - INFO - y_train after sampling: (8, 2) + +smlp_logger - INFO - Preparing training data for modeling: end + +smlp_logger - INFO - Saving data bounds into file:./Test139_smlp_toy_num_resp_noknobs_data_bounds.json + +smlp_logger - INFO - {'x1': {'min': 2.0, 'max': 4.0}, 'x2': {'min': 3.0, 'max': 8.0}, 'y1': {'min': 5.0, 'max': 9.0}, 'y2': {'min': 5.0, 'max': 9.0}} + +smlp_logger - INFO - Preparing new data for modeling: start + +smlp_logger - INFO - loading new data + +smlp_logger - INFO - data summary + y1 y2 x0 x1 x2 +count 9.000000 9.000000 8.000000 8.000000 9.000000 +mean 6.777778 7.222222 10.250000 2.875000 5.111111 +std 2.108185 2.108185 1.035098 0.834523 1.691482 +min 5.000000 5.000000 9.000000 2.000000 3.000000 +25% 5.000000 5.000000 9.750000 2.000000 4.000000 +50% 5.000000 9.000000 10.000000 3.000000 4.000000 +75% 9.000000 9.000000 11.000000 3.250000 6.000000 +max 9.000000 9.000000 12.000000 4.000000 8.000000 + +smlp_logger - INFO - new data + categ y1 y2 x0 x1 x2 +0 c0 5 9 10.0 2.0 3 +1 c12 9 9 12.0 NaN 4 +2 c2 5 9 NaN 3.0 4 +3 c17 5 5 11.0 2.0 6 +4 c18 9 5 10.0 2.0 8 +5 c8 9 9 9.0 4.0 7 +6 c7 5 5 9.0 3.0 6 +7 c3 5 5 10.0 3.0 4 +8 c12 9 9 11.0 4.0 4 + +smlp_logger - INFO - new data after imputing missing values + x1 x2 y1 y2 +0 2.0 3 5 9 +1 2.0 4 9 9 +2 3.0 4 5 9 +3 2.0 6 5 5 +4 2.0 8 9 5 +5 4.0 7 9 9 +6 3.0 6 5 5 +7 3.0 4 5 5 +8 4.0 4 9 9 + +smlp_logger - INFO - new data after processing responses + x1 x2 y1 y2 +0 2.0 3 5 9 +1 2.0 4 9 9 +2 3.0 4 5 9 +3 2.0 6 5 5 +4 2.0 8 9 5 +5 4.0 7 9 9 +6 3.0 6 5 5 +7 3.0 4 5 5 +8 4.0 4 9 9 + +smlp_logger - INFO - new data after encoding levels of categorical features with integers + x1 x2 y1 y2 +0 2.0 3 5 9 +1 2.0 4 9 9 +2 3.0 4 5 9 +3 2.0 6 5 5 +4 2.0 8 9 5 +5 4.0 7 9 9 +6 3.0 6 5 5 +7 3.0 4 5 5 +8 4.0 4 9 9 + +smlp_logger - INFO - new data after scaling (normalizing) features and responses + x1 x2 y1 y2 +0 0.0 0.0 0.0 1.0 +1 0.0 0.2 1.0 1.0 +2 0.5 0.2 0.0 1.0 +3 0.0 0.6 0.0 0.0 +4 0.0 1.0 1.0 0.0 +5 1.0 0.8 1.0 1.0 +6 0.5 0.6 0.0 0.0 +7 0.5 0.2 0.0 0.0 +8 1.0 0.2 1.0 1.0 + +smlp_logger - INFO - Preparing new data for modeling: end + +smlp_logger - INFO - TRAIN MODEL + +smlp_logger - INFO - Model training: start + +smlp_logger - INFO - Writing tree rules into file ./Test139_smlp_toy_num_resp_noknobs_dt_sklearn_y1_tree_rules.txt + +smlp_logger - INFO - Writing tree rules into file ./Test139_smlp_toy_num_resp_noknobs_dt_sklearn_y2_tree_rules.txt + +smlp_logger - INFO - Model training: end + +smlp_logger - INFO - Seving model in file ./Test139_smlp_toy_num_resp_noknobs_dt_sklearn_model_complete.pkl + +smlp_logger - INFO - PREDICT ON TRAINING DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv + +smlp_logger - INFO - Prediction on training data -- msqe: 0.000 + +smlp_logger - INFO - Prediction on training data -- r2_score: 1.000 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON TEST DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv + +smlp_logger - INFO - Prediction on test data -- msqe: 8.000 + +smlp_logger - INFO - Prediction on test data -- r2_score: -1.250 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON LABELED DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv + +smlp_logger - INFO - Prediction on labeled data -- msqe: 2.182 + +smlp_logger - INFO - Prediction on labeled data -- r2_score: 0.450 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON NEW DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv + +smlp_logger - INFO - Prediction on new data -- msqe: 2.667 + +smlp_logger - INFO - Prediction on new data -- r2_score: 0.325 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - Creating model exploration base components: Start + +smlp_logger - INFO - Parsing the SPEC: Start + +smlp_logger - INFO - Parsing the SPEC: End + +smlp_logger - INFO - Variable domains (alpha): {'y1': {'range': 'float', 'interval': None}, 'y2': {'range': 'float', 'interval': None}, 'x0': {'range': 'float', 'interval': [0, 10]}, 'x1': {'range': 'float', 'interval': [0, 10]}, 'x2': {'range': 'float', 'interval': [3, 7]}} + +smlp_logger - INFO - Input bounds (alpha): {'x0': {'min': 0, 'max': 10}, 'x1': {'min': 0, 'max': 10}, 'x2': {'min': 3, 'max': 7}} + +smlp_logger - INFO - Knob bounds (eta): {} + +smlp_logger - INFO - Knob grids (eta): {} + +smlp_logger - INFO - Alpha global constraints: (or (or (= x1 1) (= x1 4)) (= x1 7)) + +smlp_logger - INFO - Alpha ranges constraints: (and (and true (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) + +smlp_logger - INFO - Alpha combined constraints: (and (and (and true (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) (or (or (= x1 1) (= x1 4)) (= x1 7))) + +smlp_logger - INFO - Beta global constraints: true + +smlp_logger - INFO - Eta ranges constraints: true + +smlp_logger - INFO - Eta grid constraints: true + +smlp_logger - INFO - Eta global constraints: true + +smlp_logger - INFO - Eta combined constraints: true + +smlp_logger - INFO - Creating model exploration base components: End + +smlp_logger - INFO - Input and knob interface constraints are consistent + +smlp_logger - INFO - Building model terms: Start + +smlp_logger - INFO - Model operator counts for y1: {'add': 1, 'mul': 15, 'ite': 5, 'and': 9, 'prop': 14, 'const': 50, 'sub': 14, 'var': 14} + +smlp_logger - INFO - Model operator counts for y2: {'add': 1, 'mul': 4, 'ite': 2, 'and': 1, 'prop': 3, 'const': 14, 'sub': 3, 'var': 3} + +smlp_logger - INFO - Building model terms: End + +smlp_logger - INFO - Model interface constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt1: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt2: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt3: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying assertion asrt1 <-> (y2**3+x2)/2<6 + +smlp_logger - INFO - The configuration is inconsistent with assertion asrt1 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Verifying assertion asrt2 <-> y1>=9 + +smlp_logger - INFO - The configuration is consistent with assertion asrt2 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Verifying assertion asrt3 <-> y2<0 + +smlp_logger - INFO - The configuration is inconsistent with assertion asrt3 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Running SMLP in mode "verify": End + +smlp_logger - INFO - Executing run_smlp.py script: End diff --git a/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv new file mode 100644 index 00000000..7d143a2f --- /dev/null +++ b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,2.909090909090909,0.2666666666666666 +y2,1.4545454545454546,0.6333333333333333 diff --git a/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv new file mode 100644 index 00000000..7a649b6a --- /dev/null +++ b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv @@ -0,0 +1,12 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +0,5.0,9.0,5.0,9.0 +1,9.0,9.0,9.0,9.0 +2,5.0,9.0,9.0,9.0 +3,5.0,5.0,5.0,5.0 +4,9.0,5.0,9.0,5.0 +5,9.0,9.0,9.0,9.0 +6,5.0,5.0,5.0,5.0 +7,5.0,5.0,9.0,9.0 +8,9.0,9.0,9.0,9.0 +9,5.0,5.0,5.0,5.0 +10,9.0,5.0,9.0,5.0 diff --git a/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json new file mode 100644 index 00000000..6e4f6879 --- /dev/null +++ b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json @@ -0,0 +1,5 @@ +{ + "x1": [ + 1 + ] +} \ No newline at end of file diff --git a/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv new file mode 100644 index 00000000..951115f0 --- /dev/null +++ b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,3.5555555555555554,0.10000000000000009 +y2,1.7777777777777777,0.55 diff --git a/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv new file mode 100644 index 00000000..3ba8f5c9 --- /dev/null +++ b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv @@ -0,0 +1,10 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +0,5.0,9.0,5.0,9.0 +1,9.0,9.0,9.0,9.0 +2,5.0,9.0,9.0,9.0 +3,5.0,5.0,5.0,5.0 +4,9.0,5.0,9.0,5.0 +5,9.0,9.0,9.0,9.0 +6,5.0,5.0,5.0,5.0 +7,5.0,5.0,9.0,9.0 +8,9.0,9.0,9.0,9.0 diff --git a/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv new file mode 100644 index 00000000..fd947921 --- /dev/null +++ b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,10.666666666666666,-2.0 +y2,5.333333333333333,-0.5 diff --git a/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv new file mode 100644 index 00000000..3bb69252 --- /dev/null +++ b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv @@ -0,0 +1,4 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +7,5.0,5.0,9.0,9.0 +2,5.0,9.0,9.0,9.0 +8,9.0,9.0,9.0,9.0 diff --git a/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv new file mode 100644 index 00000000..12103ffa --- /dev/null +++ b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv @@ -0,0 +1,12 @@ +stage,solver,x0,x1,x2,y0,y1 +interface_consistency,sat,7.0,3.0 +model_consistency,sat,1.0,7.0,5.0,5.0 +witness_consistency,sat,1.0,7.0,5.0,5.0 +witness_consistency,sat,1.0,7.0,5.0,5.0 +witness_consistency,sat,1.0,7.0,5.0,5.0 +ca,unsat +ce,sat,4.0,6.0,5.0,9.0 +ca,sat,7.0,6.75,9.0,9.0 +ce,sat,1.0,7.0,5.0,5.0 +ca,unsat +ce,sat,1.0,7.0,5.0,5.0 diff --git a/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv new file mode 100644 index 00000000..f0c58f67 --- /dev/null +++ b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,0.0,1.0 +y2,0.0,1.0 diff --git a/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv new file mode 100644 index 00000000..74d164ca --- /dev/null +++ b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv @@ -0,0 +1,9 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +3,5.0,5.0,5.0,5.0 +4,9.0,5.0,9.0,5.0 +5,9.0,9.0,9.0,9.0 +0,5.0,9.0,5.0,9.0 +10,9.0,5.0,9.0,5.0 +9,5.0,5.0,5.0,5.0 +6,5.0,5.0,5.0,5.0 +1,9.0,9.0,9.0,9.0 diff --git a/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json new file mode 100644 index 00000000..10878543 --- /dev/null +++ b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json @@ -0,0 +1,38 @@ +{ + "asrt1": { + "configuration_consistent": "true", + "assertion_status": "FAIL", + "counter_example": { + "x1": 4.0, + "y1": 5.0, + "x2": 6.0, + "y2": 9.0 + }, + "assertion_feasible": false + }, + "asrt2": { + "configuration_consistent": "true", + "assertion_status": "FAIL", + "counter_example": { + "x1": 1.0, + "y1": 5.0, + "x2": 7.0, + "y2": 5.0 + }, + "assertion_feasible": true + }, + "asrt3": { + "configuration_consistent": "true", + "assertion_status": "FAIL", + "counter_example": { + "x1": 1.0, + "y1": 5.0, + "x2": 7.0, + "y2": 5.0 + }, + "assertion_feasible": false + }, + "smlp_execution": "completed", + "interface_consistent": "true", + "model_consistent": "true" +} \ No newline at end of file diff --git a/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_y1_smlp_full_model_term.json b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_y1_smlp_full_model_term.json new file mode 100644 index 00000000..3c7d4f33 --- /dev/null +++ b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_y1_smlp_full_model_term.json @@ -0,0 +1 @@ +"{'y1': |:0| (/ 13421773 134217728))) 1 (ite (and (and (> |:0| (/ 23488103 33554432)) (<= |:1| (/ 1 4))) (<= |:0| (/ 30198989 33554432))) 0 (ite (and (and (> |:0| (/ 23488103 33554432)) (<= |:1| (/ 1 4))) (> |:0| (/ 30198989 33554432))) 1 (ite (and (<= |:0| (/ 23488103 33554432)) (> |:0| (/ 53687093 134217728))) 0 1))))) 4) 5)))>}" \ No newline at end of file diff --git a/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_y1_smlp_model_term.json b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_y1_smlp_model_term.json new file mode 100644 index 00000000..d4363137 --- /dev/null +++ b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_y1_smlp_model_term.json @@ -0,0 +1 @@ +"{'y1_scaled': x2_scaled (/ 13421773 134217728))) 1 (ite (and (and (> x2_scaled (/ 23488103 33554432)) (<= x1_scaled (/ 1 4))) (<= x2_scaled (/ 30198989 33554432))) 0 (ite (and (and (> x2_scaled (/ 23488103 33554432)) (<= x1_scaled (/ 1 4))) (> x2_scaled (/ 30198989 33554432))) 1 (ite (and (<= x2_scaled (/ 23488103 33554432)) (> x2_scaled (/ 53687093 134217728))) 0 1)))))>}" \ No newline at end of file diff --git a/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_y2_smlp_full_model_term.json b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_y2_smlp_full_model_term.json new file mode 100644 index 00000000..cc8b6220 --- /dev/null +++ b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_y2_smlp_full_model_term.json @@ -0,0 +1 @@ +"{'y2': |:0| (/ 53687093 134217728)) (> (* (/ 1 2) (- x1 2)) (/ 3 4))) 1 (ite (<= |:0| (/ 53687093 134217728)) 1 0)) 4) 5))>}" \ No newline at end of file diff --git a/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_y2_smlp_model_term.json b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_y2_smlp_model_term.json new file mode 100644 index 00000000..56c332c4 --- /dev/null +++ b/regr_smlp/master/Test139_smlp_toy_num_resp_noknobs_y2_smlp_model_term.json @@ -0,0 +1 @@ +"{'y2_scaled': x2_scaled (/ 53687093 134217728)) (> x1_scaled (/ 3 4))) 1 (ite (<= x2_scaled (/ 53687093 134217728)) 1 0))>}" \ No newline at end of file diff --git a/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs.txt b/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs.txt new file mode 100644 index 00000000..adfed295 --- /dev/null +++ b/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs.txt @@ -0,0 +1,270 @@ + +smlp_logger - INFO - Model exploration specification: +{'version': '1.1', 'spec': [{'label': 'y1', 'type': 'response', 'range': 'float'}, {'label': 'y2', 'type': 'response', 'range': 'float'}, {'label': 'x0', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x1', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x2', 'type': 'input', 'range': 'float', 'bounds': [3, 7]}], 'alpha': 'x1==1 or x1==4 or x1==7'} + +smlp_logger - INFO - Executing run_smlp.py script: Start + +smlp_logger - INFO - Running SMLP in mode "verify": Start + +smlp_logger - INFO - Computed spec global constraint expressions: + +smlp_logger - INFO - Global alpha : x1==1 or x1==4 or x1==7 + +smlp_logger - INFO - Global beta : None + +smlp_logger - INFO - Radii theta : {} + +smlp_logger - INFO - Delta const : {'delta_abs': 0.0, 'delta_rel': 0.01} + +smlp_logger - INFO - Assertion asrt1: x0**2+y1>4.3 + +smlp_logger - INFO - Assertion asrt2: (y1+x2)/2<6 + +smlp_logger - INFO - PREPARE DATA FOR MODELING + +smlp_logger - INFO - Preparing training data for modeling: start + +smlp_logger - INFO - loading training data + +smlp_logger - INFO - data summary + y1 y2 x0 x1 x2 +count 11.000000 11.000000 10.000000 10.000000 11.000000 +mean 6.818182 6.818182 10.400000 2.800000 5.454545 +std 2.088932 2.088932 1.074968 0.788811 1.694912 +min 5.000000 5.000000 9.000000 2.000000 3.000000 +25% 5.000000 5.000000 10.000000 2.000000 4.000000 +50% 5.000000 5.000000 10.000000 3.000000 6.000000 +75% 9.000000 9.000000 11.000000 3.000000 7.000000 +max 9.000000 9.000000 12.000000 4.000000 8.000000 + +smlp_logger - INFO - training data + categ y1 y2 x0 x1 x2 +0 c14 5 9 10.0 2.0 3 +1 c15 9 9 12.0 NaN 4 +2 c1 5 9 NaN 3.0 4 +3 c9 5 5 11.0 2.0 6 +4 c5 9 5 10.0 2.0 8 +5 c10 9 9 9.0 4.0 7 +6 c13 5 5 9.0 3.0 6 +7 c4 5 5 10.0 3.0 4 +8 c15 9 9 11.0 4.0 4 +9 c11 5 5 12.0 2.0 7 +10 c19 9 5 10.0 3.0 7 + +smlp_logger - INFO - training data after imputing missing values + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - training data after processing responses + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - training data after encoding levels of categorical features with integers + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - training data after scaling (normalizing) features and responses + x0 x1 x2 y1 y2 +0 0.333333 0.0 0.0 0.0 1.0 +1 1.000000 0.0 0.2 1.0 1.0 +2 0.333333 0.5 0.2 0.0 1.0 +3 0.666667 0.0 0.6 0.0 0.0 +4 0.333333 0.0 1.0 1.0 0.0 +5 0.000000 1.0 0.8 1.0 1.0 +6 0.000000 0.5 0.6 0.0 0.0 +7 0.333333 0.5 0.2 0.0 0.0 +8 0.666667 1.0 0.2 1.0 1.0 +9 1.000000 0.0 0.8 0.0 0.0 +10 0.333333 0.5 0.8 1.0 0.0 + +smlp_logger - INFO - Sampling from training data: start + +smlp_logger - INFO - Sampling from training data: end + +smlp_logger - INFO - X_train after sampling: (8, 3) + +smlp_logger - INFO - y_train after sampling: (8, 2) + +smlp_logger - INFO - Preparing training data for modeling: end + +smlp_logger - INFO - Saving data bounds into file:./Test144_smlp_toy_num_resp_noknobs_data_bounds.json + +smlp_logger - INFO - {'x0': {'min': 9.0, 'max': 12.0}, 'x1': {'min': 2.0, 'max': 4.0}, 'x2': {'min': 3.0, 'max': 8.0}, 'y1': {'min': 5.0, 'max': 9.0}, 'y2': {'min': 5.0, 'max': 9.0}} + +smlp_logger - INFO - TRAIN MODEL + +smlp_logger - INFO - Model training: start + +smlp_logger - INFO - Writing tree rules into file ./Test144_smlp_toy_num_resp_noknobs_dt_sklearn_tree_rules.txt + +smlp_logger - INFO - Model training: end + +smlp_logger - INFO - PREDICT ON TRAINING DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test144_smlp_toy_num_resp_noknobs_training_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test144_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv + +smlp_logger - INFO - Prediction on training data -- msqe: 0.000 + +smlp_logger - INFO - Prediction on training data -- r2_score: 1.000 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON TEST DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test144_smlp_toy_num_resp_noknobs_test_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test144_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv + +smlp_logger - INFO - Prediction on test data -- msqe: 2.667 + +smlp_logger - INFO - Prediction on test data -- r2_score: 0.250 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON LABELED DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test144_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test144_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv + +smlp_logger - INFO - Prediction on labeled data -- msqe: 0.727 + +smlp_logger - INFO - Prediction on labeled data -- r2_score: 0.817 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - Creating model exploration base components: Start + +smlp_logger - INFO - Parsing the SPEC: Start + +smlp_logger - INFO - Parsing the SPEC: End + +smlp_logger - INFO - Variable domains (alpha): {'y1': {'range': 'float', 'interval': None}, 'y2': {'range': 'float', 'interval': None}, 'x0': {'range': 'float', 'interval': [0, 10]}, 'x1': {'range': 'float', 'interval': [0, 10]}, 'x2': {'range': 'float', 'interval': [3, 7]}} + +smlp_logger - INFO - Input bounds (alpha): {'x0': {'min': 0, 'max': 10}, 'x1': {'min': 0, 'max': 10}, 'x2': {'min': 3, 'max': 7}} + +smlp_logger - INFO - Knob bounds (eta): {} + +smlp_logger - INFO - Knob grids (eta): {} + +smlp_logger - INFO - Alpha global constraints: (or (or (= x1 1) (= x1 4)) (= x1 7)) + +smlp_logger - INFO - Alpha ranges constraints: (and (and (and true (and (>= x0 0) (<= x0 10))) (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) + +smlp_logger - INFO - Alpha combined constraints: (and (and (and (and true (and (>= x0 0) (<= x0 10))) (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) (or (or (= x1 1) (= x1 4)) (= x1 7))) + +smlp_logger - INFO - Beta global constraints: true + +smlp_logger - INFO - Eta ranges constraints: true + +smlp_logger - INFO - Eta grid constraints: true + +smlp_logger - INFO - Eta global constraints: true + +smlp_logger - INFO - Eta combined constraints: true + +smlp_logger - INFO - Creating model exploration base components: End + +smlp_logger - INFO - Input and knob interface constraints are consistent + +smlp_logger - INFO - Building model terms: Start + +smlp_logger - INFO - Tree rules (branches) antecedent compression statistics for response(s) y1_scaled,y2_scaled: + trees count in the model 1 + tree branches/rules count 6 + antecedent lengths before 17 + antecedent lengths after 16 + branch length counts before {3: 1, 4: 2, 2: 3} + branch length counts after {3: 2, 2: 3, 4: 1} + tree max depth before 4 + tree max depth after 4 + +smlp_logger - INFO - Model operator counts for y1: {'add': 1, 'mul': 14, 'ite': 5, 'and': 8, 'prop': 13, 'const': 47, 'sub': 13, 'var': 13} + +smlp_logger - INFO - Model operator counts for y2: {'add': 1, 'mul': 14, 'ite': 5, 'and': 8, 'prop': 13, 'const': 47, 'sub': 13, 'var': 13} + +smlp_logger - INFO - Building model terms: End + +smlp_logger - INFO - Model interface constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt1: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt2: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying assertion asrt1 <-> x0**2+y1>4.3 + +smlp_logger - INFO - The configuration is consistent with assertion asrt1 + +smlp_logger - INFO - Completed with result: PASS + +smlp_logger - INFO - Verifying assertion asrt2 <-> (y1+x2)/2<6 + +smlp_logger - INFO - The configuration is consistent with assertion asrt2 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Running SMLP in mode "verify": End + +smlp_logger - INFO - Executing run_smlp.py script: End diff --git a/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_data_bounds.json b/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_data_bounds.json new file mode 100644 index 00000000..5df59662 --- /dev/null +++ b/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_data_bounds.json @@ -0,0 +1,22 @@ +{ + "x0": { + "min": 9.0, + "max": 12.0 + }, + "x1": { + "min": 2.0, + "max": 4.0 + }, + "x2": { + "min": 3.0, + "max": 8.0 + }, + "y1": { + "min": 5.0, + "max": 9.0 + }, + "y2": { + "min": 5.0, + "max": 9.0 + } +} \ No newline at end of file diff --git a/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_dt_sklearn_tree_rules.txt b/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_dt_sklearn_tree_rules.txt new file mode 100644 index 00000000..c3f3f68c --- /dev/null +++ b/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_dt_sklearn_tree_rules.txt @@ -0,0 +1,10 @@ +#Forest semantics: majority vote +#Number of trees: 1 + +#TREE 0 +if (x2 > 0.4000000134110451) and (x1 <= 0.75) and (x0 > 0.5000000149011612) then (y1 = 0.0) and (y2 = 0.0) | based on 2 samples +if (x2 > 0.4000000134110451) and (x1 <= 0.75) and (x0 <= 0.5000000149011612) and (x2 > 0.7000000178813934) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +if (x2 > 0.4000000134110451) and (x1 > 0.75) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +if (x2 > 0.4000000134110451) and (x1 <= 0.75) and (x0 <= 0.5000000149011612) and (x2 <= 0.7000000178813934) then (y1 = 0.0) and (y2 = 0.0) | based on 1 samples +if (x2 <= 0.4000000134110451) and (x0 > 0.6666666716337204) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +if (x2 <= 0.4000000134110451) and (x0 <= 0.6666666716337204) then (y1 = 0.0) and (y2 = 1.0) | based on 1 samples diff --git a/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_features_scaler.pkl b/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_features_scaler.pkl new file mode 100644 index 00000000..d12c5399 Binary files /dev/null and b/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_features_scaler.pkl differ diff --git a/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv b/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv new file mode 100644 index 00000000..cf088bcd --- /dev/null +++ b/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,0.0,1.0 +y2,1.4545454545454546,0.6333333333333333 diff --git a/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv b/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv new file mode 100644 index 00000000..2cce12ec --- /dev/null +++ b/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv @@ -0,0 +1,12 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +0,5.0,9.0,5.0,9.0 +1,9.0,9.0,9.0,9.0 +2,5.0,9.0,5.0,9.0 +3,5.0,5.0,5.0,5.0 +4,9.0,5.0,9.0,5.0 +5,9.0,9.0,9.0,9.0 +6,5.0,5.0,5.0,5.0 +7,5.0,5.0,5.0,9.0 +8,9.0,9.0,9.0,9.0 +9,5.0,5.0,5.0,5.0 +10,9.0,5.0,9.0,5.0 diff --git a/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_missing_values_dict.json b/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_missing_values_dict.json new file mode 100644 index 00000000..553bfe33 --- /dev/null +++ b/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_missing_values_dict.json @@ -0,0 +1,8 @@ +{ + "x1": [ + 1 + ], + "x0": [ + 2 + ] +} \ No newline at end of file diff --git a/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_model_features_dict.json b/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_model_features_dict.json new file mode 100644 index 00000000..8e6a1c9a --- /dev/null +++ b/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_model_features_dict.json @@ -0,0 +1,12 @@ +{ + "y1": [ + "x0", + "x1", + "x2" + ], + "y2": [ + "x0", + "x1", + "x2" + ] +} \ No newline at end of file diff --git a/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_model_levels_dict.json b/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_model_levels_dict.json new file mode 100644 index 00000000..9e26dfee --- /dev/null +++ b/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_model_levels_dict.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_responses_scaler.pkl b/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_responses_scaler.pkl new file mode 100644 index 00000000..f2576be2 Binary files /dev/null and b/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_responses_scaler.pkl differ diff --git a/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_smlp_full_model_term.json b/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_smlp_full_model_term.json new file mode 100644 index 00000000..4724001e --- /dev/null +++ b/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_smlp_full_model_term.json @@ -0,0 +1 @@ +"{'y1': |:1| (/ 44739243 67108864))) 1 (ite (and (and (and (<= |:0| (/ 23488103 33554432)) (> |:0| (/ 53687093 134217728))) (<= |:2| (/ 3 4))) (<= |:1| (/ 33554433 67108864))) 0 (ite (and (> |:0| (/ 53687093 134217728)) (> |:2| (/ 3 4))) 1 (ite (and (and (> |:0| (/ 23488103 33554432)) (<= |:2| (/ 3 4))) (<= |:1| (/ 33554433 67108864))) 1 0))))) 4) 5))))>, 'y2': |:1| (/ 44739243 67108864))) 1 (ite (and (and (and (<= |:0| (/ 23488103 33554432)) (> |:0| (/ 53687093 134217728))) (<= |:2| (/ 3 4))) (<= |:1| (/ 33554433 67108864))) 0 (ite (and (> |:0| (/ 53687093 134217728)) (> |:2| (/ 3 4))) 1 (ite (and (and (> |:0| (/ 23488103 33554432)) (<= |:2| (/ 3 4))) (<= |:1| (/ 33554433 67108864))) 0 0))))) 4) 5))))>}" \ No newline at end of file diff --git a/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_smlp_model_term.json b/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_smlp_model_term.json new file mode 100644 index 00000000..452ad7e2 --- /dev/null +++ b/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_smlp_model_term.json @@ -0,0 +1 @@ +"{'y1_scaled': x0_scaled (/ 44739243 67108864))) 1 (ite (and (and (and (<= x2_scaled (/ 23488103 33554432)) (> x2_scaled (/ 53687093 134217728))) (<= x1_scaled (/ 3 4))) (<= x0_scaled (/ 33554433 67108864))) 0 (ite (and (> x2_scaled (/ 53687093 134217728)) (> x1_scaled (/ 3 4))) 1 (ite (and (and (> x2_scaled (/ 23488103 33554432)) (<= x1_scaled (/ 3 4))) (<= x0_scaled (/ 33554433 67108864))) 1 0)))))>, 'y2_scaled': x0_scaled (/ 44739243 67108864))) 1 (ite (and (and (and (<= x2_scaled (/ 23488103 33554432)) (> x2_scaled (/ 53687093 134217728))) (<= x1_scaled (/ 3 4))) (<= x0_scaled (/ 33554433 67108864))) 0 (ite (and (> x2_scaled (/ 53687093 134217728)) (> x1_scaled (/ 3 4))) 1 (ite (and (and (> x2_scaled (/ 23488103 33554432)) (<= x1_scaled (/ 3 4))) (<= x0_scaled (/ 33554433 67108864))) 0 0)))))>}" \ No newline at end of file diff --git a/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv b/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv new file mode 100644 index 00000000..ec56b74c --- /dev/null +++ b/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,0.0,1.0 +y2,5.333333333333333,-0.5 diff --git a/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_test_predictions_summary.csv b/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_test_predictions_summary.csv new file mode 100644 index 00000000..3477d51d --- /dev/null +++ b/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_test_predictions_summary.csv @@ -0,0 +1,4 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +7,5.0,5.0,5.0,9.0 +2,5.0,9.0,5.0,9.0 +8,9.0,9.0,9.0,9.0 diff --git a/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_trace.csv b/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_trace.csv new file mode 100644 index 00000000..4c2323a8 --- /dev/null +++ b/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_trace.csv @@ -0,0 +1,9 @@ +stage,solver,x0,x1,x2,y1,y2 +interface_consistency,sat,0,7,3 +model_consistency,sat,0,7,805306377/134217728,9,9 +witness_consistency,sat,0,7,805306377/134217728,9,9 +witness_consistency,sat,0,7,805306377/134217728,9,9 +ca,sat,1,1,27/4,9,5 +ce,unsat +ca,sat,0,1,1140850697/201326592,5,5 +ce,sat,0,4,805306377/134217728,9,9 diff --git a/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv b/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv new file mode 100644 index 00000000..f0c58f67 --- /dev/null +++ b/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,0.0,1.0 +y2,0.0,1.0 diff --git a/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_training_predictions_summary.csv b/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_training_predictions_summary.csv new file mode 100644 index 00000000..74d164ca --- /dev/null +++ b/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_training_predictions_summary.csv @@ -0,0 +1,9 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +3,5.0,5.0,5.0,5.0 +4,9.0,5.0,9.0,5.0 +5,9.0,9.0,9.0,9.0 +0,5.0,9.0,5.0,9.0 +10,9.0,5.0,9.0,5.0 +9,5.0,5.0,5.0,5.0 +6,5.0,5.0,5.0,5.0 +1,9.0,9.0,9.0,9.0 diff --git a/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_verify_results.json b/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_verify_results.json new file mode 100644 index 00000000..1579ae5d --- /dev/null +++ b/regr_smlp/master/Test144_smlp_toy_num_resp_noknobs_verify_results.json @@ -0,0 +1,23 @@ +{ + "asrt1": { + "configuration_consistent": "true", + "assertion_status": "PASS", + "counter_example": null, + "assertion_feasible": true + }, + "asrt2": { + "configuration_consistent": "true", + "assertion_status": "FAIL", + "counter_example": { + "x0": 0.0, + "x1": 4.0, + "y1": 9.0, + "x2": 6.000000067055225, + "y2": 9.0 + }, + "assertion_feasible": true + }, + "smlp_execution": "completed", + "interface_consistent": "true", + "model_consistent": "true" +} \ No newline at end of file diff --git a/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_data_bounds.json b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_data_bounds.json new file mode 100644 index 00000000..510fd80e --- /dev/null +++ b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_data_bounds.json @@ -0,0 +1,18 @@ +{ + "x1": { + "min": 2.0, + "max": 4.0 + }, + "x2": { + "min": 3.0, + "max": 8.0 + }, + "y1": { + "min": 5.0, + "max": 9.0 + }, + "y2": { + "min": 5.0, + "max": 9.0 + } +} \ No newline at end of file diff --git a/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_dt_sklearn_model_complete.pkl b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_dt_sklearn_model_complete.pkl new file mode 100644 index 00000000..9af00b41 Binary files /dev/null and b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_dt_sklearn_model_complete.pkl differ diff --git a/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_dt_sklearn_y1_tree_rules.txt b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_dt_sklearn_y1_tree_rules.txt new file mode 100644 index 00000000..9f2dbad2 --- /dev/null +++ b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_dt_sklearn_y1_tree_rules.txt @@ -0,0 +1,10 @@ +#Forest semantics: majority vote +#Number of trees: 1 + +#TREE 0 +if (x2 > 0.7000000178813934) and (x1 > 0.25) then (y1 = 1.0) | based on 2 samples +if (x2 <= 0.7000000178813934) and (x2 > 0.4000000134110451) then (y1 = 0.0) | based on 2 samples +if (x2 > 0.7000000178813934) and (x1 <= 0.25) and (x2 > 0.9000000059604645) then (y1 = 1.0) | based on 1 samples +if (x2 > 0.7000000178813934) and (x1 <= 0.25) and (x2 <= 0.9000000059604645) then (y1 = 0.0) | based on 1 samples +if (x2 <= 0.7000000178813934) and (x2 <= 0.4000000134110451) and (x2 > 0.10000000149011612) then (y1 = 1.0) | based on 1 samples +if (x2 <= 0.7000000178813934) and (x2 <= 0.4000000134110451) and (x2 <= 0.10000000149011612) then (y1 = 0.0) | based on 1 samples diff --git a/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_dt_sklearn_y2_tree_rules.txt b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_dt_sklearn_y2_tree_rules.txt new file mode 100644 index 00000000..5015336b --- /dev/null +++ b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_dt_sklearn_y2_tree_rules.txt @@ -0,0 +1,7 @@ +#Forest semantics: majority vote +#Number of trees: 1 + +#TREE 0 +if (x2 > 0.4000000134110451) and (x1 <= 0.75) then (y2 = 0.0) | based on 5 samples +if (x2 <= 0.4000000134110451) then (y2 = 1.0) | based on 2 samples +if (x2 > 0.4000000134110451) and (x1 > 0.75) then (y2 = 1.0) | based on 1 samples diff --git a/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_features_scaler.pkl b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_features_scaler.pkl new file mode 100644 index 00000000..62adbe18 Binary files /dev/null and b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_features_scaler.pkl differ diff --git a/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_model_features_dict.json b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_model_features_dict.json new file mode 100644 index 00000000..85782d17 --- /dev/null +++ b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_model_features_dict.json @@ -0,0 +1,10 @@ +{ + "y1": [ + "x1", + "x2" + ], + "y2": [ + "x1", + "x2" + ] +} \ No newline at end of file diff --git a/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_model_levels_dict.json b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_model_levels_dict.json new file mode 100644 index 00000000..9e26dfee --- /dev/null +++ b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_model_levels_dict.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_rerun_model_config.json b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_rerun_model_config.json new file mode 100644 index 00000000..dff2c3ba --- /dev/null +++ b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_rerun_model_config.json @@ -0,0 +1,172 @@ +{ + "alpha": null, + "analytics_mode": "verify", + "approximate_fractions": true, + "assertions_expressions": "(y2**3+x2)/2<6;y1>=9;y2<0", + "assertions_names": "asrt1,asrt2,asrt3", + "beta": null, + "center_offset": "0", + "compress_rules": false, + "continuous_correlation_estimators": [ + "pearson", + "spearman" + ], + "correlations_and_mutual_information": true, + "data_scaler": "min_max", + "delta_absolute": 0.0, + "delta_relative": 0.01, + "discretization_algo": "uniform", + "discretization_bins": 10, + "discretization_labels": true, + "discretization_type": "category", + "discretize_numeric_features": null, + "doe_algo": null, + "doe_box_behnken_centers": 1, + "doe_central_composite_alpha": "o", + "doe_central_composite_center": "2,2", + "doe_central_composite_face": "ccf", + "doe_design_resolution": null, + "doe_factor_level_ranges": null, + "doe_num_samples": null, + "doe_prob_distribution": "Normal", + "doe_spec_file": null, + "dt_sklearn_ccp_alpha": 0.0, + "dt_sklearn_criterion": "squared_error", + "dt_sklearn_max_depth": 15, + "dt_sklearn_max_features": null, + "dt_sklearn_max_leaf_nodes": null, + "dt_sklearn_min_impurity_decrease": 0.0, + "dt_sklearn_min_samples_leaf": 1, + "dt_sklearn_min_samples_split": 2, + "dt_sklearn_min_weight_fraction_leaf": 0.0, + "dt_sklearn_random_state": null, + "dt_sklearn_splitter": "best", + "epsilon": 0.05, + "et_sklearn_bootstrap": true, + "et_sklearn_ccp_alpha": 0.0, + "et_sklearn_criterion": "squared_error", + "et_sklearn_max_depth": null, + "et_sklearn_max_features": 1.0, + "et_sklearn_max_leaf_nodes": null, + "et_sklearn_max_samples": null, + "et_sklearn_min_impurity_decrease": 0.0, + "et_sklearn_min_samples_leaf": 1, + "et_sklearn_min_samples_split": 2, + "et_sklearn_min_weight_fraction_leaf": 0.0, + "et_sklearn_n_estimators": 100, + "et_sklearn_random_state": null, + "et_sklearn_verbose": 0, + "et_sklearn_warm_start": false, + "eta": null, + "features": "x0,x1,x2", + "fraction_precision": 64, + "impute_responses": false, + "interactive_plots": false, + "keep_features": [], + "labeled_data": null, + "lemma_precision": 0, + "load_configuration": null, + "log_files_prefix": null, + "log_level": "info", + "log_mode": "w", + "log_time": false, + "model": "dt_sklearn", + "model_caret_cross_validation": true, + "model_caret_return_train_score": false, + "model_caret_verbose": true, + "model_name": null, + "model_per_response": true, + "mrmr_feat_count_for_correlation": 15, + "mrmr_feat_count_for_prediction": 2, + "mutual_information_method": "normalized", + "negative_value": 0, + "new_data": null, + "nn_keras_batch_size": 200, + "nn_keras_batches_grid": null, + "nn_keras_epochs": 2000, + "nn_keras_hid_activation": "relu", + "nn_keras_layers": "2,1", + "nn_keras_layers_grid": null, + "nn_keras_learning_rate": 0.001, + "nn_keras_learning_rates_grid": null, + "nn_keras_loss_function": "mse", + "nn_keras_loss_functions_grid": null, + "nn_keras_metrics": [ + "mse" + ], + "nn_keras_optimizer": "adam", + "nn_keras_out_activation": "linear", + "nn_keras_sequential_api": true, + "nn_keras_tuner_algo": null, + "nn_keras_weights_precision": null, + "nnet_encoding": "nested", + "objectives_expressions": null, + "objectives_names": "None", + "optimization_strategy": "eager", + "optimize_pareto": true, + "output_directory": "./", + "poly_sklearn_copy_X": true, + "poly_sklearn_degree": 2, + "poly_sklearn_fit_intercept": true, + "poly_sklearn_n_jobs": null, + "poly_sklearn_positive": false, + "positive_value": 1, + "prediction_plots": false, + "psg_max_dimension": 3, + "psg_quality_target": "Lift", + "psg_top_ranked": 15, + "query_expressions": null, + "query_names": "None", + "radius_absolute": null, + "radius_relative": null, + "response": "y1,y2", + "response_map": null, + "response_plots": false, + "response_to_bool": null, + "rf_sklearn_bootstrap": true, + "rf_sklearn_ccp_alpha": 0.0, + "rf_sklearn_criterion": "squared_error", + "rf_sklearn_max_depth": null, + "rf_sklearn_max_features": 1.0, + "rf_sklearn_max_leaf_nodes": null, + "rf_sklearn_max_samples": null, + "rf_sklearn_min_impurity_decrease": 0.0, + "rf_sklearn_min_samples_leaf": 1, + "rf_sklearn_min_samples_split": 2, + "rf_sklearn_min_weight_fraction_leaf": 0.0, + "rf_sklearn_n_estimators": 100, + "rf_sklearn_random_state": null, + "rf_sklearn_verbose": 0, + "rf_sklearn_warm_start": false, + "sample_weights_coef": 0, + "sample_weights_exponent": 0, + "sample_weights_intercept": 0, + "save_configuration": false, + "save_model": "false", + "save_model_rerun_configuration": true, + "scale_features": true, + "scale_objectives": true, + "scale_responses": true, + "seed": 10, + "setup_caret_data_split_shuffle": true, + "setup_caret_fold": 0, + "setup_caret_session_id": null, + "setup_caret_verbose": true, + "simplify_terms": false, + "solver": "z3", + "solver_logic": "ALL", + "solver_path": null, + "spec": "../specs/smlp_toy_num_resp_noknobs_verify.spec", + "split_test": 0.2, + "trace_anonymize": true, + "trace_precision": 3, + "trace_runtime": 0, + "train_first_n": 0, + "train_random_n": 0, + "train_uniform_n": 0, + "tree_encoding": "flat", + "tuner_caret_search_algorithm": "random", + "tuner_caret_tuner_verbose": true, + "use_model": "true", + "vacuity_check": true +} \ No newline at end of file diff --git a/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_responses_scaler.pkl b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_responses_scaler.pkl new file mode 100644 index 00000000..f2576be2 Binary files /dev/null and b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_responses_scaler.pkl differ diff --git a/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled.txt b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled.txt new file mode 100644 index 00000000..e586e999 --- /dev/null +++ b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled.txt @@ -0,0 +1,392 @@ + +smlp_logger - INFO - Model exploration specification: +{'version': '1.1', 'spec': [{'label': 'y1', 'type': 'response', 'range': 'float'}, {'label': 'y2', 'type': 'response', 'range': 'float'}, {'label': 'x0', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x1', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x2', 'type': 'input', 'range': 'float', 'bounds': [3, 7]}], 'alpha': 'x1==1 or x1==4 or x1==7'} + +smlp_logger - INFO - Executing run_smlp.py script: Start + +smlp_logger - INFO - Running SMLP in mode "verify": Start + +smlp_logger - INFO - Computed spec global constraint expressions: + +smlp_logger - INFO - Global alpha : x1==1 or x1==4 or x1==7 + +smlp_logger - INFO - Global beta : None + +smlp_logger - INFO - Radii theta : {} + +smlp_logger - INFO - Delta const : {'delta_abs': 0.0, 'delta_rel': 0.01} + +smlp_logger - INFO - Assertion asrt1: (y2**3+x2)/2<6 + +smlp_logger - INFO - Assertion asrt2: y1>=9 + +smlp_logger - INFO - Assertion asrt3: y2<0 + +smlp_logger - INFO - PREPARE DATA FOR MODELING + +smlp_logger - INFO - Preparing training data for modeling: start + +smlp_logger - INFO - loading training data + +smlp_logger - INFO - data summary + y1 y2 x0 x1 x2 +count 11.000000 11.000000 10.000000 10.000000 11.000000 +mean 6.818182 6.818182 10.400000 2.800000 5.454545 +std 2.088932 2.088932 1.074968 0.788811 1.694912 +min 5.000000 5.000000 9.000000 2.000000 3.000000 +25% 5.000000 5.000000 10.000000 2.000000 4.000000 +50% 5.000000 5.000000 10.000000 3.000000 6.000000 +75% 9.000000 9.000000 11.000000 3.000000 7.000000 +max 9.000000 9.000000 12.000000 4.000000 8.000000 + +smlp_logger - INFO - training data + categ y1 y2 x0 x1 x2 +0 c14 5 9 10.0 2.0 3 +1 c15 9 9 12.0 NaN 4 +2 c1 5 9 NaN 3.0 4 +3 c9 5 5 11.0 2.0 6 +4 c5 9 5 10.0 2.0 8 +5 c10 9 9 9.0 4.0 7 +6 c13 5 5 9.0 3.0 6 +7 c4 5 5 10.0 3.0 4 +8 c15 9 9 11.0 4.0 4 +9 c11 5 5 12.0 2.0 7 +10 c19 9 5 10.0 3.0 7 + +smlp_logger - INFO - training data after imputing missing values + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - training data after processing responses + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - MRMR feature selection for response y1 : start + +smlp_logger - INFO - MRMR selected feature scores (in the ranked order) for response y1 : + Feature Score +0 x1 1.115702 +1 x2 0.944056 + +smlp_logger - INFO - MRMR feature selection for response y1 : end + +smlp_logger - INFO - MRMR feature selection for response y2 : start + +smlp_logger - INFO - MRMR selected feature scores (in the ranked order) for response y2 : + Feature Score +1 x2 4.950294 +0 x1 1.115702 + +smlp_logger - INFO - MRMR feature selection for response y2 : end + +smlp_logger - INFO - training data after encoding levels of categorical features with integers + x1 x2 y1 y2 +0 2.0 3 5 9 +1 2.0 4 9 9 +2 3.0 4 5 9 +3 2.0 6 5 5 +4 2.0 8 9 5 +5 4.0 7 9 9 +6 3.0 6 5 5 +7 3.0 4 5 5 +8 4.0 4 9 9 +9 2.0 7 5 5 +10 3.0 7 9 5 + +smlp_logger - INFO - training data after scaling (normalizing) features and responses + x1 x2 y1 y2 +0 0.0 0.0 0.0 1.0 +1 0.0 0.2 1.0 1.0 +2 0.5 0.2 0.0 1.0 +3 0.0 0.6 0.0 0.0 +4 0.0 1.0 1.0 0.0 +5 1.0 0.8 1.0 1.0 +6 0.5 0.6 0.0 0.0 +7 0.5 0.2 0.0 0.0 +8 1.0 0.2 1.0 1.0 +9 0.0 0.8 0.0 0.0 +10 0.5 0.8 1.0 0.0 + +smlp_logger - INFO - Sampling from training data: start + +smlp_logger - INFO - Sampling from training data: end + +smlp_logger - INFO - X_train after sampling: (8, 2) + +smlp_logger - INFO - y_train after sampling: (8, 2) + +smlp_logger - INFO - Preparing training data for modeling: end + +smlp_logger - INFO - Saving data bounds into file:./Test162_smlp_toy_num_resp_noknobs_data_bounds.json + +smlp_logger - INFO - {'x1': {'min': 2.0, 'max': 4.0}, 'x2': {'min': 3.0, 'max': 8.0}, 'y1': {'min': 5.0, 'max': 9.0}, 'y2': {'min': 5.0, 'max': 9.0}} + +smlp_logger - INFO - Preparing new data for modeling: start + +smlp_logger - INFO - loading new data + +smlp_logger - INFO - data summary + y1 y2 x0 x1 x2 +count 9.000000 9.000000 8.000000 8.000000 9.000000 +mean 6.777778 7.222222 10.250000 2.875000 5.111111 +std 2.108185 2.108185 1.035098 0.834523 1.691482 +min 5.000000 5.000000 9.000000 2.000000 3.000000 +25% 5.000000 5.000000 9.750000 2.000000 4.000000 +50% 5.000000 9.000000 10.000000 3.000000 4.000000 +75% 9.000000 9.000000 11.000000 3.250000 6.000000 +max 9.000000 9.000000 12.000000 4.000000 8.000000 + +smlp_logger - INFO - new data + categ y1 y2 x0 x1 x2 +0 c0 5 9 10.0 2.0 3 +1 c12 9 9 12.0 NaN 4 +2 c2 5 9 NaN 3.0 4 +3 c17 5 5 11.0 2.0 6 +4 c18 9 5 10.0 2.0 8 +5 c8 9 9 9.0 4.0 7 +6 c7 5 5 9.0 3.0 6 +7 c3 5 5 10.0 3.0 4 +8 c12 9 9 11.0 4.0 4 + +smlp_logger - INFO - new data after imputing missing values + x1 x2 y1 y2 +0 2.0 3 5 9 +1 2.0 4 9 9 +2 3.0 4 5 9 +3 2.0 6 5 5 +4 2.0 8 9 5 +5 4.0 7 9 9 +6 3.0 6 5 5 +7 3.0 4 5 5 +8 4.0 4 9 9 + +smlp_logger - INFO - new data after processing responses + x1 x2 y1 y2 +0 2.0 3 5 9 +1 2.0 4 9 9 +2 3.0 4 5 9 +3 2.0 6 5 5 +4 2.0 8 9 5 +5 4.0 7 9 9 +6 3.0 6 5 5 +7 3.0 4 5 5 +8 4.0 4 9 9 + +smlp_logger - INFO - new data after encoding levels of categorical features with integers + x1 x2 y1 y2 +0 2.0 3 5 9 +1 2.0 4 9 9 +2 3.0 4 5 9 +3 2.0 6 5 5 +4 2.0 8 9 5 +5 4.0 7 9 9 +6 3.0 6 5 5 +7 3.0 4 5 5 +8 4.0 4 9 9 + +smlp_logger - INFO - new data after scaling (normalizing) features and responses + x1 x2 y1 y2 +0 0.0 0.0 0.0 1.0 +1 0.0 0.2 1.0 1.0 +2 0.5 0.2 0.0 1.0 +3 0.0 0.6 0.0 0.0 +4 0.0 1.0 1.0 0.0 +5 1.0 0.8 1.0 1.0 +6 0.5 0.6 0.0 0.0 +7 0.5 0.2 0.0 0.0 +8 1.0 0.2 1.0 1.0 + +smlp_logger - INFO - Preparing new data for modeling: end + +smlp_logger - INFO - TRAIN MODEL + +smlp_logger - INFO - Model training: start + +smlp_logger - INFO - Writing tree rules into file ./Test162_smlp_toy_num_resp_noknobs_dt_sklearn_y1_tree_rules.txt + +smlp_logger - INFO - Writing tree rules into file ./Test162_smlp_toy_num_resp_noknobs_dt_sklearn_y2_tree_rules.txt + +smlp_logger - INFO - Model training: end + +smlp_logger - INFO - Seving model in file ./Test162_smlp_toy_num_resp_noknobs_dt_sklearn_model_complete.pkl + +smlp_logger - INFO - PREDICT ON TRAINING DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv + +smlp_logger - INFO - Prediction on training data -- msqe: 0.000 + +smlp_logger - INFO - Prediction on training data -- r2_score: 1.000 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON TEST DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv + +smlp_logger - INFO - Prediction on test data -- msqe: 8.000 + +smlp_logger - INFO - Prediction on test data -- r2_score: -1.250 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON LABELED DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv + +smlp_logger - INFO - Prediction on labeled data -- msqe: 2.182 + +smlp_logger - INFO - Prediction on labeled data -- r2_score: 0.450 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON NEW DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv + +smlp_logger - INFO - Prediction on new data -- msqe: 2.667 + +smlp_logger - INFO - Prediction on new data -- r2_score: 0.325 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - Creating model exploration base components: Start + +smlp_logger - INFO - Parsing the SPEC: Start + +smlp_logger - INFO - Parsing the SPEC: End + +smlp_logger - INFO - Variable domains (alpha): {'y1': {'range': 'float', 'interval': None}, 'y2': {'range': 'float', 'interval': None}, 'x0': {'range': 'float', 'interval': [0, 10]}, 'x1': {'range': 'float', 'interval': [0, 10]}, 'x2': {'range': 'float', 'interval': [3, 7]}} + +smlp_logger - INFO - Input bounds (alpha): {'x0': {'min': 0, 'max': 10}, 'x1': {'min': 0, 'max': 10}, 'x2': {'min': 3, 'max': 7}} + +smlp_logger - INFO - Knob bounds (eta): {} + +smlp_logger - INFO - Knob grids (eta): {} + +smlp_logger - INFO - Alpha global constraints: (or (or (= x1 1) (= x1 4)) (= x1 7)) + +smlp_logger - INFO - Alpha ranges constraints: (and (and true (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) + +smlp_logger - INFO - Alpha combined constraints: (and (and (and true (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) (or (or (= x1 1) (= x1 4)) (= x1 7))) + +smlp_logger - INFO - Beta global constraints: true + +smlp_logger - INFO - Eta ranges constraints: true + +smlp_logger - INFO - Eta grid constraints: true + +smlp_logger - INFO - Eta global constraints: true + +smlp_logger - INFO - Eta combined constraints: true + +smlp_logger - INFO - Creating model exploration base components: End + +smlp_logger - INFO - Input and knob interface constraints are consistent + +smlp_logger - INFO - Building model terms: Start + +smlp_logger - INFO - Model operator counts for dt_sklearn_y1: {'or': 6, 'not': 6, 'and': 10, 'prop': 23, 'mul': 25, 'const': 71, 'sub': 24, 'var': 24} + +smlp_logger - INFO - Model operator counts for dt_sklearn_y2: {'or': 3, 'not': 3, 'and': 2, 'prop': 9, 'mul': 11, 'const': 29, 'sub': 10, 'var': 10} + +smlp_logger - INFO - Building model terms: End + +smlp_logger - INFO - Model interface constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt1: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt2: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt3: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying assertion asrt1 <-> (y2**3+x2)/2<6 + +smlp_logger - INFO - The configuration is inconsistent with assertion asrt1 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Verifying assertion asrt2 <-> y1>=9 + +smlp_logger - INFO - The configuration is consistent with assertion asrt2 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Verifying assertion asrt3 <-> y2<0 + +smlp_logger - INFO - The configuration is inconsistent with assertion asrt3 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Running SMLP in mode "verify": End + +smlp_logger - INFO - Executing run_smlp.py script: End diff --git a/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv new file mode 100644 index 00000000..7d143a2f --- /dev/null +++ b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,2.909090909090909,0.2666666666666666 +y2,1.4545454545454546,0.6333333333333333 diff --git a/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv new file mode 100644 index 00000000..7a649b6a --- /dev/null +++ b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv @@ -0,0 +1,12 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +0,5.0,9.0,5.0,9.0 +1,9.0,9.0,9.0,9.0 +2,5.0,9.0,9.0,9.0 +3,5.0,5.0,5.0,5.0 +4,9.0,5.0,9.0,5.0 +5,9.0,9.0,9.0,9.0 +6,5.0,5.0,5.0,5.0 +7,5.0,5.0,9.0,9.0 +8,9.0,9.0,9.0,9.0 +9,5.0,5.0,5.0,5.0 +10,9.0,5.0,9.0,5.0 diff --git a/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json new file mode 100644 index 00000000..6e4f6879 --- /dev/null +++ b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json @@ -0,0 +1,5 @@ +{ + "x1": [ + 1 + ] +} \ No newline at end of file diff --git a/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv new file mode 100644 index 00000000..951115f0 --- /dev/null +++ b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,3.5555555555555554,0.10000000000000009 +y2,1.7777777777777777,0.55 diff --git a/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv new file mode 100644 index 00000000..3ba8f5c9 --- /dev/null +++ b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv @@ -0,0 +1,10 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +0,5.0,9.0,5.0,9.0 +1,9.0,9.0,9.0,9.0 +2,5.0,9.0,9.0,9.0 +3,5.0,5.0,5.0,5.0 +4,9.0,5.0,9.0,5.0 +5,9.0,9.0,9.0,9.0 +6,5.0,5.0,5.0,5.0 +7,5.0,5.0,9.0,9.0 +8,9.0,9.0,9.0,9.0 diff --git a/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv new file mode 100644 index 00000000..fd947921 --- /dev/null +++ b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,10.666666666666666,-2.0 +y2,5.333333333333333,-0.5 diff --git a/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv new file mode 100644 index 00000000..3bb69252 --- /dev/null +++ b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv @@ -0,0 +1,4 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +7,5.0,5.0,9.0,9.0 +2,5.0,9.0,9.0,9.0 +8,9.0,9.0,9.0,9.0 diff --git a/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv new file mode 100644 index 00000000..48952dda --- /dev/null +++ b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv @@ -0,0 +1,12 @@ +stage,solver,x0,x1,x2,y0,y1 +interface_consistency,sat,7.0,3.0 +model_consistency,sat,7.0,6.0,5.0,9.0 +witness_consistency,sat,7.0,6.0,5.0,9.0 +witness_consistency,sat,7.0,6.0,5.0,9.0 +witness_consistency,sat,7.0,6.0,5.0,9.0 +ca,unsat +ce,sat,1.0,4.0,9.0,9.0 +ca,sat,7.0,6.75,9.0,9.0 +ce,sat,7.0,6.0,5.0,9.0 +ca,unsat +ce,sat,7.0,6.0,5.0,9.0 diff --git a/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv new file mode 100644 index 00000000..f0c58f67 --- /dev/null +++ b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,0.0,1.0 +y2,0.0,1.0 diff --git a/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv new file mode 100644 index 00000000..74d164ca --- /dev/null +++ b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv @@ -0,0 +1,9 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +3,5.0,5.0,5.0,5.0 +4,9.0,5.0,9.0,5.0 +5,9.0,9.0,9.0,9.0 +0,5.0,9.0,5.0,9.0 +10,9.0,5.0,9.0,5.0 +9,5.0,5.0,5.0,5.0 +6,5.0,5.0,5.0,5.0 +1,9.0,9.0,9.0,9.0 diff --git a/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json new file mode 100644 index 00000000..bd094f27 --- /dev/null +++ b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json @@ -0,0 +1,38 @@ +{ + "asrt1": { + "configuration_consistent": "true", + "assertion_status": "FAIL", + "counter_example": { + "x1": 1.0, + "y1": 9.0, + "x2": 4.0, + "y2": 9.0 + }, + "assertion_feasible": false + }, + "asrt2": { + "configuration_consistent": "true", + "assertion_status": "FAIL", + "counter_example": { + "x1": 7.0, + "y1": 5.0, + "x2": 6.000000067055225, + "y2": 9.0 + }, + "assertion_feasible": true + }, + "asrt3": { + "configuration_consistent": "true", + "assertion_status": "FAIL", + "counter_example": { + "x1": 7.0, + "y1": 5.0, + "x2": 6.000000067055225, + "y2": 9.0 + }, + "assertion_feasible": false + }, + "smlp_execution": "completed", + "interface_consistent": "true", + "model_consistent": "true" +} \ No newline at end of file diff --git a/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_y1_smlp_full_model_term.json b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_y1_smlp_full_model_term.json new file mode 100644 index 00000000..c5e5f9a9 --- /dev/null +++ b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_y1_smlp_full_model_term.json @@ -0,0 +1 @@ +"{'dt_sklearn_y1': [ (* (/ 1 5) (- x2 3)) (/ 23488103 33554432)) (> (* (/ 1 2) (- x1 2)) (/ 1 4)))) (= (* (/ 1 4) (- tree_0_y1 5)) 1))>, |:0| (/ 53687093 134217728)))) (= (* (/ 1 4) (- tree_0_y1 5)) 0)))>, |:0| (/ 23488103 33554432)) (<= (* (/ 1 2) (- x1 2)) (/ 1 4))) (> |:0| (/ 30198989 33554432)))) (= (* (/ 1 4) (- tree_0_y1 5)) 1)))>, |:0| (/ 23488103 33554432)) (<= (* (/ 1 2) (- x1 2)) (/ 1 4))) (<= |:0| (/ 30198989 33554432)))) (= (* (/ 1 4) (- tree_0_y1 5)) 0)))>, |:0| (/ 13421773 134217728)))) (= (* (/ 1 4) (- tree_0_y1 5)) 1)))>, , ]}" \ No newline at end of file diff --git a/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_y1_smlp_model_term.json b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_y1_smlp_model_term.json new file mode 100644 index 00000000..813118b7 --- /dev/null +++ b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_y1_smlp_model_term.json @@ -0,0 +1 @@ +"{'dt_sklearn_y1_scaled': [ x2_scaled (/ 23488103 33554432)) (> x1_scaled (/ 1 4)))) (= tree_0_y1_scaled 1))>, x2_scaled (/ 53687093 134217728)))) (= tree_0_y1_scaled 0))>, x2_scaled (/ 23488103 33554432)) (<= x1_scaled (/ 1 4))) (> x2_scaled (/ 30198989 33554432)))) (= tree_0_y1_scaled 1))>, x2_scaled (/ 23488103 33554432)) (<= x1_scaled (/ 1 4))) (<= x2_scaled (/ 30198989 33554432)))) (= tree_0_y1_scaled 0))>, x2_scaled (/ 13421773 134217728)))) (= tree_0_y1_scaled 1))>, , ]}" \ No newline at end of file diff --git a/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_y2_smlp_full_model_term.json b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_y2_smlp_full_model_term.json new file mode 100644 index 00000000..a7291f60 --- /dev/null +++ b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_y2_smlp_full_model_term.json @@ -0,0 +1 @@ +"{'dt_sklearn_y2': [ (* (/ 1 5) (- x2 3)) (/ 53687093 134217728)) (<= (* (/ 1 2) (- x1 2)) (/ 3 4)))) (= (* (/ 1 4) (- tree_0_y2 5)) 0))>, , (* (/ 1 5) (- x2 3)) (/ 53687093 134217728)) (> (* (/ 1 2) (- x1 2)) (/ 3 4)))) (= (* (/ 1 4) (- tree_0_y2 5)) 1))>, ]}" \ No newline at end of file diff --git a/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_y2_smlp_model_term.json b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_y2_smlp_model_term.json new file mode 100644 index 00000000..b51f2381 --- /dev/null +++ b/regr_smlp/master/Test162_smlp_toy_num_resp_noknobs_y2_smlp_model_term.json @@ -0,0 +1 @@ +"{'dt_sklearn_y2_scaled': [ x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4)))) (= tree_0_y2_scaled 0))>, , x2_scaled (/ 53687093 134217728)) (> x1_scaled (/ 3 4)))) (= tree_0_y2_scaled 1))>, ]}" \ No newline at end of file diff --git a/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_data_bounds.json b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_data_bounds.json new file mode 100644 index 00000000..510fd80e --- /dev/null +++ b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_data_bounds.json @@ -0,0 +1,18 @@ +{ + "x1": { + "min": 2.0, + "max": 4.0 + }, + "x2": { + "min": 3.0, + "max": 8.0 + }, + "y1": { + "min": 5.0, + "max": 9.0 + }, + "y2": { + "min": 5.0, + "max": 9.0 + } +} \ No newline at end of file diff --git a/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_dt_sklearn_model_complete.pkl b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_dt_sklearn_model_complete.pkl new file mode 100644 index 00000000..7cef3b59 Binary files /dev/null and b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_dt_sklearn_model_complete.pkl differ diff --git a/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_dt_sklearn_tree_rules.txt b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_dt_sklearn_tree_rules.txt new file mode 100644 index 00000000..59a1eff8 --- /dev/null +++ b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_dt_sklearn_tree_rules.txt @@ -0,0 +1,11 @@ +#Forest semantics: majority vote +#Number of trees: 1 + +#TREE 0 +if (x2 > 0.4000000134110451) and (x1 <= 0.75) and (x2 <= 0.7000000178813934) then (y1 = 0.0) and (y2 = 0.0) | based on 2 samples +if (x2 > 0.4000000134110451) and (x1 > 0.75) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +if (x2 > 0.4000000134110451) and (x1 <= 0.75) and (x2 > 0.7000000178813934) and (x1 > 0.25) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples +if (x2 > 0.4000000134110451) and (x1 <= 0.75) and (x2 > 0.7000000178813934) and (x1 <= 0.25) and (x2 > 0.9000000059604645) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples +if (x2 > 0.4000000134110451) and (x1 <= 0.75) and (x2 > 0.7000000178813934) and (x1 <= 0.25) and (x2 <= 0.9000000059604645) then (y1 = 0.0) and (y2 = 0.0) | based on 1 samples +if (x2 <= 0.4000000134110451) and (x2 > 0.10000000149011612) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +if (x2 <= 0.4000000134110451) and (x2 <= 0.10000000149011612) then (y1 = 0.0) and (y2 = 1.0) | based on 1 samples diff --git a/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_features_scaler.pkl b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_features_scaler.pkl new file mode 100644 index 00000000..62adbe18 Binary files /dev/null and b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_features_scaler.pkl differ diff --git a/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_model_features_dict.json b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_model_features_dict.json new file mode 100644 index 00000000..85782d17 --- /dev/null +++ b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_model_features_dict.json @@ -0,0 +1,10 @@ +{ + "y1": [ + "x1", + "x2" + ], + "y2": [ + "x1", + "x2" + ] +} \ No newline at end of file diff --git a/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_model_levels_dict.json b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_model_levels_dict.json new file mode 100644 index 00000000..9e26dfee --- /dev/null +++ b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_model_levels_dict.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_rerun_model_config.json b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_rerun_model_config.json new file mode 100644 index 00000000..af13eb36 --- /dev/null +++ b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_rerun_model_config.json @@ -0,0 +1,172 @@ +{ + "alpha": null, + "analytics_mode": "verify", + "approximate_fractions": true, + "assertions_expressions": "(y2**3+x2)/2<6;y1>=9;y2<0", + "assertions_names": "asrt1,asrt2,asrt3", + "beta": null, + "center_offset": "0", + "compress_rules": false, + "continuous_correlation_estimators": [ + "pearson", + "spearman" + ], + "correlations_and_mutual_information": true, + "data_scaler": "min_max", + "delta_absolute": 0.0, + "delta_relative": 0.01, + "discretization_algo": "uniform", + "discretization_bins": 10, + "discretization_labels": true, + "discretization_type": "category", + "discretize_numeric_features": null, + "doe_algo": null, + "doe_box_behnken_centers": 1, + "doe_central_composite_alpha": "o", + "doe_central_composite_center": "2,2", + "doe_central_composite_face": "ccf", + "doe_design_resolution": null, + "doe_factor_level_ranges": null, + "doe_num_samples": null, + "doe_prob_distribution": "Normal", + "doe_spec_file": null, + "dt_sklearn_ccp_alpha": 0.0, + "dt_sklearn_criterion": "squared_error", + "dt_sklearn_max_depth": 15, + "dt_sklearn_max_features": null, + "dt_sklearn_max_leaf_nodes": null, + "dt_sklearn_min_impurity_decrease": 0.0, + "dt_sklearn_min_samples_leaf": 1, + "dt_sklearn_min_samples_split": 2, + "dt_sklearn_min_weight_fraction_leaf": 0.0, + "dt_sklearn_random_state": null, + "dt_sklearn_splitter": "best", + "epsilon": 0.05, + "et_sklearn_bootstrap": true, + "et_sklearn_ccp_alpha": 0.0, + "et_sklearn_criterion": "squared_error", + "et_sklearn_max_depth": null, + "et_sklearn_max_features": 1.0, + "et_sklearn_max_leaf_nodes": null, + "et_sklearn_max_samples": null, + "et_sklearn_min_impurity_decrease": 0.0, + "et_sklearn_min_samples_leaf": 1, + "et_sklearn_min_samples_split": 2, + "et_sklearn_min_weight_fraction_leaf": 0.0, + "et_sklearn_n_estimators": 100, + "et_sklearn_random_state": null, + "et_sklearn_verbose": 0, + "et_sklearn_warm_start": false, + "eta": null, + "features": "x0,x1,x2", + "fraction_precision": 64, + "impute_responses": false, + "interactive_plots": false, + "keep_features": [], + "labeled_data": null, + "lemma_precision": 0, + "load_configuration": null, + "log_files_prefix": null, + "log_level": "info", + "log_mode": "w", + "log_time": false, + "model": "dt_sklearn", + "model_caret_cross_validation": true, + "model_caret_return_train_score": false, + "model_caret_verbose": true, + "model_name": null, + "model_per_response": false, + "mrmr_feat_count_for_correlation": 15, + "mrmr_feat_count_for_prediction": 2, + "mutual_information_method": "normalized", + "negative_value": 0, + "new_data": null, + "nn_keras_batch_size": 200, + "nn_keras_batches_grid": null, + "nn_keras_epochs": 2000, + "nn_keras_hid_activation": "relu", + "nn_keras_layers": "2,1", + "nn_keras_layers_grid": null, + "nn_keras_learning_rate": 0.001, + "nn_keras_learning_rates_grid": null, + "nn_keras_loss_function": "mse", + "nn_keras_loss_functions_grid": null, + "nn_keras_metrics": [ + "mse" + ], + "nn_keras_optimizer": "adam", + "nn_keras_out_activation": "linear", + "nn_keras_sequential_api": true, + "nn_keras_tuner_algo": null, + "nn_keras_weights_precision": null, + "nnet_encoding": "nested", + "objectives_expressions": null, + "objectives_names": "None", + "optimization_strategy": "eager", + "optimize_pareto": true, + "output_directory": "./", + "poly_sklearn_copy_X": true, + "poly_sklearn_degree": 2, + "poly_sklearn_fit_intercept": true, + "poly_sklearn_n_jobs": null, + "poly_sklearn_positive": false, + "positive_value": 1, + "prediction_plots": false, + "psg_max_dimension": 3, + "psg_quality_target": "Lift", + "psg_top_ranked": 15, + "query_expressions": null, + "query_names": "None", + "radius_absolute": null, + "radius_relative": null, + "response": "y1,y2", + "response_map": null, + "response_plots": false, + "response_to_bool": null, + "rf_sklearn_bootstrap": true, + "rf_sklearn_ccp_alpha": 0.0, + "rf_sklearn_criterion": "squared_error", + "rf_sklearn_max_depth": null, + "rf_sklearn_max_features": 1.0, + "rf_sklearn_max_leaf_nodes": null, + "rf_sklearn_max_samples": null, + "rf_sklearn_min_impurity_decrease": 0.0, + "rf_sklearn_min_samples_leaf": 1, + "rf_sklearn_min_samples_split": 2, + "rf_sklearn_min_weight_fraction_leaf": 0.0, + "rf_sklearn_n_estimators": 100, + "rf_sklearn_random_state": null, + "rf_sklearn_verbose": 0, + "rf_sklearn_warm_start": false, + "sample_weights_coef": 0, + "sample_weights_exponent": 0, + "sample_weights_intercept": 0, + "save_configuration": false, + "save_model": "false", + "save_model_rerun_configuration": true, + "scale_features": true, + "scale_objectives": true, + "scale_responses": true, + "seed": 10, + "setup_caret_data_split_shuffle": true, + "setup_caret_fold": 0, + "setup_caret_session_id": null, + "setup_caret_verbose": true, + "simplify_terms": false, + "solver": "z3", + "solver_logic": "ALL", + "solver_path": null, + "spec": "../specs/smlp_toy_num_resp_noknobs_verify.spec", + "split_test": 0.2, + "trace_anonymize": true, + "trace_precision": 3, + "trace_runtime": 0, + "train_first_n": 0, + "train_random_n": 0, + "train_uniform_n": 0, + "tree_encoding": "flat", + "tuner_caret_search_algorithm": "random", + "tuner_caret_tuner_verbose": true, + "use_model": "true", + "vacuity_check": true +} \ No newline at end of file diff --git a/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_responses_scaler.pkl b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_responses_scaler.pkl new file mode 100644 index 00000000..f2576be2 Binary files /dev/null and b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_responses_scaler.pkl differ diff --git a/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_full_model_term.json b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_full_model_term.json new file mode 100644 index 00000000..489dce69 --- /dev/null +++ b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_full_model_term.json @@ -0,0 +1 @@ +"{'dt_sklearn': [ |:0| (/ 53687093 134217728)) (<= (* (/ 1 2) (- x1 2)) (/ 3 4))) (<= |:0| (/ 23488103 33554432)))) (and (= (* (/ 1 4) (- tree_0_y1 5)) 0) (= (* (/ 1 4) (- tree_0_y2 5)) 0))))>, (* (/ 1 5) (- x2 3)) (/ 53687093 134217728)) (> (* (/ 1 2) (- x1 2)) (/ 3 4)))) (and (= (* (/ 1 4) (- tree_0_y1 5)) 1) (= (* (/ 1 4) (- tree_0_y2 5)) 1)))>, |:0| (/ 53687093 134217728)) (<= |:1| (/ 3 4))) (> |:0| (/ 23488103 33554432))) (> |:1| (/ 1 4)))) (and (= (* (/ 1 4) (- tree_0_y1 5)) 1) (= (* (/ 1 4) (- tree_0_y2 5)) 0)))))>, |:0| (/ 53687093 134217728)) (<= |:1| (/ 3 4))) (> |:0| (/ 23488103 33554432))) (<= |:1| (/ 1 4))) (> |:0| (/ 30198989 33554432)))) (and (= (* (/ 1 4) (- tree_0_y1 5)) 1) (= (* (/ 1 4) (- tree_0_y2 5)) 0)))))>, |:0| (/ 53687093 134217728)) (<= |:1| (/ 3 4))) (> |:0| (/ 23488103 33554432))) (<= |:1| (/ 1 4))) (<= |:0| (/ 30198989 33554432)))) (and (= (* (/ 1 4) (- tree_0_y1 5)) 0) (= (* (/ 1 4) (- tree_0_y2 5)) 0)))))>, |:0| (/ 13421773 134217728)))) (and (= (* (/ 1 4) (- tree_0_y1 5)) 1) (= (* (/ 1 4) (- tree_0_y2 5)) 1))))>, , , ]}" \ No newline at end of file diff --git a/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_model_term.json b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_model_term.json new file mode 100644 index 00000000..1104cd11 --- /dev/null +++ b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_model_term.json @@ -0,0 +1 @@ +"{'dt_sklearn': [ x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (<= x2_scaled (/ 23488103 33554432)))) (and (= tree_0_y1_scaled 0) (= tree_0_y2_scaled 0)))>, x2_scaled (/ 53687093 134217728)) (> x1_scaled (/ 3 4)))) (and (= tree_0_y1_scaled 1) (= tree_0_y2_scaled 1)))>, x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (> x2_scaled (/ 23488103 33554432))) (> x1_scaled (/ 1 4)))) (and (= tree_0_y1_scaled 1) (= tree_0_y2_scaled 0)))>, x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (> x2_scaled (/ 23488103 33554432))) (<= x1_scaled (/ 1 4))) (> x2_scaled (/ 30198989 33554432)))) (and (= tree_0_y1_scaled 1) (= tree_0_y2_scaled 0)))>, x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (> x2_scaled (/ 23488103 33554432))) (<= x1_scaled (/ 1 4))) (<= x2_scaled (/ 30198989 33554432)))) (and (= tree_0_y1_scaled 0) (= tree_0_y2_scaled 0)))>, x2_scaled (/ 13421773 134217728)))) (and (= tree_0_y1_scaled 1) (= tree_0_y2_scaled 1)))>, , , ]}" \ No newline at end of file diff --git a/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled.txt b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled.txt new file mode 100644 index 00000000..93b08c65 --- /dev/null +++ b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled.txt @@ -0,0 +1,388 @@ + +smlp_logger - INFO - Model exploration specification: +{'version': '1.1', 'spec': [{'label': 'y1', 'type': 'response', 'range': 'float'}, {'label': 'y2', 'type': 'response', 'range': 'float'}, {'label': 'x0', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x1', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x2', 'type': 'input', 'range': 'float', 'bounds': [3, 7]}], 'alpha': 'x1==1 or x1==4 or x1==7'} + +smlp_logger - INFO - Executing run_smlp.py script: Start + +smlp_logger - INFO - Running SMLP in mode "verify": Start + +smlp_logger - INFO - Computed spec global constraint expressions: + +smlp_logger - INFO - Global alpha : x1==1 or x1==4 or x1==7 + +smlp_logger - INFO - Global beta : None + +smlp_logger - INFO - Radii theta : {} + +smlp_logger - INFO - Delta const : {'delta_abs': 0.0, 'delta_rel': 0.01} + +smlp_logger - INFO - Assertion asrt1: (y2**3+x2)/2<6 + +smlp_logger - INFO - Assertion asrt2: y1>=9 + +smlp_logger - INFO - Assertion asrt3: y2<0 + +smlp_logger - INFO - PREPARE DATA FOR MODELING + +smlp_logger - INFO - Preparing training data for modeling: start + +smlp_logger - INFO - loading training data + +smlp_logger - INFO - data summary + y1 y2 x0 x1 x2 +count 11.000000 11.000000 10.000000 10.000000 11.000000 +mean 6.818182 6.818182 10.400000 2.800000 5.454545 +std 2.088932 2.088932 1.074968 0.788811 1.694912 +min 5.000000 5.000000 9.000000 2.000000 3.000000 +25% 5.000000 5.000000 10.000000 2.000000 4.000000 +50% 5.000000 5.000000 10.000000 3.000000 6.000000 +75% 9.000000 9.000000 11.000000 3.000000 7.000000 +max 9.000000 9.000000 12.000000 4.000000 8.000000 + +smlp_logger - INFO - training data + categ y1 y2 x0 x1 x2 +0 c14 5 9 10.0 2.0 3 +1 c15 9 9 12.0 NaN 4 +2 c1 5 9 NaN 3.0 4 +3 c9 5 5 11.0 2.0 6 +4 c5 9 5 10.0 2.0 8 +5 c10 9 9 9.0 4.0 7 +6 c13 5 5 9.0 3.0 6 +7 c4 5 5 10.0 3.0 4 +8 c15 9 9 11.0 4.0 4 +9 c11 5 5 12.0 2.0 7 +10 c19 9 5 10.0 3.0 7 + +smlp_logger - INFO - training data after imputing missing values + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - training data after processing responses + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - MRMR feature selection for response y1 : start + +smlp_logger - INFO - MRMR selected feature scores (in the ranked order) for response y1 : + Feature Score +0 x1 1.115702 +1 x2 0.944056 + +smlp_logger - INFO - MRMR feature selection for response y1 : end + +smlp_logger - INFO - MRMR feature selection for response y2 : start + +smlp_logger - INFO - MRMR selected feature scores (in the ranked order) for response y2 : + Feature Score +1 x2 4.950294 +0 x1 1.115702 + +smlp_logger - INFO - MRMR feature selection for response y2 : end + +smlp_logger - INFO - training data after encoding levels of categorical features with integers + x1 x2 y1 y2 +0 2.0 3 5 9 +1 2.0 4 9 9 +2 3.0 4 5 9 +3 2.0 6 5 5 +4 2.0 8 9 5 +5 4.0 7 9 9 +6 3.0 6 5 5 +7 3.0 4 5 5 +8 4.0 4 9 9 +9 2.0 7 5 5 +10 3.0 7 9 5 + +smlp_logger - INFO - training data after scaling (normalizing) features and responses + x1 x2 y1 y2 +0 0.0 0.0 0.0 1.0 +1 0.0 0.2 1.0 1.0 +2 0.5 0.2 0.0 1.0 +3 0.0 0.6 0.0 0.0 +4 0.0 1.0 1.0 0.0 +5 1.0 0.8 1.0 1.0 +6 0.5 0.6 0.0 0.0 +7 0.5 0.2 0.0 0.0 +8 1.0 0.2 1.0 1.0 +9 0.0 0.8 0.0 0.0 +10 0.5 0.8 1.0 0.0 + +smlp_logger - INFO - Sampling from training data: start + +smlp_logger - INFO - Sampling from training data: end + +smlp_logger - INFO - X_train after sampling: (8, 2) + +smlp_logger - INFO - y_train after sampling: (8, 2) + +smlp_logger - INFO - Preparing training data for modeling: end + +smlp_logger - INFO - Saving data bounds into file:./Test163_smlp_toy_num_resp_noknobs_data_bounds.json + +smlp_logger - INFO - {'x1': {'min': 2.0, 'max': 4.0}, 'x2': {'min': 3.0, 'max': 8.0}, 'y1': {'min': 5.0, 'max': 9.0}, 'y2': {'min': 5.0, 'max': 9.0}} + +smlp_logger - INFO - Preparing new data for modeling: start + +smlp_logger - INFO - loading new data + +smlp_logger - INFO - data summary + y1 y2 x0 x1 x2 +count 9.000000 9.000000 8.000000 8.000000 9.000000 +mean 6.777778 7.222222 10.250000 2.875000 5.111111 +std 2.108185 2.108185 1.035098 0.834523 1.691482 +min 5.000000 5.000000 9.000000 2.000000 3.000000 +25% 5.000000 5.000000 9.750000 2.000000 4.000000 +50% 5.000000 9.000000 10.000000 3.000000 4.000000 +75% 9.000000 9.000000 11.000000 3.250000 6.000000 +max 9.000000 9.000000 12.000000 4.000000 8.000000 + +smlp_logger - INFO - new data + categ y1 y2 x0 x1 x2 +0 c0 5 9 10.0 2.0 3 +1 c12 9 9 12.0 NaN 4 +2 c2 5 9 NaN 3.0 4 +3 c17 5 5 11.0 2.0 6 +4 c18 9 5 10.0 2.0 8 +5 c8 9 9 9.0 4.0 7 +6 c7 5 5 9.0 3.0 6 +7 c3 5 5 10.0 3.0 4 +8 c12 9 9 11.0 4.0 4 + +smlp_logger - INFO - new data after imputing missing values + x1 x2 y1 y2 +0 2.0 3 5 9 +1 2.0 4 9 9 +2 3.0 4 5 9 +3 2.0 6 5 5 +4 2.0 8 9 5 +5 4.0 7 9 9 +6 3.0 6 5 5 +7 3.0 4 5 5 +8 4.0 4 9 9 + +smlp_logger - INFO - new data after processing responses + x1 x2 y1 y2 +0 2.0 3 5 9 +1 2.0 4 9 9 +2 3.0 4 5 9 +3 2.0 6 5 5 +4 2.0 8 9 5 +5 4.0 7 9 9 +6 3.0 6 5 5 +7 3.0 4 5 5 +8 4.0 4 9 9 + +smlp_logger - INFO - new data after encoding levels of categorical features with integers + x1 x2 y1 y2 +0 2.0 3 5 9 +1 2.0 4 9 9 +2 3.0 4 5 9 +3 2.0 6 5 5 +4 2.0 8 9 5 +5 4.0 7 9 9 +6 3.0 6 5 5 +7 3.0 4 5 5 +8 4.0 4 9 9 + +smlp_logger - INFO - new data after scaling (normalizing) features and responses + x1 x2 y1 y2 +0 0.0 0.0 0.0 1.0 +1 0.0 0.2 1.0 1.0 +2 0.5 0.2 0.0 1.0 +3 0.0 0.6 0.0 0.0 +4 0.0 1.0 1.0 0.0 +5 1.0 0.8 1.0 1.0 +6 0.5 0.6 0.0 0.0 +7 0.5 0.2 0.0 0.0 +8 1.0 0.2 1.0 1.0 + +smlp_logger - INFO - Preparing new data for modeling: end + +smlp_logger - INFO - TRAIN MODEL + +smlp_logger - INFO - Model training: start + +smlp_logger - INFO - Writing tree rules into file ./Test163_smlp_toy_num_resp_noknobs_dt_sklearn_tree_rules.txt + +smlp_logger - INFO - Model training: end + +smlp_logger - INFO - Seving model in file ./Test163_smlp_toy_num_resp_noknobs_dt_sklearn_model_complete.pkl + +smlp_logger - INFO - PREDICT ON TRAINING DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv + +smlp_logger - INFO - Prediction on training data -- msqe: 0.000 + +smlp_logger - INFO - Prediction on training data -- r2_score: 1.000 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON TEST DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv + +smlp_logger - INFO - Prediction on test data -- msqe: 8.000 + +smlp_logger - INFO - Prediction on test data -- r2_score: -1.250 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON LABELED DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv + +smlp_logger - INFO - Prediction on labeled data -- msqe: 2.182 + +smlp_logger - INFO - Prediction on labeled data -- r2_score: 0.450 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON NEW DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv + +smlp_logger - INFO - Prediction on new data -- msqe: 2.667 + +smlp_logger - INFO - Prediction on new data -- r2_score: 0.325 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - Creating model exploration base components: Start + +smlp_logger - INFO - Parsing the SPEC: Start + +smlp_logger - INFO - Parsing the SPEC: End + +smlp_logger - INFO - Variable domains (alpha): {'y1': {'range': 'float', 'interval': None}, 'y2': {'range': 'float', 'interval': None}, 'x0': {'range': 'float', 'interval': [0, 10]}, 'x1': {'range': 'float', 'interval': [0, 10]}, 'x2': {'range': 'float', 'interval': [3, 7]}} + +smlp_logger - INFO - Input bounds (alpha): {'x0': {'min': 0, 'max': 10}, 'x1': {'min': 0, 'max': 10}, 'x2': {'min': 3, 'max': 7}} + +smlp_logger - INFO - Knob bounds (eta): {} + +smlp_logger - INFO - Knob grids (eta): {} + +smlp_logger - INFO - Alpha global constraints: (or (or (= x1 1) (= x1 4)) (= x1 7)) + +smlp_logger - INFO - Alpha ranges constraints: (and (and true (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) + +smlp_logger - INFO - Alpha combined constraints: (and (and (and true (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) (or (or (= x1 1) (= x1 4)) (= x1 7))) + +smlp_logger - INFO - Beta global constraints: true + +smlp_logger - INFO - Eta ranges constraints: true + +smlp_logger - INFO - Eta grid constraints: true + +smlp_logger - INFO - Eta global constraints: true + +smlp_logger - INFO - Eta combined constraints: true + +smlp_logger - INFO - Creating model exploration base components: End + +smlp_logger - INFO - Input and knob interface constraints are consistent + +smlp_logger - INFO - Building model terms: Start + +smlp_logger - INFO - Model operator counts for dt_sklearn: {'or': 7, 'not': 7, 'and': 23, 'prop': 39, 'mul': 43, 'const': 121, 'sub': 41, 'var': 41} + +smlp_logger - INFO - Building model terms: End + +smlp_logger - INFO - Model interface constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt1: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt2: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt3: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying assertion asrt1 <-> (y2**3+x2)/2<6 + +smlp_logger - INFO - The configuration is inconsistent with assertion asrt1 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Verifying assertion asrt2 <-> y1>=9 + +smlp_logger - INFO - The configuration is consistent with assertion asrt2 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Verifying assertion asrt3 <-> y2<0 + +smlp_logger - INFO - The configuration is inconsistent with assertion asrt3 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Running SMLP in mode "verify": End + +smlp_logger - INFO - Executing run_smlp.py script: End diff --git a/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv new file mode 100644 index 00000000..7d143a2f --- /dev/null +++ b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,2.909090909090909,0.2666666666666666 +y2,1.4545454545454546,0.6333333333333333 diff --git a/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv new file mode 100644 index 00000000..7a649b6a --- /dev/null +++ b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv @@ -0,0 +1,12 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +0,5.0,9.0,5.0,9.0 +1,9.0,9.0,9.0,9.0 +2,5.0,9.0,9.0,9.0 +3,5.0,5.0,5.0,5.0 +4,9.0,5.0,9.0,5.0 +5,9.0,9.0,9.0,9.0 +6,5.0,5.0,5.0,5.0 +7,5.0,5.0,9.0,9.0 +8,9.0,9.0,9.0,9.0 +9,5.0,5.0,5.0,5.0 +10,9.0,5.0,9.0,5.0 diff --git a/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json new file mode 100644 index 00000000..6e4f6879 --- /dev/null +++ b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json @@ -0,0 +1,5 @@ +{ + "x1": [ + 1 + ] +} \ No newline at end of file diff --git a/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv new file mode 100644 index 00000000..951115f0 --- /dev/null +++ b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,3.5555555555555554,0.10000000000000009 +y2,1.7777777777777777,0.55 diff --git a/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv new file mode 100644 index 00000000..3ba8f5c9 --- /dev/null +++ b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv @@ -0,0 +1,10 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +0,5.0,9.0,5.0,9.0 +1,9.0,9.0,9.0,9.0 +2,5.0,9.0,9.0,9.0 +3,5.0,5.0,5.0,5.0 +4,9.0,5.0,9.0,5.0 +5,9.0,9.0,9.0,9.0 +6,5.0,5.0,5.0,5.0 +7,5.0,5.0,9.0,9.0 +8,9.0,9.0,9.0,9.0 diff --git a/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv new file mode 100644 index 00000000..fd947921 --- /dev/null +++ b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,10.666666666666666,-2.0 +y2,5.333333333333333,-0.5 diff --git a/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv new file mode 100644 index 00000000..3bb69252 --- /dev/null +++ b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv @@ -0,0 +1,4 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +7,5.0,5.0,9.0,9.0 +2,5.0,9.0,9.0,9.0 +8,9.0,9.0,9.0,9.0 diff --git a/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv new file mode 100644 index 00000000..af6fe955 --- /dev/null +++ b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv @@ -0,0 +1,12 @@ +stage,solver,x0,x1,x2,y0,y1 +interface_consistency,sat,7.0,3.0 +model_consistency,sat,7.0,6.75,9.0,9.0 +witness_consistency,sat,7.0,6.75,9.0,9.0 +witness_consistency,sat,7.0,6.75,9.0,9.0 +witness_consistency,sat,7.0,6.75,9.0,9.0 +ca,unsat +ce,sat,1.0,4.0,9.0,9.0 +ca,sat,7.0,4.5,9.0,9.0 +ce,sat,7.0,3.0,5.0,9.0 +ca,unsat +ce,sat,7.0,3.0,5.0,9.0 diff --git a/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv new file mode 100644 index 00000000..f0c58f67 --- /dev/null +++ b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,0.0,1.0 +y2,0.0,1.0 diff --git a/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv new file mode 100644 index 00000000..74d164ca --- /dev/null +++ b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv @@ -0,0 +1,9 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +3,5.0,5.0,5.0,5.0 +4,9.0,5.0,9.0,5.0 +5,9.0,9.0,9.0,9.0 +0,5.0,9.0,5.0,9.0 +10,9.0,5.0,9.0,5.0 +9,5.0,5.0,5.0,5.0 +6,5.0,5.0,5.0,5.0 +1,9.0,9.0,9.0,9.0 diff --git a/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json new file mode 100644 index 00000000..48d869a3 --- /dev/null +++ b/regr_smlp/master/Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json @@ -0,0 +1,38 @@ +{ + "asrt1": { + "configuration_consistent": "true", + "assertion_status": "FAIL", + "counter_example": { + "x1": 1.0, + "y1": 9.0, + "x2": 4.0, + "y2": 9.0 + }, + "assertion_feasible": false + }, + "asrt2": { + "configuration_consistent": "true", + "assertion_status": "FAIL", + "counter_example": { + "x1": 7.0, + "y1": 5.0, + "x2": 3.0, + "y2": 9.0 + }, + "assertion_feasible": true + }, + "asrt3": { + "configuration_consistent": "true", + "assertion_status": "FAIL", + "counter_example": { + "x1": 7.0, + "y1": 5.0, + "x2": 3.0, + "y2": 9.0 + }, + "assertion_feasible": false + }, + "smlp_execution": "completed", + "interface_consistent": "true", + "model_consistent": "true" +} \ No newline at end of file diff --git a/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_data_bounds.json b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_data_bounds.json new file mode 100644 index 00000000..510fd80e --- /dev/null +++ b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_data_bounds.json @@ -0,0 +1,18 @@ +{ + "x1": { + "min": 2.0, + "max": 4.0 + }, + "x2": { + "min": 3.0, + "max": 8.0 + }, + "y1": { + "min": 5.0, + "max": 9.0 + }, + "y2": { + "min": 5.0, + "max": 9.0 + } +} \ No newline at end of file diff --git a/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_dt_sklearn_model_complete.pkl b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_dt_sklearn_model_complete.pkl new file mode 100644 index 00000000..9af00b41 Binary files /dev/null and b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_dt_sklearn_model_complete.pkl differ diff --git a/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_dt_sklearn_y1_tree_rules.txt b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_dt_sklearn_y1_tree_rules.txt new file mode 100644 index 00000000..9f2dbad2 --- /dev/null +++ b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_dt_sklearn_y1_tree_rules.txt @@ -0,0 +1,10 @@ +#Forest semantics: majority vote +#Number of trees: 1 + +#TREE 0 +if (x2 > 0.7000000178813934) and (x1 > 0.25) then (y1 = 1.0) | based on 2 samples +if (x2 <= 0.7000000178813934) and (x2 > 0.4000000134110451) then (y1 = 0.0) | based on 2 samples +if (x2 > 0.7000000178813934) and (x1 <= 0.25) and (x2 > 0.9000000059604645) then (y1 = 1.0) | based on 1 samples +if (x2 > 0.7000000178813934) and (x1 <= 0.25) and (x2 <= 0.9000000059604645) then (y1 = 0.0) | based on 1 samples +if (x2 <= 0.7000000178813934) and (x2 <= 0.4000000134110451) and (x2 > 0.10000000149011612) then (y1 = 1.0) | based on 1 samples +if (x2 <= 0.7000000178813934) and (x2 <= 0.4000000134110451) and (x2 <= 0.10000000149011612) then (y1 = 0.0) | based on 1 samples diff --git a/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_dt_sklearn_y2_tree_rules.txt b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_dt_sklearn_y2_tree_rules.txt new file mode 100644 index 00000000..5015336b --- /dev/null +++ b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_dt_sklearn_y2_tree_rules.txt @@ -0,0 +1,7 @@ +#Forest semantics: majority vote +#Number of trees: 1 + +#TREE 0 +if (x2 > 0.4000000134110451) and (x1 <= 0.75) then (y2 = 0.0) | based on 5 samples +if (x2 <= 0.4000000134110451) then (y2 = 1.0) | based on 2 samples +if (x2 > 0.4000000134110451) and (x1 > 0.75) then (y2 = 1.0) | based on 1 samples diff --git a/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_features_scaler.pkl b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_features_scaler.pkl new file mode 100644 index 00000000..62adbe18 Binary files /dev/null and b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_features_scaler.pkl differ diff --git a/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_model_features_dict.json b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_model_features_dict.json new file mode 100644 index 00000000..85782d17 --- /dev/null +++ b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_model_features_dict.json @@ -0,0 +1,10 @@ +{ + "y1": [ + "x1", + "x2" + ], + "y2": [ + "x1", + "x2" + ] +} \ No newline at end of file diff --git a/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_model_levels_dict.json b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_model_levels_dict.json new file mode 100644 index 00000000..9e26dfee --- /dev/null +++ b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_model_levels_dict.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_rerun_model_config.json b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_rerun_model_config.json new file mode 100644 index 00000000..4eeebbb7 --- /dev/null +++ b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_rerun_model_config.json @@ -0,0 +1,172 @@ +{ + "alpha": null, + "analytics_mode": "verify", + "approximate_fractions": true, + "assertions_expressions": "(y2**3+x2)/2<6;y1>=9;y2<0", + "assertions_names": "asrt1,asrt2,asrt3", + "beta": null, + "center_offset": "0", + "compress_rules": false, + "continuous_correlation_estimators": [ + "pearson", + "spearman" + ], + "correlations_and_mutual_information": true, + "data_scaler": "min_max", + "delta_absolute": 0.0, + "delta_relative": 0.01, + "discretization_algo": "uniform", + "discretization_bins": 10, + "discretization_labels": true, + "discretization_type": "category", + "discretize_numeric_features": null, + "doe_algo": null, + "doe_box_behnken_centers": 1, + "doe_central_composite_alpha": "o", + "doe_central_composite_center": "2,2", + "doe_central_composite_face": "ccf", + "doe_design_resolution": null, + "doe_factor_level_ranges": null, + "doe_num_samples": null, + "doe_prob_distribution": "Normal", + "doe_spec_file": null, + "dt_sklearn_ccp_alpha": 0.0, + "dt_sklearn_criterion": "squared_error", + "dt_sklearn_max_depth": 15, + "dt_sklearn_max_features": null, + "dt_sklearn_max_leaf_nodes": null, + "dt_sklearn_min_impurity_decrease": 0.0, + "dt_sklearn_min_samples_leaf": 1, + "dt_sklearn_min_samples_split": 2, + "dt_sklearn_min_weight_fraction_leaf": 0.0, + "dt_sklearn_random_state": null, + "dt_sklearn_splitter": "best", + "epsilon": 0.05, + "et_sklearn_bootstrap": true, + "et_sklearn_ccp_alpha": 0.0, + "et_sklearn_criterion": "squared_error", + "et_sklearn_max_depth": null, + "et_sklearn_max_features": 1.0, + "et_sklearn_max_leaf_nodes": null, + "et_sklearn_max_samples": null, + "et_sklearn_min_impurity_decrease": 0.0, + "et_sklearn_min_samples_leaf": 1, + "et_sklearn_min_samples_split": 2, + "et_sklearn_min_weight_fraction_leaf": 0.0, + "et_sklearn_n_estimators": 100, + "et_sklearn_random_state": null, + "et_sklearn_verbose": 0, + "et_sklearn_warm_start": false, + "eta": null, + "features": "x0,x1,x2", + "fraction_precision": 64, + "impute_responses": false, + "interactive_plots": false, + "keep_features": [], + "labeled_data": null, + "lemma_precision": 0, + "load_configuration": null, + "log_files_prefix": null, + "log_level": "info", + "log_mode": "w", + "log_time": false, + "model": "dt_sklearn", + "model_caret_cross_validation": true, + "model_caret_return_train_score": false, + "model_caret_verbose": true, + "model_name": null, + "model_per_response": true, + "mrmr_feat_count_for_correlation": 15, + "mrmr_feat_count_for_prediction": 2, + "mutual_information_method": "normalized", + "negative_value": 0, + "new_data": null, + "nn_keras_batch_size": 200, + "nn_keras_batches_grid": null, + "nn_keras_epochs": 2000, + "nn_keras_hid_activation": "relu", + "nn_keras_layers": "2,1", + "nn_keras_layers_grid": null, + "nn_keras_learning_rate": 0.001, + "nn_keras_learning_rates_grid": null, + "nn_keras_loss_function": "mse", + "nn_keras_loss_functions_grid": null, + "nn_keras_metrics": [ + "mse" + ], + "nn_keras_optimizer": "adam", + "nn_keras_out_activation": "linear", + "nn_keras_sequential_api": true, + "nn_keras_tuner_algo": null, + "nn_keras_weights_precision": null, + "nnet_encoding": "nested", + "objectives_expressions": null, + "objectives_names": "None", + "optimization_strategy": "eager", + "optimize_pareto": true, + "output_directory": "./", + "poly_sklearn_copy_X": true, + "poly_sklearn_degree": 2, + "poly_sklearn_fit_intercept": true, + "poly_sklearn_n_jobs": null, + "poly_sklearn_positive": false, + "positive_value": 1, + "prediction_plots": false, + "psg_max_dimension": 3, + "psg_quality_target": "Lift", + "psg_top_ranked": 15, + "query_expressions": null, + "query_names": "None", + "radius_absolute": null, + "radius_relative": null, + "response": "y1,y2", + "response_map": null, + "response_plots": false, + "response_to_bool": null, + "rf_sklearn_bootstrap": true, + "rf_sklearn_ccp_alpha": 0.0, + "rf_sklearn_criterion": "squared_error", + "rf_sklearn_max_depth": null, + "rf_sklearn_max_features": 1.0, + "rf_sklearn_max_leaf_nodes": null, + "rf_sklearn_max_samples": null, + "rf_sklearn_min_impurity_decrease": 0.0, + "rf_sklearn_min_samples_leaf": 1, + "rf_sklearn_min_samples_split": 2, + "rf_sklearn_min_weight_fraction_leaf": 0.0, + "rf_sklearn_n_estimators": 100, + "rf_sklearn_random_state": null, + "rf_sklearn_verbose": 0, + "rf_sklearn_warm_start": false, + "sample_weights_coef": 0, + "sample_weights_exponent": 0, + "sample_weights_intercept": 0, + "save_configuration": false, + "save_model": "false", + "save_model_rerun_configuration": true, + "scale_features": true, + "scale_objectives": true, + "scale_responses": true, + "seed": 10, + "setup_caret_data_split_shuffle": true, + "setup_caret_fold": 0, + "setup_caret_session_id": null, + "setup_caret_verbose": true, + "simplify_terms": false, + "solver": "z3", + "solver_logic": "ALL", + "solver_path": null, + "spec": "../specs/smlp_toy_num_resp_noknobs_verify.spec", + "split_test": 0.2, + "trace_anonymize": true, + "trace_precision": 3, + "trace_runtime": 0, + "train_first_n": 0, + "train_random_n": 0, + "train_uniform_n": 0, + "tree_encoding": "branched", + "tuner_caret_search_algorithm": "random", + "tuner_caret_tuner_verbose": true, + "use_model": "true", + "vacuity_check": true +} \ No newline at end of file diff --git a/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_responses_scaler.pkl b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_responses_scaler.pkl new file mode 100644 index 00000000..f2576be2 Binary files /dev/null and b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_responses_scaler.pkl differ diff --git a/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled.txt b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled.txt new file mode 100644 index 00000000..c77250f3 --- /dev/null +++ b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled.txt @@ -0,0 +1,410 @@ + +smlp_logger - INFO - Model exploration specification: +{'version': '1.1', 'spec': [{'label': 'y1', 'type': 'response', 'range': 'float'}, {'label': 'y2', 'type': 'response', 'range': 'float'}, {'label': 'x0', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x1', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x2', 'type': 'input', 'range': 'float', 'bounds': [3, 7]}], 'alpha': 'x1==1 or x1==4 or x1==7'} + +smlp_logger - INFO - Executing run_smlp.py script: Start + +smlp_logger - INFO - Running SMLP in mode "verify": Start + +smlp_logger - INFO - Computed spec global constraint expressions: + +smlp_logger - INFO - Global alpha : x1==1 or x1==4 or x1==7 + +smlp_logger - INFO - Global beta : None + +smlp_logger - INFO - Radii theta : {} + +smlp_logger - INFO - Delta const : {'delta_abs': 0.0, 'delta_rel': 0.01} + +smlp_logger - INFO - Assertion asrt1: (y2**3+x2)/2<6 + +smlp_logger - INFO - Assertion asrt2: y1>=9 + +smlp_logger - INFO - Assertion asrt3: y2<0 + +smlp_logger - INFO - PREPARE DATA FOR MODELING + +smlp_logger - INFO - Preparing training data for modeling: start + +smlp_logger - INFO - loading training data + +smlp_logger - INFO - data summary + y1 y2 x0 x1 x2 +count 11.000000 11.000000 10.000000 10.000000 11.000000 +mean 6.818182 6.818182 10.400000 2.800000 5.454545 +std 2.088932 2.088932 1.074968 0.788811 1.694912 +min 5.000000 5.000000 9.000000 2.000000 3.000000 +25% 5.000000 5.000000 10.000000 2.000000 4.000000 +50% 5.000000 5.000000 10.000000 3.000000 6.000000 +75% 9.000000 9.000000 11.000000 3.000000 7.000000 +max 9.000000 9.000000 12.000000 4.000000 8.000000 + +smlp_logger - INFO - training data + categ y1 y2 x0 x1 x2 +0 c14 5 9 10.0 2.0 3 +1 c15 9 9 12.0 NaN 4 +2 c1 5 9 NaN 3.0 4 +3 c9 5 5 11.0 2.0 6 +4 c5 9 5 10.0 2.0 8 +5 c10 9 9 9.0 4.0 7 +6 c13 5 5 9.0 3.0 6 +7 c4 5 5 10.0 3.0 4 +8 c15 9 9 11.0 4.0 4 +9 c11 5 5 12.0 2.0 7 +10 c19 9 5 10.0 3.0 7 + +smlp_logger - INFO - training data after imputing missing values + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - training data after processing responses + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - MRMR feature selection for response y1 : start + +smlp_logger - INFO - MRMR selected feature scores (in the ranked order) for response y1 : + Feature Score +0 x1 1.115702 +1 x2 0.944056 + +smlp_logger - INFO - MRMR feature selection for response y1 : end + +smlp_logger - INFO - MRMR feature selection for response y2 : start + +smlp_logger - INFO - MRMR selected feature scores (in the ranked order) for response y2 : + Feature Score +1 x2 4.950294 +0 x1 1.115702 + +smlp_logger - INFO - MRMR feature selection for response y2 : end + +smlp_logger - INFO - training data after encoding levels of categorical features with integers + x1 x2 y1 y2 +0 2.0 3 5 9 +1 2.0 4 9 9 +2 3.0 4 5 9 +3 2.0 6 5 5 +4 2.0 8 9 5 +5 4.0 7 9 9 +6 3.0 6 5 5 +7 3.0 4 5 5 +8 4.0 4 9 9 +9 2.0 7 5 5 +10 3.0 7 9 5 + +smlp_logger - INFO - training data after scaling (normalizing) features and responses + x1 x2 y1 y2 +0 0.0 0.0 0.0 1.0 +1 0.0 0.2 1.0 1.0 +2 0.5 0.2 0.0 1.0 +3 0.0 0.6 0.0 0.0 +4 0.0 1.0 1.0 0.0 +5 1.0 0.8 1.0 1.0 +6 0.5 0.6 0.0 0.0 +7 0.5 0.2 0.0 0.0 +8 1.0 0.2 1.0 1.0 +9 0.0 0.8 0.0 0.0 +10 0.5 0.8 1.0 0.0 + +smlp_logger - INFO - Sampling from training data: start + +smlp_logger - INFO - Sampling from training data: end + +smlp_logger - INFO - X_train after sampling: (8, 2) + +smlp_logger - INFO - y_train after sampling: (8, 2) + +smlp_logger - INFO - Preparing training data for modeling: end + +smlp_logger - INFO - Saving data bounds into file:./Test184_smlp_toy_num_resp_noknobs_data_bounds.json + +smlp_logger - INFO - {'x1': {'min': 2.0, 'max': 4.0}, 'x2': {'min': 3.0, 'max': 8.0}, 'y1': {'min': 5.0, 'max': 9.0}, 'y2': {'min': 5.0, 'max': 9.0}} + +smlp_logger - INFO - Preparing new data for modeling: start + +smlp_logger - INFO - loading new data + +smlp_logger - INFO - data summary + y1 y2 x0 x1 x2 +count 9.000000 9.000000 8.000000 8.000000 9.000000 +mean 6.777778 7.222222 10.250000 2.875000 5.111111 +std 2.108185 2.108185 1.035098 0.834523 1.691482 +min 5.000000 5.000000 9.000000 2.000000 3.000000 +25% 5.000000 5.000000 9.750000 2.000000 4.000000 +50% 5.000000 9.000000 10.000000 3.000000 4.000000 +75% 9.000000 9.000000 11.000000 3.250000 6.000000 +max 9.000000 9.000000 12.000000 4.000000 8.000000 + +smlp_logger - INFO - new data + categ y1 y2 x0 x1 x2 +0 c0 5 9 10.0 2.0 3 +1 c12 9 9 12.0 NaN 4 +2 c2 5 9 NaN 3.0 4 +3 c17 5 5 11.0 2.0 6 +4 c18 9 5 10.0 2.0 8 +5 c8 9 9 9.0 4.0 7 +6 c7 5 5 9.0 3.0 6 +7 c3 5 5 10.0 3.0 4 +8 c12 9 9 11.0 4.0 4 + +smlp_logger - INFO - new data after imputing missing values + x1 x2 y1 y2 +0 2.0 3 5 9 +1 2.0 4 9 9 +2 3.0 4 5 9 +3 2.0 6 5 5 +4 2.0 8 9 5 +5 4.0 7 9 9 +6 3.0 6 5 5 +7 3.0 4 5 5 +8 4.0 4 9 9 + +smlp_logger - INFO - new data after processing responses + x1 x2 y1 y2 +0 2.0 3 5 9 +1 2.0 4 9 9 +2 3.0 4 5 9 +3 2.0 6 5 5 +4 2.0 8 9 5 +5 4.0 7 9 9 +6 3.0 6 5 5 +7 3.0 4 5 5 +8 4.0 4 9 9 + +smlp_logger - INFO - new data after encoding levels of categorical features with integers + x1 x2 y1 y2 +0 2.0 3 5 9 +1 2.0 4 9 9 +2 3.0 4 5 9 +3 2.0 6 5 5 +4 2.0 8 9 5 +5 4.0 7 9 9 +6 3.0 6 5 5 +7 3.0 4 5 5 +8 4.0 4 9 9 + +smlp_logger - INFO - new data after scaling (normalizing) features and responses + x1 x2 y1 y2 +0 0.0 0.0 0.0 1.0 +1 0.0 0.2 1.0 1.0 +2 0.5 0.2 0.0 1.0 +3 0.0 0.6 0.0 0.0 +4 0.0 1.0 1.0 0.0 +5 1.0 0.8 1.0 1.0 +6 0.5 0.6 0.0 0.0 +7 0.5 0.2 0.0 0.0 +8 1.0 0.2 1.0 1.0 + +smlp_logger - INFO - Preparing new data for modeling: end + +smlp_logger - INFO - TRAIN MODEL + +smlp_logger - INFO - Model training: start + +smlp_logger - INFO - Writing tree rules into file ./Test184_smlp_toy_num_resp_noknobs_dt_sklearn_y1_tree_rules.txt + +smlp_logger - INFO - Writing tree rules into file ./Test184_smlp_toy_num_resp_noknobs_dt_sklearn_y2_tree_rules.txt + +smlp_logger - INFO - Model training: end + +smlp_logger - INFO - Seving model in file ./Test184_smlp_toy_num_resp_noknobs_dt_sklearn_model_complete.pkl + +smlp_logger - INFO - PREDICT ON TRAINING DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv + +smlp_logger - INFO - Prediction on training data -- msqe: 0.000 + +smlp_logger - INFO - Prediction on training data -- r2_score: 1.000 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON TEST DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv + +smlp_logger - INFO - Prediction on test data -- msqe: 8.000 + +smlp_logger - INFO - Prediction on test data -- r2_score: -1.250 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON LABELED DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv + +smlp_logger - INFO - Prediction on labeled data -- msqe: 2.182 + +smlp_logger - INFO - Prediction on labeled data -- r2_score: 0.450 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON NEW DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv + +smlp_logger - INFO - Prediction on new data -- msqe: 2.667 + +smlp_logger - INFO - Prediction on new data -- r2_score: 0.325 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - Creating model exploration base components: Start + +smlp_logger - INFO - Parsing the SPEC: Start + +smlp_logger - INFO - Parsing the SPEC: End + +smlp_logger - INFO - Variable domains (alpha): {'y1': {'range': 'float', 'interval': None}, 'y2': {'range': 'float', 'interval': None}, 'x0': {'range': 'float', 'interval': [0, 10]}, 'x1': {'range': 'float', 'interval': [0, 10]}, 'x2': {'range': 'float', 'interval': [3, 7]}} + +smlp_logger - INFO - Input bounds (alpha): {'x0': {'min': 0, 'max': 10}, 'x1': {'min': 0, 'max': 10}, 'x2': {'min': 3, 'max': 7}} + +smlp_logger - INFO - Knob bounds (eta): {} + +smlp_logger - INFO - Knob grids (eta): {} + +smlp_logger - INFO - Alpha global constraints: (or (or (= x1 1) (= x1 4)) (= x1 7)) + +smlp_logger - INFO - Alpha ranges constraints: (and (and true (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) + +smlp_logger - INFO - Alpha combined constraints: (and (and (and true (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) (or (or (= x1 1) (= x1 4)) (= x1 7))) + +smlp_logger - INFO - Beta global constraints: true + +smlp_logger - INFO - Eta ranges constraints: true + +smlp_logger - INFO - Eta grid constraints: true + +smlp_logger - INFO - Eta global constraints: true + +smlp_logger - INFO - Eta combined constraints: true + +smlp_logger - INFO - Creating model exploration base components: End + +smlp_logger - INFO - Input and knob interface constraints are consistent + +smlp_logger - INFO - Building model terms: Start + +smlp_logger - INFO - Model operator counts for antecedent_y1_scaled_tree_0_rule_0: {'and': 3, 'or': 2, 'not': 2, 'prop': 6, 'var': 6, 'const': 14, 'mul': 4, 'sub': 4} + +smlp_logger - INFO - Model operator counts for antecedent_y1_scaled_tree_0_rule_1: {'and': 3, 'or': 2, 'not': 2, 'prop': 6, 'var': 6, 'const': 14, 'mul': 4, 'sub': 4} + +smlp_logger - INFO - Model operator counts for antecedent_y1_scaled_tree_0_rule_2: {'and': 5, 'or': 2, 'not': 2, 'prop': 8, 'var': 8, 'const': 20, 'mul': 6, 'sub': 6} + +smlp_logger - INFO - Model operator counts for antecedent_y1_scaled_tree_0_rule_3: {'and': 5, 'or': 2, 'not': 2, 'prop': 8, 'var': 8, 'const': 20, 'mul': 6, 'sub': 6} + +smlp_logger - INFO - Model operator counts for antecedent_y1_scaled_tree_0_rule_4: {'and': 5, 'or': 2, 'not': 2, 'prop': 8, 'var': 8, 'const': 20, 'mul': 6, 'sub': 6} + +smlp_logger - INFO - Model operator counts for antecedent_y1_scaled_tree_0_rule_5: {'and': 5, 'or': 2, 'not': 2, 'prop': 8, 'var': 8, 'const': 20, 'mul': 6, 'sub': 6} + +smlp_logger - INFO - Model operator counts for y1: {'add': 1, 'mul': 1, 'ite': 5, 'prop': 5, 'var': 5, 'const': 13} + +smlp_logger - INFO - Model operator counts for antecedent_y2_scaled_tree_0_rule_0: {'and': 3, 'or': 2, 'not': 2, 'prop': 6, 'var': 6, 'const': 14, 'mul': 4, 'sub': 4} + +smlp_logger - INFO - Model operator counts for antecedent_y2_scaled_tree_0_rule_1: {'and': 1, 'or': 2, 'not': 2, 'prop': 4, 'var': 4, 'const': 8, 'mul': 2, 'sub': 2} + +smlp_logger - INFO - Model operator counts for antecedent_y2_scaled_tree_0_rule_2: {'and': 3, 'or': 2, 'not': 2, 'prop': 6, 'var': 6, 'const': 14, 'mul': 4, 'sub': 4} + +smlp_logger - INFO - Model operator counts for y2: {'add': 1, 'mul': 1, 'ite': 2, 'prop': 2, 'var': 2, 'const': 7} + +smlp_logger - INFO - Building model terms: End + +smlp_logger - INFO - Model interface constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt1: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt2: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt3: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying assertion asrt1 <-> (y2**3+x2)/2<6 + +smlp_logger - INFO - The configuration is inconsistent with assertion asrt1 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Verifying assertion asrt2 <-> y1>=9 + +smlp_logger - INFO - The configuration is consistent with assertion asrt2 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Verifying assertion asrt3 <-> y2<0 + +smlp_logger - INFO - The configuration is inconsistent with assertion asrt3 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Running SMLP in mode "verify": End + +smlp_logger - INFO - Executing run_smlp.py script: End diff --git a/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv new file mode 100644 index 00000000..7d143a2f --- /dev/null +++ b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,2.909090909090909,0.2666666666666666 +y2,1.4545454545454546,0.6333333333333333 diff --git a/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv new file mode 100644 index 00000000..7a649b6a --- /dev/null +++ b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv @@ -0,0 +1,12 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +0,5.0,9.0,5.0,9.0 +1,9.0,9.0,9.0,9.0 +2,5.0,9.0,9.0,9.0 +3,5.0,5.0,5.0,5.0 +4,9.0,5.0,9.0,5.0 +5,9.0,9.0,9.0,9.0 +6,5.0,5.0,5.0,5.0 +7,5.0,5.0,9.0,9.0 +8,9.0,9.0,9.0,9.0 +9,5.0,5.0,5.0,5.0 +10,9.0,5.0,9.0,5.0 diff --git a/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json new file mode 100644 index 00000000..6e4f6879 --- /dev/null +++ b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json @@ -0,0 +1,5 @@ +{ + "x1": [ + 1 + ] +} \ No newline at end of file diff --git a/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv new file mode 100644 index 00000000..951115f0 --- /dev/null +++ b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,3.5555555555555554,0.10000000000000009 +y2,1.7777777777777777,0.55 diff --git a/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv new file mode 100644 index 00000000..3ba8f5c9 --- /dev/null +++ b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv @@ -0,0 +1,10 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +0,5.0,9.0,5.0,9.0 +1,9.0,9.0,9.0,9.0 +2,5.0,9.0,9.0,9.0 +3,5.0,5.0,5.0,5.0 +4,9.0,5.0,9.0,5.0 +5,9.0,9.0,9.0,9.0 +6,5.0,5.0,5.0,5.0 +7,5.0,5.0,9.0,9.0 +8,9.0,9.0,9.0,9.0 diff --git a/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv new file mode 100644 index 00000000..fd947921 --- /dev/null +++ b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,10.666666666666666,-2.0 +y2,5.333333333333333,-0.5 diff --git a/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv new file mode 100644 index 00000000..3bb69252 --- /dev/null +++ b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv @@ -0,0 +1,4 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +7,5.0,5.0,9.0,9.0 +2,5.0,9.0,9.0,9.0 +8,9.0,9.0,9.0,9.0 diff --git a/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv new file mode 100644 index 00000000..b94ebfd2 --- /dev/null +++ b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv @@ -0,0 +1,12 @@ +stage,solver,x0,x1,x2,y0,y1 +interface_consistency,sat,7.0,3.0 +model_consistency,sat,1.0,7.0,5.0,5.0 +witness_consistency,sat,1.0,7.0,5.0,5.0 +witness_consistency,sat,1.0,7.0,5.0,5.0 +witness_consistency,sat,1.0,7.0,5.0,5.0 +ca,unsat +ce,sat,1.0,3.5,5.0,9.0 +ca,sat,7.0,6.75,9.0,9.0 +ce,sat,7.0,6.0,5.0,9.0 +ca,unsat +ce,sat,7.0,6.0,5.0,9.0 diff --git a/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv new file mode 100644 index 00000000..f0c58f67 --- /dev/null +++ b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,0.0,1.0 +y2,0.0,1.0 diff --git a/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv new file mode 100644 index 00000000..74d164ca --- /dev/null +++ b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv @@ -0,0 +1,9 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +3,5.0,5.0,5.0,5.0 +4,9.0,5.0,9.0,5.0 +5,9.0,9.0,9.0,9.0 +0,5.0,9.0,5.0,9.0 +10,9.0,5.0,9.0,5.0 +9,5.0,5.0,5.0,5.0 +6,5.0,5.0,5.0,5.0 +1,9.0,9.0,9.0,9.0 diff --git a/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json new file mode 100644 index 00000000..ad4bf886 --- /dev/null +++ b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json @@ -0,0 +1,38 @@ +{ + "asrt1": { + "configuration_consistent": "true", + "assertion_status": "FAIL", + "counter_example": { + "x1": 1.0, + "y1": 5.0, + "x2": 3.5, + "y2": 9.0 + }, + "assertion_feasible": false + }, + "asrt2": { + "configuration_consistent": "true", + "assertion_status": "FAIL", + "counter_example": { + "x1": 7.0, + "y1": 5.0, + "x2": 6.000000067055225, + "y2": 9.0 + }, + "assertion_feasible": true + }, + "asrt3": { + "configuration_consistent": "true", + "assertion_status": "FAIL", + "counter_example": { + "x1": 7.0, + "y1": 5.0, + "x2": 6.000000067055225, + "y2": 9.0 + }, + "assertion_feasible": false + }, + "smlp_execution": "completed", + "interface_consistent": "true", + "model_consistent": "true" +} \ No newline at end of file diff --git a/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_y1_smlp_full_model_term.json b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_y1_smlp_full_model_term.json new file mode 100644 index 00000000..05903661 --- /dev/null +++ b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_y1_smlp_full_model_term.json @@ -0,0 +1 @@ +"{'antecedent_y1_scaled_tree_0_rule_0': = antecedent_y1_scaled_tree_0_rule_0 0))) (let ((|:1| (and (> (* (/ 1 5) (- x2 3)) (/ 23488103 33554432)) (> (* (/ 1 2) (- x1 2)) (/ 1 4))))) (and (or (not |:0|) |:1|) (or (not |:1|) |:0|))))>, 'antecedent_y1_scaled_tree_0_rule_1': = antecedent_y1_scaled_tree_0_rule_1 0))) (let ((|:1| (* (/ 1 5) (- x2 3)))) (let ((|:2| (and (<= |:1| (/ 23488103 33554432)) (> |:1| (/ 53687093 134217728))))) (and (or (not |:0|) |:2|) (or (not |:2|) |:0|)))))>, 'antecedent_y1_scaled_tree_0_rule_2': = antecedent_y1_scaled_tree_0_rule_2 0))) (let ((|:1| (* (/ 1 5) (- x2 3)))) (let ((|:2| (and (and (> |:1| (/ 23488103 33554432)) (<= (* (/ 1 2) (- x1 2)) (/ 1 4))) (> |:1| (/ 30198989 33554432))))) (and (or (not |:0|) |:2|) (or (not |:2|) |:0|)))))>, 'antecedent_y1_scaled_tree_0_rule_3': = antecedent_y1_scaled_tree_0_rule_3 0))) (let ((|:1| (* (/ 1 5) (- x2 3)))) (let ((|:2| (and (and (> |:1| (/ 23488103 33554432)) (<= (* (/ 1 2) (- x1 2)) (/ 1 4))) (<= |:1| (/ 30198989 33554432))))) (and (or (not |:0|) |:2|) (or (not |:2|) |:0|)))))>, 'antecedent_y1_scaled_tree_0_rule_4': = antecedent_y1_scaled_tree_0_rule_4 0))) (let ((|:1| (* (/ 1 5) (- x2 3)))) (let ((|:2| (and (and (<= |:1| (/ 23488103 33554432)) (<= |:1| (/ 53687093 134217728))) (> |:1| (/ 13421773 134217728))))) (and (or (not |:0|) |:2|) (or (not |:2|) |:0|)))))>, 'antecedent_y1_scaled_tree_0_rule_5': = antecedent_y1_scaled_tree_0_rule_5 0))) (let ((|:1| (* (/ 1 5) (- x2 3)))) (let ((|:2| (and (and (<= |:1| (/ 23488103 33554432)) (<= |:1| (/ 53687093 134217728))) (<= |:1| (/ 13421773 134217728))))) (and (or (not |:0|) |:2|) (or (not |:2|) |:0|)))))>, 'y1': = antecedent_y1_scaled_tree_0_rule_5 0) 0 (ite (>= antecedent_y1_scaled_tree_0_rule_4 0) 1 (ite (>= antecedent_y1_scaled_tree_0_rule_3 0) 0 (ite (>= antecedent_y1_scaled_tree_0_rule_2 0) 1 (ite (>= antecedent_y1_scaled_tree_0_rule_1 0) 0 1))))) 4) 5)>}" \ No newline at end of file diff --git a/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_y1_smlp_model_term.json b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_y1_smlp_model_term.json new file mode 100644 index 00000000..470d650f --- /dev/null +++ b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_y1_smlp_model_term.json @@ -0,0 +1 @@ +"{'y1_scaled': = antecedent_y1_scaled_tree_0_rule_5 0) 0 (ite (>= antecedent_y1_scaled_tree_0_rule_4 0) 1 (ite (>= antecedent_y1_scaled_tree_0_rule_3 0) 0 (ite (>= antecedent_y1_scaled_tree_0_rule_2 0) 1 (ite (>= antecedent_y1_scaled_tree_0_rule_1 0) 0 1)))))>, 'antecedent_y1_scaled_tree_0_rule_0': = antecedent_y1_scaled_tree_0_rule_0 0))) (let ((|:1| (and (> x2_scaled (/ 23488103 33554432)) (> x1_scaled (/ 1 4))))) (and (or (not |:0|) |:1|) (or (not |:1|) |:0|))))>, 'antecedent_y1_scaled_tree_0_rule_1': = antecedent_y1_scaled_tree_0_rule_1 0))) (let ((|:1| (and (<= x2_scaled (/ 23488103 33554432)) (> x2_scaled (/ 53687093 134217728))))) (and (or (not |:0|) |:1|) (or (not |:1|) |:0|))))>, 'antecedent_y1_scaled_tree_0_rule_2': = antecedent_y1_scaled_tree_0_rule_2 0))) (let ((|:1| (and (and (> x2_scaled (/ 23488103 33554432)) (<= x1_scaled (/ 1 4))) (> x2_scaled (/ 30198989 33554432))))) (and (or (not |:0|) |:1|) (or (not |:1|) |:0|))))>, 'antecedent_y1_scaled_tree_0_rule_3': = antecedent_y1_scaled_tree_0_rule_3 0))) (let ((|:1| (and (and (> x2_scaled (/ 23488103 33554432)) (<= x1_scaled (/ 1 4))) (<= x2_scaled (/ 30198989 33554432))))) (and (or (not |:0|) |:1|) (or (not |:1|) |:0|))))>, 'antecedent_y1_scaled_tree_0_rule_4': = antecedent_y1_scaled_tree_0_rule_4 0))) (let ((|:1| (and (and (<= x2_scaled (/ 23488103 33554432)) (<= x2_scaled (/ 53687093 134217728))) (> x2_scaled (/ 13421773 134217728))))) (and (or (not |:0|) |:1|) (or (not |:1|) |:0|))))>, 'antecedent_y1_scaled_tree_0_rule_5': = antecedent_y1_scaled_tree_0_rule_5 0))) (let ((|:1| (and (and (<= x2_scaled (/ 23488103 33554432)) (<= x2_scaled (/ 53687093 134217728))) (<= x2_scaled (/ 13421773 134217728))))) (and (or (not |:0|) |:1|) (or (not |:1|) |:0|))))>}" \ No newline at end of file diff --git a/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_y2_smlp_full_model_term.json b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_y2_smlp_full_model_term.json new file mode 100644 index 00000000..41e3b5de --- /dev/null +++ b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_y2_smlp_full_model_term.json @@ -0,0 +1 @@ +"{'antecedent_y2_scaled_tree_0_rule_0': = antecedent_y2_scaled_tree_0_rule_0 0))) (let ((|:1| (and (> (* (/ 1 5) (- x2 3)) (/ 53687093 134217728)) (<= (* (/ 1 2) (- x1 2)) (/ 3 4))))) (and (or (not |:0|) |:1|) (or (not |:1|) |:0|))))>, 'antecedent_y2_scaled_tree_0_rule_1': = antecedent_y2_scaled_tree_0_rule_1 0))) (let ((|:1| (<= (* (/ 1 5) (- x2 3)) (/ 53687093 134217728)))) (and (or (not |:0|) |:1|) (or (not |:1|) |:0|))))>, 'antecedent_y2_scaled_tree_0_rule_2': = antecedent_y2_scaled_tree_0_rule_2 0))) (let ((|:1| (and (> (* (/ 1 5) (- x2 3)) (/ 53687093 134217728)) (> (* (/ 1 2) (- x1 2)) (/ 3 4))))) (and (or (not |:0|) |:1|) (or (not |:1|) |:0|))))>, 'y2': = antecedent_y2_scaled_tree_0_rule_2 0) 1 (ite (>= antecedent_y2_scaled_tree_0_rule_1 0) 1 0)) 4) 5)>}" \ No newline at end of file diff --git a/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_y2_smlp_model_term.json b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_y2_smlp_model_term.json new file mode 100644 index 00000000..e5382ce6 --- /dev/null +++ b/regr_smlp/master/Test184_smlp_toy_num_resp_noknobs_y2_smlp_model_term.json @@ -0,0 +1 @@ +"{'y2_scaled': = antecedent_y2_scaled_tree_0_rule_2 0) 1 (ite (>= antecedent_y2_scaled_tree_0_rule_1 0) 1 0))>, 'antecedent_y2_scaled_tree_0_rule_0': = antecedent_y2_scaled_tree_0_rule_0 0))) (let ((|:1| (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))))) (and (or (not |:0|) |:1|) (or (not |:1|) |:0|))))>, 'antecedent_y2_scaled_tree_0_rule_1': = antecedent_y2_scaled_tree_0_rule_1 0))) (let ((|:1| (<= x2_scaled (/ 53687093 134217728)))) (and (or (not |:0|) |:1|) (or (not |:1|) |:0|))))>, 'antecedent_y2_scaled_tree_0_rule_2': = antecedent_y2_scaled_tree_0_rule_2 0))) (let ((|:1| (and (> x2_scaled (/ 53687093 134217728)) (> x1_scaled (/ 3 4))))) (and (or (not |:0|) |:1|) (or (not |:1|) |:0|))))>}" \ No newline at end of file diff --git a/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_data_bounds.json b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_data_bounds.json new file mode 100644 index 00000000..510fd80e --- /dev/null +++ b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_data_bounds.json @@ -0,0 +1,18 @@ +{ + "x1": { + "min": 2.0, + "max": 4.0 + }, + "x2": { + "min": 3.0, + "max": 8.0 + }, + "y1": { + "min": 5.0, + "max": 9.0 + }, + "y2": { + "min": 5.0, + "max": 9.0 + } +} \ No newline at end of file diff --git a/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_dt_sklearn_model_complete.pkl b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_dt_sklearn_model_complete.pkl new file mode 100644 index 00000000..9af00b41 Binary files /dev/null and b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_dt_sklearn_model_complete.pkl differ diff --git a/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_dt_sklearn_y1_tree_rules.txt b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_dt_sklearn_y1_tree_rules.txt new file mode 100644 index 00000000..9f2dbad2 --- /dev/null +++ b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_dt_sklearn_y1_tree_rules.txt @@ -0,0 +1,10 @@ +#Forest semantics: majority vote +#Number of trees: 1 + +#TREE 0 +if (x2 > 0.7000000178813934) and (x1 > 0.25) then (y1 = 1.0) | based on 2 samples +if (x2 <= 0.7000000178813934) and (x2 > 0.4000000134110451) then (y1 = 0.0) | based on 2 samples +if (x2 > 0.7000000178813934) and (x1 <= 0.25) and (x2 > 0.9000000059604645) then (y1 = 1.0) | based on 1 samples +if (x2 > 0.7000000178813934) and (x1 <= 0.25) and (x2 <= 0.9000000059604645) then (y1 = 0.0) | based on 1 samples +if (x2 <= 0.7000000178813934) and (x2 <= 0.4000000134110451) and (x2 > 0.10000000149011612) then (y1 = 1.0) | based on 1 samples +if (x2 <= 0.7000000178813934) and (x2 <= 0.4000000134110451) and (x2 <= 0.10000000149011612) then (y1 = 0.0) | based on 1 samples diff --git a/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_dt_sklearn_y2_tree_rules.txt b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_dt_sklearn_y2_tree_rules.txt new file mode 100644 index 00000000..5015336b --- /dev/null +++ b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_dt_sklearn_y2_tree_rules.txt @@ -0,0 +1,7 @@ +#Forest semantics: majority vote +#Number of trees: 1 + +#TREE 0 +if (x2 > 0.4000000134110451) and (x1 <= 0.75) then (y2 = 0.0) | based on 5 samples +if (x2 <= 0.4000000134110451) then (y2 = 1.0) | based on 2 samples +if (x2 > 0.4000000134110451) and (x1 > 0.75) then (y2 = 1.0) | based on 1 samples diff --git a/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_features_scaler.pkl b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_features_scaler.pkl new file mode 100644 index 00000000..62adbe18 Binary files /dev/null and b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_features_scaler.pkl differ diff --git a/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_model_features_dict.json b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_model_features_dict.json new file mode 100644 index 00000000..85782d17 --- /dev/null +++ b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_model_features_dict.json @@ -0,0 +1,10 @@ +{ + "y1": [ + "x1", + "x2" + ], + "y2": [ + "x1", + "x2" + ] +} \ No newline at end of file diff --git a/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_model_levels_dict.json b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_model_levels_dict.json new file mode 100644 index 00000000..9e26dfee --- /dev/null +++ b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_model_levels_dict.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_rerun_model_config.json b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_rerun_model_config.json new file mode 100644 index 00000000..4eeebbb7 --- /dev/null +++ b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_rerun_model_config.json @@ -0,0 +1,172 @@ +{ + "alpha": null, + "analytics_mode": "verify", + "approximate_fractions": true, + "assertions_expressions": "(y2**3+x2)/2<6;y1>=9;y2<0", + "assertions_names": "asrt1,asrt2,asrt3", + "beta": null, + "center_offset": "0", + "compress_rules": false, + "continuous_correlation_estimators": [ + "pearson", + "spearman" + ], + "correlations_and_mutual_information": true, + "data_scaler": "min_max", + "delta_absolute": 0.0, + "delta_relative": 0.01, + "discretization_algo": "uniform", + "discretization_bins": 10, + "discretization_labels": true, + "discretization_type": "category", + "discretize_numeric_features": null, + "doe_algo": null, + "doe_box_behnken_centers": 1, + "doe_central_composite_alpha": "o", + "doe_central_composite_center": "2,2", + "doe_central_composite_face": "ccf", + "doe_design_resolution": null, + "doe_factor_level_ranges": null, + "doe_num_samples": null, + "doe_prob_distribution": "Normal", + "doe_spec_file": null, + "dt_sklearn_ccp_alpha": 0.0, + "dt_sklearn_criterion": "squared_error", + "dt_sklearn_max_depth": 15, + "dt_sklearn_max_features": null, + "dt_sklearn_max_leaf_nodes": null, + "dt_sklearn_min_impurity_decrease": 0.0, + "dt_sklearn_min_samples_leaf": 1, + "dt_sklearn_min_samples_split": 2, + "dt_sklearn_min_weight_fraction_leaf": 0.0, + "dt_sklearn_random_state": null, + "dt_sklearn_splitter": "best", + "epsilon": 0.05, + "et_sklearn_bootstrap": true, + "et_sklearn_ccp_alpha": 0.0, + "et_sklearn_criterion": "squared_error", + "et_sklearn_max_depth": null, + "et_sklearn_max_features": 1.0, + "et_sklearn_max_leaf_nodes": null, + "et_sklearn_max_samples": null, + "et_sklearn_min_impurity_decrease": 0.0, + "et_sklearn_min_samples_leaf": 1, + "et_sklearn_min_samples_split": 2, + "et_sklearn_min_weight_fraction_leaf": 0.0, + "et_sklearn_n_estimators": 100, + "et_sklearn_random_state": null, + "et_sklearn_verbose": 0, + "et_sklearn_warm_start": false, + "eta": null, + "features": "x0,x1,x2", + "fraction_precision": 64, + "impute_responses": false, + "interactive_plots": false, + "keep_features": [], + "labeled_data": null, + "lemma_precision": 0, + "load_configuration": null, + "log_files_prefix": null, + "log_level": "info", + "log_mode": "w", + "log_time": false, + "model": "dt_sklearn", + "model_caret_cross_validation": true, + "model_caret_return_train_score": false, + "model_caret_verbose": true, + "model_name": null, + "model_per_response": true, + "mrmr_feat_count_for_correlation": 15, + "mrmr_feat_count_for_prediction": 2, + "mutual_information_method": "normalized", + "negative_value": 0, + "new_data": null, + "nn_keras_batch_size": 200, + "nn_keras_batches_grid": null, + "nn_keras_epochs": 2000, + "nn_keras_hid_activation": "relu", + "nn_keras_layers": "2,1", + "nn_keras_layers_grid": null, + "nn_keras_learning_rate": 0.001, + "nn_keras_learning_rates_grid": null, + "nn_keras_loss_function": "mse", + "nn_keras_loss_functions_grid": null, + "nn_keras_metrics": [ + "mse" + ], + "nn_keras_optimizer": "adam", + "nn_keras_out_activation": "linear", + "nn_keras_sequential_api": true, + "nn_keras_tuner_algo": null, + "nn_keras_weights_precision": null, + "nnet_encoding": "nested", + "objectives_expressions": null, + "objectives_names": "None", + "optimization_strategy": "eager", + "optimize_pareto": true, + "output_directory": "./", + "poly_sklearn_copy_X": true, + "poly_sklearn_degree": 2, + "poly_sklearn_fit_intercept": true, + "poly_sklearn_n_jobs": null, + "poly_sklearn_positive": false, + "positive_value": 1, + "prediction_plots": false, + "psg_max_dimension": 3, + "psg_quality_target": "Lift", + "psg_top_ranked": 15, + "query_expressions": null, + "query_names": "None", + "radius_absolute": null, + "radius_relative": null, + "response": "y1,y2", + "response_map": null, + "response_plots": false, + "response_to_bool": null, + "rf_sklearn_bootstrap": true, + "rf_sklearn_ccp_alpha": 0.0, + "rf_sklearn_criterion": "squared_error", + "rf_sklearn_max_depth": null, + "rf_sklearn_max_features": 1.0, + "rf_sklearn_max_leaf_nodes": null, + "rf_sklearn_max_samples": null, + "rf_sklearn_min_impurity_decrease": 0.0, + "rf_sklearn_min_samples_leaf": 1, + "rf_sklearn_min_samples_split": 2, + "rf_sklearn_min_weight_fraction_leaf": 0.0, + "rf_sklearn_n_estimators": 100, + "rf_sklearn_random_state": null, + "rf_sklearn_verbose": 0, + "rf_sklearn_warm_start": false, + "sample_weights_coef": 0, + "sample_weights_exponent": 0, + "sample_weights_intercept": 0, + "save_configuration": false, + "save_model": "false", + "save_model_rerun_configuration": true, + "scale_features": true, + "scale_objectives": true, + "scale_responses": true, + "seed": 10, + "setup_caret_data_split_shuffle": true, + "setup_caret_fold": 0, + "setup_caret_session_id": null, + "setup_caret_verbose": true, + "simplify_terms": false, + "solver": "z3", + "solver_logic": "ALL", + "solver_path": null, + "spec": "../specs/smlp_toy_num_resp_noknobs_verify.spec", + "split_test": 0.2, + "trace_anonymize": true, + "trace_precision": 3, + "trace_runtime": 0, + "train_first_n": 0, + "train_random_n": 0, + "train_uniform_n": 0, + "tree_encoding": "branched", + "tuner_caret_search_algorithm": "random", + "tuner_caret_tuner_verbose": true, + "use_model": "true", + "vacuity_check": true +} \ No newline at end of file diff --git a/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_responses_scaler.pkl b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_responses_scaler.pkl new file mode 100644 index 00000000..f2576be2 Binary files /dev/null and b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_responses_scaler.pkl differ diff --git a/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled.txt b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled.txt new file mode 100644 index 00000000..4a36938a --- /dev/null +++ b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled.txt @@ -0,0 +1,410 @@ + +smlp_logger - INFO - Model exploration specification: +{'version': '1.1', 'spec': [{'label': 'y1', 'type': 'response', 'range': 'float'}, {'label': 'y2', 'type': 'response', 'range': 'float'}, {'label': 'x0', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x1', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x2', 'type': 'input', 'range': 'float', 'bounds': [3, 7]}], 'alpha': 'x1==1 or x1==4 or x1==7'} + +smlp_logger - INFO - Executing run_smlp.py script: Start + +smlp_logger - INFO - Running SMLP in mode "verify": Start + +smlp_logger - INFO - Computed spec global constraint expressions: + +smlp_logger - INFO - Global alpha : x1==1 or x1==4 or x1==7 + +smlp_logger - INFO - Global beta : None + +smlp_logger - INFO - Radii theta : {} + +smlp_logger - INFO - Delta const : {'delta_abs': 0.0, 'delta_rel': 0.01} + +smlp_logger - INFO - Assertion asrt1: (y2**3+x2)/2<6 + +smlp_logger - INFO - Assertion asrt2: y1>=9 + +smlp_logger - INFO - Assertion asrt3: y2<0 + +smlp_logger - INFO - PREPARE DATA FOR MODELING + +smlp_logger - INFO - Preparing training data for modeling: start + +smlp_logger - INFO - loading training data + +smlp_logger - INFO - data summary + y1 y2 x0 x1 x2 +count 11.000000 11.000000 10.000000 10.000000 11.000000 +mean 6.818182 6.818182 10.400000 2.800000 5.454545 +std 2.088932 2.088932 1.074968 0.788811 1.694912 +min 5.000000 5.000000 9.000000 2.000000 3.000000 +25% 5.000000 5.000000 10.000000 2.000000 4.000000 +50% 5.000000 5.000000 10.000000 3.000000 6.000000 +75% 9.000000 9.000000 11.000000 3.000000 7.000000 +max 9.000000 9.000000 12.000000 4.000000 8.000000 + +smlp_logger - INFO - training data + categ y1 y2 x0 x1 x2 +0 c14 5 9 10.0 2.0 3 +1 c15 9 9 12.0 NaN 4 +2 c1 5 9 NaN 3.0 4 +3 c9 5 5 11.0 2.0 6 +4 c5 9 5 10.0 2.0 8 +5 c10 9 9 9.0 4.0 7 +6 c13 5 5 9.0 3.0 6 +7 c4 5 5 10.0 3.0 4 +8 c15 9 9 11.0 4.0 4 +9 c11 5 5 12.0 2.0 7 +10 c19 9 5 10.0 3.0 7 + +smlp_logger - INFO - training data after imputing missing values + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - training data after processing responses + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - MRMR feature selection for response y1 : start + +smlp_logger - INFO - MRMR selected feature scores (in the ranked order) for response y1 : + Feature Score +0 x1 1.115702 +1 x2 0.944056 + +smlp_logger - INFO - MRMR feature selection for response y1 : end + +smlp_logger - INFO - MRMR feature selection for response y2 : start + +smlp_logger - INFO - MRMR selected feature scores (in the ranked order) for response y2 : + Feature Score +1 x2 4.950294 +0 x1 1.115702 + +smlp_logger - INFO - MRMR feature selection for response y2 : end + +smlp_logger - INFO - training data after encoding levels of categorical features with integers + x1 x2 y1 y2 +0 2.0 3 5 9 +1 2.0 4 9 9 +2 3.0 4 5 9 +3 2.0 6 5 5 +4 2.0 8 9 5 +5 4.0 7 9 9 +6 3.0 6 5 5 +7 3.0 4 5 5 +8 4.0 4 9 9 +9 2.0 7 5 5 +10 3.0 7 9 5 + +smlp_logger - INFO - training data after scaling (normalizing) features and responses + x1 x2 y1 y2 +0 0.0 0.0 0.0 1.0 +1 0.0 0.2 1.0 1.0 +2 0.5 0.2 0.0 1.0 +3 0.0 0.6 0.0 0.0 +4 0.0 1.0 1.0 0.0 +5 1.0 0.8 1.0 1.0 +6 0.5 0.6 0.0 0.0 +7 0.5 0.2 0.0 0.0 +8 1.0 0.2 1.0 1.0 +9 0.0 0.8 0.0 0.0 +10 0.5 0.8 1.0 0.0 + +smlp_logger - INFO - Sampling from training data: start + +smlp_logger - INFO - Sampling from training data: end + +smlp_logger - INFO - X_train after sampling: (8, 2) + +smlp_logger - INFO - y_train after sampling: (8, 2) + +smlp_logger - INFO - Preparing training data for modeling: end + +smlp_logger - INFO - Saving data bounds into file:./Test185_smlp_toy_num_resp_noknobs_data_bounds.json + +smlp_logger - INFO - {'x1': {'min': 2.0, 'max': 4.0}, 'x2': {'min': 3.0, 'max': 8.0}, 'y1': {'min': 5.0, 'max': 9.0}, 'y2': {'min': 5.0, 'max': 9.0}} + +smlp_logger - INFO - Preparing new data for modeling: start + +smlp_logger - INFO - loading new data + +smlp_logger - INFO - data summary + y1 y2 x0 x1 x2 +count 9.000000 9.000000 8.000000 8.000000 9.000000 +mean 6.777778 7.222222 10.250000 2.875000 5.111111 +std 2.108185 2.108185 1.035098 0.834523 1.691482 +min 5.000000 5.000000 9.000000 2.000000 3.000000 +25% 5.000000 5.000000 9.750000 2.000000 4.000000 +50% 5.000000 9.000000 10.000000 3.000000 4.000000 +75% 9.000000 9.000000 11.000000 3.250000 6.000000 +max 9.000000 9.000000 12.000000 4.000000 8.000000 + +smlp_logger - INFO - new data + categ y1 y2 x0 x1 x2 +0 c0 5 9 10.0 2.0 3 +1 c12 9 9 12.0 NaN 4 +2 c2 5 9 NaN 3.0 4 +3 c17 5 5 11.0 2.0 6 +4 c18 9 5 10.0 2.0 8 +5 c8 9 9 9.0 4.0 7 +6 c7 5 5 9.0 3.0 6 +7 c3 5 5 10.0 3.0 4 +8 c12 9 9 11.0 4.0 4 + +smlp_logger - INFO - new data after imputing missing values + x1 x2 y1 y2 +0 2.0 3 5 9 +1 2.0 4 9 9 +2 3.0 4 5 9 +3 2.0 6 5 5 +4 2.0 8 9 5 +5 4.0 7 9 9 +6 3.0 6 5 5 +7 3.0 4 5 5 +8 4.0 4 9 9 + +smlp_logger - INFO - new data after processing responses + x1 x2 y1 y2 +0 2.0 3 5 9 +1 2.0 4 9 9 +2 3.0 4 5 9 +3 2.0 6 5 5 +4 2.0 8 9 5 +5 4.0 7 9 9 +6 3.0 6 5 5 +7 3.0 4 5 5 +8 4.0 4 9 9 + +smlp_logger - INFO - new data after encoding levels of categorical features with integers + x1 x2 y1 y2 +0 2.0 3 5 9 +1 2.0 4 9 9 +2 3.0 4 5 9 +3 2.0 6 5 5 +4 2.0 8 9 5 +5 4.0 7 9 9 +6 3.0 6 5 5 +7 3.0 4 5 5 +8 4.0 4 9 9 + +smlp_logger - INFO - new data after scaling (normalizing) features and responses + x1 x2 y1 y2 +0 0.0 0.0 0.0 1.0 +1 0.0 0.2 1.0 1.0 +2 0.5 0.2 0.0 1.0 +3 0.0 0.6 0.0 0.0 +4 0.0 1.0 1.0 0.0 +5 1.0 0.8 1.0 1.0 +6 0.5 0.6 0.0 0.0 +7 0.5 0.2 0.0 0.0 +8 1.0 0.2 1.0 1.0 + +smlp_logger - INFO - Preparing new data for modeling: end + +smlp_logger - INFO - TRAIN MODEL + +smlp_logger - INFO - Model training: start + +smlp_logger - INFO - Writing tree rules into file ./Test185_smlp_toy_num_resp_noknobs_dt_sklearn_y1_tree_rules.txt + +smlp_logger - INFO - Writing tree rules into file ./Test185_smlp_toy_num_resp_noknobs_dt_sklearn_y2_tree_rules.txt + +smlp_logger - INFO - Model training: end + +smlp_logger - INFO - Seving model in file ./Test185_smlp_toy_num_resp_noknobs_dt_sklearn_model_complete.pkl + +smlp_logger - INFO - PREDICT ON TRAINING DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv + +smlp_logger - INFO - Prediction on training data -- msqe: 0.000 + +smlp_logger - INFO - Prediction on training data -- r2_score: 1.000 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON TEST DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv + +smlp_logger - INFO - Prediction on test data -- msqe: 8.000 + +smlp_logger - INFO - Prediction on test data -- r2_score: -1.250 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON LABELED DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv + +smlp_logger - INFO - Prediction on labeled data -- msqe: 2.182 + +smlp_logger - INFO - Prediction on labeled data -- r2_score: 0.450 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON NEW DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv + +smlp_logger - INFO - Prediction on new data -- msqe: 2.667 + +smlp_logger - INFO - Prediction on new data -- r2_score: 0.325 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - Creating model exploration base components: Start + +smlp_logger - INFO - Parsing the SPEC: Start + +smlp_logger - INFO - Parsing the SPEC: End + +smlp_logger - INFO - Variable domains (alpha): {'y1': {'range': 'float', 'interval': None}, 'y2': {'range': 'float', 'interval': None}, 'x0': {'range': 'float', 'interval': [0, 10]}, 'x1': {'range': 'float', 'interval': [0, 10]}, 'x2': {'range': 'float', 'interval': [3, 7]}} + +smlp_logger - INFO - Input bounds (alpha): {'x0': {'min': 0, 'max': 10}, 'x1': {'min': 0, 'max': 10}, 'x2': {'min': 3, 'max': 7}} + +smlp_logger - INFO - Knob bounds (eta): {} + +smlp_logger - INFO - Knob grids (eta): {} + +smlp_logger - INFO - Alpha global constraints: (or (or (= x1 1) (= x1 4)) (= x1 7)) + +smlp_logger - INFO - Alpha ranges constraints: (and (and true (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) + +smlp_logger - INFO - Alpha combined constraints: (and (and (and true (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) (or (or (= x1 1) (= x1 4)) (= x1 7))) + +smlp_logger - INFO - Beta global constraints: true + +smlp_logger - INFO - Eta ranges constraints: true + +smlp_logger - INFO - Eta grid constraints: true + +smlp_logger - INFO - Eta global constraints: true + +smlp_logger - INFO - Eta combined constraints: true + +smlp_logger - INFO - Creating model exploration base components: End + +smlp_logger - INFO - Input and knob interface constraints are consistent + +smlp_logger - INFO - Building model terms: Start + +smlp_logger - INFO - Model operator counts for antecedent_y1_scaled_tree_0_rule_0: {'and': 3, 'or': 2, 'not': 2, 'prop': 6, 'var': 6, 'const': 14, 'mul': 4, 'sub': 4} + +smlp_logger - INFO - Model operator counts for antecedent_y1_scaled_tree_0_rule_1: {'and': 3, 'or': 2, 'not': 2, 'prop': 6, 'var': 6, 'const': 14, 'mul': 4, 'sub': 4} + +smlp_logger - INFO - Model operator counts for antecedent_y1_scaled_tree_0_rule_2: {'and': 5, 'or': 2, 'not': 2, 'prop': 8, 'var': 8, 'const': 20, 'mul': 6, 'sub': 6} + +smlp_logger - INFO - Model operator counts for antecedent_y1_scaled_tree_0_rule_3: {'and': 5, 'or': 2, 'not': 2, 'prop': 8, 'var': 8, 'const': 20, 'mul': 6, 'sub': 6} + +smlp_logger - INFO - Model operator counts for antecedent_y1_scaled_tree_0_rule_4: {'and': 5, 'or': 2, 'not': 2, 'prop': 8, 'var': 8, 'const': 20, 'mul': 6, 'sub': 6} + +smlp_logger - INFO - Model operator counts for antecedent_y1_scaled_tree_0_rule_5: {'and': 5, 'or': 2, 'not': 2, 'prop': 8, 'var': 8, 'const': 20, 'mul': 6, 'sub': 6} + +smlp_logger - INFO - Model operator counts for y1: {'add': 1, 'mul': 1, 'ite': 5, 'prop': 5, 'var': 5, 'const': 13} + +smlp_logger - INFO - Model operator counts for antecedent_y2_scaled_tree_0_rule_0: {'and': 3, 'or': 2, 'not': 2, 'prop': 6, 'var': 6, 'const': 14, 'mul': 4, 'sub': 4} + +smlp_logger - INFO - Model operator counts for antecedent_y2_scaled_tree_0_rule_1: {'and': 1, 'or': 2, 'not': 2, 'prop': 4, 'var': 4, 'const': 8, 'mul': 2, 'sub': 2} + +smlp_logger - INFO - Model operator counts for antecedent_y2_scaled_tree_0_rule_2: {'and': 3, 'or': 2, 'not': 2, 'prop': 6, 'var': 6, 'const': 14, 'mul': 4, 'sub': 4} + +smlp_logger - INFO - Model operator counts for y2: {'add': 1, 'mul': 1, 'ite': 2, 'prop': 2, 'var': 2, 'const': 7} + +smlp_logger - INFO - Building model terms: End + +smlp_logger - INFO - Model interface constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt1: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt2: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt3: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying assertion asrt1 <-> (y2**3+x2)/2<6 + +smlp_logger - INFO - The configuration is inconsistent with assertion asrt1 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Verifying assertion asrt2 <-> y1>=9 + +smlp_logger - INFO - The configuration is consistent with assertion asrt2 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Verifying assertion asrt3 <-> y2<0 + +smlp_logger - INFO - The configuration is inconsistent with assertion asrt3 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Running SMLP in mode "verify": End + +smlp_logger - INFO - Executing run_smlp.py script: End diff --git a/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv new file mode 100644 index 00000000..7d143a2f --- /dev/null +++ b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,2.909090909090909,0.2666666666666666 +y2,1.4545454545454546,0.6333333333333333 diff --git a/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv new file mode 100644 index 00000000..7a649b6a --- /dev/null +++ b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv @@ -0,0 +1,12 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +0,5.0,9.0,5.0,9.0 +1,9.0,9.0,9.0,9.0 +2,5.0,9.0,9.0,9.0 +3,5.0,5.0,5.0,5.0 +4,9.0,5.0,9.0,5.0 +5,9.0,9.0,9.0,9.0 +6,5.0,5.0,5.0,5.0 +7,5.0,5.0,9.0,9.0 +8,9.0,9.0,9.0,9.0 +9,5.0,5.0,5.0,5.0 +10,9.0,5.0,9.0,5.0 diff --git a/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json new file mode 100644 index 00000000..6e4f6879 --- /dev/null +++ b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json @@ -0,0 +1,5 @@ +{ + "x1": [ + 1 + ] +} \ No newline at end of file diff --git a/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv new file mode 100644 index 00000000..951115f0 --- /dev/null +++ b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,3.5555555555555554,0.10000000000000009 +y2,1.7777777777777777,0.55 diff --git a/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv new file mode 100644 index 00000000..3ba8f5c9 --- /dev/null +++ b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv @@ -0,0 +1,10 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +0,5.0,9.0,5.0,9.0 +1,9.0,9.0,9.0,9.0 +2,5.0,9.0,9.0,9.0 +3,5.0,5.0,5.0,5.0 +4,9.0,5.0,9.0,5.0 +5,9.0,9.0,9.0,9.0 +6,5.0,5.0,5.0,5.0 +7,5.0,5.0,9.0,9.0 +8,9.0,9.0,9.0,9.0 diff --git a/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv new file mode 100644 index 00000000..fd947921 --- /dev/null +++ b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,10.666666666666666,-2.0 +y2,5.333333333333333,-0.5 diff --git a/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv new file mode 100644 index 00000000..3bb69252 --- /dev/null +++ b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv @@ -0,0 +1,4 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +7,5.0,5.0,9.0,9.0 +2,5.0,9.0,9.0,9.0 +8,9.0,9.0,9.0,9.0 diff --git a/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv new file mode 100644 index 00000000..b94ebfd2 --- /dev/null +++ b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv @@ -0,0 +1,12 @@ +stage,solver,x0,x1,x2,y0,y1 +interface_consistency,sat,7.0,3.0 +model_consistency,sat,1.0,7.0,5.0,5.0 +witness_consistency,sat,1.0,7.0,5.0,5.0 +witness_consistency,sat,1.0,7.0,5.0,5.0 +witness_consistency,sat,1.0,7.0,5.0,5.0 +ca,unsat +ce,sat,1.0,3.5,5.0,9.0 +ca,sat,7.0,6.75,9.0,9.0 +ce,sat,7.0,6.0,5.0,9.0 +ca,unsat +ce,sat,7.0,6.0,5.0,9.0 diff --git a/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv new file mode 100644 index 00000000..f0c58f67 --- /dev/null +++ b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,0.0,1.0 +y2,0.0,1.0 diff --git a/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv new file mode 100644 index 00000000..74d164ca --- /dev/null +++ b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv @@ -0,0 +1,9 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +3,5.0,5.0,5.0,5.0 +4,9.0,5.0,9.0,5.0 +5,9.0,9.0,9.0,9.0 +0,5.0,9.0,5.0,9.0 +10,9.0,5.0,9.0,5.0 +9,5.0,5.0,5.0,5.0 +6,5.0,5.0,5.0,5.0 +1,9.0,9.0,9.0,9.0 diff --git a/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json new file mode 100644 index 00000000..ad4bf886 --- /dev/null +++ b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json @@ -0,0 +1,38 @@ +{ + "asrt1": { + "configuration_consistent": "true", + "assertion_status": "FAIL", + "counter_example": { + "x1": 1.0, + "y1": 5.0, + "x2": 3.5, + "y2": 9.0 + }, + "assertion_feasible": false + }, + "asrt2": { + "configuration_consistent": "true", + "assertion_status": "FAIL", + "counter_example": { + "x1": 7.0, + "y1": 5.0, + "x2": 6.000000067055225, + "y2": 9.0 + }, + "assertion_feasible": true + }, + "asrt3": { + "configuration_consistent": "true", + "assertion_status": "FAIL", + "counter_example": { + "x1": 7.0, + "y1": 5.0, + "x2": 6.000000067055225, + "y2": 9.0 + }, + "assertion_feasible": false + }, + "smlp_execution": "completed", + "interface_consistent": "true", + "model_consistent": "true" +} \ No newline at end of file diff --git a/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_y1_smlp_full_model_term.json b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_y1_smlp_full_model_term.json new file mode 100644 index 00000000..05903661 --- /dev/null +++ b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_y1_smlp_full_model_term.json @@ -0,0 +1 @@ +"{'antecedent_y1_scaled_tree_0_rule_0': = antecedent_y1_scaled_tree_0_rule_0 0))) (let ((|:1| (and (> (* (/ 1 5) (- x2 3)) (/ 23488103 33554432)) (> (* (/ 1 2) (- x1 2)) (/ 1 4))))) (and (or (not |:0|) |:1|) (or (not |:1|) |:0|))))>, 'antecedent_y1_scaled_tree_0_rule_1': = antecedent_y1_scaled_tree_0_rule_1 0))) (let ((|:1| (* (/ 1 5) (- x2 3)))) (let ((|:2| (and (<= |:1| (/ 23488103 33554432)) (> |:1| (/ 53687093 134217728))))) (and (or (not |:0|) |:2|) (or (not |:2|) |:0|)))))>, 'antecedent_y1_scaled_tree_0_rule_2': = antecedent_y1_scaled_tree_0_rule_2 0))) (let ((|:1| (* (/ 1 5) (- x2 3)))) (let ((|:2| (and (and (> |:1| (/ 23488103 33554432)) (<= (* (/ 1 2) (- x1 2)) (/ 1 4))) (> |:1| (/ 30198989 33554432))))) (and (or (not |:0|) |:2|) (or (not |:2|) |:0|)))))>, 'antecedent_y1_scaled_tree_0_rule_3': = antecedent_y1_scaled_tree_0_rule_3 0))) (let ((|:1| (* (/ 1 5) (- x2 3)))) (let ((|:2| (and (and (> |:1| (/ 23488103 33554432)) (<= (* (/ 1 2) (- x1 2)) (/ 1 4))) (<= |:1| (/ 30198989 33554432))))) (and (or (not |:0|) |:2|) (or (not |:2|) |:0|)))))>, 'antecedent_y1_scaled_tree_0_rule_4': = antecedent_y1_scaled_tree_0_rule_4 0))) (let ((|:1| (* (/ 1 5) (- x2 3)))) (let ((|:2| (and (and (<= |:1| (/ 23488103 33554432)) (<= |:1| (/ 53687093 134217728))) (> |:1| (/ 13421773 134217728))))) (and (or (not |:0|) |:2|) (or (not |:2|) |:0|)))))>, 'antecedent_y1_scaled_tree_0_rule_5': = antecedent_y1_scaled_tree_0_rule_5 0))) (let ((|:1| (* (/ 1 5) (- x2 3)))) (let ((|:2| (and (and (<= |:1| (/ 23488103 33554432)) (<= |:1| (/ 53687093 134217728))) (<= |:1| (/ 13421773 134217728))))) (and (or (not |:0|) |:2|) (or (not |:2|) |:0|)))))>, 'y1': = antecedent_y1_scaled_tree_0_rule_5 0) 0 (ite (>= antecedent_y1_scaled_tree_0_rule_4 0) 1 (ite (>= antecedent_y1_scaled_tree_0_rule_3 0) 0 (ite (>= antecedent_y1_scaled_tree_0_rule_2 0) 1 (ite (>= antecedent_y1_scaled_tree_0_rule_1 0) 0 1))))) 4) 5)>}" \ No newline at end of file diff --git a/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_y1_smlp_model_term.json b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_y1_smlp_model_term.json new file mode 100644 index 00000000..470d650f --- /dev/null +++ b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_y1_smlp_model_term.json @@ -0,0 +1 @@ +"{'y1_scaled': = antecedent_y1_scaled_tree_0_rule_5 0) 0 (ite (>= antecedent_y1_scaled_tree_0_rule_4 0) 1 (ite (>= antecedent_y1_scaled_tree_0_rule_3 0) 0 (ite (>= antecedent_y1_scaled_tree_0_rule_2 0) 1 (ite (>= antecedent_y1_scaled_tree_0_rule_1 0) 0 1)))))>, 'antecedent_y1_scaled_tree_0_rule_0': = antecedent_y1_scaled_tree_0_rule_0 0))) (let ((|:1| (and (> x2_scaled (/ 23488103 33554432)) (> x1_scaled (/ 1 4))))) (and (or (not |:0|) |:1|) (or (not |:1|) |:0|))))>, 'antecedent_y1_scaled_tree_0_rule_1': = antecedent_y1_scaled_tree_0_rule_1 0))) (let ((|:1| (and (<= x2_scaled (/ 23488103 33554432)) (> x2_scaled (/ 53687093 134217728))))) (and (or (not |:0|) |:1|) (or (not |:1|) |:0|))))>, 'antecedent_y1_scaled_tree_0_rule_2': = antecedent_y1_scaled_tree_0_rule_2 0))) (let ((|:1| (and (and (> x2_scaled (/ 23488103 33554432)) (<= x1_scaled (/ 1 4))) (> x2_scaled (/ 30198989 33554432))))) (and (or (not |:0|) |:1|) (or (not |:1|) |:0|))))>, 'antecedent_y1_scaled_tree_0_rule_3': = antecedent_y1_scaled_tree_0_rule_3 0))) (let ((|:1| (and (and (> x2_scaled (/ 23488103 33554432)) (<= x1_scaled (/ 1 4))) (<= x2_scaled (/ 30198989 33554432))))) (and (or (not |:0|) |:1|) (or (not |:1|) |:0|))))>, 'antecedent_y1_scaled_tree_0_rule_4': = antecedent_y1_scaled_tree_0_rule_4 0))) (let ((|:1| (and (and (<= x2_scaled (/ 23488103 33554432)) (<= x2_scaled (/ 53687093 134217728))) (> x2_scaled (/ 13421773 134217728))))) (and (or (not |:0|) |:1|) (or (not |:1|) |:0|))))>, 'antecedent_y1_scaled_tree_0_rule_5': = antecedent_y1_scaled_tree_0_rule_5 0))) (let ((|:1| (and (and (<= x2_scaled (/ 23488103 33554432)) (<= x2_scaled (/ 53687093 134217728))) (<= x2_scaled (/ 13421773 134217728))))) (and (or (not |:0|) |:1|) (or (not |:1|) |:0|))))>}" \ No newline at end of file diff --git a/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_y2_smlp_full_model_term.json b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_y2_smlp_full_model_term.json new file mode 100644 index 00000000..41e3b5de --- /dev/null +++ b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_y2_smlp_full_model_term.json @@ -0,0 +1 @@ +"{'antecedent_y2_scaled_tree_0_rule_0': = antecedent_y2_scaled_tree_0_rule_0 0))) (let ((|:1| (and (> (* (/ 1 5) (- x2 3)) (/ 53687093 134217728)) (<= (* (/ 1 2) (- x1 2)) (/ 3 4))))) (and (or (not |:0|) |:1|) (or (not |:1|) |:0|))))>, 'antecedent_y2_scaled_tree_0_rule_1': = antecedent_y2_scaled_tree_0_rule_1 0))) (let ((|:1| (<= (* (/ 1 5) (- x2 3)) (/ 53687093 134217728)))) (and (or (not |:0|) |:1|) (or (not |:1|) |:0|))))>, 'antecedent_y2_scaled_tree_0_rule_2': = antecedent_y2_scaled_tree_0_rule_2 0))) (let ((|:1| (and (> (* (/ 1 5) (- x2 3)) (/ 53687093 134217728)) (> (* (/ 1 2) (- x1 2)) (/ 3 4))))) (and (or (not |:0|) |:1|) (or (not |:1|) |:0|))))>, 'y2': = antecedent_y2_scaled_tree_0_rule_2 0) 1 (ite (>= antecedent_y2_scaled_tree_0_rule_1 0) 1 0)) 4) 5)>}" \ No newline at end of file diff --git a/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_y2_smlp_model_term.json b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_y2_smlp_model_term.json new file mode 100644 index 00000000..e5382ce6 --- /dev/null +++ b/regr_smlp/master/Test185_smlp_toy_num_resp_noknobs_y2_smlp_model_term.json @@ -0,0 +1 @@ +"{'y2_scaled': = antecedent_y2_scaled_tree_0_rule_2 0) 1 (ite (>= antecedent_y2_scaled_tree_0_rule_1 0) 1 0))>, 'antecedent_y2_scaled_tree_0_rule_0': = antecedent_y2_scaled_tree_0_rule_0 0))) (let ((|:1| (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))))) (and (or (not |:0|) |:1|) (or (not |:1|) |:0|))))>, 'antecedent_y2_scaled_tree_0_rule_1': = antecedent_y2_scaled_tree_0_rule_1 0))) (let ((|:1| (<= x2_scaled (/ 53687093 134217728)))) (and (or (not |:0|) |:1|) (or (not |:1|) |:0|))))>, 'antecedent_y2_scaled_tree_0_rule_2': = antecedent_y2_scaled_tree_0_rule_2 0))) (let ((|:1| (and (> x2_scaled (/ 53687093 134217728)) (> x1_scaled (/ 3 4))))) (and (or (not |:0|) |:1|) (or (not |:1|) |:0|))))>}" \ No newline at end of file diff --git a/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_data_bounds.json b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_data_bounds.json new file mode 100644 index 00000000..510fd80e --- /dev/null +++ b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_data_bounds.json @@ -0,0 +1,18 @@ +{ + "x1": { + "min": 2.0, + "max": 4.0 + }, + "x2": { + "min": 3.0, + "max": 8.0 + }, + "y1": { + "min": 5.0, + "max": 9.0 + }, + "y2": { + "min": 5.0, + "max": 9.0 + } +} \ No newline at end of file diff --git a/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_dt_sklearn_model_complete.pkl b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_dt_sklearn_model_complete.pkl new file mode 100644 index 00000000..7cef3b59 Binary files /dev/null and b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_dt_sklearn_model_complete.pkl differ diff --git a/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_dt_sklearn_tree_rules.txt b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_dt_sklearn_tree_rules.txt new file mode 100644 index 00000000..59a1eff8 --- /dev/null +++ b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_dt_sklearn_tree_rules.txt @@ -0,0 +1,11 @@ +#Forest semantics: majority vote +#Number of trees: 1 + +#TREE 0 +if (x2 > 0.4000000134110451) and (x1 <= 0.75) and (x2 <= 0.7000000178813934) then (y1 = 0.0) and (y2 = 0.0) | based on 2 samples +if (x2 > 0.4000000134110451) and (x1 > 0.75) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +if (x2 > 0.4000000134110451) and (x1 <= 0.75) and (x2 > 0.7000000178813934) and (x1 > 0.25) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples +if (x2 > 0.4000000134110451) and (x1 <= 0.75) and (x2 > 0.7000000178813934) and (x1 <= 0.25) and (x2 > 0.9000000059604645) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples +if (x2 > 0.4000000134110451) and (x1 <= 0.75) and (x2 > 0.7000000178813934) and (x1 <= 0.25) and (x2 <= 0.9000000059604645) then (y1 = 0.0) and (y2 = 0.0) | based on 1 samples +if (x2 <= 0.4000000134110451) and (x2 > 0.10000000149011612) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +if (x2 <= 0.4000000134110451) and (x2 <= 0.10000000149011612) then (y1 = 0.0) and (y2 = 1.0) | based on 1 samples diff --git a/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_features_scaler.pkl b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_features_scaler.pkl new file mode 100644 index 00000000..62adbe18 Binary files /dev/null and b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_features_scaler.pkl differ diff --git a/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_model_features_dict.json b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_model_features_dict.json new file mode 100644 index 00000000..85782d17 --- /dev/null +++ b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_model_features_dict.json @@ -0,0 +1,10 @@ +{ + "y1": [ + "x1", + "x2" + ], + "y2": [ + "x1", + "x2" + ] +} \ No newline at end of file diff --git a/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_model_levels_dict.json b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_model_levels_dict.json new file mode 100644 index 00000000..9e26dfee --- /dev/null +++ b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_model_levels_dict.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_rerun_model_config.json b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_rerun_model_config.json new file mode 100644 index 00000000..455b1f9d --- /dev/null +++ b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_rerun_model_config.json @@ -0,0 +1,172 @@ +{ + "alpha": null, + "analytics_mode": "verify", + "approximate_fractions": true, + "assertions_expressions": "(y2**3+x2)/2<6;y1>=9;y2<0", + "assertions_names": "asrt1,asrt2,asrt3", + "beta": null, + "center_offset": "0", + "compress_rules": false, + "continuous_correlation_estimators": [ + "pearson", + "spearman" + ], + "correlations_and_mutual_information": true, + "data_scaler": "min_max", + "delta_absolute": 0.0, + "delta_relative": 0.01, + "discretization_algo": "uniform", + "discretization_bins": 10, + "discretization_labels": true, + "discretization_type": "category", + "discretize_numeric_features": null, + "doe_algo": null, + "doe_box_behnken_centers": 1, + "doe_central_composite_alpha": "o", + "doe_central_composite_center": "2,2", + "doe_central_composite_face": "ccf", + "doe_design_resolution": null, + "doe_factor_level_ranges": null, + "doe_num_samples": null, + "doe_prob_distribution": "Normal", + "doe_spec_file": null, + "dt_sklearn_ccp_alpha": 0.0, + "dt_sklearn_criterion": "squared_error", + "dt_sklearn_max_depth": 15, + "dt_sklearn_max_features": null, + "dt_sklearn_max_leaf_nodes": null, + "dt_sklearn_min_impurity_decrease": 0.0, + "dt_sklearn_min_samples_leaf": 1, + "dt_sklearn_min_samples_split": 2, + "dt_sklearn_min_weight_fraction_leaf": 0.0, + "dt_sklearn_random_state": null, + "dt_sklearn_splitter": "best", + "epsilon": 0.05, + "et_sklearn_bootstrap": true, + "et_sklearn_ccp_alpha": 0.0, + "et_sklearn_criterion": "squared_error", + "et_sklearn_max_depth": null, + "et_sklearn_max_features": 1.0, + "et_sklearn_max_leaf_nodes": null, + "et_sklearn_max_samples": null, + "et_sklearn_min_impurity_decrease": 0.0, + "et_sklearn_min_samples_leaf": 1, + "et_sklearn_min_samples_split": 2, + "et_sklearn_min_weight_fraction_leaf": 0.0, + "et_sklearn_n_estimators": 100, + "et_sklearn_random_state": null, + "et_sklearn_verbose": 0, + "et_sklearn_warm_start": false, + "eta": null, + "features": "x0,x1,x2", + "fraction_precision": 64, + "impute_responses": false, + "interactive_plots": false, + "keep_features": [], + "labeled_data": null, + "lemma_precision": 0, + "load_configuration": null, + "log_files_prefix": null, + "log_level": "info", + "log_mode": "w", + "log_time": false, + "model": "dt_sklearn", + "model_caret_cross_validation": true, + "model_caret_return_train_score": false, + "model_caret_verbose": true, + "model_name": null, + "model_per_response": false, + "mrmr_feat_count_for_correlation": 15, + "mrmr_feat_count_for_prediction": 2, + "mutual_information_method": "normalized", + "negative_value": 0, + "new_data": null, + "nn_keras_batch_size": 200, + "nn_keras_batches_grid": null, + "nn_keras_epochs": 2000, + "nn_keras_hid_activation": "relu", + "nn_keras_layers": "2,1", + "nn_keras_layers_grid": null, + "nn_keras_learning_rate": 0.001, + "nn_keras_learning_rates_grid": null, + "nn_keras_loss_function": "mse", + "nn_keras_loss_functions_grid": null, + "nn_keras_metrics": [ + "mse" + ], + "nn_keras_optimizer": "adam", + "nn_keras_out_activation": "linear", + "nn_keras_sequential_api": true, + "nn_keras_tuner_algo": null, + "nn_keras_weights_precision": null, + "nnet_encoding": "nested", + "objectives_expressions": null, + "objectives_names": "None", + "optimization_strategy": "eager", + "optimize_pareto": true, + "output_directory": "./", + "poly_sklearn_copy_X": true, + "poly_sklearn_degree": 2, + "poly_sklearn_fit_intercept": true, + "poly_sklearn_n_jobs": null, + "poly_sklearn_positive": false, + "positive_value": 1, + "prediction_plots": false, + "psg_max_dimension": 3, + "psg_quality_target": "Lift", + "psg_top_ranked": 15, + "query_expressions": null, + "query_names": "None", + "radius_absolute": null, + "radius_relative": null, + "response": "y1,y2", + "response_map": null, + "response_plots": false, + "response_to_bool": null, + "rf_sklearn_bootstrap": true, + "rf_sklearn_ccp_alpha": 0.0, + "rf_sklearn_criterion": "squared_error", + "rf_sklearn_max_depth": null, + "rf_sklearn_max_features": 1.0, + "rf_sklearn_max_leaf_nodes": null, + "rf_sklearn_max_samples": null, + "rf_sklearn_min_impurity_decrease": 0.0, + "rf_sklearn_min_samples_leaf": 1, + "rf_sklearn_min_samples_split": 2, + "rf_sklearn_min_weight_fraction_leaf": 0.0, + "rf_sklearn_n_estimators": 100, + "rf_sklearn_random_state": null, + "rf_sklearn_verbose": 0, + "rf_sklearn_warm_start": false, + "sample_weights_coef": 0, + "sample_weights_exponent": 0, + "sample_weights_intercept": 0, + "save_configuration": false, + "save_model": "false", + "save_model_rerun_configuration": true, + "scale_features": true, + "scale_objectives": true, + "scale_responses": true, + "seed": 10, + "setup_caret_data_split_shuffle": true, + "setup_caret_fold": 0, + "setup_caret_session_id": null, + "setup_caret_verbose": true, + "simplify_terms": false, + "solver": "z3", + "solver_logic": "ALL", + "solver_path": null, + "spec": "../specs/smlp_toy_num_resp_noknobs_verify.spec", + "split_test": 0.2, + "trace_anonymize": true, + "trace_precision": 3, + "trace_runtime": 0, + "train_first_n": 0, + "train_random_n": 0, + "train_uniform_n": 0, + "tree_encoding": "branched", + "tuner_caret_search_algorithm": "random", + "tuner_caret_tuner_verbose": true, + "use_model": "true", + "vacuity_check": true +} \ No newline at end of file diff --git a/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_responses_scaler.pkl b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_responses_scaler.pkl new file mode 100644 index 00000000..f2576be2 Binary files /dev/null and b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_responses_scaler.pkl differ diff --git a/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_full_model_term.json b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_full_model_term.json new file mode 100644 index 00000000..72e3c1ef --- /dev/null +++ b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_full_model_term.json @@ -0,0 +1 @@ +"{'antecedent_tree_0_rule_0': = antecedent_tree_0_rule_0 0))) (let ((|:1| (* (/ 1 5) (- x2 3)))) (let ((|:2| (and (and (> |:1| (/ 53687093 134217728)) (<= (* (/ 1 2) (- x1 2)) (/ 3 4))) (<= |:1| (/ 23488103 33554432))))) (and (or (not |:0|) |:2|) (or (not |:2|) |:0|)))))>, 'antecedent_tree_0_rule_1': = antecedent_tree_0_rule_1 0))) (let ((|:1| (and (> (* (/ 1 5) (- x2 3)) (/ 53687093 134217728)) (> (* (/ 1 2) (- x1 2)) (/ 3 4))))) (and (or (not |:0|) |:1|) (or (not |:1|) |:0|))))>, 'antecedent_tree_0_rule_2': = antecedent_tree_0_rule_2 0))) (let ((|:1| (* (/ 1 5) (- x2 3)))) (let ((|:2| (* (/ 1 2) (- x1 2)))) (let ((|:3| (and (and (and (> |:1| (/ 53687093 134217728)) (<= |:2| (/ 3 4))) (> |:1| (/ 23488103 33554432))) (> |:2| (/ 1 4))))) (and (or (not |:0|) |:3|) (or (not |:3|) |:0|))))))>, 'antecedent_tree_0_rule_3': = antecedent_tree_0_rule_3 0))) (let ((|:1| (* (/ 1 5) (- x2 3)))) (let ((|:2| (* (/ 1 2) (- x1 2)))) (let ((|:3| (and (and (and (and (> |:1| (/ 53687093 134217728)) (<= |:2| (/ 3 4))) (> |:1| (/ 23488103 33554432))) (<= |:2| (/ 1 4))) (> |:1| (/ 30198989 33554432))))) (and (or (not |:0|) |:3|) (or (not |:3|) |:0|))))))>, 'antecedent_tree_0_rule_4': = antecedent_tree_0_rule_4 0))) (let ((|:1| (* (/ 1 5) (- x2 3)))) (let ((|:2| (* (/ 1 2) (- x1 2)))) (let ((|:3| (and (and (and (and (> |:1| (/ 53687093 134217728)) (<= |:2| (/ 3 4))) (> |:1| (/ 23488103 33554432))) (<= |:2| (/ 1 4))) (<= |:1| (/ 30198989 33554432))))) (and (or (not |:0|) |:3|) (or (not |:3|) |:0|))))))>, 'antecedent_tree_0_rule_5': = antecedent_tree_0_rule_5 0))) (let ((|:1| (* (/ 1 5) (- x2 3)))) (let ((|:2| (and (<= |:1| (/ 53687093 134217728)) (> |:1| (/ 13421773 134217728))))) (and (or (not |:0|) |:2|) (or (not |:2|) |:0|)))))>, 'antecedent_tree_0_rule_6': = antecedent_tree_0_rule_6 0))) (let ((|:1| (* (/ 1 5) (- x2 3)))) (let ((|:2| (and (<= |:1| (/ 53687093 134217728)) (<= |:1| (/ 13421773 134217728))))) (and (or (not |:0|) |:2|) (or (not |:2|) |:0|)))))>, 'y1': = antecedent_tree_0_rule_6 0) 0 (ite (>= antecedent_tree_0_rule_5 0) 1 (ite (>= antecedent_tree_0_rule_4 0) 0 (ite (>= antecedent_tree_0_rule_3 0) 1 (ite (>= antecedent_tree_0_rule_2 0) 1 (ite (>= antecedent_tree_0_rule_1 0) 1 0)))))) 4) 5)>, 'y2': = antecedent_tree_0_rule_6 0) 1 (ite (>= antecedent_tree_0_rule_5 0) 1 (ite (>= antecedent_tree_0_rule_4 0) 0 (ite (>= antecedent_tree_0_rule_3 0) 0 (ite (>= antecedent_tree_0_rule_2 0) 0 (ite (>= antecedent_tree_0_rule_1 0) 1 0)))))) 4) 5)>}" \ No newline at end of file diff --git a/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_model_term.json b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_model_term.json new file mode 100644 index 00000000..3e5567c7 --- /dev/null +++ b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_model_term.json @@ -0,0 +1 @@ +"{'y1_scaled': = antecedent_tree_0_rule_6 0) 0 (ite (>= antecedent_tree_0_rule_5 0) 1 (ite (>= antecedent_tree_0_rule_4 0) 0 (ite (>= antecedent_tree_0_rule_3 0) 1 (ite (>= antecedent_tree_0_rule_2 0) 1 (ite (>= antecedent_tree_0_rule_1 0) 1 0))))))>, 'y2_scaled': = antecedent_tree_0_rule_6 0) 1 (ite (>= antecedent_tree_0_rule_5 0) 1 (ite (>= antecedent_tree_0_rule_4 0) 0 (ite (>= antecedent_tree_0_rule_3 0) 0 (ite (>= antecedent_tree_0_rule_2 0) 0 (ite (>= antecedent_tree_0_rule_1 0) 1 0))))))>, 'antecedent_tree_0_rule_0': = antecedent_tree_0_rule_0 0))) (let ((|:1| (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (<= x2_scaled (/ 23488103 33554432))))) (and (or (not |:0|) |:1|) (or (not |:1|) |:0|))))>, 'antecedent_tree_0_rule_1': = antecedent_tree_0_rule_1 0))) (let ((|:1| (and (> x2_scaled (/ 53687093 134217728)) (> x1_scaled (/ 3 4))))) (and (or (not |:0|) |:1|) (or (not |:1|) |:0|))))>, 'antecedent_tree_0_rule_2': = antecedent_tree_0_rule_2 0))) (let ((|:1| (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (> x2_scaled (/ 23488103 33554432))) (> x1_scaled (/ 1 4))))) (and (or (not |:0|) |:1|) (or (not |:1|) |:0|))))>, 'antecedent_tree_0_rule_3': = antecedent_tree_0_rule_3 0))) (let ((|:1| (and (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (> x2_scaled (/ 23488103 33554432))) (<= x1_scaled (/ 1 4))) (> x2_scaled (/ 30198989 33554432))))) (and (or (not |:0|) |:1|) (or (not |:1|) |:0|))))>, 'antecedent_tree_0_rule_4': = antecedent_tree_0_rule_4 0))) (let ((|:1| (and (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (> x2_scaled (/ 23488103 33554432))) (<= x1_scaled (/ 1 4))) (<= x2_scaled (/ 30198989 33554432))))) (and (or (not |:0|) |:1|) (or (not |:1|) |:0|))))>, 'antecedent_tree_0_rule_5': = antecedent_tree_0_rule_5 0))) (let ((|:1| (and (<= x2_scaled (/ 53687093 134217728)) (> x2_scaled (/ 13421773 134217728))))) (and (or (not |:0|) |:1|) (or (not |:1|) |:0|))))>, 'antecedent_tree_0_rule_6': = antecedent_tree_0_rule_6 0))) (let ((|:1| (and (<= x2_scaled (/ 53687093 134217728)) (<= x2_scaled (/ 13421773 134217728))))) (and (or (not |:0|) |:1|) (or (not |:1|) |:0|))))>}" \ No newline at end of file diff --git a/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled.txt b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled.txt new file mode 100644 index 00000000..e53adb17 --- /dev/null +++ b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled.txt @@ -0,0 +1,404 @@ + +smlp_logger - INFO - Model exploration specification: +{'version': '1.1', 'spec': [{'label': 'y1', 'type': 'response', 'range': 'float'}, {'label': 'y2', 'type': 'response', 'range': 'float'}, {'label': 'x0', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x1', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x2', 'type': 'input', 'range': 'float', 'bounds': [3, 7]}], 'alpha': 'x1==1 or x1==4 or x1==7'} + +smlp_logger - INFO - Executing run_smlp.py script: Start + +smlp_logger - INFO - Running SMLP in mode "verify": Start + +smlp_logger - INFO - Computed spec global constraint expressions: + +smlp_logger - INFO - Global alpha : x1==1 or x1==4 or x1==7 + +smlp_logger - INFO - Global beta : None + +smlp_logger - INFO - Radii theta : {} + +smlp_logger - INFO - Delta const : {'delta_abs': 0.0, 'delta_rel': 0.01} + +smlp_logger - INFO - Assertion asrt1: (y2**3+x2)/2<6 + +smlp_logger - INFO - Assertion asrt2: y1>=9 + +smlp_logger - INFO - Assertion asrt3: y2<0 + +smlp_logger - INFO - PREPARE DATA FOR MODELING + +smlp_logger - INFO - Preparing training data for modeling: start + +smlp_logger - INFO - loading training data + +smlp_logger - INFO - data summary + y1 y2 x0 x1 x2 +count 11.000000 11.000000 10.000000 10.000000 11.000000 +mean 6.818182 6.818182 10.400000 2.800000 5.454545 +std 2.088932 2.088932 1.074968 0.788811 1.694912 +min 5.000000 5.000000 9.000000 2.000000 3.000000 +25% 5.000000 5.000000 10.000000 2.000000 4.000000 +50% 5.000000 5.000000 10.000000 3.000000 6.000000 +75% 9.000000 9.000000 11.000000 3.000000 7.000000 +max 9.000000 9.000000 12.000000 4.000000 8.000000 + +smlp_logger - INFO - training data + categ y1 y2 x0 x1 x2 +0 c14 5 9 10.0 2.0 3 +1 c15 9 9 12.0 NaN 4 +2 c1 5 9 NaN 3.0 4 +3 c9 5 5 11.0 2.0 6 +4 c5 9 5 10.0 2.0 8 +5 c10 9 9 9.0 4.0 7 +6 c13 5 5 9.0 3.0 6 +7 c4 5 5 10.0 3.0 4 +8 c15 9 9 11.0 4.0 4 +9 c11 5 5 12.0 2.0 7 +10 c19 9 5 10.0 3.0 7 + +smlp_logger - INFO - training data after imputing missing values + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - training data after processing responses + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - MRMR feature selection for response y1 : start + +smlp_logger - INFO - MRMR selected feature scores (in the ranked order) for response y1 : + Feature Score +0 x1 1.115702 +1 x2 0.944056 + +smlp_logger - INFO - MRMR feature selection for response y1 : end + +smlp_logger - INFO - MRMR feature selection for response y2 : start + +smlp_logger - INFO - MRMR selected feature scores (in the ranked order) for response y2 : + Feature Score +1 x2 4.950294 +0 x1 1.115702 + +smlp_logger - INFO - MRMR feature selection for response y2 : end + +smlp_logger - INFO - training data after encoding levels of categorical features with integers + x1 x2 y1 y2 +0 2.0 3 5 9 +1 2.0 4 9 9 +2 3.0 4 5 9 +3 2.0 6 5 5 +4 2.0 8 9 5 +5 4.0 7 9 9 +6 3.0 6 5 5 +7 3.0 4 5 5 +8 4.0 4 9 9 +9 2.0 7 5 5 +10 3.0 7 9 5 + +smlp_logger - INFO - training data after scaling (normalizing) features and responses + x1 x2 y1 y2 +0 0.0 0.0 0.0 1.0 +1 0.0 0.2 1.0 1.0 +2 0.5 0.2 0.0 1.0 +3 0.0 0.6 0.0 0.0 +4 0.0 1.0 1.0 0.0 +5 1.0 0.8 1.0 1.0 +6 0.5 0.6 0.0 0.0 +7 0.5 0.2 0.0 0.0 +8 1.0 0.2 1.0 1.0 +9 0.0 0.8 0.0 0.0 +10 0.5 0.8 1.0 0.0 + +smlp_logger - INFO - Sampling from training data: start + +smlp_logger - INFO - Sampling from training data: end + +smlp_logger - INFO - X_train after sampling: (8, 2) + +smlp_logger - INFO - y_train after sampling: (8, 2) + +smlp_logger - INFO - Preparing training data for modeling: end + +smlp_logger - INFO - Saving data bounds into file:./Test186_smlp_toy_num_resp_noknobs_data_bounds.json + +smlp_logger - INFO - {'x1': {'min': 2.0, 'max': 4.0}, 'x2': {'min': 3.0, 'max': 8.0}, 'y1': {'min': 5.0, 'max': 9.0}, 'y2': {'min': 5.0, 'max': 9.0}} + +smlp_logger - INFO - Preparing new data for modeling: start + +smlp_logger - INFO - loading new data + +smlp_logger - INFO - data summary + y1 y2 x0 x1 x2 +count 9.000000 9.000000 8.000000 8.000000 9.000000 +mean 6.777778 7.222222 10.250000 2.875000 5.111111 +std 2.108185 2.108185 1.035098 0.834523 1.691482 +min 5.000000 5.000000 9.000000 2.000000 3.000000 +25% 5.000000 5.000000 9.750000 2.000000 4.000000 +50% 5.000000 9.000000 10.000000 3.000000 4.000000 +75% 9.000000 9.000000 11.000000 3.250000 6.000000 +max 9.000000 9.000000 12.000000 4.000000 8.000000 + +smlp_logger - INFO - new data + categ y1 y2 x0 x1 x2 +0 c0 5 9 10.0 2.0 3 +1 c12 9 9 12.0 NaN 4 +2 c2 5 9 NaN 3.0 4 +3 c17 5 5 11.0 2.0 6 +4 c18 9 5 10.0 2.0 8 +5 c8 9 9 9.0 4.0 7 +6 c7 5 5 9.0 3.0 6 +7 c3 5 5 10.0 3.0 4 +8 c12 9 9 11.0 4.0 4 + +smlp_logger - INFO - new data after imputing missing values + x1 x2 y1 y2 +0 2.0 3 5 9 +1 2.0 4 9 9 +2 3.0 4 5 9 +3 2.0 6 5 5 +4 2.0 8 9 5 +5 4.0 7 9 9 +6 3.0 6 5 5 +7 3.0 4 5 5 +8 4.0 4 9 9 + +smlp_logger - INFO - new data after processing responses + x1 x2 y1 y2 +0 2.0 3 5 9 +1 2.0 4 9 9 +2 3.0 4 5 9 +3 2.0 6 5 5 +4 2.0 8 9 5 +5 4.0 7 9 9 +6 3.0 6 5 5 +7 3.0 4 5 5 +8 4.0 4 9 9 + +smlp_logger - INFO - new data after encoding levels of categorical features with integers + x1 x2 y1 y2 +0 2.0 3 5 9 +1 2.0 4 9 9 +2 3.0 4 5 9 +3 2.0 6 5 5 +4 2.0 8 9 5 +5 4.0 7 9 9 +6 3.0 6 5 5 +7 3.0 4 5 5 +8 4.0 4 9 9 + +smlp_logger - INFO - new data after scaling (normalizing) features and responses + x1 x2 y1 y2 +0 0.0 0.0 0.0 1.0 +1 0.0 0.2 1.0 1.0 +2 0.5 0.2 0.0 1.0 +3 0.0 0.6 0.0 0.0 +4 0.0 1.0 1.0 0.0 +5 1.0 0.8 1.0 1.0 +6 0.5 0.6 0.0 0.0 +7 0.5 0.2 0.0 0.0 +8 1.0 0.2 1.0 1.0 + +smlp_logger - INFO - Preparing new data for modeling: end + +smlp_logger - INFO - TRAIN MODEL + +smlp_logger - INFO - Model training: start + +smlp_logger - INFO - Writing tree rules into file ./Test186_smlp_toy_num_resp_noknobs_dt_sklearn_tree_rules.txt + +smlp_logger - INFO - Model training: end + +smlp_logger - INFO - Seving model in file ./Test186_smlp_toy_num_resp_noknobs_dt_sklearn_model_complete.pkl + +smlp_logger - INFO - PREDICT ON TRAINING DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv + +smlp_logger - INFO - Prediction on training data -- msqe: 0.000 + +smlp_logger - INFO - Prediction on training data -- r2_score: 1.000 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON TEST DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv + +smlp_logger - INFO - Prediction on test data -- msqe: 8.000 + +smlp_logger - INFO - Prediction on test data -- r2_score: -1.250 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON LABELED DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv + +smlp_logger - INFO - Prediction on labeled data -- msqe: 2.182 + +smlp_logger - INFO - Prediction on labeled data -- r2_score: 0.450 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON NEW DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv + +smlp_logger - INFO - Prediction on new data -- msqe: 2.667 + +smlp_logger - INFO - Prediction on new data -- r2_score: 0.325 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - Creating model exploration base components: Start + +smlp_logger - INFO - Parsing the SPEC: Start + +smlp_logger - INFO - Parsing the SPEC: End + +smlp_logger - INFO - Variable domains (alpha): {'y1': {'range': 'float', 'interval': None}, 'y2': {'range': 'float', 'interval': None}, 'x0': {'range': 'float', 'interval': [0, 10]}, 'x1': {'range': 'float', 'interval': [0, 10]}, 'x2': {'range': 'float', 'interval': [3, 7]}} + +smlp_logger - INFO - Input bounds (alpha): {'x0': {'min': 0, 'max': 10}, 'x1': {'min': 0, 'max': 10}, 'x2': {'min': 3, 'max': 7}} + +smlp_logger - INFO - Knob bounds (eta): {} + +smlp_logger - INFO - Knob grids (eta): {} + +smlp_logger - INFO - Alpha global constraints: (or (or (= x1 1) (= x1 4)) (= x1 7)) + +smlp_logger - INFO - Alpha ranges constraints: (and (and true (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) + +smlp_logger - INFO - Alpha combined constraints: (and (and (and true (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) (or (or (= x1 1) (= x1 4)) (= x1 7))) + +smlp_logger - INFO - Beta global constraints: true + +smlp_logger - INFO - Eta ranges constraints: true + +smlp_logger - INFO - Eta grid constraints: true + +smlp_logger - INFO - Eta global constraints: true + +smlp_logger - INFO - Eta combined constraints: true + +smlp_logger - INFO - Creating model exploration base components: End + +smlp_logger - INFO - Input and knob interface constraints are consistent + +smlp_logger - INFO - Building model terms: Start + +smlp_logger - INFO - Model operator counts for antecedent_tree_0_rule_0: {'and': 5, 'or': 2, 'not': 2, 'prop': 8, 'var': 8, 'const': 20, 'mul': 6, 'sub': 6} + +smlp_logger - INFO - Model operator counts for antecedent_tree_0_rule_1: {'and': 3, 'or': 2, 'not': 2, 'prop': 6, 'var': 6, 'const': 14, 'mul': 4, 'sub': 4} + +smlp_logger - INFO - Model operator counts for antecedent_tree_0_rule_2: {'and': 7, 'or': 2, 'not': 2, 'prop': 10, 'var': 10, 'const': 26, 'mul': 8, 'sub': 8} + +smlp_logger - INFO - Model operator counts for antecedent_tree_0_rule_3: {'and': 9, 'or': 2, 'not': 2, 'prop': 12, 'var': 12, 'const': 32, 'mul': 10, 'sub': 10} + +smlp_logger - INFO - Model operator counts for antecedent_tree_0_rule_4: {'and': 9, 'or': 2, 'not': 2, 'prop': 12, 'var': 12, 'const': 32, 'mul': 10, 'sub': 10} + +smlp_logger - INFO - Model operator counts for antecedent_tree_0_rule_5: {'and': 3, 'or': 2, 'not': 2, 'prop': 6, 'var': 6, 'const': 14, 'mul': 4, 'sub': 4} + +smlp_logger - INFO - Model operator counts for antecedent_tree_0_rule_6: {'and': 3, 'or': 2, 'not': 2, 'prop': 6, 'var': 6, 'const': 14, 'mul': 4, 'sub': 4} + +smlp_logger - INFO - Model operator counts for y1: {'add': 1, 'mul': 1, 'ite': 6, 'prop': 6, 'var': 6, 'const': 15} + +smlp_logger - INFO - Model operator counts for y2: {'add': 1, 'mul': 1, 'ite': 6, 'prop': 6, 'var': 6, 'const': 15} + +smlp_logger - INFO - Building model terms: End + +smlp_logger - INFO - Model interface constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt1: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt2: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt3: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying assertion asrt1 <-> (y2**3+x2)/2<6 + +smlp_logger - INFO - The configuration is inconsistent with assertion asrt1 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Verifying assertion asrt2 <-> y1>=9 + +smlp_logger - INFO - The configuration is consistent with assertion asrt2 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Verifying assertion asrt3 <-> y2<0 + +smlp_logger - INFO - The configuration is inconsistent with assertion asrt3 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Running SMLP in mode "verify": End + +smlp_logger - INFO - Executing run_smlp.py script: End diff --git a/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv new file mode 100644 index 00000000..7d143a2f --- /dev/null +++ b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,2.909090909090909,0.2666666666666666 +y2,1.4545454545454546,0.6333333333333333 diff --git a/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv new file mode 100644 index 00000000..7a649b6a --- /dev/null +++ b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv @@ -0,0 +1,12 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +0,5.0,9.0,5.0,9.0 +1,9.0,9.0,9.0,9.0 +2,5.0,9.0,9.0,9.0 +3,5.0,5.0,5.0,5.0 +4,9.0,5.0,9.0,5.0 +5,9.0,9.0,9.0,9.0 +6,5.0,5.0,5.0,5.0 +7,5.0,5.0,9.0,9.0 +8,9.0,9.0,9.0,9.0 +9,5.0,5.0,5.0,5.0 +10,9.0,5.0,9.0,5.0 diff --git a/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json new file mode 100644 index 00000000..6e4f6879 --- /dev/null +++ b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json @@ -0,0 +1,5 @@ +{ + "x1": [ + 1 + ] +} \ No newline at end of file diff --git a/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv new file mode 100644 index 00000000..951115f0 --- /dev/null +++ b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,3.5555555555555554,0.10000000000000009 +y2,1.7777777777777777,0.55 diff --git a/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv new file mode 100644 index 00000000..3ba8f5c9 --- /dev/null +++ b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv @@ -0,0 +1,10 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +0,5.0,9.0,5.0,9.0 +1,9.0,9.0,9.0,9.0 +2,5.0,9.0,9.0,9.0 +3,5.0,5.0,5.0,5.0 +4,9.0,5.0,9.0,5.0 +5,9.0,9.0,9.0,9.0 +6,5.0,5.0,5.0,5.0 +7,5.0,5.0,9.0,9.0 +8,9.0,9.0,9.0,9.0 diff --git a/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv new file mode 100644 index 00000000..fd947921 --- /dev/null +++ b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,10.666666666666666,-2.0 +y2,5.333333333333333,-0.5 diff --git a/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv new file mode 100644 index 00000000..3bb69252 --- /dev/null +++ b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv @@ -0,0 +1,4 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +7,5.0,5.0,9.0,9.0 +2,5.0,9.0,9.0,9.0 +8,9.0,9.0,9.0,9.0 diff --git a/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv new file mode 100644 index 00000000..3b24cc62 --- /dev/null +++ b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv @@ -0,0 +1,12 @@ +stage,solver,x0,x1,x2,y0,y1 +interface_consistency,sat,7.0,3.0 +model_consistency,sat,1.0,7.0,5.0,5.0 +witness_consistency,sat,1.0,7.0,5.0,5.0 +witness_consistency,sat,1.0,7.0,5.0,5.0 +witness_consistency,sat,1.0,7.0,5.0,5.0 +ca,unsat +ce,sat,1.0,3.5,5.0,9.0 +ca,sat,7.0,6.0,9.0,9.0 +ce,sat,1.0,6.0,5.0,5.0 +ca,unsat +ce,sat,1.0,6.0,5.0,5.0 diff --git a/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv new file mode 100644 index 00000000..f0c58f67 --- /dev/null +++ b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,0.0,1.0 +y2,0.0,1.0 diff --git a/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv new file mode 100644 index 00000000..74d164ca --- /dev/null +++ b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv @@ -0,0 +1,9 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +3,5.0,5.0,5.0,5.0 +4,9.0,5.0,9.0,5.0 +5,9.0,9.0,9.0,9.0 +0,5.0,9.0,5.0,9.0 +10,9.0,5.0,9.0,5.0 +9,5.0,5.0,5.0,5.0 +6,5.0,5.0,5.0,5.0 +1,9.0,9.0,9.0,9.0 diff --git a/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json new file mode 100644 index 00000000..011e9613 --- /dev/null +++ b/regr_smlp/master/Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json @@ -0,0 +1,38 @@ +{ + "asrt1": { + "configuration_consistent": "true", + "assertion_status": "FAIL", + "counter_example": { + "x1": 1.0, + "y1": 5.0, + "x2": 3.5, + "y2": 9.0 + }, + "assertion_feasible": false + }, + "asrt2": { + "configuration_consistent": "true", + "assertion_status": "FAIL", + "counter_example": { + "x1": 1.0, + "y1": 5.0, + "x2": 6.000000067055225, + "y2": 5.0 + }, + "assertion_feasible": true + }, + "asrt3": { + "configuration_consistent": "true", + "assertion_status": "FAIL", + "counter_example": { + "x1": 1.0, + "y1": 5.0, + "x2": 6.000000067055225, + "y2": 5.0 + }, + "assertion_feasible": false + }, + "smlp_execution": "completed", + "interface_consistent": "true", + "model_consistent": "true" +} \ No newline at end of file diff --git a/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs.txt b/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs.txt new file mode 100644 index 00000000..b42d7da2 --- /dev/null +++ b/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs.txt @@ -0,0 +1,262 @@ + +smlp_logger - INFO - Model exploration specification: +{'version': '1.1', 'spec': [{'label': 'y1', 'type': 'response', 'range': 'float'}, {'label': 'y2', 'type': 'response', 'range': 'float'}, {'label': 'x0', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x1', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x2', 'type': 'input', 'range': 'float', 'bounds': [3, 7]}], 'alpha': 'x1==1 or x1==4 or x1==7'} + +smlp_logger - INFO - Executing run_smlp.py script: Start + +smlp_logger - INFO - Running SMLP in mode "verify": Start + +smlp_logger - INFO - Computed spec global constraint expressions: + +smlp_logger - INFO - Global alpha : x1==1 or x1==4 or x1==7 + +smlp_logger - INFO - Global beta : None + +smlp_logger - INFO - Radii theta : {} + +smlp_logger - INFO - Delta const : {'delta_abs': 0.0, 'delta_rel': 0.01} + +smlp_logger - INFO - Assertion asrt_y1: y1*2+x0<=5 and y1<=10 + +smlp_logger - INFO - Assertion asrt_y2: -2*y2-1<10-x2 + +smlp_logger - INFO - PREPARE DATA FOR MODELING + +smlp_logger - INFO - Preparing training data for modeling: start + +smlp_logger - INFO - loading training data + +smlp_logger - INFO - data summary + y1 y2 x0 x1 x2 +count 11.000000 11.000000 10.000000 10.000000 11.000000 +mean 6.818182 6.818182 10.400000 2.800000 5.454545 +std 2.088932 2.088932 1.074968 0.788811 1.694912 +min 5.000000 5.000000 9.000000 2.000000 3.000000 +25% 5.000000 5.000000 10.000000 2.000000 4.000000 +50% 5.000000 5.000000 10.000000 3.000000 6.000000 +75% 9.000000 9.000000 11.000000 3.000000 7.000000 +max 9.000000 9.000000 12.000000 4.000000 8.000000 + +smlp_logger - INFO - training data + categ y1 y2 x0 x1 x2 +0 c14 5 9 10.0 2.0 3 +1 c15 9 9 12.0 NaN 4 +2 c1 5 9 NaN 3.0 4 +3 c9 5 5 11.0 2.0 6 +4 c5 9 5 10.0 2.0 8 +5 c10 9 9 9.0 4.0 7 +6 c13 5 5 9.0 3.0 6 +7 c4 5 5 10.0 3.0 4 +8 c15 9 9 11.0 4.0 4 +9 c11 5 5 12.0 2.0 7 +10 c19 9 5 10.0 3.0 7 + +smlp_logger - INFO - training data after imputing missing values + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - training data after processing responses + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - training data after encoding levels of categorical features with integers + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - training data after scaling (normalizing) features and responses + x0 x1 x2 y1 y2 +0 0.333333 0.0 0.0 0.0 1.0 +1 1.000000 0.0 0.2 1.0 1.0 +2 0.333333 0.5 0.2 0.0 1.0 +3 0.666667 0.0 0.6 0.0 0.0 +4 0.333333 0.0 1.0 1.0 0.0 +5 0.000000 1.0 0.8 1.0 1.0 +6 0.000000 0.5 0.6 0.0 0.0 +7 0.333333 0.5 0.2 0.0 0.0 +8 0.666667 1.0 0.2 1.0 1.0 +9 1.000000 0.0 0.8 0.0 0.0 +10 0.333333 0.5 0.8 1.0 0.0 + +smlp_logger - INFO - Sampling from training data: start + +smlp_logger - INFO - Sampling from training data: end + +smlp_logger - INFO - X_train after sampling: (8, 3) + +smlp_logger - INFO - y_train after sampling: (8, 2) + +smlp_logger - INFO - Preparing training data for modeling: end + +smlp_logger - INFO - Saving data bounds into file:./Test57_smlp_toy_num_resp_noknobs_data_bounds.json + +smlp_logger - INFO - {'x0': {'min': 9.0, 'max': 12.0}, 'x1': {'min': 2.0, 'max': 4.0}, 'x2': {'min': 3.0, 'max': 8.0}, 'y1': {'min': 5.0, 'max': 9.0}, 'y2': {'min': 5.0, 'max': 9.0}} + +smlp_logger - INFO - TRAIN MODEL + +smlp_logger - INFO - Model training: start + +smlp_logger - INFO - Writing tree rules into file ./Test57_smlp_toy_num_resp_noknobs_dt_sklearn_tree_rules.txt + +smlp_logger - INFO - Model training: end + +smlp_logger - INFO - Seving model in file ./Test57_smlp_toy_num_resp_noknobs_dt_sklearn_model_complete.pkl + +smlp_logger - INFO - PREDICT ON TRAINING DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test57_smlp_toy_num_resp_noknobs_training_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test57_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv + +smlp_logger - INFO - Prediction on training data -- msqe: 0.000 + +smlp_logger - INFO - Prediction on training data -- r2_score: 1.000 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON TEST DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test57_smlp_toy_num_resp_noknobs_test_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test57_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv + +smlp_logger - INFO - Prediction on test data -- msqe: 2.667 + +smlp_logger - INFO - Prediction on test data -- r2_score: 0.250 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON LABELED DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test57_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test57_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv + +smlp_logger - INFO - Prediction on labeled data -- msqe: 0.727 + +smlp_logger - INFO - Prediction on labeled data -- r2_score: 0.817 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - Creating model exploration base components: Start + +smlp_logger - INFO - Parsing the SPEC: Start + +smlp_logger - INFO - Parsing the SPEC: End + +smlp_logger - INFO - Variable domains (alpha): {'y1': {'range': 'float', 'interval': None}, 'y2': {'range': 'float', 'interval': None}, 'x0': {'range': 'float', 'interval': [0, 10]}, 'x1': {'range': 'float', 'interval': [0, 10]}, 'x2': {'range': 'float', 'interval': [3, 7]}} + +smlp_logger - INFO - Input bounds (alpha): {'x0': {'min': 0, 'max': 10}, 'x1': {'min': 0, 'max': 10}, 'x2': {'min': 3, 'max': 7}} + +smlp_logger - INFO - Knob bounds (eta): {} + +smlp_logger - INFO - Knob grids (eta): {} + +smlp_logger - INFO - Alpha global constraints: (or (or (= x1 1) (= x1 4)) (= x1 7)) + +smlp_logger - INFO - Alpha ranges constraints: (and (and (and true (and (>= x0 0) (<= x0 10))) (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) + +smlp_logger - INFO - Alpha combined constraints: (and (and (and (and true (and (>= x0 0) (<= x0 10))) (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) (or (or (= x1 1) (= x1 4)) (= x1 7))) + +smlp_logger - INFO - Beta global constraints: true + +smlp_logger - INFO - Eta ranges constraints: true + +smlp_logger - INFO - Eta grid constraints: true + +smlp_logger - INFO - Eta global constraints: true + +smlp_logger - INFO - Eta combined constraints: true + +smlp_logger - INFO - Creating model exploration base components: End + +smlp_logger - INFO - Input and knob interface constraints are consistent + +smlp_logger - INFO - Building model terms: Start + +smlp_logger - INFO - Model operator counts for y1: {'add': 1, 'mul': 15, 'ite': 5, 'and': 9, 'prop': 14, 'const': 50, 'sub': 14, 'var': 14} + +smlp_logger - INFO - Model operator counts for y2: {'add': 1, 'mul': 15, 'ite': 5, 'and': 9, 'prop': 14, 'const': 50, 'sub': 14, 'var': 14} + +smlp_logger - INFO - Building model terms: End + +smlp_logger - INFO - Model interface constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt_y1: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt_y2: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying assertion asrt_y1 <-> y1*2+x0<=5 and y1<=10 + +smlp_logger - INFO - The configuration is inconsistent with assertion asrt_y1 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Verifying assertion asrt_y2 <-> -2*y2-1<10-x2 + +smlp_logger - INFO - The configuration is consistent with assertion asrt_y2 + +smlp_logger - INFO - Completed with result: PASS + +smlp_logger - INFO - Running SMLP in mode "verify": End + +smlp_logger - INFO - Executing run_smlp.py script: End diff --git a/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_data_bounds.json b/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_data_bounds.json new file mode 100644 index 00000000..5df59662 --- /dev/null +++ b/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_data_bounds.json @@ -0,0 +1,22 @@ +{ + "x0": { + "min": 9.0, + "max": 12.0 + }, + "x1": { + "min": 2.0, + "max": 4.0 + }, + "x2": { + "min": 3.0, + "max": 8.0 + }, + "y1": { + "min": 5.0, + "max": 9.0 + }, + "y2": { + "min": 5.0, + "max": 9.0 + } +} \ No newline at end of file diff --git a/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_dt_sklearn_model_complete.pkl b/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_dt_sklearn_model_complete.pkl new file mode 100644 index 00000000..1991a9ba Binary files /dev/null and b/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_dt_sklearn_model_complete.pkl differ diff --git a/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_dt_sklearn_tree_rules.txt b/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_dt_sklearn_tree_rules.txt new file mode 100644 index 00000000..c3f3f68c --- /dev/null +++ b/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_dt_sklearn_tree_rules.txt @@ -0,0 +1,10 @@ +#Forest semantics: majority vote +#Number of trees: 1 + +#TREE 0 +if (x2 > 0.4000000134110451) and (x1 <= 0.75) and (x0 > 0.5000000149011612) then (y1 = 0.0) and (y2 = 0.0) | based on 2 samples +if (x2 > 0.4000000134110451) and (x1 <= 0.75) and (x0 <= 0.5000000149011612) and (x2 > 0.7000000178813934) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +if (x2 > 0.4000000134110451) and (x1 > 0.75) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +if (x2 > 0.4000000134110451) and (x1 <= 0.75) and (x0 <= 0.5000000149011612) and (x2 <= 0.7000000178813934) then (y1 = 0.0) and (y2 = 0.0) | based on 1 samples +if (x2 <= 0.4000000134110451) and (x0 > 0.6666666716337204) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +if (x2 <= 0.4000000134110451) and (x0 <= 0.6666666716337204) then (y1 = 0.0) and (y2 = 1.0) | based on 1 samples diff --git a/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_features_scaler.pkl b/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_features_scaler.pkl new file mode 100644 index 00000000..d12c5399 Binary files /dev/null and b/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_features_scaler.pkl differ diff --git a/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv b/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv new file mode 100644 index 00000000..cf088bcd --- /dev/null +++ b/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,0.0,1.0 +y2,1.4545454545454546,0.6333333333333333 diff --git a/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv b/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv new file mode 100644 index 00000000..2cce12ec --- /dev/null +++ b/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv @@ -0,0 +1,12 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +0,5.0,9.0,5.0,9.0 +1,9.0,9.0,9.0,9.0 +2,5.0,9.0,5.0,9.0 +3,5.0,5.0,5.0,5.0 +4,9.0,5.0,9.0,5.0 +5,9.0,9.0,9.0,9.0 +6,5.0,5.0,5.0,5.0 +7,5.0,5.0,5.0,9.0 +8,9.0,9.0,9.0,9.0 +9,5.0,5.0,5.0,5.0 +10,9.0,5.0,9.0,5.0 diff --git a/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_missing_values_dict.json b/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_missing_values_dict.json new file mode 100644 index 00000000..553bfe33 --- /dev/null +++ b/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_missing_values_dict.json @@ -0,0 +1,8 @@ +{ + "x1": [ + 1 + ], + "x0": [ + 2 + ] +} \ No newline at end of file diff --git a/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_model_features_dict.json b/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_model_features_dict.json new file mode 100644 index 00000000..8e6a1c9a --- /dev/null +++ b/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_model_features_dict.json @@ -0,0 +1,12 @@ +{ + "y1": [ + "x0", + "x1", + "x2" + ], + "y2": [ + "x0", + "x1", + "x2" + ] +} \ No newline at end of file diff --git a/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_model_levels_dict.json b/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_model_levels_dict.json new file mode 100644 index 00000000..9e26dfee --- /dev/null +++ b/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_model_levels_dict.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_responses_scaler.pkl b/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_responses_scaler.pkl new file mode 100644 index 00000000..f2576be2 Binary files /dev/null and b/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_responses_scaler.pkl differ diff --git a/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_smlp_full_model_term.json b/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_smlp_full_model_term.json new file mode 100644 index 00000000..80c624e1 --- /dev/null +++ b/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_smlp_full_model_term.json @@ -0,0 +1 @@ +"{'y1': |:1| (/ 44739243 67108864))) 1 (ite (and (and (and (> |:0| (/ 53687093 134217728)) (<= |:2| (/ 3 4))) (<= |:1| (/ 33554433 67108864))) (<= |:0| (/ 23488103 33554432))) 0 (ite (and (> |:0| (/ 53687093 134217728)) (> |:2| (/ 3 4))) 1 (ite (and (and (and (> |:0| (/ 53687093 134217728)) (<= |:2| (/ 3 4))) (<= |:1| (/ 33554433 67108864))) (> |:0| (/ 23488103 33554432))) 1 0))))) 4) 5))))>, 'y2': |:1| (/ 44739243 67108864))) 1 (ite (and (and (and (> |:0| (/ 53687093 134217728)) (<= |:2| (/ 3 4))) (<= |:1| (/ 33554433 67108864))) (<= |:0| (/ 23488103 33554432))) 0 (ite (and (> |:0| (/ 53687093 134217728)) (> |:2| (/ 3 4))) 1 (ite (and (and (and (> |:0| (/ 53687093 134217728)) (<= |:2| (/ 3 4))) (<= |:1| (/ 33554433 67108864))) (> |:0| (/ 23488103 33554432))) 0 0))))) 4) 5))))>}" \ No newline at end of file diff --git a/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_smlp_model_term.json b/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_smlp_model_term.json new file mode 100644 index 00000000..b6732e08 --- /dev/null +++ b/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_smlp_model_term.json @@ -0,0 +1 @@ +"{'y1_scaled': x0_scaled (/ 44739243 67108864))) 1 (ite (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (<= x0_scaled (/ 33554433 67108864))) (<= x2_scaled (/ 23488103 33554432))) 0 (ite (and (> x2_scaled (/ 53687093 134217728)) (> x1_scaled (/ 3 4))) 1 (ite (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (<= x0_scaled (/ 33554433 67108864))) (> x2_scaled (/ 23488103 33554432))) 1 0)))))>, 'y2_scaled': x0_scaled (/ 44739243 67108864))) 1 (ite (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (<= x0_scaled (/ 33554433 67108864))) (<= x2_scaled (/ 23488103 33554432))) 0 (ite (and (> x2_scaled (/ 53687093 134217728)) (> x1_scaled (/ 3 4))) 1 (ite (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (<= x0_scaled (/ 33554433 67108864))) (> x2_scaled (/ 23488103 33554432))) 0 0)))))>}" \ No newline at end of file diff --git a/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv b/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv new file mode 100644 index 00000000..ec56b74c --- /dev/null +++ b/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,0.0,1.0 +y2,5.333333333333333,-0.5 diff --git a/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_test_predictions_summary.csv b/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_test_predictions_summary.csv new file mode 100644 index 00000000..3477d51d --- /dev/null +++ b/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_test_predictions_summary.csv @@ -0,0 +1,4 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +7,5.0,5.0,5.0,9.0 +2,5.0,9.0,5.0,9.0 +8,9.0,9.0,9.0,9.0 diff --git a/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_trace.csv b/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_trace.csv new file mode 100644 index 00000000..76dae09a --- /dev/null +++ b/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_trace.csv @@ -0,0 +1,9 @@ +stage,solver,x0,x1,x2,y1,y2 +interface_consistency,sat,0,7,3 +model_consistency,sat,0,7,805306377/134217728,9,9 +witness_consistency,sat,0,7,805306377/134217728,9,9 +witness_consistency,sat,0,7,805306377/134217728,9,9 +ca,unsat +ce,sat,0,4,805306377/134217728,9,9 +ca,sat,0,1,805306377/134217728,5,5 +ce,unsat diff --git a/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv b/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv new file mode 100644 index 00000000..f0c58f67 --- /dev/null +++ b/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,0.0,1.0 +y2,0.0,1.0 diff --git a/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_training_predictions_summary.csv b/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_training_predictions_summary.csv new file mode 100644 index 00000000..74d164ca --- /dev/null +++ b/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_training_predictions_summary.csv @@ -0,0 +1,9 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +3,5.0,5.0,5.0,5.0 +4,9.0,5.0,9.0,5.0 +5,9.0,9.0,9.0,9.0 +0,5.0,9.0,5.0,9.0 +10,9.0,5.0,9.0,5.0 +9,5.0,5.0,5.0,5.0 +6,5.0,5.0,5.0,5.0 +1,9.0,9.0,9.0,9.0 diff --git a/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_verify_results.json b/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_verify_results.json new file mode 100644 index 00000000..65c62fc8 --- /dev/null +++ b/regr_smlp/master/Test57_smlp_toy_num_resp_noknobs_verify_results.json @@ -0,0 +1,23 @@ +{ + "asrt_y1": { + "configuration_consistent": "true", + "assertion_status": "FAIL", + "counter_example": { + "x0": 0.0, + "x1": 4.0, + "y1": 9.0, + "x2": 6.000000067055225, + "y2": 9.0 + }, + "assertion_feasible": false + }, + "asrt_y2": { + "configuration_consistent": "true", + "assertion_status": "PASS", + "counter_example": null, + "assertion_feasible": true + }, + "smlp_execution": "completed", + "interface_consistent": "true", + "model_consistent": "true" +} \ No newline at end of file diff --git a/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs.txt b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs.txt new file mode 100644 index 00000000..40d27133 --- /dev/null +++ b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs.txt @@ -0,0 +1,311 @@ + +smlp_logger - INFO - Model exploration specification: +{'version': '1.1', 'spec': [{'label': 'y1', 'type': 'response', 'range': 'float'}, {'label': 'y2', 'type': 'response', 'range': 'float'}, {'label': 'x0', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x1', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x2', 'type': 'input', 'range': 'float', 'bounds': [3, 7]}], 'alpha': 'x1==1 or x1==4 or x1==7'} + +smlp_logger - INFO - Executing run_smlp.py script: Start + +smlp_logger - INFO - Running SMLP in mode "verify": Start + +smlp_logger - INFO - Computed spec global constraint expressions: + +smlp_logger - INFO - Global alpha : x1==1 or x1==4 or x1==7 + +smlp_logger - INFO - Global beta : None + +smlp_logger - INFO - Radii theta : {} + +smlp_logger - INFO - Delta const : {'delta_abs': 0.0, 'delta_rel': 0.01} + +smlp_logger - INFO - Assertion asrt_y1: not(x25 and y1<=10) + +smlp_logger - INFO - Assertion asrt_y2: -2*y2-1<10-x2 and x2>5 and x2<8 + +smlp_logger - INFO - PREPARE DATA FOR MODELING + +smlp_logger - INFO - Preparing training data for modeling: start + +smlp_logger - INFO - loading training data + +smlp_logger - INFO - data summary + y1 y2 x0 x1 x2 +count 11.000000 11.000000 10.000000 10.000000 11.000000 +mean 6.818182 6.818182 10.400000 2.800000 5.454545 +std 2.088932 2.088932 1.074968 0.788811 1.694912 +min 5.000000 5.000000 9.000000 2.000000 3.000000 +25% 5.000000 5.000000 10.000000 2.000000 4.000000 +50% 5.000000 5.000000 10.000000 3.000000 6.000000 +75% 9.000000 9.000000 11.000000 3.000000 7.000000 +max 9.000000 9.000000 12.000000 4.000000 8.000000 + +smlp_logger - INFO - training data + categ y1 y2 x0 x1 x2 +0 c14 5 9 10.0 2.0 3 +1 c15 9 9 12.0 NaN 4 +2 c1 5 9 NaN 3.0 4 +3 c9 5 5 11.0 2.0 6 +4 c5 9 5 10.0 2.0 8 +5 c10 9 9 9.0 4.0 7 +6 c13 5 5 9.0 3.0 6 +7 c4 5 5 10.0 3.0 4 +8 c15 9 9 11.0 4.0 4 +9 c11 5 5 12.0 2.0 7 +10 c19 9 5 10.0 3.0 7 + +smlp_logger - INFO - training data after imputing missing values + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - training data after processing responses + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - training data after encoding levels of categorical features with integers + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - training data after scaling (normalizing) features and responses + x0 x1 x2 y1 y2 +0 0.333333 0.0 0.0 0.0 1.0 +1 1.000000 0.0 0.2 1.0 1.0 +2 0.333333 0.5 0.2 0.0 1.0 +3 0.666667 0.0 0.6 0.0 0.0 +4 0.333333 0.0 1.0 1.0 0.0 +5 0.000000 1.0 0.8 1.0 1.0 +6 0.000000 0.5 0.6 0.0 0.0 +7 0.333333 0.5 0.2 0.0 0.0 +8 0.666667 1.0 0.2 1.0 1.0 +9 1.000000 0.0 0.8 0.0 0.0 +10 0.333333 0.5 0.8 1.0 0.0 + +smlp_logger - INFO - Sampling from training data: start + +smlp_logger - INFO - Sampling from training data: end + +smlp_logger - INFO - X_train after sampling: (8, 3) + +smlp_logger - INFO - y_train after sampling: (8, 2) + +smlp_logger - INFO - Preparing training data for modeling: end + +smlp_logger - INFO - Saving data bounds into file:./Test61_smlp_toy_num_resp_noknobs_data_bounds.json + +smlp_logger - INFO - {'x0': {'min': 9.0, 'max': 12.0}, 'x1': {'min': 2.0, 'max': 4.0}, 'x2': {'min': 3.0, 'max': 8.0}, 'y1': {'min': 5.0, 'max': 9.0}, 'y2': {'min': 5.0, 'max': 9.0}} + +smlp_logger - INFO - TRAIN MODEL + +smlp_logger - INFO - Model training: start + +smlp_logger - INFO - keras_main: start + +smlp_logger - INFO - _keras_train_multi_response: start + +smlp_logger - INFO - layers_spec_list [2.0, 1.0] + +smlp_logger - INFO - building NN model using Keras Functional API + +smlp_logger - INFO - input layer of size 3 + +smlp_logger - INFO - dense layer of size 6 + +smlp_logger - INFO - dense layer of size 3 + +smlp_logger - INFO - output layer of size 1 + +smlp_logger - INFO - output layer of size 1 + +smlp_logger - INFO - model summary: start + +smlp_logger - INFO - Model: "model" +__________________________________________________________________________________________________ + Layer (type) Output Shape Param # Connected to +================================================================================================== + input_1 (InputLayer) [(None, 3)] 0 [] + + dense (Dense) (None, 6) 24 ['input_1[0][0]'] + + dense_1 (Dense) (None, 3) 21 ['dense[0][0]'] + + y1 (Dense) (None, 1) 4 ['dense_1[0][0]'] + + y2 (Dense) (None, 1) 4 ['dense_1[0][0]'] + +================================================================================================== +Total params: 53 (212.00 Byte) +Trainable params: 53 (212.00 Byte) +Non-trainable params: 0 (0.00 Byte) +__________________________________________________________________________________________________ + + +smlp_logger - INFO - Optimizer: {'name': 'Adam', 'weight_decay': None, 'clipnorm': None, 'global_clipnorm': None, 'clipvalue': None, 'use_ema': False, 'ema_momentum': 0.99, 'ema_overwrite_frequency': None, 'jit_compile': False, 'is_legacy_optimizer': False, 'learning_rate': 0.001, 'beta_1': 0.9, 'beta_2': 0.999, 'epsilon': 1e-07, 'amsgrad': False} + +smlp_logger - INFO - Learning rate: 0.001 + +smlp_logger - INFO - Loss function: mse + +smlp_logger - INFO - Metrics: ['mse'] + +smlp_logger - INFO - Model configuration: {'name': 'model', 'trainable': True, 'layers': [{'module': 'keras.layers', 'class_name': 'InputLayer', 'config': {'batch_input_shape': (None, 3), 'dtype': 'float32', 'sparse': False, 'ragged': False, 'name': 'input_1'}, 'registered_name': None, 'name': 'input_1', 'inbound_nodes': []}, {'module': 'keras.layers', 'class_name': 'Dense', 'config': {'name': 'dense', 'trainable': True, 'dtype': 'float32', 'units': 6, 'activation': 'relu', 'use_bias': True, 'kernel_initializer': {'module': 'keras.initializers', 'class_name': 'GlorotUniform', 'config': {'seed': None}, 'registered_name': None}, 'bias_initializer': {'module': 'keras.initializers', 'class_name': 'Zeros', 'config': {}, 'registered_name': None}, 'kernel_regularizer': None, 'bias_regularizer': None, 'activity_regularizer': None, 'kernel_constraint': None, 'bias_constraint': None}, 'registered_name': None, 'build_config': {'input_shape': (None, 3)}, 'name': 'dense', 'inbound_nodes': [[['input_1', 0, 0, {}]]]}, {'module': 'keras.layers', 'class_name': 'Dense', 'config': {'name': 'dense_1', 'trainable': True, 'dtype': 'float32', 'units': 3, 'activation': 'relu', 'use_bias': True, 'kernel_initializer': {'module': 'keras.initializers', 'class_name': 'GlorotUniform', 'config': {'seed': None}, 'registered_name': None}, 'bias_initializer': {'module': 'keras.initializers', 'class_name': 'Zeros', 'config': {}, 'registered_name': None}, 'kernel_regularizer': None, 'bias_regularizer': None, 'activity_regularizer': None, 'kernel_constraint': None, 'bias_constraint': None}, 'registered_name': None, 'build_config': {'input_shape': (None, 6)}, 'name': 'dense_1', 'inbound_nodes': [[['dense', 0, 0, {}]]]}, {'module': 'keras.layers', 'class_name': 'Dense', 'config': {'name': 'y1', 'trainable': True, 'dtype': 'float32', 'units': 1, 'activation': 'linear', 'use_bias': True, 'kernel_initializer': {'module': 'keras.initializers', 'class_name': 'GlorotUniform', 'config': {'seed': None}, 'registered_name': None}, 'bias_initializer': {'module': 'keras.initializers', 'class_name': 'Zeros', 'config': {}, 'registered_name': None}, 'kernel_regularizer': None, 'bias_regularizer': None, 'activity_regularizer': None, 'kernel_constraint': None, 'bias_constraint': None}, 'registered_name': None, 'build_config': {'input_shape': (None, 3)}, 'name': 'y1', 'inbound_nodes': [[['dense_1', 0, 0, {}]]]}, {'module': 'keras.layers', 'class_name': 'Dense', 'config': {'name': 'y2', 'trainable': True, 'dtype': 'float32', 'units': 1, 'activation': 'linear', 'use_bias': True, 'kernel_initializer': {'module': 'keras.initializers', 'class_name': 'GlorotUniform', 'config': {'seed': None}, 'registered_name': None}, 'bias_initializer': {'module': 'keras.initializers', 'class_name': 'Zeros', 'config': {}, 'registered_name': None}, 'kernel_regularizer': None, 'bias_regularizer': None, 'activity_regularizer': None, 'kernel_constraint': None, 'bias_constraint': None}, 'registered_name': None, 'build_config': {'input_shape': (None, 3)}, 'name': 'y2', 'inbound_nodes': [[['dense_1', 0, 0, {}]]]}], 'input_layers': [['input_1', 0, 0]], 'output_layers': [['y1', 0, 0], ['y2', 0, 0]]} + +smlp_logger - INFO - Epochs: 100 + +smlp_logger - INFO - Batch size: 200 + +smlp_logger - INFO - Callbacks: [""] + +smlp_logger - INFO - model summary: end + +smlp_logger - INFO - _keras_train_multi_response: end + +smlp_logger - INFO - keras_main: end + +smlp_logger - INFO - Model training: end + +smlp_logger - INFO - PREDICT ON TRAINING DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test61_smlp_toy_num_resp_noknobs_training_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test61_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv + +smlp_logger - INFO - Prediction on training data -- msqe: 5.824 + +smlp_logger - INFO - Prediction on training data -- r2_score: -0.493 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON TEST DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test61_smlp_toy_num_resp_noknobs_test_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test61_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv + +smlp_logger - INFO - Prediction on test data -- msqe: 5.280 + +smlp_logger - INFO - Prediction on test data -- r2_score: -0.485 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON LABELED DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test61_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test61_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv + +smlp_logger - INFO - Prediction on labeled data -- msqe: 5.676 + +smlp_logger - INFO - Prediction on labeled data -- r2_score: -0.431 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - Creating model exploration base components: Start + +smlp_logger - INFO - Parsing the SPEC: Start + +smlp_logger - INFO - Parsing the SPEC: End + +smlp_logger - INFO - Variable domains (alpha): {'y1': {'range': 'float', 'interval': None}, 'y2': {'range': 'float', 'interval': None}, 'x0': {'range': 'float', 'interval': [0, 10]}, 'x1': {'range': 'float', 'interval': [0, 10]}, 'x2': {'range': 'float', 'interval': [3, 7]}} + +smlp_logger - INFO - Input bounds (alpha): {'x0': {'min': 0, 'max': 10}, 'x1': {'min': 0, 'max': 10}, 'x2': {'min': 3, 'max': 7}} + +smlp_logger - INFO - Knob bounds (eta): {} + +smlp_logger - INFO - Knob grids (eta): {} + +smlp_logger - INFO - Alpha global constraints: (or (or (= x1 1) (= x1 4)) (= x1 7)) + +smlp_logger - INFO - Alpha ranges constraints: (and (and (and true (and (>= x0 0) (<= x0 10))) (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) + +smlp_logger - INFO - Alpha combined constraints: (and (and (and (and true (and (>= x0 0) (<= x0 10))) (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) (or (or (= x1 1) (= x1 4)) (= x1 7))) + +smlp_logger - INFO - Beta global constraints: true + +smlp_logger - INFO - Eta ranges constraints: true + +smlp_logger - INFO - Eta grid constraints: true + +smlp_logger - INFO - Eta global constraints: true + +smlp_logger - INFO - Eta combined constraints: true + +smlp_logger - INFO - Creating model exploration base components: End + +smlp_logger - INFO - Input and knob interface constraints are consistent + +smlp_logger - INFO - Building model terms: Start + +smlp_logger - INFO - Model operator counts for y1: {'add': 256, 'mul': 472, 'ite': 39, 'prop': 39, 'const': 846, 'sub': 216, 'var': 216} + +smlp_logger - INFO - Model operator counts for y2: {'add': 256, 'mul': 472, 'ite': 39, 'prop': 39, 'const': 846, 'sub': 216, 'var': 216} + +smlp_logger - INFO - Building model terms: End + +smlp_logger - INFO - Model interface constraints are consistent + +smlp_logger - INFO - Verifying assertion asrt_y1 <-> not(x25 and y1<=10) + +smlp_logger - INFO - The configuration is consistent with assertion asrt_y1 + +smlp_logger - INFO - Completed with result: PASS + +smlp_logger - INFO - Verifying assertion asrt_y2 <-> -2*y2-1<10-x2 and x2>5 and x2<8 + +smlp_logger - INFO - The configuration is consistent with assertion asrt_y2 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Running SMLP in mode "verify": End + +smlp_logger - INFO - Executing run_smlp.py script: End diff --git a/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_data_bounds.json b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_data_bounds.json new file mode 100644 index 00000000..5df59662 --- /dev/null +++ b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_data_bounds.json @@ -0,0 +1,22 @@ +{ + "x0": { + "min": 9.0, + "max": 12.0 + }, + "x1": { + "min": 2.0, + "max": 4.0 + }, + "x2": { + "min": 3.0, + "max": 8.0 + }, + "y1": { + "min": 5.0, + "max": 9.0 + }, + "y2": { + "min": 5.0, + "max": 9.0 + } +} \ No newline at end of file diff --git a/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_features_scaler.pkl b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_features_scaler.pkl new file mode 100644 index 00000000..d12c5399 Binary files /dev/null and b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_features_scaler.pkl differ diff --git a/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv new file mode 100644 index 00000000..b86651fc --- /dev/null +++ b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,6.276045441340474,-0.5820864550045779 +y2,5.07613741261209,-0.27960963942929795 diff --git a/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv new file mode 100644 index 00000000..219f12ea --- /dev/null +++ b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv @@ -0,0 +1,12 @@ +,y1,y2,y1_nn_keras,y2_nn_keras +0,5.0,9.0,5.342773,5.4558725 +1,9.0,9.0,4.8616014,5.949031 +2,5.0,9.0,5.4982963,5.6901507 +3,5.0,5.0,5.342773,5.4558725 +4,9.0,5.0,5.342773,5.4558725 +5,9.0,9.0,5.342773,5.4558725 +6,5.0,5.0,5.342773,5.4558725 +7,5.0,5.0,5.4982963,5.6901507 +8,9.0,9.0,5.7109795,6.0105333 +9,5.0,5.0,5.342773,5.4558725 +10,9.0,5.0,5.342773,5.4558725 diff --git a/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_missing_values_dict.json b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_missing_values_dict.json new file mode 100644 index 00000000..553bfe33 --- /dev/null +++ b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_missing_values_dict.json @@ -0,0 +1,8 @@ +{ + "x1": [ + 1 + ], + "x0": [ + 2 + ] +} \ No newline at end of file diff --git a/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_model_checkpoint.h5 b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_model_checkpoint.h5 new file mode 100644 index 00000000..108970d2 Binary files /dev/null and b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_model_checkpoint.h5 differ diff --git a/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_model_features_dict.json b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_model_features_dict.json new file mode 100644 index 00000000..8e6a1c9a --- /dev/null +++ b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_model_features_dict.json @@ -0,0 +1,12 @@ +{ + "y1": [ + "x0", + "x1", + "x2" + ], + "y2": [ + "x0", + "x1", + "x2" + ] +} \ No newline at end of file diff --git a/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_model_gen.json b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_model_gen.json new file mode 100644 index 00000000..18ffb149 --- /dev/null +++ b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_model_gen.json @@ -0,0 +1 @@ +{"train": {"layers": "2,1", "epochs": 100, "batch-size": 200, "optimizer": "adam", "learning_rate": 0.001, "loss_function": "mse", "hid_activation": "relu", "out_activation": "linear", "sequential_api": false, "seed": 10}} \ No newline at end of file diff --git a/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_model_levels_dict.json b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_model_levels_dict.json new file mode 100644 index 00000000..9e26dfee --- /dev/null +++ b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_model_levels_dict.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_nn_keras_model_complete.h5 b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_nn_keras_model_complete.h5 new file mode 100644 index 00000000..108970d2 Binary files /dev/null and b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_nn_keras_model_complete.h5 differ diff --git a/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_responses_scaler.pkl b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_responses_scaler.pkl new file mode 100644 index 00000000..f2576be2 Binary files /dev/null and b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_responses_scaler.pkl differ diff --git a/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_smlp_full_model_term.json b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_smlp_full_model_term.json new file mode 100644 index 00000000..1e427884 --- /dev/null +++ b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_smlp_full_model_term.json @@ -0,0 +1 @@ +"{'y1': = |:3| 0) |:3| 0))) (let ((|:5| (+ (+ (+ (* |:0| (/ (- 12675489) 16777216)) (* |:1| (/ (- 15700433) 33554432))) (* |:2| (/ (- 14143067) 33554432))) 0))) (let ((|:6| (ite (>= |:5| 0) |:5| 0))) (let ((|:7| (+ (+ (+ (* |:0| (/ 10432239 16777216)) (* |:1| (/ (- 5448211) 16777216))) (* |:2| (/ 10235885 16777216))) (/ (- 7545327) 134217728)))) (let ((|:8| (ite (>= |:7| 0) |:7| 0))) (let ((|:9| (+ (+ (+ (* |:0| (/ 634497 8388608)) (* |:1| (/ 211449 4194304))) (* |:2| (/ 14539073 16777216))) (/ 15597927 268435456)))) (let ((|:10| (ite (>= |:9| 0) |:9| 0))) (let ((|:11| (+ (+ (+ (* |:0| (/ 73847 524288)) (* |:1| (/ 965293 2097152))) (* |:2| (/ 12164893 268435456))) (/ 7862711 134217728)))) (let ((|:12| (ite (>= |:11| 0) |:11| 0))) (let ((|:13| (+ (+ (+ (* |:0| (/ 9113453 16777216)) (* |:1| (/ 2938155 4194304))) (* |:2| (/ (- 5855175) 16777216))) (/ 8139523 134217728)))) (let ((|:14| (ite (>= |:13| 0) |:13| 0))) (let ((|:15| (+ (+ (+ (+ (+ (+ (* |:4| (/ 1958803 8388608)) (* |:6| (/ 3285671 8388608))) (* |:8| (/ (- 1798427) 8388608))) (* |:10| (/ (- 14391915) 33554432))) (* |:12| (/ (- 5538481) 8388608))) (* |:14| (/ (- 11806169) 33554432))) 0))) (let ((|:16| (+ (+ (+ (+ (+ (+ (* |:4| (/ 2695065 4194304)) (* |:6| (/ 2493617 16777216))) (* |:8| (/ 12598207 16777216))) (* |:10| (/ (- 13203831) 16777216))) (* |:12| (/ (- 2752055) 4194304))) (* |:14| (/ (- 14256635) 33554432))) (/ (- 15332351) 268435456)))) (let ((|:17| (+ (+ (+ (+ (+ (+ (* |:4| (/ (- 13799583) 33554432)) (* |:6| (/ (- 214595) 4194304))) (* |:8| (/ (- 4053207) 16777216))) (* |:10| (/ 315509 8388608))) (* |:12| (/ (- 10656705) 16777216))) (* |:14| (/ 4727225 8388608))) 0))) (+ (* (+ (+ (+ (* (ite (>= |:15| 0) |:15| 0) (/ (- 21323) 32768)) (* (ite (>= |:16| 0) |:16| 0) (/ (- 1127807) 1048576))) (* (ite (>= |:17| 0) |:17| 0) (/ 2984211 4194304))) (/ 11501553 134217728)) 4) 5)))))))))))))))))))>, 'y2': = |:3| 0) |:3| 0))) (let ((|:5| (+ (+ (+ (* |:0| (/ (- 12675489) 16777216)) (* |:1| (/ (- 15700433) 33554432))) (* |:2| (/ (- 14143067) 33554432))) 0))) (let ((|:6| (ite (>= |:5| 0) |:5| 0))) (let ((|:7| (+ (+ (+ (* |:0| (/ 10432239 16777216)) (* |:1| (/ (- 5448211) 16777216))) (* |:2| (/ 10235885 16777216))) (/ (- 7545327) 134217728)))) (let ((|:8| (ite (>= |:7| 0) |:7| 0))) (let ((|:9| (+ (+ (+ (* |:0| (/ 634497 8388608)) (* |:1| (/ 211449 4194304))) (* |:2| (/ 14539073 16777216))) (/ 15597927 268435456)))) (let ((|:10| (ite (>= |:9| 0) |:9| 0))) (let ((|:11| (+ (+ (+ (* |:0| (/ 73847 524288)) (* |:1| (/ 965293 2097152))) (* |:2| (/ 12164893 268435456))) (/ 7862711 134217728)))) (let ((|:12| (ite (>= |:11| 0) |:11| 0))) (let ((|:13| (+ (+ (+ (* |:0| (/ 9113453 16777216)) (* |:1| (/ 2938155 4194304))) (* |:2| (/ (- 5855175) 16777216))) (/ 8139523 134217728)))) (let ((|:14| (ite (>= |:13| 0) |:13| 0))) (let ((|:15| (+ (+ (+ (+ (+ (+ (* |:4| (/ 1958803 8388608)) (* |:6| (/ 3285671 8388608))) (* |:8| (/ (- 1798427) 8388608))) (* |:10| (/ (- 14391915) 33554432))) (* |:12| (/ (- 5538481) 8388608))) (* |:14| (/ (- 11806169) 33554432))) 0))) (let ((|:16| (+ (+ (+ (+ (+ (+ (* |:4| (/ 2695065 4194304)) (* |:6| (/ 2493617 16777216))) (* |:8| (/ 12598207 16777216))) (* |:10| (/ (- 13203831) 16777216))) (* |:12| (/ (- 2752055) 4194304))) (* |:14| (/ (- 14256635) 33554432))) (/ (- 15332351) 268435456)))) (let ((|:17| (+ (+ (+ (+ (+ (+ (* |:4| (/ (- 13799583) 33554432)) (* |:6| (/ (- 214595) 4194304))) (* |:8| (/ (- 4053207) 16777216))) (* |:10| (/ 315509 8388608))) (* |:12| (/ (- 10656705) 16777216))) (* |:14| (/ 4727225 8388608))) 0))) (+ (* (+ (+ (+ (* (ite (>= |:15| 0) |:15| 0) (/ (- 1922803) 4194304)) (* (ite (>= |:16| 0) |:16| 0) (/ 9247207 8388608))) (* (ite (>= |:17| 0) |:17| 0) (/ 8990739 8388608))) (/ 15296551 134217728)) 4) 5)))))))))))))))))))>}" \ No newline at end of file diff --git a/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_smlp_model_term.json b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_smlp_model_term.json new file mode 100644 index 00000000..8dd89b59 --- /dev/null +++ b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_smlp_model_term.json @@ -0,0 +1 @@ +"{'y1_scaled': = |:0| 0) |:0| 0))) (let ((|:2| (+ (+ (+ (* x0_scaled (/ (- 12675489) 16777216)) (* x1_scaled (/ (- 15700433) 33554432))) (* x2_scaled (/ (- 14143067) 33554432))) 0))) (let ((|:3| (ite (>= |:2| 0) |:2| 0))) (let ((|:4| (+ (+ (+ (* x0_scaled (/ 10432239 16777216)) (* x1_scaled (/ (- 5448211) 16777216))) (* x2_scaled (/ 10235885 16777216))) (/ (- 7545327) 134217728)))) (let ((|:5| (ite (>= |:4| 0) |:4| 0))) (let ((|:6| (+ (+ (+ (* x0_scaled (/ 634497 8388608)) (* x1_scaled (/ 211449 4194304))) (* x2_scaled (/ 14539073 16777216))) (/ 15597927 268435456)))) (let ((|:7| (ite (>= |:6| 0) |:6| 0))) (let ((|:8| (+ (+ (+ (* x0_scaled (/ 73847 524288)) (* x1_scaled (/ 965293 2097152))) (* x2_scaled (/ 12164893 268435456))) (/ 7862711 134217728)))) (let ((|:9| (ite (>= |:8| 0) |:8| 0))) (let ((|:10| (+ (+ (+ (* x0_scaled (/ 9113453 16777216)) (* x1_scaled (/ 2938155 4194304))) (* x2_scaled (/ (- 5855175) 16777216))) (/ 8139523 134217728)))) (let ((|:11| (ite (>= |:10| 0) |:10| 0))) (let ((|:12| (+ (+ (+ (+ (+ (+ (* |:1| (/ 1958803 8388608)) (* |:3| (/ 3285671 8388608))) (* |:5| (/ (- 1798427) 8388608))) (* |:7| (/ (- 14391915) 33554432))) (* |:9| (/ (- 5538481) 8388608))) (* |:11| (/ (- 11806169) 33554432))) 0))) (let ((|:13| (+ (+ (+ (+ (+ (+ (* |:1| (/ 2695065 4194304)) (* |:3| (/ 2493617 16777216))) (* |:5| (/ 12598207 16777216))) (* |:7| (/ (- 13203831) 16777216))) (* |:9| (/ (- 2752055) 4194304))) (* |:11| (/ (- 14256635) 33554432))) (/ (- 15332351) 268435456)))) (let ((|:14| (+ (+ (+ (+ (+ (+ (* |:1| (/ (- 13799583) 33554432)) (* |:3| (/ (- 214595) 4194304))) (* |:5| (/ (- 4053207) 16777216))) (* |:7| (/ 315509 8388608))) (* |:9| (/ (- 10656705) 16777216))) (* |:11| (/ 4727225 8388608))) 0))) (+ (+ (+ (* (ite (>= |:12| 0) |:12| 0) (/ (- 21323) 32768)) (* (ite (>= |:13| 0) |:13| 0) (/ (- 1127807) 1048576))) (* (ite (>= |:14| 0) |:14| 0) (/ 2984211 4194304))) (/ 11501553 134217728)))))))))))))))))>, 'y2_scaled': = |:0| 0) |:0| 0))) (let ((|:2| (+ (+ (+ (* x0_scaled (/ (- 12675489) 16777216)) (* x1_scaled (/ (- 15700433) 33554432))) (* x2_scaled (/ (- 14143067) 33554432))) 0))) (let ((|:3| (ite (>= |:2| 0) |:2| 0))) (let ((|:4| (+ (+ (+ (* x0_scaled (/ 10432239 16777216)) (* x1_scaled (/ (- 5448211) 16777216))) (* x2_scaled (/ 10235885 16777216))) (/ (- 7545327) 134217728)))) (let ((|:5| (ite (>= |:4| 0) |:4| 0))) (let ((|:6| (+ (+ (+ (* x0_scaled (/ 634497 8388608)) (* x1_scaled (/ 211449 4194304))) (* x2_scaled (/ 14539073 16777216))) (/ 15597927 268435456)))) (let ((|:7| (ite (>= |:6| 0) |:6| 0))) (let ((|:8| (+ (+ (+ (* x0_scaled (/ 73847 524288)) (* x1_scaled (/ 965293 2097152))) (* x2_scaled (/ 12164893 268435456))) (/ 7862711 134217728)))) (let ((|:9| (ite (>= |:8| 0) |:8| 0))) (let ((|:10| (+ (+ (+ (* x0_scaled (/ 9113453 16777216)) (* x1_scaled (/ 2938155 4194304))) (* x2_scaled (/ (- 5855175) 16777216))) (/ 8139523 134217728)))) (let ((|:11| (ite (>= |:10| 0) |:10| 0))) (let ((|:12| (+ (+ (+ (+ (+ (+ (* |:1| (/ 1958803 8388608)) (* |:3| (/ 3285671 8388608))) (* |:5| (/ (- 1798427) 8388608))) (* |:7| (/ (- 14391915) 33554432))) (* |:9| (/ (- 5538481) 8388608))) (* |:11| (/ (- 11806169) 33554432))) 0))) (let ((|:13| (+ (+ (+ (+ (+ (+ (* |:1| (/ 2695065 4194304)) (* |:3| (/ 2493617 16777216))) (* |:5| (/ 12598207 16777216))) (* |:7| (/ (- 13203831) 16777216))) (* |:9| (/ (- 2752055) 4194304))) (* |:11| (/ (- 14256635) 33554432))) (/ (- 15332351) 268435456)))) (let ((|:14| (+ (+ (+ (+ (+ (+ (* |:1| (/ (- 13799583) 33554432)) (* |:3| (/ (- 214595) 4194304))) (* |:5| (/ (- 4053207) 16777216))) (* |:7| (/ 315509 8388608))) (* |:9| (/ (- 10656705) 16777216))) (* |:11| (/ 4727225 8388608))) 0))) (+ (+ (+ (* (ite (>= |:12| 0) |:12| 0) (/ (- 1922803) 4194304)) (* (ite (>= |:13| 0) |:13| 0) (/ 9247207 8388608))) (* (ite (>= |:14| 0) |:14| 0) (/ 8990739 8388608))) (/ 15296551 134217728)))))))))))))))))>}" \ No newline at end of file diff --git a/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv new file mode 100644 index 00000000..2c6d8bc4 --- /dev/null +++ b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,3.77141814289295,-0.06071135268864225 +y2,6.789440377906885,-0.9095301062863115 diff --git a/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_test_predictions_summary.csv b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_test_predictions_summary.csv new file mode 100644 index 00000000..107ed38e --- /dev/null +++ b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_test_predictions_summary.csv @@ -0,0 +1,4 @@ +,y1,y2,y1_nn_keras,y2_nn_keras +7,5.0,5.0,5.4982963,5.6901507 +2,5.0,9.0,5.4982963,5.6901507 +8,9.0,9.0,5.7109795,6.0105333 diff --git a/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_trace.csv b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_trace.csv new file mode 100644 index 00000000..05070776 --- /dev/null +++ b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_trace.csv @@ -0,0 +1,7 @@ +stage,solver,x0,x1,x2,y1,y2 +interface_consistency,sat,0,1,3 +model_consistency,sat,518763182342818807/59767095804418640,1,7,179273713/33554432,183068711/33554432 +ca,sat,676369427545304779759800936665/71922027751977055989612412928,1,1314087680626428725153770109981/287688111007908223958449651712,179273713/33554432,183068711/33554432 +ce,unsat +ca,sat,725886971/84503260,1,7,179273713/33554432,183068711/33554432 +ce,sat,139131163646518853343267/14978445805719660789760,1,7358368615260560938285/1497844580571966078976,179273713/33554432,183068711/33554432 diff --git a/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv new file mode 100644 index 00000000..d398630f --- /dev/null +++ b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,7.215280678258296,-0.8038201695645739 +y2,4.433648800626543,-0.1823063468337447 diff --git a/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_training_predictions_summary.csv b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_training_predictions_summary.csv new file mode 100644 index 00000000..9a705e18 --- /dev/null +++ b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_training_predictions_summary.csv @@ -0,0 +1,9 @@ +,y1,y2,y1_nn_keras,y2_nn_keras +3,5.0,5.0,5.342773,5.4558725 +4,9.0,5.0,5.342773,5.4558725 +5,9.0,9.0,5.342773,5.4558725 +0,5.0,9.0,5.342773,5.4558725 +10,9.0,5.0,5.342773,5.4558725 +9,5.0,5.0,5.342773,5.4558725 +6,5.0,5.0,5.342773,5.4558725 +1,9.0,9.0,4.8616014,5.949031 diff --git a/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_verify_results.json b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_verify_results.json new file mode 100644 index 00000000..fd37778a --- /dev/null +++ b/regr_smlp/master/Test61_smlp_toy_num_resp_noknobs_verify_results.json @@ -0,0 +1,23 @@ +{ + "asrt_y1": { + "configuration_consistent": "skipped", + "assertion_status": "PASS", + "counter_example": null, + "assertion_feasible": true + }, + "asrt_y2": { + "configuration_consistent": "skipped", + "assertion_status": "FAIL", + "counter_example": { + "x0": 9.288758356583985, + "x1": 1.0, + "y1": 5.342772990465164, + "x2": 4.912638274159725, + "y2": 5.455872744321823 + }, + "assertion_feasible": true + }, + "smlp_execution": "completed", + "interface_consistent": "skipped", + "model_consistent": "true" +} \ No newline at end of file diff --git a/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs.txt b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs.txt new file mode 100644 index 00000000..8a018ce0 --- /dev/null +++ b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs.txt @@ -0,0 +1,305 @@ + +smlp_logger - INFO - Model exploration specification: +{'version': '1.1', 'spec': [{'label': 'y1', 'type': 'response', 'range': 'float'}, {'label': 'y2', 'type': 'response', 'range': 'float'}, {'label': 'x0', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x1', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x2', 'type': 'input', 'range': 'float', 'bounds': [3, 7]}], 'alpha': 'x1==1 or x1==4 or x1==7'} + +smlp_logger - INFO - Executing run_smlp.py script: Start + +smlp_logger - INFO - Running SMLP in mode "verify": Start + +smlp_logger - INFO - Computed spec global constraint expressions: + +smlp_logger - INFO - Global alpha : x1==1 or x1==4 or x1==7 + +smlp_logger - INFO - Global beta : None + +smlp_logger - INFO - Radii theta : {} + +smlp_logger - INFO - Delta const : {'delta_abs': 0.0, 'delta_rel': 0.01} + +smlp_logger - INFO - Assertion asrt_y1: not(x25 and y1<=10) + +smlp_logger - INFO - Assertion asrt_y2: -2*y2-1<10-x2 and x2>5 and x2<8 + +smlp_logger - INFO - PREPARE DATA FOR MODELING + +smlp_logger - INFO - Preparing training data for modeling: start + +smlp_logger - INFO - loading training data + +smlp_logger - INFO - data summary + y1 y2 x0 x1 x2 +count 11.000000 11.000000 10.000000 10.000000 11.000000 +mean 6.818182 6.818182 10.400000 2.800000 5.454545 +std 2.088932 2.088932 1.074968 0.788811 1.694912 +min 5.000000 5.000000 9.000000 2.000000 3.000000 +25% 5.000000 5.000000 10.000000 2.000000 4.000000 +50% 5.000000 5.000000 10.000000 3.000000 6.000000 +75% 9.000000 9.000000 11.000000 3.000000 7.000000 +max 9.000000 9.000000 12.000000 4.000000 8.000000 + +smlp_logger - INFO - training data + categ y1 y2 x0 x1 x2 +0 c14 5 9 10.0 2.0 3 +1 c15 9 9 12.0 NaN 4 +2 c1 5 9 NaN 3.0 4 +3 c9 5 5 11.0 2.0 6 +4 c5 9 5 10.0 2.0 8 +5 c10 9 9 9.0 4.0 7 +6 c13 5 5 9.0 3.0 6 +7 c4 5 5 10.0 3.0 4 +8 c15 9 9 11.0 4.0 4 +9 c11 5 5 12.0 2.0 7 +10 c19 9 5 10.0 3.0 7 + +smlp_logger - INFO - training data after imputing missing values + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - training data after processing responses + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - training data after encoding levels of categorical features with integers + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - training data after scaling (normalizing) features and responses + x0 x1 x2 y1 y2 +0 0.333333 0.0 0.0 0.0 1.0 +1 1.000000 0.0 0.2 1.0 1.0 +2 0.333333 0.5 0.2 0.0 1.0 +3 0.666667 0.0 0.6 0.0 0.0 +4 0.333333 0.0 1.0 1.0 0.0 +5 0.000000 1.0 0.8 1.0 1.0 +6 0.000000 0.5 0.6 0.0 0.0 +7 0.333333 0.5 0.2 0.0 0.0 +8 0.666667 1.0 0.2 1.0 1.0 +9 1.000000 0.0 0.8 0.0 0.0 +10 0.333333 0.5 0.8 1.0 0.0 + +smlp_logger - INFO - Sampling from training data: start + +smlp_logger - INFO - Sampling from training data: end + +smlp_logger - INFO - X_train after sampling: (8, 3) + +smlp_logger - INFO - y_train after sampling: (8, 2) + +smlp_logger - INFO - Preparing training data for modeling: end + +smlp_logger - INFO - Saving data bounds into file:./Test62_smlp_toy_num_resp_noknobs_data_bounds.json + +smlp_logger - INFO - {'x0': {'min': 9.0, 'max': 12.0}, 'x1': {'min': 2.0, 'max': 4.0}, 'x2': {'min': 3.0, 'max': 8.0}, 'y1': {'min': 5.0, 'max': 9.0}, 'y2': {'min': 5.0, 'max': 9.0}} + +smlp_logger - INFO - TRAIN MODEL + +smlp_logger - INFO - Model training: start + +smlp_logger - INFO - keras_main: start + +smlp_logger - INFO - _keras_train_multi_response: start + +smlp_logger - INFO - layers_spec_list [2.0, 1.0] + +smlp_logger - INFO - building NN model using Keras Sequential API + +smlp_logger - INFO - input layer of size 3 + +smlp_logger - INFO - dense layer of size 6 + +smlp_logger - INFO - dense layer of size 3 + +smlp_logger - INFO - output layer of size 2 + +smlp_logger - INFO - model summary: start + +smlp_logger - INFO - Model: "sequential" +_________________________________________________________________ + Layer (type) Output Shape Param # +================================================================= + dense (Dense) (None, 6) 24 + + dense_1 (Dense) (None, 3) 21 + + dense_2 (Dense) (None, 2) 8 + +================================================================= +Total params: 53 (212.00 Byte) +Trainable params: 53 (212.00 Byte) +Non-trainable params: 0 (0.00 Byte) +_________________________________________________________________ + + +smlp_logger - INFO - Optimizer: {'name': 'Adam', 'weight_decay': None, 'clipnorm': None, 'global_clipnorm': None, 'clipvalue': None, 'use_ema': False, 'ema_momentum': 0.99, 'ema_overwrite_frequency': None, 'jit_compile': False, 'is_legacy_optimizer': False, 'learning_rate': 0.001, 'beta_1': 0.9, 'beta_2': 0.999, 'epsilon': 1e-07, 'amsgrad': False} + +smlp_logger - INFO - Learning rate: 0.001 + +smlp_logger - INFO - Loss function: mse + +smlp_logger - INFO - Metrics: ['mse'] + +smlp_logger - INFO - Model configuration: {'name': 'sequential', 'layers': [{'module': 'keras.layers', 'class_name': 'InputLayer', 'config': {'batch_input_shape': (None, 3), 'dtype': 'float32', 'sparse': False, 'ragged': False, 'name': 'dense_input'}, 'registered_name': None}, {'module': 'keras.layers', 'class_name': 'Dense', 'config': {'name': 'dense', 'trainable': True, 'dtype': 'float32', 'batch_input_shape': (None, 3), 'units': 6, 'activation': 'relu', 'use_bias': True, 'kernel_initializer': {'module': 'keras.initializers', 'class_name': 'GlorotUniform', 'config': {'seed': None}, 'registered_name': None}, 'bias_initializer': {'module': 'keras.initializers', 'class_name': 'Zeros', 'config': {}, 'registered_name': None}, 'kernel_regularizer': None, 'bias_regularizer': None, 'activity_regularizer': None, 'kernel_constraint': None, 'bias_constraint': None}, 'registered_name': None, 'build_config': {'input_shape': (None, 3)}}, {'module': 'keras.layers', 'class_name': 'Dense', 'config': {'name': 'dense_1', 'trainable': True, 'dtype': 'float32', 'units': 3, 'activation': 'relu', 'use_bias': True, 'kernel_initializer': {'module': 'keras.initializers', 'class_name': 'GlorotUniform', 'config': {'seed': None}, 'registered_name': None}, 'bias_initializer': {'module': 'keras.initializers', 'class_name': 'Zeros', 'config': {}, 'registered_name': None}, 'kernel_regularizer': None, 'bias_regularizer': None, 'activity_regularizer': None, 'kernel_constraint': None, 'bias_constraint': None}, 'registered_name': None, 'build_config': {'input_shape': (None, 6)}}, {'module': 'keras.layers', 'class_name': 'Dense', 'config': {'name': 'dense_2', 'trainable': True, 'dtype': 'float32', 'units': 2, 'activation': 'linear', 'use_bias': True, 'kernel_initializer': {'module': 'keras.initializers', 'class_name': 'GlorotUniform', 'config': {'seed': None}, 'registered_name': None}, 'bias_initializer': {'module': 'keras.initializers', 'class_name': 'Zeros', 'config': {}, 'registered_name': None}, 'kernel_regularizer': None, 'bias_regularizer': None, 'activity_regularizer': None, 'kernel_constraint': None, 'bias_constraint': None}, 'registered_name': None, 'build_config': {'input_shape': (None, 3)}}]} + +smlp_logger - INFO - Epochs: 100 + +smlp_logger - INFO - Batch size: 200 + +smlp_logger - INFO - Callbacks: [""] + +smlp_logger - INFO - model summary: end + +smlp_logger - INFO - _keras_train_multi_response: end + +smlp_logger - INFO - keras_main: end + +smlp_logger - INFO - Model training: end + +smlp_logger - INFO - PREDICT ON TRAINING DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test62_smlp_toy_num_resp_noknobs_training_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test62_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv + +smlp_logger - INFO - Prediction on training data -- msqe: 5.344 + +smlp_logger - INFO - Prediction on training data -- r2_score: -0.378 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON TEST DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test62_smlp_toy_num_resp_noknobs_test_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test62_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv + +smlp_logger - INFO - Prediction on test data -- msqe: 6.431 + +smlp_logger - INFO - Prediction on test data -- r2_score: -0.809 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON LABELED DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test62_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test62_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv + +smlp_logger - INFO - Prediction on labeled data -- msqe: 5.640 + +smlp_logger - INFO - Prediction on labeled data -- r2_score: -0.422 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - Creating model exploration base components: Start + +smlp_logger - INFO - Parsing the SPEC: Start + +smlp_logger - INFO - Parsing the SPEC: End + +smlp_logger - INFO - Variable domains (alpha): {'y1': {'range': 'float', 'interval': None}, 'y2': {'range': 'float', 'interval': None}, 'x0': {'range': 'float', 'interval': [0, 10]}, 'x1': {'range': 'float', 'interval': [0, 10]}, 'x2': {'range': 'float', 'interval': [3, 7]}} + +smlp_logger - INFO - Input bounds (alpha): {'x0': {'min': 0, 'max': 10}, 'x1': {'min': 0, 'max': 10}, 'x2': {'min': 3, 'max': 7}} + +smlp_logger - INFO - Knob bounds (eta): {} + +smlp_logger - INFO - Knob grids (eta): {} + +smlp_logger - INFO - Alpha global constraints: (or (or (= x1 1) (= x1 4)) (= x1 7)) + +smlp_logger - INFO - Alpha ranges constraints: (and (and (and true (and (>= x0 0) (<= x0 10))) (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) + +smlp_logger - INFO - Alpha combined constraints: (and (and (and (and true (and (>= x0 0) (<= x0 10))) (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) (or (or (= x1 1) (= x1 4)) (= x1 7))) + +smlp_logger - INFO - Beta global constraints: true + +smlp_logger - INFO - Eta ranges constraints: true + +smlp_logger - INFO - Eta grid constraints: true + +smlp_logger - INFO - Eta global constraints: true + +smlp_logger - INFO - Eta combined constraints: true + +smlp_logger - INFO - Creating model exploration base components: End + +smlp_logger - INFO - Input and knob interface constraints are consistent + +smlp_logger - INFO - Building model terms: Start + +smlp_logger - INFO - Model operator counts for y1: {'add': 256, 'mul': 472, 'ite': 39, 'prop': 39, 'const': 846, 'sub': 216, 'var': 216} + +smlp_logger - INFO - Model operator counts for y2: {'add': 256, 'mul': 472, 'ite': 39, 'prop': 39, 'const': 846, 'sub': 216, 'var': 216} + +smlp_logger - INFO - Building model terms: End + +smlp_logger - INFO - Model interface constraints are consistent + +smlp_logger - INFO - Verifying assertion asrt_y1 <-> not(x25 and y1<=10) + +smlp_logger - INFO - The configuration is consistent with assertion asrt_y1 + +smlp_logger - INFO - Completed with result: PASS + +smlp_logger - INFO - Verifying assertion asrt_y2 <-> -2*y2-1<10-x2 and x2>5 and x2<8 + +smlp_logger - INFO - The configuration is consistent with assertion asrt_y2 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Running SMLP in mode "verify": End + +smlp_logger - INFO - Executing run_smlp.py script: End diff --git a/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_data_bounds.json b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_data_bounds.json new file mode 100644 index 00000000..5df59662 --- /dev/null +++ b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_data_bounds.json @@ -0,0 +1,22 @@ +{ + "x0": { + "min": 9.0, + "max": 12.0 + }, + "x1": { + "min": 2.0, + "max": 4.0 + }, + "x2": { + "min": 3.0, + "max": 8.0 + }, + "y1": { + "min": 5.0, + "max": 9.0 + }, + "y2": { + "min": 5.0, + "max": 9.0 + } +} \ No newline at end of file diff --git a/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_features_scaler.pkl b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_features_scaler.pkl new file mode 100644 index 00000000..d12c5399 Binary files /dev/null and b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_features_scaler.pkl differ diff --git a/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv new file mode 100644 index 00000000..4b357dd5 --- /dev/null +++ b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,5.381964143430403,-0.35670346115641394 +y2,5.898617950921727,-0.48694327512818547 diff --git a/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv new file mode 100644 index 00000000..73b2c465 --- /dev/null +++ b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv @@ -0,0 +1,12 @@ +,y1,y2,y1_nn_keras,y2_nn_keras +0,5.0,9.0,5.8426776,5.3677473 +1,9.0,9.0,6.644639,5.3530006 +2,5.0,9.0,5.3453274,5.447993 +3,5.0,5.0,6.6467576,5.3529615 +4,9.0,5.0,6.6339836,5.3531966 +5,9.0,9.0,5.3769207,5.376312 +6,5.0,5.0,5.3769207,5.376312 +7,5.0,5.0,5.3453274,5.447993 +8,9.0,9.0,5.2932014,5.5662575 +9,5.0,5.0,7.063879,5.345291 +10,9.0,5.0,5.3769207,5.376312 diff --git a/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_missing_values_dict.json b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_missing_values_dict.json new file mode 100644 index 00000000..553bfe33 --- /dev/null +++ b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_missing_values_dict.json @@ -0,0 +1,8 @@ +{ + "x1": [ + 1 + ], + "x0": [ + 2 + ] +} \ No newline at end of file diff --git a/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_model_checkpoint.h5 b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_model_checkpoint.h5 new file mode 100644 index 00000000..951c94c8 Binary files /dev/null and b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_model_checkpoint.h5 differ diff --git a/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_model_features_dict.json b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_model_features_dict.json new file mode 100644 index 00000000..8e6a1c9a --- /dev/null +++ b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_model_features_dict.json @@ -0,0 +1,12 @@ +{ + "y1": [ + "x0", + "x1", + "x2" + ], + "y2": [ + "x0", + "x1", + "x2" + ] +} \ No newline at end of file diff --git a/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_model_gen.json b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_model_gen.json new file mode 100644 index 00000000..0995d138 --- /dev/null +++ b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_model_gen.json @@ -0,0 +1 @@ +{"train": {"layers": "2,1", "epochs": 100, "batch-size": 200, "optimizer": "adam", "learning_rate": 0.001, "loss_function": "mse", "hid_activation": "relu", "out_activation": "linear", "sequential_api": true, "seed": 10}} \ No newline at end of file diff --git a/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_model_levels_dict.json b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_model_levels_dict.json new file mode 100644 index 00000000..9e26dfee --- /dev/null +++ b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_model_levels_dict.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_nn_keras_model_complete.h5 b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_nn_keras_model_complete.h5 new file mode 100644 index 00000000..951c94c8 Binary files /dev/null and b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_nn_keras_model_complete.h5 differ diff --git a/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_responses_scaler.pkl b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_responses_scaler.pkl new file mode 100644 index 00000000..f2576be2 Binary files /dev/null and b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_responses_scaler.pkl differ diff --git a/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_smlp_full_model_term.json b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_smlp_full_model_term.json new file mode 100644 index 00000000..2c72d714 --- /dev/null +++ b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_smlp_full_model_term.json @@ -0,0 +1 @@ +"{'y1': = |:3| 0) |:3| 0))) (let ((|:5| (+ (+ (+ (* |:0| (/ (- 12675489) 16777216)) (* |:1| (/ (- 15700433) 33554432))) (* |:2| (/ (- 14143067) 33554432))) 0))) (let ((|:6| (ite (>= |:5| 0) |:5| 0))) (let ((|:7| (+ (+ (+ (* |:0| (/ 9900335 16777216)) (* |:1| (/ (- 5448211) 16777216))) (* |:2| (/ 12339293 16777216))) (/ 12035361 268435456)))) (let ((|:8| (ite (>= |:7| 0) |:7| 0))) (let ((|:9| (+ (+ (+ (* |:0| (/ 11088463 134217728)) (* |:1| (/ 211449 4194304))) (* |:2| (/ 12447363 16777216))) (/ (- 12828253) 134217728)))) (let ((|:10| (ite (>= |:9| 0) |:9| 0))) (let ((|:11| (+ (+ (+ (* |:0| (/ 10855307 67108864)) (* |:1| (/ 965293 2097152))) (* |:2| (/ (- 15015817) 268435456))) (/ 8969903 4294967296)))) (let ((|:12| (ite (>= |:11| 0) |:11| 0))) (let ((|:13| (+ (+ (+ (* |:0| (/ 1208381 2097152)) (* |:1| (/ 2938155 4194304))) (* |:2| (/ (- 10195595) 33554432))) (/ 3298773 33554432)))) (let ((|:14| (ite (>= |:13| 0) |:13| 0))) (let ((|:15| (+ (+ (+ (+ (+ (+ (* |:4| (/ 1958803 8388608)) (* |:6| (/ 3285671 8388608))) (* |:8| (/ (- 1798427) 8388608))) (* |:10| (/ (- 14391915) 33554432))) (* |:12| (/ (- 5538481) 8388608))) (* |:14| (/ (- 11806169) 33554432))) 0))) (let ((|:16| (+ (+ (+ (+ (+ (+ (* |:4| (/ 10348145 16777216)) (* |:6| (/ 2493617 16777216))) (* |:8| (/ 14212267 16777216))) (* |:10| (/ (- 5570941) 8388608))) (* |:12| (/ (- 11410359) 16777216))) (* |:14| (/ (- 15592633) 33554432))) (/ 5899637 134217728)))) (let ((|:17| (+ (+ (+ (+ (+ (+ (* |:4| (/ (- 13799583) 33554432)) (* |:6| (/ (- 214595) 4194304))) (* |:8| (/ (- 4053207) 16777216))) (* |:10| (/ 315509 8388608))) (* |:12| (/ (- 10656705) 16777216))) (* |:14| (/ 4727225 8388608))) 0))) (+ (* (+ (+ (+ (* (ite (>= |:15| 0) |:15| 0) (/ (- 4882399) 8388608)) (* (ite (>= |:16| 0) |:16| 0) (/ 4987691 8388608))) (* (ite (>= |:17| 0) |:17| 0) (/ (- 1315209) 8388608))) (/ 12647359 134217728)) 4) 5)))))))))))))))))))>, 'y2': = |:3| 0) |:3| 0))) (let ((|:5| (+ (+ (+ (* |:0| (/ (- 12675489) 16777216)) (* |:1| (/ (- 15700433) 33554432))) (* |:2| (/ (- 14143067) 33554432))) 0))) (let ((|:6| (ite (>= |:5| 0) |:5| 0))) (let ((|:7| (+ (+ (+ (* |:0| (/ 9900335 16777216)) (* |:1| (/ (- 5448211) 16777216))) (* |:2| (/ 12339293 16777216))) (/ 12035361 268435456)))) (let ((|:8| (ite (>= |:7| 0) |:7| 0))) (let ((|:9| (+ (+ (+ (* |:0| (/ 11088463 134217728)) (* |:1| (/ 211449 4194304))) (* |:2| (/ 12447363 16777216))) (/ (- 12828253) 134217728)))) (let ((|:10| (ite (>= |:9| 0) |:9| 0))) (let ((|:11| (+ (+ (+ (* |:0| (/ 10855307 67108864)) (* |:1| (/ 965293 2097152))) (* |:2| (/ (- 15015817) 268435456))) (/ 8969903 4294967296)))) (let ((|:12| (ite (>= |:11| 0) |:11| 0))) (let ((|:13| (+ (+ (+ (* |:0| (/ 1208381 2097152)) (* |:1| (/ 2938155 4194304))) (* |:2| (/ (- 10195595) 33554432))) (/ 3298773 33554432)))) (let ((|:14| (ite (>= |:13| 0) |:13| 0))) (let ((|:15| (+ (+ (+ (+ (+ (+ (* |:4| (/ 1958803 8388608)) (* |:6| (/ 3285671 8388608))) (* |:8| (/ (- 1798427) 8388608))) (* |:10| (/ (- 14391915) 33554432))) (* |:12| (/ (- 5538481) 8388608))) (* |:14| (/ (- 11806169) 33554432))) 0))) (let ((|:16| (+ (+ (+ (+ (+ (+ (* |:4| (/ 10348145 16777216)) (* |:6| (/ 2493617 16777216))) (* |:8| (/ 14212267 16777216))) (* |:10| (/ (- 5570941) 8388608))) (* |:12| (/ (- 11410359) 16777216))) (* |:14| (/ (- 15592633) 33554432))) (/ 5899637 134217728)))) (let ((|:17| (+ (+ (+ (+ (+ (+ (* |:4| (/ (- 13799583) 33554432)) (* |:6| (/ (- 214595) 4194304))) (* |:8| (/ (- 4053207) 16777216))) (* |:10| (/ 315509 8388608))) (* |:12| (/ (- 10656705) 16777216))) (* |:14| (/ 4727225 8388608))) 0))) (+ (* (+ (+ (+ (* (ite (>= |:15| 0) |:15| 0) (/ (- 4231593) 4194304)) (* (ite (>= |:16| 0) |:16| 0) (/ (- 11739661) 1073741824))) (* (ite (>= |:17| 0) |:17| 0) (/ 373001 1048576))) (/ 12626933 134217728)) 4) 5)))))))))))))))))))>}" \ No newline at end of file diff --git a/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_smlp_model_term.json b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_smlp_model_term.json new file mode 100644 index 00000000..1c8e0f4b --- /dev/null +++ b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_smlp_model_term.json @@ -0,0 +1 @@ +"{'y1_scaled': = |:0| 0) |:0| 0))) (let ((|:2| (+ (+ (+ (* x0_scaled (/ (- 12675489) 16777216)) (* x1_scaled (/ (- 15700433) 33554432))) (* x2_scaled (/ (- 14143067) 33554432))) 0))) (let ((|:3| (ite (>= |:2| 0) |:2| 0))) (let ((|:4| (+ (+ (+ (* x0_scaled (/ 9900335 16777216)) (* x1_scaled (/ (- 5448211) 16777216))) (* x2_scaled (/ 12339293 16777216))) (/ 12035361 268435456)))) (let ((|:5| (ite (>= |:4| 0) |:4| 0))) (let ((|:6| (+ (+ (+ (* x0_scaled (/ 11088463 134217728)) (* x1_scaled (/ 211449 4194304))) (* x2_scaled (/ 12447363 16777216))) (/ (- 12828253) 134217728)))) (let ((|:7| (ite (>= |:6| 0) |:6| 0))) (let ((|:8| (+ (+ (+ (* x0_scaled (/ 10855307 67108864)) (* x1_scaled (/ 965293 2097152))) (* x2_scaled (/ (- 15015817) 268435456))) (/ 8969903 4294967296)))) (let ((|:9| (ite (>= |:8| 0) |:8| 0))) (let ((|:10| (+ (+ (+ (* x0_scaled (/ 1208381 2097152)) (* x1_scaled (/ 2938155 4194304))) (* x2_scaled (/ (- 10195595) 33554432))) (/ 3298773 33554432)))) (let ((|:11| (ite (>= |:10| 0) |:10| 0))) (let ((|:12| (+ (+ (+ (+ (+ (+ (* |:1| (/ 1958803 8388608)) (* |:3| (/ 3285671 8388608))) (* |:5| (/ (- 1798427) 8388608))) (* |:7| (/ (- 14391915) 33554432))) (* |:9| (/ (- 5538481) 8388608))) (* |:11| (/ (- 11806169) 33554432))) 0))) (let ((|:13| (+ (+ (+ (+ (+ (+ (* |:1| (/ 10348145 16777216)) (* |:3| (/ 2493617 16777216))) (* |:5| (/ 14212267 16777216))) (* |:7| (/ (- 5570941) 8388608))) (* |:9| (/ (- 11410359) 16777216))) (* |:11| (/ (- 15592633) 33554432))) (/ 5899637 134217728)))) (let ((|:14| (+ (+ (+ (+ (+ (+ (* |:1| (/ (- 13799583) 33554432)) (* |:3| (/ (- 214595) 4194304))) (* |:5| (/ (- 4053207) 16777216))) (* |:7| (/ 315509 8388608))) (* |:9| (/ (- 10656705) 16777216))) (* |:11| (/ 4727225 8388608))) 0))) (+ (+ (+ (* (ite (>= |:12| 0) |:12| 0) (/ (- 4882399) 8388608)) (* (ite (>= |:13| 0) |:13| 0) (/ 4987691 8388608))) (* (ite (>= |:14| 0) |:14| 0) (/ (- 1315209) 8388608))) (/ 12647359 134217728)))))))))))))))))>, 'y2_scaled': = |:0| 0) |:0| 0))) (let ((|:2| (+ (+ (+ (* x0_scaled (/ (- 12675489) 16777216)) (* x1_scaled (/ (- 15700433) 33554432))) (* x2_scaled (/ (- 14143067) 33554432))) 0))) (let ((|:3| (ite (>= |:2| 0) |:2| 0))) (let ((|:4| (+ (+ (+ (* x0_scaled (/ 9900335 16777216)) (* x1_scaled (/ (- 5448211) 16777216))) (* x2_scaled (/ 12339293 16777216))) (/ 12035361 268435456)))) (let ((|:5| (ite (>= |:4| 0) |:4| 0))) (let ((|:6| (+ (+ (+ (* x0_scaled (/ 11088463 134217728)) (* x1_scaled (/ 211449 4194304))) (* x2_scaled (/ 12447363 16777216))) (/ (- 12828253) 134217728)))) (let ((|:7| (ite (>= |:6| 0) |:6| 0))) (let ((|:8| (+ (+ (+ (* x0_scaled (/ 10855307 67108864)) (* x1_scaled (/ 965293 2097152))) (* x2_scaled (/ (- 15015817) 268435456))) (/ 8969903 4294967296)))) (let ((|:9| (ite (>= |:8| 0) |:8| 0))) (let ((|:10| (+ (+ (+ (* x0_scaled (/ 1208381 2097152)) (* x1_scaled (/ 2938155 4194304))) (* x2_scaled (/ (- 10195595) 33554432))) (/ 3298773 33554432)))) (let ((|:11| (ite (>= |:10| 0) |:10| 0))) (let ((|:12| (+ (+ (+ (+ (+ (+ (* |:1| (/ 1958803 8388608)) (* |:3| (/ 3285671 8388608))) (* |:5| (/ (- 1798427) 8388608))) (* |:7| (/ (- 14391915) 33554432))) (* |:9| (/ (- 5538481) 8388608))) (* |:11| (/ (- 11806169) 33554432))) 0))) (let ((|:13| (+ (+ (+ (+ (+ (+ (* |:1| (/ 10348145 16777216)) (* |:3| (/ 2493617 16777216))) (* |:5| (/ 14212267 16777216))) (* |:7| (/ (- 5570941) 8388608))) (* |:9| (/ (- 11410359) 16777216))) (* |:11| (/ (- 15592633) 33554432))) (/ 5899637 134217728)))) (let ((|:14| (+ (+ (+ (+ (+ (+ (* |:1| (/ (- 13799583) 33554432)) (* |:3| (/ (- 214595) 4194304))) (* |:5| (/ (- 4053207) 16777216))) (* |:7| (/ 315509 8388608))) (* |:9| (/ (- 10656705) 16777216))) (* |:11| (/ 4727225 8388608))) 0))) (+ (+ (+ (* (ite (>= |:12| 0) |:12| 0) (/ (- 4231593) 4194304)) (* (ite (>= |:13| 0) |:13| 0) (/ (- 11739661) 1073741824))) (* (ite (>= |:14| 0) |:14| 0) (/ 373001 1048576))) (/ 12626933 134217728)))))))))))))))))>}" \ No newline at end of file diff --git a/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv new file mode 100644 index 00000000..548a4088 --- /dev/null +++ b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,4.659619170345347,-0.31051789165962873 +y2,8.202680134315264,-1.3070037877761678 diff --git a/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_test_predictions_summary.csv b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_test_predictions_summary.csv new file mode 100644 index 00000000..6d1abd75 --- /dev/null +++ b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_test_predictions_summary.csv @@ -0,0 +1,4 @@ +,y1,y2,y1_nn_keras,y2_nn_keras +7,5.0,5.0,5.3453274,5.447993 +2,5.0,9.0,5.3453274,5.447993 +8,9.0,9.0,5.2932014,5.5662575 diff --git a/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_trace.csv b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_trace.csv new file mode 100644 index 00000000..b5273590 --- /dev/null +++ b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_trace.csv @@ -0,0 +1,7 @@ +stage,solver,x0,x1,x2,y1,y2 +interface_consistency,sat,0,1,3 +model_consistency,sat,261883486167499698105216642400862340458138394084543237885/31340228987215820383455797183353559624951130557732186588,1,8688539606169875102404717560343127792337865336711965626729/1755052823284085941473524642267799338997263311233002448928,12567405957774479498123961652388806459074484532944246565515593654197303735848017/2072000657085627475393660876033817188258433515229246863924477250810420471529472,1422526987207249054062366449963394559765302063278719596268623270213357716086410281/265216084106960316850388592132328600097079489949343598582333088103733820355772416 +ca,sat,10,1,153562766297259232748364195706153/42322327018717815907958469730581,580608788002547407578679690268863/84644654037435631815916939461162,972313279725170195827177542822426133850125/181773010436010199177230173619051326078976 +ce,unsat +ca,sat,64840907984950032/10310323707469465,1,7,543958032077180794443143919398674132621/97378254207380196436658695762639585280,66964662649893619329048400007942715335429/12464416538544665143892313057617866915840 +ce,sat,10,1,5,255823182664445314955/36893488147419103232,25253694072079228291923/4722366482869645213696 diff --git a/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv new file mode 100644 index 00000000..c2d1fbd7 --- /dev/null +++ b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,5.652843508337298,-0.4132108770843246 +y2,5.034594632149151,-0.34255856857310696 diff --git a/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_training_predictions_summary.csv b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_training_predictions_summary.csv new file mode 100644 index 00000000..5f6b275f --- /dev/null +++ b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_training_predictions_summary.csv @@ -0,0 +1,9 @@ +,y1,y2,y1_nn_keras,y2_nn_keras +3,5.0,5.0,6.6467576,5.3529615 +4,9.0,5.0,6.6339836,5.3531966 +5,9.0,9.0,5.3769207,5.376312 +0,5.0,9.0,5.8426776,5.3677473 +10,9.0,5.0,5.3769207,5.376312 +9,5.0,5.0,7.063879,5.345291 +6,5.0,5.0,5.3769207,5.376312 +1,9.0,9.0,6.644639,5.3530006 diff --git a/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_verify_results.json b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_verify_results.json new file mode 100644 index 00000000..8905a7bd --- /dev/null +++ b/regr_smlp/master/Test62_smlp_toy_num_resp_noknobs_verify_results.json @@ -0,0 +1,23 @@ +{ + "asrt_y1": { + "configuration_consistent": "skipped", + "assertion_status": "PASS", + "counter_example": null, + "assertion_feasible": true + }, + "asrt_y2": { + "configuration_consistent": "skipped", + "assertion_status": "FAIL", + "counter_example": { + "x0": 10.0, + "x1": 1.0, + "y1": 6.934101260423692, + "x2": 5.0, + "y2": 5.34767772973293 + }, + "assertion_feasible": true + }, + "smlp_execution": "completed", + "interface_consistent": "skipped", + "model_consistent": "true" +} \ No newline at end of file diff --git a/regr_smlp/master/Test65_smlp_toy_num_resp_noknobs.txt b/regr_smlp/master/Test65_smlp_toy_num_resp_noknobs.txt new file mode 100644 index 00000000..39e87ea7 --- /dev/null +++ b/regr_smlp/master/Test65_smlp_toy_num_resp_noknobs.txt @@ -0,0 +1,262 @@ + +smlp_logger - INFO - Model exploration specification: +{'version': '1.1', 'spec': [{'label': 'y1', 'type': 'response', 'range': 'float'}, {'label': 'y2', 'type': 'response', 'range': 'float'}, {'label': 'x0', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x1', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x2', 'type': 'input', 'range': 'float', 'bounds': [3, 7]}], 'alpha': 'x1==1 or x1==4 or x1==7'} + +smlp_logger - INFO - Executing run_smlp.py script: Start + +smlp_logger - INFO - Running SMLP in mode "verify": Start + +smlp_logger - INFO - Computed spec global constraint expressions: + +smlp_logger - INFO - Global alpha : x1==1 or x1==4 or x1==7 + +smlp_logger - INFO - Global beta : None + +smlp_logger - INFO - Radii theta : {} + +smlp_logger - INFO - Delta const : {'delta_abs': 0.0, 'delta_rel': 0.01} + +smlp_logger - INFO - Assertion asrt1: x0**2+y1>4.3 + +smlp_logger - INFO - Assertion asrt2: (y1+x2)/2<6 + +smlp_logger - INFO - PREPARE DATA FOR MODELING + +smlp_logger - INFO - Preparing training data for modeling: start + +smlp_logger - INFO - loading training data + +smlp_logger - INFO - data summary + y1 y2 x0 x1 x2 +count 11.000000 11.000000 10.000000 10.000000 11.000000 +mean 6.818182 6.818182 10.400000 2.800000 5.454545 +std 2.088932 2.088932 1.074968 0.788811 1.694912 +min 5.000000 5.000000 9.000000 2.000000 3.000000 +25% 5.000000 5.000000 10.000000 2.000000 4.000000 +50% 5.000000 5.000000 10.000000 3.000000 6.000000 +75% 9.000000 9.000000 11.000000 3.000000 7.000000 +max 9.000000 9.000000 12.000000 4.000000 8.000000 + +smlp_logger - INFO - training data + categ y1 y2 x0 x1 x2 +0 c14 5 9 10.0 2.0 3 +1 c15 9 9 12.0 NaN 4 +2 c1 5 9 NaN 3.0 4 +3 c9 5 5 11.0 2.0 6 +4 c5 9 5 10.0 2.0 8 +5 c10 9 9 9.0 4.0 7 +6 c13 5 5 9.0 3.0 6 +7 c4 5 5 10.0 3.0 4 +8 c15 9 9 11.0 4.0 4 +9 c11 5 5 12.0 2.0 7 +10 c19 9 5 10.0 3.0 7 + +smlp_logger - INFO - training data after imputing missing values + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - training data after processing responses + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - training data after encoding levels of categorical features with integers + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - training data after scaling (normalizing) features and responses + x0 x1 x2 y1 y2 +0 0.333333 0.0 0.0 0.0 1.0 +1 1.000000 0.0 0.2 1.0 1.0 +2 0.333333 0.5 0.2 0.0 1.0 +3 0.666667 0.0 0.6 0.0 0.0 +4 0.333333 0.0 1.0 1.0 0.0 +5 0.000000 1.0 0.8 1.0 1.0 +6 0.000000 0.5 0.6 0.0 0.0 +7 0.333333 0.5 0.2 0.0 0.0 +8 0.666667 1.0 0.2 1.0 1.0 +9 1.000000 0.0 0.8 0.0 0.0 +10 0.333333 0.5 0.8 1.0 0.0 + +smlp_logger - INFO - Sampling from training data: start + +smlp_logger - INFO - Sampling from training data: end + +smlp_logger - INFO - X_train after sampling: (8, 3) + +smlp_logger - INFO - y_train after sampling: (8, 2) + +smlp_logger - INFO - Preparing training data for modeling: end + +smlp_logger - INFO - Saving data bounds into file:./test65_model_data_bounds.json + +smlp_logger - INFO - {'x0': {'min': 9.0, 'max': 12.0}, 'x1': {'min': 2.0, 'max': 4.0}, 'x2': {'min': 3.0, 'max': 8.0}, 'y1': {'min': 5.0, 'max': 9.0}, 'y2': {'min': 5.0, 'max': 9.0}} + +smlp_logger - INFO - TRAIN MODEL + +smlp_logger - INFO - Model training: start + +smlp_logger - INFO - Writing tree rules into file ./test65_model_dt_sklearn_tree_rules.txt + +smlp_logger - INFO - Model training: end + +smlp_logger - INFO - Seving model in file ./test65_model_dt_sklearn_model_complete.pkl + +smlp_logger - INFO - PREDICT ON TRAINING DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test65_smlp_toy_num_resp_noknobs_training_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test65_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv + +smlp_logger - INFO - Prediction on training data -- msqe: 0.000 + +smlp_logger - INFO - Prediction on training data -- r2_score: 1.000 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON TEST DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test65_smlp_toy_num_resp_noknobs_test_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test65_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv + +smlp_logger - INFO - Prediction on test data -- msqe: 2.667 + +smlp_logger - INFO - Prediction on test data -- r2_score: 0.250 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON LABELED DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test65_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test65_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv + +smlp_logger - INFO - Prediction on labeled data -- msqe: 0.727 + +smlp_logger - INFO - Prediction on labeled data -- r2_score: 0.817 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - Creating model exploration base components: Start + +smlp_logger - INFO - Parsing the SPEC: Start + +smlp_logger - INFO - Parsing the SPEC: End + +smlp_logger - INFO - Variable domains (alpha): {'y1': {'range': 'float', 'interval': None}, 'y2': {'range': 'float', 'interval': None}, 'x0': {'range': 'float', 'interval': [0, 10]}, 'x1': {'range': 'float', 'interval': [0, 10]}, 'x2': {'range': 'float', 'interval': [3, 7]}} + +smlp_logger - INFO - Input bounds (alpha): {'x0': {'min': 0, 'max': 10}, 'x1': {'min': 0, 'max': 10}, 'x2': {'min': 3, 'max': 7}} + +smlp_logger - INFO - Knob bounds (eta): {} + +smlp_logger - INFO - Knob grids (eta): {} + +smlp_logger - INFO - Alpha global constraints: (or (or (= x1 1) (= x1 4)) (= x1 7)) + +smlp_logger - INFO - Alpha ranges constraints: (and (and (and true (and (>= x0 0) (<= x0 10))) (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) + +smlp_logger - INFO - Alpha combined constraints: (and (and (and (and true (and (>= x0 0) (<= x0 10))) (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) (or (or (= x1 1) (= x1 4)) (= x1 7))) + +smlp_logger - INFO - Beta global constraints: true + +smlp_logger - INFO - Eta ranges constraints: true + +smlp_logger - INFO - Eta grid constraints: true + +smlp_logger - INFO - Eta global constraints: true + +smlp_logger - INFO - Eta combined constraints: true + +smlp_logger - INFO - Creating model exploration base components: End + +smlp_logger - INFO - Input and knob interface constraints are consistent + +smlp_logger - INFO - Building model terms: Start + +smlp_logger - INFO - Model operator counts for y1: {'add': 1, 'mul': 15, 'ite': 5, 'and': 9, 'prop': 14, 'const': 50, 'sub': 14, 'var': 14} + +smlp_logger - INFO - Model operator counts for y2: {'add': 1, 'mul': 15, 'ite': 5, 'and': 9, 'prop': 14, 'const': 50, 'sub': 14, 'var': 14} + +smlp_logger - INFO - Building model terms: End + +smlp_logger - INFO - Model interface constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt1: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt2: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying assertion asrt1 <-> x0**2+y1>4.3 + +smlp_logger - INFO - The configuration is consistent with assertion asrt1 + +smlp_logger - INFO - Completed with result: PASS + +smlp_logger - INFO - Verifying assertion asrt2 <-> (y1+x2)/2<6 + +smlp_logger - INFO - The configuration is consistent with assertion asrt2 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Running SMLP in mode "verify": End + +smlp_logger - INFO - Executing run_smlp.py script: End diff --git a/regr_smlp/master/Test65_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv b/regr_smlp/master/Test65_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv new file mode 100644 index 00000000..cf088bcd --- /dev/null +++ b/regr_smlp/master/Test65_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,0.0,1.0 +y2,1.4545454545454546,0.6333333333333333 diff --git a/regr_smlp/master/Test65_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv b/regr_smlp/master/Test65_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv new file mode 100644 index 00000000..2cce12ec --- /dev/null +++ b/regr_smlp/master/Test65_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv @@ -0,0 +1,12 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +0,5.0,9.0,5.0,9.0 +1,9.0,9.0,9.0,9.0 +2,5.0,9.0,5.0,9.0 +3,5.0,5.0,5.0,5.0 +4,9.0,5.0,9.0,5.0 +5,9.0,9.0,9.0,9.0 +6,5.0,5.0,5.0,5.0 +7,5.0,5.0,5.0,9.0 +8,9.0,9.0,9.0,9.0 +9,5.0,5.0,5.0,5.0 +10,9.0,5.0,9.0,5.0 diff --git a/regr_smlp/master/Test65_smlp_toy_num_resp_noknobs_missing_values_dict.json b/regr_smlp/master/Test65_smlp_toy_num_resp_noknobs_missing_values_dict.json new file mode 100644 index 00000000..553bfe33 --- /dev/null +++ b/regr_smlp/master/Test65_smlp_toy_num_resp_noknobs_missing_values_dict.json @@ -0,0 +1,8 @@ +{ + "x1": [ + 1 + ], + "x0": [ + 2 + ] +} \ No newline at end of file diff --git a/regr_smlp/master/Test65_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv b/regr_smlp/master/Test65_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv new file mode 100644 index 00000000..ec56b74c --- /dev/null +++ b/regr_smlp/master/Test65_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,0.0,1.0 +y2,5.333333333333333,-0.5 diff --git a/regr_smlp/master/Test65_smlp_toy_num_resp_noknobs_test_predictions_summary.csv b/regr_smlp/master/Test65_smlp_toy_num_resp_noknobs_test_predictions_summary.csv new file mode 100644 index 00000000..3477d51d --- /dev/null +++ b/regr_smlp/master/Test65_smlp_toy_num_resp_noknobs_test_predictions_summary.csv @@ -0,0 +1,4 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +7,5.0,5.0,5.0,9.0 +2,5.0,9.0,5.0,9.0 +8,9.0,9.0,9.0,9.0 diff --git a/regr_smlp/master/Test65_smlp_toy_num_resp_noknobs_trace.csv b/regr_smlp/master/Test65_smlp_toy_num_resp_noknobs_trace.csv new file mode 100644 index 00000000..4c2323a8 --- /dev/null +++ b/regr_smlp/master/Test65_smlp_toy_num_resp_noknobs_trace.csv @@ -0,0 +1,9 @@ +stage,solver,x0,x1,x2,y1,y2 +interface_consistency,sat,0,7,3 +model_consistency,sat,0,7,805306377/134217728,9,9 +witness_consistency,sat,0,7,805306377/134217728,9,9 +witness_consistency,sat,0,7,805306377/134217728,9,9 +ca,sat,1,1,27/4,9,5 +ce,unsat +ca,sat,0,1,1140850697/201326592,5,5 +ce,sat,0,4,805306377/134217728,9,9 diff --git a/regr_smlp/master/Test65_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv b/regr_smlp/master/Test65_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv new file mode 100644 index 00000000..f0c58f67 --- /dev/null +++ b/regr_smlp/master/Test65_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,0.0,1.0 +y2,0.0,1.0 diff --git a/regr_smlp/master/Test65_smlp_toy_num_resp_noknobs_training_predictions_summary.csv b/regr_smlp/master/Test65_smlp_toy_num_resp_noknobs_training_predictions_summary.csv new file mode 100644 index 00000000..74d164ca --- /dev/null +++ b/regr_smlp/master/Test65_smlp_toy_num_resp_noknobs_training_predictions_summary.csv @@ -0,0 +1,9 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +3,5.0,5.0,5.0,5.0 +4,9.0,5.0,9.0,5.0 +5,9.0,9.0,9.0,9.0 +0,5.0,9.0,5.0,9.0 +10,9.0,5.0,9.0,5.0 +9,5.0,5.0,5.0,5.0 +6,5.0,5.0,5.0,5.0 +1,9.0,9.0,9.0,9.0 diff --git a/regr_smlp/master/Test65_smlp_toy_num_resp_noknobs_verify_results.json b/regr_smlp/master/Test65_smlp_toy_num_resp_noknobs_verify_results.json new file mode 100644 index 00000000..1579ae5d --- /dev/null +++ b/regr_smlp/master/Test65_smlp_toy_num_resp_noknobs_verify_results.json @@ -0,0 +1,23 @@ +{ + "asrt1": { + "configuration_consistent": "true", + "assertion_status": "PASS", + "counter_example": null, + "assertion_feasible": true + }, + "asrt2": { + "configuration_consistent": "true", + "assertion_status": "FAIL", + "counter_example": { + "x0": 0.0, + "x1": 4.0, + "y1": 9.0, + "x2": 6.000000067055225, + "y2": 9.0 + }, + "assertion_feasible": true + }, + "smlp_execution": "completed", + "interface_consistent": "true", + "model_consistent": "true" +} \ No newline at end of file diff --git a/regr_smlp/master/Test66_test65_model.txt b/regr_smlp/master/Test66_test65_model.txt index 0903f300..ee73c4f2 100644 --- a/regr_smlp/master/Test66_test65_model.txt +++ b/regr_smlp/master/Test66_test65_model.txt @@ -24,7 +24,7 @@ smlp_logger - INFO - PREPARE DATA FOR MODELING smlp_logger - INFO - LOAD TRAINED MODEL -smlp_logger - INFO - Seving model rerun configuration in file ./../models/test65_model_rerun_model_config.json +smlp_logger - INFO - Seving model rerun configuration in file ../models/test65_model_rerun_model_config.json smlp_logger - INFO - Creating model exploration base components: Start diff --git a/regr_smlp/master/Test66_test65_model_trace.csv b/regr_smlp/master/Test66_test65_model_trace.csv new file mode 100644 index 00000000..4c2323a8 --- /dev/null +++ b/regr_smlp/master/Test66_test65_model_trace.csv @@ -0,0 +1,9 @@ +stage,solver,x0,x1,x2,y1,y2 +interface_consistency,sat,0,7,3 +model_consistency,sat,0,7,805306377/134217728,9,9 +witness_consistency,sat,0,7,805306377/134217728,9,9 +witness_consistency,sat,0,7,805306377/134217728,9,9 +ca,sat,1,1,27/4,9,5 +ce,unsat +ca,sat,0,1,1140850697/201326592,5,5 +ce,sat,0,4,805306377/134217728,9,9 diff --git a/regr_smlp/master/Test67_smlp_toy_num_resp_noknobs.txt b/regr_smlp/master/Test67_smlp_toy_num_resp_noknobs.txt new file mode 100644 index 00000000..fc976c8f --- /dev/null +++ b/regr_smlp/master/Test67_smlp_toy_num_resp_noknobs.txt @@ -0,0 +1,264 @@ + +smlp_logger - INFO - Model exploration specification: +{'version': '1.1', 'spec': [{'label': 'y1', 'type': 'response', 'range': 'float'}, {'label': 'y2', 'type': 'response', 'range': 'float'}, {'label': 'x0', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x1', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x2', 'type': 'input', 'range': 'float', 'bounds': [3, 7]}], 'alpha': 'x1==1 or x1==4 or x1==7'} + +smlp_logger - INFO - Executing run_smlp.py script: Start + +smlp_logger - INFO - Running SMLP in mode "verify": Start + +smlp_logger - INFO - Computed spec global constraint expressions: + +smlp_logger - INFO - Global alpha : x1==1 or x1==4 or x1==7 + +smlp_logger - INFO - Global beta : None + +smlp_logger - INFO - Radii theta : {} + +smlp_logger - INFO - Delta const : {'delta_abs': 0.0, 'delta_rel': 0.01} + +smlp_logger - INFO - Assertion asrt1: x0**2+y1>4.3 + +smlp_logger - INFO - Assertion asrt2: (y1+x2)/2<6 + +smlp_logger - INFO - PREPARE DATA FOR MODELING + +smlp_logger - INFO - Preparing training data for modeling: start + +smlp_logger - INFO - loading training data + +smlp_logger - INFO - data summary + y1 y2 x0 x1 x2 +count 11.000000 11.000000 10.000000 10.000000 11.000000 +mean 6.818182 6.818182 10.400000 2.800000 5.454545 +std 2.088932 2.088932 1.074968 0.788811 1.694912 +min 5.000000 5.000000 9.000000 2.000000 3.000000 +25% 5.000000 5.000000 10.000000 2.000000 4.000000 +50% 5.000000 5.000000 10.000000 3.000000 6.000000 +75% 9.000000 9.000000 11.000000 3.000000 7.000000 +max 9.000000 9.000000 12.000000 4.000000 8.000000 + +smlp_logger - INFO - training data + categ y1 y2 x0 x1 x2 +0 c14 5 9 10.0 2.0 3 +1 c15 9 9 12.0 NaN 4 +2 c1 5 9 NaN 3.0 4 +3 c9 5 5 11.0 2.0 6 +4 c5 9 5 10.0 2.0 8 +5 c10 9 9 9.0 4.0 7 +6 c13 5 5 9.0 3.0 6 +7 c4 5 5 10.0 3.0 4 +8 c15 9 9 11.0 4.0 4 +9 c11 5 5 12.0 2.0 7 +10 c19 9 5 10.0 3.0 7 + +smlp_logger - INFO - training data after imputing missing values + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - training data after processing responses + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - training data after encoding levels of categorical features with integers + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - training data after scaling (normalizing) features and responses + x0 x1 x2 y1 y2 +0 0.333333 0.0 0.0 0.0 1.0 +1 1.000000 0.0 0.2 1.0 1.0 +2 0.333333 0.5 0.2 0.0 1.0 +3 0.666667 0.0 0.6 0.0 0.0 +4 0.333333 0.0 1.0 1.0 0.0 +5 0.000000 1.0 0.8 1.0 1.0 +6 0.000000 0.5 0.6 0.0 0.0 +7 0.333333 0.5 0.2 0.0 0.0 +8 0.666667 1.0 0.2 1.0 1.0 +9 1.000000 0.0 0.8 0.0 0.0 +10 0.333333 0.5 0.8 1.0 0.0 + +smlp_logger - INFO - Sampling from training data: start + +smlp_logger - INFO - Sampling from training data: end + +smlp_logger - INFO - X_train after sampling: (8, 3) + +smlp_logger - INFO - y_train after sampling: (8, 2) + +smlp_logger - INFO - Preparing training data for modeling: end + +smlp_logger - INFO - Saving data bounds into file:./test67_model_data_bounds.json + +smlp_logger - INFO - {'x0': {'min': 9.0, 'max': 12.0}, 'x1': {'min': 2.0, 'max': 4.0}, 'x2': {'min': 3.0, 'max': 8.0}, 'y1': {'min': 5.0, 'max': 9.0}, 'y2': {'min': 5.0, 'max': 9.0}} + +smlp_logger - INFO - TRAIN MODEL + +smlp_logger - INFO - Model training: start + +smlp_logger - INFO - Writing tree rules into file ./test67_model_dt_sklearn_y1_tree_rules.txt + +smlp_logger - INFO - Writing tree rules into file ./test67_model_dt_sklearn_y2_tree_rules.txt + +smlp_logger - INFO - Model training: end + +smlp_logger - INFO - Seving model in file ./test67_model_dt_sklearn_model_complete.pkl + +smlp_logger - INFO - PREDICT ON TRAINING DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test67_smlp_toy_num_resp_noknobs_training_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test67_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv + +smlp_logger - INFO - Prediction on training data -- msqe: 0.000 + +smlp_logger - INFO - Prediction on training data -- r2_score: 1.000 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON TEST DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test67_smlp_toy_num_resp_noknobs_test_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test67_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv + +smlp_logger - INFO - Prediction on test data -- msqe: 5.333 + +smlp_logger - INFO - Prediction on test data -- r2_score: -0.500 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON LABELED DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test67_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test67_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv + +smlp_logger - INFO - Prediction on labeled data -- msqe: 1.455 + +smlp_logger - INFO - Prediction on labeled data -- r2_score: 0.633 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - Creating model exploration base components: Start + +smlp_logger - INFO - Parsing the SPEC: Start + +smlp_logger - INFO - Parsing the SPEC: End + +smlp_logger - INFO - Variable domains (alpha): {'y1': {'range': 'float', 'interval': None}, 'y2': {'range': 'float', 'interval': None}, 'x0': {'range': 'float', 'interval': [0, 10]}, 'x1': {'range': 'float', 'interval': [0, 10]}, 'x2': {'range': 'float', 'interval': [3, 7]}} + +smlp_logger - INFO - Input bounds (alpha): {'x0': {'min': 0, 'max': 10}, 'x1': {'min': 0, 'max': 10}, 'x2': {'min': 3, 'max': 7}} + +smlp_logger - INFO - Knob bounds (eta): {} + +smlp_logger - INFO - Knob grids (eta): {} + +smlp_logger - INFO - Alpha global constraints: (or (or (= x1 1) (= x1 4)) (= x1 7)) + +smlp_logger - INFO - Alpha ranges constraints: (and (and (and true (and (>= x0 0) (<= x0 10))) (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) + +smlp_logger - INFO - Alpha combined constraints: (and (and (and (and true (and (>= x0 0) (<= x0 10))) (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) (or (or (= x1 1) (= x1 4)) (= x1 7))) + +smlp_logger - INFO - Beta global constraints: true + +smlp_logger - INFO - Eta ranges constraints: true + +smlp_logger - INFO - Eta grid constraints: true + +smlp_logger - INFO - Eta global constraints: true + +smlp_logger - INFO - Eta combined constraints: true + +smlp_logger - INFO - Creating model exploration base components: End + +smlp_logger - INFO - Input and knob interface constraints are consistent + +smlp_logger - INFO - Building model terms: Start + +smlp_logger - INFO - Model operator counts for y1: {'add': 1, 'mul': 7, 'ite': 3, 'and': 3, 'prop': 6, 'const': 24, 'sub': 6, 'var': 6} + +smlp_logger - INFO - Model operator counts for y2: {'add': 1, 'mul': 4, 'ite': 2, 'and': 1, 'prop': 3, 'const': 14, 'sub': 3, 'var': 3} + +smlp_logger - INFO - Building model terms: End + +smlp_logger - INFO - Model interface constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt1: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt2: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying assertion asrt1 <-> x0**2+y1>4.3 + +smlp_logger - INFO - The configuration is consistent with assertion asrt1 + +smlp_logger - INFO - Completed with result: PASS + +smlp_logger - INFO - Verifying assertion asrt2 <-> (y1+x2)/2<6 + +smlp_logger - INFO - The configuration is consistent with assertion asrt2 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Running SMLP in mode "verify": End + +smlp_logger - INFO - Executing run_smlp.py script: End diff --git a/regr_smlp/master/Test67_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv b/regr_smlp/master/Test67_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv new file mode 100644 index 00000000..2e84754b --- /dev/null +++ b/regr_smlp/master/Test67_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,1.4545454545454546,0.6333333333333333 +y2,1.4545454545454546,0.6333333333333333 diff --git a/regr_smlp/master/Test67_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv b/regr_smlp/master/Test67_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv new file mode 100644 index 00000000..07968f7f --- /dev/null +++ b/regr_smlp/master/Test67_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv @@ -0,0 +1,12 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +0,5.0,9.0,5.0,9.0 +1,9.0,9.0,9.0,9.0 +2,5.0,9.0,5.0,9.0 +3,5.0,5.0,5.0,5.0 +4,9.0,5.0,9.0,5.0 +5,9.0,9.0,9.0,9.0 +6,5.0,5.0,5.0,5.0 +7,5.0,5.0,5.0,9.0 +8,9.0,9.0,5.0,9.0 +9,5.0,5.0,5.0,5.0 +10,9.0,5.0,9.0,5.0 diff --git a/regr_smlp/master/Test67_smlp_toy_num_resp_noknobs_missing_values_dict.json b/regr_smlp/master/Test67_smlp_toy_num_resp_noknobs_missing_values_dict.json new file mode 100644 index 00000000..553bfe33 --- /dev/null +++ b/regr_smlp/master/Test67_smlp_toy_num_resp_noknobs_missing_values_dict.json @@ -0,0 +1,8 @@ +{ + "x1": [ + 1 + ], + "x0": [ + 2 + ] +} \ No newline at end of file diff --git a/regr_smlp/master/Test67_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv b/regr_smlp/master/Test67_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv new file mode 100644 index 00000000..0a5511cf --- /dev/null +++ b/regr_smlp/master/Test67_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,5.333333333333333,-0.5 +y2,5.333333333333333,-0.5 diff --git a/regr_smlp/master/Test67_smlp_toy_num_resp_noknobs_test_predictions_summary.csv b/regr_smlp/master/Test67_smlp_toy_num_resp_noknobs_test_predictions_summary.csv new file mode 100644 index 00000000..fec92c00 --- /dev/null +++ b/regr_smlp/master/Test67_smlp_toy_num_resp_noknobs_test_predictions_summary.csv @@ -0,0 +1,4 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +7,5.0,5.0,5.0,9.0 +2,5.0,9.0,5.0,9.0 +8,9.0,9.0,5.0,9.0 diff --git a/regr_smlp/master/Test67_smlp_toy_num_resp_noknobs_trace.csv b/regr_smlp/master/Test67_smlp_toy_num_resp_noknobs_trace.csv new file mode 100644 index 00000000..fa2d5f5f --- /dev/null +++ b/regr_smlp/master/Test67_smlp_toy_num_resp_noknobs_trace.csv @@ -0,0 +1,9 @@ +stage,solver,x0,x1,x2,y1,y2 +interface_consistency,sat,0,7,3 +model_consistency,sat,0,1,7,9,5 +witness_consistency,sat,0,1,7,9,5 +witness_consistency,sat,0,1,7,9,5 +ca,sat,1,1,4,5,9 +ce,unsat +ca,sat,0,7,218103811/33554432,5,9 +ce,sat,0,7,7,9,9 diff --git a/regr_smlp/master/Test67_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv b/regr_smlp/master/Test67_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv new file mode 100644 index 00000000..f0c58f67 --- /dev/null +++ b/regr_smlp/master/Test67_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,0.0,1.0 +y2,0.0,1.0 diff --git a/regr_smlp/master/Test67_smlp_toy_num_resp_noknobs_training_predictions_summary.csv b/regr_smlp/master/Test67_smlp_toy_num_resp_noknobs_training_predictions_summary.csv new file mode 100644 index 00000000..74d164ca --- /dev/null +++ b/regr_smlp/master/Test67_smlp_toy_num_resp_noknobs_training_predictions_summary.csv @@ -0,0 +1,9 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +3,5.0,5.0,5.0,5.0 +4,9.0,5.0,9.0,5.0 +5,9.0,9.0,9.0,9.0 +0,5.0,9.0,5.0,9.0 +10,9.0,5.0,9.0,5.0 +9,5.0,5.0,5.0,5.0 +6,5.0,5.0,5.0,5.0 +1,9.0,9.0,9.0,9.0 diff --git a/regr_smlp/master/Test67_smlp_toy_num_resp_noknobs_verify_results.json b/regr_smlp/master/Test67_smlp_toy_num_resp_noknobs_verify_results.json new file mode 100644 index 00000000..1029a19f --- /dev/null +++ b/regr_smlp/master/Test67_smlp_toy_num_resp_noknobs_verify_results.json @@ -0,0 +1,23 @@ +{ + "asrt1": { + "configuration_consistent": "true", + "assertion_status": "PASS", + "counter_example": null, + "assertion_feasible": true + }, + "asrt2": { + "configuration_consistent": "true", + "assertion_status": "FAIL", + "counter_example": { + "x0": 0.0, + "x1": 7.0, + "y1": 9.0, + "x2": 7.0, + "y2": 9.0 + }, + "assertion_feasible": true + }, + "smlp_execution": "completed", + "interface_consistent": "true", + "model_consistent": "true" +} \ No newline at end of file diff --git a/regr_smlp/master/Test68_test67_model.txt b/regr_smlp/master/Test68_test67_model.txt index 86c49469..bbdc4544 100644 --- a/regr_smlp/master/Test68_test67_model.txt +++ b/regr_smlp/master/Test68_test67_model.txt @@ -24,7 +24,7 @@ smlp_logger - INFO - PREPARE DATA FOR MODELING smlp_logger - INFO - LOAD TRAINED MODEL -smlp_logger - INFO - Seving model rerun configuration in file ./../models/test67_model_rerun_model_config.json +smlp_logger - INFO - Seving model rerun configuration in file ../models/test67_model_rerun_model_config.json smlp_logger - INFO - Creating model exploration base components: Start diff --git a/regr_smlp/master/Test68_test67_model_trace.csv b/regr_smlp/master/Test68_test67_model_trace.csv new file mode 100644 index 00000000..fa2d5f5f --- /dev/null +++ b/regr_smlp/master/Test68_test67_model_trace.csv @@ -0,0 +1,9 @@ +stage,solver,x0,x1,x2,y1,y2 +interface_consistency,sat,0,7,3 +model_consistency,sat,0,1,7,9,5 +witness_consistency,sat,0,1,7,9,5 +witness_consistency,sat,0,1,7,9,5 +ca,sat,1,1,4,5,9 +ce,unsat +ca,sat,0,7,218103811/33554432,5,9 +ce,sat,0,7,7,9,9 diff --git a/regr_smlp/master/Test71_smlp_toy_num_resp_noknobs.txt b/regr_smlp/master/Test71_smlp_toy_num_resp_noknobs.txt new file mode 100644 index 00000000..267d8b07 --- /dev/null +++ b/regr_smlp/master/Test71_smlp_toy_num_resp_noknobs.txt @@ -0,0 +1,359 @@ + +smlp_logger - INFO - Model exploration specification: +{'version': '1.1', 'spec': [{'label': 'y1', 'type': 'response', 'range': 'float'}, {'label': 'y2', 'type': 'response', 'range': 'float'}, {'label': 'x0', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x1', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x2', 'type': 'input', 'range': 'float', 'bounds': [3, 7]}], 'alpha': 'x1==1 or x1==4 or x1==7'} + +smlp_logger - INFO - Executing run_smlp.py script: Start + +smlp_logger - INFO - Running SMLP in mode "verify": Start + +smlp_logger - INFO - Computed spec global constraint expressions: + +smlp_logger - INFO - Global alpha : x1==1 or x1==4 or x1==7 + +smlp_logger - INFO - Global beta : None + +smlp_logger - INFO - Radii theta : {} + +smlp_logger - INFO - Delta const : {'delta_abs': 0.0, 'delta_rel': 0.01} + +smlp_logger - INFO - Assertion asrt1: (y1**3+x2)/2<6 + +smlp_logger - INFO - PREPARE DATA FOR MODELING + +smlp_logger - INFO - Preparing training data for modeling: start + +smlp_logger - INFO - loading training data + +smlp_logger - INFO - data summary + y1 y2 x0 x1 x2 +count 11.000000 11.000000 10.000000 10.000000 11.000000 +mean 6.818182 6.818182 10.400000 2.800000 5.454545 +std 2.088932 2.088932 1.074968 0.788811 1.694912 +min 5.000000 5.000000 9.000000 2.000000 3.000000 +25% 5.000000 5.000000 10.000000 2.000000 4.000000 +50% 5.000000 5.000000 10.000000 3.000000 6.000000 +75% 9.000000 9.000000 11.000000 3.000000 7.000000 +max 9.000000 9.000000 12.000000 4.000000 8.000000 + +smlp_logger - INFO - training data + categ y1 y2 x0 x1 x2 +0 c14 5 9 10.0 2.0 3 +1 c15 9 9 12.0 NaN 4 +2 c1 5 9 NaN 3.0 4 +3 c9 5 5 11.0 2.0 6 +4 c5 9 5 10.0 2.0 8 +5 c10 9 9 9.0 4.0 7 +6 c13 5 5 9.0 3.0 6 +7 c4 5 5 10.0 3.0 4 +8 c15 9 9 11.0 4.0 4 +9 c11 5 5 12.0 2.0 7 +10 c19 9 5 10.0 3.0 7 + +smlp_logger - INFO - training data after imputing missing values + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - training data after processing responses + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - training data after encoding levels of categorical features with integers + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - training data after scaling (normalizing) features and responses + x0 x1 x2 y1 y2 +0 0.333333 0.0 0.0 0.0 1.0 +1 1.000000 0.0 0.2 1.0 1.0 +2 0.333333 0.5 0.2 0.0 1.0 +3 0.666667 0.0 0.6 0.0 0.0 +4 0.333333 0.0 1.0 1.0 0.0 +5 0.000000 1.0 0.8 1.0 1.0 +6 0.000000 0.5 0.6 0.0 0.0 +7 0.333333 0.5 0.2 0.0 0.0 +8 0.666667 1.0 0.2 1.0 1.0 +9 1.000000 0.0 0.8 0.0 0.0 +10 0.333333 0.5 0.8 1.0 0.0 + +smlp_logger - INFO - Sampling from training data: start + +smlp_logger - INFO - Sampling from training data: end + +smlp_logger - INFO - X_train after sampling: (8, 3) + +smlp_logger - INFO - y_train after sampling: (8, 2) + +smlp_logger - INFO - Preparing training data for modeling: end + +smlp_logger - INFO - Saving data bounds into file:./test71_model_data_bounds.json + +smlp_logger - INFO - {'x0': {'min': 9.0, 'max': 12.0}, 'x1': {'min': 2.0, 'max': 4.0}, 'x2': {'min': 3.0, 'max': 8.0}, 'y1': {'min': 5.0, 'max': 9.0}, 'y2': {'min': 5.0, 'max': 9.0}} + +smlp_logger - INFO - TRAIN MODEL + +smlp_logger - INFO - Model training: start + +smlp_logger - INFO - keras_main: start + +smlp_logger - INFO - _keras_train_multi_response: start + +smlp_logger - INFO - layers_spec_list [2.0, 1.0] + +smlp_logger - INFO - building NN model using Keras Functional API + +smlp_logger - INFO - input layer of size 3 + +smlp_logger - INFO - dense layer of size 6 + +smlp_logger - INFO - dense layer of size 3 + +smlp_logger - INFO - output layer of size 1 + +smlp_logger - INFO - model summary: start + +smlp_logger - INFO - Model: "model" +_________________________________________________________________ + Layer (type) Output Shape Param # +================================================================= + input_1 (InputLayer) [(None, 3)] 0 + + dense (Dense) (None, 6) 24 + + dense_1 (Dense) (None, 3) 21 + + y1 (Dense) (None, 1) 4 + +================================================================= +Total params: 49 (196.00 Byte) +Trainable params: 49 (196.00 Byte) +Non-trainable params: 0 (0.00 Byte) +_________________________________________________________________ + + +smlp_logger - INFO - Optimizer: {'name': 'Adam', 'weight_decay': None, 'clipnorm': None, 'global_clipnorm': None, 'clipvalue': None, 'use_ema': False, 'ema_momentum': 0.99, 'ema_overwrite_frequency': None, 'jit_compile': False, 'is_legacy_optimizer': False, 'learning_rate': 0.001, 'beta_1': 0.9, 'beta_2': 0.999, 'epsilon': 1e-07, 'amsgrad': False} + +smlp_logger - INFO - Learning rate: 0.001 + +smlp_logger - INFO - Loss function: mse + +smlp_logger - INFO - Metrics: ['mse'] + +smlp_logger - INFO - Model configuration: {'name': 'model', 'trainable': True, 'layers': [{'module': 'keras.layers', 'class_name': 'InputLayer', 'config': {'batch_input_shape': (None, 3), 'dtype': 'float32', 'sparse': False, 'ragged': False, 'name': 'input_1'}, 'registered_name': None, 'name': 'input_1', 'inbound_nodes': []}, {'module': 'keras.layers', 'class_name': 'Dense', 'config': {'name': 'dense', 'trainable': True, 'dtype': 'float32', 'units': 6, 'activation': 'relu', 'use_bias': True, 'kernel_initializer': {'module': 'keras.initializers', 'class_name': 'GlorotUniform', 'config': {'seed': None}, 'registered_name': None}, 'bias_initializer': {'module': 'keras.initializers', 'class_name': 'Zeros', 'config': {}, 'registered_name': None}, 'kernel_regularizer': None, 'bias_regularizer': None, 'activity_regularizer': None, 'kernel_constraint': None, 'bias_constraint': None}, 'registered_name': None, 'build_config': {'input_shape': (None, 3)}, 'name': 'dense', 'inbound_nodes': [[['input_1', 0, 0, {}]]]}, {'module': 'keras.layers', 'class_name': 'Dense', 'config': {'name': 'dense_1', 'trainable': True, 'dtype': 'float32', 'units': 3, 'activation': 'relu', 'use_bias': True, 'kernel_initializer': {'module': 'keras.initializers', 'class_name': 'GlorotUniform', 'config': {'seed': None}, 'registered_name': None}, 'bias_initializer': {'module': 'keras.initializers', 'class_name': 'Zeros', 'config': {}, 'registered_name': None}, 'kernel_regularizer': None, 'bias_regularizer': None, 'activity_regularizer': None, 'kernel_constraint': None, 'bias_constraint': None}, 'registered_name': None, 'build_config': {'input_shape': (None, 6)}, 'name': 'dense_1', 'inbound_nodes': [[['dense', 0, 0, {}]]]}, {'module': 'keras.layers', 'class_name': 'Dense', 'config': {'name': 'y1', 'trainable': True, 'dtype': 'float32', 'units': 1, 'activation': 'linear', 'use_bias': True, 'kernel_initializer': {'module': 'keras.initializers', 'class_name': 'GlorotUniform', 'config': {'seed': None}, 'registered_name': None}, 'bias_initializer': {'module': 'keras.initializers', 'class_name': 'Zeros', 'config': {}, 'registered_name': None}, 'kernel_regularizer': None, 'bias_regularizer': None, 'activity_regularizer': None, 'kernel_constraint': None, 'bias_constraint': None}, 'registered_name': None, 'build_config': {'input_shape': (None, 3)}, 'name': 'y1', 'inbound_nodes': [[['dense_1', 0, 0, {}]]]}], 'input_layers': [['input_1', 0, 0]], 'output_layers': [['y1', 0, 0]]} + +smlp_logger - INFO - Epochs: 20 + +smlp_logger - INFO - Batch size: 200 + +smlp_logger - INFO - Callbacks: [""] + +smlp_logger - INFO - model summary: end + +smlp_logger - INFO - _keras_train_multi_response: end + +smlp_logger - INFO - _keras_train_multi_response: start + +smlp_logger - INFO - layers_spec_list [2.0, 1.0] + +smlp_logger - INFO - building NN model using Keras Functional API + +smlp_logger - INFO - input layer of size 3 + +smlp_logger - INFO - dense layer of size 6 + +smlp_logger - INFO - dense layer of size 3 + +smlp_logger - INFO - output layer of size 1 + +smlp_logger - INFO - model summary: start + +smlp_logger - INFO - Model: "model_1" +_________________________________________________________________ + Layer (type) Output Shape Param # +================================================================= + input_2 (InputLayer) [(None, 3)] 0 + + dense_2 (Dense) (None, 6) 24 + + dense_3 (Dense) (None, 3) 21 + + y2 (Dense) (None, 1) 4 + +================================================================= +Total params: 49 (196.00 Byte) +Trainable params: 49 (196.00 Byte) +Non-trainable params: 0 (0.00 Byte) +_________________________________________________________________ + + +smlp_logger - INFO - Optimizer: {'name': 'Adam', 'weight_decay': None, 'clipnorm': None, 'global_clipnorm': None, 'clipvalue': None, 'use_ema': False, 'ema_momentum': 0.99, 'ema_overwrite_frequency': None, 'jit_compile': False, 'is_legacy_optimizer': False, 'learning_rate': 0.001, 'beta_1': 0.9, 'beta_2': 0.999, 'epsilon': 1e-07, 'amsgrad': False} + +smlp_logger - INFO - Learning rate: 0.001 + +smlp_logger - INFO - Loss function: mse + +smlp_logger - INFO - Metrics: ['mse'] + +smlp_logger - INFO - Model configuration: {'name': 'model_1', 'trainable': True, 'layers': [{'module': 'keras.layers', 'class_name': 'InputLayer', 'config': {'batch_input_shape': (None, 3), 'dtype': 'float32', 'sparse': False, 'ragged': False, 'name': 'input_2'}, 'registered_name': None, 'name': 'input_2', 'inbound_nodes': []}, {'module': 'keras.layers', 'class_name': 'Dense', 'config': {'name': 'dense_2', 'trainable': True, 'dtype': 'float32', 'units': 6, 'activation': 'relu', 'use_bias': True, 'kernel_initializer': {'module': 'keras.initializers', 'class_name': 'GlorotUniform', 'config': {'seed': None}, 'registered_name': None}, 'bias_initializer': {'module': 'keras.initializers', 'class_name': 'Zeros', 'config': {}, 'registered_name': None}, 'kernel_regularizer': None, 'bias_regularizer': None, 'activity_regularizer': None, 'kernel_constraint': None, 'bias_constraint': None}, 'registered_name': None, 'build_config': {'input_shape': (None, 3)}, 'name': 'dense_2', 'inbound_nodes': [[['input_2', 0, 0, {}]]]}, {'module': 'keras.layers', 'class_name': 'Dense', 'config': {'name': 'dense_3', 'trainable': True, 'dtype': 'float32', 'units': 3, 'activation': 'relu', 'use_bias': True, 'kernel_initializer': {'module': 'keras.initializers', 'class_name': 'GlorotUniform', 'config': {'seed': None}, 'registered_name': None}, 'bias_initializer': {'module': 'keras.initializers', 'class_name': 'Zeros', 'config': {}, 'registered_name': None}, 'kernel_regularizer': None, 'bias_regularizer': None, 'activity_regularizer': None, 'kernel_constraint': None, 'bias_constraint': None}, 'registered_name': None, 'build_config': {'input_shape': (None, 6)}, 'name': 'dense_3', 'inbound_nodes': [[['dense_2', 0, 0, {}]]]}, {'module': 'keras.layers', 'class_name': 'Dense', 'config': {'name': 'y2', 'trainable': True, 'dtype': 'float32', 'units': 1, 'activation': 'linear', 'use_bias': True, 'kernel_initializer': {'module': 'keras.initializers', 'class_name': 'GlorotUniform', 'config': {'seed': None}, 'registered_name': None}, 'bias_initializer': {'module': 'keras.initializers', 'class_name': 'Zeros', 'config': {}, 'registered_name': None}, 'kernel_regularizer': None, 'bias_regularizer': None, 'activity_regularizer': None, 'kernel_constraint': None, 'bias_constraint': None}, 'registered_name': None, 'build_config': {'input_shape': (None, 3)}, 'name': 'y2', 'inbound_nodes': [[['dense_3', 0, 0, {}]]]}], 'input_layers': [['input_2', 0, 0]], 'output_layers': [['y2', 0, 0]]} + +smlp_logger - INFO - Epochs: 20 + +smlp_logger - INFO - Batch size: 200 + +smlp_logger - INFO - Callbacks: [""] + +smlp_logger - INFO - model summary: end + +smlp_logger - INFO - _keras_train_multi_response: end + +smlp_logger - INFO - keras_main: end + +smlp_logger - INFO - Model training: end + +smlp_logger - INFO - PREDICT ON TRAINING DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test71_smlp_toy_num_resp_noknobs_training_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test71_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv + +smlp_logger - INFO - Prediction on training data -- msqe: 10.617 + +smlp_logger - INFO - Prediction on training data -- r2_score: -1.737 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON TEST DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test71_smlp_toy_num_resp_noknobs_test_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test71_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv + +smlp_logger - INFO - Prediction on test data -- msqe: 7.020 + +smlp_logger - INFO - Prediction on test data -- r2_score: -0.974 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON LABELED DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test71_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test71_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv + +smlp_logger - INFO - Prediction on labeled data -- msqe: 9.636 + +smlp_logger - INFO - Prediction on labeled data -- r2_score: -1.429 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - Creating model exploration base components: Start + +smlp_logger - INFO - Parsing the SPEC: Start + +smlp_logger - INFO - Parsing the SPEC: End + +smlp_logger - INFO - Variable domains (alpha): {'y1': {'range': 'float', 'interval': None}, 'y2': {'range': 'float', 'interval': None}, 'x0': {'range': 'float', 'interval': [0, 10]}, 'x1': {'range': 'float', 'interval': [0, 10]}, 'x2': {'range': 'float', 'interval': [3, 7]}} + +smlp_logger - INFO - Input bounds (alpha): {'x0': {'min': 0, 'max': 10}, 'x1': {'min': 0, 'max': 10}, 'x2': {'min': 3, 'max': 7}} + +smlp_logger - INFO - Knob bounds (eta): {} + +smlp_logger - INFO - Knob grids (eta): {} + +smlp_logger - INFO - Alpha global constraints: (or (or (= x1 1) (= x1 4)) (= x1 7)) + +smlp_logger - INFO - Alpha ranges constraints: (and (and (and true (and (>= x0 0) (<= x0 10))) (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) + +smlp_logger - INFO - Alpha combined constraints: (and (and (and (and true (and (>= x0 0) (<= x0 10))) (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) (or (or (= x1 1) (= x1 4)) (= x1 7))) + +smlp_logger - INFO - Beta global constraints: true + +smlp_logger - INFO - Eta ranges constraints: true + +smlp_logger - INFO - Eta grid constraints: true + +smlp_logger - INFO - Eta global constraints: true + +smlp_logger - INFO - Eta combined constraints: true + +smlp_logger - INFO - Creating model exploration base components: End + +smlp_logger - INFO - Input and knob interface constraints are consistent + +smlp_logger - INFO - Building model terms: Start + +smlp_logger - INFO - Model operator counts for y1: {'add': 256, 'mul': 472, 'ite': 39, 'prop': 39, 'const': 846, 'sub': 216, 'var': 216} + +smlp_logger - INFO - Model operator counts for y2: {'add': 256, 'mul': 472, 'ite': 39, 'prop': 39, 'const': 846, 'sub': 216, 'var': 216} + +smlp_logger - INFO - Building model terms: End + +smlp_logger - INFO - Model interface constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt1: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying assertion asrt1 <-> (y1**3+x2)/2<6 + +smlp_logger - INFO - The configuration is consistent with assertion asrt1 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Running SMLP in mode "verify": End + +smlp_logger - INFO - Executing run_smlp.py script: End diff --git a/regr_smlp/master/Test71_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv b/regr_smlp/master/Test71_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv new file mode 100644 index 00000000..b9f8b97d --- /dev/null +++ b/regr_smlp/master/Test71_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,9.459757099022092,-1.3846471020451525 +y2,9.812356030356097,-1.4735314159855997 diff --git a/regr_smlp/master/Test71_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv b/regr_smlp/master/Test71_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv new file mode 100644 index 00000000..f613999e --- /dev/null +++ b/regr_smlp/master/Test71_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv @@ -0,0 +1,12 @@ +,y1,y2,y1_nn_keras,y2_nn_keras +0,5.0,9.0,4.5473785,4.5473723 +1,9.0,9.0,2.800819,2.7999172 +2,5.0,9.0,5.1303916,5.130361 +3,5.0,5.0,3.789467,3.786426 +4,9.0,5.0,5.0789886,5.0787845 +5,9.0,9.0,5.0789886,5.0787845 +6,5.0,5.0,5.0789886,5.0787845 +7,5.0,5.0,5.1303916,5.130361 +8,9.0,9.0,5.3193283,5.3193192 +9,5.0,5.0,2.9392607,2.9352548 +10,9.0,5.0,5.0789886,5.0787845 diff --git a/regr_smlp/master/Test71_smlp_toy_num_resp_noknobs_missing_values_dict.json b/regr_smlp/master/Test71_smlp_toy_num_resp_noknobs_missing_values_dict.json new file mode 100644 index 00000000..553bfe33 --- /dev/null +++ b/regr_smlp/master/Test71_smlp_toy_num_resp_noknobs_missing_values_dict.json @@ -0,0 +1,8 @@ +{ + "x1": [ + 1 + ], + "x0": [ + 2 + ] +} \ No newline at end of file diff --git a/regr_smlp/master/Test71_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv b/regr_smlp/master/Test71_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv new file mode 100644 index 00000000..fb32d32f --- /dev/null +++ b/regr_smlp/master/Test71_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,4.527116013680067,-0.2732513788475188 +y2,9.512836725921185,-1.6754853291653333 diff --git a/regr_smlp/master/Test71_smlp_toy_num_resp_noknobs_test_predictions_summary.csv b/regr_smlp/master/Test71_smlp_toy_num_resp_noknobs_test_predictions_summary.csv new file mode 100644 index 00000000..181af7c0 --- /dev/null +++ b/regr_smlp/master/Test71_smlp_toy_num_resp_noknobs_test_predictions_summary.csv @@ -0,0 +1,4 @@ +,y1,y2,y1_nn_keras,y2_nn_keras +7,5.0,5.0,5.1303916,5.130361 +2,5.0,9.0,5.1303916,5.130361 +8,9.0,9.0,5.3193283,5.3193192 diff --git a/regr_smlp/master/Test71_smlp_toy_num_resp_noknobs_trace.csv b/regr_smlp/master/Test71_smlp_toy_num_resp_noknobs_trace.csv new file mode 100644 index 00000000..7bd72ced --- /dev/null +++ b/regr_smlp/master/Test71_smlp_toy_num_resp_noknobs_trace.csv @@ -0,0 +1,6 @@ +stage,solver,x0,x1,x2,y1,y2 +interface_consistency,sat,0,1,3 +model_consistency,sat,335865014418181538/39045391051627415,1,7,10651411/2097152,681662925/134217728 +witness_consistency,sat,335865014418181538/39045391051627415,1,7,10651411/2097152,681662925/134217728 +ca,sat,6620280383692978272588900391513/2792544730528104800601395382730,1,149829899645819429662281060723/42962226623509304624636852042,1139036899059459709076824500537/558508946105620960120279076546,320594959794944162063971076295705147292952553/157206292597772686818588679219324560317874176 +ce,sat,335865014418181538/39045391051627415,1,7,10651411/2097152,681662925/134217728 diff --git a/regr_smlp/master/Test71_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv b/regr_smlp/master/Test71_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv new file mode 100644 index 00000000..f7c3418d --- /dev/null +++ b/regr_smlp/master/Test71_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,11.309497506025352,-1.827374376506338 +y2,9.92467576951919,-1.6465802052051175 diff --git a/regr_smlp/master/Test71_smlp_toy_num_resp_noknobs_training_predictions_summary.csv b/regr_smlp/master/Test71_smlp_toy_num_resp_noknobs_training_predictions_summary.csv new file mode 100644 index 00000000..0a8836f5 --- /dev/null +++ b/regr_smlp/master/Test71_smlp_toy_num_resp_noknobs_training_predictions_summary.csv @@ -0,0 +1,9 @@ +,y1,y2,y1_nn_keras,y2_nn_keras +3,5.0,5.0,3.789467,3.786426 +4,9.0,5.0,5.0789886,5.0787845 +5,9.0,9.0,5.0789886,5.0787845 +0,5.0,9.0,4.5473785,4.5473723 +10,9.0,5.0,5.0789886,5.0787845 +9,5.0,5.0,2.9392607,2.9352548 +6,5.0,5.0,5.0789886,5.0787845 +1,9.0,9.0,2.800819,2.7999172 diff --git a/regr_smlp/master/Test71_smlp_toy_num_resp_noknobs_verify_results.json b/regr_smlp/master/Test71_smlp_toy_num_resp_noknobs_verify_results.json new file mode 100644 index 00000000..4fd78e0c --- /dev/null +++ b/regr_smlp/master/Test71_smlp_toy_num_resp_noknobs_verify_results.json @@ -0,0 +1,17 @@ +{ + "asrt1": { + "configuration_consistent": "true", + "assertion_status": "FAIL", + "counter_example": { + "x0": 8.601911912575982, + "x1": 1.0, + "y1": 5.078988552093506, + "x2": 7.0, + "y2": 5.078784562647343 + }, + "assertion_feasible": true + }, + "smlp_execution": "completed", + "interface_consistent": "true", + "model_consistent": "true" +} \ No newline at end of file diff --git a/regr_smlp/master/Test72_test71_model.txt b/regr_smlp/master/Test72_test71_model.txt index 559fc640..73e18ff3 100644 --- a/regr_smlp/master/Test72_test71_model.txt +++ b/regr_smlp/master/Test72_test71_model.txt @@ -22,7 +22,7 @@ smlp_logger - INFO - PREPARE DATA FOR MODELING smlp_logger - INFO - LOAD TRAINED MODEL -smlp_logger - INFO - Seving model rerun configuration in file ./../models/test71_model_rerun_model_config.json +smlp_logger - INFO - Seving model rerun configuration in file ../models/test71_model_rerun_model_config.json smlp_logger - INFO - Creating model exploration base components: Start diff --git a/regr_smlp/master/Test72_test71_model_trace.csv b/regr_smlp/master/Test72_test71_model_trace.csv new file mode 100644 index 00000000..dd5802bc --- /dev/null +++ b/regr_smlp/master/Test72_test71_model_trace.csv @@ -0,0 +1,6 @@ +stage,solver,x0,x1,x2,y1,y2 +interface_consistency,sat,0,1,3 +model_consistency,sat,335865014418181538/39045391051627415,1,7,10651411/2097152,681662925/134217728 +witness_consistency,sat,335865014418181538/39045391051627415,1,7,10651411/2097152,681662925/134217728 +ca,sat,123722618417780711807298165032/54373292127912369088951862159,1,1139945589298010346211983867637/326239752767474214533711172954,6163226111796703126611665298945400963472615264808055/3081247347697599620049750445846427059655048291155968,2 +ce,sat,335865014418181538/39045391051627415,1,7,10651411/2097152,681662925/134217728 diff --git a/regr_smlp/master/Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled.txt b/regr_smlp/master/Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled.txt new file mode 100644 index 00000000..34dc0611 --- /dev/null +++ b/regr_smlp/master/Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled.txt @@ -0,0 +1,392 @@ + +smlp_logger - INFO - Model exploration specification: +{'version': '1.1', 'spec': [{'label': 'y1', 'type': 'response', 'range': 'float'}, {'label': 'y2', 'type': 'response', 'range': 'float'}, {'label': 'x0', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x1', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x2', 'type': 'input', 'range': 'float', 'bounds': [3, 7]}], 'alpha': 'x1==1 or x1==4 or x1==7'} + +smlp_logger - INFO - Executing run_smlp.py script: Start + +smlp_logger - INFO - Running SMLP in mode "verify": Start + +smlp_logger - INFO - Computed spec global constraint expressions: + +smlp_logger - INFO - Global alpha : x1==1 or x1==4 or x1==7 + +smlp_logger - INFO - Global beta : None + +smlp_logger - INFO - Radii theta : {} + +smlp_logger - INFO - Delta const : {'delta_abs': 0.0, 'delta_rel': 0.01} + +smlp_logger - INFO - Assertion asrt1: (y2**3+x2)/2<6 + +smlp_logger - INFO - Assertion asrt2: y1>=9 + +smlp_logger - INFO - Assertion asrt3: y2<0 + +smlp_logger - INFO - PREPARE DATA FOR MODELING + +smlp_logger - INFO - Preparing training data for modeling: start + +smlp_logger - INFO - loading training data + +smlp_logger - INFO - data summary + y1 y2 x0 x1 x2 +count 11.000000 11.000000 10.000000 10.000000 11.000000 +mean 6.818182 6.818182 10.400000 2.800000 5.454545 +std 2.088932 2.088932 1.074968 0.788811 1.694912 +min 5.000000 5.000000 9.000000 2.000000 3.000000 +25% 5.000000 5.000000 10.000000 2.000000 4.000000 +50% 5.000000 5.000000 10.000000 3.000000 6.000000 +75% 9.000000 9.000000 11.000000 3.000000 7.000000 +max 9.000000 9.000000 12.000000 4.000000 8.000000 + +smlp_logger - INFO - training data + categ y1 y2 x0 x1 x2 +0 c14 5 9 10.0 2.0 3 +1 c15 9 9 12.0 NaN 4 +2 c1 5 9 NaN 3.0 4 +3 c9 5 5 11.0 2.0 6 +4 c5 9 5 10.0 2.0 8 +5 c10 9 9 9.0 4.0 7 +6 c13 5 5 9.0 3.0 6 +7 c4 5 5 10.0 3.0 4 +8 c15 9 9 11.0 4.0 4 +9 c11 5 5 12.0 2.0 7 +10 c19 9 5 10.0 3.0 7 + +smlp_logger - INFO - training data after imputing missing values + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - training data after processing responses + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - MRMR feature selection for response y1 : start + +smlp_logger - INFO - MRMR selected feature scores (in the ranked order) for response y1 : + Feature Score +0 x1 1.115702 +1 x2 0.944056 + +smlp_logger - INFO - MRMR feature selection for response y1 : end + +smlp_logger - INFO - MRMR feature selection for response y2 : start + +smlp_logger - INFO - MRMR selected feature scores (in the ranked order) for response y2 : + Feature Score +1 x2 4.950294 +0 x1 1.115702 + +smlp_logger - INFO - MRMR feature selection for response y2 : end + +smlp_logger - INFO - training data after encoding levels of categorical features with integers + x1 x2 y1 y2 +0 2.0 3 5 9 +1 2.0 4 9 9 +2 3.0 4 5 9 +3 2.0 6 5 5 +4 2.0 8 9 5 +5 4.0 7 9 9 +6 3.0 6 5 5 +7 3.0 4 5 5 +8 4.0 4 9 9 +9 2.0 7 5 5 +10 3.0 7 9 5 + +smlp_logger - INFO - training data after scaling (normalizing) features and responses + x1 x2 y1 y2 +0 0.0 0.0 0.0 1.0 +1 0.0 0.2 1.0 1.0 +2 0.5 0.2 0.0 1.0 +3 0.0 0.6 0.0 0.0 +4 0.0 1.0 1.0 0.0 +5 1.0 0.8 1.0 1.0 +6 0.5 0.6 0.0 0.0 +7 0.5 0.2 0.0 0.0 +8 1.0 0.2 1.0 1.0 +9 0.0 0.8 0.0 0.0 +10 0.5 0.8 1.0 0.0 + +smlp_logger - INFO - Sampling from training data: start + +smlp_logger - INFO - Sampling from training data: end + +smlp_logger - INFO - X_train after sampling: (8, 2) + +smlp_logger - INFO - y_train after sampling: (8, 2) + +smlp_logger - INFO - Preparing training data for modeling: end + +smlp_logger - INFO - Saving data bounds into file:./test73_model_data_bounds.json + +smlp_logger - INFO - {'x1': {'min': 2.0, 'max': 4.0}, 'x2': {'min': 3.0, 'max': 8.0}, 'y1': {'min': 5.0, 'max': 9.0}, 'y2': {'min': 5.0, 'max': 9.0}} + +smlp_logger - INFO - Preparing new data for modeling: start + +smlp_logger - INFO - loading new data + +smlp_logger - INFO - data summary + y1 y2 x0 x1 x2 +count 9.000000 9.000000 8.000000 8.000000 9.000000 +mean 6.777778 7.222222 10.250000 2.875000 5.111111 +std 2.108185 2.108185 1.035098 0.834523 1.691482 +min 5.000000 5.000000 9.000000 2.000000 3.000000 +25% 5.000000 5.000000 9.750000 2.000000 4.000000 +50% 5.000000 9.000000 10.000000 3.000000 4.000000 +75% 9.000000 9.000000 11.000000 3.250000 6.000000 +max 9.000000 9.000000 12.000000 4.000000 8.000000 + +smlp_logger - INFO - new data + categ y1 y2 x0 x1 x2 +0 c0 5 9 10.0 2.0 3 +1 c12 9 9 12.0 NaN 4 +2 c2 5 9 NaN 3.0 4 +3 c17 5 5 11.0 2.0 6 +4 c18 9 5 10.0 2.0 8 +5 c8 9 9 9.0 4.0 7 +6 c7 5 5 9.0 3.0 6 +7 c3 5 5 10.0 3.0 4 +8 c12 9 9 11.0 4.0 4 + +smlp_logger - INFO - new data after imputing missing values + x1 x2 y1 y2 +0 2.0 3 5 9 +1 2.0 4 9 9 +2 3.0 4 5 9 +3 2.0 6 5 5 +4 2.0 8 9 5 +5 4.0 7 9 9 +6 3.0 6 5 5 +7 3.0 4 5 5 +8 4.0 4 9 9 + +smlp_logger - INFO - new data after processing responses + x1 x2 y1 y2 +0 2.0 3 5 9 +1 2.0 4 9 9 +2 3.0 4 5 9 +3 2.0 6 5 5 +4 2.0 8 9 5 +5 4.0 7 9 9 +6 3.0 6 5 5 +7 3.0 4 5 5 +8 4.0 4 9 9 + +smlp_logger - INFO - new data after encoding levels of categorical features with integers + x1 x2 y1 y2 +0 2.0 3 5 9 +1 2.0 4 9 9 +2 3.0 4 5 9 +3 2.0 6 5 5 +4 2.0 8 9 5 +5 4.0 7 9 9 +6 3.0 6 5 5 +7 3.0 4 5 5 +8 4.0 4 9 9 + +smlp_logger - INFO - new data after scaling (normalizing) features and responses + x1 x2 y1 y2 +0 0.0 0.0 0.0 1.0 +1 0.0 0.2 1.0 1.0 +2 0.5 0.2 0.0 1.0 +3 0.0 0.6 0.0 0.0 +4 0.0 1.0 1.0 0.0 +5 1.0 0.8 1.0 1.0 +6 0.5 0.6 0.0 0.0 +7 0.5 0.2 0.0 0.0 +8 1.0 0.2 1.0 1.0 + +smlp_logger - INFO - Preparing new data for modeling: end + +smlp_logger - INFO - TRAIN MODEL + +smlp_logger - INFO - Model training: start + +smlp_logger - INFO - Writing tree rules into file ./test73_model_dt_sklearn_y1_tree_rules.txt + +smlp_logger - INFO - Writing tree rules into file ./test73_model_dt_sklearn_y2_tree_rules.txt + +smlp_logger - INFO - Model training: end + +smlp_logger - INFO - Seving model in file ./test73_model_dt_sklearn_model_complete.pkl + +smlp_logger - INFO - PREDICT ON TRAINING DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv + +smlp_logger - INFO - Prediction on training data -- msqe: 0.000 + +smlp_logger - INFO - Prediction on training data -- r2_score: 1.000 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON TEST DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv + +smlp_logger - INFO - Prediction on test data -- msqe: 8.000 + +smlp_logger - INFO - Prediction on test data -- r2_score: -1.250 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON LABELED DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv + +smlp_logger - INFO - Prediction on labeled data -- msqe: 2.182 + +smlp_logger - INFO - Prediction on labeled data -- r2_score: 0.450 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON NEW DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv + +smlp_logger - INFO - Prediction on new data -- msqe: 2.667 + +smlp_logger - INFO - Prediction on new data -- r2_score: 0.325 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - Creating model exploration base components: Start + +smlp_logger - INFO - Parsing the SPEC: Start + +smlp_logger - INFO - Parsing the SPEC: End + +smlp_logger - INFO - Variable domains (alpha): {'y1': {'range': 'float', 'interval': None}, 'y2': {'range': 'float', 'interval': None}, 'x0': {'range': 'float', 'interval': [0, 10]}, 'x1': {'range': 'float', 'interval': [0, 10]}, 'x2': {'range': 'float', 'interval': [3, 7]}} + +smlp_logger - INFO - Input bounds (alpha): {'x0': {'min': 0, 'max': 10}, 'x1': {'min': 0, 'max': 10}, 'x2': {'min': 3, 'max': 7}} + +smlp_logger - INFO - Knob bounds (eta): {} + +smlp_logger - INFO - Knob grids (eta): {} + +smlp_logger - INFO - Alpha global constraints: (or (or (= x1 1) (= x1 4)) (= x1 7)) + +smlp_logger - INFO - Alpha ranges constraints: (and (and true (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) + +smlp_logger - INFO - Alpha combined constraints: (and (and (and true (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) (or (or (= x1 1) (= x1 4)) (= x1 7))) + +smlp_logger - INFO - Beta global constraints: true + +smlp_logger - INFO - Eta ranges constraints: true + +smlp_logger - INFO - Eta grid constraints: true + +smlp_logger - INFO - Eta global constraints: true + +smlp_logger - INFO - Eta combined constraints: true + +smlp_logger - INFO - Creating model exploration base components: End + +smlp_logger - INFO - Input and knob interface constraints are consistent + +smlp_logger - INFO - Building model terms: Start + +smlp_logger - INFO - Model operator counts for y1: {'add': 1, 'mul': 15, 'ite': 5, 'and': 9, 'prop': 14, 'const': 50, 'sub': 14, 'var': 14} + +smlp_logger - INFO - Model operator counts for y2: {'add': 1, 'mul': 4, 'ite': 2, 'and': 1, 'prop': 3, 'const': 14, 'sub': 3, 'var': 3} + +smlp_logger - INFO - Building model terms: End + +smlp_logger - INFO - Model interface constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt1: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt2: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt3: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying assertion asrt1 <-> (y2**3+x2)/2<6 + +smlp_logger - INFO - The configuration is inconsistent with assertion asrt1 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Verifying assertion asrt2 <-> y1>=9 + +smlp_logger - INFO - The configuration is consistent with assertion asrt2 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Verifying assertion asrt3 <-> y2<0 + +smlp_logger - INFO - The configuration is inconsistent with assertion asrt3 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Running SMLP in mode "verify": End + +smlp_logger - INFO - Executing run_smlp.py script: End diff --git a/regr_smlp/master/Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv b/regr_smlp/master/Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv new file mode 100644 index 00000000..7d143a2f --- /dev/null +++ b/regr_smlp/master/Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,2.909090909090909,0.2666666666666666 +y2,1.4545454545454546,0.6333333333333333 diff --git a/regr_smlp/master/Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv b/regr_smlp/master/Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv new file mode 100644 index 00000000..7a649b6a --- /dev/null +++ b/regr_smlp/master/Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv @@ -0,0 +1,12 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +0,5.0,9.0,5.0,9.0 +1,9.0,9.0,9.0,9.0 +2,5.0,9.0,9.0,9.0 +3,5.0,5.0,5.0,5.0 +4,9.0,5.0,9.0,5.0 +5,9.0,9.0,9.0,9.0 +6,5.0,5.0,5.0,5.0 +7,5.0,5.0,9.0,9.0 +8,9.0,9.0,9.0,9.0 +9,5.0,5.0,5.0,5.0 +10,9.0,5.0,9.0,5.0 diff --git a/regr_smlp/master/Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json b/regr_smlp/master/Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json new file mode 100644 index 00000000..6e4f6879 --- /dev/null +++ b/regr_smlp/master/Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json @@ -0,0 +1,5 @@ +{ + "x1": [ + 1 + ] +} \ No newline at end of file diff --git a/regr_smlp/master/Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv b/regr_smlp/master/Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv new file mode 100644 index 00000000..951115f0 --- /dev/null +++ b/regr_smlp/master/Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,3.5555555555555554,0.10000000000000009 +y2,1.7777777777777777,0.55 diff --git a/regr_smlp/master/Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv b/regr_smlp/master/Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv new file mode 100644 index 00000000..3ba8f5c9 --- /dev/null +++ b/regr_smlp/master/Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv @@ -0,0 +1,10 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +0,5.0,9.0,5.0,9.0 +1,9.0,9.0,9.0,9.0 +2,5.0,9.0,9.0,9.0 +3,5.0,5.0,5.0,5.0 +4,9.0,5.0,9.0,5.0 +5,9.0,9.0,9.0,9.0 +6,5.0,5.0,5.0,5.0 +7,5.0,5.0,9.0,9.0 +8,9.0,9.0,9.0,9.0 diff --git a/regr_smlp/master/Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv b/regr_smlp/master/Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv new file mode 100644 index 00000000..fd947921 --- /dev/null +++ b/regr_smlp/master/Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,10.666666666666666,-2.0 +y2,5.333333333333333,-0.5 diff --git a/regr_smlp/master/Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv b/regr_smlp/master/Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv new file mode 100644 index 00000000..3bb69252 --- /dev/null +++ b/regr_smlp/master/Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv @@ -0,0 +1,4 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +7,5.0,5.0,9.0,9.0 +2,5.0,9.0,9.0,9.0 +8,9.0,9.0,9.0,9.0 diff --git a/regr_smlp/master/Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv b/regr_smlp/master/Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv new file mode 100644 index 00000000..69a38a5f --- /dev/null +++ b/regr_smlp/master/Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv @@ -0,0 +1,12 @@ +stage,solver,x0,x1,x2,y1,y2 +interface_consistency,sat,7,3 +model_consistency,sat,1,7,5,5 +witness_consistency,sat,1,7,5,5 +witness_consistency,sat,1,7,5,5 +witness_consistency,sat,1,7,5,5 +ca,unsat +ce,sat,4,6,5,9 +ca,sat,7,452984835/67108864,9,9 +ce,sat,1,7,5,5 +ca,unsat +ce,sat,1,7,5,5 diff --git a/regr_smlp/master/Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv b/regr_smlp/master/Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv new file mode 100644 index 00000000..f0c58f67 --- /dev/null +++ b/regr_smlp/master/Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,0.0,1.0 +y2,0.0,1.0 diff --git a/regr_smlp/master/Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv b/regr_smlp/master/Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv new file mode 100644 index 00000000..74d164ca --- /dev/null +++ b/regr_smlp/master/Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv @@ -0,0 +1,9 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +3,5.0,5.0,5.0,5.0 +4,9.0,5.0,9.0,5.0 +5,9.0,9.0,9.0,9.0 +0,5.0,9.0,5.0,9.0 +10,9.0,5.0,9.0,5.0 +9,5.0,5.0,5.0,5.0 +6,5.0,5.0,5.0,5.0 +1,9.0,9.0,9.0,9.0 diff --git a/regr_smlp/master/Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json b/regr_smlp/master/Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json new file mode 100644 index 00000000..10878543 --- /dev/null +++ b/regr_smlp/master/Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json @@ -0,0 +1,38 @@ +{ + "asrt1": { + "configuration_consistent": "true", + "assertion_status": "FAIL", + "counter_example": { + "x1": 4.0, + "y1": 5.0, + "x2": 6.0, + "y2": 9.0 + }, + "assertion_feasible": false + }, + "asrt2": { + "configuration_consistent": "true", + "assertion_status": "FAIL", + "counter_example": { + "x1": 1.0, + "y1": 5.0, + "x2": 7.0, + "y2": 5.0 + }, + "assertion_feasible": true + }, + "asrt3": { + "configuration_consistent": "true", + "assertion_status": "FAIL", + "counter_example": { + "x1": 1.0, + "y1": 5.0, + "x2": 7.0, + "y2": 5.0 + }, + "assertion_feasible": false + }, + "smlp_execution": "completed", + "interface_consistent": "true", + "model_consistent": "true" +} \ No newline at end of file diff --git a/regr_smlp/master/Test74_test73_model_smlp_toy_num_resp_noknobs_pred_labeled.txt b/regr_smlp/master/Test74_test73_model_smlp_toy_num_resp_noknobs_pred_labeled.txt new file mode 100644 index 00000000..2598ff98 --- /dev/null +++ b/regr_smlp/master/Test74_test73_model_smlp_toy_num_resp_noknobs_pred_labeled.txt @@ -0,0 +1,207 @@ + +smlp_logger - INFO - Model exploration specification: +{'version': '1.1', 'spec': [{'label': 'y1', 'type': 'response', 'range': 'float'}, {'label': 'y2', 'type': 'response', 'range': 'float'}, {'label': 'x0', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x1', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x2', 'type': 'input', 'range': 'float', 'bounds': [3, 7]}], 'alpha': 'x1==1 or x1==4 or x1==7'} + +smlp_logger - INFO - Executing run_smlp.py script: Start + +smlp_logger - INFO - Running SMLP in mode "verify": Start + +smlp_logger - INFO - Computed spec global constraint expressions: + +smlp_logger - INFO - Global alpha : x1==1 or x1==4 or x1==7 + +smlp_logger - INFO - Global beta : None + +smlp_logger - INFO - Radii theta : {} + +smlp_logger - INFO - Delta const : {'delta_abs': 0.0, 'delta_rel': 0.01} + +smlp_logger - INFO - Assertion asrt1: (y2**3+x2)/2<6 + +smlp_logger - INFO - Assertion asrt2: y1>=9 + +smlp_logger - INFO - Assertion asrt3: y2<0 + +smlp_logger - INFO - PREPARE DATA FOR MODELING + +smlp_logger - INFO - Preparing new data for modeling: start + +smlp_logger - INFO - loading new data + +smlp_logger - INFO - data summary + y1 y2 x0 x1 x2 +count 9.000000 9.000000 8.000000 8.000000 9.000000 +mean 6.777778 7.222222 10.250000 2.875000 5.111111 +std 2.108185 2.108185 1.035098 0.834523 1.691482 +min 5.000000 5.000000 9.000000 2.000000 3.000000 +25% 5.000000 5.000000 9.750000 2.000000 4.000000 +50% 5.000000 9.000000 10.000000 3.000000 4.000000 +75% 9.000000 9.000000 11.000000 3.250000 6.000000 +max 9.000000 9.000000 12.000000 4.000000 8.000000 + +smlp_logger - INFO - new data + categ y1 y2 x0 x1 x2 +0 c0 5 9 10.0 2.0 3 +1 c12 9 9 12.0 NaN 4 +2 c2 5 9 NaN 3.0 4 +3 c17 5 5 11.0 2.0 6 +4 c18 9 5 10.0 2.0 8 +5 c8 9 9 9.0 4.0 7 +6 c7 5 5 9.0 3.0 6 +7 c3 5 5 10.0 3.0 4 +8 c12 9 9 11.0 4.0 4 + +smlp_logger - INFO - new data after imputing missing values + x1 x2 y1 y2 +0 2.0 3 5 9 +1 2.0 4 9 9 +2 3.0 4 5 9 +3 2.0 6 5 5 +4 2.0 8 9 5 +5 4.0 7 9 9 +6 3.0 6 5 5 +7 3.0 4 5 5 +8 4.0 4 9 9 + +smlp_logger - INFO - new data after processing responses + x1 x2 y1 y2 +0 2.0 3 5 9 +1 2.0 4 9 9 +2 3.0 4 5 9 +3 2.0 6 5 5 +4 2.0 8 9 5 +5 4.0 7 9 9 +6 3.0 6 5 5 +7 3.0 4 5 5 +8 4.0 4 9 9 + +smlp_logger - INFO - new data after encoding levels of categorical features with integers + x1 x2 y1 y2 +0 2.0 3 5 9 +1 2.0 4 9 9 +2 3.0 4 5 9 +3 2.0 6 5 5 +4 2.0 8 9 5 +5 4.0 7 9 9 +6 3.0 6 5 5 +7 3.0 4 5 5 +8 4.0 4 9 9 + +smlp_logger - INFO - new data after scaling (normalizing) features and responses + x1 x2 y1 y2 +0 0.0 0.0 0.0 1.0 +1 0.0 0.2 1.0 1.0 +2 0.5 0.2 0.0 1.0 +3 0.0 0.6 0.0 0.0 +4 0.0 1.0 1.0 0.0 +5 1.0 0.8 1.0 1.0 +6 0.5 0.6 0.0 0.0 +7 0.5 0.2 0.0 0.0 +8 1.0 0.2 1.0 1.0 + +smlp_logger - INFO - Preparing new data for modeling: end + +smlp_logger - INFO - LOAD TRAINED MODEL + +smlp_logger - INFO - Seving model rerun configuration in file ../models/test73_model_rerun_model_config.json + +smlp_logger - INFO - PREDICT ON NEW DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test74_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test74_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv + +smlp_logger - INFO - Prediction on new data -- msqe: 2.667 + +smlp_logger - INFO - Prediction on new data -- r2_score: 0.325 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - Creating model exploration base components: Start + +smlp_logger - INFO - Parsing the SPEC: Start + +smlp_logger - INFO - Parsing the SPEC: End + +smlp_logger - INFO - Variable domains (alpha): {'y1': {'range': 'float', 'interval': None}, 'y2': {'range': 'float', 'interval': None}, 'x0': {'range': 'float', 'interval': [0, 10]}, 'x1': {'range': 'float', 'interval': [0, 10]}, 'x2': {'range': 'float', 'interval': [3, 7]}} + +smlp_logger - INFO - Input bounds (alpha): {'x0': {'min': 0, 'max': 10}, 'x1': {'min': 0, 'max': 10}, 'x2': {'min': 3, 'max': 7}} + +smlp_logger - INFO - Knob bounds (eta): {} + +smlp_logger - INFO - Knob grids (eta): {} + +smlp_logger - INFO - Alpha global constraints: (or (or (= x1 1) (= x1 4)) (= x1 7)) + +smlp_logger - INFO - Alpha ranges constraints: (and (and true (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) + +smlp_logger - INFO - Alpha combined constraints: (and (and (and true (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) (or (or (= x1 1) (= x1 4)) (= x1 7))) + +smlp_logger - INFO - Beta global constraints: true + +smlp_logger - INFO - Eta ranges constraints: true + +smlp_logger - INFO - Eta grid constraints: true + +smlp_logger - INFO - Eta global constraints: true + +smlp_logger - INFO - Eta combined constraints: true + +smlp_logger - INFO - Creating model exploration base components: End + +smlp_logger - INFO - Input and knob interface constraints are consistent + +smlp_logger - INFO - Building model terms: Start + +smlp_logger - INFO - Model operator counts for y1: {'add': 1, 'mul': 15, 'ite': 5, 'and': 9, 'prop': 14, 'const': 50, 'sub': 14, 'var': 14} + +smlp_logger - INFO - Model operator counts for y2: {'add': 1, 'mul': 4, 'ite': 2, 'and': 1, 'prop': 3, 'const': 14, 'sub': 3, 'var': 3} + +smlp_logger - INFO - Building model terms: End + +smlp_logger - INFO - Model interface constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt1: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt2: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt3: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying assertion asrt1 <-> (y2**3+x2)/2<6 + +smlp_logger - INFO - The configuration is inconsistent with assertion asrt1 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Verifying assertion asrt2 <-> y1>=9 + +smlp_logger - INFO - The configuration is consistent with assertion asrt2 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Verifying assertion asrt3 <-> y2<0 + +smlp_logger - INFO - The configuration is inconsistent with assertion asrt3 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Running SMLP in mode "verify": End + +smlp_logger - INFO - Executing run_smlp.py script: End diff --git a/regr_smlp/master/Test74_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json b/regr_smlp/master/Test74_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json new file mode 100644 index 00000000..6e4f6879 --- /dev/null +++ b/regr_smlp/master/Test74_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json @@ -0,0 +1,5 @@ +{ + "x1": [ + 1 + ] +} \ No newline at end of file diff --git a/regr_smlp/master/Test74_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv b/regr_smlp/master/Test74_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv new file mode 100644 index 00000000..951115f0 --- /dev/null +++ b/regr_smlp/master/Test74_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,3.5555555555555554,0.10000000000000009 +y2,1.7777777777777777,0.55 diff --git a/regr_smlp/master/Test74_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv b/regr_smlp/master/Test74_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv new file mode 100644 index 00000000..3ba8f5c9 --- /dev/null +++ b/regr_smlp/master/Test74_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv @@ -0,0 +1,10 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +0,5.0,9.0,5.0,9.0 +1,9.0,9.0,9.0,9.0 +2,5.0,9.0,9.0,9.0 +3,5.0,5.0,5.0,5.0 +4,9.0,5.0,9.0,5.0 +5,9.0,9.0,9.0,9.0 +6,5.0,5.0,5.0,5.0 +7,5.0,5.0,9.0,9.0 +8,9.0,9.0,9.0,9.0 diff --git a/regr_smlp/master/Test74_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv b/regr_smlp/master/Test74_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv new file mode 100644 index 00000000..69a38a5f --- /dev/null +++ b/regr_smlp/master/Test74_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv @@ -0,0 +1,12 @@ +stage,solver,x0,x1,x2,y1,y2 +interface_consistency,sat,7,3 +model_consistency,sat,1,7,5,5 +witness_consistency,sat,1,7,5,5 +witness_consistency,sat,1,7,5,5 +witness_consistency,sat,1,7,5,5 +ca,unsat +ce,sat,4,6,5,9 +ca,sat,7,452984835/67108864,9,9 +ce,sat,1,7,5,5 +ca,unsat +ce,sat,1,7,5,5 diff --git a/regr_smlp/master/Test74_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json b/regr_smlp/master/Test74_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json new file mode 100644 index 00000000..10878543 --- /dev/null +++ b/regr_smlp/master/Test74_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json @@ -0,0 +1,38 @@ +{ + "asrt1": { + "configuration_consistent": "true", + "assertion_status": "FAIL", + "counter_example": { + "x1": 4.0, + "y1": 5.0, + "x2": 6.0, + "y2": 9.0 + }, + "assertion_feasible": false + }, + "asrt2": { + "configuration_consistent": "true", + "assertion_status": "FAIL", + "counter_example": { + "x1": 1.0, + "y1": 5.0, + "x2": 7.0, + "y2": 5.0 + }, + "assertion_feasible": true + }, + "asrt3": { + "configuration_consistent": "true", + "assertion_status": "FAIL", + "counter_example": { + "x1": 1.0, + "y1": 5.0, + "x2": 7.0, + "y2": 5.0 + }, + "assertion_feasible": false + }, + "smlp_execution": "completed", + "interface_consistent": "true", + "model_consistent": "true" +} \ No newline at end of file diff --git a/regr_smlp/master/Test75_test73_model_smlp_toy_num_resp_noknobs_pred_labeled.txt b/regr_smlp/master/Test75_test73_model_smlp_toy_num_resp_noknobs_pred_labeled.txt new file mode 100644 index 00000000..f327d37d --- /dev/null +++ b/regr_smlp/master/Test75_test73_model_smlp_toy_num_resp_noknobs_pred_labeled.txt @@ -0,0 +1,207 @@ + +smlp_logger - INFO - Model exploration specification: +{'version': '1.1', 'spec': [{'label': 'y1', 'type': 'response', 'range': 'float'}, {'label': 'y2', 'type': 'response', 'range': 'float'}, {'label': 'x0', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x1', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x2', 'type': 'input', 'range': 'float', 'bounds': [3, 7]}], 'alpha': 'x1==1 or x1==4 or x1==7'} + +smlp_logger - INFO - Executing run_smlp.py script: Start + +smlp_logger - INFO - Running SMLP in mode "verify": Start + +smlp_logger - INFO - Computed spec global constraint expressions: + +smlp_logger - INFO - Global alpha : x1==1 or x1==4 or x1==7 + +smlp_logger - INFO - Global beta : None + +smlp_logger - INFO - Radii theta : {} + +smlp_logger - INFO - Delta const : {'delta_abs': 0.0, 'delta_rel': 0.01} + +smlp_logger - INFO - Assertion asrt1: (y2**3+x2)/2<6 + +smlp_logger - INFO - Assertion asrt2: y1>=9 + +smlp_logger - INFO - Assertion asrt3: y2<0 + +smlp_logger - INFO - PREPARE DATA FOR MODELING + +smlp_logger - INFO - Preparing new data for modeling: start + +smlp_logger - INFO - loading new data + +smlp_logger - INFO - data summary + y1 y2 x0 x1 x2 +count 9.000000 9.000000 8.000000 8.000000 9.000000 +mean 6.777778 7.222222 10.250000 2.875000 5.111111 +std 2.108185 2.108185 1.035098 0.834523 1.691482 +min 5.000000 5.000000 9.000000 2.000000 3.000000 +25% 5.000000 5.000000 9.750000 2.000000 4.000000 +50% 5.000000 9.000000 10.000000 3.000000 4.000000 +75% 9.000000 9.000000 11.000000 3.250000 6.000000 +max 9.000000 9.000000 12.000000 4.000000 8.000000 + +smlp_logger - INFO - new data + categ y1 y2 x0 x1 x2 +0 c0 5 9 10.0 2.0 3 +1 c12 9 9 12.0 NaN 4 +2 c2 5 9 NaN 3.0 4 +3 c17 5 5 11.0 2.0 6 +4 c18 9 5 10.0 2.0 8 +5 c8 9 9 9.0 4.0 7 +6 c7 5 5 9.0 3.0 6 +7 c3 5 5 10.0 3.0 4 +8 c12 9 9 11.0 4.0 4 + +smlp_logger - INFO - new data after imputing missing values + x1 x2 y1 y2 +0 2.0 3 5 9 +1 2.0 4 9 9 +2 3.0 4 5 9 +3 2.0 6 5 5 +4 2.0 8 9 5 +5 4.0 7 9 9 +6 3.0 6 5 5 +7 3.0 4 5 5 +8 4.0 4 9 9 + +smlp_logger - INFO - new data after processing responses + x1 x2 y1 y2 +0 2.0 3 5 9 +1 2.0 4 9 9 +2 3.0 4 5 9 +3 2.0 6 5 5 +4 2.0 8 9 5 +5 4.0 7 9 9 +6 3.0 6 5 5 +7 3.0 4 5 5 +8 4.0 4 9 9 + +smlp_logger - INFO - new data after encoding levels of categorical features with integers + x1 x2 y1 y2 +0 2.0 3 5 9 +1 2.0 4 9 9 +2 3.0 4 5 9 +3 2.0 6 5 5 +4 2.0 8 9 5 +5 4.0 7 9 9 +6 3.0 6 5 5 +7 3.0 4 5 5 +8 4.0 4 9 9 + +smlp_logger - INFO - new data after scaling (normalizing) features and responses + x1 x2 y1 y2 +0 0.0 0.0 0.0 1.0 +1 0.0 0.2 1.0 1.0 +2 0.5 0.2 0.0 1.0 +3 0.0 0.6 0.0 0.0 +4 0.0 1.0 1.0 0.0 +5 1.0 0.8 1.0 1.0 +6 0.5 0.6 0.0 0.0 +7 0.5 0.2 0.0 0.0 +8 1.0 0.2 1.0 1.0 + +smlp_logger - INFO - Preparing new data for modeling: end + +smlp_logger - INFO - LOAD TRAINED MODEL + +smlp_logger - INFO - Seving model rerun configuration in file ../models/test73_model_rerun_model_config.json + +smlp_logger - INFO - PREDICT ON NEW DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test75_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test75_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv + +smlp_logger - INFO - Prediction on new data -- msqe: 2.667 + +smlp_logger - INFO - Prediction on new data -- r2_score: 0.325 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - Creating model exploration base components: Start + +smlp_logger - INFO - Parsing the SPEC: Start + +smlp_logger - INFO - Parsing the SPEC: End + +smlp_logger - INFO - Variable domains (alpha): {'y1': {'range': 'float', 'interval': None}, 'y2': {'range': 'float', 'interval': None}, 'x0': {'range': 'float', 'interval': [0, 10]}, 'x1': {'range': 'float', 'interval': [0, 10]}, 'x2': {'range': 'float', 'interval': [3, 7]}} + +smlp_logger - INFO - Input bounds (alpha): {'x0': {'min': 0, 'max': 10}, 'x1': {'min': 0, 'max': 10}, 'x2': {'min': 3, 'max': 7}} + +smlp_logger - INFO - Knob bounds (eta): {} + +smlp_logger - INFO - Knob grids (eta): {} + +smlp_logger - INFO - Alpha global constraints: (or (or (= x1 1) (= x1 4)) (= x1 7)) + +smlp_logger - INFO - Alpha ranges constraints: (and (and true (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) + +smlp_logger - INFO - Alpha combined constraints: (and (and (and true (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) (or (or (= x1 1) (= x1 4)) (= x1 7))) + +smlp_logger - INFO - Beta global constraints: true + +smlp_logger - INFO - Eta ranges constraints: true + +smlp_logger - INFO - Eta grid constraints: true + +smlp_logger - INFO - Eta global constraints: true + +smlp_logger - INFO - Eta combined constraints: true + +smlp_logger - INFO - Creating model exploration base components: End + +smlp_logger - INFO - Input and knob interface constraints are consistent + +smlp_logger - INFO - Building model terms: Start + +smlp_logger - INFO - Model operator counts for y1: {'add': 1, 'mul': 15, 'ite': 5, 'and': 9, 'prop': 14, 'const': 50, 'sub': 14, 'var': 14} + +smlp_logger - INFO - Model operator counts for y2: {'add': 1, 'mul': 4, 'ite': 2, 'and': 1, 'prop': 3, 'const': 14, 'sub': 3, 'var': 3} + +smlp_logger - INFO - Building model terms: End + +smlp_logger - INFO - Model interface constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt1: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt2: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt3: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying assertion asrt1 <-> (y2**3+x2)/2<6 + +smlp_logger - INFO - The configuration is inconsistent with assertion asrt1 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Verifying assertion asrt2 <-> y1>=9 + +smlp_logger - INFO - The configuration is consistent with assertion asrt2 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Verifying assertion asrt3 <-> y2<0 + +smlp_logger - INFO - The configuration is inconsistent with assertion asrt3 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Running SMLP in mode "verify": End + +smlp_logger - INFO - Executing run_smlp.py script: End diff --git a/regr_smlp/master/Test75_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json b/regr_smlp/master/Test75_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json new file mode 100644 index 00000000..6e4f6879 --- /dev/null +++ b/regr_smlp/master/Test75_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json @@ -0,0 +1,5 @@ +{ + "x1": [ + 1 + ] +} \ No newline at end of file diff --git a/regr_smlp/master/Test75_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv b/regr_smlp/master/Test75_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv new file mode 100644 index 00000000..951115f0 --- /dev/null +++ b/regr_smlp/master/Test75_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,3.5555555555555554,0.10000000000000009 +y2,1.7777777777777777,0.55 diff --git a/regr_smlp/master/Test75_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv b/regr_smlp/master/Test75_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv new file mode 100644 index 00000000..3ba8f5c9 --- /dev/null +++ b/regr_smlp/master/Test75_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv @@ -0,0 +1,10 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +0,5.0,9.0,5.0,9.0 +1,9.0,9.0,9.0,9.0 +2,5.0,9.0,9.0,9.0 +3,5.0,5.0,5.0,5.0 +4,9.0,5.0,9.0,5.0 +5,9.0,9.0,9.0,9.0 +6,5.0,5.0,5.0,5.0 +7,5.0,5.0,9.0,9.0 +8,9.0,9.0,9.0,9.0 diff --git a/regr_smlp/master/Test75_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv b/regr_smlp/master/Test75_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv new file mode 100644 index 00000000..69a38a5f --- /dev/null +++ b/regr_smlp/master/Test75_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv @@ -0,0 +1,12 @@ +stage,solver,x0,x1,x2,y1,y2 +interface_consistency,sat,7,3 +model_consistency,sat,1,7,5,5 +witness_consistency,sat,1,7,5,5 +witness_consistency,sat,1,7,5,5 +witness_consistency,sat,1,7,5,5 +ca,unsat +ce,sat,4,6,5,9 +ca,sat,7,452984835/67108864,9,9 +ce,sat,1,7,5,5 +ca,unsat +ce,sat,1,7,5,5 diff --git a/regr_smlp/master/Test75_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json b/regr_smlp/master/Test75_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json new file mode 100644 index 00000000..10878543 --- /dev/null +++ b/regr_smlp/master/Test75_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json @@ -0,0 +1,38 @@ +{ + "asrt1": { + "configuration_consistent": "true", + "assertion_status": "FAIL", + "counter_example": { + "x1": 4.0, + "y1": 5.0, + "x2": 6.0, + "y2": 9.0 + }, + "assertion_feasible": false + }, + "asrt2": { + "configuration_consistent": "true", + "assertion_status": "FAIL", + "counter_example": { + "x1": 1.0, + "y1": 5.0, + "x2": 7.0, + "y2": 5.0 + }, + "assertion_feasible": true + }, + "asrt3": { + "configuration_consistent": "true", + "assertion_status": "FAIL", + "counter_example": { + "x1": 1.0, + "y1": 5.0, + "x2": 7.0, + "y2": 5.0 + }, + "assertion_feasible": false + }, + "smlp_execution": "completed", + "interface_consistent": "true", + "model_consistent": "true" +} \ No newline at end of file diff --git a/regr_smlp/master/Test76_smlp_toy_num_resp_noknobs.txt b/regr_smlp/master/Test76_smlp_toy_num_resp_noknobs.txt new file mode 100644 index 00000000..4d4eb429 --- /dev/null +++ b/regr_smlp/master/Test76_smlp_toy_num_resp_noknobs.txt @@ -0,0 +1,293 @@ + +smlp_logger - INFO - Model exploration specification: +{'version': '1.1', 'spec': [{'label': 'y1', 'type': 'response', 'range': 'float'}, {'label': 'y2', 'type': 'response', 'range': 'float'}, {'label': 'x0', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x1', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x2', 'type': 'input', 'range': 'float', 'bounds': [3, 7]}], 'alpha': 'x1==1 or x1==4 or x1==7'} + +smlp_logger - INFO - Executing run_smlp.py script: Start + +smlp_logger - INFO - Running SMLP in mode "verify": Start + +smlp_logger - INFO - Computed spec global constraint expressions: + +smlp_logger - INFO - Global alpha : x1==1 or x1==4 or x1==7 + +smlp_logger - INFO - Global beta : None + +smlp_logger - INFO - Radii theta : {} + +smlp_logger - INFO - Delta const : {'delta_abs': 0.0, 'delta_rel': 0.01} + +smlp_logger - INFO - Assertion asrt1: (y2**3+x2)/2<6 + +smlp_logger - INFO - Assertion asrt2: y1>=9 + +smlp_logger - INFO - Assertion asrt3: y2<0 + +smlp_logger - INFO - PREPARE DATA FOR MODELING + +smlp_logger - INFO - Preparing training data for modeling: start + +smlp_logger - INFO - loading training data + +smlp_logger - INFO - data summary + y1 y2 x0 x1 x2 +count 11.000000 11.000000 10.000000 10.000000 11.000000 +mean 6.818182 6.818182 10.400000 2.800000 5.454545 +std 2.088932 2.088932 1.074968 0.788811 1.694912 +min 5.000000 5.000000 9.000000 2.000000 3.000000 +25% 5.000000 5.000000 10.000000 2.000000 4.000000 +50% 5.000000 5.000000 10.000000 3.000000 6.000000 +75% 9.000000 9.000000 11.000000 3.000000 7.000000 +max 9.000000 9.000000 12.000000 4.000000 8.000000 + +smlp_logger - INFO - training data + categ y1 y2 x0 x1 x2 +0 c14 5 9 10.0 2.0 3 +1 c15 9 9 12.0 NaN 4 +2 c1 5 9 NaN 3.0 4 +3 c9 5 5 11.0 2.0 6 +4 c5 9 5 10.0 2.0 8 +5 c10 9 9 9.0 4.0 7 +6 c13 5 5 9.0 3.0 6 +7 c4 5 5 10.0 3.0 4 +8 c15 9 9 11.0 4.0 4 +9 c11 5 5 12.0 2.0 7 +10 c19 9 5 10.0 3.0 7 + +smlp_logger - INFO - training data after imputing missing values + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - training data after processing responses + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - MRMR feature selection for response y1 : start + +smlp_logger - INFO - MRMR selected feature scores (in the ranked order) for response y1 : + Feature Score +0 x1 1.115702 +1 x2 0.944056 + +smlp_logger - INFO - MRMR feature selection for response y1 : end + +smlp_logger - INFO - MRMR feature selection for response y2 : start + +smlp_logger - INFO - MRMR selected feature scores (in the ranked order) for response y2 : + Feature Score +1 x2 4.950294 +0 x1 1.115702 + +smlp_logger - INFO - MRMR feature selection for response y2 : end + +smlp_logger - INFO - training data after encoding levels of categorical features with integers + x1 x2 y1 y2 +0 2.0 3 5 9 +1 2.0 4 9 9 +2 3.0 4 5 9 +3 2.0 6 5 5 +4 2.0 8 9 5 +5 4.0 7 9 9 +6 3.0 6 5 5 +7 3.0 4 5 5 +8 4.0 4 9 9 +9 2.0 7 5 5 +10 3.0 7 9 5 + +smlp_logger - INFO - training data after scaling (normalizing) features and responses + x1 x2 y1 y2 +0 0.0 0.0 0.0 1.0 +1 0.0 0.2 1.0 1.0 +2 0.5 0.2 0.0 1.0 +3 0.0 0.6 0.0 0.0 +4 0.0 1.0 1.0 0.0 +5 1.0 0.8 1.0 1.0 +6 0.5 0.6 0.0 0.0 +7 0.5 0.2 0.0 0.0 +8 1.0 0.2 1.0 1.0 +9 0.0 0.8 0.0 0.0 +10 0.5 0.8 1.0 0.0 + +smlp_logger - INFO - Sampling from training data: start + +smlp_logger - INFO - Sampling from training data: end + +smlp_logger - INFO - X_train after sampling: (8, 2) + +smlp_logger - INFO - y_train after sampling: (8, 2) + +smlp_logger - INFO - Preparing training data for modeling: end + +smlp_logger - INFO - Saving data bounds into file:./test76_model_data_bounds.json + +smlp_logger - INFO - {'x1': {'min': 2.0, 'max': 4.0}, 'x2': {'min': 3.0, 'max': 8.0}, 'y1': {'min': 5.0, 'max': 9.0}, 'y2': {'min': 5.0, 'max': 9.0}} + +smlp_logger - INFO - TRAIN MODEL + +smlp_logger - INFO - Model training: start + +smlp_logger - INFO - Writing tree rules into file ./test76_model_dt_sklearn_tree_rules.txt + +smlp_logger - INFO - Model training: end + +smlp_logger - INFO - Seving model in file ./test76_model_dt_sklearn_model_complete.pkl + +smlp_logger - INFO - PREDICT ON TRAINING DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test76_smlp_toy_num_resp_noknobs_training_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test76_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv + +smlp_logger - INFO - Prediction on training data -- msqe: 0.000 + +smlp_logger - INFO - Prediction on training data -- r2_score: 1.000 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON TEST DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test76_smlp_toy_num_resp_noknobs_test_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test76_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv + +smlp_logger - INFO - Prediction on test data -- msqe: 8.000 + +smlp_logger - INFO - Prediction on test data -- r2_score: -1.250 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON LABELED DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test76_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test76_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv + +smlp_logger - INFO - Prediction on labeled data -- msqe: 2.182 + +smlp_logger - INFO - Prediction on labeled data -- r2_score: 0.450 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - Creating model exploration base components: Start + +smlp_logger - INFO - Parsing the SPEC: Start + +smlp_logger - INFO - Parsing the SPEC: End + +smlp_logger - INFO - Variable domains (alpha): {'y1': {'range': 'float', 'interval': None}, 'y2': {'range': 'float', 'interval': None}, 'x0': {'range': 'float', 'interval': [0, 10]}, 'x1': {'range': 'float', 'interval': [0, 10]}, 'x2': {'range': 'float', 'interval': [3, 7]}} + +smlp_logger - INFO - Input bounds (alpha): {'x0': {'min': 0, 'max': 10}, 'x1': {'min': 0, 'max': 10}, 'x2': {'min': 3, 'max': 7}} + +smlp_logger - INFO - Knob bounds (eta): {} + +smlp_logger - INFO - Knob grids (eta): {} + +smlp_logger - INFO - Alpha global constraints: (or (or (= x1 1) (= x1 4)) (= x1 7)) + +smlp_logger - INFO - Alpha ranges constraints: (and (and true (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) + +smlp_logger - INFO - Alpha combined constraints: (and (and (and true (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) (or (or (= x1 1) (= x1 4)) (= x1 7))) + +smlp_logger - INFO - Beta global constraints: true + +smlp_logger - INFO - Eta ranges constraints: true + +smlp_logger - INFO - Eta grid constraints: true + +smlp_logger - INFO - Eta global constraints: true + +smlp_logger - INFO - Eta combined constraints: true + +smlp_logger - INFO - Creating model exploration base components: End + +smlp_logger - INFO - Input and knob interface constraints are consistent + +smlp_logger - INFO - Building model terms: Start + +smlp_logger - INFO - Model operator counts for y1: {'add': 1, 'mul': 21, 'ite': 6, 'and': 14, 'prop': 20, 'const': 69, 'sub': 20, 'var': 20} + +smlp_logger - INFO - Model operator counts for y2: {'add': 1, 'mul': 21, 'ite': 6, 'and': 14, 'prop': 20, 'const': 69, 'sub': 20, 'var': 20} + +smlp_logger - INFO - Building model terms: End + +smlp_logger - INFO - Model interface constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt1: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt2: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt3: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying assertion asrt1 <-> (y2**3+x2)/2<6 + +smlp_logger - INFO - The configuration is inconsistent with assertion asrt1 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Verifying assertion asrt2 <-> y1>=9 + +smlp_logger - INFO - The configuration is consistent with assertion asrt2 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Verifying assertion asrt3 <-> y2<0 + +smlp_logger - INFO - The configuration is inconsistent with assertion asrt3 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Running SMLP in mode "verify": End + +smlp_logger - INFO - Executing run_smlp.py script: End diff --git a/regr_smlp/master/Test76_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv b/regr_smlp/master/Test76_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv new file mode 100644 index 00000000..7d143a2f --- /dev/null +++ b/regr_smlp/master/Test76_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,2.909090909090909,0.2666666666666666 +y2,1.4545454545454546,0.6333333333333333 diff --git a/regr_smlp/master/Test76_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv b/regr_smlp/master/Test76_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv new file mode 100644 index 00000000..7a649b6a --- /dev/null +++ b/regr_smlp/master/Test76_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv @@ -0,0 +1,12 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +0,5.0,9.0,5.0,9.0 +1,9.0,9.0,9.0,9.0 +2,5.0,9.0,9.0,9.0 +3,5.0,5.0,5.0,5.0 +4,9.0,5.0,9.0,5.0 +5,9.0,9.0,9.0,9.0 +6,5.0,5.0,5.0,5.0 +7,5.0,5.0,9.0,9.0 +8,9.0,9.0,9.0,9.0 +9,5.0,5.0,5.0,5.0 +10,9.0,5.0,9.0,5.0 diff --git a/regr_smlp/master/Test76_smlp_toy_num_resp_noknobs_missing_values_dict.json b/regr_smlp/master/Test76_smlp_toy_num_resp_noknobs_missing_values_dict.json new file mode 100644 index 00000000..553bfe33 --- /dev/null +++ b/regr_smlp/master/Test76_smlp_toy_num_resp_noknobs_missing_values_dict.json @@ -0,0 +1,8 @@ +{ + "x1": [ + 1 + ], + "x0": [ + 2 + ] +} \ No newline at end of file diff --git a/regr_smlp/master/Test76_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv b/regr_smlp/master/Test76_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv new file mode 100644 index 00000000..fd947921 --- /dev/null +++ b/regr_smlp/master/Test76_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,10.666666666666666,-2.0 +y2,5.333333333333333,-0.5 diff --git a/regr_smlp/master/Test76_smlp_toy_num_resp_noknobs_test_predictions_summary.csv b/regr_smlp/master/Test76_smlp_toy_num_resp_noknobs_test_predictions_summary.csv new file mode 100644 index 00000000..3bb69252 --- /dev/null +++ b/regr_smlp/master/Test76_smlp_toy_num_resp_noknobs_test_predictions_summary.csv @@ -0,0 +1,4 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +7,5.0,5.0,9.0,9.0 +2,5.0,9.0,9.0,9.0 +8,9.0,9.0,9.0,9.0 diff --git a/regr_smlp/master/Test76_smlp_toy_num_resp_noknobs_trace.csv b/regr_smlp/master/Test76_smlp_toy_num_resp_noknobs_trace.csv new file mode 100644 index 00000000..912b1a49 --- /dev/null +++ b/regr_smlp/master/Test76_smlp_toy_num_resp_noknobs_trace.csv @@ -0,0 +1,12 @@ +stage,solver,x0,x1,x2,y1,y2 +interface_consistency,sat,7,3 +model_consistency,sat,1,7,5,5 +witness_consistency,sat,1,7,5,5 +witness_consistency,sat,1,7,5,5 +witness_consistency,sat,1,7,5,5 +ca,unsat +ce,sat,1,27/4,5,5 +ca,sat,7,805306377/134217728,9,9 +ce,sat,1,7,5,5 +ca,unsat +ce,sat,7,805306377/134217728,9,9 diff --git a/regr_smlp/master/Test76_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv b/regr_smlp/master/Test76_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv new file mode 100644 index 00000000..f0c58f67 --- /dev/null +++ b/regr_smlp/master/Test76_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,0.0,1.0 +y2,0.0,1.0 diff --git a/regr_smlp/master/Test76_smlp_toy_num_resp_noknobs_training_predictions_summary.csv b/regr_smlp/master/Test76_smlp_toy_num_resp_noknobs_training_predictions_summary.csv new file mode 100644 index 00000000..74d164ca --- /dev/null +++ b/regr_smlp/master/Test76_smlp_toy_num_resp_noknobs_training_predictions_summary.csv @@ -0,0 +1,9 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +3,5.0,5.0,5.0,5.0 +4,9.0,5.0,9.0,5.0 +5,9.0,9.0,9.0,9.0 +0,5.0,9.0,5.0,9.0 +10,9.0,5.0,9.0,5.0 +9,5.0,5.0,5.0,5.0 +6,5.0,5.0,5.0,5.0 +1,9.0,9.0,9.0,9.0 diff --git a/regr_smlp/master/Test76_smlp_toy_num_resp_noknobs_verify_results.json b/regr_smlp/master/Test76_smlp_toy_num_resp_noknobs_verify_results.json new file mode 100644 index 00000000..87425ddb --- /dev/null +++ b/regr_smlp/master/Test76_smlp_toy_num_resp_noknobs_verify_results.json @@ -0,0 +1,38 @@ +{ + "asrt1": { + "configuration_consistent": "true", + "assertion_status": "FAIL", + "counter_example": { + "x1": 1.0, + "y1": 5.0, + "x2": 6.75, + "y2": 5.0 + }, + "assertion_feasible": false + }, + "asrt2": { + "configuration_consistent": "true", + "assertion_status": "FAIL", + "counter_example": { + "x1": 1.0, + "y1": 5.0, + "x2": 7.0, + "y2": 5.0 + }, + "assertion_feasible": true + }, + "asrt3": { + "configuration_consistent": "true", + "assertion_status": "FAIL", + "counter_example": { + "x1": 7.0, + "y1": 9.0, + "x2": 6.000000067055225, + "y2": 9.0 + }, + "assertion_feasible": false + }, + "smlp_execution": "completed", + "interface_consistent": "true", + "model_consistent": "true" +} \ No newline at end of file diff --git a/regr_smlp/master/Test77_test76_model.txt b/regr_smlp/master/Test77_test76_model.txt index 289b9ae6..b9a2ef57 100644 --- a/regr_smlp/master/Test77_test76_model.txt +++ b/regr_smlp/master/Test77_test76_model.txt @@ -26,7 +26,7 @@ smlp_logger - INFO - PREPARE DATA FOR MODELING smlp_logger - INFO - LOAD TRAINED MODEL -smlp_logger - INFO - Seving model rerun configuration in file ./../models/test76_model_rerun_model_config.json +smlp_logger - INFO - Seving model rerun configuration in file ../models/test76_model_rerun_model_config.json smlp_logger - INFO - Creating model exploration base components: Start diff --git a/regr_smlp/master/Test77_test76_model_trace.csv b/regr_smlp/master/Test77_test76_model_trace.csv new file mode 100644 index 00000000..912b1a49 --- /dev/null +++ b/regr_smlp/master/Test77_test76_model_trace.csv @@ -0,0 +1,12 @@ +stage,solver,x0,x1,x2,y1,y2 +interface_consistency,sat,7,3 +model_consistency,sat,1,7,5,5 +witness_consistency,sat,1,7,5,5 +witness_consistency,sat,1,7,5,5 +witness_consistency,sat,1,7,5,5 +ca,unsat +ce,sat,1,27/4,5,5 +ca,sat,7,805306377/134217728,9,9 +ce,sat,1,7,5,5 +ca,unsat +ce,sat,7,805306377/134217728,9,9 diff --git a/regr_smlp/master/Test78_smlp_toy_num_resp_noknobs.txt b/regr_smlp/master/Test78_smlp_toy_num_resp_noknobs.txt new file mode 100644 index 00000000..dd941ff1 --- /dev/null +++ b/regr_smlp/master/Test78_smlp_toy_num_resp_noknobs.txt @@ -0,0 +1,278 @@ + +smlp_logger - INFO - Model exploration specification: +{'version': '1.1', 'spec': [{'label': 'y1', 'type': 'response', 'range': 'float'}, {'label': 'y2', 'type': 'response', 'range': 'float'}, {'label': 'x0', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x1', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x2', 'type': 'input', 'range': 'float', 'bounds': [3, 7]}], 'alpha': 'x1==1 or x1==4 or x1==7'} + +smlp_logger - INFO - Executing run_smlp.py script: Start + +smlp_logger - INFO - Running SMLP in mode "verify": Start + +smlp_logger - INFO - Computed spec global constraint expressions: + +smlp_logger - INFO - Global alpha : x1==1 or x1==4 or x1==7 + +smlp_logger - INFO - Global beta : None + +smlp_logger - INFO - Radii theta : {} + +smlp_logger - INFO - Delta const : {'delta_abs': 0.0, 'delta_rel': 0.01} + +smlp_logger - INFO - Assertion asrt1: y1==9 + +smlp_logger - INFO - Assertion asrt2: y2>0 + +smlp_logger - INFO - PREPARE DATA FOR MODELING + +smlp_logger - INFO - Preparing training data for modeling: start + +smlp_logger - INFO - loading training data + +smlp_logger - INFO - data summary + y1 y2 x0 x1 x2 +count 11.000000 11.000000 10.000000 10.000000 11.000000 +mean 6.818182 6.818182 10.400000 2.800000 5.454545 +std 2.088932 2.088932 1.074968 0.788811 1.694912 +min 5.000000 5.000000 9.000000 2.000000 3.000000 +25% 5.000000 5.000000 10.000000 2.000000 4.000000 +50% 5.000000 5.000000 10.000000 3.000000 6.000000 +75% 9.000000 9.000000 11.000000 3.000000 7.000000 +max 9.000000 9.000000 12.000000 4.000000 8.000000 + +smlp_logger - INFO - training data + categ y1 y2 x0 x1 x2 +0 c14 5 9 10.0 2.0 3 +1 c15 9 9 12.0 NaN 4 +2 c1 5 9 NaN 3.0 4 +3 c9 5 5 11.0 2.0 6 +4 c5 9 5 10.0 2.0 8 +5 c10 9 9 9.0 4.0 7 +6 c13 5 5 9.0 3.0 6 +7 c4 5 5 10.0 3.0 4 +8 c15 9 9 11.0 4.0 4 +9 c11 5 5 12.0 2.0 7 +10 c19 9 5 10.0 3.0 7 + +smlp_logger - INFO - training data after imputing missing values + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - training data after processing responses + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - MRMR feature selection for response y1 : start + +smlp_logger - INFO - MRMR selected feature scores (in the ranked order) for response y1 : + Feature Score +0 x1 1.115702 + +smlp_logger - INFO - MRMR feature selection for response y1 : end + +smlp_logger - INFO - MRMR feature selection for response y2 : start + +smlp_logger - INFO - MRMR selected feature scores (in the ranked order) for response y2 : + Feature Score +0 x2 4.950294 + +smlp_logger - INFO - MRMR feature selection for response y2 : end + +smlp_logger - INFO - training data after encoding levels of categorical features with integers + x1 x2 y1 y2 +0 2.0 3 5 9 +1 2.0 4 9 9 +2 3.0 4 5 9 +3 2.0 6 5 5 +4 2.0 8 9 5 +5 4.0 7 9 9 +6 3.0 6 5 5 +7 3.0 4 5 5 +8 4.0 4 9 9 +9 2.0 7 5 5 +10 3.0 7 9 5 + +smlp_logger - INFO - training data after scaling (normalizing) features and responses + x1 x2 y1 y2 +0 0.0 0.0 0.0 1.0 +1 0.0 0.2 1.0 1.0 +2 0.5 0.2 0.0 1.0 +3 0.0 0.6 0.0 0.0 +4 0.0 1.0 1.0 0.0 +5 1.0 0.8 1.0 1.0 +6 0.5 0.6 0.0 0.0 +7 0.5 0.2 0.0 0.0 +8 1.0 0.2 1.0 1.0 +9 0.0 0.8 0.0 0.0 +10 0.5 0.8 1.0 0.0 + +smlp_logger - INFO - Sampling from training data: start + +smlp_logger - INFO - Sampling from training data: end + +smlp_logger - INFO - X_train after sampling: (8, 2) + +smlp_logger - INFO - y_train after sampling: (8, 2) + +smlp_logger - INFO - Preparing training data for modeling: end + +smlp_logger - INFO - Saving data bounds into file:./test78_model_data_bounds.json + +smlp_logger - INFO - {'x1': {'min': 2.0, 'max': 4.0}, 'x2': {'min': 3.0, 'max': 8.0}, 'y1': {'min': 5.0, 'max': 9.0}, 'y2': {'min': 5.0, 'max': 9.0}} + +smlp_logger - INFO - TRAIN MODEL + +smlp_logger - INFO - Model training: start + +smlp_logger - INFO - Writing tree rules into file ./test78_model_dt_sklearn_tree_rules.txt + +smlp_logger - INFO - Model training: end + +smlp_logger - INFO - Seving model in file ./test78_model_dt_sklearn_model_complete.pkl + +smlp_logger - INFO - PREDICT ON TRAINING DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test78_smlp_toy_num_resp_noknobs_training_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test78_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv + +smlp_logger - INFO - Prediction on training data -- msqe: 0.000 + +smlp_logger - INFO - Prediction on training data -- r2_score: 1.000 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON TEST DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test78_smlp_toy_num_resp_noknobs_test_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test78_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv + +smlp_logger - INFO - Prediction on test data -- msqe: 8.000 + +smlp_logger - INFO - Prediction on test data -- r2_score: -1.250 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON LABELED DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test78_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test78_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv + +smlp_logger - INFO - Prediction on labeled data -- msqe: 2.182 + +smlp_logger - INFO - Prediction on labeled data -- r2_score: 0.450 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - Creating model exploration base components: Start + +smlp_logger - INFO - Parsing the SPEC: Start + +smlp_logger - INFO - Parsing the SPEC: End + +smlp_logger - INFO - Variable domains (alpha): {'y1': {'range': 'float', 'interval': None}, 'y2': {'range': 'float', 'interval': None}, 'x0': {'range': 'float', 'interval': [0, 10]}, 'x1': {'range': 'float', 'interval': [0, 10]}, 'x2': {'range': 'float', 'interval': [3, 7]}} + +smlp_logger - INFO - Input bounds (alpha): {'x0': {'min': 0, 'max': 10}, 'x1': {'min': 0, 'max': 10}, 'x2': {'min': 3, 'max': 7}} + +smlp_logger - INFO - Knob bounds (eta): {} + +smlp_logger - INFO - Knob grids (eta): {} + +smlp_logger - INFO - Alpha global constraints: (or (or (= x1 1) (= x1 4)) (= x1 7)) + +smlp_logger - INFO - Alpha ranges constraints: (and (and true (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) + +smlp_logger - INFO - Alpha combined constraints: (and (and (and true (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) (or (or (= x1 1) (= x1 4)) (= x1 7))) + +smlp_logger - INFO - Beta global constraints: true + +smlp_logger - INFO - Eta ranges constraints: true + +smlp_logger - INFO - Eta grid constraints: true + +smlp_logger - INFO - Eta global constraints: true + +smlp_logger - INFO - Eta combined constraints: true + +smlp_logger - INFO - Creating model exploration base components: End + +smlp_logger - INFO - Input and knob interface constraints are consistent + +smlp_logger - INFO - Building model terms: Start + +smlp_logger - INFO - Model operator counts for y1: {'add': 1, 'mul': 21, 'ite': 6, 'and': 14, 'prop': 20, 'const': 69, 'sub': 20, 'var': 20} + +smlp_logger - INFO - Model operator counts for y2: {'add': 1, 'mul': 21, 'ite': 6, 'and': 14, 'prop': 20, 'const': 69, 'sub': 20, 'var': 20} + +smlp_logger - INFO - Building model terms: End + +smlp_logger - INFO - Model interface constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt1: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt2: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying assertion asrt1 <-> y1==9 + +smlp_logger - INFO - The configuration is consistent with assertion asrt1 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Verifying assertion asrt2 <-> y2>0 + +smlp_logger - INFO - The configuration is consistent with assertion asrt2 + +smlp_logger - INFO - Completed with result: PASS + +smlp_logger - INFO - Running SMLP in mode "verify": End + +smlp_logger - INFO - Executing run_smlp.py script: End diff --git a/regr_smlp/master/Test78_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv b/regr_smlp/master/Test78_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv new file mode 100644 index 00000000..7d143a2f --- /dev/null +++ b/regr_smlp/master/Test78_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,2.909090909090909,0.2666666666666666 +y2,1.4545454545454546,0.6333333333333333 diff --git a/regr_smlp/master/Test78_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv b/regr_smlp/master/Test78_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv new file mode 100644 index 00000000..7a649b6a --- /dev/null +++ b/regr_smlp/master/Test78_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv @@ -0,0 +1,12 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +0,5.0,9.0,5.0,9.0 +1,9.0,9.0,9.0,9.0 +2,5.0,9.0,9.0,9.0 +3,5.0,5.0,5.0,5.0 +4,9.0,5.0,9.0,5.0 +5,9.0,9.0,9.0,9.0 +6,5.0,5.0,5.0,5.0 +7,5.0,5.0,9.0,9.0 +8,9.0,9.0,9.0,9.0 +9,5.0,5.0,5.0,5.0 +10,9.0,5.0,9.0,5.0 diff --git a/regr_smlp/master/Test78_smlp_toy_num_resp_noknobs_missing_values_dict.json b/regr_smlp/master/Test78_smlp_toy_num_resp_noknobs_missing_values_dict.json new file mode 100644 index 00000000..553bfe33 --- /dev/null +++ b/regr_smlp/master/Test78_smlp_toy_num_resp_noknobs_missing_values_dict.json @@ -0,0 +1,8 @@ +{ + "x1": [ + 1 + ], + "x0": [ + 2 + ] +} \ No newline at end of file diff --git a/regr_smlp/master/Test78_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv b/regr_smlp/master/Test78_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv new file mode 100644 index 00000000..fd947921 --- /dev/null +++ b/regr_smlp/master/Test78_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,10.666666666666666,-2.0 +y2,5.333333333333333,-0.5 diff --git a/regr_smlp/master/Test78_smlp_toy_num_resp_noknobs_test_predictions_summary.csv b/regr_smlp/master/Test78_smlp_toy_num_resp_noknobs_test_predictions_summary.csv new file mode 100644 index 00000000..3bb69252 --- /dev/null +++ b/regr_smlp/master/Test78_smlp_toy_num_resp_noknobs_test_predictions_summary.csv @@ -0,0 +1,4 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +7,5.0,5.0,9.0,9.0 +2,5.0,9.0,9.0,9.0 +8,9.0,9.0,9.0,9.0 diff --git a/regr_smlp/master/Test78_smlp_toy_num_resp_noknobs_trace.csv b/regr_smlp/master/Test78_smlp_toy_num_resp_noknobs_trace.csv new file mode 100644 index 00000000..c22e67e6 --- /dev/null +++ b/regr_smlp/master/Test78_smlp_toy_num_resp_noknobs_trace.csv @@ -0,0 +1,9 @@ +stage,solver,x0,x1,x2,y1,y2 +interface_consistency,sat,7,3 +model_consistency,sat,1,7,5,5 +witness_consistency,sat,1,7,5,5 +witness_consistency,sat,1,7,5,5 +ca,sat,1,603979777/134217728,9,9 +ce,sat,7,3,5,9 +ca,sat,7,805306377/134217728,9,9 +ce,unsat diff --git a/regr_smlp/master/Test78_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv b/regr_smlp/master/Test78_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv new file mode 100644 index 00000000..f0c58f67 --- /dev/null +++ b/regr_smlp/master/Test78_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,0.0,1.0 +y2,0.0,1.0 diff --git a/regr_smlp/master/Test78_smlp_toy_num_resp_noknobs_training_predictions_summary.csv b/regr_smlp/master/Test78_smlp_toy_num_resp_noknobs_training_predictions_summary.csv new file mode 100644 index 00000000..74d164ca --- /dev/null +++ b/regr_smlp/master/Test78_smlp_toy_num_resp_noknobs_training_predictions_summary.csv @@ -0,0 +1,9 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +3,5.0,5.0,5.0,5.0 +4,9.0,5.0,9.0,5.0 +5,9.0,9.0,9.0,9.0 +0,5.0,9.0,5.0,9.0 +10,9.0,5.0,9.0,5.0 +9,5.0,5.0,5.0,5.0 +6,5.0,5.0,5.0,5.0 +1,9.0,9.0,9.0,9.0 diff --git a/regr_smlp/master/Test78_smlp_toy_num_resp_noknobs_verify_results.json b/regr_smlp/master/Test78_smlp_toy_num_resp_noknobs_verify_results.json new file mode 100644 index 00000000..69481125 --- /dev/null +++ b/regr_smlp/master/Test78_smlp_toy_num_resp_noknobs_verify_results.json @@ -0,0 +1,22 @@ +{ + "asrt1": { + "configuration_consistent": "true", + "assertion_status": "FAIL", + "counter_example": { + "x1": 7.0, + "y1": 5.0, + "x2": 3.0, + "y2": 9.0 + }, + "assertion_feasible": true + }, + "asrt2": { + "configuration_consistent": "true", + "assertion_status": "PASS", + "counter_example": null, + "assertion_feasible": true + }, + "smlp_execution": "completed", + "interface_consistent": "true", + "model_consistent": "true" +} \ No newline at end of file diff --git a/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs.txt b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs.txt new file mode 100644 index 00000000..5be4da4e --- /dev/null +++ b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs.txt @@ -0,0 +1,293 @@ + +smlp_logger - INFO - Model exploration specification: +{'version': '1.1', 'spec': [{'label': 'y1', 'type': 'response', 'range': 'float'}, {'label': 'y2', 'type': 'response', 'range': 'float'}, {'label': 'x0', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x1', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x2', 'type': 'input', 'range': 'float', 'bounds': [3, 7]}], 'alpha': 'x1==1 or x1==4 or x1==7'} + +smlp_logger - INFO - Executing run_smlp.py script: Start + +smlp_logger - INFO - Running SMLP in mode "verify": Start + +smlp_logger - INFO - Computed spec global constraint expressions: + +smlp_logger - INFO - Global alpha : x2==7.0 and x0==0 and x1==2.5 + +smlp_logger - INFO - Global beta : None + +smlp_logger - INFO - Radii theta : {} + +smlp_logger - INFO - Delta const : {'delta_abs': 0.0, 'delta_rel': 0.01} + +smlp_logger - INFO - Assertion asrt1: (y2**3+x2)/2<6 + +smlp_logger - INFO - Assertion asrt2: y1>=9 + +smlp_logger - INFO - Assertion asrt3: y2<0 + +smlp_logger - INFO - PREPARE DATA FOR MODELING + +smlp_logger - INFO - Preparing training data for modeling: start + +smlp_logger - INFO - loading training data + +smlp_logger - INFO - data summary + y1 y2 x0 x1 x2 +count 11.000000 11.000000 10.000000 10.000000 11.000000 +mean 6.818182 6.818182 10.400000 2.800000 5.454545 +std 2.088932 2.088932 1.074968 0.788811 1.694912 +min 5.000000 5.000000 9.000000 2.000000 3.000000 +25% 5.000000 5.000000 10.000000 2.000000 4.000000 +50% 5.000000 5.000000 10.000000 3.000000 6.000000 +75% 9.000000 9.000000 11.000000 3.000000 7.000000 +max 9.000000 9.000000 12.000000 4.000000 8.000000 + +smlp_logger - INFO - training data + categ y1 y2 x0 x1 x2 +0 c14 5 9 10.0 2.0 3 +1 c15 9 9 12.0 NaN 4 +2 c1 5 9 NaN 3.0 4 +3 c9 5 5 11.0 2.0 6 +4 c5 9 5 10.0 2.0 8 +5 c10 9 9 9.0 4.0 7 +6 c13 5 5 9.0 3.0 6 +7 c4 5 5 10.0 3.0 4 +8 c15 9 9 11.0 4.0 4 +9 c11 5 5 12.0 2.0 7 +10 c19 9 5 10.0 3.0 7 + +smlp_logger - INFO - training data after imputing missing values + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - training data after processing responses + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - MRMR feature selection for response y1 : start + +smlp_logger - INFO - MRMR selected feature scores (in the ranked order) for response y1 : + Feature Score +0 x1 1.115702 +1 x2 0.944056 + +smlp_logger - INFO - MRMR feature selection for response y1 : end + +smlp_logger - INFO - MRMR feature selection for response y2 : start + +smlp_logger - INFO - MRMR selected feature scores (in the ranked order) for response y2 : + Feature Score +1 x2 4.950294 +0 x1 1.115702 + +smlp_logger - INFO - MRMR feature selection for response y2 : end + +smlp_logger - INFO - training data after encoding levels of categorical features with integers + x0 x1 x2 y1 y2 +0 10.0 2.0 3 5 9 +1 12.0 2.0 4 9 9 +2 10.0 3.0 4 5 9 +3 11.0 2.0 6 5 5 +4 10.0 2.0 8 9 5 +5 9.0 4.0 7 9 9 +6 9.0 3.0 6 5 5 +7 10.0 3.0 4 5 5 +8 11.0 4.0 4 9 9 +9 12.0 2.0 7 5 5 +10 10.0 3.0 7 9 5 + +smlp_logger - INFO - training data after scaling (normalizing) features and responses + x0 x1 x2 y1 y2 +0 0.333333 0.0 0.0 0.0 1.0 +1 1.000000 0.0 0.2 1.0 1.0 +2 0.333333 0.5 0.2 0.0 1.0 +3 0.666667 0.0 0.6 0.0 0.0 +4 0.333333 0.0 1.0 1.0 0.0 +5 0.000000 1.0 0.8 1.0 1.0 +6 0.000000 0.5 0.6 0.0 0.0 +7 0.333333 0.5 0.2 0.0 0.0 +8 0.666667 1.0 0.2 1.0 1.0 +9 1.000000 0.0 0.8 0.0 0.0 +10 0.333333 0.5 0.8 1.0 0.0 + +smlp_logger - INFO - Sampling from training data: start + +smlp_logger - INFO - Sampling from training data: end + +smlp_logger - INFO - X_train after sampling: (8, 3) + +smlp_logger - INFO - y_train after sampling: (8, 2) + +smlp_logger - INFO - Preparing training data for modeling: end + +smlp_logger - INFO - Saving data bounds into file:./Test84_smlp_toy_num_resp_noknobs_data_bounds.json + +smlp_logger - INFO - {'x0': {'min': 9.0, 'max': 12.0}, 'x1': {'min': 2.0, 'max': 4.0}, 'x2': {'min': 3.0, 'max': 8.0}, 'y1': {'min': 5.0, 'max': 9.0}, 'y2': {'min': 5.0, 'max': 9.0}} + +smlp_logger - INFO - TRAIN MODEL + +smlp_logger - INFO - Model training: start + +smlp_logger - INFO - Writing tree rules into file ./Test84_smlp_toy_num_resp_noknobs_dt_sklearn_tree_rules.txt + +smlp_logger - INFO - Model training: end + +smlp_logger - INFO - Seving model in file ./Test84_smlp_toy_num_resp_noknobs_dt_sklearn_model_complete.pkl + +smlp_logger - INFO - PREDICT ON TRAINING DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test84_smlp_toy_num_resp_noknobs_training_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test84_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv + +smlp_logger - INFO - Prediction on training data -- msqe: 0.000 + +smlp_logger - INFO - Prediction on training data -- r2_score: 1.000 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON TEST DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test84_smlp_toy_num_resp_noknobs_test_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test84_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv + +smlp_logger - INFO - Prediction on test data -- msqe: 2.667 + +smlp_logger - INFO - Prediction on test data -- r2_score: 0.250 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - PREDICT ON LABELED DATA + +smlp_logger - INFO - Model prediction: start + +smlp_logger - INFO - Model prediction: end + +smlp_logger - INFO - Reporting prediction results: start + +smlp_logger - INFO - Saving predictions summary into file: +./Test84_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv + +smlp_logger - INFO - Saving prediction precisions into file: +./Test84_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv + +smlp_logger - INFO - Prediction on labeled data -- msqe: 0.727 + +smlp_logger - INFO - Prediction on labeled data -- r2_score: 0.817 + +smlp_logger - INFO - Reporting prediction results: end + +smlp_logger - INFO - Creating model exploration base components: Start + +smlp_logger - INFO - Parsing the SPEC: Start + +smlp_logger - INFO - Parsing the SPEC: End + +smlp_logger - INFO - Variable domains (alpha): {'y1': {'range': 'float', 'interval': None}, 'y2': {'range': 'float', 'interval': None}, 'x0': {'range': 'float', 'interval': [0, 10]}, 'x1': {'range': 'float', 'interval': [0, 10]}, 'x2': {'range': 'float', 'interval': [3, 7]}} + +smlp_logger - INFO - Input bounds (alpha): {'x0': {'min': 0, 'max': 10}, 'x1': {'min': 0, 'max': 10}, 'x2': {'min': 3, 'max': 7}} + +smlp_logger - INFO - Knob bounds (eta): {} + +smlp_logger - INFO - Knob grids (eta): {} + +smlp_logger - INFO - Alpha global constraints: (and (and (= x2 7) (= x0 0)) (= x1 (/ 5 2))) + +smlp_logger - INFO - Alpha ranges constraints: (and (and (and true (and (>= x0 0) (<= x0 10))) (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) + +smlp_logger - INFO - Alpha combined constraints: (and (and (and (and true (and (>= x0 0) (<= x0 10))) (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) (and (and (= x2 7) (= x0 0)) (= x1 (/ 5 2)))) + +smlp_logger - INFO - Beta global constraints: true + +smlp_logger - INFO - Eta ranges constraints: true + +smlp_logger - INFO - Eta grid constraints: true + +smlp_logger - INFO - Eta global constraints: true + +smlp_logger - INFO - Eta combined constraints: true + +smlp_logger - INFO - Creating model exploration base components: End + +smlp_logger - INFO - Input and knob interface constraints are consistent + +smlp_logger - INFO - Building model terms: Start + +smlp_logger - INFO - Model operator counts for y1: {'add': 1, 'mul': 15, 'ite': 5, 'and': 9, 'prop': 14, 'const': 50, 'sub': 14, 'var': 14} + +smlp_logger - INFO - Model operator counts for y2: {'add': 1, 'mul': 15, 'ite': 5, 'and': 9, 'prop': 14, 'const': 50, 'sub': 14, 'var': 14} + +smlp_logger - INFO - Building model terms: End + +smlp_logger - INFO - Model interface constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt1: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt2: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying consistency of configuration for assertion asrt3: + true + +smlp_logger - INFO - Input, knob and configuration constraints are consistent + +smlp_logger - INFO - Verifying assertion asrt1 <-> (y2**3+x2)/2<6 + +smlp_logger - INFO - The configuration is inconsistent with assertion asrt1 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Verifying assertion asrt2 <-> y1>=9 + +smlp_logger - INFO - The configuration is consistent with assertion asrt2 + +smlp_logger - INFO - Completed with result: PASS + +smlp_logger - INFO - Verifying assertion asrt3 <-> y2<0 + +smlp_logger - INFO - The configuration is inconsistent with assertion asrt3 + +smlp_logger - INFO - Completed with result: FAIL + +smlp_logger - INFO - Running SMLP in mode "verify": End + +smlp_logger - INFO - Executing run_smlp.py script: End diff --git a/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_data_bounds.json b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_data_bounds.json new file mode 100644 index 00000000..5df59662 --- /dev/null +++ b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_data_bounds.json @@ -0,0 +1,22 @@ +{ + "x0": { + "min": 9.0, + "max": 12.0 + }, + "x1": { + "min": 2.0, + "max": 4.0 + }, + "x2": { + "min": 3.0, + "max": 8.0 + }, + "y1": { + "min": 5.0, + "max": 9.0 + }, + "y2": { + "min": 5.0, + "max": 9.0 + } +} \ No newline at end of file diff --git a/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_dt_sklearn_model_complete.pkl b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_dt_sklearn_model_complete.pkl new file mode 100644 index 00000000..3b6c85e5 Binary files /dev/null and b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_dt_sklearn_model_complete.pkl differ diff --git a/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_dt_sklearn_tree_rules.txt b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_dt_sklearn_tree_rules.txt new file mode 100644 index 00000000..c3f3f68c --- /dev/null +++ b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_dt_sklearn_tree_rules.txt @@ -0,0 +1,10 @@ +#Forest semantics: majority vote +#Number of trees: 1 + +#TREE 0 +if (x2 > 0.4000000134110451) and (x1 <= 0.75) and (x0 > 0.5000000149011612) then (y1 = 0.0) and (y2 = 0.0) | based on 2 samples +if (x2 > 0.4000000134110451) and (x1 <= 0.75) and (x0 <= 0.5000000149011612) and (x2 > 0.7000000178813934) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +if (x2 > 0.4000000134110451) and (x1 > 0.75) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +if (x2 > 0.4000000134110451) and (x1 <= 0.75) and (x0 <= 0.5000000149011612) and (x2 <= 0.7000000178813934) then (y1 = 0.0) and (y2 = 0.0) | based on 1 samples +if (x2 <= 0.4000000134110451) and (x0 > 0.6666666716337204) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +if (x2 <= 0.4000000134110451) and (x0 <= 0.6666666716337204) then (y1 = 0.0) and (y2 = 1.0) | based on 1 samples diff --git a/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_features_scaler.pkl b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_features_scaler.pkl new file mode 100644 index 00000000..1e8ed87e Binary files /dev/null and b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_features_scaler.pkl differ diff --git a/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv new file mode 100644 index 00000000..cf088bcd --- /dev/null +++ b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,0.0,1.0 +y2,1.4545454545454546,0.6333333333333333 diff --git a/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv new file mode 100644 index 00000000..2cce12ec --- /dev/null +++ b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv @@ -0,0 +1,12 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +0,5.0,9.0,5.0,9.0 +1,9.0,9.0,9.0,9.0 +2,5.0,9.0,5.0,9.0 +3,5.0,5.0,5.0,5.0 +4,9.0,5.0,9.0,5.0 +5,9.0,9.0,9.0,9.0 +6,5.0,5.0,5.0,5.0 +7,5.0,5.0,5.0,9.0 +8,9.0,9.0,9.0,9.0 +9,5.0,5.0,5.0,5.0 +10,9.0,5.0,9.0,5.0 diff --git a/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_missing_values_dict.json b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_missing_values_dict.json new file mode 100644 index 00000000..553bfe33 --- /dev/null +++ b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_missing_values_dict.json @@ -0,0 +1,8 @@ +{ + "x1": [ + 1 + ], + "x0": [ + 2 + ] +} \ No newline at end of file diff --git a/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_model_features_dict.json b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_model_features_dict.json new file mode 100644 index 00000000..8e6a1c9a --- /dev/null +++ b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_model_features_dict.json @@ -0,0 +1,12 @@ +{ + "y1": [ + "x0", + "x1", + "x2" + ], + "y2": [ + "x0", + "x1", + "x2" + ] +} \ No newline at end of file diff --git a/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_model_levels_dict.json b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_model_levels_dict.json new file mode 100644 index 00000000..9e26dfee --- /dev/null +++ b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_model_levels_dict.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_rerun_model_config.json b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_rerun_model_config.json new file mode 100644 index 00000000..c7a3a170 --- /dev/null +++ b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_rerun_model_config.json @@ -0,0 +1,172 @@ +{ + "alpha": "x2==7.0 and x0==0 and x1==2.5", + "analytics_mode": "verify", + "approximate_fractions": true, + "assertions_expressions": "(y2**3+x2)/2<6;y1>=9;y2<0", + "assertions_names": "asrt1,asrt2,asrt3", + "beta": null, + "center_offset": "0", + "compress_rules": false, + "continuous_correlation_estimators": [ + "pearson", + "spearman" + ], + "correlations_and_mutual_information": true, + "data_scaler": "min_max", + "delta_absolute": 0.0, + "delta_relative": 0.01, + "discretization_algo": "uniform", + "discretization_bins": 10, + "discretization_labels": true, + "discretization_type": "category", + "discretize_numeric_features": null, + "doe_algo": null, + "doe_box_behnken_centers": 1, + "doe_central_composite_alpha": "o", + "doe_central_composite_center": "2,2", + "doe_central_composite_face": "ccf", + "doe_design_resolution": null, + "doe_factor_level_ranges": null, + "doe_num_samples": null, + "doe_prob_distribution": "Normal", + "doe_spec_file": null, + "dt_sklearn_ccp_alpha": 0.0, + "dt_sklearn_criterion": "squared_error", + "dt_sklearn_max_depth": 15, + "dt_sklearn_max_features": null, + "dt_sklearn_max_leaf_nodes": null, + "dt_sklearn_min_impurity_decrease": 0.0, + "dt_sklearn_min_samples_leaf": 1, + "dt_sklearn_min_samples_split": 2, + "dt_sklearn_min_weight_fraction_leaf": 0.0, + "dt_sklearn_random_state": null, + "dt_sklearn_splitter": "best", + "epsilon": 0.05, + "et_sklearn_bootstrap": true, + "et_sklearn_ccp_alpha": 0.0, + "et_sklearn_criterion": "squared_error", + "et_sklearn_max_depth": null, + "et_sklearn_max_features": 1.0, + "et_sklearn_max_leaf_nodes": null, + "et_sklearn_max_samples": null, + "et_sklearn_min_impurity_decrease": 0.0, + "et_sklearn_min_samples_leaf": 1, + "et_sklearn_min_samples_split": 2, + "et_sklearn_min_weight_fraction_leaf": 0.0, + "et_sklearn_n_estimators": 100, + "et_sklearn_random_state": null, + "et_sklearn_verbose": 0, + "et_sklearn_warm_start": false, + "eta": null, + "features": "x0,x1,x2", + "fraction_precision": 64, + "impute_responses": false, + "interactive_plots": false, + "keep_features": [], + "labeled_data": null, + "lemma_precision": 0, + "load_configuration": null, + "log_files_prefix": null, + "log_level": "info", + "log_mode": "w", + "log_time": false, + "model": "dt_sklearn", + "model_caret_cross_validation": true, + "model_caret_return_train_score": false, + "model_caret_verbose": true, + "model_name": null, + "model_per_response": false, + "mrmr_feat_count_for_correlation": 15, + "mrmr_feat_count_for_prediction": 2, + "mutual_information_method": "normalized", + "negative_value": 0, + "new_data": null, + "nn_keras_batch_size": 200, + "nn_keras_batches_grid": null, + "nn_keras_epochs": 2000, + "nn_keras_hid_activation": "relu", + "nn_keras_layers": "2,1", + "nn_keras_layers_grid": null, + "nn_keras_learning_rate": 0.001, + "nn_keras_learning_rates_grid": null, + "nn_keras_loss_function": "mse", + "nn_keras_loss_functions_grid": null, + "nn_keras_metrics": [ + "mse" + ], + "nn_keras_optimizer": "adam", + "nn_keras_out_activation": "linear", + "nn_keras_sequential_api": true, + "nn_keras_tuner_algo": null, + "nn_keras_weights_precision": null, + "nnet_encoding": "nested", + "objectives_expressions": null, + "objectives_names": "None", + "optimization_strategy": "eager", + "optimize_pareto": true, + "output_directory": "./", + "poly_sklearn_copy_X": true, + "poly_sklearn_degree": 2, + "poly_sklearn_fit_intercept": true, + "poly_sklearn_n_jobs": null, + "poly_sklearn_positive": false, + "positive_value": 1, + "prediction_plots": false, + "psg_max_dimension": 3, + "psg_quality_target": "Lift", + "psg_top_ranked": 15, + "query_expressions": null, + "query_names": "None", + "radius_absolute": null, + "radius_relative": null, + "response": "y1,y2", + "response_map": null, + "response_plots": false, + "response_to_bool": null, + "rf_sklearn_bootstrap": true, + "rf_sklearn_ccp_alpha": 0.0, + "rf_sklearn_criterion": "squared_error", + "rf_sklearn_max_depth": null, + "rf_sklearn_max_features": 1.0, + "rf_sklearn_max_leaf_nodes": null, + "rf_sklearn_max_samples": null, + "rf_sklearn_min_impurity_decrease": 0.0, + "rf_sklearn_min_samples_leaf": 1, + "rf_sklearn_min_samples_split": 2, + "rf_sklearn_min_weight_fraction_leaf": 0.0, + "rf_sklearn_n_estimators": 100, + "rf_sklearn_random_state": null, + "rf_sklearn_verbose": 0, + "rf_sklearn_warm_start": false, + "sample_weights_coef": 0, + "sample_weights_exponent": 0, + "sample_weights_intercept": 0, + "save_configuration": false, + "save_model": "false", + "save_model_rerun_configuration": true, + "scale_features": true, + "scale_objectives": true, + "scale_responses": true, + "seed": 10, + "setup_caret_data_split_shuffle": true, + "setup_caret_fold": 0, + "setup_caret_session_id": null, + "setup_caret_verbose": true, + "simplify_terms": false, + "solver": "z3", + "solver_logic": "ALL", + "solver_path": null, + "spec": "../specs/smlp_toy_num_resp_noknobs_verify.spec", + "split_test": 0.2, + "trace_anonymize": false, + "trace_precision": 0, + "trace_runtime": 0, + "train_first_n": 0, + "train_random_n": 0, + "train_uniform_n": 0, + "tree_encoding": "nested", + "tuner_caret_search_algorithm": "random", + "tuner_caret_tuner_verbose": true, + "use_model": "true", + "vacuity_check": true +} \ No newline at end of file diff --git a/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_responses_scaler.pkl b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_responses_scaler.pkl new file mode 100644 index 00000000..07d03657 Binary files /dev/null and b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_responses_scaler.pkl differ diff --git a/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_smlp_full_model_term.json b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_smlp_full_model_term.json new file mode 100644 index 00000000..80c624e1 --- /dev/null +++ b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_smlp_full_model_term.json @@ -0,0 +1 @@ +"{'y1': |:1| (/ 44739243 67108864))) 1 (ite (and (and (and (> |:0| (/ 53687093 134217728)) (<= |:2| (/ 3 4))) (<= |:1| (/ 33554433 67108864))) (<= |:0| (/ 23488103 33554432))) 0 (ite (and (> |:0| (/ 53687093 134217728)) (> |:2| (/ 3 4))) 1 (ite (and (and (and (> |:0| (/ 53687093 134217728)) (<= |:2| (/ 3 4))) (<= |:1| (/ 33554433 67108864))) (> |:0| (/ 23488103 33554432))) 1 0))))) 4) 5))))>, 'y2': |:1| (/ 44739243 67108864))) 1 (ite (and (and (and (> |:0| (/ 53687093 134217728)) (<= |:2| (/ 3 4))) (<= |:1| (/ 33554433 67108864))) (<= |:0| (/ 23488103 33554432))) 0 (ite (and (> |:0| (/ 53687093 134217728)) (> |:2| (/ 3 4))) 1 (ite (and (and (and (> |:0| (/ 53687093 134217728)) (<= |:2| (/ 3 4))) (<= |:1| (/ 33554433 67108864))) (> |:0| (/ 23488103 33554432))) 0 0))))) 4) 5))))>}" \ No newline at end of file diff --git a/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_smlp_model_term.json b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_smlp_model_term.json new file mode 100644 index 00000000..ea7ac531 --- /dev/null +++ b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_smlp_model_term.json @@ -0,0 +1 @@ +"{'y1_scaled': x_scaled (/ 44739243 67108864))) 1 (ite (and (and (and (> p2_scaled (/ 53687093 134217728)) (<= p1_scaled (/ 3 4))) (<= x_scaled (/ 33554433 67108864))) (<= p2_scaled (/ 23488103 33554432))) 0 (ite (and (> p2_scaled (/ 53687093 134217728)) (> p1_scaled (/ 3 4))) 1 (ite (and (and (and (> p2_scaled (/ 53687093 134217728)) (<= p1_scaled (/ 3 4))) (<= x_scaled (/ 33554433 67108864))) (> p2_scaled (/ 23488103 33554432))) 1 0)))))>, 'y2_scaled': x_scaled (/ 44739243 67108864))) 1 (ite (and (and (and (> p2_scaled (/ 53687093 134217728)) (<= p1_scaled (/ 3 4))) (<= x_scaled (/ 33554433 67108864))) (<= p2_scaled (/ 23488103 33554432))) 0 (ite (and (> p2_scaled (/ 53687093 134217728)) (> p1_scaled (/ 3 4))) 1 (ite (and (and (and (> p2_scaled (/ 53687093 134217728)) (<= p1_scaled (/ 3 4))) (<= x_scaled (/ 33554433 67108864))) (> p2_scaled (/ 23488103 33554432))) 0 0)))))>}" \ No newline at end of file diff --git a/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv new file mode 100644 index 00000000..ec56b74c --- /dev/null +++ b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,0.0,1.0 +y2,5.333333333333333,-0.5 diff --git a/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_test_predictions_summary.csv b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_test_predictions_summary.csv new file mode 100644 index 00000000..3477d51d --- /dev/null +++ b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_test_predictions_summary.csv @@ -0,0 +1,4 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +7,5.0,5.0,5.0,9.0 +2,5.0,9.0,5.0,9.0 +8,9.0,9.0,9.0,9.0 diff --git a/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_trace.csv b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_trace.csv new file mode 100644 index 00000000..c7df48b0 --- /dev/null +++ b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_trace.csv @@ -0,0 +1,12 @@ +stage,solver,x0,x1,x2,y1,y2 +interface_consistency,sat,0,5/2,7 +model_consistency,sat,0,5/2,7,9,5 +witness_consistency,sat,0,5/2,7,9,5 +witness_consistency,sat,0,5/2,7,9,5 +witness_consistency,sat,0,5/2,7,9,5 +ca,unsat +ce,sat,0,5/2,7,9,5 +ca,sat,0,5/2,7,9,5 +ce,unsat +ca,unsat +ce,sat,0,5/2,7,9,5 diff --git a/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv new file mode 100644 index 00000000..f0c58f67 --- /dev/null +++ b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv @@ -0,0 +1,3 @@ +response,msqe,r2_score +y1,0.0,1.0 +y2,0.0,1.0 diff --git a/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_training_predictions_summary.csv b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_training_predictions_summary.csv new file mode 100644 index 00000000..74d164ca --- /dev/null +++ b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_training_predictions_summary.csv @@ -0,0 +1,9 @@ +,y1,y2,y1_dt_sklearn,y2_dt_sklearn +3,5.0,5.0,5.0,5.0 +4,9.0,5.0,9.0,5.0 +5,9.0,9.0,9.0,9.0 +0,5.0,9.0,5.0,9.0 +10,9.0,5.0,9.0,5.0 +9,5.0,5.0,5.0,5.0 +6,5.0,5.0,5.0,5.0 +1,9.0,9.0,9.0,9.0 diff --git a/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_verify_results.json b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_verify_results.json new file mode 100644 index 00000000..df922ae4 --- /dev/null +++ b/regr_smlp/master/Test84_smlp_toy_num_resp_noknobs_verify_results.json @@ -0,0 +1,35 @@ +{ + "asrt1": { + "configuration_consistent": "true", + "assertion_status": "FAIL", + "counter_example": { + "x0": 0.0, + "x1": 2.5, + "y1": 9.0, + "x2": 7.0, + "y2": 5.0 + }, + "assertion_feasible": false + }, + "asrt2": { + "configuration_consistent": "true", + "assertion_status": "PASS", + "counter_example": null, + "assertion_feasible": true + }, + "asrt3": { + "configuration_consistent": "true", + "assertion_status": "FAIL", + "counter_example": { + "x0": 0.0, + "x1": 2.5, + "y1": 9.0, + "x2": 7.0, + "y2": 5.0 + }, + "assertion_feasible": false + }, + "smlp_execution": "completed", + "interface_consistent": "true", + "model_consistent": "true" +} \ No newline at end of file diff --git a/regr_smlp/master/test65_model_smlp_full_model_term.json b/regr_smlp/master/test65_model_smlp_full_model_term.json new file mode 100644 index 00000000..80c624e1 --- /dev/null +++ b/regr_smlp/master/test65_model_smlp_full_model_term.json @@ -0,0 +1 @@ +"{'y1': |:1| (/ 44739243 67108864))) 1 (ite (and (and (and (> |:0| (/ 53687093 134217728)) (<= |:2| (/ 3 4))) (<= |:1| (/ 33554433 67108864))) (<= |:0| (/ 23488103 33554432))) 0 (ite (and (> |:0| (/ 53687093 134217728)) (> |:2| (/ 3 4))) 1 (ite (and (and (and (> |:0| (/ 53687093 134217728)) (<= |:2| (/ 3 4))) (<= |:1| (/ 33554433 67108864))) (> |:0| (/ 23488103 33554432))) 1 0))))) 4) 5))))>, 'y2': |:1| (/ 44739243 67108864))) 1 (ite (and (and (and (> |:0| (/ 53687093 134217728)) (<= |:2| (/ 3 4))) (<= |:1| (/ 33554433 67108864))) (<= |:0| (/ 23488103 33554432))) 0 (ite (and (> |:0| (/ 53687093 134217728)) (> |:2| (/ 3 4))) 1 (ite (and (and (and (> |:0| (/ 53687093 134217728)) (<= |:2| (/ 3 4))) (<= |:1| (/ 33554433 67108864))) (> |:0| (/ 23488103 33554432))) 0 0))))) 4) 5))))>}" \ No newline at end of file diff --git a/regr_smlp/master/test65_model_smlp_model_term.json b/regr_smlp/master/test65_model_smlp_model_term.json index 3c75bd69..b6732e08 100644 --- a/regr_smlp/master/test65_model_smlp_model_term.json +++ b/regr_smlp/master/test65_model_smlp_model_term.json @@ -1 +1 @@ -"{'y1_scaled': x0_scaled (/ 44739243 67108864))) 1 (ite (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (<= x0_scaled (/ 33554433 67108864))) (<= x2_scaled (/ 23488103 33554432))) 0 (ite (and (> x2_scaled (/ 53687093 134217728)) (> x1_scaled (/ 3 4))) 1 (ite (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (<= x0_scaled (/ 33554433 67108864))) (> x2_scaled (/ 23488103 33554432))) 1 0)))))>, 'y2_scaled': x0_scaled (/ 44739243 67108864))) 1 (ite (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (<= x0_scaled (/ 33554433 67108864))) (<= x2_scaled (/ 23488103 33554432))) 0 (ite (and (> x2_scaled (/ 53687093 134217728)) (> x1_scaled (/ 3 4))) 1 (ite (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (<= x0_scaled (/ 33554433 67108864))) (> x2_scaled (/ 23488103 33554432))) 0 0)))))>}" \ No newline at end of file +"{'y1_scaled': x0_scaled (/ 44739243 67108864))) 1 (ite (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (<= x0_scaled (/ 33554433 67108864))) (<= x2_scaled (/ 23488103 33554432))) 0 (ite (and (> x2_scaled (/ 53687093 134217728)) (> x1_scaled (/ 3 4))) 1 (ite (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (<= x0_scaled (/ 33554433 67108864))) (> x2_scaled (/ 23488103 33554432))) 1 0)))))>, 'y2_scaled': x0_scaled (/ 44739243 67108864))) 1 (ite (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (<= x0_scaled (/ 33554433 67108864))) (<= x2_scaled (/ 23488103 33554432))) 0 (ite (and (> x2_scaled (/ 53687093 134217728)) (> x1_scaled (/ 3 4))) 1 (ite (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (<= x0_scaled (/ 33554433 67108864))) (> x2_scaled (/ 23488103 33554432))) 0 0)))))>}" \ No newline at end of file diff --git a/regr_smlp/master/test67_model_dt_sklearn_y1_tree_rules.txt b/regr_smlp/master/test67_model_dt_sklearn_y1_tree_rules.txt new file mode 100644 index 00000000..f1ff1d7a --- /dev/null +++ b/regr_smlp/master/test67_model_dt_sklearn_y1_tree_rules.txt @@ -0,0 +1,8 @@ +#Forest semantics: majority vote +#Number of trees: 1 + +#TREE 0 +if (x2 > 0.7000000178813934) and (x0 <= 0.6666666716337204) then (y1 = 1.0) | based on 3 samples +if (x2 <= 0.7000000178813934) and (x0 <= 0.8333333432674408) then (y1 = 0.0) | based on 3 samples +if (x2 > 0.7000000178813934) and (x0 > 0.6666666716337204) then (y1 = 0.0) | based on 1 samples +if (x2 <= 0.7000000178813934) and (x0 > 0.8333333432674408) then (y1 = 1.0) | based on 1 samples diff --git a/regr_smlp/master/test67_model_dt_sklearn_y2_tree_rules.txt b/regr_smlp/master/test67_model_dt_sklearn_y2_tree_rules.txt new file mode 100644 index 00000000..5015336b --- /dev/null +++ b/regr_smlp/master/test67_model_dt_sklearn_y2_tree_rules.txt @@ -0,0 +1,7 @@ +#Forest semantics: majority vote +#Number of trees: 1 + +#TREE 0 +if (x2 > 0.4000000134110451) and (x1 <= 0.75) then (y2 = 0.0) | based on 5 samples +if (x2 <= 0.4000000134110451) then (y2 = 1.0) | based on 2 samples +if (x2 > 0.4000000134110451) and (x1 > 0.75) then (y2 = 1.0) | based on 1 samples diff --git a/regr_smlp/master/test67_model_y1_smlp_full_model_term.json b/regr_smlp/master/test67_model_y1_smlp_full_model_term.json new file mode 100644 index 00000000..b383dbe4 --- /dev/null +++ b/regr_smlp/master/test67_model_y1_smlp_full_model_term.json @@ -0,0 +1 @@ +"{'y1': |:1| (/ 27962027 33554432))) 1 (ite (and (> |:0| (/ 23488103 33554432)) (> |:1| (/ 44739243 67108864))) 0 (ite (and (<= |:0| (/ 23488103 33554432)) (<= |:1| (/ 27962027 33554432))) 0 1))) 4) 5)))>}" \ No newline at end of file diff --git a/regr_smlp/master/test67_model_y1_smlp_model_term.json b/regr_smlp/master/test67_model_y1_smlp_model_term.json new file mode 100644 index 00000000..e69e78f6 --- /dev/null +++ b/regr_smlp/master/test67_model_y1_smlp_model_term.json @@ -0,0 +1 @@ +"{'y1_scaled': x0_scaled (/ 27962027 33554432))) 1 (ite (and (> x2_scaled (/ 23488103 33554432)) (> x0_scaled (/ 44739243 67108864))) 0 (ite (and (<= x2_scaled (/ 23488103 33554432)) (<= x0_scaled (/ 27962027 33554432))) 0 1)))>}" \ No newline at end of file diff --git a/regr_smlp/master/test67_model_y2_smlp_full_model_term.json b/regr_smlp/master/test67_model_y2_smlp_full_model_term.json new file mode 100644 index 00000000..cc8b6220 --- /dev/null +++ b/regr_smlp/master/test67_model_y2_smlp_full_model_term.json @@ -0,0 +1 @@ +"{'y2': |:0| (/ 53687093 134217728)) (> (* (/ 1 2) (- x1 2)) (/ 3 4))) 1 (ite (<= |:0| (/ 53687093 134217728)) 1 0)) 4) 5))>}" \ No newline at end of file diff --git a/regr_smlp/master/test67_model_y2_smlp_model_term.json b/regr_smlp/master/test67_model_y2_smlp_model_term.json new file mode 100644 index 00000000..56c332c4 --- /dev/null +++ b/regr_smlp/master/test67_model_y2_smlp_model_term.json @@ -0,0 +1 @@ +"{'y2_scaled': x2_scaled (/ 53687093 134217728)) (> x1_scaled (/ 3 4))) 1 (ite (<= x2_scaled (/ 53687093 134217728)) 1 0))>}" \ No newline at end of file diff --git a/regr_smlp/master/test71_model_model_checkpoint.h5 b/regr_smlp/master/test71_model_model_checkpoint.h5 index f62b6a09..840f24be 100644 Binary files a/regr_smlp/master/test71_model_model_checkpoint.h5 and b/regr_smlp/master/test71_model_model_checkpoint.h5 differ diff --git a/regr_smlp/master/test71_model_y1_nn_keras_model_complete.h5 b/regr_smlp/master/test71_model_y1_nn_keras_model_complete.h5 index d42a2363..d2ad081f 100644 Binary files a/regr_smlp/master/test71_model_y1_nn_keras_model_complete.h5 and b/regr_smlp/master/test71_model_y1_nn_keras_model_complete.h5 differ diff --git a/regr_smlp/master/test71_model_y1_smlp_full_model_term.json b/regr_smlp/master/test71_model_y1_smlp_full_model_term.json new file mode 100644 index 00000000..3b060676 --- /dev/null +++ b/regr_smlp/master/test71_model_y1_smlp_full_model_term.json @@ -0,0 +1 @@ +"{'y1': = |:3| 0) |:3| 0))) (let ((|:5| (+ (+ (+ (* |:0| (/ (- 12675489) 16777216)) (* |:1| (/ (- 15700433) 33554432))) (* |:2| (/ (- 14143067) 33554432))) 0))) (let ((|:6| (ite (>= |:5| 0) |:5| 0))) (let ((|:7| (+ (+ (+ (* |:0| (/ 1386949 2097152)) (* |:1| (/ (- 5448211) 16777216))) (* |:2| (/ 1350605 2097152))) (/ (- 5256753) 268435456)))) (let ((|:8| (ite (>= |:7| 0) |:7| 0))) (let ((|:9| (+ (+ (+ (* |:0| (/ 9167379 268435456)) (* |:1| (/ 211449 4194304))) (* |:2| (/ 6971083 8388608))) (/ 10550525 536870912)))) (let ((|:10| (ite (>= |:9| 0) |:9| 0))) (let ((|:11| (+ (+ (+ (* |:0| (/ 13268843 134217728)) (* |:1| (/ 965293 2097152))) (* |:2| (/ 5006791 536870912))) (/ 329869 16777216)))) (let ((|:12| (ite (>= |:11| 0) |:11| 0))) (let ((|:13| (+ (+ (+ (* |:0| (/ 16774547 33554432)) (* |:1| (/ 2938155 4194304))) (* |:2| (/ (- 6516567) 16777216))) (/ 10549879 536870912)))) (let ((|:14| (ite (>= |:13| 0) |:13| 0))) (let ((|:15| (+ (+ (+ (+ (+ (+ (* |:4| (/ 1958803 8388608)) (* |:6| (/ 3285671 8388608))) (* |:8| (/ (- 1798427) 8388608))) (* |:10| (/ (- 14391915) 33554432))) (* |:12| (/ (- 5538481) 8388608))) (* |:14| (/ (- 11806169) 33554432))) 0))) (let ((|:16| (+ (+ (+ (+ (+ (+ (* |:4| (/ 2850953 4194304)) (* |:6| (/ 2493617 16777216))) (* |:8| (/ 13211545 16777216))) (* |:10| (/ (- 12550421) 16777216))) (* |:12| (/ (- 10076651) 16777216))) (* |:14| (/ (- 12590041) 33554432))) (/ (- 658205) 33554432)))) (let ((|:17| (+ (+ (+ (+ (+ (+ (* |:4| (/ (- 13799583) 33554432)) (* |:6| (/ (- 214595) 4194304))) (* |:8| (/ (- 4053207) 16777216))) (* |:10| (/ 315509 8388608))) (* |:12| (/ (- 10656705) 16777216))) (* |:14| (/ 4727225 8388608))) 0))) (+ (* (+ (+ (+ (* (ite (>= |:15| 0) |:15| 0) (/ (- 21323) 32768)) (* (ite (>= |:16| 0) |:16| 0) (/ (- 9302501) 8388608))) (* (ite (>= |:17| 0) |:17| 0) (/ 2984211 4194304))) (/ 165651 8388608)) 4) 5)))))))))))))))))))>}" \ No newline at end of file diff --git a/regr_smlp/master/test71_model_y1_smlp_model_term.json b/regr_smlp/master/test71_model_y1_smlp_model_term.json new file mode 100644 index 00000000..5535cf73 --- /dev/null +++ b/regr_smlp/master/test71_model_y1_smlp_model_term.json @@ -0,0 +1 @@ +"{'y1_scaled': = |:0| 0) |:0| 0))) (let ((|:2| (+ (+ (+ (* x0_scaled (/ (- 12675489) 16777216)) (* x1_scaled (/ (- 15700433) 33554432))) (* x2_scaled (/ (- 14143067) 33554432))) 0))) (let ((|:3| (ite (>= |:2| 0) |:2| 0))) (let ((|:4| (+ (+ (+ (* x0_scaled (/ 1386949 2097152)) (* x1_scaled (/ (- 5448211) 16777216))) (* x2_scaled (/ 1350605 2097152))) (/ (- 5256753) 268435456)))) (let ((|:5| (ite (>= |:4| 0) |:4| 0))) (let ((|:6| (+ (+ (+ (* x0_scaled (/ 9167379 268435456)) (* x1_scaled (/ 211449 4194304))) (* x2_scaled (/ 6971083 8388608))) (/ 10550525 536870912)))) (let ((|:7| (ite (>= |:6| 0) |:6| 0))) (let ((|:8| (+ (+ (+ (* x0_scaled (/ 13268843 134217728)) (* x1_scaled (/ 965293 2097152))) (* x2_scaled (/ 5006791 536870912))) (/ 329869 16777216)))) (let ((|:9| (ite (>= |:8| 0) |:8| 0))) (let ((|:10| (+ (+ (+ (* x0_scaled (/ 16774547 33554432)) (* x1_scaled (/ 2938155 4194304))) (* x2_scaled (/ (- 6516567) 16777216))) (/ 10549879 536870912)))) (let ((|:11| (ite (>= |:10| 0) |:10| 0))) (let ((|:12| (+ (+ (+ (+ (+ (+ (* |:1| (/ 1958803 8388608)) (* |:3| (/ 3285671 8388608))) (* |:5| (/ (- 1798427) 8388608))) (* |:7| (/ (- 14391915) 33554432))) (* |:9| (/ (- 5538481) 8388608))) (* |:11| (/ (- 11806169) 33554432))) 0))) (let ((|:13| (+ (+ (+ (+ (+ (+ (* |:1| (/ 2850953 4194304)) (* |:3| (/ 2493617 16777216))) (* |:5| (/ 13211545 16777216))) (* |:7| (/ (- 12550421) 16777216))) (* |:9| (/ (- 10076651) 16777216))) (* |:11| (/ (- 12590041) 33554432))) (/ (- 658205) 33554432)))) (let ((|:14| (+ (+ (+ (+ (+ (+ (* |:1| (/ (- 13799583) 33554432)) (* |:3| (/ (- 214595) 4194304))) (* |:5| (/ (- 4053207) 16777216))) (* |:7| (/ 315509 8388608))) (* |:9| (/ (- 10656705) 16777216))) (* |:11| (/ 4727225 8388608))) 0))) (+ (+ (+ (* (ite (>= |:12| 0) |:12| 0) (/ (- 21323) 32768)) (* (ite (>= |:13| 0) |:13| 0) (/ (- 9302501) 8388608))) (* (ite (>= |:14| 0) |:14| 0) (/ 2984211 4194304))) (/ 165651 8388608)))))))))))))))))>}" \ No newline at end of file diff --git a/regr_smlp/master/test71_model_y2_nn_keras_model_complete.h5 b/regr_smlp/master/test71_model_y2_nn_keras_model_complete.h5 index f62b6a09..840f24be 100644 Binary files a/regr_smlp/master/test71_model_y2_nn_keras_model_complete.h5 and b/regr_smlp/master/test71_model_y2_nn_keras_model_complete.h5 differ diff --git a/regr_smlp/master/test71_model_y2_smlp_full_model_term.json b/regr_smlp/master/test71_model_y2_smlp_full_model_term.json new file mode 100644 index 00000000..94c17030 --- /dev/null +++ b/regr_smlp/master/test71_model_y2_smlp_full_model_term.json @@ -0,0 +1 @@ +"{'y2': = |:3| 0) |:3| 0))) (let ((|:5| (+ (+ (+ (* |:0| (/ (- 12675489) 16777216)) (* |:1| (/ (- 15700433) 33554432))) (* |:2| (/ (- 14143067) 33554432))) 0))) (let ((|:6| (ite (>= |:5| 0) |:5| 0))) (let ((|:7| (+ (+ (+ (* |:0| (/ 5547791 8388608)) (* |:1| (/ (- 5448211) 16777216))) (* |:2| (/ 2702563 4194304))) (/ (- 2628463) 134217728)))) (let ((|:8| (ite (>= |:7| 0) |:7| 0))) (let ((|:9| (+ (+ (+ (* |:0| (/ 9167453 268435456)) (* |:1| (/ 211449 4194304))) (* |:2| (/ 6968413 8388608))) (/ 5275359 268435456)))) (let ((|:10| (ite (>= |:9| 0) |:9| 0))) (let ((|:11| (+ (+ (+ (* |:0| (/ 13268893 134217728)) (* |:1| (/ 965293 2097152))) (* |:2| (/ 4836293 536870912))) (/ 10556057 536870912)))) (let ((|:12| (ite (>= |:11| 0) |:11| 0))) (let ((|:13| (+ (+ (+ (* |:0| (/ 8387941 16777216)) (* |:1| (/ 2938155 4194304))) (* |:2| (/ (- 6516551) 16777216))) (/ 10604597 536870912)))) (let ((|:14| (ite (>= |:13| 0) |:13| 0))) (let ((|:15| (+ (+ (+ (+ (+ (+ (* |:4| (/ 1958803 8388608)) (* |:6| (/ 3285671 8388608))) (* |:8| (/ (- 1798427) 8388608))) (* |:10| (/ (- 14391915) 33554432))) (* |:12| (/ (- 5538481) 8388608))) (* |:14| (/ (- 11806169) 33554432))) 0))) (let ((|:16| (+ (+ (+ (+ (+ (+ (* |:4| (/ 11402887 16777216)) (* |:6| (/ 2493617 16777216))) (* |:8| (/ 13213105 16777216))) (* |:10| (/ (- 12546881) 16777216))) (* |:12| (/ (- 5037737) 8388608))) (* |:14| (/ (- 786973) 2097152))) (/ (- 10531581) 536870912)))) (let ((|:17| (+ (+ (+ (+ (+ (+ (* |:4| (/ (- 13799583) 33554432)) (* |:6| (/ (- 214595) 4194304))) (* |:8| (/ (- 4053207) 16777216))) (* |:10| (/ 315509 8388608))) (* |:12| (/ (- 10656705) 16777216))) (* |:14| (/ 4727225 8388608))) 0))) (+ (* (+ (+ (+ (* (ite (>= |:15| 0) |:15| 0) (/ (- 21323) 32768)) (* (ite (>= |:16| 0) |:16| 0) (/ (- 4650829) 4194304))) (* (ite (>= |:17| 0) |:17| 0) (/ 2984211 4194304))) (/ 10574285 536870912)) 4) 5)))))))))))))))))))>}" \ No newline at end of file diff --git a/regr_smlp/master/test71_model_y2_smlp_model_term.json b/regr_smlp/master/test71_model_y2_smlp_model_term.json new file mode 100644 index 00000000..ae859b9f --- /dev/null +++ b/regr_smlp/master/test71_model_y2_smlp_model_term.json @@ -0,0 +1 @@ +"{'y2_scaled': = |:0| 0) |:0| 0))) (let ((|:2| (+ (+ (+ (* x0_scaled (/ (- 12675489) 16777216)) (* x1_scaled (/ (- 15700433) 33554432))) (* x2_scaled (/ (- 14143067) 33554432))) 0))) (let ((|:3| (ite (>= |:2| 0) |:2| 0))) (let ((|:4| (+ (+ (+ (* x0_scaled (/ 5547791 8388608)) (* x1_scaled (/ (- 5448211) 16777216))) (* x2_scaled (/ 2702563 4194304))) (/ (- 2628463) 134217728)))) (let ((|:5| (ite (>= |:4| 0) |:4| 0))) (let ((|:6| (+ (+ (+ (* x0_scaled (/ 9167453 268435456)) (* x1_scaled (/ 211449 4194304))) (* x2_scaled (/ 6968413 8388608))) (/ 5275359 268435456)))) (let ((|:7| (ite (>= |:6| 0) |:6| 0))) (let ((|:8| (+ (+ (+ (* x0_scaled (/ 13268893 134217728)) (* x1_scaled (/ 965293 2097152))) (* x2_scaled (/ 4836293 536870912))) (/ 10556057 536870912)))) (let ((|:9| (ite (>= |:8| 0) |:8| 0))) (let ((|:10| (+ (+ (+ (* x0_scaled (/ 8387941 16777216)) (* x1_scaled (/ 2938155 4194304))) (* x2_scaled (/ (- 6516551) 16777216))) (/ 10604597 536870912)))) (let ((|:11| (ite (>= |:10| 0) |:10| 0))) (let ((|:12| (+ (+ (+ (+ (+ (+ (* |:1| (/ 1958803 8388608)) (* |:3| (/ 3285671 8388608))) (* |:5| (/ (- 1798427) 8388608))) (* |:7| (/ (- 14391915) 33554432))) (* |:9| (/ (- 5538481) 8388608))) (* |:11| (/ (- 11806169) 33554432))) 0))) (let ((|:13| (+ (+ (+ (+ (+ (+ (* |:1| (/ 11402887 16777216)) (* |:3| (/ 2493617 16777216))) (* |:5| (/ 13213105 16777216))) (* |:7| (/ (- 12546881) 16777216))) (* |:9| (/ (- 5037737) 8388608))) (* |:11| (/ (- 786973) 2097152))) (/ (- 10531581) 536870912)))) (let ((|:14| (+ (+ (+ (+ (+ (+ (* |:1| (/ (- 13799583) 33554432)) (* |:3| (/ (- 214595) 4194304))) (* |:5| (/ (- 4053207) 16777216))) (* |:7| (/ 315509 8388608))) (* |:9| (/ (- 10656705) 16777216))) (* |:11| (/ 4727225 8388608))) 0))) (+ (+ (+ (* (ite (>= |:12| 0) |:12| 0) (/ (- 21323) 32768)) (* (ite (>= |:13| 0) |:13| 0) (/ (- 4650829) 4194304))) (* (ite (>= |:14| 0) |:14| 0) (/ 2984211 4194304))) (/ 10574285 536870912)))))))))))))))))>}" \ No newline at end of file diff --git a/regr_smlp/master/test73_model_dt_sklearn_y1_tree_rules.txt b/regr_smlp/master/test73_model_dt_sklearn_y1_tree_rules.txt new file mode 100644 index 00000000..9f2dbad2 --- /dev/null +++ b/regr_smlp/master/test73_model_dt_sklearn_y1_tree_rules.txt @@ -0,0 +1,10 @@ +#Forest semantics: majority vote +#Number of trees: 1 + +#TREE 0 +if (x2 > 0.7000000178813934) and (x1 > 0.25) then (y1 = 1.0) | based on 2 samples +if (x2 <= 0.7000000178813934) and (x2 > 0.4000000134110451) then (y1 = 0.0) | based on 2 samples +if (x2 > 0.7000000178813934) and (x1 <= 0.25) and (x2 > 0.9000000059604645) then (y1 = 1.0) | based on 1 samples +if (x2 > 0.7000000178813934) and (x1 <= 0.25) and (x2 <= 0.9000000059604645) then (y1 = 0.0) | based on 1 samples +if (x2 <= 0.7000000178813934) and (x2 <= 0.4000000134110451) and (x2 > 0.10000000149011612) then (y1 = 1.0) | based on 1 samples +if (x2 <= 0.7000000178813934) and (x2 <= 0.4000000134110451) and (x2 <= 0.10000000149011612) then (y1 = 0.0) | based on 1 samples diff --git a/regr_smlp/master/test73_model_dt_sklearn_y2_tree_rules.txt b/regr_smlp/master/test73_model_dt_sklearn_y2_tree_rules.txt new file mode 100644 index 00000000..5015336b --- /dev/null +++ b/regr_smlp/master/test73_model_dt_sklearn_y2_tree_rules.txt @@ -0,0 +1,7 @@ +#Forest semantics: majority vote +#Number of trees: 1 + +#TREE 0 +if (x2 > 0.4000000134110451) and (x1 <= 0.75) then (y2 = 0.0) | based on 5 samples +if (x2 <= 0.4000000134110451) then (y2 = 1.0) | based on 2 samples +if (x2 > 0.4000000134110451) and (x1 > 0.75) then (y2 = 1.0) | based on 1 samples diff --git a/regr_smlp/master/test73_model_y1_smlp_full_model_term.json b/regr_smlp/master/test73_model_y1_smlp_full_model_term.json new file mode 100644 index 00000000..3c7d4f33 --- /dev/null +++ b/regr_smlp/master/test73_model_y1_smlp_full_model_term.json @@ -0,0 +1 @@ +"{'y1': |:0| (/ 13421773 134217728))) 1 (ite (and (and (> |:0| (/ 23488103 33554432)) (<= |:1| (/ 1 4))) (<= |:0| (/ 30198989 33554432))) 0 (ite (and (and (> |:0| (/ 23488103 33554432)) (<= |:1| (/ 1 4))) (> |:0| (/ 30198989 33554432))) 1 (ite (and (<= |:0| (/ 23488103 33554432)) (> |:0| (/ 53687093 134217728))) 0 1))))) 4) 5)))>}" \ No newline at end of file diff --git a/regr_smlp/master/test73_model_y1_smlp_model_term.json b/regr_smlp/master/test73_model_y1_smlp_model_term.json new file mode 100644 index 00000000..d4363137 --- /dev/null +++ b/regr_smlp/master/test73_model_y1_smlp_model_term.json @@ -0,0 +1 @@ +"{'y1_scaled': x2_scaled (/ 13421773 134217728))) 1 (ite (and (and (> x2_scaled (/ 23488103 33554432)) (<= x1_scaled (/ 1 4))) (<= x2_scaled (/ 30198989 33554432))) 0 (ite (and (and (> x2_scaled (/ 23488103 33554432)) (<= x1_scaled (/ 1 4))) (> x2_scaled (/ 30198989 33554432))) 1 (ite (and (<= x2_scaled (/ 23488103 33554432)) (> x2_scaled (/ 53687093 134217728))) 0 1)))))>}" \ No newline at end of file diff --git a/regr_smlp/master/test73_model_y2_smlp_full_model_term.json b/regr_smlp/master/test73_model_y2_smlp_full_model_term.json new file mode 100644 index 00000000..cc8b6220 --- /dev/null +++ b/regr_smlp/master/test73_model_y2_smlp_full_model_term.json @@ -0,0 +1 @@ +"{'y2': |:0| (/ 53687093 134217728)) (> (* (/ 1 2) (- x1 2)) (/ 3 4))) 1 (ite (<= |:0| (/ 53687093 134217728)) 1 0)) 4) 5))>}" \ No newline at end of file diff --git a/regr_smlp/master/test73_model_y2_smlp_model_term.json b/regr_smlp/master/test73_model_y2_smlp_model_term.json new file mode 100644 index 00000000..56c332c4 --- /dev/null +++ b/regr_smlp/master/test73_model_y2_smlp_model_term.json @@ -0,0 +1 @@ +"{'y2_scaled': x2_scaled (/ 53687093 134217728)) (> x1_scaled (/ 3 4))) 1 (ite (<= x2_scaled (/ 53687093 134217728)) 1 0))>}" \ No newline at end of file diff --git a/regr_smlp/master/test76_model_dt_sklearn_tree_rules.txt b/regr_smlp/master/test76_model_dt_sklearn_tree_rules.txt index 5f89d9ed..59a1eff8 100644 --- a/regr_smlp/master/test76_model_dt_sklearn_tree_rules.txt +++ b/regr_smlp/master/test76_model_dt_sklearn_tree_rules.txt @@ -3,8 +3,8 @@ #TREE 0 if (x2 > 0.4000000134110451) and (x1 <= 0.75) and (x2 <= 0.7000000178813934) then (y1 = 0.0) and (y2 = 0.0) | based on 2 samples -if (x2 > 0.4000000134110451) and (x1 <= 0.75) and (x2 > 0.7000000178813934) and (x1 > 0.25) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples if (x2 > 0.4000000134110451) and (x1 > 0.75) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +if (x2 > 0.4000000134110451) and (x1 <= 0.75) and (x2 > 0.7000000178813934) and (x1 > 0.25) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples if (x2 > 0.4000000134110451) and (x1 <= 0.75) and (x2 > 0.7000000178813934) and (x1 <= 0.25) and (x2 > 0.9000000059604645) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples if (x2 > 0.4000000134110451) and (x1 <= 0.75) and (x2 > 0.7000000178813934) and (x1 <= 0.25) and (x2 <= 0.9000000059604645) then (y1 = 0.0) and (y2 = 0.0) | based on 1 samples if (x2 <= 0.4000000134110451) and (x2 > 0.10000000149011612) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples diff --git a/regr_smlp/master/test76_model_smlp_full_model_term.json b/regr_smlp/master/test76_model_smlp_full_model_term.json new file mode 100644 index 00000000..d046fc1f --- /dev/null +++ b/regr_smlp/master/test76_model_smlp_full_model_term.json @@ -0,0 +1 @@ +"{'y1': |:0| (/ 13421773 134217728))) 1 (ite (and (and (and (and (> |:0| (/ 53687093 134217728)) (<= |:1| (/ 3 4))) (> |:0| (/ 23488103 33554432))) (<= |:1| (/ 1 4))) (<= |:0| (/ 30198989 33554432))) 0 (ite (and (and (and (and (> |:0| (/ 53687093 134217728)) (<= |:1| (/ 3 4))) (> |:0| (/ 23488103 33554432))) (<= |:1| (/ 1 4))) (> |:0| (/ 30198989 33554432))) 1 (ite (and (and (and (> |:0| (/ 53687093 134217728)) (<= |:1| (/ 3 4))) (> |:0| (/ 23488103 33554432))) (> |:1| (/ 1 4))) 1 (ite (and (> |:0| (/ 53687093 134217728)) (> |:1| (/ 3 4))) 1 0)))))) 4) 5)))>, 'y2': |:0| (/ 13421773 134217728))) 1 (ite (and (and (and (and (> |:0| (/ 53687093 134217728)) (<= |:1| (/ 3 4))) (> |:0| (/ 23488103 33554432))) (<= |:1| (/ 1 4))) (<= |:0| (/ 30198989 33554432))) 0 (ite (and (and (and (and (> |:0| (/ 53687093 134217728)) (<= |:1| (/ 3 4))) (> |:0| (/ 23488103 33554432))) (<= |:1| (/ 1 4))) (> |:0| (/ 30198989 33554432))) 0 (ite (and (and (and (> |:0| (/ 53687093 134217728)) (<= |:1| (/ 3 4))) (> |:0| (/ 23488103 33554432))) (> |:1| (/ 1 4))) 0 (ite (and (> |:0| (/ 53687093 134217728)) (> |:1| (/ 3 4))) 1 0)))))) 4) 5)))>}" \ No newline at end of file diff --git a/regr_smlp/master/test76_model_smlp_model_term.json b/regr_smlp/master/test76_model_smlp_model_term.json index 8fd79688..cd219aa4 100644 --- a/regr_smlp/master/test76_model_smlp_model_term.json +++ b/regr_smlp/master/test76_model_smlp_model_term.json @@ -1 +1 @@ -"{'y1_scaled': x2_scaled (/ 13421773 134217728))) 1 (ite (and (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (> x2_scaled (/ 23488103 33554432))) (<= x1_scaled (/ 1 4))) (<= x2_scaled (/ 30198989 33554432))) 0 (ite (and (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (> x2_scaled (/ 23488103 33554432))) (<= x1_scaled (/ 1 4))) (> x2_scaled (/ 30198989 33554432))) 1 (ite (and (> x2_scaled (/ 53687093 134217728)) (> x1_scaled (/ 3 4))) 1 (ite (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (> x2_scaled (/ 23488103 33554432))) (> x1_scaled (/ 1 4))) 1 0))))))>, 'y2_scaled': x2_scaled (/ 13421773 134217728))) 1 (ite (and (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (> x2_scaled (/ 23488103 33554432))) (<= x1_scaled (/ 1 4))) (<= x2_scaled (/ 30198989 33554432))) 0 (ite (and (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (> x2_scaled (/ 23488103 33554432))) (<= x1_scaled (/ 1 4))) (> x2_scaled (/ 30198989 33554432))) 0 (ite (and (> x2_scaled (/ 53687093 134217728)) (> x1_scaled (/ 3 4))) 1 (ite (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (> x2_scaled (/ 23488103 33554432))) (> x1_scaled (/ 1 4))) 0 0))))))>}" \ No newline at end of file +"{'y1_scaled': x2_scaled (/ 13421773 134217728))) 1 (ite (and (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (> x2_scaled (/ 23488103 33554432))) (<= x1_scaled (/ 1 4))) (<= x2_scaled (/ 30198989 33554432))) 0 (ite (and (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (> x2_scaled (/ 23488103 33554432))) (<= x1_scaled (/ 1 4))) (> x2_scaled (/ 30198989 33554432))) 1 (ite (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (> x2_scaled (/ 23488103 33554432))) (> x1_scaled (/ 1 4))) 1 (ite (and (> x2_scaled (/ 53687093 134217728)) (> x1_scaled (/ 3 4))) 1 0))))))>, 'y2_scaled': x2_scaled (/ 13421773 134217728))) 1 (ite (and (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (> x2_scaled (/ 23488103 33554432))) (<= x1_scaled (/ 1 4))) (<= x2_scaled (/ 30198989 33554432))) 0 (ite (and (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (> x2_scaled (/ 23488103 33554432))) (<= x1_scaled (/ 1 4))) (> x2_scaled (/ 30198989 33554432))) 0 (ite (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (> x2_scaled (/ 23488103 33554432))) (> x1_scaled (/ 1 4))) 0 (ite (and (> x2_scaled (/ 53687093 134217728)) (> x1_scaled (/ 3 4))) 1 0))))))>}" \ No newline at end of file diff --git a/regr_smlp/master/test78_model_dt_sklearn_tree_rules.txt b/regr_smlp/master/test78_model_dt_sklearn_tree_rules.txt index 5f89d9ed..59a1eff8 100644 --- a/regr_smlp/master/test78_model_dt_sklearn_tree_rules.txt +++ b/regr_smlp/master/test78_model_dt_sklearn_tree_rules.txt @@ -3,8 +3,8 @@ #TREE 0 if (x2 > 0.4000000134110451) and (x1 <= 0.75) and (x2 <= 0.7000000178813934) then (y1 = 0.0) and (y2 = 0.0) | based on 2 samples -if (x2 > 0.4000000134110451) and (x1 <= 0.75) and (x2 > 0.7000000178813934) and (x1 > 0.25) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples if (x2 > 0.4000000134110451) and (x1 > 0.75) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +if (x2 > 0.4000000134110451) and (x1 <= 0.75) and (x2 > 0.7000000178813934) and (x1 > 0.25) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples if (x2 > 0.4000000134110451) and (x1 <= 0.75) and (x2 > 0.7000000178813934) and (x1 <= 0.25) and (x2 > 0.9000000059604645) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples if (x2 > 0.4000000134110451) and (x1 <= 0.75) and (x2 > 0.7000000178813934) and (x1 <= 0.25) and (x2 <= 0.9000000059604645) then (y1 = 0.0) and (y2 = 0.0) | based on 1 samples if (x2 <= 0.4000000134110451) and (x2 > 0.10000000149011612) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples diff --git a/regr_smlp/master/test78_model_smlp_full_model_term.json b/regr_smlp/master/test78_model_smlp_full_model_term.json new file mode 100644 index 00000000..d046fc1f --- /dev/null +++ b/regr_smlp/master/test78_model_smlp_full_model_term.json @@ -0,0 +1 @@ +"{'y1': |:0| (/ 13421773 134217728))) 1 (ite (and (and (and (and (> |:0| (/ 53687093 134217728)) (<= |:1| (/ 3 4))) (> |:0| (/ 23488103 33554432))) (<= |:1| (/ 1 4))) (<= |:0| (/ 30198989 33554432))) 0 (ite (and (and (and (and (> |:0| (/ 53687093 134217728)) (<= |:1| (/ 3 4))) (> |:0| (/ 23488103 33554432))) (<= |:1| (/ 1 4))) (> |:0| (/ 30198989 33554432))) 1 (ite (and (and (and (> |:0| (/ 53687093 134217728)) (<= |:1| (/ 3 4))) (> |:0| (/ 23488103 33554432))) (> |:1| (/ 1 4))) 1 (ite (and (> |:0| (/ 53687093 134217728)) (> |:1| (/ 3 4))) 1 0)))))) 4) 5)))>, 'y2': |:0| (/ 13421773 134217728))) 1 (ite (and (and (and (and (> |:0| (/ 53687093 134217728)) (<= |:1| (/ 3 4))) (> |:0| (/ 23488103 33554432))) (<= |:1| (/ 1 4))) (<= |:0| (/ 30198989 33554432))) 0 (ite (and (and (and (and (> |:0| (/ 53687093 134217728)) (<= |:1| (/ 3 4))) (> |:0| (/ 23488103 33554432))) (<= |:1| (/ 1 4))) (> |:0| (/ 30198989 33554432))) 0 (ite (and (and (and (> |:0| (/ 53687093 134217728)) (<= |:1| (/ 3 4))) (> |:0| (/ 23488103 33554432))) (> |:1| (/ 1 4))) 0 (ite (and (> |:0| (/ 53687093 134217728)) (> |:1| (/ 3 4))) 1 0)))))) 4) 5)))>}" \ No newline at end of file diff --git a/regr_smlp/models/Test84_smlp_toy_num_resp_noknobs_rerun_model_config.json b/regr_smlp/models/Test84_smlp_toy_num_resp_noknobs_rerun_model_config.json new file mode 100644 index 00000000..c7a3a170 --- /dev/null +++ b/regr_smlp/models/Test84_smlp_toy_num_resp_noknobs_rerun_model_config.json @@ -0,0 +1,172 @@ +{ + "alpha": "x2==7.0 and x0==0 and x1==2.5", + "analytics_mode": "verify", + "approximate_fractions": true, + "assertions_expressions": "(y2**3+x2)/2<6;y1>=9;y2<0", + "assertions_names": "asrt1,asrt2,asrt3", + "beta": null, + "center_offset": "0", + "compress_rules": false, + "continuous_correlation_estimators": [ + "pearson", + "spearman" + ], + "correlations_and_mutual_information": true, + "data_scaler": "min_max", + "delta_absolute": 0.0, + "delta_relative": 0.01, + "discretization_algo": "uniform", + "discretization_bins": 10, + "discretization_labels": true, + "discretization_type": "category", + "discretize_numeric_features": null, + "doe_algo": null, + "doe_box_behnken_centers": 1, + "doe_central_composite_alpha": "o", + "doe_central_composite_center": "2,2", + "doe_central_composite_face": "ccf", + "doe_design_resolution": null, + "doe_factor_level_ranges": null, + "doe_num_samples": null, + "doe_prob_distribution": "Normal", + "doe_spec_file": null, + "dt_sklearn_ccp_alpha": 0.0, + "dt_sklearn_criterion": "squared_error", + "dt_sklearn_max_depth": 15, + "dt_sklearn_max_features": null, + "dt_sklearn_max_leaf_nodes": null, + "dt_sklearn_min_impurity_decrease": 0.0, + "dt_sklearn_min_samples_leaf": 1, + "dt_sklearn_min_samples_split": 2, + "dt_sklearn_min_weight_fraction_leaf": 0.0, + "dt_sklearn_random_state": null, + "dt_sklearn_splitter": "best", + "epsilon": 0.05, + "et_sklearn_bootstrap": true, + "et_sklearn_ccp_alpha": 0.0, + "et_sklearn_criterion": "squared_error", + "et_sklearn_max_depth": null, + "et_sklearn_max_features": 1.0, + "et_sklearn_max_leaf_nodes": null, + "et_sklearn_max_samples": null, + "et_sklearn_min_impurity_decrease": 0.0, + "et_sklearn_min_samples_leaf": 1, + "et_sklearn_min_samples_split": 2, + "et_sklearn_min_weight_fraction_leaf": 0.0, + "et_sklearn_n_estimators": 100, + "et_sklearn_random_state": null, + "et_sklearn_verbose": 0, + "et_sklearn_warm_start": false, + "eta": null, + "features": "x0,x1,x2", + "fraction_precision": 64, + "impute_responses": false, + "interactive_plots": false, + "keep_features": [], + "labeled_data": null, + "lemma_precision": 0, + "load_configuration": null, + "log_files_prefix": null, + "log_level": "info", + "log_mode": "w", + "log_time": false, + "model": "dt_sklearn", + "model_caret_cross_validation": true, + "model_caret_return_train_score": false, + "model_caret_verbose": true, + "model_name": null, + "model_per_response": false, + "mrmr_feat_count_for_correlation": 15, + "mrmr_feat_count_for_prediction": 2, + "mutual_information_method": "normalized", + "negative_value": 0, + "new_data": null, + "nn_keras_batch_size": 200, + "nn_keras_batches_grid": null, + "nn_keras_epochs": 2000, + "nn_keras_hid_activation": "relu", + "nn_keras_layers": "2,1", + "nn_keras_layers_grid": null, + "nn_keras_learning_rate": 0.001, + "nn_keras_learning_rates_grid": null, + "nn_keras_loss_function": "mse", + "nn_keras_loss_functions_grid": null, + "nn_keras_metrics": [ + "mse" + ], + "nn_keras_optimizer": "adam", + "nn_keras_out_activation": "linear", + "nn_keras_sequential_api": true, + "nn_keras_tuner_algo": null, + "nn_keras_weights_precision": null, + "nnet_encoding": "nested", + "objectives_expressions": null, + "objectives_names": "None", + "optimization_strategy": "eager", + "optimize_pareto": true, + "output_directory": "./", + "poly_sklearn_copy_X": true, + "poly_sklearn_degree": 2, + "poly_sklearn_fit_intercept": true, + "poly_sklearn_n_jobs": null, + "poly_sklearn_positive": false, + "positive_value": 1, + "prediction_plots": false, + "psg_max_dimension": 3, + "psg_quality_target": "Lift", + "psg_top_ranked": 15, + "query_expressions": null, + "query_names": "None", + "radius_absolute": null, + "radius_relative": null, + "response": "y1,y2", + "response_map": null, + "response_plots": false, + "response_to_bool": null, + "rf_sklearn_bootstrap": true, + "rf_sklearn_ccp_alpha": 0.0, + "rf_sklearn_criterion": "squared_error", + "rf_sklearn_max_depth": null, + "rf_sklearn_max_features": 1.0, + "rf_sklearn_max_leaf_nodes": null, + "rf_sklearn_max_samples": null, + "rf_sklearn_min_impurity_decrease": 0.0, + "rf_sklearn_min_samples_leaf": 1, + "rf_sklearn_min_samples_split": 2, + "rf_sklearn_min_weight_fraction_leaf": 0.0, + "rf_sklearn_n_estimators": 100, + "rf_sklearn_random_state": null, + "rf_sklearn_verbose": 0, + "rf_sklearn_warm_start": false, + "sample_weights_coef": 0, + "sample_weights_exponent": 0, + "sample_weights_intercept": 0, + "save_configuration": false, + "save_model": "false", + "save_model_rerun_configuration": true, + "scale_features": true, + "scale_objectives": true, + "scale_responses": true, + "seed": 10, + "setup_caret_data_split_shuffle": true, + "setup_caret_fold": 0, + "setup_caret_session_id": null, + "setup_caret_verbose": true, + "simplify_terms": false, + "solver": "z3", + "solver_logic": "ALL", + "solver_path": null, + "spec": "../specs/smlp_toy_num_resp_noknobs_verify.spec", + "split_test": 0.2, + "trace_anonymize": false, + "trace_precision": 0, + "trace_runtime": 0, + "train_first_n": 0, + "train_random_n": 0, + "train_uniform_n": 0, + "tree_encoding": "nested", + "tuner_caret_search_algorithm": "random", + "tuner_caret_tuner_verbose": true, + "use_model": "true", + "vacuity_check": true +} \ No newline at end of file diff --git a/regr_smlp/models/test65_model_smlp_full_model_term.json b/regr_smlp/models/test65_model_smlp_full_model_term.json new file mode 100644 index 00000000..80c624e1 --- /dev/null +++ b/regr_smlp/models/test65_model_smlp_full_model_term.json @@ -0,0 +1 @@ +"{'y1': |:1| (/ 44739243 67108864))) 1 (ite (and (and (and (> |:0| (/ 53687093 134217728)) (<= |:2| (/ 3 4))) (<= |:1| (/ 33554433 67108864))) (<= |:0| (/ 23488103 33554432))) 0 (ite (and (> |:0| (/ 53687093 134217728)) (> |:2| (/ 3 4))) 1 (ite (and (and (and (> |:0| (/ 53687093 134217728)) (<= |:2| (/ 3 4))) (<= |:1| (/ 33554433 67108864))) (> |:0| (/ 23488103 33554432))) 1 0))))) 4) 5))))>, 'y2': |:1| (/ 44739243 67108864))) 1 (ite (and (and (and (> |:0| (/ 53687093 134217728)) (<= |:2| (/ 3 4))) (<= |:1| (/ 33554433 67108864))) (<= |:0| (/ 23488103 33554432))) 0 (ite (and (> |:0| (/ 53687093 134217728)) (> |:2| (/ 3 4))) 1 (ite (and (and (and (> |:0| (/ 53687093 134217728)) (<= |:2| (/ 3 4))) (<= |:1| (/ 33554433 67108864))) (> |:0| (/ 23488103 33554432))) 0 0))))) 4) 5))))>}" \ No newline at end of file diff --git a/regr_smlp/models/test65_model_smlp_model_term.json b/regr_smlp/models/test65_model_smlp_model_term.json index 3c75bd69..b6732e08 100644 --- a/regr_smlp/models/test65_model_smlp_model_term.json +++ b/regr_smlp/models/test65_model_smlp_model_term.json @@ -1 +1 @@ -"{'y1_scaled': x0_scaled (/ 44739243 67108864))) 1 (ite (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (<= x0_scaled (/ 33554433 67108864))) (<= x2_scaled (/ 23488103 33554432))) 0 (ite (and (> x2_scaled (/ 53687093 134217728)) (> x1_scaled (/ 3 4))) 1 (ite (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (<= x0_scaled (/ 33554433 67108864))) (> x2_scaled (/ 23488103 33554432))) 1 0)))))>, 'y2_scaled': x0_scaled (/ 44739243 67108864))) 1 (ite (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (<= x0_scaled (/ 33554433 67108864))) (<= x2_scaled (/ 23488103 33554432))) 0 (ite (and (> x2_scaled (/ 53687093 134217728)) (> x1_scaled (/ 3 4))) 1 (ite (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (<= x0_scaled (/ 33554433 67108864))) (> x2_scaled (/ 23488103 33554432))) 0 0)))))>}" \ No newline at end of file +"{'y1_scaled': x0_scaled (/ 44739243 67108864))) 1 (ite (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (<= x0_scaled (/ 33554433 67108864))) (<= x2_scaled (/ 23488103 33554432))) 0 (ite (and (> x2_scaled (/ 53687093 134217728)) (> x1_scaled (/ 3 4))) 1 (ite (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (<= x0_scaled (/ 33554433 67108864))) (> x2_scaled (/ 23488103 33554432))) 1 0)))))>, 'y2_scaled': x0_scaled (/ 44739243 67108864))) 1 (ite (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (<= x0_scaled (/ 33554433 67108864))) (<= x2_scaled (/ 23488103 33554432))) 0 (ite (and (> x2_scaled (/ 53687093 134217728)) (> x1_scaled (/ 3 4))) 1 (ite (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (<= x0_scaled (/ 33554433 67108864))) (> x2_scaled (/ 23488103 33554432))) 0 0)))))>}" \ No newline at end of file diff --git a/regr_smlp/models/test67_model_dt_sklearn_tree_rules.txt b/regr_smlp/models/test67_model_dt_sklearn_tree_rules.txt index 5015336b..30a6a1be 100644 --- a/regr_smlp/models/test67_model_dt_sklearn_tree_rules.txt +++ b/regr_smlp/models/test67_model_dt_sklearn_tree_rules.txt @@ -2,6 +2,6 @@ #Number of trees: 1 #TREE 0 -if (x2 > 0.4000000134110451) and (x1 <= 0.75) then (y2 = 0.0) | based on 5 samples -if (x2 <= 0.4000000134110451) then (y2 = 1.0) | based on 2 samples -if (x2 > 0.4000000134110451) and (x1 > 0.75) then (y2 = 1.0) | based on 1 samples +if (p2 > 0.4000000134110451) and (p1 <= 0.75) then (y2 = 0.0) | based on 5 samples +if (p2 <= 0.4000000134110451) then (y2 = 1.0) | based on 2 samples +if (p2 > 0.4000000134110451) and (p1 > 0.75) then (y2 = 1.0) | based on 1 samples diff --git a/regr_smlp/models/test67_model_dt_sklearn_y1_tree_rules.txt b/regr_smlp/models/test67_model_dt_sklearn_y1_tree_rules.txt new file mode 100644 index 00000000..f1ff1d7a --- /dev/null +++ b/regr_smlp/models/test67_model_dt_sklearn_y1_tree_rules.txt @@ -0,0 +1,8 @@ +#Forest semantics: majority vote +#Number of trees: 1 + +#TREE 0 +if (x2 > 0.7000000178813934) and (x0 <= 0.6666666716337204) then (y1 = 1.0) | based on 3 samples +if (x2 <= 0.7000000178813934) and (x0 <= 0.8333333432674408) then (y1 = 0.0) | based on 3 samples +if (x2 > 0.7000000178813934) and (x0 > 0.6666666716337204) then (y1 = 0.0) | based on 1 samples +if (x2 <= 0.7000000178813934) and (x0 > 0.8333333432674408) then (y1 = 1.0) | based on 1 samples diff --git a/regr_smlp/models/test67_model_dt_sklearn_y2_tree_rules.txt b/regr_smlp/models/test67_model_dt_sklearn_y2_tree_rules.txt new file mode 100644 index 00000000..5015336b --- /dev/null +++ b/regr_smlp/models/test67_model_dt_sklearn_y2_tree_rules.txt @@ -0,0 +1,7 @@ +#Forest semantics: majority vote +#Number of trees: 1 + +#TREE 0 +if (x2 > 0.4000000134110451) and (x1 <= 0.75) then (y2 = 0.0) | based on 5 samples +if (x2 <= 0.4000000134110451) then (y2 = 1.0) | based on 2 samples +if (x2 > 0.4000000134110451) and (x1 > 0.75) then (y2 = 1.0) | based on 1 samples diff --git a/regr_smlp/models/test67_model_smlp_model_term.json b/regr_smlp/models/test67_model_smlp_model_term.json index dd21c3ce..475d42a2 100644 --- a/regr_smlp/models/test67_model_smlp_model_term.json +++ b/regr_smlp/models/test67_model_smlp_model_term.json @@ -1 +1 @@ -"{'y2_scaled': x2_scaled (/ 53687093 134217728)) (> x1_scaled (/ 3 4))) 1 (ite (<= x2_scaled (/ 53687093 134217728)) 1 0))>}" \ No newline at end of file +"{'y2_scaled': p2_scaled (/ 53687093 134217728)) (> p1_scaled (/ 3 4))) 1 (ite (<= p2_scaled (/ 53687093 134217728)) 1 0))>}" \ No newline at end of file diff --git a/regr_smlp/models/test67_model_y1_smlp_full_model_term.json b/regr_smlp/models/test67_model_y1_smlp_full_model_term.json new file mode 100644 index 00000000..b383dbe4 --- /dev/null +++ b/regr_smlp/models/test67_model_y1_smlp_full_model_term.json @@ -0,0 +1 @@ +"{'y1': |:1| (/ 27962027 33554432))) 1 (ite (and (> |:0| (/ 23488103 33554432)) (> |:1| (/ 44739243 67108864))) 0 (ite (and (<= |:0| (/ 23488103 33554432)) (<= |:1| (/ 27962027 33554432))) 0 1))) 4) 5)))>}" \ No newline at end of file diff --git a/regr_smlp/models/test67_model_y1_smlp_model_term.json b/regr_smlp/models/test67_model_y1_smlp_model_term.json new file mode 100644 index 00000000..e69e78f6 --- /dev/null +++ b/regr_smlp/models/test67_model_y1_smlp_model_term.json @@ -0,0 +1 @@ +"{'y1_scaled': x0_scaled (/ 27962027 33554432))) 1 (ite (and (> x2_scaled (/ 23488103 33554432)) (> x0_scaled (/ 44739243 67108864))) 0 (ite (and (<= x2_scaled (/ 23488103 33554432)) (<= x0_scaled (/ 27962027 33554432))) 0 1)))>}" \ No newline at end of file diff --git a/regr_smlp/models/test67_model_y2_smlp_full_model_term.json b/regr_smlp/models/test67_model_y2_smlp_full_model_term.json new file mode 100644 index 00000000..cc8b6220 --- /dev/null +++ b/regr_smlp/models/test67_model_y2_smlp_full_model_term.json @@ -0,0 +1 @@ +"{'y2': |:0| (/ 53687093 134217728)) (> (* (/ 1 2) (- x1 2)) (/ 3 4))) 1 (ite (<= |:0| (/ 53687093 134217728)) 1 0)) 4) 5))>}" \ No newline at end of file diff --git a/regr_smlp/models/test67_model_y2_smlp_model_term.json b/regr_smlp/models/test67_model_y2_smlp_model_term.json new file mode 100644 index 00000000..56c332c4 --- /dev/null +++ b/regr_smlp/models/test67_model_y2_smlp_model_term.json @@ -0,0 +1 @@ +"{'y2_scaled': x2_scaled (/ 53687093 134217728)) (> x1_scaled (/ 3 4))) 1 (ite (<= x2_scaled (/ 53687093 134217728)) 1 0))>}" \ No newline at end of file diff --git a/regr_smlp/models/test71_model_model_checkpoint.h5 b/regr_smlp/models/test71_model_model_checkpoint.h5 index f62b6a09..840f24be 100644 Binary files a/regr_smlp/models/test71_model_model_checkpoint.h5 and b/regr_smlp/models/test71_model_model_checkpoint.h5 differ diff --git a/regr_smlp/models/test71_model_smlp_model_term.json b/regr_smlp/models/test71_model_smlp_model_term.json index 1689b2b1..f0a5a188 100644 --- a/regr_smlp/models/test71_model_smlp_model_term.json +++ b/regr_smlp/models/test71_model_smlp_model_term.json @@ -1 +1 @@ -"{'y2_scaled': = |:0| 0) |:0| 0))) (let ((|:2| (+ (+ (+ (* x0_scaled (/ (- 12675489) 16777216)) (* x1_scaled (/ (- 15700433) 33554432))) (* x2_scaled (/ (- 14143067) 33554432))) 0))) (let ((|:3| (ite (>= |:2| 0) |:2| 0))) (let ((|:4| (+ (+ (+ (* x0_scaled (/ 5547791 8388608)) (* x1_scaled (/ (- 5448211) 16777216))) (* x2_scaled (/ 2702563 4194304))) (/ (- 2628463) 134217728)))) (let ((|:5| (ite (>= |:4| 0) |:4| 0))) (let ((|:6| (+ (+ (+ (* x0_scaled (/ 9167453 268435456)) (* x1_scaled (/ 211449 4194304))) (* x2_scaled (/ 6968413 8388608))) (/ 5275359 268435456)))) (let ((|:7| (ite (>= |:6| 0) |:6| 0))) (let ((|:8| (+ (+ (+ (* x0_scaled (/ 6634447 67108864)) (* x1_scaled (/ 965293 2097152))) (* x2_scaled (/ 4836293 536870912))) (/ 10556057 536870912)))) (let ((|:9| (ite (>= |:8| 0) |:8| 0))) (let ((|:10| (+ (+ (+ (* x0_scaled (/ 8387941 16777216)) (* x1_scaled (/ 2938155 4194304))) (* x2_scaled (/ (- 6516551) 16777216))) (/ 10604597 536870912)))) (let ((|:11| (ite (>= |:10| 0) |:10| 0))) (let ((|:12| (+ (+ (+ (+ (+ (+ (* |:1| (/ 1958803 8388608)) (* |:3| (/ 3285671 8388608))) (* |:5| (/ (- 1798427) 8388608))) (* |:7| (/ (- 14391915) 33554432))) (* |:9| (/ (- 5538481) 8388608))) (* |:11| (/ (- 11806169) 33554432))) 0))) (let ((|:13| (+ (+ (+ (+ (+ (+ (* |:1| (/ 11402887 16777216)) (* |:3| (/ 2493617 16777216))) (* |:5| (/ 13213105 16777216))) (* |:7| (/ (- 12546881) 16777216))) (* |:9| (/ (- 5037737) 8388608))) (* |:11| (/ (- 786973) 2097152))) (/ (- 10531581) 536870912)))) (let ((|:14| (+ (+ (+ (+ (+ (+ (* |:1| (/ (- 13799583) 33554432)) (* |:3| (/ (- 214595) 4194304))) (* |:5| (/ (- 4053207) 16777216))) (* |:7| (/ 315509 8388608))) (* |:9| (/ (- 10656705) 16777216))) (* |:11| (/ 4727225 8388608))) 0))) (+ (+ (+ (* (ite (>= |:12| 0) |:12| 0) (/ (- 21323) 32768)) (* (ite (>= |:13| 0) |:13| 0) (/ (- 4650829) 4194304))) (* (ite (>= |:14| 0) |:14| 0) (/ 2984211 4194304))) (/ 10574285 536870912)))))))))))))))))>}" \ No newline at end of file +"{'y2_scaled': = |:0| 0) |:0| 0))) (let ((|:2| (+ (+ (+ (* x_scaled (/ 16226559 33554432)) (* p1_scaled (/ (- 3173377) 4194304))) (* p2_scaled (/ (- 11004215) 16777216))) (/ (- 2655009) 134217728)))) (let ((|:3| (ite (>= |:2| 0) |:2| 0))) (let ((|:4| (+ (+ (+ (* x_scaled (/ 11226163 16777216)) (* p1_scaled (/ 10798807 67108864))) (* p2_scaled (/ 13595525 67108864))) (/ 10620895 536870912)))) (let ((|:5| (ite (>= |:4| 0) |:4| 0))) (let ((|:6| (+ (+ (+ (* x_scaled (/ 3418331 16777216)) (* p1_scaled (/ 1014123 8388608))) (* p2_scaled (/ (- 2623577) 4194304))) (/ (- 664505) 33554432)))) (let ((|:7| (ite (>= |:6| 0) |:6| 0))) (let ((|:8| (+ (+ (+ (* x_scaled (/ 3153319 16777216)) (* p1_scaled (/ (- 11570481) 16777216))) (* p2_scaled (/ 5273493 8388608))) (/ (- 5270219) 268435456)))) (let ((|:9| (ite (>= |:8| 0) |:8| 0))) (let ((|:10| (+ (+ (+ (* x_scaled (/ 103071 131072)) (* p1_scaled (/ 2069995 16777216))) (* p2_scaled (/ (- 12245549) 16777216))) (/ (- 10580837) 536870912)))) (let ((|:11| (ite (>= |:10| 0) |:10| 0))) (let ((|:12| (+ (+ (+ (+ (+ (+ (* |:1| (/ 9405017 134217728)) (* |:3| (/ 10501567 33554432))) (* |:5| (/ (- 15264105) 33554432))) (* |:7| (/ 6042365 8388608))) (* |:9| (/ 9912413 16777216))) (* |:11| (/ 12593515 16777216))) (/ (- 2644615) 134217728)))) (let ((|:13| (+ (+ (+ (+ (+ (+ (* |:1| (/ (- 6717881) 67108864)) (* |:3| (/ (- 6305147) 33554432))) (* |:5| (/ 4216389 33554432))) (* |:7| (/ 168083 8388608))) (* |:9| (/ 8516555 16777216))) (* |:11| (/ (- 5350265) 33554432))) (/ 10769195 536870912)))) (let ((|:14| (+ (+ (+ (+ (+ (+ (* |:1| (/ (- 4912199) 8388608)) (* |:3| (/ 677045 2097152))) (* |:5| (/ (- 8260447) 16777216))) (* |:7| (/ (- 481099) 1048576))) (* |:9| (/ (- 3619967) 8388608))) (* |:11| (/ (- 5030151) 16777216))) 0))) (+ (+ (+ (* (ite (>= |:12| 0) |:12| 0) (/ (- 4199133) 8388608)) (* (ite (>= |:13| 0) |:13| 0) (/ 13074671 134217728))) (* (ite (>= |:14| 0) |:14| 0) (/ 1706327 4194304))) (/ 10628649 536870912)))))))))))))))))>}" \ No newline at end of file diff --git a/regr_smlp/models/test71_model_y1_nn_keras_model_complete.h5 b/regr_smlp/models/test71_model_y1_nn_keras_model_complete.h5 index d42a2363..d2ad081f 100644 Binary files a/regr_smlp/models/test71_model_y1_nn_keras_model_complete.h5 and b/regr_smlp/models/test71_model_y1_nn_keras_model_complete.h5 differ diff --git a/regr_smlp/models/test71_model_y1_smlp_full_model_term.json b/regr_smlp/models/test71_model_y1_smlp_full_model_term.json new file mode 100644 index 00000000..3b060676 --- /dev/null +++ b/regr_smlp/models/test71_model_y1_smlp_full_model_term.json @@ -0,0 +1 @@ +"{'y1': = |:3| 0) |:3| 0))) (let ((|:5| (+ (+ (+ (* |:0| (/ (- 12675489) 16777216)) (* |:1| (/ (- 15700433) 33554432))) (* |:2| (/ (- 14143067) 33554432))) 0))) (let ((|:6| (ite (>= |:5| 0) |:5| 0))) (let ((|:7| (+ (+ (+ (* |:0| (/ 1386949 2097152)) (* |:1| (/ (- 5448211) 16777216))) (* |:2| (/ 1350605 2097152))) (/ (- 5256753) 268435456)))) (let ((|:8| (ite (>= |:7| 0) |:7| 0))) (let ((|:9| (+ (+ (+ (* |:0| (/ 9167379 268435456)) (* |:1| (/ 211449 4194304))) (* |:2| (/ 6971083 8388608))) (/ 10550525 536870912)))) (let ((|:10| (ite (>= |:9| 0) |:9| 0))) (let ((|:11| (+ (+ (+ (* |:0| (/ 13268843 134217728)) (* |:1| (/ 965293 2097152))) (* |:2| (/ 5006791 536870912))) (/ 329869 16777216)))) (let ((|:12| (ite (>= |:11| 0) |:11| 0))) (let ((|:13| (+ (+ (+ (* |:0| (/ 16774547 33554432)) (* |:1| (/ 2938155 4194304))) (* |:2| (/ (- 6516567) 16777216))) (/ 10549879 536870912)))) (let ((|:14| (ite (>= |:13| 0) |:13| 0))) (let ((|:15| (+ (+ (+ (+ (+ (+ (* |:4| (/ 1958803 8388608)) (* |:6| (/ 3285671 8388608))) (* |:8| (/ (- 1798427) 8388608))) (* |:10| (/ (- 14391915) 33554432))) (* |:12| (/ (- 5538481) 8388608))) (* |:14| (/ (- 11806169) 33554432))) 0))) (let ((|:16| (+ (+ (+ (+ (+ (+ (* |:4| (/ 2850953 4194304)) (* |:6| (/ 2493617 16777216))) (* |:8| (/ 13211545 16777216))) (* |:10| (/ (- 12550421) 16777216))) (* |:12| (/ (- 10076651) 16777216))) (* |:14| (/ (- 12590041) 33554432))) (/ (- 658205) 33554432)))) (let ((|:17| (+ (+ (+ (+ (+ (+ (* |:4| (/ (- 13799583) 33554432)) (* |:6| (/ (- 214595) 4194304))) (* |:8| (/ (- 4053207) 16777216))) (* |:10| (/ 315509 8388608))) (* |:12| (/ (- 10656705) 16777216))) (* |:14| (/ 4727225 8388608))) 0))) (+ (* (+ (+ (+ (* (ite (>= |:15| 0) |:15| 0) (/ (- 21323) 32768)) (* (ite (>= |:16| 0) |:16| 0) (/ (- 9302501) 8388608))) (* (ite (>= |:17| 0) |:17| 0) (/ 2984211 4194304))) (/ 165651 8388608)) 4) 5)))))))))))))))))))>}" \ No newline at end of file diff --git a/regr_smlp/models/test71_model_y1_smlp_model_term.json b/regr_smlp/models/test71_model_y1_smlp_model_term.json new file mode 100644 index 00000000..5535cf73 --- /dev/null +++ b/regr_smlp/models/test71_model_y1_smlp_model_term.json @@ -0,0 +1 @@ +"{'y1_scaled': = |:0| 0) |:0| 0))) (let ((|:2| (+ (+ (+ (* x0_scaled (/ (- 12675489) 16777216)) (* x1_scaled (/ (- 15700433) 33554432))) (* x2_scaled (/ (- 14143067) 33554432))) 0))) (let ((|:3| (ite (>= |:2| 0) |:2| 0))) (let ((|:4| (+ (+ (+ (* x0_scaled (/ 1386949 2097152)) (* x1_scaled (/ (- 5448211) 16777216))) (* x2_scaled (/ 1350605 2097152))) (/ (- 5256753) 268435456)))) (let ((|:5| (ite (>= |:4| 0) |:4| 0))) (let ((|:6| (+ (+ (+ (* x0_scaled (/ 9167379 268435456)) (* x1_scaled (/ 211449 4194304))) (* x2_scaled (/ 6971083 8388608))) (/ 10550525 536870912)))) (let ((|:7| (ite (>= |:6| 0) |:6| 0))) (let ((|:8| (+ (+ (+ (* x0_scaled (/ 13268843 134217728)) (* x1_scaled (/ 965293 2097152))) (* x2_scaled (/ 5006791 536870912))) (/ 329869 16777216)))) (let ((|:9| (ite (>= |:8| 0) |:8| 0))) (let ((|:10| (+ (+ (+ (* x0_scaled (/ 16774547 33554432)) (* x1_scaled (/ 2938155 4194304))) (* x2_scaled (/ (- 6516567) 16777216))) (/ 10549879 536870912)))) (let ((|:11| (ite (>= |:10| 0) |:10| 0))) (let ((|:12| (+ (+ (+ (+ (+ (+ (* |:1| (/ 1958803 8388608)) (* |:3| (/ 3285671 8388608))) (* |:5| (/ (- 1798427) 8388608))) (* |:7| (/ (- 14391915) 33554432))) (* |:9| (/ (- 5538481) 8388608))) (* |:11| (/ (- 11806169) 33554432))) 0))) (let ((|:13| (+ (+ (+ (+ (+ (+ (* |:1| (/ 2850953 4194304)) (* |:3| (/ 2493617 16777216))) (* |:5| (/ 13211545 16777216))) (* |:7| (/ (- 12550421) 16777216))) (* |:9| (/ (- 10076651) 16777216))) (* |:11| (/ (- 12590041) 33554432))) (/ (- 658205) 33554432)))) (let ((|:14| (+ (+ (+ (+ (+ (+ (* |:1| (/ (- 13799583) 33554432)) (* |:3| (/ (- 214595) 4194304))) (* |:5| (/ (- 4053207) 16777216))) (* |:7| (/ 315509 8388608))) (* |:9| (/ (- 10656705) 16777216))) (* |:11| (/ 4727225 8388608))) 0))) (+ (+ (+ (* (ite (>= |:12| 0) |:12| 0) (/ (- 21323) 32768)) (* (ite (>= |:13| 0) |:13| 0) (/ (- 9302501) 8388608))) (* (ite (>= |:14| 0) |:14| 0) (/ 2984211 4194304))) (/ 165651 8388608)))))))))))))))))>}" \ No newline at end of file diff --git a/regr_smlp/models/test71_model_y2_nn_keras_model_complete.h5 b/regr_smlp/models/test71_model_y2_nn_keras_model_complete.h5 index f62b6a09..840f24be 100644 Binary files a/regr_smlp/models/test71_model_y2_nn_keras_model_complete.h5 and b/regr_smlp/models/test71_model_y2_nn_keras_model_complete.h5 differ diff --git a/regr_smlp/models/test71_model_y2_smlp_full_model_term.json b/regr_smlp/models/test71_model_y2_smlp_full_model_term.json new file mode 100644 index 00000000..94c17030 --- /dev/null +++ b/regr_smlp/models/test71_model_y2_smlp_full_model_term.json @@ -0,0 +1 @@ +"{'y2': = |:3| 0) |:3| 0))) (let ((|:5| (+ (+ (+ (* |:0| (/ (- 12675489) 16777216)) (* |:1| (/ (- 15700433) 33554432))) (* |:2| (/ (- 14143067) 33554432))) 0))) (let ((|:6| (ite (>= |:5| 0) |:5| 0))) (let ((|:7| (+ (+ (+ (* |:0| (/ 5547791 8388608)) (* |:1| (/ (- 5448211) 16777216))) (* |:2| (/ 2702563 4194304))) (/ (- 2628463) 134217728)))) (let ((|:8| (ite (>= |:7| 0) |:7| 0))) (let ((|:9| (+ (+ (+ (* |:0| (/ 9167453 268435456)) (* |:1| (/ 211449 4194304))) (* |:2| (/ 6968413 8388608))) (/ 5275359 268435456)))) (let ((|:10| (ite (>= |:9| 0) |:9| 0))) (let ((|:11| (+ (+ (+ (* |:0| (/ 13268893 134217728)) (* |:1| (/ 965293 2097152))) (* |:2| (/ 4836293 536870912))) (/ 10556057 536870912)))) (let ((|:12| (ite (>= |:11| 0) |:11| 0))) (let ((|:13| (+ (+ (+ (* |:0| (/ 8387941 16777216)) (* |:1| (/ 2938155 4194304))) (* |:2| (/ (- 6516551) 16777216))) (/ 10604597 536870912)))) (let ((|:14| (ite (>= |:13| 0) |:13| 0))) (let ((|:15| (+ (+ (+ (+ (+ (+ (* |:4| (/ 1958803 8388608)) (* |:6| (/ 3285671 8388608))) (* |:8| (/ (- 1798427) 8388608))) (* |:10| (/ (- 14391915) 33554432))) (* |:12| (/ (- 5538481) 8388608))) (* |:14| (/ (- 11806169) 33554432))) 0))) (let ((|:16| (+ (+ (+ (+ (+ (+ (* |:4| (/ 11402887 16777216)) (* |:6| (/ 2493617 16777216))) (* |:8| (/ 13213105 16777216))) (* |:10| (/ (- 12546881) 16777216))) (* |:12| (/ (- 5037737) 8388608))) (* |:14| (/ (- 786973) 2097152))) (/ (- 10531581) 536870912)))) (let ((|:17| (+ (+ (+ (+ (+ (+ (* |:4| (/ (- 13799583) 33554432)) (* |:6| (/ (- 214595) 4194304))) (* |:8| (/ (- 4053207) 16777216))) (* |:10| (/ 315509 8388608))) (* |:12| (/ (- 10656705) 16777216))) (* |:14| (/ 4727225 8388608))) 0))) (+ (* (+ (+ (+ (* (ite (>= |:15| 0) |:15| 0) (/ (- 21323) 32768)) (* (ite (>= |:16| 0) |:16| 0) (/ (- 4650829) 4194304))) (* (ite (>= |:17| 0) |:17| 0) (/ 2984211 4194304))) (/ 10574285 536870912)) 4) 5)))))))))))))))))))>}" \ No newline at end of file diff --git a/regr_smlp/models/test71_model_y2_smlp_model_term.json b/regr_smlp/models/test71_model_y2_smlp_model_term.json new file mode 100644 index 00000000..ae859b9f --- /dev/null +++ b/regr_smlp/models/test71_model_y2_smlp_model_term.json @@ -0,0 +1 @@ +"{'y2_scaled': = |:0| 0) |:0| 0))) (let ((|:2| (+ (+ (+ (* x0_scaled (/ (- 12675489) 16777216)) (* x1_scaled (/ (- 15700433) 33554432))) (* x2_scaled (/ (- 14143067) 33554432))) 0))) (let ((|:3| (ite (>= |:2| 0) |:2| 0))) (let ((|:4| (+ (+ (+ (* x0_scaled (/ 5547791 8388608)) (* x1_scaled (/ (- 5448211) 16777216))) (* x2_scaled (/ 2702563 4194304))) (/ (- 2628463) 134217728)))) (let ((|:5| (ite (>= |:4| 0) |:4| 0))) (let ((|:6| (+ (+ (+ (* x0_scaled (/ 9167453 268435456)) (* x1_scaled (/ 211449 4194304))) (* x2_scaled (/ 6968413 8388608))) (/ 5275359 268435456)))) (let ((|:7| (ite (>= |:6| 0) |:6| 0))) (let ((|:8| (+ (+ (+ (* x0_scaled (/ 13268893 134217728)) (* x1_scaled (/ 965293 2097152))) (* x2_scaled (/ 4836293 536870912))) (/ 10556057 536870912)))) (let ((|:9| (ite (>= |:8| 0) |:8| 0))) (let ((|:10| (+ (+ (+ (* x0_scaled (/ 8387941 16777216)) (* x1_scaled (/ 2938155 4194304))) (* x2_scaled (/ (- 6516551) 16777216))) (/ 10604597 536870912)))) (let ((|:11| (ite (>= |:10| 0) |:10| 0))) (let ((|:12| (+ (+ (+ (+ (+ (+ (* |:1| (/ 1958803 8388608)) (* |:3| (/ 3285671 8388608))) (* |:5| (/ (- 1798427) 8388608))) (* |:7| (/ (- 14391915) 33554432))) (* |:9| (/ (- 5538481) 8388608))) (* |:11| (/ (- 11806169) 33554432))) 0))) (let ((|:13| (+ (+ (+ (+ (+ (+ (* |:1| (/ 11402887 16777216)) (* |:3| (/ 2493617 16777216))) (* |:5| (/ 13213105 16777216))) (* |:7| (/ (- 12546881) 16777216))) (* |:9| (/ (- 5037737) 8388608))) (* |:11| (/ (- 786973) 2097152))) (/ (- 10531581) 536870912)))) (let ((|:14| (+ (+ (+ (+ (+ (+ (* |:1| (/ (- 13799583) 33554432)) (* |:3| (/ (- 214595) 4194304))) (* |:5| (/ (- 4053207) 16777216))) (* |:7| (/ 315509 8388608))) (* |:9| (/ (- 10656705) 16777216))) (* |:11| (/ 4727225 8388608))) 0))) (+ (+ (+ (* (ite (>= |:12| 0) |:12| 0) (/ (- 21323) 32768)) (* (ite (>= |:13| 0) |:13| 0) (/ (- 4650829) 4194304))) (* (ite (>= |:14| 0) |:14| 0) (/ 2984211 4194304))) (/ 10574285 536870912)))))))))))))))))>}" \ No newline at end of file diff --git a/regr_smlp/models/test73_model_dt_sklearn_tree_rules.txt b/regr_smlp/models/test73_model_dt_sklearn_tree_rules.txt index 5015336b..30a6a1be 100644 --- a/regr_smlp/models/test73_model_dt_sklearn_tree_rules.txt +++ b/regr_smlp/models/test73_model_dt_sklearn_tree_rules.txt @@ -2,6 +2,6 @@ #Number of trees: 1 #TREE 0 -if (x2 > 0.4000000134110451) and (x1 <= 0.75) then (y2 = 0.0) | based on 5 samples -if (x2 <= 0.4000000134110451) then (y2 = 1.0) | based on 2 samples -if (x2 > 0.4000000134110451) and (x1 > 0.75) then (y2 = 1.0) | based on 1 samples +if (p2 > 0.4000000134110451) and (p1 <= 0.75) then (y2 = 0.0) | based on 5 samples +if (p2 <= 0.4000000134110451) then (y2 = 1.0) | based on 2 samples +if (p2 > 0.4000000134110451) and (p1 > 0.75) then (y2 = 1.0) | based on 1 samples diff --git a/regr_smlp/models/test73_model_dt_sklearn_y1_tree_rules.txt b/regr_smlp/models/test73_model_dt_sklearn_y1_tree_rules.txt new file mode 100644 index 00000000..9f2dbad2 --- /dev/null +++ b/regr_smlp/models/test73_model_dt_sklearn_y1_tree_rules.txt @@ -0,0 +1,10 @@ +#Forest semantics: majority vote +#Number of trees: 1 + +#TREE 0 +if (x2 > 0.7000000178813934) and (x1 > 0.25) then (y1 = 1.0) | based on 2 samples +if (x2 <= 0.7000000178813934) and (x2 > 0.4000000134110451) then (y1 = 0.0) | based on 2 samples +if (x2 > 0.7000000178813934) and (x1 <= 0.25) and (x2 > 0.9000000059604645) then (y1 = 1.0) | based on 1 samples +if (x2 > 0.7000000178813934) and (x1 <= 0.25) and (x2 <= 0.9000000059604645) then (y1 = 0.0) | based on 1 samples +if (x2 <= 0.7000000178813934) and (x2 <= 0.4000000134110451) and (x2 > 0.10000000149011612) then (y1 = 1.0) | based on 1 samples +if (x2 <= 0.7000000178813934) and (x2 <= 0.4000000134110451) and (x2 <= 0.10000000149011612) then (y1 = 0.0) | based on 1 samples diff --git a/regr_smlp/models/test73_model_dt_sklearn_y2_tree_rules.txt b/regr_smlp/models/test73_model_dt_sklearn_y2_tree_rules.txt new file mode 100644 index 00000000..5015336b --- /dev/null +++ b/regr_smlp/models/test73_model_dt_sklearn_y2_tree_rules.txt @@ -0,0 +1,7 @@ +#Forest semantics: majority vote +#Number of trees: 1 + +#TREE 0 +if (x2 > 0.4000000134110451) and (x1 <= 0.75) then (y2 = 0.0) | based on 5 samples +if (x2 <= 0.4000000134110451) then (y2 = 1.0) | based on 2 samples +if (x2 > 0.4000000134110451) and (x1 > 0.75) then (y2 = 1.0) | based on 1 samples diff --git a/regr_smlp/models/test73_model_smlp_model_term.json b/regr_smlp/models/test73_model_smlp_model_term.json index dd21c3ce..475d42a2 100644 --- a/regr_smlp/models/test73_model_smlp_model_term.json +++ b/regr_smlp/models/test73_model_smlp_model_term.json @@ -1 +1 @@ -"{'y2_scaled': x2_scaled (/ 53687093 134217728)) (> x1_scaled (/ 3 4))) 1 (ite (<= x2_scaled (/ 53687093 134217728)) 1 0))>}" \ No newline at end of file +"{'y2_scaled': p2_scaled (/ 53687093 134217728)) (> p1_scaled (/ 3 4))) 1 (ite (<= p2_scaled (/ 53687093 134217728)) 1 0))>}" \ No newline at end of file diff --git a/regr_smlp/models/test73_model_y1_smlp_full_model_term.json b/regr_smlp/models/test73_model_y1_smlp_full_model_term.json new file mode 100644 index 00000000..3c7d4f33 --- /dev/null +++ b/regr_smlp/models/test73_model_y1_smlp_full_model_term.json @@ -0,0 +1 @@ +"{'y1': |:0| (/ 13421773 134217728))) 1 (ite (and (and (> |:0| (/ 23488103 33554432)) (<= |:1| (/ 1 4))) (<= |:0| (/ 30198989 33554432))) 0 (ite (and (and (> |:0| (/ 23488103 33554432)) (<= |:1| (/ 1 4))) (> |:0| (/ 30198989 33554432))) 1 (ite (and (<= |:0| (/ 23488103 33554432)) (> |:0| (/ 53687093 134217728))) 0 1))))) 4) 5)))>}" \ No newline at end of file diff --git a/regr_smlp/models/test73_model_y1_smlp_model_term.json b/regr_smlp/models/test73_model_y1_smlp_model_term.json new file mode 100644 index 00000000..d4363137 --- /dev/null +++ b/regr_smlp/models/test73_model_y1_smlp_model_term.json @@ -0,0 +1 @@ +"{'y1_scaled': x2_scaled (/ 13421773 134217728))) 1 (ite (and (and (> x2_scaled (/ 23488103 33554432)) (<= x1_scaled (/ 1 4))) (<= x2_scaled (/ 30198989 33554432))) 0 (ite (and (and (> x2_scaled (/ 23488103 33554432)) (<= x1_scaled (/ 1 4))) (> x2_scaled (/ 30198989 33554432))) 1 (ite (and (<= x2_scaled (/ 23488103 33554432)) (> x2_scaled (/ 53687093 134217728))) 0 1)))))>}" \ No newline at end of file diff --git a/regr_smlp/models/test73_model_y2_smlp_full_model_term.json b/regr_smlp/models/test73_model_y2_smlp_full_model_term.json new file mode 100644 index 00000000..cc8b6220 --- /dev/null +++ b/regr_smlp/models/test73_model_y2_smlp_full_model_term.json @@ -0,0 +1 @@ +"{'y2': |:0| (/ 53687093 134217728)) (> (* (/ 1 2) (- x1 2)) (/ 3 4))) 1 (ite (<= |:0| (/ 53687093 134217728)) 1 0)) 4) 5))>}" \ No newline at end of file diff --git a/regr_smlp/models/test73_model_y2_smlp_model_term.json b/regr_smlp/models/test73_model_y2_smlp_model_term.json new file mode 100644 index 00000000..56c332c4 --- /dev/null +++ b/regr_smlp/models/test73_model_y2_smlp_model_term.json @@ -0,0 +1 @@ +"{'y2_scaled': x2_scaled (/ 53687093 134217728)) (> x1_scaled (/ 3 4))) 1 (ite (<= x2_scaled (/ 53687093 134217728)) 1 0))>}" \ No newline at end of file diff --git a/regr_smlp/models/test76_model_dt_sklearn_tree_rules.txt b/regr_smlp/models/test76_model_dt_sklearn_tree_rules.txt index 5f89d9ed..59a1eff8 100644 --- a/regr_smlp/models/test76_model_dt_sklearn_tree_rules.txt +++ b/regr_smlp/models/test76_model_dt_sklearn_tree_rules.txt @@ -3,8 +3,8 @@ #TREE 0 if (x2 > 0.4000000134110451) and (x1 <= 0.75) and (x2 <= 0.7000000178813934) then (y1 = 0.0) and (y2 = 0.0) | based on 2 samples -if (x2 > 0.4000000134110451) and (x1 <= 0.75) and (x2 > 0.7000000178813934) and (x1 > 0.25) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples if (x2 > 0.4000000134110451) and (x1 > 0.75) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +if (x2 > 0.4000000134110451) and (x1 <= 0.75) and (x2 > 0.7000000178813934) and (x1 > 0.25) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples if (x2 > 0.4000000134110451) and (x1 <= 0.75) and (x2 > 0.7000000178813934) and (x1 <= 0.25) and (x2 > 0.9000000059604645) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples if (x2 > 0.4000000134110451) and (x1 <= 0.75) and (x2 > 0.7000000178813934) and (x1 <= 0.25) and (x2 <= 0.9000000059604645) then (y1 = 0.0) and (y2 = 0.0) | based on 1 samples if (x2 <= 0.4000000134110451) and (x2 > 0.10000000149011612) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples diff --git a/regr_smlp/models/test76_model_smlp_full_model_term.json b/regr_smlp/models/test76_model_smlp_full_model_term.json new file mode 100644 index 00000000..d046fc1f --- /dev/null +++ b/regr_smlp/models/test76_model_smlp_full_model_term.json @@ -0,0 +1 @@ +"{'y1': |:0| (/ 13421773 134217728))) 1 (ite (and (and (and (and (> |:0| (/ 53687093 134217728)) (<= |:1| (/ 3 4))) (> |:0| (/ 23488103 33554432))) (<= |:1| (/ 1 4))) (<= |:0| (/ 30198989 33554432))) 0 (ite (and (and (and (and (> |:0| (/ 53687093 134217728)) (<= |:1| (/ 3 4))) (> |:0| (/ 23488103 33554432))) (<= |:1| (/ 1 4))) (> |:0| (/ 30198989 33554432))) 1 (ite (and (and (and (> |:0| (/ 53687093 134217728)) (<= |:1| (/ 3 4))) (> |:0| (/ 23488103 33554432))) (> |:1| (/ 1 4))) 1 (ite (and (> |:0| (/ 53687093 134217728)) (> |:1| (/ 3 4))) 1 0)))))) 4) 5)))>, 'y2': |:0| (/ 13421773 134217728))) 1 (ite (and (and (and (and (> |:0| (/ 53687093 134217728)) (<= |:1| (/ 3 4))) (> |:0| (/ 23488103 33554432))) (<= |:1| (/ 1 4))) (<= |:0| (/ 30198989 33554432))) 0 (ite (and (and (and (and (> |:0| (/ 53687093 134217728)) (<= |:1| (/ 3 4))) (> |:0| (/ 23488103 33554432))) (<= |:1| (/ 1 4))) (> |:0| (/ 30198989 33554432))) 0 (ite (and (and (and (> |:0| (/ 53687093 134217728)) (<= |:1| (/ 3 4))) (> |:0| (/ 23488103 33554432))) (> |:1| (/ 1 4))) 0 (ite (and (> |:0| (/ 53687093 134217728)) (> |:1| (/ 3 4))) 1 0)))))) 4) 5)))>}" \ No newline at end of file diff --git a/regr_smlp/models/test76_model_smlp_model_term.json b/regr_smlp/models/test76_model_smlp_model_term.json index 8fd79688..cd219aa4 100644 --- a/regr_smlp/models/test76_model_smlp_model_term.json +++ b/regr_smlp/models/test76_model_smlp_model_term.json @@ -1 +1 @@ -"{'y1_scaled': x2_scaled (/ 13421773 134217728))) 1 (ite (and (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (> x2_scaled (/ 23488103 33554432))) (<= x1_scaled (/ 1 4))) (<= x2_scaled (/ 30198989 33554432))) 0 (ite (and (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (> x2_scaled (/ 23488103 33554432))) (<= x1_scaled (/ 1 4))) (> x2_scaled (/ 30198989 33554432))) 1 (ite (and (> x2_scaled (/ 53687093 134217728)) (> x1_scaled (/ 3 4))) 1 (ite (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (> x2_scaled (/ 23488103 33554432))) (> x1_scaled (/ 1 4))) 1 0))))))>, 'y2_scaled': x2_scaled (/ 13421773 134217728))) 1 (ite (and (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (> x2_scaled (/ 23488103 33554432))) (<= x1_scaled (/ 1 4))) (<= x2_scaled (/ 30198989 33554432))) 0 (ite (and (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (> x2_scaled (/ 23488103 33554432))) (<= x1_scaled (/ 1 4))) (> x2_scaled (/ 30198989 33554432))) 0 (ite (and (> x2_scaled (/ 53687093 134217728)) (> x1_scaled (/ 3 4))) 1 (ite (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (> x2_scaled (/ 23488103 33554432))) (> x1_scaled (/ 1 4))) 0 0))))))>}" \ No newline at end of file +"{'y1_scaled': x2_scaled (/ 13421773 134217728))) 1 (ite (and (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (> x2_scaled (/ 23488103 33554432))) (<= x1_scaled (/ 1 4))) (<= x2_scaled (/ 30198989 33554432))) 0 (ite (and (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (> x2_scaled (/ 23488103 33554432))) (<= x1_scaled (/ 1 4))) (> x2_scaled (/ 30198989 33554432))) 1 (ite (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (> x2_scaled (/ 23488103 33554432))) (> x1_scaled (/ 1 4))) 1 (ite (and (> x2_scaled (/ 53687093 134217728)) (> x1_scaled (/ 3 4))) 1 0))))))>, 'y2_scaled': x2_scaled (/ 13421773 134217728))) 1 (ite (and (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (> x2_scaled (/ 23488103 33554432))) (<= x1_scaled (/ 1 4))) (<= x2_scaled (/ 30198989 33554432))) 0 (ite (and (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (> x2_scaled (/ 23488103 33554432))) (<= x1_scaled (/ 1 4))) (> x2_scaled (/ 30198989 33554432))) 0 (ite (and (and (and (> x2_scaled (/ 53687093 134217728)) (<= x1_scaled (/ 3 4))) (> x2_scaled (/ 23488103 33554432))) (> x1_scaled (/ 1 4))) 0 (ite (and (> x2_scaled (/ 53687093 134217728)) (> x1_scaled (/ 3 4))) 1 0))))))>}" \ No newline at end of file diff --git a/regr_smlp/models/test78_model_dt_sklearn_tree_rules.txt b/regr_smlp/models/test78_model_dt_sklearn_tree_rules.txt index 5f89d9ed..59a1eff8 100644 --- a/regr_smlp/models/test78_model_dt_sklearn_tree_rules.txt +++ b/regr_smlp/models/test78_model_dt_sklearn_tree_rules.txt @@ -3,8 +3,8 @@ #TREE 0 if (x2 > 0.4000000134110451) and (x1 <= 0.75) and (x2 <= 0.7000000178813934) then (y1 = 0.0) and (y2 = 0.0) | based on 2 samples -if (x2 > 0.4000000134110451) and (x1 <= 0.75) and (x2 > 0.7000000178813934) and (x1 > 0.25) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples if (x2 > 0.4000000134110451) and (x1 > 0.75) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +if (x2 > 0.4000000134110451) and (x1 <= 0.75) and (x2 > 0.7000000178813934) and (x1 > 0.25) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples if (x2 > 0.4000000134110451) and (x1 <= 0.75) and (x2 > 0.7000000178813934) and (x1 <= 0.25) and (x2 > 0.9000000059604645) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples if (x2 > 0.4000000134110451) and (x1 <= 0.75) and (x2 > 0.7000000178813934) and (x1 <= 0.25) and (x2 <= 0.9000000059604645) then (y1 = 0.0) and (y2 = 0.0) | based on 1 samples if (x2 <= 0.4000000134110451) and (x2 > 0.10000000149011612) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples diff --git a/regr_smlp/specs/smlp_toy_num_resp_noknobs_verify.spec b/regr_smlp/specs/smlp_toy_num_resp_noknobs_verify.spec new file mode 100644 index 00000000..88dab785 --- /dev/null +++ b/regr_smlp/specs/smlp_toy_num_resp_noknobs_verify.spec @@ -0,0 +1,8 @@ +{"version": "1.1", + "spec":[ + {"label": "y1", "type": "response", "range": "float"}, + {"label": "y2", "type": "response", "range": "float"}, + {"label": "x0", "type": "input", "range": "float", "bounds": [0,10]}, + {"label": "x1", "type": "input", "range": "float", "bounds": [0,10]}, + {"label": "x2", "type": "input", "range": "float", "bounds": [3,7]}], + "alpha": "x1==1 or x1==4 or x1==7"} diff --git a/tests/smlp_regression/run_smlp_regression_expected.log b/tests/smlp_regression/run_smlp_regression_expected.log index 1abcad8d..eab2d72a 100644 --- a/tests/smlp_regression/run_smlp_regression_expected.log +++ b/tests/smlp_regression/run_smlp_regression_expected.log @@ -1,830 +1,923 @@ Calling 8 workers for multiprocessing... Initiating 0 worker... -Running test 1 test type: train, description: basic dt_caret training and test on labeled data with single numeric response -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test1 -mode train -resp y1 -feat x,p1,p2 -model dt_caret -save_model_config f -mrmr_pred 0 -plots f -seed 10 -log_time f - Initiating 1 worker... -Running test 2 test type: prediction, description: basic rf_sklearn prediction test on labeled and new data with numeric labelsInitiating 2 worker... - -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test2 -mode predict -resp y1 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 15 -save_model_config f -mrmr_pred 0 -plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - +Initiating 2 worker... Initiating 3 worker... -Running test 3 test type: prediction, description: basic poly_sklearn prediction test on labeled and new data with numeric response in training/test data only -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test3 -mode predict -resp y1 -feat x,p1,p2 -model poly_sklearn -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_unlabeled.csv" - Initiating 4 worker... Initiating 5 worker... -Running test 4 test type: prediction, description: basic nn_keras prediction test on labeled and new data with numeric labels and one response -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test4 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -nn_keras_weights_precision 2 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - Initiating 6 worker... -Running test 6 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test6 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - -Initiating 7 worker... -Running test 5 test type: prediction, description: basic dt_caret prediction test on labeled and new data with numeric labels -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test5 -mode predict -resp y1 -feat x,p1,p2 -model dt_caret -save_model t -use_model f -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - -Running test 7 test type: prediction, description: basic rf_sklearn prediction test on labeled and new data with numeric labels -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test7 -mode predict -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 15 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - -Running test 8 test type: prediction, description: basic nn_keras prediction test on labeled and new data with numeric labels and two responses -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test8 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -nn_keras_epochs 20 -nn_keras_seq_api f -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - -Running test 9 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test9 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model t -model_name test20_model -data_scaler none -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -save_config t -save_model_config t -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - -Running test 10 test type: prediction, description: basic et_sklearn prediction test on labeled and new data with numeric labels -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test10 -mode predict -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 15 -et_sklearn_bootstrap f -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - -Running test 11 test type: prediction, description: basic poly_sklearn prediction test on labeled and new data with numeric labels -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test11 -mode predict -resp y1,y2 -feat x,p1,p2 -model poly_sklearn -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 2 test type: prediction, description: basic rf_sklearn prediction test on labeled and new data with numeric labels +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test2 -mode predict -resp y1 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 15 -save_model_config f -mrmr_pred 0 -plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" Running test 12 test type: train, description: EV-SI real life dt_sklearn predict test on labeled and new data with numeric labels -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test12 -mode train -resp y1,y2 -feat x1,x2,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -Running test 13 test type: train, description: EV-SI real life nn_keras prediction test on labeled and new data with numeric labels -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test13 -mode train -resp y1,y2 -feat x1,x2,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api f -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -Running test 14 test type: train, description: EV-SI real life poly_sklearn prediction test on labeled and new data with numeric labels -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test14 -mode train -resp y1,y2 -feat x1,x2,p1,p2 -model poly_sklearn -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -Running test 15 test type: prediction, description: basic dt_caret prediction test from saved model on new data with numeric labels -../../src/run_smlp.py -model_name "../models/Test5_smlp_toy_num_resp_mult" -out_dir ./ -pref Test15 -mode predict -resp y1 -feat x,p1,p2 -model dt_caret -save_model f -use_model t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - -Running test 16 test type: prediction, description: basic nn_keras prediction test from saved model on new data with numeric labels and two responses -../../src/run_smlp.py -model_name "../models/Test8_smlp_toy_num_resp_mult" -out_dir ./ -pref Test16 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -save_model f -use_model t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - -Running test 17 test type: prediction, description: basic poly_sklearn prediction test from saved model on new data with numeric labels and two responses -../../src/run_smlp.py -model_name "../models/Test11_smlp_toy_num_resp_mult" -out_dir ./ -pref Test17 -mode predict -resp y1,y2 -feat x,p1,p2 -model poly_sklearn -save_model f -use_model t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - -Running test 18 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels and saving model using name specified through model_name option - adapts Test6 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test18 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model t -use_model f -model_name test19_model -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - -Running test 19 test type: prediction, description: basic dt_sklearn prediction test using a model saved under a name specified through model_name option on new data with numeric labels -../../src/run_smlp.py -model_name "../models/test19_model" -out_dir ./ -pref Test19 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test12 -mode train -resp y1,y2 -feat x1,x2,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f Running test 20 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels -../../src/run_smlp.py -model_name "../models/test20_model" -out_dir ./ -pref Test20 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model t -data_scaler none -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +smlp -model_name "../models/test20_model" -out_dir ./ -pref Test20 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model t -data_scaler none -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 21 test type: prediction, description: test for illegal symbols in column names -../../src/run_smlp.py -data "../data/smlp_toy_num_metasymbol_mult_reg.csv" -out_dir ./ -pref Test21 -mode predict -resp "PF ,|PF |" -model poly_sklearn -save_model t -use_model f -model_name test22_model -pred_plots t -resp_plots t -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_metasymbol_mult_reg_pred_labeled.csv" - -Running test 22 test type: prediction, description: test for illegal symbols in column names -../../src/run_smlp.py -model_name "../models/test22_model" -out_dir ./ -pref Test22 -mode predict -resp "PF ,|PF |" -model poly_sklearn -save_model f -use_model t -pred_plots t -resp_plots t -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_metasymbol_mult_reg_pred_labeled.csv" - -Running test 23 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels and saving model using name specified through model_name option - adapts Test6 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test23 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model t -use_model f -model_name test24_model -model_per_response t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 28 test type: prediction, description: checks nn_keras prediction with sw_coef 0.8 and functional API +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test28 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 24 test type: prediction, description: basic dt_sklearn prediction test using a model saved under a name specified through model_name option on new data with numeric labels -../../src/run_smlp.py -model_name "../models/test24_model" -out_dir ./ -pref Test24 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model t -model_per_response t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 35 test type: doe, description: doe test with four levels with plackett_burman +smlp -doe_spec "../grids/doe_four_levels_real.csv" -out_dir ./ -pref Test35 -mode doe -doe_algo plackett_burman -log_time f -Running test 25 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels and saving model using name specified through model_name option - adapts Test6 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test25 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model t -use_model f -model_name test26_model -mrmr_pred 2 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 42 test type: doe, description: doe test with four levels with maximin_reconstruction +smlp -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test42 -mode doe -doe_algo maximin_reconstruction -doe_samples 20 -log_time f -Running test 26 test type: prediction, description: basic dt_sklearn prediction test using a model saved under a name specified through model_name option on new data with numeric labels -../../src/run_smlp.py -model_name "../models/test26_model" -out_dir ./ -pref Test26 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model t -mrmr_pred 2 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 50 test type: discretization, description: tests discretization options +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test50 -mode discretize -resp "PF,PF1" -discr_algo kmeans -discr_bins 6 -discr_labels t -discr_type ordered -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass -Running test 27 test type: prediction, description: checks nn_keras prediction with nn_keras_seq_api t -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test27 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +spec_fn smlp_toy_num_resp_mult.spec +specs_path ../specs +Running test 58 test type: optimize, description: basic dt_sklearn optimization test with numeric labels and integer grid as domain and without scaling objectives +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test58 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -objv_names objv_y1,objv_y2 -objv_exprs "y1;y2" -epsilon 0.01 -delta_rel 0.01 -data_scaler none -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 28 test type: prediction, description: checks nn_keras prediction with sw_coef 0.8 and functional API -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test28 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +spec_fn smlp_toy_num_resp_noknobs_verify.spec +specs_path ../specs +Running test 66 test type: verify, description: basic dt_sklearn assertion verification test on data with one numeric response +smlp -model_name "../models/test65_model" -out_dir ./ -pref Test66 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x0**2+y1>4.3;(y1+x2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 29 test type: subgroups, description: basic test for subgroup discovery for pass-fail responses -../../src/run_smlp.py -data "../data/smlp_toy_cls_metasymbol_colnames_mult.csv" -out_dir ./ -pref Test29 -mode subgroups -psg_dim 3 -psg_top 10 -resp "PF 1,PF#" -plots t -seed 10 -log_time f +spec_fn smlp_toy_num_resp_noknobs_verify.spec +specs_path ../specs +Running test 72 test type: verify, description: nn_keras verification test with re-using saved model_per_response trained model +smlp -model_name "../models/test71_model" -out_dir ./ -pref Test72 -mode verify -resp y1,y2 -feat x0,x1,x2 -model nn_keras -nnet_encoding nested -save_model f -use_model t -model_per_response t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1 -asrt_exprs "(y2**3+x2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -Running test 30 test type: subgroups, description: basic test for subgroup discovery for numric responses -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test30 -mode subgroups -psg_dim 3 -psg_top 10 -resp y1,y2 -feat x,p1,p2 -plots t -seed 10 -log_time f +spec_fn smlp_toy_num_resp_noknobs_verify.spec +specs_path ../specs +Running test 78 test type: verify, description: tests model term formation when mrmr_pred is activated and not all features are selected for training the model +smlp -data "../data/smlp_toy_num_resp_noknobs.csv" -out_dir ./ -pref Test78 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -use_model f -model_name test78_model -mrmr_pred 1 -model_per_response f -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "y1==9;y2>0" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 31 test type: subgroups, description: testing resp2b in subgroup discovery mode -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test31 -mode subgroups -psg_dim 3 -psg_top 10 -resp y1,y2 -resp2b "y1<6;y2>6" -feat x,p1,p2 -plots t -seed 10 -log_time f -save_config t +spec_fn smlp_toy_num_resp_mult.spec +specs_path ../specs +Running test 85 test type: optimize, description: tests alpha and eta constraints specified in command line in optimization mode +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test85 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -data_scaler min_max -objv_names obj1,objv2 -objv_exprs "(y1+y2)/2;y1" -alpha "p2<5 and x==10 and x<12" -eta "p1==4" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 32 test type: unknown, description: test reusing saved model by using configuration file -../../src/run_smlp.py -model_name "../models/test20_model" -out_dir ./ -pref Test32 -config ../models/test20_model_rerun_model_config.json -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +spec_fn smlp_toy_num_resp_mult_verify_vacuous.spec +specs_path ../specs +Running test 92 test type: verify, description: test to detect contradictory constraints in verification mode +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test92 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_verify_vacuous.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 33 test type: unknown, description: testing -config option with subgroups mode -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test33 -config ../models/Test31_smlp_toy_num_resp_mult_args_config.json +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +specs_path ../specs +Running test 100 test type: optimize, description: basic test for sat_threshold option enabing usage of objectve values in SAT assignments that prove optimization thresholds +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test100 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 34 test type: doe, description: doe test with four levels with full_factorial method -../../src/run_smlp.py -doe_spec "../grids/doe_four_levels_real.csv" -out_dir ./ -pref Test34 -mode doe -doe_algo full_factorial -log_time f +spec_fn smlp_toy_num_resp_mult.spec +specs_path ../specs +Running test 104 test type: verify, description: assertion verfication test with wrong spec that does not assign a single value using a singleton grid or range with equal max and min +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test104 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "y1*2+x<=5 and y1<=10;-2*y2-1<10-p2" -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 35 test type: doe, description: doe test with four levels with plackett_burman -../../src/run_smlp.py -doe_spec "../grids/doe_four_levels_real.csv" -out_dir ./ -pref Test35 -mode doe -doe_algo plackett_burman -log_time f +spec_fn smlp_toy_num_resp_mult_cannot_synthesize.spec +specs_path ../specs +Running test 109 test type: synthesize, description: basic test for mode synthesize where synthesis fails +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test109 -mode synthesize -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_cannot_synthesize.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 3 test type: prediction, description: basic poly_sklearn prediction test on labeled and new data with numeric response in training/test data only +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test3 -mode predict -resp y1 -feat x,p1,p2 -model poly_sklearn -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_unlabeled.csv" -Running test 36 test type: doe, description: doe test with four levels with sukharev_grid -../../src/run_smlp.py -doe_spec "../grids/doe_four_levels_real.csv" -out_dir ./ -pref Test36 -mode doe -doe_algo sukharev_grid -doe_samples 125 -log_time f +Running test 9 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test9 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model t -model_name test20_model -data_scaler none -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -save_config t -save_model_config t -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 37 test type: doe, description: doe test with four levels with box_behnken -../../src/run_smlp.py -doe_spec "../grids/doe_three_levels_real_nan.csv" -out_dir ./ -pref Test37 -mode doe -doe_algo box_behnken -log_time f +Running test 17 test type: prediction, description: basic poly_sklearn prediction test from saved model on new data with numeric labels and two responses +smlp -model_name "../models/Test11_smlp_toy_num_resp_mult" -out_dir ./ -pref Test17 -mode predict -resp y1,y2 -feat x,p1,p2 -model poly_sklearn -save_model f -use_model t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 38 test type: doe, description: doe test with four levels with box_wilson -../../src/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test38 -mode doe -doe_algo box_wilson -doe_cc_face ccc -doe_cc_alpha r -doe_cc_center 2,3 -log_time f +Running test 22 test type: prediction, description: test for illegal symbols in column names +smlp -model_name "../models/test22_model" -out_dir ./ -pref Test22 -mode predict -resp "PF ,|PF |" -model poly_sklearn -save_model f -use_model t -pred_plots t -resp_plots t -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_metasymbol_mult_reg_pred_labeled.csv" -Running test 39 test type: doe, description: doe test with four levels with latin_hypercube -../../src/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test39 -mode doe -doe_algo latin_hypercube -doe_prob_distr Exponential -doe_samples 30 -log_time f +Running test 32 test type: unknown, description: test reusing saved model by using configuration file +smlp -model_name "../models/test20_model" -out_dir ./ -pref Test32 -config ../models/test20_model_rerun_model_config.json -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" Running test 40 test type: doe, description: doe test with four levels with latin_hypercube_space_filling -../../src/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test40 -mode doe -doe_algo latin_hypercube_sf -doe_samples 20 -log_time f +smlp -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test40 -mode doe -doe_algo latin_hypercube_sf -doe_samples 20 -log_time f -Running test 41 test type: doe, description: doe test with four levels with random_k_means -../../src/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test41 -mode doe -doe_algo random_k_means -doe_samples 20 -log_time f +Running test 48 test type: discretization, description: tests discretization options +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test48 -mode discretize -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass -Running test 42 test type: doe, description: doe test with four levels with maximin_reconstruction -../../src/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test42 -mode doe -doe_algo maximin_reconstruction -doe_samples 20 -log_time f +Running test 56 test type: discretization, description: tests discretization options +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test56 -mode discretize -resp "PF,PF1" -discr_algo ranks -discr_bins 6 -discr_labels f -discr_type object -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass -Running test 43 test type: doe, description: doe test with four levels with halton_sequence -../../src/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test43 -mode doe -doe_algo halton_sequence -doe_samples 20 -log_time f +spec_fn smlp_toy_num_resp_mult_y1_verify.spec +specs_path ../specs +Running test 64 test type: verify, description: basic dt_sklearn assertion verification test on data with one numeric response +smlp -model_name "../models/test63_model" -out_dir ./ -pref Test64 -mode verify -resp y1 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model t -spec ../specs/smlp_toy_num_resp_mult_y1_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x/2+y1>4.3;(y1+p2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 44 test type: doe, description: doe test with four levels with uniform_random_matrix -../../src/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test44 -mode doe -doe_algo uniform_random_matrix -doe_samples 20 -log_time f +spec_fn smlp_toy_num_resp_noknobs_verify.spec +specs_path ../specs +Running test 71 test type: verify, description: nn_keras verification test with model_per_response training +smlp -data "../data/smlp_toy_num_resp_noknobs.csv" -out_dir ./ -pref Test71 -mode verify -resp y1,y2 -feat x0,x1,x2 -model nn_keras -nnet_encoding nested -save_model t -use_model f -model_name test71_model -model_per_response t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1 -asrt_exprs "(y1**3+x2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -Running test 45 test type: doe, description: doe test with four levels with fractional_factorial -../../src/run_smlp.py -doe_spec "../grids/doe_two_levels_real.csv" -out_dir ./ -pref Test45 -mode doe -doe_algo fractional_factorial -doe_resolution 5 -log_time f +spec_fn smlp_toy_num_resp_mult_free_inps.spec +specs_path ../specs +Running test 81 test type: optimize, description: basic dt_sklearn single objective optimization test with numeric labels and integer grid as domain and with scaling objectives +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test81 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps.spec -data_scaler min_max -objv_names obj1 -objv_exprs "(y1+y2)/2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -Running test 46 test type: prediction, description: tests options -pos_val and -neg_val -../../src/run_smlp.py -data "../data/smlp_toy_pf_mult.csv" -out_dir ./ -pref Test46 -mode predict -resp "PF,PF1" -model poly_sklearn -save_model t -save_model_config f -use_model f -model_name test47_model -data_scaler none -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -pos_val fail -neg_val pass -new_dat "../data/smlp_toy_pf_mult.csv" +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +specs_path ../specs +Running test 88 test type: optimize, description: basic dt_sklearn multi objective pareto optimization test with beta and objectives specified in spec file +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test88 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 47 test type: prediction, description: tests options -pos_val and -neg_val when re-using saved model -../../src/run_smlp.py -model_name "../models/test47_model" -out_dir ./ -pref Test47 -mode predict -resp "PF,PF1" -model poly_sklearn -save_model f -use_model t -data_scaler none -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -pos_val fail -neg_val pass -new_dat "../data/smlp_toy_pf_mult.csv" +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 93 test type: optsyn, description: basic test for mode optsyn +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test93 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 48 test type: discretization, description: tests discretization options -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test48 -mode discretize -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +spec_fn smlp_toy_num_resp_mult_witness.spec +specs_path ../specs +Running test 101 test type: certify, description: basic test in certify mode to test stability (theta) and guard (eta) constraint generation +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test101 -mode certify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -use_model f -model_name test101_model -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_witness.spec -quer_names query1,query2,query3 -quer_exprs "(y2**3+p2)/2<6;y1>=9;y2<20" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 49 test type: discretization, description: tests discretization options -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test49 -mode discretize -resp "PF,PF1" -discr_algo quantile -discr_bins 6 -discr_labels t -discr_type category -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +spec_fn smlp_toy_num_resp_mult_beta_verify.spec +specs_path ../specs +Running test 107 test type: verify, description: test for verification mode to check that eta contraints are not contradictory and as otherwise verification problem is not well defined +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test107 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_beta_verify.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "y1*2+x<=5 and y1<=10;-2*y2-1<10-p2" -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 50 test type: discretization, description: tests discretization options -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test50 -mode discretize -resp "PF,PF1" -discr_algo kmeans -discr_bins 6 -discr_labels t -discr_type ordered -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +spec_fn smlp_toy_basic.spec +specs_path ../specs +Running test 114 test type: optimize, description: smlp toy basic test for mode optimize from SMLP manual without specifying resp and feat in command line +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test114 -mode optimize -pareto t -opt_strategy lazy -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -mrmr_pred 0 -epsilon 0.05 -delta_rel 0.01 -save_model f -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec ../specs/smlp_toy_basic.spec +Running test 4 test type: prediction, description: basic nn_keras prediction test on labeled and new data with numeric labels and one response +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test4 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -nn_keras_weights_precision 2 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 51 test type: discretization, description: tests discretization options -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test51 -mode discretize -resp "PF,PF1" -discr_algo jenks -discr_bins 6 -discr_labels f -discr_type integer -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +Running test 13 test type: train, description: EV-SI real life nn_keras prediction test on labeled and new data with numeric labels +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test13 -mode train -resp y1,y2 -feat x1,x2,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api f -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 52 test type: discretization, description: tests discretization options -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test52 -mode discretize -resp "PF,PF1" -discr_algo jenks -discr_bins 6 -discr_labels t -discr_type ordered -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +Running test 24 test type: prediction, description: basic dt_sklearn prediction test using a model saved under a name specified through model_name option on new data with numeric labels +smlp -model_name "../models/test24_model" -out_dir ./ -pref Test24 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model t -model_per_response t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 53 test type: discretization, description: tests discretization options -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test53 -mode discretize -resp "PF,PF1" -discr_algo ordinals -discr_bins 6 -discr_labels f -discr_type integer -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +Running test 31 test type: subgroups, description: testing resp2b in subgroup discovery mode +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test31 -mode subgroups -psg_dim 3 -psg_top 10 -resp y1,y2 -resp2b "y1<6;y2>6" -feat x,p1,p2 -plots t -seed 10 -log_time f -save_config t -Running test 54 test type: discretization, description: tests discretization options -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test54 -mode discretize -resp "PF,PF1" -discr_algo ordinals -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +Running test 39 test type: doe, description: doe test with four levels with latin_hypercube +smlp -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test39 -mode doe -doe_algo latin_hypercube -doe_prob_distr Exponential -doe_samples 30 -log_time f -Running test 55 test type: discretization, description: tests discretization options -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test55 -mode discretize -resp "PF,PF1" -discr_algo ranks -discr_bins 6 -discr_labels t -discr_type category -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +Running test 47 test type: prediction, description: tests options -pos_val and -neg_val when re-using saved model +smlp -model_name "../models/test47_model" -out_dir ./ -pref Test47 -mode predict -resp "PF,PF1" -model poly_sklearn -save_model f -use_model t -data_scaler none -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -pos_val fail -neg_val pass -new_dat "../data/smlp_toy_pf_mult.csv" -Running test 56 test type: discretization, description: tests discretization options -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test56 -mode discretize -resp "PF,PF1" -discr_algo ranks -discr_bins 6 -discr_labels f -discr_type object -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +Running test 55 test type: discretization, description: tests discretization options +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test55 -mode discretize -resp "PF,PF1" -discr_algo ranks -discr_bins 6 -discr_labels t -discr_type category -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass -spec_fn smlp_toy_num_resp_mult.spec +spec_fn smlp_toy_num_resp_mult_y1_verify.spec specs_path ../specs -Running test 58 test type: optimize, description: basic dt_sklearn optimization test with numeric labels and integer grid as domain and without scaling objectives -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test58 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -objv_names objv_y1,objv_y2 -objv_exprs "y1;y2" -epsilon 0.01 -delta_rel 0.01 -data_scaler none -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 63 test type: verify, description: basic dt_sklearn assertion verification test on data with numeric labels +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test63 -mode verify -resp y1 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -use_model f -model_name test63_model -spec ../specs/smlp_toy_num_resp_mult_y1_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x/2+y1>4.3;(y1+p2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_y2_verify.spec specs_path ../specs -Running test 59 test type: verify, description: basic nn_keras assertion verification test for functional nn_keras model -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test59 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat +Running test 69 test type: verify, description: nn_keras verification test with model_per_response training +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test69 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model t -use_model f -model_name test69_model -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "(y2**3+p2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -spec_fn smlp_toy_num_resp_mult_y2_verify.spec +spec_fn smlp_toy_num_resp_mult.spec specs_path ../specs -Running test 60 test type: verify, description: basic nn_keras assertion verification test for functional nn_keras model -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test60 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat +Running test 80 test type: optimize, description: basic dt_sklearn single objective optimization test with numeric labels and integer grid as domain and with scaling objectives +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test80 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -data_scaler min_max -objv_names obj1 -objv_exprs "(y1+y2)/2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_y1_verify.spec +spec_fn smlp_toy_num_resp_mult.spec specs_path ../specs -Running test 63 test type: verify, description: basic dt_sklearn assertion verification test on data with numeric labels -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test63 -mode verify -resp y1 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -use_model f -model_name test63_model -spec ../specs/smlp_toy_num_resp_mult_y1_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x/2+y1>4.3;(y1+p2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 86 test type: optimize, description: tests alpha +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test86 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -data_scaler min_max -objv_names obj1,objv2 -objv_exprs "(y1+y2)/2;y1" -asrt_names asrt1,asrt2,asrt3 -asrt_exprs "(y2**3+p2)/2<6;y1>=9;y2<0" -alpha "p2<5 and x==10 and x<12" -eta "p1==4" -epsilon 0.05 -delta_rel 0.01 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_y1_verify.spec +spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 64 test type: verify, description: basic dt_sklearn assertion verification test on data with one numeric response -../../src/run_smlp.py -model_name "../models/test63_model" -out_dir ./ -pref Test64 -mode verify -resp y1 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model t -spec ../specs/smlp_toy_num_resp_mult_y1_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x/2+y1>4.3;(y1+p2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 94 test type: optsyn, description: basic test for rf_sklearn in model exploration mode optsyn +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test94 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_noknobs_verify.spec +spec_fn smlp_toy_num_resp_mult_witness.spec specs_path ../specs -Running test 66 test type: verify, description: basic dt_sklearn assertion verification test on data with one numeric response -../../src/run_smlp.py -model_name "../models/test65_model" -out_dir ./ -pref Test66 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x0**2+y1>4.3;(y1+x2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 102 test type: certify, description: basic test in certify mode to test stability (theta) and guard (eta) constraint generation +smlp -model_name "../models/test101_model" -out_dir ./ -pref Test102 -mode certify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model t -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_witness.spec -quer_names query1,query2,query3 -quer_exprs "(y2**3+p2)/2<6;y1>=9;y2<20" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_noknobs_verify.spec +spec_fn smlp_toy_num_resp_mult_stable_verify.spec specs_path ../specs -Running test 68 test type: verify, description: basic dt_sklearn assertion verification test on data with one numeric response -../../src/run_smlp.py -model_name "../models/test67_model" -out_dir ./ -pref Test68 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -model_per_response t -save_model f -use_model t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x0**2+y1>4.3;(y1+x2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 105 test type: verify, description: basic dt_sklearn assertion verfication test with numeric labels and integer grid as domain +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test105 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_stable_verify.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "y1*2+x<=5 and y1<=10;-2*y2-1<10-p2" -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_y2_verify.spec +Running test 112 test type: prediction, description: smlp toy basic test from SMLP manual +smlp -model_name "../models/test110_model" -out_dir ./ -pref Test112 -mode predict -resp y1,y2 -feat x1,x2,p1,p2 -model poly_sklearn -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -use_model t -save_model f -new_dat "../data/smlp_toy_basic_pred_unlabeled.csv" + +spec_fn smlp_toy_system_stable_constant_query.spec specs_path ../specs -Running test 69 test type: verify, description: nn_keras verification test with model_per_response training -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test69 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model t -use_model f -model_name test69_model -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "(y2**3+p2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat +Running test 119 test type: query, description: query test with knobs only where query is satisfiable without stability and fails with stability +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test119 -mode query -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_query.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 1 test type: train, description: basic dt_caret training and test on labeled data with single numeric response +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test1 -mode train -resp y1 -feat x,p1,p2 -model dt_caret -save_model_config f -mrmr_pred 0 -plots f -seed 10 -log_time f + +Running test 19 test type: prediction, description: basic dt_sklearn prediction test using a model saved under a name specified through model_name option on new data with numeric labels +smlp -model_name "../models/test19_model" -out_dir ./ -pref Test19 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +Running test 27 test type: prediction, description: checks nn_keras prediction with nn_keras_seq_api t +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test27 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +Running test 34 test type: doe, description: doe test with four levels with full_factorial method +smlp -doe_spec "../grids/doe_four_levels_real.csv" -out_dir ./ -pref Test34 -mode doe -doe_algo full_factorial -log_time f + +Running test 43 test type: doe, description: doe test with four levels with halton_sequence +smlp -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test43 -mode doe -doe_algo halton_sequence -doe_samples 20 -log_time f + +Running test 51 test type: discretization, description: tests discretization options +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test51 -mode discretize -resp "PF,PF1" -discr_algo jenks -discr_bins 6 -discr_labels f -discr_type integer -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass spec_fn smlp_toy_num_resp_mult_y2_verify.spec specs_path ../specs -Running test 70 test type: verify, description: nn_keras verification test with re-using saved model_per_response trained model -../../src/run_smlp.py -model_name "../models/test69_model" -out_dir ./ -pref Test70 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model f -use_model t -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "(y2**3+p2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat +Running test 59 test type: verify, description: basic nn_keras assertion verification test for functional nn_keras model +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test59 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat spec_fn smlp_toy_num_resp_noknobs_verify.spec specs_path ../specs -Running test 72 test type: verify, description: nn_keras verification test with re-using saved model_per_response trained model -../../src/run_smlp.py -model_name "../models/test71_model" -out_dir ./ -pref Test72 -mode verify -resp y1,y2 -feat x0,x1,x2 -model nn_keras -nnet_encoding nested -save_model f -use_model t -model_per_response t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1 -asrt_exprs "(y2**3+x2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat +Running test 68 test type: verify, description: basic dt_sklearn assertion verification test on data with one numeric response +smlp -model_name "../models/test67_model" -out_dir ./ -pref Test68 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -model_per_response t -save_model f -use_model t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x0**2+y1>4.3;(y1+x2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 77 test type: unknown, description: verification test run using model_rerun config covering the case when mrmr selcts only a subset of features specified through the command line or config file -../../src/run_smlp.py -model_name "../models/test76_model" -out_dir ./ -pref Test77 -config ../models/test76_model_rerun_model_config.json +spec_fn smlp_toy_num_resp_noknobs_verify.spec +specs_path ../specs +Running test 74 test type: verify, description: tests model term formation when mrmr_pred is activated and not all features are selected for training the model. with separate model for each response +smlp -model_name "../models/test73_model" -out_dir ./ -pref Test74 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model t -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2,asrt3 -asrt_exprs "(y2**3+x2)/2<6;y1>=9;y2<0" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_noknobs_pred_labeled.csv" spec_fn smlp_toy_num_resp_mult.spec specs_path ../specs Running test 79 test type: query, description: basic test in query mode to test stability (theta) and guard (eta) constraint generation -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test79 -mode query -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult.spec -quer_names query1,query2,query3 -quer_exprs "(y2**3+p2)/2<6;y1>=9;y2<0" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test79 -mode query -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult.spec -quer_names query1,query2,query3 -quer_exprs "(y2**3+p2)/2<6;y1>=9;y2<0" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult.spec +spec_fn smlp_toy_num_resp_mult_query_vacuous.spec specs_path ../specs -Running test 80 test type: optimize, description: basic dt_sklearn single objective optimization test with numeric labels and integer grid as domain and with scaling objectives -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test80 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -data_scaler min_max -objv_names obj1 -objv_exprs "(y1+y2)/2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 91 test type: query, description: test to detect contradictory constraints in optimization mode due to contradictory alpha global and alpha bounds constraints on FMAX_xyx +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test91 -mode query -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_query_vacuous.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_free_inps.spec +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec specs_path ../specs -Running test 81 test type: optimize, description: basic dt_sklearn single objective optimization test with numeric labels and integer grid as domain and with scaling objectives -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test81 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps.spec -data_scaler min_max -objv_names obj1 -objv_exprs "(y1+y2)/2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat +Running test 99 test type: optimize, description: testing that the response and feature names can be taken from spec file in model exploration modes when the responses and/or features are not specified in the command line +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test99 -mode optimize -pareto t -opt_strategy lazy -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_free_inps.spec +spec_fn smlp_toy_num_resp_mult_certify_witness.spec specs_path ../specs -Running test 82 test type: optimize, description: basic dt_sklearn single objective optimization test with numeric labels and integer grid as domain and with scaling objectives -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test82 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps.spec -data_scaler min_max -objv_names obj1,objv2,objv3 -objv_exprs "(y1+y2)/2;y1;y2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat +Running test 103 test type: certify, description: +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test103 -mode certify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -use_model f -model_name test103_model -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_certify_witness.spec -quer_names valid_candidate,grid_conflict,range_conflict -quer_exprs "True;True;True" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_free_inps.spec -specs_path ../specs -Running test 83 test type: optimize, description: basic dt_sklearn multi objective pareto optimization test with numeric labels and integer grid as domain and with scaling objectives -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test83 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps.spec -data_scaler min_max -beta "y1>7 and y2>6" -objv_names obj1,objv2,objv3 -objv_exprs "(y1+y2)/2;y1/2-y2;y2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 110 test type: prediction, description: smlp toy basic example for predict mode from SMLP user manual +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test110 -mode predict -resp y1,y2 -feat x1,x2,p1,p2 -model poly_sklearn -save_model t -model_name test110_model -save_model_config t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_basic_pred_unlabeled.csv" -spec_fn smlp_toy_num_resp_mult.spec +spec_fn smlp_toy_system_stable_constant_certify.spec specs_path ../specs -Running test 85 test type: optimize, description: tests alpha and eta constraints specified in command line in optimization mode -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test85 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -data_scaler min_max -objv_names obj1,objv2 -objv_exprs "(y1+y2)/2;y1" -alpha "p2<5 and x==10 and x<12" -eta "p1==4" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 117 test type: certify, description: certification test with knobs only where assertion is valid without stability and fails with stability +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test117 -mode certify -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_certify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult.spec +spec_fn smlp_toy_system_stable_constant_synth_feasible.spec specs_path ../specs -Running test 86 test type: optimize, description: tests alpha -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test86 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -data_scaler min_max -objv_names obj1,objv2 -objv_exprs "(y1+y2)/2;y1" -asrt_names asrt1,asrt2,asrt3 -asrt_exprs "(y2**3+p2)/2<6;y1>=9;y2<0" -alpha "p2<5 and x==10 and x<12" -eta "p1==4" -epsilon 0.05 -delta_rel 0.01 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 123 test type: optimize, description: optimization test with constant knob and no inputs where synthesis is feasible and optimization is performed +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test123 -mode optimize -pareto t -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 8 test type: prediction, description: basic nn_keras prediction test on labeled and new data with numeric labels and two responses +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test8 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -nn_keras_epochs 20 -nn_keras_seq_api f -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -spec_fn smlp_toy_num_resp_mult_alpha_asrt_verify.spec +Running test 14 test type: train, description: EV-SI real life poly_sklearn prediction test on labeled and new data with numeric labels +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test14 -mode train -resp y1,y2 -feat x1,x2,p1,p2 -model poly_sklearn -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +Running test 21 test type: prediction, description: test for illegal symbols in column names +smlp -data "../data/smlp_toy_num_metasymbol_mult_reg.csv" -out_dir ./ -pref Test21 -mode predict -resp "PF ,|PF |" -model poly_sklearn -save_model t -use_model f -model_name test22_model -pred_plots t -resp_plots t -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_metasymbol_mult_reg_pred_labeled.csv" + +Running test 29 test type: subgroups, description: basic test for subgroup discovery for pass-fail responses +smlp -data "../data/smlp_toy_cls_metasymbol_colnames_mult.csv" -out_dir ./ -pref Test29 -mode subgroups -psg_dim 3 -psg_top 10 -resp "PF 1,PF#" -plots t -seed 10 -log_time f + +Running test 38 test type: doe, description: doe test with four levels with box_wilson +smlp -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test38 -mode doe -doe_algo box_wilson -doe_cc_face ccc -doe_cc_alpha r -doe_cc_center 2,3 -log_time f + +Running test 45 test type: doe, description: doe test with four levels with fractional_factorial +smlp -doe_spec "../grids/doe_two_levels_real.csv" -out_dir ./ -pref Test45 -mode doe -doe_algo fractional_factorial -doe_resolution 5 -log_time f + +Running test 53 test type: discretization, description: tests discretization options +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test53 -mode discretize -resp "PF,PF1" -discr_algo ordinals -discr_bins 6 -discr_labels f -discr_type integer -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +spec_fn smlp_toy_num_resp_noknobs_verify.spec specs_path ../specs -Running test 87 test type: verify, description: tests global alpha constraints and assertions specified in spec file -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test87 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_alpha_asrt_verify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 62 test type: verify, description: tests verificaion mode for NN with nn_keras_seq_api t +smlp -data "../data/smlp_toy_num_resp_noknobs.csv" -out_dir ./ -pref Test62 -mode verify -resp y1,y2 -feat x0,x1,x2 -model nn_keras -nnet_encoding nested -nn_keras_epochs 100 -nn_keras_seq_api t -save_model_config f -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "not(x25 and y1<=10);-2*y2-1<10-x2 and x2>5 and x2<8" -vacuity f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +Running test 75 test type: unknown, description: verification test run using model_rerun config covering the case when mrmr selcts only a subset of features specified through the command line or config file +smlp -model_name "../models/test73_model" -out_dir ./ -pref Test75 -config ../models/test73_model_rerun_model_config.json -new_dat "../data/smlp_toy_num_resp_noknobs_pred_labeled.csv" + +spec_fn smlp_toy_num_resp_mult_free_inps.spec specs_path ../specs -Running test 88 test type: optimize, description: basic dt_sklearn multi objective pareto optimization test with beta and objectives specified in spec file -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test88 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 82 test type: optimize, description: basic dt_sklearn single objective optimization test with numeric labels and integer grid as domain and with scaling objectives +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test82 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps.spec -data_scaler min_max -objv_names obj1,objv2,objv3 -objv_exprs "(y1+y2)/2;y1;y2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat spec_fn smlp_toy_num_resp_mult_query.spec specs_path ../specs Running test 89 test type: query, description: basic test in query mode to test stability (theta) and guard (eta) constraint generation -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test89 -mode query -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_query.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test89 -mode query -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_query.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn_vacuous.spec +spec_fn smlp_toy_num_resp_mult_query.spec specs_path ../specs -Running test 90 test type: optsyn, description: test to detect contradictory constraints in optsyn mode -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test90 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn_vacuous.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 97 test type: query, description: basic test for rf_sklearn in model exploration mode optsyn +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test97 -mode query -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_bootstrap f -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_query.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_query_vacuous.spec +spec_fn smlp_toy_num_resp_mult_unsat_eta_verify.spec specs_path ../specs -Running test 91 test type: query, description: test to detect contradictory constraints in optimization mode due to contradictory alpha global and alpha bounds constraints on FMAX_xyx -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test91 -mode query -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_query_vacuous.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 106 test type: verify, description: test for verification mode to check that eta contraints are not contradictory and as otherwise verification problem is not well defined +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test106 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_unsat_eta_verify.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "y1*2+x<=5 and y1<=10;-2*y2-1<10-p2" -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_verify_vacuous.spec +spec_fn smlp_toy_basic.spec specs_path ../specs -Running test 92 test type: verify, description: test to detect contradictory constraints in verification mode -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test92 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_verify_vacuous.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 113 test type: optimize, description: smlp toy basic test for mode optimize from SMLP manual +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test113 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x1,x2,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -mrmr_pred 0 -epsilon 0.05 -delta_rel 0.01 -save_model t -model_name test113_model -save_model_config t -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec ../specs/smlp_toy_basic.spec -spec_fn smlp_toy_num_resp_mult_optsyn.spec +spec_fn smlp_toy_system_stable_constant_synth_fail.spec specs_path ../specs -Running test 93 test type: optsyn, description: basic test for mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test93 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 120 test type: synthesize, description: synthesis test with constant knob and no inputs where synthesis is not feasible because the assertion is not feasible +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test120 -mode synthesize -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_fail.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn.spec +spec_fn smlp_toy_system_stable_certify.spec specs_path ../specs -Running test 94 test type: optsyn, description: basic test for rf_sklearn in model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test94 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 127 test type: certify, description: certification example with knobs only and fictitious inputs with values fixed through their ranges +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test127 -mode certify -model system -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_certify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 95 test type: optsyn, description: basic test for dt_caret in model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test95 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_caret -save_model f -use_model f -tree_encoding nested -compress_rules f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 142 test type: optsyn, description: basic test for compress_rules option for rf_sklearn in optsin mode +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test142 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 15 -tree_encoding nested -compress_rules t -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 6 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test6 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -spec_fn smlp_toy_num_resp_mult_optsyn.spec -specs_path ../specs -Running test 96 test type: optsyn, description: basic test for rf_sklearn in model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test96 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_caret -save_model f -use_model f -tree_encoding nested -compress_rules f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 11 test type: prediction, description: basic poly_sklearn prediction test on labeled and new data with numeric labels +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test11 -mode predict -resp y1,y2 -feat x,p1,p2 -model poly_sklearn -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -spec_fn smlp_toy_num_resp_mult_query.spec -specs_path ../specs -Running test 97 test type: query, description: basic test for rf_sklearn in model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test97 -mode query -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_bootstrap f -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_query.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 16 test type: prediction, description: basic nn_keras prediction test from saved model on new data with numeric labels and two responses +smlp -model_name "../models/Test8_smlp_toy_num_resp_mult" -out_dir ./ -pref Test16 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -save_model f -use_model t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -spec_fn smlp_toy_num_resp_mult_optsyn.spec -specs_path ../specs -Running test 98 test type: optsyn, description: basic test for et_caret in model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test98 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_caret -save_model f -use_model f -tree_encoding nested -compress_rules f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 26 test type: prediction, description: basic dt_sklearn prediction test using a model saved under a name specified through model_name option on new data with numeric labels +smlp -model_name "../models/test26_model" -out_dir ./ -pref Test26 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model t -mrmr_pred 2 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec -specs_path ../specs -Running test 99 test type: optimize, description: testing that the response and feature names can be taken from spec file in model exploration modes when the responses and/or features are not specified in the command line -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test99 -mode optimize -pareto t -opt_strategy lazy -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 33 test type: unknown, description: testing -config option with subgroups mode +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test33 -config ../models/Test31_smlp_toy_num_resp_mult_args_config.json -spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec -specs_path ../specs -Running test 100 test type: optimize, description: basic test for sat_threshold option enabing usage of objectve values in SAT assignments that prove optimization thresholds -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test100 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 41 test type: doe, description: doe test with four levels with random_k_means +smlp -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test41 -mode doe -doe_algo random_k_means -doe_samples 20 -log_time f -spec_fn smlp_toy_num_resp_mult_witness.spec -specs_path ../specs -Running test 101 test type: certify, description: basic test in certify mode to test stability (theta) and guard (eta) constraint generation -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test101 -mode certify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -use_model f -model_name test101_model -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_witness.spec -quer_names query1,query2,query3 -quer_exprs "(y2**3+p2)/2<6;y1>=9;y2<20" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 49 test type: discretization, description: tests discretization options +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test49 -mode discretize -resp "PF,PF1" -discr_algo quantile -discr_bins 6 -discr_labels t -discr_type category -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass -spec_fn smlp_toy_num_resp_mult_witness.spec +spec_fn smlp_toy_num_resp_noknobs_verify.spec specs_path ../specs -Running test 102 test type: certify, description: basic test in certify mode to test stability (theta) and guard (eta) constraint generation -../../src/run_smlp.py -model_name "../models/test101_model" -out_dir ./ -pref Test102 -mode certify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model t -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_witness.spec -quer_names query1,query2,query3 -quer_exprs "(y2**3+p2)/2<6;y1>=9;y2<20" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 57 test type: verify, description: basic dt_sklearn assertion verfication test with numeric labels and integer grid as domain +smlp -data "../data/smlp_toy_num_resp_noknobs.csv" -out_dir ./ -pref Test57 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "y1*2+x0<=5 and y1<=10;-2*y2-1<10-x2" -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_certify_witness.spec +spec_fn smlp_toy_num_resp_noknobs_verify.spec specs_path ../specs -Running test 103 test type: certify, description: -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test103 -mode certify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -use_model f -model_name test103_model -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_certify_witness.spec -quer_names valid_candidate,grid_conflict,range_conflict -quer_exprs "True;True;True" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 65 test type: verify, description: basic dt_sklearn assertion verification test on data with numeric labels +smlp -data "../data/smlp_toy_num_resp_noknobs.csv" -out_dir ./ -pref Test65 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -use_model f -model_name test65_model -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x0**2+y1>4.3;(y1+x2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult.spec +spec_fn smlp_toy_num_resp_mult_y2_verify.spec specs_path ../specs -Running test 104 test type: verify, description: assertion verfication test with wrong spec that does not assign a single value using a singleton grid or range with equal max and min -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test104 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "y1*2+x<=5 and y1<=10;-2*y2-1<10-p2" -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 70 test type: verify, description: nn_keras verification test with re-using saved model_per_response trained model +smlp -model_name "../models/test69_model" -out_dir ./ -pref Test70 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model f -use_model t -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "(y2**3+p2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -spec_fn smlp_toy_num_resp_mult_stable_verify.spec +Running test 77 test type: unknown, description: verification test run using model_rerun config covering the case when mrmr selcts only a subset of features specified through the command line or config file +smlp -model_name "../models/test76_model" -out_dir ./ -pref Test77 -config ../models/test76_model_rerun_model_config.json + +spec_fn smlp_toy_num_resp_mult_free_inps.spec specs_path ../specs -Running test 105 test type: verify, description: basic dt_sklearn assertion verfication test with numeric labels and integer grid as domain -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test105 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_stable_verify.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "y1*2+x<=5 and y1<=10;-2*y2-1<10-p2" -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 83 test type: optimize, description: basic dt_sklearn multi objective pareto optimization test with numeric labels and integer grid as domain and with scaling objectives +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test83 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps.spec -data_scaler min_max -beta "y1>7 and y2>6" -objv_names obj1,objv2,objv3 -objv_exprs "(y1+y2)/2;y1/2-y2;y2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_unsat_eta_verify.spec +spec_fn smlp_toy_num_resp_mult_optsyn_vacuous.spec specs_path ../specs -Running test 106 test type: verify, description: test for verification mode to check that eta contraints are not contradictory and as otherwise verification problem is not well defined -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test106 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_unsat_eta_verify.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "y1*2+x<=5 and y1<=10;-2*y2-1<10-p2" -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 90 test type: optsyn, description: test to detect contradictory constraints in optsyn mode +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test90 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn_vacuous.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_beta_verify.spec +spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 107 test type: verify, description: test for verification mode to check that eta contraints are not contradictory and as otherwise verification problem is not well defined -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test107 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_beta_verify.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "y1*2+x<=5 and y1<=10;-2*y2-1<10-p2" -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 98 test type: optsyn, description: basic test for et_caret in model exploration mode optsyn +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test98 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_caret -save_model f -use_model f -tree_encoding nested -compress_rules f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_synthesize.spec +spec_fn smlp_toy_system.spec specs_path ../specs -Running test 108 test type: synthesize, description: basic test for dt_sklearn in model exploration mode synthesize where synthesis succeeds -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test108 -mode synthesize -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_synthesize.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 116 test type: certify, description: basic test in certify mode when system is specified and is used as the model; p2 rel-rad needs to be 0 or very close to it the witness to first query to be stable +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test116 -mode certify -resp y1,y2 -feat x1,x2,p1,p2 -model system -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_system.spec -quer_names query1,query2 -quer_exprs "y1>0;y2<=0" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_cannot_synthesize.spec +spec_fn smlp_toy_witness_certify.spec specs_path ../specs -Running test 109 test type: synthesize, description: basic test for mode synthesize where synthesis fails -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test109 -mode synthesize -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_cannot_synthesize.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 128 test type: certify, description: Basic regression test in certify mode covering all four possible outcomes when certifying a witness for a query: the witness is stable +smlp -data "../data/smlp_toy_ctg_num_resp.csv" -out_dir ./ -pref Test128 -mode certify -resp y1,y2 -feat x,p1,p2 -model poly_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model f -model_per_response f -spec ../specs/smlp_toy_witness_certify.spec -quer_names query_stable_witness,query_grid_conflict,query_unstable_witness,query_infeasible_witness,query_poly_intercept_sensitive -quer_exprs "y2<=90;y1>=9;y1>=(-13);y1>9;y1>=(-10)" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 110 test type: prediction, description: smlp toy basic example for predict mode from SMLP user manual -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test110 -mode predict -resp y1,y2 -feat x1,x2,p1,p2 -model poly_sklearn -save_model t -model_name test110_model -save_model_config t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_basic_pred_unlabeled.csv" +spec_fn smlp_toy_system_stable_constant_synth_feasible.specRunning test 5 test type: prediction, description: basic dt_caret prediction test on labeled and new data with numeric labels +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test5 -mode predict -resp y1 -feat x,p1,p2 -model dt_caret -save_model t -use_model f -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 111 test type: unknown, description: smlp toy basic test to rerun saved model using the model rerun config file saved during model training -../../src/run_smlp.py -model_name "../models/test110_model" -out_dir ./ -pref Test111 -config ../models/test110_model_rerun_model_config.json -new_dat "../data/smlp_toy_basic_pred_unlabeled.csv" +Running test 18 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels and saving model using name specified through model_name option - adapts Test6 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test18 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model t -use_model f -model_name test19_model -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 112 test type: prediction, description: smlp toy basic test from SMLP manual -../../src/run_smlp.py -model_name "../models/test110_model" -out_dir ./ -pref Test112 -mode predict -resp y1,y2 -feat x1,x2,p1,p2 -model poly_sklearn -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -use_model t -save_model f -new_dat "../data/smlp_toy_basic_pred_unlabeled.csv" +Running test 25 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels and saving model using name specified through model_name option - adapts Test6 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test25 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model t -use_model f -model_name test26_model -mrmr_pred 2 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -spec_fn smlp_toy_basic.spec -specs_path ../specs -Running test 113 test type: optimize, description: smlp toy basic test for mode optimize from SMLP manual -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test113 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x1,x2,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -mrmr_pred 0 -epsilon 0.05 -delta_rel 0.01 -save_model t -model_name test113_model -save_model_config t -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec ../specs/smlp_toy_basic.spec +Running test 36 test type: doe, description: doe test with four levels with sukharev_grid +smlp -doe_spec "../grids/doe_four_levels_real.csv" -out_dir ./ -pref Test36 -mode doe -doe_algo sukharev_grid -doe_samples 125 -log_time f -spec_fn smlp_toy_basic.spec -specs_path ../specs -Running test 114 test type: optimize, description: smlp toy basic test for mode optimize from SMLP manual without specifying resp and feat in command line -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test114 -mode optimize -pareto t -opt_strategy lazy -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -mrmr_pred 0 -epsilon 0.05 -delta_rel 0.01 -save_model f -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec ../specs/smlp_toy_basic.spec +Running test 44 test type: doe, description: doe test with four levels with uniform_random_matrix +smlp -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test44 -mode doe -doe_algo uniform_random_matrix -doe_samples 20 -log_time f -spec_fn smlp_toy_system.spec +Running test 52 test type: discretization, description: tests discretization options +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test52 -mode discretize -resp "PF,PF1" -discr_algo jenks -discr_bins 6 -discr_labels t -discr_type ordered -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +spec_fn smlp_toy_num_resp_mult_y2_verify.spec specs_path ../specs -Running test 115 test type: certify, description: basic test in certify mode -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test115 -mode certify -resp y1,y2 -feat x1,x2,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_system.spec -quer_names query1,query2 -quer_exprs "y1>0;y2<=0" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 60 test type: verify, description: basic nn_keras assertion verification test for functional nn_keras model +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test60 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -spec_fn smlp_toy_system.spec +spec_fn smlp_toy_num_resp_noknobs_verify.spec specs_path ../specs -Running test 116 test type: certify, description: basic test in certify mode when system is specified and is used as the model; p2 rel-rad needs to be 0 or very close to it the witness to first query to be stable -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test116 -mode certify -resp y1,y2 -feat x1,x2,p1,p2 -model system -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_system.spec -quer_names query1,query2 -quer_exprs "y1>0;y2<=0" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 67 test type: verify, description: basic dt_sklearn assertion verification test on data with numeric labels +smlp -data "../data/smlp_toy_num_resp_noknobs.csv" -out_dir ./ -pref Test67 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -model_per_response t -save_model t -use_model f -model_name test67_model -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x0**2+y1>4.3;(y1+x2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_system_stable_constant_certify.spec +spec_fn smlp_toy_num_resp_noknobs_verify.spec specs_path ../specs -Running test 117 test type: certify, description: certification test with knobs only where assertion is valid without stability and fails with stability -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test117 -mode certify -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_certify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 73 test type: verify, description: tests model term formation when mrmr_pred is activated and not all features are selected for training the model +smlp -data "../data/smlp_toy_num_resp_noknobs.csv" -out_dir ./ -pref Test73 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -use_model f -model_name test73_model -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2,asrt3 -asrt_exprs "(y2**3+x2)/2<6;y1>=9;y2<0" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_noknobs_pred_labeled.csv" -spec_fn smlp_toy_system_stable_constant_verify.spec +spec_fn smlp_toy_num_resp_noknobs_verify.spec specs_path ../specs -Running test 118 test type: verify, description: verification test with knobs only where assertion is valid without stability and fails with stability -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test118 -mode verify -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_verify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 84 test type: verify, description: tests global alpha constraints specified using option -alpha on inputs +smlp -data "../data/smlp_toy_num_resp_noknobs.csv" -out_dir ./ -pref Test84 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2,asrt3 -asrt_exprs "(y2**3+x2)/2<6;y1>=9;y2<0" -alpha "x2==7.0 and x0==0 and x1==2.5" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_system_stable_constant_query.spec +spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 119 test type: query, description: query test with knobs only where query is satisfiable without stability and fails with stability -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test119 -mode query -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_query.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 95 test type: optsyn, description: basic test for dt_caret in model exploration mode optsyn +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test95 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_caret -save_model f -use_model f -tree_encoding nested -compress_rules f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_system_stable_constant_synth_fail.spec +spec_fn smlp_toy_num_resp_mult_synthesize.spec specs_path ../specs -Running test 120 test type: synthesize, description: synthesis test with constant knob and no inputs where synthesis is not feasible because the assertion is not feasible -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test120 -mode synthesize -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_fail.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 108 test type: synthesize, description: basic test for dt_sklearn in model exploration mode synthesize where synthesis succeeds +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test108 -mode synthesize -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_synthesize.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_system_stable_constant_synth_feasible.spec +spec_fn smlp_toy_system_stable_constant_verify.spec specs_path ../specs -Running test 121 test type: synthesize, description: synthesis test with constant knob and no inputs where synthesis is feasible -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test121 -mode synthesize -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 118 test type: verify, description: verification test with knobs only where assertion is valid without stability and fails with stability +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test118 -mode verify -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_verify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_system_stable_constant_synth_fail.spec specs_path ../specs -Running test 122 test type: optimize, description: optimization test with constant knob and no inputs where synthesis is not feasible because the assertion is not feasible but beta constraint is feasible therefore optimization is performed -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test122 -mode optimize -pareto f -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_fail.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 124 test type: optsyn, description: optimized synthesis test with constant knob and no inputs where synthesis is not feasible because while beta constraint is feasible the assertion is not feasible therefore optimization is not performed +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test124 -mode optsyn -pareto f -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_fail.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_system_stable_constant_synth_feasible.spec +spec_fn smlp_toy_system_stable_verify.spec specs_path ../specs -Running test 123 test type: optimize, description: optimization test with constant knob and no inputs where synthesis is feasible and optimization is performed -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test123 -mode optimize -pareto t -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 140 test type: verify, description: verification example with knobs only and fictitious inputs that have no effect where proparty is valid without stability and fails with stability +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test140 -mode verify -model system -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_verify.spec -trace_prec 1 -trace_anonym t -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_system_stable_constant_synth_fail.spec -specs_path ../specs -Running test 124 test type: optsyn, description: optimized synthesis test with constant knob and no inputs where synthesis is not feasible because while beta constraint is feasible the assertion is not feasible therefore optimization is not performed -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test124 -mode optsyn -pareto f -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_fail.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 147 test type: prediction, description: checks nn_keras prediction with sw_coef 0.8 and sequential API +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test147 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 7 test type: prediction, description: basic rf_sklearn prediction test on labeled and new data with numeric labels +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test7 -mode predict -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 15 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -spec_fn smlp_toy_system_stable_constant_synth_feasible.spec +Running test 10 test type: prediction, description: basic et_sklearn prediction test on labeled and new data with numeric labels +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test10 -mode predict -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 15 -et_sklearn_bootstrap f -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +Running test 15 test type: prediction, description: basic dt_caret prediction test from saved model on new data with numeric labels +smlp -model_name "../models/Test5_smlp_toy_num_resp_mult" -out_dir ./ -pref Test15 -mode predict -resp y1 -feat x,p1,p2 -model dt_caret -save_model f -use_model t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +Running test 23 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels and saving model using name specified through model_name option - adapts Test6 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test23 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model t -use_model f -model_name test24_model -model_per_response t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +Running test 30 test type: subgroups, description: basic test for subgroup discovery for numric responses +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test30 -mode subgroups -psg_dim 3 -psg_top 10 -resp y1,y2 -feat x,p1,p2 -plots t -seed 10 -log_time f + +Running test 37 test type: doe, description: doe test with four levels with box_behnken +smlp -doe_spec "../grids/doe_three_levels_real_nan.csv" -out_dir ./ -pref Test37 -mode doe -doe_algo box_behnken -log_time f + +Running test 46 test type: prediction, description: tests options -pos_val and -neg_val +smlp -data "../data/smlp_toy_pf_mult.csv" -out_dir ./ -pref Test46 -mode predict -resp "PF,PF1" -model poly_sklearn -save_model t -save_model_config f -use_model f -model_name test47_model -data_scaler none -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -pos_val fail -neg_val pass -new_dat "../data/smlp_toy_pf_mult.csv" + +Running test 54 test type: discretization, description: tests discretization options +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test54 -mode discretize -resp "PF,PF1" -discr_algo ordinals -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +spec_fn smlp_toy_num_resp_noknobs_verify.spec specs_path ../specs -Running test 125 test type: optsyn, description: optimized synthesis test with constant knob and no inputs where synthesis is feasible and optimization is performed -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test125 -mode optsyn -pareto t -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 61 test type: verify, description: tests verificaion mode for NN with nn_keras_seq_api f +smlp -data "../data/smlp_toy_num_resp_noknobs.csv" -out_dir ./ -pref Test61 -mode verify -resp y1,y2 -feat x0,x1,x2 -model nn_keras -nnet_encoding nested -nn_keras_epochs 100 -save_model_config f --spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "not(x25 and y1<=10);-2*y2-1<10-x2 and x2>5 and x2<8" -vacuity f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_seq_api f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -spec_fn smlp_toy_system_stable_verify.spec +spec_fn smlp_toy_num_resp_noknobs_verify.spec specs_path ../specs -Running test 126 test type: verify, description: verification example with knobs only and fictitious inputs that have no effect where proparty is valid without stability and fails with stability -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test126 -mode verify -model system -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_verify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 76 test type: verify, description: tests model term formation when mrmr_pred is activated and not all features are selected for training the model +smlp -data "../data/smlp_toy_num_resp_noknobs.csv" -out_dir ./ -pref Test76 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -use_model f -model_name test76_model -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2,asrt3 -asrt_exprs "(y2**3+x2)/2<6;y1>=9;y2<0" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_system_stable_certify.spec +spec_fn smlp_toy_num_resp_mult_alpha_asrt_verify.spec specs_path ../specs -Running test 127 test type: certify, description: certification example with knobs only and fictitious inputs with values fixed through their ranges -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test127 -mode certify -model system -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_certify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 87 test type: verify, description: tests global alpha constraints and assertions specified in spec file +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test87 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_alpha_asrt_verify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_witness_certify.spec +spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 128 test type: certify, description: Basic regression test in certify mode covering all four possible outcomes when certifying a witness for a query: the witness is stable -../../src/run_smlp.py -data "../data/smlp_toy_ctg_num_resp.csv" -out_dir ./ -pref Test128 -mode certify -resp y1,y2 -feat x,p1,p2 -model poly_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model f -model_per_response f -spec ../specs/smlp_toy_witness_certify.spec -quer_names query_stable_witness,query_grid_conflict,query_unstable_witness,query_infeasible_witness,query_poly_intercept_sensitive -quer_exprs "y2<=90;y1>=9;y1>=(-13);y1>9;y1>=(-10)" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 96 test type: optsyn, description: basic test for rf_sklearn in model exploration mode optsyn +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test96 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_caret -save_model f -use_model f -tree_encoding nested -compress_rules f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_configuration_verify.spec +Running test 111 test type: unknown, description: smlp toy basic test to rerun saved model using the model rerun config file saved during model training +smlp -model_name "../models/test110_model" -out_dir ./ -pref Test111 -config ../models/test110_model_rerun_model_config.json -new_dat "../data/smlp_toy_basic_pred_unlabeled.csv" + +spec_fn smlp_toy_system.spec specs_path ../specs -Running test 129 test type: verify, description: verification example with demonstrating all basic result scenarious for assertions -../../src/run_smlp.py -data "../data/smlp_toy_ctg_num_resp.csv" -out_dir ./ -pref Test129 -mode verify -resp y1,y2 -feat x,p1,p2 -model poly_sklearn -save_model f -use_model f -model_per_response f -spec ../specs/smlp_toy_configuration_verify.spec -asrt_names assert_stable_config,assert_grid_conflict,assert_unstable_config,assert_infeasible -asrt_exprs "y2<=90;y1>=9;y1>=(-10);y1>20" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 115 test type: certify, description: basic test in certify mode +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test115 -mode certify -resp y1,y2 -feat x1,x2,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_system.spec -quer_names query1,query2 -quer_exprs "y1>0;y2<=0" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_system_stable_verify.spec specs_path ../specs -Running test 140 test type: verify, description: verification example with knobs only and fictitious inputs that have no effect where proparty is valid without stability and fails with stability -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test140 -mode verify -model system -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_verify.spec -trace_prec 1 -trace_anonym t -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 126 test type: verify, description: verification example with knobs only and fictitious inputs that have no effect where proparty is valid without stability and fails with stability +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test126 -mode verify -model system -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_verify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult.spec specs_path ../specs Running test 141 test type: optimize, description: basic test for compress_rules option for dt_sklearn in optimization mode -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test141 -mode optimize -opt_strategy lazy -pareto f -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules t -spec ../specs/smlp_toy_num_resp_mult.spec -objv_names objv_y1,objv_y2 -objv_exprs "y1;y2" -epsilon 0.01 -delta_rel 0.01 -data_scaler none -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test141 -mode optimize -opt_strategy lazy -pareto f -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules t -spec ../specs/smlp_toy_num_resp_mult.spec -objv_names objv_y1,objv_y2 -objv_exprs "y1;y2" -epsilon 0.01 -delta_rel 0.01 -data_scaler none -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn.spec -specs_path ../specs -Running test 142 test type: optsyn, description: basic test for compress_rules option for rf_sklearn in optsin mode -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test142 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 15 -tree_encoding nested -compress_rules t -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 148 test type: prediction, description: checks nn_keras prediction with sw_coef 0.8 and sequential API +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test148 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -spec_fn smlp_toy_num_resp_mult_query.spec +spec_fn smlp_toy_system_stable_constant_synth_fail.spec specs_path ../specs -Running test 143 test type: query, description: basic test for compress_rules for et_sklearn in mode query -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test143 -mode query -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_bootstrap f -tree_encoding nested -compress_rules t -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_query.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 122 test type: optimize, description: optimization test with constant knob and no inputs where synthesis is not feasible because the assertion is not feasible but beta constraint is feasible therefore optimization is performed +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test122 -mode optimize -pareto f -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_fail.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_system_stable_constant_synth_feasible.spec +spec_fn smlp_toy_num_resp_noknobs_verify.spec specs_path ../specs -Running test 145 test type: optimize, description: optimization test with constant knob and no inputs where synthesis is feasible and optimization is performed -../../src/run_smlp.py -out_dir ./ -pref Test145 -mode optimize -pareto t -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -doe_spec ../grids/doe_two_levels_opt.csv -doe_algo latin_hypercube -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 139 test type: verify, description: tests model term formation when mrmr_pred is activated and not all features are selected for training the model +smlp -data "../data/smlp_toy_num_resp_noknobs.csv" -out_dir ./ -pref Test139 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2,asrt3 -asrt_exprs "(y2**3+x2)/2<6;y1>=9;y2<0" -trace_anonym t -trace_prec 3 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_noknobs_pred_labeled.csv" -spec_fn smlp_toy_system.spec -specs_path ../specs -Running test 146 test type: optimize, description: optimization test with constant knob and no inputs where synthesis is feasible and optimization is performed -../../src/run_smlp.py -out_dir ./ -pref Test146 -mode optimize -pareto t -opt_strategy lazy -model poly_sklearn -resp y1,y2 -feat p1,p2,x1,x2 -save_model t -use_model f -mrmr_pred 0 -model_per_response t -split 1 -spec ../specs/smlp_toy_system.spec -doe_spec ../grids/explore_doe_two_levels.csv -doe_algo latin_hypercube -epsilon 0.99999999 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 149 test type: prediction, description: tests the mae loss function MeanAbsoluteError and sample weoghts +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test149 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss mae -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 147 test type: prediction, description: checks nn_keras prediction with sw_coef 0.8 and sequential API -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test147 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +spec_fn smlp_toy_num_resp_mult_verify.spec +specs_path ../specs +Running test 156 test type: verify, description: basic nn_keras assertion verification test that uses keras tuner for functional model training; adapts test 154 by consdering multiple responses +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test156 -mode verify -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics rmse -Running test 148 test type: prediction, description: checks nn_keras prediction with sw_coef 0.8 and sequential API -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test148 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +spec_fn smlp_toy_num_resp_noknobs_verify.spec +specs_path ../specs +Running test 162 test type: verify, description: tests model term construction with flat_encoding of tress and model per reponse when mrmr_pred is activated and not all features are selected for training the model +smlp -data "../data/smlp_toy_num_resp_noknobs.csv" -out_dir ./ -pref Test162 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2,asrt3 -asrt_exprs "(y2**3+x2)/2<6;y1>=9;y2<0" -trace_anonym t -trace_prec 3 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_noknobs_pred_labeled.csv" -Running test 149 test type: prediction, description: tests the mae loss function MeanAbsoluteError and sample weoghts -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test149 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss mae -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 167 test type: optsyn, description: basic flat tree encoding test with model_per_response t for rf_sklearn in model exploration mode optsyn +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test167 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 4 -rf_sklearn_n_estimators 3 -tree_encoding flat -compress_rules t -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 150 test type: prediction, description: tests the mape loss function MeanAbsolutePercentageError and sample weights -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test150 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss mape -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 171 test type: optimize, description: basic test for et_caret with flat tree_encoding in model exploration mode optimize +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test171 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_caret -tree_encoding flat -model_per_response t -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 151 test type: prediction, description: tests msle loss function MeanSquaredLogarithmicError and and sample weoghts -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test151 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss msle -sw_coef 3 -sw_exp 10 -sw_int 0 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 176 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response t nn_keras_seq_api f for nn_keras in model exploration mode optsyn +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test176 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api f -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 152 test type: prediction, description: tests the huber loss function Huber and sample weights -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test152 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss huber -sw_coef 8 -sw_exp 5 -sw_int 0.5 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +specs_path ../specs +Running test 181 test type: optimize, description: basic flat tree encoding test for dt_sklearn multi objective pareto optimization when features are not scaled modifies test 164 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test181 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 153 test type: prediction, description: tests the logcosh loss function LogCosh and sample weights -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test153 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_loss logcosh -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics mse -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +specs_path ../specs +Running test 182 test type: optimize, description: basic flat tree encoding test for dt_sklearn multi objective pareto optimization when responses are not scaled modifies test 164 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test182 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -scale_resp f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat"" -spec_fn smlp_toy_num_resp_mult_y2_verify.spec +spec_fn smlp_toy_num_resp_noknobs_verify.spec specs_path ../specs -Running test 154 test type: verify, description: basic nn_keras assertion verification test that uses keras tuner for functional model training -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test154 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3,3,3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 +Running test 186 test type: verify, description: tests model term formation when mrmr_pred is activated and not all features are selected for training the model +smlp -data "../data/smlp_toy_num_resp_noknobs.csv" -out_dir ./ -pref Test186 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2,asrt3 -asrt_exprs "(y2**3+x2)/2<6;y1>=9;y2<0" -trace_anonym t -trace_prec 3 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_noknobs_pred_labeled.csv" -spec_fn smlp_toy_num_resp_mult_y2_verify.spec +spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 155 test type: verify, description: basic nn_keras assertion verification test that uses keras tuner with sequrntial models for model training -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test155 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3,3,3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics mae +Running test 192 test type: optimize, description: basic test for et_sklearn with branched tree_encoding and model_per_response f in model exploration mode optimize adapts test 170 !!!!!!!!! in this test z3 result differs from mathsat and yices results (the latter two give sma results +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test192 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 100 -et_sklearn_bootstrap f -tree_encoding branched -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_verify.spec +spec_fn smlp_toy_system_stable_constant_synth_feasible.spec specs_path ../specs -Running test 156 test type: verify, description: basic nn_keras assertion verification test that uses keras tuner for functional model training; adapts test 154 by consdering multiple responses -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test156 -mode verify -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics rmse +Running test 125 test type: optsyn, description: optimized synthesis test with constant knob and no inputs where synthesis is feasible and optimization is performed +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test125 -mode optsyn -pareto t -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_verify.spec +spec_fn smlp_toy_num_resp_noknobs_verify.spec specs_path ../specs -Running test 157 test type: verify, description: basic nn_keras assertion verification test that uses keras tuner with sequrntial models for model training; adapts test 155 by consdering multiple responses -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test157 -mode verify -resp y1,y2 -feat x,p1,p2 --model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics rmse,logcosh +Running test 144 test type: verify, description: basic test for compress_rules for dt_sklearn in mode verify and re-using saved model +smlp -data "../data/smlp_toy_num_resp_noknobs.csv" -out_dir ./ -pref Test144 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules t -save_model f -use_model f -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x0**2+y1>4.3;(y1+x2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 158 test type: prediction, description: tests the mape loss function and sample weights with model_per_response t -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test158 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss mape -model_per_response t -sw_coef 8 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics rmse -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 150 test type: prediction, description: tests the mape loss function MeanAbsolutePercentageError and sample weights +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test150 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss mape -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" Running test 159 test type: prediction, description: tests the msle loss function and sample weights with model_per_response t -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test159 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_loss msle -model_per_response t -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics mae,cosine -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - -Running test 160 test type: prediction, description: tests nn keras tuner bayesian -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test160 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss mape -nn_keras_metrics msle -nn_keras_tuner bayesian -nn_keras_layers_grid "2,3" -nn_keras_losses_grid "mse,mae,huber" -model_per_response f -sw_coef 8 -sw_exp 5 -sw_int 0.5 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - -Running test 161 test type: prediction, description: tests nn keras tuner bayesian -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test161 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_loss msle -nn_keras_metrics mape,logcosh -nn_keras_tuner random -nn_keras_lrates_grid "0.01,0.001" -nn_keras_batches_grid "32,64" -model_per_response f -sw_coef 4 -sw_exp 5 -sw_int 0.5 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test159 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_loss msle -model_per_response t -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics mae,cosine -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec specs_path ../specs Running test 164 test type: optimize, description: basic flat tree encoding test for dt_sklearn multi objective pareto optimization -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test164 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test164 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 165 test type: optsyn, description: basic flat tree encoding test for dt_caretin model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test165 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_caret -tree_encoding flat -save_model f -use_model f -compress_rules f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 166 test type: optsyn, description: basic flat tree encoding test with model_per_response f for rf_sklearn in model exploration mode optsyn +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test166 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 4 -rf_sklearn_n_estimators 3 -tree_encoding flat -compress_rules t -save_model f -use_model f -compress_rules t -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 166 test type: optsyn, description: basic flat tree encoding test with model_per_response f for rf_sklearn in model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test166 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 4 -rf_sklearn_n_estimators 3 -tree_encoding flat -compress_rules t -save_model f -use_model f -compress_rules t -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 170 test type: optimize, description: basic test for et_sklearn with flat tree_encoding and model_per_response f in model exploration mode optimize +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test170 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -rf_sklearn_n_estimators 3 -et_sklearn_bootstrap f -tree_encoding flat -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 167 test type: optsyn, description: basic flat tree encoding test with model_per_response t for rf_sklearn in model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test167 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 4 -rf_sklearn_n_estimators 3 -tree_encoding flat -compress_rules t -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 175 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response f nn_keras_seq_api t for nn_keras in model exploration mode optsyn +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test175 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 168 test type: optimize, description: basic test for rf_caret with flat tree_encoding and modelper_response in model exploration mode optimize -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test168 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_caret -model_per_response t -compress_rules t -tree_encoding flat -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 180 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response f nn_keras_seq_api t for nn_keras in model exploration mode optsyn when features and responses are not scaled adapts test 175 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test180 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response f -scale_feat f -scale_resp f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn.spec +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec specs_path ../specs -Running test 169 test type: optimize, description: basic test for et_sklearn with flat tree_encoding and model_per_response t in model exploration mode optimize -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test169 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -rf_sklearn_n_estimators 3 -et_sklearn_bootstrap f -tree_encoding flat -model_per_response t -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 183 test type: optimize, description: basic flat tree encoding test for dt_sklearn multi objective pareto optimization when features and responses are not scaled modifies test 164 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test183 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -scale_resp f -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn.spec +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec specs_path ../specs -Running test 170 test type: optimize, description: basic test for et_sklearn with flat tree_encoding and model_per_response f in model exploration mode optimize -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test170 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -rf_sklearn_n_estimators 3 -et_sklearn_bootstrap f -tree_encoding flat -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 187 test type: optimize, description: basic branched tree encoding test for dt_sklearn multi objective pareto optimization adapts test 164 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test187 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 171 test type: optimize, description: basic test for et_caret with flat tree_encoding in model exploration mode optimize -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test171 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_caret -tree_encoding flat -model_per_response t -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 193 test type: optimize, description: basic test for et_caret with branched tree_encoding in model exploration mode optimize adapts test 171 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test193 -mode optimize -resp y1,y2 -feat x,p1,p2 -model et_caret -tree_encoding branched -model_per_response t -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_y2_verify.spec +Running test 216 test type: correlate, description: basic test for correlate mode +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test216 -mode correlate -resp y1,y2 -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f + +Running test 222 test type: correlate, description: basic test for correlate mode +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test222 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method adjusted -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +spec_fn smlp_toy_missing_radii.spec specs_path ../specs -Running test 172 test type: verify, description: basic test for nn_keras flat encoding for functional api -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test172 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nnet_encoding layered -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3,3,3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat"" +Running test 229 test type: certify, description: basic test for checking that each knob must have either absolute or relative radius specified in the spec file (even if radii are specified in the command line) +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test229 -mode certify -resp y1,y2 -feat x1,x2,p1,p2 -model system -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_missing_radii.spec -rad_rel 0.005 -rad_abs 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_y2_verify.spec + +Running test 153 test type: prediction, description: tests the logcosh loss function LogCosh and sample weights +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test153 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_loss logcosh -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics mse -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +Running test 161 test type: prediction, description: tests nn keras tuner bayesian +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test161 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_loss msle -nn_keras_metrics mape,logcosh -nn_keras_tuner random -nn_keras_lrates_grid "0.01,0.001" -nn_keras_batches_grid "32,64" -model_per_response f -sw_coef 4 -sw_exp 5 -sw_int 0.5 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 173 test type: verify, description: basic test for nn_keras flat encoding for sequential api -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test173 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3,3,3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics mae -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat"" +Running test 168 test type: optimize, description: basic test for rf_caret with flat tree_encoding and modelper_response in model exploration mode optimize +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test168 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_caret -model_per_response t -compress_rules t -tree_encoding flat -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs Running test 174 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response f nn_keras_seq_api f for nn_keras in model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test174 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api f -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test174 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api f -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 175 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response f nn_keras_seq_api t for nn_keras in model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test175 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 190 test type: optimize, description: basic test for rf_caret with branched tree_encoding and modelper_response in model exploration mode optimize adapts test 168 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test190 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_caret -model_per_response t -compress_rules t -tree_encoding branched -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn.spec +spec_fn smlp_toy_num_resp_mult_no_input.spec specs_path ../specs -Running test 176 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response t nn_keras_seq_api f for nn_keras in model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test176 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api f -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 202 test type: optimize, description: basic dt_sklearn single objective optimization with the eager algorithm when there are no inputs and no beta constraints +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test202 -mode optimize -pareto t -opt_strategy eager -resp y1,y2 -feat p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_no_input.spec -data_scaler min_max -objv_names obj1 -objv_exprs "(y1+y2)/2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn.spec +Running test 215 test type: correlate, description: basic test for correlate mode +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test215 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +Running test 221 test type: correlate, description: basic test for correlate mode +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test221 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method shannon -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +spec_fn smlp_toy_system_radii_update_certify.spec specs_path ../specs -Running test 177 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response t nn_keras_seq_api t for nn_keras in model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test177 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 228 test type: certify, description: test that radii specified in command line properly override the radii specified in the spec file. Here we override both ansolute and relative radii and one can observe that the certification results also change compared to test 116 +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test228 -mode certify -resp y1,y2 -feat x1,x2,p1,p2 -model system -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_system_radii_update_certify.spec -rad_rel 0.005 -rad_abs 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn.spec + +spec_fn smlp_toy_system_stable_constant_synth_feasible.spec specs_path ../specs -Running test 178 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response t nn_keras_seq_api t for nn_keras in model exploration mode optsyn when features are not scaled adapts test 177 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test178 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response t -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 205 test type: optimize, description: optimization test with eager strategy and with constant knob and no inputs where synthesis is feasible and optimization is performed adapts test 145 +smlp -out_dir ./ -pref Test205 -mode optimize -pareto t -opt_strategy eager -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -doe_spec ../grids/doe_two_levels_opt.csv -doe_algo latin_hypercube -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +Running test 219 test type: correlate, description: basic test for correlate mode +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test219 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type integer -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +Running test 225 test type: correlate, description: basic test for correlate mode and tests the adjusted mutual information +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test225 -mode correlate -resp y1,y2 -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method adjusted -mrmr_pred 0 -plots f -seed 10 -log_time f + +Running test 233 test type: subgroups, description: tests subgroup discovery mode when the response has string values +smlp -data "../data/smlp_toy_string_response.csv" -out_dir ./ -pref Test233 -mode subgroups -resp str_resp1 -feat num,int,str -pos_val no -neg_val yes -seed 10 -log_time f + + +specs_path ../specs +Running test 145 test type: optimize, description: optimization test with constant knob and no inputs where synthesis is feasible and optimization is performed +smlp -out_dir ./ -pref Test145 -mode optimize -pareto t -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -doe_spec ../grids/doe_two_levels_opt.csv -doe_algo latin_hypercube -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +Running test 152 test type: prediction, description: tests the huber loss function Huber and sample weights +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test152 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss huber -sw_coef 8 -sw_exp 5 -sw_int 0.5 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +Running test 160 test type: prediction, description: tests nn keras tuner bayesian +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test160 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss mape -nn_keras_metrics msle -nn_keras_tuner bayesian -nn_keras_layers_grid "2,3" -nn_keras_losses_grid "mse,mae,huber" -model_per_response f -sw_coef 8 -sw_exp 5 -sw_int 0.5 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 179 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response f nn_keras_seq_api f for nn_keras in model exploration mode optsyn when resposes are not scaled adapts test 174 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test179 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api f -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response f -scale_resp f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 177 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response t nn_keras_seq_api t for nn_keras in model exploration mode optsyn +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test177 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 180 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response f nn_keras_seq_api t for nn_keras in model exploration mode optsyn when features and responses are not scaled adapts test 175 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test180 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response f -scale_feat f -scale_resp f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 191 test type: optimize, description: basic test for et_sklearn with branched tree_encoding and model_per_response t in model exploration mode optimize adapts test 169 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test191 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 3 -et_sklearn_bootstrap t -tree_encoding branched -model_per_response t -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec specs_path ../specs -Running test 181 test type: optimize, description: basic flat tree encoding test for dt_sklearn multi objective pareto optimization when features are not scaled modifies test 164 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test181 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 196 test type: optimize, description: basic branched tree encoding test for dt_sklearn multi objective pareto optimization when features are not scaled modifies test 164 and test 181 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test196 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +spec_fn smlp_toy_system_stable_constant_synth_fail.spec specs_path ../specs -Running test 182 test type: optimize, description: basic flat tree encoding test for dt_sklearn multi objective pareto optimization when responses are not scaled modifies test 164 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test182 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -scale_resp f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat"" +Running test 203 test type: optimize, description: optimization test with eager strategy and with constant knob and no inputs where synthesis is not feasible because the assertion is not feasible but beta constraint is feasible therefore optimization is performed adapts test 122 +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test203 -mode optimize -pareto f -opt_strategy eager -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_fail.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +Running test 217 test type: correlate, description: basic test for correlate mode +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test217 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type category -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +Running test 223 test type: correlate, description: basic test for correlate mode and tests the normalized mutual information +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test223 -mode correlate -resp y1,y2 -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method normalized -mrmr_pred 0 -plots f -seed 10 -log_time f + +spec_fn smlp_toy_system_monotone_knob.05_verify.spec specs_path ../specs -Running test 183 test type: optimize, description: basic flat tree encoding test for dt_sklearn multi objective pareto optimization when features and responses are not scaled modifies test 164 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test183 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -scale_resp f -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 230 test type: verify, description: tests that outputs in system specificaation might depend on different inuts (knobs and free inputs) +smlp -data "../data/smlp_toy_monotone_basic.csv" -out_dir ./ -pref Test230 -mode verify -spec ../specs/smlp_toy_system_monotone_knob.05_verify.spec -model system -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec + +spec_fn smlp_toy_num_resp_mult_y2_verify.spec specs_path ../specs -Running test 187 test type: optimize, description: basic branched tree encoding test for dt_sklearn multi objective pareto optimization adapts test 164 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test187 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 154 test type: verify, description: basic nn_keras assertion verification test that uses keras tuner for functional model training +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test154 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3,3,3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 188 test type: optsyn, description: basic branched tree encoding test for dt_caretin model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test188 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_caret -tree_encoding branched -save_model f -use_model f -compress_rules f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 178 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response t nn_keras_seq_api t for nn_keras in model exploration mode optsyn when features are not scaled adapts test 177 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test178 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response t -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_noknobs_verify.spec +specs_path ../specs +Running test 185 test type: verify, description: tests model term construction with branched_encoding of tress and model per reponse when mrmr_pred is activated and not all features are selected for training the model +smlp -data "../data/smlp_toy_num_resp_noknobs.csv" -out_dir ./ -pref Test185 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2,asrt3 -asrt_exprs "(y2**3+x2)/2<6;y1>=9;y2<0" -trace_anonym t -trace_prec 3 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_noknobs_pred_labeled.csv" spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs Running test 189 test type: optsyn, description: basic branched tree encoding test with model_per_response f for rf_sklearn in model exploration mode optsyn adapts test 166 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test189 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 4 -rf_sklearn_n_estimators 3 -tree_encoding branched -compress_rules t -save_model f -use_model f -compress_rules t -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test189 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 4 -rf_sklearn_n_estimators 3 -tree_encoding branched -compress_rules t -save_model f -use_model f -compress_rules t -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 190 test type: optimize, description: basic test for rf_caret with branched tree_encoding and modelper_response in model exploration mode optimize adapts test 168 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test190 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_caret -model_per_response t -compress_rules t -tree_encoding branched -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 194 test type: optsyn, description: basic branched tree encoding test with model_per_response t for rf_sklearn in model exploration mode optsyn +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test194 -mode optsyn -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 4 -rf_sklearn_n_estimators 3 -tree_encoding branched -compress_rules t -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn.spec +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec specs_path ../specs -Running test 191 test type: optimize, description: basic test for et_sklearn with branched tree_encoding and model_per_response t in model exploration mode optimize adapts test 169 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test191 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 3 -et_sklearn_bootstrap t -tree_encoding branched -model_per_response t -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 198 test type: optimize, description: basic branched tree encoding test for dt_sklearn multi objective pareto optimization when features and responses are not scaled modifies test 164 and test 183 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test198 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -scale_resp f -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -spec_fn smlp_toy_num_resp_mult_optsyn.spec +spec_fn smlp_toy_system_stable_constant_synth_feasible.spec specs_path ../specs -Running test 192 test type: optimize, description: basic test for et_sklearn with branched tree_encoding and model_per_response f in model exploration mode optimize adapts test 170 !!!!!!!!! in this test z3 result differs from mathsat and yices results (the latter two give sma results -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test192 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 100 -et_sklearn_bootstrap f -tree_encoding branched -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 204 test type: optimize, description: optimization test with eager strategy and with constant knob and no inputs where synthesis is feasible and optimization is performed adapts test 123 +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test204 -mode optimize -pareto t -opt_strategy eager -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn.spec +Running test 218 test type: correlate, description: basic test for correlate mode +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test218 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type ordered -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +Running test 226 test type: correlate, description: basic test for correlate mode +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test226 -mode correlate -resp y1,y2 -discr_algo uniform -discret_num t -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f + +spec_fn smlp_toy_system_running_example_certify.spec specs_path ../specs -Running test 193 test type: optimize, description: basic test for et_caret with branched tree_encoding in model exploration mode optimize adapts test 171 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test193 -mode optimize -resp y1,y2 -feat x,p1,p2 -model et_caret -tree_encoding branched -model_per_response t -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 232 test type: certify, description: running example from smlp manual +smlp -data "../data/smlp_toy_system_running_example_certify.csv" -out_dir ./ -pref Test232 -mode certify -spec ../specs/smlp_toy_system_running_example_certify.spec -model system -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn.spec + +spec_fn smlp_toy_num_resp_mult_query.spec specs_path ../specs -Running test 194 test type: optsyn, description: basic branched tree encoding test with model_per_response t for rf_sklearn in model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test194 -mode optsyn -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 4 -rf_sklearn_n_estimators 3 -tree_encoding branched -compress_rules t -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 143 test type: query, description: basic test for compress_rules for et_sklearn in mode query +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test143 -mode query -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_bootstrap f -tree_encoding nested -compress_rules t -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_query.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn.spec +spec_fn smlp_toy_num_resp_mult_verify.spec specs_path ../specs -Running test 195 test type: optimize, description: basic test for et_sklearn with branched tree_encoding and model_per_response f in model exploration mode optimize adapts test 192 by setting n_estimators 3 and then discrepancy between z3 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test195 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 3 -et_sklearn_bootstrap f -tree_encoding branched -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 157 test type: verify, description: basic nn_keras assertion verification test that uses keras tuner with sequrntial models for model training; adapts test 155 by consdering multiple responses +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test157 -mode verify -resp y1,y2 -feat x,p1,p2 --model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics rmse,logcosh -spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +spec_fn smlp_toy_num_resp_noknobs_verify.spec specs_path ../specs -Running test 196 test type: optimize, description: basic branched tree encoding test for dt_sklearn multi objective pareto optimization when features are not scaled modifies test 164 and test 181 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test196 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 163 test type: verify, description: tests model term formation when mrmr_pred is activated and not all features are selected for training the model +smlp -data "../data/smlp_toy_num_resp_noknobs.csv" -out_dir ./ -pref Test163 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2,asrt3 -asrt_exprs "(y2**3+x2)/2<6;y1>=9;y2<0" -trace_anonym t -trace_prec 3 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_noknobs_pred_labeled.csv" -spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 197 test type: optimize, description: basic branched tree encoding test for dt_sklearn multi objective pareto optimization when responses are not scaled modifies test 164 and test 182 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test197 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -scale_resp f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat"" +Running test 169 test type: optimize, description: basic test for et_sklearn with flat tree_encoding and model_per_response t in model exploration mode optimize +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test169 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -rf_sklearn_n_estimators 3 -et_sklearn_bootstrap f -tree_encoding flat -model_per_response t -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +spec_fn smlp_toy_num_resp_mult_y2_verify.spec specs_path ../specs -Running test 198 test type: optimize, description: basic branched tree encoding test for dt_sklearn multi objective pareto optimization when features and responses are not scaled modifies test 164 and test 183 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test198 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -scale_resp f -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat +Running test 173 test type: verify, description: basic test for nn_keras flat encoding for sequential api +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test173 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3,3,3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics mae -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat"" spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs Running test 199 test type: optimize, description: test to demonstrate that in pareto optimization and optsyn modes with multiple objectives when beta constraints are not present SMLP results are not consistent when different solvers are used; this is due to fact that when a subset of objectoves are exemined in pareto algo -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test199 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 100 -et_sklearn_bootstrap f -tree_encoding branched -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test199 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 100 -et_sklearn_bootstrap f -tree_encoding branched -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn.spec +Running test 224 test type: correlate, description: basic test for correlate mode and tests the Shannon mutual information +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test224 -mode correlate -resp y1,y2 -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method shannon -mrmr_pred 0 -plots f -seed 10 -log_time f + +spec_fn smlp_toy_system_decreasing_knob.05_certify.spec specs_path ../specs -Running test 200 test type: optimize, description: basic test for et_sklearn with branched tree_encoding and model_per_response f in model exploration mode optimize adapts test 170 !!!!!!!!! in this test z3 result differs from mathsat and yices results (the latter two give sma results -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test200 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 100 -et_sklearn_bootstrap f -tree_encoding branched -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 231 test type: certify, description: certification test with monotonicity query with a knob with a grid point +smlp -data "../data/smlp_toy_monotone_basic.csv" -out_dir ./ -pref Test231 -mode certify -spec ../specs/smlp_toy_system_decreasing_knob.05_certify.spec -model system -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_no_input_beta.spec + +spec_fn smlp_toy_num_resp_mult_y2_verify.spec specs_path ../specs -Running test 201 test type: optimize, description: basic dt_sklearn single objective optimization with the eager algorithm when there are no inputs and there are beta constraints -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test201 -mode optimize -pareto t -opt_strategy eager -resp y1,y2 -feat p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_no_input_beta.spec -data_scaler min_max -objv_names obj1 -objv_exprs "(y1+y2)/2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 155 test type: verify, description: basic nn_keras assertion verification test that uses keras tuner with sequrntial models for model training +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test155 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3,3,3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics mae -spec_fn smlp_toy_num_resp_mult_no_input.spec +spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 202 test type: optimize, description: basic dt_sklearn single objective optimization with the eager algorithm when there are no inputs and no beta constraints -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test202 -mode optimize -pareto t -opt_strategy eager -resp y1,y2 -feat p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_no_input.spec -data_scaler min_max -objv_names obj1 -objv_exprs "(y1+y2)/2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 179 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response f nn_keras_seq_api f for nn_keras in model exploration mode optsyn when resposes are not scaled adapts test 174 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test179 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api f -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response f -scale_resp f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_system_stable_constant_synth_fail.spec +spec_fn smlp_toy_num_resp_noknobs_verify.spec specs_path ../specs -Running test 203 test type: optimize, description: optimization test with eager strategy and with constant knob and no inputs where synthesis is not feasible because the assertion is not feasible but beta constraint is feasible therefore optimization is performed adapts test 122 -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test203 -mode optimize -pareto f -opt_strategy eager -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_fail.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 184 test type: verify, description: tests model term formation when mrmr_pred is activated and not all features are selected for training the model adapts test 139 +smlp -data "../data/smlp_toy_num_resp_noknobs.csv" -out_dir ./ -pref Test184 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2,asrt3 -asrt_exprs "(y2**3+x2)/2<6;y1>=9;y2<0" -trace_anonym t -trace_prec 3 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_noknobs_pred_labeled.csv" -spec_fn smlp_toy_system_stable_constant_synth_feasible.spec +spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 204 test type: optimize, description: optimization test with eager strategy and with constant knob and no inputs where synthesis is feasible and optimization is performed adapts test 123 -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test204 -mode optimize -pareto t -opt_strategy eager -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 188 test type: optsyn, description: basic branched tree encoding test for dt_caretin model exploration mode optsyn +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test188 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_caret -tree_encoding branched -save_model f -use_model f -compress_rules f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_system_stable_constant_synth_feasible.spec +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec specs_path ../specs -Running test 205 test type: optimize, description: optimization test with eager strategy and with constant knob and no inputs where synthesis is feasible and optimization is performed adapts test 145 -../../src/run_smlp.py -out_dir ./ -pref Test205 -mode optimize -pareto t -opt_strategy eager -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -doe_spec ../grids/doe_two_levels_opt.csv -doe_algo latin_hypercube -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 197 test type: optimize, description: basic branched tree encoding test for dt_sklearn multi objective pareto optimization when responses are not scaled modifies test 164 and test 182 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test197 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -scale_resp f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat"" + +spec_fn smlp_toy_num_resp_mult_no_input_beta.spec +specs_path ../specs +Running test 201 test type: optimize, description: basic dt_sklearn single objective optimization with the eager algorithm when there are no inputs and there are beta constraints +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test201 -mode optimize -pareto t -opt_strategy eager -resp y1,y2 -feat p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_no_input_beta.spec -data_scaler min_max -objv_names obj1 -objv_exprs "(y1+y2)/2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_system_stable_constant_synth_feasible.spec specs_path ../specs Running test 206 test type: optsyn, description: optimized synthesis test with eager strategy and with constant knob and no inputs where synthesis is feasible and optimization is performed adapts test 125 -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test206 -mode optsyn -pareto t -opt_strategy eager -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -Running test 215 test type: correlate, description: basic test for correlate mode -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test215 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass - -Running test 216 test type: correlate, description: basic test for correlate mode -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test216 -mode correlate -resp y1,y2 -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f - -Running test 217 test type: correlate, description: basic test for correlate mode -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test217 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type category -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass - -Running test 218 test type: correlate, description: basic test for correlate mode -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test218 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type ordered -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass - -Running test 219 test type: correlate, description: basic test for correlate mode -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test219 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type integer -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test206 -mode optsyn -pareto t -opt_strategy eager -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f Running test 220 test type: correlate, description: basic test for correlate mode -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test220 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method normalized -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test220 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method normalized -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass -Running test 221 test type: correlate, description: basic test for correlate mode -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test221 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method shannon -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +Running test 227 test type: correlate, description: basic test for correlate mode and tests the normalized mutual information +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test227 -mode correlate -resp y1,y2 -discr_algo uniform -discret_num t -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method normalized -mrmr_pred 0 -plots f -seed 10 -log_time f -Running test 222 test type: correlate, description: basic test for correlate mode -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test222 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method adjusted -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +Running test 234 test type: subgroups, description: tests subgroup discovery mode when there are two responses with string values +smlp -data "../data/smlp_toy_string_response.csv" -out_dir ./ -pref Test234 -mode subgroups -resp str_resp1,str_resp2 -feat num,int,str -pos_val no -neg_val yes -seed 10 -log_time f -Running test 223 test type: correlate, description: basic test for correlate mode and tests the normalized mutual information -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test223 -mode correlate -resp y1,y2 -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method normalized -mrmr_pred 0 -plots f -seed 10 -log_time f -Running test 224 test type: correlate, description: basic test for correlate mode and tests the Shannon mutual information -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test224 -mode correlate -resp y1,y2 -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method shannon -mrmr_pred 0 -plots f -seed 10 -log_time f +spec_fn smlp_toy_system_stable_constant_synth_feasible.spec +specs_path ../specs +Running test 121 test type: synthesize, description: synthesis test with constant knob and no inputs where synthesis is feasible +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test121 -mode synthesize -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 225 test type: correlate, description: basic test for correlate mode and tests the adjusted mutual information -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test225 -mode correlate -resp y1,y2 -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method adjusted -mrmr_pred 0 -plots f -seed 10 -log_time f +spec_fn smlp_toy_configuration_verify.spec +specs_path ../specs +Running test 129 test type: verify, description: verification example with demonstrating all basic result scenarious for assertions +smlp -data "../data/smlp_toy_ctg_num_resp.csv" -out_dir ./ -pref Test129 -mode verify -resp y1,y2 -feat x,p1,p2 -model poly_sklearn -save_model f -use_model f -model_per_response f -spec ../specs/smlp_toy_configuration_verify.spec -asrt_names assert_stable_config,assert_grid_conflict,assert_unstable_config,assert_infeasible -asrt_exprs "y2<=90;y1>=9;y1>=(-10);y1>20" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 226 test type: correlate, description: basic test for correlate mode -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test226 -mode correlate -resp y1,y2 -discr_algo uniform -discret_num t -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f +spec_fn smlp_toy_system.spec +specs_path ../specs +Running test 146 test type: optimize, description: optimization test with constant knob and no inputs where synthesis is feasible and optimization is performed +smlp -out_dir ./ -pref Test146 -mode optimize -pareto t -opt_strategy lazy -model poly_sklearn -resp y1,y2 -feat p1,p2,x1,x2 -save_model t -use_model f -mrmr_pred 0 -model_per_response t -split 1 -spec ../specs/smlp_toy_system.spec -doe_spec ../grids/explore_doe_two_levels.csv -doe_algo latin_hypercube -epsilon 0.99999999 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 227 test type: correlate, description: basic test for correlate mode and tests the normalized mutual information -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test227 -mode correlate -resp y1,y2 -discr_algo uniform -discret_num t -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method normalized -mrmr_pred 0 -plots f -seed 10 -log_time f +Running test 151 test type: prediction, description: tests msle loss function MeanSquaredLogarithmicError and and sample weoghts +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test151 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss msle -sw_coef 3 -sw_exp 10 -sw_int 0 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -spec_fn smlp_toy_system_radii_update_certify.spec -specs_path ../specs -Running test 228 test type: certify, description: test that radii specified in command line properly override the radii specified in the spec file. Here we override both ansolute and relative radii and one can observe that the certification results also change compared to test 116 -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test228 -mode certify -resp y1,y2 -feat x1,x2,p1,p2 -model system -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_system_radii_update_certify.spec -rad_rel 0.005 -rad_abs 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 158 test type: prediction, description: tests the mape loss function and sample weights with model_per_response t +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test158 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss mape -model_per_response t -sw_coef 8 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics rmse -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -spec_fn smlp_toy_missing_radii.spec +spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 229 test type: certify, description: basic test for checking that each knob must have either absolute or relative radius specified in the spec file (even if radii are specified in the command line) -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test229 -mode certify -resp y1,y2 -feat x1,x2,p1,p2 -model system -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_missing_radii.spec -rad_rel 0.005 -rad_abs 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 165 test type: optsyn, description: basic flat tree encoding test for dt_caretin model exploration mode optsyn +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test165 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_caret -tree_encoding flat -save_model f -use_model f -compress_rules f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_system_monotone_knob.05_verify.spec +spec_fn smlp_toy_num_resp_mult_y2_verify.spec specs_path ../specs -Running test 230 test type: verify, description: tests that outputs in system specificaation might depend on different inuts (knobs and free inputs) -../../src/run_smlp.py -data "../data/smlp_toy_monotone_basic.csv" -out_dir ./ -pref Test230 -mode verify -spec ../specs/smlp_toy_system_monotone_knob.05_verify.spec -model system -seed 10 -log_time f +Running test 172 test type: verify, description: basic test for nn_keras flat encoding for functional api +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test172 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nnet_encoding layered -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3,3,3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat"" -spec_fn smlp_toy_system_decreasing_knob.05_certify.spec +spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 231 test type: certify, description: certification test with monotonicity query with a knob with a grid point -../../src/run_smlp.py -data "../data/smlp_toy_monotone_basic.csv" -out_dir ./ -pref Test231 -mode certify -spec ../specs/smlp_toy_system_decreasing_knob.05_certify.spec -model system -seed 10 -log_time f +Running test 195 test type: optimize, description: basic test for et_sklearn with branched tree_encoding and model_per_response f in model exploration mode optimize adapts test 192 by setting n_estimators 3 and then discrepancy between z3 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test195 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 3 -et_sklearn_bootstrap f -tree_encoding branched -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_system_running_example_certify.spec +spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 232 test type: certify, description: running example from smlp manual -../../src/run_smlp.py -data "../data/smlp_toy_system_running_example_certify.csv" -out_dir ./ -pref Test232 -mode certify -spec ../specs/smlp_toy_system_running_example_certify.spec -model system -seed 10 -log_time f - -Running test 233 test type: subgroups, description: tests subgroup discovery mode when the response has string values -../../src/run_smlp.py -data "../data/smlp_toy_string_response.csv" -out_dir ./ -pref Test233 -mode subgroups -resp str_resp1 -feat num,int,str -pos_val no -neg_val yes -seed 10 -log_time f - -Running test 234 test type: subgroups, description: tests subgroup discovery mode when there are two responses with string values -../../src/run_smlp.py -data "../data/smlp_toy_string_response.csv" -out_dir ./ -pref Test234 -mode subgroups -resp str_resp1,str_resp2 -feat num,int,str -pos_val no -neg_val yes -seed 10 -log_time f +Running test 200 test type: optimize, description: basic test for et_sklearn with branched tree_encoding and model_per_response f in model exploration mode optimize adapts test 170 !!!!!!!!! in this test z3 result differs from mathsat and yices results (the latter two give sma results +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test200 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 100 -et_sklearn_bootstrap f -tree_encoding branched -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Initiating 7 worker... comparing Test1_smlp_toy_num_resp_mult_y1_dt_caret_tree_rules.txt to master Passed! comparing Test1_smlp_toy_num_resp_mult.txt to master @@ -1611,9 +1704,34 @@ comparing Test56_smlp_toy_mult_discr.txt to master Passed! comparing Test56_smlp_toy_mult_discr_missing_values_dict.json to master Passed! -Test 57 Failed: -Error in Build stage: -Data file does not exist +comparing Test57_smlp_toy_num_resp_noknobs_dt_sklearn_tree_rules.txt to master +Passed! +comparing Test57_smlp_toy_num_resp_noknobs.txt to master +Passed! +comparing Test57_smlp_toy_num_resp_noknobs_data_bounds.json to master +Passed! +comparing Test57_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv to master +Passed! +comparing Test57_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv to master +Passed! +comparing Test57_smlp_toy_num_resp_noknobs_missing_values_dict.json to master +Passed! +comparing Test57_smlp_toy_num_resp_noknobs_model_features_dict.json to master +Passed! +comparing Test57_smlp_toy_num_resp_noknobs_model_levels_dict.json to master +Passed! +comparing Test57_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv to master +Passed! +comparing Test57_smlp_toy_num_resp_noknobs_test_predictions_summary.csv to master +Passed! +comparing Test57_smlp_toy_num_resp_noknobs_trace.csv to master +Passed! +comparing Test57_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv to master +Passed! +comparing Test57_smlp_toy_num_resp_noknobs_training_predictions_summary.csv to master +Passed! +comparing Test57_smlp_toy_num_resp_noknobs_verify_results.json to master +Passed! comparing Test58_smlp_toy_num_resp_mult.txt to master Passed! comparing Test58_smlp_toy_num_resp_mult_data_bounds.json to master @@ -1664,158 +1782,415 @@ comparing Test59_smlp_toy_num_resp_mult_model_levels_dict.json to master Passed! comparing Test59_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master Passed! -comparing Test59_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +comparing Test59_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test59_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test59_smlp_toy_num_resp_mult_train-reg_y2_mse.png does not exist +comparing Test59_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test59_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +comparing Test59_smlp_toy_num_resp_mult_verify_results.json to master +Passed! +File master Test59_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist +File master Test59_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist +comparing Test60_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test60_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +comparing Test60_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test60_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test60_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test60_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test60_smlp_toy_num_resp_mult_model_gen.json to master +Passed! +comparing Test60_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test60_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test60_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test60_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test60_smlp_toy_num_resp_mult_train-reg_y2_mse.png does not exist +comparing Test60_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test60_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +comparing Test60_smlp_toy_num_resp_mult_verify_results.json to master +Passed! +File master Test60_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist +File master Test60_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist +comparing Test61_smlp_toy_num_resp_noknobs.txt to master +Passed! +comparing Test61_smlp_toy_num_resp_noknobs_data_bounds.json to master +Passed! +comparing Test61_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv to master +Passed! +comparing Test61_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv to master +Passed! +comparing Test61_smlp_toy_num_resp_noknobs_missing_values_dict.json to master +Passed! +comparing Test61_smlp_toy_num_resp_noknobs_model_features_dict.json to master +Passed! +comparing Test61_smlp_toy_num_resp_noknobs_model_gen.json to master +Passed! +comparing Test61_smlp_toy_num_resp_noknobs_model_levels_dict.json to master +Passed! +comparing Test61_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv to master +Passed! +comparing Test61_smlp_toy_num_resp_noknobs_test_predictions_summary.csv to master +Passed! +comparing Test61_smlp_toy_num_resp_noknobs_trace.csv to master +Passed! +File master Test61_smlp_toy_num_resp_noknobs_train-reg_y1_mse.png does not exist +File master Test61_smlp_toy_num_resp_noknobs_train-reg_y2_mse.png does not exist +comparing Test61_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv to master +Passed! +comparing Test61_smlp_toy_num_resp_noknobs_training_predictions_summary.csv to master +Passed! +comparing Test61_smlp_toy_num_resp_noknobs_verify_results.json to master +Passed! +comparing Test62_smlp_toy_num_resp_noknobs.txt to master +Passed! +comparing Test62_smlp_toy_num_resp_noknobs_data_bounds.json to master +Passed! +comparing Test62_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv to master +Passed! +comparing Test62_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv to master +Passed! +comparing Test62_smlp_toy_num_resp_noknobs_missing_values_dict.json to master +Passed! +comparing Test62_smlp_toy_num_resp_noknobs_model_features_dict.json to master +Passed! +comparing Test62_smlp_toy_num_resp_noknobs_model_gen.json to master +Passed! +comparing Test62_smlp_toy_num_resp_noknobs_model_levels_dict.json to master +Passed! +comparing Test62_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv to master +Passed! +comparing Test62_smlp_toy_num_resp_noknobs_test_predictions_summary.csv to master +Passed! +comparing Test62_smlp_toy_num_resp_noknobs_trace.csv to master +Passed! +File master Test62_smlp_toy_num_resp_noknobs_train-reg_all_responses_mse.png does not exist +comparing Test62_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv to master +Passed! +comparing Test62_smlp_toy_num_resp_noknobs_training_predictions_summary.csv to master +Passed! +comparing Test62_smlp_toy_num_resp_noknobs_verify_results.json to master +Passed! +File master test63_model_dt_sklearn_y1_tree_rules.txt does not exist +comparing Test63_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test63_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test63_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test63_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test63_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test63_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test63_smlp_toy_num_resp_mult_trace.csv does not exist +comparing Test63_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test63_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +comparing Test63_smlp_toy_num_resp_mult_verify_results.json to master +Passed! +comparing test63_model_data_bounds.json to master +Passed! +comparing test63_model_model_features_dict.json to master +Passed! +comparing test63_model_model_levels_dict.json to master +Passed! +comparing test63_model_rerun_model_config.json to master +Passed! +File master test63_model_y1_smlp_full_model_term.json does not exist +File master test63_model_y1_smlp_model_term.json does not exist +comparing Test64_test63_model.txt to master +File master Test64_test63_model_trace.csv does not exist +comparing Test64_test63_model_verify_results.json to master +Passed! +comparing Test65_smlp_toy_num_resp_noknobs.txt to master +Passed! +comparing Test65_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv to master +Passed! +comparing Test65_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv to master +Passed! +comparing Test65_smlp_toy_num_resp_noknobs_missing_values_dict.json to master +Passed! +comparing Test65_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv to master +Passed! +comparing Test65_smlp_toy_num_resp_noknobs_test_predictions_summary.csv to master +Passed! +comparing Test65_smlp_toy_num_resp_noknobs_trace.csv to master +Passed! +comparing Test65_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv to master +Passed! +comparing Test65_smlp_toy_num_resp_noknobs_training_predictions_summary.csv to master +Passed! +comparing Test65_smlp_toy_num_resp_noknobs_verify_results.json to master +Passed! +comparing test65_model_data_bounds.json to master +Passed! +comparing test65_model_dt_sklearn_tree_rules.txt to master +Passed! +comparing test65_model_model_features_dict.json to master +Passed! +comparing test65_model_model_levels_dict.json to master +Passed! +comparing test65_model_rerun_model_config.json to master +Passed! +comparing Test66_test65_model.txt to master +Passed! +comparing Test66_test65_model_trace.csv to master +Passed! +comparing Test66_test65_model_verify_results.json to master +Passed! +comparing test67_model_dt_sklearn_y1_tree_rules.txt to master +Passed! +comparing Test67_smlp_toy_num_resp_noknobs.txt to master +Passed! +comparing Test67_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv to master +Passed! +comparing Test67_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv to master +Passed! +comparing Test67_smlp_toy_num_resp_noknobs_missing_values_dict.json to master +Passed! +comparing Test67_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv to master +Passed! +comparing Test67_smlp_toy_num_resp_noknobs_test_predictions_summary.csv to master +Passed! +comparing Test67_smlp_toy_num_resp_noknobs_trace.csv to master +Passed! +comparing Test67_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv to master +Passed! +comparing Test67_smlp_toy_num_resp_noknobs_training_predictions_summary.csv to master +Passed! +comparing Test67_smlp_toy_num_resp_noknobs_verify_results.json to master +Passed! +comparing test67_model_data_bounds.json to master +Passed! +comparing test67_model_dt_sklearn_y2_tree_rules.txt to master +Passed! +comparing test67_model_model_features_dict.json to master +Passed! +comparing test67_model_model_levels_dict.json to master +Passed! +comparing test67_model_rerun_model_config.json to master +Passed! +comparing Test68_test67_model.txt to master +Passed! +comparing Test68_test67_model_trace.csv to master +Passed! +comparing Test68_test67_model_verify_results.json to master +Passed! +comparing Test69_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test69_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test69_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test69_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test69_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test69_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test69_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test69_smlp_toy_num_resp_mult_train-reg_y2_mse.png does not exist +comparing Test69_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test69_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +comparing Test69_smlp_toy_num_resp_mult_verify_results.json to master +Passed! +comparing test69_model_data_bounds.json to master +Passed! +comparing test69_model_model_features_dict.json to master +Passed! +comparing test69_model_model_gen.json to master +Passed! +comparing test69_model_model_levels_dict.json to master +Passed! +comparing test69_model_rerun_model_config.json to master +Passed! +File master test69_model_y2_smlp_full_model_term.json does not exist +File master test69_model_y2_smlp_model_term.json does not exist +comparing Test70_test69_model.txt to master +File master Test70_test69_model_trace.csv does not exist +comparing Test70_test69_model_verify_results.json to master +Passed! +comparing Test71_smlp_toy_num_resp_noknobs.txt to master +Passed! +comparing Test71_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv to master +Passed! +comparing Test71_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv to master +Passed! +comparing Test71_smlp_toy_num_resp_noknobs_missing_values_dict.json to master +Passed! +comparing Test71_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv to master +Passed! +comparing Test71_smlp_toy_num_resp_noknobs_test_predictions_summary.csv to master +Passed! +comparing Test71_smlp_toy_num_resp_noknobs_trace.csv to master +Passed! +File master Test71_smlp_toy_num_resp_noknobs_train-reg_y1_mse.png does not exist +File master Test71_smlp_toy_num_resp_noknobs_train-reg_y2_mse.png does not exist +comparing Test71_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv to master +Passed! +comparing Test71_smlp_toy_num_resp_noknobs_training_predictions_summary.csv to master +Passed! +comparing Test71_smlp_toy_num_resp_noknobs_verify_results.json to master +Passed! +comparing test71_model_data_bounds.json to master +Passed! +comparing test71_model_model_features_dict.json to master +Passed! +comparing test71_model_model_gen.json to master +Passed! +comparing test71_model_model_levels_dict.json to master +Passed! +comparing test71_model_rerun_model_config.json to master +Passed! +comparing Test72_test71_model.txt to master +Passed! +comparing Test72_test71_model_trace.csv to master +Passed! +comparing Test72_test71_model_verify_results.json to master +Passed! +comparing test73_model_dt_sklearn_y2_tree_rules.txt to master +Passed! +comparing Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled.txt to master +Passed! +comparing Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv to master +Passed! +comparing Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv to master +Passed! +comparing Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json to master +Passed! +comparing Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv to master +Passed! +comparing Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv to master +Passed! +comparing Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv to master +Passed! +comparing Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv to master +Passed! +comparing Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv to master +Passed! +comparing Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv to master +Passed! +comparing Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json to master +Passed! +comparing test73_model_data_bounds.json to master +Passed! +comparing test73_model_dt_sklearn_y1_tree_rules.txt to master +Passed! +comparing test73_model_model_features_dict.json to master +Passed! +comparing test73_model_model_levels_dict.json to master +Passed! +comparing test73_model_rerun_model_config.json to master Passed! -File master Test59_smlp_toy_num_resp_mult_trace.csv does not exist -File master Test59_smlp_toy_num_resp_mult_train-reg_y2_mse.png does not exist -comparing Test59_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +comparing Test74_test73_model_smlp_toy_num_resp_noknobs_pred_labeled.txt to master Passed! -comparing Test59_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +comparing Test74_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json to master Passed! -comparing Test59_smlp_toy_num_resp_mult_verify_results.json to master +comparing Test74_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv to master Passed! -File master Test59_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist -File master Test59_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist -comparing Test60_smlp_toy_num_resp_mult.txt to master +comparing Test74_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv to master Passed! -comparing Test60_smlp_toy_num_resp_mult_data_bounds.json to master +comparing Test74_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv to master Passed! -comparing Test60_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +comparing Test74_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json to master Passed! -comparing Test60_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +comparing Test75_test73_model_smlp_toy_num_resp_noknobs_pred_labeled.txt to master Passed! -comparing Test60_smlp_toy_num_resp_mult_missing_values_dict.json to master +comparing Test75_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json to master Passed! -comparing Test60_smlp_toy_num_resp_mult_model_features_dict.json to master +comparing Test75_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv to master Passed! -comparing Test60_smlp_toy_num_resp_mult_model_gen.json to master +comparing Test75_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv to master Passed! -comparing Test60_smlp_toy_num_resp_mult_model_levels_dict.json to master +comparing Test75_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv to master Passed! -comparing Test60_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +comparing Test75_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json to master Passed! -comparing Test60_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +comparing Test76_smlp_toy_num_resp_noknobs.txt to master Passed! -File master Test60_smlp_toy_num_resp_mult_trace.csv does not exist -File master Test60_smlp_toy_num_resp_mult_train-reg_y2_mse.png does not exist -comparing Test60_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +comparing Test76_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv to master Passed! -comparing Test60_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +comparing Test76_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv to master Passed! -comparing Test60_smlp_toy_num_resp_mult_verify_results.json to master +comparing Test76_smlp_toy_num_resp_noknobs_missing_values_dict.json to master Passed! -File master Test60_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist -File master Test60_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist -Test 61 Failed: -Error in Build stage: -Data file does not exist -Test 62 Failed: -Error in Build stage: -Data file does not exist -File master test63_model_dt_sklearn_y1_tree_rules.txt does not exist -comparing Test63_smlp_toy_num_resp_mult.txt to master +comparing Test76_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv to master Passed! -comparing Test63_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +comparing Test76_smlp_toy_num_resp_noknobs_test_predictions_summary.csv to master Passed! -comparing Test63_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +comparing Test76_smlp_toy_num_resp_noknobs_trace.csv to master Passed! -comparing Test63_smlp_toy_num_resp_mult_missing_values_dict.json to master +comparing Test76_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv to master Passed! -comparing Test63_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +comparing Test76_smlp_toy_num_resp_noknobs_training_predictions_summary.csv to master Passed! -comparing Test63_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +comparing Test76_smlp_toy_num_resp_noknobs_verify_results.json to master Passed! -File master Test63_smlp_toy_num_resp_mult_trace.csv does not exist -comparing Test63_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +comparing test76_model_data_bounds.json to master Passed! -comparing Test63_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +comparing test76_model_dt_sklearn_tree_rules.txt to master Passed! -comparing Test63_smlp_toy_num_resp_mult_verify_results.json to master +comparing test76_model_model_features_dict.json to master Passed! -comparing test63_model_data_bounds.json to master +comparing test76_model_model_levels_dict.json to master Passed! -comparing test63_model_model_features_dict.json to master +comparing test76_model_rerun_model_config.json to master Passed! -comparing test63_model_model_levels_dict.json to master +comparing Test77_test76_model.txt to master Passed! -comparing test63_model_rerun_model_config.json to master +comparing Test77_test76_model_trace.csv to master Passed! -File master test63_model_y1_smlp_full_model_term.json does not exist -File master test63_model_y1_smlp_model_term.json does not exist -comparing Test64_test63_model.txt to master -File master Test64_test63_model_trace.csv does not exist -comparing Test64_test63_model_verify_results.json to master +comparing Test77_test76_model_verify_results.json to master Passed! -Test 65 Failed: -Error in Build stage: -Data file does not exist -comparing Test66_test65_model.txt to master -File new Test66_test65_model_verify_results.json does not exist -Test 67 Failed: -Error in Build stage: -Data file does not exist -comparing Test68_test67_model.txt to master -File new Test68_test67_model_verify_results.json does not exist -comparing Test69_smlp_toy_num_resp_mult.txt to master +comparing test78_model_dt_sklearn_tree_rules.txt to master Passed! -comparing Test69_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +comparing Test78_smlp_toy_num_resp_noknobs.txt to master Passed! -comparing Test69_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +comparing Test78_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv to master Passed! -comparing Test69_smlp_toy_num_resp_mult_missing_values_dict.json to master +comparing Test78_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv to master Passed! -comparing Test69_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +comparing Test78_smlp_toy_num_resp_noknobs_missing_values_dict.json to master Passed! -comparing Test69_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +comparing Test78_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv to master Passed! -File master Test69_smlp_toy_num_resp_mult_trace.csv does not exist -File master Test69_smlp_toy_num_resp_mult_train-reg_y2_mse.png does not exist -comparing Test69_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +comparing Test78_smlp_toy_num_resp_noknobs_test_predictions_summary.csv to master Passed! -comparing Test69_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +comparing Test78_smlp_toy_num_resp_noknobs_trace.csv to master Passed! -comparing Test69_smlp_toy_num_resp_mult_verify_results.json to master +comparing Test78_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv to master Passed! -comparing test69_model_data_bounds.json to master +comparing Test78_smlp_toy_num_resp_noknobs_training_predictions_summary.csv to master Passed! -comparing test69_model_model_features_dict.json to master +comparing Test78_smlp_toy_num_resp_noknobs_verify_results.json to master Passed! -comparing test69_model_model_gen.json to master +comparing test78_model_data_bounds.json to master Passed! -comparing test69_model_model_levels_dict.json to master +comparing test78_model_model_features_dict.json to master Passed! -comparing test69_model_rerun_model_config.json to master +comparing test78_model_model_levels_dict.json to master Passed! -File master test69_model_y2_smlp_full_model_term.json does not exist -File master test69_model_y2_smlp_model_term.json does not exist -comparing Test70_test69_model.txt to master -File master Test70_test69_model_trace.csv does not exist -comparing Test70_test69_model_verify_results.json to master +comparing test78_model_rerun_model_config.json to master Passed! -Test 71 Failed: -Error in Build stage: -Data file does not exist -comparing Test72_test71_model.txt to master -File new Test72_test71_model_verify_results.json does not exist -Test 73 Failed: -Error in Build stage: -Data file does not exist -Error in Build stage: -New data file does not exist -Test 74 Failed: -Error in Build stage: -New data file does not exist -Test 75 Failed: -Error in Build stage: -New data file does not exist -Test 76 Failed: -Error in Build stage: -Data file does not exist -comparing Test77_test76_model.txt to master -File new Test77_test76_model_verify_results.json does not exist -Test 78 Failed: -Error in Build stage: -Data file does not exist comparing Test79_smlp_toy_num_resp_mult.txt to master Passed! comparing Test79_smlp_toy_num_resp_mult_data_bounds.json to master @@ -1979,9 +2354,36 @@ comparing Test83_smlp_toy_num_resp_mult_training_prediction_precisions.csv to ma Passed! comparing Test83_smlp_toy_num_resp_mult_training_predictions_summary.csv to master Passed! -Test 84 Failed: -Error in Build stage: -Data file does not exist +comparing Test84_smlp_toy_num_resp_noknobs_dt_sklearn_tree_rules.txt to master +Passed! +comparing Test84_smlp_toy_num_resp_noknobs.txt to master +Passed! +comparing Test84_smlp_toy_num_resp_noknobs_data_bounds.json to master +Passed! +comparing Test84_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv to master +Passed! +comparing Test84_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv to master +Passed! +comparing Test84_smlp_toy_num_resp_noknobs_missing_values_dict.json to master +Passed! +comparing Test84_smlp_toy_num_resp_noknobs_model_features_dict.json to master +Passed! +comparing Test84_smlp_toy_num_resp_noknobs_model_levels_dict.json to master +Passed! +comparing Test84_smlp_toy_num_resp_noknobs_rerun_model_config.json to master +Passed! +comparing Test84_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv to master +Passed! +comparing Test84_smlp_toy_num_resp_noknobs_test_predictions_summary.csv to master +Passed! +comparing Test84_smlp_toy_num_resp_noknobs_trace.csv to master +Passed! +comparing Test84_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv to master +Passed! +comparing Test84_smlp_toy_num_resp_noknobs_training_predictions_summary.csv to master +Passed! +comparing Test84_smlp_toy_num_resp_noknobs_verify_results.json to master +Passed! comparing Test85_smlp_toy_num_resp_mult_dt_sklearn_tree_rules.txt to master Passed! comparing Test85_smlp_toy_num_resp_mult.txt to master @@ -3110,11 +3512,42 @@ New data file does not exist Test 138 Failed: Error in Build stage: Data file does not exist -Test 139 Failed: -Error in Build stage: -Data file does not exist -Error in Build stage: -New data file does not exist +comparing Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled.txt to master +Passed! +comparing Test139_smlp_toy_num_resp_noknobs_data_bounds.json to master +Passed! +comparing Test139_smlp_toy_num_resp_noknobs_dt_sklearn_y1_tree_rules.txt to master +Passed! +comparing Test139_smlp_toy_num_resp_noknobs_dt_sklearn_y2_tree_rules.txt to master +Passed! +comparing Test139_smlp_toy_num_resp_noknobs_model_features_dict.json to master +Passed! +comparing Test139_smlp_toy_num_resp_noknobs_model_levels_dict.json to master +Passed! +comparing Test139_smlp_toy_num_resp_noknobs_rerun_model_config.json to master +Passed! +comparing Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv to master +Passed! +comparing Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv to master +Passed! +comparing Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json to master +Passed! +comparing Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv to master +Passed! +comparing Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv to master +Passed! +comparing Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv to master +Passed! +comparing Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv to master +Passed! +comparing Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv to master +Passed! +comparing Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv to master +Passed! +comparing Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json to master +Passed! File master Test140_smlp_toy_basic.txt does not exist File master Test140_smlp_toy_basic_data_bounds.json does not exist File master Test140_smlp_toy_basic_features_scaler.pkl does not exist @@ -3189,9 +3622,34 @@ File master Test143_smlp_toy_num_resp_mult_y1_smlp_full_model_term.json does not File master Test143_smlp_toy_num_resp_mult_y1_smlp_model_term.json does not exist File master Test143_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist File master Test143_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist -Test 144 Failed: -Error in Build stage: -Data file does not exist +comparing Test144_smlp_toy_num_resp_noknobs_dt_sklearn_tree_rules.txt to master +Passed! +comparing Test144_smlp_toy_num_resp_noknobs.txt to master +Passed! +comparing Test144_smlp_toy_num_resp_noknobs_data_bounds.json to master +Passed! +comparing Test144_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv to master +Passed! +comparing Test144_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv to master +Passed! +comparing Test144_smlp_toy_num_resp_noknobs_missing_values_dict.json to master +Passed! +comparing Test144_smlp_toy_num_resp_noknobs_model_features_dict.json to master +Passed! +comparing Test144_smlp_toy_num_resp_noknobs_model_levels_dict.json to master +Passed! +comparing Test144_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv to master +Passed! +comparing Test144_smlp_toy_num_resp_noknobs_test_predictions_summary.csv to master +Passed! +comparing Test144_smlp_toy_num_resp_noknobs_trace.csv to master +Passed! +comparing Test144_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv to master +Passed! +comparing Test144_smlp_toy_num_resp_noknobs_training_predictions_summary.csv to master +Passed! +comparing Test144_smlp_toy_num_resp_noknobs_verify_results.json to master +Passed! File master Test145_doe_two_levels_opt.txt does not exist File master Test145_doe_two_levels_opt_trace.csv does not exist File master Test146_explore_doe_two_levels.txt does not exist @@ -3489,16 +3947,76 @@ File master Test161_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_t File master Test161_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_train-reg_all_responses_mape.png does not exist File master Test161_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_prediction_precisions.csv does not exist File master Test161_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_predictions_summary.csv does not exist -Test 162 Failed: -Error in Build stage: -Data file does not exist -Error in Build stage: -New data file does not exist -Test 163 Failed: -Error in Build stage: -Data file does not exist -Error in Build stage: -New data file does not exist +comparing Test162_smlp_toy_num_resp_noknobs_dt_sklearn_y2_tree_rules.txt to master +Passed! +comparing Test162_smlp_toy_num_resp_noknobs_data_bounds.json to master +Passed! +comparing Test162_smlp_toy_num_resp_noknobs_dt_sklearn_y1_tree_rules.txt to master +Passed! +comparing Test162_smlp_toy_num_resp_noknobs_model_features_dict.json to master +Passed! +comparing Test162_smlp_toy_num_resp_noknobs_model_levels_dict.json to master +Passed! +comparing Test162_smlp_toy_num_resp_noknobs_rerun_model_config.json to master +Passed! +comparing Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled.txt to master +Passed! +comparing Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv to master +Passed! +comparing Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv to master +Passed! +comparing Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json to master +Passed! +comparing Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv to master +Passed! +comparing Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv to master +Passed! +comparing Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv to master +Passed! +comparing Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv to master +Passed! +comparing Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv to master +Passed! +comparing Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv to master +Passed! +comparing Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json to master +Passed! +comparing Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled.txt to master +Passed! +comparing Test163_smlp_toy_num_resp_noknobs_data_bounds.json to master +Passed! +comparing Test163_smlp_toy_num_resp_noknobs_dt_sklearn_tree_rules.txt to master +Passed! +comparing Test163_smlp_toy_num_resp_noknobs_model_features_dict.json to master +Passed! +comparing Test163_smlp_toy_num_resp_noknobs_model_levels_dict.json to master +Passed! +comparing Test163_smlp_toy_num_resp_noknobs_rerun_model_config.json to master +Passed! +comparing Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv to master +Passed! +comparing Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv to master +Passed! +comparing Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json to master +Passed! +comparing Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv to master +Passed! +comparing Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv to master +Passed! +comparing Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv to master +Passed! +comparing Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv to master +Passed! +comparing Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv to master +Passed! +comparing Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv to master +Passed! +comparing Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json to master +Passed! File master Test164_smlp_toy_num_resp_mult.txt does not exist File master Test164_smlp_toy_num_resp_mult_data_bounds.json does not exist File master Test164_smlp_toy_num_resp_mult_dt_sklearn_model_complete.pkl does not exist @@ -3945,21 +4463,112 @@ File master Test183_smlp_toy_num_resp_mult_test_predictions_summary.csv does not File master Test183_smlp_toy_num_resp_mult_trace.csv does not exist File master Test183_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist File master Test183_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist -Test 184 Failed: -Error in Build stage: -Data file does not exist -Error in Build stage: -New data file does not exist -Test 185 Failed: -Error in Build stage: -Data file does not exist -Error in Build stage: -New data file does not exist -Test 186 Failed: -Error in Build stage: -Data file does not exist -Error in Build stage: -New data file does not exist +comparing Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled.txt to master +Passed! +comparing Test184_smlp_toy_num_resp_noknobs_data_bounds.json to master +Passed! +comparing Test184_smlp_toy_num_resp_noknobs_dt_sklearn_y1_tree_rules.txt to master +Passed! +comparing Test184_smlp_toy_num_resp_noknobs_dt_sklearn_y2_tree_rules.txt to master +Passed! +comparing Test184_smlp_toy_num_resp_noknobs_model_features_dict.json to master +Passed! +comparing Test184_smlp_toy_num_resp_noknobs_model_levels_dict.json to master +Passed! +comparing Test184_smlp_toy_num_resp_noknobs_rerun_model_config.json to master +Passed! +comparing Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv to master +Passed! +comparing Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv to master +Passed! +comparing Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json to master +Passed! +comparing Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv to master +Passed! +comparing Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv to master +Passed! +comparing Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv to master +Passed! +comparing Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv to master +Passed! +comparing Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv to master +Passed! +comparing Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv to master +Passed! +comparing Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json to master +Passed! +comparing Test185_smlp_toy_num_resp_noknobs_dt_sklearn_y1_tree_rules.txt to master +Passed! +comparing Test185_smlp_toy_num_resp_noknobs_data_bounds.json to master +Passed! +comparing Test185_smlp_toy_num_resp_noknobs_dt_sklearn_y2_tree_rules.txt to master +Passed! +comparing Test185_smlp_toy_num_resp_noknobs_model_features_dict.json to master +Passed! +comparing Test185_smlp_toy_num_resp_noknobs_model_levels_dict.json to master +Passed! +comparing Test185_smlp_toy_num_resp_noknobs_rerun_model_config.json to master +Passed! +comparing Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled.txt to master +Passed! +comparing Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv to master +Passed! +comparing Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv to master +Passed! +comparing Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json to master +Passed! +comparing Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv to master +Passed! +comparing Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv to master +Passed! +comparing Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv to master +Passed! +comparing Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv to master +Passed! +comparing Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv to master +Passed! +comparing Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv to master +Passed! +comparing Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json to master +Passed! +comparing Test186_smlp_toy_num_resp_noknobs_dt_sklearn_tree_rules.txt to master +Passed! +comparing Test186_smlp_toy_num_resp_noknobs_data_bounds.json to master +Passed! +comparing Test186_smlp_toy_num_resp_noknobs_model_features_dict.json to master +Passed! +comparing Test186_smlp_toy_num_resp_noknobs_model_levels_dict.json to master +Passed! +comparing Test186_smlp_toy_num_resp_noknobs_rerun_model_config.json to master +Passed! +comparing Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled.txt to master +Passed! +comparing Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv to master +Passed! +comparing Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv to master +Passed! +comparing Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json to master +Passed! +comparing Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv to master +Passed! +comparing Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv to master +Passed! +comparing Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv to master +Passed! +comparing Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv to master +Passed! +comparing Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv to master +Passed! +comparing Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv to master +Passed! +comparing Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json to master +Passed! File master Test187_smlp_toy_num_resp_mult.txt does not exist File master Test187_smlp_toy_num_resp_mult_data_bounds.json does not exist File master Test187_smlp_toy_num_resp_mult_dt_sklearn_model_complete.pkl does not exist @@ -4429,5 +5038,5 @@ Passed! master log file does not exist! Do you wish to copy the new log file to master? (yes/no|y/n): No new tests crashed (not in the masters) -Time: 35.39404908021291 minutes +Time: 28.481274580955507 minutes End of regression diff --git a/tests/smlp_regression/run_smlp_regression_expected_diff_report.log b/tests/smlp_regression/run_smlp_regression_expected_diff_report.log index 952a5d6a..125c558f 100644 --- a/tests/smlp_regression/run_smlp_regression_expected_diff_report.log +++ b/tests/smlp_regression/run_smlp_regression_expected_diff_report.log @@ -386,424 +386,12 @@ --- > smlp_logger - INFO - Seving model rerun configuration in file ./../models/test63_model_rerun_model_config.json =================== End of Test64_test63_model.txt diff report ================================ -=================== Diff report for: Test66_test65_model.txt ================================== -0a1,97 -> -> smlp_logger - INFO - Model exploration specification: -> {'version': '1.1', 'spec': [{'label': 'y1', 'type': 'response', 'range': 'float'}, {'label': 'y2', 'type': 'response', 'range': 'float'}, {'label': 'x0', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x1', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x2', 'type': 'input', 'range': 'float', 'bounds': [3, 7]}], 'alpha': 'x1==1 or x1==4 or x1==7'} -> -> smlp_logger - INFO - Executing run_smlp.py script: Start -> -> smlp_logger - INFO - Running SMLP in mode "verify": Start -> -> smlp_logger - INFO - Computed spec global constraint expressions: -> -> smlp_logger - INFO - Global alpha : x1==1 or x1==4 or x1==7 -> -> smlp_logger - INFO - Global beta : None -> -> smlp_logger - INFO - Radii theta : {} -> -> smlp_logger - INFO - Delta const : {'delta_abs': 0.0, 'delta_rel': 0.01} -> -> smlp_logger - INFO - Assertion asrt1: x0**2+y1>4.3 -> -> smlp_logger - INFO - Assertion asrt2: (y1+x2)/2<6 -> -> smlp_logger - INFO - PREPARE DATA FOR MODELING -> -> smlp_logger - INFO - LOAD TRAINED MODEL -> -> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test65_model_rerun_model_config.json -> -> smlp_logger - INFO - Creating model exploration base components: Start -> -> smlp_logger - INFO - Parsing the SPEC: Start -> -> smlp_logger - INFO - Parsing the SPEC: End -> -> smlp_logger - INFO - Variable domains (alpha): {'y1': {'range': 'float', 'interval': None}, 'y2': {'range': 'float', 'interval': None}, 'x0': {'range': 'float', 'interval': [0, 10]}, 'x1': {'range': 'float', 'interval': [0, 10]}, 'x2': {'range': 'float', 'interval': [3, 7]}} -> -> smlp_logger - INFO - Input bounds (alpha): {'x0': {'min': 0, 'max': 10}, 'x1': {'min': 0, 'max': 10}, 'x2': {'min': 3, 'max': 7}} -> -> smlp_logger - INFO - Knob bounds (eta): {} -> -> smlp_logger - INFO - Knob grids (eta): {} -> -> smlp_logger - INFO - Alpha global constraints: (or (or (= x1 1) (= x1 4)) (= x1 7)) -> -> smlp_logger - INFO - Alpha ranges constraints: (and (and (and true (and (>= x0 0) (<= x0 10))) (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) -> -> smlp_logger - INFO - Alpha combined constraints: (and (and (and (and true (and (>= x0 0) (<= x0 10))) (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) (or (or (= x1 1) (= x1 4)) (= x1 7))) -> -> smlp_logger - INFO - Beta global constraints: true -> -> smlp_logger - INFO - Eta ranges constraints: true -> -> smlp_logger - INFO - Eta grid constraints: true -> -> smlp_logger - INFO - Eta global constraints: true -> -> smlp_logger - INFO - Eta combined constraints: true -> -> smlp_logger - INFO - Creating model exploration base components: End -> -> smlp_logger - INFO - Input and knob interface constraints are consistent -> -> smlp_logger - INFO - Building model terms: Start -> -> smlp_logger - INFO - Model operator counts for y1: {'add': 1, 'mul': 15, 'ite': 5, 'and': 9, 'prop': 14, 'const': 50, 'sub': 14, 'var': 14} -> -> smlp_logger - INFO - Model operator counts for y2: {'add': 1, 'mul': 15, 'ite': 5, 'and': 9, 'prop': 14, 'const': 50, 'sub': 14, 'var': 14} -> -> smlp_logger - INFO - Building model terms: End -> -> smlp_logger - INFO - Model interface constraints are consistent -> -> smlp_logger - INFO - Verifying consistency of configuration for assertion asrt1: -> true -> -> smlp_logger - INFO - Input, knob and configuration constraints are consistent -> -> smlp_logger - INFO - Verifying consistency of configuration for assertion asrt2: -> true -> -> smlp_logger - INFO - Input, knob and configuration constraints are consistent -> -> smlp_logger - INFO - Verifying assertion asrt1 <-> x0**2+y1>4.3 -> -> smlp_logger - INFO - The configuration is consistent with assertion asrt1 -> -> smlp_logger - INFO - Completed with result: PASS -> -> smlp_logger - INFO - Verifying assertion asrt2 <-> (y1+x2)/2<6 -> -> smlp_logger - INFO - The configuration is consistent with assertion asrt2 -> -> smlp_logger - INFO - Completed with result: FAIL -> -> smlp_logger - INFO - Running SMLP in mode "verify": End -> -> smlp_logger - INFO - Executing run_smlp.py script: End -=================== End of Test66_test65_model.txt diff report ================================ -=================== Diff report for: Test66_test65_model_verify_results.json ================================== -diff: /app/smlp/regr_smlp/code/Test66_test65_model_verify_results.json: No such file or directory -=================== End of Test66_test65_model_verify_results.json diff report ================================ -=================== Diff report for: Test68_test67_model.txt ================================== -0a1,97 -> -> smlp_logger - INFO - Model exploration specification: -> {'version': '1.1', 'spec': [{'label': 'y1', 'type': 'response', 'range': 'float'}, {'label': 'y2', 'type': 'response', 'range': 'float'}, {'label': 'x0', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x1', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x2', 'type': 'input', 'range': 'float', 'bounds': [3, 7]}], 'alpha': 'x1==1 or x1==4 or x1==7'} -> -> smlp_logger - INFO - Executing run_smlp.py script: Start -> -> smlp_logger - INFO - Running SMLP in mode "verify": Start -> -> smlp_logger - INFO - Computed spec global constraint expressions: -> -> smlp_logger - INFO - Global alpha : x1==1 or x1==4 or x1==7 -> -> smlp_logger - INFO - Global beta : None -> -> smlp_logger - INFO - Radii theta : {} -> -> smlp_logger - INFO - Delta const : {'delta_abs': 0.0, 'delta_rel': 0.01} -> -> smlp_logger - INFO - Assertion asrt1: x0**2+y1>4.3 -> -> smlp_logger - INFO - Assertion asrt2: (y1+x2)/2<6 -> -> smlp_logger - INFO - PREPARE DATA FOR MODELING -> -> smlp_logger - INFO - LOAD TRAINED MODEL -> -> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test67_model_rerun_model_config.json -> -> smlp_logger - INFO - Creating model exploration base components: Start -> -> smlp_logger - INFO - Parsing the SPEC: Start -> -> smlp_logger - INFO - Parsing the SPEC: End -> -> smlp_logger - INFO - Variable domains (alpha): {'y1': {'range': 'float', 'interval': None}, 'y2': {'range': 'float', 'interval': None}, 'x0': {'range': 'float', 'interval': [0, 10]}, 'x1': {'range': 'float', 'interval': [0, 10]}, 'x2': {'range': 'float', 'interval': [3, 7]}} -> -> smlp_logger - INFO - Input bounds (alpha): {'x0': {'min': 0, 'max': 10}, 'x1': {'min': 0, 'max': 10}, 'x2': {'min': 3, 'max': 7}} -> -> smlp_logger - INFO - Knob bounds (eta): {} -> -> smlp_logger - INFO - Knob grids (eta): {} -> -> smlp_logger - INFO - Alpha global constraints: (or (or (= x1 1) (= x1 4)) (= x1 7)) -> -> smlp_logger - INFO - Alpha ranges constraints: (and (and (and true (and (>= x0 0) (<= x0 10))) (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) -> -> smlp_logger - INFO - Alpha combined constraints: (and (and (and (and true (and (>= x0 0) (<= x0 10))) (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) (or (or (= x1 1) (= x1 4)) (= x1 7))) -> -> smlp_logger - INFO - Beta global constraints: true -> -> smlp_logger - INFO - Eta ranges constraints: true -> -> smlp_logger - INFO - Eta grid constraints: true -> -> smlp_logger - INFO - Eta global constraints: true -> -> smlp_logger - INFO - Eta combined constraints: true -> -> smlp_logger - INFO - Creating model exploration base components: End -> -> smlp_logger - INFO - Input and knob interface constraints are consistent -> -> smlp_logger - INFO - Building model terms: Start -> -> smlp_logger - INFO - Model operator counts for y1: {'add': 1, 'mul': 7, 'ite': 3, 'and': 3, 'prop': 6, 'const': 24, 'sub': 6, 'var': 6} -> -> smlp_logger - INFO - Model operator counts for y2: {'add': 1, 'mul': 4, 'ite': 2, 'and': 1, 'prop': 3, 'const': 14, 'sub': 3, 'var': 3} -> -> smlp_logger - INFO - Building model terms: End -> -> smlp_logger - INFO - Model interface constraints are consistent -> -> smlp_logger - INFO - Verifying consistency of configuration for assertion asrt1: -> true -> -> smlp_logger - INFO - Input, knob and configuration constraints are consistent -> -> smlp_logger - INFO - Verifying consistency of configuration for assertion asrt2: -> true -> -> smlp_logger - INFO - Input, knob and configuration constraints are consistent -> -> smlp_logger - INFO - Verifying assertion asrt1 <-> x0**2+y1>4.3 -> -> smlp_logger - INFO - The configuration is consistent with assertion asrt1 -> -> smlp_logger - INFO - Completed with result: PASS -> -> smlp_logger - INFO - Verifying assertion asrt2 <-> (y1+x2)/2<6 -> -> smlp_logger - INFO - The configuration is consistent with assertion asrt2 -> -> smlp_logger - INFO - Completed with result: FAIL -> -> smlp_logger - INFO - Running SMLP in mode "verify": End -> -> smlp_logger - INFO - Executing run_smlp.py script: End -=================== End of Test68_test67_model.txt diff report ================================ -=================== Diff report for: Test68_test67_model_verify_results.json ================================== -diff: /app/smlp/regr_smlp/code/Test68_test67_model_verify_results.json: No such file or directory -=================== End of Test68_test67_model_verify_results.json diff report ================================ =================== Diff report for: Test70_test69_model.txt ================================== 25c25 < smlp_logger - INFO - Seving model rerun configuration in file ../models/test69_model_rerun_model_config.json --- > smlp_logger - INFO - Seving model rerun configuration in file ./../models/test69_model_rerun_model_config.json =================== End of Test70_test69_model.txt diff report ================================ -=================== Diff report for: Test72_test71_model.txt ================================== -0a1,84 -> -> smlp_logger - INFO - Model exploration specification: -> {'version': '1.1', 'spec': [{'label': 'y1', 'type': 'response', 'range': 'float'}, {'label': 'y2', 'type': 'response', 'range': 'float'}, {'label': 'x0', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x1', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x2', 'type': 'input', 'range': 'float', 'bounds': [3, 7]}], 'alpha': 'x1==1 or x1==4 or x1==7'} -> -> smlp_logger - INFO - Executing run_smlp.py script: Start -> -> smlp_logger - INFO - Running SMLP in mode "verify": Start -> -> smlp_logger - INFO - Computed spec global constraint expressions: -> -> smlp_logger - INFO - Global alpha : x1==1 or x1==4 or x1==7 -> -> smlp_logger - INFO - Global beta : None -> -> smlp_logger - INFO - Radii theta : {} -> -> smlp_logger - INFO - Delta const : {'delta_abs': 0.0, 'delta_rel': 0.01} -> -> smlp_logger - INFO - Assertion asrt1: (y2**3+x2)/2<6 -> -> smlp_logger - INFO - PREPARE DATA FOR MODELING -> -> smlp_logger - INFO - LOAD TRAINED MODEL -> -> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test71_model_rerun_model_config.json -> -> smlp_logger - INFO - Creating model exploration base components: Start -> -> smlp_logger - INFO - Parsing the SPEC: Start -> -> smlp_logger - INFO - Parsing the SPEC: End -> -> smlp_logger - INFO - Variable domains (alpha): {'y1': {'range': 'float', 'interval': None}, 'y2': {'range': 'float', 'interval': None}, 'x0': {'range': 'float', 'interval': [0, 10]}, 'x1': {'range': 'float', 'interval': [0, 10]}, 'x2': {'range': 'float', 'interval': [3, 7]}} -> -> smlp_logger - INFO - Input bounds (alpha): {'x0': {'min': 0, 'max': 10}, 'x1': {'min': 0, 'max': 10}, 'x2': {'min': 3, 'max': 7}} -> -> smlp_logger - INFO - Knob bounds (eta): {} -> -> smlp_logger - INFO - Knob grids (eta): {} -> -> smlp_logger - INFO - Alpha global constraints: (or (or (= x1 1) (= x1 4)) (= x1 7)) -> -> smlp_logger - INFO - Alpha ranges constraints: (and (and (and true (and (>= x0 0) (<= x0 10))) (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) -> -> smlp_logger - INFO - Alpha combined constraints: (and (and (and (and true (and (>= x0 0) (<= x0 10))) (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) (or (or (= x1 1) (= x1 4)) (= x1 7))) -> -> smlp_logger - INFO - Beta global constraints: true -> -> smlp_logger - INFO - Eta ranges constraints: true -> -> smlp_logger - INFO - Eta grid constraints: true -> -> smlp_logger - INFO - Eta global constraints: true -> -> smlp_logger - INFO - Eta combined constraints: true -> -> smlp_logger - INFO - Creating model exploration base components: End -> -> smlp_logger - INFO - Input and knob interface constraints are consistent -> -> smlp_logger - INFO - Building model terms: Start -> -> smlp_logger - INFO - Model operator counts for y1: {'add': 256, 'mul': 472, 'ite': 39, 'prop': 39, 'const': 846, 'sub': 216, 'var': 216} -> -> smlp_logger - INFO - Model operator counts for y2: {'add': 256, 'mul': 472, 'ite': 39, 'prop': 39, 'const': 846, 'sub': 216, 'var': 216} -> -> smlp_logger - INFO - Building model terms: End -> -> smlp_logger - INFO - Model interface constraints are consistent -> -> smlp_logger - INFO - Verifying consistency of configuration for assertion asrt1: -> true -> -> smlp_logger - INFO - Input, knob and configuration constraints are consistent -> -> smlp_logger - INFO - Verifying assertion asrt1 <-> (y2**3+x2)/2<6 -> -> smlp_logger - INFO - The configuration is consistent with assertion asrt1 -> -> smlp_logger - INFO - Completed with result: FAIL -> -> smlp_logger - INFO - Running SMLP in mode "verify": End -> -> smlp_logger - INFO - Executing run_smlp.py script: End -=================== End of Test72_test71_model.txt diff report ================================ -=================== Diff report for: Test72_test71_model_verify_results.json ================================== -diff: /app/smlp/regr_smlp/code/Test72_test71_model_verify_results.json: No such file or directory -=================== End of Test72_test71_model_verify_results.json diff report ================================ -=================== Diff report for: Test77_test76_model.txt ================================== -0a1,110 -> -> smlp_logger - INFO - Model exploration specification: -> {'version': '1.1', 'spec': [{'label': 'y1', 'type': 'response', 'range': 'float'}, {'label': 'y2', 'type': 'response', 'range': 'float'}, {'label': 'x0', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x1', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x2', 'type': 'input', 'range': 'float', 'bounds': [3, 7]}], 'alpha': 'x1==1 or x1==4 or x1==7'} -> -> smlp_logger - INFO - Executing run_smlp.py script: Start -> -> smlp_logger - INFO - Running SMLP in mode "verify": Start -> -> smlp_logger - INFO - Computed spec global constraint expressions: -> -> smlp_logger - INFO - Global alpha : x1==1 or x1==4 or x1==7 -> -> smlp_logger - INFO - Global beta : None -> -> smlp_logger - INFO - Radii theta : {} -> -> smlp_logger - INFO - Delta const : {'delta_abs': 0.0, 'delta_rel': 0.01} -> -> smlp_logger - INFO - Assertion asrt1: (y2**3+x2)/2<6 -> -> smlp_logger - INFO - Assertion asrt2: y1>=9 -> -> smlp_logger - INFO - Assertion asrt3: y2<0 -> -> smlp_logger - INFO - PREPARE DATA FOR MODELING -> -> smlp_logger - INFO - LOAD TRAINED MODEL -> -> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test76_model_rerun_model_config.json -> -> smlp_logger - INFO - Creating model exploration base components: Start -> -> smlp_logger - INFO - Parsing the SPEC: Start -> -> smlp_logger - INFO - Parsing the SPEC: End -> -> smlp_logger - INFO - Variable domains (alpha): {'y1': {'range': 'float', 'interval': None}, 'y2': {'range': 'float', 'interval': None}, 'x0': {'range': 'float', 'interval': [0, 10]}, 'x1': {'range': 'float', 'interval': [0, 10]}, 'x2': {'range': 'float', 'interval': [3, 7]}} -> -> smlp_logger - INFO - Input bounds (alpha): {'x0': {'min': 0, 'max': 10}, 'x1': {'min': 0, 'max': 10}, 'x2': {'min': 3, 'max': 7}} -> -> smlp_logger - INFO - Knob bounds (eta): {} -> -> smlp_logger - INFO - Knob grids (eta): {} -> -> smlp_logger - INFO - Alpha global constraints: (or (or (= x1 1) (= x1 4)) (= x1 7)) -> -> smlp_logger - INFO - Alpha ranges constraints: (and (and true (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) -> -> smlp_logger - INFO - Alpha combined constraints: (and (and (and true (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) (or (or (= x1 1) (= x1 4)) (= x1 7))) -> -> smlp_logger - INFO - Beta global constraints: true -> -> smlp_logger - INFO - Eta ranges constraints: true -> -> smlp_logger - INFO - Eta grid constraints: true -> -> smlp_logger - INFO - Eta global constraints: true -> -> smlp_logger - INFO - Eta combined constraints: true -> -> smlp_logger - INFO - Creating model exploration base components: End -> -> smlp_logger - INFO - Input and knob interface constraints are consistent -> -> smlp_logger - INFO - Building model terms: Start -> -> smlp_logger - INFO - Model operator counts for y1: {'add': 1, 'mul': 21, 'ite': 6, 'and': 14, 'prop': 20, 'const': 69, 'sub': 20, 'var': 20} -> -> smlp_logger - INFO - Model operator counts for y2: {'add': 1, 'mul': 21, 'ite': 6, 'and': 14, 'prop': 20, 'const': 69, 'sub': 20, 'var': 20} -> -> smlp_logger - INFO - Building model terms: End -> -> smlp_logger - INFO - Model interface constraints are consistent -> -> smlp_logger - INFO - Verifying consistency of configuration for assertion asrt1: -> true -> -> smlp_logger - INFO - Input, knob and configuration constraints are consistent -> -> smlp_logger - INFO - Verifying consistency of configuration for assertion asrt2: -> true -> -> smlp_logger - INFO - Input, knob and configuration constraints are consistent -> -> smlp_logger - INFO - Verifying consistency of configuration for assertion asrt3: -> true -> -> smlp_logger - INFO - Input, knob and configuration constraints are consistent -> -> smlp_logger - INFO - Verifying assertion asrt1 <-> (y2**3+x2)/2<6 -> -> smlp_logger - INFO - The configuration is inconsistent with assertion asrt1 -> -> smlp_logger - INFO - Completed with result: FAIL -> -> smlp_logger - INFO - Verifying assertion asrt2 <-> y1>=9 -> -> smlp_logger - INFO - The configuration is consistent with assertion asrt2 -> -> smlp_logger - INFO - Completed with result: FAIL -> -> smlp_logger - INFO - Verifying assertion asrt3 <-> y2<0 -> -> smlp_logger - INFO - The configuration is inconsistent with assertion asrt3 -> -> smlp_logger - INFO - Completed with result: FAIL -> -> smlp_logger - INFO - Running SMLP in mode "verify": End -> -> smlp_logger - INFO - Executing run_smlp.py script: End -=================== End of Test77_test76_model.txt diff report ================================ -=================== Diff report for: Test77_test76_model_verify_results.json ================================== -diff: /app/smlp/regr_smlp/code/Test77_test76_model_verify_results.json: No such file or directory -=================== End of Test77_test76_model_verify_results.json diff report ================================ =================== Diff report for: Test97_smlp_toy_num_resp_mult.txt ================================== 252c252 < smlp_logger - INFO - Model operator counts for y2: {'add': 100, 'mul': 715, 'const': 2547, 'ite': 305, 'and': 408, 'prop': 713, 'sub': 713, 'var': 713} @@ -817,7 +405,7 @@ diff: /app/smlp/regr_smlp/code/Test77_test76_model_verify_results.json: No such > smlp_logger - INFO - Seving model rerun configuration in file ./../models/test101_model_rerun_model_config.json =================== End of Test102_test101_model.txt diff report ================================ =================== Diff report for: test110_model_poly_sklearn_formula.txt ================================== -diff: /app/smlp/regr_smlp/code/test110_model_poly_sklearn_formula.txt: No such file or directory +diff: /home/testuser/smlp/regr_smlp/code/test110_model_poly_sklearn_formula.txt: No such file or directory =================== End of test110_model_poly_sklearn_formula.txt diff report ================================ =================== Diff report for: Test111_test110_model_smlp_toy_basic_pred_unlabeled.txt ================================== 79c79 diff --git a/tests/smlp_regression/run_smlp_regression_venv_expected.log b/tests/smlp_regression/run_smlp_regression_venv_expected.log index ce736b01..0779c92b 100644 --- a/tests/smlp_regression/run_smlp_regression_venv_expected.log +++ b/tests/smlp_regression/run_smlp_regression_venv_expected.log @@ -7,822 +7,915 @@ Initiating 4 worker... Initiating 5 worker... Initiating 6 worker... Running test 7 test type: prediction, description: basic rf_sklearn prediction test on labeled and new data with numeric labels -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test7 -mode predict -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 15 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test7 -mode predict -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 15 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" Running test 11 test type: prediction, description: basic poly_sklearn prediction test on labeled and new data with numeric labels -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test11 -mode predict -resp y1,y2 -feat x,p1,p2 -model poly_sklearn -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test11 -mode predict -resp y1,y2 -feat x,p1,p2 -model poly_sklearn -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 16 test type: prediction, description: basic nn_keras prediction test from saved model on new data with numeric labels and two responses -../../src/run_smlp.py -model_name "../models/Test8_smlp_toy_num_resp_mult" -out_dir ./ -pref Test16 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -save_model f -use_model t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 18 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels and saving model using name specified through model_name option - adapts Test6 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test18 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model t -use_model f -model_name test19_model -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 25 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels and saving model using name specified through model_name option - adapts Test6 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test25 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model t -use_model f -model_name test26_model -mrmr_pred 2 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 26 test type: prediction, description: basic dt_sklearn prediction test using a model saved under a name specified through model_name option on new data with numeric labels +smlp -model_name "../models/test26_model" -out_dir ./ -pref Test26 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model t -mrmr_pred 2 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 36 test type: doe, description: doe test with four levels with sukharev_grid -../../src/run_smlp.py -doe_spec "../grids/doe_four_levels_real.csv" -out_dir ./ -pref Test36 -mode doe -doe_algo sukharev_grid -doe_samples 125 -log_time f +Running test 32 test type: unknown, description: test reusing saved model by using configuration file +smlp -model_name "../models/test20_model" -out_dir ./ -pref Test32 -config ../models/test20_model_rerun_model_config.json -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 44 test type: doe, description: doe test with four levels with uniform_random_matrix -../../src/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test44 -mode doe -doe_algo uniform_random_matrix -doe_samples 20 -log_time f +Running test 37 test type: doe, description: doe test with four levels with box_behnken +smlp -doe_spec "../grids/doe_three_levels_real_nan.csv" -out_dir ./ -pref Test37 -mode doe -doe_algo box_behnken -log_time f -Running test 52 test type: discretization, description: tests discretization options -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test52 -mode discretize -resp "PF,PF1" -discr_algo jenks -discr_bins 6 -discr_labels t -discr_type ordered -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +Running test 48 test type: discretization, description: tests discretization options +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test48 -mode discretize -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +Running test 55 test type: discretization, description: tests discretization options +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test55 -mode discretize -resp "PF,PF1" -discr_algo ranks -discr_bins 6 -discr_labels t -discr_type category -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass spec_fn smlp_toy_num_resp_mult_y1_verify.spec specs_path ../specs Running test 63 test type: verify, description: basic dt_sklearn assertion verification test on data with numeric labels -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test63 -mode verify -resp y1 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -use_model f -model_name test63_model -spec ../specs/smlp_toy_num_resp_mult_y1_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x/2+y1>4.3;(y1+p2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test63 -mode verify -resp y1 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -use_model f -model_name test63_model -spec ../specs/smlp_toy_num_resp_mult_y1_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x/2+y1>4.3;(y1+p2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_noknobs_verify.spec specs_path ../specs -Running test 72 test type: verify, description: nn_keras verification test with re-using saved model_per_response trained model -../../src/run_smlp.py -model_name "../models/test71_model" -out_dir ./ -pref Test72 -mode verify -resp y1,y2 -feat x0,x1,x2 -model nn_keras -nnet_encoding nested -save_model f -use_model t -model_per_response t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1 -asrt_exprs "(y2**3+x2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat +Running test 67 test type: verify, description: basic dt_sklearn assertion verification test on data with numeric labels +smlp -data "../data/smlp_toy_num_resp_noknobs.csv" -out_dir ./ -pref Test67 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -model_per_response t -save_model t -use_model f -model_name test67_model -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x0**2+y1>4.3;(y1+x2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_noknobs_verify.spec +specs_path ../specs +Running test 73 test type: verify, description: tests model term formation when mrmr_pred is activated and not all features are selected for training the model +smlp -data "../data/smlp_toy_num_resp_noknobs.csv" -out_dir ./ -pref Test73 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -use_model f -model_name test73_model -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2,asrt3 -asrt_exprs "(y2**3+x2)/2<6;y1>=9;y2<0" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_noknobs_pred_labeled.csv" spec_fn smlp_toy_num_resp_mult_free_inps.spec specs_path ../specs -Running test 83 test type: optimize, description: basic dt_sklearn multi objective pareto optimization test with numeric labels and integer grid as domain and with scaling objectives -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test83 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps.spec -data_scaler min_max -beta "y1>7 and y2>6" -objv_names obj1,objv2,objv3 -objv_exprs "(y1+y2)/2;y1/2-y2;y2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 82 test type: optimize, description: basic dt_sklearn single objective optimization test with numeric labels and integer grid as domain and with scaling objectives +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test82 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps.spec -data_scaler min_max -objv_names obj1,objv2,objv3 -objv_exprs "(y1+y2)/2;y1;y2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -spec_fn smlp_toy_num_resp_mult_verify_vacuous.spec +spec_fn smlp_toy_num_resp_mult_query_vacuous.spec specs_path ../specs -Running test 92 test type: verify, description: test to detect contradictory constraints in verification mode -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test92 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_verify_vacuous.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 91 test type: query, description: test to detect contradictory constraints in optimization mode due to contradictory alpha global and alpha bounds constraints on FMAX_xyx +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test91 -mode query -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_query_vacuous.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec specs_path ../specs Running test 99 test type: optimize, description: testing that the response and feature names can be taken from spec file in model exploration modes when the responses and/or features are not specified in the command line -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test99 -mode optimize -pareto t -opt_strategy lazy -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test99 -mode optimize -pareto t -opt_strategy lazy -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_certify_witness.spec specs_path ../specs Running test 103 test type: certify, description: -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test103 -mode certify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -use_model f -model_name test103_model -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_certify_witness.spec -quer_names valid_candidate,grid_conflict,range_conflict -quer_exprs "True;True;True" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test103 -mode certify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -use_model f -model_name test103_model -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_certify_witness.spec -quer_names valid_candidate,grid_conflict,range_conflict -quer_exprs "True;True;True" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_cannot_synthesize.spec specs_path ../specs Running test 109 test type: synthesize, description: basic test for mode synthesize where synthesis fails -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test109 -mode synthesize -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_cannot_synthesize.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -spec_fn smlp_toy_system_stable_constant_synth_fail.spec -specs_path ../specs -Running test 122 test type: optimize, description: optimization test with constant knob and no inputs where synthesis is not feasible because the assertion is not feasible but beta constraint is feasible therefore optimization is performed -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test122 -mode optimize -pareto f -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_fail.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -spec_fn smlp_toy_system_stable_verify.spec -specs_path ../specs -Running test 140 test type: verify, description: verification example with knobs only and fictitious inputs that have no effect where proparty is valid without stability and fails with stabilityRunning test 3 test type: prediction, description: basic poly_sklearn prediction test on labeled and new data with numeric response in training/test data only -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test3 -mode predict -resp y1 -feat x,p1,p2 -model poly_sklearn -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_unlabeled.csv" +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test109 -mode synthesize -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_cannot_synthesize.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 3 test type: prediction, description: basic poly_sklearn prediction test on labeled and new data with numeric response in training/test data only +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test3 -mode predict -resp y1 -feat x,p1,p2 -model poly_sklearn -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_unlabeled.csv" Running test 10 test type: prediction, description: basic et_sklearn prediction test on labeled and new data with numeric labels -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test10 -mode predict -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 15 -et_sklearn_bootstrap f -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test10 -mode predict -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 15 -et_sklearn_bootstrap f -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" Running test 17 test type: prediction, description: basic poly_sklearn prediction test from saved model on new data with numeric labels and two responses -../../src/run_smlp.py -model_name "../models/Test11_smlp_toy_num_resp_mult" -out_dir ./ -pref Test17 -mode predict -resp y1,y2 -feat x,p1,p2 -model poly_sklearn -save_model f -use_model t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +smlp -model_name "../models/Test11_smlp_toy_num_resp_mult" -out_dir ./ -pref Test17 -mode predict -resp y1,y2 -feat x,p1,p2 -model poly_sklearn -save_model f -use_model t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 24 test type: prediction, description: basic dt_sklearn prediction test using a model saved under a name specified through model_name option on new data with numeric labels -../../src/run_smlp.py -model_name "../models/test24_model" -out_dir ./ -pref Test24 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model t -model_per_response t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 23 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels and saving model using name specified through model_name option - adapts Test6 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test23 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model t -use_model f -model_name test24_model -model_per_response t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 30 test type: subgroups, description: basic test for subgroup discovery for numric responses -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test30 -mode subgroups -psg_dim 3 -psg_top 10 -resp y1,y2 -feat x,p1,p2 -plots t -seed 10 -log_time f +Running test 31 test type: subgroups, description: testing resp2b in subgroup discovery mode +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test31 -mode subgroups -psg_dim 3 -psg_top 10 -resp y1,y2 -resp2b "y1<6;y2>6" -feat x,p1,p2 -plots t -seed 10 -log_time f -save_config t -Running test 37 test type: doe, description: doe test with four levels with box_behnken -../../src/run_smlp.py -doe_spec "../grids/doe_three_levels_real_nan.csv" -out_dir ./ -pref Test37 -mode doe -doe_algo box_behnken -log_time f +Running test 40 test type: doe, description: doe test with four levels with latin_hypercube_space_filling +smlp -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test40 -mode doe -doe_algo latin_hypercube_sf -doe_samples 20 -log_time f -Running test 45 test type: doe, description: doe test with four levels with fractional_factorial -../../src/run_smlp.py -doe_spec "../grids/doe_two_levels_real.csv" -out_dir ./ -pref Test45 -mode doe -doe_algo fractional_factorial -doe_resolution 5 -log_time f +Running test 44 test type: doe, description: doe test with four levels with uniform_random_matrix +smlp -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test44 -mode doe -doe_algo uniform_random_matrix -doe_samples 20 -log_time f -Running test 55 test type: discretization, description: tests discretization options -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test55 -mode discretize -resp "PF,PF1" -discr_algo ranks -discr_bins 6 -discr_labels t -discr_type category -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +Running test 51 test type: discretization, description: tests discretization options +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test51 -mode discretize -resp "PF,PF1" -discr_algo jenks -discr_bins 6 -discr_labels f -discr_type integer -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass spec_fn smlp_toy_num_resp_mult_y2_verify.spec specs_path ../specs -Running test 69 test type: verify, description: nn_keras verification test with model_per_response training -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test69 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model t -use_model f -model_name test69_model -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "(y2**3+p2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat +Running test 59 test type: verify, description: basic nn_keras assertion verification test for functional nn_keras model +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test59 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat + +spec_fn smlp_toy_num_resp_mult_y2_verify.spec +specs_path ../specs +Running test 70 test type: verify, description: nn_keras verification test with re-using saved model_per_response trained model +smlp -model_name "../models/test69_model" -out_dir ./ -pref Test70 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model f -use_model t -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "(y2**3+p2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat + +spec_fn smlp_toy_num_resp_mult_free_inps.spec +specs_path ../specs +Running test 81 test type: optimize, description: basic dt_sklearn single objective optimization test with numeric labels and integer grid as domain and with scaling objectives +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test81 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps.spec -data_scaler min_max -objv_names obj1 -objv_exprs "(y1+y2)/2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec specs_path ../specs Running test 88 test type: optimize, description: basic dt_sklearn multi objective pareto optimization test with beta and objectives specified in spec file -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test88 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test88 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs Running test 94 test type: optsyn, description: basic test for rf_sklearn in model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test94 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test94 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_witness.spec specs_path ../specs Running test 102 test type: certify, description: basic test in certify mode to test stability (theta) and guard (eta) constraint generation -../../src/run_smlp.py -model_name "../models/test101_model" -out_dir ./ -pref Test102 -mode certify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model t -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_witness.spec -quer_names query1,query2,query3 -quer_exprs "(y2**3+p2)/2<6;y1>=9;y2<20" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +smlp -model_name "../models/test101_model" -out_dir ./ -pref Test102 -mode certify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model t -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_witness.spec -quer_names query1,query2,query3 -quer_exprs "(y2**3+p2)/2<6;y1>=9;y2<20" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_unsat_eta_verify.spec +spec_fn smlp_toy_num_resp_mult_stable_verify.spec specs_path ../specs -Running test 106 test type: verify, description: test for verification mode to check that eta contraints are not contradictory and as otherwise verification problem is not well defined -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test106 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_unsat_eta_verify.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "y1*2+x<=5 and y1<=10;-2*y2-1<10-p2" -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 105 test type: verify, description: basic dt_sklearn assertion verfication test with numeric labels and integer grid as domain +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test105 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_stable_verify.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "y1*2+x<=5 and y1<=10;-2*y2-1<10-p2" -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_basic.spec -specs_path ../specs -Running test 113 test type: optimize, description: smlp toy basic test for mode optimize from SMLP manual -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test113 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x1,x2,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -mrmr_pred 0 -epsilon 0.05 -delta_rel 0.01 -save_model t -model_name test113_model -save_model_config t -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec ../specs/smlp_toy_basic.spec +Running test 111 test type: unknown, description: smlp toy basic test to rerun saved model using the model rerun config file saved during model training +smlp -model_name "../models/test110_model" -out_dir ./ -pref Test111 -config ../models/test110_model_rerun_model_config.json -new_dat "../data/smlp_toy_basic_pred_unlabeled.csv" spec_fn smlp_toy_system_stable_constant_verify.spec specs_path ../specs -Running test 118 test type: verify, description: verification test with knobs only where assertion is valid without stability and fails with stability -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test118 -mode verify -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_verify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -spec_fn smlp_toy_system_stable_verify.spec -specs_path ../specs -Running test 126 test type: verify, description: verification example with knobs only and fictitious inputs that have no effect where proparty is valid without stability and fails with stability -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test126 -mode verify -model system -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_verify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 118 test type: verify, description: verification test with knobs only where assertion is valid without stability and fails with stabilityRunning test 6 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test6 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -spec_fn smlp_toy_num_resp_mult_optsyn.spec -specs_pathRunning test 5 test type: prediction, description: basic dt_caret prediction test on labeled and new data with numeric labels -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test5 -mode predict -resp y1 -feat x,p1,p2 -model dt_caret -save_model t -use_model f -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 9 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test9 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model t -model_name test20_model -data_scaler none -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -save_config t -save_model_config t -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 18 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels and saving model using name specified through model_name option - adapts Test6 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test18 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model t -use_model f -model_name test19_model -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 15 test type: prediction, description: basic dt_caret prediction test from saved model on new data with numeric labels +smlp -model_name "../models/Test5_smlp_toy_num_resp_mult" -out_dir ./ -pref Test15 -mode predict -resp y1 -feat x,p1,p2 -model dt_caret -save_model f -use_model t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 26 test type: prediction, description: basic dt_sklearn prediction test using a model saved under a name specified through model_name option on new data with numeric labels -../../src/run_smlp.py -model_name "../models/test26_model" -out_dir ./ -pref Test26 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model t -mrmr_pred 2 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 22 test type: prediction, description: test for illegal symbols in column names +smlp -model_name "../models/test22_model" -out_dir ./ -pref Test22 -mode predict -resp "PF ,|PF |" -model poly_sklearn -save_model f -use_model t -pred_plots t -resp_plots t -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_metasymbol_mult_reg_pred_labeled.csv" Running test 33 test type: unknown, description: testing -config option with subgroups mode -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test33 -config ../models/Test31_smlp_toy_num_resp_mult_args_config.json +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test33 -config ../models/Test31_smlp_toy_num_resp_mult_args_config.json Running test 41 test type: doe, description: doe test with four levels with random_k_means -../../src/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test41 -mode doe -doe_algo random_k_means -doe_samples 20 -log_time f +smlp -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test41 -mode doe -doe_algo random_k_means -doe_samples 20 -log_time f Running test 49 test type: discretization, description: tests discretization options -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test49 -mode discretize -resp "PF,PF1" -discr_algo quantile -discr_bins 6 -discr_labels t -discr_type category -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test49 -mode discretize -resp "PF,PF1" -discr_algo quantile -discr_bins 6 -discr_labels t -discr_type category -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass -spec_fn smlp_toy_num_resp_mult.spec +spec_fn smlp_toy_num_resp_noknobs_verify.spec specs_path ../specs -Running test 58 test type: optimize, description: basic dt_sklearn optimization test with numeric labels and integer grid as domain and without scaling objectives -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test58 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -objv_names objv_y1,objv_y2 -objv_exprs "y1;y2" -epsilon 0.01 -delta_rel 0.01 -data_scaler none -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 57 test type: verify, description: basic dt_sklearn assertion verfication test with numeric labels and integer grid as domain +smlp -data "../data/smlp_toy_num_resp_noknobs.csv" -out_dir ./ -pref Test57 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "y1*2+x0<=5 and y1<=10;-2*y2-1<10-x2" -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_y2_verify.spec +spec_fn smlp_toy_num_resp_noknobs_verify.spec specs_path ../specs -Running test 70 test type: verify, description: nn_keras verification test with re-using saved model_per_response trained model -../../src/run_smlp.py -model_name "../models/test69_model" -out_dir ./ -pref Test70 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model f -use_model t -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "(y2**3+p2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat +Running test 65 test type: verify, description: basic dt_sklearn assertion verification test on data with numeric labels +smlp -data "../data/smlp_toy_num_resp_noknobs.csv" -out_dir ./ -pref Test65 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -use_model f -model_name test65_model -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x0**2+y1>4.3;(y1+x2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult.spec +spec_fn smlp_toy_num_resp_mult_y2_verify.spec specs_path ../specs -Running test 85 test type: optimize, description: tests alpha and eta constraints specified in command line in optimization mode -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test85 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -data_scaler min_max -objv_names obj1,objv2 -objv_exprs "(y1+y2)/2;y1" -alpha "p2<5 and x==10 and x<12" -eta "p1==4" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 69 test type: verify, description: nn_keras verification test with model_per_response training +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test69 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model t -use_model f -model_name test69_model -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "(y2**3+p2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -spec_fn smlp_toy_num_resp_mult_optsyn.spec +spec_fn smlp_toy_num_resp_mult.spec specs_path ../specs -Running test 93 test type: optsyn, description: basic test for mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test93 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 80 test type: optimize, description: basic dt_sklearn single objective optimization test with numeric labels and integer grid as domain and with scaling objectives +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test80 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -data_scaler min_max -objv_names obj1 -objv_exprs "(y1+y2)/2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_witness.spec +spec_fn smlp_toy_num_resp_mult.spec specs_path ../specs -Running test 101 test type: certify, description: basic test in certify mode to test stability (theta) and guard (eta) constraint generation -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test101 -mode certify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -use_model f -model_name test101_model -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_witness.spec -quer_names query1,query2,query3 -quer_exprs "(y2**3+p2)/2<6;y1>=9;y2<20" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 86 test type: optimize, description: tests alpha +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test86 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -data_scaler min_max -objv_names obj1,objv2 -objv_exprs "(y1+y2)/2;y1" -asrt_names asrt1,asrt2,asrt3 -asrt_exprs "(y2**3+p2)/2<6;y1>=9;y2<0" -alpha "p2<5 and x==10 and x<12" -eta "p1==4" -epsilon 0.05 -delta_rel 0.01 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_stable_verify.spec +spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 105 test type: verify, description: basic dt_sklearn assertion verfication test with numeric labels and integer grid as domain -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test105 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_stable_verify.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "y1*2+x<=5 and y1<=10;-2*y2-1<10-p2" -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -Running test 111 test type: unknown, description: smlp toy basic test to rerun saved model using the model rerun config file saved during model training -../../src/run_smlp.py -model_name "../models/test110_model" -out_dir ./ -pref Test111 -config ../models/test110_model_rerun_model_config.json -new_dat "../data/smlp_toy_basic_pred_unlabeled.csv" +Running test 95 test type: optsyn, description: basic test for dt_caret in model exploration mode optsyn +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test95 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_caret -save_model f -use_model f -tree_encoding nested -compress_rules f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_system_stable_constant_certify.spec +spec_fn smlp_toy_num_resp_mult_beta_verify.spec specs_path ../specs -Running test 117 test type: certify, description: certification test with knobs only where assertion is valid without stability and fails with stability -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test117 -mode certify -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_certify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 107 test type: verify, description: test for verification mode to check that eta contraints are not contradictory and as otherwise verification problem is not well defined +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test107 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_beta_verify.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "y1*2+x<=5 and y1<=10;-2*y2-1<10-p2" -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_system_stable_constant_synth_feasible.spec +spec_fn smlp_toy_basic.spec specs_path ../specs -Running test 123 test type: optimize, description: optimization test with constant knob and no inputs where synthesis is feasible and optimization is performed -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test123 -mode optimize -pareto t -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 113 test type: optimize, description: smlp toy basic test for mode optimize from SMLP manual +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test113 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x1,x2,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -mrmr_pred 0 -epsilon 0.05 -delta_rel 0.01 -save_model t -model_name test113_model -save_model_config t -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec ../specs/smlp_toy_basic.spec -spec_fn smlp_toy_system_stable_constant_synth_feasible.spec +spec_fn smlp_toy_system_stable_constant_synth_fail.spec specs_path ../specs -Running test 145 test type: optimize, description: optimization test with constant knob and no inputs where synthesis is feasible and optimization is performed -../../src/run_smlp.py -out_dir ./ -pref Test145 -mode optimize -pareto t -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -doe_spec ../grids/doe_two_levels_opt.csv -doe_algo latin_hypercube -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 6 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test6 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - -Running test 9 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test9 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model t -model_name test20_model -data_scaler none -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -save_config t -save_model_config t -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 120 test type: synthesize, description: synthesis test with constant knob and no inputs where synthesis is not feasible because the assertion is not feasible +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test120 -mode synthesize -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_fail.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 4 test type: prediction, description: basic nn_keras prediction test on labeled and new data with numeric labels and one response +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test4 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -nn_keras_weights_precision 2 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 15 test type: prediction, description: basic dt_caret prediction test from saved model on new data with numeric labels -../../src/run_smlp.py -model_name "../models/Test5_smlp_toy_num_resp_mult" -out_dir ./ -pref Test15 -mode predict -resp y1 -feat x,p1,p2 -model dt_caret -save_model f -use_model t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 13 test type: train, description: EV-SI real life nn_keras prediction test on labeled and new data with numeric labels +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test13 -mode train -resp y1,y2 -feat x1,x2,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api f -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 23 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels and saving model using name specified through model_name option - adapts Test6 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test23 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model t -use_model f -model_name test24_model -model_per_response t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 25 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels and saving model using name specified through model_name option - adapts Test6 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test25 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model t -use_model f -model_name test26_model -mrmr_pred 2 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 29 test type: subgroups, description: basic test for subgroup discovery for pass-fail responses -../../src/run_smlp.py -data "../data/smlp_toy_cls_metasymbol_colnames_mult.csv" -out_dir ./ -pref Test29 -mode subgroups -psg_dim 3 -psg_top 10 -resp "PF 1,PF#" -plots t -seed 10 -log_time f +Running test 34 test type: doe, description: doe test with four levels with full_factorial method +smlp -doe_spec "../grids/doe_four_levels_real.csv" -out_dir ./ -pref Test34 -mode doe -doe_algo full_factorial -log_time f -Running test 38 test type: doe, description: doe test with four levels with box_wilson -../../src/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test38 -mode doe -doe_algo box_wilson -doe_cc_face ccc -doe_cc_alpha r -doe_cc_center 2,3 -log_time f +Running test 42 test type: doe, description: doe test with four levels with maximin_reconstruction +smlp -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test42 -mode doe -doe_algo maximin_reconstruction -doe_samples 20 -log_time f -Running test 46 test type: prediction, description: tests options -pos_val and -neg_val -../../src/run_smlp.py -data "../data/smlp_toy_pf_mult.csv" -out_dir ./ -pref Test46 -mode predict -resp "PF,PF1" -model poly_sklearn -save_model t -save_model_config f -use_model f -model_name test47_model -data_scaler none -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -pos_val fail -neg_val pass -new_dat "../data/smlp_toy_pf_mult.csv" +Running test 50 test type: discretization, description: tests discretization options +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test50 -mode discretize -resp "PF,PF1" -discr_algo kmeans -discr_bins 6 -discr_labels t -discr_type ordered -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass -Running test 56 test type: discretization, description: tests discretization options -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test56 -mode discretize -resp "PF,PF1" -discr_algo ranks -discr_bins 6 -discr_labels f -discr_type object -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +spec_fn smlp_toy_num_resp_mult.spec +specs_path ../specs +Running test 58 test type: optimize, description: basic dt_sklearn optimization test with numeric labels and integer grid as domain and without scaling objectives +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test58 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -objv_names objv_y1,objv_y2 -objv_exprs "y1;y2" -epsilon 0.01 -delta_rel 0.01 -data_scaler none -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_noknobs_verify.spec specs_path ../specs -Running test 68 test type: verify, description: basic dt_sklearn assertion verification test on data with one numeric response -../../src/run_smlp.py -model_name "../models/test67_model" -out_dir ./ -pref Test68 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -model_per_response t -save_model f -use_model t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x0**2+y1>4.3;(y1+x2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 66 test type: verify, description: basic dt_sklearn assertion verification test on data with one numeric response +smlp -model_name "../models/test65_model" -out_dir ./ -pref Test66 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x0**2+y1>4.3;(y1+x2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult.spec +spec_fn smlp_toy_num_resp_noknobs_verify.spec specs_path ../specs -Running test 80 test type: optimize, description: basic dt_sklearn single objective optimization test with numeric labels and integer grid as domain and with scaling objectives -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test80 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -data_scaler min_max -objv_names obj1 -objv_exprs "(y1+y2)/2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 72 test type: verify, description: nn_keras verification test with re-using saved model_per_response trained model +smlp -model_name "../models/test71_model" -out_dir ./ -pref Test72 -mode verify -resp y1,y2 -feat x0,x1,x2 -model nn_keras -nnet_encoding nested -save_model f -use_model t -model_per_response t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1 -asrt_exprs "(y2**3+x2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -spec_fn smlp_toy_num_resp_mult_alpha_asrt_verify.spec +Running test 77 test type: unknown, description: verification test run using model_rerun config covering the case when mrmr selcts only a subset of features specified through the command line or config file +smlp -model_name "../models/test76_model" -out_dir ./ -pref Test77 -config ../models/test76_model_rerun_model_config.json + +spec_fn smlp_toy_num_resp_noknobs_verify.spec specs_path ../specs -Running test 87 test type: verify, description: tests global alpha constraints and assertions specified in spec file -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test87 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_alpha_asrt_verify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 84 test type: verify, description: tests global alpha constraints specified using option -alpha on inputs +smlp -data "../data/smlp_toy_num_resp_noknobs.csv" -out_dir ./ -pref Test84 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2,asrt3 -asrt_exprs "(y2**3+x2)/2<6;y1>=9;y2<0" -alpha "x2==7.0 and x0==0 and x1==2.5" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 96 test type: optsyn, description: basic test for rf_sklearn in model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test96 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_caret -save_model f -use_model f -tree_encoding nested -compress_rules f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 93 test type: optsyn, description: basic test for mode optsyn +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test93 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 112 test type: prediction, description: smlp toy basic test from SMLP manual -../../src/run_smlp.py -model_name "../models/test110_model" -out_dir ./ -pref Test112 -mode predict -resp y1,y2 -feat x1,x2,p1,p2 -model poly_sklearn -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -use_model t -save_model f -new_dat "../data/smlp_toy_basic_pred_unlabeled.csv" +spec_fn smlp_toy_num_resp_mult_witness.spec +specs_path ../specs +Running test 101 test type: certify, description: basic test in certify mode to test stability (theta) and guard (eta) constraint generation +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test101 -mode certify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -use_model f -model_name test101_model -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_witness.spec -quer_names query1,query2,query3 -quer_exprs "(y2**3+p2)/2<6;y1>=9;y2<20" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_system_stable_constant_query.spec +spec_fn smlp_toy_num_resp_mult_unsat_eta_verify.spec specs_path ../specs -Running test 119 test type: query, description: query test with knobs only where query is satisfiable without stability and fails with stability -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test119 -mode query -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_query.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 106 test type: verify, description: test for verification mode to check that eta contraints are not contradictory and as otherwise verification problem is not well defined +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test106 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_unsat_eta_verify.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "y1*2+x<=5 and y1<=10;-2*y2-1<10-p2" -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_system_stable_constant_synth_feasible.spec +spec_fn smlp_toy_basic.spec specs_path ../specs -Running test 125 test type: optsyn, description: optimized synthesis test with constant knob and no inputs where synthesis is feasible and optimization is performed -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test125 -mode optsyn -pareto t -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 114 test type: optimize, description: smlp toy basic test for mode optimize from SMLP manual without specifying resp and feat in command line +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test114 -mode optimize -pareto t -opt_strategy lazy -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -mrmr_pred 0 -epsilon 0.05 -delta_rel 0.01 -save_model f -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec ../specs/smlp_toy_basic.spec -spec_fn smlp_toy_system.spec +spec_fn smlp_toy_system_stable_constant_synth_feasible.spec specs_path ../specs -Running test 146 test type: optimize, description: optimization test with constant knob and no inputs where synthesis is feasible and optimization is performed -../../src/run_smlp.py -out_dir ./ -pref Test146 -mode optimize -pareto t -opt_strategy lazy -model poly_sklearn -resp y1,y2 -feat p1,p2,x1,x2 -save_model t -use_model f -mrmr_pred 0 -model_per_response t -split 1 -spec ../specs/smlp_toy_system.spec -doe_spec ../grids/explore_doe_two_levels.csv -doe_algo latin_hypercube -epsilon 0.99999999 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 121 test type: synthesize, description: synthesis test with constant knob and no inputs where synthesis is feasible +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test121 -mode synthesize -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 152 test type: prediction, description: tests the huber loss function Huber and sample weights -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test152 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss huber -sw_coef 8 -sw_exp 5 -sw_int 0.5 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 4 test type: prediction, description: basic nn_keras prediction test on labeled and new data with numeric labels and one response -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test4 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -nn_keras_weights_precision 2 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +spec_fnRunning test 2 test type: prediction, description: basic rf_sklearn prediction test on labeled and new data with numeric labels +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test2 -mode predict -resp y1 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 15 -save_model_config f -mrmr_pred 0 -plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 13 test type: train, description: EV-SI real life nn_keras prediction test on labeled and new data with numeric labels -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test13 -mode train -resp y1,y2 -feat x1,x2,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api f -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 12 test type: train, description: EV-SI real life dt_sklearn predict test on labeled and new data with numeric labels +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test12 -mode train -resp y1,y2 -feat x1,x2,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 22 test type: prediction, description: test for illegal symbols in column names -../../src/run_smlp.py -model_name "../models/test22_model" -out_dir ./ -pref Test22 -mode predict -resp "PF ,|PF |" -model poly_sklearn -save_model f -use_model t -pred_plots t -resp_plots t -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_metasymbol_mult_reg_pred_labeled.csv" +Running test 20 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels +smlp -model_name "../models/test20_model" -out_dir ./ -pref Test20 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model t -data_scaler none -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 32 test type: unknown, description: test reusing saved model by using configuration file -../../src/run_smlp.py -model_name "../models/test20_model" -out_dir ./ -pref Test32 -config ../models/test20_model_rerun_model_config.json -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 28 test type: prediction, description: checks nn_keras prediction with sw_coef 0.8 and functional API +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test28 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 39 test type: doe, description: doe test with four levels with latin_hypercube -../../src/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test39 -mode doe -doe_algo latin_hypercube -doe_prob_distr Exponential -doe_samples 30 -log_time f +Running test 38 test type: doe, description: doe test with four levels with box_wilson +smlp -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test38 -mode doe -doe_algo box_wilson -doe_cc_face ccc -doe_cc_alpha r -doe_cc_center 2,3 -log_time f -Running test 47 test type: prediction, description: tests options -pos_val and -neg_val when re-using saved model -../../src/run_smlp.py -model_name "../models/test47_model" -out_dir ./ -pref Test47 -mode predict -resp "PF,PF1" -model poly_sklearn -save_model f -use_model t -data_scaler none -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -pos_val fail -neg_val pass -new_dat "../data/smlp_toy_pf_mult.csv" +Running test 46 test type: prediction, description: tests options -pos_val and -neg_val +smlp -data "../data/smlp_toy_pf_mult.csv" -out_dir ./ -pref Test46 -mode predict -resp "PF,PF1" -model poly_sklearn -save_model t -save_model_config f -use_model f -model_name test47_model -data_scaler none -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -pos_val fail -neg_val pass -new_dat "../data/smlp_toy_pf_mult.csv" Running test 54 test type: discretization, description: tests discretization options -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test54 -mode discretize -resp "PF,PF1" -discr_algo ordinals -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test54 -mode discretize -resp "PF,PF1" -discr_algo ordinals -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass spec_fn smlp_toy_num_resp_mult_y1_verify.spec specs_path ../specs Running test 64 test type: verify, description: basic dt_sklearn assertion verification test on data with one numeric response -../../src/run_smlp.py -model_name "../models/test63_model" -out_dir ./ -pref Test64 -mode verify -resp y1 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model t -spec ../specs/smlp_toy_num_resp_mult_y1_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x/2+y1>4.3;(y1+p2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -Running test 77 test type: unknown, description: verification test run using model_rerun config covering the case when mrmr selcts only a subset of features specified through the command line or config file -../../src/run_smlp.py -model_name "../models/test76_model" -out_dir ./ -pref Test77 -config ../models/test76_model_rerun_model_config.json +smlp -model_name "../models/test63_model" -out_dir ./ -pref Test64 -mode verify -resp y1 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model t -spec ../specs/smlp_toy_num_resp_mult_y1_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x/2+y1>4.3;(y1+p2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult.spec +spec_fn smlp_toy_num_resp_noknobs_verify.spec specs_path ../specs -Running test 86 test type: optimize, description: tests alpha -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test86 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -data_scaler min_max -objv_names obj1,objv2 -objv_exprs "(y1+y2)/2;y1" -asrt_names asrt1,asrt2,asrt3 -asrt_exprs "(y2**3+p2)/2<6;y1>=9;y2<0" -alpha "p2<5 and x==10 and x<12" -eta "p1==4" -epsilon 0.05 -delta_rel 0.01 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 68 test type: verify, description: basic dt_sklearn assertion verification test on data with one numeric response +smlp -model_name "../models/test67_model" -out_dir ./ -pref Test68 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -model_per_response t -save_model f -use_model t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x0**2+y1>4.3;(y1+x2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn.spec +spec_fn smlp_toy_num_resp_noknobs_verify.spec specs_path ../specs -Running test 95 test type: optsyn, description: basic test for dt_caret in model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test95 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_caret -save_model f -use_model f -tree_encoding nested -compress_rules f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 74 test type: verify, description: tests model term formation when mrmr_pred is activated and not all features are selected for training the model. with separate model for each response +smlp -model_name "../models/test73_model" -out_dir ./ -pref Test74 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model t -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2,asrt3 -asrt_exprs "(y2**3+x2)/2<6;y1>=9;y2<0" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_noknobs_pred_labeled.csv" -spec_fn smlp_toy_num_resp_mult_beta_verify.spec +spec_fn smlp_toy_num_resp_noknobs_verify.spec specs_path ../specs -Running test 107 test type: verify, description: test for verification mode to check that eta contraints are not contradictory and as otherwise verification problem is not well defined -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test107 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_beta_verify.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "y1*2+x<=5 and y1<=10;-2*y2-1<10-p2" -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 78 test type: verify, description: tests model term formation when mrmr_pred is activated and not all features are selected for training the model +smlp -data "../data/smlp_toy_num_resp_noknobs.csv" -out_dir ./ -pref Test78 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -use_model f -model_name test78_model -mrmr_pred 1 -model_per_response f -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "y1==9;y2>0" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_basic.spec +spec_fn smlp_toy_num_resp_mult_alpha_asrt_verify.spec specs_path ../specs -Running test 114 test type: optimize, description: smlp toy basic test for mode optimize from SMLP manual without specifying resp and feat in command line -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test114 -mode optimize -pareto t -opt_strategy lazy -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -mrmr_pred 0 -epsilon 0.05 -delta_rel 0.01 -save_model f -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec ../specs/smlp_toy_basic.spec +Running test 87 test type: verify, description: tests global alpha constraints and assertions specified in spec file +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test87 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_alpha_asrt_verify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_system_stable_constant_synth_fail.spec +spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 120 test type: synthesize, description: synthesis test with constant knob and no inputs where synthesis is not feasible because the assertion is not feasible -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test120 -mode synthesize -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_fail.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 96 test type: optsyn, description: basic test for rf_sklearn in model exploration mode optsyn +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test96 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_caret -save_model f -use_model f -tree_encoding nested -compress_rules f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +Running test 112 test type: prediction, description: smlp toy basic test from SMLP manual +smlp -model_name "../models/test110_model" -out_dir ./ -pref Test112 -mode predict -resp y1,y2 -feat x1,x2,p1,p2 -model poly_sklearn -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -use_model t -save_model f -new_dat "../data/smlp_toy_basic_pred_unlabeled.csv" -spec_fn smlp_toy_witness_certify.spec +spec_fn smlp_toy_system_stable_constant_query.spec specs_path ../specs -Running test 128 test type: certify, description: Basic regression test in certify mode covering all four possible outcomes when certifying a witness for a query: the witness is stable -../../src/run_smlp.py -data "../data/smlp_toy_ctg_num_resp.csv" -out_dir ./ -pref Test128 -mode certify -resp y1,y2 -feat x,p1,p2 -model poly_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model f -model_per_response f -spec ../specs/smlp_toy_witness_certify.spec -quer_names query_stable_witness,query_grid_conflict,query_unstable_witness,query_infeasible_witness,query_poly_intercept_sensitive -quer_exprs "y2<=90;y1>=9;y1>=(-13);y1>9;y1>=(-10)" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 119 test type: query, description: query test with knobs only where query is satisfiable without stability and fails with stability +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test119 -mode query -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_query.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 149 test type: prediction, description: tests the mae loss function MeanAbsoluteError and sample weoghts -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test149 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss mae -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +spec_fn smlp_toy_system_stable_verify.spec +specs_path ../specs +Running test 126 test type: verify, description: verification example with knobs only and fictitious inputs that have no effect where proparty is valid without stability and fails with stability +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test126 -mode verify -model system -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_verify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_y2_verify.specRunning test 8 test type: prediction, description: basic nn_keras prediction test on labeled and new data with numeric labels and two responses -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test8 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -nn_keras_epochs 20 -nn_keras_seq_api f -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +spec_fn smlp_toy_num_resp_mult.spec +specs_path ../specs +Running test 141 test type: optimize, description: basic test for compress_rules option for dt_sklearn in optimization mode +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test141 -mode optimize -opt_strategy lazy -pareto f -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules t -spec ../specs/smlp_toy_num_resp_mult.spec -objv_names objv_y1,objv_y2 -objv_exprs "y1;y2" -epsilon 0.01 -delta_rel 0.01 -data_scaler none -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 8 test type: prediction, description: basic nn_keras prediction test on labeled and new data with numeric labels and two responses +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test8 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -nn_keras_epochs 20 -nn_keras_seq_api f -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" Running test 14 test type: train, description: EV-SI real life poly_sklearn prediction test on labeled and new data with numeric labels -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test14 -mode train -resp y1,y2 -feat x1,x2,p1,p2 -model poly_sklearn -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test14 -mode train -resp y1,y2 -feat x1,x2,p1,p2 -model poly_sklearn -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f Running test 21 test type: prediction, description: test for illegal symbols in column names -../../src/run_smlp.py -data "../data/smlp_toy_num_metasymbol_mult_reg.csv" -out_dir ./ -pref Test21 -mode predict -resp "PF ,|PF |" -model poly_sklearn -save_model t -use_model f -model_name test22_model -pred_plots t -resp_plots t -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_metasymbol_mult_reg_pred_labeled.csv" +smlp -data "../data/smlp_toy_num_metasymbol_mult_reg.csv" -out_dir ./ -pref Test21 -mode predict -resp "PF ,|PF |" -model poly_sklearn -save_model t -use_model f -model_name test22_model -pred_plots t -resp_plots t -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_metasymbol_mult_reg_pred_labeled.csv" -Running test 31 test type: subgroups, description: testing resp2b in subgroup discovery mode -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test31 -mode subgroups -psg_dim 3 -psg_top 10 -resp y1,y2 -resp2b "y1<6;y2>6" -feat x,p1,p2 -plots t -seed 10 -log_time f -save_config t +Running test 29 test type: subgroups, description: basic test for subgroup discovery for pass-fail responses +smlp -data "../data/smlp_toy_cls_metasymbol_colnames_mult.csv" -out_dir ./ -pref Test29 -mode subgroups -psg_dim 3 -psg_top 10 -resp "PF 1,PF#" -plots t -seed 10 -log_time f -Running test 40 test type: doe, description: doe test with four levels with latin_hypercube_space_filling -../../src/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test40 -mode doe -doe_algo latin_hypercube_sf -doe_samples 20 -log_time f +Running test 35 test type: doe, description: doe test with four levels with plackett_burman +smlp -doe_spec "../grids/doe_four_levels_real.csv" -out_dir ./ -pref Test35 -mode doe -doe_algo plackett_burman -log_time f -Running test 48 test type: discretization, description: tests discretization options -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test48 -mode discretize -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +Running test 43 test type: doe, description: doe test with four levels with halton_sequence +smlp -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test43 -mode doe -doe_algo halton_sequence -doe_samples 20 -log_time f Running test 53 test type: discretization, description: tests discretization options -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test53 -mode discretize -resp "PF,PF1" -discr_algo ordinals -discr_bins 6 -discr_labels f -discr_type integer -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test53 -mode discretize -resp "PF,PF1" -discr_algo ordinals -discr_bins 6 -discr_labels f -discr_type integer -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass spec_fn smlp_toy_num_resp_noknobs_verify.spec specs_path ../specs -Running test 66 test type: verify, description: basic dt_sklearn assertion verification test on data with one numeric response -../../src/run_smlp.py -model_name "../models/test65_model" -out_dir ./ -pref Test66 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x0**2+y1>4.3;(y1+x2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 61 test type: verify, description: tests verificaion mode for NN with nn_keras_seq_api f +smlp -data "../data/smlp_toy_num_resp_noknobs.csv" -out_dir ./ -pref Test61 -mode verify -resp y1,y2 -feat x0,x1,x2 -model nn_keras -nnet_encoding nested -nn_keras_epochs 100 -save_model_config f --spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "not(x25 and y1<=10);-2*y2-1<10-x2 and x2>5 and x2<8" -vacuity f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_seq_api f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -spec_fn smlp_toy_num_resp_mult.spec +spec_fn smlp_toy_num_resp_noknobs_verify.spec specs_path ../specs -Running test 79 test type: query, description: basic test in query mode to test stability (theta) and guard (eta) constraint generation -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test79 -mode query -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult.spec -quer_names query1,query2,query3 -quer_exprs "(y2**3+p2)/2<6;y1>=9;y2<0" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 76 test type: verify, description: tests model term formation when mrmr_pred is activated and not all features are selected for training the model +smlp -data "../data/smlp_toy_num_resp_noknobs.csv" -out_dir ./ -pref Test76 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -use_model f -model_name test76_model -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2,asrt3 -asrt_exprs "(y2**3+x2)/2<6;y1>=9;y2<0" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn_vacuous.spec +spec_fn smlp_toy_num_resp_mult.spec specs_path ../specs -Running test 90 test type: optsyn, description: test to detect contradictory constraints in optsyn mode -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test90 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn_vacuous.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 85 test type: optimize, description: tests alpha and eta constraints specified in command line in optimization mode +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test85 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -data_scaler min_max -objv_names obj1,objv2 -objv_exprs "(y1+y2)/2;y1" -alpha "p2<5 and x==10 and x<12" -eta "p1==4" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn.spec +spec_fn smlp_toy_num_resp_mult_verify_vacuous.spec specs_path ../specs -Running test 98 test type: optsyn, description: basic test for et_caret in model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test98 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_caret -save_model f -use_model f -tree_encoding nested -compress_rules f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 92 test type: verify, description: test to detect contradictory constraints in verification mode +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test92 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_verify_vacuous.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_system.spec +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec specs_path ../specs -Running test 115 test type: certify, description: basic test in certify mode -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test115 -mode certify -resp y1,y2 -feat x1,x2,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_system.spec -quer_names query1,query2 -quer_exprs "y1>0;y2<=0" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 100 test type: optimize, description: basic test for sat_threshold option enabing usage of objectve values in SAT assignments that prove optimization thresholds +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test100 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_system_stable_constant_synth_fail.spec +spec_fn smlp_toy_num_resp_mult.spec specs_path ../specs -Running test 124 test type: optsyn, description: optimized synthesis test with constant knob and no inputs where synthesis is not feasible because while beta constraint is feasible the assertion is not feasible therefore optimization is not performed -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test124 -mode optsyn -pareto f -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_fail.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 104 test type: verify, description: assertion verfication test with wrong spec that does not assign a single value using a singleton grid or range with equal max and min +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test104 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "y1*2+x<=5 and y1<=10;-2*y2-1<10-p2" -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult.spec +Running test 110 test type: prediction, description: smlp toy basic example for predict mode from SMLP user manual +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test110 -mode predict -resp y1,y2 -feat x1,x2,p1,p2 -model poly_sklearn -save_model t -model_name test110_model -save_model_config t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_basic_pred_unlabeled.csv" + +spec_fn smlp_toy_system.spec specs_path ../specs -Running test 141 test type: optimize, description: basic test for compress_rules option for dt_sklearn in optimization mode -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test141 -mode optimize -opt_strategy lazy -pareto f -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules t -spec ../specs/smlp_toy_num_resp_mult.spec -objv_names objv_y1,objv_y2 -objv_exprs "y1;y2" -epsilon 0.01 -delta_rel 0.01 -data_scaler none -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 116 test type: certify, description: basic test in certify mode when system is specified and is used as the model; p2 rel-rad needs to be 0 or very close to it the witness to first query to be stable +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test116 -mode certify -resp y1,y2 -feat x1,x2,p1,p2 -model system -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_system.spec -quer_names query1,query2 -quer_exprs "y1>0;y2<=0" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 148 test type: prediction, description: checks nn_keras prediction with sw_coef 0.8 and sequential API -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test148 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +spec_fn smlp_toy_system_stable_certify.spec +specs_path ../specs +Running test 127 test type: certify, description: certification example with knobs only and fictitious inputs with values fixed through their ranges +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test127 -mode certify -model system -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_certify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_verify.spec +spec_fn smlp_toy_num_resp_mult_query.spec specs_path ../specs -Running test 157 test type: verify, description: basic nn_keras assertion verification test that uses keras tuner with sequrntial models for model training; adapts test 155 by consdering multiple responses -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test157 -mode verify -resp y1,y2 -feat x,p1,p2 --model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics rmse,logcosh +Running test 143 test type: query, description: basic test for compress_rules for et_sklearn in mode query +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test143 -mode query -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_bootstrap f -tree_encoding nested -compress_rules t -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_query.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f Running test 1 test type: train, description: basic dt_caret training and test on labeled data with single numeric response -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test1 -mode train -resp y1 -feat x,p1,p2 -model dt_caret -save_model_config f -mrmr_pred 0 -plots f -seed 10 -log_time f +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test1 -mode train -resp y1 -feat x,p1,p2 -model dt_caret -save_model_config f -mrmr_pred 0 -plots f -seed 10 -log_time f -Running test 20 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels -../../src/run_smlp.py -model_name "../models/test20_model" -out_dir ./ -pref Test20 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model t -data_scaler none -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 19 test type: prediction, description: basic dt_sklearn prediction test using a model saved under a name specified through model_name option on new data with numeric labels +smlp -model_name "../models/test19_model" -out_dir ./ -pref Test19 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 28 test type: prediction, description: checks nn_keras prediction with sw_coef 0.8 and functional API -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test28 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 27 test type: prediction, description: checks nn_keras prediction with nn_keras_seq_api t +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test27 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 34 test type: doe, description: doe test with four levels with full_factorial method -../../src/run_smlp.py -doe_spec "../grids/doe_four_levels_real.csv" -out_dir ./ -pref Test34 -mode doe -doe_algo full_factorial -log_time f +Running test 36 test type: doe, description: doe test with four levels with sukharev_grid +smlp -doe_spec "../grids/doe_four_levels_real.csv" -out_dir ./ -pref Test36 -mode doe -doe_algo sukharev_grid -doe_samples 125 -log_time f -Running test 42 test type: doe, description: doe test with four levels with maximin_reconstruction -../../src/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test42 -mode doe -doe_algo maximin_reconstruction -doe_samples 20 -log_time f +Running test 45 test type: doe, description: doe test with four levels with fractional_factorial +smlp -doe_spec "../grids/doe_two_levels_real.csv" -out_dir ./ -pref Test45 -mode doe -doe_algo fractional_factorial -doe_resolution 5 -log_time f -Running test 50 test type: discretization, description: tests discretization options -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test50 -mode discretize -resp "PF,PF1" -discr_algo kmeans -discr_bins 6 -discr_labels t -discr_type ordered -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +Running test 52 test type: discretization, description: tests discretization options +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test52 -mode discretize -resp "PF,PF1" -discr_algo jenks -discr_bins 6 -discr_labels t -discr_type ordered -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass spec_fn smlp_toy_num_resp_mult_y2_verify.spec specs_path ../specs Running test 60 test type: verify, description: basic nn_keras assertion verification test for functional nn_keras model -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test60 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test60 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -spec_fn smlp_toy_num_resp_mult_free_inps.spec +spec_fn smlp_toy_num_resp_noknobs_verify.spec specs_path ../specs -Running test 82 test type: optimize, description: basic dt_sklearn single objective optimization test with numeric labels and integer grid as domain and with scaling objectives -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test82 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps.spec -data_scaler min_max -objv_names obj1,objv2,objv3 -objv_exprs "(y1+y2)/2;y1;y2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat +Running test 71 test type: verify, description: nn_keras verification test with model_per_response training +smlp -data "../data/smlp_toy_num_resp_noknobs.csv" -out_dir ./ -pref Test71 -mode verify -resp y1,y2 -feat x0,x1,x2 -model nn_keras -nnet_encoding nested -save_model t -use_model f -model_name test71_model -model_per_response t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1 -asrt_exprs "(y1**3+x2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -spec_fn smlp_toy_num_resp_mult_query_vacuous.spec +spec_fn smlp_toy_num_resp_mult_free_inps.spec specs_path ../specs -Running test 91 test type: query, description: test to detect contradictory constraints in optimization mode due to contradictory alpha global and alpha bounds constraints on FMAX_xyx -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test91 -mode query -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_query_vacuous.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 83 test type: optimize, description: basic dt_sklearn multi objective pareto optimization test with numeric labels and integer grid as domain and with scaling objectives +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test83 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps.spec -data_scaler min_max -beta "y1>7 and y2>6" -objv_names obj1,objv2,objv3 -objv_exprs "(y1+y2)/2;y1/2-y2;y2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +spec_fn smlp_toy_num_resp_mult_query.spec specs_path ../specs -Running test 100 test type: optimize, description: basic test for sat_threshold option enabing usage of objectve values in SAT assignments that prove optimization thresholds -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test100 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 89 test type: query, description: basic test in query mode to test stability (theta) and guard (eta) constraint generation +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test89 -mode query -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_query.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult.spec +spec_fn smlp_toy_num_resp_mult_query.spec specs_path ../specs -Running test 104 test type: verify, description: assertion verfication test with wrong spec that does not assign a single value using a singleton grid or range with equal max and min -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test104 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "y1*2+x<=5 and y1<=10;-2*y2-1<10-p2" -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 97 test type: query, description: basic test for rf_sklearn in model exploration mode optsyn +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test97 -mode query -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_bootstrap f -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_query.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_synthesize.spec specs_path ../specs Running test 108 test type: synthesize, description: basic test for dt_sklearn in model exploration mode synthesize where synthesis succeeds -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test108 -mode synthesize -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_synthesize.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test108 -mode synthesize -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_synthesize.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_system_stable_constant_synth_feasible.spec +spec_fn smlp_toy_system_stable_constant_certify.spec specs_path ../specs -Running test 121 test type: synthesize, description: synthesis test with constant knob and no inputs where synthesis is feasible -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test121 -mode synthesize -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 117 test type: certify, description: certification test with knobs only where assertion is valid without stability and fails with stability +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test117 -mode certify -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_certify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_configuration_verify.spec +spec_fn smlp_toy_system_stable_constant_synth_fail.spec specs_path ../specs -Running test 129 test type: verify, description: verification example with demonstrating all basic result scenarious for assertions -../../src/run_smlp.py -data "../data/smlp_toy_ctg_num_resp.csv" -out_dir ./ -pref Test129 -mode verify -resp y1,y2 -feat x,p1,p2 -model poly_sklearn -save_model f -use_model f -model_per_response f -spec ../specs/smlp_toy_configuration_verify.spec -asrt_names assert_stable_config,assert_grid_conflict,assert_unstable_config,assert_infeasible -asrt_exprs "y2<=90;y1>=9;y1>=(-10);y1>20" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 124 test type: optsyn, description: optimized synthesis test with constant knob and no inputs where synthesis is not feasible because while beta constraint is feasible the assertion is not feasible therefore optimization is not performed +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test124 -mode optsyn -pareto f -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_fail.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 150 test type: prediction, description: tests the mape loss function MeanAbsolutePercentageError and sample weights -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test150 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss mape -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +spec_fn smlp_toy_system_stable_verify.spec +specs_path ../specs +Running test 140 test type: verify, description: verification example with knobs only and fictitious inputs that have no effect where proparty is valid without stability and fails with stability +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test140 -mode verify -model system -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_verify.spec -trace_prec 1 -trace_anonym t -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 158 test type: prediction, description: tests the mape loss function and sample weights with model_per_response t -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test158 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss mape -model_per_response t -sw_coef 8 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics rmse -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 2 test type: prediction, description: basic rf_sklearn prediction test on labeled and new data with numeric labels -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test2 -mode predict -resp y1 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 15 -save_model_config f -mrmr_pred 0 -plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 147 test type: prediction, description: checks nn_keras prediction with sw_coef 0.8 and sequential API +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test147 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 5 test type: prediction, description: basic dt_caret prediction test on labeled and new data with numeric labels +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test5 -mode predict -resp y1 -feat x,p1,p2 -model dt_caret -save_model t -use_model f -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 12 test type: train, description: EV-SI real life dt_sklearn predict test on labeled and new data with numeric labels -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test12 -mode train -resp y1,y2 -feat x1,x2,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 16 test type: prediction, description: basic nn_keras prediction test from saved model on new data with numeric labels and two responses +smlp -model_name "../models/Test8_smlp_toy_num_resp_mult" -out_dir ./ -pref Test16 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -save_model f -use_model t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 19 test type: prediction, description: basic dt_sklearn prediction test using a model saved under a name specified through model_name option on new data with numeric labels -../../src/run_smlp.py -model_name "../models/test19_model" -out_dir ./ -pref Test19 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 24 test type: prediction, description: basic dt_sklearn prediction test using a model saved under a name specified through model_name option on new data with numeric labels +smlp -model_name "../models/test24_model" -out_dir ./ -pref Test24 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model t -model_per_response t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 27 test type: prediction, description: checks nn_keras prediction with nn_keras_seq_api t -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test27 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 30 test type: subgroups, description: basic test for subgroup discovery for numric responses +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test30 -mode subgroups -psg_dim 3 -psg_top 10 -resp y1,y2 -feat x,p1,p2 -plots t -seed 10 -log_time f -Running test 35 test type: doe, description: doe test with four levels with plackett_burman -../../src/run_smlp.py -doe_spec "../grids/doe_four_levels_real.csv" -out_dir ./ -pref Test35 -mode doe -doe_algo plackett_burman -log_time f +Running test 39 test type: doe, description: doe test with four levels with latin_hypercube +smlp -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test39 -mode doe -doe_algo latin_hypercube -doe_prob_distr Exponential -doe_samples 30 -log_time f -Running test 43 test type: doe, description: doe test with four levels with halton_sequence -../../src/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test43 -mode doe -doe_algo halton_sequence -doe_samples 20 -log_time f +Running test 47 test type: prediction, description: tests options -pos_val and -neg_val when re-using saved model +smlp -model_name "../models/test47_model" -out_dir ./ -pref Test47 -mode predict -resp "PF,PF1" -model poly_sklearn -save_model f -use_model t -data_scaler none -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -pos_val fail -neg_val pass -new_dat "../data/smlp_toy_pf_mult.csv" -Running test 51 test type: discretization, description: tests discretization options -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test51 -mode discretize -resp "PF,PF1" -discr_algo jenks -discr_bins 6 -discr_labels f -discr_type integer -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +Running test 56 test type: discretization, description: tests discretization options +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test56 -mode discretize -resp "PF,PF1" -discr_algo ranks -discr_bins 6 -discr_labels f -discr_type object -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass -spec_fn smlp_toy_num_resp_mult_y2_verify.spec +spec_fn smlp_toy_num_resp_noknobs_verify.spec specs_path ../specs -Running test 59 test type: verify, description: basic nn_keras assertion verification test for functional nn_keras model -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test59 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat +Running test 62 test type: verify, description: tests verificaion mode for NN with nn_keras_seq_api t +smlp -data "../data/smlp_toy_num_resp_noknobs.csv" -out_dir ./ -pref Test62 -mode verify -resp y1,y2 -feat x0,x1,x2 -model nn_keras -nnet_encoding nested -nn_keras_epochs 100 -nn_keras_seq_api t -save_model_config f -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "not(x25 and y1<=10);-2*y2-1<10-x2 and x2>5 and x2<8" -vacuity f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -spec_fn smlp_toy_num_resp_mult_free_inps.spec -specs_path ../specs -Running test 81 test type: optimize, description: basic dt_sklearn single objective optimization test with numeric labels and integer grid as domain and with scaling objectives -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test81 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps.spec -data_scaler min_max -objv_names obj1 -objv_exprs "(y1+y2)/2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat +Running test 75 test type: unknown, description: verification test run using model_rerun config covering the case when mrmr selcts only a subset of features specified through the command line or config file +smlp -model_name "../models/test73_model" -out_dir ./ -pref Test75 -config ../models/test73_model_rerun_model_config.json -new_dat "../data/smlp_toy_num_resp_noknobs_pred_labeled.csv" -spec_fn smlp_toy_num_resp_mult_query.spec +spec_fn smlp_toy_num_resp_mult.spec specs_path ../specs -Running test 89 test type: query, description: basic test in query mode to test stability (theta) and guard (eta) constraint generation -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test89 -mode query -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_query.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 79 test type: query, description: basic test in query mode to test stability (theta) and guard (eta) constraint generation +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test79 -mode query -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult.spec -quer_names query1,query2,query3 -quer_exprs "(y2**3+p2)/2<6;y1>=9;y2<0" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_query.spec +spec_fn smlp_toy_num_resp_mult_optsyn_vacuous.spec specs_path ../specs -Running test 97 test type: query, description: basic test for rf_sklearn in model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test97 -mode query -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_bootstrap f -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_query.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 90 test type: optsyn, description: test to detect contradictory constraints in optsyn mode +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test90 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn_vacuous.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 110 test type: prediction, description: smlp toy basic example for predict mode from SMLP user manual -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test110 -mode predict -resp y1,y2 -feat x1,x2,p1,p2 -model poly_sklearn -save_model t -model_name test110_model -save_model_config t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_basic_pred_unlabeled.csv" +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 98 test type: optsyn, description: basic test for et_caret in model exploration mode optsyn +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test98 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_caret -save_model f -use_model f -tree_encoding nested -compress_rules f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_system.spec specs_path ../specs -Running test 116 test type: certify, description: basic test in certify mode when system is specified and is used as the model; p2 rel-rad needs to be 0 or very close to it the witness to first query to be stable -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test116 -mode certify -resp y1,y2 -feat x1,x2,p1,p2 -model system -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_system.spec -quer_names query1,query2 -quer_exprs "y1>0;y2<=0" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 115 test type: certify, description: basic test in certify mode +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test115 -mode certify -resp y1,y2 -feat x1,x2,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_system.spec -quer_names query1,query2 -quer_exprs "y1>0;y2<=0" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_system_stable_certify.spec +spec_fn smlp_toy_system_stable_constant_synth_feasible.spec specs_path ../specs -Running test 127 test type: certify, description: certification example with knobs only and fictitious inputs with values fixed through their ranges -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test127 -mode certify -model system -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_certify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 123 test type: optimize, description: optimization test with constant knob and no inputs where synthesis is feasible and optimization is performed +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test123 -mode optimize -pareto t -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_query.spec +spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 143 test type: query, description: basic test for compress_rules for et_sklearn in mode query -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test143 -mode query -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_bootstrap f -tree_encoding nested -compress_rules t -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_query.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 142 test type: optsyn, description: basic test for compress_rules option for rf_sklearn in optsin mode +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test142 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 15 -tree_encoding nested -compress_rules t -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 160 test type: prediction, description: tests nn keras tuner bayesian -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test160 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss mape -nn_keras_metrics msle -nn_keras_tuner bayesian -nn_keras_layers_grid "2,3" -nn_keras_losses_grid "mse,mae,huber" -model_per_response f -sw_coef 8 -sw_exp 5 -sw_int 0.5 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 153 test type: prediction, description: tests the logcosh loss function LogCosh and sample weights +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test153 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_loss logcosh -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics mse -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -spec_fn -Running test 151 test type: prediction, description: tests msle loss function MeanSquaredLogarithmicError and and sample weoghts -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test151 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss msle -sw_coef 3 -sw_exp 10 -sw_int 0 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 161 test type: prediction, description: tests nn keras tuner bayesian +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test161 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_loss msle -nn_keras_metrics mape,logcosh -nn_keras_tuner random -nn_keras_lrates_grid "0.01,0.001" -nn_keras_batches_grid "32,64" -model_per_response f -sw_coef 4 -sw_exp 5 -sw_int 0.5 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test118 -mode verify -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_verify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_system_stable_constant_synth_feasible.spec +specs_path ../specs +Running test 125 test type: optsyn, description: optimized synthesis test with constant knob and no inputs where synthesis is feasible and optimization is performed +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test125 -mode optsyn -pareto t -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_noknobs_verify.spec +specs_path ../specs +Running test 144 test type: verify, description: basic test for compress_rules for dt_sklearn in mode verify and re-using saved model +smlp -data "../data/smlp_toy_num_resp_noknobs.csv" -out_dir ./ -pref Test144 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules t -save_model f -use_model f -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x0**2+y1>4.3;(y1+x2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +Running test 150 test type: prediction, description: tests the mape loss function MeanAbsolutePercentageError and sample weights +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test150 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss mape -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" spec_fn smlp_toy_num_resp_mult_verify.spec specs_path ../specs -Running test 156 test type: verify, description: basic nn_keras assertion verification test that uses keras tuner for functional model training; adapts test 154 by consdering multiple responses -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test156 -mode verify -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics rmse +Running test 157 test type: verify, description: basic nn_keras assertion verification test that uses keras tuner with sequrntial models for model training; adapts test 155 by consdering multiple responses +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test157 -mode verify -resp y1,y2 -feat x,p1,p2 --model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics rmse,logcosh + +spec_fn smlp_toy_num_resp_noknobs_verify.spec +specs_path ../specs +Running test 162 test type: verify, description: tests model term construction with flat_encoding of tress and model per reponse when mrmr_pred is activated and not all features are selected for training the model +smlp -data "../data/smlp_toy_num_resp_noknobs.csv" -out_dir ./ -pref Test162 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2,asrt3 -asrt_exprs "(y2**3+x2)/2<6;y1>=9;y2<0" -trace_anonym t -trace_prec 3 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_noknobs_pred_labeled.csv" spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs Running test 167 test type: optsyn, description: basic flat tree encoding test with model_per_response t for rf_sklearn in model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test167 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 4 -rf_sklearn_n_estimators 3 -tree_encoding flat -compress_rules t -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test167 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 4 -rf_sklearn_n_estimators 3 -tree_encoding flat -compress_rules t -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs Running test 171 test type: optimize, description: basic test for et_caret with flat tree_encoding in model exploration mode optimize -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test171 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_caret -tree_encoding flat -model_per_response t -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test171 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_caret -tree_encoding flat -model_per_response t -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 175 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response f nn_keras_seq_api t for nn_keras in model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test175 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 176 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response t nn_keras_seq_api f for nn_keras in model exploration mode optsyn +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test176 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api f -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn.spec +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec specs_path ../specs -Running test 178 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response t nn_keras_seq_api t for nn_keras in model exploration mode optsyn when features are not scaled adapts test 177 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test178 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response t -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 181 test type: optimize, description: basic flat tree encoding test for dt_sklearn multi objective pareto optimization when features are not scaled modifies test 164 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test181 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn.spec +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec specs_path ../specs -Running test 188 test type: optsyn, description: basic branched tree encoding test for dt_caretin model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test188 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_caret -tree_encoding branched -save_model f -use_model f -compress_rules f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 183 test type: optimize, description: basic flat tree encoding test for dt_sklearn multi objective pareto optimization when features and responses are not scaled modifies test 164 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test183 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -scale_resp f -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +spec_fn smlp_toy_num_resp_noknobs_verify.spec specs_path ../specs -Running test 196 test type: optimize, description: basic branched tree encoding test for dt_sklearn multi objective pareto optimization when features are not scaled modifies test 164 and test 181 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test196 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 185 test type: verify, description: tests model term construction with branched_encoding of tress and model per reponse when mrmr_pred is activated and not all features are selected for training the model +smlp -data "../data/smlp_toy_num_resp_noknobs.csv" -out_dir ./ -pref Test185 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2,asrt3 -asrt_exprs "(y2**3+x2)/2<6;y1>=9;y2<0" -trace_anonym t -trace_prec 3 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_noknobs_pred_labeled.csv" -spec_fn smlp_toy_num_resp_mult_no_input_beta.spec +spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 201 test type: optimize, description: basic dt_sklearn single objective optimization with the eager algorithm when there are no inputs and there are beta constraints -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test201 -mode optimize -pareto t -opt_strategy eager -resp y1,y2 -feat p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_no_input_beta.spec -data_scaler min_max -objv_names obj1 -objv_exprs "(y1+y2)/2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 191 test type: optimize, description: basic test for et_sklearn with branched tree_encoding and model_per_response t in model exploration mode optimize adapts test 169 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test191 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 3 -et_sklearn_bootstrap t -tree_encoding branched -model_per_response t -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_no_input.spec +spec_fn smlp_toy_system_stable_constant_synth_fail.spec specs_path ../specs -Running test 202 test type: optimize, description: basic dt_sklearn single objective optimization with the eager algorithm when there are no inputs and no beta constraints -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test202 -mode optimize -pareto t -opt_strategy eager -resp y1,y2 -feat p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_no_input.spec -data_scaler min_max -objv_names obj1 -objv_exprs "(y1+y2)/2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 122 test type: optimize, description: optimization test with constant knob and no inputs where synthesis is not feasible because the assertion is not feasible but beta constraint is feasible therefore optimization is performed +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test122 -mode optimize -pareto f -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_fail.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_system_stable_constant_synth_feasible.spec +spec_fn smlp_toy_num_resp_noknobs_verify.spec specs_path ../specs -Running test 206 test type: optsyn, description: optimized synthesis test with eager strategy and with constant knob and no inputs where synthesis is feasible and optimization is performed adapts test 125 -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test206 -mode optsyn -pareto t -opt_strategy eager -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 139 test type: verify, description: tests model term formation when mrmr_pred is activated and not all features are selected for training the model +smlp -data "../data/smlp_toy_num_resp_noknobs.csv" -out_dir ./ -pref Test139 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2,asrt3 -asrt_exprs "(y2**3+x2)/2<6;y1>=9;y2<0" -trace_anonym t -trace_prec 3 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_noknobs_pred_labeled.csv" -Running test 219 test type: correlate, description: basic test for correlate mode -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test219 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type integer -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +Running test 149 test type: prediction, description: tests the mae loss function MeanAbsoluteError and sample weoghts +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test149 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss mae -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 224 test type: correlate, description: basic test for correlate mode and tests the Shannon mutual information -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test224 -mode correlate -resp y1,y2 -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method shannon -mrmr_pred 0 -plots f -seed 10 -log_time f +spec_fn smlp_toy_num_resp_mult_verify.spec +specs_path ../specs +Running test 156 test type: verify, description: basic nn_keras assertion verification test that uses keras tuner for functional model training; adapts test 154 by consdering multiple responses +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test156 -mode verify -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics rmse spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec specs_path ../specs Running test 164 test type: optimize, description: basic flat tree encoding test for dt_sklearn multi objective pareto optimization -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test164 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test164 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 168 test type: optimize, description: basic test for rf_caret with flat tree_encoding and modelper_response in model exploration mode optimize -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test168 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_caret -model_per_response t -compress_rules t -tree_encoding flat -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 166 test type: optsyn, description: basic flat tree encoding test with model_per_response f for rf_sklearn in model exploration mode optsyn +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test166 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 4 -rf_sklearn_n_estimators 3 -tree_encoding flat -compress_rules t -save_model f -use_model f -compress_rules t -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_y2_verify.spec +spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 173 test type: verify, description: basic test for nn_keras flat encoding for sequential api -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test173 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3,3,3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics mae -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat"" +Running test 170 test type: optimize, description: basic test for et_sklearn with flat tree_encoding and model_per_response f in model exploration mode optimize +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test170 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -rf_sklearn_n_estimators 3 -et_sklearn_bootstrap f -tree_encoding flat -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 183 test type: optimize, description: basic flat tree encoding test for dt_sklearn multi objective pareto optimization when features and responses are not scaled modifies test 164 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test183 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -scale_resp f -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 175 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response f nn_keras_seq_api t for nn_keras in model exploration mode optsyn +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test175 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 191 test type: optimize, description: basic test for et_sklearn with branched tree_encoding and model_per_response t in model exploration mode optimize adapts test 169 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test191 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 3 -et_sklearn_bootstrap t -tree_encoding branched -model_per_response t -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 179 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response f nn_keras_seq_api f for nn_keras in model exploration mode optsyn when resposes are not scaled adapts test 174 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test179 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api f -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response f -scale_resp f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_noknobs_verify.spec +specs_path ../specs +Running test 184 test type: verify, description: tests model term formation when mrmr_pred is activated and not all features are selected for training the model adapts test 139 +smlp -data "../data/smlp_toy_num_resp_noknobs.csv" -out_dir ./ -pref Test184 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2,asrt3 -asrt_exprs "(y2**3+x2)/2<6;y1>=9;y2<0" -trace_anonym t -trace_prec 3 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_noknobs_pred_labeled.csv" spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 195 test type: optimize, description: basic test for et_sklearn with branched tree_encoding and model_per_response f in model exploration mode optimize adapts test 192 by setting n_estimators 3 and then discrepancy between z3 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test195 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 3 -et_sklearn_bootstrap f -tree_encoding branched -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 189 test type: optsyn, description: basic branched tree encoding test with model_per_response f for rf_sklearn in model exploration mode optsyn adapts test 166 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test189 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 4 -rf_sklearn_n_estimators 3 -tree_encoding branched -compress_rules t -save_model f -use_model f -compress_rules t -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 199 test type: optimize, description: test to demonstrate that in pareto optimization and optsyn modes with multiple objectives when beta constraints are not present SMLP results are not consistent when different solvers are used; this is due to fact that when a subset of objectoves are exemined in pareto algo -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test199 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 100 -et_sklearn_bootstrap f -tree_encoding branched -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 193 test type: optimize, description: basic test for et_caret with branched tree_encoding in model exploration mode optimize adapts test 171 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test193 -mode optimize -resp y1,y2 -feat x,p1,p2 -model et_caret -tree_encoding branched -model_per_response t -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 218 test type: correlate, description: basic test for correlate mode -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test218 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type ordered -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +spec_fn smlp_toy_system_stable_constant_synth_fail.spec +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 168 test type: optimize, description: basic test for rf_caret with flat tree_encoding and modelper_response in model exploration mode optimize +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test168 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_caret -model_per_response t -compress_rules t -tree_encoding flat -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_y2_verify.spec +specs_path ../specs +Running test 173 test type: verify, description: basic test for nn_keras flat encoding for sequential api +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test173 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3,3,3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics mae -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat"" + +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 199 test type: optimize, description: test to demonstrate that in pareto optimization and optsyn modes with multiple objectives when beta constraints are not present SMLP results are not consistent when different solvers are used; this is due to fact that when a subset of objectoves are exemined in pareto algo +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test199 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 100 -et_sklearn_bootstrap f -tree_encoding branched -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f Running test 223 test type: correlate, description: basic test for correlate mode and tests the normalized mutual information -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test223 -mode correlate -resp y1,y2 -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method normalized -mrmr_pred 0 -plots f -seed 10 -log_time f +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test223 -mode correlate -resp y1,y2 -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method normalized -mrmr_pred 0 -plots f -seed 10 -log_time f -spec_fn smlp_toy_system_radii_update_certify.spec +spec_fn smlp_toy_missing_radii.spec specs_path ../specs -Running test 228 test type: certify, description: test that radii specified in command line properly override the radii specified in the spec file. Here we override both ansolute and relative radii and one can observe that the certification results also change compared to test 116 -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test228 -mode certify -resp y1,y2 -feat x1,x2,p1,p2 -model system -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_system_radii_update_certify.spec -rad_rel 0.005 -rad_abs 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 229 test type: certify, description: basic test for checking that each knob must have either absolute or relative radius specified in the spec file (even if radii are specified in the command line) +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test229 -mode certify -resp y1,y2 -feat x1,x2,p1,p2 -model system -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_missing_radii.spec -rad_rel 0.005 -rad_abs 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 233 test type: subgroups, description: tests subgroup discovery mode when the response has string values -../../src/run_smlp.py -data "../data/smlp_toy_string_response.csv" -out_dir ./ -pref Test233 -mode subgroups -resp str_resp1 -feat num,int,str -pos_val no -neg_val yes -seed 10 -log_time f - smlp_toy_num_resp_mult_optsyn.spec +Running test 148 test type: prediction, description: checks nn_keras prediction with sw_coef 0.8 and sequential API +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test148 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +spec_fn smlp_toy_num_resp_mult_y2_verify.spec specs_path ../specs -Running test 177 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response t nn_keras_seq_api t for nn_keras in model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test177 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 155 test type: verify, description: basic nn_keras assertion verification test that uses keras tuner with sequrntial models for model training +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test155 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3,3,3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics mae + +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 180 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response f nn_keras_seq_api t for nn_keras in model exploration mode optsyn when features and responses are not scaled adapts test 175 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test180 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response f -scale_feat f -scale_resp f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec specs_path ../specs Running test 182 test type: optimize, description: basic flat tree encoding test for dt_sklearn multi objective pareto optimization when responses are not scaled modifies test 164 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test182 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -scale_resp f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat"" +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test182 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -scale_resp f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat"" + +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +specs_path ../specs +Running test 187 test type: optimize, description: basic branched tree encoding test for dt_sklearn multi objective pareto optimization adapts test 164 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test187 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 192 test type: optimize, description: basic test for et_sklearn with branched tree_encoding and model_per_response f in model exploration mode optimize adapts test 170 !!!!!!!!! in this test z3 result differs from mathsat and yices results (the latter two give sma results -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test192 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 100 -et_sklearn_bootstrap f -tree_encoding branched -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 194 test type: optsyn, description: basic branched tree encoding test with model_per_response t for rf_sklearn in model exploration mode optsyn +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test194 -mode optsyn -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 4 -rf_sklearn_n_estimators 3 -tree_encoding branched -compress_rules t -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +specs_path ../specs +Running test 198 test type: optimize, description: basic branched tree encoding test for dt_sklearn multi objective pareto optimization when features and responses are not scaled modifies test 164 and test 183 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test198 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -scale_resp f -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat + +Running test 216 test type: correlate, description: basic test for correlate mode +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test216 -mode correlate -resp y1,y2 -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f + +Running test 221 test type: correlate, description: basic test for correlate mode +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test221 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method shannon -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +spec_fn smlp_toy_system_radii_update_certify.spec +specs_path ../specs +Running test 228 test type: certify, description: test that radii specified in command line properly override the radii specified in the spec file. Here we override both ansolute and relative radii and one can observe that the certification results also change compared to test 116 +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test228 -mode certify -resp y1,y2 -feat x1,x2,p1,p2 -model system -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_system_radii_update_certify.spec -rad_rel 0.005 -rad_abs 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + + +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +specs_path ../specs +Running test 196 test type: optimize, description: basic branched tree encoding test for dt_sklearn multi objective pareto optimization when features are not scaled modifies test 164 and test 181 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test196 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_no_input.spec +specs_path ../specs +Running test 202 test type: optimize, description: basic dt_sklearn single objective optimization with the eager algorithm when there are no inputs and no beta constraints +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test202 -mode optimize -pareto t -opt_strategy eager -resp y1,y2 -feat p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_no_input.spec -data_scaler min_max -objv_names obj1 -objv_exprs "(y1+y2)/2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_system_stable_constant_synth_feasible.spec specs_path ../specs -Running test 204 test type: optimize, description: optimization test with eager strategy and with constant knob and no inputs where synthesis is feasible and optimization is performed adapts test 123 -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test204 -mode optimize -pareto t -opt_strategy eager -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 205 test type: optimize, description: optimization test with eager strategy and with constant knob and no inputs where synthesis is feasible and optimization is performed adapts test 145 +smlp -out_dir ./ -pref Test205 -mode optimize -pareto t -opt_strategy eager -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -doe_spec ../grids/doe_two_levels_opt.csv -doe_algo latin_hypercube -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 217 test type: correlate, description: basic test for correlate mode -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test217 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type category -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +Running test 218 test type: correlate, description: basic test for correlate mode +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test218 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type ordered -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass -Running test 222 test type: correlate, description: basic test for correlate mode -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test222 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method adjusted -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +Running test 226 test type: correlate, description: basic test for correlate mode +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test226 -mode correlate -resp y1,y2 -discr_algo uniform -discret_num t -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f -Running test 227 test type: correlate, description: basic test for correlate mode and tests the normalized mutual information -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test227 -mode correlate -resp y1,y2 -discr_algo uniform -discret_num t -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method normalized -mrmr_pred 0 -plots f -seed 10 -log_time f +Running test 233 test type: subgroups, description: tests subgroup discovery mode when the response has string values +smlp -data "../data/smlp_toy_string_response.csv" -out_dir ./ -pref Test233 -mode subgroups -resp str_resp1 -feat num,int,str -pos_val no -neg_val yes -seed 10 -log_time f -Running test 234 test type: subgroups, description: tests subgroup discovery mode when there are two responses with string values -../../src/run_smlp.py -data "../data/smlp_toy_string_response.csv" -out_dir ./ -pref Test234 -mode subgroups -resp str_resp1,str_resp2 -feat num,int,str -pos_val no -neg_val yes -seed 10 -log_time f +Running test 158 test type: prediction, description: tests the mape loss function and sample weights with model_per_response t +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test158 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss mape -model_per_response t -sw_coef 8 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics rmse -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -spec_fn smlp_toy_num_resp_mult_optsyn.spec +spec_fn smlp_toy_num_resp_noknobs_verify.spec specs_path ../specs -Running test 166 test type: optsyn, description: basic flat tree encoding test with model_per_response f for rf_sklearn in model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test166 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 4 -rf_sklearn_n_estimators 3 -tree_encoding flat -compress_rules t -save_model f -use_model f -compress_rules t -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 163 test type: verify, description: tests model term formation when mrmr_pred is activated and not all features are selected for training the model +smlp -data "../data/smlp_toy_num_resp_noknobs.csv" -out_dir ./ -pref Test163 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2,asrt3 -asrt_exprs "(y2**3+x2)/2<6;y1>=9;y2<0" -trace_anonym t -trace_prec 3 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_noknobs_pred_labeled.csv" spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs Running test 169 test type: optimize, description: basic test for et_sklearn with flat tree_encoding and model_per_response t in model exploration mode optimize -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test169 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -rf_sklearn_n_estimators 3 -et_sklearn_bootstrap f -tree_encoding flat -model_per_response t -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test169 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -rf_sklearn_n_estimators 3 -et_sklearn_bootstrap f -tree_encoding flat -model_per_response t -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 176 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response t nn_keras_seq_api f for nn_keras in model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test176 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api f -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 174 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response f nn_keras_seq_api f for nn_keras in model exploration mode optsyn +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test174 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api f -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 179 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response f nn_keras_seq_api f for nn_keras in model exploration mode optsyn when resposes are not scaled adapts test 174 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test179 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api f -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response f -scale_resp f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 192 test type: optimize, description: basic test for et_sklearn with branched tree_encoding and model_per_response f in model exploration mode optimize adapts test 170 !!!!!!!!! in this test z3 result differs from mathsat and yices results (the latter two give sma results +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test192 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 100 -et_sklearn_bootstrap f -tree_encoding branched -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +Running test 215 test type: correlate, description: basic test for correlate mode +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test215 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +Running test 222 test type: correlate, description: basic test for correlate mode +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test222 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method adjusted -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +spec_fn smlp_toy_system_monotone_knob.05_verify.spec specs_path ../specs -Running test 181 test type: optimize, description: basic flat tree encoding test for dt_sklearn multi objective pareto optimization when features are not scaled modifies test 164 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test181 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 230 test type: verify, description: tests that outputs in system specificaation might depend on different inuts (knobs and free inputs) +smlp -data "../data/smlp_toy_monotone_basic.csv" -out_dir ./ -pref Test230 -mode verify -spec ../specs/smlp_toy_system_monotone_knob.05_verify.spec -model system -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec + +spec_fn smlp_toy_num_resp_mult_y2_verify.spec specs_path ../specs -Running test 187 test type: optimize, description: basic branched tree encoding test for dt_sklearn multi objective pareto optimization adapts test 164 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test187 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 154 test type: verify, description: basic nn_keras assertion verification test that uses keras tuner for functional model training +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test154 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3,3,3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 194 test type: optsyn, description: basic branched tree encoding test with model_per_response t for rf_sklearn in model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test194 -mode optsyn -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 4 -rf_sklearn_n_estimators 3 -tree_encoding branched -compress_rules t -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 178 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response t nn_keras_seq_api t for nn_keras in model exploration mode optsyn when features are not scaled adapts test 177 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test178 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response t -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +spec_fn smlp_toy_num_resp_noknobs_verify.spec specs_path ../specs -Running test 197 test type: optimize, description: basic branched tree encoding test for dt_sklearn multi objective pareto optimization when responses are not scaled modifies test 164 and test 182 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test197 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -scale_resp f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat"" +Running test 186 test type: verify, description: tests model term formation when mrmr_pred is activated and not all features are selected for training the model +smlp -data "../data/smlp_toy_num_resp_noknobs.csv" -out_dir ./ -pref Test186 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2,asrt3 -asrt_exprs "(y2**3+x2)/2<6;y1>=9;y2<0" -trace_anonym t -trace_prec 3 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_noknobs_pred_labeled.csv" -spec_fn smlp_toy_system_stable_constant_synth_fail.spec +spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 203 test type: optimize, description: optimization test with eager strategy and with constant knob and no inputs where synthesis is not feasible because the assertion is not feasible but beta constraint is feasible therefore optimization is performed adapts test 122 -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test203 -mode optimize -pareto f -opt_strategy eager -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_fail.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 190 test type: optimize, description: basic test for rf_caret with branched tree_encoding and modelper_response in model exploration mode optimize adapts test 168 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test190 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_caret -model_per_response t -compress_rules t -tree_encoding branched -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 216 test type: correlate, description: basic test for correlate mode -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test216 -mode correlate -resp y1,y2 -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f +spec_fn smlp_toy_num_resp_mult_no_input_beta.spec +specs_path ../specs +Running test 201 test type: optimize, description: basic dt_sklearn single objective optimization with the eager algorithm when there are no inputs and there are beta constraints +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test201 -mode optimize -pareto t -opt_strategy eager -resp y1,y2 -feat p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_no_input_beta.spec -data_scaler min_max -objv_names obj1 -objv_exprs "(y1+y2)/2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 220 test type: correlate, description: basic test for correlate mode -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test220 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method normalized -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +spec_fn smlp_toy_system_stable_constant_synth_feasible.spec +specs_path ../specs +Running test 204 test type: optimize, description: optimization test with eager strategy and with constant knob and no inputs where synthesis is feasible and optimization is performed adapts test 123 +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test204 -mode optimize -pareto t -opt_strategy eager -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +Running test 219 test type: correlate, description: basic test for correlate mode +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test219 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type integer -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass Running test 225 test type: correlate, description: basic test for correlate mode and tests the adjusted mutual information -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test225 -mode correlate -resp y1,y2 -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method adjusted -mrmr_pred 0 -plots f -seed 10 -log_time f +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test225 -mode correlate -resp y1,y2 -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method adjusted -mrmr_pred 0 -plots f -seed 10 -log_time f -spec_fn smlp_toy_system_monotone_knob.05_verify.spec +spec_fn smlp_toy_system_running_example_certify.spec specs_path ../specs -Running test 230 test type: verify, description: tests that outputs in system specificaation might depend on different inuts (knobs and free inputs) -../../src/run_smlp.py -data "../data/smlp_toy_monotone_basic.csv" -out_dir ./ -pref Test230 -mode verify -spec ../specs/smlp_toy_system_monotone_knob.05_verify.spec -model system -seed 10 -log_time f +Running test 232 test type: certify, description: running example from smlp manual +smlp -data "../data/smlp_toy_system_running_example_certify.csv" -out_dir ./ -pref Test232 -mode certify -spec ../specs/smlp_toy_system_running_example_certify.spec -model system -seed 10 -log_time f -Running test 159 test type: prediction, description: tests the msle loss function and sample weights with model_per_response t -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test159 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_loss msle -model_per_response t -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics mae,cosine -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +specs_path ../specs +Running test 203 test type: optimize, description: optimization test with eager strategy and with constant knob and no inputs where synthesis is not feasible because the assertion is not feasible but beta constraint is feasible therefore optimization is performed adapts test 122 +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test203 -mode optimize -pareto f -opt_strategy eager -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_fail.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn.spec +Running test 217 test type: correlate, description: basic test for correlate mode +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test217 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type category -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +Running test 224 test type: correlate, description: basic test for correlate mode and tests the Shannon mutual information +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test224 -mode correlate -resp y1,y2 -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method shannon -mrmr_pred 0 -plots f -seed 10 -log_time f + +spec_fn smlp_toy_system_decreasing_knob.05_certify.spec specs_path ../specs -Running test 165 test type: optsyn, description: basic flat tree encoding test for dt_caretin model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test165 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_caret -tree_encoding flat -save_model f -use_model f -compress_rules f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 231 test type: certify, description: certification test with monotonicity query with a knob with a grid point +smlp -data "../data/smlp_toy_monotone_basic.csv" -out_dir ./ -pref Test231 -mode certify -spec ../specs/smlp_toy_system_decreasing_knob.05_certify.spec -model system -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_y2_verify.spec + smlp_toy_witness_certify.spec specs_path ../specs -Running test 172 test type: verify, description: basic test for nn_keras flat encoding for functional api -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test172 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nnet_encoding layered -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3,3,3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat"" +Running test 128 test type: certify, description: Basic regression test in certify mode covering all four possible outcomes when certifying a witness for a query: the witness is stable +smlp -data "../data/smlp_toy_ctg_num_resp.csv" -out_dir ./ -pref Test128 -mode certify -resp y1,y2 -feat x,p1,p2 -model poly_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model f -model_per_response f -spec ../specs/smlp_toy_witness_certify.spec -quer_names query_stable_witness,query_grid_conflict,query_unstable_witness,query_infeasible_witness,query_poly_intercept_sensitive -quer_exprs "y2<=90;y1>=9;y1>=(-13);y1>9;y1>=(-10)" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_system.spec +specs_path ../specs +Running test 146 test type: optimize, description: optimization test with constant knob and no inputs where synthesis is feasible and optimization is performed +smlp -out_dir ./ -pref Test146 -mode optimize -pareto t -opt_strategy lazy -model poly_sklearn -resp y1,y2 -feat p1,p2,x1,x2 -save_model t -use_model f -mrmr_pred 0 -model_per_response t -split 1 -spec ../specs/smlp_toy_system.spec -doe_spec ../grids/explore_doe_two_levels.csv -doe_algo latin_hypercube -epsilon 0.99999999 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +Running test 152 test type: prediction, description: tests the huber loss function Huber and sample weights +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test152 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss huber -sw_coef 8 -sw_exp 5 -sw_int 0.5 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +Running test 160 test type: prediction, description: tests nn keras tuner bayesian +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test160 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss mape -nn_keras_metrics msle -nn_keras_tuner bayesian -nn_keras_layers_grid "2,3" -nn_keras_losses_grid "mse,mae,huber" -model_per_response f -sw_coef 8 -sw_exp 5 -sw_int 0.5 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 180 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response f nn_keras_seq_api t for nn_keras in model exploration mode optsyn when features and responses are not scaled adapts test 175 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test180 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response f -scale_feat f -scale_resp f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 177 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response t nn_keras_seq_api t for nn_keras in model exploration mode optsyn +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test177 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 190 test type: optimize, description: basic test for rf_caret with branched tree_encoding and modelper_response in model exploration mode optimize adapts test 168 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test190 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_caret -model_per_response t -compress_rules t -tree_encoding branched -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 188 test type: optsyn, description: basic branched tree encoding test for dt_caretin model exploration mode optsyn +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test188 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_caret -tree_encoding branched -save_model f -use_model f -compress_rules f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec specs_path ../specs -Running test 198 test type: optimize, description: basic branched tree encoding test for dt_sklearn multi objective pareto optimization when features and responses are not scaled modifies test 164 and test 183 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test198 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -scale_resp f -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat +Running test 197 test type: optimize, description: basic branched tree encoding test for dt_sklearn multi objective pareto optimization when responses are not scaled modifies test 164 and test 182 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test197 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -scale_resp f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat"" spec_fn smlp_toy_system_stable_constant_synth_feasible.spec specs_path ../specs -Running test 205 test type: optimize, description: optimization test with eager strategy and with constant knob and no inputs where synthesis is feasible and optimization is performed adapts test 145 -../../src/run_smlp.py -out_dir ./ -pref Test205 -mode optimize -pareto t -opt_strategy eager -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -doe_spec ../grids/doe_two_levels_opt.csv -doe_algo latin_hypercube -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -Running test 215 test type: correlate, description: basic test for correlate mode -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test215 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +Running test 206 test type: optsyn, description: optimized synthesis test with eager strategy and with constant knob and no inputs where synthesis is feasible and optimization is performed adapts test 125 +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test206 -mode optsyn -pareto t -opt_strategy eager -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 221 test type: correlate, description: basic test for correlate mode -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test221 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method shannon -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +Running test 220 test type: correlate, description: basic test for correlate mode +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test220 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method normalized -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass -Running test 226 test type: correlate, description: basic test for correlate mode -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test226 -mode correlate -resp y1,y2 -discr_algo uniform -discret_num t -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f +Running test 227 test type: correlate, description: basic test for correlate mode and tests the normalized mutual information +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test227 -mode correlate -resp y1,y2 -discr_algo uniform -discret_num t -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method normalized -mrmr_pred 0 -plots f -seed 10 -log_time f -spec_fn smlp_toy_system_running_example_certify.spec -specs_path ../specs -Running test 232 test type: certify, description: running example from smlp manual -../../src/run_smlp.py -data "../data/smlp_toy_system_running_example_certify.csv" -out_dir ./ -pref Test232 -mode certify -spec ../specs/smlp_toy_system_running_example_certify.spec -model system -seed 10 -log_time f +Running test 234 test type: subgroups, description: tests subgroup discovery mode when there are two responses with string values +smlp -data "../data/smlp_toy_string_response.csv" -out_dir ./ -pref Test234 -mode subgroups -resp str_resp1,str_resp2 -feat num,int,str -pos_val no -neg_val yes -seed 10 -log_time f -spec_fn smlp_toy_missing_radii.spec +spec_fn smlp_toy_configuration_verify.spec specs_path ../specs -Running test 229 test type: certify, description: basic test for checking that each knob must have either absolute or relative radius specified in the spec file (even if radii are specified in the command line) -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test229 -mode certify -resp y1,y2 -feat x1,x2,p1,p2 -model system -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_missing_radii.spec -rad_rel 0.005 -rad_abs 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 129 test type: verify, description: verification example with demonstrating all basic result scenarious for assertions +smlp -data "../data/smlp_toy_ctg_num_resp.csv" -out_dir ./ -pref Test129 -mode verify -resp y1,y2 -feat x,p1,p2 -model poly_sklearn -save_model f -use_model f -model_per_response f -spec ../specs/smlp_toy_configuration_verify.spec -asrt_names assert_stable_config,assert_grid_conflict,assert_unstable_config,assert_infeasible -asrt_exprs "y2<=90;y1>=9;y1>=(-10);y1>20" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_system_decreasing_knob.05_certify.spec +spec_fn smlp_toy_system_stable_constant_synth_feasible.spec specs_path ../specs -Running test 231 test type: certify, description: certification test with monotonicity query with a knob with a grid point -../../src/run_smlp.py -data "../data/smlp_toy_monotone_basic.csv" -out_dir ./ -pref Test231 -mode certify -spec ../specs/smlp_toy_system_decreasing_knob.05_certify.spec -model system -seed 10 -log_time f - - -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test140 -mode verify -model system -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_verify.spec -trace_prec 1 -trace_anonym t -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -Running test 147 test type: prediction, description: checks nn_keras prediction with sw_coef 0.8 and sequential API -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test147 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - -Running test 153 test type: prediction, description: tests the logcosh loss function LogCosh and sample weights -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test153 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_loss logcosh -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics mse -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 145 test type: optimize, description: optimization test with constant knob and no inputs where synthesis is feasible and optimization is performed +smlp -out_dir ./ -pref Test145 -mode optimize -pareto t -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -doe_spec ../grids/doe_two_levels_opt.csv -doe_algo latin_hypercube -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 161 test type: prediction, description: tests nn keras tuner bayesian -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test161 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_loss msle -nn_keras_metrics mape,logcosh -nn_keras_tuner random -nn_keras_lrates_grid "0.01,0.001" -nn_keras_batches_grid "32,64" -model_per_response f -sw_coef 4 -sw_exp 5 -sw_int 0.5 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 151 test type: prediction, description: tests msle loss function MeanSquaredLogarithmicError and and sample weoghts +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test151 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss msle -sw_coef 3 -sw_exp 10 -sw_int 0 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -spec_fn smlp_toy_num_resp_mult_optsyn.spec -specs_path ../specs -Running test 170 test type: optimize, description: basic test for et_sklearn with flat tree_encoding and model_per_response f in model exploration mode optimize -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test170 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -rf_sklearn_n_estimators 3 -et_sklearn_bootstrap f -tree_encoding flat -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 159 test type: prediction, description: tests the msle loss function and sample weights with model_per_response t +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test159 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_loss msle -model_per_response t -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics mae,cosine -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 174 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response f nn_keras_seq_api f for nn_keras in model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test174 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api f -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 165 test type: optsyn, description: basic flat tree encoding test for dt_caretin model exploration mode optsyn +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test165 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_caret -tree_encoding flat -save_model f -use_model f -compress_rules f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn.spec +spec_fn smlp_toy_num_resp_mult_y2_verify.spec specs_path ../specs -Running test 189 test type: optsyn, description: basic branched tree encoding test with model_per_response f for rf_sklearn in model exploration mode optsyn adapts test 166 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test189 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 4 -rf_sklearn_n_estimators 3 -tree_encoding branched -compress_rules t -save_model f -use_model f -compress_rules t -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 172 test type: verify, description: basic test for nn_keras flat encoding for functional api +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test172 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nnet_encoding layered -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3,3,3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat"" spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 193 test type: optimize, description: basic test for et_caret with branched tree_encoding in model exploration mode optimize adapts test 171 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test193 -mode optimize -resp y1,y2 -feat x,p1,p2 -model et_caret -tree_encoding branched -model_per_response t -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 195 test type: optimize, description: basic test for et_sklearn with branched tree_encoding and model_per_response f in model exploration mode optimize adapts test 192 by setting n_estimators 3 and then discrepancy between z3 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test195 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 3 -et_sklearn_bootstrap f -tree_encoding branched -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs Running test 200 test type: optimize, description: basic test for et_sklearn with branched tree_encoding and model_per_response f in model exploration mode optimize adapts test 170 !!!!!!!!! in this test z3 result differs from mathsat and yices results (the latter two give sma results -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test200 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 100 -et_sklearn_bootstrap f -tree_encoding branched -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - - ../specs -Running test 142 test type: optsyn, description: basic test for compress_rules option for rf_sklearn in optsin mode -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test142 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 15 -tree_encoding nested -compress_rules t -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -spec_fn smlp_toy_num_resp_mult_y2_verify.spec -specs_path ../specs -Running test 154 test type: verify, description: basic nn_keras assertion verification test that uses keras tuner for functional model training -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test154 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3,3,3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 - - -specs_path ../specs -Running test 155 test type: verify, description: basic nn_keras assertion verification test that uses keras tuner with sequrntial models for model training -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test155 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3,3,3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics mae +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test200 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 100 -et_sklearn_bootstrap f -tree_encoding branched -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f Initiating 7 worker... comparing Test1_smlp_toy_num_resp_mult_y1_dt_caret_tree_rules.txt to master @@ -1611,9 +1704,34 @@ comparing Test56_smlp_toy_mult_discr.txt to master Passed! comparing Test56_smlp_toy_mult_discr_missing_values_dict.json to master Passed! -Test 57 Failed: -Error in Build stage: -Data file does not exist +comparing Test57_smlp_toy_num_resp_noknobs_dt_sklearn_tree_rules.txt to master +Passed! +comparing Test57_smlp_toy_num_resp_noknobs.txt to master +Passed! +comparing Test57_smlp_toy_num_resp_noknobs_data_bounds.json to master +Passed! +comparing Test57_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv to master +Passed! +comparing Test57_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv to master +Passed! +comparing Test57_smlp_toy_num_resp_noknobs_missing_values_dict.json to master +Passed! +comparing Test57_smlp_toy_num_resp_noknobs_model_features_dict.json to master +Passed! +comparing Test57_smlp_toy_num_resp_noknobs_model_levels_dict.json to master +Passed! +comparing Test57_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv to master +Passed! +comparing Test57_smlp_toy_num_resp_noknobs_test_predictions_summary.csv to master +Passed! +comparing Test57_smlp_toy_num_resp_noknobs_trace.csv to master +Passed! +comparing Test57_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv to master +Passed! +comparing Test57_smlp_toy_num_resp_noknobs_training_predictions_summary.csv to master +Passed! +comparing Test57_smlp_toy_num_resp_noknobs_verify_results.json to master +Passed! comparing Test58_smlp_toy_num_resp_mult.txt to master Passed! comparing Test58_smlp_toy_num_resp_mult_data_bounds.json to master @@ -1664,158 +1782,415 @@ comparing Test59_smlp_toy_num_resp_mult_model_levels_dict.json to master Passed! comparing Test59_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master Passed! -comparing Test59_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +comparing Test59_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test59_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test59_smlp_toy_num_resp_mult_train-reg_y2_mse.png does not exist +comparing Test59_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test59_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +comparing Test59_smlp_toy_num_resp_mult_verify_results.json to master +Passed! +File master Test59_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist +File master Test59_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist +comparing Test60_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test60_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +comparing Test60_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test60_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test60_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test60_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test60_smlp_toy_num_resp_mult_model_gen.json to master +Passed! +comparing Test60_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test60_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test60_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test60_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test60_smlp_toy_num_resp_mult_train-reg_y2_mse.png does not exist +comparing Test60_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test60_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +comparing Test60_smlp_toy_num_resp_mult_verify_results.json to master +Passed! +File master Test60_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist +File master Test60_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist +comparing Test61_smlp_toy_num_resp_noknobs.txt to master +Passed! +comparing Test61_smlp_toy_num_resp_noknobs_data_bounds.json to master +Passed! +comparing Test61_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv to master +Passed! +comparing Test61_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv to master +Passed! +comparing Test61_smlp_toy_num_resp_noknobs_missing_values_dict.json to master +Passed! +comparing Test61_smlp_toy_num_resp_noknobs_model_features_dict.json to master +Passed! +comparing Test61_smlp_toy_num_resp_noknobs_model_gen.json to master +Passed! +comparing Test61_smlp_toy_num_resp_noknobs_model_levels_dict.json to master +Passed! +comparing Test61_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv to master +Passed! +comparing Test61_smlp_toy_num_resp_noknobs_test_predictions_summary.csv to master +Passed! +comparing Test61_smlp_toy_num_resp_noknobs_trace.csv to master +Passed! +File master Test61_smlp_toy_num_resp_noknobs_train-reg_y1_mse.png does not exist +File master Test61_smlp_toy_num_resp_noknobs_train-reg_y2_mse.png does not exist +comparing Test61_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv to master +Passed! +comparing Test61_smlp_toy_num_resp_noknobs_training_predictions_summary.csv to master +Passed! +comparing Test61_smlp_toy_num_resp_noknobs_verify_results.json to master +Passed! +comparing Test62_smlp_toy_num_resp_noknobs.txt to master +Passed! +comparing Test62_smlp_toy_num_resp_noknobs_data_bounds.json to master +Passed! +comparing Test62_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv to master +Passed! +comparing Test62_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv to master +Passed! +comparing Test62_smlp_toy_num_resp_noknobs_missing_values_dict.json to master +Passed! +comparing Test62_smlp_toy_num_resp_noknobs_model_features_dict.json to master +Passed! +comparing Test62_smlp_toy_num_resp_noknobs_model_gen.json to master +Passed! +comparing Test62_smlp_toy_num_resp_noknobs_model_levels_dict.json to master +Passed! +comparing Test62_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv to master +Passed! +comparing Test62_smlp_toy_num_resp_noknobs_test_predictions_summary.csv to master +Passed! +comparing Test62_smlp_toy_num_resp_noknobs_trace.csv to master +Passed! +File master Test62_smlp_toy_num_resp_noknobs_train-reg_all_responses_mse.png does not exist +comparing Test62_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv to master +Passed! +comparing Test62_smlp_toy_num_resp_noknobs_training_predictions_summary.csv to master +Passed! +comparing Test62_smlp_toy_num_resp_noknobs_verify_results.json to master +Passed! +File master test63_model_dt_sklearn_y1_tree_rules.txt does not exist +comparing Test63_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test63_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test63_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test63_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test63_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test63_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test63_smlp_toy_num_resp_mult_trace.csv does not exist +comparing Test63_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test63_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +comparing Test63_smlp_toy_num_resp_mult_verify_results.json to master +Passed! +comparing test63_model_data_bounds.json to master +Passed! +comparing test63_model_model_features_dict.json to master +Passed! +comparing test63_model_model_levels_dict.json to master +Passed! +comparing test63_model_rerun_model_config.json to master +Passed! +File master test63_model_y1_smlp_full_model_term.json does not exist +File master test63_model_y1_smlp_model_term.json does not exist +comparing Test64_test63_model.txt to master +File master Test64_test63_model_trace.csv does not exist +comparing Test64_test63_model_verify_results.json to master +Passed! +comparing Test65_smlp_toy_num_resp_noknobs.txt to master +Passed! +comparing Test65_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv to master +Passed! +comparing Test65_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv to master +Passed! +comparing Test65_smlp_toy_num_resp_noknobs_missing_values_dict.json to master +Passed! +comparing Test65_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv to master +Passed! +comparing Test65_smlp_toy_num_resp_noknobs_test_predictions_summary.csv to master +Passed! +comparing Test65_smlp_toy_num_resp_noknobs_trace.csv to master +Passed! +comparing Test65_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv to master +Passed! +comparing Test65_smlp_toy_num_resp_noknobs_training_predictions_summary.csv to master +Passed! +comparing Test65_smlp_toy_num_resp_noknobs_verify_results.json to master +Passed! +comparing test65_model_data_bounds.json to master +Passed! +comparing test65_model_dt_sklearn_tree_rules.txt to master +Passed! +comparing test65_model_model_features_dict.json to master +Passed! +comparing test65_model_model_levels_dict.json to master +Passed! +comparing test65_model_rerun_model_config.json to master +Passed! +comparing Test66_test65_model.txt to master +Passed! +comparing Test66_test65_model_trace.csv to master +Passed! +comparing Test66_test65_model_verify_results.json to master +Passed! +comparing test67_model_dt_sklearn_y1_tree_rules.txt to master +Passed! +comparing Test67_smlp_toy_num_resp_noknobs.txt to master +Passed! +comparing Test67_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv to master +Passed! +comparing Test67_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv to master +Passed! +comparing Test67_smlp_toy_num_resp_noknobs_missing_values_dict.json to master +Passed! +comparing Test67_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv to master +Passed! +comparing Test67_smlp_toy_num_resp_noknobs_test_predictions_summary.csv to master +Passed! +comparing Test67_smlp_toy_num_resp_noknobs_trace.csv to master +Passed! +comparing Test67_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv to master +Passed! +comparing Test67_smlp_toy_num_resp_noknobs_training_predictions_summary.csv to master +Passed! +comparing Test67_smlp_toy_num_resp_noknobs_verify_results.json to master +Passed! +comparing test67_model_data_bounds.json to master +Passed! +comparing test67_model_dt_sklearn_y2_tree_rules.txt to master +Passed! +comparing test67_model_model_features_dict.json to master +Passed! +comparing test67_model_model_levels_dict.json to master +Passed! +comparing test67_model_rerun_model_config.json to master +Passed! +comparing Test68_test67_model.txt to master +Passed! +comparing Test68_test67_model_trace.csv to master +Passed! +comparing Test68_test67_model_verify_results.json to master +Passed! +comparing Test69_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test69_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test69_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test69_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test69_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test69_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test69_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test69_smlp_toy_num_resp_mult_train-reg_y2_mse.png does not exist +comparing Test69_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test69_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +comparing Test69_smlp_toy_num_resp_mult_verify_results.json to master +Passed! +comparing test69_model_data_bounds.json to master +Passed! +comparing test69_model_model_features_dict.json to master +Passed! +comparing test69_model_model_gen.json to master +Passed! +comparing test69_model_model_levels_dict.json to master +Passed! +comparing test69_model_rerun_model_config.json to master +Passed! +File master test69_model_y2_smlp_full_model_term.json does not exist +File master test69_model_y2_smlp_model_term.json does not exist +comparing Test70_test69_model.txt to master +File master Test70_test69_model_trace.csv does not exist +comparing Test70_test69_model_verify_results.json to master +Passed! +comparing Test71_smlp_toy_num_resp_noknobs.txt to master +Passed! +comparing Test71_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv to master +Passed! +comparing Test71_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv to master +Passed! +comparing Test71_smlp_toy_num_resp_noknobs_missing_values_dict.json to master +Passed! +comparing Test71_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv to master +Passed! +comparing Test71_smlp_toy_num_resp_noknobs_test_predictions_summary.csv to master +Passed! +comparing Test71_smlp_toy_num_resp_noknobs_trace.csv to master +Passed! +File master Test71_smlp_toy_num_resp_noknobs_train-reg_y1_mse.png does not exist +File master Test71_smlp_toy_num_resp_noknobs_train-reg_y2_mse.png does not exist +comparing Test71_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv to master +Passed! +comparing Test71_smlp_toy_num_resp_noknobs_training_predictions_summary.csv to master +Passed! +comparing Test71_smlp_toy_num_resp_noknobs_verify_results.json to master +Passed! +comparing test71_model_data_bounds.json to master +Passed! +comparing test71_model_model_features_dict.json to master +Passed! +comparing test71_model_model_gen.json to master +Passed! +comparing test71_model_model_levels_dict.json to master +Passed! +comparing test71_model_rerun_model_config.json to master +Passed! +comparing Test72_test71_model.txt to master +Passed! +comparing Test72_test71_model_trace.csv to master +Passed! +comparing Test72_test71_model_verify_results.json to master +Passed! +comparing test73_model_dt_sklearn_y2_tree_rules.txt to master +Passed! +comparing Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled.txt to master +Passed! +comparing Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv to master +Passed! +comparing Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv to master +Passed! +comparing Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json to master +Passed! +comparing Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv to master +Passed! +comparing Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv to master +Passed! +comparing Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv to master +Passed! +comparing Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv to master +Passed! +comparing Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv to master +Passed! +comparing Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv to master +Passed! +comparing Test73_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json to master +Passed! +comparing test73_model_data_bounds.json to master +Passed! +comparing test73_model_dt_sklearn_y1_tree_rules.txt to master +Passed! +comparing test73_model_model_features_dict.json to master +Passed! +comparing test73_model_model_levels_dict.json to master +Passed! +comparing test73_model_rerun_model_config.json to master Passed! -File master Test59_smlp_toy_num_resp_mult_trace.csv does not exist -File master Test59_smlp_toy_num_resp_mult_train-reg_y2_mse.png does not exist -comparing Test59_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +comparing Test74_test73_model_smlp_toy_num_resp_noknobs_pred_labeled.txt to master Passed! -comparing Test59_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +comparing Test74_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json to master Passed! -comparing Test59_smlp_toy_num_resp_mult_verify_results.json to master +comparing Test74_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv to master Passed! -File master Test59_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist -File master Test59_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist -comparing Test60_smlp_toy_num_resp_mult.txt to master +comparing Test74_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv to master Passed! -comparing Test60_smlp_toy_num_resp_mult_data_bounds.json to master +comparing Test74_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv to master Passed! -comparing Test60_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +comparing Test74_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json to master Passed! -comparing Test60_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +comparing Test75_test73_model_smlp_toy_num_resp_noknobs_pred_labeled.txt to master Passed! -comparing Test60_smlp_toy_num_resp_mult_missing_values_dict.json to master +comparing Test75_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json to master Passed! -comparing Test60_smlp_toy_num_resp_mult_model_features_dict.json to master +comparing Test75_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv to master Passed! -comparing Test60_smlp_toy_num_resp_mult_model_gen.json to master +comparing Test75_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv to master Passed! -comparing Test60_smlp_toy_num_resp_mult_model_levels_dict.json to master +comparing Test75_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv to master Passed! -comparing Test60_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +comparing Test75_test73_model_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json to master Passed! -comparing Test60_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +comparing Test76_smlp_toy_num_resp_noknobs.txt to master Passed! -File master Test60_smlp_toy_num_resp_mult_trace.csv does not exist -File master Test60_smlp_toy_num_resp_mult_train-reg_y2_mse.png does not exist -comparing Test60_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +comparing Test76_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv to master Passed! -comparing Test60_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +comparing Test76_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv to master Passed! -comparing Test60_smlp_toy_num_resp_mult_verify_results.json to master +comparing Test76_smlp_toy_num_resp_noknobs_missing_values_dict.json to master Passed! -File master Test60_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist -File master Test60_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist -Test 61 Failed: -Error in Build stage: -Data file does not exist -Test 62 Failed: -Error in Build stage: -Data file does not exist -File master test63_model_dt_sklearn_y1_tree_rules.txt does not exist -comparing Test63_smlp_toy_num_resp_mult.txt to master +comparing Test76_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv to master Passed! -comparing Test63_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +comparing Test76_smlp_toy_num_resp_noknobs_test_predictions_summary.csv to master Passed! -comparing Test63_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +comparing Test76_smlp_toy_num_resp_noknobs_trace.csv to master Passed! -comparing Test63_smlp_toy_num_resp_mult_missing_values_dict.json to master +comparing Test76_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv to master Passed! -comparing Test63_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +comparing Test76_smlp_toy_num_resp_noknobs_training_predictions_summary.csv to master Passed! -comparing Test63_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +comparing Test76_smlp_toy_num_resp_noknobs_verify_results.json to master Passed! -File master Test63_smlp_toy_num_resp_mult_trace.csv does not exist -comparing Test63_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +comparing test76_model_data_bounds.json to master Passed! -comparing Test63_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +comparing test76_model_dt_sklearn_tree_rules.txt to master Passed! -comparing Test63_smlp_toy_num_resp_mult_verify_results.json to master +comparing test76_model_model_features_dict.json to master Passed! -comparing test63_model_data_bounds.json to master +comparing test76_model_model_levels_dict.json to master Passed! -comparing test63_model_model_features_dict.json to master +comparing test76_model_rerun_model_config.json to master Passed! -comparing test63_model_model_levels_dict.json to master +comparing Test77_test76_model.txt to master Passed! -comparing test63_model_rerun_model_config.json to master +comparing Test77_test76_model_trace.csv to master Passed! -File master test63_model_y1_smlp_full_model_term.json does not exist -File master test63_model_y1_smlp_model_term.json does not exist -comparing Test64_test63_model.txt to master -File master Test64_test63_model_trace.csv does not exist -comparing Test64_test63_model_verify_results.json to master +comparing Test77_test76_model_verify_results.json to master Passed! -Test 65 Failed: -Error in Build stage: -Data file does not exist -comparing Test66_test65_model.txt to master -File new Test66_test65_model_verify_results.json does not exist -Test 67 Failed: -Error in Build stage: -Data file does not exist -comparing Test68_test67_model.txt to master -File new Test68_test67_model_verify_results.json does not exist -comparing Test69_smlp_toy_num_resp_mult.txt to master +comparing test78_model_dt_sklearn_tree_rules.txt to master Passed! -comparing Test69_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +comparing Test78_smlp_toy_num_resp_noknobs.txt to master Passed! -comparing Test69_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +comparing Test78_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv to master Passed! -comparing Test69_smlp_toy_num_resp_mult_missing_values_dict.json to master +comparing Test78_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv to master Passed! -comparing Test69_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +comparing Test78_smlp_toy_num_resp_noknobs_missing_values_dict.json to master Passed! -comparing Test69_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +comparing Test78_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv to master Passed! -File master Test69_smlp_toy_num_resp_mult_trace.csv does not exist -File master Test69_smlp_toy_num_resp_mult_train-reg_y2_mse.png does not exist -comparing Test69_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +comparing Test78_smlp_toy_num_resp_noknobs_test_predictions_summary.csv to master Passed! -comparing Test69_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +comparing Test78_smlp_toy_num_resp_noknobs_trace.csv to master Passed! -comparing Test69_smlp_toy_num_resp_mult_verify_results.json to master +comparing Test78_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv to master Passed! -comparing test69_model_data_bounds.json to master +comparing Test78_smlp_toy_num_resp_noknobs_training_predictions_summary.csv to master Passed! -comparing test69_model_model_features_dict.json to master +comparing Test78_smlp_toy_num_resp_noknobs_verify_results.json to master Passed! -comparing test69_model_model_gen.json to master +comparing test78_model_data_bounds.json to master Passed! -comparing test69_model_model_levels_dict.json to master +comparing test78_model_model_features_dict.json to master Passed! -comparing test69_model_rerun_model_config.json to master +comparing test78_model_model_levels_dict.json to master Passed! -File master test69_model_y2_smlp_full_model_term.json does not exist -File master test69_model_y2_smlp_model_term.json does not exist -comparing Test70_test69_model.txt to master -File master Test70_test69_model_trace.csv does not exist -comparing Test70_test69_model_verify_results.json to master +comparing test78_model_rerun_model_config.json to master Passed! -Test 71 Failed: -Error in Build stage: -Data file does not exist -comparing Test72_test71_model.txt to master -File new Test72_test71_model_verify_results.json does not exist -Test 73 Failed: -Error in Build stage: -Data file does not exist -Error in Build stage: -New data file does not exist -Test 74 Failed: -Error in Build stage: -New data file does not exist -Test 75 Failed: -Error in Build stage: -New data file does not exist -Test 76 Failed: -Error in Build stage: -Data file does not exist -comparing Test77_test76_model.txt to master -File new Test77_test76_model_verify_results.json does not exist -Test 78 Failed: -Error in Build stage: -Data file does not exist comparing Test79_smlp_toy_num_resp_mult.txt to master Passed! comparing Test79_smlp_toy_num_resp_mult_data_bounds.json to master @@ -1979,9 +2354,36 @@ comparing Test83_smlp_toy_num_resp_mult_training_prediction_precisions.csv to ma Passed! comparing Test83_smlp_toy_num_resp_mult_training_predictions_summary.csv to master Passed! -Test 84 Failed: -Error in Build stage: -Data file does not exist +comparing Test84_smlp_toy_num_resp_noknobs_dt_sklearn_tree_rules.txt to master +Passed! +comparing Test84_smlp_toy_num_resp_noknobs.txt to master +Passed! +comparing Test84_smlp_toy_num_resp_noknobs_data_bounds.json to master +Passed! +comparing Test84_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv to master +Passed! +comparing Test84_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv to master +Passed! +comparing Test84_smlp_toy_num_resp_noknobs_missing_values_dict.json to master +Passed! +comparing Test84_smlp_toy_num_resp_noknobs_model_features_dict.json to master +Passed! +comparing Test84_smlp_toy_num_resp_noknobs_model_levels_dict.json to master +Passed! +comparing Test84_smlp_toy_num_resp_noknobs_rerun_model_config.json to master +Passed! +comparing Test84_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv to master +Passed! +comparing Test84_smlp_toy_num_resp_noknobs_test_predictions_summary.csv to master +Passed! +comparing Test84_smlp_toy_num_resp_noknobs_trace.csv to master +Passed! +comparing Test84_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv to master +Passed! +comparing Test84_smlp_toy_num_resp_noknobs_training_predictions_summary.csv to master +Passed! +comparing Test84_smlp_toy_num_resp_noknobs_verify_results.json to master +Passed! comparing Test85_smlp_toy_num_resp_mult_dt_sklearn_tree_rules.txt to master Passed! comparing Test85_smlp_toy_num_resp_mult.txt to master @@ -3110,11 +3512,42 @@ New data file does not exist Test 138 Failed: Error in Build stage: Data file does not exist -Test 139 Failed: -Error in Build stage: -Data file does not exist -Error in Build stage: -New data file does not exist +comparing Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled.txt to master +Passed! +comparing Test139_smlp_toy_num_resp_noknobs_data_bounds.json to master +Passed! +comparing Test139_smlp_toy_num_resp_noknobs_dt_sklearn_y1_tree_rules.txt to master +Passed! +comparing Test139_smlp_toy_num_resp_noknobs_dt_sklearn_y2_tree_rules.txt to master +Passed! +comparing Test139_smlp_toy_num_resp_noknobs_model_features_dict.json to master +Passed! +comparing Test139_smlp_toy_num_resp_noknobs_model_levels_dict.json to master +Passed! +comparing Test139_smlp_toy_num_resp_noknobs_rerun_model_config.json to master +Passed! +comparing Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv to master +Passed! +comparing Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv to master +Passed! +comparing Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json to master +Passed! +comparing Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv to master +Passed! +comparing Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv to master +Passed! +comparing Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv to master +Passed! +comparing Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv to master +Passed! +comparing Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv to master +Passed! +comparing Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv to master +Passed! +comparing Test139_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json to master +Passed! File master Test140_smlp_toy_basic.txt does not exist File master Test140_smlp_toy_basic_data_bounds.json does not exist File master Test140_smlp_toy_basic_features_scaler.pkl does not exist @@ -3189,9 +3622,34 @@ File master Test143_smlp_toy_num_resp_mult_y1_smlp_full_model_term.json does not File master Test143_smlp_toy_num_resp_mult_y1_smlp_model_term.json does not exist File master Test143_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist File master Test143_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist -Test 144 Failed: -Error in Build stage: -Data file does not exist +comparing Test144_smlp_toy_num_resp_noknobs_dt_sklearn_tree_rules.txt to master +Passed! +comparing Test144_smlp_toy_num_resp_noknobs.txt to master +Passed! +comparing Test144_smlp_toy_num_resp_noknobs_data_bounds.json to master +Passed! +comparing Test144_smlp_toy_num_resp_noknobs_labeled_prediction_precisions.csv to master +Passed! +comparing Test144_smlp_toy_num_resp_noknobs_labeled_predictions_summary.csv to master +Passed! +comparing Test144_smlp_toy_num_resp_noknobs_missing_values_dict.json to master +Passed! +comparing Test144_smlp_toy_num_resp_noknobs_model_features_dict.json to master +Passed! +comparing Test144_smlp_toy_num_resp_noknobs_model_levels_dict.json to master +Passed! +comparing Test144_smlp_toy_num_resp_noknobs_test_prediction_precisions.csv to master +Passed! +comparing Test144_smlp_toy_num_resp_noknobs_test_predictions_summary.csv to master +Passed! +comparing Test144_smlp_toy_num_resp_noknobs_trace.csv to master +Passed! +comparing Test144_smlp_toy_num_resp_noknobs_training_prediction_precisions.csv to master +Passed! +comparing Test144_smlp_toy_num_resp_noknobs_training_predictions_summary.csv to master +Passed! +comparing Test144_smlp_toy_num_resp_noknobs_verify_results.json to master +Passed! File master Test145_doe_two_levels_opt.txt does not exist File master Test145_doe_two_levels_opt_trace.csv does not exist File master Test146_explore_doe_two_levels.txt does not exist @@ -3489,16 +3947,76 @@ File master Test161_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_t File master Test161_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_train-reg_all_responses_mape.png does not exist File master Test161_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_prediction_precisions.csv does not exist File master Test161_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_predictions_summary.csv does not exist -Test 162 Failed: -Error in Build stage: -Data file does not exist -Error in Build stage: -New data file does not exist -Test 163 Failed: -Error in Build stage: -Data file does not exist -Error in Build stage: -New data file does not exist +comparing Test162_smlp_toy_num_resp_noknobs_dt_sklearn_y2_tree_rules.txt to master +Passed! +comparing Test162_smlp_toy_num_resp_noknobs_data_bounds.json to master +Passed! +comparing Test162_smlp_toy_num_resp_noknobs_dt_sklearn_y1_tree_rules.txt to master +Passed! +comparing Test162_smlp_toy_num_resp_noknobs_model_features_dict.json to master +Passed! +comparing Test162_smlp_toy_num_resp_noknobs_model_levels_dict.json to master +Passed! +comparing Test162_smlp_toy_num_resp_noknobs_rerun_model_config.json to master +Passed! +comparing Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled.txt to master +Passed! +comparing Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv to master +Passed! +comparing Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv to master +Passed! +comparing Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json to master +Passed! +comparing Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv to master +Passed! +comparing Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv to master +Passed! +comparing Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv to master +Passed! +comparing Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv to master +Passed! +comparing Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv to master +Passed! +comparing Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv to master +Passed! +comparing Test162_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json to master +Passed! +comparing Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled.txt to master +Passed! +comparing Test163_smlp_toy_num_resp_noknobs_data_bounds.json to master +Passed! +comparing Test163_smlp_toy_num_resp_noknobs_dt_sklearn_tree_rules.txt to master +Passed! +comparing Test163_smlp_toy_num_resp_noknobs_model_features_dict.json to master +Passed! +comparing Test163_smlp_toy_num_resp_noknobs_model_levels_dict.json to master +Passed! +comparing Test163_smlp_toy_num_resp_noknobs_rerun_model_config.json to master +Passed! +comparing Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv to master +Passed! +comparing Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv to master +Passed! +comparing Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json to master +Passed! +comparing Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv to master +Passed! +comparing Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv to master +Passed! +comparing Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv to master +Passed! +comparing Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv to master +Passed! +comparing Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv to master +Passed! +comparing Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv to master +Passed! +comparing Test163_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json to master +Passed! File master Test164_smlp_toy_num_resp_mult.txt does not exist File master Test164_smlp_toy_num_resp_mult_data_bounds.json does not exist File master Test164_smlp_toy_num_resp_mult_dt_sklearn_model_complete.pkl does not exist @@ -3945,21 +4463,112 @@ File master Test183_smlp_toy_num_resp_mult_test_predictions_summary.csv does not File master Test183_smlp_toy_num_resp_mult_trace.csv does not exist File master Test183_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist File master Test183_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist -Test 184 Failed: -Error in Build stage: -Data file does not exist -Error in Build stage: -New data file does not exist -Test 185 Failed: -Error in Build stage: -Data file does not exist -Error in Build stage: -New data file does not exist -Test 186 Failed: -Error in Build stage: -Data file does not exist -Error in Build stage: -New data file does not exist +comparing Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled.txt to master +Passed! +comparing Test184_smlp_toy_num_resp_noknobs_data_bounds.json to master +Passed! +comparing Test184_smlp_toy_num_resp_noknobs_dt_sklearn_y1_tree_rules.txt to master +Passed! +comparing Test184_smlp_toy_num_resp_noknobs_dt_sklearn_y2_tree_rules.txt to master +Passed! +comparing Test184_smlp_toy_num_resp_noknobs_model_features_dict.json to master +Passed! +comparing Test184_smlp_toy_num_resp_noknobs_model_levels_dict.json to master +Passed! +comparing Test184_smlp_toy_num_resp_noknobs_rerun_model_config.json to master +Passed! +comparing Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv to master +Passed! +comparing Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv to master +Passed! +comparing Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json to master +Passed! +comparing Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv to master +Passed! +comparing Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv to master +Passed! +comparing Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv to master +Passed! +comparing Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv to master +Passed! +comparing Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv to master +Passed! +comparing Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv to master +Passed! +comparing Test184_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json to master +Passed! +comparing Test185_smlp_toy_num_resp_noknobs_dt_sklearn_y1_tree_rules.txt to master +Passed! +comparing Test185_smlp_toy_num_resp_noknobs_data_bounds.json to master +Passed! +comparing Test185_smlp_toy_num_resp_noknobs_dt_sklearn_y2_tree_rules.txt to master +Passed! +comparing Test185_smlp_toy_num_resp_noknobs_model_features_dict.json to master +Passed! +comparing Test185_smlp_toy_num_resp_noknobs_model_levels_dict.json to master +Passed! +comparing Test185_smlp_toy_num_resp_noknobs_rerun_model_config.json to master +Passed! +comparing Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled.txt to master +Passed! +comparing Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv to master +Passed! +comparing Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv to master +Passed! +comparing Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json to master +Passed! +comparing Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv to master +Passed! +comparing Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv to master +Passed! +comparing Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv to master +Passed! +comparing Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv to master +Passed! +comparing Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv to master +Passed! +comparing Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv to master +Passed! +comparing Test185_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json to master +Passed! +comparing Test186_smlp_toy_num_resp_noknobs_dt_sklearn_tree_rules.txt to master +Passed! +comparing Test186_smlp_toy_num_resp_noknobs_data_bounds.json to master +Passed! +comparing Test186_smlp_toy_num_resp_noknobs_model_features_dict.json to master +Passed! +comparing Test186_smlp_toy_num_resp_noknobs_model_levels_dict.json to master +Passed! +comparing Test186_smlp_toy_num_resp_noknobs_rerun_model_config.json to master +Passed! +comparing Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled.txt to master +Passed! +comparing Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_prediction_precisions.csv to master +Passed! +comparing Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_labeled_predictions_summary.csv to master +Passed! +comparing Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_missing_values_dict.json to master +Passed! +comparing Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_new_predictions_summary.csv to master +Passed! +comparing Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_prediction_precisions.csv to master +Passed! +comparing Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_test_predictions_summary.csv to master +Passed! +comparing Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_trace.csv to master +Passed! +comparing Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_prediction_precisions.csv to master +Passed! +comparing Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_training_predictions_summary.csv to master +Passed! +comparing Test186_smlp_toy_num_resp_noknobs_smlp_toy_num_resp_noknobs_pred_labeled_verify_results.json to master +Passed! File master Test187_smlp_toy_num_resp_mult.txt does not exist File master Test187_smlp_toy_num_resp_mult_data_bounds.json does not exist File master Test187_smlp_toy_num_resp_mult_dt_sklearn_model_complete.pkl does not exist @@ -4429,5 +5038,5 @@ Passed! master log file does not exist! Do you wish to copy the new log file to master? (yes/no|y/n): No new tests crashed (not in the masters) -Time: 32.59507596492767 minutes +Time: 28.443782750765482 minutes End of regression diff --git a/tests/smlp_regression/run_smlp_regression_venv_expected_diff_report.log b/tests/smlp_regression/run_smlp_regression_venv_expected_diff_report.log index 5d8bc15b..8c813c01 100644 --- a/tests/smlp_regression/run_smlp_regression_venv_expected_diff_report.log +++ b/tests/smlp_regression/run_smlp_regression_venv_expected_diff_report.log @@ -386,424 +386,12 @@ --- > smlp_logger - INFO - Seving model rerun configuration in file ./../models/test63_model_rerun_model_config.json =================== End of Test64_test63_model.txt diff report ================================ -=================== Diff report for: Test66_test65_model.txt ================================== -0a1,97 -> -> smlp_logger - INFO - Model exploration specification: -> {'version': '1.1', 'spec': [{'label': 'y1', 'type': 'response', 'range': 'float'}, {'label': 'y2', 'type': 'response', 'range': 'float'}, {'label': 'x0', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x1', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x2', 'type': 'input', 'range': 'float', 'bounds': [3, 7]}], 'alpha': 'x1==1 or x1==4 or x1==7'} -> -> smlp_logger - INFO - Executing run_smlp.py script: Start -> -> smlp_logger - INFO - Running SMLP in mode "verify": Start -> -> smlp_logger - INFO - Computed spec global constraint expressions: -> -> smlp_logger - INFO - Global alpha : x1==1 or x1==4 or x1==7 -> -> smlp_logger - INFO - Global beta : None -> -> smlp_logger - INFO - Radii theta : {} -> -> smlp_logger - INFO - Delta const : {'delta_abs': 0.0, 'delta_rel': 0.01} -> -> smlp_logger - INFO - Assertion asrt1: x0**2+y1>4.3 -> -> smlp_logger - INFO - Assertion asrt2: (y1+x2)/2<6 -> -> smlp_logger - INFO - PREPARE DATA FOR MODELING -> -> smlp_logger - INFO - LOAD TRAINED MODEL -> -> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test65_model_rerun_model_config.json -> -> smlp_logger - INFO - Creating model exploration base components: Start -> -> smlp_logger - INFO - Parsing the SPEC: Start -> -> smlp_logger - INFO - Parsing the SPEC: End -> -> smlp_logger - INFO - Variable domains (alpha): {'y1': {'range': 'float', 'interval': None}, 'y2': {'range': 'float', 'interval': None}, 'x0': {'range': 'float', 'interval': [0, 10]}, 'x1': {'range': 'float', 'interval': [0, 10]}, 'x2': {'range': 'float', 'interval': [3, 7]}} -> -> smlp_logger - INFO - Input bounds (alpha): {'x0': {'min': 0, 'max': 10}, 'x1': {'min': 0, 'max': 10}, 'x2': {'min': 3, 'max': 7}} -> -> smlp_logger - INFO - Knob bounds (eta): {} -> -> smlp_logger - INFO - Knob grids (eta): {} -> -> smlp_logger - INFO - Alpha global constraints: (or (or (= x1 1) (= x1 4)) (= x1 7)) -> -> smlp_logger - INFO - Alpha ranges constraints: (and (and (and true (and (>= x0 0) (<= x0 10))) (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) -> -> smlp_logger - INFO - Alpha combined constraints: (and (and (and (and true (and (>= x0 0) (<= x0 10))) (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) (or (or (= x1 1) (= x1 4)) (= x1 7))) -> -> smlp_logger - INFO - Beta global constraints: true -> -> smlp_logger - INFO - Eta ranges constraints: true -> -> smlp_logger - INFO - Eta grid constraints: true -> -> smlp_logger - INFO - Eta global constraints: true -> -> smlp_logger - INFO - Eta combined constraints: true -> -> smlp_logger - INFO - Creating model exploration base components: End -> -> smlp_logger - INFO - Input and knob interface constraints are consistent -> -> smlp_logger - INFO - Building model terms: Start -> -> smlp_logger - INFO - Model operator counts for y1: {'add': 1, 'mul': 15, 'ite': 5, 'and': 9, 'prop': 14, 'const': 50, 'sub': 14, 'var': 14} -> -> smlp_logger - INFO - Model operator counts for y2: {'add': 1, 'mul': 15, 'ite': 5, 'and': 9, 'prop': 14, 'const': 50, 'sub': 14, 'var': 14} -> -> smlp_logger - INFO - Building model terms: End -> -> smlp_logger - INFO - Model interface constraints are consistent -> -> smlp_logger - INFO - Verifying consistency of configuration for assertion asrt1: -> true -> -> smlp_logger - INFO - Input, knob and configuration constraints are consistent -> -> smlp_logger - INFO - Verifying consistency of configuration for assertion asrt2: -> true -> -> smlp_logger - INFO - Input, knob and configuration constraints are consistent -> -> smlp_logger - INFO - Verifying assertion asrt1 <-> x0**2+y1>4.3 -> -> smlp_logger - INFO - The configuration is consistent with assertion asrt1 -> -> smlp_logger - INFO - Completed with result: PASS -> -> smlp_logger - INFO - Verifying assertion asrt2 <-> (y1+x2)/2<6 -> -> smlp_logger - INFO - The configuration is consistent with assertion asrt2 -> -> smlp_logger - INFO - Completed with result: FAIL -> -> smlp_logger - INFO - Running SMLP in mode "verify": End -> -> smlp_logger - INFO - Executing run_smlp.py script: End -=================== End of Test66_test65_model.txt diff report ================================ -=================== Diff report for: Test66_test65_model_verify_results.json ================================== -diff: /home/mdmitry/github/smlp_subgroups_assertion_fix/scripts/venv/smlp_regression_venv/smlp/regr_smlp/code/Test66_test65_model_verify_results.json: No such file or directory -=================== End of Test66_test65_model_verify_results.json diff report ================================ -=================== Diff report for: Test68_test67_model.txt ================================== -0a1,97 -> -> smlp_logger - INFO - Model exploration specification: -> {'version': '1.1', 'spec': [{'label': 'y1', 'type': 'response', 'range': 'float'}, {'label': 'y2', 'type': 'response', 'range': 'float'}, {'label': 'x0', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x1', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x2', 'type': 'input', 'range': 'float', 'bounds': [3, 7]}], 'alpha': 'x1==1 or x1==4 or x1==7'} -> -> smlp_logger - INFO - Executing run_smlp.py script: Start -> -> smlp_logger - INFO - Running SMLP in mode "verify": Start -> -> smlp_logger - INFO - Computed spec global constraint expressions: -> -> smlp_logger - INFO - Global alpha : x1==1 or x1==4 or x1==7 -> -> smlp_logger - INFO - Global beta : None -> -> smlp_logger - INFO - Radii theta : {} -> -> smlp_logger - INFO - Delta const : {'delta_abs': 0.0, 'delta_rel': 0.01} -> -> smlp_logger - INFO - Assertion asrt1: x0**2+y1>4.3 -> -> smlp_logger - INFO - Assertion asrt2: (y1+x2)/2<6 -> -> smlp_logger - INFO - PREPARE DATA FOR MODELING -> -> smlp_logger - INFO - LOAD TRAINED MODEL -> -> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test67_model_rerun_model_config.json -> -> smlp_logger - INFO - Creating model exploration base components: Start -> -> smlp_logger - INFO - Parsing the SPEC: Start -> -> smlp_logger - INFO - Parsing the SPEC: End -> -> smlp_logger - INFO - Variable domains (alpha): {'y1': {'range': 'float', 'interval': None}, 'y2': {'range': 'float', 'interval': None}, 'x0': {'range': 'float', 'interval': [0, 10]}, 'x1': {'range': 'float', 'interval': [0, 10]}, 'x2': {'range': 'float', 'interval': [3, 7]}} -> -> smlp_logger - INFO - Input bounds (alpha): {'x0': {'min': 0, 'max': 10}, 'x1': {'min': 0, 'max': 10}, 'x2': {'min': 3, 'max': 7}} -> -> smlp_logger - INFO - Knob bounds (eta): {} -> -> smlp_logger - INFO - Knob grids (eta): {} -> -> smlp_logger - INFO - Alpha global constraints: (or (or (= x1 1) (= x1 4)) (= x1 7)) -> -> smlp_logger - INFO - Alpha ranges constraints: (and (and (and true (and (>= x0 0) (<= x0 10))) (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) -> -> smlp_logger - INFO - Alpha combined constraints: (and (and (and (and true (and (>= x0 0) (<= x0 10))) (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) (or (or (= x1 1) (= x1 4)) (= x1 7))) -> -> smlp_logger - INFO - Beta global constraints: true -> -> smlp_logger - INFO - Eta ranges constraints: true -> -> smlp_logger - INFO - Eta grid constraints: true -> -> smlp_logger - INFO - Eta global constraints: true -> -> smlp_logger - INFO - Eta combined constraints: true -> -> smlp_logger - INFO - Creating model exploration base components: End -> -> smlp_logger - INFO - Input and knob interface constraints are consistent -> -> smlp_logger - INFO - Building model terms: Start -> -> smlp_logger - INFO - Model operator counts for y1: {'add': 1, 'mul': 7, 'ite': 3, 'and': 3, 'prop': 6, 'const': 24, 'sub': 6, 'var': 6} -> -> smlp_logger - INFO - Model operator counts for y2: {'add': 1, 'mul': 4, 'ite': 2, 'and': 1, 'prop': 3, 'const': 14, 'sub': 3, 'var': 3} -> -> smlp_logger - INFO - Building model terms: End -> -> smlp_logger - INFO - Model interface constraints are consistent -> -> smlp_logger - INFO - Verifying consistency of configuration for assertion asrt1: -> true -> -> smlp_logger - INFO - Input, knob and configuration constraints are consistent -> -> smlp_logger - INFO - Verifying consistency of configuration for assertion asrt2: -> true -> -> smlp_logger - INFO - Input, knob and configuration constraints are consistent -> -> smlp_logger - INFO - Verifying assertion asrt1 <-> x0**2+y1>4.3 -> -> smlp_logger - INFO - The configuration is consistent with assertion asrt1 -> -> smlp_logger - INFO - Completed with result: PASS -> -> smlp_logger - INFO - Verifying assertion asrt2 <-> (y1+x2)/2<6 -> -> smlp_logger - INFO - The configuration is consistent with assertion asrt2 -> -> smlp_logger - INFO - Completed with result: FAIL -> -> smlp_logger - INFO - Running SMLP in mode "verify": End -> -> smlp_logger - INFO - Executing run_smlp.py script: End -=================== End of Test68_test67_model.txt diff report ================================ -=================== Diff report for: Test68_test67_model_verify_results.json ================================== -diff: /home/mdmitry/github/smlp_subgroups_assertion_fix/scripts/venv/smlp_regression_venv/smlp/regr_smlp/code/Test68_test67_model_verify_results.json: No such file or directory -=================== End of Test68_test67_model_verify_results.json diff report ================================ =================== Diff report for: Test70_test69_model.txt ================================== 25c25 < smlp_logger - INFO - Seving model rerun configuration in file ../models/test69_model_rerun_model_config.json --- > smlp_logger - INFO - Seving model rerun configuration in file ./../models/test69_model_rerun_model_config.json =================== End of Test70_test69_model.txt diff report ================================ -=================== Diff report for: Test72_test71_model.txt ================================== -0a1,84 -> -> smlp_logger - INFO - Model exploration specification: -> {'version': '1.1', 'spec': [{'label': 'y1', 'type': 'response', 'range': 'float'}, {'label': 'y2', 'type': 'response', 'range': 'float'}, {'label': 'x0', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x1', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x2', 'type': 'input', 'range': 'float', 'bounds': [3, 7]}], 'alpha': 'x1==1 or x1==4 or x1==7'} -> -> smlp_logger - INFO - Executing run_smlp.py script: Start -> -> smlp_logger - INFO - Running SMLP in mode "verify": Start -> -> smlp_logger - INFO - Computed spec global constraint expressions: -> -> smlp_logger - INFO - Global alpha : x1==1 or x1==4 or x1==7 -> -> smlp_logger - INFO - Global beta : None -> -> smlp_logger - INFO - Radii theta : {} -> -> smlp_logger - INFO - Delta const : {'delta_abs': 0.0, 'delta_rel': 0.01} -> -> smlp_logger - INFO - Assertion asrt1: (y2**3+x2)/2<6 -> -> smlp_logger - INFO - PREPARE DATA FOR MODELING -> -> smlp_logger - INFO - LOAD TRAINED MODEL -> -> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test71_model_rerun_model_config.json -> -> smlp_logger - INFO - Creating model exploration base components: Start -> -> smlp_logger - INFO - Parsing the SPEC: Start -> -> smlp_logger - INFO - Parsing the SPEC: End -> -> smlp_logger - INFO - Variable domains (alpha): {'y1': {'range': 'float', 'interval': None}, 'y2': {'range': 'float', 'interval': None}, 'x0': {'range': 'float', 'interval': [0, 10]}, 'x1': {'range': 'float', 'interval': [0, 10]}, 'x2': {'range': 'float', 'interval': [3, 7]}} -> -> smlp_logger - INFO - Input bounds (alpha): {'x0': {'min': 0, 'max': 10}, 'x1': {'min': 0, 'max': 10}, 'x2': {'min': 3, 'max': 7}} -> -> smlp_logger - INFO - Knob bounds (eta): {} -> -> smlp_logger - INFO - Knob grids (eta): {} -> -> smlp_logger - INFO - Alpha global constraints: (or (or (= x1 1) (= x1 4)) (= x1 7)) -> -> smlp_logger - INFO - Alpha ranges constraints: (and (and (and true (and (>= x0 0) (<= x0 10))) (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) -> -> smlp_logger - INFO - Alpha combined constraints: (and (and (and (and true (and (>= x0 0) (<= x0 10))) (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) (or (or (= x1 1) (= x1 4)) (= x1 7))) -> -> smlp_logger - INFO - Beta global constraints: true -> -> smlp_logger - INFO - Eta ranges constraints: true -> -> smlp_logger - INFO - Eta grid constraints: true -> -> smlp_logger - INFO - Eta global constraints: true -> -> smlp_logger - INFO - Eta combined constraints: true -> -> smlp_logger - INFO - Creating model exploration base components: End -> -> smlp_logger - INFO - Input and knob interface constraints are consistent -> -> smlp_logger - INFO - Building model terms: Start -> -> smlp_logger - INFO - Model operator counts for y1: {'add': 256, 'mul': 472, 'ite': 39, 'prop': 39, 'const': 846, 'sub': 216, 'var': 216} -> -> smlp_logger - INFO - Model operator counts for y2: {'add': 256, 'mul': 472, 'ite': 39, 'prop': 39, 'const': 846, 'sub': 216, 'var': 216} -> -> smlp_logger - INFO - Building model terms: End -> -> smlp_logger - INFO - Model interface constraints are consistent -> -> smlp_logger - INFO - Verifying consistency of configuration for assertion asrt1: -> true -> -> smlp_logger - INFO - Input, knob and configuration constraints are consistent -> -> smlp_logger - INFO - Verifying assertion asrt1 <-> (y2**3+x2)/2<6 -> -> smlp_logger - INFO - The configuration is consistent with assertion asrt1 -> -> smlp_logger - INFO - Completed with result: FAIL -> -> smlp_logger - INFO - Running SMLP in mode "verify": End -> -> smlp_logger - INFO - Executing run_smlp.py script: End -=================== End of Test72_test71_model.txt diff report ================================ -=================== Diff report for: Test72_test71_model_verify_results.json ================================== -diff: /home/mdmitry/github/smlp_subgroups_assertion_fix/scripts/venv/smlp_regression_venv/smlp/regr_smlp/code/Test72_test71_model_verify_results.json: No such file or directory -=================== End of Test72_test71_model_verify_results.json diff report ================================ -=================== Diff report for: Test77_test76_model.txt ================================== -0a1,110 -> -> smlp_logger - INFO - Model exploration specification: -> {'version': '1.1', 'spec': [{'label': 'y1', 'type': 'response', 'range': 'float'}, {'label': 'y2', 'type': 'response', 'range': 'float'}, {'label': 'x0', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x1', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x2', 'type': 'input', 'range': 'float', 'bounds': [3, 7]}], 'alpha': 'x1==1 or x1==4 or x1==7'} -> -> smlp_logger - INFO - Executing run_smlp.py script: Start -> -> smlp_logger - INFO - Running SMLP in mode "verify": Start -> -> smlp_logger - INFO - Computed spec global constraint expressions: -> -> smlp_logger - INFO - Global alpha : x1==1 or x1==4 or x1==7 -> -> smlp_logger - INFO - Global beta : None -> -> smlp_logger - INFO - Radii theta : {} -> -> smlp_logger - INFO - Delta const : {'delta_abs': 0.0, 'delta_rel': 0.01} -> -> smlp_logger - INFO - Assertion asrt1: (y2**3+x2)/2<6 -> -> smlp_logger - INFO - Assertion asrt2: y1>=9 -> -> smlp_logger - INFO - Assertion asrt3: y2<0 -> -> smlp_logger - INFO - PREPARE DATA FOR MODELING -> -> smlp_logger - INFO - LOAD TRAINED MODEL -> -> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test76_model_rerun_model_config.json -> -> smlp_logger - INFO - Creating model exploration base components: Start -> -> smlp_logger - INFO - Parsing the SPEC: Start -> -> smlp_logger - INFO - Parsing the SPEC: End -> -> smlp_logger - INFO - Variable domains (alpha): {'y1': {'range': 'float', 'interval': None}, 'y2': {'range': 'float', 'interval': None}, 'x0': {'range': 'float', 'interval': [0, 10]}, 'x1': {'range': 'float', 'interval': [0, 10]}, 'x2': {'range': 'float', 'interval': [3, 7]}} -> -> smlp_logger - INFO - Input bounds (alpha): {'x0': {'min': 0, 'max': 10}, 'x1': {'min': 0, 'max': 10}, 'x2': {'min': 3, 'max': 7}} -> -> smlp_logger - INFO - Knob bounds (eta): {} -> -> smlp_logger - INFO - Knob grids (eta): {} -> -> smlp_logger - INFO - Alpha global constraints: (or (or (= x1 1) (= x1 4)) (= x1 7)) -> -> smlp_logger - INFO - Alpha ranges constraints: (and (and true (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) -> -> smlp_logger - INFO - Alpha combined constraints: (and (and (and true (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) (or (or (= x1 1) (= x1 4)) (= x1 7))) -> -> smlp_logger - INFO - Beta global constraints: true -> -> smlp_logger - INFO - Eta ranges constraints: true -> -> smlp_logger - INFO - Eta grid constraints: true -> -> smlp_logger - INFO - Eta global constraints: true -> -> smlp_logger - INFO - Eta combined constraints: true -> -> smlp_logger - INFO - Creating model exploration base components: End -> -> smlp_logger - INFO - Input and knob interface constraints are consistent -> -> smlp_logger - INFO - Building model terms: Start -> -> smlp_logger - INFO - Model operator counts for y1: {'add': 1, 'mul': 21, 'ite': 6, 'and': 14, 'prop': 20, 'const': 69, 'sub': 20, 'var': 20} -> -> smlp_logger - INFO - Model operator counts for y2: {'add': 1, 'mul': 21, 'ite': 6, 'and': 14, 'prop': 20, 'const': 69, 'sub': 20, 'var': 20} -> -> smlp_logger - INFO - Building model terms: End -> -> smlp_logger - INFO - Model interface constraints are consistent -> -> smlp_logger - INFO - Verifying consistency of configuration for assertion asrt1: -> true -> -> smlp_logger - INFO - Input, knob and configuration constraints are consistent -> -> smlp_logger - INFO - Verifying consistency of configuration for assertion asrt2: -> true -> -> smlp_logger - INFO - Input, knob and configuration constraints are consistent -> -> smlp_logger - INFO - Verifying consistency of configuration for assertion asrt3: -> true -> -> smlp_logger - INFO - Input, knob and configuration constraints are consistent -> -> smlp_logger - INFO - Verifying assertion asrt1 <-> (y2**3+x2)/2<6 -> -> smlp_logger - INFO - The configuration is inconsistent with assertion asrt1 -> -> smlp_logger - INFO - Completed with result: FAIL -> -> smlp_logger - INFO - Verifying assertion asrt2 <-> y1>=9 -> -> smlp_logger - INFO - The configuration is consistent with assertion asrt2 -> -> smlp_logger - INFO - Completed with result: FAIL -> -> smlp_logger - INFO - Verifying assertion asrt3 <-> y2<0 -> -> smlp_logger - INFO - The configuration is inconsistent with assertion asrt3 -> -> smlp_logger - INFO - Completed with result: FAIL -> -> smlp_logger - INFO - Running SMLP in mode "verify": End -> -> smlp_logger - INFO - Executing run_smlp.py script: End -=================== End of Test77_test76_model.txt diff report ================================ -=================== Diff report for: Test77_test76_model_verify_results.json ================================== -diff: /home/mdmitry/github/smlp_subgroups_assertion_fix/scripts/venv/smlp_regression_venv/smlp/regr_smlp/code/Test77_test76_model_verify_results.json: No such file or directory -=================== End of Test77_test76_model_verify_results.json diff report ================================ =================== Diff report for: Test97_smlp_toy_num_resp_mult.txt ================================== 252c252 < smlp_logger - INFO - Model operator counts for y2: {'add': 100, 'mul': 715, 'const': 2547, 'ite': 305, 'and': 408, 'prop': 713, 'sub': 713, 'var': 713} @@ -817,7 +405,7 @@ diff: /home/mdmitry/github/smlp_subgroups_assertion_fix/scripts/venv/smlp_regres > smlp_logger - INFO - Seving model rerun configuration in file ./../models/test101_model_rerun_model_config.json =================== End of Test102_test101_model.txt diff report ================================ =================== Diff report for: test110_model_poly_sklearn_formula.txt ================================== -diff: /home/mdmitry/github/smlp_subgroups_assertion_fix/scripts/venv/smlp_regression_venv/smlp/regr_smlp/code/test110_model_poly_sklearn_formula.txt: No such file or directory +diff: /home/mdmitry/github/smlp_noknobs_tests_recovery/scripts/venv/smlp_package_venv/smlp/regr_smlp/code/test110_model_poly_sklearn_formula.txt: No such file or directory =================== End of test110_model_poly_sklearn_formula.txt diff report ================================ =================== Diff report for: Test111_test110_model_smlp_toy_basic_pred_unlabeled.txt ================================== 79c79