diff --git a/tests/conftest.py b/tests/conftest.py index c96a0089..35498c6e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,52 +9,29 @@ from dice_ml.utils import helpers -@pytest.fixture -def binary_classification_exp_object(method="random"): - backend = 'sklearn' - dataset = helpers.load_custom_testing_dataset_binary() - d = dice_ml.Data(dataframe=dataset, continuous_features=['Numerical'], outcome_name='Outcome') - ML_modelpath = helpers.get_custom_dataset_modelpath_pipeline_binary() - m = dice_ml.Model(model_path=ML_modelpath, backend=backend) - exp = dice_ml.Dice(d, m, method=method) - return exp - - -@pytest.fixture -def binary_classification_exp_object_out_of_order(method="random"): - backend = 'sklearn' +@pytest.fixture(scope='session') +def custom_public_data_interface_binary_out_of_order(): dataset = helpers.load_outcome_not_last_column_dataset() d = dice_ml.Data(dataframe=dataset, continuous_features=['Numerical'], outcome_name='Outcome') - ML_modelpath = helpers.get_custom_dataset_modelpath_pipeline_binary() - m = dice_ml.Model(model_path=ML_modelpath, backend=backend) - exp = dice_ml.Dice(d, m, method=method) - return exp + return d -@pytest.fixture -def multi_classification_exp_object(method="random"): - backend = 'sklearn' - dataset = helpers.load_custom_testing_dataset_multiclass() +@pytest.fixture(scope='session') +def custom_public_data_interface_binary(): + dataset = helpers.load_custom_testing_dataset_binary() d = dice_ml.Data(dataframe=dataset, continuous_features=['Numerical'], outcome_name='Outcome') - ML_modelpath = helpers.get_custom_dataset_modelpath_pipeline_multiclass() - m = dice_ml.Model(model_path=ML_modelpath, backend=backend) - exp = dice_ml.Dice(d, m, method=method) - return exp + return d -@pytest.fixture -def regression_exp_object(method="random"): - backend = 'sklearn' - dataset = helpers.load_custom_testing_dataset_regression() +@pytest.fixture(scope='session') +def custom_public_data_interface_multicalss(): + dataset = helpers.load_custom_testing_dataset_multiclass() d = dice_ml.Data(dataframe=dataset, continuous_features=['Numerical'], outcome_name='Outcome') - ML_modelpath = helpers.get_custom_dataset_modelpath_pipeline_regression() - m = dice_ml.Model(model_path=ML_modelpath, backend=backend, model_type='regressor') - exp = dice_ml.Dice(d, m, method=method) - return exp + return d @pytest.fixture(scope='session') -def custom_public_data_interface(): +def custom_public_data_interface_regression(): dataset = helpers.load_custom_testing_dataset_regression() d = dice_ml.Data(dataframe=dataset, continuous_features=['Numerical'], outcome_name='Outcome') return d @@ -77,7 +54,7 @@ def sklearn_multiclass_classification_model_interface(): @pytest.fixture(scope='session') def sklearn_regression_model_interface(): ML_modelpath = helpers.get_custom_dataset_modelpath_pipeline_regression() - m = dice_ml.Model(model_path=ML_modelpath, backend='sklearn', model_type='regression') + m = dice_ml.Model(model_path=ML_modelpath, backend='sklearn', model_type='regressor') return m diff --git a/tests/test_dice_interface/test_dice_KD.py b/tests/test_dice_interface/test_dice_KD.py index 9185835b..06fdaa74 100644 --- a/tests/test_dice_interface/test_dice_KD.py +++ b/tests/test_dice_interface/test_dice_KD.py @@ -2,8 +2,6 @@ import pytest import dice_ml -from dice_ml.counterfactual_explanations import CounterfactualExplanations -from dice_ml.diverse_counterfactuals import CounterfactualExamples from dice_ml.utils import helpers @@ -18,46 +16,12 @@ def KD_binary_classification_exp_object(): return exp -@pytest.fixture -def KD_multi_classification_exp_object(): - backend = 'sklearn' - dataset = helpers.load_custom_testing_dataset_multiclass() - d = dice_ml.Data(dataframe=dataset, continuous_features=['Numerical'], outcome_name='Outcome') - ML_modelpath = helpers.get_custom_dataset_modelpath_pipeline_multiclass() - m = dice_ml.Model(model_path=ML_modelpath, backend=backend) - exp = dice_ml.Dice(d, m, method='kdtree') - return exp - - -@pytest.fixture -def KD_regression_exp_object(): - backend = 'sklearn' - dataset = helpers.load_custom_testing_dataset_regression() - d = dice_ml.Data(dataframe=dataset, continuous_features=['Numerical'], outcome_name='Outcome') - ML_modelpath = helpers.get_custom_dataset_modelpath_pipeline_regression() - m = dice_ml.Model(model_path=ML_modelpath, backend=backend, model_type='regressor') - exp = dice_ml.Dice(d, m, method='kdtree') - return exp - - class TestDiceKDBinaryClassificationMethods: @pytest.fixture(autouse=True) def _initiate_exp_object(self, KD_binary_classification_exp_object): self.exp = KD_binary_classification_exp_object # explainer object self.data_df_copy = self.exp.data_interface.data_df.copy() - # When a query's feature value is not within the permitted range and the feature is not allowed to vary - @pytest.mark.parametrize("desired_range, desired_class, total_CFs, features_to_vary, permitted_range", - [(None, 0, 4, ['Numerical'], {'Categorical': ['b', 'c']})]) - def test_invalid_query_instance(self, desired_range, desired_class, sample_custom_query_1, total_CFs, - features_to_vary, permitted_range): - self.exp.dataset_with_predictions, self.exp.KD_tree, self.exp.predictions = \ - self.exp.build_KD_tree(self.data_df_copy, desired_range, desired_class, self.exp.predicted_outcome_name) - - with pytest.raises(ValueError): - self.exp._generate_counterfactuals(query_instance=sample_custom_query_1, total_CFs=total_CFs, - features_to_vary=features_to_vary, permitted_range=permitted_range) - # Verifying the output of the KD tree @pytest.mark.parametrize("desired_class, total_CFs", [(0, 1)]) @pytest.mark.parametrize('posthoc_sparsity_algorithm', ['linear', 'binary', None]) @@ -71,26 +35,6 @@ def test_KD_tree_output(self, desired_class, sample_custom_query_1, total_CFs, p assert all(self.exp.final_cfs_df.Numerical == expected_output.Numerical[0]) and \ all(self.exp.final_cfs_df.Categorical == expected_output.Categorical[0]) - # Verifying the output of the KD tree - @pytest.mark.parametrize("desired_class, total_CFs", [(0, 1)]) - def test_KD_tree_counterfactual_explanations_output(self, desired_class, sample_custom_query_1, total_CFs): - counterfactual_explanations = self.exp.generate_counterfactuals( - query_instances=sample_custom_query_1, desired_class=desired_class, - total_CFs=total_CFs) - - assert counterfactual_explanations is not None - - # Testing that the features_to_vary argument actually varies only the features that you wish to vary - @pytest.mark.parametrize("desired_class, total_CFs, features_to_vary", [(0, 1, ["Numerical"])]) - def test_features_to_vary(self, desired_class, sample_custom_query_2, total_CFs, features_to_vary): - self.exp._generate_counterfactuals(query_instance=sample_custom_query_2, desired_class=desired_class, - total_CFs=total_CFs, features_to_vary=features_to_vary) - self.exp.final_cfs_df.Numerical = self.exp.final_cfs_df.Numerical.astype(int) - expected_output = self.exp.data_interface.data_df - - assert all(self.exp.final_cfs_df.Numerical == expected_output.Numerical[1]) and \ - all(self.exp.final_cfs_df.Categorical == expected_output.Categorical[1]) - # Testing that the permitted_range argument actually varies the features only within the permitted_range @pytest.mark.parametrize("desired_class, total_CFs, permitted_range", [(0, 1, {'Numerical': [1000, 10000]})]) def test_permitted_range(self, desired_class, sample_custom_query_2, total_CFs, permitted_range): @@ -101,13 +45,6 @@ def test_permitted_range(self, desired_class, sample_custom_query_2, total_CFs, assert all(self.exp.final_cfs_df.Numerical == expected_output.Numerical[1]) and \ all(self.exp.final_cfs_df.Categorical == expected_output.Categorical[1]) - # Testing if you can provide permitted_range for categorical variables - @pytest.mark.parametrize("desired_class, total_CFs, permitted_range", [(0, 4, {'Categorical': ['b', 'c']})]) - def test_permitted_range_categorical(self, desired_class, sample_custom_query_2, total_CFs, permitted_range): - self.exp._generate_counterfactuals(query_instance=sample_custom_query_2, desired_class=desired_class, - total_CFs=total_CFs, permitted_range=permitted_range) - assert all(i in permitted_range["Categorical"] for i in self.exp.final_cfs_df.Categorical.values) - # Ensuring that there are no duplicates in the resulting counterfactuals even if the dataset has duplicates @pytest.mark.parametrize("desired_class, total_CFs", [(0, 2)]) def test_duplicates(self, desired_class, sample_custom_query_4, total_CFs): @@ -130,66 +67,3 @@ def test_index(self, desired_class, sample_custom_query_index, total_CFs, postho desired_class=desired_class, posthoc_sparsity_algorithm=posthoc_sparsity_algorithm) assert self.exp.final_cfs_df.index[0] == 3 - - -class TestDiceKDMultiClassificationMethods: - @pytest.fixture(autouse=True) - def _initiate_exp_object(self, KD_multi_classification_exp_object): - self.exp_multi = KD_multi_classification_exp_object # explainer object - self.data_df_copy = self.exp_multi.data_interface.data_df.copy() - - # Testing that the output of multiclass classification lies in the desired_class - @pytest.mark.parametrize("desired_class, total_CFs", [(2, 3)]) - @pytest.mark.parametrize('posthoc_sparsity_algorithm', ['linear', 'binary', None]) - def test_KD_tree_output(self, desired_class, sample_custom_query_2, total_CFs, - posthoc_sparsity_algorithm): - self.exp_multi._generate_counterfactuals(query_instance=sample_custom_query_2, total_CFs=total_CFs, - desired_class=desired_class, - posthoc_sparsity_algorithm=posthoc_sparsity_algorithm) - assert all(i == desired_class for i in self.exp_multi.cfs_preds) - - -class TestDiceKDRegressionMethods: - @pytest.fixture(autouse=True) - def _initiate_exp_object(self, KD_regression_exp_object): - self.exp_regr = KD_regression_exp_object # explainer object - self.data_df_copy = self.exp_regr.data_interface.data_df.copy() - - # Testing that the output of regression lies in the desired_range - @pytest.mark.parametrize("desired_range, total_CFs", [([1, 2.8], 6)]) - @pytest.mark.parametrize("version", ['2.0', '1.0']) - @pytest.mark.parametrize('posthoc_sparsity_algorithm', ['linear', 'binary', None]) - def test_KD_tree_output(self, desired_range, sample_custom_query_2, total_CFs, version, posthoc_sparsity_algorithm): - cf_examples = self.exp_regr._generate_counterfactuals(query_instance=sample_custom_query_2, total_CFs=total_CFs, - desired_range=desired_range, - posthoc_sparsity_algorithm=posthoc_sparsity_algorithm) - assert all(desired_range[0] <= i <= desired_range[1] for i in self.exp_regr.cfs_preds) - - assert cf_examples is not None - json_str = cf_examples.to_json(version) - assert json_str is not None - - recovered_cf_examples = CounterfactualExamples.from_json(json_str) - assert recovered_cf_examples is not None - assert cf_examples == recovered_cf_examples - - @pytest.mark.parametrize("desired_range, total_CFs", [([1, 2.8], 6)]) - def test_KD_tree_counterfactual_explanations_output(self, desired_range, sample_custom_query_2, - total_CFs): - counterfactual_explanations = self.exp_regr.generate_counterfactuals( - query_instances=sample_custom_query_2, total_CFs=total_CFs, - desired_range=desired_range) - - assert counterfactual_explanations is not None - json_str = counterfactual_explanations.to_json() - assert json_str is not None - - recovered_counterfactual_explanations = CounterfactualExplanations.from_json(json_str) - assert recovered_counterfactual_explanations is not None - assert counterfactual_explanations == recovered_counterfactual_explanations - - # Testing for 0 CFs needed - @pytest.mark.parametrize("desired_class, desired_range, total_CFs", [(0, [1, 2.8], 0)]) - def test_zero_cfs(self, desired_class, desired_range, sample_custom_query_4, total_CFs): - self.exp_regr._generate_counterfactuals(query_instance=sample_custom_query_4, total_CFs=total_CFs, - desired_range=desired_range) diff --git a/tests/test_dice_interface/test_dice_genetic.py b/tests/test_dice_interface/test_dice_genetic.py index 2da55d73..27095bd5 100644 --- a/tests/test_dice_interface/test_dice_genetic.py +++ b/tests/test_dice_interface/test_dice_genetic.py @@ -2,7 +2,6 @@ import dice_ml from dice_ml.utils import helpers -from dice_ml.utils.exception import UserConfigValidationException @pytest.fixture @@ -43,36 +42,6 @@ class TestDiceGeneticBinaryClassificationMethods: def _initiate_exp_object(self, genetic_binary_classification_exp_object): self.exp = genetic_binary_classification_exp_object # explainer object - # When invalid desired_class is given - @pytest.mark.parametrize("desired_class, total_CFs", [(7, 3)]) - def test_no_cfs(self, desired_class, sample_custom_query_1, total_CFs): - with pytest.raises(UserConfigValidationException): - self.exp.generate_counterfactuals(query_instances=sample_custom_query_1, total_CFs=total_CFs, - desired_class=desired_class) - - # When a query's feature value is not within the permitted range and the feature is not allowed to vary - @pytest.mark.parametrize("features_to_vary, permitted_range, feature_weights", - [(['Numerical'], {'Categorical': ['b', 'c']}, "inverse_mad")]) - def test_invalid_query_instance(self, sample_custom_query_1, features_to_vary, permitted_range, feature_weights): - with pytest.raises(ValueError): - self.exp.setup(features_to_vary, permitted_range, sample_custom_query_1, feature_weights) - - # Testing that the features_to_vary argument actually varies only the features that you wish to vary - @pytest.mark.parametrize("desired_class, total_CFs, features_to_vary, initialization", - [(1, 2, ["Numerical"], "kdtree"), (1, 2, ["Numerical"], "random")]) - def test_features_to_vary(self, desired_class, sample_custom_query_2, total_CFs, features_to_vary, initialization): - ans = self.exp.generate_counterfactuals(query_instances=sample_custom_query_2, - features_to_vary=features_to_vary, - total_CFs=total_CFs, desired_class=desired_class, - initialization=initialization) - - for cfs_example in ans.cf_examples_list: - for feature in self.exp.data_interface.feature_names: - if feature not in features_to_vary: - assert all( - cfs_example.final_cfs_df[feature].values[i] == sample_custom_query_2[feature].values[0] for i in - range(total_CFs)) - # Testing that the permitted_range argument actually varies the features only within the permitted_range @pytest.mark.parametrize("desired_class, total_CFs, features_to_vary, permitted_range, initialization", [(1, 2, "all", {'Numerical': [10, 15]}, "kdtree"), @@ -91,32 +60,6 @@ def test_permitted_range(self, desired_class, sample_custom_query_2, total_CFs, permitted_range[feature][1] for i in range(total_CFs)) - # Testing if you can provide permitted_range for categorical variables - @pytest.mark.parametrize("desired_class, total_CFs, features_to_vary, permitted_range, initialization", - [(1, 2, "all", {'Categorical': ['a', 'c']}, "kdtree"), - (1, 2, "all", {'Categorical': ['a', 'c']}, "random")]) - def test_permitted_range_categorical(self, desired_class, total_CFs, features_to_vary, sample_custom_query_2, - permitted_range, - initialization): - ans = self.exp.generate_counterfactuals(query_instances=sample_custom_query_2, - features_to_vary=features_to_vary, permitted_range=permitted_range, - total_CFs=total_CFs, desired_class=desired_class, - initialization=initialization) - - for cfs_example in ans.cf_examples_list: - for feature in permitted_range: - assert all( - permitted_range[feature][0] <= cfs_example.final_cfs_df[feature].values[i] <= - permitted_range[feature][1] for i - in range(total_CFs)) - - # Testing if an error is thrown when the query instance has outcome variable - def test_query_instance_with_target_column(self, sample_custom_query_6): - with pytest.raises(ValueError) as ve: - self.exp.setup("all", None, sample_custom_query_6, "inverse_mad") - - assert "present in query instance" in str(ve) - # Testing if only valid cfs are found after maxiterations @pytest.mark.parametrize("desired_class, total_CFs, initialization, maxiterations", [(0, 7, "kdtree", 0), (0, 7, "random", 0)]) diff --git a/tests/test_dice_interface/test_dice_random.py b/tests/test_dice_interface/test_dice_random.py deleted file mode 100644 index c1d09281..00000000 --- a/tests/test_dice_interface/test_dice_random.py +++ /dev/null @@ -1,164 +0,0 @@ -import pytest - -import dice_ml -from dice_ml.counterfactual_explanations import CounterfactualExplanations -from dice_ml.diverse_counterfactuals import CounterfactualExamples -from dice_ml.utils import helpers -from dice_ml.utils.exception import UserConfigValidationException - - -@pytest.fixture -def random_binary_classification_exp_object(): - backend = 'sklearn' - dataset = helpers.load_custom_testing_dataset_binary() - d = dice_ml.Data(dataframe=dataset, continuous_features=['Numerical'], outcome_name='Outcome') - ML_modelpath = helpers.get_custom_dataset_modelpath_pipeline_binary() - m = dice_ml.Model(model_path=ML_modelpath, backend=backend) - exp = dice_ml.Dice(d, m, method='random') - return exp - - -@pytest.fixture -def random_multi_classification_exp_object(): - backend = 'sklearn' - dataset = helpers.load_custom_testing_dataset_multiclass() - d = dice_ml.Data(dataframe=dataset, continuous_features=['Numerical'], outcome_name='Outcome') - ML_modelpath = helpers.get_custom_dataset_modelpath_pipeline_multiclass() - m = dice_ml.Model(model_path=ML_modelpath, backend=backend) - exp = dice_ml.Dice(d, m, method='random') - return exp - - -@pytest.fixture -def random_regression_exp_object(): - backend = 'sklearn' - dataset = helpers.load_custom_testing_dataset_regression() - d = dice_ml.Data(dataframe=dataset, continuous_features=['Numerical'], outcome_name='Outcome') - ML_modelpath = helpers.get_custom_dataset_modelpath_pipeline_regression() - m = dice_ml.Model(model_path=ML_modelpath, backend=backend, model_type='regressor') - exp = dice_ml.Dice(d, m, method='random') - return exp - - -class TestDiceRandomBinaryClassificationMethods: - @pytest.fixture(autouse=True) - def _initiate_exp_object(self, random_binary_classification_exp_object): - self.exp = random_binary_classification_exp_object # explainer object - - @pytest.mark.parametrize("desired_class, total_CFs", [(0, 1)]) - def test_random_counterfactual_explanations_output(self, desired_class, sample_custom_query_1, total_CFs): - counterfactual_explanations = self.exp.generate_counterfactuals( - query_instances=sample_custom_query_1, desired_class=desired_class, - total_CFs=total_CFs) - - assert counterfactual_explanations is not None - assert len(counterfactual_explanations.cf_examples_list) == sample_custom_query_1.shape[0] - assert counterfactual_explanations.cf_examples_list[0].final_cfs_df.shape[0] == total_CFs - - # When invalid desired_class is given - @pytest.mark.parametrize("desired_class, desired_range, total_CFs, features_to_vary, permitted_range", - [(7, None, 3, "all", None)]) - def test_no_cfs(self, desired_class, desired_range, sample_custom_query_1, total_CFs, features_to_vary, - permitted_range): - with pytest.raises(UserConfigValidationException): - self.exp._generate_counterfactuals(features_to_vary=features_to_vary, query_instance=sample_custom_query_1, - total_CFs=total_CFs, - desired_class=desired_class, desired_range=desired_range, - permitted_range=permitted_range) - - # When a query's feature value is not within the permitted range and the feature is not allowed to vary - @pytest.mark.parametrize("features_to_vary, permitted_range, feature_weights", - [(['Numerical'], {'Categorical': ['b', 'c']}, "inverse_mad")]) - def test_invalid_query_instance(self, sample_custom_query_1, features_to_vary, permitted_range, feature_weights): - with pytest.raises(ValueError): - self.exp.setup(features_to_vary, permitted_range, sample_custom_query_1, feature_weights) - - # Testing that the features_to_vary argument actually varies only the features that you wish to vary - @pytest.mark.parametrize("desired_class, desired_range, total_CFs, features_to_vary, permitted_range", - [(1, None, 2, ["Numerical"], None)]) - def test_features_to_vary(self, desired_class, desired_range, sample_custom_query_2, total_CFs, features_to_vary, - permitted_range): - features_to_vary = self.exp.setup(features_to_vary, None, sample_custom_query_2, "inverse_mad") - ans = self.exp._generate_counterfactuals(query_instance=sample_custom_query_2, - features_to_vary=features_to_vary, - total_CFs=total_CFs, desired_class=desired_class, - desired_range=desired_range, permitted_range=permitted_range) - - for feature in self.exp.data_interface.feature_names: - if feature not in features_to_vary: - assert all(ans.final_cfs_df[feature].values[i] == sample_custom_query_2[feature].values[0] for i in - range(total_CFs)) - - # Testing if you can provide permitted_range for categorical variables - @pytest.mark.parametrize("desired_class, desired_range, total_CFs, permitted_range", - [(1, None, 2, {'Categorical': ['a', 'c']})]) - def test_permitted_range_categorical(self, desired_class, desired_range, total_CFs, sample_custom_query_2, - permitted_range): - features_to_vary = self.exp.setup("all", permitted_range, sample_custom_query_2, "inverse_mad") - ans = self.exp._generate_counterfactuals(query_instance=sample_custom_query_2, - features_to_vary=features_to_vary, permitted_range=permitted_range, - total_CFs=total_CFs, desired_class=desired_class, - desired_range=desired_range) - - for feature in permitted_range: - assert all( - permitted_range[feature][0] <= ans.final_cfs_df[feature].values[i] <= permitted_range[feature][1] for i - in range(total_CFs)) - - -class TestDiceRandomRegressionMethods: - @pytest.fixture(autouse=True) - def _initiate_exp_object(self, random_regression_exp_object): - self.exp = random_regression_exp_object # explainer object - - # features_range - @pytest.mark.parametrize("features_to_vary, desired_class, desired_range, total_CFs, permitted_range", - [("all", None, [1, 2.8], 2, None)]) - def test_desired_range(self, features_to_vary, desired_class, desired_range, sample_custom_query_2, total_CFs, - permitted_range): - ans = self.exp._generate_counterfactuals(features_to_vary=features_to_vary, - query_instance=sample_custom_query_2, - total_CFs=total_CFs, desired_class=desired_class, - desired_range=desired_range, permitted_range=permitted_range) - assert all( - [desired_range[0]] * total_CFs <= ans.final_cfs_df[self.exp.data_interface.outcome_name].values) and all( - ans.final_cfs_df[self.exp.data_interface.outcome_name].values <= [desired_range[1]] * total_CFs) - - # Testing that the output of regression lies in the desired_range - @pytest.mark.parametrize("desired_range, total_CFs", [([1, 2.8], 6)]) - @pytest.mark.parametrize("version", ['2.0', '1.0']) - def test_random_output(self, desired_range, sample_custom_query_2, total_CFs, version): - cf_examples = self.exp._generate_counterfactuals(query_instance=sample_custom_query_2, total_CFs=total_CFs, - desired_range=desired_range) - assert all(desired_range[0] <= i <= desired_range[1] for i in self.exp.cfs_preds) - - assert cf_examples is not None - json_str = cf_examples.to_json(version) - assert json_str is not None - - recovered_cf_examples = CounterfactualExamples.from_json(json_str) - assert recovered_cf_examples is not None - assert cf_examples == recovered_cf_examples - - @pytest.mark.parametrize("desired_range, total_CFs", [([1, 2.8], 6)]) - def test_random_counterfactual_explanations_output(self, desired_range, sample_custom_query_2, total_CFs): - counterfactual_explanations = self.exp.generate_counterfactuals( - query_instances=sample_custom_query_2, total_CFs=total_CFs, - desired_range=desired_range) - - assert counterfactual_explanations is not None - json_str = counterfactual_explanations.to_json() - assert json_str is not None - - recovered_counterfactual_explanations = CounterfactualExplanations.from_json(json_str) - assert recovered_counterfactual_explanations is not None - assert counterfactual_explanations == recovered_counterfactual_explanations - - # Testing for 0 CFs needed - @pytest.mark.parametrize("features_to_vary, desired_class, desired_range, total_CFs, permitted_range", - [("all", None, [1, 2.8], 0, None)]) - def test_zero_cfs(self, features_to_vary, desired_class, desired_range, sample_custom_query_2, total_CFs, - permitted_range): - self.exp._generate_counterfactuals(features_to_vary=features_to_vary, query_instance=sample_custom_query_2, - total_CFs=total_CFs, desired_class=desired_class, - desired_range=desired_range, permitted_range=permitted_range) diff --git a/tests/test_dice_interface/test_explainer_base.py b/tests/test_dice_interface/test_explainer_base.py index 90d02cea..182f88af 100644 --- a/tests/test_dice_interface/test_explainer_base.py +++ b/tests/test_dice_interface/test_explainer_base.py @@ -3,6 +3,7 @@ from sklearn.ensemble import RandomForestRegressor import dice_ml +from dice_ml.counterfactual_explanations import CounterfactualExplanations from dice_ml.diverse_counterfactuals import CounterfactualExamples from dice_ml.explainer_interfaces.explainer_base import ExplainerBase from dice_ml.utils.exception import UserConfigValidationException @@ -18,17 +19,17 @@ def _verify_feature_importance(self, feature_importance): def test_check_any_counterfactuals_computed( self, method, - custom_public_data_interface, + custom_public_data_interface_binary, sklearn_binary_classification_model_interface ): exp = dice_ml.Dice( - custom_public_data_interface, + custom_public_data_interface_binary, sklearn_binary_classification_model_interface, method=method) - sample_custom_query = custom_public_data_interface.data_df[0:1] + sample_custom_query = custom_public_data_interface_binary.data_df[0:1] cf_example = CounterfactualExamples( - data_interface=custom_public_data_interface, + data_interface=custom_public_data_interface_binary, test_instance_df=sample_custom_query) cf_examples_arr = [cf_example] @@ -38,11 +39,11 @@ def test_check_any_counterfactuals_computed( exp._check_any_counterfactuals_computed(cf_examples_arr=cf_examples_arr) cf_example_has_cf = CounterfactualExamples( - data_interface=custom_public_data_interface, + data_interface=custom_public_data_interface_binary, final_cfs_df=sample_custom_query, test_instance_df=sample_custom_query) cf_example_no_cf = CounterfactualExamples( - data_interface=custom_public_data_interface, + data_interface=custom_public_data_interface_binary, test_instance_df=sample_custom_query) cf_examples_arr = [cf_example_has_cf, cf_example_no_cf] exp._check_any_counterfactuals_computed(cf_examples_arr=cf_examples_arr) @@ -50,11 +51,11 @@ def test_check_any_counterfactuals_computed( @pytest.mark.parametrize("desired_class", [1]) def test_zero_totalcfs( self, desired_class, method, sample_custom_query_1, - custom_public_data_interface, + custom_public_data_interface_binary, sklearn_binary_classification_model_interface ): exp = dice_ml.Dice( - custom_public_data_interface, + custom_public_data_interface_binary, sklearn_binary_classification_model_interface, method=method) @@ -68,10 +69,10 @@ def test_zero_totalcfs( def test_local_feature_importance( self, desired_class, method, sample_custom_query_1, sample_counterfactual_example_dummy, - custom_public_data_interface, + custom_public_data_interface_binary, sklearn_binary_classification_model_interface): exp = dice_ml.Dice( - custom_public_data_interface, + custom_public_data_interface_binary, sklearn_binary_classification_model_interface, method=method) sample_custom_query = pd.concat([sample_custom_query_1, sample_custom_query_1]) @@ -101,10 +102,10 @@ def test_local_feature_importance( def test_global_feature_importance( self, desired_class, method, sample_custom_query_10, sample_counterfactual_example_dummy, - custom_public_data_interface, + custom_public_data_interface_binary, sklearn_binary_classification_model_interface): exp = dice_ml.Dice( - custom_public_data_interface, + custom_public_data_interface_binary, sklearn_binary_classification_model_interface, method=method) @@ -128,14 +129,35 @@ def test_global_feature_importance( self._verify_feature_importance(global_importance.summary_importance) + # @pytest.mark.parametrize("desired_class", [1]) + # def test_columns_out_of_order( + # self, desired_class, method, sample_custom_query_1, + # custom_public_data_interface_binary_out_of_order, + # sklearn_binary_classification_model_interface): + # if method == 'genetic': + # pytest.skip('DiceGenetic takes a very long time to run this test') + + # exp = dice_ml.Dice( + # custom_public_data_interface_binary_out_of_order, + # sklearn_binary_classification_model_interface, + # method=method) + + # cf_explanation = exp.generate_counterfactuals( + # query_instances=sample_custom_query_1, + # total_CFs=1, + # desired_class=desired_class, + # features_to_vary='all') + + # assert cf_explanation is not None + @pytest.mark.parametrize("desired_class", [1]) def test_global_feature_importance_error_conditions_with_insufficient_query_points( self, desired_class, method, sample_custom_query_1, - custom_public_data_interface, + custom_public_data_interface_binary, sklearn_binary_classification_model_interface): exp = dice_ml.Dice( - custom_public_data_interface, + custom_public_data_interface_binary, sklearn_binary_classification_model_interface, method=method) @@ -166,10 +188,10 @@ def test_global_feature_importance_error_conditions_with_insufficient_query_poin def test_global_feature_importance_error_conditions_with_insufficient_cfs_per_query_point( self, desired_class, method, sample_custom_query_10, - custom_public_data_interface, + custom_public_data_interface_binary, sklearn_binary_classification_model_interface): exp = dice_ml.Dice( - custom_public_data_interface, + custom_public_data_interface_binary, sklearn_binary_classification_model_interface, method=method) @@ -201,10 +223,10 @@ def test_global_feature_importance_error_conditions_with_insufficient_cfs_per_qu def test_local_feature_importance_error_conditions_with_insufficient_cfs_per_query_point( self, desired_class, method, sample_custom_query_1, - custom_public_data_interface, + custom_public_data_interface_binary, sklearn_binary_classification_model_interface): exp = dice_ml.Dice( - custom_public_data_interface, + custom_public_data_interface_binary, sklearn_binary_classification_model_interface, method=method) @@ -231,26 +253,13 @@ def test_local_feature_importance_error_conditions_with_insufficient_cfs_per_que total_CFs=1, desired_class=desired_class) - # @pytest.mark.parametrize("desired_class, binary_classification_exp_object_out_of_order", - # [(1, 'random'), (1, 'genetic'), (1, 'kdtree')], - # indirect=['binary_classification_exp_object_out_of_order']) - # def test_columns_out_of_order(self, desired_class, binary_classification_exp_object_out_of_order, sample_custom_query_1): - # exp = binary_classification_exp_object_out_of_order # explainer object - # exp._generate_counterfactuals( - # query_instance=sample_custom_query_1, - # total_CFs=0, - # desired_class=desired_class, - # desired_range=None, - # permitted_range=None, - # features_to_vary='all') - @pytest.mark.parametrize("desired_class", [1]) def test_incorrect_features_to_vary_list( self, desired_class, method, sample_custom_query_1, - custom_public_data_interface, + custom_public_data_interface_binary, sklearn_binary_classification_model_interface): exp = dice_ml.Dice( - custom_public_data_interface, + custom_public_data_interface_binary, sklearn_binary_classification_model_interface, method=method) with pytest.raises( @@ -267,10 +276,10 @@ def test_incorrect_features_to_vary_list( @pytest.mark.parametrize("desired_class", [1]) def test_incorrect_features_permitted_range( self, desired_class, method, sample_custom_query_1, - custom_public_data_interface, + custom_public_data_interface_binary, sklearn_binary_classification_model_interface): exp = dice_ml.Dice( - custom_public_data_interface, + custom_public_data_interface_binary, sklearn_binary_classification_model_interface, method=method) with pytest.raises( @@ -287,10 +296,10 @@ def test_incorrect_features_permitted_range( @pytest.mark.parametrize("desired_class", [1]) def test_incorrect_values_permitted_range( self, desired_class, method, sample_custom_query_1, - custom_public_data_interface, + custom_public_data_interface_binary, sklearn_binary_classification_model_interface): exp = dice_ml.Dice( - custom_public_data_interface, + custom_public_data_interface_binary, sklearn_binary_classification_model_interface, method=method) with pytest.raises(UserConfigValidationException) as ucve: @@ -309,10 +318,10 @@ def test_incorrect_values_permitted_range( @pytest.mark.parametrize("desired_class", [100, 'a']) def test_unsupported_binary_class( self, desired_class, method, sample_custom_query_1, - custom_public_data_interface, + custom_public_data_interface_binary, sklearn_binary_classification_model_interface): exp = dice_ml.Dice( - custom_public_data_interface, + custom_public_data_interface_binary, sklearn_binary_classification_model_interface, method=method) with pytest.raises(UserConfigValidationException) as ucve: @@ -327,10 +336,10 @@ def test_unsupported_binary_class( @pytest.mark.parametrize("desired_class", [1]) def test_query_instance_unknown_column( self, desired_class, method, sample_custom_query_5, - custom_public_data_interface, + custom_public_data_interface_binary, sklearn_binary_classification_model_interface): exp = dice_ml.Dice( - custom_public_data_interface, + custom_public_data_interface_binary, sklearn_binary_classification_model_interface, method=method) with pytest.raises(ValueError, match='not present in training data'): @@ -342,10 +351,10 @@ def test_query_instance_unknown_column( @pytest.mark.parametrize("desired_class", [1]) def test_query_instance_outside_bounds( self, desired_class, method, sample_custom_query_3, - custom_public_data_interface, + custom_public_data_interface_binary, sklearn_binary_classification_model_interface): exp = dice_ml.Dice( - custom_public_data_interface, + custom_public_data_interface_binary, sklearn_binary_classification_model_interface, method=method) with pytest.raises(ValueError, match='has a value outside the dataset'): @@ -356,16 +365,21 @@ def test_query_instance_outside_bounds( @pytest.mark.parametrize("desired_class", [1]) def test_desired_class( self, desired_class, method, sample_custom_query_2, - custom_public_data_interface, + custom_public_data_interface_binary, sklearn_binary_classification_model_interface): exp = dice_ml.Dice( - custom_public_data_interface, + custom_public_data_interface_binary, sklearn_binary_classification_model_interface, method=method) ans = exp.generate_counterfactuals(query_instances=sample_custom_query_2, features_to_vary='all', total_CFs=2, desired_class=desired_class, permitted_range=None) + + assert ans is not None + assert len(ans.cf_examples_list) == sample_custom_query_2.shape[0] + assert ans.cf_examples_list[0].final_cfs_df.shape[0] == 2 + if method != 'kdtree': assert all(ans.cf_examples_list[0].final_cfs_df[exp.data_interface.outcome_name].values == [desired_class] * 2) else: @@ -376,10 +390,13 @@ def test_desired_class( [(1, 1, {'Numerical': [10, 150]})]) def test_permitted_range( self, desired_class, method, total_CFs, permitted_range, sample_custom_query_2, - custom_public_data_interface, + custom_public_data_interface_binary, sklearn_binary_classification_model_interface): + if method == 'kdtree': + pytest.skip('DiceKD cannot seem to handle permitted_range') + exp = dice_ml.Dice( - custom_public_data_interface, + custom_public_data_interface_binary, sklearn_binary_classification_model_interface, method=method) ans = exp.generate_counterfactuals(query_instances=sample_custom_query_2, @@ -401,11 +418,11 @@ def test_permitted_range( [("all", 0, None, 0, None)]) def test_zero_cfs_internal( self, method, features_to_vary, desired_class, desired_range, sample_custom_query_2, total_CFs, - permitted_range, custom_public_data_interface, sklearn_binary_classification_model_interface): + permitted_range, custom_public_data_interface_binary, sklearn_binary_classification_model_interface): if method == 'genetic': pytest.skip('DiceGenetic explainer does not handle the total counterfactuals as zero') exp = dice_ml.Dice( - custom_public_data_interface, + custom_public_data_interface_binary, sklearn_binary_classification_model_interface, method=method) features_to_vary = exp.setup(features_to_vary, None, sample_custom_query_2, "inverse_mad") @@ -413,6 +430,100 @@ def test_zero_cfs_internal( total_CFs=total_CFs, desired_class=desired_class, desired_range=desired_range, permitted_range=permitted_range) + # Testing if you can provide permitted_range for categorical variables + @pytest.mark.parametrize("desired_class", [1]) + @pytest.mark.parametrize("permitted_range", [{'Categorical': ['b', 'c']}]) + @pytest.mark.parametrize("genetic_initialization", ['kdtree', 'random']) + def test_permitted_range_categorical( + self, method, desired_class, permitted_range, genetic_initialization, + sample_custom_query_2, custom_public_data_interface_binary, + sklearn_binary_classification_model_interface): + exp = dice_ml.Dice( + custom_public_data_interface_binary, + sklearn_binary_classification_model_interface, + method=method) + + if method != 'genetic': + ans = exp.generate_counterfactuals( + query_instances=sample_custom_query_2, + permitted_range=permitted_range, + total_CFs=2, desired_class=desired_class) + else: + ans = exp.generate_counterfactuals( + query_instances=sample_custom_query_2, + initialization=genetic_initialization, + permitted_range=permitted_range, + total_CFs=2, desired_class=desired_class) + + assert all(i in permitted_range["Categorical"] for i in ans.cf_examples_list[0].final_cfs_df.Categorical.values) + + # Testing that the features_to_vary argument actually varies only the features that you wish to vary + @pytest.mark.parametrize("desired_class, total_CFs, features_to_vary", + [(1, 1, ["Numerical"])]) + @pytest.mark.parametrize("genetic_initialization", ['kdtree', 'random']) + def test_features_to_vary( + self, method, desired_class, sample_custom_query_2, + total_CFs, features_to_vary, genetic_initialization, + custom_public_data_interface_binary, + sklearn_binary_classification_model_interface): + exp = dice_ml.Dice( + custom_public_data_interface_binary, + sklearn_binary_classification_model_interface, + method=method) + if method != 'genetic': + ans = exp.generate_counterfactuals( + query_instances=sample_custom_query_2, + features_to_vary=features_to_vary, + total_CFs=total_CFs, desired_class=desired_class) + else: + ans = exp.generate_counterfactuals( + query_instances=sample_custom_query_2, + features_to_vary=features_to_vary, + total_CFs=total_CFs, desired_class=desired_class, + initialization=genetic_initialization) + + for feature in exp.data_interface.feature_names: + if feature not in features_to_vary: + if method != 'kdtree': + assert all( + ans.cf_examples_list[0].final_cfs_df[feature].values[i] == sample_custom_query_2[feature].values[0] + for i in range(total_CFs)) + else: + assert all( + ans.cf_examples_list[0].final_cfs_df_sparse[feature].values[i] == + sample_custom_query_2[feature].values[0] + for i in range(total_CFs)) + + # When a query's feature value is not within the permitted range and the feature is not allowed to vary + @pytest.mark.parametrize("features_to_vary, permitted_range, feature_weights", + [(['Numerical'], {'Categorical': ['b', 'c']}, "inverse_mad")]) + def test_invalid_query_instance( + self, method, sample_custom_query_1, + features_to_vary, permitted_range, + feature_weights, + custom_public_data_interface_binary, + sklearn_binary_classification_model_interface): + exp = dice_ml.Dice( + custom_public_data_interface_binary, + sklearn_binary_classification_model_interface, + method=method) + with pytest.raises(ValueError): + exp.setup(features_to_vary, permitted_range, sample_custom_query_1, feature_weights) + + # Testing if an error is thrown when the query instance has outcome variable + def test_query_instance_with_target_column( + self, method, sample_custom_query_6, + custom_public_data_interface_binary, + sklearn_binary_classification_model_interface): + exp = dice_ml.Dice( + custom_public_data_interface_binary, + sklearn_binary_classification_model_interface, + method=method) + with pytest.raises(ValueError) as ve: + exp.setup("all", None, sample_custom_query_6, "inverse_mad") + + assert "present in query instance" in str(ve) + @pytest.mark.parametrize("method", ['random', 'genetic', 'kdtree']) class TestExplainerBaseMultiClassClassification: @@ -420,10 +531,10 @@ class TestExplainerBaseMultiClassClassification: @pytest.mark.parametrize("desired_class", [1]) def test_zero_totalcfs( self, desired_class, method, sample_custom_query_1, - custom_public_data_interface, + custom_public_data_interface_multicalss, sklearn_multiclass_classification_model_interface): exp = dice_ml.Dice( - custom_public_data_interface, + custom_public_data_interface_multicalss, sklearn_multiclass_classification_model_interface, method=method) with pytest.raises(UserConfigValidationException): @@ -435,13 +546,15 @@ def test_zero_totalcfs( # Testing that the counterfactuals are in the desired class @pytest.mark.parametrize("desired_class, total_CFs", [(2, 2)]) @pytest.mark.parametrize("genetic_initialization", ['kdtree', 'random']) + @pytest.mark.parametrize('posthoc_sparsity_algorithm', ['linear', 'binary', None]) def test_desired_class( self, desired_class, total_CFs, method, genetic_initialization, + posthoc_sparsity_algorithm, sample_custom_query_2, - custom_public_data_interface, + custom_public_data_interface_multicalss, sklearn_multiclass_classification_model_interface): exp = dice_ml.Dice( - custom_public_data_interface, + custom_public_data_interface_multicalss, sklearn_multiclass_classification_model_interface, method=method) @@ -453,7 +566,8 @@ def test_desired_class( ans = exp.generate_counterfactuals( query_instances=sample_custom_query_2, total_CFs=total_CFs, desired_class=desired_class, - initialization=genetic_initialization) + initialization=genetic_initialization, + posthoc_sparsity_algorithm=posthoc_sparsity_algorithm) assert ans is not None if method != 'kdtree': @@ -469,10 +583,10 @@ def test_desired_class( @pytest.mark.parametrize("desired_class, total_CFs", [(100, 3), ('opposite', 3)]) def test_unsupported_multiclass( self, desired_class, total_CFs, method, sample_custom_query_4, - custom_public_data_interface, + custom_public_data_interface_multicalss, sklearn_multiclass_classification_model_interface): exp = dice_ml.Dice( - custom_public_data_interface, + custom_public_data_interface_multicalss, sklearn_multiclass_classification_model_interface, method=method) with pytest.raises(UserConfigValidationException) as ucve: @@ -488,11 +602,11 @@ def test_unsupported_multiclass( [("all", 0, None, 0, None)]) def test_zero_cfs_internal( self, method, features_to_vary, desired_class, desired_range, sample_custom_query_2, total_CFs, - permitted_range, custom_public_data_interface, sklearn_multiclass_classification_model_interface): + permitted_range, custom_public_data_interface_multicalss, sklearn_multiclass_classification_model_interface): if method == 'genetic': pytest.skip('DiceGenetic explainer does not handle the total counterfactuals as zero') exp = dice_ml.Dice( - custom_public_data_interface, + custom_public_data_interface_multicalss, sklearn_multiclass_classification_model_interface, method=method) features_to_vary = exp.setup(features_to_vary, None, sample_custom_query_2, "inverse_mad") @@ -501,22 +615,31 @@ def test_zero_cfs_internal( desired_range=desired_range, permitted_range=permitted_range) +@pytest.mark.parametrize("method", ['random', 'genetic', 'kdtree']) class TestExplainerBaseRegression: - @pytest.mark.parametrize("desired_range, regression_exp_object", - [([10, 100], 'random'), ([10, 100], 'genetic'), ([10, 100], 'kdtree')], - indirect=['regression_exp_object']) - def test_zero_totalcfs(self, desired_range, regression_exp_object, sample_custom_query_1): - exp = regression_exp_object # explainer object + @pytest.mark.parametrize("desired_range", [[10, 100]]) + def test_zero_cfs( + self, desired_range, method, + custom_public_data_interface_regression, + sklearn_regression_model_interface, + sample_custom_query_1): + exp = dice_ml.Dice( + custom_public_data_interface_regression, + sklearn_regression_model_interface, + method=method) + with pytest.raises(UserConfigValidationException): exp.generate_counterfactuals( query_instances=[sample_custom_query_1], total_CFs=0, desired_range=desired_range) - @pytest.mark.parametrize("desired_range, method", - [([10, 100], 'random')]) + @pytest.mark.parametrize("desired_range", [[10, 100]]) def test_numeric_categories(self, desired_range, method, create_boston_data): + if method == 'genetic' or method == 'kdtree': + pytest.skip('DiceGenetic/DiceKD explainer does not handle numeric categories') + x_train, x_test, y_train, y_test, feature_names = \ create_boston_data @@ -539,6 +662,78 @@ def test_numeric_categories(self, desired_range, method, create_boston_data): assert cf_explanation is not None + # Testing for 0 CFs needed + @pytest.mark.parametrize("desired_range, total_CFs", [([1, 2.8], 0)]) + def test_zero_cfs_internal( + self, desired_range, method, total_CFs, + custom_public_data_interface_regression, + sklearn_regression_model_interface, + sample_custom_query_4): + if method == 'genetic': + pytest.skip('DiceGenetic explainer does not handle the total counterfactuals as zero') + exp = dice_ml.Dice( + custom_public_data_interface_regression, + sklearn_regression_model_interface, + method=method) + + exp._generate_counterfactuals(query_instance=sample_custom_query_4, total_CFs=total_CFs, + desired_range=desired_range) + + @pytest.mark.parametrize("desired_range, total_CFs", [([1, 2.8], 6)]) + @pytest.mark.parametrize("version", ['2.0', '1.0']) + @pytest.mark.parametrize('posthoc_sparsity_algorithm', ['linear', 'binary', None]) + def test_counterfactual_explanations_output( + self, desired_range, total_CFs, method, version, + posthoc_sparsity_algorithm, sample_custom_query_2, + custom_public_data_interface_regression, + sklearn_regression_model_interface): + if method == 'genetic' and version == '1.0': + pytest.skip('DiceGenetic cannot be serialized using version 1.0 serialization logic') + + exp = dice_ml.Dice( + custom_public_data_interface_regression, + sklearn_regression_model_interface, + method=method) + + counterfactual_explanations = exp.generate_counterfactuals( + query_instances=sample_custom_query_2, total_CFs=total_CFs, + desired_range=desired_range, + posthoc_sparsity_algorithm=posthoc_sparsity_algorithm) + + counterfactual_examples = counterfactual_explanations.cf_examples_list[0] + + if method != 'kdtree': + assert all( + [desired_range[0]] * counterfactual_examples.final_cfs_df.shape[0] <= + counterfactual_examples.final_cfs_df[exp.data_interface.outcome_name].values) and \ + all(counterfactual_examples.final_cfs_df[exp.data_interface.outcome_name].values <= + [desired_range[1]] * counterfactual_examples.final_cfs_df.shape[0]) + else: + assert all( + [desired_range[0]] * counterfactual_examples.final_cfs_df_sparse.shape[0] <= + counterfactual_examples.final_cfs_df_sparse[exp.data_interface.outcome_name].values) and \ + all( + counterfactual_examples.final_cfs_df_sparse[exp.data_interface.outcome_name].values <= + [desired_range[1]] * counterfactual_examples.final_cfs_df_sparse.shape[0]) + + assert all(desired_range[0] <= i <= desired_range[1] for i in exp.cfs_preds) + + assert counterfactual_explanations is not None + json_str = counterfactual_explanations.to_json() + assert json_str is not None + + recovered_counterfactual_explanations = CounterfactualExplanations.from_json(json_str) + assert recovered_counterfactual_explanations is not None + assert counterfactual_explanations == recovered_counterfactual_explanations + + assert counterfactual_examples is not None + json_str = counterfactual_examples.to_json(version) + assert json_str is not None + + recovered_counterfactual_examples = CounterfactualExamples.from_json(json_str) + assert recovered_counterfactual_examples is not None + assert counterfactual_examples == recovered_counterfactual_examples + class TestExplainerBase: