Unify sklearn explainer tests - part 2

tests/conftest.py

@@ -9,52 +9,29 @@
 from dice_ml.utils import helpers
 
 
-@pytest.fixture
-def binary_classification_exp_object(method="random"):
-    backend = 'sklearn'
-    dataset = helpers.load_custom_testing_dataset_binary()
-    d = dice_ml.Data(dataframe=dataset, continuous_features=['Numerical'], outcome_name='Outcome')
-    ML_modelpath = helpers.get_custom_dataset_modelpath_pipeline_binary()
-    m = dice_ml.Model(model_path=ML_modelpath, backend=backend)
-    exp = dice_ml.Dice(d, m, method=method)
-    return exp
-
-
-@pytest.fixture
-def binary_classification_exp_object_out_of_order(method="random"):
-    backend = 'sklearn'
+@pytest.fixture(scope='session')
+def custom_public_data_interface_binary_out_of_order():
     dataset = helpers.load_outcome_not_last_column_dataset()
     d = dice_ml.Data(dataframe=dataset, continuous_features=['Numerical'], outcome_name='Outcome')
-    ML_modelpath = helpers.get_custom_dataset_modelpath_pipeline_binary()
-    m = dice_ml.Model(model_path=ML_modelpath, backend=backend)
-    exp = dice_ml.Dice(d, m, method=method)
-    return exp
+    return d
 
 
-@pytest.fixture
-def multi_classification_exp_object(method="random"):
-    backend = 'sklearn'
-    dataset = helpers.load_custom_testing_dataset_multiclass()
+@pytest.fixture(scope='session')
+def custom_public_data_interface_binary():
+    dataset = helpers.load_custom_testing_dataset_binary()
     d = dice_ml.Data(dataframe=dataset, continuous_features=['Numerical'], outcome_name='Outcome')
-    ML_modelpath = helpers.get_custom_dataset_modelpath_pipeline_multiclass()
-    m = dice_ml.Model(model_path=ML_modelpath, backend=backend)
-    exp = dice_ml.Dice(d, m, method=method)
-    return exp
+    return d
 
 
-@pytest.fixture
-def regression_exp_object(method="random"):
-    backend = 'sklearn'
-    dataset = helpers.load_custom_testing_dataset_regression()
+@pytest.fixture(scope='session')
+def custom_public_data_interface_multicalss():
+    dataset = helpers.load_custom_testing_dataset_multiclass()
     d = dice_ml.Data(dataframe=dataset, continuous_features=['Numerical'], outcome_name='Outcome')
-    ML_modelpath = helpers.get_custom_dataset_modelpath_pipeline_regression()
-    m = dice_ml.Model(model_path=ML_modelpath, backend=backend, model_type='regressor')
-    exp = dice_ml.Dice(d, m, method=method)
-    return exp
+    return d
 
 
 @pytest.fixture(scope='session')
-def custom_public_data_interface():
+def custom_public_data_interface_regression():
     dataset = helpers.load_custom_testing_dataset_regression()
     d = dice_ml.Data(dataframe=dataset, continuous_features=['Numerical'], outcome_name='Outcome')
     return d
@@ -77,7 +54,7 @@ def sklearn_multiclass_classification_model_interface():
 @pytest.fixture(scope='session')
 def sklearn_regression_model_interface():
     ML_modelpath = helpers.get_custom_dataset_modelpath_pipeline_regression()
-    m = dice_ml.Model(model_path=ML_modelpath, backend='sklearn', model_type='regression')
+    m = dice_ml.Model(model_path=ML_modelpath, backend='sklearn', model_type='regressor')
     return m
 
 

tests/test_dice_interface/test_dice_KD.py

@@ -2,8 +2,6 @@
 import pytest
 
 import dice_ml
-from dice_ml.counterfactual_explanations import CounterfactualExplanations
-from dice_ml.diverse_counterfactuals import CounterfactualExamples
 from dice_ml.utils import helpers
 
 
@@ -18,46 +16,12 @@ def KD_binary_classification_exp_object():
     return exp
 
 
-@pytest.fixture
-def KD_multi_classification_exp_object():
-    backend = 'sklearn'
-    dataset = helpers.load_custom_testing_dataset_multiclass()
-    d = dice_ml.Data(dataframe=dataset, continuous_features=['Numerical'], outcome_name='Outcome')
-    ML_modelpath = helpers.get_custom_dataset_modelpath_pipeline_multiclass()
-    m = dice_ml.Model(model_path=ML_modelpath, backend=backend)
-    exp = dice_ml.Dice(d, m, method='kdtree')
-    return exp
-
-
-@pytest.fixture
-def KD_regression_exp_object():
-    backend = 'sklearn'
-    dataset = helpers.load_custom_testing_dataset_regression()
-    d = dice_ml.Data(dataframe=dataset, continuous_features=['Numerical'], outcome_name='Outcome')
-    ML_modelpath = helpers.get_custom_dataset_modelpath_pipeline_regression()
-    m = dice_ml.Model(model_path=ML_modelpath, backend=backend, model_type='regressor')
-    exp = dice_ml.Dice(d, m, method='kdtree')
-    return exp
-
-
 class TestDiceKDBinaryClassificationMethods:
     @pytest.fixture(autouse=True)
     def _initiate_exp_object(self, KD_binary_classification_exp_object):
         self.exp = KD_binary_classification_exp_object  # explainer object
         self.data_df_copy = self.exp.data_interface.data_df.copy()
 
-    # When a query's feature value is not within the permitted range and the feature is not allowed to vary
-    @pytest.mark.parametrize("desired_range, desired_class, total_CFs, features_to_vary, permitted_range",
-                             [(None, 0, 4, ['Numerical'], {'Categorical': ['b', 'c']})])
-    def test_invalid_query_instance(self, desired_range, desired_class, sample_custom_query_1, total_CFs,
-                                    features_to_vary, permitted_range):
-        self.exp.dataset_with_predictions, self.exp.KD_tree, self.exp.predictions = \
-            self.exp.build_KD_tree(self.data_df_copy, desired_range, desired_class, self.exp.predicted_outcome_name)
-
-        with pytest.raises(ValueError):
-            self.exp._generate_counterfactuals(query_instance=sample_custom_query_1, total_CFs=total_CFs,
-                                               features_to_vary=features_to_vary, permitted_range=permitted_range)
-
     # Verifying the output of the KD tree
     @pytest.mark.parametrize("desired_class, total_CFs", [(0, 1)])
     @pytest.mark.parametrize('posthoc_sparsity_algorithm', ['linear', 'binary', None])
@@ -71,26 +35,6 @@ def test_KD_tree_output(self, desired_class, sample_custom_query_1, total_CFs, p
         assert all(self.exp.final_cfs_df.Numerical == expected_output.Numerical[0]) and \
                all(self.exp.final_cfs_df.Categorical == expected_output.Categorical[0])
 
-    # Verifying the output of the KD tree
-    @pytest.mark.parametrize("desired_class, total_CFs", [(0, 1)])
-    def test_KD_tree_counterfactual_explanations_output(self, desired_class, sample_custom_query_1, total_CFs):
-        counterfactual_explanations = self.exp.generate_counterfactuals(
-            query_instances=sample_custom_query_1, desired_class=desired_class,
-            total_CFs=total_CFs)
-
-        assert counterfactual_explanations is not None
-
-    # Testing that the features_to_vary argument actually varies only the features that you wish to vary
-    @pytest.mark.parametrize("desired_class, total_CFs, features_to_vary", [(0, 1, ["Numerical"])])
-    def test_features_to_vary(self, desired_class, sample_custom_query_2, total_CFs, features_to_vary):
-        self.exp._generate_counterfactuals(query_instance=sample_custom_query_2, desired_class=desired_class,
-                                           total_CFs=total_CFs, features_to_vary=features_to_vary)
-        self.exp.final_cfs_df.Numerical = self.exp.final_cfs_df.Numerical.astype(int)
-        expected_output = self.exp.data_interface.data_df
-
-        assert all(self.exp.final_cfs_df.Numerical == expected_output.Numerical[1]) and \
-               all(self.exp.final_cfs_df.Categorical == expected_output.Categorical[1])
-
     # Testing that the permitted_range argument actually varies the features only within the permitted_range
     @pytest.mark.parametrize("desired_class, total_CFs, permitted_range", [(0, 1, {'Numerical': [1000, 10000]})])
     def test_permitted_range(self, desired_class, sample_custom_query_2, total_CFs, permitted_range):
@@ -101,13 +45,6 @@ def test_permitted_range(self, desired_class, sample_custom_query_2, total_CFs,
         assert all(self.exp.final_cfs_df.Numerical == expected_output.Numerical[1]) and \
                all(self.exp.final_cfs_df.Categorical == expected_output.Categorical[1])
 
-    # Testing if you can provide permitted_range for categorical variables
-    @pytest.mark.parametrize("desired_class, total_CFs, permitted_range", [(0, 4, {'Categorical': ['b', 'c']})])
-    def test_permitted_range_categorical(self, desired_class, sample_custom_query_2, total_CFs, permitted_range):
-        self.exp._generate_counterfactuals(query_instance=sample_custom_query_2, desired_class=desired_class,
-                                           total_CFs=total_CFs, permitted_range=permitted_range)
-        assert all(i in permitted_range["Categorical"] for i in self.exp.final_cfs_df.Categorical.values)
-
     # Ensuring that there are no duplicates in the resulting counterfactuals even if the dataset has duplicates
     @pytest.mark.parametrize("desired_class, total_CFs", [(0, 2)])
     def test_duplicates(self, desired_class, sample_custom_query_4, total_CFs):
@@ -130,66 +67,3 @@ def test_index(self, desired_class, sample_custom_query_index, total_CFs, postho
                                            desired_class=desired_class,
                                            posthoc_sparsity_algorithm=posthoc_sparsity_algorithm)
         assert self.exp.final_cfs_df.index[0] == 3
-
-
-class TestDiceKDMultiClassificationMethods:
-    @pytest.fixture(autouse=True)
-    def _initiate_exp_object(self, KD_multi_classification_exp_object):
-        self.exp_multi = KD_multi_classification_exp_object  # explainer object
-        self.data_df_copy = self.exp_multi.data_interface.data_df.copy()
-
-    # Testing that the output of multiclass classification lies in the desired_class
-    @pytest.mark.parametrize("desired_class, total_CFs", [(2, 3)])
-    @pytest.mark.parametrize('posthoc_sparsity_algorithm', ['linear', 'binary', None])
-    def test_KD_tree_output(self, desired_class, sample_custom_query_2, total_CFs,
-                            posthoc_sparsity_algorithm):
-        self.exp_multi._generate_counterfactuals(query_instance=sample_custom_query_2, total_CFs=total_CFs,
-                                                 desired_class=desired_class,
-                                                 posthoc_sparsity_algorithm=posthoc_sparsity_algorithm)
-        assert all(i == desired_class for i in self.exp_multi.cfs_preds)
-
-
-class TestDiceKDRegressionMethods:
-    @pytest.fixture(autouse=True)
-    def _initiate_exp_object(self, KD_regression_exp_object):
-        self.exp_regr = KD_regression_exp_object  # explainer object
-        self.data_df_copy = self.exp_regr.data_interface.data_df.copy()
-
-    # Testing that the output of regression lies in the desired_range
-    @pytest.mark.parametrize("desired_range, total_CFs", [([1, 2.8], 6)])
-    @pytest.mark.parametrize("version", ['2.0', '1.0'])
-    @pytest.mark.parametrize('posthoc_sparsity_algorithm', ['linear', 'binary', None])
-    def test_KD_tree_output(self, desired_range, sample_custom_query_2, total_CFs, version, posthoc_sparsity_algorithm):
-        cf_examples = self.exp_regr._generate_counterfactuals(query_instance=sample_custom_query_2, total_CFs=total_CFs,
-                                                              desired_range=desired_range,
-                                                              posthoc_sparsity_algorithm=posthoc_sparsity_algorithm)
-        assert all(desired_range[0] <= i <= desired_range[1] for i in self.exp_regr.cfs_preds)
-
-        assert cf_examples is not None
-        json_str = cf_examples.to_json(version)
-        assert json_str is not None
-
-        recovered_cf_examples = CounterfactualExamples.from_json(json_str)
-        assert recovered_cf_examples is not None
-        assert cf_examples == recovered_cf_examples
-
-    @pytest.mark.parametrize("desired_range, total_CFs", [([1, 2.8], 6)])
-    def test_KD_tree_counterfactual_explanations_output(self, desired_range, sample_custom_query_2,
-                                                        total_CFs):
-        counterfactual_explanations = self.exp_regr.generate_counterfactuals(
-                                            query_instances=sample_custom_query_2, total_CFs=total_CFs,
-                                            desired_range=desired_range)
-
-        assert counterfactual_explanations is not None
-        json_str = counterfactual_explanations.to_json()
-        assert json_str is not None
-
-        recovered_counterfactual_explanations = CounterfactualExplanations.from_json(json_str)
-        assert recovered_counterfactual_explanations is not None
-        assert counterfactual_explanations == recovered_counterfactual_explanations
-
-    # Testing for 0 CFs needed
-    @pytest.mark.parametrize("desired_class, desired_range, total_CFs", [(0, [1, 2.8], 0)])
-    def test_zero_cfs(self, desired_class, desired_range, sample_custom_query_4, total_CFs):
-        self.exp_regr._generate_counterfactuals(query_instance=sample_custom_query_4, total_CFs=total_CFs,
-                                                desired_range=desired_range)

tests/test_dice_interface/test_dice_genetic.py

@@ -2,7 +2,6 @@
 
 import dice_ml
 from dice_ml.utils import helpers
-from dice_ml.utils.exception import UserConfigValidationException
 
 
 @pytest.fixture
@@ -43,36 +42,6 @@ class TestDiceGeneticBinaryClassificationMethods:
     def _initiate_exp_object(self, genetic_binary_classification_exp_object):
         self.exp = genetic_binary_classification_exp_object  # explainer object
 
-    # When invalid desired_class is given
-    @pytest.mark.parametrize("desired_class, total_CFs", [(7, 3)])
-    def test_no_cfs(self, desired_class, sample_custom_query_1, total_CFs):
-        with pytest.raises(UserConfigValidationException):
-            self.exp.generate_counterfactuals(query_instances=sample_custom_query_1, total_CFs=total_CFs,
-                                              desired_class=desired_class)
-
-    # When a query's feature value is not within the permitted range and the feature is not allowed to vary
-    @pytest.mark.parametrize("features_to_vary, permitted_range, feature_weights",
-                             [(['Numerical'], {'Categorical': ['b', 'c']}, "inverse_mad")])
-    def test_invalid_query_instance(self, sample_custom_query_1, features_to_vary, permitted_range, feature_weights):
-        with pytest.raises(ValueError):
-            self.exp.setup(features_to_vary, permitted_range, sample_custom_query_1, feature_weights)
-
-    # Testing that the features_to_vary argument actually varies only the features that you wish to vary
-    @pytest.mark.parametrize("desired_class, total_CFs, features_to_vary, initialization",
-                             [(1, 2, ["Numerical"], "kdtree"), (1, 2, ["Numerical"], "random")])
-    def test_features_to_vary(self, desired_class, sample_custom_query_2, total_CFs, features_to_vary, initialization):
-        ans = self.exp.generate_counterfactuals(query_instances=sample_custom_query_2,
-                                                features_to_vary=features_to_vary,
-                                                total_CFs=total_CFs, desired_class=desired_class,
-                                                initialization=initialization)
-
-        for cfs_example in ans.cf_examples_list:
-            for feature in self.exp.data_interface.feature_names:
-                if feature not in features_to_vary:
-                    assert all(
-                        cfs_example.final_cfs_df[feature].values[i] == sample_custom_query_2[feature].values[0] for i in
-                        range(total_CFs))
-
     # Testing that the permitted_range argument actually varies the features only within the permitted_range
     @pytest.mark.parametrize("desired_class, total_CFs, features_to_vary, permitted_range, initialization",
                              [(1, 2, "all", {'Numerical': [10, 15]}, "kdtree"),
@@ -91,32 +60,6 @@ def test_permitted_range(self, desired_class, sample_custom_query_2, total_CFs,
                     permitted_range[feature][1] for i
                     in range(total_CFs))
 
-    # Testing if you can provide permitted_range for categorical variables
-    @pytest.mark.parametrize("desired_class, total_CFs, features_to_vary, permitted_range, initialization",
-                             [(1, 2, "all", {'Categorical': ['a', 'c']}, "kdtree"),
-                              (1, 2, "all", {'Categorical': ['a', 'c']}, "random")])
-    def test_permitted_range_categorical(self, desired_class, total_CFs, features_to_vary, sample_custom_query_2,
-                                         permitted_range,
-                                         initialization):
-        ans = self.exp.generate_counterfactuals(query_instances=sample_custom_query_2,
-                                                features_to_vary=features_to_vary, permitted_range=permitted_range,
-                                                total_CFs=total_CFs, desired_class=desired_class,
-                                                initialization=initialization)
-
-        for cfs_example in ans.cf_examples_list:
-            for feature in permitted_range:
-                assert all(
-                    permitted_range[feature][0] <= cfs_example.final_cfs_df[feature].values[i] <=
-                    permitted_range[feature][1] for i
-                    in range(total_CFs))
-
-    # Testing if an error is thrown when the query instance has outcome variable
-    def test_query_instance_with_target_column(self, sample_custom_query_6):
-        with pytest.raises(ValueError) as ve:
-            self.exp.setup("all", None, sample_custom_query_6, "inverse_mad")
-
-        assert "present in query instance" in str(ve)
-
     # Testing if only valid cfs are found after maxiterations
     @pytest.mark.parametrize("desired_class, total_CFs, initialization, maxiterations",
                              [(0, 7, "kdtree", 0), (0, 7, "random", 0)])