Custom Check Templates#

This page supplies templates for the different types of custom checks that you can create using the deepchecks package. For more information on custom checks, please see the Creating a Custom Check

Templates:

Single Dataset Check#

Check type for cases when running on a single dataset and optional model, for example integrity checks. When in suite if 2 datasets are supplied it will run on both independently.

import pandas as pd

from deepchecks.core import CheckResult, ConditionCategory, ConditionResult, DatasetKind
from deepchecks.tabular import SingleDatasetCheck, Dataset, Context

class SingleDatasetCustomCheck(SingleDatasetCheck):
    """Description of the check. The name of the check will be the class name split by upper case letters."""

    # OPTIONAL: we can add different properties in the init
    def __init__(self, prop_a: str, prop_b: str, **kwargs):
        super().__init__(**kwargs)
        self.prop_a = prop_a
        self.prop_b = prop_b

    def run_logic(self, context: Context, dataset_kind: DatasetKind) -> CheckResult:
        # Get the dataset by its type (train/test)
        dataset: Dataset = context.get_data_by_kind(dataset_kind)
        # Get the model (optional, if needed for check logic)
        model = context.model
        # Get from the dataset the data
        data: pd.DataFrame = dataset.data

        # LOGIC HERE - possible to add validations on inputs and properties like prop_a and prop_b
        failing_rows = some_calc_fn(self.prop_a, self.prop_b)

        # Define result value: Adding any info that we might want to know later
        result = {
            'ratio': failing_rows.shape[0] / data.shape[0],
            'indices': failing_rows.index
        }

        # Define result display: list of either plotly-figure/dataframe/html
        display = [
            # Showing in the display only sample of 5 rows
            failing_rows[:5]
        ]

        return CheckResult(result, display=display)

    # OPTIONAL: add condition to check
    def add_condition_ratio_less_than(self, threshold: float = 0.01):
        # Define condition function: the function accepts as input the result value we defined in the run_logic
        def condition(result):
            ratio = result['ratio']
            category = ConditionCategory.PASS if ratio < threshold else ConditionCategory.FAIL
            message = f'Found X ratio of {ratio}'
            return ConditionResult(category, message)

        # Define the name of the condition
        name = f'Custom check ratio is less than {threshold}'
        # Now add it on the class instance
        return self.add_condition(name, condition)

Train Test Check#

Check type for cases when running on two datasets and optional model, for example drift checks.

import pandas as pd

from deepchecks.core import CheckResult, ConditionCategory, ConditionResult
from deepchecks.tabular import TrainTestCheck, Dataset, Context


class TrainTestCustomCheck(TrainTestCheck):
    """Description of the check. The name of the check will be the class name split by upper case letters."""

    # OPTIONAL: we can add different properties in the init
    def __init__(self, prop_a: str, prop_b: str, **kwargs):
        super().__init__(**kwargs)
        self.prop_a = prop_a
        self.prop_b = prop_b

    def run_logic(self, context: Context) -> CheckResult:
        # Get the 2 datasets
        train_dataset: Dataset = context.train
        test_dataset: Dataset = context.test
        # Get the model (optional, if needed for check logic)
        model = context.model
        # Get from the datasets the data
        train_df: pd.DataFrame = train_dataset.data
        test_df: pd.DataFrame = test_dataset.data

        # LOGIC HERE - possible to add validations on inputs and properties like prop_a and prop_b
        test_failing_rows = some_calc_fn(self.prop_a, self.prop_b)

        # Define result value: Adding any info that we might want to know later
        result = {
            'ratio': test_failing_rows.shape[0] / test_df.shape[0],
            'indices': test_failing_rows.index
        }

        # Define result display: list of either plotly-figure/dataframe/html
        display = [
            # Showing in the display only sample of 5 rows
            test_failing_rows[:5]
        ]

        return CheckResult(result, display=display)

    # OPTIONAL: add condition to check
    def add_condition_ratio_less_than(self, threshold: float = 0.01):
        # Define condition function: the function accepts as input the result value we defined in the run_logic
        def condition(result):
            ratio = result['ratio']
            category = ConditionCategory.PASS if ratio < threshold else ConditionCategory.FAIL
            message = f'Found X ratio of {ratio}'
            return ConditionResult(category, message)

        # Define the name of the condition
        name = f'Custom check ratio is less than {threshold}'
        # Now add it on the class instance
        return self.add_condition(name, condition)

Model Only Check#

Check type for cases when running only on a model, for example model parameters check.

from deepchecks.core import CheckResult, ConditionCategory, ConditionResult
from deepchecks.tabular import ModelOnlyCheck, Context


class ModelOnlyCustomCheck(ModelOnlyCheck):
    """Description of the check. The name of the check will be the class name split by upper case letters."""

    # OPTIONAL: we can add different properties in the init
    def __init__(self, prop_a: str, prop_b: str, **kwargs):
        super().__init__(**kwargs)
        self.prop_a = prop_a
        self.prop_b = prop_b

    def run_logic(self, context: Context) -> CheckResult:
        # Get the model
        model = context.model

        # LOGIC HERE - possible to add validations on inputs and properties like prop_a and prop_b
        some_score = some_calc_fn(model, self.prop_a, self.prop_b)

        # Define result value: Adding any info that we might want to know later
        result = some_score

        # Define result display: list of either plotly-figure/dataframe/html, or Nothing if we have no display
        display = None

        return CheckResult(result, display=display)

    # OPTIONAL: add condition to check
    def add_condition_score_more_than(self, threshold: float = 1):
        # Define condition function: the function accepts as input the result value we defined in the run_logic
        def condition(result):
            category = ConditionCategory.PASS if result > 1 else ConditionCategory.FAIL
            message = f'Found X score of {result}'
            return ConditionResult(category, message)

        # Define the name of the condition
        name = f'Custom check score is more than {threshold}'
        # Now add it on the class instance
        return self.add_condition(name, condition)