Skip to content

Main

Warning

This section is a work in progress

check_and_export(schema, data, file, format, hard_check=True, custom_checks=None)

function to create validation object, run validation and export log.

Parameters

schema : dict The schema to validate against. data : pd.DataFrame | pl.DataFrame The data to validate. file : str The file path to export the validation log. format : str The format to use when exporting the log. hard_check : bool, optional Whether to perform strict validation checks, by default True custom_checks : list, optional A list of custom validation checks to apply, by default None

Returns

DataValidator Returns data validator object after validation and export.

Source code in datachecker/main.py 
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
def check_and_export(schema, data, file, format, hard_check=True, custom_checks=None) -> Validator:
    """
    function to create validation object, run validation and export log.

    Parameters
    ----------
    schema : dict
        The schema to validate against.
    data : pd.DataFrame | pl.DataFrame
        The data to validate.
    file : str
        The file path to export the validation log.
    format : str
        The format to use when exporting the log.
    hard_check : bool, optional
        Whether to perform strict validation checks, by default True
    custom_checks : list, optional
        A list of custom validation checks to apply, by default None

    Returns
    -------
    DataValidator
        Returns data validator object after validation and export.
    """
    mod, name = _type_id(data)

    if (mod, name) == ("polars.dataframe.frame", "DataFrame"):
        validator = PolarsValidator(
            schema=schema,
            data=data,
            file=file,
            format=format,
            hard_check=hard_check,
            custom_checks=custom_checks,
        )

    if (mod, name) == ("pandas.core.frame", "DataFrame"):
        validator = DataValidator(
            schema=schema,
            data=data,
            file=file,
            format=format,
            hard_check=hard_check,
            custom_checks=custom_checks,
        )

    if (mod, name) == ("pyspark.sql.dataframe", "DataFrame") or (mod, name) == (
        "pyspark.sql.classic.dataframe",
        "DataFrame",
    ):
        validator = PySparkValidator(
            schema=schema,
            data=data,
            file=file,
            format=format,
            hard_check=hard_check,
            custom_checks=custom_checks,
        )

    validator.validate()
    validator.export()
    return validator