Skip to content

Spec Version Checker Plugin

A pytest plugin that verifies the tested version of an EIP specification against the latest version from the ethereum/EIPs Github repository.

A pytest plugin that checks that the spec version specified in test/filler modules matches that of https://github.com/ethereum/EIPs.

pytest_configure(config)

Register the plugin's custom markers and process command-line options.

Custom marker registration: https://docs.pytest.org/en/7.1.x/how-to/writing_plugins.html#registering-custom-markers

Source code in /home/dtopz/code/github/danceratopz/execution-spec-tests/src/pytest_plugins/spec_version_checker/spec_version_checker.py 
16
17
18
19
20
21
22
23
24
25
26
27
@pytest.hookimpl(tryfirst=True)
def pytest_configure(config):
    """
    Register the plugin's custom markers and process command-line options.

    Custom marker registration:
    https://docs.pytest.org/en/7.1.x/how-to/writing_plugins.html#registering-custom-markers
    """
    config.addinivalue_line(
        "markers",
        "eip_version_check: a test that tests the reference spec defined in an EIP test module.",
    )

ReferenceSpec

Reference Specification Description Abstract Class.

Source code in /home/dtopz/code/github/danceratopz/execution-spec-tests/src/ethereum_test_tools/reference_spec/reference_spec.py 
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
class ReferenceSpec:
    """
    Reference Specification Description Abstract Class.
    """

    @abstractmethod
    def name(self) -> str:
        """
        Returns the name of the spec.
        """
        pass

    @abstractmethod
    def has_known_version(self) -> bool:
        """
        Returns true if the reference spec object is hard-coded with a latest
        known version.
        """
        pass

    @abstractmethod
    def known_version(self) -> str:
        """
        Returns the latest known version in the reference.
        """
        pass

    @abstractmethod
    def api_url(self) -> str:
        """
        Returns the URL required to poll the version from an API, if needed.
        """
        pass

    @abstractmethod
    def latest_version(self) -> str:
        """
        Returns a digest that points to the latest version of the spec.
        """
        pass

    @abstractmethod
    def is_outdated(self) -> bool:
        """
        Checks whether the reference specification has been updated since the
        test was last updated.
        """
        pass

    @abstractmethod
    def write_info(self, info: Dict[str, str]):
        """
        Writes info about the reference specification used into the output
        fixture.
        """
        pass

    @staticmethod
    @abstractmethod
    def parseable_from_module(module_dict: Dict[str, Any]) -> bool:
        """
        Checks whether the module's dict contains required reference spec
        information.
        """
        pass

    @staticmethod
    @abstractmethod
    def parse_from_module(module_dict: Dict[str, Any]) -> "ReferenceSpec":
        """
        Parses the module's dict into a reference spec.
        """
        pass

name() abstractmethod

Returns the name of the spec.

Source code in /home/dtopz/code/github/danceratopz/execution-spec-tests/src/ethereum_test_tools/reference_spec/reference_spec.py 
35
36
37
38
39
40
@abstractmethod
def name(self) -> str:
    """
    Returns the name of the spec.
    """
    pass

has_known_version() abstractmethod

Returns true if the reference spec object is hard-coded with a latest known version.

Source code in /home/dtopz/code/github/danceratopz/execution-spec-tests/src/ethereum_test_tools/reference_spec/reference_spec.py 
42
43
44
45
46
47
48
@abstractmethod
def has_known_version(self) -> bool:
    """
    Returns true if the reference spec object is hard-coded with a latest
    known version.
    """
    pass

known_version() abstractmethod

Returns the latest known version in the reference.

Source code in /home/dtopz/code/github/danceratopz/execution-spec-tests/src/ethereum_test_tools/reference_spec/reference_spec.py 
50
51
52
53
54
55
@abstractmethod
def known_version(self) -> str:
    """
    Returns the latest known version in the reference.
    """
    pass

api_url() abstractmethod

Returns the URL required to poll the version from an API, if needed.

Source code in /home/dtopz/code/github/danceratopz/execution-spec-tests/src/ethereum_test_tools/reference_spec/reference_spec.py 
57
58
59
60
61
62
@abstractmethod
def api_url(self) -> str:
    """
    Returns the URL required to poll the version from an API, if needed.
    """
    pass

latest_version() abstractmethod

Returns a digest that points to the latest version of the spec.

Source code in /home/dtopz/code/github/danceratopz/execution-spec-tests/src/ethereum_test_tools/reference_spec/reference_spec.py 
64
65
66
67
68
69
@abstractmethod
def latest_version(self) -> str:
    """
    Returns a digest that points to the latest version of the spec.
    """
    pass

is_outdated() abstractmethod

Checks whether the reference specification has been updated since the test was last updated.

Source code in /home/dtopz/code/github/danceratopz/execution-spec-tests/src/ethereum_test_tools/reference_spec/reference_spec.py 
71
72
73
74
75
76
77
@abstractmethod
def is_outdated(self) -> bool:
    """
    Checks whether the reference specification has been updated since the
    test was last updated.
    """
    pass

write_info(info) abstractmethod

Writes info about the reference specification used into the output fixture.

Source code in /home/dtopz/code/github/danceratopz/execution-spec-tests/src/ethereum_test_tools/reference_spec/reference_spec.py 
79
80
81
82
83
84
85
@abstractmethod
def write_info(self, info: Dict[str, str]):
    """
    Writes info about the reference specification used into the output
    fixture.
    """
    pass

parseable_from_module(module_dict) staticmethod abstractmethod

Checks whether the module's dict contains required reference spec information.

Source code in /home/dtopz/code/github/danceratopz/execution-spec-tests/src/ethereum_test_tools/reference_spec/reference_spec.py 
87
88
89
90
91
92
93
94
@staticmethod
@abstractmethod
def parseable_from_module(module_dict: Dict[str, Any]) -> bool:
    """
    Checks whether the module's dict contains required reference spec
    information.
    """
    pass

parse_from_module(module_dict) staticmethod abstractmethod

Parses the module's dict into a reference spec.

Source code in /home/dtopz/code/github/danceratopz/execution-spec-tests/src/ethereum_test_tools/reference_spec/reference_spec.py 
 96
 97
 98
 99
100
101
102
@staticmethod
@abstractmethod
def parse_from_module(module_dict: Dict[str, Any]) -> "ReferenceSpec":
    """
    Parses the module's dict into a reference spec.
    """
    pass

get_ref_spec_from_module(module)

Return the reference spec object defined in a module.

Raises:

Type Description
Exception

If the module path contains "eip" and the module does not define a reference spec.

Returns:

Name Type Description
spec_obj None | ReferenceSpec

Return None if the module path does not contain "eip", i.e., the module is not required to define a reference spec, otherwise, return the ReferenceSpec object as defined by the module.
Source code in /home/dtopz/code/github/danceratopz/execution-spec-tests/src/pytest_plugins/spec_version_checker/spec_version_checker.py 
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
def get_ref_spec_from_module(module: ModuleType) -> None | ReferenceSpec:
    """
    Return the reference spec object defined in a module.

    Raises:
        Exception: If the module path contains "eip" and the module
            does not define a reference spec.

    Returns:
        spec_obj: Return None if the module path does not contain "eip",
            i.e., the module is not required to define a reference spec,
            otherwise, return the ReferenceSpec object as defined by the
            module.
    """
    if not is_test_for_an_eip(str(module.__file__)):
        return None
    module_dict = module.__dict__
    parseable_ref_specs = [
        ref_spec_type
        for ref_spec_type in ReferenceSpecTypes
        if ref_spec_type.parseable_from_module(module_dict)
    ]
    if len(parseable_ref_specs) > 0:
        module_dict = module.__dict__
        try:
            spec_obj = parseable_ref_specs[0].parse_from_module(module_dict)
        except Exception as e:
            raise Exception(f"Error in spec_version_checker: {e} (this test is generated).")
    else:
        raise Exception("Test doesn't define REFERENCE_SPEC_GIT_PATH and REFERENCE_SPEC_VERSION")
    return spec_obj

reference_spec(request)

Pytest fixture that returns the reference spec defined in a module.

See get_ref_spec_from_module.

Source code in /home/dtopz/code/github/danceratopz/execution-spec-tests/src/pytest_plugins/spec_version_checker/spec_version_checker.py 
63
64
65
66
67
68
69
70
@pytest.fixture(autouse=True, scope="module")
def reference_spec(request) -> None | ReferenceSpec:
    """
    Pytest fixture that returns the reference spec defined in a module.

    See `get_ref_spec_from_module`.
    """
    return get_ref_spec_from_module(request.module)

is_test_for_an_eip(input_string)

Return True if input_string contains an EIP number, i.e., eipNNNN.

Source code in /home/dtopz/code/github/danceratopz/execution-spec-tests/src/pytest_plugins/spec_version_checker/spec_version_checker.py 
73
74
75
76
77
78
79
80
def is_test_for_an_eip(input_string: str) -> bool:
    """
    Return True if `input_string` contains an EIP number, i.e., eipNNNN.
    """
    pattern = re.compile(r".*eip\d{1,4}", re.IGNORECASE)
    if pattern.match(input_string):
        return True
    return False

test_eip_spec_version(module)

Test that the ReferenceSpec object as defined in the test module is not outdated when compared to the remote hash from ethereum/EIPs.

Source code in /home/dtopz/code/github/danceratopz/execution-spec-tests/src/pytest_plugins/spec_version_checker/spec_version_checker.py 
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
def test_eip_spec_version(module: ModuleType):
    """
    Test that the ReferenceSpec object as defined in the test module
    is not outdated when compared to the remote hash from
    ethereum/EIPs.
    """
    ref_spec = get_ref_spec_from_module(module)
    assert ref_spec, "No reference spec object defined"

    message = (
        "The version of the spec referenced in "
        f"{module} does not match that from ethereum/EIPs, "
        f"tests might be outdated: Spec: {ref_spec.name()}. "
        f"Referenced version: {ref_spec.known_version()}. "
        f"Latest version: {ref_spec.latest_version()}. The "
        f"version was retrieved from {ref_spec.api_url()}."
    )
    try:
        is_up_to_date = not ref_spec.is_outdated()
    except Exception as e:
        raise Exception(
            f"Error in spec_version_checker: {e} (this test is generated). "
            f"Reference spec URL: {ref_spec.api_url()}."
        )

    assert is_up_to_date, message

EIPSpecTestItem

Bases: Item

Custom pytest test item to test EIP spec versions.

Source code in /home/dtopz/code/github/danceratopz/execution-spec-tests/src/pytest_plugins/spec_version_checker/spec_version_checker.py 
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
class EIPSpecTestItem(Item):
    """
    Custom pytest test item to test EIP spec versions.
    """

    def __init__(self, name, parent, module):
        super().__init__(name, parent)
        self.module = module

    @classmethod
    def from_parent(cls, parent, module):
        """
        Public constructor to define new tests.
        https://docs.pytest.org/en/latest/reference/reference.html#pytest.nodes.Node.from_parent
        """
        return super().from_parent(parent=parent, name="test_eip_spec_version", module=module)

    def runtest(self):
        """
        Define the test to execute for this item.
        """
        test_eip_spec_version(self.module)

    def reportinfo(self):
        """
        Get location information for this test item to use test reports.
        """
        return "spec_version_checker", 0, f"{self.name}"

from_parent(parent, module) classmethod

Public constructor to define new tests. https://docs.pytest.org/en/latest/reference/reference.html#pytest.nodes.Node.from_parent

Source code in /home/dtopz/code/github/danceratopz/execution-spec-tests/src/pytest_plugins/spec_version_checker/spec_version_checker.py 
120
121
122
123
124
125
126
@classmethod
def from_parent(cls, parent, module):
    """
    Public constructor to define new tests.
    https://docs.pytest.org/en/latest/reference/reference.html#pytest.nodes.Node.from_parent
    """
    return super().from_parent(parent=parent, name="test_eip_spec_version", module=module)

runtest()

Define the test to execute for this item.

Source code in /home/dtopz/code/github/danceratopz/execution-spec-tests/src/pytest_plugins/spec_version_checker/spec_version_checker.py 
128
129
130
131
132
def runtest(self):
    """
    Define the test to execute for this item.
    """
    test_eip_spec_version(self.module)

reportinfo()

Get location information for this test item to use test reports.

Source code in /home/dtopz/code/github/danceratopz/execution-spec-tests/src/pytest_plugins/spec_version_checker/spec_version_checker.py 
134
135
136
137
138
def reportinfo(self):
    """
    Get location information for this test item to use test reports.
    """
    return "spec_version_checker", 0, f"{self.name}"

pytest_collection_modifyitems(session, config, items)

Insert a new test EIPSpecTestItem for every test modules that contains 'eip' in its path.

Source code in /home/dtopz/code/github/danceratopz/execution-spec-tests/src/pytest_plugins/spec_version_checker/spec_version_checker.py 
141
142
143
144
145
146
147
148
149
150
151
152
153
154
def pytest_collection_modifyitems(session, config, items):
    """
    Insert a new test EIPSpecTestItem for every test modules that
    contains 'eip' in its path.
    """
    modules = set(item.parent for item in items if isinstance(item.parent, Module))
    new_test_eip_spec_version_items = [
        EIPSpecTestItem.from_parent(module, module.obj)
        for module in modules
        if is_test_for_an_eip(str(module.path))
    ]
    for item in new_test_eip_spec_version_items:
        item.add_marker("eip_version_check", append=True)
    items.extend(new_test_eip_spec_version_items)