Compare ethdebug output to program schema

Author: clonker

.circleci/config.yml

@@ -951,6 +951,8 @@ jobs:
   chk_pylint:
     <<: *base_ubuntu2404_small
     steps:
+      - install_python3:
+          packages: pyyaml jsonschema pytest
       - checkout
       - run: pylint --version
       - run:
@@ -1534,6 +1536,19 @@ jobs:
             - reports/externalTests/
       - matrix_notify_failure_unless_pr
 
+  t_ethdebug_output_validity:
+    <<: *base_node_small
+    steps:
+      - checkout
+      - attach_workspace:
+          at: /tmp/workspace
+      - install_python3:
+            packages: pyyaml jsonschema pytest
+      - run:
+          name: Ethdebug validity tests
+          command: |
+            pytest test/ethdebugSchemaTests --solc-binary-path=/tmp/workspace/solc/solc-static-linux -v
+
   c_ext_benchmarks:
     <<: *base_node_small
     steps:
@@ -1927,6 +1942,9 @@ workflows:
       #- t_ext: *job_native_compile_ext_chainlink
       #- t_ext: *job_native_compile_ext_bleeps
 
+      - t_ethdebug_output_validity:
+          <<: *requires_b_ubu_static
+
       - c_ext_benchmarks:
           <<: *requires_nothing
           requires:

test/ethdebugSchemaTests/conftest.py

@@ -0,0 +1,48 @@
+import shutil
+import subprocess
+from pathlib import Path
+
+import pytest
+import referencing
+import yaml
+from referencing.jsonschema import DRAFT202012
+
+
+def pytest_addoption(parser):
+    parser.addoption("--solc_binary_path", type=Path, required=True, help="Path to the solidity compiler binary.")
+
+
+@pytest.fixture
+def solc_path(request):
+    solc_path = request.config.getoption("--solc_binary_path")
+    assert solc_path.is_file()
+    assert solc_path.exists()
+    return solc_path
+
+
+@pytest.fixture(scope="module")
+def ethdebug_clone_dir(tmpdir_factory):
+    temporary_dir = tmpdir_factory.mktemp("data")
+    yield Path(temporary_dir)
+    shutil.rmtree(str(temporary_dir))
+
+@pytest.fixture(scope="module")
+def ethdebug_schema_repository(ethdebug_clone_dir):
+    process = subprocess.run(
+        ["git", "clone", "https://github.com/ethdebug/format.git", ethdebug_clone_dir],
+        encoding='utf8',
+        capture_output=True,
+        check=True
+    )
+    assert process.returncode == 0
+
+    registry = referencing.Registry()
+    for path in (ethdebug_clone_dir / "schemas").rglob("*.yaml"):
+        with open(path, "r") as f:
+            schema = yaml.safe_load(f)
+            if "$id" in schema:
+                resource = referencing.Resource.from_contents(schema, DRAFT202012)
+                registry = resource @ registry
+            else:
+                raise ValueError(f"Schema did not define an $id: {path}")
+    return registry

test/ethdebugSchemaTests/input_file.json

@@ -0,0 +1,27 @@
+{
+  "language": "Solidity",
+  "sources": {
+    "a.sol": {
+      "content": "//SPDX-License-Identifier: GPL-3.0\npragma solidity >=0.0;\ncontract A1 { function a(uint x) public pure { assert(x > 0); } } contract A2 { function a(uint x) public pure { assert(x > 0); } }"
+    },
+    "b.sol": {
+      "content": "//SPDX-License-Identifier: GPL-3.0\npragma solidity >=0.0;\ncontract A1 { function b(uint x) public pure { assert(x > 0); } } contract B2 { function b(uint x) public pure { assert(x > 0); } }"
+    }
+  },
+  "settings": {
+    "viaIR": true,
+    "debug": {
+      "debugInfo": [
+        "ethdebug"
+      ]
+    },
+    "outputSelection": {
+      "*": {
+        "*": [
+          "evm.bytecode.ethdebug",
+          "evm.deployedBytecode.ethdebug"
+        ]
+      }
+    }
+  }
+}

test/ethdebugSchemaTests/input_file_eof.json

@@ -0,0 +1,29 @@
+{
+  "language": "Solidity",
+  "sources": {
+    "a.sol": {
+      "content": "//SPDX-License-Identifier: GPL-3.0\npragma solidity >=0.0;\ncontract A1 { function a(uint x) public pure { assert(x > 0); } } contract A2 { function a(uint x) public pure { assert(x > 0); } }"
+    },
+    "b.sol": {
+      "content": "//SPDX-License-Identifier: GPL-3.0\npragma solidity >=0.0;\ncontract A1 { function b(uint x) public pure { assert(x > 0); } } contract B2 { function b(uint x) public pure { assert(x > 0); } }"
+    }
+  },
+  "settings": {
+    "eofVersion": 1,
+    "evmVersion": "osaka",
+    "viaIR": true,
+    "debug": {
+      "debugInfo": [
+        "ethdebug"
+      ]
+    },
+    "outputSelection": {
+      "*": {
+        "*": [
+          "evm.bytecode.ethdebug",
+          "evm.deployedBytecode.ethdebug"
+        ]
+      }
+    }
+  }
+}

test/ethdebugSchemaTests/test_ethdebug_schema_conformity.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+
+import json
+import subprocess
+from pathlib import Path
+
+import jsonschema
+import pytest
+
+def get_nested_value(dictionary, *keys):
+    for key in keys:
+        dictionary = dictionary[key]
+    return dictionary
+
+@pytest.fixture(params=["input_file.json", "input_file_eof.json"])
+def solc_output(request, solc_path):
+    testfile_dir = Path(__file__).parent
+    with open(testfile_dir / request.param, encoding='utf8') as f:
+        source = json.load(f)
+
+    process = subprocess.run(
+        [solc_path, "--standard-json"],
+        input=json.dumps(source),
+        encoding='utf8',
+        capture_output=True,
+        check=True,
+    )
+    assert process.returncode == 0
+    return json.loads(process.stdout)
+
+
+@pytest.mark.parametrize("output_selection", ["evm.bytecode.ethdebug", "evm.deployedBytecode.ethdebug"], ids=str)
+def test_ethdebug_program_schema_conformity(
+    output_selection,
+    ethdebug_schema_repository,
+    solc_output
+):
+    validator = jsonschema.Draft202012Validator(
+        schema={"$ref": "schema:ethdebug/format/program"},
+        registry=ethdebug_schema_repository
+    )
+    assert "contracts" in solc_output
+    for contract in solc_output["contracts"].keys():
+        contract_output = solc_output["contracts"][contract]
+        assert len(contract_output) > 0
+        for source in contract_output.keys():
+            source_output = contract_output[source]
+            ethdebug_data = get_nested_value(source_output, *(output_selection.split(".")))
+            validator.validate(ethdebug_data)