test: add webtest marker to tests that use the internet (#2295)

aucampia · web-flow · commit cfe6e378e6b0 · 2023-03-21T22:31:52.000+01:00
This is being done so that it is easier for downstream packagers to run the test suite without requiring internet access. To run only tests that does not use the internet, run `pytest -m "not webtest"`. The validation workflow validates that test run without internet access by running the tests inside `firejail --net=none`. - Closes <#2293>.
diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml
@@ -52,6 +52,10 @@ jobs:
             os: ubuntu-latest
             TOX_EXTRA_COMMAND: "flake8 --exit-zero rdflib"
             TOXENV_SUFFIX: "-docs"
+            PREPARATION: "sudo apt-get install -y firejail"
+            extensive-tests: true
+            TOX_TEST_HARNESS: "firejail --net=none --"
+            TOX_PYTEST_EXTRA_ARGS: "-m 'not webtest'"
           - python-version: "3.11"
             os: ubuntu-latest
             TOXENV_SUFFIX: "-docs"
@@ -82,11 +86,15 @@ jobs:
         uses: arduino/setup-task@v1
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Run preparation
+        if: ${{ matrix.PREPARATION }}
+        shell: bash
+        run: |
+          ${{ matrix.PREPARATION }}
       - name: Run validation
         shell: bash
         run: |
           task \
-            TOX_EXTRA_COMMAND="${{ matrix.TOX_EXTRA_COMMAND }}" \
             OS=${{ matrix.os }} \
             MATRIX_SUFFIX=${{ matrix.suffix }} \
             EXTENSIVE=${{ matrix.extensive-tests || 'false' }} \
@@ -96,6 +104,9 @@ jobs:
             gha:validate
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          TOX_PYTEST_EXTRA_ARGS: ${{ matrix.TOX_PYTEST_EXTRA_ARGS }}
+          TOX_TEST_HARNESS: ${{ matrix.TOX_TEST_HARNESS }}
+          TOX_EXTRA_COMMAND: ${{ matrix.TOX_EXTRA_COMMAND }}
       - uses: actions/upload-artifact@v3
         if: ${{ (success() || failure()) }}
         with:
diff --git a/Taskfile.yml b/Taskfile.yml
@@ -98,7 +98,6 @@ tasks:
       - echo "TOXENV=${TOXENV}"
       - |
         {{if .TOX_PYTEST_ARGS}}TOX_PYTEST_ARGS={{shellQuote .TOX_PYTEST_ARGS}}{{end}} \
-        {{if .TOX_EXTRA_COMMAND}}TOX_EXTRA_COMMAND={{shellQuote .TOX_EXTRA_COMMAND}}{{end}} \
         {{if .TOX_JUNIT_XML_PREFIX}}TOX_JUNIT_XML_PREFIX={{shellQuote .TOX_JUNIT_XML_PREFIX}}{{end}} \
         {{if .COVERAGE_FILE}}COVERAGE_FILE={{shellQuote .COVERAGE_FILE}}{{end}} \
         {{.TEST_HARNESS}} \
@@ -359,6 +358,12 @@ tasks:
         poetry run mypy --show-error-context --show-error-codes -p rdflib
         poetry run sphinx-build -T -W -b html -d docs/_build/doctree docs docs/_build/html
         poetry run pytest
+
+  test:no_internet:
+    desc: Run tests without internet access
+    cmds:
+      - |
+        {{.TEST_HARNESS}}{{.RUN_PREFIX}} firejail --net=none -- pytest -m "not webtest" {{.CLI_ARGS}}
   _rimraf:
     # This task is a utility task for recursively removing directories, it is
     # similar to rm -rf but not identical and it should work wherever there is
diff --git a/pyproject.toml b/pyproject.toml
@@ -156,6 +156,7 @@ addopts = [
    "--ignore=rdflib/extras/external_graph_libs.py",
    "--ignore-glob=docs/*.py",
    "--doctest-glob=docs/*.rst",
+   "--strict-markers",
 ]
 doctest_optionflags = "ALLOW_UNICODE"
 filterwarnings = [
@@ -164,6 +165,9 @@ filterwarnings = [
     # The below warning is a consequence of how pytest detects fixtures and how DefinedNamespace behaves when an undefined attribute is being accessed.
     "ignore:Code. _pytestfixturefunction is not defined in namespace .*:UserWarning",
 ]
+markers = [
+    "webtest: mark a test as using the internet",
+]
 # log_cli = true
 # log_cli_level = "DEBUG"
 log_format = "%(asctime)s.%(msecs)03d %(levelname)-8s %(name)-12s %(filename)s:%(lineno)s:%(funcName)s %(message)s"
diff --git a/test/conftest.py b/test/conftest.py
@@ -5,10 +5,19 @@
 
 pytest.register_assert_rewrite("test.utils")
 
+from pathlib import Path  # noqa: E402
 from test.utils.audit import AuditHookDispatcher  # noqa: E402
 from test.utils.http import ctx_http_server  # noqa: E402
 from test.utils.httpfileserver import HTTPFileServer  # noqa: E402
-from typing import Generator, Optional  # noqa: E402
+from typing import (  # noqa: E402
+    Collection,
+    Dict,
+    Generator,
+    Iterable,
+    Optional,
+    Tuple,
+    Union,
+)
 
 from rdflib import Graph
 
@@ -67,3 +76,31 @@ def audit_hook_dispatcher() -> Generator[Optional[AuditHookDispatcher], None, No
 def exit_stack() -> Generator[ExitStack, None, None]:
     with ExitStack() as stack:
         yield stack
+
+
+EXTRA_MARKERS: Dict[
+    Tuple[Optional[str], str], Collection[Union[pytest.MarkDecorator, str]]
+] = {
+    ("rdflib/__init__.py", "rdflib"): [pytest.mark.webtest],
+    ("rdflib/term.py", "rdflib.term.Literal.normalize"): [pytest.mark.webtest],
+    ("rdflib/extras/infixowl.py", "rdflib.extras.infixowl"): [pytest.mark.webtest],
+}
+
+
+PROJECT_ROOT = Path(__file__).parent.parent
+
+
+@pytest.hookimpl(tryfirst=True)
+def pytest_collection_modifyitems(items: Iterable[pytest.Item]):
+    for item in items:
+        parent_name = (
+            str(Path(item.parent.module.__file__).relative_to(PROJECT_ROOT))
+            if item.parent is not None
+            and isinstance(item.parent, pytest.Module)
+            and item.parent.module is not None
+            else None
+        )
+        if (parent_name, item.name) in EXTRA_MARKERS:
+            extra_markers = EXTRA_MARKERS[(parent_name, item.name)]
+            for extra_marker in extra_markers:
+                item.add_marker(extra_marker)
diff --git a/test/jsonld/test_onedotone.py b/test/jsonld/test_onedotone.py
@@ -231,6 +231,10 @@ def global_state():
     chdir(old_cwd)
 
 
+@pytest.mark.webtest
+# TODO: apply webtest marker to individual tests
+# Marking this whole function as webtest is too broad, as many tests don't
+# require the web, but making it narrower requires more refactoring.
 @pytest.mark.parametrize(
     "rdf_test_uri, func, suite_base, cat, num, inputpath, expectedpath, context, options",
     get_test_suite_cases(),
diff --git a/test/test_examples.py b/test/test_examples.py
@@ -19,6 +19,7 @@ def generate_example_cases() -> Iterable[ParameterSet]:
         yield pytest.param(example_file, id=f"{example_file.relative_to(EXAMPLES_DIR)}")
 
 
+@pytest.mark.webtest
 @pytest.mark.parametrize(["example_file"], generate_example_cases())
 def test_example(example_file: Path) -> None:
     """
diff --git a/test/test_extras/test_infixowl/test_basic.py b/test/test_extras/test_infixowl/test_basic.py
@@ -1,5 +1,7 @@
 from test.data import context0
 
+import pytest
+
 from rdflib import OWL, Graph, Literal, Namespace
 from rdflib.extras.infixowl import (
     Class,
@@ -79,6 +81,7 @@ def test_infixowl_serialization():
     )
 
 
+@pytest.mark.webtest
 def test_infix_owl_example1():
     g = Graph(identifier=context0)
     g.bind("ex", EXNS)
diff --git a/test/test_extras/test_infixowl/test_context.py b/test/test_extras/test_infixowl/test_context.py
@@ -28,6 +28,7 @@ def graph():
     del g
 
 
+@pytest.mark.webtest
 def test_context(graph):
     # Now we have an empty graph, we can construct OWL classes in it
     # using the Python classes defined in this module
diff --git a/test/test_sparql/test_service.py b/test/test_sparql/test_service.py
@@ -25,6 +25,7 @@
 from rdflib.term import BNode, Identifier
 
 
+@pytest.mark.webtest
 def test_service():
     g = Graph()
     q = """select ?sameAs ?dbpComment
@@ -47,6 +48,7 @@ def test_service():
         assert len(r) == 2
 
 
+@pytest.mark.webtest
 def test_service_with_bind():
     g = Graph()
     q = """select ?sameAs ?dbpComment ?subject
@@ -69,6 +71,7 @@ def test_service_with_bind():
         assert len(r) == 3
 
 
+@pytest.mark.webtest
 def test_service_with_bound_solutions():
     g = Graph()
     g.update(
@@ -104,6 +107,7 @@ def test_service_with_bound_solutions():
         assert len(r) == 3
 
 
+@pytest.mark.webtest
 def test_service_with_values():
     g = Graph()
     q = """select ?sameAs ?dbpComment ?subject
@@ -126,6 +130,7 @@ def test_service_with_values():
         assert len(r) == 3
 
 
+@pytest.mark.webtest
 def test_service_with_implicit_select():
     g = Graph()
     q = """select ?s ?p ?o
@@ -142,6 +147,7 @@ def test_service_with_implicit_select():
         assert len(r) == 3
 
 
+@pytest.mark.webtest
 def test_service_with_implicit_select_and_prefix():
     g = Graph()
     q = """prefix ex:<https://example.org/>
@@ -159,6 +165,7 @@ def test_service_with_implicit_select_and_prefix():
         assert len(r) == 3
 
 
+@pytest.mark.webtest
 def test_service_with_implicit_select_and_base():
     g = Graph()
     q = """base <https://example.org/>
@@ -176,6 +183,7 @@ def test_service_with_implicit_select_and_base():
         assert len(r) == 3
 
 
+@pytest.mark.webtest
 def test_service_with_implicit_select_and_allcaps():
     g = Graph()
     q = """SELECT ?s
@@ -199,6 +207,7 @@ def freeze_bindings(
     return frozenset(result)
 
 
+@pytest.mark.webtest
 def test_simple_not_null():
     """Test service returns simple literals not as NULL.
 
@@ -216,6 +225,7 @@ def test_simple_not_null():
     assert results.bindings[0].get(Variable("o")) == Literal("c")
 
 
+@pytest.mark.webtest
 def test_service_node_types():
     """Test if SERVICE properly returns different types of nodes:
     - URI;
diff --git a/tox.ini b/tox.ini
@@ -24,7 +24,7 @@ commands_pre =
 commands =
     {env:TOX_EXTRA_COMMAND:}
     {env:TOX_MYPY_COMMAND:poetry run python -m mypy --show-error-context --show-error-codes --junit-xml=test_reports/{env:TOX_JUNIT_XML_PREFIX:}mypy-junit.xml}
-    {posargs:poetry run pytest -ra --tb=native {env:TOX_PYTEST_ARGS:--junit-xml=test_reports/{env:TOX_JUNIT_XML_PREFIX:}pytest-junit.xml --cov --cov-report=}}
+    {posargs:poetry run {env:TOX_TEST_HARNESS:} pytest -ra --tb=native {env:TOX_PYTEST_ARGS:--junit-xml=test_reports/{env:TOX_JUNIT_XML_PREFIX:}pytest-junit.xml --cov --cov-report=} {env:TOX_PYTEST_EXTRA_ARGS:}}
     docs: poetry run sphinx-build -T -W -b html -d {envdir}/doctree docs docs/_build/html
 
 [testenv:covreport]
