From 7744983fabc561fd8bfc51ab48882c1ad54b3dda Mon Sep 17 00:00:00 2001 From: George Weale Date: Wed, 2 Sep 2026 17:28:42 +0000 Subject: [PATCH] Add a CI check for the Python code samples on the docs pages Nothing currently checks the code in the documentation. python-lint.yaml and python-tests.yaml are both filtered to samples/python/**, and no samples directory exists, so on pull requests neither can run. Samples that do not parse, and samples importing names google-adk no longer provides, have shipped as a result. This adds a checker that reads the fenced Python blocks straight out of the markdown and asks two things of each: does it parse, and do its google.adk imports resolve against the installed package. No sample is executed. They build agents and call live endpoints, so running one in CI would be slow and billable; importing the library is enough to answer both questions. Samples are frequently excerpts rather than whole modules, so a plain parse failure is not automatically a defect. Method bodies, blocks that only make sense indented under something shown above them, pseudocode using ... as a placeholder, and signature listings with no body are all recognised and allowed through. The checker ships with tests, half of which plant a defect of each class it claims to catch. A checker that reports nothing because it is broken looks exactly like one that reports nothing because the samples are fine, and only the planted cases tell those apart. --- .github/workflows/code-samples.yaml | 63 +++++++ tools/code_samples/check.py | 250 ++++++++++++++++++++++++++++ tools/code_samples/check_test.py | 237 ++++++++++++++++++++++++++ tools/code_samples/fences.py | 159 ++++++++++++++++++ 4 files changed, 709 insertions(+) create mode 100644 .github/workflows/code-samples.yaml create mode 100644 tools/code_samples/check.py create mode 100644 tools/code_samples/check_test.py create mode 100644 tools/code_samples/fences.py diff --git a/.github/workflows/code-samples.yaml b/.github/workflows/code-samples.yaml new file mode 100644 index 0000000000..d3f1fe3791 --- /dev/null +++ b/.github/workflows/code-samples.yaml @@ -0,0 +1,63 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Code samples + +# Checks that the Python samples on the docs pages parse, and that the names +# they import from google.adk still exist. No sample is executed; the samples +# build agents and call live endpoints. + +on: + pull_request: + paths: + - 'docs/**/*.md' + - 'examples/python/**' + - 'tools/code_samples/**' + - '.github/workflows/code-samples.yaml' + push: + branches: + - main + paths: + - 'docs/**/*.md' + - 'examples/python/**' + - 'tools/code_samples/**' + workflow_dispatch: + +permissions: + contents: read + +jobs: + check: + runs-on: ubuntu-latest + steps: + - name: Check out the repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.12' + + - name: Install the ADK the samples are written against + run: | + python -m pip install --upgrade pip + # The extras matter: without them the optional integrations resolve + # as missing and every sample importing one looks broken. + pip install 'google-adk[all]' pytest + + - name: Test the checker + run: python -m pytest tools/code_samples/check_test.py -q + + - name: Check the code samples + run: python tools/code_samples/check.py diff --git a/tools/code_samples/check.py b/tools/code_samples/check.py new file mode 100644 index 0000000000..0f7e8ef4f8 --- /dev/null +++ b/tools/code_samples/check.py @@ -0,0 +1,250 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Check the Python code samples in the documentation. + +Two questions are asked of every sample: + + Does it parse? A sample a reader cannot paste into a file and run is a bug, + and several have shipped that way. + + Do the names it imports from google.adk still exist? This is what rots + quietly when the library renames or moves something, because the prose keeps + reading correctly while the code stops working. + +Nothing is imported or executed from the samples themselves. They construct +agents and call live endpoints, so running one in CI would be both slow and +billable. Only the library is imported, which is what any reader does. + +A sample lifted out of prose is often not a whole module, and that is fine. +Three shapes are recognised and excused: + + fragment a method body, or a block that only makes sense indented under + something the page shows above it + pseudocode "..." or "<...>" standing in for omitted code, or a notebook + magic such as !pip + signature a def or a class listed with no body, to describe an interface + +Run it over the whole tree, or pass paths to check only those files. +""" + +from __future__ import annotations + +import argparse +import ast +import builtins +import collections +import importlib +import pathlib +import re +import sys +import textwrap +import warnings + +import fences + +# Importing parts of google.adk emits experimental-feature and deprecation +# notices. They are about the library, not about the samples, and they bury the +# output that matters. +warnings.filterwarnings("ignore") + +DOCS = pathlib.Path("docs") +PY_LANGUAGES = {"python", "py"} +WATCHED_PACKAGE = "google.adk" + +ELISION = re.compile(r"^\s*\.\.\.\s*,?\s*$|<\.\.\.|\.\.\.\s*>|^\s*[!%][A-Za-z]") +ELISION_INLINE = re.compile(r"[(,]\s*\.\.\.\s*[,)]") +NEEDS_BODY = re.compile(r"expected an indented block") +BUILTINS = frozenset(dir(builtins)) + + +class Defect(collections.namedtuple("Defect", "path line kind detail")): + def __str__(self) -> str: + return f"{self.path}:{self.line}: {self.kind}: {self.detail}" + + +# -------------------------------------------------------------------------- +# Does it parse? +# -------------------------------------------------------------------------- + + +def _compiles(source: str) -> str | None: + try: + compile(source, "", "exec") + return None + except SyntaxError as exc: + return f"line {exc.lineno}: {exc.msg}" + except ValueError as exc: + return str(exc) + + +def _is_pseudocode(source: str) -> bool: + return any( + ELISION.search(line) or ELISION_INLINE.search(line) + for line in source.split("\n") + ) + + +def _body_omitted(source: str, error: str) -> bool: + """True when a block's body was written as comments, or left out.""" + if not NEEDS_BODY.search(error): + return False + match = re.search(r"line (\d+):", error) + if not match: + return False + lines = source.split("\n") + index = int(match.group(1)) - 1 + if index >= len(lines): + return True + rest = [l for l in lines[index:] if l.strip()] + # The reported line is often the header's own closing paren. + if rest and re.sub(r"\s+#.*$", "", rest[0]).rstrip().endswith(":"): + rest = rest[1:] + return not rest or all(l.lstrip().startswith("#") for l in rest) + + +def check_parses(source: str) -> str | None: + """Return a defect description, or None if the sample is acceptable.""" + error = _compiles(source) + if error is None: + return None + + flat = textwrap.dedent(source) + if flat != source and _compiles(flat) is None: + return None + + indented = textwrap.indent(flat, " ") + for wrapper in ("if True:\n", "class _C:\n", "async def _f():\n"): + if _compiles(wrapper + indented) is None: + return None + # An excerpt from inside an async loop may use await, break and continue. + deeper = textwrap.indent(flat, " ") + if _compiles(f"async def _f():\n while True:\n{deeper}") is None: + return None + + if _is_pseudocode(source) or _body_omitted(source, error): + return None + return error + + +# -------------------------------------------------------------------------- +# Do the imported names exist? +# -------------------------------------------------------------------------- + +_module_cache: dict[str, object | None] = {} + + +def _load(name: str) -> object | None: + if name not in _module_cache: + try: + _module_cache[name] = importlib.import_module(name) + except Exception: # noqa: BLE001 - any failure means "cannot resolve" + _module_cache[name] = None + return _module_cache[name] + + +def _has_attribute(module_name: str, module: object, attribute: str) -> bool: + try: + if hasattr(module, attribute): + return True + except ImportError: + # A lazy loader that raises has still declared the name; it only needs + # an optional dependency that is not installed here. + return True + except Exception: # noqa: BLE001 + return True + return _load(f"{module_name}.{attribute}") is not None + + +def check_imports(source: str) -> list[str]: + """Return descriptions of any google.adk import that cannot be resolved.""" + try: + tree = ast.parse(source) + except SyntaxError: + return [] # the parse check already reported on this sample + + problems: list[str] = [] + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom): + if node.level or not (node.module or "").startswith(WATCHED_PACKAGE): + continue + module = _load(node.module) + if module is None: + problems.append(f"no module named {node.module!r}") + continue + for alias in node.names: + if alias.name != "*" and not _has_attribute(node.module, module, alias.name): + problems.append(f"{node.module} has no {alias.name!r}") + elif isinstance(node, ast.Import): + for alias in node.names: + if alias.name.startswith(WATCHED_PACKAGE) and _load(alias.name) is None: + problems.append(f"no module named {alias.name!r}") + return problems + + +# -------------------------------------------------------------------------- + + +def check_file(path: pathlib.Path, root: pathlib.Path) -> list[Defect]: + defects: list[Defect] = [] + for fence in fences.parse(path): + if fence.language not in PY_LANGUAGES or fence.is_pointer: + continue + source = fences.resolve_includes(fence.text, root).strip("\n") + if not source: + continue + error = check_parses(source) + if error: + defects.append(Defect(path, fence.start_line, "does not parse", error)) + continue + for problem in check_imports(source): + defects.append(Defect(path, fence.start_line, "unresolved import", problem)) + return defects + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("paths", nargs="*", type=pathlib.Path, + help="markdown files to check; defaults to all of docs/") + parser.add_argument("--root", type=pathlib.Path, default=pathlib.Path("."), + help="repository root, used to resolve snippet includes") + args = parser.parse_args(argv) + + targets = args.paths or sorted((args.root / DOCS).rglob("*.md")) + targets = [p for p in targets if p.suffix == ".md" and p.is_file()] + + defects: list[Defect] = [] + for path in targets: + defects.extend(check_file(path, args.root)) + + print(f"checked {len(targets)} page(s)") + if not defects: + print("no defects") + return 0 + + by_kind = collections.Counter(d.kind for d in defects) + print(f"{len(defects)} defect(s): " + ", ".join(f"{v} {k}" for k, v in by_kind.items())) + print() + for defect in defects: + print(defect) + print() + print("Each sample above either cannot be parsed, or imports a name that") + print("google-adk no longer provides. Fix the sample, or if it is meant to") + print("be illustrative rather than runnable, mark the omitted part with") + print("'...' so it is recognised as pseudocode.") + return 1 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/tools/code_samples/check_test.py b/tools/code_samples/check_test.py new file mode 100644 index 0000000000..7649e92822 --- /dev/null +++ b/tools/code_samples/check_test.py @@ -0,0 +1,237 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the code sample checker. + +Two halves, and the second matters as much as the first. A checker that +reports nothing because it is broken looks exactly like a checker that reports +nothing because the samples are fine, so every defect class it claims to catch +is planted here and asserted on. + +Run with: python -m pytest tools/code_samples/check_test.py +""" + +from __future__ import annotations + +import pathlib +import sys +import textwrap + +sys.path.insert(0, str(pathlib.Path(__file__).parent)) + +import check # noqa: E402 +import fences # noqa: E402 + + +def page(tmp_path: pathlib.Path, markdown: str) -> pathlib.Path: + path = tmp_path / "page.md" + path.write_text(textwrap.dedent(markdown), encoding="utf-8") + return path + + +def defects(tmp_path: pathlib.Path, markdown: str) -> list[check.Defect]: + return check.check_file(page(tmp_path, markdown), tmp_path) + + +# ---------------------------------------------------------------- caught --- + + +def test_catches_unparseable_sample(tmp_path): + found = defects(tmp_path, """ + ```python + def broken( + ``` + """) + assert len(found) == 1 + assert found[0].kind == "does not parse" + + +def test_catches_unindented_class_body(tmp_path): + found = defects(tmp_path, """ + ```python + class Thing: + def method(self): + return 1 + ``` + """) + assert [d.kind for d in found] == ["does not parse"] + + +def test_catches_await_in_sync_function(tmp_path): + found = defects(tmp_path, """ + ```python + def tool(): + result = await something() + return result + ``` + """) + assert [d.kind for d in found] == ["does not parse"] + + +def test_catches_module_that_does_not_exist(tmp_path): + found = defects(tmp_path, """ + ```python + from google.adk.no_such_module import Thing + ``` + """) + assert [d.kind for d in found] == ["unresolved import"] + + +def test_catches_name_that_does_not_exist(tmp_path): + found = defects(tmp_path, """ + ```python + from google.adk.agents import NoSuchAgentClass + ``` + """) + assert [d.kind for d in found] == ["unresolved import"] + + +def test_reports_the_line_of_the_opening_fence(tmp_path): + found = defects(tmp_path, """ + Some prose. + + ```python + from google.adk.agents import NoSuchAgentClass + ``` + """) + assert found[0].line == 4 + + +def test_catches_a_sample_nested_in_a_tab(tmp_path): + found = defects(tmp_path, """ + === "Python" + + ```python + from google.adk.agents import NoSuchAgentClass + ``` + """) + assert [d.kind for d in found] == ["unresolved import"] + + +# ---------------------------------------------------------------- passed --- + + +def test_accepts_a_whole_module(tmp_path): + assert not defects(tmp_path, """ + ```python + from google.adk.agents import LlmAgent + + agent = LlmAgent(name="demo", model="gemini-flash-latest") + ``` + """) + + +def test_accepts_a_method_body_fragment(tmp_path): + assert not defects(tmp_path, """ + ```python + self.counter += 1 + print(self.counter) + ``` + """) + + +def test_accepts_pseudocode_with_elision(tmp_path): + assert not defects(tmp_path, """ + ```python + agent = LlmAgent( + name="demo", + ... + ) + ``` + """) + + +def test_accepts_a_signature_listing(tmp_path): + assert not defects(tmp_path, """ + ```python + async def before_model_callback( + self, *, callback_context, llm_request + ) -> None: + ``` + """) + + +def test_accepts_an_await_excerpt_from_a_loop(tmp_path): + assert not defects(tmp_path, """ + ```python + async for event in runner.run_async(...): + if event.is_final_response(): + break + await handle(event) + ``` + """) + + +def test_ignores_non_python_blocks(tmp_path): + assert not defects(tmp_path, """ + ```go + this is not python at all ((( + ``` + + ```bash + pip install google-adk + ``` + """) + + +def test_ignores_a_block_that_only_includes_a_snippet(tmp_path): + assert not defects(tmp_path, """ + ```python + --8<-- "examples/python/does/not/matter.py" + ``` + """) + + +# ------------------------------------------------------------ fence parse --- + + +def test_fence_survives_backticks_inside_a_string(tmp_path): + found = fences.parse(page(tmp_path, ''' + ```python + PROMPT = """ + ```tool_code + nested and indented + ``` + """ + ``` + ''')) + assert len(found) == 1 + assert "nested and indented" in found[0].text + + +def test_fence_skips_a_block_that_never_closes_cleanly(tmp_path): + """A fence closed at the wrong indentation swallows what follows. + + Opening at six columns and "closing" at eight means the block never + terminates, so the next opener gets absorbed into its body. Rather than + guess a boundary, the parser reports nothing for the whole ambiguous run. + Reporting nothing is right: any guess would produce defects against code + the author never wrote. + """ + path = tmp_path / "page.md" + path.write_text( + " ```python\n x = 1\n ```\n\n ```python\n y = 2\n ```\n", + encoding="utf-8", + ) + assert fences.parse(path) == [] + + +def test_dedent_measures_columns_not_characters(tmp_path): + path = tmp_path / "page.md" + # Fence at four columns; body line indented with a tab, which is four + # columns, then real code indentation on top. + path.write_text(" ```python\n\tif True:\n\t pass\n ```\n", encoding="utf-8") + body = fences.parse(path)[0].text + assert body.split("\n")[0] == "if True:" + assert body.split("\n")[1] == " pass" diff --git a/tools/code_samples/fences.py b/tools/code_samples/fences.py new file mode 100644 index 0000000000..d443f6dfae --- /dev/null +++ b/tools/code_samples/fences.py @@ -0,0 +1,159 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Pull fenced code blocks out of the documentation. + +Blocks are frequently nested inside tabbed sections, so a fence can be indented +by four or eight columns, and its body carries that indentation on every line. +Both are handled here so the rest of the tooling sees the code as an author +wrote it. +""" + +from __future__ import annotations + +import dataclasses +import pathlib +import re + +FENCE = re.compile(r"^(?P[ \t]*)(?P`{3,}|~{3,})(?P.*)$") +SNIPPET_INCLUDE = "--8<--" + + +def _columns(text: str, upto: int) -> int: + """Display width of the first `upto` characters, tabs expanding to four.""" + col = 0 + for ch in text[:upto]: + col = col + (4 - col % 4) if ch == "\t" else col + 1 + return col + + +@dataclasses.dataclass(frozen=True) +class Fence: + """One fenced code block.""" + + path: pathlib.Path + start_line: int # 1-based line of the opening fence + language: str + body: tuple[str, ...] # dedented to column zero + + @property + def text(self) -> str: + return "\n".join(self.body) + + @property + def is_pointer(self) -> bool: + """True when the block only pulls in an external snippet file.""" + lines = [line for line in self.body if line.strip()] + return bool(lines) and all(l.startswith(SNIPPET_INCLUDE) for l in lines) + + +def _dedent(body: list[str], indent: str) -> list[str]: + """Strip the fence's own indentation, never the code's.""" + width = _columns(indent, len(indent)) + if not width: + return body + out = [] + for line in body: + cut = 0 + while cut < len(line) and line[cut] in " \t" and _columns(line, cut + 1) <= width: + cut += 1 + out.append(line[cut:]) + return out + + +def parse(path: pathlib.Path) -> list[Fence]: + """Return every fenced block in one markdown file.""" + lines = path.read_text(encoding="utf-8").split("\n") + fences: list[Fence] = [] + i = 0 + while i < len(lines): + opening = FENCE.match(lines[i]) + if not opening or opening.group("ticks")[0] in opening.group("info"): + i += 1 + continue + indent, ticks = opening.group("indent"), opening.group("ticks") + + # A closing fence sits at the same indentation. A more deeply indented + # run of backticks is content, which is how a sample that quotes fenced + # markdown inside a string survives intact. + close = None + j = i + 1 + while j < len(lines): + line = lines[j] + if line.startswith(indent): + rest = line[len(indent):].rstrip() + if rest and set(rest) == {ticks[0]} and len(rest) >= len(ticks): + close = j + break + j += 1 + if close is None: + i += 1 + continue + + body = lines[i + 1 : close] + # Another fence opening at this indentation means the block was never + # closed cleanly and any boundary here is a guess, so skip it. + ambiguous = any( + (m := FENCE.match(l)) and m.group("indent") == indent and m.group("info").strip() + for l in body + ) + if not ambiguous: + info = opening.group("info").strip() + language = info.split(" ", 1)[0].split("{", 1)[0].strip().lower() + fences.append( + Fence( + path=path, + start_line=i + 1, + language=language, + body=tuple(_dedent(body, indent)), + ) + ) + i = close + 1 + return fences + + +def resolve_includes(text: str, root: pathlib.Path, depth: int = 0) -> str: + """Expand `--8<--` snippet includes so a block can be read as real code.""" + if depth > 5: + return text + section_marker = re.compile(r"--8<--\s+\[(start|end):([^\]]+)\]") + include = re.compile(r'^(?P\s*)--8<--\s+"(?P[^"]+)"\s*$') + + out: list[str] = [] + for line in text.split("\n"): + m = include.match(line) + if not m: + if not section_marker.search(line): + out.append(line) + continue + target, _, section = m.group("spec").partition(":") + ref = root / target + if not ref.is_file(): + out.append(line) + continue + inner = ref.read_text(encoding="utf-8").split("\n") + if section: + picked, inside = [], False + for l in inner: + sm = section_marker.search(l) + if sm and sm.group(2).strip() == section.strip(): + inside = sm.group(1) == "start" + continue + if inside: + picked.append(l) + inner = picked + expanded = resolve_includes("\n".join(inner), root, depth + 1).split("\n") + pad = m.group("indent") + out.extend(pad + l if l.strip() else l for l in expanded) + return "\n".join(out)