diff --git a/.github/workflows/code-samples.yaml b/.github/workflows/code-samples.yaml new file mode 100644 index 0000000000..d3f1fe3791 --- /dev/null +++ b/.github/workflows/code-samples.yaml @@ -0,0 +1,63 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Code samples + +# Checks that the Python samples on the docs pages parse, and that the names +# they import from google.adk still exist. No sample is executed; the samples +# build agents and call live endpoints. + +on: + pull_request: + paths: + - 'docs/**/*.md' + - 'examples/python/**' + - 'tools/code_samples/**' + - '.github/workflows/code-samples.yaml' + push: + branches: + - main + paths: + - 'docs/**/*.md' + - 'examples/python/**' + - 'tools/code_samples/**' + workflow_dispatch: + +permissions: + contents: read + +jobs: + check: + runs-on: ubuntu-latest + steps: + - name: Check out the repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.12' + + - name: Install the ADK the samples are written against + run: | + python -m pip install --upgrade pip + # The extras matter: without them the optional integrations resolve + # as missing and every sample importing one looks broken. + pip install 'google-adk[all]' pytest + + - name: Test the checker + run: python -m pytest tools/code_samples/check_test.py -q + + - name: Check the code samples + run: python tools/code_samples/check.py diff --git a/tools/code_samples/check.py b/tools/code_samples/check.py new file mode 100644 index 0000000000..0f7e8ef4f8 --- /dev/null +++ b/tools/code_samples/check.py @@ -0,0 +1,250 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Check the Python code samples in the documentation. + +Two questions are asked of every sample: + + Does it parse? A sample a reader cannot paste into a file and run is a bug, + and several have shipped that way. + + Do the names it imports from google.adk still exist? This is what rots + quietly when the library renames or moves something, because the prose keeps + reading correctly while the code stops working. + +Nothing is imported or executed from the samples themselves. They construct +agents and call live endpoints, so running one in CI would be both slow and +billable. Only the library is imported, which is what any reader does. + +A sample lifted out of prose is often not a whole module, and that is fine. +Three shapes are recognised and excused: + + fragment a method body, or a block that only makes sense indented under + something the page shows above it + pseudocode "..." or "<...>" standing in for omitted code, or a notebook + magic such as !pip + signature a def or a class listed with no body, to describe an interface + +Run it over the whole tree, or pass paths to check only those files. +""" + +from __future__ import annotations + +import argparse +import ast +import builtins +import collections +import importlib +import pathlib +import re +import sys +import textwrap +import warnings + +import fences + +# Importing parts of google.adk emits experimental-feature and deprecation +# notices. They are about the library, not about the samples, and they bury the +# output that matters. +warnings.filterwarnings("ignore") + +DOCS = pathlib.Path("docs") +PY_LANGUAGES = {"python", "py"} +WATCHED_PACKAGE = "google.adk" + +ELISION = re.compile(r"^\s*\.\.\.\s*,?\s*$|<\.\.\.|\.\.\.\s*>|^\s*[!%][A-Za-z]") +ELISION_INLINE = re.compile(r"[(,]\s*\.\.\.\s*[,)]") +NEEDS_BODY = re.compile(r"expected an indented block") +BUILTINS = frozenset(dir(builtins)) + + +class Defect(collections.namedtuple("Defect", "path line kind detail")): + def __str__(self) -> str: + return f"{self.path}:{self.line}: {self.kind}: {self.detail}" + + +# -------------------------------------------------------------------------- +# Does it parse? +# -------------------------------------------------------------------------- + + +def _compiles(source: str) -> str | None: + try: + compile(source, "", "exec") + return None + except SyntaxError as exc: + return f"line {exc.lineno}: {exc.msg}" + except ValueError as exc: + return str(exc) + + +def _is_pseudocode(source: str) -> bool: + return any( + ELISION.search(line) or ELISION_INLINE.search(line) + for line in source.split("\n") + ) + + +def _body_omitted(source: str, error: str) -> bool: + """True when a block's body was written as comments, or left out.""" + if not NEEDS_BODY.search(error): + return False + match = re.search(r"line (\d+):", error) + if not match: + return False + lines = source.split("\n") + index = int(match.group(1)) - 1 + if index >= len(lines): + return True + rest = [l for l in lines[index:] if l.strip()] + # The reported line is often the header's own closing paren. + if rest and re.sub(r"\s+#.*$", "", rest[0]).rstrip().endswith(":"): + rest = rest[1:] + return not rest or all(l.lstrip().startswith("#") for l in rest) + + +def check_parses(source: str) -> str | None: + """Return a defect description, or None if the sample is acceptable.""" + error = _compiles(source) + if error is None: + return None + + flat = textwrap.dedent(source) + if flat != source and _compiles(flat) is None: + return None + + indented = textwrap.indent(flat, " ") + for wrapper in ("if True:\n", "class _C:\n", "async def _f():\n"): + if _compiles(wrapper + indented) is None: + return None + # An excerpt from inside an async loop may use await, break and continue. + deeper = textwrap.indent(flat, " ") + if _compiles(f"async def _f():\n while True:\n{deeper}") is None: + return None + + if _is_pseudocode(source) or _body_omitted(source, error): + return None + return error + + +# -------------------------------------------------------------------------- +# Do the imported names exist? +# -------------------------------------------------------------------------- + +_module_cache: dict[str, object | None] = {} + + +def _load(name: str) -> object | None: + if name not in _module_cache: + try: + _module_cache[name] = importlib.import_module(name) + except Exception: # noqa: BLE001 - any failure means "cannot resolve" + _module_cache[name] = None + return _module_cache[name] + + +def _has_attribute(module_name: str, module: object, attribute: str) -> bool: + try: + if hasattr(module, attribute): + return True + except ImportError: + # A lazy loader that raises has still declared the name; it only needs + # an optional dependency that is not installed here. + return True + except Exception: # noqa: BLE001 + return True + return _load(f"{module_name}.{attribute}") is not None + + +def check_imports(source: str) -> list[str]: + """Return descriptions of any google.adk import that cannot be resolved.""" + try: + tree = ast.parse(source) + except SyntaxError: + return [] # the parse check already reported on this sample + + problems: list[str] = [] + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom): + if node.level or not (node.module or "").startswith(WATCHED_PACKAGE): + continue + module = _load(node.module) + if module is None: + problems.append(f"no module named {node.module!r}") + continue + for alias in node.names: + if alias.name != "*" and not _has_attribute(node.module, module, alias.name): + problems.append(f"{node.module} has no {alias.name!r}") + elif isinstance(node, ast.Import): + for alias in node.names: + if alias.name.startswith(WATCHED_PACKAGE) and _load(alias.name) is None: + problems.append(f"no module named {alias.name!r}") + return problems + + +# -------------------------------------------------------------------------- + + +def check_file(path: pathlib.Path, root: pathlib.Path) -> list[Defect]: + defects: list[Defect] = [] + for fence in fences.parse(path): + if fence.language not in PY_LANGUAGES or fence.is_pointer: + continue + source = fences.resolve_includes(fence.text, root).strip("\n") + if not source: + continue + error = check_parses(source) + if error: + defects.append(Defect(path, fence.start_line, "does not parse", error)) + continue + for problem in check_imports(source): + defects.append(Defect(path, fence.start_line, "unresolved import", problem)) + return defects + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("paths", nargs="*", type=pathlib.Path, + help="markdown files to check; defaults to all of docs/") + parser.add_argument("--root", type=pathlib.Path, default=pathlib.Path("."), + help="repository root, used to resolve snippet includes") + args = parser.parse_args(argv) + + targets = args.paths or sorted((args.root / DOCS).rglob("*.md")) + targets = [p for p in targets if p.suffix == ".md" and p.is_file()] + + defects: list[Defect] = [] + for path in targets: + defects.extend(check_file(path, args.root)) + + print(f"checked {len(targets)} page(s)") + if not defects: + print("no defects") + return 0 + + by_kind = collections.Counter(d.kind for d in defects) + print(f"{len(defects)} defect(s): " + ", ".join(f"{v} {k}" for k, v in by_kind.items())) + print() + for defect in defects: + print(defect) + print() + print("Each sample above either cannot be parsed, or imports a name that") + print("google-adk no longer provides. Fix the sample, or if it is meant to") + print("be illustrative rather than runnable, mark the omitted part with") + print("'...' so it is recognised as pseudocode.") + return 1 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/tools/code_samples/check_test.py b/tools/code_samples/check_test.py new file mode 100644 index 0000000000..7649e92822 --- /dev/null +++ b/tools/code_samples/check_test.py @@ -0,0 +1,237 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the code sample checker. + +Two halves, and the second matters as much as the first. A checker that +reports nothing because it is broken looks exactly like a checker that reports +nothing because the samples are fine, so every defect class it claims to catch +is planted here and asserted on. + +Run with: python -m pytest tools/code_samples/check_test.py +""" + +from __future__ import annotations + +import pathlib +import sys +import textwrap + +sys.path.insert(0, str(pathlib.Path(__file__).parent)) + +import check # noqa: E402 +import fences # noqa: E402 + + +def page(tmp_path: pathlib.Path, markdown: str) -> pathlib.Path: + path = tmp_path / "page.md" + path.write_text(textwrap.dedent(markdown), encoding="utf-8") + return path + + +def defects(tmp_path: pathlib.Path, markdown: str) -> list[check.Defect]: + return check.check_file(page(tmp_path, markdown), tmp_path) + + +# ---------------------------------------------------------------- caught --- + + +def test_catches_unparseable_sample(tmp_path): + found = defects(tmp_path, """ + ```python + def broken( + ``` + """) + assert len(found) == 1 + assert found[0].kind == "does not parse" + + +def test_catches_unindented_class_body(tmp_path): + found = defects(tmp_path, """ + ```python + class Thing: + def method(self): + return 1 + ``` + """) + assert [d.kind for d in found] == ["does not parse"] + + +def test_catches_await_in_sync_function(tmp_path): + found = defects(tmp_path, """ + ```python + def tool(): + result = await something() + return result + ``` + """) + assert [d.kind for d in found] == ["does not parse"] + + +def test_catches_module_that_does_not_exist(tmp_path): + found = defects(tmp_path, """ + ```python + from google.adk.no_such_module import Thing + ``` + """) + assert [d.kind for d in found] == ["unresolved import"] + + +def test_catches_name_that_does_not_exist(tmp_path): + found = defects(tmp_path, """ + ```python + from google.adk.agents import NoSuchAgentClass + ``` + """) + assert [d.kind for d in found] == ["unresolved import"] + + +def test_reports_the_line_of_the_opening_fence(tmp_path): + found = defects(tmp_path, """ + Some prose. + + ```python + from google.adk.agents import NoSuchAgentClass + ``` + """) + assert found[0].line == 4 + + +def test_catches_a_sample_nested_in_a_tab(tmp_path): + found = defects(tmp_path, """ + === "Python" + + ```python + from google.adk.agents import NoSuchAgentClass + ``` + """) + assert [d.kind for d in found] == ["unresolved import"] + + +# ---------------------------------------------------------------- passed --- + + +def test_accepts_a_whole_module(tmp_path): + assert not defects(tmp_path, """ + ```python + from google.adk.agents import LlmAgent + + agent = LlmAgent(name="demo", model="gemini-flash-latest") + ``` + """) + + +def test_accepts_a_method_body_fragment(tmp_path): + assert not defects(tmp_path, """ + ```python + self.counter += 1 + print(self.counter) + ``` + """) + + +def test_accepts_pseudocode_with_elision(tmp_path): + assert not defects(tmp_path, """ + ```python + agent = LlmAgent( + name="demo", + ... + ) + ``` + """) + + +def test_accepts_a_signature_listing(tmp_path): + assert not defects(tmp_path, """ + ```python + async def before_model_callback( + self, *, callback_context, llm_request + ) -> None: + ``` + """) + + +def test_accepts_an_await_excerpt_from_a_loop(tmp_path): + assert not defects(tmp_path, """ + ```python + async for event in runner.run_async(...): + if event.is_final_response(): + break + await handle(event) + ``` + """) + + +def test_ignores_non_python_blocks(tmp_path): + assert not defects(tmp_path, """ + ```go + this is not python at all ((( + ``` + + ```bash + pip install google-adk + ``` + """) + + +def test_ignores_a_block_that_only_includes_a_snippet(tmp_path): + assert not defects(tmp_path, """ + ```python + --8<-- "examples/python/does/not/matter.py" + ``` + """) + + +# ------------------------------------------------------------ fence parse --- + + +def test_fence_survives_backticks_inside_a_string(tmp_path): + found = fences.parse(page(tmp_path, ''' + ```python + PROMPT = """ + ```tool_code + nested and indented + ``` + """ + ``` + ''')) + assert len(found) == 1 + assert "nested and indented" in found[0].text + + +def test_fence_skips_a_block_that_never_closes_cleanly(tmp_path): + """A fence closed at the wrong indentation swallows what follows. + + Opening at six columns and "closing" at eight means the block never + terminates, so the next opener gets absorbed into its body. Rather than + guess a boundary, the parser reports nothing for the whole ambiguous run. + Reporting nothing is right: any guess would produce defects against code + the author never wrote. + """ + path = tmp_path / "page.md" + path.write_text( + " ```python\n x = 1\n ```\n\n ```python\n y = 2\n ```\n", + encoding="utf-8", + ) + assert fences.parse(path) == [] + + +def test_dedent_measures_columns_not_characters(tmp_path): + path = tmp_path / "page.md" + # Fence at four columns; body line indented with a tab, which is four + # columns, then real code indentation on top. + path.write_text(" ```python\n\tif True:\n\t pass\n ```\n", encoding="utf-8") + body = fences.parse(path)[0].text + assert body.split("\n")[0] == "if True:" + assert body.split("\n")[1] == " pass" diff --git a/tools/code_samples/fences.py b/tools/code_samples/fences.py new file mode 100644 index 0000000000..d443f6dfae --- /dev/null +++ b/tools/code_samples/fences.py @@ -0,0 +1,159 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Pull fenced code blocks out of the documentation. + +Blocks are frequently nested inside tabbed sections, so a fence can be indented +by four or eight columns, and its body carries that indentation on every line. +Both are handled here so the rest of the tooling sees the code as an author +wrote it. +""" + +from __future__ import annotations + +import dataclasses +import pathlib +import re + +FENCE = re.compile(r"^(?P[ \t]*)(?P`{3,}|~{3,})(?P.*)$") +SNIPPET_INCLUDE = "--8<--" + + +def _columns(text: str, upto: int) -> int: + """Display width of the first `upto` characters, tabs expanding to four.""" + col = 0 + for ch in text[:upto]: + col = col + (4 - col % 4) if ch == "\t" else col + 1 + return col + + +@dataclasses.dataclass(frozen=True) +class Fence: + """One fenced code block.""" + + path: pathlib.Path + start_line: int # 1-based line of the opening fence + language: str + body: tuple[str, ...] # dedented to column zero + + @property + def text(self) -> str: + return "\n".join(self.body) + + @property + def is_pointer(self) -> bool: + """True when the block only pulls in an external snippet file.""" + lines = [line for line in self.body if line.strip()] + return bool(lines) and all(l.startswith(SNIPPET_INCLUDE) for l in lines) + + +def _dedent(body: list[str], indent: str) -> list[str]: + """Strip the fence's own indentation, never the code's.""" + width = _columns(indent, len(indent)) + if not width: + return body + out = [] + for line in body: + cut = 0 + while cut < len(line) and line[cut] in " \t" and _columns(line, cut + 1) <= width: + cut += 1 + out.append(line[cut:]) + return out + + +def parse(path: pathlib.Path) -> list[Fence]: + """Return every fenced block in one markdown file.""" + lines = path.read_text(encoding="utf-8").split("\n") + fences: list[Fence] = [] + i = 0 + while i < len(lines): + opening = FENCE.match(lines[i]) + if not opening or opening.group("ticks")[0] in opening.group("info"): + i += 1 + continue + indent, ticks = opening.group("indent"), opening.group("ticks") + + # A closing fence sits at the same indentation. A more deeply indented + # run of backticks is content, which is how a sample that quotes fenced + # markdown inside a string survives intact. + close = None + j = i + 1 + while j < len(lines): + line = lines[j] + if line.startswith(indent): + rest = line[len(indent):].rstrip() + if rest and set(rest) == {ticks[0]} and len(rest) >= len(ticks): + close = j + break + j += 1 + if close is None: + i += 1 + continue + + body = lines[i + 1 : close] + # Another fence opening at this indentation means the block was never + # closed cleanly and any boundary here is a guess, so skip it. + ambiguous = any( + (m := FENCE.match(l)) and m.group("indent") == indent and m.group("info").strip() + for l in body + ) + if not ambiguous: + info = opening.group("info").strip() + language = info.split(" ", 1)[0].split("{", 1)[0].strip().lower() + fences.append( + Fence( + path=path, + start_line=i + 1, + language=language, + body=tuple(_dedent(body, indent)), + ) + ) + i = close + 1 + return fences + + +def resolve_includes(text: str, root: pathlib.Path, depth: int = 0) -> str: + """Expand `--8<--` snippet includes so a block can be read as real code.""" + if depth > 5: + return text + section_marker = re.compile(r"--8<--\s+\[(start|end):([^\]]+)\]") + include = re.compile(r'^(?P\s*)--8<--\s+"(?P[^"]+)"\s*$') + + out: list[str] = [] + for line in text.split("\n"): + m = include.match(line) + if not m: + if not section_marker.search(line): + out.append(line) + continue + target, _, section = m.group("spec").partition(":") + ref = root / target + if not ref.is_file(): + out.append(line) + continue + inner = ref.read_text(encoding="utf-8").split("\n") + if section: + picked, inside = [], False + for l in inner: + sm = section_marker.search(l) + if sm and sm.group(2).strip() == section.strip(): + inside = sm.group(1) == "start" + continue + if inside: + picked.append(l) + inner = picked + expanded = resolve_includes("\n".join(inner), root, depth + 1).split("\n") + pad = m.group("indent") + out.extend(pad + l if l.strip() else l for l in expanded) + return "\n".join(out)