diff --git a/compose_api/environments/compliance.py b/compose_api/environments/compliance.py index e6ae73cc..cf10d052 100644 --- a/compose_api/environments/compliance.py +++ b/compose_api/environments/compliance.py @@ -137,7 +137,8 @@ class Expectation(BaseModel): - ``reference``: every column of ``reference`` (a CSV in the case, with a time column first) matches the result at the reference's times, within the tolerance. - ``scalar``: the emitted number ``field`` in row ``row`` (the last when omitted) is ``expected``, within the - tolerance: for a component whose stores are plain numbers rather than a ``numeric_result``. + tolerance: for a component whose stores are plain numbers rather than a ``numeric_result``. ``field`` may be a + dotted path into lists and maps, ``values.2.0`` (composition 7c-4: an ensemble's per-seed values). """ kind: Literal["rows", "value", "reference", "scalar"] @@ -237,8 +238,10 @@ def _check_scalar(expectation: Expectation, rows: list[Any]) -> list[str]: if expectation.expected is None: return ["a scalar expectation needs `expected`"] try: - row = rows[expectation.row] - actual = float(row[expectation.field]) + found: Any = rows[expectation.row] + for step in expectation.field.split("."): + found = found[int(step)] if isinstance(found, list) else found[step] + actual = float(found) except (IndexError, KeyError, TypeError, ValueError) as e: return [f"{where}: no number ({type(e).__name__}: {e})"] if _close(actual, expectation.expected, expectation.rel, expectation.abs): diff --git a/compose_api/registry/compliance/viva-pde-particle/fokker-planck-ensemble/case.yaml b/compose_api/registry/compliance/viva-pde-particle/fokker-planck-ensemble/case.yaml new file mode 100644 index 00000000..207e7f3f --- /dev/null +++ b/compose_api/registry/compliance/viva-pde-particle/fokker-planck-ensemble/case.yaml @@ -0,0 +1,18 @@ +description: >- + viva-pde-particle's own example (compose_runner.write_example): FokkerPlanckEnsemble runs four trials, seeds 1 to 4, of + the Fokker-Planck benchmark's trial_rho (test4, tau 0.2, dtau 1e-3), each a PDE solve co-simulated with Smoldyn (built + with OPTION_VCELL), and emits each seed's two values. The emitter is named fully qualified (composition D7c.2). The + values are seeded, so exact: these are what its prebuilt image's runner and compose-api's runner gave in the + registry_env@5 + viva-pde-particle environment, identically (2026-10-27, docs/plan-environment-composition.md 7c). +document: document.pbg +interval: 1 +expect: + - {kind: rows, emitter: emitter, min: 2} + - {kind: scalar, emitter: emitter, field: seeds.0, expected: 1} + - {kind: scalar, emitter: emitter, field: seeds.3, expected: 4} + - {kind: scalar, emitter: emitter, field: values.0.0, expected: 0.07137369415367853, rel: 1.0e-6} + - {kind: scalar, emitter: emitter, field: values.0.1, expected: 0.0004004746655585104, rel: 1.0e-6} + - {kind: scalar, emitter: emitter, field: values.1.0, expected: 0.0, abs: 1.0e-12} + - {kind: scalar, emitter: emitter, field: values.2.0, expected: 0.09234150979599469, rel: 1.0e-6} + - {kind: scalar, emitter: emitter, field: values.2.1, expected: 0.0020975747341526988, rel: 1.0e-6} + - {kind: scalar, emitter: emitter, field: values.3.0, expected: 0.0, abs: 1.0e-12} diff --git a/compose_api/registry/compliance/viva-pde-particle/fokker-planck-ensemble/document.pbg b/compose_api/registry/compliance/viva-pde-particle/fokker-planck-ensemble/document.pbg new file mode 100644 index 00000000..911a9677 --- /dev/null +++ b/compose_api/registry/compliance/viva-pde-particle/fokker-planck-ensemble/document.pbg @@ -0,0 +1,46 @@ +{ + "state": { + "seeds": [], + "values": [], + "ensemble": { + "_type": "process", + "address": "local:viva_pde_particle.ensemble.FokkerPlanckEnsemble", + "config": { + "params": "{\"test\": \"test4\", \"tau\": 0.2, \"dtau\": 0.001}", + "seeds": [ + 1, + 4 + ], + "workers": 0 + }, + "interval": 1.0, + "inputs": {}, + "outputs": { + "seeds": [ + "seeds" + ], + "values": [ + "values" + ] + } + }, + "emitter": { + "_type": "step", + "address": "local:process_bigraph.emitter.RAMEmitter", + "config": { + "emit": { + "seeds": "list[integer]", + "values": "list[list[float]]" + } + }, + "inputs": { + "seeds": [ + "seeds" + ], + "values": [ + "values" + ] + } + } + } +} diff --git a/compose_api/registry/environments.py b/compose_api/registry/environments.py index 37647318..fdb40a43 100644 --- a/compose_api/registry/environments.py +++ b/compose_api/registry/environments.py @@ -100,11 +100,13 @@ def spec_problems(spec: EnvironmentSpec, manifest: Manifest, *, bundled: bool = def members_of(spec: EnvironmentSpec, manifest: Manifest) -> frozenset[str]: """The registry entries an environment provides: those it is composed of, or, for a prebuilt image (which lists - none), the entries bundled in it. Core entries are left out: every environment that can run their classes + none), the entries bundled in it and what they require, which the image installs its own way (viva-pde-particle's + Smoldyn and engines: composition 7c-4). Core entries are left out: every environment that can run their classes provides them, so they never tell environments apart (composition 7c-1).""" - return frozenset(spec.entries) or frozenset( - e.id for e in manifest.entries if spec.name in e.bundles and e.kind != "core" - ) + if spec.entries: + return frozenset(spec.entries) + bundled = {e.id for e in manifest.entries if spec.name in e.bundles and e.kind != "core"} + return frozenset(manifest.with_requirements(bundled)) def _unlisted_requirements(spec: EnvironmentSpec, by_id: dict[str, RegistryEntry]) -> list[str]: diff --git a/compose_api/registry/manifest.yaml b/compose_api/registry/manifest.yaml index 7cf1f636..5759b47b 100644 --- a/compose_api/registry/manifest.yaml +++ b/compose_api/registry/manifest.yaml @@ -77,12 +77,40 @@ entries: - id: viva-pde-particle package: viva-pde-particle source: https://github.com/vivarium-collective/viva-pde-particle - # the commit behind the deployed image (PREBUILT_SIMULATORS pins its digest, tagged sha-8c16827) - commit: 8c168271e3b48ac667cde9092ca025be0f444d65 + # viva-pde-particle#67's merge (2026-10-27): each trial function its own Ensemble class, so no document's config + # names what is imported. What a composed environment installs, and the commit of the prebuilt image + # (viva-pde-particle-compose:sha-21dfe46) registered as viva-pde-particle@2 + commit: 21dfe468455b3a8746b184469651494f4b06b4a4 + # composed too (docs/plan-environment-composition.md 7c, D7c.1-D7c.3): its engines from conda-forge, as its own + # pixi.toml's `compose` environment has them, less the Smoldyn toolchain; Smoldyn is the smoldyn-vcell wheel + install: + git: https://github.com/vivarium-collective/viva-pde-particle + commit: 21dfe468455b3a8746b184469651494f4b06b4a4 + extras: [reference] + conda: + fenics-dolfinx: "0.10.*" + mpich: "*" + gmsh: "*" + python-gmsh: "*" + h5py: "*" + netgen: ">=6.2.2602,<7" + scikit-image: "*" + pydantic: ">=2.10,<3" + vtk: "*" + pyvista: ">=0.49.0,<0.50" + imageio: ">=2.38.0,<3" + # what its pixi.toml adds beyond its package metadata: Smoldyn, the VCell solver and geometry, results bundles + requires: + - smoldyn-vcell + - vcell-fenics + - pyvcell-fvsolver + - zarr + - pyvcell + - libvcell module_roots: - viva_pde_particle level: tested - # its own prebuilt image, not registry_env: it runs only with `simulator=viva-pde-particle` + # its own prebuilt image runs a document of its classes alone (D7c.3); composed, they join a mash-up bundles: - viva-pde-particle evidence: @@ -91,6 +119,11 @@ entries: package's own CI on every push to main (.github/workflows/compose-image.yml): a smoke ensemble in Docker, then this service's exact job contract under `apptainer run --compat` with /experiment bound, before the image is pushed. The pinned digest passed both. + - >- + Composed on 2026-10-27 (7c-4): with registry_env@5's entries in one pixi lock (472 packages, no conflict), its + example ensemble (four PDE + Smoldyn trials) gave values identical to its prebuilt image's through + compose-api's runner. In a composed version its compliance case (registry/compliance/viva-pde-particle) + decides its level. - id: biosim-toolkit package: biosim-toolkit @@ -160,6 +193,58 @@ entries: # Pinned to what registry_env@1 actually ran (read back from its image, 2026-10-25), not to the old library list, # which said tellurium 2.2.11.1 and python-copasi 4.46.300 while the image held these. + - id: smoldyn-vcell + kind: library + package: smoldyn + # Smoldyn's Python module built with OPTION_VCELL (the hybrid ValueProvider hooks) from the virtualcell fork's + # pyhybrid branch, by its .github/workflows/vcell-hybrid-wheel.yml (composition 7c-3, D7c.1): on no index + source: https://github.com/virtualcell/Smoldyn/releases/tag/vcell-pyhybrid-e86d552 + version: "2.76.dev0+vcell.e86d552" + install: + url: https://github.com/virtualcell/Smoldyn/releases/download/vcell-pyhybrid-e86d552/smoldyn-2.76.dev0%2Bvcell.e86d552-cp312-cp312-linux_x86_64.whl + sha256: bb033bccff9481a075603d4a79fefe249b152fac5544a716adf16761adc29d14 + level: installable + evidence: + - >- + Built at e86d552 by the fork's workflow and released 2026-10-27; the hash is the release asset's, checked on + download. CPython 3.12 on Linux x86-64; it links libpython, which composed images put on LD_LIBRARY_PATH. + + - id: vcell-fenics + kind: library + package: vcell-fenics + source: https://github.com/virtualcell/vcell-fenics/tree/4dab281e1ed3f7c2b2bb60c6c2d1a8ccb251ca17 + commit: 4dab281e1ed3f7c2b2bb60c6c2d1a8ccb251ca17 + install: + git: https://github.com/virtualcell/vcell-fenics + commit: 4dab281e1ed3f7c2b2bb60c6c2d1a8ccb251ca17 + level: installable + evidence: + - The commit viva-pde-particle's pixi.toml pins (main, the merge of #211); its geometry formalism and Netgen realization. + + - id: pyvcell-fvsolver + kind: library + package: pyvcell-fvsolver + source: https://pypi.org/project/pyvcell-fvsolver/0.11.1/ + version: "0.11.1" + install: + pypi: pyvcell-fvsolver + version: "0.11.1" + level: installable + evidence: + - viva-pde-particle's pixi.toml floor (>=0.11.1, the hybrid fixes vcell-fvsolver#32-#34); what the 2026-10-27 lock resolved. + + - id: zarr + kind: library + package: zarr + source: https://pypi.org/project/zarr/3.4.1/ + version: "3.4.1" + install: + pypi: zarr + version: "3.4.1" + level: installable + evidence: + - viva-pde-particle's results bundles are zarr groups (its pixi.toml, >=3.1,<4); what the 2026-10-27 lock resolved. + - id: tellurium kind: library package: tellurium diff --git a/docs/plan-environment-composition.md b/docs/plan-environment-composition.md index 1f11642f..60dc8d2e 100644 --- a/docs/plan-environment-composition.md +++ b/docs/plan-environment-composition.md @@ -480,10 +480,42 @@ - a compliance case kept with it: a short PDE + Smoldyn ensemble with fixed seeds, the emitter fully qualified, and its expected values; - tested locally (lock, install, the case) before merge. + - *As built (2026-10-27):* + - **The wheel:** release `vcell-pyhybrid-e86d552` of `virtualcell/Smoldyn`, built by its workflow (#1, 7c-3). + sha256 `bb033bcc…`, checked on download. Builds are not bit-reproducible, so the entry takes the release + asset's hash. + - **The entries:** + - `smoldyn-vcell` (by URL), `vcell-fenics` (git `4dab281`), `pyvcell-fvsolver` 0.11.1 and `zarr` 3.4.1; + - viva-pde-particle gains `install` (git `21dfe46`, viva-pde-particle#67's merge, below), the + `reference` extra, its conda engines, and `requires`. + - **`members_of` for a prebuilt image** now includes its bundled entries' requirements, which the image + installs its own way. Without that, a document of viva-pde-particle's classes alone would no longer be + covered by its image and would make a mash-up, breaking D7c.3. + - **Compliance case `viva-pde-particle/fokker-planck-ensemble`:** its own example (four seeded PDE + Smoldyn + trials) with the emitter fully qualified. A `scalar` expectation's `field` may now be a path + (`values.2.0`). + - **Checked with compose-api's own code, in Docker:** + - a document of its ensemble, `CopasiUTCStep` and RAMEmitter needs ten entries; no declared environment + covers them; they are composable, so `mash-05956395cf`; + - its generated workspace locked in 45 s (462 packages); + - the definition's URL check passed against the release; + - `pixi install --frozen` succeeded; + - compose-api's runner ran both cases, and its evaluator passed both: viva-pde-particle's nine + expectations, and pbsim-common's COPASI case against its reference; + - validation admits the case document in the mash-up. + - **`EnsembleRunner` imported its config (decided 2026-10-27).** Its `trial: "module:function"` imported + and called whatever a document named, which the address check never sees. Jim chose a fix in the class + (vivarium-collective/viva-pde-particle#67, merged as `21dfe46`): each trial function has its own + `Ensemble` subclass naming it in code (`FokkerPlanckEnsemble` for `trial_rho`), so the document picks the + trial by its address, its config is only `params`, `seeds` and `workers`, and `EnsembleRunner` is gone. + The same PR names the emitter fully qualified in `ensemble_document`. The case document moved to + `local:viva_pde_particle.ensemble.FokkerPlanckEnsemble`, and its values are unchanged. + - **The prebuilt image** (viva-pde-particle@1, `sha-8c16827`) still has `EnsembleRunner`, and a prebuilt + run's classes are not looked up. After the release, `environments create viva-pde-particle --image + …viva-pde-particle-compose@sha256:` makes @2; it becomes the default, `environments + inspect` reads it back, and @1 is retired. - **7c-5. Live:** a release, then a document combining viva-pde-particle's processes with `CopasiUTCStep`, submitted signed in. It creates its mash-up, locks and builds it (expect a large image), and runs. - - **Optional, vivarium-collective with Jim's go-ahead:** viva-pde-particle's `ensemble_document` writes the fully - qualified emitter (`ensemble.py`, one line), so documents it generates pass validation as they are. 8. **Retire** what composition replaced (surveyed 2026-10-26, when registry_env@5, composed, became the default). diff --git a/tests/api/test_submission_rejection.py b/tests/api/test_submission_rejection.py index 27df89a4..3481ede2 100644 --- a/tests/api/test_submission_rejection.py +++ b/tests/api/test_submission_rejection.py @@ -131,7 +131,8 @@ async def test_a_run_naming_no_environment_goes_where_its_classes_are( http_api_client: httpx.AsyncClient, database_service: DatabaseService ) -> None: """Step 7a (D7.1): a document using only viva-pde-particle's classes is inferred to run in its image when the - deployment offers it, as if it had named it; offered nowhere, the shared environment refuses it as before.""" + deployment offers it, as if it had named it. Offered nowhere, its classes are composable since composition 7c-4, + so the run needs a mash-up, which only a signed-in caller may cause (D7.4): an anonymous one is told to sign in.""" await register_prebuilt(database_service, PREBUILT, PDE_IMAGE) inferred = await http_api_client.post( "/simulation/run", files={"uploaded_file": ("doc.pbg", pbg(PDE), "application/json")} @@ -141,9 +142,9 @@ async def test_a_run_naming_no_environment_goes_where_its_classes_are( refused = await http_api_client.post( "/simulation/run", files={"uploaded_file": ("doc.pbg", pbg(PDE), "application/json")} ) - assert refused.status_code == 400 - (violation,) = refused.json()["detail"]["violations"] - assert "not installed in the environment `registry_env`" in violation["reason"] + assert refused.status_code == 401, refused.text + detail = refused.json()["detail"] + assert "viva-pde-particle" in detail and "sign in to have one made" in detail @pytest.mark.asyncio diff --git a/tests/environments/test_compliance.py b/tests/environments/test_compliance.py index aa6c4858..22afb637 100644 --- a/tests/environments/test_compliance.py +++ b/tests/environments/test_compliance.py @@ -140,3 +140,23 @@ def test_viva_basic_processes_case_checks_plain_numbers() -> None: assert not outcome.passed and outcome.failures == ["emitter.y in row -1: 33.0, expected 34.0"] missing = evaluate(case, {"emitter": [{"x": 1.0}]}) assert any("no number" in f for f in missing.failures) + + +def test_viva_pde_particles_case_reads_an_ensembles_values_by_path() -> None: + """Its emitter records each seed's values as lists, so `scalar` reads `values.2.0`: a path into lists and maps + (composition 7c-4). The rows are what compose-api's runner emitted for this document in the registry_env@5 + + viva-pde-particle environment on 2026-10-27, identical to its prebuilt image's.""" + (case,) = load_cases("viva-pde-particle") + assert compliance.case_problems(case) == [] + values = [[0.07137369415367853, 0.0004004746655585104], [0.0, 0.0], + [0.09234150979599469, 0.0020975747341526988], [0.0, 0.0]] # fmt: skip + rows = [{"seeds": [], "values": []}, {"seeds": [1, 2, 3, 4], "values": values}] + assert evaluate(case, {"emitter": rows}).passed + off = [rows[0], {"seeds": [1, 2, 3, 4], "values": [values[0], values[1], [0.09, 0.002], values[3]]}] + outcome = evaluate(case, {"emitter": off}) + assert not outcome.passed and any(f.startswith("emitter.values.2.0 in row -1: 0.09") for f in outcome.failures) + short = evaluate(case, {"emitter": [rows[0], {"seeds": [1], "values": [values[0]]}]}) + assert any("no number (IndexError" in f for f in short.failures) + # the document names its emitter fully qualified (D7c.2), never viva-pde-particle's own short `ram-emitter` + document = json.loads(case.path(case.document).read_text()) + assert document["state"]["emitter"]["address"] == "local:process_bigraph.emitter.RAMEmitter" diff --git a/tests/environments/test_mash_up_creation.py b/tests/environments/test_mash_up_creation.py index 7c47dc6c..fd1259ba 100644 --- a/tests/environments/test_mash_up_creation.py +++ b/tests/environments/test_mash_up_creation.py @@ -115,7 +115,8 @@ def test_a_mash_up_is_named_by_its_entries_and_only_composable_ones_qualify() -> assert lifecycle.composable(NEEDED, manifest) == [] (listed,) = lifecycle.composable({"viva-tellurium"}, manifest) # a catalog wrapper: no install, `listed` assert "does not say how it installs" in listed - assert any("cannot be composed" in p for p in lifecycle.composable({"viva-pde-particle"}, manifest)) + # viva-pde-particle is composable since composition 7c-4: its classes may join a mash-up + assert lifecycle.composable(manifest.with_requirements({"viva-pde-particle"}), manifest) == [] def test_declared_environments_cover_before_mash_ups() -> None: diff --git a/tests/environments/test_url_installs.py b/tests/environments/test_url_installs.py index ba4ef3e7..5bb62f33 100644 --- a/tests/environments/test_url_installs.py +++ b/tests/environments/test_url_installs.py @@ -83,7 +83,8 @@ def test_the_check_passes_the_wheel_or_fails_the_build(tmp_path: Path, matches: def test_a_mash_up_may_list_a_url_entry() -> None: - manifest = Manifest(entries=[*load_manifest().entries, _smoldyn()]) + others = [e for e in load_manifest().entries if e.id != "smoldyn-vcell"] + manifest = Manifest(entries=[*others, _smoldyn()]) spec = EnvironmentSpec(name="mash-x", kind="mash-up", python=["python"], entries=["smoldyn-vcell"]) composed = ComposedSpec.of(spec, manifest, COMPOSITION, WHEN) assert composed.as_json()["entries"][0]["install"] == {"url": WHEEL, "sha256": SHA} diff --git a/tests/registry/test_core_entries.py b/tests/registry/test_core_entries.py index a1a86a09..92ce0236 100644 --- a/tests/registry/test_core_entries.py +++ b/tests/registry/test_core_entries.py @@ -92,3 +92,22 @@ def test_a_core_entry_needs_module_roots() -> None: RegistryEntry( id="x", package="x", source="s", kind="core", version="1", level=CurationLevel.TESTED, module_roots=[] ) + + +def test_viva_pde_particle_and_copasi_make_a_mash_up_that_admits_them() -> None: + """The motivating mash-up (composition 7c): viva-pde-particle's ensemble beside pbsim-common's COPASI step, with + process-bigraph's emitter. No declared environment covers both, so a mash-up of exactly what they need is made, + and its version admits every address. Locked, built and run in Docker on 2026-10-27 (7c-4).""" + from compose_api.environments import lifecycle + from compose_api.registry.resolver import covering_environment + + manifest = load_manifest() + targets = ["viva_pde_particle.ensemble.FokkerPlanckEnsemble", COPASI[6:], RAM[6:]] + needed = needed_entries(targets, manifest) + assert {"viva-pde-particle", "smoldyn-vcell", "pbsim-common", "python-copasi"} <= needed + assert "process-bigraph-emitters" not in needed + assert covering_environment(needed, manifest, load_environments()) is None + assert lifecycle.composable(needed, manifest) == [] + standing = Standing(label=f"{lifecycle.mash_up_name(needed)}@1", members=frozenset(needed)) + for address in ("local:viva_pde_particle.ensemble.FokkerPlanckEnsemble", COPASI, RAM): + assert _reason(address, standing) is None, address diff --git a/tests/registry/test_environments.py b/tests/registry/test_environments.py index 81402354..c21b3fdd 100644 --- a/tests/registry/test_environments.py +++ b/tests/registry/test_environments.py @@ -154,7 +154,9 @@ def test_summaries_describe_the_default_versions_manifest() -> None: current = summarize(spec, {"viva-pde-particle": PDE_IMAGE}, manifest) assert current is not None assert (current.simulator, current.components, current.errors, current.stale) == ("viva-pde-particle", 2, 1, False) - assert current.process_bigraph == "1.8.5" and [e.id for e in current.entries] == ["viva-pde-particle"] + # its entry and what it requires, which its image installs (composition 7c-4), as registry_env lists its libraries + assert current.process_bigraph == "1.8.5" and "viva-pde-particle" in [e.id for e in current.entries] + assert {"smoldyn-vcell", "vcell-fenics"} <= {e.id for e in current.entries} shared = next(e for e in load_environments().environments if e.kind == "shared") listed = summarize(shared, {}, None) diff --git a/tests/registry/test_resolver.py b/tests/registry/test_resolver.py index d0fad80a..a57ded0b 100644 --- a/tests/registry/test_resolver.py +++ b/tests/registry/test_resolver.py @@ -38,7 +38,9 @@ def test_every_class_registry_env_provides_still_runs_there() -> None: def test_a_prebuilt_environments_classes_go_to_it_when_it_is_offered() -> None: manifest = load_manifest() needed = needed_entries([PDE], manifest) - assert needed == {"viva-pde-particle"} + # its requirements come with it (composition 7c-4: it is composable now), and its image provides them, so a + # document of its classes alone still runs in its prebuilt image (D7c.3), not in a mash-up + assert "viva-pde-particle" in needed and "smoldyn-vcell" in needed assert choose_environment(needed, manifest, EVERYWHERE).name == "viva-pde-particle" # not offered here: the shared environment, whose validation then says the entry is not installed in it assert choose_environment(needed, manifest, EVERYWHERE, offered=lambda spec: False).name == "registry_env"