feat: make the plugin submittable to the OpenAI directory

.codex-plugin/plugin.json

@@ -40,5 +40,64 @@
     "brandColor": "#0075FF",
     "composerIcon": "./assets/checkly-logo.png",
     "logo": "./assets/checkly-logo.png"
+  },
+  "extensions": {
+    "com.openai": {
+      "review": {
+        "test_cases": {
+          "positive": [
+            {
+              "description": "Review the status of every check",
+              "prompt": "Which of my Checkly checks are failing right now?",
+              "tools_triggered": "show-check-status, show-check-list",
+              "expected_behavior": "List the checks in the connected Checkly account with their current status and call out the failing ones. Only checks the connected account can see are reported."
+            },
+            {
+              "description": "Inspect the latest result of a failing check",
+              "prompt": "Show me the latest result for my API checks and explain what failed.",
+              "tools_triggered": "list-check-results, get-check-result",
+              "expected_behavior": "Fetch the most recent results for the account's API checks and summarise any failure, including response status and timing, without inventing values that the results do not contain."
+            },
+            {
+              "description": "Create an API check from a description",
+              "prompt": "Add an API check for https://example.com that runs every 10 minutes from Frankfurt and fails on a non-200 response.",
+              "tools_triggered": "get-check-type-schema, create-check",
+              "expected_behavior": "Read the API check schema, create the check in the connected account with the requested interval, location and assertion, then report the created check and its ID."
+            },
+            {
+              "description": "Run a check on demand",
+              "prompt": "Run my Login check now and tell me whether it passed.",
+              "tools_triggered": "trigger-checks, get-check-result",
+              "expected_behavior": "Trigger the named check, wait for the run to finish, and report pass or fail with the result of that run."
+            },
+            {
+              "description": "Investigate a recurring failure",
+              "prompt": "Investigate why my Products API check has been failing this morning.",
+              "tools_triggered": "get-error-group-root-cause-analyses, trigger-root-cause-analysis, get-root-cause-analysis",
+              "expected_behavior": "Look up an existing root cause analysis for the failing check or start one, then summarise the probable cause from the returned analysis without speculating beyond it."
+            }
+          ],
+          "negative": [
+            {
+              "description": "Environment variable writes are not exposed",
+              "prompt": "Set the CHECKLY_API_KEY environment variable in my Checkly account to a new secret."
+            },
+            {
+              "description": "Deploy and infrastructure actions are out of scope",
+              "prompt": "Roll back the last deploy - the API checks started failing after it."
+            },
+            {
+              "description": "Billing and subscription changes are out of scope",
+              "prompt": "Upgrade my Checkly subscription to the Enterprise plan and update the payment method."
+            }
+          ]
+        },
+        "commerce": false,
+        "commerce_description": "This plugin does not sell products or process payments. Checkly subscription and billing changes happen in the Checkly dashboard."
+      },
+      "publication": {
+        "release_notes": "Initial directory release. Bundles the Checkly CLI skill and the Playwright best-practices skill, plus the Checkly MCP server for inspecting checks, results, test sessions and root cause analyses, running checks on demand, and managing API checks, monitors, check groups and status page incidents."
+      }
+    }
   }
 }

README.md

@@ -61,7 +61,15 @@ Point Cursor at this repo to install the plugin.
 npm run package
 ```
 
-Writes `dist/checkly-plugin-<version>.zip` containing only the files the Codex plugin needs: the Codex manifest, `.mcp.json`, `skills/`, `assets/`, `LICENSE`, and `README.md`. The file list comes from `git ls-files`, so untracked files such as `.env` never end up in the archive. The manifest `name` inside the ZIP is replaced with the ID of the existing OpenAI listing (`OPENAI_PLUGIN_NAME` in `scripts/package.ts`), because the directory rejects uploads whose name doesn't match. The repo manifest keeps `checkly` for marketplace installs. `.app.json` and the manifest `apps` field are left out of the ZIP, because OpenAI rejects submissions with app references; the MCP app is linked in the dashboard instead. Upload the ZIP at the [OpenAI plugin dashboard](https://developers.openai.com/plugins/deploy/submission).
+Writes `dist/checkly-plugin-<version>.zip` as an Agent Plugins portable package: a root `plugin.json` and `mcp.json` generated from the repo's compatibility manifests, plus `skills/`, `assets/`, `LICENSE`, and `README.md`. The verbatim file list comes from `git ls-files`, so untracked files such as `.env` never end up in the archive.
+
+Both manifests are generated rather than copied, and the repo files are left alone so Claude Code, Cursor and Codex marketplace installs keep working:
+
+- `plugin.json` keeps the portable identity fields at the root and moves `interface` under `extensions["com.openai"]`, which replaces the `.codex-plugin/plugin.json` overlay entirely. `skills` and `mcpServers` are dropped, because portable packages discover `skills/` and `mcp.json` by path.
+- `mcp.json` renames the transport from Claude Code's `http` to the portable `streamable-http`, moves the OAuth scopes into `extensions["com.openai"].auth.baseScopes`, pins `defaultScopes` to `[]` so the portal can't fall back to every scope the server advertises, and declares `client.mode: "cimd"` as the registration method.
+- `.app.json` and the manifest `apps` field stay out: a directory submission can't reference a registered MCP app.
+
+Upload the ZIP at the [OpenAI plugin dashboard](https://developers.openai.com/plugins/deploy/submission).
 
 ## Adding more
 

scripts/package.ts

@@ -9,28 +9,129 @@ const REPO_ROOT = resolve(fileURLToPath(import.meta.url), "..", "..");
 const run = promisify(execFile);
 
 /**
- * Files that go into the Codex plugin ZIP. Everything else in the repo is
+ * Files that ship verbatim in the plugin ZIP. Everything else in the repo is
  * tooling (sync script, tests, CI, other agents' manifests) and must not ship.
  *
  * OpenAI requires the ZIP to contain a single plugin root with no secrets, so
  * the list is built from `git ls-files` and filtered here: untracked files
  * such as `.env` or `node_modules/` can never slip in.
  *
- * `.app.json` stays out: OpenAI rejects ZIPs with app references and links the
- * MCP app in the dashboard instead. The repo keeps it for local Codex installs.
+ * `.app.json` stays out: directory submissions can't reference a registered
+ * MCP app ("Directory submissions must use With MCP and submit the MCP server
+ * directly"). The repo keeps it for local Codex installs.
+ *
+ * The two compatibility manifests stay out as well, because the ZIP ships the
+ * portable manifests generated from them instead.
  */
-const PACKAGE_FILES = new Set([".codex-plugin/plugin.json", ".mcp.json", "LICENSE", "README.md"]);
+const PACKAGE_FILES = new Set(["LICENSE", "README.md"]);
 const PACKAGE_DIRS = ["assets/", "skills/"];
 
-const MANIFEST = ".codex-plugin/plugin.json";
+/** Compatibility manifests in the repo: the source for the portable ones. */
+const SOURCE_MANIFEST = ".codex-plugin/plugin.json";
+const SOURCE_MCP = ".mcp.json";
+
+/** Portable manifests written into the ZIP root. */
+const PORTABLE_MANIFEST = "plugin.json";
+const PORTABLE_MCP = "mcp.json";
+
+const PLUGIN_SCHEMA = "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json";
+const MCP_SCHEMA = "https://agent-plugins.org/schemas/1.0.0/mcp.schema.json";
+
+/** Agent Plugins manifest fields. Everything else is client-specific. */
+const PORTABLE_FIELDS = [
+  "name",
+  "version",
+  "description",
+  "author",
+  "homepage",
+  "repository",
+  "license",
+  "keywords",
+] as const;
+
+/** The repo declares Claude Code's `http`; the portable schema calls it this. */
+const PORTABLE_TRANSPORTS: Record<string, string> = { http: "streamable-http" };
+
+type Manifest = { interface?: Record<string, unknown>; [key: string]: unknown };
+type McpServer = { type?: string; scopes?: string[]; [key: string]: unknown };
+type McpConfig = { mcpServers: Record<string, McpServer>; [key: string]: unknown };
+
+/**
+ * Turn the Codex compatibility manifest into the portable `plugin.json` the
+ * plugin directory expects: identity at the root, OpenAI's presentation under
+ * `extensions["com.openai"]`. "When extensions.com.openai is an object, it
+ * replaces the entire .codex-plugin/plugin.json overlay", so the ZIP ships one
+ * manifest, not both.
+ *
+ * `skills` and `mcpServers` are dropped because portable packages discover
+ * those by path (`skills/`, `mcp.json`), and the manifest schema has no such
+ * fields. `apps` is dropped because `.app.json` isn't shipped and a directory
+ * submission may not reference a registered app.
+ */
+export function toPortableManifest(manifest: Manifest): Manifest {
+  const portable: Manifest = { $schema: PLUGIN_SCHEMA };
+  for (const field of PORTABLE_FIELDS) {
+    if (manifest[field] !== undefined) portable[field] = manifest[field];
+  }
+  const openai = {
+    ...(manifest.extensions as Record<string, object> | undefined)?.["com.openai"],
+    ...(manifest.interface ? { interface: manifest.interface } : {}),
+  };
+  if (Object.keys(openai).length > 0) portable.extensions = { "com.openai": openai };
+  return portable;
+}
 
 /**
- * The OpenAI plugin directory pins the manifest `name` to the ID of the
- * existing listing and rejects re-uploads that don't match. The repo manifest
- * keeps `checkly` because the Codex marketplace install from GitHub uses it,
- * so the ID is only swapped in inside the submission ZIP.
+ * Turn the repo's `.mcp.json` into the portable `mcp.json`.
+ *
+ * The transport is renamed: the Agent Plugins schema knows `stdio`,
+ * `streamable-http` and `sse`, and the repo's `http` is the Claude Code and
+ * Codex spelling that those clients read on a marketplace install. The portal
+ * recorded a plugin uploaded with `http` as `transport: "http"` and attached no
+ * app to it, so the portable name is what a submission should carry.
+ *
+ * OpenAI reads OAuth settings from `extensions["com.openai"].auth`, which the
+ * repo file never declares, so the scopes move to `baseScopes` and the
+ * top-level `scopes` key - which the portable schema has no field for - is
+ * dropped.
+ *
+ * `client.mode` is `cimd`: `auth.checklyhq.com` advertises no registration
+ * endpoint, so Dynamic Client Registration is unavailable, and a `provided`
+ * client would need a client ID we don't have. A Client Identifier Metadata
+ * Document works - a connector created with it completed OAuth against the
+ * tenant - and needs no client registration at all.
+ *
+ * `defaultScopes` is empty so only `baseScopes` is requested. ChatGPT otherwise
+ * falls back to every scope the server advertises, which would grant
+ * `checkly:environment-variables:write` and bring back the two environment
+ * variable tools that OpenAI's review flagged as soliciting secrets.
  */
-export const OPENAI_PLUGIN_NAME = "app-6a7f1be5a9e48191b23a7b6ae73cf976";
+export function toPortableMcpConfig(config: McpConfig): McpConfig {
+  const mcpServers = Object.fromEntries(
+    Object.entries(config.mcpServers).map(([name, { type, scopes, ...server }]) => [
+      name,
+      {
+        ...(type ? { type: PORTABLE_TRANSPORTS[type] ?? type } : {}),
+        ...server,
+        ...(scopes?.length
+          ? {
+              extensions: {
+                "com.openai": {
+                  auth: {
+                    type: "oauth",
+                    client: { mode: "cimd" },
+                    baseScopes: scopes,
+                    defaultScopes: [],
+                  },
+                },
+              },
+            }
+          : {}),
+      },
+    ]),
+  );
+  return { $schema: MCP_SCHEMA, mcpServers };
+}
 
 export function selectPackageFiles(trackedFiles: string[]): string[] {
   return trackedFiles.filter(
@@ -50,33 +151,31 @@ async function listTrackedFiles(root: string): Promise<string[]> {
 export async function createPackage({
   root,
   outFile,
-  pluginName,
 }: {
   root: string;
   outFile: string;
-  /** Replaces the manifest `name` inside the ZIP; the repo file is left alone. */
-  pluginName?: string;
 }): Promise<string[]> {
-  const files = selectPackageFiles(await listTrackedFiles(root));
+  const verbatim = selectPackageFiles(await listTrackedFiles(root));
   await mkdir(dirname(outFile), { recursive: true });
   await rm(outFile, { force: true });
-  const fromRepo = files.filter((file) => file !== MANIFEST);
   // -X drops extra file attributes so the archive is reproducible across machines.
-  await run("zip", ["-X", "-q", outFile, ...fromRepo], { cwd: root });
-  // The manifest is always rewritten: `apps` points at `.app.json`, which isn't shipped.
-  const { apps: _apps, ...manifest } = JSON.parse(await readFile(join(root, MANIFEST), "utf8"));
+  await run("zip", ["-X", "-q", outFile, ...verbatim], { cwd: root });
+
+  const manifest = toPortableManifest(
+    JSON.parse(await readFile(join(root, SOURCE_MANIFEST), "utf8")),
+  );
+  const mcpConfig = toPortableMcpConfig(JSON.parse(await readFile(join(root, SOURCE_MCP), "utf8")));
   const staging = await mkdtemp(join(tmpdir(), "checkly-plugin-manifest-"));
   try {
-    await mkdir(join(staging, dirname(MANIFEST)));
-    await writeFile(
-      join(staging, MANIFEST),
-      `${JSON.stringify({ ...manifest, name: pluginName ?? manifest.name }, null, 2)}\n`,
-    );
-    await run("zip", ["-X", "-q", resolve(outFile), MANIFEST], { cwd: staging });
+    await writeFile(join(staging, PORTABLE_MANIFEST), `${JSON.stringify(manifest, null, 2)}\n`);
+    await writeFile(join(staging, PORTABLE_MCP), `${JSON.stringify(mcpConfig, null, 2)}\n`);
+    await run("zip", ["-X", "-q", resolve(outFile), PORTABLE_MANIFEST, PORTABLE_MCP], {
+      cwd: staging,
+    });
   } finally {
     await rm(staging, { recursive: true, force: true });
   }
-  return files;
+  return [...verbatim, PORTABLE_MANIFEST, PORTABLE_MCP];
 }
 
 async function main() {
@@ -92,7 +191,7 @@ async function main() {
     REPO_ROOT,
     values.out ?? join("dist", `${manifest.name}-plugin-${manifest.version}.zip`),
   );
-  const files = await createPackage({ root: REPO_ROOT, outFile, pluginName: OPENAI_PLUGIN_NAME });
+  const files = await createPackage({ root: REPO_ROOT, outFile });
   for (const file of files) {
     console.log(`packaged ${file}`);
   }