v0.6.0 — verbatim filter extraction + per-flag probe gate

.claude-plugin/marketplace.json

@@ -8,7 +8,7 @@
     {
       "name": "clify",
       "description": "Generate A+ Node.js CLIs from API documentation. Copies a hand-crafted exemplar (structurally inspired by google/agents-cli), mechanically substitutes API-specific content, and verifies via a deterministic validation gate.",
-      "version": "0.5.0",
+      "version": "0.6.0",
       "source": "./"
     }
   ]

.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "clify",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "description": "Generate A+ Node.js CLIs from API documentation. Copies a hand-crafted exemplar (structurally inspired by google/agents-cli), mechanically substitutes API-specific content, and verifies via a deterministic validation gate.",
   "author": {
     "name": "codeyogi911",

examples/exemplar-cli/.clify.json

@@ -26,5 +26,8 @@
     "totalParsed": 11,
     "totalIncluded": 11,
     "totalDropped": 0
-  }
+  },
+  "filterProbes": [
+    { "resource": "orders", "filter": "status", "baselineCount": 5, "filteredCount": 1, "status": "verified", "note": "verified against the bundled mock server in test/integration.test.mjs" }
+  ]
 }

examples/exemplar-cli/bin/exemplar-cli.mjs

@@ -11,7 +11,7 @@ import { dirname, join, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 
 import { loadEnv } from "../lib/env.mjs";
-import { splitGlobal, hasHelp, toParseArgs, checkRequired } from "../lib/args.mjs";
+import { splitGlobal, hasHelp, toParseArgs, checkRequired, checkEnum } from "../lib/args.mjs";
 import { output, errorOut } from "../lib/output.mjs";
 import { apiRequest, paginate } from "../lib/api.mjs";
 import { showRootHelp, showResourceHelp, showActionHelp } from "../lib/help.mjs";
@@ -73,6 +73,12 @@ async function runResourceAction(resourceArg, actionArg, remaining, global, rest
   const missing = checkRequired(parsed.values, def.flags);
   if (missing.length) errorOut("validation_error", `Missing required flag(s): ${missing.map((m) => `--${m}`).join(", ")}`);
 
+  const enumViolations = checkEnum(parsed.values, def.flags);
+  if (enumViolations.length) {
+    const v = enumViolations[0];
+    errorOut("validation_error", `--${v.flag}=${v.value} is not allowed; expected one of: ${v.allowed.join(", ")}`);
+  }
+
   const path = interpolatePath(def.path, parsed.values);
   const buildPayload = PAYLOAD_BUILDERS[resourceArg];
 

examples/exemplar-cli/knowledge/query-flags-and-broken-list-filters.md

@@ -1,7 +1,7 @@
 ---
 type: contract
-source: clify v0.5
-applies-to: ["bin/<api>-cli.mjs", "commands/*.mjs", "lib/quirks.mjs"]
+source: clify v0.6
+applies-to: ["bin/<api>-cli.mjs", "commands/*.mjs", "lib/quirks.mjs", "lib/help.mjs", "lib/args.mjs"]
 ---
 
 # `queryFlags` and `brokenListFilters` action annotations
@@ -45,40 +45,99 @@ The runtime in `bin/<api>-cli.mjs`:
 
 Some `GET …/list` endpoints accept filter query parameters that the server
 silently ignores: `200 OK`, no error header, full unfiltered list returned.
-Detection requires comparing baseline (no filter) and `FAKE-NONEXISTENT`
-(impossible value) row counts — equal counts means the filter is broken.
+**Detection requires per-flag probing** — running one probe with the
+wrong name and blanket-marking the rest is the v0.5 anti-pattern this
+contract exists to prevent.
+
+### Discovery checklist (per list endpoint)
+
+When generating or auditing a CLI:
+
+1. **Read the docs page properly.** Every documented query-parameter
+   becomes its own flag, **using the documented name verbatim**:
+   `--customer_name_startswith`, `--customer_name_contains`,
+   `--reference_number_startswith`, `--filter_by`, `--date_start`,
+   `--date_end`. Do NOT collapse variants — the suffix (`_startswith`,
+   `_contains`) is the server-side match mode and is not optional.
+2. **Surface enum values.** When the docs say a param accepts an enum
+   (e.g. `filter_by: All|NotShipped|Shipped|Delivered`), set
+   `flags.<name>.enum: [...]` AND mention the allowed values in
+   `description`. The exemplar's help generator renders inline:
+   `--filter_by <All|NotShipped|Shipped|Delivered>`. The bin runtime
+   rejects out-of-enum values with a `validation_error`.
+3. **Auto-emit `sort_column` + `sort_order`** for any list endpoint
+   whose docs mention sorting. Most paginated APIs (Zoho, Shopify,
+   Stripe-style) accept these even when not tabled.
+4. **Per-flag probe** (when the API can be reached at generation time):
+   - Make one unfiltered call → record `baselineCount` (response row count).
+   - For each filter, make one call with a value sampled from the
+     unfiltered response (or the first enum value).
+   - Record `filteredCount`.
+   - Status:
+     - `filteredCount < baselineCount` → `verified` (server honored it).
+     - `filteredCount === baselineCount` → `broken` (server ignored it).
+     - probe failed (no creds, network blocked, rate limit) → `untested`.
+       Leave the flag working; do NOT add to `brokenListFilters`.
+5. **Write the probe log to `.clify.json.filterProbes`** — one entry per
+   filter probed (or skipped). The validator's `filter-coverage` check
+   reads this:
+   - List action declares filter-shaped flags but `filterProbes` has
+     zero entries for that resource → HARD-FAIL.
+   - Every filter on a list action is in `brokenListFilters` AND
+     `filterProbes` shows no individual probes → HARD-FAIL (the
+     blanket-mark anti-pattern).
+   - Filter is `untested` → WARN, not fail.
+
+For each filter the probe marked `broken`, declare it in
+`brokenListFilters` on the action def. The v0.6+ form takes
+per-filter `match` modes so the client-side fallback mirrors the
+documented match semantics:
 
 ```js
 list: {
   method: "GET",
   path: "/orders",
-  flags: { customerId: { type: "string", description: "Filter (BROKEN upstream — client-side fallback)" } },
-  brokenListFilters: ["customerId"],
+  flags: {
+    customer_name_startswith: { type: "string", description: "Filter by customer name prefix (BROKEN upstream — client-side fallback)" },
+    filter_by: { type: "string", enum: ["All", "Shipped", "NotShipped"], description: "Status filter (BROKEN upstream)" },
+  },
+  brokenListFilters: [
+    { name: "customer_name_startswith", match: "startswith" },
+    { name: "filter_by", match: "equals" },
+  ],
 }
 ```
 
+(The legacy v0.5 string-list form `brokenListFilters: ["customer_name"]` is
+still accepted and defaults to `match: "equals"` — but new generations
+should use the object form so the fallback semantics match the docs.)
+
 Runtime behavior when the user passes a broken filter:
 1. Strip the filter from the wire query.
 2. Pull the full list via cursor pagination.
-3. Filter client-side (case-insensitive equals OR substring) on the
-   row's same-named field.
+3. Filter client-side with the per-filter `match` mode (`equals`,
+   `startswith`, `contains`).
 4. Write a one-line `note: …` to stderr.
 
 Cost goes up to the full-list response size. For small datasets this is
 trivial; for larger ones, callers should pull once and filter themselves.
 
-## Discovery checklist
-
-When generating or auditing a CLI:
-
-1. For every documented `POST <resource>` create — does the API doc list a
-   foreign-key parameter under "Query parameters" (not "Body parameters")?
-   If yes, add it to `queryFlags`. (Common APIs that do this:
-   Zoho Inventory/Books, Zoho CRM, Xero, Razorpay X.)
-2. For every list endpoint — pick one declared filter and compare the
-   row count with `--<filter> FAKE-NONEXISTENT-XYZ` against an
-   unfiltered call. Equal counts → broken; declare in `brokenListFilters`
-   AND clarify the help text.
+### Worked example: Zoho Inventory `packages list`
+
+Docs declare nine query parameters: `filter_by` (enum), `customer_name_startswith`,
+`customer_name_contains`, `reference_number_startswith`, `reference_number_contains`,
+`date_start`, `date_end`, `sort_column`, `sort_order`. The pre-v0.6 generator
+collapsed these into a bare `--status`, probed once, saw the API ignore `status`,
+and marked all nine as broken.
+
+Correct generation:
+- Emit each flag verbatim.
+- Probe `--filter_by Shipped` → `filteredCount=83 vs baselineCount=319` → `verified`.
+- Probe `--customer_name_startswith Acme` → `filteredCount=12 vs 319` → `verified`.
+- Probe `--status Shipped` (if mistakenly emitted) → `filteredCount=319` → `broken`.
+- `.clify.json.filterProbes` records all nine — verified or broken or untested.
+- Only the genuinely-broken ones land in `brokenListFilters` with the right
+  `match` mode.
 
 A new clify version should re-run these probes whenever the upstream
 docs change (`clify sync-check` flags doc drift; the manual probe is the

examples/exemplar-cli/lib/args.mjs

@@ -37,3 +37,17 @@ export function checkRequired(values, flagSpec) {
   }
   return missing;
 }
+
+// For flags that declare `enum: [...]`, reject values that aren't in the
+// allowed set. Returns a list of `{flag, value, allowed}` for each
+// violator (empty list = no violations). Skips flags that weren't passed.
+export function checkEnum(values, flagSpec) {
+  const violations = [];
+  for (const [name, spec] of Object.entries(flagSpec)) {
+    if (!Array.isArray(spec.enum) || spec.enum.length === 0) continue;
+    const v = values[name];
+    if (v === undefined || v === "") continue;
+    if (!spec.enum.includes(v)) violations.push({ flag: name, value: v, allowed: spec.enum });
+  }
+  return violations;
+}

examples/exemplar-cli/lib/help.mjs

@@ -30,8 +30,13 @@ export function showActionHelp(resource, action, registry) {
   if (entries.length === 0) out += `  (none)\n`;
   for (const [name, spec] of entries) {
     const req = spec.required ? "required" : "optional";
+    // Render enum values inline so users see allowed values without
+    // bouncing to the docs: `--filter_by <All|NotShipped|Shipped>`.
+    const flagLabel = Array.isArray(spec.enum) && spec.enum.length
+      ? `--${name} <${spec.enum.join("|")}>`
+      : `--${name}`;
     const desc = spec.description || "";
-    out += `  --${name.padEnd(20)} ${spec.type.padEnd(8)} ${req.padEnd(8)} ${desc}\n`;
+    out += `  ${flagLabel.padEnd(22)} ${spec.type.padEnd(8)} ${req.padEnd(8)} ${desc}\n`;
   }
   return out;
 }

examples/exemplar-cli/lib/quirks.mjs

@@ -26,34 +26,64 @@ export function stripQueryFlags(values, def) {
   return out;
 }
 
+// `brokenListFilters` may be either a flat list of flag names (legacy v0.5
+// shape) OR a list of `{ name, match }` objects (v0.6+, per-flag match mode).
+// Normalize both into `{ name, match }`. Default match is "equals".
+function normalizeBrokenFilters(def) {
+  if (!def?.brokenListFilters?.length) return [];
+  return def.brokenListFilters.map((entry) =>
+    typeof entry === "string"
+      ? { name: entry, match: "equals" }
+      : { name: entry.name, match: entry.match || "equals" },
+  );
+}
+
 // Detect which of `def.brokenListFilters` the user populated on this call.
-// Returns a {flag: value} map (empty when none). Callers should drop these
-// from the wire query, fetch the full list, and run `clientFilter` on the
-// result.
+// Returns a `{ flag: { value, match } }` map (empty when none). Callers
+// should drop these from the wire query, fetch the full list, and run
+// `clientFilter` on the result.
 export function pickBrokenFilters(values, def) {
-  if (!def?.brokenListFilters?.length) return {};
   const out = {};
-  for (const k of def.brokenListFilters) {
-    const v = values[k];
-    if (v !== undefined && v !== "") out[k] = String(v);
+  for (const { name, match } of normalizeBrokenFilters(def)) {
+    const v = values[name];
+    if (v !== undefined && v !== "") out[name] = { value: String(v), match };
   }
   return out;
 }
 
 // Client-side fallback for list endpoints whose upstream-API filter is
-// silently ignored. Compares each requested value against the row's
-// same-named field with case-insensitive equality OR substring match (the
-// substring covers prefix-style number lookups users typically want).
+// silently ignored. Per-filter match modes:
+//   - "equals"     — case-insensitive exact equality.
+//   - "startswith" — case-insensitive prefix match.
+//   - "contains"   — case-insensitive substring match.
+// `filters` is the `{ name: { value, match } }` map from pickBrokenFilters.
+// Backwards-compatible: callers passing the legacy `{ name: value }` shape
+// are coerced to `{ value, match: "equals-or-contains" }` for the v0.5
+// hybrid behavior (case-insensitive equals OR substring).
 export function clientFilter(items, filters) {
   const checks = Object.entries(filters);
   if (!checks.length) return items;
+  const normalized = checks.map(([k, raw]) => {
+    if (raw && typeof raw === "object" && "value" in raw) return [k, raw];
+    return [k, { value: raw, match: "equals-or-contains" }];
+  });
   return items.filter((row) =>
-    checks.every(([k, target]) => {
+    normalized.every(([k, { value, match }]) => {
       const v = row?.[k];
       if (v === undefined || v === null) return false;
       const a = String(v).toLowerCase();
-      const b = String(target).toLowerCase();
-      return a === b || a.includes(b);
+      const b = String(value).toLowerCase();
+      switch (match) {
+        case "equals":
+          return a === b;
+        case "startswith":
+          return a.startsWith(b);
+        case "contains":
+          return a.includes(b);
+        case "equals-or-contains":
+        default:
+          return a === b || a.includes(b);
+      }
     }),
   );
 }

examples/exemplar-cli/test/integration.test.mjs

@@ -139,6 +139,46 @@ test("orders list filters by status query", async () => {
   });
 });
 
+// ---------- filter-probe pattern (v0.6+) ----------
+//
+// This is the canonical test pattern any generated CLI inherits — it
+// proves a declared list-filter flag actually filters server-side.
+// Catches the Zoho `filter_by` miss the v0.6 generator-side fix targets:
+// pre-v0.6, generated CLIs would declare `--status` on a list endpoint,
+// then mark the WHOLE filter set as broken when one probe returned the
+// unfiltered count. With per-flag probing, the verified-server-side
+// filter (this test) and the broken-fallback (unit-tested in quirks.test.mjs)
+// stay distinct, and `.clify.json.filterProbes` records the truth.
+test("filter-probe pattern: declared --status flag filters server-side (verified)", async () => {
+  // Mock returns 5 rows unfiltered, 2 rows when ?status=shipped.
+  const allRows = [
+    { id: "o1", status: "pending" },
+    { id: "o2", status: "shipped" },
+    { id: "o3", status: "shipped" },
+    { id: "o4", status: "cancelled" },
+    { id: "o5", status: "pending" },
+  ];
+  await withMock({
+    "GET /orders": (req) => {
+      const filter = req.query.status;
+      const rows = filter ? allRows.filter((r) => r.status === filter) : allRows;
+      return { status: 200, body: { items: rows, nextCursor: null } };
+    },
+  }, async (server) => {
+    // Baseline (unfiltered).
+    const baseline = await runJson(["orders", "list"], { env: { ...ENV, EXEMPLAR_BASE_URL: server.url } });
+    assert.equal(baseline.exitCode, 0, baseline.stderr);
+    assert.equal(baseline.json.items.length, 5, "baseline returns the full list");
+
+    // Filtered call — must (a) put status on the wire, (b) return fewer rows.
+    const filtered = await runJson(["orders", "list", "--status", "shipped"], { env: { ...ENV, EXEMPLAR_BASE_URL: server.url } });
+    assert.equal(filtered.exitCode, 0, filtered.stderr);
+    assert.equal(filtered.json.items.length, 2, "server-side filter took effect");
+    const filteredReq = server.requests[server.requests.length - 1];
+    assert.equal(filteredReq.query.status, "shipped", "wire URL contained ?status=shipped (server-side filter, not client-side fallback)");
+  });
+});
+
 test("orders upload posts multipart/form-data with --file", async () => {
   const tmp = mkdtempSync(join(tmpdir(), "exemplar-mp-"));
   const filePath = join(tmp, "receipt.txt");