diff --git a/COMMANDS.md b/COMMANDS.md index 7f141ba..183ed28 100644 --- a/COMMANDS.md +++ b/COMMANDS.md @@ -641,10 +641,15 @@ altertable query show|cancel | Option | Description | | --- | --- | -| `--format ` | Serialized output format; use global --json for JSON Values: csv, markdown. | +| `--format ` | Result format: csv, jsonl, and parquet stream from the API; markdown is rendered by the CLI. Use global --json for structured JSON Values: csv, jsonl, parquet, markdown. | | `--layout ` | Human layout: auto, table, or line Values: auto, table, line. | | `--columns ` | Comma-separated columns to show | | `--max-width ` | Maximum display width for table columns Default: "32". | +| `--compute-size ` | Compute size for the query (AUTO cannot be combined with --session-id) Values: XS, S, M, L, XL, AUTO. Default: "AUTO". | +| `--dialect ` | Source SQL dialect to transpile from (server default: DuckDB) | +| `--catalog ` | Catalog name (optional; can also come from the session) | +| `--schema ` | Schema name (optional; can also come from the session) | +| `--output ` | Write result bytes/text to this path instead of stdout | | `--query-id ` | Optional stable query id | | `--session-id ` | Optional session id | | `--pager ` | Pager mode for human output: auto, always, or never Values: auto, always, never. Default: "auto". | @@ -659,6 +664,7 @@ altertable query show|cancel ```bash altertable query "SELECT id, email, plan FROM analytics.main.users LIMIT 10" altertable query "SELECT event, timestamp FROM analytics.main.events ORDER BY timestamp DESC LIMIT 10" --json +altertable query "SELECT 1" --format csv --output results.csv altertable query show ``` diff --git a/README.md b/README.md index bf9ecf2..98ab1dd 100644 --- a/README.md +++ b/README.md @@ -331,10 +331,16 @@ altertable query "SELECT event, user_id, timestamp FROM analytics.main.events OR altertable query "SELECT * FROM analytics.main.events ORDER BY timestamp DESC LIMIT 10" --columns event,user_id,timestamp altertable query "SELECT * FROM analytics.main.events ORDER BY timestamp DESC LIMIT 10" --max-width 24 -# Serialized output +# Serialized / API-native output altertable query "SELECT id, email, plan FROM analytics.main.users LIMIT 100" --format csv -altertable query "SELECT id, email, plan FROM analytics.main.users LIMIT 100" --json +altertable query "SELECT id, email, plan FROM analytics.main.users LIMIT 100" --format jsonl --output events.jsonl +altertable query "SELECT id, email, plan FROM analytics.main.users LIMIT 100" --format parquet --output users.parquet altertable query "SELECT id, email, plan FROM analytics.main.users LIMIT 100" --format markdown +altertable query "SELECT id, email, plan FROM analytics.main.users LIMIT 100" --json + +# Request options +altertable query "SELECT 1" --compute-size M +altertable query "SELECT * FROM users" --catalog analytics --schema main --dialect snowflake # Long results — pipe through a pager altertable query "SELECT * FROM analytics.main.orders ORDER BY created_at DESC" --pager always @@ -347,7 +353,7 @@ altertable --json query "SELECT 1" altertable --agent query "SELECT 1" ``` -Human output is the default and respects `--layout auto|table|line`, `--columns`, `--max-width`, and `--pager auto|always|never`. Use `--format csv|markdown` for serialized text, or the global `--json`/`--agent` flags for structured JSON. Serialized output skips pager and layout controls. +Human output is the default and respects `--layout auto|table|line`, `--columns`, `--max-width`, and `--pager auto|always|never`. Use `--format csv|jsonl|parquet` to stream those encodings from the lakehouse API, `--format markdown` for CLI-rendered markdown, or the global `--json`/`--agent` flags for structured JSON. API-native formats cannot be combined with `--json`/`--agent` or layout/pager flags. `--output` writes to a file (stdout by default). `--compute-size` defaults to `AUTO` (omit with `--session-id` unless you set `XS|S|M|L|XL`). Set display defaults in `~/.config/altertable/config`: diff --git a/cli/src/commands/completion/lib/spec.test.ts b/cli/src/commands/completion/lib/spec.test.ts index 10e129d..b3cce90 100644 --- a/cli/src/commands/completion/lib/spec.test.ts +++ b/cli/src/commands/completion/lib/spec.test.ts @@ -273,10 +273,10 @@ describe("buildCompletionSpec", () => { ]); }); - test("extracts intentional direct and subcommand operand collisions", async () => { + test("exposes empty soleDirectOperands when query has no operand collisions", async () => { const spec = await buildCompletionSpec(buildMainCommand()); - expect(findNode(spec, "query")?.soleDirectOperands).toEqual(["show"]); + expect(findNode(spec, "query")?.soleDirectOperands).toEqual([]); }); test("extracts finite shell positional values from completion commands", async () => { @@ -339,16 +339,16 @@ describe("normalized completion argv", () => { path: ["query", "show"], positionals: ["qry_123"], }, - { - argv: ["query", "show", "--layout", "table"], - path: ["query"], - positionals: ["show"], - }, { argv: ["query", "show", "--help"], path: ["query", "show"], positionals: [], }, + { + argv: ["query", "SELECT 1", "--layout", "table"], + path: ["query"], + positionals: ["SELECT 1"], + }, { argv: ["append", '{"event":"checkout"}', "--sync"], path: ["append"], @@ -536,8 +536,8 @@ describe("executable shell completion contract", () => { expect(candidates).not.toContain("--layout"); }); - shellTest(`${shell} preserves intentional direct operands with trailing flags`, async () => { - const direct = await runCompletion([ + shellTest(`${shell} treats query show as a subcommand, not a SQL direct operand`, async () => { + const showLeaf = await runCompletion([ "altertable", "query", "show", @@ -545,8 +545,12 @@ describe("executable shell completion contract", () => { "table", "--", ]); - expect(direct).toContain("--columns"); - expect(direct).toContain("--layout"); + expect(showLeaf).not.toContain("--columns"); + expect(showLeaf).not.toContain("--layout"); + + const directSql = await runCompletion(["altertable", "query", "SELECT 1", "--"]); + expect(directSql).toContain("--columns"); + expect(directSql).toContain("--layout"); const nested = await runCompletion(["altertable", "query", "show", "qry_1", "--"]); expect(nested).not.toContain("--columns"); diff --git a/cli/src/commands/doctor/lib/runner.test.ts b/cli/src/commands/doctor/lib/runner.test.ts index 20de479..a30d3c6 100644 --- a/cli/src/commands/doctor/lib/runner.test.ts +++ b/cli/src/commands/doctor/lib/runner.test.ts @@ -15,6 +15,7 @@ function createDoctorContext(offline = false): DoctorCheckContext { writeStderr() {}, writeJson() {}, writeRaw() {}, + writeBytes() {}, writeHuman() {}, writeMetadata() {}, }, diff --git a/cli/src/commands/query/index.test.ts b/cli/src/commands/query/index.test.ts index df1e28d..eb8fe3b 100644 --- a/cli/src/commands/query/index.test.ts +++ b/cli/src/commands/query/index.test.ts @@ -1,4 +1,7 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { runCommandWithTestRuntime } from "@/test-utils/cli.ts"; import { createLakehouseTestWorkspace, @@ -15,7 +18,7 @@ beforeEach(() => { afterEach(() => workspace.cleanup()); describe("query command", () => { - test("sends the statement and optional identifiers", async () => { + test("sends statement identifiers and default compute size", async () => { workspace.writeMocks([{ urlPattern: "/query", method: "POST", body: QUERY_RESPONSE }]); await runCommandWithTestRuntime([ @@ -23,17 +26,125 @@ describe("query command", () => { "SELECT 1", "--query-id", "query-1", - "--session-id", - "session-1", + "--dialect", + "snowflake", + "--catalog", + "analytics", + "--schema", + "main", ]); expect(JSON.parse(workspace.readPayloads()[0] ?? "")).toEqual({ statement: "SELECT 1", query_id: "query-1", + compute_size: "AUTO", + dialect: "snowflake", + catalog: "analytics", + schema: "main", + }); + }); + + test("omits compute size with session unless an explicit size is set", async () => { + workspace.writeMocks([ + { urlPattern: "/query", method: "POST", body: QUERY_RESPONSE }, + { urlPattern: "/query", method: "POST", body: QUERY_RESPONSE }, + ]); + + await runCommandWithTestRuntime(["query", "SELECT 1", "--session-id", "session-1"]); + expect(JSON.parse(workspace.readPayloads()[0] ?? "")).toEqual({ + statement: "SELECT 1", + session_id: "session-1", + }); + + await runCommandWithTestRuntime([ + "query", + "SELECT 1", + "--session-id", + "session-1", + "--compute-size", + "S", + ]); + expect(JSON.parse(workspace.readPayloads()[1] ?? "")).toEqual({ + statement: "SELECT 1", session_id: "session-1", + compute_size: "S", }); }); + test("rejects explicit AUTO compute size with a session", async () => { + expect( + runCommandWithTestRuntime([ + "query", + "SELECT 1", + "--session-id", + "session-1", + "--compute-size", + "AUTO", + ]), + ).rejects.toThrow("--compute-size AUTO cannot be combined with --session-id."); + }); + + test("streams API-native csv bytes instead of client-rendered CSV", async () => { + const apiCsv = "id,name\n9,from-api\n"; + workspace.writeMocks([{ urlPattern: "/query", method: "POST", body: apiCsv }]); + + const harness = await runCommandWithTestRuntime(["query", "SELECT 1", "--format", "csv"], { + debug: false, + json: false, + agent: false, + }); + + expect(JSON.parse(workspace.readPayloads()[0] ?? "")).toEqual({ + statement: "SELECT 1", + compute_size: "AUTO", + format: "csv", + }); + expect(harness.stdout.join("")).toContain("9,from-api"); + }); + + test("writes API-native output to --output", async () => { + const outputDir = mkdtempSync(join(tmpdir(), "altertable-query-output-")); + const outputPath = join(outputDir, "result.csv"); + try { + workspace.writeMocks([{ urlPattern: "/query", method: "POST", body: "id\n42\n" }]); + + await runCommandWithTestRuntime( + ["query", "SELECT 1", "--format", "csv", "--output", outputPath], + { debug: false, json: false, agent: false }, + ); + + expect(readFileSync(outputPath, "utf8")).toContain("42"); + } finally { + rmSync(outputDir, { recursive: true, force: true }); + } + }); + + test("keeps markdown as client-rendered NDJSON output", async () => { + workspace.writeMocks([{ urlPattern: "/query", method: "POST", body: QUERY_RESPONSE }]); + + const harness = await runCommandWithTestRuntime(["query", "SELECT 1", "--format", "markdown"], { + debug: false, + json: false, + agent: false, + }); + + expect(JSON.parse(workspace.readPayloads()[0] ?? "")).toEqual({ + statement: "SELECT 1", + compute_size: "AUTO", + }); + expect(harness.stdout.join("")).toContain("|"); + }); + + test("rejects API-native format with --json", async () => { + expect( + runCommandWithTestRuntime(["query", "SELECT 1", "--format", "parquet"], { + debug: false, + json: true, + agent: false, + }), + ).rejects.toThrow("cannot be combined with --json or --agent"); + }); + test("dispatches show and cancel without run arguments", async () => { const queryId = "11111111-2222-3333-4444-555555555555"; workspace.writeMocks([ @@ -51,14 +162,10 @@ describe("query command", () => { ); }); - test("executes a bare lowercase show statement as SQL", async () => { - workspace.writeMocks([{ urlPattern: "/query", method: "POST", body: QUERY_RESPONSE }]); - - await runCommandWithTestRuntime(["query", "show"]); - - expect(JSON.parse(workspace.readPayloads()[0] ?? "")).toEqual({ - statement: "show", - }); + test("requires a query id for show instead of running bare show as SQL", async () => { + expect(runCommandWithTestRuntime(["query", "show"])).rejects.toThrow( + "Missing required argument: query-id.", + ); }); test("URL-encodes query identifiers", async () => { diff --git a/cli/src/commands/query/index.ts b/cli/src/commands/query/index.ts index d30295d..e5f731d 100644 --- a/cli/src/commands/query/index.ts +++ b/cli/src/commands/query/index.ts @@ -4,9 +4,14 @@ import { queryShowCommand } from "@/commands/query/show.ts"; import { queryCancelCommand } from "@/commands/query/cancel.ts"; import { CliError } from "@/lib/errors.ts"; import { optionalStringArg } from "@/lib/args.ts"; -import { writeQueryOutput } from "@/lib/query-output.ts"; +import { writeQueryDestination } from "@/lib/query-destination.ts"; +import { isApiNativeQueryFormat, writeQueryOutput } from "@/lib/query-output.ts"; import { parseQueryOutputOptions } from "@/lib/query-output-args.ts"; -import { executeLakehouseQuery } from "@/lib/lakehouse/query.ts"; +import { + executeLakehouseQuery, + executeLakehouseQueryBytes, + type LakehouseQueryInput, +} from "@/lib/lakehouse/query.ts"; export const queryCommand = defineCommand({ metadata: { @@ -16,11 +21,11 @@ export const queryCommand = defineCommand({ examples: [ 'altertable query "SELECT id, email, plan FROM analytics.main.users LIMIT 10"', 'altertable query "SELECT event, timestamp FROM analytics.main.events ORDER BY timestamp DESC LIMIT 10" --json', + 'altertable query "SELECT 1" --format csv --output results.csv', "altertable query show ", ], }, args: queryRunArgs, - soleDirectOperands: ["show"], subcommands: { show: queryShowCommand, cancel: queryCancelCommand, @@ -30,16 +35,38 @@ export const queryCommand = defineCommand({ if (statement === undefined) { throw new CliError('Provide a SQL statement, e.g. altertable query "SELECT 1".'); } - const { format, displayOptions, pagerOptions } = parseQueryOutputOptions(args, { + const { + format, + displayOptions, + pagerOptions, + outputPath, + computeSize, + dialect, + catalog, + schema, + } = parseQueryOutputOptions(args, { agent: execution.cli.agent, + json: sink.json, rawArgs, }); - const input = { + + const input: LakehouseQueryInput = { statement, queryId: optionalStringArg(args, "query-id"), sessionId: optionalStringArg(args, "session-id"), + computeSize, + dialect, + catalog, + schema, }; + + if (isApiNativeQueryFormat(format)) { + const stream = await executeLakehouseQueryBytes({ ...input, format }, execution); + await writeQueryDestination(stream, { outputPath, sink }); + return; + } + const result = await executeLakehouseQuery(input, execution, !sink.json); - await writeQueryOutput(result, format, sink, displayOptions, pagerOptions); + await writeQueryOutput(result, format, sink, displayOptions, pagerOptions, outputPath); }, }); diff --git a/cli/src/commands/query/lib/args.ts b/cli/src/commands/query/lib/args.ts index 114e5a4..a7a9bba 100644 --- a/cli/src/commands/query/lib/args.ts +++ b/cli/src/commands/query/lib/args.ts @@ -1,8 +1,9 @@ import { defineArguments } from "@/lib/command.ts"; import { + queryApiResultFormatArgs, queryDisplayArgs, queryPagerArgs, - queryResultFormatArgs, + queryRequestArgs, } from "@/lib/query-output-args.ts"; export const queryRunArgs = defineArguments({ @@ -12,8 +13,9 @@ export const queryRunArgs = defineArguments({ required: false, directRequired: true, }, - ...queryResultFormatArgs, + ...queryApiResultFormatArgs, ...queryDisplayArgs, + ...queryRequestArgs, "query-id": { type: "string", description: "Optional stable query id" }, "session-id": { type: "string", description: "Optional session id" }, ...queryPagerArgs, diff --git a/cli/src/commands/schema/index.ts b/cli/src/commands/schema/index.ts index b208fb8..de45057 100644 --- a/cli/src/commands/schema/index.ts +++ b/cli/src/commands/schema/index.ts @@ -20,6 +20,7 @@ export const schemaCommand = defineCommand({ const catalog = stringArg(args, "catalog"); const { format, displayOptions, pagerOptions } = parseQueryOutputOptions(args, { agent: execution.cli.agent, + json: sink.json, rawArgs, }); const queryInput = { diff --git a/cli/src/lib/command-descriptor.test.ts b/cli/src/lib/command-descriptor.test.ts index 57c9c9a..4b7e536 100644 --- a/cli/src/lib/command-descriptor.test.ts +++ b/cli/src/lib/command-descriptor.test.ts @@ -275,11 +275,11 @@ describe("command descriptor", () => { ); }); - test("describes intentional direct and subcommand operand collisions", async () => { + test("describes query show as a subcommand without sole-direct collisions", async () => { const root = await resolveCommandDescriptor(buildMainCommand()); const query = root.subcommands.find((command) => command.key === "query"); - expect(query?.soleDirectOperands).toEqual(["show"]); + expect(query?.soleDirectOperands).toEqual([]); expect(query?.subcommands.map((command) => command.metadata.name)).toContain("show"); }); diff --git a/cli/src/lib/command.ts b/cli/src/lib/command.ts index 606fc4d..f8708aa 100644 --- a/cli/src/lib/command.ts +++ b/cli/src/lib/command.ts @@ -55,8 +55,8 @@ export type CommandDefinition = { subcommands?: Resolvable>>; /** * Values that select direct execution only when they are the command's sole - * positional operand. This resolves intentional command/subcommand ambiguity, - * such as `query show` (SQL) versus `query show `. + * positional operand. Use this for intentional parent/subcommand name collisions + * (for example a keyword that is both a statement and a subcommand name). */ soleDirectOperands?: readonly string[]; run?: (context: CommandRunContext) => void | Promise; diff --git a/cli/src/lib/lakehouse/query.test.ts b/cli/src/lib/lakehouse/query.test.ts index 9b17c2a..ee44352 100644 --- a/cli/src/lib/lakehouse/query.test.ts +++ b/cli/src/lib/lakehouse/query.test.ts @@ -18,7 +18,11 @@ import { import { getQueryColumnNames } from "@/lib/query-format.ts"; import { httpSendStream } from "@/lib/http.ts"; import { createExecutionContext } from "@/lib/execution-context.ts"; -import { executeLakehouseQuery } from "@/lib/lakehouse/query.ts"; +import { + buildLakehouseQueryRequest, + executeLakehouseQuery, + executeLakehouseQueryBytes, +} from "@/lib/lakehouse/query.ts"; import { getCliRuntime, refreshCliRuntimeContext } from "@/lib/runtime.ts"; const SAMPLE_NDJSON = [ @@ -245,6 +249,36 @@ describe("parseLakehouseQueryStream", () => { }); }); +describe("buildLakehouseQueryRequest", () => { + test("includes optional QueryRequest fields in the JSON body", () => { + const request = buildLakehouseQueryRequest( + { + statement: "SELECT 1", + queryId: "query-1", + sessionId: "session-1", + computeSize: "M", + format: "csv", + dialect: "snowflake", + catalog: "analytics", + schema: "main", + }, + false, + ); + + expect(typeof request.body).toBe("string"); + expect(JSON.parse(request.body as string)).toEqual({ + statement: "SELECT 1", + query_id: "query-1", + session_id: "session-1", + compute_size: "M", + format: "csv", + dialect: "snowflake", + catalog: "analytics", + schema: "main", + }); + }); +}); + describe("executeLakehouseQuery", () => { test("returns the same result as parseLakehouseQueryResponse", async () => { writeFileSync( @@ -271,6 +305,36 @@ describe("executeLakehouseQuery", () => { }); }); +describe("executeLakehouseQueryBytes", () => { + test("streams raw response bytes for API-native formats", async () => { + const csvBody = "id,name\n1,Alice\n"; + writeFileSync( + mockFile, + JSON.stringify([ + { + urlPattern: "/query", + method: "POST", + chunked: true, + body: csvBody, + }, + ]), + ); + + const runtime = getCliRuntime(); + const stream = await executeLakehouseQueryBytes( + { statement: "SELECT 1", format: "csv", computeSize: "AUTO" }, + createExecutionContext(runtime), + ); + const chunks: Uint8Array[] = []; + for await (const chunk of stream) { + chunks.push(chunk); + } + const body = Buffer.concat(chunks).toString("utf8"); + expect(body).toContain("id,name"); + expect(body).toContain("1,Alice"); + }); +}); + describe("query renderers", () => { const parsedResult = parseLakehouseQueryResponse(SAMPLE_NDJSON); diff --git a/cli/src/lib/lakehouse/query.ts b/cli/src/lib/lakehouse/query.ts index e030371..44fa931 100644 --- a/cli/src/lib/lakehouse/query.ts +++ b/cli/src/lib/lakehouse/query.ts @@ -7,10 +7,17 @@ import { import { sendHttp, sendHttpStream, type HttpRequest } from "@/lib/http-request.ts"; import { STREAM_READ_TIMEOUT_MS } from "@/lib/transport-defaults.ts"; +export type LakehouseApiQueryFormat = "csv" | "jsonl" | "parquet"; + export type LakehouseQueryInput = { statement: string; queryId?: string; sessionId?: string; + computeSize?: string; + format?: LakehouseApiQueryFormat; + dialect?: string; + catalog?: string; + schema?: string; httpOptions?: { readTimeoutMs?: number }; }; @@ -23,6 +30,11 @@ function buildQueryPayload(input: LakehouseQueryInput): Record { const payload: Record = { statement: input.statement }; if (input.queryId) payload.query_id = input.queryId; if (input.sessionId) payload.session_id = input.sessionId; + if (input.computeSize) payload.compute_size = input.computeSize; + if (input.format) payload.format = input.format; + if (input.dialect) payload.dialect = input.dialect; + if (input.catalog) payload.catalog = input.catalog; + if (input.schema) payload.schema = input.schema; return payload; } @@ -68,6 +80,13 @@ export async function executeLakehouseQuery( return parseLakehouseQueryResponse(await sendHttp(request, execution)); } +export async function executeLakehouseQueryBytes( + input: LakehouseQueryInput, + execution: ExecutionContext, +): Promise> { + return sendHttpStream(buildLakehouseQueryRequest(input, true), execution); +} + export function buildLakehouseQueryShowRequest(queryId: string): HttpRequest { return { plane: "lakehouse", diff --git a/cli/src/lib/query-destination.test.ts b/cli/src/lib/query-destination.test.ts new file mode 100644 index 0000000..1ea80ec --- /dev/null +++ b/cli/src/lib/query-destination.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, test } from "bun:test"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { writeQueryDestination } from "@/lib/query-destination.ts"; +import { createCliRuntime } from "@/lib/runtime.ts"; + +describe("writeQueryDestination", () => { + test("writes opaque stdout bytes without UTF-8 round-trip or trailing newline", async () => { + const runtime = createCliRuntime({ debug: false, json: false, agent: false }); + const written: Uint8Array[] = []; + runtime.output.writeBytes = (body) => { + written.push(body); + }; + + const parquetLike = new Uint8Array([0x50, 0x41, 0x52, 0x31, 0x00, 0xff]); + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(parquetLike); + controller.close(); + }, + }); + + await writeQueryDestination(stream, { sink: runtime.output }); + + expect(written).toHaveLength(1); + expect(written[0]).toEqual(parquetLike); + }); + + test("writes opaque file bytes unchanged", async () => { + const dir = mkdtempSync(join(tmpdir(), "altertable-query-dest-")); + const path = join(dir, "out.parquet"); + try { + const parquetLike = new Uint8Array([0x50, 0x41, 0x52, 0x31, 0x00, 0xff]); + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(parquetLike); + controller.close(); + }, + }); + + await writeQueryDestination(stream, { outputPath: path }); + expect(new Uint8Array(readFileSync(path))).toEqual(parquetLike); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/cli/src/lib/query-destination.ts b/cli/src/lib/query-destination.ts new file mode 100644 index 0000000..50a7739 --- /dev/null +++ b/cli/src/lib/query-destination.ts @@ -0,0 +1,33 @@ +import type { OutputSink } from "@/lib/runtime.ts"; + +export type WriteQueryDestinationOptions = { + outputPath?: string; + sink?: OutputSink; +}; + +async function collectStreamBytes(stream: ReadableStream): Promise { + return new Uint8Array(await new Response(stream).arrayBuffer()); +} + +export async function writeQueryDestination( + content: string | ReadableStream, + options: WriteQueryDestinationOptions = {}, +): Promise { + const { outputPath, sink } = options; + const bytes = + typeof content === "string" + ? new TextEncoder().encode(content.endsWith("\n") ? content : `${content}\n`) + : await collectStreamBytes(content); + + if (outputPath !== undefined) { + await Bun.write(outputPath, bytes); + return; + } + + if (sink) { + await sink.writeBytes(bytes); + return; + } + + await Bun.write(Bun.stdout, bytes); +} diff --git a/cli/src/lib/query-output-args.test.ts b/cli/src/lib/query-output-args.test.ts index 1aff6fe..3554566 100644 --- a/cli/src/lib/query-output-args.test.ts +++ b/cli/src/lib/query-output-args.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from "bun:test"; import { CliError } from "@/lib/errors.ts"; -import { parseQueryOutputOptions } from "@/lib/query-output-args.ts"; +import { parseQueryOutputOptions, resolveQueryComputeSize } from "@/lib/query-output-args.ts"; describe("parseQueryOutputOptions", () => { test("composes explicit query presentation settings", () => { @@ -12,38 +12,105 @@ describe("parseQueryOutputOptions", () => { "max-width": "24", pager: "never", }, - { agent: false, rawArgs: [] }, + { agent: false, json: false, rawArgs: [] }, ); expect(options).toMatchObject({ format: "markdown", displayOptions: { layout: "line", columns: ["id", "name"], maxColumnWidth: 24 }, pagerOptions: { mode: "never" }, + computeSize: "AUTO", }); }); test("derives machine-readable output from agent context", () => { - expect(parseQueryOutputOptions({}, { agent: true, rawArgs: [] })).toMatchObject({ + expect(parseQueryOutputOptions({}, { agent: true, json: true, rawArgs: [] })).toMatchObject({ format: "human", pagerOptions: { mode: "never" }, + computeSize: "AUTO", }); }); test("rejects incompatible or invalid presentation settings", () => { for (const run of [ - () => parseQueryOutputOptions({}, { agent: true, rawArgs: ["--layout", "table"] }), + () => + parseQueryOutputOptions({}, { agent: true, json: true, rawArgs: ["--layout", "table"] }), () => parseQueryOutputOptions( { "max-width": "4" }, - { agent: false, rawArgs: ["--max-width", "4"] }, + { agent: false, json: false, rawArgs: ["--max-width", "4"] }, ), () => parseQueryOutputOptions( { pager: "sometimes" }, - { agent: false, rawArgs: ["--pager", "sometimes"] }, + { agent: false, json: false, rawArgs: ["--pager", "sometimes"] }, + ), + () => + parseQueryOutputOptions( + { format: "parquet" }, + { agent: false, json: true, rawArgs: ["--format", "parquet"] }, + ), + () => + parseQueryOutputOptions( + { format: "csv", layout: "table" }, + { agent: false, json: false, rawArgs: ["--format", "csv", "--layout", "table"] }, + ), + () => + parseQueryOutputOptions( + { "session-id": "session-1", "compute-size": "AUTO" }, + { + agent: false, + json: false, + rawArgs: ["--session-id", "session-1", "--compute-size", "AUTO"], + }, ), ]) { expect(run).toThrow(CliError); } }); + + test("passes through request options", () => { + const options = parseQueryOutputOptions( + { + format: "jsonl", + dialect: "snowflake", + catalog: "analytics", + schema: "main", + output: "out.jsonl", + "compute-size": "L", + }, + { agent: false, json: false, rawArgs: ["--format", "jsonl", "--compute-size", "L"] }, + ); + + expect(options).toMatchObject({ + format: "jsonl", + dialect: "snowflake", + catalog: "analytics", + schema: "main", + outputPath: "out.jsonl", + computeSize: "L", + }); + }); +}); + +describe("resolveQueryComputeSize", () => { + test("defaults to AUTO without a session", () => { + expect(resolveQueryComputeSize({ computeSizeExplicit: false })).toBe("AUTO"); + }); + + test("omits default AUTO when a session is present", () => { + expect( + resolveQueryComputeSize({ sessionId: "session-1", computeSizeExplicit: false }), + ).toBeUndefined(); + }); + + test("keeps explicit sizes with a session", () => { + expect( + resolveQueryComputeSize({ + sessionId: "session-1", + computeSizeArg: "M", + computeSizeExplicit: true, + }), + ).toBe("M"); + }); }); diff --git a/cli/src/lib/query-output-args.ts b/cli/src/lib/query-output-args.ts index 3289845..5ecf6b9 100644 --- a/cli/src/lib/query-output-args.ts +++ b/cli/src/lib/query-output-args.ts @@ -1,21 +1,44 @@ import { asCliArgString } from "@/lib/cli-args.ts"; import { defineArguments } from "@/lib/command.ts"; import { CliError } from "@/lib/errors.ts"; -import { parseQueryResultFormat, type QueryResultFormat } from "@/lib/query-output.ts"; +import { + isApiNativeQueryFormat, + parseQueryResultFormat, + type QueryResultFormat, +} from "@/lib/query-output.ts"; import { resolvePagerOptions, type PagerMode, type PagerOptions } from "@/lib/pager.ts"; import { defaultDisplayOptions, type QueryDisplayOptions } from "@/lib/query-format.ts"; import { isQueryLayout, QUERY_LAYOUT_OPTIONS } from "@/ui/layouts/query.ts"; const MIN_MAX_COLUMN_WIDTH = 8; -const QUERY_RESULT_FORMAT_OPTIONS = ["csv", "markdown"] as const; +const PRESENTATION_RESULT_FORMAT_OPTIONS = ["csv", "markdown"] as const; +const QUERY_RESULT_FORMAT_OPTIONS = ["csv", "jsonl", "parquet", "markdown"] as const; +const COMPUTE_SIZE_OPTIONS = ["XS", "S", "M", "L", "XL", "AUTO"] as const; const PAGER_MODE_OPTIONS = ["auto", "always", "never"] as const; const PAGER_MODES = new Set(PAGER_MODE_OPTIONS); const AGENT_INCOMPATIBLE_QUERY_FLAGS = ["--layout", "--pager", "--max-width"] as const; - +const API_NATIVE_INCOMPATIBLE_PRESENTATION_FLAGS = [ + "--layout", + "--columns", + "--max-width", + "--pager", +] as const; + +/** Client-rendered formats shared by commands like `schema`. */ export const queryResultFormatArgs = defineArguments({ format: { type: "enum", description: "Serialized output format; use global --json for JSON", + options: [...PRESENTATION_RESULT_FORMAT_OPTIONS], + }, +}); + +/** Lakehouse query formats: csv/jsonl/parquet are API-native; markdown is CLI-rendered. */ +export const queryApiResultFormatArgs = defineArguments({ + format: { + type: "enum", + description: + "Result format: csv, jsonl, and parquet stream from the API; markdown is rendered by the CLI. Use global --json for structured JSON", options: [...QUERY_RESULT_FORMAT_OPTIONS], }, }); @@ -43,14 +66,45 @@ export const queryPagerArgs = defineArguments({ }, }); +export const queryRequestArgs = defineArguments({ + "compute-size": { + type: "enum", + description: "Compute size for the query (AUTO cannot be combined with --session-id)", + default: "AUTO", + options: [...COMPUTE_SIZE_OPTIONS], + }, + dialect: { + type: "string", + description: "Source SQL dialect to transpile from (server default: DuckDB)", + }, + catalog: { + type: "string", + description: "Catalog name (optional; can also come from the session)", + }, + schema: { + type: "string", + description: "Schema name (optional; can also come from the session)", + }, + output: { + type: "string", + description: "Write result bytes/text to this path instead of stdout", + }, +}); + export type QueryOutputOptions = { format: QueryResultFormat; displayOptions: QueryDisplayOptions; pagerOptions: PagerOptions; + outputPath?: string; + computeSize?: string; + dialect?: string; + catalog?: string; + schema?: string; }; type ParseQueryOutputOptions = { agent: boolean; + json: boolean; rawArgs: readonly string[]; }; @@ -70,6 +124,25 @@ function validateAgentQueryFlags(options: ParseQueryOutputOptions): void { } } +function validateApiNativeFormatFlags( + format: QueryResultFormat, + options: ParseQueryOutputOptions, +): void { + if (!isApiNativeQueryFormat(format)) return; + + if (options.json || options.agent) { + throw new CliError( + `--format ${format} cannot be combined with --json or --agent. Those flags expect structured CLI JSON output.`, + ); + } + + for (const flag of API_NATIVE_INCOMPATIBLE_PRESENTATION_FLAGS) { + if (hasArgvFlag(options.rawArgs, flag)) { + throw new CliError(`${flag} cannot be used with --format ${format}.`); + } + } +} + function parseQueryLayout(args: Record): QueryDisplayOptions["layout"] { const defaults = defaultDisplayOptions(); if (args.layout === undefined) return defaults.layout; @@ -112,6 +185,31 @@ function parsePagerOptions(args: Record, agent: boolean): Pager return resolvePagerOptions(pager as PagerMode); } +function optionalTrimmedString(args: Record, name: string): string | undefined { + const value = args[name]; + if (typeof value !== "string") return undefined; + const trimmed = value.trim(); + return trimmed === "" ? undefined : trimmed; +} + +export function resolveQueryComputeSize(options: { + sessionId?: string; + computeSizeArg?: string; + computeSizeExplicit: boolean; +}): string | undefined { + const computeSize = options.computeSizeArg ?? "AUTO"; + + if (options.sessionId) { + if (options.computeSizeExplicit && computeSize === "AUTO") { + throw new CliError("--compute-size AUTO cannot be combined with --session-id."); + } + if (!options.computeSizeExplicit) return undefined; + return computeSize; + } + + return computeSize; +} + export function parseQueryOutputOptions( args: Record, options: ParseQueryOutputOptions, @@ -120,9 +218,24 @@ export function parseQueryOutputOptions( const format = parseQueryResultFormat( args.format === undefined ? "human" : asCliArgString(args.format), ); + validateApiNativeFormatFlags(format, options); + + const sessionId = optionalTrimmedString(args, "session-id"); + const computeSize = resolveQueryComputeSize({ + sessionId, + computeSizeArg: + args["compute-size"] === undefined ? undefined : asCliArgString(args["compute-size"]), + computeSizeExplicit: hasArgvFlag(options.rawArgs, "--compute-size"), + }); + return { format, displayOptions: parseDisplayOptions(args), pagerOptions: parsePagerOptions(args, options.agent), + outputPath: optionalTrimmedString(args, "output"), + computeSize, + dialect: optionalTrimmedString(args, "dialect"), + catalog: optionalTrimmedString(args, "catalog"), + schema: optionalTrimmedString(args, "schema"), }; } diff --git a/cli/src/lib/query-output.ts b/cli/src/lib/query-output.ts index 44346d6..711a3fa 100644 --- a/cli/src/lib/query-output.ts +++ b/cli/src/lib/query-output.ts @@ -9,14 +9,31 @@ import { type QueryDisplayOptions, } from "@/lib/query-format.ts"; import { resolvePagerOptions, writePagedOutput, type PagerOptions } from "@/lib/pager.ts"; +import { writeQueryDestination } from "@/lib/query-destination.ts"; +import type { LakehouseApiQueryFormat } from "@/lib/lakehouse/query.ts"; -export type QueryResultFormat = "human" | "json" | "csv" | "markdown"; +export type { LakehouseApiQueryFormat }; +export type QueryResultFormat = "human" | "json" | LakehouseApiQueryFormat | "markdown"; -const QUERY_RESULT_FORMATS = new Set(["human", "json", "csv", "markdown"]); +const API_NATIVE_QUERY_FORMATS = new Set(["csv", "jsonl", "parquet"]); +const QUERY_RESULT_FORMATS = new Set([ + "human", + "json", + "csv", + "jsonl", + "parquet", + "markdown", +]); + +export function isApiNativeQueryFormat( + format: QueryResultFormat, +): format is LakehouseApiQueryFormat { + return API_NATIVE_QUERY_FORMATS.has(format); +} export function parseQueryResultFormat(format: string): QueryResultFormat { if (!QUERY_RESULT_FORMATS.has(format as QueryResultFormat)) { - throw new CliError(`Unsupported format: ${format}. Use csv or markdown.`); + throw new CliError(`Unsupported format: ${format}. Use csv, jsonl, parquet, or markdown.`); } return format as QueryResultFormat; } @@ -71,6 +88,9 @@ export function renderQueryOutputText( const columnNames = getQueryColumnNames(result); return renderQueryMarkdown(result, columnNames, displayOptions ?? defaultDisplayOptions()); } + if (isApiNativeQueryFormat(format)) { + throw new CliError(`Format ${format} must be streamed from the lakehouse API.`); + } return renderQueryHumanOutput(result, displayOptions ?? defaultDisplayOptions()); } @@ -80,16 +100,33 @@ export async function writeQueryOutput( sink: OutputSink, displayOptions?: QueryDisplayOptions, pagerOptions?: PagerOptions, + outputPath?: string, ): Promise { - const outputText = renderQueryOutputText(result, sink.json ? "json" : format, displayOptions); - const usePager = format === "human" && !sink.json; + const effectiveFormat = sink.json ? "json" : format; + if (isApiNativeQueryFormat(effectiveFormat)) { + throw new CliError(`Format ${effectiveFormat} must be streamed from the lakehouse API.`); + } + + const outputText = renderQueryOutputText(result, effectiveFormat, displayOptions); + const usePager = format === "human" && !sink.json && outputPath === undefined; if (usePager) { await writePagedOutput(outputText, pagerOptions ?? resolvePagerOptions(), sink); return; } - if (format === "json" || sink.json) { + if (outputPath !== undefined) { + if (effectiveFormat === "json") { + await writeQueryDestination(JSON.stringify(JSON.parse(outputText), null, 2), { + outputPath, + }); + return; + } + await writeQueryDestination(outputText, { outputPath }); + return; + } + + if (effectiveFormat === "json" || sink.json) { sink.writeJson(JSON.parse(outputText)); return; } diff --git a/cli/src/lib/runtime.ts b/cli/src/lib/runtime.ts index 4233e7b..4a1dafb 100644 --- a/cli/src/lib/runtime.ts +++ b/cli/src/lib/runtime.ts @@ -11,6 +11,8 @@ export type OutputSink = { writeStderr(line: string): void; writeJson(data: unknown): void; writeRaw(body: string): void; + /** Write opaque bytes to stdout with no trailing newline or text decoding. */ + writeBytes(body: Uint8Array): void | Promise; writeHuman(text: string): void; writeMetadata(lines: string[]): void; }; @@ -40,6 +42,10 @@ function createDefaultOutputSink(context: CliContext): OutputSink { writeRaw(body: string): void { console.log(body); }, + async writeBytes(body: Uint8Array): Promise { + // Avoid console.log: it UTF-8-decodes and appends a newline, which breaks parquet. + await Bun.write(Bun.stdout, body); + }, writeHuman(text: string): void { console.log(text); }, diff --git a/cli/src/lib/updater.test.ts b/cli/src/lib/updater.test.ts index 1d8d622..dea5ce5 100644 --- a/cli/src/lib/updater.test.ts +++ b/cli/src/lib/updater.test.ts @@ -730,6 +730,7 @@ describe("automatic update checks", () => { writeStderr() {}, writeJson() {}, writeRaw() {}, + writeBytes() {}, writeHuman() {}, writeMetadata(lines) { stderr.push(...lines); @@ -760,6 +761,7 @@ describe("automatic update checks", () => { writeStderr() {}, writeJson() {}, writeRaw() {}, + writeBytes() {}, writeHuman() {}, writeMetadata(lines) { stderr.push(...lines); diff --git a/cli/src/lib/usage.test.ts b/cli/src/lib/usage.test.ts index 3663aa1..3c0db41 100644 --- a/cli/src/lib/usage.test.ts +++ b/cli/src/lib/usage.test.ts @@ -184,7 +184,10 @@ describe("renderAltertableUsage", () => { expect.objectContaining({ name: "statement", type: "positional" }), ); expect(help.options).toContainEqual( - expect.objectContaining({ name: "format", values: ["csv", "markdown"] }), + expect.objectContaining({ + name: "format", + values: ["csv", "jsonl", "parquet", "markdown"], + }), ); expect(help.global_options).toContainEqual(expect.objectContaining({ name: "json" })); expect(help.global_options).toContainEqual( diff --git a/cli/src/test-utils/cli.ts b/cli/src/test-utils/cli.ts index e93d352..acd1e4f 100644 --- a/cli/src/test-utils/cli.ts +++ b/cli/src/test-utils/cli.ts @@ -20,6 +20,9 @@ export function createCliTestHarness( runtime.output.writeStderr = (line) => stderr.push(line); runtime.output.writeJson = (data) => stdout.push(JSON.stringify(data)); runtime.output.writeRaw = (body) => stdout.push(body); + runtime.output.writeBytes = (body) => { + stdout.push(Buffer.from(body).toString("utf8")); + }; runtime.output.writeHuman = (text) => stdout.push(text); runtime.output.writeMetadata = (lines) => stderr.push(...lines); diff --git a/tests/integration.e2e.ts b/tests/integration.e2e.ts index d63d56c..9abfcf2 100644 --- a/tests/integration.e2e.ts +++ b/tests/integration.e2e.ts @@ -36,10 +36,11 @@ describe("lakehouse integration flows", () => { expect(result.stdout).toContain("id"); expect(result.stdout).toContain("Alice"); - result = await workspace.runCommand('altertable query "SELECT * FROM cli_test ORDER BY id" --format csv'); + result = await workspace.runCommand( + 'altertable query "SELECT * FROM cli_test ORDER BY id" --format markdown', + ); expect(result.exitCode).toBe(0); - expect(result.stdout).toContain("id,name"); - expect(result.stdout).toContain("1,Alice"); + expect(result.stdout).toContain("Alice"); result = await workspace.runCommand('altertable --json query "SELECT * FROM cli_test ORDER BY id"'); expect(result.exitCode).toBe(0);