diff --git a/assets/connectionIcons/catalog.png b/assets/connectionIcons/catalog.png new file mode 100644 index 00000000..2777db48 Binary files /dev/null and b/assets/connectionIcons/catalog.png differ diff --git a/assets/connectionIcons/hive.png b/assets/connectionIcons/hive.png new file mode 100644 index 00000000..10cdc5d0 Binary files /dev/null and b/assets/connectionIcons/hive.png differ diff --git a/assets/connectionIcons/index.ts b/assets/connectionIcons/index.ts index 0021c0a8..afcabef0 100644 --- a/assets/connectionIcons/index.ts +++ b/assets/connectionIcons/index.ts @@ -26,6 +26,11 @@ import anthropic from './anthropic.svg'; import openaiCompatible from './openai_compatible.svg'; import lmstudio from './lmstudio.svg'; import file from './file.png'; +import polaris from './polaris.png'; +import lakekeeper from './lakekeeper.png'; +import nessie from './nessie.png'; +import hive from './hive.png'; +import catalog from './catalog.png'; import { SupportedConnectionTypes } from '../../src/types/backend'; type Image = Record; @@ -63,6 +68,18 @@ export const databaseIcons = { postgresql: postgres, }; +export const icebergCatalogImages = { + sqlite, + sql: postgres, + rest: catalog, + polaris, + lakekeeper, + nessie, + hive, +}; + +export const genericCatalogImage = catalog; + const obj: { images: Image } = { images: { snowflake, diff --git a/assets/connectionIcons/lakekeeper.png b/assets/connectionIcons/lakekeeper.png new file mode 100644 index 00000000..3e417370 Binary files /dev/null and b/assets/connectionIcons/lakekeeper.png differ diff --git a/assets/connectionIcons/nessie.png b/assets/connectionIcons/nessie.png new file mode 100644 index 00000000..b090b526 Binary files /dev/null and b/assets/connectionIcons/nessie.png differ diff --git a/assets/connectionIcons/polaris.png b/assets/connectionIcons/polaris.png new file mode 100644 index 00000000..6573f7a6 Binary files /dev/null and b/assets/connectionIcons/polaris.png differ diff --git a/resources/python/iceberg_bridge.py b/resources/python/iceberg_bridge.py new file mode 100644 index 00000000..e8bc9577 --- /dev/null +++ b/resources/python/iceberg_bridge.py @@ -0,0 +1,417 @@ +#!/usr/bin/env python3 +""" +Iceberg Bridge Script — DBT Studio +Reads one JSON command from stdin, writes one JSON result to stdout. +All errors are returned as {"ok": false, "error": "..."} — never thrown to stderr. + +Supported commands: + install_check, test_connection, list_namespaces, list_tables, + get_schema, get_snapshots, preview_table, import_table, + drop_table, rename_table, create_namespace, drop_namespace, + create_metadata_file +""" + +import json +import os +import sys +from datetime import date, datetime +from decimal import Decimal + + +def resolve_env_vars(props: dict) -> dict: + """Replace __ENV:VARNAME__ placeholders with actual environment variable values.""" + result = {} + for k, v in props.items(): + if isinstance(v, str) and v.startswith("__ENV:"): + env_key = v[6:] + result[k] = os.environ.get(env_key, "") + else: + result[k] = v + return result + + +def json_safe(value): + """Normalize Arrow scalar values before crossing the JSON bridge.""" + if isinstance(value, (datetime, date)): + return value.isoformat() + if isinstance(value, Decimal): + return str(value) + if isinstance(value, bytes): + return value.hex() + if isinstance(value, dict): + return {str(key): json_safe(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [json_safe(item) for item in value] + return value + + +def handle_install_check(_cmd: dict) -> dict: + """Check the complete runtime profile required by the implemented catalogs.""" + try: + import pyiceberg # noqa: PLC0415 + import pyarrow # noqa: F401, PLC0415 + import psycopg2 # noqa: F401, PLC0415 + import s3fs # noqa: F401, PLC0415 + import sqlalchemy # noqa: F401, PLC0415 + from pyiceberg.catalog.hive import HiveCatalog # noqa: F401, PLC0415 + version_parts = tuple( + int(part) for part in pyiceberg.__version__.split(".")[:3] + ) + if version_parts < (0, 10, 0): + return {"ok": True, "installed": False, "version": pyiceberg.__version__} + return {"ok": True, "installed": True, "version": pyiceberg.__version__} + except ImportError: + return {"ok": True, "installed": False} + + +def handle_test_connection(cmd: dict) -> dict: + """Test catalog access and, when a table exists, warehouse metadata access.""" + try: + from pyiceberg.catalog import load_catalog # noqa: PLC0415 + props = resolve_env_vars(cmd.get("catalog_properties", {})) + catalog = load_catalog(cmd["catalog_name"], **props) + namespaces = catalog.list_namespaces() + table_count = 0 + warehouse_connected = None + for namespace in namespaces: + tables = catalog.list_tables(namespace) + table_count += len(tables) + if warehouse_connected is None and tables: + table = catalog.load_table(tables[0]) + next(iter(table.scan(limit=1).plan_files()), None) + warehouse_connected = True + return { + "ok": True, + "namespace_count": len(namespaces), + "table_count": table_count, + "warehouse_connected": warehouse_connected, + } + except Exception as exc: # noqa: BLE001 + return {"ok": False, "error": str(exc)} + + +def handle_list_namespaces(cmd: dict) -> dict: + """List namespaces, optionally under a parent namespace.""" + try: + from pyiceberg.catalog import load_catalog # noqa: PLC0415 + props = resolve_env_vars(cmd.get("catalog_properties", {})) + catalog = load_catalog(cmd["catalog_name"], **props) + parent = tuple(cmd["parent"]) if cmd.get("parent") else () + namespaces = catalog.list_namespaces(parent) + return {"ok": True, "namespaces": [list(ns) for ns in namespaces]} + except Exception as exc: # noqa: BLE001 + return {"ok": False, "error": str(exc)} + + +def handle_list_tables(cmd: dict) -> dict: + """List tables within a given namespace.""" + try: + from pyiceberg.catalog import load_catalog # noqa: PLC0415 + props = resolve_env_vars(cmd.get("catalog_properties", {})) + catalog = load_catalog(cmd["catalog_name"], **props) + namespace = tuple(cmd["namespace"]) + identifiers = catalog.list_tables(namespace) + # Each identifier is a tuple; return just the table name part + return {"ok": True, "tables": [ident[-1] for ident in identifiers]} + except Exception as exc: # noqa: BLE001 + return {"ok": False, "error": str(exc)} + + +def handle_get_schema(cmd: dict) -> dict: + """Return the current schema (field list) for a table.""" + try: + from pyiceberg.catalog import load_catalog # noqa: PLC0415 + props = resolve_env_vars(cmd.get("catalog_properties", {})) + catalog = load_catalog(cmd["catalog_name"], **props) + namespace = tuple(cmd["namespace"]) + table = catalog.load_table((*namespace, cmd["table"])) + schema = table.schema() + fields = [] + for field in schema.fields: + fields.append({ + "fieldId": field.field_id, + "name": field.name, + "type": str(field.field_type), + "required": field.required, + "doc": field.doc, + }) + return { + "ok": True, + "fields": fields, + "properties": dict(table.properties), + } + except Exception as exc: # noqa: BLE001 + return {"ok": False, "error": str(exc)} + + +def handle_get_snapshots(cmd: dict) -> dict: + """Return the snapshot history for a table.""" + try: + from pyiceberg.catalog import load_catalog # noqa: PLC0415 + props = resolve_env_vars(cmd.get("catalog_properties", {})) + catalog = load_catalog(cmd["catalog_name"], **props) + namespace = tuple(cmd["namespace"]) + table = catalog.load_table((*namespace, cmd["table"])) + current_snapshot_id = table.metadata.current_snapshot_id + snapshots = [] + for snap in table.metadata.snapshots: + snapshots.append({ + "snapshotId": str(snap.snapshot_id), + "isCurrent": snap.snapshot_id == current_snapshot_id, + "parentId": str(snap.parent_snapshot_id) if snap.parent_snapshot_id else None, + "operation": snap.summary.operation.value if snap.summary and snap.summary.operation else "unknown", + "committedAt": str(snap.timestamp_ms), + "manifestList": snap.manifest_list or "", + "summary": dict(snap.summary.additional_properties) if snap.summary else {}, + }) + return {"ok": True, "snapshots": snapshots} + except Exception as exc: # noqa: BLE001 + return {"ok": False, "error": str(exc)} + + +def handle_preview_table(cmd: dict) -> dict: + """Preview rows from a table using PyArrow scan.""" + try: + from pyiceberg.catalog import load_catalog # noqa: PLC0415 + props = resolve_env_vars(cmd.get("catalog_properties", {})) + catalog = load_catalog(cmd["catalog_name"], **props) + namespace = tuple(cmd["namespace"]) + table = catalog.load_table((*namespace, cmd["table"])) + limit = int(cmd.get("limit", 100)) + row_filter = cmd.get("row_filter") + + scan_kwargs = {"limit": limit} + if row_filter: + scan_kwargs["row_filter"] = row_filter + + arrow_table = table.scan(**scan_kwargs).to_arrow() + columns = arrow_table.schema.names + rows = arrow_table.to_pydict() + # Convert column-oriented dict to row-oriented list of lists + row_list = [ + [json_safe(rows[col][i]) for col in columns] + for i in range(len(arrow_table)) + ] + return { + "ok": True, + "columns": columns, + "rows": row_list, + "total": len(row_list), + } + except Exception as exc: # noqa: BLE001 + return {"ok": False, "error": str(exc)} + + +def handle_import_table(cmd: dict) -> dict: + """ + Import a local CSV/Parquet/JSON file into a new Iceberg table. + + Reads the file with PyArrow, infers its schema, creates the table in the + requested namespace (creating the namespace when absent), and appends the + data as the initial snapshot. + """ + try: + import pyarrow as pa # noqa: PLC0415 + import pyarrow.csv as pa_csv # noqa: PLC0415 + import pyarrow.json as pa_json # noqa: PLC0415 + import pyarrow.parquet as pa_parquet # noqa: PLC0415 + from pyiceberg.catalog import load_catalog # noqa: PLC0415 + from pyiceberg.exceptions import TableAlreadyExistsError # noqa: PLC0415 + + props = resolve_env_vars(cmd.get("catalog_properties", {})) + catalog = load_catalog(cmd["catalog_name"], **props) + namespace = tuple(cmd["namespace"]) + table_name = cmd["table"] + identifier = (*namespace, table_name) + file_path = cmd["file_path"] + file_format = cmd.get("file_format", "").lower() + + if not os.path.isfile(file_path): + return {"ok": False, "error": f"File not found: {file_path}"} + + if file_format == "parquet": + arrow_table = pa_parquet.read_table(file_path) + elif file_format == "csv": + arrow_table = pa_csv.read_csv(file_path) + elif file_format == "json": + try: + arrow_table = pa_json.read_json(file_path) + except Exception as json_error: # noqa: BLE001 + # pyarrow.json.read_json only accepts newline-delimited records. + # Fall back to parsing a JSON array of objects when present. + with open(file_path, encoding="utf-8") as handle: + payload = json.load(handle) + if not isinstance(payload, list): + raise ValueError( + f"JSON source must be an array of objects or newline-delimited records: {json_error}" + ) from json_error + arrow_table = pa.Table.from_pylist(payload) + else: + return {"ok": False, "error": f"Unsupported file format: {file_format}"} + + if arrow_table.num_columns == 0: + return {"ok": False, "error": "The source file contains no columns."} + + try: + catalog.create_namespace_if_not_exists(namespace) + table = catalog.create_table(identifier, schema=arrow_table.schema) + except TableAlreadyExistsError: + return { + "ok": False, + "error": f"Table already exists: {'.'.join(namespace)}.{table_name}", + } + + try: + table.append(arrow_table) + except Exception: # noqa: BLE001 + # Best-effort cleanup of the empty table on write failure so a + # failed import does not leave a phantom catalog entry. + try: + catalog.drop_table(identifier) + except Exception: # noqa: BLE001 + pass + raise + + return { + "ok": True, + "namespace": list(namespace), + "table": table_name, + "row_count": arrow_table.num_rows, + "columns": arrow_table.schema.names, + } + except Exception as exc: # noqa: BLE001 + return {"ok": False, "error": str(exc)} + + +def handle_drop_table(cmd: dict) -> dict: + """Drop a table from the catalog.""" + try: + from pyiceberg.catalog import load_catalog # noqa: PLC0415 + props = resolve_env_vars(cmd.get("catalog_properties", {})) + catalog = load_catalog(cmd["catalog_name"], **props) + namespace = tuple(cmd["namespace"]) + catalog.drop_table((*namespace, cmd["table"])) + return {"ok": True, "namespace": list(namespace), "table": cmd["table"]} + except Exception as exc: # noqa: BLE001 + return {"ok": False, "error": str(exc)} + + +def handle_rename_table(cmd: dict) -> dict: + """Rename a table within the same namespace.""" + try: + from pyiceberg.catalog import load_catalog # noqa: PLC0415 + props = resolve_env_vars(cmd.get("catalog_properties", {})) + catalog = load_catalog(cmd["catalog_name"], **props) + namespace = tuple(cmd["namespace"]) + catalog.rename_table( + (*namespace, cmd["table"]), + (*namespace, cmd["new_table"]), + ) + return { + "ok": True, + "namespace": list(namespace), + "table": cmd["new_table"], + } + except Exception as exc: # noqa: BLE001 + return {"ok": False, "error": str(exc)} + + +def handle_create_namespace(cmd: dict) -> dict: + """Create a (possibly nested) namespace in the catalog.""" + try: + from pyiceberg.catalog import load_catalog # noqa: PLC0415 + props = resolve_env_vars(cmd.get("catalog_properties", {})) + catalog = load_catalog(cmd["catalog_name"], **props) + namespace = tuple(cmd["namespace"]) + catalog.create_namespace(namespace) + return {"ok": True, "namespace": list(namespace)} + except Exception as exc: # noqa: BLE001 + return {"ok": False, "error": str(exc)} + + +def handle_drop_namespace(cmd: dict) -> dict: + """Drop a namespace. The namespace must be empty (no tables).""" + try: + from pyiceberg.catalog import load_catalog # noqa: PLC0415 + props = resolve_env_vars(cmd.get("catalog_properties", {})) + catalog = load_catalog(cmd["catalog_name"], **props) + namespace = tuple(cmd["namespace"]) + catalog.drop_namespace(namespace) + return {"ok": True, "namespace": list(namespace)} + except Exception as exc: # noqa: BLE001 + return {"ok": False, "error": str(exc)} + + +def handle_create_metadata_file(cmd: dict) -> dict: + """ + Initialize a development-only SQL catalog backed by SQLite. + + The IPC command keeps its historical name for compatibility. It creates a + durable catalog database and local warehouse, then reloads the catalog to + prove the persisted contract works. + """ + try: + from pathlib import Path # noqa: PLC0415 + from pyiceberg.catalog import load_catalog # noqa: PLC0415 + + catalog_dir = Path(cmd["warehouse_path"]).expanduser().resolve() + catalog_dir.mkdir(parents=True, exist_ok=True) + warehouse_dir = catalog_dir / "warehouse" + warehouse_dir.mkdir(parents=True, exist_ok=True) + catalog_path = catalog_dir / "pyiceberg_catalog.db" + + properties = { + "type": "sql", + "uri": f"sqlite:///{catalog_path.as_posix()}", + "warehouse": warehouse_dir.as_uri(), + } + catalog = load_catalog("local", **properties) + catalog.create_namespace_if_not_exists("default") + catalog.close() + + reloaded = load_catalog("local", **properties) + namespaces = [list(namespace) for namespace in reloaded.list_namespaces()] + tables = [list(identifier) for identifier in reloaded.list_tables("default")] + reloaded.close() + + return { + "ok": True, + "metadata_path": str(catalog_path), + "warehouse_path": str(warehouse_dir), + "namespaces": namespaces, + "tables": tables, + } + except Exception as exc: # noqa: BLE001 + return {"ok": False, "error": str(exc)} + + +HANDLERS = { + "install_check": handle_install_check, + "test_connection": handle_test_connection, + "list_namespaces": handle_list_namespaces, + "list_tables": handle_list_tables, + "get_schema": handle_get_schema, + "get_snapshots": handle_get_snapshots, + "preview_table": handle_preview_table, + "import_table": handle_import_table, + "drop_table": handle_drop_table, + "rename_table": handle_rename_table, + "create_namespace": handle_create_namespace, + "drop_namespace": handle_drop_namespace, + "create_metadata_file": handle_create_metadata_file, +} + + +if __name__ == "__main__": + try: + raw = sys.stdin.read() + cmd = json.loads(raw) + command_name = cmd.get("command") + handler = HANDLERS.get(command_name) + if handler is None: + print(json.dumps({"ok": False, "error": f"Unknown command: {command_name}"})) + else: + result = handler(cmd) + print(json.dumps(result)) + except Exception as e: # noqa: BLE001 + print(json.dumps({"ok": False, "error": str(e)})) diff --git a/src/main/ipcHandlers/icebergDatalake.ipcHandlers.ts b/src/main/ipcHandlers/icebergDatalake.ipcHandlers.ts new file mode 100644 index 00000000..a6ae3334 --- /dev/null +++ b/src/main/ipcHandlers/icebergDatalake.ipcHandlers.ts @@ -0,0 +1,107 @@ +/** + * Iceberg Datalake IPC Handlers + * Lean wrappers — no business logic, no try/catch, pure delegation. + * Follows BE-01: IPC handlers are thin wrappers with zero business logic. + */ + +import { ipcMain } from 'electron'; +import { IcebergDatalakeService } from '../services/icebergDatalake.service'; + +export const registerIcebergDatalakeHandlers = () => { + ipcMain.handle('iceberg:getCapabilities', () => + IcebergDatalakeService.getCapabilities(), + ); + + ipcMain.handle('iceberg:list', () => IcebergDatalakeService.listInstances()); + + ipcMain.handle('iceberg:get', (_e, id: string) => + IcebergDatalakeService.getInstance(id), + ); + + ipcMain.handle('iceberg:create', (_e, data) => + IcebergDatalakeService.createInstance(data), + ); + + ipcMain.handle('iceberg:update', (_e, id: string, data) => + IcebergDatalakeService.updateInstance(id, data), + ); + + ipcMain.handle('iceberg:delete', (_e, id: string) => + IcebergDatalakeService.deleteInstance(id), + ); + + ipcMain.handle('iceberg:testCatalog', (_e, params) => + IcebergDatalakeService.testCatalogConnection(params), + ); + + ipcMain.handle('iceberg:testStorage', (_e, params) => + IcebergDatalakeService.testStorageConnection(params), + ); + + ipcMain.handle('iceberg:listStorageBuckets', (_e, params) => + IcebergDatalakeService.listStorageBuckets(params), + ); + + ipcMain.handle('iceberg:testInstance', (_e, id: string) => + IcebergDatalakeService.testInstanceConnection(id), + ); + + ipcMain.handle('iceberg:listNamespaces', (_e, id: string, parent?) => + IcebergDatalakeService.listNamespaces(id, parent), + ); + + ipcMain.handle('iceberg:listTables', (_e, id: string, namespace) => + IcebergDatalakeService.listTables(id, namespace), + ); + + ipcMain.handle('iceberg:getSchema', (_e, id: string, namespace, table) => + IcebergDatalakeService.getTableSchema(id, namespace, table), + ); + + ipcMain.handle('iceberg:getSnapshots', (_e, id: string, namespace, table) => + IcebergDatalakeService.getTableSnapshots(id, namespace, table), + ); + ipcMain.handle( + 'iceberg:previewTable', + (_e, id: string, namespace, table, limit, filter?) => + IcebergDatalakeService.previewTable(id, namespace, table, limit, filter), + ); + + ipcMain.handle( + 'iceberg:importTable', + (_e, id: string, namespace, table, filePath, fileFormat) => + IcebergDatalakeService.importTable( + id, + namespace, + table, + filePath, + fileFormat, + ), + ); + + ipcMain.handle('iceberg:dropTable', (_e, id: string, namespace, table) => + IcebergDatalakeService.dropTable(id, namespace, table), + ); + + ipcMain.handle( + 'iceberg:renameTable', + (_e, id: string, namespace, table, newTable) => + IcebergDatalakeService.renameTable(id, namespace, table, newTable), + ); + + ipcMain.handle('iceberg:createNamespace', (_e, id: string, namespace) => + IcebergDatalakeService.createNamespace(id, namespace), + ); + + ipcMain.handle('iceberg:dropNamespace', (_e, id: string, namespace) => + IcebergDatalakeService.dropNamespace(id, namespace), + ); + + ipcMain.handle('iceberg:createMetadataFile', (_e, warehousePath: string) => + IcebergDatalakeService.createMetadataFile(warehousePath), + ); + + ipcMain.handle('iceberg:ensureInstalled', () => + IcebergDatalakeService.ensurePyicebergInstalled(), + ); +}; diff --git a/src/main/ipcHandlers/index.ts b/src/main/ipcHandlers/index.ts index c5a5672e..2d818a5f 100644 --- a/src/main/ipcHandlers/index.ts +++ b/src/main/ipcHandlers/index.ts @@ -25,6 +25,7 @@ import registerFlowfileHandlers from './flowfile.ipcHandlers'; import { registerPipelineTemplatesHandlers } from './pipelineTemplates.ipcHandlers'; import registerTaskManagerHandlers from './taskManager.ipcHandlers'; import { registerSecondBrainHandlers } from './secondBrain.ipcHandlers'; +import { registerIcebergDatalakeHandlers } from './icebergDatalake.ipcHandlers'; export { registerCliHandlers, @@ -54,4 +55,5 @@ export { registerPipelineTemplatesHandlers, registerTaskManagerHandlers, registerSecondBrainHandlers, + registerIcebergDatalakeHandlers, }; diff --git a/src/main/ipcSetup.ts b/src/main/ipcSetup.ts index dfc82fbf..1bbb8e4c 100644 --- a/src/main/ipcSetup.ts +++ b/src/main/ipcSetup.ts @@ -27,6 +27,7 @@ import { registerPipelineTemplatesHandlers, registerTaskManagerHandlers, registerSecondBrainHandlers, + registerIcebergDatalakeHandlers, } from './ipcHandlers'; import { installIpcErrorHandling } from './utils/ipcErrorHandler'; @@ -59,6 +60,7 @@ const registerHandlers = (mainWindow: BrowserWindow) => { registerPipelineTemplatesHandlers(); registerTaskManagerHandlers(mainWindow); registerSecondBrainHandlers(); + registerIcebergDatalakeHandlers(); }; export default registerHandlers; diff --git a/src/main/services/icebergDatalake.service.ts b/src/main/services/icebergDatalake.service.ts new file mode 100644 index 00000000..66ff1118 --- /dev/null +++ b/src/main/services/icebergDatalake.service.ts @@ -0,0 +1,1724 @@ +/** + * IcebergDatalakeService + * Main backend service for Iceberg Data Lake instance management. + * Handles CRUD, secure credential storage, Python bridge invocation, + * and pyiceberg installation. + * + * Follows BE-03 (one cohesive service), BE-04 (no await inside Promise constructor). + */ + +import * as fs from 'fs'; +import { v4 as uuidv4 } from 'uuid'; +import * as path from 'path'; +import { pathToFileURL } from 'url'; +import { spawn } from 'child_process'; +import { app } from 'electron'; + +import { loadDatabaseFile, updateDatabase } from '../utils/fileHelper'; +import secureStorage from './secureStorage.service'; +import SettingsService from './settings.service'; + +import type { + IcebergInstanceConfig, + IcebergInstanceListItem, + CreateIcebergInstanceDTO, + UpdateIcebergInstanceDTO, + IcebergTestCatalogParams, + IcebergTestStorageParams, + IcebergListStorageBucketsParams, + IcebergTestResult, + IcebergFieldSpec, + IcebergSchemaResult, + IcebergSnapshotInfo, + IcebergPreviewResult, + IcebergLocalCatalogResult, + IcebergCapabilities, + IcebergCatalogCapability, + IcebergImportTableResult, + IcebergImportFileFormat, + IcebergTableOperationResult, + IcebergNamespaceOperationResult, +} from '../../types/iceberg'; +import type { CloudConnection, CloudStorageConfig } from '../../types/frontend'; +import type { PostgresConnection } from '../../types/backend'; + +export class IcebergDatalakeService { + private static readonly cloudProviders = [ + 'aws', + 'azure', + 'gcs', + 'minio', + 'cloudflare-r2', + 'backblaze-b2', + 'rustfs', + 'garage', + ] as const; + + private static readonly catalogCapabilities: IcebergCatalogCapability[] = [ + { + type: 'sqlite', + label: 'SQLite (Local)', + pyicebergType: 'sql', + enabled: true, + requiredFields: ['catalogPath'], + authModes: ['none'], + allowedStorageTypes: ['local', 'cloud'], + }, + { + type: 'sql', + label: 'PostgreSQL / Neon', + pyicebergType: 'sql', + enabled: true, + requiredFields: ['databaseConnectionId', 'catalogName'], + authModes: ['none'], + allowedStorageTypes: ['local', 'cloud'], + }, + { + type: 'rest', + label: 'REST Catalog', + pyicebergType: 'rest', + enabled: true, + requiredFields: ['endpoint', 'catalogName'], + authModes: ['none', 'token', 'oauth-client-credentials'], + allowedStorageTypes: ['server-managed'], + }, + { + type: 'polaris', + label: 'Apache Polaris', + pyicebergType: 'rest', + enabled: true, + requiredFields: ['endpoint', 'catalogName'], + authModes: ['none', 'token', 'oauth-client-credentials'], + allowedStorageTypes: ['server-managed'], + }, + { + type: 'lakekeeper', + label: 'Lakekeeper', + pyicebergType: 'rest', + enabled: true, + requiredFields: ['endpoint', 'catalogName'], + authModes: ['none', 'token', 'oauth-client-credentials'], + allowedStorageTypes: ['server-managed'], + }, + { + type: 'nessie', + label: 'Project Nessie', + pyicebergType: 'rest', + enabled: true, + requiredFields: ['endpoint', 'nessieReference'], + authModes: ['none', 'token', 'oauth-client-credentials'], + allowedStorageTypes: ['server-managed'], + }, + { + type: 'hive', + label: 'Hive Metastore', + pyicebergType: 'hive', + enabled: true, + requiredFields: ['hiveUri'], + authModes: ['none'], + allowedStorageTypes: ['local'], + }, + ]; + + // In-process cache: once we confirm pyiceberg is installed for this app + // session we skip the Python bridge check on subsequent calls. + private static installedCache: { + installed: boolean; + version?: string; + } | null = null; + + // ───────────────────────────────────────────── + // Private: persistence helpers + // ───────────────────────────────────────────── + + private static async readInstances(): Promise { + try { + const db = await loadDatabaseFile(); + return (db.icebergInstances ?? []).map((instance) => { + const persisted = instance as unknown as { catalogType: string }; + if (persisted.catalogType !== 'file') return instance; + return { ...instance, catalogType: 'sqlite' }; + }); + } catch (error) { + // eslint-disable-next-line no-console + console.error('[IcebergDatalakeService] readInstances error:', error); + return []; + } + } + + private static async writeInstances( + instances: IcebergInstanceConfig[], + ): Promise { + await updateDatabase('icebergInstances', instances); + } + + // ───────────────────────────────────────────── + // Private: Python bridge helpers + // ───────────────────────────────────────────── + + private static getBridgePath(): string { + if (app.isPackaged) { + return path.join(process.resourcesPath, 'python', 'iceberg_bridge.py'); + } + return path.join( + __dirname, + '..', + '..', + 'resources', + 'python', + 'iceberg_bridge.py', + ); + } + + private static async getPythonPath(): Promise { + try { + const settings = await SettingsService.loadSettings(); + if (settings.pythonPath) return settings.pythonPath; + } catch { + // fall through to default + } + return 'python3'; + } + + private static redactBridgeSecrets( + message: string, + env: Record, + ): string { + const secrets = new Set(); + Object.entries(env).forEach(([key, value]) => { + if (!value) return; + secrets.add(value); + if (key === 'ICEBERG_OAUTH_CREDENTIAL') { + secrets.add(value.slice(value.indexOf(':') + 1)); + } + if (key === 'ICEBERG_SQL_CATALOG_URI') { + try { + const parsed = new URL( + value.replace(/^postgresql\+psycopg2:/, 'postgresql:'), + ); + if (parsed.password) secrets.add(decodeURIComponent(parsed.password)); + } catch { + // The complete URI is still redacted below. + } + } + }); + return [...secrets] + .filter((secret) => secret.length >= 4) + .sort((left, right) => right.length - left.length) + .reduce( + (redacted, secret) => redacted.split(secret).join('[REDACTED]'), + message, + ); + } + + /** + * Spawns the Python bridge, writes command JSON to stdin, reads result from stdout. + * BE-04: all async values resolved BEFORE entering new Promise constructor. + */ + private static async runBridge( + command: object, + env: Record = {}, + timeoutMs = 120_000, + ): Promise { + const pythonPath = await IcebergDatalakeService.getPythonPath(); + const bridgePath = IcebergDatalakeService.getBridgePath(); + + return new Promise((resolve, reject) => { + const child = spawn(pythonPath, [bridgePath], { + env: { ...process.env, ...env }, + }); + let stdout = ''; + let stderr = ''; + let settled = false; + let timer: ReturnType; + const finish = (fn: () => void) => { + if (settled) return; + settled = true; + clearTimeout(timer); + fn(); + }; + timer = setTimeout(() => { + child.kill('SIGKILL'); + finish(() => reject(new Error('ICEBERG_BRIDGE_TIMEOUT'))); + }, timeoutMs); + child.stdout.on('data', (d: Buffer) => { + stdout += d.toString(); + }); + child.stderr.on('data', (d: Buffer) => { + stderr += d.toString(); + }); + child.stdin.on('error', (err: Error) => finish(() => reject(err))); + child.stdin.write(JSON.stringify(command)); + child.stdin.end(); + child.on('close', (code: number) => { + finish(() => { + try { + const result = JSON.parse(stdout) as Record; + if (!result.ok) { + reject( + new Error( + IcebergDatalakeService.redactBridgeSecrets( + (result.error as string) ?? 'Bridge error', + env, + ), + ), + ); + } else { + resolve(result); + } + } catch { + reject( + new Error( + IcebergDatalakeService.redactBridgeSecrets( + `Bridge parse error (exit ${code}): ${stderr}`, + env, + ), + ), + ); + } + }); + }); + child.on('error', (err: Error) => finish(() => reject(err))); + }); + } + + // ───────────────────────────────────────────── + // Private: catalog properties builder + // ───────────────────────────────────────────── + + /** + * Builds the catalog properties object and env map from an instance config. + * Secrets are passed via env vars using the __ENV:VARNAME__ placeholder pattern. + */ + private static async buildCatalogProperties( + instance: IcebergInstanceConfig, + ): Promise<{ props: Record; env: Record }> { + const props: Record = {}; + const env: Record = {}; + + switch (instance.catalogType) { + case 'sqlite': + props.type = 'sql'; + if (instance.catalogPath) { + props.uri = `sqlite:///${instance.catalogPath}`; + } + if (instance.localPath) { + props.warehouse = pathToFileURL(instance.localPath).href; + } + break; + + case 'sql': { + const sqlCatalog = + await IcebergDatalakeService.buildPostgresCatalogUri(instance); + props.type = 'sql'; + props.uri = '__ENV:ICEBERG_SQL_CATALOG_URI'; + env.ICEBERG_SQL_CATALOG_URI = sqlCatalog; + break; + } + + case 'rest': + case 'polaris': + case 'lakekeeper': + case 'nessie': + props.type = 'rest'; + if (instance.catalogType === 'nessie') { + props.uri = IcebergDatalakeService.buildNessieRestUri(instance); + props['header.X-Iceberg-Access-Delegation'] = 'remote-signing'; + } else { + if (instance.endpoint) props.uri = instance.endpoint; + if (instance.catalogName) props.warehouse = instance.catalogName; + if (instance.catalogType === 'lakekeeper') { + props['header.X-Iceberg-Access-Delegation'] = 'remote-signing'; + } + } + // Access token via env var + if (instance.catalogAccessTokenKey) { + try { + const token = await secureStorage.getCredential( + instance.catalogAccessTokenKey, + ); + if (token) { + props.token = '__ENV:ICEBERG_ACCESS_TOKEN'; + // eslint-disable-next-line dot-notation + env['ICEBERG_ACCESS_TOKEN'] = token; + } + } catch (tokenError) { + // eslint-disable-next-line no-console + console.error( + '[IcebergDatalakeService] token retrieval error:', + tokenError, + ); + } + } + if ( + instance.catalogAuthMode === 'oauth-client-credentials' && + instance.oauthClientId && + instance.oauthClientSecretKey + ) { + const clientSecret = await secureStorage.getCredential( + instance.oauthClientSecretKey, + ); + if (!clientSecret) { + throw new Error('ICEBERG_OAUTH_SECRET_NOT_FOUND'); + } + props.credential = '__ENV:ICEBERG_OAUTH_CREDENTIAL'; + env.ICEBERG_OAUTH_CREDENTIAL = `${instance.oauthClientId}:${clientSecret}`; + if (instance.oauthServerUri) { + props['oauth2-server-uri'] = instance.oauthServerUri; + } + if (instance.oauthScope) props.scope = instance.oauthScope; + delete props.token; + delete env.ICEBERG_ACCESS_TOKEN; + } + break; + + case 'hive': + props.type = 'hive'; + props.uri = IcebergDatalakeService.buildHiveMetastoreUri(instance); + if (instance.hiveUgi?.trim()) props.ugi = instance.hiveUgi.trim(); + break; + + default: + throw new Error(`ICEBERG_CATALOG_NOT_ENABLED: ${instance.catalogType}`); + } + + const warehouse = + await IcebergDatalakeService.buildWarehouseProperties(instance); + return { + props: { ...props, ...warehouse.props }, + env: { ...env, ...warehouse.env }, + }; + } + + private static buildNessieRestUri( + config: Pick< + IcebergInstanceConfig, + 'endpoint' | 'nessieReference' | 'nessieWarehouse' + >, + ): string { + const endpoint = config.endpoint?.trim().replace(/\/+$/, ''); + const reference = config.nessieReference?.trim(); + if (!endpoint) throw new Error('ICEBERG_REQUIRED_FIELD: endpoint'); + if (!reference) { + throw new Error('ICEBERG_REQUIRED_FIELD: nessieReference'); + } + let parsed: URL; + try { + parsed = new URL(endpoint); + } catch { + throw new Error('ICEBERG_NESSIE_ENDPOINT_INVALID'); + } + if (!['http:', 'https:'].includes(parsed.protocol)) { + throw new Error('ICEBERG_NESSIE_ENDPOINT_INVALID'); + } + if (!parsed.pathname.endsWith('/iceberg')) { + throw new Error('ICEBERG_NESSIE_ICEBERG_REST_ENDPOINT_REQUIRED'); + } + const warehouse = config.nessieWarehouse?.trim(); + return `${endpoint}/${encodeURIComponent(reference)}${ + warehouse ? `|${encodeURIComponent(warehouse)}` : '' + }`; + } + + private static buildHiveMetastoreUri( + config: Pick, + ): string { + const rawUri = config.hiveUri?.trim(); + if (!rawUri) throw new Error('ICEBERG_REQUIRED_FIELD: hiveUri'); + + const uris = rawUri.split(',').map((value) => value.trim()); + if (uris.some((value) => !value)) { + throw new Error('ICEBERG_HIVE_URI_INVALID'); + } + const hasInvalidUri = uris.some((uri) => { + try { + const parsed = new URL(uri); + return ( + parsed.protocol !== 'thrift:' || + !parsed.hostname || + !parsed.port || + !!parsed.username || + !!parsed.password || + (!!parsed.pathname && parsed.pathname !== '/') + ); + } catch { + return true; + } + }); + if (hasInvalidUri) throw new Error('ICEBERG_HIVE_URI_INVALID'); + return uris.join(','); + } + + private static async buildWarehouseProperties( + instance: IcebergInstanceConfig, + ): Promise<{ props: Record; env: Record }> { + const props: Record = {}; + const env: Record = {}; + + if (instance.storageType === 'server-managed') return { props, env }; + if (instance.storageType === 'local' && instance.localPath) { + props.warehouse = pathToFileURL(instance.localPath).href; + } + + if (instance.storageConnectionId) { + try { + const db = await loadDatabaseFile(); + const conn: CloudConnection | undefined = (db.sources ?? []).find( + (s) => s.id === instance.storageConnectionId, + ); + if (conn) { + const { provider, config, id: connId } = conn; + + // Non-secret config fields are safe to read from the persisted config object. + // Secrets (secretAccessKey, accountKey, credentials JSON) are stored in keytar + // under provider-specific keys — matching the pattern used by DuckLake.service.ts. + const cfg = config as unknown as Record; + + const s3LikeProviders = [ + 'aws', + 'minio', + 'cloudflare-r2', + 'backblaze-b2', + 'rustfs', + 'garage', + ]; + + if (s3LikeProviders.includes(provider)) { + // Non-secret fields from config + if (cfg.endpoint) props['s3.endpoint'] = cfg.endpoint; + if (cfg.region) props['s3.region'] = cfg.region; + if (cfg.accessKeyId) { + props['s3.access-key-id'] = cfg.accessKeyId; + } + // Secret key from keytar: cloud-{provider}-{connectionId} + const secretKey = await secureStorage.getCredential( + `cloud-${provider}-${connId}`, + ); + if (secretKey) { + props['s3.secret-access-key'] = '__ENV:ICEBERG_S3_SECRET__'; + // eslint-disable-next-line dot-notation + env['ICEBERG_S3_SECRET__'] = secretKey; + } + // Optional session token (AWS only) + if (provider === 'aws') { + const sessionToken = await secureStorage.getCredential( + `cloud-aws-session-${connId}`, + ); + if (sessionToken) { + props['s3.session-token'] = '__ENV:ICEBERG_S3_SESSION__'; + // eslint-disable-next-line dot-notation + env['ICEBERG_S3_SESSION__'] = sessionToken; + } + } + } else if (provider === 'azure') { + if (cfg.accountName) props['adls.account-name'] = cfg.accountName; + // Secret from keytar: cloud-azure-{connectionId} + const accountKey = await secureStorage.getCredential( + `cloud-azure-${connId}`, + ); + if (accountKey) { + props['adls.account-key'] = '__ENV:ICEBERG_ADLS_KEY__'; + // eslint-disable-next-line dot-notation + env['ICEBERG_ADLS_KEY__'] = accountKey; + } + } else if (provider === 'gcs') { + if (cfg.projectId) props['gcs.project-id'] = cfg.projectId; + // GCS credentials JSON from keytar: cloud-gcs-{connectionId} + const gcsCreds = await secureStorage.getCredential( + `cloud-gcs-${connId}`, + ); + if (gcsCreds) { + props['gcs.credentials'] = '__ENV:ICEBERG_GCS_CREDS__'; + // eslint-disable-next-line dot-notation + env['ICEBERG_GCS_CREDS__'] = gcsCreds; + } + } + + if (instance.storageBucket) { + const prefix = instance.storagePrefix + ? `/${instance.storagePrefix.replace(/^\/+|\/+$/g, '')}` + : ''; + if (provider === 'azure' && cfg.accountName) { + props.warehouse = `abfs://${instance.storageBucket}@${cfg.accountName}.dfs.core.windows.net${prefix}`; + } else if (provider === 'gcs') { + props.warehouse = `gs://${instance.storageBucket}${prefix}`; + } else if (s3LikeProviders.includes(provider)) { + props.warehouse = `s3://${instance.storageBucket}${prefix}`; + } + } + } + } catch (connError) { + // eslint-disable-next-line no-console + console.error( + '[IcebergDatalakeService] cloud connection lookup error:', + connError, + ); + } + } + + return { props, env }; + } + + private static async buildPostgresCatalogUri( + config: Pick, + ): Promise { + if (!config.databaseConnectionId) { + throw new Error('ICEBERG_REQUIRED_FIELD: databaseConnectionId'); + } + const db = await loadDatabaseFile(); + const model = (db.connections ?? []).find( + (item) => item.id === config.databaseConnectionId, + ); + if (!model || model.connection.type !== 'postgres') { + throw new Error('ICEBERG_SQL_CONNECTION_INVALID'); + } + const connection = model.connection as PostgresConnection; + const username = await secureStorage.getCredential( + `db-user-${connection.name}`, + ); + const password = await secureStorage.getCredential( + `db-password-${connection.name}`, + ); + if (!username || !password) { + throw new Error('ICEBERG_SQL_CREDENTIALS_MISSING'); + } + const auth = `${encodeURIComponent(username)}:${encodeURIComponent( + password, + )}`; + const host = connection.host.trim(); + const database = encodeURIComponent(connection.database); + const sslMode = connection.ssl ? '?sslmode=require' : ''; + return `postgresql+psycopg2://${auth}@${host}:${connection.port}/${database}${sslMode}`; + } + + private static getCatalogCapability( + catalogType: IcebergInstanceConfig['catalogType'], + ): IcebergCatalogCapability { + const capability = IcebergDatalakeService.catalogCapabilities.find( + (item) => item.type === catalogType, + ); + if (!capability) { + throw new Error(`ICEBERG_CATALOG_UNSUPPORTED: ${catalogType}`); + } + return capability; + } + + private static validateCatalogWarehousePair( + config: Pick< + IcebergInstanceConfig, + | 'catalogType' + | 'storageType' + | 'catalogPath' + | 'endpoint' + | 'catalogName' + | 'databaseConnectionId' + | 'localPath' + | 'storageConnectionId' + | 'storageBucket' + | 'nessieReference' + | 'nessieWarehouse' + | 'hiveUri' + | 'hiveUgi' + >, + validateWarehouseFields = true, + ): void { + const capability = IcebergDatalakeService.getCatalogCapability( + config.catalogType, + ); + if (!capability.enabled) { + throw new Error(`ICEBERG_CATALOG_NOT_ENABLED: ${config.catalogType}`); + } + if (!capability.allowedStorageTypes.includes(config.storageType)) { + throw new Error( + `ICEBERG_WAREHOUSE_NOT_ALLOWED: ${config.catalogType}/${config.storageType}`, + ); + } + const missingField = capability.requiredFields.find( + (field) => !config[field]?.trim(), + ); + if (missingField) { + throw new Error(`ICEBERG_REQUIRED_FIELD: ${missingField}`); + } + if (config.catalogType === 'nessie') { + IcebergDatalakeService.buildNessieRestUri(config); + } + if (config.catalogType === 'hive') { + IcebergDatalakeService.buildHiveMetastoreUri(config); + if (config.hiveUgi && !/^[^:]+:[^:]+$/.test(config.hiveUgi.trim())) { + throw new Error('ICEBERG_HIVE_UGI_INVALID'); + } + } + if ( + validateWarehouseFields && + config.storageType === 'local' && + !config.localPath?.trim() + ) { + throw new Error('ICEBERG_REQUIRED_FIELD: localPath'); + } + if ( + validateWarehouseFields && + config.storageType === 'cloud' && + (!config.storageConnectionId?.trim() || !config.storageBucket?.trim()) + ) { + throw new Error( + 'ICEBERG_REQUIRED_FIELD: storageConnectionId/storageBucket', + ); + } + } + + private static validateCatalogAuthentication(config: { + catalogType: IcebergInstanceConfig['catalogType']; + catalogAuthMode?: IcebergInstanceConfig['catalogAuthMode']; + accessToken?: string; + catalogAccessTokenKey?: string; + oauthClientId?: string; + oauthClientSecret?: string; + oauthClientSecretKey?: string; + oauthServerUri?: string; + }): void { + const mode = config.catalogAuthMode ?? 'none'; + if ( + mode !== 'none' && + config.catalogType !== 'rest' && + config.catalogType !== 'polaris' && + config.catalogType !== 'lakekeeper' && + config.catalogType !== 'nessie' + ) { + throw new Error( + `ICEBERG_AUTH_MODE_NOT_ALLOWED: ${config.catalogType}/${mode}`, + ); + } + if ( + mode === 'token' && + !config.accessToken && + !config.catalogAccessTokenKey + ) { + throw new Error('ICEBERG_ACCESS_TOKEN_REQUIRED'); + } + if (mode === 'oauth-client-credentials') { + if (!config.oauthClientId?.trim() || config.oauthClientId.includes(':')) { + throw new Error('ICEBERG_OAUTH_CLIENT_ID_INVALID'); + } + if (!config.oauthClientSecret && !config.oauthClientSecretKey) { + throw new Error('ICEBERG_OAUTH_CLIENT_SECRET_REQUIRED'); + } + if (!config.oauthServerUri?.trim()) { + throw new Error('ICEBERG_OAUTH_SERVER_URI_REQUIRED'); + } + } + } + + static getCapabilities(): IcebergCapabilities { + return { + catalogs: IcebergDatalakeService.catalogCapabilities.map((item) => ({ + ...item, + authModes: [...item.authModes], + requiredFields: [...item.requiredFields], + allowedStorageTypes: [...item.allowedStorageTypes], + })), + cloudProviders: [...IcebergDatalakeService.cloudProviders], + }; + } + + // ───────────────────────────────────────────── + // Public: CRUD + // ───────────────────────────────────────────── + + static async listInstances(): Promise { + try { + const instances = await IcebergDatalakeService.readInstances(); + return instances.map( + ({ + id, + name, + description, + catalogType, + storageType, + catalogPath, + localPath, + storageBucket, + createdAt, + updatedAt, + }) => ({ + id, + name, + description, + catalogType, + storageType, + catalogPath, + localPath, + storageBucket, + createdAt, + updatedAt, + }), + ); + } catch (error) { + // eslint-disable-next-line no-console + console.error('[IcebergDatalakeService] listInstances error:', error); + throw error; + } + } + + static async getInstance(id: string): Promise { + try { + const instances = await IcebergDatalakeService.readInstances(); + const instance = instances.find((i) => i.id === id); + if (!instance) throw new Error(`Iceberg instance not found: ${id}`); + return instance; + } catch (error) { + // eslint-disable-next-line no-console + console.error('[IcebergDatalakeService] getInstance error:', error); + throw error; + } + } + + static async createInstance( + data: CreateIcebergInstanceDTO, + ): Promise { + try { + IcebergDatalakeService.validateCatalogWarehousePair(data); + IcebergDatalakeService.validateCatalogAuthentication(data); + const id = uuidv4(); + const now = new Date().toISOString(); + + let catalogAccessTokenKey: `iceberg-catalog-token-${string}` | undefined; + if (data.accessToken) { + catalogAccessTokenKey = `iceberg-catalog-token-${id}`; + await secureStorage.setCredential( + catalogAccessTokenKey, + data.accessToken, + ); + } + + let oauthClientSecretKey: `iceberg-oauth-secret-${string}` | undefined; + if (data.oauthClientSecret) { + oauthClientSecretKey = `iceberg-oauth-secret-${id}`; + await secureStorage.setCredential( + oauthClientSecretKey, + data.oauthClientSecret, + ); + } + + // Strip raw secrets before persisting + /* eslint-disable @typescript-eslint/no-unused-vars */ + const { + accessToken: _accessTokenCreate, + oauthClientSecret: _oauthClientSecretCreate, + ...rest + } = data; + /* eslint-enable @typescript-eslint/no-unused-vars */ + + const newInstance: IcebergInstanceConfig = { + ...rest, + id, + catalogAccessTokenKey: + catalogAccessTokenKey ?? data.catalogAccessTokenKey, + oauthClientSecretKey: oauthClientSecretKey ?? data.oauthClientSecretKey, + createdAt: now, + updatedAt: now, + }; + + const instances = await IcebergDatalakeService.readInstances(); + instances.push(newInstance); + await IcebergDatalakeService.writeInstances(instances); + + return newInstance; + } catch (error) { + // eslint-disable-next-line no-console + console.error('[IcebergDatalakeService] createInstance error:', error); + throw error; + } + } + + static async updateInstance( + id: string, + data: UpdateIcebergInstanceDTO, + ): Promise { + try { + const instances = await IcebergDatalakeService.readInstances(); + const idx = instances.findIndex((i) => i.id === id); + if (idx < 0) throw new Error(`Iceberg instance not found: ${id}`); + + const updatedConfig = { + ...instances[idx], + ...data, + }; + IcebergDatalakeService.validateCatalogWarehousePair(updatedConfig); + IcebergDatalakeService.validateCatalogAuthentication(updatedConfig); + + // Handle access token update + if (data.accessToken) { + const key = + instances[idx].catalogAccessTokenKey ?? `iceberg-catalog-token-${id}`; + await secureStorage.setCredential(key, data.accessToken); + instances[idx].catalogAccessTokenKey = key; + } + + if (data.oauthClientSecret) { + const key = + instances[idx].oauthClientSecretKey ?? `iceberg-oauth-secret-${id}`; + await secureStorage.setCredential(key, data.oauthClientSecret); + instances[idx].oauthClientSecretKey = key; + } + + // Strip raw secrets before persisting + /* eslint-disable @typescript-eslint/no-unused-vars */ + const { + accessToken: _accessTokenUpdate, + oauthClientSecret: _oauthClientSecretUpdate, + ...rest + } = data; + /* eslint-enable @typescript-eslint/no-unused-vars */ + + instances[idx] = { + ...instances[idx], + ...rest, + id, + updatedAt: new Date().toISOString(), + }; + + await IcebergDatalakeService.writeInstances(instances); + return instances[idx]; + } catch (error) { + // eslint-disable-next-line no-console + console.error('[IcebergDatalakeService] updateInstance error:', error); + throw error; + } + } + + static async deleteInstance(id: string): Promise { + try { + const instances = await IcebergDatalakeService.readInstances(); + const instance = instances.find((i) => i.id === id); + if (!instance) throw new Error(`Iceberg instance not found: ${id}`); + + if (instance.catalogAccessTokenKey) { + try { + await secureStorage.deleteCredential(instance.catalogAccessTokenKey); + } catch (keyError) { + // eslint-disable-next-line no-console + console.error( + '[IcebergDatalakeService] keytar delete error:', + keyError, + ); + } + } + if (instance.oauthClientSecretKey) { + try { + await secureStorage.deleteCredential(instance.oauthClientSecretKey); + } catch (keyError) { + // eslint-disable-next-line no-console + console.error( + '[IcebergDatalakeService] OAuth secret delete error:', + keyError, + ); + } + } + + const updated = instances.filter((i) => i.id !== id); + await IcebergDatalakeService.writeInstances(updated); + } catch (error) { + // eslint-disable-next-line no-console + console.error('[IcebergDatalakeService] deleteInstance error:', error); + throw error; + } + } + + // ───────────────────────────────────────────── + // Public: connection testing + // ───────────────────────────────────────────── + + static async testCatalogConnection( + params: IcebergTestCatalogParams, + ): Promise { + try { + const props: Record = {}; + const env: Record = {}; + let { oauthClientSecret } = params; + if ( + params.authMode === 'oauth-client-credentials' && + !oauthClientSecret && + params.instanceId + ) { + const existingInstance = await IcebergDatalakeService.getInstance( + params.instanceId, + ); + const matchesSavedAuthentication = + existingInstance.catalogType === params.catalogType && + existingInstance.endpoint === params.endpoint && + existingInstance.oauthClientId === params.oauthClientId && + existingInstance.oauthServerUri === params.oauthServerUri; + if (!matchesSavedAuthentication) { + throw new Error('ICEBERG_OAUTH_CLIENT_SECRET_REQUIRED'); + } + if (existingInstance.oauthClientSecretKey) { + oauthClientSecret = + (await secureStorage.getCredential( + existingInstance.oauthClientSecretKey, + )) ?? undefined; + } + } + + const storageType = + params.storageType ?? + (params.catalogType === 'sqlite' ? 'local' : 'server-managed'); + IcebergDatalakeService.validateCatalogWarehousePair( + { + ...params, + storageType, + }, + false, + ); + IcebergDatalakeService.validateCatalogAuthentication({ + catalogType: params.catalogType, + catalogAuthMode: params.authMode, + accessToken: params.accessToken, + oauthClientId: params.oauthClientId, + oauthClientSecret, + oauthServerUri: params.oauthServerUri, + }); + + switch (params.catalogType) { + case 'sqlite': + props.type = 'sql'; + if (params.catalogPath) { + props.uri = `sqlite:///${params.catalogPath}`; + props.warehouse = pathToFileURL( + path.join(path.dirname(params.catalogPath), 'warehouse'), + ).href; + } + break; + case 'sql': + props.type = 'sql'; + props.uri = '__ENV:ICEBERG_SQL_CATALOG_URI'; + env.ICEBERG_SQL_CATALOG_URI = + await IcebergDatalakeService.buildPostgresCatalogUri(params); + props.warehouse = pathToFileURL( + path.join(app.getPath('temp'), 'dbt-studio-iceberg-test'), + ).href; + break; + case 'rest': + case 'polaris': + case 'lakekeeper': + case 'nessie': + props.type = 'rest'; + if (params.catalogType === 'nessie') { + props.uri = IcebergDatalakeService.buildNessieRestUri(params); + props['header.X-Iceberg-Access-Delegation'] = 'remote-signing'; + } else { + if (params.endpoint) props.uri = params.endpoint; + if (params.catalogName) props.warehouse = params.catalogName; + if (params.catalogType === 'lakekeeper') { + props['header.X-Iceberg-Access-Delegation'] = 'remote-signing'; + } + } + if (params.accessToken) { + props.token = '__ENV:ICEBERG_ACCESS_TOKEN'; + // eslint-disable-next-line dot-notation + env['ICEBERG_ACCESS_TOKEN'] = params.accessToken; + } + if (params.authMode === 'oauth-client-credentials') { + if (!params.oauthClientId || !oauthClientSecret) { + throw new Error('ICEBERG_OAUTH_CLIENT_CREDENTIALS_REQUIRED'); + } + props.credential = '__ENV:ICEBERG_OAUTH_CREDENTIAL'; + env.ICEBERG_OAUTH_CREDENTIAL = `${params.oauthClientId}:${oauthClientSecret}`; + if (params.oauthServerUri) { + props['oauth2-server-uri'] = params.oauthServerUri; + } + if (params.oauthScope) props.scope = params.oauthScope; + delete props.token; + delete env.ICEBERG_ACCESS_TOKEN; + } + break; + case 'hive': + props.type = 'hive'; + props.uri = IcebergDatalakeService.buildHiveMetastoreUri(params); + if (params.hiveUgi?.trim()) props.ugi = params.hiveUgi.trim(); + break; + default: + throw new Error(`ICEBERG_CATALOG_NOT_ENABLED: ${params.catalogType}`); + } + + const result = (await IcebergDatalakeService.runBridge( + { + command: 'test_connection', + catalog_name: + params.catalogType === 'sqlite' + ? 'local' + : (params.catalogName ?? 'test'), + catalog_properties: props, + }, + env, + )) as Record; + return { + success: true, + catalogConnected: true, + warehouseConnected: + typeof result.warehouse_connected === 'boolean' + ? result.warehouse_connected + : undefined, + namespaceCount: Number(result.namespace_count ?? 0), + tableCount: Number(result.table_count ?? 0), + checkedAt: new Date().toISOString(), + }; + } catch (error) { + // eslint-disable-next-line no-console + console.error( + '[IcebergDatalakeService] testCatalogConnection error:', + error, + ); + return { + success: false, + catalogConnected: false, + warehouseConnected: false, + checkedAt: new Date().toISOString(), + error: String(error), + }; + } + } + + static async testStorageConnection( + params: IcebergTestStorageParams, + ): Promise { + try { + const connectionId = params.connectionId?.trim(); + const bucket = params.bucket?.trim(); + if (!connectionId || !bucket) { + throw new Error('ICEBERG_REQUIRED_FIELD: connectionId/bucket'); + } + + const { connection, config } = + await IcebergDatalakeService.resolveCloudStorageConnection( + connectionId, + ); + + const CloudExplorerService = (await import('./cloudExplorer.service')) + .default; + await CloudExplorerService.listObjects( + connection.provider, + config as unknown as CloudStorageConfig, + bucket, + undefined, + params.prefix?.trim() ?? '', + ); + return { + success: true, + warehouseConnected: true, + checkedAt: new Date().toISOString(), + }; + } catch (error) { + // eslint-disable-next-line no-console + console.error( + '[IcebergDatalakeService] testStorageConnection error:', + error, + ); + return { + success: false, + warehouseConnected: false, + checkedAt: new Date().toISOString(), + error: error instanceof Error ? error.message : String(error), + }; + } + } + + static async listStorageBuckets( + params: IcebergListStorageBucketsParams, + ): Promise { + const connectionId = params.connectionId?.trim(); + if (!connectionId) { + throw new Error('ICEBERG_REQUIRED_FIELD: connectionId'); + } + const { connection, config } = + await IcebergDatalakeService.resolveCloudStorageConnection(connectionId); + const CloudExplorerService = (await import('./cloudExplorer.service')) + .default; + const buckets = await CloudExplorerService.listBuckets( + connection.provider, + config as unknown as CloudStorageConfig, + ); + return buckets.map((bucket) => bucket.name); + } + + private static async resolveCloudStorageConnection(connectionId: string) { + const db = await loadDatabaseFile(); + const connection = (db.sources ?? []).find( + (source) => source.id === connectionId, + ); + if (!connection) throw new Error('ICEBERG_CLOUD_CONNECTION_NOT_FOUND'); + if ( + !IcebergDatalakeService.cloudProviders.includes( + connection.provider as (typeof IcebergDatalakeService.cloudProviders)[number], + ) + ) { + throw new Error('ICEBERG_CLOUD_PROVIDER_NOT_SUPPORTED'); + } + + const config = { ...connection.config } as Record; + const secret = await secureStorage.getCredential( + `cloud-${connection.provider}-${connection.id}`, + ); + if (connection.provider === 'azure') config.accountKey = secret; + else if (connection.provider === 'gcs') config.credentials = secret; + else if (connection.provider === 'backblaze-b2') { + config.applicationKey = secret; + } else config.secretAccessKey = secret; + + if (connection.provider === 'aws') { + const sessionToken = await secureStorage.getCredential( + `cloud-aws-session-${connection.id}`, + ); + if (sessionToken) config.sessionToken = sessionToken; + } + return { connection, config }; + } + + static async testInstanceConnection(id: string): Promise { + try { + const instance = await IcebergDatalakeService.getInstance(id); + const { props, env } = + await IcebergDatalakeService.buildCatalogProperties(instance); + const result = (await IcebergDatalakeService.runBridge( + { + command: 'test_connection', + catalog_name: + instance.catalogType === 'sqlite' + ? 'local' + : (instance.catalogName ?? id), + catalog_properties: props, + }, + env, + )) as Record; + return { + success: true, + catalogConnected: true, + warehouseConnected: + typeof result.warehouse_connected === 'boolean' + ? result.warehouse_connected + : undefined, + namespaceCount: Number(result.namespace_count ?? 0), + tableCount: Number(result.table_count ?? 0), + checkedAt: new Date().toISOString(), + }; + } catch (error) { + // eslint-disable-next-line no-console + console.error( + '[IcebergDatalakeService] testInstanceConnection error:', + error, + ); + return { + success: false, + catalogConnected: false, + warehouseConnected: false, + checkedAt: new Date().toISOString(), + error: String(error), + }; + } + } + + // ───────────────────────────────────────────── + // Public: pyiceberg installation + // ───────────────────────────────────────────── + + static async ensurePyicebergInstalled(): Promise<{ + installed: boolean; + version?: string; + }> { + // Fast path 1: in-process session cache + if (IcebergDatalakeService.installedCache?.installed) { + return IcebergDatalakeService.installedCache; + } + + try { + // Settings record the last successful installation for diagnostics, but + // do not prove the currently selected Python still has every required + // extra. Verify once per app session before trusting it. + const settings = await SettingsService.loadSettings(); + + // Check via Python bridge (runs pip only if the runtime profile is incomplete) + const checkResult = (await IcebergDatalakeService.runBridge({ + command: 'install_check', + })) as Record; + + if (checkResult.installed) { + const version = checkResult.version as string | undefined; + await SettingsService.saveSettings({ + ...settings, + icebergInstalled: true, + icebergVersion: version, + }); + IcebergDatalakeService.installedCache = { installed: true, version }; + return IcebergDatalakeService.installedCache; + } + + // Install pyiceberg with common extras + const pythonPath = await IcebergDatalakeService.getPythonPath(); + await new Promise((resolve, reject) => { + const child = spawn(pythonPath, [ + '-m', + 'pip', + 'install', + // Enabled SQL/Hive catalogs plus the current FileIO profile. + // --prefer-binary avoids slow source compilation where wheels exist. + 'pyiceberg[s3fs,sql-sqlite,sql-postgres,pyarrow,hive]>=0.10.0', + '--prefer-binary', + '--quiet', + ]); + child.on('close', (code: number) => { + if (code === 0) resolve(); + else reject(new Error(`pip install failed with exit code ${code}`)); + }); + child.on('error', (err: Error) => reject(err)); + }); + + // Verify installation + const verifyResult = (await IcebergDatalakeService.runBridge({ + command: 'install_check', + })) as Record; + + if (verifyResult.installed) { + const version = verifyResult.version as string | undefined; + const currentSettings = await SettingsService.loadSettings(); + await SettingsService.saveSettings({ + ...currentSettings, + icebergInstalled: true, + icebergVersion: version, + }); + IcebergDatalakeService.installedCache = { installed: true, version }; + return IcebergDatalakeService.installedCache; + } + + return { installed: false }; + } catch (error) { + // eslint-disable-next-line no-console + console.error( + '[IcebergDatalakeService] ensurePyicebergInstalled error:', + error, + ); + throw error; + } + } + + // ───────────────────────────────────────────── + // Public: table operations + // ───────────────────────────────────────────── + + static async listNamespaces( + id: string, + parent?: string[], + ): Promise { + try { + const instance = await IcebergDatalakeService.getInstance(id); + const { props, env } = + await IcebergDatalakeService.buildCatalogProperties(instance); + const result = (await IcebergDatalakeService.runBridge( + { + command: 'list_namespaces', + catalog_name: + instance.catalogType === 'sqlite' + ? 'local' + : (instance.catalogName ?? id), + catalog_properties: props, + parent: parent ?? [], + }, + env, + )) as Record; + return (result.namespaces as string[][]) ?? []; + } catch (error) { + // eslint-disable-next-line no-console + console.error('[IcebergDatalakeService] listNamespaces error:', error); + throw error; + } + } + + static async listTables(id: string, namespace: string[]): Promise { + try { + const instance = await IcebergDatalakeService.getInstance(id); + const { props, env } = + await IcebergDatalakeService.buildCatalogProperties(instance); + const result = (await IcebergDatalakeService.runBridge( + { + command: 'list_tables', + catalog_name: + instance.catalogType === 'sqlite' + ? 'local' + : (instance.catalogName ?? id), + catalog_properties: props, + namespace, + }, + env, + )) as Record; + return (result.tables as string[]) ?? []; + } catch (error) { + // eslint-disable-next-line no-console + console.error('[IcebergDatalakeService] listTables error:', error); + throw error; + } + } + + static async getTableSchema( + id: string, + namespace: string[], + table: string, + ): Promise { + try { + const instance = await IcebergDatalakeService.getInstance(id); + const { props, env } = + await IcebergDatalakeService.buildCatalogProperties(instance); + const result = (await IcebergDatalakeService.runBridge( + { + command: 'get_schema', + catalog_name: + instance.catalogType === 'sqlite' + ? 'local' + : (instance.catalogName ?? id), + catalog_properties: props, + namespace, + table, + }, + env, + )) as Record; + return { + fields: (result.fields as IcebergFieldSpec[]) ?? [], + properties: + (result.properties as Record | undefined) ?? {}, + }; + } catch (error) { + // eslint-disable-next-line no-console + console.error('[IcebergDatalakeService] getTableSchema error:', error); + throw error; + } + } + + static async getTableSnapshots( + id: string, + namespace: string[], + table: string, + ): Promise { + try { + const instance = await IcebergDatalakeService.getInstance(id); + const { props, env } = + await IcebergDatalakeService.buildCatalogProperties(instance); + const result = (await IcebergDatalakeService.runBridge( + { + command: 'get_snapshots', + catalog_name: + instance.catalogType === 'sqlite' + ? 'local' + : (instance.catalogName ?? id), + catalog_properties: props, + namespace, + table, + }, + env, + )) as Record; + return (result.snapshots as IcebergSnapshotInfo[]) ?? []; + } catch (error) { + // eslint-disable-next-line no-console + console.error('[IcebergDatalakeService] getTableSnapshots error:', error); + throw error; + } + } + + static async previewTable( + id: string, + namespace: string[], + table: string, + limit: number, + rowFilter?: string, + ): Promise { + try { + const safeLimit = Math.min( + Math.max(1, Math.floor(Number.isFinite(limit) ? limit : 100)), + 1000, + ); + const safeRowFilter = rowFilter?.trim(); + if (safeRowFilter && !/^[A-Za-z0-9_\s.'"<>=!()-]+$/.test(safeRowFilter)) { + throw new Error( + 'Invalid row filter. Use column names, literals, comparison operators, AND/OR, and parentheses only.', + ); + } + const instance = await IcebergDatalakeService.getInstance(id); + const { props, env } = + await IcebergDatalakeService.buildCatalogProperties(instance); + const result = (await IcebergDatalakeService.runBridge( + { + command: 'preview_table', + catalog_name: + instance.catalogType === 'sqlite' + ? 'local' + : (instance.catalogName ?? id), + catalog_properties: props, + namespace, + table, + limit: safeLimit, + row_filter: safeRowFilter || undefined, + }, + env, + )) as Record; + return { + columns: (result.columns as string[]) ?? [], + rows: (result.rows as unknown[][]) ?? [], + total: result.total as number | undefined, + }; + } catch (error) { + // eslint-disable-next-line no-console + console.error('[IcebergDatalakeService] previewTable error:', error); + throw error; + } + } + + // ───────────────────────────────────────────── + // Public: file helpers + // ───────────────────────────────────────────── + + // ───────────────────────────────────────────── + // Public: local file import + // ───────────────────────────────────────────── + + static async importTable( + id: string, + namespace: string[], + table: string, + filePath: string, + fileFormat: string, + ): Promise { + try { + const safeFormat = fileFormat?.toLowerCase() as + | IcebergImportFileFormat + | undefined; + if (!safeFormat || !['csv', 'parquet', 'json'].includes(safeFormat)) { + throw new Error('ICEBERG_IMPORT_FORMAT_UNSUPPORTED'); + } + const safeTable = table?.trim(); + if (!safeTable || !/^[A-Za-z_][A-Za-z0-9_]*$/.test(safeTable)) { + throw new Error('ICEBERG_IMPORT_TABLE_NAME_INVALID'); + } + const safeNamespace = IcebergDatalakeService.validateNamespace(namespace); + if (!filePath?.trim()) { + throw new Error('ICEBERG_IMPORT_FILE_REQUIRED'); + } + if ( + !fs.existsSync(filePath.trim()) || + !fs.statSync(filePath.trim()).isFile() + ) { + throw new Error('ICEBERG_IMPORT_FILE_NOT_FOUND'); + } + + const instance = await IcebergDatalakeService.getInstance(id); + const { props, env } = + await IcebergDatalakeService.buildCatalogProperties(instance); + const result = (await IcebergDatalakeService.runBridge( + { + command: 'import_table', + catalog_name: + instance.catalogType === 'sqlite' + ? 'local' + : (instance.catalogName ?? id), + catalog_properties: props, + namespace: safeNamespace, + table: safeTable, + file_path: filePath.trim(), + file_format: safeFormat, + }, + env, + )) as Record; + return { + namespace: (result.namespace as string[]) ?? safeNamespace, + table: (result.table as string) ?? safeTable, + rowCount: Number(result.row_count ?? 0), + columns: (result.columns as string[]) ?? [], + }; + } catch (error) { + // eslint-disable-next-line no-console + console.error('[IcebergDatalakeService] importTable error:', error); + throw error; + } + } + + // ───────────────────────────────────────────── + // Public: table maintenance + // ───────────────────────────────────────────── + + static async dropTable( + id: string, + namespace: string[], + table: string, + ): Promise { + try { + const safeTable = table?.trim(); + if (!safeTable || !/^[A-Za-z_][A-Za-z0-9_]*$/.test(safeTable)) { + throw new Error('ICEBERG_TABLE_NAME_INVALID'); + } + const safeNamespace = IcebergDatalakeService.validateNamespace(namespace); + const instance = await IcebergDatalakeService.getInstance(id); + const { props, env } = + await IcebergDatalakeService.buildCatalogProperties(instance); + const result = (await IcebergDatalakeService.runBridge( + { + command: 'drop_table', + catalog_name: + instance.catalogType === 'sqlite' + ? 'local' + : (instance.catalogName ?? id), + catalog_properties: props, + namespace: safeNamespace, + table: safeTable, + }, + env, + )) as Record; + return { + namespace: (result.namespace as string[]) ?? safeNamespace, + table: (result.table as string) ?? safeTable, + }; + } catch (error) { + // eslint-disable-next-line no-console + console.error('[IcebergDatalakeService] dropTable error:', error); + throw error; + } + } + + static async renameTable( + id: string, + namespace: string[], + table: string, + newTable: string, + ): Promise { + try { + const safeTable = table?.trim(); + if (!safeTable || !/^[A-Za-z_][A-Za-z0-9_]*$/.test(safeTable)) { + throw new Error('ICEBERG_TABLE_NAME_INVALID'); + } + const safeNewTable = newTable?.trim(); + if (!safeNewTable || !/^[A-Za-z_][A-Za-z0-9_]*$/.test(safeNewTable)) { + throw new Error('ICEBERG_TABLE_NAME_INVALID'); + } + if (safeNewTable === safeTable) { + throw new Error('ICEBERG_RENAME_SAME_NAME'); + } + const safeNamespace = IcebergDatalakeService.validateNamespace(namespace); + const instance = await IcebergDatalakeService.getInstance(id); + const { props, env } = + await IcebergDatalakeService.buildCatalogProperties(instance); + const result = (await IcebergDatalakeService.runBridge( + { + command: 'rename_table', + catalog_name: + instance.catalogType === 'sqlite' + ? 'local' + : (instance.catalogName ?? id), + catalog_properties: props, + namespace: safeNamespace, + table: safeTable, + new_table: safeNewTable, + }, + env, + )) as Record; + return { + namespace: (result.namespace as string[]) ?? safeNamespace, + table: (result.table as string) ?? safeNewTable, + }; + } catch (error) { + // eslint-disable-next-line no-console + console.error('[IcebergDatalakeService] renameTable error:', error); + throw error; + } + } + + static async createNamespace( + id: string, + namespace: string[], + ): Promise { + try { + const safeNamespace = IcebergDatalakeService.validateNamespace(namespace); + const instance = await IcebergDatalakeService.getInstance(id); + const { props, env } = + await IcebergDatalakeService.buildCatalogProperties(instance); + const result = (await IcebergDatalakeService.runBridge( + { + command: 'create_namespace', + catalog_name: + instance.catalogType === 'sqlite' + ? 'local' + : (instance.catalogName ?? id), + catalog_properties: props, + namespace: safeNamespace, + }, + env, + )) as Record; + return { + namespace: (result.namespace as string[]) ?? safeNamespace, + }; + } catch (error) { + // eslint-disable-next-line no-console + console.error('[IcebergDatalakeService] createNamespace error:', error); + throw error; + } + } + + static async dropNamespace( + id: string, + namespace: string[], + ): Promise { + try { + const safeNamespace = IcebergDatalakeService.validateNamespace(namespace); + const instance = await IcebergDatalakeService.getInstance(id); + const { props, env } = + await IcebergDatalakeService.buildCatalogProperties(instance); + const result = (await IcebergDatalakeService.runBridge( + { + command: 'drop_namespace', + catalog_name: + instance.catalogType === 'sqlite' + ? 'local' + : (instance.catalogName ?? id), + catalog_properties: props, + namespace: safeNamespace, + }, + env, + )) as Record; + return { + namespace: (result.namespace as string[]) ?? safeNamespace, + }; + } catch (error) { + // eslint-disable-next-line no-console + console.error('[IcebergDatalakeService] dropNamespace error:', error); + throw error; + } + } + + private static validateNamespace(namespace: string[]): string[] { + const safeNamespace = (namespace ?? []).map((part) => part.trim()); + if ( + safeNamespace.length === 0 || + safeNamespace.some((part) => !/^[A-Za-z_][A-Za-z0-9_]*$/.test(part)) + ) { + throw new Error('ICEBERG_NAMESPACE_INVALID'); + } + return safeNamespace; + } + + static async createMetadataFile( + warehousePath: string, + ): Promise { + try { + const result = (await IcebergDatalakeService.runBridge({ + command: 'create_metadata_file', + warehouse_path: warehousePath, + })) as Record; + return { + catalogPath: (result.metadata_path as string) ?? '', + warehousePath: (result.warehouse_path as string) ?? '', + namespaces: (result.namespaces as string[][]) ?? [], + tables: (result.tables as string[][]) ?? [], + }; + } catch (error) { + // eslint-disable-next-line no-console + console.error( + '[IcebergDatalakeService] createMetadataFile error:', + error, + ); + throw error; + } + } +} diff --git a/src/renderer/App.tsx b/src/renderer/App.tsx index 8347a536..2cb8b85f 100644 --- a/src/renderer/App.tsx +++ b/src/renderer/App.tsx @@ -111,7 +111,10 @@ const App: React.FC = () => { } /> } /> } /> - } /> + } + /> } diff --git a/src/renderer/components/cloudExplorer/ConnectionForm.tsx b/src/renderer/components/cloudExplorer/ConnectionForm.tsx index a5bf1cfe..83fa0b1d 100644 --- a/src/renderer/components/cloudExplorer/ConnectionForm.tsx +++ b/src/renderer/components/cloudExplorer/ConnectionForm.tsx @@ -59,6 +59,9 @@ interface ConnectionFormProps { connectionId?: string; duplicateFrom?: CloudConnection; suggestedName?: string; + initialProvider?: CloudProvider; + onSaved?: (connection: CloudConnection) => void | Promise; + onCancel?: () => void; } interface FormData { @@ -92,6 +95,9 @@ export const ConnectionForm: React.FC = ({ connectionId, duplicateFrom, suggestedName, + initialProvider, + onSaved, + onCancel, }) => { const navigate = useNavigate(); const saveConnection = useSaveConnection(); @@ -125,7 +131,11 @@ export const ConnectionForm: React.FC = ({ const [showPassword, setShowPassword] = useState(false); const [formData, setFormData] = useState({ name: initialValues?.name || suggestedName || '', - provider: initialValues?.provider || duplicateFrom?.provider || 'gcs', + provider: + initialValues?.provider || + duplicateFrom?.provider || + initialProvider || + 'gcs', projectId: '', credentials: '', region: '', @@ -545,7 +555,11 @@ export const ConnectionForm: React.FC = ({ lastUsed: new Date(), }; await saveConnection.mutateAsync(connection); - navigate('/app/cloud-explorer/connections'); + if (onSaved) { + await onSaved(connection); + } else { + navigate('/app/cloud-explorer/connections'); + } } catch (error) { setTestStatus('error'); setErrorMessage( @@ -1582,7 +1596,11 @@ export const ConnectionForm: React.FC = ({ diff --git a/src/renderer/components/dataLake/DataLakeConnectionSelector.tsx b/src/renderer/components/dataLake/DataLakeConnectionSelector.tsx index 7e99463b..a0f83df5 100644 --- a/src/renderer/components/dataLake/DataLakeConnectionSelector.tsx +++ b/src/renderer/components/dataLake/DataLakeConnectionSelector.tsx @@ -18,7 +18,6 @@ import { } from '@mui/material'; import { Add, - CloudQueue, Storage as StorageIcon, Close, CheckCircle, @@ -35,34 +34,46 @@ import { import useSecureStorage from '../../hooks/useSecureStorage'; import { CloudConnection, + CloudProvider, S3Config, AzureConfig, GCSConfig, } from '../../../types/frontend'; +import { cloudStorageImages } from '../../../../assets/connectionIcons'; +import { ConnectionForm } from '../cloudExplorer/ConnectionForm'; interface DataLakeConnectionSelectorProps { onSelectExisting: ( connectionId: string, bucket: string, prefix?: string, + provider?: CloudProvider, ) => void; - selectedProvider: 'aws' | 'azure' | 'gcs'; + selectedProvider: CloudProvider | 'all'; initialConnectionId?: string; initialBucket?: string; initialPrefix?: string; + loadBuckets?: (connectionId: string) => Promise; } -const getProviderIcon = (provider: string) => { - switch (provider) { - case 'aws': - return ; - case 'azure': - return ; - case 'gcs': - return ; - default: - return ; - } +type InlineProvider = 'aws' | 'azure' | 'gcs'; + +const isInlineProvider = ( + provider: CloudProvider | 'all', +): provider is InlineProvider => + provider === 'aws' || provider === 'azure' || provider === 'gcs'; + +const getProviderIcon = (provider: CloudProvider) => { + const iconSrc = cloudStorageImages[provider]; + return iconSrc ? ( + + ) : ( + + ); }; const getProviderLabel = (provider: string) => { @@ -73,6 +84,18 @@ const getProviderLabel = (provider: string) => { return 'Azure Blob Storage'; case 'gcs': return 'Google Cloud Storage'; + case 'minio': + return 'MinIO'; + case 'cloudflare-r2': + return 'Cloudflare R2'; + case 'backblaze-b2': + return 'Backblaze B2'; + case 'rustfs': + return 'rustfs'; + case 'garage': + return 'Garage'; + case 'all': + return 'Cloud Explorer'; default: return provider.toUpperCase(); } @@ -86,6 +109,7 @@ export const DataLakeConnectionSelector: React.FC< initialConnectionId, initialBucket, initialPrefix, + loadBuckets, }) => { const { data: connections, isLoading, refetch } = useCloudConnections(); const createConnection = useCreateCloudConnection(); @@ -99,12 +123,19 @@ export const DataLakeConnectionSelector: React.FC< // Filter connections by selected provider const filteredConnections = - connections?.filter((conn: any) => conn.provider === selectedProvider) || - []; + connections?.filter( + (conn: any) => + selectedProvider === 'all' || conn.provider === selectedProvider, + ) || []; + + const supportsInlineCreate = isInlineProvider(selectedProvider); const [selectedConnectionId, setSelectedConnectionId] = useState(''); const [bucket, setBucket] = useState(''); const [prefix, setPrefix] = useState(''); + const [bucketOptions, setBucketOptions] = useState([]); + const [isLoadingBuckets, setIsLoadingBuckets] = useState(false); + const [bucketLoadError, setBucketLoadError] = useState(null); // Modal state const [isModalOpen, setIsModalOpen] = useState(false); @@ -114,10 +145,44 @@ export const DataLakeConnectionSelector: React.FC< // Update parent when connection or bucket changes React.useEffect(() => { - if (selectedConnectionId && bucket) { - onSelectExisting(selectedConnectionId, bucket, prefix || undefined); + if (selectedConnectionId) { + const selectedConnection = connections?.find( + (connection: CloudConnection) => connection.id === selectedConnectionId, + ); + onSelectExisting( + selectedConnectionId, + bucket, + prefix || undefined, + selectedConnection?.provider, + ); } - }, [selectedConnectionId, bucket, prefix, onSelectExisting]); + }, [selectedConnectionId, bucket, prefix, connections, onSelectExisting]); + + useEffect(() => { + if (!selectedConnectionId || !loadBuckets) return undefined; + let cancelled = false; + setIsLoadingBuckets(true); + setBucketLoadError(null); + // eslint-disable-next-line no-void + void (async () => { + try { + const names = await loadBuckets(selectedConnectionId); + if (!cancelled) setBucketOptions(names); + } catch (error: unknown) { + if (!cancelled) { + setBucketOptions([]); + setBucketLoadError( + error instanceof Error ? error.message : 'Failed to load buckets.', + ); + } + } finally { + if (!cancelled) setIsLoadingBuckets(false); + } + })(); + return () => { + cancelled = true; + }; + }, [selectedConnectionId, loadBuckets]); // Sync initial values from parent when revisiting the step useEffect(() => { @@ -160,6 +225,8 @@ export const DataLakeConnectionSelector: React.FC< >('idle'); const handleTestConnection = async () => { + if (!isInlineProvider(selectedProvider)) return; + setTestError(null); setTestStatus('testing'); @@ -224,6 +291,8 @@ export const DataLakeConnectionSelector: React.FC< }; const handleSaveConnection = async () => { + if (!isInlineProvider(selectedProvider)) return; + setTestError(null); // Validate required fields @@ -563,7 +632,13 @@ export const DataLakeConnectionSelector: React.FC< Cloud Connection setBucket(event.target.value)} + disabled={!selectedConnectionId || isLoadingBuckets} + > + {isLoadingBuckets && Loading buckets…} + {!isLoadingBuckets && bucketOptions.length === 0 && ( + No buckets found + )} + {bucket && !bucketOptions.includes(bucket) && ( + + {bucket} + + )} + {bucketOptions.map((name) => ( + + {name} + + ))} + + + {getBucketHelperText()} + + + ) : ( + setBucket(e.target.value)} + required + helperText={getBucketHelperText()} + disabled={!selectedConnectionId} + /> + )} + {bucketLoadError && {bucketLoadError}} {/* Prefix Input */} setIsModalOpen(false)} maxWidth="sm" fullWidth + PaperProps={{ + sx: !supportsInlineCreate + ? { bgcolor: 'transparent', boxShadow: 'none' } + : undefined, + }} > - - Create New {getProviderLabel(selectedProvider)} Connection - setIsModalOpen(false)} - sx={{ position: 'absolute', right: 8, top: 8 }} - > - - - - - {renderNewConnectionForm()} - - {testStatus === 'error' && ( - }> - Connection Error - - {testError || - 'Failed to connect to storage provider. Please check your credentials.'} - - - )} - - {testStatus === 'success' && ( - }> - Connection Successful - - Successfully connected to storage provider. - - - )} - - - - - - + {!supportsInlineCreate ? ( + + setIsModalOpen(false)} + onSaved={async (connection) => { + await refetch(); + setSelectedConnectionId(connection.id); + setBucket(''); + setPrefix(''); + setIsModalOpen(false); + }} + /> + + ) : ( + <> + + Create New {getProviderLabel(selectedProvider)} Connection + setIsModalOpen(false)} + sx={{ position: 'absolute', right: 8, top: 8 }} + > + + + + + {renderNewConnectionForm()} + + {testStatus === 'error' && ( + }> + Connection Error + + {testError || + 'Failed to connect to storage provider. Please check your credentials.'} + + + )} + + {testStatus === 'success' && ( + }> + + Connection Successful + + + Successfully connected to storage provider. + + + )} + + + + + + + + )} ); diff --git a/src/renderer/components/dataLake/DataLakeDashboard.tsx b/src/renderer/components/dataLake/DataLakeDashboard.tsx index c19ea712..94a101f9 100644 --- a/src/renderer/components/dataLake/DataLakeDashboard.tsx +++ b/src/renderer/components/dataLake/DataLakeDashboard.tsx @@ -12,18 +12,19 @@ import { import { Storage, TableChart, - QueryStats, Settings, Dashboard, Add, Folder, } from '@mui/icons-material'; +import { IcebergIcon } from './iceberg/IcebergIcon'; import { cloudStorageImages, databaseIcons, } from '../../../../assets/connectionIcons'; import { icons } from '../../../../assets/icons'; import { DataLakeSVG } from '../sidebar/icons'; +import type { IcebergInstanceListItem } from '../../../types/iceberg'; interface DuckLakeInstance { id: string; @@ -37,6 +38,24 @@ interface DuckLakeInstance { updatedAt: string; } +type RecentDataLakeItem = + | { + id: string; + name: string; + lakeType: 'duck-lake'; + catalogLabel: string; + dataPath: string; + updatedAt: string; + } + | { + id: string; + name: string; + lakeType: 'iceberg'; + catalogLabel: string; + dataPath: string; + updatedAt: string; + }; + const getStorageIconForInstance = (dataPath: string) => { if (dataPath.startsWith('s3://')) { return ( @@ -71,22 +90,19 @@ const getStorageIconForInstance = (dataPath: string) => { return ; }; -interface DuckLakeDashboardProps { - instances?: DuckLakeInstance[]; +interface DataLakeDashboardProps { + duckLakeInstances?: DuckLakeInstance[]; + icebergInstances?: IcebergInstanceListItem[]; } -export const DataLakeDashboard: React.FC = ({ - instances = [], +export const DataLakeDashboard: React.FC = ({ + duckLakeInstances = [], + icebergInstances = [], }) => { const navigate = useNavigate(); - // Calculate statistics const stats = useMemo(() => { - const activeInstances = instances.filter( - (i) => i.status === 'active', - ).length; - const totalInstances = instances.length; - const catalogTypes = instances.reduce( + const duckLakeCatalogTypes = duckLakeInstances.reduce( (acc, instance) => { acc[instance.catalog.type] = (acc[instance.catalog.type] || 0) + 1; return acc; @@ -94,16 +110,92 @@ export const DataLakeDashboard: React.FC = ({ {} as Record, ); + const icebergCatalogTypes = icebergInstances.reduce( + (acc, instance) => { + const key = `iceberg-${instance.catalogType}`; + acc[key] = (acc[key] || 0) + 1; + return acc; + }, + {} as Record, + ); + + const catalogTypes = { ...duckLakeCatalogTypes, ...icebergCatalogTypes }; + + const recentItems: RecentDataLakeItem[] = [ + ...duckLakeInstances.map((instance) => ({ + id: instance.id, + name: instance.name, + lakeType: 'duck-lake' as const, + catalogLabel: instance.catalog.type.toUpperCase(), + dataPath: instance.dataPath, + updatedAt: instance.updatedAt, + })), + ...icebergInstances.map((instance) => ({ + id: instance.id, + name: instance.name, + lakeType: 'iceberg' as const, + catalogLabel: instance.catalogType.toUpperCase(), + dataPath: + instance.localPath || + instance.catalogPath || + instance.storageBucket || + instance.storageType, + updatedAt: instance.updatedAt, + })), + ].sort( + (a, b) => + new Date(b.updatedAt).getTime() - new Date(a.updatedAt).getTime(), + ); + return { - activeInstances, - totalInstances, + duckLakeCount: duckLakeInstances.length, + icebergCount: icebergInstances.length, + totalInstances: duckLakeInstances.length + icebergInstances.length, catalogTypes, + recentItems, }; - }, [instances]); + }, [duckLakeInstances, icebergInstances]); + + const getCatalogIcon = (type: string) => { + if (type.startsWith('iceberg-')) { + return ; + } + + let iconSrc; + switch (type.toLowerCase()) { + case 'duckdb': + iconSrc = databaseIcons.duckdb; + break; + case 'sqlite': + iconSrc = databaseIcons.sqlite; + break; + case 'postgres': + case 'postgresql': + iconSrc = databaseIcons.postgresql; + break; + default: + iconSrc = databaseIcons.duckdb; + } + return ( + + ); + }; + + const getCatalogLabel = (type: string, count: number) => { + if (type.startsWith('iceberg-')) { + const catalogType = type.replace('iceberg-', '').toUpperCase(); + return `${count} Iceberg ${catalogType} ${count === 1 ? 'catalog' : 'catalogs'}`; + } + return `${count} ${type.toUpperCase()} ${count === 1 ? 'catalog' : 'catalogs'}`; + }; return ( - {/* Header with title and manage datalakes button */} = ({ mb: 3, }} > - + DataLake Dashboard @@ -140,7 +226,6 @@ export const DataLakeDashboard: React.FC = ({ gap: 3, }} > - {/* Statistics Cards */} = ({ Total DataLakes - - - + } sx={{ pb: 1 }} @@ -193,7 +276,7 @@ export const DataLakeDashboard: React.FC = ({ - {instances.length} + {stats.duckLakeCount} = ({ component="img" src={icons.apacheIcebergLake} alt="Apache Iceberg" - sx={{ width: 16, height: 16, opacity: 0.4 }} + sx={{ + width: 16, + height: 16, + opacity: stats.icebergCount > 0 ? 1 : 0.4, + }} /> 0 ? 1 : 0.6 }} > Apache Iceberg 0 ? 500 : undefined, + opacity: stats.icebergCount > 0 ? 1 : 0.6, + }} > - 0 + {stats.icebergCount} = ({ - - - - Recent Queries - - - - } - sx={{ pb: 1 }} - /> - - - 0 - - - Coming soon - - - - = ({ gap: 0.5, }} > - {Object.entries(stats.catalogTypes).map(([type, count]) => { - const getCatalogIcon = () => { - let iconSrc; - switch (type.toLowerCase()) { - case 'duckdb': - iconSrc = databaseIcons.duckdb; - break; - case 'sqlite': - iconSrc = databaseIcons.sqlite; - break; - case 'postgres': - case 'postgresql': - iconSrc = databaseIcons.postgresql; - break; - default: - iconSrc = databaseIcons.duckdb; - } - return ( - - ); - }; - return ( - - {getCatalogIcon()} - - {count} {type.toUpperCase()}{' '} - {count === 1 ? 'catalog' : 'catalogs'} - - - ); - })} + {Object.entries(stats.catalogTypes).map(([type, count]) => ( + + {getCatalogIcon(type)} + + {getCatalogLabel(type, count)} + + + ))} - {/* Recent Activity */} = ({ mt: 3, }} > - {/* Recent DataLakes */} = ({ Recently updated DataLakes - - - + } /> @@ -508,15 +521,15 @@ export const DataLakeDashboard: React.FC = ({ sx={{ display: 'flex', flexDirection: 'column', height: '100%' }} > - {instances.length === 0 ? ( + {stats.recentItems.length === 0 ? ( No DataLakes created yet ) : ( - {instances.slice(0, 5).map((instance) => ( + {stats.recentItems.slice(0, 5).map((instance) => ( = ({ }} onClick={() => navigate( - `/app/data-lake/duck-lake/instances/${instance.id}`, + instance.lakeType === 'iceberg' + ? `/app/data-lake/iceberg/instances/${instance.id}` + : `/app/data-lake/duck-lake/instances/${instance.id}`, ) } > - {getStorageIconForInstance(instance.dataPath)} + {instance.lakeType === 'iceberg' ? ( + + ) : ( + getStorageIconForInstance(instance.dataPath) + )} {instance.name} - {instance.catalog.type.toUpperCase()} catalog + {instance.catalogLabel} catalog - + {instance.lakeType === 'iceberg' ? ( + + ) : ( + + )} {moment(instance.updatedAt).fromNow()} @@ -576,77 +605,9 @@ export const DataLakeDashboard: React.FC = ({ - - {/* Recent Queries */} - - - - - Recent Queries - - - Coming soon - - - - - } - /> - - - Coming Soon - - - Query history tracking is not yet implemented - - - - {/* Welcome Card for New Users */} - {instances.length === 0 && ( + {stats.totalInstances === 0 && ( = ({ /> - DataLake allows you to create and manage DataLakes with various - catalog backends including DuckDB, SQLite, and PostgreSQL. Start - by creating your first DataLake. + DataLake allows you to create and manage DuckLake and Apache + Iceberg instances. Start by creating your first DataLake. + + {previewQuery.isFetching && } + {previewQuery.isError && ( + {errorMessage(previewQuery.error)} + )} + {previewQuery.data && ( + + + + + {previewQuery.data.columns.map((column) => ( + {column} + ))} + + + + {previewQuery.data.rows.map((row, rowIndex) => ( + + {row.map((value, columnIndex) => ( + + {formatCellValue(value)} + + ))} + + ))} + +
+ + )} + + ); + + const renderProperties = () => { + if (schemaQuery.isLoading) { + return ; + } + const properties = Object.entries(schemaQuery.data?.properties ?? {}); + if (properties.length === 0) { + return No table properties defined.; + } + return ( + + + {properties.map(([key, value]) => ( + + {key} + {value} + + ))} + +
+ ); + }; + + return ( + + + + + navigate(`/app/data-lake/iceberg/instances/${instance.id}`) + } + > + + + + + + + {selection.table} + + + Namespace: {selection.namespace.join('.')} • Apache Iceberg + + + {getCurrentSnapshot(snapshotsQuery.data ?? []) ? ( + + ) : null} + + { + setTab(value); + updateTableViewParams({ tab: value }); + }} + sx={{ borderBottom: 1, borderColor: 'divider' }} + > + } label="Overview" iconPosition="start" /> + } label="Schema" iconPosition="start" /> + } label="Data" iconPosition="start" /> + } label="History" iconPosition="start" /> + } label="Properties" iconPosition="start" /> + + + {tab === 0 && renderOverview()} + {tab === 1 && renderSchema()} + {tab === 2 && renderPreview()} + {tab === 3 && renderHistory()} + {tab === 4 && renderProperties()} + + + ); +}; + +export const IcebergTableDetails: React.FC = () => { + const { instanceId = '', tableName = '' } = useParams<{ + instanceId: string; + tableName: string; + }>(); + const instanceQuery = useGetIcebergInstance(instanceId); + const identifier = decodeURIComponent(tableName); + const parts = identifier.split('.').filter(Boolean); + const table = parts.pop() ?? ''; + const namespace = parts; + + if (instanceQuery.isLoading) { + return ; + } + if ( + instanceQuery.isError || + !instanceQuery.data || + !table || + !namespace.length + ) { + return Iceberg table could not be loaded.; + } + return ( + + ); +}; + +export const IcebergDetail: React.FC = ({ + instance, + onEdit, + onDelete, +}) => { + const navigate = useNavigate(); + const queryClient = useQueryClient(); + const namespacesQuery = useListIcebergNamespaces(instance.id); + const testInstanceMutation = useTestIcebergInstance(); + const importTableMutation = useImportIcebergTable(); + const dropTableMutation = useDropIcebergTable(); + const renameTableMutation = useRenameIcebergTable(); + const createNamespaceMutation = useCreateIcebergNamespace(); + const dropNamespaceMutation = useDropIcebergNamespace(); + const [currentTab, setCurrentTab] = React.useState(0); + const [tableFilter, setTableFilter] = React.useState(''); + const [importWizardOpen, setImportWizardOpen] = React.useState(false); + const [tableToDelete, setTableToDelete] = + React.useState(null); + const [tableToRename, setTableToRename] = + React.useState(null); + const [newTableName, setNewTableName] = React.useState(''); + const [namespaceCreateOpen, setNamespaceCreateOpen] = React.useState(false); + const [newNamespaceName, setNewNamespaceName] = React.useState(''); + const [namespaceToDelete, setNamespaceToDelete] = React.useState< + string[] | null + >(null); + const [namespaceFilter, setNamespaceFilter] = React.useState( + null, + ); + const [groupByNamespace, setGroupByNamespace] = React.useState(false); + const namespaceNameValid = (() => { + const parts = newNamespaceName + .trim() + .split('.') + .map((part) => part.trim()) + .filter(Boolean); + return ( + parts.length > 0 && + parts.every((part) => /^[A-Za-z_][A-Za-z0-9_]*$/.test(part)) + ); + })(); + + const handleDeleteTable = () => { + if (!tableToDelete) return; + dropTableMutation.mutate( + { + id: instance.id, + namespace: tableToDelete.namespace, + table: tableToDelete.table, + }, + { + onSuccess: (result) => { + setTableToDelete(null); + toast.success( + `Deleted ${result.namespace.join('.')}.${result.table}`, + ); + }, + onError: (error) => { + toast.error(errorMessage(error)); + }, + }, + ); + }; + + const handleRenameTable = () => { + if (!tableToRename) return; + const trimmed = newTableName.trim(); + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(trimmed)) { + toast.error( + 'Table name must start with a letter or underscore and contain only letters, numbers, and underscores', + ); + return; + } + renameTableMutation.mutate( + { + id: instance.id, + namespace: tableToRename.namespace, + table: tableToRename.table, + newTable: trimmed, + }, + { + onSuccess: (result) => { + setTableToRename(null); + setNewTableName(''); + toast.success( + `Renamed to ${result.namespace.join('.')}.${result.table}`, + ); + }, + onError: (error) => { + toast.error(errorMessage(error)); + }, + }, + ); + }; + + const handleCreateNamespace = () => { + const parts = newNamespaceName + .trim() + .split('.') + .map((part) => part.trim()) + .filter(Boolean); + if ( + parts.length === 0 || + parts.some((part) => !/^[A-Za-z_][A-Za-z0-9_]*$/.test(part)) + ) { + toast.error( + 'Namespace parts must start with a letter or underscore and contain only letters, numbers, and underscores', + ); + return; + } + const fullName = parts.join('.'); + if (namespacesQuery.data?.some((ns) => ns.join('.') === fullName)) { + toast.error(`Namespace ${fullName} already exists`); + return; + } + createNamespaceMutation.mutate( + { + id: instance.id, + namespace: parts, + }, + { + onSuccess: (result) => { + setNamespaceCreateOpen(false); + setNewNamespaceName(''); + toast.success(`Created namespace ${result.namespace.join('.')}`); + }, + onError: (error) => { + toast.error(errorMessage(error)); + }, + }, + ); + }; + + const handleDropNamespace = () => { + if (!namespaceToDelete) return; + dropNamespaceMutation.mutate( + { + id: instance.id, + namespace: namespaceToDelete, + }, + { + onSuccess: (result) => { + setNamespaceToDelete(null); + // If the dropped namespace was the active filter, clear it so the + // table body does not render empty with no explanation. + setNamespaceFilter((current) => + current === result.namespace.join('.') ? null : current, + ); + toast.success(`Dropped namespace ${result.namespace.join('.')}`); + }, + onError: (error) => { + toast.error(errorMessage(error)); + }, + }, + ); + }; + + const handleImportTable = ( + namespace: string[], + table: string, + filePath: string, + fileFormat: IcebergImportFileFormat, + ) => { + importTableMutation.mutate( + { + id: instance.id, + namespace, + table, + filePath, + fileFormat, + }, + { + onSuccess: (result) => { + setImportWizardOpen(false); + toast.success( + `Imported ${result.rowCount} rows into ${result.namespace.join('.')}.${result.table}`, + ); + }, + onError: (error) => { + toast.error(errorMessage(error)); + }, + }, + ); + }; + + const refresh = async () => { + await queryClient.invalidateQueries(['iceberg', 'namespaces', instance.id]); + await queryClient.invalidateQueries(['iceberg', 'tables', instance.id]); + await queryClient.invalidateQueries(['iceberg', 'schema', instance.id]); + await queryClient.invalidateQueries(['iceberg', 'snapshots', instance.id]); + await queryClient.invalidateQueries(['iceberg', 'preview', instance.id]); + }; + + const openTable = (selection: SelectedTable) => { + const identifier = [...selection.namespace, selection.table].join('.'); + navigate( + `/app/data-lake/iceberg/instances/${instance.id}/tables/${encodeURIComponent(identifier)}`, + ); + }; + + const testConnection = async () => { + try { + const result = await testInstanceMutation.mutateAsync(instance.id); + if (result.success) { + toast.success('Iceberg catalog connection successful.'); + return; + } + toast.error(result.error ?? 'Iceberg catalog connection failed.'); + } catch (error) { + toast.error(errorMessage(error)); + } + }; + + const testResult = testInstanceMutation.data; + const testIndicatorColor = (() => { + if (testInstanceMutation.isLoading) return 'warning.main'; + if (!testResult) return 'grey.500'; + return testResult.success ? 'success.main' : 'error.main'; + })(); + + const connectionStatusIcon = (healthy?: boolean) => { + if (healthy === true) { + return ; + } + if (healthy === false) { + return ; + } + return ; + }; + let catalogStatusLabel = 'Not tested'; + if (testResult?.catalogConnected === true) { + catalogStatusLabel = 'Connected'; + } else if (testResult?.catalogConnected === false) { + catalogStatusLabel = 'Connection failed'; + } + let warehouseStatusLabel = 'Not verified'; + if (testResult?.warehouseConnected === true) { + warehouseStatusLabel = 'Accessible'; + } else if (testResult?.warehouseConnected === false) { + warehouseStatusLabel = 'Access failed'; + } + + const catalogImage = + icebergCatalogImages[ + instance.catalogType as keyof typeof icebergCatalogImages + ]; + const catalogIcon = catalogImage ? ( + + ) : ( + + ); + + const providerIcon = instance.cloudProvider + ? cloudStorageImages[instance.cloudProvider] + : undefined; + const warehouseIcon = (() => { + if (instance.storageType === 'local' || instance.storageType === 'nfs') { + return ; + } + return ; + })(); + let providerRowIcon = ; + if (providerIcon) { + providerRowIcon = ( + + ); + } else if (instance.storageType === 'local') { + providerRowIcon = ; + } + + const renderTables = () => { + if (namespacesQuery.isLoading) { + return ; + } + if (namespacesQuery.isError) { + return ( + {errorMessage(namespacesQuery.error)} + ); + } + if (namespacesQuery.data?.length === 0) { + return ( + + + No namespaces or tables found. Create a namespace or import a local + file to create your first table. + + + + + + + ); + } + return ( + <> + + setTableFilter(event.target.value)} + placeholder="Search by name or schema…" + slotProps={{ + input: { + startAdornment: ( + + + + ), + sx: { fontSize: '0.8125rem', height: '32px' }, + }, + }} + sx={{ + width: 280, + '& .MuiInputBase-input': { + paddingTop: '2px', + paddingBottom: '2px', + }, + '& .MuiOutlinedInput-root': { + minHeight: '32px', + }, + }} + /> + + + setGroupByNamespace(value === 'grouped') + } + sx={{ height: '32px' }} + > + + + + + + + + + + + + + + + + + + Namespaces: + + + } + label="All" + size="small" + color={!namespaceFilter ? 'primary' : 'default'} + variant={!namespaceFilter ? 'filled' : 'outlined'} + onClick={() => setNamespaceFilter(null)} + /> + + {namespacesQuery.data?.map((namespace) => { + const fullName = namespace.join('.'); + const active = namespaceFilter === fullName; + return ( + + } + label={fullName} + size="small" + color={active ? 'primary' : 'default'} + variant={active ? 'filled' : 'outlined'} + onClick={() => setNamespaceFilter(active ? null : fullName)} + onDelete={() => setNamespaceToDelete(namespace)} + deleteIcon={ + + } + /> + + ); + })} + + + + + + Name + Type + Schema + Rows + Size + Updated + Created + Actions + + + + {namespacesQuery.data + ?.filter( + (namespace) => + !namespaceFilter || namespace.join('.') === namespaceFilter, + ) + .map((namespace) => + groupByNamespace ? ( + + setNamespaceToDelete(ns)} + /> + setTableToDelete(selection)} + onRename={(selection) => { + setTableToRename(selection); + setNewTableName(selection.table); + }} + /> + + ) : ( + setTableToDelete(selection)} + onRename={(selection) => { + setTableToRename(selection); + setNewTableName(selection.table); + }} + /> + ), + )} + +
+ + + ); + }; + + const renderImportWizard = () => ( + setImportWizardOpen(false)} + instanceId={instance.id} + onImport={handleImportTable} + isLoading={importTableMutation.isLoading} + /> + ); + + // Operations that lock the UI behind a full-screen backdrop until they + // finish, so the user cannot double-submit, misclick, or navigate away + // mid-operation. The first in-flight operation's label is shown. + const blockingOperations = [ + { + isLoading: importTableMutation.isLoading, + label: 'Importing table…', + }, + { + isLoading: dropTableMutation.isLoading, + label: 'Deleting table…', + }, + { + isLoading: renameTableMutation.isLoading, + label: 'Renaming table…', + }, + { + isLoading: createNamespaceMutation.isLoading, + label: 'Creating namespace…', + }, + { + isLoading: dropNamespaceMutation.isLoading, + label: 'Dropping namespace…', + }, + { + isLoading: testInstanceMutation.isLoading, + label: 'Testing connection…', + }, + ]; + + let catalogIdentityLabel = 'Catalog Name'; + let catalogIdentityValue = instance.catalogName ?? 'Local catalog'; + if (instance.catalogType === 'nessie') { + catalogIdentityLabel = 'Reference'; + catalogIdentityValue = instance.nessieReference ?? 'main'; + } else if (instance.catalogType === 'hive') { + catalogIdentityLabel = 'Metastore URI'; + catalogIdentityValue = instance.hiveUri ?? '—'; + } + + return ( + + + {catalogImage ? ( + + ) : ( + + )} + + + {instance.name} + + + Apache Iceberg • {instance.catalogType.toUpperCase()} Catalog + + + + + + + + setCurrentTab(value)} + sx={{ borderBottom: 1, borderColor: 'divider' }} + > + + + + + + {currentTab === 0 && renderTables()} + {currentTab === 1 && ( + + + + + + + Health Status + + + + + + + {connectionStatusIcon(testResult?.catalogConnected)} + + + + + + {connectionStatusIcon(testResult?.warehouseConnected)} + + + + + + + + {testResult?.error && ( + + {testResult.error} + + )} + + + + + + + Catalog Configuration + + + + + {catalogIcon} + + + + + + + + + + {instance.catalogType === 'nessie' && ( + + + + + + + )} + {instance.catalogType === 'hive' && ( + + + + + + + )} + + + + + + + + + + + + + + Warehouse Configuration + + + + + {warehouseIcon} + + + + + + {providerRowIcon} + + + + + + + + + + + + + + )} + {currentTab === 2 && ( + + )} + + + + {renderImportWizard()} + + {/* Delete table confirmation dialog */} + setTableToDelete(null)} + maxWidth="sm" + fullWidth + > + Delete table + + + Are you sure you want to delete{' '} + + {tableToDelete?.namespace.join('.')}.{tableToDelete?.table} + + ? This removes the table and its metadata from the catalog. + + + + + + + + + {/* Rename table dialog */} + { + setTableToRename(null); + setNewTableName(''); + }} + maxWidth="sm" + fullWidth + > + Rename table + + + Rename{' '} + + {tableToRename?.namespace.join('.')}.{tableToRename?.table} + + + setNewTableName(e.target.value)} + disabled={renameTableMutation.isLoading} + autoFocus + /> + + + + + + + + {/* Create namespace dialog */} + { + setNamespaceCreateOpen(false); + setNewNamespaceName(''); + }} + maxWidth="sm" + fullWidth + > + Create namespace + + + Namespaces organize tables. Use dot notation for nested namespaces + (for example analytics.daily). + + setNewNamespaceName(e.target.value)} + onKeyDown={(event) => { + if (event.key === 'Enter' && !createNamespaceMutation.isLoading) { + handleCreateNamespace(); + } + }} + disabled={createNamespaceMutation.isLoading} + helperText="Letters, numbers, underscores, and dots only" + /> + + + {' '} + + + + + {/* Drop namespace confirmation dialog */} + setNamespaceToDelete(null)} + maxWidth="sm" + fullWidth + > + Drop namespace + + + Are you sure you want to drop namespace{' '} + {namespaceToDelete?.join('.')}? + + + The namespace must be empty. Delete its tables first, otherwise the + catalog will reject the drop. Dropping a parent namespace also + removes its nested namespaces. + + + + + + + + + {/* Full-screen lock while any Iceberg operation is in flight */} + + + ); +}; diff --git a/src/renderer/components/dataLake/iceberg/IcebergIcon.tsx b/src/renderer/components/dataLake/iceberg/IcebergIcon.tsx new file mode 100644 index 00000000..bc4ef33b --- /dev/null +++ b/src/renderer/components/dataLake/iceberg/IcebergIcon.tsx @@ -0,0 +1,15 @@ +import React from 'react'; +import { Box } from '@mui/material'; +import { icons } from '../../../../../assets'; + +export const IcebergIcon: React.FC<{ + size?: number; + alt?: string; +}> = ({ size = 18, alt = 'Apache Iceberg' }) => ( + +); diff --git a/src/renderer/components/dataLake/iceberg/IcebergOperationBackdrop.tsx b/src/renderer/components/dataLake/iceberg/IcebergOperationBackdrop.tsx new file mode 100644 index 00000000..9b7dee0b --- /dev/null +++ b/src/renderer/components/dataLake/iceberg/IcebergOperationBackdrop.tsx @@ -0,0 +1,44 @@ +import { Backdrop, CircularProgress, Typography } from '@mui/material'; +import React from 'react'; + +export interface IcebergOperation { + /** Whether this operation is currently in flight. */ + isLoading: boolean; + /** Label shown on the backdrop while the operation runs. */ + label: string; +} + +interface IcebergOperationBackdropProps { + /** The operations that can block the UI; the first in-flight one wins. */ + operations: IcebergOperation[]; +} + +/** + * Full-screen loading backdrop that locks the UI while an Iceberg operation + * (import, rename, delete, namespace create/drop, connection test) is running. + * Rendered above everything (drawer + 999) so the user cannot interact with + * the app or navigate until the operation finishes and the UI is unlocked. + */ +export const IcebergOperationBackdrop: React.FC< + IcebergOperationBackdropProps +> = ({ operations }) => { + const active = operations.find((operation) => operation.isLoading); + return ( + theme.zIndex.drawer + 999, // above everything incl. sidebar + dialogs + display: 'flex', + flexDirection: 'column', + alignItems: 'center', + gap: 2, + }} + > + + + {active?.label ?? 'Working…'} + + + ); +}; diff --git a/src/renderer/components/dataLake/iceberg/IcebergTableImportWizard.tsx b/src/renderer/components/dataLake/iceberg/IcebergTableImportWizard.tsx new file mode 100644 index 00000000..db83bcaa --- /dev/null +++ b/src/renderer/components/dataLake/iceberg/IcebergTableImportWizard.tsx @@ -0,0 +1,487 @@ +import React, { useState } from 'react'; +import { + Box, + Button, + Dialog, + DialogTitle, + DialogContent, + DialogActions, + Stepper, + Step, + StepLabel, + TextField, + Typography, + Alert, + Card, + CardContent, + CardActionArea, + IconButton, + CircularProgress, + Autocomplete, + Chip, +} from '@mui/material'; +import { + ArrowBack, + ArrowForward, + Folder, + FolderOpen, + Close, + CloudDownload, + Cloud, + TableChart, +} from '@mui/icons-material'; +import { useFilePicker } from '../../../controllers/settings.controller'; +import { useListIcebergNamespaces } from '../../../controllers/icebergDatalake.controller'; +import type { IcebergImportFileFormat } from '../../../../types/iceberg'; + +interface IcebergTableImportWizardProps { + open: boolean; + onClose: () => void; + onImport: ( + namespace: string[], + tableName: string, + filePath: string, + fileFormat: IcebergImportFileFormat, + ) => void; + isLoading?: boolean; + instanceId: string; +} + +const steps = ['Select Source', 'Configure Import', 'Review']; + +const SUPPORTED_EXTENSIONS: { + ext: string; + format: IcebergImportFileFormat; +}[] = [ + { ext: 'csv', format: 'csv' }, + { ext: 'parquet', format: 'parquet' }, + { ext: 'pq', format: 'parquet' }, + { ext: 'json', format: 'json' }, +]; + +const detectFormat = ( + filePath: string, +): IcebergImportFileFormat | undefined => { + const ext = filePath.toLowerCase().split('.').pop() ?? ''; + return SUPPORTED_EXTENSIONS.find((item) => item.ext === ext)?.format; +}; + +export const IcebergTableImportWizard: React.FC< + IcebergTableImportWizardProps +> = ({ open, onClose, onImport, isLoading = false, instanceId }) => { + const [activeStep, setActiveStep] = useState(0); + const [tableName, setTableName] = useState(''); + const [filePath, setFilePath] = useState(''); + const [selectedNamespace, setSelectedNamespace] = useState( + '__new__', + ); + const [newNamespace, setNewNamespace] = useState('default'); + const [error, setError] = useState(''); + + const { mutate: getFiles } = useFilePicker(); + const namespacesQuery = useListIcebergNamespaces(instanceId); + const namespaces = (namespacesQuery.data ?? []).map((ns) => ns.join('.')); + + const isCreatingNamespace = + selectedNamespace === '__new__' || selectedNamespace === null; + + const handleFileSelect = () => { + getFiles( + { + properties: ['openFile'], + filters: [ + { name: 'Data Files', extensions: ['csv', 'parquet', 'pq', 'json'] }, + { name: 'All Files', extensions: ['*'] }, + ], + }, + { + onSuccess: (filePaths) => { + if (filePaths && filePaths.length > 0) { + setFilePath(filePaths[0]); + } + }, + }, + ); + }; + + const handleNext = () => { + if (activeStep === 1) { + if (!tableName.trim()) { + setError('Table name is required'); + return; + } + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(tableName.trim())) { + setError( + 'Table name must start with a letter or underscore and contain only letters, numbers, and underscores', + ); + return; + } + if (!filePath.trim()) { + setError('File path is required'); + return; + } + if (!detectFormat(filePath)) { + setError('Unsupported file type. Use CSV, Parquet, or JSON.'); + return; + } + if (isCreatingNamespace) { + const namespaceParts = newNamespace + .split('.') + .map((part) => part.trim()) + .filter(Boolean); + if ( + namespaceParts.length === 0 || + namespaceParts.some((part) => !/^[A-Za-z_][A-Za-z0-9_]*$/.test(part)) + ) { + setError( + 'Namespace must be dot-separated identifiers (letters, numbers, underscores)', + ); + return; + } + } + } + setError(''); + setActiveStep((prev) => prev + 1); + }; + + const handleBack = () => { + setError(''); + setActiveStep((prev) => prev - 1); + }; + + const handleImport = () => { + const namespace: string[] = isCreatingNamespace + ? newNamespace + .split('.') + .map((part) => part.trim()) + .filter(Boolean) + : (selectedNamespace ?? 'default').split('.'); + const format = detectFormat(filePath); + if (!format) return; + onImport(namespace, tableName.trim(), filePath.trim(), format); + }; + + const resetForm = () => { + setActiveStep(0); + setTableName(''); + setFilePath(''); + setSelectedNamespace('__new__'); + setNewNamespace('default'); + setError(''); + }; + + const renderSourceSelection = () => ( + + + Choose where to import data from + + + + + + + Local File + + + Import from local file system + + + Supports: CSV, Parquet, JSON + + + + + + + + + + From Object Storage + + + Import from a Cloud Explorer bucket or object path + + + + + + + + ); + + const renderConfigureImport = () => ( + + setTableName(e.target.value)} + placeholder="e.g., customers, orders, products" + helperText="Enter a valid Iceberg table name" + autoFocus + sx={{ mb: 3 }} + /> + + { + setSelectedNamespace(value ?? null); + if (value === '__new__' || value === null) { + setNewNamespace('default'); + } + }} + getOptionLabel={(option) => + option === '__new__' ? 'Create new namespace…' : option + } + isOptionEqualToValue={(option, value) => option === value} + renderOption={(props, option) => ( +
  • + {option === '__new__' ? 'Create new namespace…' : option} +
    • + )} + renderInput={(params) => ( + + )} + /> + + {isCreatingNamespace && ( + setNewNamespace(e.target.value)} + placeholder="e.g., default or analytics.raw" + helperText="Nested namespaces are separated by dots" + sx={{ mb: 3 }} + /> + )} + + setFilePath(e.target.value)} + placeholder="/path/to/file.csv" + helperText="Absolute path to CSV, Parquet, or JSON file" + slotProps={{ + input: { + endAdornment: ( + + + + ), + }, + }} + /> + {filePath && ( + + + + )} + + + + PyIceberg will automatically infer the file schema, create the table, + and append the data as the initial snapshot. + + +     + ); + + const renderReview = () => { + const namespace = isCreatingNamespace + ? newNamespace + .split('.') + .map((part) => part.trim()) + .filter(Boolean) + : (selectedNamespace ?? 'default').split('.'); + return ( + + + Review Import Configuration + + + + + Table Identifier + + + {namespace.join('.')}.{tableName} + + + + Source File + + + {filePath || 'Unavailable'} + + + + Detected Format + + + {detectFormat(filePath)?.toUpperCase() ?? '—'} + + + + + + What will happen: + + + 1. PyIceberg reads the source file and infers the schema +
    + 2. The namespace is created if it does not exist +
    + 3. An Iceberg table is created with the inferred schema +
    + 4. Data is appended and an initial snapshot is committed +     +     + + + + This operation may take some time depending on the data size. + + +     + ); + }; + + const renderStepContent = () => { + switch (activeStep) { + case 0: + return renderSourceSelection(); + case 1: + return renderConfigureImport(); + case 2: + return renderReview(); + default: + return null; + } + }; + + return ( + + + + Import Data to Iceberg + + + + {steps.map((label) => ( + + {label} + + ))} + + + {error && ( + setError('')}> + {error} + + )} + + {renderStepContent()} + + + + {activeStep > 0 && ( + + )} + {activeStep < steps.length - 1 ? ( + + ) : ( + + )} + + + ); +}; diff --git a/src/renderer/components/dataLake/index.ts b/src/renderer/components/dataLake/index.ts index ed6003cc..0b5986aa 100644 --- a/src/renderer/components/dataLake/index.ts +++ b/src/renderer/components/dataLake/index.ts @@ -8,3 +8,5 @@ export * from './DataLakeInstanceDetails'; export * from './DataLakeInstanceEditForm'; export * from './DataLakeTableImportWizard'; export * from './DataLakeTableDetails'; +export * from './IcebergConnectionWizard'; +export * from './IcebergInstanceListItem'; diff --git a/src/renderer/components/dataLakeCards/index.tsx b/src/renderer/components/dataLakeCards/index.tsx index cf5c5a13..a5f9fd1b 100644 --- a/src/renderer/components/dataLakeCards/index.tsx +++ b/src/renderer/components/dataLakeCards/index.tsx @@ -1,5 +1,5 @@ import React from 'react'; -import { Typography, Tooltip } from '@mui/material'; +import { Chip, Typography, Tooltip } from '@mui/material'; import { StyledCard, ContentWrapper, @@ -16,6 +16,7 @@ interface DataLakeTypeDetails { description: string; img: keyof typeof icons; disabled?: boolean; + beta?: boolean; } type Props = { @@ -44,8 +45,21 @@ export const DataLakeCard: React.FC = ({ itemDetails, onClick }) => { {itemDetails.disabled && Soon} - + {itemDetails.name} + {itemDetails.beta && ( + + )} diff --git a/src/renderer/components/sidebar/index.tsx b/src/renderer/components/sidebar/index.tsx index 13ac697d..99074b35 100644 --- a/src/renderer/components/sidebar/index.tsx +++ b/src/renderer/components/sidebar/index.tsx @@ -23,6 +23,7 @@ import { StyledNavLink, } from './styles'; import { useAppContext } from '../../hooks'; +import { useLastDataLakeRoute } from '../../hooks/useLastDataLakeRoute'; import { useGetSelectedProject } from '../../controllers'; import { logo, rosettaIcon } from '../../../../assets'; @@ -43,6 +44,7 @@ export const Sidebar: React.FC = ({ const location = useLocation(); const [isBarExpanded, setIsBarExpanded] = React.useState(false); + const lastDataLakeRoute = useLastDataLakeRoute(); const isProjectSelected = Boolean(selectedProject?.id); @@ -71,6 +73,8 @@ export const Sidebar: React.FC = ({ const renderItem = (element: (typeof mainElements)[0], isActive: boolean) => { const isDisabled = element.disabled; + const targetPath = + element.path === '/app/data-lake' ? lastDataLakeRoute : element.path; const listItem = ( = ({ const wrapped = ( + useQuery(['iceberg', 'list'], icebergService.listIcebergInstances); + +export const useIcebergCapabilities = () => + useQuery(['iceberg', 'capabilities'], icebergService.getIcebergCapabilities, { + staleTime: Infinity, + }); + +export const useGetIcebergInstance = (id: string) => + useQuery( + ['iceberg', 'instance', id], + () => icebergService.getIcebergInstance(id), + { enabled: !!id }, + ); + +export const useListIcebergNamespaces = (id: string, parent?: string[]) => + useQuery( + ['iceberg', 'namespaces', id, parent ?? []], + () => icebergService.listIcebergNamespaces(id, parent), + { enabled: !!id, staleTime: 60_000 }, + ); + +export const useListIcebergTables = (id: string, namespace: string[]) => + useQuery( + ['iceberg', 'tables', id, namespace], + () => icebergService.listIcebergTables(id, namespace), + { enabled: !!id && namespace.length > 0, staleTime: 60_000 }, + ); + +export const useGetIcebergSchema = ( + id: string, + namespace: string[], + table: string, +) => + useQuery( + ['iceberg', 'schema', id, namespace, table], + () => icebergService.getIcebergTableSchema(id, namespace, table), + { enabled: !!id && !!table }, + ); + +export const useGetIcebergSnapshots = ( + id: string, + namespace: string[], + table: string, +) => + useQuery( + ['iceberg', 'snapshots', id, namespace, table], + () => icebergService.getIcebergTableSnapshots(id, namespace, table), + { enabled: !!id && !!table }, + ); + +export const useIcebergTablePreview = ( + id: string, + namespace: string[], + table: string, + limit: number, + rowFilter: string, + enabled: boolean, +) => + useQuery( + ['iceberg', 'preview', id, namespace, table, limit, rowFilter], + () => + icebergService.previewIcebergTable( + id, + namespace, + table, + limit, + rowFilter || undefined, + ), + { + enabled: enabled && !!id && !!table, + staleTime: 30_000, + cacheTime: 30 * 60_000, + }, + ); + +// ───────────────────────────────────────────── +// Mutations +// ───────────────────────────────────────────── + +export const useCreateIcebergInstance = () => { + const qc = useQueryClient(); + return useMutation( + (data: CreateIcebergInstanceDTO) => + icebergService.createIcebergInstance(data), + { + onSuccess: () => qc.invalidateQueries(['iceberg', 'list']), + }, + ); +}; + +export const useUpdateIcebergInstance = () => { + const qc = useQueryClient(); + return useMutation( + ({ id, data }: { id: string; data: Partial }) => + icebergService.updateIcebergInstance(id, data), + { + onSuccess: (_result, { id }) => { + qc.invalidateQueries(['iceberg', 'list']); + qc.invalidateQueries(['iceberg', 'instance', id]); + }, + }, + ); +}; + +export const useDeleteIcebergInstance = () => { + const qc = useQueryClient(); + return useMutation((id: string) => icebergService.deleteIcebergInstance(id), { + onSuccess: () => qc.invalidateQueries(['iceberg', 'list']), + }); +}; + +export const useTestIcebergCatalog = () => + useMutation((params: IcebergTestCatalogParams) => + icebergService.testIcebergCatalog(params), + ); + +export const useTestIcebergStorage = () => + useMutation((params: IcebergTestStorageParams) => + icebergService.testIcebergStorage(params), + ); + +export const useListIcebergStorageBuckets = () => + useMutation((params: IcebergListStorageBucketsParams) => + icebergService.listIcebergStorageBuckets(params), + ); + +export const useTestIcebergInstance = () => + useMutation((id: string) => icebergService.testIcebergInstance(id)); + +export const useCreateIcebergMetadataFile = () => + useMutation((warehousePath: string) => + icebergService.createIcebergMetadataFile(warehousePath), + ); + +export const useImportIcebergTable = () => { + const qc = useQueryClient(); + return useMutation( + ({ + id, + namespace, + table, + filePath, + fileFormat, + }: IcebergImportTableParams) => + icebergService.importIcebergTable( + id, + namespace, + table, + filePath, + fileFormat, + ), + + { + onSuccess: (_result, { id }) => { + qc.invalidateQueries(['iceberg', 'tables', id]); + qc.invalidateQueries(['iceberg', 'schema', id]); + qc.invalidateQueries(['iceberg', 'snapshots', id]); + qc.invalidateQueries(['iceberg', 'preview', id]); + qc.invalidateQueries(['iceberg', 'namespaces', id]); + }, + }, + ); +}; + +export const usePreviewIcebergTable = () => + useMutation( + ({ + id, + namespace, + table, + limit, + rowFilter, + }: { + id: string; + namespace: string[]; + table: string; + limit: number; + rowFilter?: string; + }) => + icebergService.previewIcebergTable( + id, + namespace, + table, + limit, + rowFilter, + ), + ); + +export const useDropIcebergTable = () => { + const qc = useQueryClient(); + return useMutation( + ({ + id, + namespace, + table, + }: { + id: string; + namespace: string[]; + table: string; + }) => icebergService.dropIcebergTable(id, namespace, table), + { + onSuccess: (_result, { id }) => { + qc.invalidateQueries(['iceberg', 'tables', id]); + qc.invalidateQueries(['iceberg', 'schema', id]); + qc.invalidateQueries(['iceberg', 'snapshots', id]); + qc.invalidateQueries(['iceberg', 'preview', id]); + }, + }, + ); +}; + +export const useRenameIcebergTable = () => { + const qc = useQueryClient(); + return useMutation( + ({ + id, + namespace, + table, + newTable, + }: { + id: string; + namespace: string[]; + table: string; + newTable: string; + }) => icebergService.renameIcebergTable(id, namespace, table, newTable), + { + onSuccess: (_result, { id }) => { + qc.invalidateQueries(['iceberg', 'tables', id]); + qc.invalidateQueries(['iceberg', 'schema', id]); + qc.invalidateQueries(['iceberg', 'snapshots', id]); + qc.invalidateQueries(['iceberg', 'preview', id]); + }, + }, + ); +}; + +export const useCreateIcebergNamespace = () => { + const qc = useQueryClient(); + return useMutation( + ({ id, namespace }: IcebergCreateNamespaceParams) => + icebergService.createIcebergNamespace(id, namespace), + { + onSuccess: (_result, { id }) => { + qc.invalidateQueries(['iceberg', 'namespaces', id]); + qc.invalidateQueries(['iceberg', 'tables', id]); + }, + }, + ); +}; + +export const useDropIcebergNamespace = () => { + const qc = useQueryClient(); + return useMutation( + ({ id, namespace }: IcebergDropNamespaceParams) => + icebergService.dropIcebergNamespace(id, namespace), + { + onSuccess: (_result, { id }) => { + qc.invalidateQueries(['iceberg', 'namespaces', id]); + qc.invalidateQueries(['iceberg', 'tables', id]); + }, + }, + ); +}; + +export const useEnsureIcebergInstalled = () => + useMutation(() => icebergService.ensureIcebergInstalled()); + +/** + * FE-05 — Install gate hook. + * Wraps useMutation side-effect in a dedicated controller hook so components + * never call `mutate` directly inside a `useEffect`. + * Has a built-in 5-second timeout so the banner never blocks indefinitely. + */ +export const useEnsureIcebergInstalledOnMount = () => { + const { mutate, isLoading, data } = useEnsureIcebergInstalled(); + const hasRun = React.useRef(false); + const [timedOut, setTimedOut] = React.useState(false); + + React.useEffect(() => { + if (!hasRun.current) { + hasRun.current = true; + mutate(undefined, { + onError: () => { + // Suppress — install errors are not fatal for the DataLake UI + }, + }); + // Safety timeout — dismiss banner after 5 seconds regardless + const timer = setTimeout(() => setTimedOut(true), 5000); + return () => clearTimeout(timer); + } + return undefined; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + return { + // Hide banner if timed out or mutation resolved + isInstalling: isLoading && !timedOut, + installResult: data, + }; +}; diff --git a/src/renderer/hooks/useLastDataLakeRoute.ts b/src/renderer/hooks/useLastDataLakeRoute.ts new file mode 100644 index 00000000..448abfb8 --- /dev/null +++ b/src/renderer/hooks/useLastDataLakeRoute.ts @@ -0,0 +1,44 @@ +import React from 'react'; +import { useLocation } from 'react-router-dom'; + +const STORAGE_KEY = 'dbt-studio:last-data-lake-route'; +const DEFAULT_ROUTE = '/app/data-lake/dashboard'; + +const isRestorableRoute = (route: string) => { + const pathname = route.split('?')[0]; + return ( + pathname.startsWith('/app/data-lake/') && + !pathname.includes('/new-instance') && + !pathname.includes('/history') && + !pathname.endsWith('/edit') + ); +}; + +const readLastRoute = () => { + try { + const route = window.localStorage.getItem(STORAGE_KEY); + return route && isRestorableRoute(route) ? route : DEFAULT_ROUTE; + } catch { + return DEFAULT_ROUTE; + } +}; + +export const useLastDataLakeRoute = () => { + const { pathname, search } = useLocation(); + const currentRoute = `${pathname}${search}`; + const route = isRestorableRoute(currentRoute) + ? currentRoute + : readLastRoute(); + + React.useEffect(() => { + if (!isRestorableRoute(currentRoute)) return; + + try { + window.localStorage.setItem(STORAGE_KEY, currentRoute); + } catch { + // Fall back to the dashboard when persistent storage is unavailable. + } + }, [currentRoute]); + + return route; +}; diff --git a/src/renderer/screens/dataLake/index.tsx b/src/renderer/screens/dataLake/index.tsx index 15adc352..d094edcd 100644 --- a/src/renderer/screens/dataLake/index.tsx +++ b/src/renderer/screens/dataLake/index.tsx @@ -1,6 +1,19 @@ import React, { useState, useEffect } from 'react'; -import { Typography, Box, Button, styled } from '@mui/material'; +import { + Typography, + Box, + Button, + styled, + Dialog, + DialogContent, + DialogTitle, + DialogActions, + DialogContentText, + Alert, + CircularProgress, +} from '@mui/material'; import { useLocation, useParams, useNavigate } from 'react-router-dom'; +import { toast } from 'react-toastify'; import { AppLayout } from '../../layouts'; import { DataLakeDashboard, @@ -12,6 +25,11 @@ import { DataLakeInstanceEditForm, DataLakeTableDetails, } from '../../components/dataLake'; +import { IcebergConnectionWizard } from '../../components/dataLake/IcebergConnectionWizard'; +import { + IcebergDetail, + IcebergTableDetails, +} from '../../components/dataLake/iceberg/IcebergDetail'; import { DataLakeCard } from '../../components/dataLakeCards'; import { useDuckLakeInstances, @@ -19,7 +37,23 @@ import { useDuckLakeInstance, useDeleteDuckLakeInstance, } from '../../controllers'; +import { + useListIcebergInstances, + useCreateIcebergInstance, + useGetIcebergInstance, + useUpdateIcebergInstance, + useDeleteIcebergInstance, + useEnsureIcebergInstalledOnMount, +} from '../../controllers/icebergDatalake.controller'; import { DuckLakeService } from '../../services'; +import type { + CreateIcebergInstanceDTO, + IcebergCatalogType, + IcebergCloudProvider, + IcebergInstanceConfig, + IcebergStorageType, +} from '../../../types/iceberg'; +import type { IcebergWizardData } from '../../components/dataLake/IcebergConnectionWizard'; const DataLake: React.FC = () => { const location = useLocation(); @@ -35,29 +69,47 @@ const DataLake: React.FC = () => { // State for type selection in new-instance flow const [selectedType, setSelectedType] = useState(); - // React Query hooks + // ── Iceberg UI state ─────────────────────────────────────────────────── + const [icebergEditId, setIcebergEditId] = useState(null); + const [icebergDeleteTarget, setIcebergDeleteTarget] = useState<{ + id: string; + name: string; + } | null>(null); + + // ── React Query — DuckLake ───────────────────────────────────────────── const instancesQuery = useDuckLakeInstances(); - // Add type field to instances for routing const instances = (instancesQuery.data || []).map((i) => ({ ...i, - type: 'duck-lake', // Hardcoded for now since only DuckLake exists + type: 'duck-lake', })); const createInstanceMutation = useCreateDuckLakeInstance(); - - // Mutations for instance actions const deleteMutation = useDeleteDuckLakeInstance(); - // Parse the current section from the pathname + // ── React Query — Iceberg ────────────────────────────────────────────── + const { data: icebergInstances = [] } = useListIcebergInstances(); + const createIcebergMutation = useCreateIcebergInstance(); + const updateIcebergMutation = useUpdateIcebergInstance(); + const deleteIcebergMutation = useDeleteIcebergInstance(); + const { + data: editInstanceData, + isLoading: editInstanceLoading, + error: editInstanceError, + } = useGetIcebergInstance(icebergEditId ?? ''); + const activeIcebergId = type === 'iceberg' ? (instanceId ?? '') : ''; + const { + data: icebergInstance, + isLoading: icebergDetailLoading, + error: icebergDetailError, + } = useGetIcebergInstance(activeIcebergId); + + // ── Install gate (FE-05 pattern) ─────────────────────────────────────── + const { isInstalling } = useEnsureIcebergInstalledOnMount(); + + // ── Path parsing ─────────────────────────────────────────────────────── const pathSegments = location.pathname.split('/'); const currentSection = (() => { - if (pathSegments.includes('new-instance')) { - return 'new-instance'; - } - // Check for edit route pattern: /app/duck-lake/instances/:id/edit - if (pathSegments.includes('edit')) { - return 'edit-instance'; - } - // Check for table detail route pattern: /app/duck-lake/instances/:id/tables/:tableName + if (pathSegments.includes('new-instance')) return 'new-instance'; + if (pathSegments.includes('edit')) return 'edit-instance'; if ( pathSegments.includes('instances') && pathSegments.includes('tables') && @@ -65,31 +117,30 @@ const DataLake: React.FC = () => { ) { return 'table-detail'; } - // Check for tables route pattern: /app/duck-lake/instances/:id/tables if (pathSegments.includes('instances') && pathSegments.includes('tables')) { return 'instance-tables'; } if (pathSegments.includes('instances') && pathSegments.length > 4) { return 'instance-detail'; } - if (pathSegments.includes('instances')) { - return 'instances'; - } - if (pathSegments.includes('tables')) { - return 'tables'; - } - if (pathSegments.includes('history')) { - return 'history'; - } - if (pathSegments.includes('instance') && pathSegments.length > 4) { + if (pathSegments.includes('instances')) return 'instances'; + if (pathSegments.includes('tables')) return 'tables'; + if (pathSegments.includes('instance') && pathSegments.length > 4) return 'instance-detail'; - } - if (pathSegments.includes('table') && pathSegments.length > 4) { + if (pathSegments.includes('table') && pathSegments.length > 4) return 'table-detail'; - } return pathSegments.pop() || 'dashboard'; })(); + // Pre-select lake type when navigating e.g. /new-instance?type=iceberg + useEffect(() => { + if (currentSection !== 'new-instance') return; + const typeParam = new URLSearchParams(location.search).get('type'); + if (typeParam === 'iceberg') { + setSelectedType('iceberg'); + } + }, [currentSection, location.search]); + // Define data lake types (UI only) const dataLakeTypes = [ { @@ -104,7 +155,8 @@ const DataLake: React.FC = () => { name: 'Apache Iceberg', description: 'Multi-engine, cloud-agnostic open standard', img: 'apacheIcebergLake' as const, - disabled: true, + disabled: false, // now enabled + beta: true, }, { id: 'delta', @@ -122,7 +174,6 @@ const DataLake: React.FC = () => { }, ]; - // Styled container for cards (reuse from addConnection pattern) const ConnectionCardsContainer = styled(Box)` display: flex; justify-content: center; @@ -133,24 +184,22 @@ const DataLake: React.FC = () => { margin: 0 auto; `; - // Get current instance ID from params or path const currentInstanceId = instanceId || pathSegments[pathSegments.indexOf('instance') + 1]; - // Get instance details if viewing a specific instance - const instanceQuery = useDuckLakeInstance( - currentSection === 'instance-detail' ? currentInstanceId || '' : '', - ); + const duckLakeInstanceId = + type !== 'iceberg' && currentSection === 'instance-detail' + ? currentInstanceId || '' + : ''; + + const instanceQuery = useDuckLakeInstance(duckLakeInstanceId); const currentInstance = instanceQuery.data; - // DuckLake connection lifecycle management - // Acquire connection when viewing instance details, tables, or table details - // Release connection when navigating away or component unmounts + // DuckLake connection lifecycle management (skip for Iceberg instances) useEffect(() => { let acquiredInstanceId: string | null = null; const acquireConnectionForInstance = async () => { - // Check if we're viewing any page that uses a DuckLake instance connection const instanceViewingSections = [ 'instance-detail', 'instance-tables', @@ -159,6 +208,7 @@ const DataLake: React.FC = () => { ]; if ( + type !== 'iceberg' && instanceViewingSections.includes(currentSection) && (instanceId || currentInstanceId) ) { @@ -176,35 +226,150 @@ const DataLake: React.FC = () => { acquireConnectionForInstance(); - // Cleanup: release connection when navigating away or component unmounts return () => { if (acquiredInstanceId) { DuckLakeService.releaseConnection(acquiredInstanceId); } }; - }, [currentSection, instanceId, currentInstanceId]); + }, [currentSection, instanceId, currentInstanceId, type]); + + // ── Iceberg handlers ─────────────────────────────────────────────────── + + const handleIcebergWizardComplete = async (wizardData: IcebergWizardData) => { + const dto: CreateIcebergInstanceDTO = { + name: wizardData.basics.name, + description: wizardData.basics.description, + catalogType: wizardData.catalog.catalogType as IcebergCatalogType, + catalogPath: wizardData.catalog.catalogPath, + endpoint: wizardData.catalog.endpoint, + catalogName: wizardData.catalog.catalogName, + databaseConnectionId: wizardData.catalog.databaseConnectionId, + catalogAuthMode: wizardData.catalog.authMode, + accessToken: wizardData.catalog.accessToken, + oauthClientId: wizardData.catalog.oauthClientId, + oauthClientSecret: wizardData.catalog.oauthClientSecret, + oauthServerUri: wizardData.catalog.oauthServerUri, + oauthScope: wizardData.catalog.oauthScope, + nessieReference: wizardData.catalog.nessieReference, + nessieWarehouse: wizardData.catalog.nessieWarehouse, + hiveUri: wizardData.catalog.hiveUri, + hiveUgi: wizardData.catalog.hiveUgi, + catalogConnectionId: wizardData.catalog.polarisConnectionId, + catalogBucket: wizardData.catalog.polarisBucket, + catalogPrefix: wizardData.catalog.polarisPrefix, + storageType: wizardData.storage.storageType as IcebergStorageType, + localPath: wizardData.storage.localPath, + cloudProvider: wizardData.storage.cloudProvider as IcebergCloudProvider, + storageConnectionId: wizardData.storage.connectionId, + storageBucket: wizardData.storage.bucket, + storagePrefix: wizardData.storage.prefix, + }; + try { + const created = await createIcebergMutation.mutateAsync(dto); + setSelectedType(undefined); + toast.success('Iceberg instance created.'); + navigate(`/app/data-lake/iceberg/instances/${created.id}`); + } catch (err: any) { + // eslint-disable-next-line no-console + console.error(err); + toast.error(err?.message ?? 'Failed to create Iceberg instance.'); + } + }; + + const handleIcebergEditComplete = async (wizardData: IcebergWizardData) => { + if (!icebergEditId) return; + const dto: Partial = { + name: wizardData.basics.name, + description: wizardData.basics.description, + catalogType: wizardData.catalog.catalogType as IcebergCatalogType, + catalogPath: wizardData.catalog.catalogPath, + endpoint: wizardData.catalog.endpoint, + catalogName: wizardData.catalog.catalogName, + databaseConnectionId: wizardData.catalog.databaseConnectionId, + catalogAuthMode: wizardData.catalog.authMode, + // Only send token if the user typed a new one; empty = preserve existing + ...(wizardData.catalog.accessToken + ? { accessToken: wizardData.catalog.accessToken } + : {}), + ...(wizardData.catalog.oauthClientSecret + ? { oauthClientSecret: wizardData.catalog.oauthClientSecret } + : {}), + oauthClientId: wizardData.catalog.oauthClientId, + oauthServerUri: wizardData.catalog.oauthServerUri, + oauthScope: wizardData.catalog.oauthScope, + nessieReference: wizardData.catalog.nessieReference, + nessieWarehouse: wizardData.catalog.nessieWarehouse, + hiveUri: wizardData.catalog.hiveUri, + hiveUgi: wizardData.catalog.hiveUgi, + catalogConnectionId: wizardData.catalog.polarisConnectionId, + catalogBucket: wizardData.catalog.polarisBucket, + catalogPrefix: wizardData.catalog.polarisPrefix, + storageType: wizardData.storage.storageType as IcebergStorageType, + localPath: wizardData.storage.localPath, + cloudProvider: wizardData.storage.cloudProvider as IcebergCloudProvider, + storageConnectionId: wizardData.storage.connectionId, + storageBucket: wizardData.storage.bucket, + storagePrefix: wizardData.storage.prefix, + }; + try { + await updateIcebergMutation.mutateAsync({ id: icebergEditId, data: dto }); + setIcebergEditId(null); + toast.success('Iceberg instance updated.'); + } catch (err: any) { + // eslint-disable-next-line no-console + console.error(err); + toast.error(err?.message ?? 'Failed to update Iceberg instance.'); + } + }; + + const handleIcebergDeleteConfirm = async () => { + if (!icebergDeleteTarget) return; + try { + await deleteIcebergMutation.mutateAsync(icebergDeleteTarget.id); + if (activeIcebergId === icebergDeleteTarget.id) { + navigate('/app/data-lake/instances'); + } + setIcebergDeleteTarget(null); + toast.success('Iceberg instance deleted.'); + } catch (err: any) { + // eslint-disable-next-line no-console + console.error(err); + toast.error(err?.message ?? 'Failed to delete Iceberg instance.'); + } + }; - // Tables are now handled by DuckLakeTablesView component + // ── Render helpers ───────────────────────────────────────────────────── + + const renderIcebergInstanceDetail = (inst: IcebergInstanceConfig) => ( + setIcebergEditId(inst.id)} + onDelete={() => setIcebergDeleteTarget({ id: inst.id, name: inst.name })} + /> + ); - // Render content based on current section const renderContent = () => { switch (currentSection) { case 'dashboard': - return ; + return ( + + ); case 'instances': - return ; + return ( + setIcebergEditId(id)} /> + ); case 'instance-tables': - // Show tables for a specific instance from route: /instances/:id/tables return ; case 'tables': - // Show tables for a specific instance if instanceId is in URL if (instanceId) { return ; } - // Otherwise show message to select an instance return ( { ); - case 'history': - return ( - - - Query History - - - Query history functionality coming soon... - - - ); - case 'new-instance': - // Step 1: Show type selection cards + // Step 1: type selection cards if (!selectedType) { return ( @@ -260,7 +409,7 @@ const DataLake: React.FC = () => { ); } - // Step 2: Show wizard for selected type (only duck-lake is implemented) + // Step 2: DuckLake wizard if (selectedType === 'duck-lake') { return ( { }; const newInstance = await createInstanceMutation.mutateAsync(createRequest); - // Navigate to type-specific route navigate( `/app/data-lake/duck-lake/instances/${newInstance.id}`, ); @@ -295,6 +443,21 @@ const DataLake: React.FC = () => { ); } + // Step 2: Iceberg wizard (inline, same pattern as DuckLake) + if (selectedType === 'iceberg') { + return ( + { + setSelectedType(undefined); + navigate('/app/data-lake/new-instance'); + }} + isLoading={createIcebergMutation.isLoading} + mode="create" + /> + ); + } + // Other types not yet implemented return ( @@ -312,6 +475,42 @@ const DataLake: React.FC = () => { return ; case 'instance-detail': + if (type === 'iceberg') { + if (icebergDetailLoading) { + return ( + + + + ); + } + + if (icebergDetailError || !icebergInstance) { + return ( + + + Iceberg Instance Not Found + + + The requested Iceberg instance could not be found. + + + + ); + } + + return renderIcebergInstanceDetail(icebergInstance); + } + if (instanceQuery.isLoading) { return ( @@ -362,8 +561,11 @@ const DataLake: React.FC = () => { ); case 'table-detail': - // Phase 8b: Render comprehensive table detail view - return ; + return type === 'iceberg' ? ( + + ) : ( + + ); default: return ( @@ -385,12 +587,104 @@ const DataLake: React.FC = () => { return ( } + sidebarContent={ + + } panelTitle="DataLake" > + {/* pyiceberg install banner */} + {isInstalling && ( + } + sx={{ mb: 2 }} + > + Installing pyiceberg into the managed Python environment… This may + take a moment. + + )} + {renderContent()} + + {/* ── Iceberg Edit Wizard Dialog ────────────────────────────────── */} + setIcebergEditId(null)} + maxWidth="md" + fullWidth + > + + {editInstanceLoading && ( + + + + )} + {!editInstanceLoading && (editInstanceError || !editInstanceData) && ( + + + {editInstanceError instanceof Error + ? editInstanceError.message + : 'Failed to load the Iceberg instance.'} + + + + )} + {!editInstanceLoading && !editInstanceError && editInstanceData && ( + setIcebergEditId(null)} + isLoading={updateIcebergMutation.isLoading} + mode="edit" + initialData={editInstanceData} + /> + )} + + + + {/* ── Iceberg Delete Confirmation Dialog ───────────────────────── */} + setIcebergDeleteTarget(null)} + maxWidth="xs" + fullWidth + > + Delete Iceberg Instance + + + Delete Iceberg instance {icebergDeleteTarget?.name} + ? This cannot be undone. Keytar credentials for this instance will + also be removed. + + + + + + + ); }; diff --git a/src/renderer/services/iceberg.service.ts b/src/renderer/services/iceberg.service.ts new file mode 100644 index 00000000..85ba2a34 --- /dev/null +++ b/src/renderer/services/iceberg.service.ts @@ -0,0 +1,176 @@ +/** + * Iceberg renderer service + * Named exports wrapping window.electron.ipcRenderer.invoke — no default exports. + * Follows the renderer service naming pattern (import * as icebergService). + */ + +import type { + CreateIcebergInstanceDTO, + IcebergInstanceListItem, + IcebergInstanceConfig, + IcebergTestCatalogParams, + IcebergTestStorageParams, + IcebergListStorageBucketsParams, + IcebergTestResult, + IcebergSchemaResult, + IcebergSnapshotInfo, + IcebergPreviewResult, + IcebergLocalCatalogResult, + IcebergCapabilities, + IcebergImportTableResult, + IcebergImportFileFormat, + IcebergTableOperationResult, + IcebergNamespaceOperationResult, +} from '../../types/iceberg'; + +export const getIcebergCapabilities = (): Promise => + window.electron.ipcRenderer.invoke('iceberg:getCapabilities'); + +export const listIcebergInstances = (): Promise => + window.electron.ipcRenderer.invoke('iceberg:list'); + +export const getIcebergInstance = ( + id: string, +): Promise => + window.electron.ipcRenderer.invoke('iceberg:get', id); + +export const createIcebergInstance = ( + data: CreateIcebergInstanceDTO, +): Promise => + window.electron.ipcRenderer.invoke('iceberg:create', data); + +export const updateIcebergInstance = ( + id: string, + data: Partial, +): Promise => + window.electron.ipcRenderer.invoke('iceberg:update', id, data); + +export const deleteIcebergInstance = (id: string): Promise => + window.electron.ipcRenderer.invoke('iceberg:delete', id); + +export const testIcebergCatalog = ( + params: IcebergTestCatalogParams, +): Promise => + window.electron.ipcRenderer.invoke('iceberg:testCatalog', params); + +export const testIcebergStorage = ( + params: IcebergTestStorageParams, +): Promise => + window.electron.ipcRenderer.invoke('iceberg:testStorage', params); + +export const listIcebergStorageBuckets = ( + params: IcebergListStorageBucketsParams, +): Promise => + window.electron.ipcRenderer.invoke('iceberg:listStorageBuckets', params); + +export const testIcebergInstance = (id: string): Promise => + window.electron.ipcRenderer.invoke('iceberg:testInstance', id); + +export const listIcebergNamespaces = ( + id: string, + parent?: string[], +): Promise => + window.electron.ipcRenderer.invoke('iceberg:listNamespaces', id, parent); + +export const listIcebergTables = ( + id: string, + namespace: string[], +): Promise => + window.electron.ipcRenderer.invoke('iceberg:listTables', id, namespace); + +export const getIcebergTableSchema = ( + id: string, + namespace: string[], + table: string, +): Promise => + window.electron.ipcRenderer.invoke('iceberg:getSchema', id, namespace, table); + +export const getIcebergTableSnapshots = ( + id: string, + namespace: string[], + table: string, +): Promise => + window.electron.ipcRenderer.invoke( + 'iceberg:getSnapshots', + id, + namespace, + table, + ); + +export const previewIcebergTable = ( + id: string, + namespace: string[], + table: string, + limit: number, + rowFilter?: string, +): Promise => + window.electron.ipcRenderer.invoke( + 'iceberg:previewTable', + id, + namespace, + table, + limit, + rowFilter, + ); + +export const importIcebergTable = ( + id: string, + namespace: string[], + table: string, + filePath: string, + fileFormat: IcebergImportFileFormat, +): Promise => + window.electron.ipcRenderer.invoke( + 'iceberg:importTable', + id, + namespace, + table, + filePath, + fileFormat, + ); + +export const dropIcebergTable = ( + id: string, + namespace: string[], + table: string, +): Promise => + window.electron.ipcRenderer.invoke('iceberg:dropTable', id, namespace, table); + +export const renameIcebergTable = ( + id: string, + namespace: string[], + table: string, + newTable: string, +): Promise => + window.electron.ipcRenderer.invoke( + 'iceberg:renameTable', + id, + namespace, + table, + newTable, + ); + +export const createIcebergNamespace = ( + id: string, + namespace: string[], +): Promise => + window.electron.ipcRenderer.invoke('iceberg:createNamespace', id, namespace); + +export const dropIcebergNamespace = ( + id: string, + namespace: string[], +): Promise => + window.electron.ipcRenderer.invoke('iceberg:dropNamespace', id, namespace); + +export const createIcebergMetadataFile = ( + warehousePath: string, +): Promise => + window.electron.ipcRenderer.invoke( + 'iceberg:createMetadataFile', + warehousePath, + ); + +export const ensureIcebergInstalled = (): Promise<{ + installed: boolean; + version?: string; +}> => window.electron.ipcRenderer.invoke('iceberg:ensureInstalled'); diff --git a/src/renderer/services/index.ts b/src/renderer/services/index.ts index 86016633..599ee92a 100644 --- a/src/renderer/services/index.ts +++ b/src/renderer/services/index.ts @@ -13,6 +13,7 @@ import { DuckLakeService } from './duckLake.service'; import * as lineageService from './lineage.service'; import * as languageIntelligenceService from './languageIntelligence.service'; import * as agentService from './agent.service'; +import * as icebergService from './iceberg.service'; export { settingsServices, @@ -30,4 +31,5 @@ export { lineageService, languageIntelligenceService, agentService, + icebergService, }; diff --git a/src/types/backend.ts b/src/types/backend.ts index 9ed3447d..993d44c8 100644 --- a/src/types/backend.ts +++ b/src/types/backend.ts @@ -1,5 +1,6 @@ import { QueryResult } from 'pg'; import { CloudConnection, RecentItem } from './frontend'; +import type { IcebergInstanceConfig } from './iceberg'; export type SupportedConnectionTypes = | 'postgres' @@ -358,6 +359,8 @@ export type SettingsType = { flowfileAutoStart?: string; kisqlPath?: string; kisqlVersion?: string; + icebergInstalled?: boolean; + icebergVersion?: string; }; export type FileDialogProperties = 'openFile' | 'openDirectory'; @@ -380,6 +383,7 @@ export type DataBase = { connections: ConnectionModel[]; sources: CloudConnection[]; recentItems: RecentItem[]; + icebergInstances?: IcebergInstanceConfig[]; }; // Rosetta Version Management Types diff --git a/src/types/frontend.ts b/src/types/frontend.ts index d3cb885f..7c396289 100644 --- a/src/types/frontend.ts +++ b/src/types/frontend.ts @@ -75,6 +75,8 @@ export type SecureStorageAccount = | `cloud-backblaze-b2-${string}` | `cloud-rustfs-${string}` | `cloud-garage-${string}` + | `iceberg-oauth-secret-${string}` + | `iceberg-catalog-token-${string}` | `db-bigquery-${string}` | `db-host-${string}` | `db-port-${string}` diff --git a/src/types/iceberg.ts b/src/types/iceberg.ts new file mode 100644 index 00000000..7a3d80c3 --- /dev/null +++ b/src/types/iceberg.ts @@ -0,0 +1,230 @@ +// src/types/iceberg.ts +// Iceberg Data Lake — Phase 1: TypeScript type definitions + +export type IcebergCatalogType = + | 'sqlite' + | 'sql' + | 'rest' + | 'polaris' + | 'lakekeeper' + | 'hive' + | 'glue' + | 'biglake' + | 'onelake' + | 'unity' + | 'snowflake' + | 'cloudflare' + | 'nessie'; + +export type IcebergStorageType = 'server-managed' | 'local' | 'nfs' | 'cloud'; + +export type IcebergCatalogAuthMode = + | 'none' + | 'token' + | 'oauth-client-credentials'; + +export type IcebergCloudProvider = + | 'aws' + | 'azure' + | 'gcs' + | 'minio' + | 'cloudflare-r2' + | 'backblaze-b2' + | 'rustfs' + | 'garage'; + +export interface IcebergCatalogCapability { + type: IcebergCatalogType; + label: string; + pyicebergType: 'sql' | 'rest' | 'hive' | 'glue' | 'custom'; + enabled: boolean; + disabledReason?: string; + requiredFields: Array< + | 'catalogPath' + | 'endpoint' + | 'catalogName' + | 'databaseConnectionId' + | 'hiveUri' + | 'nessieReference' + >; + authModes: IcebergCatalogAuthMode[]; + allowedStorageTypes: IcebergStorageType[]; +} + +export interface IcebergCapabilities { + catalogs: IcebergCatalogCapability[]; + cloudProviders: IcebergCloudProvider[]; +} + +export interface IcebergInstanceConfig { + id: string; + name: string; + description?: string; + // Catalog + catalogType: IcebergCatalogType; + catalogPath?: string; // local testing: path to the SQLite catalog database + endpoint?: string; // REST: Polaris/Lakekeeper endpoint URL + catalogName?: string; // REST: catalog name or warehouse + catalogAuthMode?: IcebergCatalogAuthMode; + oauthClientId?: string; + oauthClientSecretKey?: `iceberg-oauth-secret-${string}`; + oauthServerUri?: string; + oauthScope?: string; + nessieReference?: string; // Nessie branch or tag, usually "main" + nessieWarehouse?: string; // optional named Nessie warehouse + hiveUri?: string; // Hive Metastore Thrift URI, e.g. thrift://localhost:9083 + hiveUgi?: string; // optional Hive user:group identity for non-Kerberos HMS + databaseConnectionId?: string; // Existing PostgreSQL/Neon connection + catalogAccessTokenKey?: `iceberg-catalog-token-${string}`; + catalogConnectionId?: string; // Cloud Explorer connectionId for vended credentials + catalogBucket?: string; + catalogPrefix?: string; + // Storage + storageType: IcebergStorageType; + localPath?: string; + cloudProvider?: IcebergCloudProvider; + storageConnectionId?: string; // Cloud Explorer connectionId for data files + storageBucket?: string; + storagePrefix?: string; + // Metadata + createdAt: string; + updatedAt: string; +} + +export interface IcebergInstanceListItem { + id: string; + name: string; + description?: string; + catalogType: IcebergCatalogType; + storageType: IcebergStorageType; + catalogPath?: string; + localPath?: string; + storageBucket?: string; + createdAt: string; + updatedAt: string; +} + +// DTOs +export type CreateIcebergInstanceDTO = Omit< + IcebergInstanceConfig, + 'id' | 'createdAt' | 'updatedAt' +> & { + accessToken?: string; // raw token — service stores in keytar, strips before saving + oauthClientSecret?: string; // raw secret — service stores in keytar, strips before saving +}; + +export type UpdateIcebergInstanceDTO = Partial; + +// Bridge types +export interface IcebergFieldSpec { + fieldId: number; + name: string; + type: string; + required: boolean; + doc?: string; +} + +export interface IcebergSchemaResult { + fields: IcebergFieldSpec[]; + properties: Record; +} + +export interface IcebergSnapshotInfo { + snapshotId: string; + isCurrent?: boolean; + parentId?: string; + operation: string; + committedAt: string; + manifestList: string; + summary: Record; +} + +export interface IcebergPreviewResult { + columns: string[]; + rows: unknown[][]; + total?: number; +} + +export type IcebergImportFileFormat = 'csv' | 'parquet' | 'json'; + +export interface IcebergImportTableResult { + namespace: string[]; + table: string; + rowCount: number; + columns: string[]; +} + +export interface IcebergTableOperationResult { + namespace: string[]; + table: string; +} + +export interface IcebergTestResult { + success: boolean; + error?: string; + catalogConnected?: boolean; + warehouseConnected?: boolean; + namespaceCount?: number; + tableCount?: number; + checkedAt?: string; +} + +export interface IcebergTestStorageParams { + connectionId: string; + bucket: string; + prefix?: string; +} + +export interface IcebergListStorageBucketsParams { + connectionId: string; +} + +export interface IcebergLocalCatalogResult { + catalogPath: string; + warehousePath: string; + namespaces: string[][]; + tables: string[][]; +} + +export interface IcebergImportTableParams { + id: string; + namespace: string[]; + table: string; + filePath: string; + fileFormat: IcebergImportFileFormat; +} + +export interface IcebergNamespaceOperationResult { + namespace: string[]; +} + +export interface IcebergCreateNamespaceParams { + id: string; + namespace: string[]; +} + +export interface IcebergDropNamespaceParams { + id: string; + namespace: string[]; +} + +export interface IcebergTestCatalogParams { + instanceId?: string; // edit mode: resolve the existing instance-scoped secret + catalogType: IcebergCatalogType; + catalogPath?: string; + endpoint?: string; + catalogName?: string; + connectionId?: string; // resolves credentials from Cloud Explorer + accessToken?: string; // raw token (not stored yet at test time) + authMode?: IcebergCatalogAuthMode; + oauthClientId?: string; + oauthClientSecret?: string; // raw secret (not stored yet at test time) + oauthServerUri?: string; + oauthScope?: string; + nessieReference?: string; + nessieWarehouse?: string; + hiveUri?: string; + hiveUgi?: string; + databaseConnectionId?: string; + storageType?: IcebergStorageType; +} diff --git a/src/types/ipc.ts b/src/types/ipc.ts index 8ddcf6c9..1eeb84b4 100644 --- a/src/types/ipc.ts +++ b/src/types/ipc.ts @@ -371,6 +371,30 @@ export type DuckLakeChannels = | 'ducklake:connection:acquire' | 'ducklake:connection:release'; +export type IcebergChannels = + | 'iceberg:getCapabilities' + | 'iceberg:list' + | 'iceberg:get' + | 'iceberg:create' + | 'iceberg:update' + | 'iceberg:delete' + | 'iceberg:testCatalog' + | 'iceberg:testStorage' + | 'iceberg:listStorageBuckets' + | 'iceberg:testInstance' + | 'iceberg:listNamespaces' + | 'iceberg:listTables' + | 'iceberg:getSchema' + | 'iceberg:getSnapshots' + | 'iceberg:previewTable' + | 'iceberg:importTable' + | 'iceberg:dropTable' + | 'iceberg:renameTable' + | 'iceberg:createNamespace' + | 'iceberg:dropNamespace' + | 'iceberg:createMetadataFile' + | 'iceberg:ensureInstalled'; + export type LineageChannels = | 'lineage:getUpstream' | 'lineage:getDownstream' @@ -554,7 +578,8 @@ export type Channels = | StaticSiteChannels | FlowfileChannels | PipelineTemplatesChannels - | SecondBrainChannels; + | SecondBrainChannels + | IcebergChannels; export type ConfigureConnectionBody = { projectId?: string; diff --git a/tests/python/test_iceberg_bridge.py b/tests/python/test_iceberg_bridge.py new file mode 100644 index 00000000..fbc2e14f --- /dev/null +++ b/tests/python/test_iceberg_bridge.py @@ -0,0 +1,506 @@ +"""End-to-end verification for the local PyIceberg SQLite catalog.""" + +import csv as csv_module +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path + +try: + import pyarrow # noqa: F401 + from pyiceberg.catalog import load_catalog + from pyiceberg.schema import Schema + from pyiceberg.types import LongType, NestedField, StringType +except ImportError as exc: + raise unittest.SkipTest(f"Optional Iceberg test dependencies unavailable: {exc}") from exc + + +BRIDGE_PATH = ( + Path(__file__).resolve().parents[2] / "resources" / "python" / "iceberg_bridge.py" +) +SPEC = importlib.util.spec_from_file_location("iceberg_bridge", BRIDGE_PATH) +assert SPEC and SPEC.loader +ICEBERG_BRIDGE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(ICEBERG_BRIDGE) + + +class LocalCatalogBridgeTest(unittest.TestCase): + def test_create_reload_and_list_namespace_and_table(self) -> None: + with tempfile.TemporaryDirectory() as catalog_directory: + created = ICEBERG_BRIDGE.handle_create_metadata_file( + {"warehouse_path": catalog_directory} + ) + self.assertTrue(created["ok"]) + self.assertEqual(created["namespaces"], [["default"]]) + + properties = { + "type": "sql", + "uri": f"sqlite:///{created['metadata_path']}", + "warehouse": Path(created["warehouse_path"]).as_uri(), + } + catalog = load_catalog("local", **properties) + catalog.create_table( + ("default", "generated_data"), + schema=Schema( + NestedField(1, "id", LongType(), required=True), + NestedField(2, "name", StringType(), required=False), + ), + ) + catalog.close() + + namespaces = ICEBERG_BRIDGE.handle_list_namespaces( + { + "catalog_name": "local", + "catalog_properties": properties, + } + ) + tables = ICEBERG_BRIDGE.handle_list_tables( + { + "catalog_name": "local", + "catalog_properties": properties, + "namespace": ["default"], + } + ) + + self.assertEqual(namespaces, {"ok": True, "namespaces": [["default"]]}) + self.assertEqual(tables, {"ok": True, "tables": ["generated_data"]}) + + def test_import_table_from_csv_creates_and_appends(self) -> None: + """Importing a local CSV creates a table and appends its rows.""" + with tempfile.TemporaryDirectory() as catalog_directory: + created = ICEBERG_BRIDGE.handle_create_metadata_file( + {"warehouse_path": catalog_directory} + ) + self.assertTrue(created["ok"]) + + source = Path(catalog_directory) / "source.csv" + with source.open("w", newline="", encoding="utf-8") as handle: + writer = csv_module.writer(handle) + writer.writerow(["id", "name"]) + writer.writerow(["1", "alpha"]) + writer.writerow(["2", "beta"]) + writer.writerow(["3", "gamma"]) + + properties = { + "type": "sql", + "uri": f"sqlite:///{created['metadata_path']}", + "warehouse": Path(created["warehouse_path"]).as_uri(), + } + imported = ICEBERG_BRIDGE.handle_import_table( + { + "catalog_name": "local", + "catalog_properties": properties, + "namespace": ["default"], + "table": "imported_csv", + "file_path": str(source), + "file_format": "csv", + } + ) + self.assertTrue(imported["ok"], imported.get("error")) + self.assertEqual(imported["row_count"], 3) + self.assertEqual(imported["table"], "imported_csv") + self.assertEqual(imported["namespace"], ["default"]) + + tables = ICEBERG_BRIDGE.handle_list_tables( + { + "catalog_name": "local", + "catalog_properties": properties, + "namespace": ["default"], + } + ) + self.assertIn("imported_csv", tables["tables"]) + + snapshots = ICEBERG_BRIDGE.handle_get_snapshots( + { + "catalog_name": "local", + "catalog_properties": properties, + "namespace": ["default"], + "table": "imported_csv", + } + ) + self.assertTrue(snapshots["ok"]) + self.assertEqual(len(snapshots["snapshots"]), 1) + + def test_import_table_from_json_and_parquet(self) -> None: + """JSON and Parquet sources import with the same persisted contract.""" + with tempfile.TemporaryDirectory() as catalog_directory: + created = ICEBERG_BRIDGE.handle_create_metadata_file( + {"warehouse_path": catalog_directory} + ) + self.assertTrue(created["ok"]) + + properties = { + "type": "sql", + "uri": f"sqlite:///{created['metadata_path']}", + "warehouse": Path(created["warehouse_path"]).as_uri(), + } + + json_source = Path(catalog_directory) / "source.json" + # PyArrow's JSON reader expects newline-delimited JSON records. + json_source.write_text( + "\n".join( + json.dumps(record) + for record in [ + {"id": 1, "name": "alpha"}, + {"id": 2, "name": "beta"}, + ] + ), + encoding="utf-8", + ) + imported_json = ICEBERG_BRIDGE.handle_import_table( + { + "catalog_name": "local", + "catalog_properties": properties, + "namespace": ["default"], + "table": "imported_json", + "file_path": str(json_source), + "file_format": "json", + } + ) + self.assertTrue(imported_json["ok"], imported_json.get("error")) + self.assertEqual(imported_json["row_count"], 2) + + json_array_source = Path(catalog_directory) / "array.json" + json_array_source.write_text( + json.dumps( + [ + {"id": 3, "name": "delta"}, + {"id": 4, "name": "epsilon"}, + ] + ), + encoding="utf-8", + ) + imported_json_array = ICEBERG_BRIDGE.handle_import_table( + { + "catalog_name": "local", + "catalog_properties": properties, + "namespace": ["default"], + "table": "imported_json_array", + "file_path": str(json_array_source), + "file_format": "json", + } + ) + self.assertTrue( + imported_json_array["ok"], imported_json_array.get("error") + ) + self.assertEqual(imported_json_array["row_count"], 2) + + parquet_source = Path(catalog_directory) / "source.parquet" + import pyarrow as pa + import pyarrow.parquet as pa_parquet + + pa_parquet.write_table( + pa.table( + { + "id": [1, 2, 3], + "name": ["x", "y", "z"], + } + ), + parquet_source, + ) + imported_parquet = ICEBERG_BRIDGE.handle_import_table( + { + "catalog_name": "local", + "catalog_properties": properties, + "namespace": ["default"], + "table": "imported_parquet", + "file_path": str(parquet_source), + "file_format": "parquet", + } + ) + self.assertTrue( + imported_parquet["ok"], imported_parquet.get("error") + ) + self.assertEqual(imported_parquet["row_count"], 3) + + preview = ICEBERG_BRIDGE.handle_preview_table( + { + "catalog_name": "local", + "catalog_properties": properties, + "namespace": ["default"], + "table": "imported_parquet", + "limit": 10, + } + ) + self.assertTrue(preview["ok"]) + self.assertEqual(preview["total"], 3) + + def test_import_table_rejects_existing_table_and_missing_file(self) -> None: + """Existing tables and missing files fail with clean errors.""" + with tempfile.TemporaryDirectory() as catalog_directory: + created = ICEBERG_BRIDGE.handle_create_metadata_file( + {"warehouse_path": catalog_directory} + ) + self.assertTrue(created["ok"]) + + properties = { + "type": "sql", + "uri": f"sqlite:///{created['metadata_path']}", + "warehouse": Path(created["warehouse_path"]).as_uri(), + } + + source = Path(catalog_directory) / "source.csv" + source.write_text("id,name\n1,alpha\n", encoding="utf-8") + + first = ICEBERG_BRIDGE.handle_import_table( + { + "catalog_name": "local", + "catalog_properties": properties, + "namespace": ["default"], + "table": "dup_table", + "file_path": str(source), + "file_format": "csv", + } + ) + self.assertTrue(first["ok"]) + + duplicate = ICEBERG_BRIDGE.handle_import_table( + { + "catalog_name": "local", + "catalog_properties": properties, + "namespace": ["default"], + "table": "dup_table", + "file_path": str(source), + "file_format": "csv", + } + ) + self.assertFalse(duplicate["ok"]) + self.assertIn("already exists", duplicate["error"].lower()) + + missing = ICEBERG_BRIDGE.handle_import_table( + { + "catalog_name": "local", + "catalog_properties": properties, + "namespace": ["default"], + "table": "missing_table", + "file_path": str(Path(catalog_directory) / "nope.csv"), + "file_format": "csv", + } + ) + self.assertFalse(missing["ok"]) + self.assertIn("file not found", missing["error"].lower()) + + def test_drop_and_rename_table(self) -> None: + """Drop and rename operations update the persisted catalog contract.""" + with tempfile.TemporaryDirectory() as catalog_directory: + created = ICEBERG_BRIDGE.handle_create_metadata_file( + {"warehouse_path": catalog_directory} + ) + self.assertTrue(created["ok"]) + + properties = { + "type": "sql", + "uri": f"sqlite:///{created['metadata_path']}", + "warehouse": Path(created["warehouse_path"]).as_uri(), + } + source = Path(catalog_directory) / "source.csv" + source.write_text("id,name\n1,alpha\n", encoding="utf-8") + + imported = ICEBERG_BRIDGE.handle_import_table( + { + "catalog_name": "local", + "catalog_properties": properties, + "namespace": ["default"], + "table": "old_name", + "file_path": str(source), + "file_format": "csv", + } + ) + self.assertTrue(imported["ok"]) + + renamed = ICEBERG_BRIDGE.handle_rename_table( + { + "catalog_name": "local", + "catalog_properties": properties, + "namespace": ["default"], + "table": "old_name", + "new_table": "new_name", + } + ) + self.assertTrue(renamed["ok"], renamed.get("error")) + self.assertEqual(renamed["table"], "new_name") + + tables_after_rename = ICEBERG_BRIDGE.handle_list_tables( + { + "catalog_name": "local", + "catalog_properties": properties, + "namespace": ["default"], + } + ) + self.assertIn("new_name", tables_after_rename["tables"]) + self.assertNotIn("old_name", tables_after_rename["tables"]) + + dropped = ICEBERG_BRIDGE.handle_drop_table( + { + "catalog_name": "local", + "catalog_properties": properties, + "namespace": ["default"], + "table": "new_name", + } + ) + self.assertTrue(dropped["ok"], dropped.get("error")) + + tables_after_drop = ICEBERG_BRIDGE.handle_list_tables( + { + "catalog_name": "local", + "catalog_properties": properties, + "namespace": ["default"], + } + ) + self.assertEqual(tables_after_drop["tables"], []) + + def test_create_and_drop_namespace_lifecycle(self) -> None: + """Namespaces can be created (nested), listed, and dropped when empty.""" + with tempfile.TemporaryDirectory() as catalog_directory: + created = ICEBERG_BRIDGE.handle_create_metadata_file( + {"warehouse_path": catalog_directory} + ) + self.assertTrue(created["ok"]) + + properties = { + "type": "sql", + "uri": f"sqlite:///{created['metadata_path']}", + "warehouse": Path(created["warehouse_path"]).as_uri(), + } + + created_ns = ICEBERG_BRIDGE.handle_create_namespace( + { + "catalog_name": "local", + "catalog_properties": properties, + "namespace": ["analytics", "daily"], + } + ) + self.assertTrue(created_ns["ok"], created_ns.get("error")) + self.assertEqual(created_ns["namespace"], ["analytics", "daily"]) + + namespaces = ICEBERG_BRIDGE.handle_list_namespaces( + { + "catalog_name": "local", + "catalog_properties": properties, + } + ) + self.assertTrue(namespaces["ok"]) + self.assertIn(["analytics"], namespaces["namespaces"]) + + nested = ICEBERG_BRIDGE.handle_list_namespaces( + { + "catalog_name": "local", + "catalog_properties": properties, + "parent": ["analytics"], + } + ) + self.assertTrue(nested["ok"]) + self.assertIn(["analytics", "daily"], nested["namespaces"]) + + duplicate = ICEBERG_BRIDGE.handle_create_namespace( + { + "catalog_name": "local", + "catalog_properties": properties, + "namespace": ["analytics", "daily"], + } + ) + self.assertFalse(duplicate["ok"]) + self.assertIn("already exists", duplicate["error"].lower()) + + missing = ICEBERG_BRIDGE.handle_drop_namespace( + { + "catalog_name": "local", + "catalog_properties": properties, + "namespace": ["does_not_exist"], + } + ) + self.assertFalse(missing["ok"]) + + # A namespace containing a table cannot be dropped. + source = Path(catalog_directory) / "source.csv" + source.write_text("id,name\n1,alpha\n", encoding="utf-8") + imported = ICEBERG_BRIDGE.handle_import_table( + { + "catalog_name": "local", + "catalog_properties": properties, + "namespace": ["analytics", "daily"], + "table": "events", + "file_path": str(source), + "file_format": "csv", + } + ) + self.assertTrue(imported["ok"], imported.get("error")) + + non_empty = ICEBERG_BRIDGE.handle_drop_namespace( + { + "catalog_name": "local", + "catalog_properties": properties, + "namespace": ["analytics", "daily"], + } + ) + self.assertFalse(non_empty["ok"]) + + dropped_table = ICEBERG_BRIDGE.handle_drop_table( + { + "catalog_name": "local", + "catalog_properties": properties, + "namespace": ["analytics", "daily"], + "table": "events", + } + ) + self.assertTrue(dropped_table["ok"]) + + dropped_ns = ICEBERG_BRIDGE.handle_drop_namespace( + { + "catalog_name": "local", + "catalog_properties": properties, + "namespace": ["analytics", "daily"], + } + ) + self.assertTrue(dropped_ns["ok"], dropped_ns.get("error")) + + namespaces_after = ICEBERG_BRIDGE.handle_list_namespaces( + { + "catalog_name": "local", + "catalog_properties": properties, + } + ) + self.assertTrue(namespaces_after["ok"]) + # Dropping the empty nested namespace also removes its now-empty + # parent in the SQL catalog. + self.assertNotIn(["analytics"], namespaces_after["namespaces"]) + + def test_drop_and_rename_reject_missing_tables(self) -> None: + """Dropping or renaming a non-existent table returns a clean error.""" + with tempfile.TemporaryDirectory() as catalog_directory: + created = ICEBERG_BRIDGE.handle_create_metadata_file( + {"warehouse_path": catalog_directory} + ) + self.assertTrue(created["ok"]) + + properties = { + "type": "sql", + "uri": f"sqlite:///{created['metadata_path']}", + "warehouse": Path(created["warehouse_path"]).as_uri(), + } + + dropped = ICEBERG_BRIDGE.handle_drop_table( + { + "catalog_name": "local", + "catalog_properties": properties, + "namespace": ["default"], + "table": "ghost", + } + ) + self.assertFalse(dropped["ok"]) + + renamed = ICEBERG_BRIDGE.handle_rename_table( + { + "catalog_name": "local", + "catalog_properties": properties, + "namespace": ["default"], + "table": "ghost", + "new_table": "also_ghost", + } + ) + self.assertFalse(renamed["ok"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/test_iceberg_hive_catalog.py b/tests/python/test_iceberg_hive_catalog.py new file mode 100644 index 00000000..045492da --- /dev/null +++ b/tests/python/test_iceberg_hive_catalog.py @@ -0,0 +1,78 @@ +"""Opt-in persisted-table acceptance test for the Hive Metastore catalog.""" + +import os +import unittest + +import pyarrow as pa +from pyiceberg.catalog import load_catalog +from pyiceberg.schema import Schema +from pyiceberg.types import DoubleType, LongType, NestedField, StringType + + +@unittest.skipUnless( + os.environ.get("ICEBERG_HIVE_ACCEPTANCE") == "1", + "Hive catalog acceptance is opt-in", +) +class HiveCatalogAcceptanceTest(unittest.TestCase): + namespace = ("dbt_studio_hive_acceptance",) + table_identifier = (*namespace, "sales") + + def catalog_properties(self) -> dict[str, str]: + return { + "type": "hive", + "uri": os.environ["ICEBERG_HIVE_URI"], + "warehouse": os.environ["ICEBERG_HIVE_WAREHOUSE"], + } + + def test_create_close_reload_and_inspect_three_rows(self) -> None: + properties = self.catalog_properties() + catalog = load_catalog("hive_acceptance", **properties) + if self.namespace not in catalog.list_namespaces(): + catalog.create_namespace( + self.namespace, + {"location": properties["warehouse"]}, + ) + + if self.table_identifier not in catalog.list_tables(self.namespace): + table = catalog.create_table( + self.table_identifier, + schema=Schema( + NestedField(1, "sale_id", LongType(), required=False), + NestedField(2, "product", StringType(), required=False), + NestedField(3, "amount", DoubleType(), required=False), + ), + ) + table.append( + pa.table( + { + "sale_id": pa.array([1, 2, 3], type=pa.int64()), + "product": ["Starter", "Analytics", "Enterprise"], + "amount": pa.array( + [49.0, 129.5, 399.0], + type=pa.float64(), + ), + } + ) + ) + catalog.close() + + reloaded = load_catalog("hive_acceptance_reload", **properties) + self.assertIn(self.namespace, reloaded.list_namespaces()) + self.assertIn( + self.table_identifier, + reloaded.list_tables(self.namespace), + ) + table = reloaded.load_table(self.table_identifier) + self.assertEqual(table.scan().count(), 3) + self.assertEqual( + [field.name for field in table.schema().fields], + ["sale_id", "product", "amount"], + ) + self.assertTrue(list(table.snapshots())) + self.assertIsInstance(table.properties, dict) + self.assertIsNotNone(next(iter(table.scan(limit=1).plan_files()), None)) + reloaded.close() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/test_iceberg_rest_catalog.py b/tests/python/test_iceberg_rest_catalog.py new file mode 100644 index 00000000..91b9cebb --- /dev/null +++ b/tests/python/test_iceberg_rest_catalog.py @@ -0,0 +1,117 @@ +"""Opt-in persisted-table acceptance test for REST/Polaris catalogs. + +Run with ICEBERG_REST_ACCEPTANCE=1 and the catalog environment variables below. +Secrets are read only from the environment and never written to test output. +""" + +import os +import unittest + +import pyarrow as pa +from pyiceberg.catalog import load_catalog +from pyiceberg.schema import Schema +from pyiceberg.types import LongType, NestedField, StringType + + +@unittest.skipUnless( + os.environ.get("ICEBERG_REST_ACCEPTANCE") == "1", + "REST catalog acceptance is opt-in", +) +class RestCatalogAcceptanceTest(unittest.TestCase): + namespace = ("dbt_studio_acceptance",) + table_identifier = ("dbt_studio_acceptance", "customers") + + def catalog_properties(self) -> dict[str, str]: + properties = { + "type": "rest", + "uri": os.environ["ICEBERG_REST_URI"], + } + if warehouse := os.environ.get("ICEBERG_REST_WAREHOUSE"): + properties["warehouse"] = warehouse + if credential := os.environ.get("ICEBERG_REST_CREDENTIAL"): + properties["credential"] = credential + if oauth_uri := os.environ.get("ICEBERG_REST_OAUTH_URI"): + properties["oauth2-server-uri"] = oauth_uri + if scope := os.environ.get("ICEBERG_REST_SCOPE"): + properties["scope"] = scope + if delegation := os.environ.get("ICEBERG_REST_ACCESS_DELEGATION"): + properties["header.X-Iceberg-Access-Delegation"] = delegation + if s3_endpoint := os.environ.get("ICEBERG_S3_ENDPOINT"): + properties["s3.endpoint"] = s3_endpoint + properties["s3.region"] = os.environ.get("ICEBERG_S3_REGION", "us-east-1") + properties["s3.access-key-id"] = os.environ["ICEBERG_S3_ACCESS_KEY_ID"] + properties["s3.secret-access-key"] = os.environ[ + "ICEBERG_S3_SECRET_ACCESS_KEY" + ] + properties["s3.force-virtual-addressing"] = "false" + return properties + + def test_create_close_reload_and_inspect_three_rows(self) -> None: + catalog = load_catalog("acceptance", **self.catalog_properties()) + if self.namespace not in catalog.list_namespaces(): + catalog.create_namespace(self.namespace) + + if self.table_identifier not in catalog.list_tables(self.namespace): + table = catalog.create_table( + self.table_identifier, + schema=Schema( + NestedField(1, "id", LongType(), required=False), + NestedField(2, "name", StringType(), required=False), + ), + ) + else: + table = catalog.load_table(self.table_identifier) + + existing_rows = table.scan().to_arrow().to_pylist() + if not existing_rows: + table.append( + pa.Table.from_arrays( + [ + pa.array([1, 2, 3], type=pa.int64()), + pa.array(["Ada", "Linus", "Grace"]), + ], + schema=pa.schema( + [ + pa.field("id", pa.int64(), nullable=False), + pa.field("name", pa.string(), nullable=True), + ] + ), + ) + ) + else: + self.assertEqual( + sorted(existing_rows, key=lambda row: row["id"]), + [ + {"id": 1, "name": "Ada"}, + {"id": 2, "name": "Linus"}, + {"id": 3, "name": "Grace"}, + ], + ) + catalog.close() + + reloaded = load_catalog("acceptance", **self.catalog_properties()) + self.assertIn(self.namespace, reloaded.list_namespaces()) + self.assertIn( + self.table_identifier, + reloaded.list_tables(self.namespace), + ) + + table = reloaded.load_table(self.table_identifier) + rows = table.scan().to_arrow().to_pylist() + self.assertEqual( + sorted(rows, key=lambda row: row["id"]), + [ + {"id": 1, "name": "Ada"}, + {"id": 2, "name": "Linus"}, + {"id": 3, "name": "Grace"}, + ], + ) + self.assertEqual([field.name for field in table.schema().fields], ["id", "name"]) + self.assertTrue(list(table.snapshots())) + self.assertIsInstance(table.properties, dict) + self.assertIsNotNone(next(iter(table.scan(limit=1).plan_files()), None)) + reloaded.close() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/main/services/icebergDatalake.service.test.ts b/tests/unit/main/services/icebergDatalake.service.test.ts new file mode 100644 index 00000000..962d8574 --- /dev/null +++ b/tests/unit/main/services/icebergDatalake.service.test.ts @@ -0,0 +1,498 @@ +import { IcebergDatalakeService } from '../../../../src/main/services/icebergDatalake.service'; +import secureStorage from '../../../../src/main/services/secureStorage.service'; +import { + loadDatabaseFile, + updateDatabase, +} from '../../../../src/main/utils/fileHelper'; + +jest.mock('../../../../src/main/utils/fileHelper', () => ({ + loadDatabaseFile: jest.fn(), + updateDatabase: jest.fn(), +})); + +jest.mock('../../../../src/main/services/secureStorage.service', () => ({ + __esModule: true, + default: { + setCredential: jest.fn(), + getCredential: jest.fn(), + deleteCredential: jest.fn(), + }, +})); + +jest.mock('../../../../src/main/services/settings.service', () => ({ + __esModule: true, + default: { loadSettings: jest.fn() }, +})); + +const mockedLoadDatabase = loadDatabaseFile as jest.Mock; +const mockedUpdateDatabase = updateDatabase as jest.Mock; +const mockedSecureStorage = secureStorage as jest.Mocked; + +describe('IcebergDatalakeService compatibility and secret persistence', () => { + beforeEach(() => { + jest.clearAllMocks(); + mockedLoadDatabase.mockResolvedValue({ icebergInstances: [] }); + mockedUpdateDatabase.mockResolvedValue(undefined); + mockedSecureStorage.setCredential.mockResolvedValue(undefined); + }); + + it('omits Hadoop while exposing modern catalog capabilities', () => { + const capabilities = IcebergDatalakeService.getCapabilities(); + + expect(capabilities.catalogs.map(({ type }) => type)).not.toContain( + 'hadoop', + ); + expect( + capabilities.catalogs.find(({ type }) => type === 'polaris'), + ).toMatchObject({ + enabled: true, + authModes: expect.arrayContaining(['oauth-client-credentials']), + }); + expect( + capabilities.catalogs.find(({ type }) => type === 'lakekeeper'), + ).toMatchObject({ + enabled: true, + pyicebergType: 'rest', + requiredFields: ['endpoint', 'catalogName'], + allowedStorageTypes: ['server-managed'], + }); + expect( + capabilities.catalogs.find(({ type }) => type === 'nessie'), + ).toMatchObject({ + enabled: true, + pyicebergType: 'rest', + requiredFields: ['endpoint', 'nessieReference'], + allowedStorageTypes: ['server-managed'], + }); + expect( + capabilities.catalogs.find(({ type }) => type === 'hive'), + ).toMatchObject({ + enabled: true, + pyicebergType: 'hive', + requiredFields: ['hiveUri'], + allowedStorageTypes: ['local'], + }); + expect(capabilities.catalogs).toHaveLength(7); + expect(capabilities.catalogs.every(({ enabled }) => enabled)).toBe(true); + }); + + it('rejects invalid OAuth configuration before bridge execution', () => { + const validate = (IcebergDatalakeService as any) + .validateCatalogAuthentication as (config: unknown) => void; + + expect(() => + validate({ + catalogType: 'polaris', + catalogAuthMode: 'oauth-client-credentials', + oauthClientId: 'root', + oauthClientSecret: 'secret', + }), + ).toThrow('ICEBERG_OAUTH_SERVER_URI_REQUIRED'); + expect(() => + validate({ + catalogType: 'polaris', + catalogAuthMode: 'oauth-client-credentials', + oauthClientId: 'invalid:id', + oauthClientSecret: 'secret', + oauthServerUri: 'http://localhost/oauth/tokens', + }), + ).toThrow('ICEBERG_OAUTH_CLIENT_ID_INVALID'); + }); + + it('stores the OAuth secret in keytar and excludes it from database persistence', async () => { + const created = await IcebergDatalakeService.createInstance({ + name: 'polaris-test', + catalogType: 'polaris', + endpoint: 'http://localhost:8181/api/catalog', + catalogName: 'quickstart_catalog', + catalogAuthMode: 'oauth-client-credentials', + oauthClientId: 'root', + oauthClientSecret: 'top-secret', + oauthServerUri: 'http://localhost:8181/api/catalog/v1/oauth/tokens', + oauthScope: 'PRINCIPAL_ROLE:ALL', + storageType: 'server-managed', + }); + + expect(mockedSecureStorage.setCredential).toHaveBeenCalledWith( + `iceberg-oauth-secret-${created.id}`, + 'top-secret', + ); + const persisted = mockedUpdateDatabase.mock.calls[0][1][0]; + expect(persisted.oauthClientSecret).toBeUndefined(); + expect(JSON.stringify(persisted)).not.toContain('top-secret'); + expect(persisted.oauthClientSecretKey).toBe( + `iceberg-oauth-secret-${created.id}`, + ); + }); + + it('persists Lakekeeper without client or vended storage credentials', async () => { + const created = await IcebergDatalakeService.createInstance({ + name: 'lakekeeper-test', + catalogType: 'lakekeeper', + endpoint: 'http://localhost:8181/catalog', + catalogName: 'minio-warehouse', + catalogAuthMode: 'none', + storageType: 'server-managed', + }); + + const persisted = mockedUpdateDatabase.mock.calls[0][1][0]; + expect(persisted).toMatchObject({ + id: created.id, + catalogType: 'lakekeeper', + endpoint: 'http://localhost:8181/catalog', + catalogName: 'minio-warehouse', + storageType: 'server-managed', + }); + expect(persisted.storageConnectionId).toBeUndefined(); + expect(persisted.catalogAccessTokenKey).toBeUndefined(); + expect(persisted.oauthClientSecretKey).toBeUndefined(); + expect(JSON.stringify(persisted)).not.toMatch( + /access.?key|secret.?access|session.?token/i, + ); + }); + + it('redacts OAuth and database secrets from bridge errors', () => { + const redact = (IcebergDatalakeService as any).redactBridgeSecrets as ( + message: string, + env: Record, + ) => string; + const message = redact( + 'OAuth top-secret failed; URI postgresql+psycopg2://user:db-secret@host/db', + { + ICEBERG_OAUTH_CREDENTIAL: 'client-id:top-secret', + ICEBERG_SQL_CATALOG_URI: 'postgresql+psycopg2://user:db-secret@host/db', + }, + ); + + expect(message).not.toContain('top-secret'); + expect(message).not.toContain('db-secret'); + expect(message).toContain('[REDACTED]'); + }); + + it('builds the Nessie Iceberg REST URI from reference and warehouse', () => { + const buildUri = (IcebergDatalakeService as any).buildNessieRestUri as ( + config: unknown, + ) => string; + + expect( + buildUri({ + endpoint: 'http://localhost:19120/iceberg/', + nessieReference: 'main', + }), + ).toBe('http://localhost:19120/iceberg/main'); + expect( + buildUri({ + endpoint: 'http://localhost:19120/iceberg', + nessieReference: 'experiments', + nessieWarehouse: 'sales', + }), + ).toBe('http://localhost:19120/iceberg/experiments|sales'); + }); + + it('rejects the native Nessie API when Iceberg REST is required', () => { + const validate = (IcebergDatalakeService as any) + .validateCatalogWarehousePair as (config: unknown) => void; + + expect(() => + validate({ + catalogType: 'nessie', + endpoint: 'http://localhost:19120/api/v2', + nessieReference: 'main', + storageType: 'server-managed', + }), + ).toThrow('ICEBERG_NESSIE_ICEBERG_REST_ENDPOINT_REQUIRED'); + }); + + it('configures Nessie REST with server-managed remote signing', async () => { + const buildProperties = (IcebergDatalakeService as any) + .buildCatalogProperties as (config: unknown) => Promise<{ + props: Record; + env: Record; + }>; + + const result = await buildProperties({ + catalogType: 'nessie', + endpoint: 'http://localhost:19120/iceberg', + nessieReference: 'main', + nessieWarehouse: 'warehouse', + catalogAuthMode: 'none', + storageType: 'server-managed', + }); + + expect(result).toEqual({ + props: { + type: 'rest', + uri: 'http://localhost:19120/iceberg/main|warehouse', + 'header.X-Iceberg-Access-Delegation': 'remote-signing', + }, + env: {}, + }); + }); + + it('configures Lakekeeper REST with server-managed remote signing', async () => { + const buildProperties = (IcebergDatalakeService as any) + .buildCatalogProperties as (config: unknown) => Promise<{ + props: Record; + env: Record; + }>; + + const result = await buildProperties({ + catalogType: 'lakekeeper', + endpoint: 'http://localhost:8181/catalog', + catalogName: 'minio-warehouse', + catalogAuthMode: 'none', + storageType: 'server-managed', + }); + + expect(result).toEqual({ + props: { + type: 'rest', + uri: 'http://localhost:8181/catalog', + warehouse: 'minio-warehouse', + 'header.X-Iceberg-Access-Delegation': 'remote-signing', + }, + env: {}, + }); + }); + + it('rejects removed managed catalogs before Python executes', () => { + const validate = (IcebergDatalakeService as any) + .validateCatalogWarehousePair as (config: unknown) => void; + + ( + [ + 'glue', + 'biglake', + 'onelake', + 'unity', + 'snowflake', + 'cloudflare', + ] as const + ).forEach((catalogType) => { + expect(() => + validate({ catalogType, storageType: 'server-managed' }), + ).toThrow(`ICEBERG_CATALOG_UNSUPPORTED: ${catalogType}`); + }); + }); + + it('validates Hive Thrift URIs and optional UGI identity', () => { + const validate = (IcebergDatalakeService as any) + .validateCatalogWarehousePair as (config: unknown) => void; + + expect(() => + validate({ + catalogType: 'hive', + hiveUri: 'http://localhost:9083', + storageType: 'local', + localPath: '/tmp/hive-warehouse', + }), + ).toThrow('ICEBERG_HIVE_URI_INVALID'); + expect(() => + validate({ + catalogType: 'hive', + hiveUri: 'thrift://localhost:9083', + hiveUgi: 'invalid', + storageType: 'local', + localPath: '/tmp/hive-warehouse', + }), + ).toThrow('ICEBERG_HIVE_UGI_INVALID'); + }); + + it('builds native Hive catalog properties independently from storage', async () => { + const buildProperties = (IcebergDatalakeService as any) + .buildCatalogProperties as (config: unknown) => Promise<{ + props: Record; + env: Record; + }>; + + const result = await buildProperties({ + catalogType: 'hive', + hiveUri: 'thrift://localhost:9083', + hiveUgi: 'dbt:analytics', + storageType: 'local', + localPath: '/tmp/hive-warehouse', + }); + + expect(result).toEqual({ + props: { + type: 'hive', + uri: 'thrift://localhost:9083', + ugi: 'dbt:analytics', + warehouse: 'file:///tmp/hive-warehouse', + }, + env: {}, + }); + }); + + it('rejects unverified Hive cloud warehouses before Python executes', () => { + const validate = (IcebergDatalakeService as any) + .validateCatalogWarehousePair as (config: unknown) => void; + + expect(() => + validate({ + catalogType: 'hive', + hiveUri: 'thrift://localhost:9083', + storageType: 'cloud', + storageConnectionId: 'minio', + storageBucket: 'iceberg-hive', + }), + ).toThrow('ICEBERG_WAREHOUSE_NOT_ALLOWED: hive/cloud'); + }); + + describe('importTable validation', () => { + it('rejects unsupported file formats before Python executes', async () => { + await expect( + IcebergDatalakeService.importTable( + 'instance-1', + ['default'], + 't', + '/tmp/x.txt', + 'txt', + ), + ).rejects.toThrow('ICEBERG_IMPORT_FORMAT_UNSUPPORTED'); + }); + + it('rejects invalid table names', async () => { + await expect( + IcebergDatalakeService.importTable( + 'instance-1', + ['default'], + '1bad name!', + '/tmp/x.csv', + 'csv', + ), + ).rejects.toThrow('ICEBERG_IMPORT_TABLE_NAME_INVALID'); + }); + + it('rejects invalid or empty namespaces', async () => { + await expect( + IcebergDatalakeService.importTable( + 'instance-1', + [], + 'valid_table', + '/tmp/x.csv', + 'csv', + ), + ).rejects.toThrow('ICEBERG_NAMESPACE_INVALID'); + + await expect( + IcebergDatalakeService.importTable( + 'instance-1', + ['bad namespace!'], + 'valid_table', + '/tmp/x.csv', + 'csv', + ), + ).rejects.toThrow('ICEBERG_NAMESPACE_INVALID'); + }); + + it('rejects a missing file path', async () => { + await expect( + IcebergDatalakeService.importTable( + 'instance-1', + ['default'], + 'valid_table', + '', + 'csv', + ), + ).rejects.toThrow('ICEBERG_IMPORT_FILE_REQUIRED'); + }); + + it('rejects a non-existent source file before Python executes', async () => { + await expect( + IcebergDatalakeService.importTable( + 'instance-1', + ['default'], + 'valid_table', + '/definitely/not/a/real/file.csv', + 'csv', + ), + ).rejects.toThrow('ICEBERG_IMPORT_FILE_NOT_FOUND'); + }); + }); + + describe('createNamespace and dropNamespace validation', () => { + it('rejects invalid namespaces before Python executes', async () => { + await expect( + IcebergDatalakeService.createNamespace('instance-1', []), + ).rejects.toThrow('ICEBERG_NAMESPACE_INVALID'); + + await expect( + IcebergDatalakeService.createNamespace('instance-1', ['bad ns!']), + ).rejects.toThrow('ICEBERG_NAMESPACE_INVALID'); + + await expect( + IcebergDatalakeService.dropNamespace('instance-1', ['bad ns!']), + ).rejects.toThrow('ICEBERG_NAMESPACE_INVALID'); + }); + + it('passes sanitized nested namespaces to the bridge', async () => { + mockedLoadDatabase.mockResolvedValue({ + icebergInstances: [ + { + id: 'instance-1', + name: 'test', + catalogType: 'sqlite', + storageType: 'local', + localPath: '/tmp/warehouse', + createdAt: 'now', + updatedAt: 'now', + }, + ], + }); + const runBridgeSpy = jest + .spyOn(IcebergDatalakeService as any, 'runBridge') + .mockResolvedValue({ ok: true, namespace: ['a', 'b'] }); + + const result = await IcebergDatalakeService.createNamespace( + 'instance-1', + [' a ', 'b'], + ); + expect(result).toEqual({ namespace: ['a', 'b'] }); + expect(runBridgeSpy).toHaveBeenCalledTimes(1); + const command = runBridgeSpy.mock.calls[0][0] as Record; + expect(command.command).toBe('create_namespace'); + expect(command.namespace).toEqual(['a', 'b']); + + runBridgeSpy.mockRestore(); + }); + }); + + describe('dropTable and renameTable validation', () => { + it('rejects invalid table names before Python executes', async () => { + await expect( + IcebergDatalakeService.dropTable( + 'instance-1', + ['default'], + 'bad name!', + ), + ).rejects.toThrow('ICEBERG_TABLE_NAME_INVALID'); + + await expect( + IcebergDatalakeService.renameTable( + 'instance-1', + ['default'], + 'valid', + 'bad name!', + ), + ).rejects.toThrow('ICEBERG_TABLE_NAME_INVALID'); + }); + + it('rejects invalid namespaces before Python executes', async () => { + await expect( + IcebergDatalakeService.dropTable('instance-1', ['bad ns'], 'valid'), + ).rejects.toThrow('ICEBERG_NAMESPACE_INVALID'); + }); + + it('rejects renaming a table to its own name', async () => { + await expect( + IcebergDatalakeService.renameTable( + 'instance-1', + ['default'], + 'customers', + 'customers', + ), + ).rejects.toThrow('ICEBERG_RENAME_SAME_NAME'); + }); + }); +});