feat: expose lineage uncertainty and expand dialects

sqllineage-python/sqllineage.pyi

@@ -17,7 +17,8 @@ class ColumnOrigin:
 
     Check ``kind`` to determine the variant:
     - ``"concrete"``: ``table`` and ``column`` are set.
-    - ``"ambiguous"``: ``column`` and ``candidates`` are set.
+    - ``"ambiguous"``: ``column`` and ``candidates`` are set. ``candidates``
+      may be an empty list when the column is unresolved.
     - ``"wildcard"``: ``table`` is set.
     - ``"recursive"``: ``base_sources`` is set.
     """

sqllineage-python/src/lib.rs

@@ -111,6 +111,15 @@ impl PyColumnOrigin {
                 "ColumnOrigin.wildcard({}.*)",
                 self.table.as_ref().map_or("?", |t| &t.table),
             ),
+            "named_wildcard" => format!(
+                "ColumnOrigin.named_wildcard({}.{})",
+                self.table.as_ref().map_or("?", |t| &t.table),
+                self.column.as_deref().unwrap_or("?"),
+            ),
+            "source_free" => format!(
+                "ColumnOrigin.source_free({})",
+                self.column.as_deref().unwrap_or("?"),
+            ),
             "ambiguous" => format!(
                 "ColumnOrigin.ambiguous({})",
                 self.column.as_deref().unwrap_or("?"),
@@ -144,13 +153,34 @@ fn convert_origin(o: &sqllineage_core::ColumnOrigin) -> PyColumnOrigin {
             candidates: None,
             base_sources: None,
         },
+        sqllineage_core::ColumnOrigin::NamedWildcard { table, column } => PyColumnOrigin {
+            kind: "named_wildcard".into(),
+            table: Some(PyTableRef::from(table)),
+            column: Some(column.clone()),
+            candidates: None,
+            base_sources: None,
+        },
+        sqllineage_core::ColumnOrigin::SourceFree { column } => PyColumnOrigin {
+            kind: "source_free".into(),
+            table: None,
+            column: Some(column.clone()),
+            candidates: None,
+            base_sources: None,
+        },
         sqllineage_core::ColumnOrigin::Recursive { base_sources } => PyColumnOrigin {
             kind: "recursive".into(),
             table: None,
             column: None,
             candidates: None,
             base_sources: Some(base_sources.iter().map(convert_origin).collect()),
         },
+        _ => PyColumnOrigin {
+            kind: "unknown".into(),
+            table: None,
+            column: None,
+            candidates: None,
+            base_sources: None,
+        },
     }
 }
 
@@ -210,6 +240,8 @@ struct PyLineageResult {
     tables: PyTableLineage,
     #[pyo3(get)]
     columns: Vec<PyColumnMapping>,
+    #[pyo3(get)]
+    has_unresolved_stars: bool,
 }
 
 #[pymethods]
@@ -282,22 +314,15 @@ fn analyze(
     catalog: Option<Py<PyAny>>,
     normalize_case: bool,
 ) -> PyResult<Vec<PyLineageResult>> {
-    let d = match dialect.to_lowercase().as_str() {
-        "generic" => sqllineage_core::Dialect::Generic,
-        "ansi" => sqllineage_core::Dialect::Ansi,
-        "postgresql" | "postgres" => sqllineage_core::Dialect::PostgreSql,
-        "mysql" => sqllineage_core::Dialect::MySql,
-        "hive" => sqllineage_core::Dialect::Hive,
-        "databricks" => sqllineage_core::Dialect::Databricks,
-        "snowflake" => sqllineage_core::Dialect::Snowflake,
-        "bigquery" => sqllineage_core::Dialect::BigQuery,
-        other => {
+    let d = match parse_dialect_name(dialect) {
+        Some(dialect) => dialect,
+        None => {
+            let normalized = dialect.to_lowercase();
             return Err(pyo3::exceptions::PyValueError::new_err(format!(
-                "unknown dialect: '{other}'"
+                "unknown dialect: '{normalized}'"
             )));
         }
     };
-
     let catalog_box: Option<Box<dyn sqllineage_core::CatalogProvider>> =
         catalog.map(|obj| Box::new(PyCatalog { obj }) as Box<dyn sqllineage_core::CatalogProvider>);
 
@@ -329,10 +354,58 @@ fn analyze(
                     transform: convert_transform(&m.transform).into(),
                 })
                 .collect(),
+            has_unresolved_stars: result.columns.has_unresolved_stars,
         })
         .collect())
 }
 
+fn parse_dialect_name(name: &str) -> Option<sqllineage_core::Dialect> {
+    match name.to_lowercase().as_str() {
+        "generic" => Some(sqllineage_core::Dialect::Generic),
+        "ansi" => Some(sqllineage_core::Dialect::Ansi),
+        "postgresql" | "postgres" => Some(sqllineage_core::Dialect::PostgreSql),
+        "mysql" => Some(sqllineage_core::Dialect::MySql),
+        "hive" => Some(sqllineage_core::Dialect::Hive),
+        "databricks" => Some(sqllineage_core::Dialect::Databricks),
+        "snowflake" => Some(sqllineage_core::Dialect::Snowflake),
+        "bigquery" => Some(sqllineage_core::Dialect::BigQuery),
+        "duckdb" | "duck_db" => Some(sqllineage_core::Dialect::DuckDb),
+        "redshift" => Some(sqllineage_core::Dialect::Redshift),
+        "trino" => Some(sqllineage_core::Dialect::Trino),
+        "spark" | "spark2" | "sparksql" => Some(sqllineage_core::Dialect::Spark),
+        "clickhouse" | "click_house" => Some(sqllineage_core::Dialect::ClickHouse),
+        "sqlite" => Some(sqllineage_core::Dialect::SQLite),
+        "mssql" | "ms_sql" | "tsql" | "t-sql" | "sqlserver" => {
+            Some(sqllineage_core::Dialect::MsSql)
+        }
+        _ => None,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::parse_dialect_name;
+    use sqllineage_core::Dialect;
+
+    #[test]
+    fn parses_added_dialect_names_and_tsql_aliases() {
+        for (name, expected) in [
+            ("duckdb", Dialect::DuckDb),
+            ("redshift", Dialect::Redshift),
+            ("trino", Dialect::Trino),
+            ("spark", Dialect::Spark),
+            ("clickhouse", Dialect::ClickHouse),
+            ("sqlite", Dialect::SQLite),
+            ("tsql", Dialect::MsSql),
+        ] {
+            assert!(
+                matches!(parse_dialect_name(name), Some(actual) if std::mem::discriminant(&actual) == std::mem::discriminant(&expected))
+            );
+        }
+        assert!(parse_dialect_name("not-a-dialect").is_none());
+    }
+}
+
 #[pymodule]
 mod sqllineage {
     #[pymodule_export]

sqllineage/src/bin/sqllineage.rs

@@ -34,7 +34,7 @@ fn main() {
         Some(d) => d,
         None => {
             eprintln!(
-                "error: unknown dialect '{}'. valid: generic, ansi, postgresql, mysql, hive, databricks, snowflake, bigquery",
+                "error: unknown dialect '{}'. valid: generic, ansi, postgresql, mysql, hive, databricks, snowflake, bigquery, duckdb, redshift, trino, spark, clickhouse, sqlite, mssql/tsql",
                 cli.dialect
             );
             process::exit(1);
@@ -82,6 +82,13 @@ fn parse_dialect(s: &str) -> Option<Dialect> {
         "databricks" => Some(Dialect::Databricks),
         "snowflake" => Some(Dialect::Snowflake),
         "bigquery" => Some(Dialect::BigQuery),
+        "duckdb" | "duck_db" => Some(Dialect::DuckDb),
+        "redshift" => Some(Dialect::Redshift),
+        "trino" => Some(Dialect::Trino),
+        "spark" | "spark2" | "sparksql" => Some(Dialect::Spark),
+        "clickhouse" | "click_house" => Some(Dialect::ClickHouse),
+        "sqlite" => Some(Dialect::SQLite),
+        "mssql" | "ms_sql" | "tsql" | "t-sql" | "sqlserver" => Some(Dialect::MsSql),
         _ => None,
     }
 }
@@ -158,10 +165,13 @@ fn format_origin(origin: &ColumnOrigin) -> String {
         ColumnOrigin::Concrete { table, column } => format!("{table}.{column}"),
         ColumnOrigin::Ambiguous { column, .. } => format!("?{column}?"),
         ColumnOrigin::Wildcard { table } => format!("{table}.*"),
+        ColumnOrigin::NamedWildcard { table, column } => format!("{table}.*({column})"),
+        ColumnOrigin::SourceFree { column } => format!("<literal:{column}>"),
         ColumnOrigin::Recursive { base_sources } => {
             let inner: Vec<String> = base_sources.iter().map(format_origin).collect();
             format!("recursive({})", inner.join(", "))
         }
+        _ => "<unknown>".to_string(),
     }
 }
 
@@ -205,3 +215,24 @@ fn format_dot(result: &AnalyzeResult, columns: bool) -> String {
     out.push('}');
     out
 }
+
+#[cfg(test)]
+mod tests {
+    use super::parse_dialect;
+    use sqllineage::Dialect;
+
+    #[test]
+    fn parses_added_dialects_and_tsql_aliases() {
+        assert!(matches!(parse_dialect("duckdb"), Some(Dialect::DuckDb)));
+        assert!(matches!(parse_dialect("redshift"), Some(Dialect::Redshift)));
+        assert!(matches!(parse_dialect("trino"), Some(Dialect::Trino)));
+        assert!(matches!(parse_dialect("spark"), Some(Dialect::Spark)));
+        assert!(matches!(
+            parse_dialect("clickhouse"),
+            Some(Dialect::ClickHouse)
+        ));
+        assert!(matches!(parse_dialect("sqlite"), Some(Dialect::SQLite)));
+        assert!(matches!(parse_dialect("tsql"), Some(Dialect::MsSql)));
+        assert!(parse_dialect("not-a-dialect").is_none());
+    }
+}