chore(sec): bump deps + drop Python 3.9 to clear all OSV findings (5.0.0)

.github/workflows/code-quality-checks.yml

@@ -34,7 +34,7 @@ jobs:
       labels: linux-ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
         dependency-version: ["default", "min"]
         exclude:
           - python-version: "3.12"
@@ -96,7 +96,7 @@ jobs:
       labels: linux-ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
         dependency-version: ["default", "min"]
         exclude:
           - python-version: "3.12"
@@ -246,7 +246,7 @@ jobs:
       labels: linux-ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
     steps:
       - name: Check out repository
         uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
@@ -273,7 +273,7 @@ jobs:
       labels: linux-ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
     steps:
       - name: Check out repository
         uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4

CHANGELOG.md

@@ -1,5 +1,29 @@
 # Release History
 
+# 5.0.0 (TBD)
+
+**Breaking changes — security & language-version cleanup.**
+
+- Minimum Python is now **3.10** (was 3.8). Python 3.8 (EOL 2024-10) and 3.9 (EOL 2025-10) are no longer supported. Users on these versions should stay on the 4.x line.
+- Bumped `urllib3` to the **2.x** series (was `>=1.26`). Customers pinning `urllib3<2` must either lift the pin or stay on the 4.x line. urllib3 2.x requires Python 3.10+.
+- Bumped several other runtime dependency floors to versions that clear all open CVEs in `poetry.lock`:
+  - `requests`: `>=2.18.1` → `>=2.33.0`
+  - `pyjwt`: `>=2.0` → `>=2.12.0`
+  - `pyarrow`: `>=14.0.1` / `>=18.0.0` / `>=22.0.0` (Python-version-gated blocks) → `>=23.0.1` everywhere
+  - `urllib3`: `>=1.26` → `>=2.7.0,<3.0.0`
+- Bumped dev dependencies for the same reason:
+  - `pytest`: `^7.1.2` → `^9.0.3`
+  - `black`: `^22.3.0` → `^26.3.1` (codebase has been reformatted in a single commit; downstream forks should expect to reformat once on the merge)
+- Transitive cleanups (no `pyproject.toml` change, lockfile only):
+  - `cryptography` 45.0.6 → 48.0.0
+  - `idna` 3.10 → 3.16
+  - `python-dotenv` 1.0.1 → 1.2.2
+- **`thrift` stays pinned at `~=0.22.0`** (not re-widened). thrift 0.23.0 ships sdist-only and breaks `pip install` on the old setuptools bundled by DBR LTS (the SEV0 ES-1960554 incident, PR #840). It will be re-widened on a fixed thrift release (THRIFT-6067 / apache/thrift#3584). As a result the one thrift advisory (and the disputed pyjwt PYSEC-2025-183, which has no upstream fix) remain until that lands.
+
+**Migration**: For most users, `pip install -U databricks-sql-connector` is the only required change. Users on Python 3.9 must update their interpreter before upgrading. Users with hard pins on `urllib3<2` must lift the pin before upgrading.
+
+**OSV-Scanner status**: all advisories cleared on `poetry.lock` except the two noted above (thrift, blocked upstream; and the disputed pyjwt PYSEC-2025-183).
+
 # Unreleased
 - Fix: `REMOVE` staging operations no longer require `staging_allowed_local_path` to be set, since removing a remote file does not touch the local filesystem (databricks/databricks-sql-python#726)
 - Report `cursor.rowcount` for DML on the Thrift backend: INSERT/UPDATE/DELETE/MERGE now set `rowcount` to the server's affected-row count instead of the hardcoded `-1`; SELECT (and statements the server does not report a count for) still return `-1`. `executemany` aggregates the count across all parameter sets per PEP 249 ([#784](https://github.com/databricks/databricks-sql-python/issues/784))

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "databricks-sql-connector"
-version = "4.3.0"
+version = "5.0.0"
 description = "Databricks SQL Connector for Python"
 authors = ["Databricks <databricks-sql-connector-maintainers@databricks.com>"]
 license = "Apache-2.0"
@@ -9,7 +9,7 @@ packages = [{ include = "databricks", from = "src" }]
 include = ["CHANGELOG.md"]
 
 [tool.poetry.dependencies]
-python = "^3.8.0"
+python = "^3.10"
 # Keep the cap at <0.23.0. thrift ships sdist-only, and thrift 0.23.0's
 # setup.py calls sys.exit(0) on the build-success path, which kills the PEP
 # 517 build backend before pip writes output.json. On the OLD setuptools
@@ -21,35 +21,36 @@ python = "^3.8.0"
 # LTS clusters and catches this regression when dependency pins change.
 thrift = "~=0.22.0"
 pandas = [
-    { version = ">=1.2.5,<4.0.0", python = ">=3.8,<3.13" },
+    { version = ">=1.2.5,<4.0.0", python = ">=3.10,<3.13" },
     { version = ">=2.2.3,<4.0.0", python = ">=3.13" }
 ]
 lz4 = [
-    { version = "^4.0.2", python = ">=3.8,<3.14" },
+    { version = "^4.0.2", python = ">=3.10,<3.14" },
     { version = "^4.4.5", python = ">=3.14" }
 ]
-requests = "^2.18.1"
+requests = "^2.33.0"
 oauthlib = "^3.1.0"
 openpyxl = "^3.0.10"
-urllib3 = ">=1.26"
+urllib3 = ">=2.7.0,<3.0.0"
 python-dateutil = "^2.8.0"
 pyarrow = [
-    # The <3.10 band is capped at <23 because pyarrow>=23 dropped
-    # Python 3.9 (it requires >=3.10). Without the cap, poetry tries to
-    # unify this entry with the kernel's transitive pyarrow>=23.0.1,<24
-    # across the 3.8–3.10 slice and `poetry lock` fails ("pyarrow is
-    # forbidden"). The cap removes no installable version — the newest
-    # pyarrow with a 3.9 wheel is 21.x — it just makes that explicit to
-    # the solver so the optional [kernel] extra (python>=3.10) can
-    # coexist. See the kernel dep + [kernel] extra below.
-    { version = ">=14.0.1,<23", python = ">=3.8,<3.10", optional=true },
-    { version = ">=14.0.1", python = ">=3.10,<3.13", optional=true },
-    { version = ">=18.0.0", python = ">=3.13,<3.14", optional=true },
-    { version = ">=22.0.0", python = ">=3.14", optional=true }
+    { version = ">=23.0.1", python = ">=3.10,<3.13", optional=true },
+    { version = ">=23.0.1", python = ">=3.13,<3.14", optional=true },
+    { version = ">=23.0.1", python = ">=3.14", optional=true }
 ]
-pyjwt = "^2.0.0"
+pyjwt = "^2.12.0"
 pybreaker = "^1.0.0"
 requests-kerberos = {version = "^0.15.0", optional = true}
+# Transitive-only dependencies pinned to a security floor. Nothing in our
+# direct dep set constrains these, so without an explicit floor here `poetry
+# lock` floats them to whatever satisfies the parents -- which reintroduces
+# known CVEs (e.g. cryptography 45.0.6 carries the 9.8 PYSEC-2026-36). A
+# lock-only bump is not durable: the next `poetry lock` silently reverts it.
+# Declaring the floor here makes the CVE clear survive re-locks and documents
+# why. Raise these as new advisories land; the OSV-Scanner gate is the backstop.
+cryptography = ">=48.0.0"
+idna = ">=3.16"
+python-dotenv = ">=1.2.2"
 # Optional Rust kernel backend for ``use_kernel=True`` (PyO3 wheel).
 # Pulled in only via the ``[kernel]`` extra below. The published wheel
 # is ``abi3`` with ``Requires-Python: >=3.10`` (built ``abi3-py310``),
@@ -86,15 +87,15 @@ pyarrow = ["pyarrow"]
 kernel = ["databricks-sql-kernel"]
 
 [tool.poetry.group.dev.dependencies]
-pytest = "^7.1.2"
+pytest = "^9.0.3"
 mypy = "^1.10.1"
 pylint = ">=2.12.0"
-black = "^22.3.0"
+black = "^26.3.1"
 pytest-dotenv = "^0.5.2"
 pytest-cov = "^4.0.0"
 pytest-xdist = "^3.0.0"
 numpy = [
-    { version = ">=1.16.6", python = ">=3.8,<3.11" },
+    { version = ">=1.16.6", python = ">=3.10,<3.11" },
     { version = ">=1.23.4", python = ">=3.11" },
 ]
 

src/databricks/sql/auth/auth.py

@@ -102,7 +102,7 @@ def get_python_sql_connector_auth_provider(hostname: str, http_client, **kwargs)
     # TODO : unify all the auth mechanisms with the Python SDK
 
     auth_type = kwargs.get("auth_type")
-    (client_id, redirect_port_range) = get_client_id_and_redirect_port(
+    client_id, redirect_port_range = get_client_id_and_redirect_port(
         auth_type == AuthType.AZURE_OAUTH.value
     )
 
@@ -124,9 +124,11 @@ def get_python_sql_connector_auth_provider(hostname: str, http_client, **kwargs)
         azure_client_secret=kwargs.get("azure_client_secret"),
         azure_tenant_id=kwargs.get("azure_tenant_id"),
         azure_workspace_resource_id=kwargs.get("azure_workspace_resource_id"),
-        oauth_redirect_port_range=[kwargs["oauth_redirect_port"]]
-        if kwargs.get("oauth_client_id") and kwargs.get("oauth_redirect_port")
-        else redirect_port_range,
+        oauth_redirect_port_range=(
+            [kwargs["oauth_redirect_port"]]
+            if kwargs.get("oauth_client_id") and kwargs.get("oauth_redirect_port")
+            else redirect_port_range
+        ),
         oauth_persistence=kwargs.get("experimental_oauth_persistence"),
         credentials_provider=kwargs.get("credentials_provider"),
         identity_federation_client_id=kwargs.get("identity_federation_client_id"),

src/databricks/sql/auth/authenticators.py

@@ -33,12 +33,10 @@ class CredentialsProvider(abc.ABC):
     for authenticating requests to Databricks REST APIs"""
 
     @abc.abstractmethod
-    def auth_type(self) -> str:
-        ...
+    def auth_type(self) -> str: ...
 
     @abc.abstractmethod
-    def __call__(self, *args, **kwargs) -> HeaderFactory:
-        ...
+    def __call__(self, *args, **kwargs) -> HeaderFactory: ...
 
 
 # Private API: this is an evolving interface and it will change in the future.
@@ -109,7 +107,7 @@ def _initial_get_token(self):
             if self._access_token and self._refresh_token:
                 self._update_token_if_expired()
             else:
-                (access_token, refresh_token) = self.oauth_manager.get_tokens(
+                access_token, refresh_token = self.oauth_manager.get_tokens(
                     hostname=self._hostname, scope=self._scopes_as_str
                 )
                 self._access_token = access_token
@@ -231,9 +229,9 @@ def header_factory() -> Dict[str, str]:
             }
 
             if self.azure_workspace_resource_id:
-                headers[
-                    self.DATABRICKS_AZURE_WORKSPACE_RESOURCE_ID_HEADER
-                ] = self.azure_workspace_resource_id
+                headers[self.DATABRICKS_AZURE_WORKSPACE_RESOURCE_ID_HEADER] = (
+                    self.azure_workspace_resource_id
+                )
 
             return headers
 

src/databricks/sql/auth/oauth.py

@@ -43,9 +43,11 @@ def is_expired(self) -> bool:
                 self.access_token, options={"verify_signature": False}
             )
             exp_time = decoded_token.get("exp")
+            if exp_time is None:
+                return False
             current_time = time.time()
             buffer_time = 30  # 30 seconds buffer
-            return exp_time and (exp_time - buffer_time) <= current_time
+            return (exp_time - buffer_time) <= current_time
         except Exception as e:
             logger.error("Failed to decode token: %s", e)
             raise e
@@ -134,7 +136,7 @@ def __get_authorization_code(self, client, auth_url, scope, state, challenge):
             try:
                 with HTTPServer(("", port), handler) as httpd:
                     redirect_url = OAuthManager.__get_redirect_url(port)
-                    (auth_req_uri, _, _) = client.prepare_authorization_request(
+                    auth_req_uri, _, _ = client.prepare_authorization_request(
                         authorization_url=auth_url,
                         redirect_url=redirect_url,
                         scope=scope,
@@ -269,7 +271,7 @@ def get_tokens(self, hostname: str, scope=None):
         auth_url = self.idp_endpoint.get_authorization_url(hostname)
 
         state = OAuthManager.__token_urlsafe(16)
-        (verifier, challenge) = OAuthManager.__get_challenge()
+        verifier, challenge = OAuthManager.__get_challenge()
         client = oauthlib.oauth2.WebApplicationClient(self.client_id)
 
         try:

src/databricks/sql/auth/thrift_http_client.py

@@ -122,9 +122,11 @@ def open(self):
             pool_class = HTTPSConnectionPool
             _pool_kwargs.update(
                 {
-                    "cert_reqs": ssl.CERT_REQUIRED
-                    if self._ssl_options.tls_verify
-                    else ssl.CERT_NONE,
+                    "cert_reqs": (
+                        ssl.CERT_REQUIRED
+                        if self._ssl_options.tls_verify
+                        else ssl.CERT_NONE
+                    ),
                     "ca_certs": self._ssl_options.tls_trusted_ca_file,
                     "cert_file": self._ssl_options.tls_client_cert_file,
                     "key_file": self._ssl_options.tls_client_cert_key_file,

src/databricks/sql/auth/token_federation.py

@@ -111,9 +111,9 @@ def add_headers(self, request_headers: Dict[str, str]):
         """Add authentication headers to the request."""
 
         if self._cached_token and not self._cached_token.is_expired():
-            request_headers[
-                "Authorization"
-            ] = f"{self._cached_token.token_type} {self._cached_token.access_token}"
+            request_headers["Authorization"] = (
+                f"{self._cached_token.token_type} {self._cached_token.access_token}"
+            )
             return
 
         # Get the external headers first to check if we need token federation

src/databricks/sql/backend/sea/utils/filters.py

@@ -227,10 +227,8 @@ def _filter_json_result_set(
         if not case_sensitive:
             allowed_values = [v.upper() for v in allowed_values]
         # Helper lambda to get column value based on case sensitivity
-        get_column_value = (
-            lambda row: row[column_index].upper()
-            if not case_sensitive
-            else row[column_index]
+        get_column_value = lambda row: (
+            row[column_index].upper() if not case_sensitive else row[column_index]
         )
 
         # Filter rows based on allowed values

src/databricks/sql/backend/sea/utils/http_client.py

@@ -161,9 +161,11 @@ def _open(self):
             pool_class = HTTPSConnectionPool
             pool_kwargs.update(
                 {
-                    "cert_reqs": ssl.CERT_REQUIRED
-                    if self.ssl_options.tls_verify
-                    else ssl.CERT_NONE,
+                    "cert_reqs": (
+                        ssl.CERT_REQUIRED
+                        if self.ssl_options.tls_verify
+                        else ssl.CERT_NONE
+                    ),
                     "ca_certs": self.ssl_options.tls_trusted_ca_file,
                     "cert_file": self.ssl_options.tls_client_cert_file,
                     "key_file": self.ssl_options.tls_client_cert_key_file,

src/databricks/sql/backend/thrift_backend.py

@@ -12,7 +12,6 @@
 from databricks.sql.result_set import ThriftResultSet
 from databricks.sql.telemetry.models.event import StatementType
 
-
 if TYPE_CHECKING:
     from databricks.sql.client import Cursor
     from databricks.sql.result_set import ResultSet
@@ -680,7 +679,10 @@ def _create_arrow_table(self, t_row_set, lz4_compressed, schema_bytes, descripti
                 num_rows,
             ) = convert_column_based_set_to_arrow_table(t_row_set.columns, description)
         elif t_row_set.arrowBatches is not None:
-            (arrow_table, num_rows,) = convert_arrow_based_set_to_arrow_table(
+            (
+                arrow_table,
+                num_rows,
+            ) = convert_arrow_based_set_to_arrow_table(
                 t_row_set.arrowBatches, lz4_compressed, schema_bytes
             )
         else:
@@ -1053,11 +1055,13 @@ def execute_command(
             statement=operation,
             runAsync=True,
             # For async operation we don't want the direct results
-            getDirectResults=None
-            if async_op
-            else ttypes.TSparkGetDirectResults(
-                maxRows=max_rows,
-                maxBytes=max_bytes,
+            getDirectResults=(
+                None
+                if async_op
+                else ttypes.TSparkGetDirectResults(
+                    maxRows=max_rows,
+                    maxBytes=max_bytes,
+                )
             ),
             canReadArrowResult=True if pyarrow else False,
             canDecompressLZ4Result=lz4_compression,

src/databricks/sql/client.py

@@ -340,9 +340,9 @@ def read(self) -> Optional[OAuthToken]:
                 http_path=http_path,
                 port=kwargs.get("_port", 443),
                 client_context=client_context,
-                user_agent=self.session.useragent_header
-                if hasattr(self, "session")
-                else None,
+                user_agent=(
+                    self.session.useragent_header if hasattr(self, "session") else None
+                ),
                 enable_telemetry=enable_telemetry,
             )
             raise e
@@ -390,9 +390,11 @@ def read(self) -> Optional[OAuthToken]:
 
         driver_connection_params = DriverConnectionParameters(
             http_path=http_path,
-            mode=DatabricksClientType.SEA
-            if self.session.use_sea
-            else DatabricksClientType.THRIFT,
+            mode=(
+                DatabricksClientType.SEA
+                if self.session.use_sea
+                else DatabricksClientType.THRIFT
+            ),
             host_info=HostDetails(host_url=server_hostname, port=self.session.port),
             auth_mech=TelemetryHelper.get_auth_mechanism(self.session.auth_provider),
             auth_flow=TelemetryHelper.get_auth_flow(self.session.auth_provider),

src/databricks/sql/common/unified_http_client.py

@@ -148,11 +148,13 @@ def _setup_pool_managers(self):
             "num_pools": self.config.pool_connections,
             "maxsize": self.config.pool_maxsize,
             "retries": self._retry_policy,
-            "timeout": urllib3.Timeout(
-                connect=self.config.socket_timeout, read=self.config.socket_timeout
-            )
-            if self.config.socket_timeout
-            else None,
+            "timeout": (
+                urllib3.Timeout(
+                    connect=self.config.socket_timeout, read=self.config.socket_timeout
+                )
+                if self.config.socket_timeout
+                else None
+            ),
             "ssl_context": ssl_context,
         }