feat(client): retry transient GET failures with backoff

packages/tangle-cli/src/tangle_cli/artifacts_cli.py

@@ -80,6 +80,7 @@ def artifacts_get(
                 header=args.header,
                 include_env_credentials=include_env_credentials_for_args(args, base_url),
                 command_name="artifact commands",
+                logger=logger,
             )
             if require_available := getattr(client, "require_available", None):
                 require_available()

packages/tangle-cli/src/tangle_cli/client.py

@@ -53,6 +53,10 @@ class TangleApiClient(GeneratedTangleApiOperations):
     _MAX_RATE_LIMIT_RETRIES = 3
     _RATE_LIMIT_BACKOFF_SECONDS = 1.0
     _MAX_RETRY_AFTER_SECONDS = 60.0
+    _RETRYABLE_GET_STATUSES = frozenset({500, 502, 503, 504})
+    _MAX_GET_RETRIES = 6
+    _GET_RETRY_BACKOFF_SECONDS = 1.0
+    _MAX_GET_RETRY_BACKOFF_SECONDS = 30.0
 
     def __init__(
         self,
@@ -157,7 +161,7 @@ def _request_with_rate_limit_retries(
     ) -> requests.Response:
         response: requests.Response | None = None
         for attempt in range(self._MAX_RATE_LIMIT_RETRIES + 1):
-            response = self._request_with_same_origin_redirects(
+            response = self._request_with_transient_retries(
                 method,
                 url,
                 params=params,
@@ -171,6 +175,95 @@ def _request_with_rate_limit_retries(
             self._sleep_for_rate_limit(response, attempt)
         return response
 
+    def _request_with_transient_retries(
+        self,
+        method: str,
+        url: str,
+        *,
+        params: Mapping[str, Any] | None,
+        json_data: Any,
+        extra_headers: Mapping[str, str] | None,
+        timeout: float,
+        request_kwargs: Mapping[str, Any],
+    ) -> requests.Response:
+        """Retry idempotent GETs on transient 5xx and transport errors.
+
+        Mutating methods are sent once (never duplicated). Streamed GETs bypass
+        this layer so their consumer owns any stream-open retries. 429s are left
+        to the rate-limit layer, whose retries re-enter this layer with a fresh
+        backoff. ``SSLError`` raises immediately: certificate failures are
+        deterministic, so retrying only delays the report. Each doubling
+        sleep is capped at ``_MAX_GET_RETRY_BACKOFF_SECONDS`` and announced
+        through ``self.logger`` (a null logger on non-verbose clients built
+        without one), so a stalled GET is bounded.
+        """
+
+        if method.upper() != "GET" or request_kwargs.get("stream"):
+            return self._request_with_same_origin_redirects(
+                method,
+                url,
+                params=params,
+                json_data=json_data,
+                extra_headers=extra_headers,
+                timeout=timeout,
+                request_kwargs=request_kwargs,
+            )
+
+        backoff = self._GET_RETRY_BACKOFF_SECONDS
+        for attempt in range(1, self._MAX_GET_RETRIES + 1):
+            try:
+                response = self._request_with_same_origin_redirects(
+                    method,
+                    url,
+                    params=params,
+                    json_data=json_data,
+                    extra_headers=extra_headers,
+                    timeout=timeout,
+                    request_kwargs=request_kwargs,
+                )
+            # Transient transport failures (reset/refused, timeout, truncated or
+            # corrupt body) can succeed on retry; other request errors surface.
+            except (
+                requests.ConnectionError,
+                requests.Timeout,
+                requests.exceptions.ChunkedEncodingError,
+                requests.exceptions.ContentDecodingError,
+            ) as exc:
+                # SSLError subclasses ConnectionError but signals a certificate
+                # or TLS configuration problem that no retry can fix.
+                if isinstance(exc, requests.exceptions.SSLError):
+                    raise
+                self._sleep_for_transient_retry(backoff, attempt, type(exc).__name__)
+            else:
+                if response.status_code not in self._RETRYABLE_GET_STATUSES:
+                    return response
+                # Release the intermediate response so its connection returns to the pool.
+                try:
+                    response.close()
+                except Exception:
+                    pass
+                self._sleep_for_transient_retry(backoff, attempt, f"HTTP {response.status_code}")
+            backoff *= 2.0
+        # Final attempt surfaces its own outcome: a transport error raises, and
+        # any response (5xx included) returns for the caller's raise_for_status.
+        return self._request_with_same_origin_redirects(
+            method,
+            url,
+            params=params,
+            json_data=json_data,
+            extra_headers=extra_headers,
+            timeout=timeout,
+            request_kwargs=request_kwargs,
+        )
+
+    def _sleep_for_transient_retry(self, backoff: float, attempt: int, reason: str) -> None:
+        delay = min(backoff, self._MAX_GET_RETRY_BACKOFF_SECONDS)
+        self.logger.warn(
+            f"transient {reason} on GET; retrying in {delay:.1f}s "
+            f"(attempt {attempt + 1}/{self._MAX_GET_RETRIES + 1})"
+        )
+        time.sleep(delay)
+
     def _sleep_for_rate_limit(self, response: requests.Response, attempt: int) -> None:
         retry_after = response.headers.get("Retry-After")
         delay = self._retry_after_delay(retry_after)

packages/tangle-cli/src/tangle_cli/pipeline_runs_cli.py

@@ -69,20 +69,25 @@ def _allow_all_hydration_for_args(args: ArgsContainer) -> bool:
     return bool(config.get("allow_all", False))
 
 
-def _api_client(args: ArgsContainer, *, cli_base_url: str | None, command_name: str) -> LazyTangleApiClient:
+def _api_client(
+    args: ArgsContainer, *, cli_base_url: str | None, command_name: str, logger: Logger | None = None
+) -> LazyTangleApiClient:
     return LazyTangleApiClient(
         base_url=args.base_url,
         token=args.token,
         auth_header=args.auth_header,
         header=args.header,
         include_env_credentials=include_env_credentials_for_args(args, cli_base_url),
         command_name=command_name,
+        logger=logger,
     )
 
 
 def _manager(args: ArgsContainer, *, cli_base_url: str | None, logger: Logger) -> PipelineRunManager:
     return PipelineRunManager(
-        client=_api_client(args, cli_base_url=cli_base_url, command_name="pipeline-run commands"),
+        client=_api_client(
+            args, cli_base_url=cli_base_url, command_name="pipeline-run commands", logger=logger
+        ),
         hooks=PipelineRunHooks(
             logger=logger,
             trusted_python_sources=_trusted_sources_for_args(args),
@@ -117,7 +122,12 @@ def _run_annotation_action(config: str | None, cli_base_url: str | None, specs:
             raise SystemExit(str(exc)) from exc
         try:
             manager = AnnotationManager(
-                client=_api_client(args, cli_base_url=cli_base_url, command_name="pipeline-run annotation commands"),
+                client=_api_client(
+                    args,
+                    cli_base_url=cli_base_url,
+                    command_name="pipeline-run annotation commands",
+                    logger=logger,
+                ),
                 logger=logger,
             )
             print_json(fn(manager, args))

packages/tangle-cli/src/tangle_cli/pipelines_cli.py

@@ -220,6 +220,7 @@ def pipelines_hydrate(
                 ),
                 header=_header_entries(header, config_values),
                 include_env_credentials=include_env_credentials,
+                logger=logger,
             ),
         )
     except PipelineValidationError as exc:

packages/tangle-cli/src/tangle_cli/published_components_cli.py

@@ -24,7 +24,7 @@
     TokenOption,
 )
 from .component_publisher import ComponentPublisher, deprecate_component
-from .logger import logger_for_log_type
+from .logger import Logger, logger_for_log_type
 
 
 def _client_from_options(
@@ -35,6 +35,7 @@ def _client_from_options(
     header: list[str] | str | None = None,
     include_env_credentials: bool = True,
     command_name: str = "published-component commands",
+    logger: Logger | None = None,
 ) -> LazyTangleApiClient:
     """Create a lazy static client proxy for published-component commands.
 
@@ -49,6 +50,7 @@ def _client_from_options(
         header=header,
         include_env_credentials=include_env_credentials,
         command_name=command_name,
+        logger=logger,
     )
 
 
@@ -97,6 +99,7 @@ def published_components_search(
                 header=args.header,
                 include_env_credentials=include_env_credentials_for_args(args, base_url),
                 command_name="published-component commands",
+                logger=logger,
             )
             if require_available := getattr(client, "require_available", None):
                 require_available()
@@ -163,6 +166,7 @@ def published_components_inspect(
                 header=args.header,
                 include_env_credentials=include_env_credentials_for_args(args, base_url),
                 command_name="published-component commands",
+                logger=logger,
             )
             if require_available := getattr(client, "require_available", None):
                 require_available()
@@ -219,6 +223,7 @@ def published_components_library(
                 header=args.header,
                 include_env_credentials=include_env_credentials_for_args(args, base_url),
                 command_name="published-component commands",
+                logger=logger,
             )
             if require_available := getattr(client, "require_available", None):
                 require_available()
@@ -288,6 +293,7 @@ def published_components_publish(
                 header=args.header,
                 include_env_credentials=include_env_credentials_for_args(args, base_url),
                 command_name="published-component commands",
+                logger=logger,
             )
             publisher = ComponentPublisher(
                 dry_run=bool(args.dry_run),
@@ -358,6 +364,7 @@ def published_components_deprecate(
                 header=args.header,
                 include_env_credentials=include_env_credentials_for_args(args, base_url),
                 command_name="published-component commands",
+                logger=logger,
             )
             result = deprecate_component(
                 client,

packages/tangle-cli/src/tangle_cli/secrets_cli.py

@@ -58,14 +58,17 @@
 app = App(name="secrets", help="Manage Tangle secrets.")
 
 
-def _client(args: ArgsContainer, *, cli_base_url: str | None, command_name: str) -> LazyTangleApiClient:
+def _client(
+    args: ArgsContainer, *, cli_base_url: str | None, command_name: str, logger: Logger | None = None
+) -> LazyTangleApiClient:
     return LazyTangleApiClient(
         base_url=args.base_url,
         token=args.token,
         auth_header=args.auth_header,
         header=args.header,
         include_env_credentials=include_env_credentials_for_args(args, cli_base_url),
         command_name=command_name,
+        logger=logger,
     )
 
 
@@ -79,7 +82,7 @@ def _run_secret_action(
     for args in load_args_or_exit(config, **specs):
         logger, finalize_logs = logger_for_log_type(getattr(args, "log_type", "console"))
         try:
-            client = _client(args, cli_base_url=cli_base_url, command_name="secret commands")
+            client = _client(args, cli_base_url=cli_base_url, command_name="secret commands", logger=logger)
             try:
                 results.append(fn(client, args, logger))
             except SecretValueError as exc:

tests/test_api_cli.py

@@ -1,6 +1,7 @@
 import importlib
 import json
 import sys
+from unittest.mock import ANY
 
 import httpx
 import pytest
@@ -460,6 +461,7 @@ def fake_client_from_options(**kwargs):
         "header": ["X-Config: yes"],
         "include_env_credentials": False,
         "command_name": "published-component commands",
+        "logger": ANY,
     }
 
 

tests/test_artifacts_cli.py

@@ -5,6 +5,7 @@
 import json
 import sys
 from typing import Any
+from unittest.mock import ANY
 
 from tangle_cli import artifacts as artifacts_module
 from tangle_cli import artifacts_cli, cli
@@ -80,6 +81,7 @@ def fake_get_artifacts(self, run_id: str, query: dict[str, Any]) -> dict[str, ob
             "header": ["X-Config: yes"],
             "include_env_credentials": False,
             "command_name": "artifact commands",
+            "logger": ANY,
         }
     ]
     assert get_calls == [