Ppolewicz/verify 503 integration retry

.github/workflows/ci.yml

@@ -35,8 +35,18 @@ jobs:
         run: uv run nox -vs lint
       - name: Validate new changelog entries
         if: (contains(github.event.pull_request.labels.*.name, '-changelog') == false) && (github.event.pull_request.base.ref != '')
-        run: if [ -z "$(git diff --diff-filter=A --name-only origin/${{ github.event.pull_request.base.ref }} changelog.d)" ];
-          then echo no changelog item added; exit 1; fi
+        run: |
+          mapfile -t changed_files < <(git diff --diff-filter=ACMR --name-only origin/${{ github.event.pull_request.base.ref }}...HEAD)
+
+          if [ "${#changed_files[@]}" -eq 1 ] && [ "${changed_files[0]}" = "uv.lock" ]; then
+            echo "Skipping changelog validation for lockfile-only change."
+            exit 0
+          fi
+
+          if [ -z "$(git diff --diff-filter=A --name-only origin/${{ github.event.pull_request.base.ref }}...HEAD -- changelog.d)" ]; then
+            echo "No changelog item added."
+            exit 1
+          fi
       - name: Changelog validation
         run: uv run nox -vs towncrier_check
   build:
@@ -129,6 +139,72 @@ jobs:
       - name: Run integration tests
         if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }}
         run: uv run nox -vs integration -- --dont-cleanup-old-buckets -v
+  cli-compat:
+    timeout-minutes: 90
+    needs: cleanup_buckets
+    env:
+      B2_TEST_APPLICATION_KEY: ${{ secrets.B2_TEST_APPLICATION_KEY }}
+      B2_TEST_APPLICATION_KEY_ID: ${{ secrets.B2_TEST_APPLICATION_KEY_ID }}
+      INSTALL_SDK_FROM: ../b2-sdk-python
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-latest
+            python-version: "3.10"
+          - os: ubuntu-latest
+            python-version: "3.14"
+          - os: macos-latest
+            python-version: "3.12"
+          - os: windows-latest
+            python-version: "3.12"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          path: b2-sdk-python
+      - uses: actions/checkout@v4
+        with:
+          repository: Backblaze/B2_Command_Line_Tool
+          path: B2_Command_Line_Tool
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v6
+        with:
+          python-version: ${{ matrix.python-version }}
+      - uses: astral-sh/setup-uv@v7
+        with:
+          version: ${{ env.UV_VERSION }}
+          enable-cache: true
+      - name: Install test binary dependencies
+        if: startsWith(matrix.os, 'ubuntu')
+        run: |
+          sudo apt-get -y update
+          sudo apt-get -y install zsh fish
+          sudo chmod -R 755 /usr/share/zsh/vendor-completions /usr/share/zsh
+      - name: Install test binary dependencies (macOS)
+        if: startsWith(matrix.os, 'macos')
+        run: brew install fish
+      - name: Install dependencies
+        run: uv sync --directory ./B2_Command_Line_Tool --locked --group nox
+      - name: Run CLI unit tests
+        run: uv run --directory ./B2_Command_Line_Tool nox -vs unit -p ${{ matrix.python-version }}
+      - name: Run CLI integration tests (without secrets)
+        run: uv run --directory ./B2_Command_Line_Tool nox -vs integration -p ${{ matrix.python-version }} -- -m "not require_secrets"
+      - name: Run CLI integration tests (with secrets)
+        if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' && matrix.os == 'ubuntu-latest' && matrix.python-version == '3.14' }}
+        working-directory: B2_Command_Line_Tool
+        run: |
+          export VIRTUAL_ENV="$PWD/.nox/integration-3-14"
+          export PATH="$VIRTUAL_ENV/bin:$PATH"
+          python -m pytest \
+            test/integration \
+            -n 1 \
+            --log-level INFO \
+            -W ignore::DeprecationWarning:rst2ansi.visitor: \
+            -m "require_secrets" \
+            --cleanup \
+            --sut "$VIRTUAL_ENV/bin/b2v4"
   doc:
     timeout-minutes: 30
     needs: build

.readthedocs.yml

@@ -2,30 +2,25 @@
 # Read the Docs configuration file
 # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
 
-# Required
 version: 2
 
 build:
-  os: ubuntu-22.04
+  os: ubuntu-24.04
   tools:
-    python: "3.12"
+    python: "3.14"
   apt_packages:
     - graphviz
-  jobs:
-    post_create_environment:
-      - python -m pip install uv==0.8.4
-      - uv export --format requirements-txt --group doc --output-file requirements-doc.txt
+    - plantuml
 
-# Build documentation in the docs/ directory with Sphinx
 sphinx:
   configuration: doc/source/conf.py
+  fail_on_warning: true
 
-# Optionally build your docs in additional formats such as PDF and ePub
 formats: all
 
-# Optionally set the version of Python and requirements required to build your docs
 python:
   install:
-    - requirements: requirements-doc.txt
-    - method: pip
-      path: .
+    - method: uv
+      command: sync
+      groups:
+        - doc

b2sdk/_internal/exception.py

@@ -433,6 +433,15 @@ class ServiceError(TransientErrorMixin, B2Error):
     Used for HTTP status codes 500 through 599.
     """
 
+    def __init__(self, status, code, message):
+        super().__init__()
+        self._status = status
+        self._code = code
+        self._message = message
+
+    def __str__(self):
+        return f'{self._status} {self._code} {self._message}'
+
 
 class CapExceeded(B2Error):
     def __str__(self):
@@ -744,5 +753,5 @@ def interpret_b2_error(
     elif status == 429:
         return TooManyRequests(retry_after_seconds=response_headers.get('retry-after'))
     elif 500 <= status < 600:
-        return ServiceError('%d %s %s' % (status, code, message))
+        return ServiceError(status, code, message)
     return UnknownError('%d %s %s' % (status, code, message))

b2sdk/_internal/testing/helpers/bucket_manager.py

@@ -25,6 +25,7 @@
     BucketIdNotFound,
     DuplicateBucketName,
     FileNotPresent,
+    ServiceError,
     TooManyRequests,
 )
 from b2sdk._internal.file_lock import NO_RETENTION_FILE_SETTING, LegalHold, RetentionMode
@@ -43,6 +44,11 @@
 logger = logging.getLogger(__name__)
 
 
+def _retry_bucket_test_operation(exc: BaseException) -> bool:
+    # Retry on TooManyRequests as well as all 5xx errors (covered by ServiceError)
+    return isinstance(exc, (TooManyRequests, ServiceError))
+
+
 class BucketManager:
     def __init__(
         self,
@@ -83,7 +89,7 @@ def new_bucket_info(self) -> dict:
         }
 
     @tenacity.retry(
-        retry=tenacity.retry_if_exception_type(TooManyRequests),
+        retry=tenacity.retry_if_exception(_retry_bucket_test_operation),
         wait=tenacity.wait_exponential(),
         stop=tenacity.stop_after_attempt(8),
     )
@@ -152,7 +158,7 @@ def clean_buckets(self, quick=False):
             print(bucket)
 
     @tenacity.retry(
-        retry=tenacity.retry_if_exception_type(TooManyRequests),
+        retry=tenacity.retry_if_exception(_retry_bucket_test_operation),
         wait=tenacity.wait_exponential(),
         stop=tenacity.stop_after_attempt(8),
     )

changelog.d/+built-sdist-import-check.infrastructure.md

@@ -0,0 +1 @@
+Add a build-time smoke test that installs the freshly built source distribution and imports the public API shims.

changelog.d/+cli-compat.infrastructure.md

@@ -0,0 +1 @@
+Run B2 CLI unit tests and integration tests in CI against the SDK checkout.

changelog.d/+readthedocs-config.infrastructure.md

@@ -0,0 +1 @@
+Add a checked-in Read the Docs build configuration so docs builds install the required doc dependencies and system packages.

doc/source/conf.py

@@ -57,6 +57,11 @@
     'sphinxcontrib.plantuml',
 ]
 
+# graphviz emits .png.map imagemap files that the epub builder cannot classify;
+# epub does not support imagemaps, so silence the un-actionable mimetype warning
+# (otherwise fail_on_warning breaks the epub build from `formats: all`).
+suppress_warnings = ['epub.unknown_project_files']
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 

noxfile.py

@@ -178,6 +178,7 @@ def cover(session):
 @nox.session(python=PYTHON_DEFAULT_VERSION)
 def build(session):
     """Build the distribution."""
+    session.run('rm', '-rf', 'build', 'dist', external=True)
     session.run('uv', 'build', external=True)
 
     # Set outputs for GitHub Actions
@@ -190,6 +191,32 @@ def build(session):
             version = os.environ['GITHUB_REF'].replace('refs/tags/v', '')
             print(f'version={version}', file=github_output)
 
+    sdists = sorted(pathlib.Path('dist').glob('b2sdk-*.tar.gz'))
+    if len(sdists) != 1:
+        session.error(f'Expected exactly one source distribution, found {len(sdists)}: {sdists!r}')
+
+    session.install(str(sdists[0]))
+    session.cd('dist')  # avoid importing from the checkout instead of the built package
+    session.run(
+        'python',
+        '-c',
+        (
+            'import pathlib; '
+            'from b2sdk import v0, v1, v2, v3; '
+            'repo_root = pathlib.Path.cwd().parent.resolve(); '
+            'source_root = repo_root / "b2sdk"; '
+            "module_files = {'v0': pathlib.Path(v0.__file__).resolve(), "
+            "'v1': pathlib.Path(v1.__file__).resolve(), "
+            "'v2': pathlib.Path(v2.__file__).resolve(), "
+            "'v3': pathlib.Path(v3.__file__).resolve()}; "
+            'print(module_files); '
+            "assert all(not path.is_relative_to(source_root) for path in module_files.values()), "
+            "f'Imported modules from checkout: {module_files!r}'; "
+            "assert all('site-packages' in path.parts for path in module_files.values()), "
+            "f'Imported modules from an unexpected location: {module_files!r}'"
+        ),
+    )
+
 
 @nox.session(python=PYTHON_DEFAULT_VERSION)
 def doc(session):

test/integration/conftest.py

@@ -7,9 +7,42 @@
 # License https://www.backblaze.com/using_b2_code.html
 #
 ######################################################################
+from __future__ import annotations
+
 import pytest
 
+from b2sdk._internal.exception import ServiceError, TooManyRequests
+
+INTEGRATION_TEST_RETRY_COUNT = 4
+
 
 @pytest.fixture(scope='session', autouse=True)
 def auto_change_account_info_dir(change_account_info_dir):
     pass
+
+
+@pytest.hookimpl(tryfirst=True)
+def pytest_pyfunc_call(pyfuncitem):
+    testfunction = pyfuncitem.obj
+    funcargs = pyfuncitem.funcargs
+    testargs = {arg: funcargs[arg] for arg in pyfuncitem._fixtureinfo.argnames}
+
+    for attempt in range(INTEGRATION_TEST_RETRY_COUNT + 1):
+        try:
+            testfunction(**testargs)
+            return True
+        except ServiceError as exc:
+            # Retry on all 5xx errors, which ServiceError covers
+            if attempt >= INTEGRATION_TEST_RETRY_COUNT:
+                raise
+            print(
+                f'Retrying {pyfuncitem.nodeid} after transient service error {exc._status}:'
+                f' attempt {attempt + 1} of {INTEGRATION_TEST_RETRY_COUNT}'
+            )
+        except TooManyRequests:
+            if attempt >= INTEGRATION_TEST_RETRY_COUNT:
+                raise
+            print(
+                f'Retrying {pyfuncitem.nodeid} after transient too many requests:'
+                f' attempt {attempt + 1} of {INTEGRATION_TEST_RETRY_COUNT}'
+            )

test/integration/test_upload.py

@@ -10,15 +10,27 @@
 from __future__ import annotations
 
 import io
+import logging
+import secrets
 
-from b2sdk._internal.b2http import B2Http
+from b2sdk._internal.b2http import B2Http, HttpCallback
 from b2sdk._internal.encryption.setting import EncryptionKey, EncryptionSetting
 from b2sdk._internal.encryption.types import EncryptionAlgorithm, EncryptionMode
+from b2sdk._internal.utils import hex_sha1_of_stream
 from b2sdk.v2 import B2RawHTTPApi
 from b2sdk.v3.testing import IntegrationTestBase
 
 from .test_raw_api import authorize_raw_api
 
+logger = logging.getLogger(__name__)
+
+
+class FailSomeUploads(HttpCallback):
+    def pre_request(self, method, url, headers):
+        if method == 'POST' and 'b2_upload_file' in url:
+            headers['X-Bz-Test-Mode'] = 'fail_some_uploads'
+            logger.info('Added X-Bz-Test-Mode=fail_some_uploads header to %s', url)
+
 
 class TestUnboundStreamUpload(IntegrationTestBase):
     def assert_data_uploaded_via_stream(self, data: bytes, part_size: int | None = None):
@@ -46,6 +58,34 @@ def test_streamed_large_buffer_small_part_size(self):
 
 
 class TestUploadLargeFile(IntegrationTestBase):
+    def test_raw_upload_with_intermittent_failures(self):
+        bucket = self.create_bucket()
+        raw_api = self.b2_api.session.raw_api
+        b2_http = raw_api.b2_http
+        account_info = self.b2_api.account_info
+        callback = FailSomeUploads()
+        b2_http.add_callback(callback)
+        try:
+            payload = b'payload'
+            file_name = f'fail-some-uploads-{secrets.token_hex(4)}'
+            upload_url = raw_api.get_upload_url(
+                account_info.get_api_url(),
+                account_info.get_account_auth_token(),
+                bucket.id_,
+            )
+            raw_api.upload_file(
+                upload_url['uploadUrl'],
+                upload_url['authorizationToken'],
+                file_name,
+                len(payload),
+                'text/plain',
+                hex_sha1_of_stream(io.BytesIO(payload), len(payload)),
+                {},
+                io.BytesIO(payload),
+            )
+        finally:
+            b2_http.callbacks.remove(callback)
+
     def test_ssec_key_id(self):
         sse_c = EncryptionSetting(
             mode=EncryptionMode.SSE_C,

test/unit/test_exception.py

@@ -159,6 +159,9 @@ def test_bad_bucket_id(self):
     def test_service_error(self):
         error = interpret_b2_error(500, 'code', 'message', {})
         assert isinstance(error, ServiceError)
+        assert error._status == 500
+        assert error._code == 'code'
+        assert error._message == 'message'
         assert '500 code message' == str(error)
 
     def test_unknown_error(self):