diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3bff6598..e8e7ce08 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,8 +35,18 @@ jobs: run: uv run nox -vs lint - name: Validate new changelog entries if: (contains(github.event.pull_request.labels.*.name, '-changelog') == false) && (github.event.pull_request.base.ref != '') - run: if [ -z "$(git diff --diff-filter=A --name-only origin/${{ github.event.pull_request.base.ref }} changelog.d)" ]; - then echo no changelog item added; exit 1; fi + run: | + mapfile -t changed_files < <(git diff --diff-filter=ACMR --name-only origin/${{ github.event.pull_request.base.ref }}...HEAD) + + if [ "${#changed_files[@]}" -eq 1 ] && [ "${changed_files[0]}" = "uv.lock" ]; then + echo "Skipping changelog validation for lockfile-only change." + exit 0 + fi + + if [ -z "$(git diff --diff-filter=A --name-only origin/${{ github.event.pull_request.base.ref }}...HEAD -- changelog.d)" ]; then + echo "No changelog item added." + exit 1 + fi - name: Changelog validation run: uv run nox -vs towncrier_check build: @@ -129,6 +139,72 @@ jobs: - name: Run integration tests if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }} run: uv run nox -vs integration -- --dont-cleanup-old-buckets -v + cli-compat: + timeout-minutes: 90 + needs: cleanup_buckets + env: + B2_TEST_APPLICATION_KEY: ${{ secrets.B2_TEST_APPLICATION_KEY }} + B2_TEST_APPLICATION_KEY_ID: ${{ secrets.B2_TEST_APPLICATION_KEY_ID }} + INSTALL_SDK_FROM: ../b2-sdk-python + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + python-version: "3.10" + - os: ubuntu-latest + python-version: "3.14" + - os: macos-latest + python-version: "3.12" + - os: windows-latest + python-version: "3.12" + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + path: b2-sdk-python + - uses: actions/checkout@v4 + with: + repository: Backblaze/B2_Command_Line_Tool + path: B2_Command_Line_Tool + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + - uses: astral-sh/setup-uv@v7 + with: + version: ${{ env.UV_VERSION }} + enable-cache: true + - name: Install test binary dependencies + if: startsWith(matrix.os, 'ubuntu') + run: | + sudo apt-get -y update + sudo apt-get -y install zsh fish + sudo chmod -R 755 /usr/share/zsh/vendor-completions /usr/share/zsh + - name: Install test binary dependencies (macOS) + if: startsWith(matrix.os, 'macos') + run: brew install fish + - name: Install dependencies + run: uv sync --directory ./B2_Command_Line_Tool --locked --group nox + - name: Run CLI unit tests + run: uv run --directory ./B2_Command_Line_Tool nox -vs unit -p ${{ matrix.python-version }} + - name: Run CLI integration tests (without secrets) + run: uv run --directory ./B2_Command_Line_Tool nox -vs integration -p ${{ matrix.python-version }} -- -m "not require_secrets" + - name: Run CLI integration tests (with secrets) + if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' && matrix.os == 'ubuntu-latest' && matrix.python-version == '3.14' }} + working-directory: B2_Command_Line_Tool + run: | + export VIRTUAL_ENV="$PWD/.nox/integration-3-14" + export PATH="$VIRTUAL_ENV/bin:$PATH" + python -m pytest \ + test/integration \ + -n 1 \ + --log-level INFO \ + -W ignore::DeprecationWarning:rst2ansi.visitor: \ + -m "require_secrets" \ + --cleanup \ + --sut "$VIRTUAL_ENV/bin/b2v4" doc: timeout-minutes: 30 needs: build diff --git a/.readthedocs.yml b/.readthedocs.yml index c2c1c68b..4ffc2bff 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -2,30 +2,25 @@ # Read the Docs configuration file # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details -# Required version: 2 build: - os: ubuntu-22.04 + os: ubuntu-24.04 tools: - python: "3.12" + python: "3.14" apt_packages: - graphviz - jobs: - post_create_environment: - - python -m pip install uv==0.8.4 - - uv export --format requirements-txt --group doc --output-file requirements-doc.txt + - plantuml -# Build documentation in the docs/ directory with Sphinx sphinx: configuration: doc/source/conf.py + fail_on_warning: true -# Optionally build your docs in additional formats such as PDF and ePub formats: all -# Optionally set the version of Python and requirements required to build your docs python: install: - - requirements: requirements-doc.txt - - method: pip - path: . + - method: uv + command: sync + groups: + - doc diff --git a/b2sdk/_internal/exception.py b/b2sdk/_internal/exception.py index 11a57475..2cf50dd7 100644 --- a/b2sdk/_internal/exception.py +++ b/b2sdk/_internal/exception.py @@ -433,6 +433,15 @@ class ServiceError(TransientErrorMixin, B2Error): Used for HTTP status codes 500 through 599. """ + def __init__(self, status, code, message): + super().__init__() + self._status = status + self._code = code + self._message = message + + def __str__(self): + return f'{self._status} {self._code} {self._message}' + class CapExceeded(B2Error): def __str__(self): @@ -744,5 +753,5 @@ def interpret_b2_error( elif status == 429: return TooManyRequests(retry_after_seconds=response_headers.get('retry-after')) elif 500 <= status < 600: - return ServiceError('%d %s %s' % (status, code, message)) + return ServiceError(status, code, message) return UnknownError('%d %s %s' % (status, code, message)) diff --git a/b2sdk/_internal/testing/helpers/bucket_manager.py b/b2sdk/_internal/testing/helpers/bucket_manager.py index bc76ef3b..78c966d1 100644 --- a/b2sdk/_internal/testing/helpers/bucket_manager.py +++ b/b2sdk/_internal/testing/helpers/bucket_manager.py @@ -25,6 +25,7 @@ BucketIdNotFound, DuplicateBucketName, FileNotPresent, + ServiceError, TooManyRequests, ) from b2sdk._internal.file_lock import NO_RETENTION_FILE_SETTING, LegalHold, RetentionMode @@ -43,6 +44,11 @@ logger = logging.getLogger(__name__) +def _retry_bucket_test_operation(exc: BaseException) -> bool: + # Retry on TooManyRequests as well as all 5xx errors (covered by ServiceError) + return isinstance(exc, (TooManyRequests, ServiceError)) + + class BucketManager: def __init__( self, @@ -83,7 +89,7 @@ def new_bucket_info(self) -> dict: } @tenacity.retry( - retry=tenacity.retry_if_exception_type(TooManyRequests), + retry=tenacity.retry_if_exception(_retry_bucket_test_operation), wait=tenacity.wait_exponential(), stop=tenacity.stop_after_attempt(8), ) @@ -152,7 +158,7 @@ def clean_buckets(self, quick=False): print(bucket) @tenacity.retry( - retry=tenacity.retry_if_exception_type(TooManyRequests), + retry=tenacity.retry_if_exception(_retry_bucket_test_operation), wait=tenacity.wait_exponential(), stop=tenacity.stop_after_attempt(8), ) diff --git a/changelog.d/+built-sdist-import-check.infrastructure.md b/changelog.d/+built-sdist-import-check.infrastructure.md new file mode 100644 index 00000000..ba7469b9 --- /dev/null +++ b/changelog.d/+built-sdist-import-check.infrastructure.md @@ -0,0 +1 @@ +Add a build-time smoke test that installs the freshly built source distribution and imports the public API shims. diff --git a/changelog.d/+cli-compat.infrastructure.md b/changelog.d/+cli-compat.infrastructure.md new file mode 100644 index 00000000..7be7e94e --- /dev/null +++ b/changelog.d/+cli-compat.infrastructure.md @@ -0,0 +1 @@ +Run B2 CLI unit tests and integration tests in CI against the SDK checkout. diff --git a/changelog.d/+readthedocs-config.infrastructure.md b/changelog.d/+readthedocs-config.infrastructure.md new file mode 100644 index 00000000..eb9fb5c6 --- /dev/null +++ b/changelog.d/+readthedocs-config.infrastructure.md @@ -0,0 +1 @@ +Add a checked-in Read the Docs build configuration so docs builds install the required doc dependencies and system packages. diff --git a/doc/source/conf.py b/doc/source/conf.py index 7c8a742c..29af0074 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -57,6 +57,11 @@ 'sphinxcontrib.plantuml', ] +# graphviz emits .png.map imagemap files that the epub builder cannot classify; +# epub does not support imagemaps, so silence the un-actionable mimetype warning +# (otherwise fail_on_warning breaks the epub build from `formats: all`). +suppress_warnings = ['epub.unknown_project_files'] + # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/noxfile.py b/noxfile.py index 4458bf30..fedc60dd 100644 --- a/noxfile.py +++ b/noxfile.py @@ -178,6 +178,7 @@ def cover(session): @nox.session(python=PYTHON_DEFAULT_VERSION) def build(session): """Build the distribution.""" + session.run('rm', '-rf', 'build', 'dist', external=True) session.run('uv', 'build', external=True) # Set outputs for GitHub Actions @@ -190,6 +191,32 @@ def build(session): version = os.environ['GITHUB_REF'].replace('refs/tags/v', '') print(f'version={version}', file=github_output) + sdists = sorted(pathlib.Path('dist').glob('b2sdk-*.tar.gz')) + if len(sdists) != 1: + session.error(f'Expected exactly one source distribution, found {len(sdists)}: {sdists!r}') + + session.install(str(sdists[0])) + session.cd('dist') # avoid importing from the checkout instead of the built package + session.run( + 'python', + '-c', + ( + 'import pathlib; ' + 'from b2sdk import v0, v1, v2, v3; ' + 'repo_root = pathlib.Path.cwd().parent.resolve(); ' + 'source_root = repo_root / "b2sdk"; ' + "module_files = {'v0': pathlib.Path(v0.__file__).resolve(), " + "'v1': pathlib.Path(v1.__file__).resolve(), " + "'v2': pathlib.Path(v2.__file__).resolve(), " + "'v3': pathlib.Path(v3.__file__).resolve()}; " + 'print(module_files); ' + "assert all(not path.is_relative_to(source_root) for path in module_files.values()), " + "f'Imported modules from checkout: {module_files!r}'; " + "assert all('site-packages' in path.parts for path in module_files.values()), " + "f'Imported modules from an unexpected location: {module_files!r}'" + ), + ) + @nox.session(python=PYTHON_DEFAULT_VERSION) def doc(session): diff --git a/test/integration/conftest.py b/test/integration/conftest.py index d700e2fd..178e7a77 100644 --- a/test/integration/conftest.py +++ b/test/integration/conftest.py @@ -7,9 +7,42 @@ # License https://www.backblaze.com/using_b2_code.html # ###################################################################### +from __future__ import annotations + import pytest +from b2sdk._internal.exception import ServiceError, TooManyRequests + +INTEGRATION_TEST_RETRY_COUNT = 4 + @pytest.fixture(scope='session', autouse=True) def auto_change_account_info_dir(change_account_info_dir): pass + + +@pytest.hookimpl(tryfirst=True) +def pytest_pyfunc_call(pyfuncitem): + testfunction = pyfuncitem.obj + funcargs = pyfuncitem.funcargs + testargs = {arg: funcargs[arg] for arg in pyfuncitem._fixtureinfo.argnames} + + for attempt in range(INTEGRATION_TEST_RETRY_COUNT + 1): + try: + testfunction(**testargs) + return True + except ServiceError as exc: + # Retry on all 5xx errors, which ServiceError covers + if attempt >= INTEGRATION_TEST_RETRY_COUNT: + raise + print( + f'Retrying {pyfuncitem.nodeid} after transient service error {exc._status}:' + f' attempt {attempt + 1} of {INTEGRATION_TEST_RETRY_COUNT}' + ) + except TooManyRequests: + if attempt >= INTEGRATION_TEST_RETRY_COUNT: + raise + print( + f'Retrying {pyfuncitem.nodeid} after transient too many requests:' + f' attempt {attempt + 1} of {INTEGRATION_TEST_RETRY_COUNT}' + ) diff --git a/test/integration/test_upload.py b/test/integration/test_upload.py index 8dbebcae..15e8e01a 100644 --- a/test/integration/test_upload.py +++ b/test/integration/test_upload.py @@ -10,15 +10,27 @@ from __future__ import annotations import io +import logging +import secrets -from b2sdk._internal.b2http import B2Http +from b2sdk._internal.b2http import B2Http, HttpCallback from b2sdk._internal.encryption.setting import EncryptionKey, EncryptionSetting from b2sdk._internal.encryption.types import EncryptionAlgorithm, EncryptionMode +from b2sdk._internal.utils import hex_sha1_of_stream from b2sdk.v2 import B2RawHTTPApi from b2sdk.v3.testing import IntegrationTestBase from .test_raw_api import authorize_raw_api +logger = logging.getLogger(__name__) + + +class FailSomeUploads(HttpCallback): + def pre_request(self, method, url, headers): + if method == 'POST' and 'b2_upload_file' in url: + headers['X-Bz-Test-Mode'] = 'fail_some_uploads' + logger.info('Added X-Bz-Test-Mode=fail_some_uploads header to %s', url) + class TestUnboundStreamUpload(IntegrationTestBase): def assert_data_uploaded_via_stream(self, data: bytes, part_size: int | None = None): @@ -46,6 +58,34 @@ def test_streamed_large_buffer_small_part_size(self): class TestUploadLargeFile(IntegrationTestBase): + def test_raw_upload_with_intermittent_failures(self): + bucket = self.create_bucket() + raw_api = self.b2_api.session.raw_api + b2_http = raw_api.b2_http + account_info = self.b2_api.account_info + callback = FailSomeUploads() + b2_http.add_callback(callback) + try: + payload = b'payload' + file_name = f'fail-some-uploads-{secrets.token_hex(4)}' + upload_url = raw_api.get_upload_url( + account_info.get_api_url(), + account_info.get_account_auth_token(), + bucket.id_, + ) + raw_api.upload_file( + upload_url['uploadUrl'], + upload_url['authorizationToken'], + file_name, + len(payload), + 'text/plain', + hex_sha1_of_stream(io.BytesIO(payload), len(payload)), + {}, + io.BytesIO(payload), + ) + finally: + b2_http.callbacks.remove(callback) + def test_ssec_key_id(self): sse_c = EncryptionSetting( mode=EncryptionMode.SSE_C, diff --git a/test/unit/test_exception.py b/test/unit/test_exception.py index 258e301a..5e44af7c 100644 --- a/test/unit/test_exception.py +++ b/test/unit/test_exception.py @@ -159,6 +159,9 @@ def test_bad_bucket_id(self): def test_service_error(self): error = interpret_b2_error(500, 'code', 'message', {}) assert isinstance(error, ServiceError) + assert error._status == 500 + assert error._code == 'code' + assert error._message == 'message' assert '500 code message' == str(error) def test_unknown_error(self): diff --git a/uv.lock b/uv.lock index 2cdfa526..b8bb3e64 100644 --- a/uv.lock +++ b/uv.lock @@ -557,11 +557,11 @@ wheels = [ [[package]] name = "idna" -version = "3.11" +version = "3.15" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" } +sdist = { url = "https://files.pythonhosted.org/packages/82/77/7b3966d0b9d1d31a36ddf1746926a11dface89a83409bf1483f0237aa758/idna-3.15.tar.gz", hash = "sha256:ca962446ea538f7092a95e057da437618e886f4d349216d2b1e294abfdb65fdc", size = 199245, upload-time = "2026-05-12T22:45:57.011Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, + { url = "https://files.pythonhosted.org/packages/d2/23/408243171aa9aaba178d3e2559159c24c1171a641aa83b67bdd3394ead8e/idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8", size = 72340, upload-time = "2026-05-12T22:45:55.733Z" }, ] [[package]]