From 1adc6884fd98e6b047f0647f5a0e43d2af66e61b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2026 11:14:12 +0000 Subject: [PATCH 1/2] Bump ruff from 0.15.22 to 0.16.0 Bumps [ruff](https://github.com/astral-sh/ruff) from 0.15.22 to 0.16.0. - [Release notes](https://github.com/astral-sh/ruff/releases) - [Changelog](https://github.com/astral-sh/ruff/blob/main/CHANGELOG.md) - [Commits](https://github.com/astral-sh/ruff/compare/0.15.22...0.16.0) --- updated-dependencies: - dependency-name: ruff dependency-version: 0.16.0 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- uv.lock | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/uv.lock b/uv.lock index 1ab794c..c9e6991 100644 --- a/uv.lock +++ b/uv.lock @@ -358,27 +358,27 @@ wheels = [ [[package]] name = "ruff" -version = "0.15.22" +version = "0.16.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3a/06/ae069393fc66e8ff33036d4b368003833bf6e88ccf182e17e7a2f1c754fd/ruff-0.15.22.tar.gz", hash = "sha256:3f15175b1fb580126f58285a5dae6b2ea89000136d980c64499211f116b54809", size = 4785063, upload-time = "2026-07-16T15:14:13.244Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4d/94/1e5e4967626faf12fa56999cd6222dff6992ceb086ad7945756baf70c7a7/ruff-0.16.0.tar.gz", hash = "sha256:e460aafd5495ec89efaa6ced2e4a9a581116451e1c88b9d37ef497e0f8e93982", size = 4790557, upload-time = "2026-07-23T19:11:30.981Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/23/18/ee54b7ae1e121be7a28ea6da4b67564ebb0530e183a54415ab7e3bcd2c4e/ruff-0.15.22-py3-none-linux_armv6l.whl", hash = "sha256:44423e73493737f5e7c5b41d475483898ff37afcdae38bc3da5085e29af1c2d8", size = 10781258, upload-time = "2026-07-16T15:13:19.452Z" }, - { url = "https://files.pythonhosted.org/packages/2f/d2/2520cb14761ddbeaf57642a76942fc36adcbdbe53b4532241995f6fc485c/ruff-0.15.22-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b82c6482946e9eda7ff2e091d25b8bad3f718684e1916d41bd56873cee05b697", size = 10999477, upload-time = "2026-07-16T15:13:23.318Z" }, - { url = "https://files.pythonhosted.org/packages/c9/10/74e53572aa758dfaa678c2a2646b5c5515d884b7ca56be4d2ce03ca4b560/ruff-0.15.22-py3-none-macosx_11_0_arm64.whl", hash = "sha256:11c1c715af53a09f714e011106bffc419751ec8232fcb5da42173284ea3fec6f", size = 10466716, upload-time = "2026-07-16T15:13:26.162Z" }, - { url = "https://files.pythonhosted.org/packages/1e/cc/44eaaf0844e028182f2d0a8f2190d0f359159aed0a9e5ab861d892f1ae2a/ruff-0.15.22-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:742a29cf29bddb7c8327895d6a10e0e6c5b38a96dd407af9b5d0857f809c0576", size = 10892644, upload-time = "2026-07-16T15:13:29.229Z" }, - { url = "https://files.pythonhosted.org/packages/9f/21/8edf559014d2b0f82beea19cfb713993ad802ccda16868769979c6090a84/ruff-0.15.22-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72af58b951b0ae395935ae79763dc349bc0eb706319d28f7a33ad2cfb3cfc178", size = 10576719, upload-time = "2026-07-16T15:13:32.35Z" }, - { url = "https://files.pythonhosted.org/packages/bf/1e/3a13abd392a3b50b62e5938a831f9ab6e588358cacad5c18545b716d2182/ruff-0.15.22-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:62d425005c1835eb24e2ee4161cb90e8db263415f4a71c8c72c33abaa6c0c224", size = 11376494, upload-time = "2026-07-16T15:13:35.958Z" }, - { url = "https://files.pythonhosted.org/packages/bf/3e/422d3d95bcf04dd78e1aeac22184d4f9a8fb2c01865d39d44618484a0317/ruff-0.15.22-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e8b9b3f8779a4f08c969defc3c8c35abffaa757e601ed5ae66d6d1db6519969a", size = 12208370, upload-time = "2026-07-16T15:13:39.185Z" }, - { url = "https://files.pythonhosted.org/packages/1e/91/5d065a0e0a02bf4813f5119ad278462eed081d2b832eb7c021ade0ec9e65/ruff-0.15.22-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1e0dd1b2e4d3d585f897a0d137cbf4eaf6223bef4e8ce34d6bb12556c5f9249e", size = 11581098, upload-time = "2026-07-16T15:13:42.132Z" }, - { url = "https://files.pythonhosted.org/packages/f6/f9/a0d4871d12fae702eb1f41b686caf05f1f8b124dc6db6f784f53d74918fa/ruff-0.15.22-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:365523eb91d9224e1bcb03b022fbf0facb8f9e23792a2c53d9d4b3924bdbdebb", size = 11399422, upload-time = "2026-07-16T15:13:45.2Z" }, - { url = "https://files.pythonhosted.org/packages/18/80/c843a5176cddbceb0b7e8dd41cf9993490796c1c469348d384f5a5c13c56/ruff-0.15.22-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:fabfd168afdf29fee5be98b831efa9683c94d7c5a3b58b9ce5a2e38444589a74", size = 11381683, upload-time = "2026-07-16T15:13:48.46Z" }, - { url = "https://files.pythonhosted.org/packages/d4/00/8485de0ae92239438a36cfc51350db9b9e85c9ebdfaea91b18e422706662/ruff-0.15.22-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:225dbf095a87f1d9f90f5fd7924d2613ee452a75a4308c63a8f50f761787aa7c", size = 10850295, upload-time = "2026-07-16T15:13:51.655Z" }, - { url = "https://files.pythonhosted.org/packages/fa/91/24977ec2ec72eaf15e4394ace2959fdff2dd1e14f03e005e838023407169/ruff-0.15.22-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:1877d63b9d24ed278744f1523fd11b85540566d54641f97c566d7d9dc5ca5296", size = 10579640, upload-time = "2026-07-16T15:13:54.79Z" }, - { url = "https://files.pythonhosted.org/packages/9c/47/9b51216951974df1f263ac19da550d34252e0ed7218c25f10c5ef9ed7517/ruff-0.15.22-py3-none-musllinux_1_2_i686.whl", hash = "sha256:a1606c510bd7215680d32efab38965f7cdec3ef69f5170a3f4791404ffdd5262", size = 11105077, upload-time = "2026-07-16T15:13:57.915Z" }, - { url = "https://files.pythonhosted.org/packages/c2/47/20e9d4a3b8016778acea5fc32bb50d35d207500a17ddb529ffa6996feef8/ruff-0.15.22-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:630479b18625f5ffc373f77603a22a9f8ac0acd7ff0501178b5db28ec71e9c64", size = 11490980, upload-time = "2026-07-16T15:14:01.032Z" }, - { url = "https://files.pythonhosted.org/packages/4d/76/3f72d8fc38c1cb77b38c56a70da9d0c17700cc1cc50f9649c9d3c8f5ba71/ruff-0.15.22-py3-none-win32.whl", hash = "sha256:e5ba0e4a13fd14abbed2a77b517a3911290c6c6c59ef67784328d1668fab76cf", size = 10789165, upload-time = "2026-07-16T15:14:04.16Z" }, - { url = "https://files.pythonhosted.org/packages/cb/46/4965251734c2b6fcdca1b1b187d20bcac3af0ee5b083b89c910bb961ce3a/ruff-0.15.22-py3-none-win_amd64.whl", hash = "sha256:9be63ba1eb936acd2d1342fb8337c356353706fce233b2a15a09a97037e6acde", size = 11938297, upload-time = "2026-07-16T15:14:07.316Z" }, - { url = "https://files.pythonhosted.org/packages/57/c9/e69b1ff4c8b69093ef08b8919ab767af0569666865b39c30a8795d88d3c6/ruff-0.15.22-py3-none-win_arm64.whl", hash = "sha256:e1168075b72158510839f250027659cdd78476f40507dd517892304c41318661", size = 11298172, upload-time = "2026-07-16T15:14:10.51Z" }, + { url = "https://files.pythonhosted.org/packages/4b/81/1c8818fee7ce1a04cd7d1b3172e0a8f8e4f1dc4feb7fc390e16daa8af323/ruff-0.16.0-py3-none-linux_armv6l.whl", hash = "sha256:e5115729eb08c585e5121978ba5d5b60caeae394ce21b9fb5e6cd33a1c6c9b1e", size = 10754633, upload-time = "2026-07-23T19:10:46.415Z" }, + { url = "https://files.pythonhosted.org/packages/23/df/beaf59c09d68db84304d555f188b276a77132a5d5b0b67a5c762aa143628/ruff-0.16.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:3c954b1d580bfa035b41654f7858cc7e71d5fc3ac5b723dd62bd9133830ed522", size = 10969164, upload-time = "2026-07-23T19:10:50.271Z" }, + { url = "https://files.pythonhosted.org/packages/42/ce/741cd197496a1abbf51352710fd15ed995d2a2be87189c1da26a450d6e83/ruff-0.16.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e01c21d10eb1b29f47b7454e1f4056db9a3f0260c646aa88457c610291db9f81", size = 10488846, upload-time = "2026-07-23T19:10:52.639Z" }, + { url = "https://files.pythonhosted.org/packages/52/2a/a2db8e88cade358f5cdcb05674a917751074109315d014eb6352d9a893f7/ruff-0.16.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6e364e5ed22ed8dc05082fd78e35308618260907ac2d3c1d637b2e682415b6c9", size = 10889729, upload-time = "2026-07-23T19:10:54.89Z" }, + { url = "https://files.pythonhosted.org/packages/42/65/62a771694ebd63029dc953e27dbad40e1588bd4860ff9fe881018fddaa49/ruff-0.16.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d327b8fc113a1d4421a04f3839d3752057c8dd1ee320223a6f3f52d04ada462a", size = 10568275, upload-time = "2026-07-23T19:10:56.993Z" }, + { url = "https://files.pythonhosted.org/packages/3f/e2/ced249fe8af5f086c5c58cc21cc3356d50f32f7401c5df87050c999620a7/ruff-0.16.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9b50c55e263103586b3dcf5f73d479eb8cb5fdb6098fec59a62891dab653717", size = 11385112, upload-time = "2026-07-23T19:10:59.615Z" }, + { url = "https://files.pythonhosted.org/packages/87/0b/05154977a8fd69eeb6c103271f55403bfd8711f5c0f8ed07489d95a504e7/ruff-0.16.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0ff4a79ce3ec0172f3241943835de1c4cb4e2dcd07f0f8c2d02603dbbbee4b17", size = 12207008, upload-time = "2026-07-23T19:11:02.154Z" }, + { url = "https://files.pythonhosted.org/packages/fb/29/98225831a3a1eab0e02f4acc6ca6559a98611dcc68b6965ff4b7234627c1/ruff-0.16.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e95c448fca1fb2a18372a9440926c5a6ee789639bb975c72e7ae6d0b04218ab4", size = 11650842, upload-time = "2026-07-23T19:11:04.557Z" }, + { url = "https://files.pythonhosted.org/packages/91/66/6bd3cf90500653d55dc0ffc8507aa8300bd49d0214b2e8cb4d3fef2943ba/ruff-0.16.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f11a8d11010301d0a398a2fdef67691feca7294da6aef55e2150e8fa2cd520b", size = 11400718, upload-time = "2026-07-23T19:11:09.233Z" }, + { url = "https://files.pythonhosted.org/packages/8e/a2/a54eb4eae05d66364050a5d3b8a9c5ef88196531b3cbe7109d873f87f819/ruff-0.16.0-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:48044c678e9cb8698246c99b14aaccfa6601dea7379eb48a6f8f73f7a6d86cd0", size = 11426177, upload-time = "2026-07-23T19:11:11.994Z" }, + { url = "https://files.pythonhosted.org/packages/1a/be/16e3eea4b2a478a496919f5e36f17c4559e54620bd3bbac5d6affa068006/ruff-0.16.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:7aa0959bad8eb8bef50340154fc9b58678dae31fa4293afa38b44b6e552c0213", size = 10856126, upload-time = "2026-07-23T19:11:14.221Z" }, + { url = "https://files.pythonhosted.org/packages/a2/84/252eb8b868a16eec7257c14f504f77537e734b2d69c762e639e588e304a3/ruff-0.16.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:28ea2b7df8ebf7f9da6b7d47b230ab48f387c0a29be3b474c4d0740e197bb9af", size = 10571208, upload-time = "2026-07-23T19:11:16.378Z" }, + { url = "https://files.pythonhosted.org/packages/21/09/817a482f542f7570cbb4554b26e896610c7114f539b1d9e2d2145bf6bef6/ruff-0.16.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:33a3dfac8c35f81498dea9181bccc2f4c4bc8f1521a1dd9406e77643e0f0fb09", size = 11063329, upload-time = "2026-07-23T19:11:19.173Z" }, + { url = "https://files.pythonhosted.org/packages/2e/23/9403c180ca1cb9b1f7335f5c3e5305c09d49ea5b345196682a36028bde4a/ruff-0.16.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:a5237a0bda500d30d81b8e07a6973a5cbc772864cbf746ae2f4e8a2e01c9f4ed", size = 11489751, upload-time = "2026-07-23T19:11:21.74Z" }, + { url = "https://files.pythonhosted.org/packages/b2/1d/1b2ef7bcde851c78d7f17f1cca13fd6dc695fc4b3d6197941e72cae5b132/ruff-0.16.0-py3-none-win32.whl", hash = "sha256:7fab76fa065c873f41ff744347c6e77bcc3dfec4bcc754dc26b63d23c0f7f5fb", size = 10785885, upload-time = "2026-07-23T19:11:23.947Z" }, + { url = "https://files.pythonhosted.org/packages/b2/a3/d5e4ef7a56be3f928ffb90b94c25ba7d3cb9c7fe0736aeaaedf361770712/ruff-0.16.0-py3-none-win_amd64.whl", hash = "sha256:429c117f022bf481fabd9d551e7a3952b24c65e6ef44337ea09d90bebef14472", size = 11923141, upload-time = "2026-07-23T19:11:26.409Z" }, + { url = "https://files.pythonhosted.org/packages/cb/9a/8415f2657cbe200f41a4531ccededf135505a92d4a012229121f885b26f9/ruff-0.16.0-py3-none-win_arm64.whl", hash = "sha256:14296fedcd2705c77ab8235439278bbb38f285cf7da5528b00b3e330c3d4872d", size = 11273407, upload-time = "2026-07-23T19:11:28.705Z" }, ] [[package]] From 1f457505ec9cb04b35cc0488893592f33fb2b150 Mon Sep 17 00:00:00 2001 From: Steven Maude Date: Mon, 3 Aug 2026 14:48:56 +0100 Subject: [PATCH 2/2] Apply fixes for Ruff v0.16 Some of these are automated Ruff fixes; others suggested by Gemini. --- scraperwiki/__init__.py | 13 +++---- scraperwiki/sql.py | 23 +++++------- scraperwiki/utils.py | 36 +++++++++---------- tests/test_scraperwiki.py | 73 ++++++++++++++++++--------------------- 4 files changed, 63 insertions(+), 82 deletions(-) diff --git a/scraperwiki/__init__.py b/scraperwiki/__init__.py index bb072d3..5ed2da3 100644 --- a/scraperwiki/__init__.py +++ b/scraperwiki/__init__.py @@ -3,20 +3,19 @@ https://scraperwiki.com/docs/python/python_help_documentation/ """ -from .utils import scrape, pdftoxml, status, swimport -from . import utils -from . import sql +from . import sql, utils +from .utils import pdftoxml, scrape, status, swimport # Compatibility sqlite = sql __all__ = [ - "scrape", "pdftoxml", + "scrape", + "sql", "status", "swimport", "utils", - "sql", ] @@ -24,10 +23,6 @@ class Error(Exception): """All ScraperWiki exceptions are instances of this class (usually via a subclass).""" - pass - class CPUTimeExceededError(Error): """CPU time limit exceeded.""" - - pass diff --git a/scraperwiki/sql.py b/scraperwiki/sql.py index d97461f..971b7eb 100644 --- a/scraperwiki/sql.py +++ b/scraperwiki/sql.py @@ -1,11 +1,10 @@ -from collections.abc import Iterable, Mapping - import atexit import datetime -import time import os import re +import time import warnings +from collections.abc import Iterable, Mapping import alembic.ddl import sqlalchemy @@ -14,7 +13,7 @@ "SCRAPERWIKI_DATABASE_NAME", "sqlite:///scraperwiki.sqlite" ) -DATABASE_TIMEOUT = float(os.environ.get("SCRAPERWIKI_DATABASE_TIMEOUT", 300)) +DATABASE_TIMEOUT = float(os.environ.get("SCRAPERWIKI_DATABASE_TIMEOUT", "300")) SECONDS_BETWEEN_COMMIT = 2 # The scraperwiki.sqlite.SqliteError exception @@ -26,8 +25,6 @@ class Blob(bytes): Represents a blob as a string. """ - pass - PYTHON_SQLITE_TYPE_MAP = { str: sqlalchemy.types.Text, @@ -201,9 +198,7 @@ def save(unique_keys, data, table_name="swdata"): insert = sqlalchemy.insert(_State.table).prefix_with("OR REPLACE") for row in data: if not isinstance(row, Mapping): - raise TypeError( - "Elements of data must be mappings, got {}".format(type(row)) - ) + raise TypeError(f"Elements of data must be mappings, got {type(row)}") fit_row(connection, row, unique_keys) connection.execute(insert.values(row)) _State.check_last_committed() @@ -261,12 +256,12 @@ def save_var(name, value): else: value_blob = str(value).encode("utf-8") - values = dict( - name=name, - value_blob=value_blob, + values = { + "name": name, + "value_blob": value_blob, # value_blob=Blob(value), - type=column_type.__visit_name__.lower(), - ) + "type": column_type.__visit_name__.lower(), + } stmt = sqlalchemy.insert(vars_table).prefix_with("OR REPLACE").values(**values) connection.execute(stmt) diff --git a/scraperwiki/utils.py b/scraperwiki/utils.py index b4d6124..3416b2b 100644 --- a/scraperwiki/utils.py +++ b/scraperwiki/utils.py @@ -5,10 +5,10 @@ import os import shutil -import warnings import tempfile import urllib.parse import urllib.request +import warnings def scrape(url, params=None, user_agent=None): @@ -43,23 +43,22 @@ def pdftoxml(pdfdata, options=""): "install pdftohtml." ) return None - pdffout = tempfile.NamedTemporaryFile(suffix=".pdf") - pdffout.write(pdfdata) - pdffout.flush() - - xmlin = tempfile.NamedTemporaryFile(mode="r", suffix=".xml", encoding="utf-8") - tmpxml = xmlin.name # "temph.xml" - cmd = 'pdftohtml -xml -nodrm -zoom 1.5 -enc UTF-8 -noframes {} "{}" "{}"'.format( - options, pdffout.name, os.path.splitext(tmpxml)[0] - ) - # can't turn off output, so throw away even stderr yeuch - cmd = cmd + " >/dev/null 2>&1" - os.system(cmd) - - pdffout.close() - # xmlfin = open(tmpxml) - xmldata = xmlin.read() - xmlin.close() + + with ( + tempfile.NamedTemporaryFile(suffix=".pdf") as pdffout, + tempfile.NamedTemporaryFile(mode="r", suffix=".xml", encoding="utf-8") as xmlin, + ): + pdffout.write(pdfdata) + pdffout.flush() + + tmpxml = xmlin.name # "temph.xml" + cmd = f'pdftohtml -xml -nodrm -zoom 1.5 -enc UTF-8 -noframes {options} "{pdffout.name}" "{os.path.splitext(tmpxml)[0]}"' + # can't turn off output, so throw away even stderr yeuch + cmd = cmd + " >/dev/null 2>&1" + os.system(cmd) + + xmldata = xmlin.read() + return xmldata @@ -70,7 +69,6 @@ def status(type, message=None): DeprecationWarning, stacklevel=2, ) - return def swimport(scrapername): diff --git a/tests/test_scraperwiki.py b/tests/test_scraperwiki.py index d09f42a..566abf3 100644 --- a/tests/test_scraperwiki.py +++ b/tests/test_scraperwiki.py @@ -2,17 +2,14 @@ import json import os import sqlite3 +import sys import warnings - -from subprocess import Popen, PIPE +from subprocess import PIPE, Popen from textwrap import dedent - from unittest import TestCase import scraperwiki -import sys - # scraperwiki.sql._State.echo = True DB_NAME = "scraperwiki.sqlite" @@ -57,7 +54,7 @@ def test_save_no_warn(self): with warnings.catch_warnings(): warnings.simplefilter("error") scraperwiki.sql.save( - ["id"], dict(id=4, tumble="weed"), table_name="warning_test" + ["id"], {"id": 4, "tumble": "weed"}, table_name="warning_test" ) @@ -85,8 +82,8 @@ def test_bytes(self): self.savegetvar(b"asodpa\x00\x22") def test_date(self): - date1 = datetime.datetime.now() - date2 = datetime.date.today() + date1 = datetime.datetime.now(tz=datetime.timezone.utc) + date2 = datetime.datetime.now(tz=datetime.timezone.utc).date() scraperwiki.sql.save_var("weird\u1234", date1) self.assertEqual(scraperwiki.sql.get_var("weird\u1234"), str(date1)) scraperwiki.sql.save_var("weird\u1234", date2) @@ -107,7 +104,7 @@ def test_get(self): class TestSaveVar(DBTestCase): def setUp(self): - super(TestSaveVar, self).setUp() + super().setUp() scraperwiki.sql.save_var("birthday\xfe", "\u1234November 30, 1888") connection = sqlite3.connect(DB_NAME) self.cursor = connection.cursor() @@ -142,13 +139,13 @@ def save_and_check(self, dataIn, tableIn, dataOut, tableOut=None, twice=True): # Observe with pysqlite connection = sqlite3.connect(DB_NAME) cursor = connection.cursor() - cursor.execute("SELECT * FROM %s" % tableOut) + cursor.execute(f"SELECT * FROM {tableOut}") observed1 = cursor.fetchall() connection.close() if twice: # Observe using this module - observed2 = scraperwiki.sql.select("* FROM %s" % tableOut) + observed2 = scraperwiki.sql.select(f"* FROM {tableOut}") # Check expected1 = dataOut @@ -200,7 +197,7 @@ def test_two(self): } try: self.assertDictEqual(observed, expected1) - except Exception: + except AssertionError: self.assertDictEqual(observed, expected2) # Uniqueness @@ -241,8 +238,8 @@ def test_add_column(self): ) stdout, stderr = process.communicate() assert process.returncode == 0 - self.assertEqual(stdout, "".encode("utf-8")) - self.assertEqual(stderr, "".encode("utf-8")) + self.assertEqual(stdout, b"") + self.assertEqual(stderr, b"") class TestSave(SaveAndCheck): @@ -280,12 +277,12 @@ def test_save_table_name(self): subsequent .save without table_name= uses the `swdata` table again. """ - scraperwiki.sql.save(["id"], dict(id=1, stuff=1), table_name="sticky\u1234") - scraperwiki.sql.save(["id"], dict(id=2, stuff=2)) + scraperwiki.sql.save(["id"], {"id": 1, "stuff": 1}, table_name="sticky\u1234") + scraperwiki.sql.save(["id"], {"id": 2, "stuff": 2}) results = scraperwiki.sql.select("* FROM sticky\u1234") self.assertEqual(1, len(results)) (row,) = results - self.assertDictEqual(dict(id=1, stuff=1), row) + self.assertDictEqual({"id": 1, "stuff": 1}, row) def test_lxml_string(self): """Save lxml string.""" @@ -301,9 +298,9 @@ def test_lxml_string(self): self.save_and_check({"text": s}, "lxml", [(str(s),)]) def test_save_and_drop(self): - scraperwiki.sql.save([], dict(foo=7), table_name="dropper\xaa") + scraperwiki.sql.save([], {"foo": 7}, table_name="dropper\xaa") scraperwiki.sql.execute("DROP TABLE dropper\xaa") - scraperwiki.sql.save([], dict(foo=9), table_name="dropper\xaa") + scraperwiki.sql.save([], {"foo": 9}, table_name="dropper\xaa") class TestQuestionMark(DBTestCase): @@ -333,13 +330,17 @@ class TestDateTime(DBTestCase): def rawdate(self, table="swdata", column="datetime"): connection = sqlite3.connect(DB_NAME) cursor = connection.cursor() - cursor.execute("SELECT {} FROM {}".format(column, table)) + cursor.execute(f"SELECT {column} FROM {table}") rawdate = cursor.fetchall()[0][0] connection.close() return rawdate def test_save_date(self): - d = datetime.datetime.strptime("1991-03-30", "%Y-%m-%d").date() + d = ( + datetime.datetime.strptime("1991-03-30", "%Y-%m-%d") + .replace(tzinfo=datetime.timezone.utc) + .date() + ) with scraperwiki.sql.Transaction(): scraperwiki.sql.save([], {"birthday\xaa": d}) @@ -355,24 +356,16 @@ def test_save_date(self): self.assertEqual(str(d), self.rawdate(column="birthday\xaa")) def test_save_datetime(self): - d = datetime.datetime.strptime("1990-03-30", "%Y-%m-%d") + d = datetime.datetime.strptime("1990-03-30", "%Y-%m-%d").replace( + tzinfo=datetime.timezone.utc + ) with scraperwiki.sql.Transaction(): scraperwiki.sql.save([], {"birthday": d}, table_name="datetimetest") - exemplar = str(d) - # SQLAlchemy appears to convert with extended precision. - exemplar += ".000000" - - self.assertEqual( - [{"birthday": exemplar}], scraperwiki.sql.select("* FROM datetimetest") - ) - self.assertDictEqual( - {"keys": ["birthday"], "data": [(exemplar,)]}, - scraperwiki.sql.execute("SELECT * FROM datetimetest"), - ) + exemplar = d.strftime("%Y-%m-%d %H:%M:%S.000000") self.assertEqual( - exemplar, self.rawdate(table="datetimetest", column="birthday") + [{"birthday": exemplar}], scraperwiki.sql.select("* FROM datetimetest") ) @@ -401,19 +394,19 @@ def setUp(self): self.sw = __import__("scraperwiki") def test_import_scraperwiki_root(self): - self.sw.scrape + _ = self.sw.scrape def test_import_scraperwiki_sqlite(self): - self.sw.sqlite + _ = self.sw.sqlite def test_import_scraperwiki_sql(self): - self.sw.sql + _ = self.sw.sql def test_import_scraperwiki_status(self): - self.sw.status + _ = self.sw.status def test_import_scraperwiki_utils(self): - self.sw.utils + _ = self.sw.utils def test_import_scraperwiki_special_utils(self): - self.sw.pdftoxml + _ = self.sw.pdftoxml