archive: add gitea generic package registry backend

doc/manual/configuration.rst

@@ -2560,7 +2560,7 @@ The default is ``[download, upload]``.
 ``managed``
     This archive is managed, meaning the files can be iterated and deleted.
     This is required for the archive command to work. Only supported with the
-    ``file`` and ``http`` backends so far.
+    ``file``, ``http`` and ``gitea`` backends so far.
 ``cache``
     Use this archive to cache downloaded artifacts from other archives. If a
     binary artifact was successfully downloaded from another archive it will
@@ -2602,6 +2602,14 @@ file        Use a local directory as binary artifact repository. The directory
             directory. The optional ``fileMode`` and ``directoryMode`` keys
             take the desired access modes as numeric value to override the
             default umask derived modes.
+gitea       Uses a `Gitea generic package registry`_ as binary artifact
+            repository. Only the HEAD, GET, PUT and DELETE methods are used.
+            The base server URL is given in ``url``, the registry owner (user
+            or organization) in ``owner`` and the generic package name in
+            ``package``. As with the ``http`` backend the credentials are part
+            of the URL. The optional ``sslVerify`` boolean key controls whether
+            to verify the SSL certificate and ``retries`` sets the number of
+            retries on transient errors.
 http        Uses a HTTP server as binary artifact repository. The server has to
             support the HEAD, PUT and GET methods. The base URL is given in the
             ``url`` key. The optional ``sslVerify`` boolean key controls
@@ -2680,6 +2688,37 @@ the anonymous access to the container can be used like this::
 The ``flags: [download]`` makes sure that Bob does not try to upload artifacts
 in case other backends are configured too.
 
+The ``gitea`` backend stores the artifacts in a Gitea generic package registry.
+The artifacts are put below ``{url}/api/packages/{owner}/generic/{package}/``
+with one package version per artifact::
+
+    archive:
+        -
+            backend: gitea
+            url: "https://user:passw%40rd@gitea.example.com"
+            owner: "bob-artifacts"
+            package: "myproject"
+            flags: [download, upload, managed]
+
+As for the ``http`` backend the credentials for the HTTP basic authentication
+are taken from the URL and have to be percent encoded. Gitea accepts a personal
+access token instead of the password. Uploading requires the ``write:package``
+scope, the ``managed`` flag additionally needs ``read:package``.
+
+.. warning::
+   The password will be part of the Jenkins job configuration. Anybody who can
+   read the jobs ``config.xml`` will be able to retrieve the password!
+
+.. note::
+   The registry cannot enumerate its content. The managed operations of
+   :ref:`manpage-archive` therefore use the package API of the server
+   (``{url}/api/v1/packages/...``) to list the artifacts. As with the ``http``
+   backend only the artifact itself is deleted, the build-id and fingerprint
+   files are left behind. The package version is dropped by the server as soon
+   as its last file is gone.
+
+.. _Gitea generic package registry: https://docs.gitea.com/usage/packages/generic
+
 .. _configuration-config-archive-prepend-append:
 
 archive{Prepend,Append}

pym/bob/archive.py

@@ -24,14 +24,16 @@
 from .tty import stepAction, stepMessage, \
     SKIPPED, EXECUTED, WARNING, INFO, TRACE, ERROR, IMPORTANT
 from .utils import asHexStr, removePath, isWindows, getBashPath, tarfileOpen, binStat
-from .webdav import WebDav, WebdavError, WebdavNotFoundError, WebdavAlreadyExistsError
+from .webdav import WebDav, WebdavError, WebdavNotFoundError, WebdavAlreadyExistsError, \
+    getNetLoc
 from tempfile import mkstemp, NamedTemporaryFile, TemporaryFile, gettempdir
 import asyncio
 import concurrent.futures
 import concurrent.futures.process
 import errno
 import gzip
 import io
+import json
 import os
 import os.path
 import shutil
@@ -861,6 +863,31 @@ def __exit__(self, exc_type, exc_value, traceback):
         return False
 
 
+def retryWebdavRequest(request, retries):
+    """Run a WebDav request, retrying transient transport errors."""
+    while True:
+        try:
+            return request()
+        except (WebdavError, OSError) as e:
+            if retries == 0: raise
+            retries -= 1
+
+
+def getWebdavAudit(webdav, path, retries, extract):
+    """Read the audit trail of a remote artifact.
+
+    Only the beginning of the artifact is fetched. If the audit trail is not
+    part of it yet, more data is requested until it can be extracted.
+    """
+    downloader = retryWebdavRequest(lambda: webdav.getPartialDownloader(path), retries)
+    while True:
+        try:
+            return extract(io.BytesIO(downloader.get()))
+        except (EOFError, tarfile.ReadError):
+            # partial downloader reached EOF or could not extract the audit from the tarfile, so we get more data
+            retryWebdavRequest(lambda: downloader.more(), retries)
+
+
 class HttpArchive(BaseArchive):
     def __init__(self, spec):
         super().__init__(spec)
@@ -874,16 +901,10 @@ def getArchiveName(self):
             return name
 
         url = self.__url
-        return urllib.parse.urlunparse((url.scheme, url.netloc, url.path, '', '', ''))
+        return urllib.parse.urlunparse((url.scheme, getNetLoc(url), url.path, '', '', ''))
 
     def __retry(self, request):
-        retries = self._retries
-        while True:
-            try:
-                return request()
-            except (WebdavError, OSError) as e:
-                if retries == 0: raise
-                retries -= 1
+        return retryWebdavRequest(request, self._retries)
 
     def _canManage(self):
         return True
@@ -899,7 +920,7 @@ def _makeParentDirs(self, path):
 
     def _remoteName(self, buildId, suffix):
         url = self.__url
-        return urllib.parse.urlunparse((url.scheme, url.netloc, self._makePath(buildId, suffix), '', '', ''))
+        return urllib.parse.urlunparse((url.scheme, getNetLoc(url), self._makePath(buildId, suffix), '', '', ''))
 
     def _exists(self, path):
         return self.__retry(lambda: self._webdav.exists(path))
@@ -952,18 +973,11 @@ def _stat(self, filename):
         return struct.pack('=dL', parsedate_to_datetime(stats['mdate']).timestamp(), stats['len'])
 
     def _getAudit(self, filename):
-        downloader = self.__retry(lambda: self._webdav.getPartialDownloader("/".join([self.__url.path, filename])))
-        while True:
-            try:
-                file = io.BytesIO(downloader.get())
-                return self._extractAudit(fileobj=file)
-            except (EOFError, tarfile.ReadError):
-                # partial downloader reached EOF or could not extract the audit from the tarfile, so we get more data
-                self.__retry(lambda: downloader.more())
-                pass
+        return getWebdavAudit(self._webdav, "/".join([self.__url.path, filename]),
+            self._retries, lambda f: self._extractAudit(fileobj=f))
 
     def getArchiveUri(self):
-        return self.__url.netloc + self.__url.path
+        return getNetLoc(self.__url) + self.__url.path
 
 
 class HttpDownloader:
@@ -1211,6 +1225,216 @@ def __upload(self):
             raise ArtifactError(str(e))
 
 
+class GiteaArchive(BaseArchive):
+    """Bob artifact backend for the Gitea 'generic' package registry.
+
+    Artifacts are stored using the generic package API:
+
+        {url}/api/packages/{owner}/generic/{package}/{version}/{filename}
+
+    with ``version`` and ``filename`` both derived from the build-id. All
+    artifact types belonging to the same key (``.tgz``, ``.buildid``,
+    ``.fprnt``) share a single package version.
+
+    The registry is a plain HTTP server as far as Bob is concerned. Only
+    HEAD, GET, PUT and DELETE are used, so the WebDav class does the actual
+    transport. As with the http backend the HTTP basic authentication
+    credentials are part of the URL. Gitea accepts a personal access token in
+    place of the password.
+
+    The managed operations additionally use the package API of the server
+    (``{url}/api/v1/packages/...``) because the registry itself cannot
+    enumerate its content.
+    """
+
+    # The package API does not return more entries than that per request.
+    PAGE_SIZE = 50
+
+    def __init__(self, spec):
+        super().__init__(spec)
+        self.__url = urllib.parse.urlparse(spec["url"])
+        self.__owner = spec["owner"]
+        self.__package = spec["package"]
+        self._webdav = WebDav(self.__url, spec.get("sslVerify", True))
+        self._retries = spec.get("retries", 1)
+        # Caches of the managed operations. They are filled on demand and are
+        # only relevant for the lifetime of a "bob archive" invocation.
+        self.__versions = None
+        self.__files = {}
+
+    def __basePath(self):
+        return "/".join([self.__url.path.rstrip("/"), "api", "packages",
+            self.__owner, "generic", self.__package])
+
+    def getArchiveName(self):
+        name = super().getArchiveName()
+        if name:
+            return name
+        return urllib.parse.urlunparse((self.__url.scheme, getNetLoc(self.__url),
+            self.__basePath(), '', '', ''))
+
+    def _canManage(self):
+        return True
+
+    def __packagePath(self, version, name):
+        return "/".join([self.__basePath(), version, name])
+
+    def _makePath(self, buildId, suffix):
+        # One package version per artifact. The build-id/fingerprint/tarball
+        # files of an artifact all live in that single version.
+        packageResultId = buildIdToName(buildId)
+        return self.__packagePath(packageResultId, packageResultId + suffix)
+
+    def _remoteName(self, buildId, suffix):
+        return urllib.parse.urlunparse((self.__url.scheme, getNetLoc(self.__url),
+            self._makePath(buildId, suffix), '', '', ''))
+
+    def __retry(self, request):
+        return retryWebdavRequest(request, self._retries)
+
+    def _exists(self, path):
+        return self.__retry(lambda: self._webdav.exists(path))
+
+    def _openDownloadFile(self, buildId, suffix):
+        path = self._makePath(buildId, suffix)
+        return self.__retry(lambda: HttpDownloader(self, self._webdav.download(path)))
+
+    def _openUploadFile(self, buildId, suffix, overwrite):
+        path = self._makePath(buildId, suffix)
+        if overwrite:
+            # The generic registry refuses to overwrite an existing file but
+            # the meta data files (build-id, fingerprint) must be replaced.
+            # Delete a possibly existing file first. This is inherently racy:
+            # a concurrent upload may still squeeze in between the delete and
+            # the PUT below and let the latter fail with a conflict.
+            self.__retry(lambda: self._webdav.deletePath(path))
+        elif self._exists(path):
+            raise ArtifactExistsError()
+        return HttpUploader(self, path, overwrite)
+
+    def _putUploadFile(self, path, tmp, overwrite):
+        return self.__retry(lambda: self._webdav.upload(path, tmp, overwrite))
+
+    #
+    # Managed operations. The registry itself cannot enumerate its content.
+    # That is done by the package API of the server instead, which is not part
+    # of the registry but knows about all package types.
+    #
+
+    def __apiPath(self, *parts):
+        return "/".join([self.__url.path.rstrip("/"), "api", "v1", "packages",
+            self.__owner, "generic", self.__package, *parts])
+
+    def __apiGet(self, path, query=None):
+        def request():
+            with self._webdav.download(path, query=query) as reply:
+                return reply.read()
+
+        try:
+            return json.loads(self.__retry(request).decode("utf-8"))
+        except (UnicodeDecodeError, ValueError) as e:
+            raise WebdavError("Invalid reply of package API: " + str(e))
+
+    def __allVersions(self):
+        """All package versions, that is all artifacts of the archive."""
+        if self.__versions is None:
+            versions = []
+            page = 1
+            while True:
+                try:
+                    reply = self.__apiGet(self.__apiPath(),
+                        "page={}&limit={}".format(page, self.PAGE_SIZE))
+                except WebdavNotFoundError:
+                    # The package is created with the first upload. Until then
+                    # the archive is simply empty.
+                    break
+                try:
+                    versions.extend(i["version"] for i in reply)
+                except (KeyError, TypeError):
+                    raise WebdavError("Unexpected reply of package API")
+                if len(reply) < self.PAGE_SIZE: break
+                page += 1
+            self.__versions = versions
+
+        return self.__versions
+
+    def __versionFiles(self, version):
+        """The files of a single package version, indexed by their name.
+
+        A version holds at most the tarball, the build-id and the fingerprint
+        of one artifact, so the reply always fits into a single page.
+        """
+        files = self.__files.get(version)
+        if files is None:
+            try:
+                reply = self.__apiGet(self.__apiPath(version, "files"))
+            except WebdavNotFoundError:
+                reply = []
+            try:
+                files = { i["name"] : i for i in reply }
+            except (KeyError, TypeError):
+                raise WebdavError("Unexpected reply of package API")
+            self.__files[version] = files
+
+        return files
+
+    @staticmethod
+    def __splitPath(filename):
+        """Translate an archive command path back to version and file name.
+
+        The archive command works on the "<xx>/<yy>/<rest>.tgz" layout of the
+        file and http backends. Our version is the file name without the
+        separators and without the suffix.
+        """
+        name = filename.replace("\\", "/").replace("/", "")
+        return name.rpartition(".")[0], name
+
+    def _listDir(self, path):
+        prefix = path.replace("\\", "/").strip("/")
+        if prefix == ".": prefix = ""
+        versions = self.__allVersions()
+
+        # The first two levels are just the first four characters of the
+        # version, that is of the build-id.
+        if prefix == "":
+            return sorted({ v[0:2] for v in versions })
+        elif len(prefix) == 2:
+            return sorted({ v[2:4] for v in versions if v.startswith(prefix) })
+
+        # Only on the last level the files matter. Asking for them is
+        # required: uploaded live-build-ids and fingerprints create versions
+        # that hold no artifact at all.
+        prefix = prefix.replace("/", "")
+        return sorted(name[4:] for v in versions if v.startswith(prefix)
+                                for name in self.__versionFiles(v))
+
+    def _stat(self, filename):
+        version, name = self.__splitPath(filename)
+        info = self.__versionFiles(version).get(name)
+        if info is None:
+            raise ArtifactNotFoundError()
+        # Unlike the other backends we get a real content hash instead of a
+        # modification time.
+        stat = info.get("sha256")
+        if not stat:
+            raise ArtifactError("Missing stats for file " + filename)
+        return stat
+
+    def _getAudit(self, filename):
+        version, name = self.__splitPath(filename)
+        return getWebdavAudit(self._webdav, self.__packagePath(version, name),
+            self._retries, lambda f: self._extractAudit(fileobj=f))
+
+    def _delete(self, filename):
+        version, name = self.__splitPath(filename)
+        self.__retry(lambda: self._webdav.deletePath(self.__packagePath(version, name)))
+        # The server drops the version together with its last file.
+        self.__files.pop(version, None)
+
+    def getArchiveUri(self):
+        return getNetLoc(self.__url) + self.__url.path
+
+
 class MultiArchive:
     def __init__(self, archives):
         self.__archives = archives
@@ -1283,6 +1507,8 @@ def getSingleArchiver(recipes, archiveSpec):
         return CustomArchive(archiveSpec, recipes.envWhiteList())
     elif archiveBackend == "azure":
         return AzureArchive(archiveSpec)
+    elif archiveBackend == "gitea":
+        return GiteaArchive(archiveSpec)
     elif archiveBackend == "none":
         return DummyArchive()
     elif archiveBackend == "__jenkins":