From d41422252979cc97b158e7eb32876110697ce1a0 Mon Sep 17 00:00:00 2001 From: "Moises Lopez - https://www.vauxoo.com/" Date: Fri, 7 Aug 2026 12:32:47 -0600 Subject: [PATCH 1/9] [REF] travis2docker: Drop .travis.yml support, keep only deployv images The .travis.yml based generation has not been used for years since the deployv images built from the CI (variables.sh) became the only workflow. Remove the travis yml parsing (env matrix, addons, python versions and apt-source-whitelist submodule), the travis templates, the examples and the PyYAML dependency, fixing the Dependabot alert about PyYAML < 5.1 arbitrary code execution (GHSA-rprw-h62v-c2w7). The --deployv flag is kept as a deprecated no-op for backward compatibility. --- .github/workflows/github-actions.yml | 1 - .gitmodules | 3 - .travis.yml | 68 ---- MANIFEST.in | 1 - README.rst | 31 +- examples/example_1.yml | 22 -- examples/example_2.yml | 30 -- examples/example_3.yml | 37 -- examples/example_4.yml | 41 --- examples/example_5.yml | 42 --- requirements.txt | 2 - setup.py | 2 +- src/travis2docker/cli.py | 103 +----- src/travis2docker/templates/Dockerfile | 89 ----- src/travis2docker/templates/entrypoint.sh | 10 - src/travis2docker/templates/rvm_env.sh | 18 - .../travis-ci-apt-source-whitelist | 1 - src/travis2docker/travis2docker.py | 339 +++--------------- tests/test_travis2docker.py | 202 +++++------ 19 files changed, 156 insertions(+), 886 deletions(-) delete mode 100644 .gitmodules delete mode 100644 .travis.yml delete mode 100644 examples/example_1.yml delete mode 100644 examples/example_2.yml delete mode 100644 examples/example_3.yml delete mode 100644 examples/example_4.yml delete mode 100644 examples/example_5.yml delete mode 100644 src/travis2docker/templates/Dockerfile delete mode 100644 src/travis2docker/templates/entrypoint.sh delete mode 100644 src/travis2docker/templates/rvm_env.sh delete mode 160000 src/travis2docker/travis-ci-apt-source-whitelist diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index 5a204ba..e8f59a1 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -45,7 +45,6 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 - submodules: recursive - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 38bf737..0000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "src/travis2docker/travis-ci-apt-source-whitelist"] - path = src/travis2docker/travis-ci-apt-source-whitelist - url = https://github.com/travis-ci/apt-source-whitelist.git diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 71a6a65..0000000 --- a/.travis.yml +++ /dev/null @@ -1,68 +0,0 @@ -language: python -sudo: false -env: - global: - - LD_PRELOAD=/lib/x86_64-linux-gnu/libSegFault.so - - SEGFAULT_SIGNALS=all -matrix: - include: - - python: 2.7 - env: - TOXENV=py27,codecov - # TODO: Fix coverage diff version issue - # - python: 3.4 - # env: - # TOXENV=py34,codecov - - python: 3.5 - env: - TOXENV=py35,codecov - - python: 3.6 - env: - TOXENV=py36,codecov - - python: 3.7 - env: - TOXENV=py37,codecov - - python: 3.8 - env: - TOXENV=py38,codecov - - python: 3.9 - env: - TOXENV=py39,codecov - - python: pypy - env: - TOXENV=pypy,codecov - - python: 3.5 - env: - TOXENV=check - - python: 3.5 - env: - TOXENV=docs -before_install: - - python --version - - uname -a - - lsb_release -a -install: - - pip install tox - - virtualenv --version - - easy_install --version - - pip --version - - tox --version -script: - - tox -ve $TOXENV -after_failure: - - more .tox/log/* | cat - - more .tox/*/log/* | cat -before_cache: - - rm -rf $HOME/.cache/pip/log -cache: - directories: - - $HOME/.cache/pip -deploy: - provider: pypi - user: vauxoo - password: - secure: dScWMD2+phGrPeufO2TTQLa3ZW5olWrSf1xXwki2UcrPjF4vIHmFUFoHZ9zK3BPVnY114XArb7O5N0pGO+j9b8G3jXD0HA+jlkMEtrbRJ1kiKUAb4KzF3xs9S/qLNZ2I3+pcE29+faDEV4CErsmlVK/h2HEYVdseH+V0qXUGxEk2bY6NKQ5LGkhb4Ole3wWLah11vvAEGcYQLd0K/hRoNup0bDgmjacH4gfCXLfhNJkz3hw1D/FPYv2Z7AIVvo/UMQS62LisMKKcw7DltzMg7rp1tuQdt7gUBD/EGiqlyVqSgj7d0XOO3HucJK9KATQRDoetVwCQSiR5GwMu39zqijpd1AH5T4mxm9w7xAG9KGlog12dDjZe5eneGGXxfxApZ5K12huiSfxAMHiF6Yhm3BDNTh3WeWAYxH2o3XoqoB5DZowWWAw9Fdl3iG7QdNaboVZ6iOxPUxERJYkle5a74RraQyPEdv7xHyNZhjMrc1NOIzYswGsvMgW/TyCKpBqidSFrkOf6hCGOdMbRXmqtM+M2mAV4WuIJx3fbXT3nvpHmdMMMnwyAYY35CQaz2RcQ3mV7WcyKi0zyrys3wcgsZ2lBMxiqpmBJzDAZQDhBtX8xIKvtdZHf8lbUQFrQttwBgsZIL3eKVFE37iKJEIhMnWXvPcYyhhBYdeu4E9MarSQ= - on: - tags: true - condition: $TOXENV == py37,codecov - repo: Vauxoo/travis2docker diff --git a/MANIFEST.in b/MANIFEST.in index 9dcd891..62e24dd 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,4 @@ graft docs -graft examples graft src graft tests diff --git a/README.rst b/README.rst index bba43f2..db44839 100644 --- a/README.rst +++ b/README.rst @@ -53,7 +53,7 @@ Overview .. end-badges -Script to generate Dockerfile from .travis.yml file +Script to generate a development Dockerfile from the deployv image of a repository (based on its ``variables.sh`` file) * Free software: BSD license @@ -81,19 +81,19 @@ Example: `travisfile2dockerfile --root-path=$HOME/t2d git@github.com:Vauxoo/forecast.git 8.0` The output is: - `${HOME}/t2d/script/git_github.com_Vauxoo_forecast.git/8.0/1` - `${HOME}/t2d/script/git_github.com_Vauxoo_forecast.git/8.0/2` - -The first one is the build for env `TESTS=1`, the second one is for env with `LINT_CHECK=1` + `${HOME}/t2d/script/git_github.com_Vauxoo_forecast.git/8.0` To build image: - `${HOME}/t2d/script/git_github.com_Vauxoo_forecast.git/8.0/1/10-build.sh` + `${HOME}/t2d/script/git_github.com_Vauxoo_forecast.git/8.0/10-build.sh` To create container: - `${HOME}/t2d/script/git_github.com_Vauxoo_forecast.git/8.0/1/20-run.sh --entrypoint=bash` + `${HOME}/t2d/script/git_github.com_Vauxoo_forecast.git/8.0/20-run.sh --entrypoint=bash` -To run the test (into of container): - `/entrypoint.sh` +The repository needs a ``variables.sh`` file in its root path (the one used by +the deployv images built from the CI). By default the docker image is built +from its values as ``DOCKER_IMAGE_REPO:MAIN_APP-VERSION-SHA_SHORT``; use +``--docker-image=quay.io/vauxoo/PROJECT:TAG`` to pick the image pushed by the +``build_docker`` pipeline instead. Optional tools (``--build-env-args``) ===================================== @@ -152,19 +152,6 @@ Dockerfile doesn't support a prompt to enter your password, so you need to remov openssl rsa -in ${fname} -out ${fname}_without_pwd cp ${fname}_without_pwd ${fname} -Download the big image -********************** - -Travis2docker uses a default image with many packages pre-installed. - -`docker pull vauxoo/odoo-80-image-shippable-auto` - -Note: You can define a custom image to use with `--docker-image` parameter. - -For example if you want use the original image of travis you can add the following parameters: - -`--docker-image=quay.io/travisci/travis-python --docker-user=travis` - Install docker ************** diff --git a/examples/example_1.yml b/examples/example_1.yml deleted file mode 100644 index 6aa3401..0000000 --- a/examples/example_1.yml +++ /dev/null @@ -1,22 +0,0 @@ -language: python - -sudo: false -cache: - apt: true - directories: - - $HOME/.cache/pip - -python: - - "2.7" - -virtualenv: - system_site_packages: true - -install: - - touch install - -script: - - touch script - -after_success: - - touch after_success diff --git a/examples/example_2.yml b/examples/example_2.yml deleted file mode 100644 index 164d987..0000000 --- a/examples/example_2.yml +++ /dev/null @@ -1,30 +0,0 @@ -language: python - -sudo: false -cache: - apt: true - directories: - - $HOME/.cache/pip - -addons: - apt: - packages: - - python-lxml - -python: - - "2.7" - -env: - - VARIABLE="value" - -virtualenv: - system_site_packages: true - -install: - - touch install - -script: - - touch script - -after_success: - - touch after_success diff --git a/examples/example_3.yml b/examples/example_3.yml deleted file mode 100644 index 34e2089..0000000 --- a/examples/example_3.yml +++ /dev/null @@ -1,37 +0,0 @@ -language: python - -sudo: false -cache: - apt: true - directories: - - $HOME/.cache/pip - -addons: - apt: - sources: - - pov-wkhtmltopdf - packages: - - python-lxml - - wkhtmltopdf - -python: - - "2.7" - -env: - global: - - VARIABLE_GLOBAL="value global" - matrix: - - VARIABLE_MATRIX_1="value matrix 1" - - VARIABLE_MATRIX_2="value matrix 2" - -virtualenv: - system_site_packages: true - -install: - - touch install - -script: - - touch script - -after_success: - - touch after_success diff --git a/examples/example_4.yml b/examples/example_4.yml deleted file mode 100644 index b1f80d6..0000000 --- a/examples/example_4.yml +++ /dev/null @@ -1,41 +0,0 @@ -language: python - -sudo: false -cache: - apt: true - directories: - - $HOME/.cache/pip - -addons: - apt: - sources: - - pov-wkhtmltopdf - - chef-stable-trusty - packages: - - python-lxml - - wkhtmltopdf - - chefdk - -python: - - "2.7" - -matrix: - include: - - python: 2.7 - env: - VARIABLE_INCLUDE_1="value include 1" - - python: 2.7 - env: - VARIABLE_INCLUDE_2="value include 2" - -virtualenv: - system_site_packages: true - -install: - - touch install - -script: - - touch script - -after_success: - - touch after_success diff --git a/examples/example_5.yml b/examples/example_5.yml deleted file mode 100644 index d12b736..0000000 --- a/examples/example_5.yml +++ /dev/null @@ -1,42 +0,0 @@ -language: python - -sudo: false -cache: - apt: true - directories: - - $HOME/.cache/pip - -addons: - postgresql: "9.5" - apt: - sources: - - pov-wkhtmltopdf - - chef-stable-trusty - packages: - - python-lxml - - wkhtmltopdf - - chefdk - -python: - - "2.7" - -matrix: - include: - - python: 2.7 - env: - VARIABLE_INCLUDE_1="value include 1" - - python: 2.7 - env: - VARIABLE_INCLUDE_2="value include 2" - -virtualenv: - system_site_packages: true - -install: - - touch install - -script: - - touch script - -after_success: - - touch after_success diff --git a/requirements.txt b/requirements.txt index c83de08..b8b84ee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,2 @@ -PyYAML==3.12 ; python_version < '3.7' -PyYAML==6.0.* ; python_version >= '3.7' Jinja2==2.11.3; python_version < '3.7' Jinja2==3.1.*; python_version >= '3.7' diff --git a/setup.py b/setup.py index 1a86bc5..c3b2800 100755 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ def read(*names, **kwargs): name="travis2docker", version="6.4.28", license="BSD-3-Clause", - description="Script to generate Dockerfile from .travis.yml file", + description="Script to generate a development Dockerfile from the deployv image of a repository", long_description="%s\n%s" % ( re.compile("^.. start-badges.*^.. end-badges", re.M | re.S).sub("", read("README.rst")), diff --git a/src/travis2docker/cli.py b/src/travis2docker/cli.py index 9816c2f..1bd0cc3 100644 --- a/src/travis2docker/cli.py +++ b/src/travis2docker/cli.py @@ -15,7 +15,7 @@ import argparse import os -from os.path import expanduser, expandvars, isdir, isfile, join +from os.path import expanduser, join from sys import stdout from . import __version__ @@ -29,7 +29,6 @@ def get_git_data(project, path, revision): git_obj.update() data = { "sha": git_obj.get_sha(revision), - "content": git_obj.show_file(".travis.yml", revision) or git_obj.show_file(".t2d.yml", revision), "variables_sh": git_obj.show_file("variables.sh", revision), "repo_owner": git_obj.owner, "repo_project": git_obj.repo, @@ -41,28 +40,13 @@ def get_git_data(project, path, revision): return data -def yml_read(yml_path): - yml_path_expanded = expandvars(expanduser(yml_path)) - alt_yml_path_expanded = None - if isdir(yml_path_expanded): - yml_path_expanded = join(yml_path_expanded, ".travis.yml") - alt_yml_path_expanded = join(yml_path_expanded, ".t2d.yml") - if not isfile(yml_path_expanded): - if alt_yml_path_expanded and isfile(alt_yml_path_expanded): - yml_path_expanded = alt_yml_path_expanded - else: - return - with open(yml_path_expanded) as f_yml: - return f_yml.read() - - def main(return_result=False): parser = argparse.ArgumentParser() parser.add_argument( "git_repo_url", help="Specify repository git of work." "\nThis is used to clone it " - "and get file .travis.yml or .shippable.yml" + "and get the variables.sh file of the deployv image" "\nIf your repository is private, " "don't use https url, " "use ssh url", @@ -78,16 +62,14 @@ def main(return_result=False): parser.add_argument( "--docker-user", dest="docker_user", - help="User of work into Dockerfile." "\nBased on your docker image." "\nDefault: root", + help="User of work into Dockerfile." "\nBased on your docker image." "\nDefault: odoo", ) parser.add_argument( "--docker-image", dest="default_docker_image", help="Docker image to use by default in Dockerfile." - "\nUse this parameter if don't " - "exists value: 'build_image: IMAGE_NAME' " - "in .travis.yml" - "\nDefault: 'vauxoo/odoo-80-image-shippable-auto'", + "\nDefault: built from variables.sh as " + "'DOCKER_IMAGE_REPO:MAIN_APP-VERSION-SHA_SHORT'", ) default_root_path = os.environ.get("TRAVIS2DOCKER_ROOT_PATH") if not default_root_path: @@ -104,13 +86,6 @@ def main(return_result=False): dest="remotes", help="Add git remote to git of build path, separated by a comma." "\nUse remote name. E.g. 'Vauxoo,moylop260'", ) - parser.add_argument( - "--exclude-after-success", - dest="exclude_after_success", - action="store_true", - default=False, - help="Exclude `travis_after_success` section to entrypoint", - ) parser.add_argument( "--run-extra-args", dest="run_extra_args", @@ -139,19 +114,6 @@ def main(return_result=False): default="", help='Extra commands to run after "build" script. ' "Note: You can use \\$IMAGE escaped environment variable.", ) - parser.add_argument( - "--travis-yml-path", - dest="travis_yml_path", - help="Optional path of file .travis.yml to use.\n" "Default: Extracted from git repo and git revision.", - default=None, - ) - parser.add_argument( - "--no-clone", - dest="no_clone", - action="store_true", - help="Avoid clone the repository. It will require travis-yml-path", - default=False, - ) parser.add_argument( "--add-rcfile", dest="add_rcfile", @@ -160,13 +122,6 @@ def main(return_result=False): "copy for user's HOME path into container, separated by a comma.", ) parser.add_argument("-v", "--version", action="version", version="%(prog)s " + __version__) - parser.add_argument( - "--runs-at-the-end-script", - dest="runs_at_the_end_script", - nargs="*", - default="", - help='Extra commands to run after "script" file. ' "Note: You can use \\$IMAGE escaped environment variable.", - ) parser.add_argument( "--build-env-args", dest="build_env_args", @@ -183,8 +138,8 @@ def main(return_result=False): "--deployv", dest="deployv", action="store_true", - default=False, - help="Use the image generated from the CI and used in deployV", + default=True, + help="Deprecated. The deployv image is now the only supported mode", ) parser.add_argument( "--build-extra-steps", @@ -202,63 +157,33 @@ def main(return_result=False): root_path = args.root_path default_docker_image = args.default_docker_image remotes = args.remotes and args.remotes.split(",") - exclude_after_success = args.exclude_after_success run_extra_args = args.run_extra_args build_extra_args = args.build_extra_args - travis_yml_path = args.travis_yml_path build_extra_cmds = "\n".join(args.build_extra_cmds) run_extra_cmds = "\n".join(args.run_extra_cmds) - no_clone = args.no_clone - deployv = args.deployv rcfiles_args = args.add_rcfile and args.add_rcfile.split(",") - runs_at_the_end_script = args.runs_at_the_end_script or None build_env_args = [build_env_args[0] for build_env_args in args.build_env_args] rcfiles = [(expanduser(rc_file), os.path.join("$HOME", os.path.basename(rc_file))) for rc_file in rcfiles_args] - if no_clone: - os_kwargs = { - "repo_owner": "local_file", - "repo_project": "local_file", - "revision": revision, - "sha": "local_file", - "project": git_repo, - } - else: - os_kwargs = get_git_data(git_repo, join(root_path, "repo"), revision) - - if travis_yml_path: - yml_content = yml_read(travis_yml_path) - else: - yml_content = os_kwargs["content"] - - if not yml_content and os_kwargs.get("variables_sh"): - deployv = True - yml_content = "deployv: True" - if not default_docker_image and not deployv: - default_docker_image = "vauxoo/odoo-80-image-shippable-auto" + os_kwargs = get_git_data(git_repo, join(root_path, "repo"), revision) - if not yml_content: + if not os_kwargs.get("variables_sh"): msg = ( - "The file %s is empty." % travis_yml_path - if travis_yml_path - else "The repo or the branch is incorrect value, because " - + "It can not got the .travis.yml or variables.sh content from %s %s. " % (git_repo, revision) + "The repo or the branch is incorrect value, because " + + "It can not got the variables.sh content from %s %s. " % (git_repo, revision) + "\nPlease, verify access repository," + "\nverify exists url and revision, " - + "\nverify exists .travis.yml" + + "\nverify exists variables.sh" ) raise InvalidRepoBranchError(msg) os_kwargs.update({"remotes": remotes, "git_base": git_base}) if docker_user: os_kwargs.update({"user": docker_user}) t2d = Travis2Docker( - yml_buffer=yml_content, work_path=join(root_path, "script", GitRun.url2dirname(git_repo), revision), image=default_docker_image, os_kwargs=os_kwargs, copy_paths=[(expanduser("~/.ssh"), "$HOME/.ssh")] + rcfiles, - runs_at_the_end_script=runs_at_the_end_script, build_env_args=build_env_args, - deployv=deployv, build_extra_steps=args.build_extra_steps, ) t2d.build_extra_params = { @@ -269,11 +194,11 @@ def main(return_result=False): "extra_params": run_extra_args, "extra_cmds": run_extra_cmds, } - fname_scripts = t2d.compute_dockerfile(skip_after_success=exclude_after_success) + fname_scripts = t2d.compute_dockerfile() if fname_scripts: fname_list = "- " + "\n- ".join(fname_scripts) stdout.write("\nGenerated scripts:\n%s\n" % fname_list) - if deployv and not default_docker_image: + if not default_docker_image: stdout.write("=" * 80) # TODO: Add the URL to open the pipelines stdout.write( diff --git a/src/travis2docker/templates/Dockerfile b/src/travis2docker/templates/Dockerfile deleted file mode 100644 index ba7304c..0000000 --- a/src/travis2docker/templates/Dockerfile +++ /dev/null @@ -1,89 +0,0 @@ -FROM {{ image }} -COPY {{ entrypoint_path }} /entrypoint.sh -RUN chown -R {{ user }}:{{ user }} /entrypoint.sh -COPY {{ rvm_env_path }} /rvm_env.sh -RUN chown -R {{ user }}:{{ user }} /rvm_env.sh - -{%- for build_env_arg in build_env_args %} -ARG {{ build_env_arg }} -ENV {{ build_env_arg }}=${{ build_env_arg }} -{% endfor %} - -ENV HOME= -{%- if user == 'root' -%} -/root -{%- else -%} -/home/{{ user }} -{%- endif %} - -{% if image == 'quay.io/travisci/travis-python' -%} -ENV PATH=${PATH}:/home/travis/.nvm/v0.10.36/bin:/home/travis/.nvm/v0.10.36/lib/node_modules/npm/bin -{%- endif %} - -{%- for src, dest in copies or [] %} -COPY {{ src }} {{ dest }} -{% endfor -%} - -{% if copies -%} -RUN {% for src, dest in copies or [] -%} chown -R {{ user }}:{{ user }} {{dest}}; -{%- endfor -%} -{%- endif %} - -{% if sources -%} -RUN {{ ' && '.join(sources) }} -{%- endif %} - -{% if packages -%} -RUN apt-get update; apt-get install {{ ' '.join(packages) }} -{%- endif %} - -RUN echo "TRAVIS_PYTHON_VERSION={{ python_version }}" >> /etc/environment - -{% for step in build_extra_steps %} -RUN {{ step }} -{% endfor %} - -USER {{ user }} -ENV TRAVIS_PYTHON_VERSION={{ python_version }} -ENV TRAVIS_REPO_SLUG={{ repo_owner }}/{{ repo_project }} -ENV TRAVIS_BUILD_DIR=${HOME}/build/${TRAVIS_REPO_SLUG} -RUN git init ${TRAVIS_BUILD_DIR} \ - && cd ${TRAVIS_BUILD_DIR} \ - && git remote add origin {{ project }} \ - && git fetch --update-head-ok -p origin \ -{% if revision.startswith('pull/') -%} - '+refs/{{ revision }}/head:refs/{{ revision }}' || true && \ - git fetch --update-head-ok -p origin \ - '+refs/{{ revision.replace('pull/', 'merge-requests/') }}/head:refs/{{ revision }}' || true -{%- else -%} - '+refs/heads/{{ revision }}:refs/heads/{{ revision }}' -{%- endif %} \ - && git checkout -qf {{ revision }} \ - && git config --global user.email "{{ git_email }}" \ - && git config --global user.name "{{ git_user }}" \ -{%- for remote in remotes or [] %} - && git remote add {{ remote }} {{git_base}}:{{ remote }}/{{ repo_project }}.git \ -{%- endfor %} - || true - -{% if add_self_rsa_pub -%} -RUN cat ${HOME}/.ssh/id_rsa.pub | tee -a ${HOME}/.ssh/authorized_keys -{%- endif %} - -{% if env -%} -ENV {{ env }} -{%- endif %} - -WORKDIR ${TRAVIS_BUILD_DIR} - - -{% if runs -%} -{% if image == 'quay.io/travisci/travis-python' -%} -RUN /bin/bash -c "source $HOME/virtualenv/python{{ python_version }}/bin/activate && source /rvm_env.sh && {{ ' && '.join(runs) }}" -{% elif image == 'vauxoo/odoo-80-image-shippable-auto' -%} -RUN /bin/bash -c "source ${REPO_REQUIREMENTS}/virtualenv/python{{ python_version }}/bin/activate && source ${REPO_REQUIREMENTS}/virtualenv/nodejs/bin/activate && source /rvm_env.sh && {{ ' && '.join(runs) }}" -{% else %} -RUN /bin/bash -c "source /rvm_env.sh && {{ ' && '.join(runs) }}" -{%- endif %} -{%- endif %} -ENTRYPOINT /entrypoint.sh diff --git a/src/travis2docker/templates/entrypoint.sh b/src/travis2docker/templates/entrypoint.sh deleted file mode 100644 index 38ccbc6..0000000 --- a/src/travis2docker/templates/entrypoint.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -{% if image == 'quay.io/travisci/travis-python' -%} -source /home/travis/virtualenv/python{{ python_version }}/bin/activate -{% elif image == 'vauxoo/odoo-80-image-shippable-auto' -%} -source ${REPO_REQUIREMENTS}/virtualenv/python{{ python_version }}/bin/activate && source ${REPO_REQUIREMENTS}/virtualenv/nodejs/bin/activate -{%- endif %} -source /rvm_env.sh -{% for entrypoint in entrypoints %} -{{ entrypoint }} -{% endfor %} diff --git a/src/travis2docker/templates/rvm_env.sh b/src/travis2docker/templates/rvm_env.sh deleted file mode 100644 index 2ba94d8..0000000 --- a/src/travis2docker/templates/rvm_env.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -# Load RVM into a shell session *as a function* -if [[ -s "$HOME/.rvm/scripts/rvm" ]] ; then - - # First try to load from a user install - source "$HOME/.rvm/scripts/rvm" - -elif [[ -s "/usr/local/rvm/scripts/rvm" ]] ; then - - # Then try to load from a root install - source "/usr/local/rvm/scripts/rvm" - -else - - printf "An RVM installation was not found.\n" - -fi diff --git a/src/travis2docker/travis-ci-apt-source-whitelist b/src/travis2docker/travis-ci-apt-source-whitelist deleted file mode 160000 index 92dbd15..0000000 --- a/src/travis2docker/travis-ci-apt-source-whitelist +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 92dbd15f6a12e5f24c2f31f314060a349c37dc68 diff --git a/src/travis2docker/travis2docker.py b/src/travis2docker/travis2docker.py index 94b455b..268225f 100644 --- a/src/travis2docker/travis2docker.py +++ b/src/travis2docker/travis2docker.py @@ -1,7 +1,4 @@ -# pylint: disable=useless-object-inheritance,consider-using-with,too-complex,print-used -import collections -import errno -import json +# pylint: disable=useless-object-inheritance,consider-using-with,print-used import os import re import shutil @@ -10,11 +7,6 @@ import jinja2 -try: - from yaml import full_load as yaml_load -except ImportError: - from yaml import load as yaml_load - RE_ENV_STR = r"(?P[\w]*)[ ]*[\=][ ]*[\"\']{0,1}" + r"(?P[\w\.\-\_/\$\{\}\:,\(\)\#\* ]*)[\"\']{0,1}" RE_EXPORT_STR = r"^(?Pexport|EXPORT)( )+" + RE_ENV_STR @@ -24,10 +16,7 @@ class Travis2Docker: @property def dockerfile_template(self): - dockerfile = "Dockerfile" - if self.deployv: - dockerfile += "_deployv" - return self.jinja_env.get_template(dockerfile) + return self.jinja_env.get_template("Dockerfile_deployv") @property def new_image(self): @@ -38,10 +27,6 @@ def new_image(self): revision = revision.replace(invalid_char, "_") return ("%s:%s" % (image_name, revision)).lower() - @property - def entrypoint_template(self): - return self.jinja_env.get_template("entrypoint.sh") - @property def build_template(self): return self.jinja_env.get_template("10-build.sh") @@ -56,88 +41,54 @@ def chmod_execution(file_path): @staticmethod def mkdir_p(path): - try: - os.makedirs(path) - except OSError as os_error: - if os_error.errno == errno.EEXIST and os.path.isdir(path): - pass - else: - raise + os.makedirs(path, exist_ok=True) def __init__( self, - yml_buffer, image=None, work_path=None, dockerfile=None, templates_path=None, os_kwargs=None, copy_paths=None, - runs_at_the_end_script=None, build_env_args=None, - deployv=None, build_extra_steps=None, ): - self._python_versions = [] self.curr_work_path = None - self.curr_exports = [] self.build_extra_params = {} self.run_extra_params = {} self.build_env_args = build_env_args - self.deployv = deployv - self.runs_at_the_end_script = ["sleep 2"] if runs_at_the_end_script is None else runs_at_the_end_script - self.variables_sh_data = {} self.build_extra_steps = build_extra_steps - if deployv: - self.variables_sh_data = { - var.lower(): value for _, _, var, value in self.re_export.findall(os_kwargs["variables_sh"]) - } - self.variables_sh_data.update({"sha_short": os_kwargs["sha"][:7]}) - image = ( - "%(docker_image_repo)s:%(main_app)s-%(version)s-%(sha_short)s" % self.variables_sh_data - if not image - else image - ) - build_sh = os.path.join(os.path.dirname(os.path.realpath(__file__)), "templates", "build.sh") - entrypoint_sh = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "templates", - "entrypoint_deployv.sh", - ) - docker_helper = os.path.join(os.path.dirname(os.path.realpath(__file__)), "docker_helper") - vscode_conf = os.path.join(os.path.dirname(os.path.realpath(__file__)), "templates", ".vscode") - coveragerc = os.path.join(os.path.dirname(os.path.realpath(__file__)), "templates", ".coveragerc") - copy_paths.append([build_sh, "/home/odoo/build.sh"]) - copy_paths.append([entrypoint_sh, "/entrypoint.sh"]) - copy_paths.append([docker_helper, "/home/odoo/build"]) - copy_paths.append([vscode_conf, "/home/odoo/.vscode"]) - copy_paths.append([coveragerc, "/home/odoo/.coveragerc"]) - if image is None: - image = "vauxoo/odoo-80-image-shippable-auto" if os_kwargs is None: os_kwargs = {} - default_user = "root" - if image == "vauxoo/odoo-80-image-shippable-auto" or deployv: - default_user = "odoo" - elif image == "quay.io/travisci/travis-python": - default_user = "travis" - os_kwargs.setdefault("user", default_user) + if copy_paths is None: + copy_paths = [] + self.variables_sh_data = { + var.lower(): value for _, _, var, value in self.re_export.findall(os_kwargs["variables_sh"]) + } + self.variables_sh_data.update({"sha_short": os_kwargs["sha"][:7]}) + if not image: + image = "%(docker_image_repo)s:%(main_app)s-%(version)s-%(sha_short)s" % self.variables_sh_data + templates_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "templates") + build_sh = os.path.join(templates_dir, "build.sh") + entrypoint_sh = os.path.join(templates_dir, "entrypoint_deployv.sh") + docker_helper = os.path.join(os.path.dirname(os.path.realpath(__file__)), "docker_helper") + vscode_conf = os.path.join(templates_dir, ".vscode") + coveragerc = os.path.join(templates_dir, ".coveragerc") + copy_paths.append([build_sh, "/home/odoo/build.sh"]) + copy_paths.append([entrypoint_sh, "/entrypoint.sh"]) + copy_paths.append([docker_helper, "/home/odoo/build"]) + copy_paths.append([vscode_conf, "/home/odoo/.vscode"]) + copy_paths.append([coveragerc, "/home/odoo/.coveragerc"]) + os_kwargs.setdefault("user", "odoo") if dockerfile is None: dockerfile = "Dockerfile" if templates_path is None: - templates_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "templates") + templates_path = templates_dir self.copy_paths = copy_paths self.os_kwargs = os_kwargs self.jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(templates_path)) self.image = image - self._sections = collections.OrderedDict() - self._sections["env"] = "env" - self._sections["addons"] = "addons" - self._sections["before_install"] = "run" - self._sections["install"] = "run" - self._sections["script"] = "entrypoint" - self._sections["after_success"] = "entrypoint" - self.yml = yaml_load(yml_buffer) if work_path is None: base_name = os.path.splitext(os.path.basename(__file__))[0] self.work_path = os.path.join(gettempdir(), base_name) @@ -145,230 +96,40 @@ def __init__( self.work_path = os.path.expandvars(os.path.expanduser(work_path)) self.dockerfile = dockerfile - travis_ci_apt_src = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "travis-ci-apt-source-whitelist", - ) - self.ubuntu_json = json.load( - open(os.path.join(travis_ci_apt_src, "ubuntu.json")) - ) # pylint: disable=consider-using-with - - def _compute(self, section, yml=None): - if yml is None: - yml = self.yml - section_type = self._sections.get(section) - if not section_type: - return None - section_data = yml.get(section, "") - if section != "env" and not section_data: - return None - if not isinstance(section_data, (list, dict, tuple)): - section_data = [section_data] - job_method = getattr(self, "_compute_" + section_type) - return job_method(section_data, section, yml) - - def _compute_env(self, data, _, yml): - if isinstance(data, list): - # old version without matrix - data = {"matrix": data} - env_globals = "" - for env_global in data.get("global", []): - if isinstance(env_global, dict): - # we can't use the secure encrypted variables - continue - env_globals += " " + env_global - env_globals = env_globals.strip() - psql_version = (yml.get("addons") or {}).get("postgresql") - if psql_version: - env_globals += ' PSQL_VERSION="%s"' % psql_version - - for env_matrix in data.get("matrix", [""]): - yield (env_globals + " " + env_matrix).strip() - - def _compute_run(self, data, section, _): - args = self._make_script(data, section, add_run=True, prefix="files") - return args - - def _compute_entrypoint(self, data, section, _): - args = self._make_script(data, section, add_entrypoint=True, prefix="files") - return args - - def _compute_addons(self, data, section, _): - if "apt" not in data: - return - sources = [] - for alias in data["apt"].get("sources") or []: - for ubuntu_source in self.ubuntu_json: - if alias == ubuntu_source["alias"]: - if ubuntu_source["key_url"]: - sources.append('curl -sSL "' + ubuntu_source["key_url"] + '" | apt-key add -') - if ubuntu_source["sourceline"].startswith("ppa:"): - sources.append('apt-add-repository -y "' + ubuntu_source["sourceline"] + '"') - else: - sources.append( - 'echo "' + ubuntu_source["sourceline"] + '" | tee -a /etc/apt/sources.list > /dev/null' - ) - new_data = data["apt"].copy() - new_data["sources"] = sources - return new_data - - def _make_script(self, data, section, add_entrypoint=False, add_run=False, prefix=""): - file_path = os.path.join(self.curr_work_path, prefix, section) - self.mkdir_p(os.path.dirname(file_path)) - with open(file_path, "w") as f_section: - f_section.write("#!/bin/bash\n") - for var, value in self.curr_exports: - f_section.write("\nexport %s=%s" % (var, value)) - for line in data: - self.curr_exports.extend([(var, value) for _, _, var, value in self.re_export.findall(line)]) - f_section.write("\n" + line) - if section == "script": - for run_at_the_end_script in self.runs_at_the_end_script: - f_section.write("\n%s" % run_at_the_end_script) - src = "./" + os.path.relpath(file_path, self.curr_work_path) - dest = "/" + section - args = { - "copies": [(src, dest)], - "entrypoints": [dest] if add_entrypoint else [], - "runs": [dest] if add_run else [], - } - self.chmod_execution(file_path) - return args - - def reset(self): - self.curr_work_path = None - self.curr_exports = [] - - def compute_build_scripts(self, prefix_build, version): + def compute_build_scripts(self): build_path = os.path.join(self.curr_work_path, "10-build.sh") run_path = os.path.join(self.curr_work_path, "20-run.sh") - new_image = self.new_image + "_" + version.replace(".", "_") + "_" + str(prefix_build) with open(build_path, "w") as f_build, open(run_path, "w") as f_run: build_content = self.build_template.render( - image=new_image, dirname_dockerfile=self.curr_work_path, **self.build_extra_params + image=self.new_image, dirname_dockerfile=self.curr_work_path, **self.build_extra_params ).strip("\n ") - try: - f_build.write(build_content.encode("utf-8")) - except TypeError: - f_build.write(build_content) - - run_content = self.run_template.render(image=new_image, **self.run_extra_params).strip("\n ") - try: - f_run.write(run_content.encode("utf-8")) - except TypeError: - f_run.write(run_content) + f_build.write(build_content) + run_content = self.run_template.render(image=self.new_image, **self.run_extra_params).strip("\n ") + f_run.write(run_content) self.chmod_execution(build_path) self.chmod_execution(run_path) - def _python_version_env(self): - versions = self.yml.pop("python", {}) - if not versions: - self._python_versions = ["3.5"] # 3.5 by default - return - if not isinstance(versions, list): - versions = [versions] - # TODO: Use full version if in the default base image are installed - new_versions = {".".join(version.split(".")[:2]) for version in versions} - self._python_versions = list(set(self._python_versions) | new_versions) - - def _transform_yml_matrix2env(self): - matrix = self.yml.pop("matrix", {}) - envs = [include["env"] for include in matrix.get("include", []) if include.get("env")] - if envs: - self.yml["env"] = envs - - def compute_dockerfile(self, skip_after_success=False): - work_paths = [] - self._transform_yml_matrix2env() - self._python_version_env() - jobs_stages = self.yml.pop("jobs", {}).get("include", {}) - for global_version in self._python_versions: - for count, global_env in enumerate(self._compute("env"), 1): - for job_count, job_stage in enumerate(jobs_stages or [{}], 1): - job_env = self._compute("env", job_stage) or "" - if job_env is not None: - job_env = next(job_env) - env = "%s %s" % (global_env, job_env) - env = env.strip() - version = global_version - try: - version = job_stage["python"] - except KeyError: # pylint: disable=except-pass - pass - version = "%s" % version - - self.reset() - self.curr_work_path = os.path.join( - self.work_path, - version.replace(".", "_"), - "env_%d_job_%d" % (count, job_count), - ) - curr_dockerfile = os.path.join(self.curr_work_path, self.dockerfile) - entryp_path = os.path.join(self.curr_work_path, "files", "entrypoint.sh") - self.mkdir_p(os.path.dirname(entryp_path)) - entryp_relpath = os.path.relpath(entryp_path, self.curr_work_path) - rvm_env_path = os.path.join(self.curr_work_path, "files", "rvm_env.sh") - rvm_env_relpath = os.path.relpath(rvm_env_path, self.curr_work_path) - copies = [] - for copy_path, dest in self.copy_paths: - copies.append((self.copy_path(copy_path), dest)) - self.set_authorized_key() - kwargs = { - "runs": [], - "copies": copies, - "entrypoints": [], - "entrypoint_path": entryp_relpath, - "python_version": version, - "image": self.image, - "env": env, - "packages": [], - "sources": [], - "rvm_env_path": rvm_env_relpath, - "build_env_args": self.build_env_args, - "build_extra_steps": self.build_extra_steps, - } - with open(curr_dockerfile, "w") as f_dockerfile, open(entryp_path, "w") as f_entrypoint, open( - rvm_env_path, "w" - ) as f_rvm: - for section, _ in self._sections.items(): - if section == "env": - continue - if skip_after_success and section == "after_success": - continue - # job section replace global one - result = self._compute(section, job_stage) - if not result: - result = self._compute(section) - if not result: - continue - keys_to_extend = ( - ["copies", "runs", "entrypoints", "packages", "sources"] - if isinstance(result, dict) - else [] - ) - for key_to_extend in keys_to_extend: - if key_to_extend in result: - kwargs[key_to_extend].extend(result[key_to_extend]) - kwargs.update(self.os_kwargs) - dockerfile_content = self.dockerfile_template.render(kwargs).strip("\n ") - try: - f_dockerfile.write(dockerfile_content.encode("utf-8")) - except TypeError: - f_dockerfile.write(dockerfile_content) - entrypoint_content = self.entrypoint_template.render(kwargs).strip("\n ") - try: - f_entrypoint.write(entrypoint_content.encode("utf-8")) - except TypeError: - f_entrypoint.write(entrypoint_content) - rvm_env_content = self.jinja_env.get_template("rvm_env.sh").render(kwargs).strip("\n ") - try: - f_rvm.write(rvm_env_content.encode("UTF-8")) - except TypeError: - f_rvm.write(rvm_env_content) - self.compute_build_scripts(count, version) - self.chmod_execution(entryp_path) - work_paths.append(self.curr_work_path) - self.reset() + def compute_dockerfile(self): + self.curr_work_path = self.work_path + curr_dockerfile = os.path.join(self.curr_work_path, self.dockerfile) + self.mkdir_p(self.curr_work_path) + copies = [] + for copy_path, dest in self.copy_paths: + copies.append((self.copy_path(copy_path), dest)) + self.set_authorized_key() + kwargs = { + "copies": copies, + "image": self.image, + "build_env_args": self.build_env_args, + "build_extra_steps": self.build_extra_steps, + } + kwargs.update(self.os_kwargs) + with open(curr_dockerfile, "w") as f_dockerfile: + dockerfile_content = self.dockerfile_template.render(kwargs).strip("\n ") + f_dockerfile.write(dockerfile_content) + self.compute_build_scripts() + work_paths = [self.curr_work_path] + self.curr_work_path = None return work_paths def copy_path(self, path): diff --git a/tests/test_travis2docker.py b/tests/test_travis2docker.py index da70622..478588f 100644 --- a/tests/test_travis2docker.py +++ b/tests/test_travis2docker.py @@ -4,20 +4,21 @@ import os import subprocess import sys +from shutil import which -from travis2docker.cli import main as cli_main - -try: - from shutil import which # python3.x -except ImportError: - from whichcraft import which +import pytest +from travis2docker.cli import main as cli_main +from travis2docker.exceptions import InvalidRepoBranchError -def main(): - return cli_main(return_result=True) +VARIABLES_SH = """export DOCKER_IMAGE_REPO=quay.io/vauxoo/myproject +export MAIN_APP=myproject +export VERSION=16.0 +export CUSTOM_VAR="custom value" +""" -def check_failed_dockerfile(scripts, lines_required=None): +def check_dockerfile_lint(scripts): npm_bin = which("npm") npm_bin_path = subprocess.check_output([npm_bin, "list"]).decode("UTF-8").strip("\n") if npm_bin else "" npm_bin_path_g = subprocess.check_output([npm_bin, "list", "-g"]).decode("UTF-8").strip("\n") if npm_bin else "" @@ -34,123 +35,84 @@ def check_failed_dockerfile(scripts, lines_required=None): output = pipe.stdout.read().decode("utf-8") assert "Check passed" in output, fname_dkr print("Check dockerfile output", output) - if not lines_required: - continue - with open(fname_dkr) as fdkr: - fdkr_lines = fdkr.readlines() - fdkr_lines[-1] = fdkr_lines[-1].strip("\n") + "\n" - for line_required in lines_required: - assert line_required + "\n" in fdkr_lines - print(fdkr_lines) -def test_main(): - # TODO: fix duplicated code - dirname_example = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "examples") - argv = ["travis2docker", "foo", "bar", "--no-clone"] - sources_py = "source ${REPO_REQUIREMENTS}/virtualenv/" + "python2.7/bin/activate" - sources_js = "source ${REPO_REQUIREMENTS}/virtualenv/nodejs/bin/activate" - lines_required = [ - 'RUN /bin/bash -c "{source_py} && {source_js} ' - '&& source /rvm_env.sh && /install"'.format(source_py=sources_py, source_js=sources_js), - "ENTRYPOINT /entrypoint.sh", - ] - - example = os.path.join(dirname_example, "example_1.yml") - sys.argv = argv + [ - "--travis-yml-path", - example, - "--add-rcfile=%s,%s" % (example, dirname_example), +def create_repo(base_path, files): + repo_path = os.path.join(str(base_path), "myrepo") + os.makedirs(repo_path) + subprocess.check_call(["git", "init", "-b", "main", repo_path]) + for fname, content in files.items(): + with open(os.path.join(repo_path, fname), "w") as f_repo: + f_repo.write(content) + subprocess.check_call(["git", "-C", repo_path, "add", "-A"]) + subprocess.check_call( + ["git", "-C", repo_path, "-c", "user.email=test@test.com", "-c", "user.name=test", "commit", "-m", "initial"] + ) + return repo_path + + +def test_main_deployv(tmp_path): + repo = create_repo(tmp_path, {"variables.sh": VARIABLES_SH}) + sys.argv = [ + "travis2docker", + repo, + "main", + "--root-path", + os.path.join(str(tmp_path), "t2d"), + "--build-env-args", + "BUILD_ENV1", + "--build-env-args", + "BUILD_ENV2", + "--build-extra-steps", + "touch /home/odoo/extra_step_done", ] - scripts = main() - assert len(scripts) == 1, "Scripts returned should be 1 for %s" % example - check_failed_dockerfile(scripts, lines_required) - assert os.path.isdir(os.path.join(scripts[0], os.path.basename(dirname_example))) - assert os.path.isfile(os.path.join(scripts[0], os.path.basename(example))) - - sys.argv = argv + [ - "--travis-yml-path", - example, + scripts = cli_main(return_result=True) + assert len(scripts) == 1, "Scripts returned should be 1" + fname_dkr = os.path.join(scripts[0], "Dockerfile") + with open(fname_dkr) as f_dkr: + dkr_content = f_dkr.read() + sha_short = subprocess.check_output(["git", "-C", repo, "rev-parse", "HEAD"]).decode("UTF-8")[:7] + assert "FROM quay.io/vauxoo/myproject:myproject-16.0-%s" % sha_short in dkr_content + assert "ENV BUILD_ENV1=TRUE" in dkr_content + assert "ENV BUILD_ENV2=TRUE" in dkr_content + assert "RUN touch /home/odoo/extra_step_done" in dkr_content + assert "ENTRYPOINT /entrypoint.sh" in dkr_content + assert "COPY build.sh /home/odoo/build.sh" in dkr_content + assert "COPY entrypoint_deployv.sh /entrypoint.sh" in dkr_content + assert "COPY docker_helper /home/odoo/build" in dkr_content + for script in ("10-build.sh", "20-run.sh"): + script_path = os.path.join(scripts[0], script) + assert os.path.isfile(script_path) + assert os.access(script_path, os.X_OK), "%s should be executable" % script + check_dockerfile_lint(scripts) + + +def test_main_docker_image_parameter(tmp_path): + repo = create_repo(tmp_path, {"variables.sh": VARIABLES_SH}) + sys.argv = [ + "travis2docker", + repo, + "main", + "--root-path", + os.path.join(str(tmp_path), "t2d"), "--docker-image", - "quay.io/travisci/travis-python", + "quay.io/vauxoo/myproject:custom-tag", ] - scripts = main() - assert len(scripts) == 1, "Scripts returned should be 1 for %s" % example - check_failed_dockerfile(scripts, ["FROM quay.io/travisci/travis-python"]) - - example = os.path.join(dirname_example, "example_2.yml") - sys.argv = argv + ["--travis-yml-path", example] - scripts = main() - assert len(scripts) == 1, "Scripts returned should be 1 for %s" % example - check_failed_dockerfile(scripts, lines_required + ['ENV VARIABLE="value"']) - - example = os.path.join(dirname_example, "example_3.yml") - sys.argv = argv + ["--travis-yml-path", example] - scripts = main() - assert len(scripts) == 2, "Scripts returned should be 2 for %s" % example - check_failed_dockerfile(scripts, lines_required) + scripts = cli_main(return_result=True) + assert len(scripts) == 1, "Scripts returned should be 1" with open(os.path.join(scripts[0], "Dockerfile")) as f_dkr: dkr_content = f_dkr.read() - assert 'VARIABLE_MATRIX_1="value matrix 1"' in dkr_content - assert 'ENV VARIABLE_GLOBAL="value global"' in dkr_content - assert "RUN apt-add-repository" in dkr_content - with open(os.path.join(scripts[1], "Dockerfile")) as f_dkr: - dkr_content = f_dkr.read() - assert 'VARIABLE_MATRIX_2="value matrix 2"' in dkr_content - assert 'ENV VARIABLE_GLOBAL="value global"' in dkr_content - assert "RUN apt-add-repository" in dkr_content + assert "FROM quay.io/vauxoo/myproject:custom-tag" in dkr_content - example = os.path.join(dirname_example, "example_4.yml") - sys.argv = argv + ["--travis-yml-path", example] - scripts = main() - assert len(scripts) == 2, "Scripts returned should be 2 for %s" % example - check_failed_dockerfile(scripts) - with open(os.path.join(scripts[0], "Dockerfile")) as f_dkr: - dkr_content = f_dkr.read() - assert 'VARIABLE_INCLUDE_1="value include 1"' in dkr_content - with open(os.path.join(scripts[1], "Dockerfile")) as f_dkr: - dkr_content = f_dkr.read() - assert 'VARIABLE_INCLUDE_2="value include 2"' in dkr_content - - # Tests that, when specified, the postgresql key sets - # automatically the environment variable $PSQL_VERSION - example = os.path.join(dirname_example, "example_5.yml") - sys.argv = argv + ["--travis-yml-path", example] - scripts = main() - assert len(scripts) == 2, "Scripts returned should be 2 for %s" % example - check_failed_dockerfile(scripts) - with open(os.path.join(scripts[0], "Dockerfile")) as f_dkr: - dkr_content = f_dkr.read() - assert ' PSQL_VERSION="9.5" ' in dkr_content - with open(os.path.join(scripts[1], "Dockerfile")) as f_dkr: - dkr_content = f_dkr.read() - assert ' PSQL_VERSION="9.5" ' in dkr_content - url = "https://github.com/Vauxoo/travis2docker.git" - sys.argv = ["travis2docker", url, "main"] - scripts = main() - sources_py = "source ${REPO_REQUIREMENTS}/virtualenv/" + "python3.5/bin/activate" - lines_required.pop(0) - lines_required.append( - 'RUN /bin/bash -c "{source_py} && {source_js} && ' - "source /rvm_env.sh && " - '/before_install && /install"'.format(source_py=sources_py, source_js=sources_js), - ) - check_failed_dockerfile(scripts, lines_required + ["ENV TRAVIS_REPO_SLUG=Vauxoo/travis2docker"]) - - sys.argv = ["travis2docker", url, "pull/54"] - scripts = main() - check_failed_dockerfile(scripts, lines_required + ["ENV TRAVIS_REPO_SLUG=Vauxoo/travis2docker"]) - - sys.argv += ["--build-env-args", "BUILD_ENV1", "--build-env-args", "BUILD_ENV2"] - scripts = main() - check_failed_dockerfile( - scripts, - lines_required - + [ - "ARG BUILD_ENV1", - "ENV BUILD_ENV1=$BUILD_ENV1", - "ARG BUILD_ENV2", - "ENV BUILD_ENV2=$BUILD_ENV2", - ], - ) +def test_main_without_variables_sh(tmp_path): + repo = create_repo(tmp_path, {"README.md": "no variables.sh here"}) + sys.argv = [ + "travis2docker", + repo, + "main", + "--root-path", + os.path.join(str(tmp_path), "t2d"), + ] + with pytest.raises(InvalidRepoBranchError): + cli_main(return_result=True) From 1608562f2cfbad17f17298d1acae3040c1256da5 Mon Sep 17 00:00:00 2001 From: "Moises Lopez - https://www.vauxoo.com/" Date: Fri, 7 Aug 2026 12:33:21 -0600 Subject: [PATCH 2/9] [REF] travis2docker: Drop support for Python 3.9 and older Minimum supported version is now Python 3.10: update the GitHub Actions matrix, add python_requires, remove obsolete classifiers, the py2/py<3.7 requirement markers and the universal wheel flag. --- .github/workflows/github-actions.yml | 6 +----- requirements.txt | 3 +-- setup.cfg | 3 --- setup.py | 3 +-- tox.ini | 1 - 5 files changed, 3 insertions(+), 13 deletions(-) diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index e8f59a1..b55f744 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -18,7 +18,7 @@ jobs: strategy: fail-fast: false matrix: - python: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13-dev'] + python: ['3.10', '3.11', '3.12', '3.13'] os: [ubuntu-latest, windows-latest, macos-latest] tox_env: ["py,codecov"] include: @@ -34,10 +34,6 @@ jobs: exclude: # macos-14 AKA macos-latest has switched to being an ARM runner, only supporting newer versions of Python # https://github.com/actions/setup-python/issues/825#issuecomment-2096792396 - - python: '3.8' - os: macos-latest - - python: '3.9' - os: macos-latest - python: '3.10' os: macos-latest diff --git a/requirements.txt b/requirements.txt index b8b84ee..74663e0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1 @@ -Jinja2==2.11.3; python_version < '3.7' -Jinja2==3.1.*; python_version >= '3.7' +Jinja2==3.1.* diff --git a/setup.cfg b/setup.cfg index db1f18a..e45796b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,3 @@ -[bdist_wheel] -universal = 1 - [flake8] max-line-length = 140 exclude = tests/*,*/migrations/*,*/south_migrations/* diff --git a/setup.py b/setup.py index c3b2800..361a6ed 100755 --- a/setup.py +++ b/setup.py @@ -39,8 +39,6 @@ def read(*names, **kwargs): "Operating System :: Microsoft :: Windows", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", @@ -56,6 +54,7 @@ def read(*names, **kwargs): keywords=[ # eg: 'keyword1', 'keyword2', 'keyword3', ], + python_requires=">=3.10", install_requires=read("requirements.txt").split("\n"), extras_require={ # eg: diff --git a/tox.ini b/tox.ini index 8cc5b92..2670403 100644 --- a/tox.ini +++ b/tox.ini @@ -20,7 +20,6 @@ deps = -r{toxinidir}/requirements.txt # test deps pytest pytest-cov - whichcraft nodeenv commands = nodeenv --python-virtualenv From e13a2862ffbeb35cb2e4b52b1a3b8e629897376a Mon Sep 17 00:00:00 2001 From: "Moises Lopez - https://www.vauxoo.com/" Date: Fri, 7 Aug 2026 12:33:32 -0600 Subject: [PATCH 3/9] Revert "[REF] tox.ini: Freeze pre-commit-vauxoo autofixes incompatibles with py3.8" This reverts commit 7891fd6cc55a192979e567310eb638960a7e1787. The pin to compatibility-version=20.20.* is no longer needed now that Python 3.9 and older are not supported anymore. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 2670403..01dd54d 100644 --- a/tox.ini +++ b/tox.ini @@ -69,7 +69,7 @@ commands = python setup.py check --strict --metadata --restructuredtext ; We are using git submodule in the package ; check-manifest {toxinidir} - pre-commit-vauxoo --compatibility-version=20.20.20.20.20.20.20.20.20.20 -t all -p src/ -p tests/ -p setup.py + pre-commit-vauxoo --compatibility-version=900.900.900.900.900.900.900.900.900.900 -t all -p src/ -p tests/ -p setup.py [testenv:build] skip_install = true From 892fcaa74d691a1244c8b0a9de1b8804c6069a72 Mon Sep 17 00:00:00 2001 From: "Moises Lopez - https://www.vauxoo.com/" Date: Fri, 7 Aug 2026 12:33:53 -0600 Subject: [PATCH 4/9] [REF] tox.ini: Rename check environment to lint Align the lint environment name with the rest of the Vauxoo PyPI packages (pylint-odoo, odoo-pre-commit-hooks). --- .github/workflows/github-actions.yml | 2 +- tox.ini | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index b55f744..5c02470 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -24,7 +24,7 @@ jobs: include: - python: '3.12' os: ubuntu-latest - tox_env: 'check' + tox_env: 'lint' - python: '3.12' os: ubuntu-latest tox_env: 'docs' diff --git a/tox.ini b/tox.ini index 01dd54d..a7a34e6 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ envlist = build, clean, - check, + lint, report, docs @@ -55,7 +55,7 @@ commands = sphinx-build {posargs:-E} -b html docs dist/docs sphinx-build -b linkcheck docs dist/docs -[testenv:check] +[testenv:lint] deps = docutils check-manifest From d93a9ee91b4ef430ebeacd69f9cc65f67cfda6d0 Mon Sep 17 00:00:00 2001 From: "Moises Lopez - https://www.vauxoo.com/" Date: Fri, 7 Aug 2026 12:35:00 -0600 Subject: [PATCH 5/9] [REF] travis2docker: Apply pre-commit-vauxoo autofixes Autofixes applied after unpinning --compatibility-version: raw regex strings, contextlib.suppress, partial pathlib usage and formatting. The prints are kept on purpose: variables.sh disables the ruff print check via RUFF_DISABLE_CHECKS. --- setup.py | 13 ++++--- src/travis2docker/cli.py | 18 ++++++---- src/travis2docker/docker_helper/build.py | 9 ++--- src/travis2docker/git_run.py | 22 ++++++------ src/travis2docker/travis2docker.py | 46 ++++++++++++------------ tests/test_travis2docker.py | 30 +++++++++------- variables.sh | 1 + 7 files changed, 76 insertions(+), 63 deletions(-) create mode 100644 variables.sh diff --git a/setup.py b/setup.py index 361a6ed..8a772d5 100755 --- a/setup.py +++ b/setup.py @@ -1,14 +1,17 @@ #!/usr/bin/env python +import pathlib import re from glob import glob -from os.path import basename, dirname, join, splitext +from os.path import join, splitext from setuptools import find_packages, setup def read(*names, **kwargs): - return open(join(dirname(__file__), *names), encoding=kwargs.get("encoding", "utf8")).read() + return ( + pathlib.Path(join(pathlib.Path(__file__).parent, *names)).open(encoding=kwargs.get("encoding", "utf8")).read() + ) setup( @@ -18,8 +21,8 @@ def read(*names, **kwargs): description="Script to generate a development Dockerfile from the deployv image of a repository", long_description="%s\n%s" % ( - re.compile("^.. start-badges.*^.. end-badges", re.M | re.S).sub("", read("README.rst")), - re.sub(":[a-z]+:`~?(.*?)`", r"``\1``", read("CHANGELOG.rst")), + re.compile(r"^.. start-badges.*^.. end-badges", re.MULTILINE | re.DOTALL).sub("", read("README.rst")), + re.sub(r":[a-z]+:`~?(.*?)`", r"``\1``", read("CHANGELOG.rst")), ), long_description_content_type="text/x-rst", author="Vauxoo", @@ -27,7 +30,7 @@ def read(*names, **kwargs): url="https://github.com/vauxoo/travis2docker", packages=find_packages("src"), package_dir={"": "src"}, - py_modules=[splitext(basename(path))[0] for path in glob("src/*.py")], + py_modules=[splitext(pathlib.Path(path).name)[0] for path in glob("src/*.py")], include_package_data=True, zip_safe=False, classifiers=[ diff --git a/src/travis2docker/cli.py b/src/travis2docker/cli.py index 1bd0cc3..0220de0 100644 --- a/src/travis2docker/cli.py +++ b/src/travis2docker/cli.py @@ -15,7 +15,8 @@ import argparse import os -from os.path import expanduser, join +import pathlib +from os.path import join from sys import stdout from . import __version__ @@ -62,7 +63,7 @@ def main(return_result=False): parser.add_argument( "--docker-user", dest="docker_user", - help="User of work into Dockerfile." "\nBased on your docker image." "\nDefault: odoo", + help="User of work into Dockerfile.\nBased on your docker image.\nDefault: odoo", ) parser.add_argument( "--docker-image", @@ -73,7 +74,7 @@ def main(return_result=False): ) default_root_path = os.environ.get("TRAVIS2DOCKER_ROOT_PATH") if not default_root_path: - default_root_path = os.path.expanduser("~") + default_root_path = pathlib.Path("~").expanduser() default_root_path = join(default_root_path, ".t2d") parser.add_argument( "--root-path", @@ -84,7 +85,7 @@ def main(return_result=False): parser.add_argument( "--add-remote", dest="remotes", - help="Add git remote to git of build path, separated by a comma." "\nUse remote name. E.g. 'Vauxoo,moylop260'", + help="Add git remote to git of build path, separated by a comma.\nUse remote name. E.g. 'Vauxoo,moylop260'", ) parser.add_argument( "--run-extra-args", @@ -112,7 +113,7 @@ def main(return_result=False): dest="build_extra_cmds", nargs="*", default="", - help='Extra commands to run after "build" script. ' "Note: You can use \\$IMAGE escaped environment variable.", + help='Extra commands to run after "build" script. Note: You can use \\$IMAGE escaped environment variable.', ) parser.add_argument( "--add-rcfile", @@ -163,7 +164,10 @@ def main(return_result=False): run_extra_cmds = "\n".join(args.run_extra_cmds) rcfiles_args = args.add_rcfile and args.add_rcfile.split(",") build_env_args = [build_env_args[0] for build_env_args in args.build_env_args] - rcfiles = [(expanduser(rc_file), os.path.join("$HOME", os.path.basename(rc_file))) for rc_file in rcfiles_args] + rcfiles = [ + (pathlib.Path(rc_file).expanduser(), os.path.join("$HOME", pathlib.Path(rc_file).name)) + for rc_file in rcfiles_args + ] os_kwargs = get_git_data(git_repo, join(root_path, "repo"), revision) if not os_kwargs.get("variables_sh"): @@ -182,7 +186,7 @@ def main(return_result=False): work_path=join(root_path, "script", GitRun.url2dirname(git_repo), revision), image=default_docker_image, os_kwargs=os_kwargs, - copy_paths=[(expanduser("~/.ssh"), "$HOME/.ssh")] + rcfiles, + copy_paths=[(pathlib.Path("~/.ssh").expanduser(), "$HOME/.ssh")] + rcfiles, build_env_args=build_env_args, build_extra_steps=args.build_extra_steps, ) diff --git a/src/travis2docker/docker_helper/build.py b/src/travis2docker/docker_helper/build.py index a843130..665bd38 100644 --- a/src/travis2docker/docker_helper/build.py +++ b/src/travis2docker/docker_helper/build.py @@ -3,6 +3,7 @@ import glob import os +import pathlib import re import subprocess import sys @@ -11,7 +12,7 @@ def ssh_keyscan2known_hosts(url, known_hosts_path=None): # python3 -c "import build;build.ssh_keyscan2known_hosts('url')" if known_hosts_path is None: - known_hosts_path = os.path.join(os.path.expanduser("~"), ".ssh", "known_hosts") + known_hosts_path = os.path.join(pathlib.Path("~").expanduser(), ".ssh", "known_hosts") # Clear current known hosts cmd = ["ssh-keygen", "-R", url] @@ -22,7 +23,7 @@ def ssh_keyscan2known_hosts(url, known_hosts_path=None): cmd = ["ssh-keyscan", "-p", "22", url] print(" ".join(cmd)) keys_scanned = subprocess.check_output(cmd).decode(sys.stdout.encoding).strip() - with open(known_hosts_path, "r+") as known_hosts_f: + with pathlib.Path(known_hosts_path).open("r+") as known_hosts_f: known_hosts_f.write("\n" + keys_scanned) @@ -31,14 +32,14 @@ def git_set_remote(path=None): if path is None: # /home/odoo/instance/odoo spends a lot of time path = "/home/odoo/instance/extra_addons" - git_re = re.compile("([^/|@]+)/([^/]+)/([^/.]+(.git)?)") + git_re = re.compile(r"([^/|@]+)/([^/]+)/([^/.]+(.git)?)") # TODO: Support ssh url path = os.path.join(path, "*", "**", ".git") hosts_scanned = set() for git_dir in glob.glob(path, recursive=True): git_cmd = [ "git", - "--work-tree=%s" % os.path.dirname(git_dir), + "--work-tree=%s" % pathlib.Path(git_dir).parent, "--git-dir=%s" % git_dir, ] cmd = git_cmd + ["remote", "get-url", "--push", "origin"] diff --git a/src/travis2docker/git_run.py b/src/travis2docker/git_run.py index 8d4aa83..3c94929 100644 --- a/src/travis2docker/git_run.py +++ b/src/travis2docker/git_run.py @@ -1,6 +1,8 @@ # pylint: disable=useless-object-inheritance,print-used,except-pass +import contextlib import os +import pathlib import re import subprocess @@ -25,17 +27,17 @@ def get_data_url(repo_git, no_user=True): host, owner, repo = False, False, False repo_git_sub = repo_git.replace(":", "/") if no_user: - repo_git_sub = re.sub(".+@", "", repo_git_sub) - repo_git_sub = re.sub(".git$", "", repo_git_sub) + repo_git_sub = re.sub(r".+@", "", repo_git_sub) + repo_git_sub = re.sub(r".git$", "", repo_git_sub) match_object = re.search(r"(?P[^/]+)/(?P[^/]+)/(?P[^/]+)", repo_git_sub) if match_object: host = match_object.group("host") owner = match_object.group("owner") repo = match_object.group("repo") - elif os.path.isdir(repo_git): + elif pathlib.Path(repo_git).is_dir(): host = "local" - owner = os.path.basename(repo_git) - repo = os.path.basename(os.path.dirname(repo_git)) + owner = pathlib.Path(repo_git).name + repo = pathlib.Path(pathlib.Path(repo_git).parent).name return host, owner, repo @staticmethod @@ -61,10 +63,8 @@ def run(self, cmd): print("cmd list", cmd) print("cmd", " ".join(cmd)) res = None - try: + with contextlib.suppress(BaseException): res = subprocess.check_output(cmd) - except BaseException: - pass if res: try: res = res.decode() @@ -94,9 +94,9 @@ def get_ref_data(self, refs=None, fields=None): def update(self): """Get a repository git or update it""" - if not os.path.isdir(os.path.join(self.path)): - os.makedirs(self.path) - if not os.path.isdir(os.path.join(self.path, "refs")): + if not pathlib.Path(os.path.join(self.path)).is_dir(): + pathlib.Path(self.path).mkdir(parents=True) + if not pathlib.Path(os.path.join(self.path, "refs")).is_dir(): subprocess.check_output(["git", "clone", "--bare", self.repo_git, self.path]) self.run(["gc", "--auto", "--prune=all"]) self.run(["fetch", "-p", "origin", "+refs/heads/*:refs/heads/*"]) diff --git a/src/travis2docker/travis2docker.py b/src/travis2docker/travis2docker.py index 268225f..88bdb11 100644 --- a/src/travis2docker/travis2docker.py +++ b/src/travis2docker/travis2docker.py @@ -1,5 +1,6 @@ # pylint: disable=useless-object-inheritance,consider-using-with,print-used import os +import pathlib import re import shutil import stat @@ -12,7 +13,7 @@ class Travis2Docker: - re_export = re.compile(RE_EXPORT_STR, re.M) + re_export = re.compile(RE_EXPORT_STR, re.MULTILINE) @property def dockerfile_template(self): @@ -37,11 +38,11 @@ def run_template(self): @staticmethod def chmod_execution(file_path): - os.chmod(file_path, os.stat(file_path).st_mode | stat.S_IEXEC) + pathlib.Path(file_path).chmod(os.stat(file_path).st_mode | stat.S_IEXEC) @staticmethod def mkdir_p(path): - os.makedirs(path, exist_ok=True) + pathlib.Path(path).mkdir(exist_ok=True, parents=True) def __init__( self, @@ -69,10 +70,10 @@ def __init__( self.variables_sh_data.update({"sha_short": os_kwargs["sha"][:7]}) if not image: image = "%(docker_image_repo)s:%(main_app)s-%(version)s-%(sha_short)s" % self.variables_sh_data - templates_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "templates") + templates_dir = os.path.join(pathlib.Path(os.path.realpath(__file__)).parent, "templates") build_sh = os.path.join(templates_dir, "build.sh") entrypoint_sh = os.path.join(templates_dir, "entrypoint_deployv.sh") - docker_helper = os.path.join(os.path.dirname(os.path.realpath(__file__)), "docker_helper") + docker_helper = os.path.join(pathlib.Path(os.path.realpath(__file__)).parent, "docker_helper") vscode_conf = os.path.join(templates_dir, ".vscode") coveragerc = os.path.join(templates_dir, ".coveragerc") copy_paths.append([build_sh, "/home/odoo/build.sh"]) @@ -90,16 +91,16 @@ def __init__( self.jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(templates_path)) self.image = image if work_path is None: - base_name = os.path.splitext(os.path.basename(__file__))[0] + base_name = os.path.splitext(pathlib.Path(__file__).name)[0] self.work_path = os.path.join(gettempdir(), base_name) else: - self.work_path = os.path.expandvars(os.path.expanduser(work_path)) + self.work_path = os.path.expandvars(pathlib.Path(work_path).expanduser()) self.dockerfile = dockerfile def compute_build_scripts(self): build_path = os.path.join(self.curr_work_path, "10-build.sh") run_path = os.path.join(self.curr_work_path, "20-run.sh") - with open(build_path, "w") as f_build, open(run_path, "w") as f_run: + with pathlib.Path(build_path).open("w") as f_build, pathlib.Path(run_path).open("w") as f_run: build_content = self.build_template.render( image=self.new_image, dirname_dockerfile=self.curr_work_path, **self.build_extra_params ).strip("\n ") @@ -124,7 +125,7 @@ def compute_dockerfile(self): "build_extra_steps": self.build_extra_steps, } kwargs.update(self.os_kwargs) - with open(curr_dockerfile, "w") as f_dockerfile: + with pathlib.Path(curr_dockerfile).open("w") as f_dockerfile: dockerfile_content = self.dockerfile_template.render(kwargs).strip("\n ") f_dockerfile.write(dockerfile_content) self.compute_build_scripts() @@ -134,31 +135,31 @@ def compute_dockerfile(self): def copy_path(self, path): """:param paths list: List of paths to copy""" - src = os.path.expandvars(os.path.expanduser(path)) - basename = os.path.basename(src) - dest_path = os.path.expandvars(os.path.expanduser(os.path.join(self.curr_work_path, basename))) - if os.path.isdir(dest_path): + src = os.path.expandvars(pathlib.Path(path).expanduser()) + basename = pathlib.Path(src).name + dest_path = os.path.expandvars(pathlib.Path(os.path.join(self.curr_work_path, basename)).expanduser()) + if pathlib.Path(dest_path).is_dir(): shutil.rmtree(dest_path) - if os.path.isdir(src): + if pathlib.Path(src).is_dir(): try: shutil.copytree(src, dest_path) except shutil.Error: # pylint: disable=except-pass pass # There are permissions errors to copy - elif os.path.isfile(src): + elif pathlib.Path(src).is_file(): shutil.copy(src, dest_path) else: raise UserWarning("Just directory or file is supported to copy [%s]" % src) return os.path.relpath(dest_path, self.curr_work_path) def set_authorized_key(self): - ssh_dir = os.path.expanduser("~/.ssh") + ssh_dir = pathlib.Path("~/.ssh").expanduser() ed_key = os.path.join(ssh_dir, "id_ed25519.pub") rsa_key = os.path.join(ssh_dir, "id_rsa.pub") to_copy = False - if os.path.isfile(ed_key): + if pathlib.Path(ed_key).is_file(): to_copy = ed_key - elif os.path.isfile(rsa_key): + elif pathlib.Path(rsa_key).is_file(): print("RSA keys are deprecated, consider changing to ed25519") to_copy = rsa_key @@ -166,12 +167,9 @@ def set_authorized_key(self): print("No public key found. No key added to ~/.ssh/authorized_keys. SSH login won't work.") return - with open(to_copy, encoding="utf-8") as key_fd: - pub_key = key_fd.read() + pub_key = pathlib.Path(to_copy).read_text(encoding="utf-8") - with open( - os.path.join(self.curr_work_path, ".ssh", "authorized_keys"), - "a", - encoding="utf-8", + with pathlib.Path(os.path.join(self.curr_work_path, ".ssh", "authorized_keys")).open( + "a", encoding="utf-8" ) as auth_fd: auth_fd.write(pub_key) diff --git a/tests/test_travis2docker.py b/tests/test_travis2docker.py index 478588f..00c8b43 100644 --- a/tests/test_travis2docker.py +++ b/tests/test_travis2docker.py @@ -2,6 +2,7 @@ # pylint: disable=print-used,consider-using-with import os +import pathlib import subprocess import sys from shutil import which @@ -34,20 +35,27 @@ def check_dockerfile_lint(scripts): ) output = pipe.stdout.read().decode("utf-8") assert "Check passed" in output, fname_dkr - print("Check dockerfile output", output) def create_repo(base_path, files): repo_path = os.path.join(str(base_path), "myrepo") - os.makedirs(repo_path) + pathlib.Path(repo_path).mkdir(parents=True) subprocess.check_call(["git", "init", "-b", "main", repo_path]) for fname, content in files.items(): - with open(os.path.join(repo_path, fname), "w") as f_repo: - f_repo.write(content) + pathlib.Path(os.path.join(repo_path, fname)).write_text(content) subprocess.check_call(["git", "-C", repo_path, "add", "-A"]) - subprocess.check_call( - ["git", "-C", repo_path, "-c", "user.email=test@test.com", "-c", "user.name=test", "commit", "-m", "initial"] - ) + subprocess.check_call([ + "git", + "-C", + repo_path, + "-c", + "user.email=test@test.com", + "-c", + "user.name=test", + "commit", + "-m", + "initial", + ]) return repo_path @@ -69,8 +77,7 @@ def test_main_deployv(tmp_path): scripts = cli_main(return_result=True) assert len(scripts) == 1, "Scripts returned should be 1" fname_dkr = os.path.join(scripts[0], "Dockerfile") - with open(fname_dkr) as f_dkr: - dkr_content = f_dkr.read() + dkr_content = pathlib.Path(fname_dkr).read_text() sha_short = subprocess.check_output(["git", "-C", repo, "rev-parse", "HEAD"]).decode("UTF-8")[:7] assert "FROM quay.io/vauxoo/myproject:myproject-16.0-%s" % sha_short in dkr_content assert "ENV BUILD_ENV1=TRUE" in dkr_content @@ -82,7 +89,7 @@ def test_main_deployv(tmp_path): assert "COPY docker_helper /home/odoo/build" in dkr_content for script in ("10-build.sh", "20-run.sh"): script_path = os.path.join(scripts[0], script) - assert os.path.isfile(script_path) + assert pathlib.Path(script_path).is_file() assert os.access(script_path, os.X_OK), "%s should be executable" % script check_dockerfile_lint(scripts) @@ -100,8 +107,7 @@ def test_main_docker_image_parameter(tmp_path): ] scripts = cli_main(return_result=True) assert len(scripts) == 1, "Scripts returned should be 1" - with open(os.path.join(scripts[0], "Dockerfile")) as f_dkr: - dkr_content = f_dkr.read() + dkr_content = pathlib.Path(os.path.join(scripts[0], "Dockerfile")).read_text() assert "FROM quay.io/vauxoo/myproject:custom-tag" in dkr_content diff --git a/variables.sh b/variables.sh new file mode 100644 index 0000000..36b8d8d --- /dev/null +++ b/variables.sh @@ -0,0 +1 @@ +export RUFF_DISABLE_CHECKS="print" From 4d2a47c64fd160eb97dc8b46e8b1a84c2e05db66 Mon Sep 17 00:00:00 2001 From: "Moises Lopez - https://www.vauxoo.com/" Date: Fri, 7 Aug 2026 12:39:29 -0600 Subject: [PATCH 6/9] [REF] travis2docker: Migrate os.path usage to pathlib Complete the pathlib migration started by the autofixes: Path objects end to end in git_run, cli, travis2docker, docker_helper, tests and setup.py. os.path.expandvars is dropped too: the shell already expands environment variables before they reach argv. --- setup.py | 7 +-- src/travis2docker/cli.py | 10 ++-- src/travis2docker/docker_helper/build.py | 5 +- src/travis2docker/git_run.py | 12 ++-- src/travis2docker/travis2docker.py | 73 ++++++++++-------------- tests/test_travis2docker.py | 27 +++++---- 6 files changed, 57 insertions(+), 77 deletions(-) diff --git a/setup.py b/setup.py index 8a772d5..7eff669 100755 --- a/setup.py +++ b/setup.py @@ -3,15 +3,12 @@ import pathlib import re from glob import glob -from os.path import join, splitext from setuptools import find_packages, setup def read(*names, **kwargs): - return ( - pathlib.Path(join(pathlib.Path(__file__).parent, *names)).open(encoding=kwargs.get("encoding", "utf8")).read() - ) + return pathlib.Path(__file__).parent.joinpath(*names).read_text(encoding=kwargs.get("encoding", "utf8")) setup( @@ -30,7 +27,7 @@ def read(*names, **kwargs): url="https://github.com/vauxoo/travis2docker", packages=find_packages("src"), package_dir={"": "src"}, - py_modules=[splitext(pathlib.Path(path).name)[0] for path in glob("src/*.py")], + py_modules=[pathlib.Path(path).stem for path in glob("src/*.py")], include_package_data=True, zip_safe=False, classifiers=[ diff --git a/src/travis2docker/cli.py b/src/travis2docker/cli.py index 0220de0..4f646a0 100644 --- a/src/travis2docker/cli.py +++ b/src/travis2docker/cli.py @@ -16,7 +16,6 @@ import argparse import os import pathlib -from os.path import join from sys import stdout from . import __version__ @@ -75,7 +74,7 @@ def main(return_result=False): default_root_path = os.environ.get("TRAVIS2DOCKER_ROOT_PATH") if not default_root_path: default_root_path = pathlib.Path("~").expanduser() - default_root_path = join(default_root_path, ".t2d") + default_root_path = str(pathlib.Path(default_root_path) / ".t2d") parser.add_argument( "--root-path", dest="root_path", @@ -165,10 +164,9 @@ def main(return_result=False): rcfiles_args = args.add_rcfile and args.add_rcfile.split(",") build_env_args = [build_env_args[0] for build_env_args in args.build_env_args] rcfiles = [ - (pathlib.Path(rc_file).expanduser(), os.path.join("$HOME", pathlib.Path(rc_file).name)) - for rc_file in rcfiles_args + (pathlib.Path(rc_file).expanduser(), "$HOME/%s" % pathlib.Path(rc_file).name) for rc_file in rcfiles_args ] - os_kwargs = get_git_data(git_repo, join(root_path, "repo"), revision) + os_kwargs = get_git_data(git_repo, pathlib.Path(root_path) / "repo", revision) if not os_kwargs.get("variables_sh"): msg = ( @@ -183,7 +181,7 @@ def main(return_result=False): if docker_user: os_kwargs.update({"user": docker_user}) t2d = Travis2Docker( - work_path=join(root_path, "script", GitRun.url2dirname(git_repo), revision), + work_path=pathlib.Path(root_path) / "script" / GitRun.url2dirname(git_repo) / revision, image=default_docker_image, os_kwargs=os_kwargs, copy_paths=[(pathlib.Path("~/.ssh").expanduser(), "$HOME/.ssh")] + rcfiles, diff --git a/src/travis2docker/docker_helper/build.py b/src/travis2docker/docker_helper/build.py index 665bd38..e68c993 100644 --- a/src/travis2docker/docker_helper/build.py +++ b/src/travis2docker/docker_helper/build.py @@ -2,7 +2,6 @@ # pylint: disable=print-used import glob -import os import pathlib import re import subprocess @@ -12,7 +11,7 @@ def ssh_keyscan2known_hosts(url, known_hosts_path=None): # python3 -c "import build;build.ssh_keyscan2known_hosts('url')" if known_hosts_path is None: - known_hosts_path = os.path.join(pathlib.Path("~").expanduser(), ".ssh", "known_hosts") + known_hosts_path = pathlib.Path("~/.ssh/known_hosts").expanduser() # Clear current known hosts cmd = ["ssh-keygen", "-R", url] @@ -34,7 +33,7 @@ def git_set_remote(path=None): path = "/home/odoo/instance/extra_addons" git_re = re.compile(r"([^/|@]+)/([^/]+)/([^/.]+(.git)?)") # TODO: Support ssh url - path = os.path.join(path, "*", "**", ".git") + path = str(pathlib.Path(path) / "*" / "**" / ".git") hosts_scanned = set() for git_dir in glob.glob(path, recursive=True): git_cmd = [ diff --git a/src/travis2docker/git_run.py b/src/travis2docker/git_run.py index 3c94929..2b3da93 100644 --- a/src/travis2docker/git_run.py +++ b/src/travis2docker/git_run.py @@ -1,7 +1,6 @@ # pylint: disable=useless-object-inheritance,print-used,except-pass import contextlib -import os import pathlib import re import subprocess @@ -17,8 +16,9 @@ def decode_utf(field): class GitRun: def __init__(self, repo_git, path, path_prefix_repo=False): self.repo_git = repo_git + path = pathlib.Path(path) if path_prefix_repo: - path = os.path.join(path, self.url2dirname(repo_git)) + path /= self.url2dirname(repo_git) self.path = path self.host, self.owner, self.repo = self.get_data_url(repo_git) @@ -37,7 +37,7 @@ def get_data_url(repo_git, no_user=True): elif pathlib.Path(repo_git).is_dir(): host = "local" owner = pathlib.Path(repo_git).name - repo = pathlib.Path(pathlib.Path(repo_git).parent).name + repo = pathlib.Path(repo_git).parent.name return host, owner, repo @staticmethod @@ -94,9 +94,9 @@ def get_ref_data(self, refs=None, fields=None): def update(self): """Get a repository git or update it""" - if not pathlib.Path(os.path.join(self.path)).is_dir(): - pathlib.Path(self.path).mkdir(parents=True) - if not pathlib.Path(os.path.join(self.path, "refs")).is_dir(): + if not self.path.is_dir(): + self.path.mkdir(parents=True) + if not (self.path / "refs").is_dir(): subprocess.check_output(["git", "clone", "--bare", self.repo_git, self.path]) self.run(["gc", "--auto", "--prune=all"]) self.run(["fetch", "-p", "origin", "+refs/heads/*:refs/heads/*"]) diff --git a/src/travis2docker/travis2docker.py b/src/travis2docker/travis2docker.py index 88bdb11..14b20f8 100644 --- a/src/travis2docker/travis2docker.py +++ b/src/travis2docker/travis2docker.py @@ -1,5 +1,4 @@ # pylint: disable=useless-object-inheritance,consider-using-with,print-used -import os import pathlib import re import shutil @@ -38,11 +37,7 @@ def run_template(self): @staticmethod def chmod_execution(file_path): - pathlib.Path(file_path).chmod(os.stat(file_path).st_mode | stat.S_IEXEC) - - @staticmethod - def mkdir_p(path): - pathlib.Path(path).mkdir(exist_ok=True, parents=True) + file_path.chmod(file_path.stat().st_mode | stat.S_IEXEC) def __init__( self, @@ -70,17 +65,13 @@ def __init__( self.variables_sh_data.update({"sha_short": os_kwargs["sha"][:7]}) if not image: image = "%(docker_image_repo)s:%(main_app)s-%(version)s-%(sha_short)s" % self.variables_sh_data - templates_dir = os.path.join(pathlib.Path(os.path.realpath(__file__)).parent, "templates") - build_sh = os.path.join(templates_dir, "build.sh") - entrypoint_sh = os.path.join(templates_dir, "entrypoint_deployv.sh") - docker_helper = os.path.join(pathlib.Path(os.path.realpath(__file__)).parent, "docker_helper") - vscode_conf = os.path.join(templates_dir, ".vscode") - coveragerc = os.path.join(templates_dir, ".coveragerc") - copy_paths.append([build_sh, "/home/odoo/build.sh"]) - copy_paths.append([entrypoint_sh, "/entrypoint.sh"]) - copy_paths.append([docker_helper, "/home/odoo/build"]) - copy_paths.append([vscode_conf, "/home/odoo/.vscode"]) - copy_paths.append([coveragerc, "/home/odoo/.coveragerc"]) + module_dir = pathlib.Path(__file__).resolve().parent + templates_dir = module_dir / "templates" + copy_paths.append([templates_dir / "build.sh", "/home/odoo/build.sh"]) + copy_paths.append([templates_dir / "entrypoint_deployv.sh", "/entrypoint.sh"]) + copy_paths.append([module_dir / "docker_helper", "/home/odoo/build"]) + copy_paths.append([templates_dir / ".vscode", "/home/odoo/.vscode"]) + copy_paths.append([templates_dir / ".coveragerc", "/home/odoo/.coveragerc"]) os_kwargs.setdefault("user", "odoo") if dockerfile is None: dockerfile = "Dockerfile" @@ -91,16 +82,15 @@ def __init__( self.jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(templates_path)) self.image = image if work_path is None: - base_name = os.path.splitext(pathlib.Path(__file__).name)[0] - self.work_path = os.path.join(gettempdir(), base_name) + self.work_path = pathlib.Path(gettempdir()) / pathlib.Path(__file__).stem else: - self.work_path = os.path.expandvars(pathlib.Path(work_path).expanduser()) + self.work_path = pathlib.Path(work_path).expanduser() self.dockerfile = dockerfile def compute_build_scripts(self): - build_path = os.path.join(self.curr_work_path, "10-build.sh") - run_path = os.path.join(self.curr_work_path, "20-run.sh") - with pathlib.Path(build_path).open("w") as f_build, pathlib.Path(run_path).open("w") as f_run: + build_path = self.curr_work_path / "10-build.sh" + run_path = self.curr_work_path / "20-run.sh" + with build_path.open("w") as f_build, run_path.open("w") as f_run: build_content = self.build_template.render( image=self.new_image, dirname_dockerfile=self.curr_work_path, **self.build_extra_params ).strip("\n ") @@ -112,8 +102,8 @@ def compute_build_scripts(self): def compute_dockerfile(self): self.curr_work_path = self.work_path - curr_dockerfile = os.path.join(self.curr_work_path, self.dockerfile) - self.mkdir_p(self.curr_work_path) + curr_dockerfile = self.curr_work_path / self.dockerfile + self.curr_work_path.mkdir(exist_ok=True, parents=True) copies = [] for copy_path, dest in self.copy_paths: copies.append((self.copy_path(copy_path), dest)) @@ -125,41 +115,40 @@ def compute_dockerfile(self): "build_extra_steps": self.build_extra_steps, } kwargs.update(self.os_kwargs) - with pathlib.Path(curr_dockerfile).open("w") as f_dockerfile: + with curr_dockerfile.open("w") as f_dockerfile: dockerfile_content = self.dockerfile_template.render(kwargs).strip("\n ") f_dockerfile.write(dockerfile_content) self.compute_build_scripts() - work_paths = [self.curr_work_path] + work_paths = [str(self.curr_work_path)] self.curr_work_path = None return work_paths def copy_path(self, path): """:param paths list: List of paths to copy""" - src = os.path.expandvars(pathlib.Path(path).expanduser()) - basename = pathlib.Path(src).name - dest_path = os.path.expandvars(pathlib.Path(os.path.join(self.curr_work_path, basename)).expanduser()) - if pathlib.Path(dest_path).is_dir(): + src = pathlib.Path(path).expanduser() + dest_path = self.curr_work_path / src.name + if dest_path.is_dir(): shutil.rmtree(dest_path) - if pathlib.Path(src).is_dir(): + if src.is_dir(): try: shutil.copytree(src, dest_path) except shutil.Error: # pylint: disable=except-pass pass # There are permissions errors to copy - elif pathlib.Path(src).is_file(): + elif src.is_file(): shutil.copy(src, dest_path) else: raise UserWarning("Just directory or file is supported to copy [%s]" % src) - return os.path.relpath(dest_path, self.curr_work_path) + return str(dest_path.relative_to(self.curr_work_path)) def set_authorized_key(self): ssh_dir = pathlib.Path("~/.ssh").expanduser() - ed_key = os.path.join(ssh_dir, "id_ed25519.pub") - rsa_key = os.path.join(ssh_dir, "id_rsa.pub") + ed_key = ssh_dir / "id_ed25519.pub" + rsa_key = ssh_dir / "id_rsa.pub" - to_copy = False - if pathlib.Path(ed_key).is_file(): + to_copy = None + if ed_key.is_file(): to_copy = ed_key - elif pathlib.Path(rsa_key).is_file(): + elif rsa_key.is_file(): print("RSA keys are deprecated, consider changing to ed25519") to_copy = rsa_key @@ -167,9 +156,7 @@ def set_authorized_key(self): print("No public key found. No key added to ~/.ssh/authorized_keys. SSH login won't work.") return - pub_key = pathlib.Path(to_copy).read_text(encoding="utf-8") + pub_key = to_copy.read_text(encoding="utf-8") - with pathlib.Path(os.path.join(self.curr_work_path, ".ssh", "authorized_keys")).open( - "a", encoding="utf-8" - ) as auth_fd: + with (self.curr_work_path / ".ssh" / "authorized_keys").open("a", encoding="utf-8") as auth_fd: auth_fd.write(pub_key) diff --git a/tests/test_travis2docker.py b/tests/test_travis2docker.py index 00c8b43..069f36d 100644 --- a/tests/test_travis2docker.py +++ b/tests/test_travis2docker.py @@ -27,7 +27,7 @@ def check_dockerfile_lint(scripts): lint_bin = which(lint_bin_name) or which(lint_bin_name, path=npm_bin_path + os.pathsep + npm_bin_path_g) assert lint_bin, "'%s' not found." % lint_bin_name for script in scripts: - fname_dkr = os.path.join(script, "Dockerfile") + fname_dkr = str(pathlib.Path(script) / "Dockerfile") pipe = subprocess.Popen( [lint_bin, "-f", fname_dkr], stderr=subprocess.STDOUT, @@ -38,11 +38,11 @@ def check_dockerfile_lint(scripts): def create_repo(base_path, files): - repo_path = os.path.join(str(base_path), "myrepo") - pathlib.Path(repo_path).mkdir(parents=True) - subprocess.check_call(["git", "init", "-b", "main", repo_path]) + repo_path = pathlib.Path(base_path) / "myrepo" + repo_path.mkdir(parents=True) + subprocess.check_call(["git", "init", "-b", "main", str(repo_path)]) for fname, content in files.items(): - pathlib.Path(os.path.join(repo_path, fname)).write_text(content) + (repo_path / fname).write_text(content) subprocess.check_call(["git", "-C", repo_path, "add", "-A"]) subprocess.check_call([ "git", @@ -56,7 +56,7 @@ def create_repo(base_path, files): "-m", "initial", ]) - return repo_path + return str(repo_path) def test_main_deployv(tmp_path): @@ -66,7 +66,7 @@ def test_main_deployv(tmp_path): repo, "main", "--root-path", - os.path.join(str(tmp_path), "t2d"), + str(tmp_path / "t2d"), "--build-env-args", "BUILD_ENV1", "--build-env-args", @@ -76,8 +76,7 @@ def test_main_deployv(tmp_path): ] scripts = cli_main(return_result=True) assert len(scripts) == 1, "Scripts returned should be 1" - fname_dkr = os.path.join(scripts[0], "Dockerfile") - dkr_content = pathlib.Path(fname_dkr).read_text() + dkr_content = (pathlib.Path(scripts[0]) / "Dockerfile").read_text() sha_short = subprocess.check_output(["git", "-C", repo, "rev-parse", "HEAD"]).decode("UTF-8")[:7] assert "FROM quay.io/vauxoo/myproject:myproject-16.0-%s" % sha_short in dkr_content assert "ENV BUILD_ENV1=TRUE" in dkr_content @@ -88,8 +87,8 @@ def test_main_deployv(tmp_path): assert "COPY entrypoint_deployv.sh /entrypoint.sh" in dkr_content assert "COPY docker_helper /home/odoo/build" in dkr_content for script in ("10-build.sh", "20-run.sh"): - script_path = os.path.join(scripts[0], script) - assert pathlib.Path(script_path).is_file() + script_path = pathlib.Path(scripts[0]) / script + assert script_path.is_file() assert os.access(script_path, os.X_OK), "%s should be executable" % script check_dockerfile_lint(scripts) @@ -101,13 +100,13 @@ def test_main_docker_image_parameter(tmp_path): repo, "main", "--root-path", - os.path.join(str(tmp_path), "t2d"), + str(tmp_path / "t2d"), "--docker-image", "quay.io/vauxoo/myproject:custom-tag", ] scripts = cli_main(return_result=True) assert len(scripts) == 1, "Scripts returned should be 1" - dkr_content = pathlib.Path(os.path.join(scripts[0], "Dockerfile")).read_text() + dkr_content = (pathlib.Path(scripts[0]) / "Dockerfile").read_text() assert "FROM quay.io/vauxoo/myproject:custom-tag" in dkr_content @@ -118,7 +117,7 @@ def test_main_without_variables_sh(tmp_path): repo, "main", "--root-path", - os.path.join(str(tmp_path), "t2d"), + str(tmp_path / "t2d"), ] with pytest.raises(InvalidRepoBranchError): cli_main(return_result=True) From 60be23c330a1bd596b96f103e522d3d00af524ce Mon Sep 17 00:00:00 2001 From: "Moises Lopez - https://www.vauxoo.com/" Date: Fri, 7 Aug 2026 12:55:36 -0600 Subject: [PATCH 7/9] [FIX] git_run: Flatten Windows paths in url2dirname to unbreak CI Backslashes in a local Windows repo path were not replaced, so the bare clone destination expanded into a deeply nested tree exceeding the 260 chars MAX_PATH limit (fatal: Filename too long). Flatten them like the rest of the separators and enable core.longpaths in the Windows runner. The tests also clone the fixture repo from a relative path: the absolute path was duplicated inside the flattened clone dirname, blowing past the Windows git dir length limit (fatal: '$GIT_DIR' too big). --- .github/workflows/github-actions.yml | 3 +++ src/travis2docker/git_run.py | 2 +- tests/test_travis2docker.py | 13 +++++++++---- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index 5c02470..c8fe2e7 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -60,6 +60,9 @@ jobs: pip --version tox --version pip list --format=freeze + - name: Enable git long paths (Windows) + if: runner.os == 'Windows' + run: git config --global core.longpaths true - name: test run: | mkdir -p ~/.ssh diff --git a/src/travis2docker/git_run.py b/src/travis2docker/git_run.py index 2b3da93..858565a 100644 --- a/src/travis2docker/git_run.py +++ b/src/travis2docker/git_run.py @@ -42,7 +42,7 @@ def get_data_url(repo_git, no_user=True): @staticmethod def url2dirname(url): - for invalid_char in "@:/#": + for invalid_char in "@:/#\\": url = url.replace(invalid_char, "_") return url diff --git a/tests/test_travis2docker.py b/tests/test_travis2docker.py index 069f36d..b89bcf9 100644 --- a/tests/test_travis2docker.py +++ b/tests/test_travis2docker.py @@ -56,10 +56,13 @@ def create_repo(base_path, files): "-m", "initial", ]) - return str(repo_path) + # Return the path relative to base_path: an absolute path would be flattened + # into the clone dirname, exceeding the Windows MAX_PATH limit + return repo_path.name -def test_main_deployv(tmp_path): +def test_main_deployv(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) repo = create_repo(tmp_path, {"variables.sh": VARIABLES_SH}) sys.argv = [ "travis2docker", @@ -93,7 +96,8 @@ def test_main_deployv(tmp_path): check_dockerfile_lint(scripts) -def test_main_docker_image_parameter(tmp_path): +def test_main_docker_image_parameter(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) repo = create_repo(tmp_path, {"variables.sh": VARIABLES_SH}) sys.argv = [ "travis2docker", @@ -110,7 +114,8 @@ def test_main_docker_image_parameter(tmp_path): assert "FROM quay.io/vauxoo/myproject:custom-tag" in dkr_content -def test_main_without_variables_sh(tmp_path): +def test_main_without_variables_sh(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) repo = create_repo(tmp_path, {"README.md": "no variables.sh here"}) sys.argv = [ "travis2docker", From 3f5448a7d40dfa67c5cdd0bdec2a39f1ff39ca28 Mon Sep 17 00:00:00 2001 From: "Moises Lopez - https://www.vauxoo.com/" Date: Fri, 7 Aug 2026 13:19:14 -0600 Subject: [PATCH 8/9] [IMP] cli: Keep removed parameters as deprecated no-ops and restore --no-clone Existing aliases and scripts still pass --exclude-after-success, --travis-yml-path and --runs-at-the-end-script; accept them like --deployv, print a WARNING when a value is going to be ignored and document them as deprecated in the help, keeping each parameter in its original position for a minimal diff. --no-clone stays functional modernized to the deployv reality: the new --variables-sh-path points to the variables.sh file (or the directory containing it) and skips cloning the repository, the same logic --travis-yml-path provided for .travis.yml. --- src/travis2docker/cli.py | 69 +++++++++++++++++++++++++++++++++++-- tests/test_travis2docker.py | 42 ++++++++++++++++++++++ 2 files changed, 109 insertions(+), 2 deletions(-) diff --git a/src/travis2docker/cli.py b/src/travis2docker/cli.py index 4f646a0..f077b9f 100644 --- a/src/travis2docker/cli.py +++ b/src/travis2docker/cli.py @@ -24,6 +24,15 @@ from .travis2docker import Travis2Docker +def variables_sh_read(variables_sh_path): + variables_sh_path = pathlib.Path(variables_sh_path).expanduser() + if variables_sh_path.is_dir(): + variables_sh_path /= "variables.sh" + if not variables_sh_path.is_file(): + return None + return variables_sh_path.read_text() + + def get_git_data(project, path, revision): git_obj = GitRun(project, path, path_prefix_repo=True) git_obj.update() @@ -86,6 +95,13 @@ def main(return_result=False): dest="remotes", help="Add git remote to git of build path, separated by a comma.\nUse remote name. E.g. 'Vauxoo,moylop260'", ) + parser.add_argument( + "--exclude-after-success", + dest="exclude_after_success", + action="store_true", + default=False, + help="Deprecated. Ignored: the travis_after_success section does not exist anymore", + ) parser.add_argument( "--run-extra-args", dest="run_extra_args", @@ -114,6 +130,26 @@ def main(return_result=False): default="", help='Extra commands to run after "build" script. Note: You can use \\$IMAGE escaped environment variable.', ) + parser.add_argument( + "--travis-yml-path", + dest="travis_yml_path", + default=None, + help="Deprecated. Ignored: the .travis.yml file is not used anymore", + ) + parser.add_argument( + "--variables-sh-path", + dest="variables_sh_path", + default=None, + help="Optional path of the variables.sh file (or the directory containing it) to use.\n" + "Default: Extracted from git repo and git revision.", + ) + parser.add_argument( + "--no-clone", + dest="no_clone", + action="store_true", + default=False, + help="Avoid cloning the repository. It requires --variables-sh-path", + ) parser.add_argument( "--add-rcfile", dest="add_rcfile", @@ -122,6 +158,13 @@ def main(return_result=False): "copy for user's HOME path into container, separated by a comma.", ) parser.add_argument("-v", "--version", action="version", version="%(prog)s " + __version__) + parser.add_argument( + "--runs-at-the-end-script", + dest="runs_at_the_end_script", + nargs="*", + default="", + help="Deprecated. Ignored: the script section of .travis.yml does not exist anymore", + ) parser.add_argument( "--build-env-args", dest="build_env_args", @@ -150,6 +193,14 @@ def main(return_result=False): ) args = parser.parse_args() + deprecated_args = { + "--exclude-after-success": args.exclude_after_success, + "--travis-yml-path": args.travis_yml_path, + "--runs-at-the-end-script": args.runs_at_the_end_script, + } + for deprecated_arg, value in deprecated_args.items(): + if value: + stdout.write("WARNING: %s is deprecated and its value will be ignored\n" % deprecated_arg) revision = args.git_revision git_repo = args.git_repo_url git_base = GitRun.get_data_url(git_repo, False)[0] @@ -166,11 +217,25 @@ def main(return_result=False): rcfiles = [ (pathlib.Path(rc_file).expanduser(), "$HOME/%s" % pathlib.Path(rc_file).name) for rc_file in rcfiles_args ] - os_kwargs = get_git_data(git_repo, pathlib.Path(root_path) / "repo", revision) + if args.no_clone: + os_kwargs = { + "repo_owner": "local_file", + "repo_project": "local_file", + "revision": revision, + "sha": "local_file", + "project": git_repo, + } + else: + os_kwargs = get_git_data(git_repo, pathlib.Path(root_path) / "repo", revision) + + if args.variables_sh_path: + os_kwargs["variables_sh"] = variables_sh_read(args.variables_sh_path) if not os_kwargs.get("variables_sh"): msg = ( - "The repo or the branch is incorrect value, because " + "The file %s is empty or does not exist." % args.variables_sh_path + if args.variables_sh_path + else "The repo or the branch is incorrect value, because " + "It can not got the variables.sh content from %s %s. " % (git_repo, revision) + "\nPlease, verify access repository," + "\nverify exists url and revision, " diff --git a/tests/test_travis2docker.py b/tests/test_travis2docker.py index b89bcf9..630dfd8 100644 --- a/tests/test_travis2docker.py +++ b/tests/test_travis2docker.py @@ -76,6 +76,13 @@ def test_main_deployv(tmp_path, monkeypatch): "BUILD_ENV2", "--build-extra-steps", "touch /home/odoo/extra_step_done", + # Deprecated parameters must still be accepted (and ignored) + "--deployv", + "--exclude-after-success", + "--travis-yml-path", + "foo.yml", + "--runs-at-the-end-script", + "echo done", ] scripts = cli_main(return_result=True) assert len(scripts) == 1, "Scripts returned should be 1" @@ -114,6 +121,41 @@ def test_main_docker_image_parameter(tmp_path, monkeypatch): assert "FROM quay.io/vauxoo/myproject:custom-tag" in dkr_content +def test_main_no_clone(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + (tmp_path / "variables.sh").write_text(VARIABLES_SH) + sys.argv = [ + "travis2docker", + "foo", + "bar", + "--no-clone", + "--variables-sh-path", + str(tmp_path), + "--root-path", + str(tmp_path / "t2d"), + ] + scripts = cli_main(return_result=True) + assert len(scripts) == 1, "Scripts returned should be 1" + dkr_content = (pathlib.Path(scripts[0]) / "Dockerfile").read_text() + assert "FROM quay.io/vauxoo/myproject:myproject-16.0-local_f" in dkr_content + + +def test_main_no_clone_missing_variables_sh(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + sys.argv = [ + "travis2docker", + "foo", + "bar", + "--no-clone", + "--variables-sh-path", + str(tmp_path / "missing.sh"), + "--root-path", + str(tmp_path / "t2d"), + ] + with pytest.raises(InvalidRepoBranchError): + cli_main(return_result=True) + + def test_main_without_variables_sh(tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) repo = create_repo(tmp_path, {"README.md": "no variables.sh here"}) From c4fb6395496344429aecf9dffa89fffa5a3f4f62 Mon Sep 17 00:00:00 2001 From: "Moises Lopez - https://www.vauxoo.com/" Date: Fri, 7 Aug 2026 13:28:56 -0600 Subject: [PATCH 9/9] [IMP] github-actions: Test modern python versions up to 3.15-dev Run the lint, docs and build jobs with python 3.14, the most recent stable version, and add the python 3.14 and 3.15 classifiers to setup.py. --- .github/workflows/github-actions.yml | 8 ++++---- setup.py | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index c8fe2e7..04325a3 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -18,17 +18,17 @@ jobs: strategy: fail-fast: false matrix: - python: ['3.10', '3.11', '3.12', '3.13'] + python: ['3.10', '3.11', '3.12', '3.13', '3.14', '3.15-dev'] os: [ubuntu-latest, windows-latest, macos-latest] tox_env: ["py,codecov"] include: - - python: '3.12' + - python: '3.14' os: ubuntu-latest tox_env: 'lint' - - python: '3.12' + - python: '3.14' os: ubuntu-latest tox_env: 'docs' - - python: '3.12' + - python: '3.14' os: ubuntu-latest tox_env: 'build' exclude: diff --git a/setup.py b/setup.py index 7eff669..e711195 100755 --- a/setup.py +++ b/setup.py @@ -43,6 +43,8 @@ def read(*names, **kwargs): "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3.15", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", # uncomment if you test on these interpreters: