diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index 5a204ba..04325a3 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -18,26 +18,22 @@ jobs: strategy: fail-fast: false matrix: - python: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13-dev'] + python: ['3.10', '3.11', '3.12', '3.13', '3.14', '3.15-dev'] os: [ubuntu-latest, windows-latest, macos-latest] tox_env: ["py,codecov"] include: - - python: '3.12' + - python: '3.14' os: ubuntu-latest - tox_env: 'check' - - python: '3.12' + tox_env: 'lint' + - python: '3.14' os: ubuntu-latest tox_env: 'docs' - - python: '3.12' + - python: '3.14' os: ubuntu-latest tox_env: 'build' exclude: # macos-14 AKA macos-latest has switched to being an ARM runner, only supporting newer versions of Python # https://github.com/actions/setup-python/issues/825#issuecomment-2096792396 - - python: '3.8' - os: macos-latest - - python: '3.9' - os: macos-latest - python: '3.10' os: macos-latest @@ -45,7 +41,6 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 - submodules: recursive - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} @@ -65,6 +60,9 @@ jobs: pip --version tox --version pip list --format=freeze + - name: Enable git long paths (Windows) + if: runner.os == 'Windows' + run: git config --global core.longpaths true - name: test run: | mkdir -p ~/.ssh diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 38bf737..0000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "src/travis2docker/travis-ci-apt-source-whitelist"] - path = src/travis2docker/travis-ci-apt-source-whitelist - url = https://github.com/travis-ci/apt-source-whitelist.git diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 71a6a65..0000000 --- a/.travis.yml +++ /dev/null @@ -1,68 +0,0 @@ -language: python -sudo: false -env: - global: - - LD_PRELOAD=/lib/x86_64-linux-gnu/libSegFault.so - - SEGFAULT_SIGNALS=all -matrix: - include: - - python: 2.7 - env: - TOXENV=py27,codecov - # TODO: Fix coverage diff version issue - # - python: 3.4 - # env: - # TOXENV=py34,codecov - - python: 3.5 - env: - TOXENV=py35,codecov - - python: 3.6 - env: - TOXENV=py36,codecov - - python: 3.7 - env: - TOXENV=py37,codecov - - python: 3.8 - env: - TOXENV=py38,codecov - - python: 3.9 - env: - TOXENV=py39,codecov - - python: pypy - env: - TOXENV=pypy,codecov - - python: 3.5 - env: - TOXENV=check - - python: 3.5 - env: - TOXENV=docs -before_install: - - python --version - - uname -a - - lsb_release -a -install: - - pip install tox - - virtualenv --version - - easy_install --version - - pip --version - - tox --version -script: - - tox -ve $TOXENV -after_failure: - - more .tox/log/* | cat - - more .tox/*/log/* | cat -before_cache: - - rm -rf $HOME/.cache/pip/log -cache: - directories: - - $HOME/.cache/pip -deploy: - provider: pypi - user: vauxoo - password: - secure: dScWMD2+phGrPeufO2TTQLa3ZW5olWrSf1xXwki2UcrPjF4vIHmFUFoHZ9zK3BPVnY114XArb7O5N0pGO+j9b8G3jXD0HA+jlkMEtrbRJ1kiKUAb4KzF3xs9S/qLNZ2I3+pcE29+faDEV4CErsmlVK/h2HEYVdseH+V0qXUGxEk2bY6NKQ5LGkhb4Ole3wWLah11vvAEGcYQLd0K/hRoNup0bDgmjacH4gfCXLfhNJkz3hw1D/FPYv2Z7AIVvo/UMQS62LisMKKcw7DltzMg7rp1tuQdt7gUBD/EGiqlyVqSgj7d0XOO3HucJK9KATQRDoetVwCQSiR5GwMu39zqijpd1AH5T4mxm9w7xAG9KGlog12dDjZe5eneGGXxfxApZ5K12huiSfxAMHiF6Yhm3BDNTh3WeWAYxH2o3XoqoB5DZowWWAw9Fdl3iG7QdNaboVZ6iOxPUxERJYkle5a74RraQyPEdv7xHyNZhjMrc1NOIzYswGsvMgW/TyCKpBqidSFrkOf6hCGOdMbRXmqtM+M2mAV4WuIJx3fbXT3nvpHmdMMMnwyAYY35CQaz2RcQ3mV7WcyKi0zyrys3wcgsZ2lBMxiqpmBJzDAZQDhBtX8xIKvtdZHf8lbUQFrQttwBgsZIL3eKVFE37iKJEIhMnWXvPcYyhhBYdeu4E9MarSQ= - on: - tags: true - condition: $TOXENV == py37,codecov - repo: Vauxoo/travis2docker diff --git a/MANIFEST.in b/MANIFEST.in index 9dcd891..62e24dd 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,4 @@ graft docs -graft examples graft src graft tests diff --git a/README.rst b/README.rst index bba43f2..db44839 100644 --- a/README.rst +++ b/README.rst @@ -53,7 +53,7 @@ Overview .. end-badges -Script to generate Dockerfile from .travis.yml file +Script to generate a development Dockerfile from the deployv image of a repository (based on its ``variables.sh`` file) * Free software: BSD license @@ -81,19 +81,19 @@ Example: `travisfile2dockerfile --root-path=$HOME/t2d git@github.com:Vauxoo/forecast.git 8.0` The output is: - `${HOME}/t2d/script/git_github.com_Vauxoo_forecast.git/8.0/1` - `${HOME}/t2d/script/git_github.com_Vauxoo_forecast.git/8.0/2` - -The first one is the build for env `TESTS=1`, the second one is for env with `LINT_CHECK=1` + `${HOME}/t2d/script/git_github.com_Vauxoo_forecast.git/8.0` To build image: - `${HOME}/t2d/script/git_github.com_Vauxoo_forecast.git/8.0/1/10-build.sh` + `${HOME}/t2d/script/git_github.com_Vauxoo_forecast.git/8.0/10-build.sh` To create container: - `${HOME}/t2d/script/git_github.com_Vauxoo_forecast.git/8.0/1/20-run.sh --entrypoint=bash` + `${HOME}/t2d/script/git_github.com_Vauxoo_forecast.git/8.0/20-run.sh --entrypoint=bash` -To run the test (into of container): - `/entrypoint.sh` +The repository needs a ``variables.sh`` file in its root path (the one used by +the deployv images built from the CI). By default the docker image is built +from its values as ``DOCKER_IMAGE_REPO:MAIN_APP-VERSION-SHA_SHORT``; use +``--docker-image=quay.io/vauxoo/PROJECT:TAG`` to pick the image pushed by the +``build_docker`` pipeline instead. Optional tools (``--build-env-args``) ===================================== @@ -152,19 +152,6 @@ Dockerfile doesn't support a prompt to enter your password, so you need to remov openssl rsa -in ${fname} -out ${fname}_without_pwd cp ${fname}_without_pwd ${fname} -Download the big image -********************** - -Travis2docker uses a default image with many packages pre-installed. - -`docker pull vauxoo/odoo-80-image-shippable-auto` - -Note: You can define a custom image to use with `--docker-image` parameter. - -For example if you want use the original image of travis you can add the following parameters: - -`--docker-image=quay.io/travisci/travis-python --docker-user=travis` - Install docker ************** diff --git a/examples/example_1.yml b/examples/example_1.yml deleted file mode 100644 index 6aa3401..0000000 --- a/examples/example_1.yml +++ /dev/null @@ -1,22 +0,0 @@ -language: python - -sudo: false -cache: - apt: true - directories: - - $HOME/.cache/pip - -python: - - "2.7" - -virtualenv: - system_site_packages: true - -install: - - touch install - -script: - - touch script - -after_success: - - touch after_success diff --git a/examples/example_2.yml b/examples/example_2.yml deleted file mode 100644 index 164d987..0000000 --- a/examples/example_2.yml +++ /dev/null @@ -1,30 +0,0 @@ -language: python - -sudo: false -cache: - apt: true - directories: - - $HOME/.cache/pip - -addons: - apt: - packages: - - python-lxml - -python: - - "2.7" - -env: - - VARIABLE="value" - -virtualenv: - system_site_packages: true - -install: - - touch install - -script: - - touch script - -after_success: - - touch after_success diff --git a/examples/example_3.yml b/examples/example_3.yml deleted file mode 100644 index 34e2089..0000000 --- a/examples/example_3.yml +++ /dev/null @@ -1,37 +0,0 @@ -language: python - -sudo: false -cache: - apt: true - directories: - - $HOME/.cache/pip - -addons: - apt: - sources: - - pov-wkhtmltopdf - packages: - - python-lxml - - wkhtmltopdf - -python: - - "2.7" - -env: - global: - - VARIABLE_GLOBAL="value global" - matrix: - - VARIABLE_MATRIX_1="value matrix 1" - - VARIABLE_MATRIX_2="value matrix 2" - -virtualenv: - system_site_packages: true - -install: - - touch install - -script: - - touch script - -after_success: - - touch after_success diff --git a/examples/example_4.yml b/examples/example_4.yml deleted file mode 100644 index b1f80d6..0000000 --- a/examples/example_4.yml +++ /dev/null @@ -1,41 +0,0 @@ -language: python - -sudo: false -cache: - apt: true - directories: - - $HOME/.cache/pip - -addons: - apt: - sources: - - pov-wkhtmltopdf - - chef-stable-trusty - packages: - - python-lxml - - wkhtmltopdf - - chefdk - -python: - - "2.7" - -matrix: - include: - - python: 2.7 - env: - VARIABLE_INCLUDE_1="value include 1" - - python: 2.7 - env: - VARIABLE_INCLUDE_2="value include 2" - -virtualenv: - system_site_packages: true - -install: - - touch install - -script: - - touch script - -after_success: - - touch after_success diff --git a/examples/example_5.yml b/examples/example_5.yml deleted file mode 100644 index d12b736..0000000 --- a/examples/example_5.yml +++ /dev/null @@ -1,42 +0,0 @@ -language: python - -sudo: false -cache: - apt: true - directories: - - $HOME/.cache/pip - -addons: - postgresql: "9.5" - apt: - sources: - - pov-wkhtmltopdf - - chef-stable-trusty - packages: - - python-lxml - - wkhtmltopdf - - chefdk - -python: - - "2.7" - -matrix: - include: - - python: 2.7 - env: - VARIABLE_INCLUDE_1="value include 1" - - python: 2.7 - env: - VARIABLE_INCLUDE_2="value include 2" - -virtualenv: - system_site_packages: true - -install: - - touch install - -script: - - touch script - -after_success: - - touch after_success diff --git a/requirements.txt b/requirements.txt index c83de08..74663e0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1 @@ -PyYAML==3.12 ; python_version < '3.7' -PyYAML==6.0.* ; python_version >= '3.7' -Jinja2==2.11.3; python_version < '3.7' -Jinja2==3.1.*; python_version >= '3.7' +Jinja2==3.1.* diff --git a/setup.cfg b/setup.cfg index db1f18a..e45796b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,3 @@ -[bdist_wheel] -universal = 1 - [flake8] max-line-length = 140 exclude = tests/*,*/migrations/*,*/south_migrations/* diff --git a/setup.py b/setup.py index 1a86bc5..e711195 100755 --- a/setup.py +++ b/setup.py @@ -1,25 +1,25 @@ #!/usr/bin/env python +import pathlib import re from glob import glob -from os.path import basename, dirname, join, splitext from setuptools import find_packages, setup def read(*names, **kwargs): - return open(join(dirname(__file__), *names), encoding=kwargs.get("encoding", "utf8")).read() + return pathlib.Path(__file__).parent.joinpath(*names).read_text(encoding=kwargs.get("encoding", "utf8")) setup( name="travis2docker", version="6.4.28", license="BSD-3-Clause", - description="Script to generate Dockerfile from .travis.yml file", + description="Script to generate a development Dockerfile from the deployv image of a repository", long_description="%s\n%s" % ( - re.compile("^.. start-badges.*^.. end-badges", re.M | re.S).sub("", read("README.rst")), - re.sub(":[a-z]+:`~?(.*?)`", r"``\1``", read("CHANGELOG.rst")), + re.compile(r"^.. start-badges.*^.. end-badges", re.MULTILINE | re.DOTALL).sub("", read("README.rst")), + re.sub(r":[a-z]+:`~?(.*?)`", r"``\1``", read("CHANGELOG.rst")), ), long_description_content_type="text/x-rst", author="Vauxoo", @@ -27,7 +27,7 @@ def read(*names, **kwargs): url="https://github.com/vauxoo/travis2docker", packages=find_packages("src"), package_dir={"": "src"}, - py_modules=[splitext(basename(path))[0] for path in glob("src/*.py")], + py_modules=[pathlib.Path(path).stem for path in glob("src/*.py")], include_package_data=True, zip_safe=False, classifiers=[ @@ -39,12 +39,12 @@ def read(*names, **kwargs): "Operating System :: Microsoft :: Windows", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3.15", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", # uncomment if you test on these interpreters: @@ -56,6 +56,7 @@ def read(*names, **kwargs): keywords=[ # eg: 'keyword1', 'keyword2', 'keyword3', ], + python_requires=">=3.10", install_requires=read("requirements.txt").split("\n"), extras_require={ # eg: diff --git a/src/travis2docker/cli.py b/src/travis2docker/cli.py index 9816c2f..f077b9f 100644 --- a/src/travis2docker/cli.py +++ b/src/travis2docker/cli.py @@ -15,7 +15,7 @@ import argparse import os -from os.path import expanduser, expandvars, isdir, isfile, join +import pathlib from sys import stdout from . import __version__ @@ -24,12 +24,20 @@ from .travis2docker import Travis2Docker +def variables_sh_read(variables_sh_path): + variables_sh_path = pathlib.Path(variables_sh_path).expanduser() + if variables_sh_path.is_dir(): + variables_sh_path /= "variables.sh" + if not variables_sh_path.is_file(): + return None + return variables_sh_path.read_text() + + def get_git_data(project, path, revision): git_obj = GitRun(project, path, path_prefix_repo=True) git_obj.update() data = { "sha": git_obj.get_sha(revision), - "content": git_obj.show_file(".travis.yml", revision) or git_obj.show_file(".t2d.yml", revision), "variables_sh": git_obj.show_file("variables.sh", revision), "repo_owner": git_obj.owner, "repo_project": git_obj.repo, @@ -41,28 +49,13 @@ def get_git_data(project, path, revision): return data -def yml_read(yml_path): - yml_path_expanded = expandvars(expanduser(yml_path)) - alt_yml_path_expanded = None - if isdir(yml_path_expanded): - yml_path_expanded = join(yml_path_expanded, ".travis.yml") - alt_yml_path_expanded = join(yml_path_expanded, ".t2d.yml") - if not isfile(yml_path_expanded): - if alt_yml_path_expanded and isfile(alt_yml_path_expanded): - yml_path_expanded = alt_yml_path_expanded - else: - return - with open(yml_path_expanded) as f_yml: - return f_yml.read() - - def main(return_result=False): parser = argparse.ArgumentParser() parser.add_argument( "git_repo_url", help="Specify repository git of work." "\nThis is used to clone it " - "and get file .travis.yml or .shippable.yml" + "and get the variables.sh file of the deployv image" "\nIf your repository is private, " "don't use https url, " "use ssh url", @@ -78,21 +71,19 @@ def main(return_result=False): parser.add_argument( "--docker-user", dest="docker_user", - help="User of work into Dockerfile." "\nBased on your docker image." "\nDefault: root", + help="User of work into Dockerfile.\nBased on your docker image.\nDefault: odoo", ) parser.add_argument( "--docker-image", dest="default_docker_image", help="Docker image to use by default in Dockerfile." - "\nUse this parameter if don't " - "exists value: 'build_image: IMAGE_NAME' " - "in .travis.yml" - "\nDefault: 'vauxoo/odoo-80-image-shippable-auto'", + "\nDefault: built from variables.sh as " + "'DOCKER_IMAGE_REPO:MAIN_APP-VERSION-SHA_SHORT'", ) default_root_path = os.environ.get("TRAVIS2DOCKER_ROOT_PATH") if not default_root_path: - default_root_path = os.path.expanduser("~") - default_root_path = join(default_root_path, ".t2d") + default_root_path = pathlib.Path("~").expanduser() + default_root_path = str(pathlib.Path(default_root_path) / ".t2d") parser.add_argument( "--root-path", dest="root_path", @@ -102,14 +93,14 @@ def main(return_result=False): parser.add_argument( "--add-remote", dest="remotes", - help="Add git remote to git of build path, separated by a comma." "\nUse remote name. E.g. 'Vauxoo,moylop260'", + help="Add git remote to git of build path, separated by a comma.\nUse remote name. E.g. 'Vauxoo,moylop260'", ) parser.add_argument( "--exclude-after-success", dest="exclude_after_success", action="store_true", default=False, - help="Exclude `travis_after_success` section to entrypoint", + help="Deprecated. Ignored: the travis_after_success section does not exist anymore", ) parser.add_argument( "--run-extra-args", @@ -137,20 +128,27 @@ def main(return_result=False): dest="build_extra_cmds", nargs="*", default="", - help='Extra commands to run after "build" script. ' "Note: You can use \\$IMAGE escaped environment variable.", + help='Extra commands to run after "build" script. Note: You can use \\$IMAGE escaped environment variable.', ) parser.add_argument( "--travis-yml-path", dest="travis_yml_path", - help="Optional path of file .travis.yml to use.\n" "Default: Extracted from git repo and git revision.", default=None, + help="Deprecated. Ignored: the .travis.yml file is not used anymore", + ) + parser.add_argument( + "--variables-sh-path", + dest="variables_sh_path", + default=None, + help="Optional path of the variables.sh file (or the directory containing it) to use.\n" + "Default: Extracted from git repo and git revision.", ) parser.add_argument( "--no-clone", dest="no_clone", action="store_true", - help="Avoid clone the repository. It will require travis-yml-path", default=False, + help="Avoid cloning the repository. It requires --variables-sh-path", ) parser.add_argument( "--add-rcfile", @@ -165,7 +163,7 @@ def main(return_result=False): dest="runs_at_the_end_script", nargs="*", default="", - help='Extra commands to run after "script" file. ' "Note: You can use \\$IMAGE escaped environment variable.", + help="Deprecated. Ignored: the script section of .travis.yml does not exist anymore", ) parser.add_argument( "--build-env-args", @@ -183,8 +181,8 @@ def main(return_result=False): "--deployv", dest="deployv", action="store_true", - default=False, - help="Use the image generated from the CI and used in deployV", + default=True, + help="Deprecated. The deployv image is now the only supported mode", ) parser.add_argument( "--build-extra-steps", @@ -195,6 +193,14 @@ def main(return_result=False): ) args = parser.parse_args() + deprecated_args = { + "--exclude-after-success": args.exclude_after_success, + "--travis-yml-path": args.travis_yml_path, + "--runs-at-the-end-script": args.runs_at_the_end_script, + } + for deprecated_arg, value in deprecated_args.items(): + if value: + stdout.write("WARNING: %s is deprecated and its value will be ignored\n" % deprecated_arg) revision = args.git_revision git_repo = args.git_repo_url git_base = GitRun.get_data_url(git_repo, False)[0] @@ -202,19 +208,16 @@ def main(return_result=False): root_path = args.root_path default_docker_image = args.default_docker_image remotes = args.remotes and args.remotes.split(",") - exclude_after_success = args.exclude_after_success run_extra_args = args.run_extra_args build_extra_args = args.build_extra_args - travis_yml_path = args.travis_yml_path build_extra_cmds = "\n".join(args.build_extra_cmds) run_extra_cmds = "\n".join(args.run_extra_cmds) - no_clone = args.no_clone - deployv = args.deployv rcfiles_args = args.add_rcfile and args.add_rcfile.split(",") - runs_at_the_end_script = args.runs_at_the_end_script or None build_env_args = [build_env_args[0] for build_env_args in args.build_env_args] - rcfiles = [(expanduser(rc_file), os.path.join("$HOME", os.path.basename(rc_file))) for rc_file in rcfiles_args] - if no_clone: + rcfiles = [ + (pathlib.Path(rc_file).expanduser(), "$HOME/%s" % pathlib.Path(rc_file).name) for rc_file in rcfiles_args + ] + if args.no_clone: os_kwargs = { "repo_owner": "local_file", "repo_project": "local_file", @@ -223,42 +226,31 @@ def main(return_result=False): "project": git_repo, } else: - os_kwargs = get_git_data(git_repo, join(root_path, "repo"), revision) - - if travis_yml_path: - yml_content = yml_read(travis_yml_path) - else: - yml_content = os_kwargs["content"] + os_kwargs = get_git_data(git_repo, pathlib.Path(root_path) / "repo", revision) - if not yml_content and os_kwargs.get("variables_sh"): - deployv = True - yml_content = "deployv: True" - if not default_docker_image and not deployv: - default_docker_image = "vauxoo/odoo-80-image-shippable-auto" + if args.variables_sh_path: + os_kwargs["variables_sh"] = variables_sh_read(args.variables_sh_path) - if not yml_content: + if not os_kwargs.get("variables_sh"): msg = ( - "The file %s is empty." % travis_yml_path - if travis_yml_path + "The file %s is empty or does not exist." % args.variables_sh_path + if args.variables_sh_path else "The repo or the branch is incorrect value, because " - + "It can not got the .travis.yml or variables.sh content from %s %s. " % (git_repo, revision) + + "It can not got the variables.sh content from %s %s. " % (git_repo, revision) + "\nPlease, verify access repository," + "\nverify exists url and revision, " - + "\nverify exists .travis.yml" + + "\nverify exists variables.sh" ) raise InvalidRepoBranchError(msg) os_kwargs.update({"remotes": remotes, "git_base": git_base}) if docker_user: os_kwargs.update({"user": docker_user}) t2d = Travis2Docker( - yml_buffer=yml_content, - work_path=join(root_path, "script", GitRun.url2dirname(git_repo), revision), + work_path=pathlib.Path(root_path) / "script" / GitRun.url2dirname(git_repo) / revision, image=default_docker_image, os_kwargs=os_kwargs, - copy_paths=[(expanduser("~/.ssh"), "$HOME/.ssh")] + rcfiles, - runs_at_the_end_script=runs_at_the_end_script, + copy_paths=[(pathlib.Path("~/.ssh").expanduser(), "$HOME/.ssh")] + rcfiles, build_env_args=build_env_args, - deployv=deployv, build_extra_steps=args.build_extra_steps, ) t2d.build_extra_params = { @@ -269,11 +261,11 @@ def main(return_result=False): "extra_params": run_extra_args, "extra_cmds": run_extra_cmds, } - fname_scripts = t2d.compute_dockerfile(skip_after_success=exclude_after_success) + fname_scripts = t2d.compute_dockerfile() if fname_scripts: fname_list = "- " + "\n- ".join(fname_scripts) stdout.write("\nGenerated scripts:\n%s\n" % fname_list) - if deployv and not default_docker_image: + if not default_docker_image: stdout.write("=" * 80) # TODO: Add the URL to open the pipelines stdout.write( diff --git a/src/travis2docker/docker_helper/build.py b/src/travis2docker/docker_helper/build.py index a843130..e68c993 100644 --- a/src/travis2docker/docker_helper/build.py +++ b/src/travis2docker/docker_helper/build.py @@ -2,7 +2,7 @@ # pylint: disable=print-used import glob -import os +import pathlib import re import subprocess import sys @@ -11,7 +11,7 @@ def ssh_keyscan2known_hosts(url, known_hosts_path=None): # python3 -c "import build;build.ssh_keyscan2known_hosts('url')" if known_hosts_path is None: - known_hosts_path = os.path.join(os.path.expanduser("~"), ".ssh", "known_hosts") + known_hosts_path = pathlib.Path("~/.ssh/known_hosts").expanduser() # Clear current known hosts cmd = ["ssh-keygen", "-R", url] @@ -22,7 +22,7 @@ def ssh_keyscan2known_hosts(url, known_hosts_path=None): cmd = ["ssh-keyscan", "-p", "22", url] print(" ".join(cmd)) keys_scanned = subprocess.check_output(cmd).decode(sys.stdout.encoding).strip() - with open(known_hosts_path, "r+") as known_hosts_f: + with pathlib.Path(known_hosts_path).open("r+") as known_hosts_f: known_hosts_f.write("\n" + keys_scanned) @@ -31,14 +31,14 @@ def git_set_remote(path=None): if path is None: # /home/odoo/instance/odoo spends a lot of time path = "/home/odoo/instance/extra_addons" - git_re = re.compile("([^/|@]+)/([^/]+)/([^/.]+(.git)?)") + git_re = re.compile(r"([^/|@]+)/([^/]+)/([^/.]+(.git)?)") # TODO: Support ssh url - path = os.path.join(path, "*", "**", ".git") + path = str(pathlib.Path(path) / "*" / "**" / ".git") hosts_scanned = set() for git_dir in glob.glob(path, recursive=True): git_cmd = [ "git", - "--work-tree=%s" % os.path.dirname(git_dir), + "--work-tree=%s" % pathlib.Path(git_dir).parent, "--git-dir=%s" % git_dir, ] cmd = git_cmd + ["remote", "get-url", "--push", "origin"] diff --git a/src/travis2docker/git_run.py b/src/travis2docker/git_run.py index 8d4aa83..858565a 100644 --- a/src/travis2docker/git_run.py +++ b/src/travis2docker/git_run.py @@ -1,6 +1,7 @@ # pylint: disable=useless-object-inheritance,print-used,except-pass -import os +import contextlib +import pathlib import re import subprocess @@ -15,8 +16,9 @@ def decode_utf(field): class GitRun: def __init__(self, repo_git, path, path_prefix_repo=False): self.repo_git = repo_git + path = pathlib.Path(path) if path_prefix_repo: - path = os.path.join(path, self.url2dirname(repo_git)) + path /= self.url2dirname(repo_git) self.path = path self.host, self.owner, self.repo = self.get_data_url(repo_git) @@ -25,22 +27,22 @@ def get_data_url(repo_git, no_user=True): host, owner, repo = False, False, False repo_git_sub = repo_git.replace(":", "/") if no_user: - repo_git_sub = re.sub(".+@", "", repo_git_sub) - repo_git_sub = re.sub(".git$", "", repo_git_sub) + repo_git_sub = re.sub(r".+@", "", repo_git_sub) + repo_git_sub = re.sub(r".git$", "", repo_git_sub) match_object = re.search(r"(?P[^/]+)/(?P[^/]+)/(?P[^/]+)", repo_git_sub) if match_object: host = match_object.group("host") owner = match_object.group("owner") repo = match_object.group("repo") - elif os.path.isdir(repo_git): + elif pathlib.Path(repo_git).is_dir(): host = "local" - owner = os.path.basename(repo_git) - repo = os.path.basename(os.path.dirname(repo_git)) + owner = pathlib.Path(repo_git).name + repo = pathlib.Path(repo_git).parent.name return host, owner, repo @staticmethod def url2dirname(url): - for invalid_char in "@:/#": + for invalid_char in "@:/#\\": url = url.replace(invalid_char, "_") return url @@ -61,10 +63,8 @@ def run(self, cmd): print("cmd list", cmd) print("cmd", " ".join(cmd)) res = None - try: + with contextlib.suppress(BaseException): res = subprocess.check_output(cmd) - except BaseException: - pass if res: try: res = res.decode() @@ -94,9 +94,9 @@ def get_ref_data(self, refs=None, fields=None): def update(self): """Get a repository git or update it""" - if not os.path.isdir(os.path.join(self.path)): - os.makedirs(self.path) - if not os.path.isdir(os.path.join(self.path, "refs")): + if not self.path.is_dir(): + self.path.mkdir(parents=True) + if not (self.path / "refs").is_dir(): subprocess.check_output(["git", "clone", "--bare", self.repo_git, self.path]) self.run(["gc", "--auto", "--prune=all"]) self.run(["fetch", "-p", "origin", "+refs/heads/*:refs/heads/*"]) diff --git a/src/travis2docker/templates/Dockerfile b/src/travis2docker/templates/Dockerfile deleted file mode 100644 index ba7304c..0000000 --- a/src/travis2docker/templates/Dockerfile +++ /dev/null @@ -1,89 +0,0 @@ -FROM {{ image }} -COPY {{ entrypoint_path }} /entrypoint.sh -RUN chown -R {{ user }}:{{ user }} /entrypoint.sh -COPY {{ rvm_env_path }} /rvm_env.sh -RUN chown -R {{ user }}:{{ user }} /rvm_env.sh - -{%- for build_env_arg in build_env_args %} -ARG {{ build_env_arg }} -ENV {{ build_env_arg }}=${{ build_env_arg }} -{% endfor %} - -ENV HOME= -{%- if user == 'root' -%} -/root -{%- else -%} -/home/{{ user }} -{%- endif %} - -{% if image == 'quay.io/travisci/travis-python' -%} -ENV PATH=${PATH}:/home/travis/.nvm/v0.10.36/bin:/home/travis/.nvm/v0.10.36/lib/node_modules/npm/bin -{%- endif %} - -{%- for src, dest in copies or [] %} -COPY {{ src }} {{ dest }} -{% endfor -%} - -{% if copies -%} -RUN {% for src, dest in copies or [] -%} chown -R {{ user }}:{{ user }} {{dest}}; -{%- endfor -%} -{%- endif %} - -{% if sources -%} -RUN {{ ' && '.join(sources) }} -{%- endif %} - -{% if packages -%} -RUN apt-get update; apt-get install {{ ' '.join(packages) }} -{%- endif %} - -RUN echo "TRAVIS_PYTHON_VERSION={{ python_version }}" >> /etc/environment - -{% for step in build_extra_steps %} -RUN {{ step }} -{% endfor %} - -USER {{ user }} -ENV TRAVIS_PYTHON_VERSION={{ python_version }} -ENV TRAVIS_REPO_SLUG={{ repo_owner }}/{{ repo_project }} -ENV TRAVIS_BUILD_DIR=${HOME}/build/${TRAVIS_REPO_SLUG} -RUN git init ${TRAVIS_BUILD_DIR} \ - && cd ${TRAVIS_BUILD_DIR} \ - && git remote add origin {{ project }} \ - && git fetch --update-head-ok -p origin \ -{% if revision.startswith('pull/') -%} - '+refs/{{ revision }}/head:refs/{{ revision }}' || true && \ - git fetch --update-head-ok -p origin \ - '+refs/{{ revision.replace('pull/', 'merge-requests/') }}/head:refs/{{ revision }}' || true -{%- else -%} - '+refs/heads/{{ revision }}:refs/heads/{{ revision }}' -{%- endif %} \ - && git checkout -qf {{ revision }} \ - && git config --global user.email "{{ git_email }}" \ - && git config --global user.name "{{ git_user }}" \ -{%- for remote in remotes or [] %} - && git remote add {{ remote }} {{git_base}}:{{ remote }}/{{ repo_project }}.git \ -{%- endfor %} - || true - -{% if add_self_rsa_pub -%} -RUN cat ${HOME}/.ssh/id_rsa.pub | tee -a ${HOME}/.ssh/authorized_keys -{%- endif %} - -{% if env -%} -ENV {{ env }} -{%- endif %} - -WORKDIR ${TRAVIS_BUILD_DIR} - - -{% if runs -%} -{% if image == 'quay.io/travisci/travis-python' -%} -RUN /bin/bash -c "source $HOME/virtualenv/python{{ python_version }}/bin/activate && source /rvm_env.sh && {{ ' && '.join(runs) }}" -{% elif image == 'vauxoo/odoo-80-image-shippable-auto' -%} -RUN /bin/bash -c "source ${REPO_REQUIREMENTS}/virtualenv/python{{ python_version }}/bin/activate && source ${REPO_REQUIREMENTS}/virtualenv/nodejs/bin/activate && source /rvm_env.sh && {{ ' && '.join(runs) }}" -{% else %} -RUN /bin/bash -c "source /rvm_env.sh && {{ ' && '.join(runs) }}" -{%- endif %} -{%- endif %} -ENTRYPOINT /entrypoint.sh diff --git a/src/travis2docker/templates/entrypoint.sh b/src/travis2docker/templates/entrypoint.sh deleted file mode 100644 index 38ccbc6..0000000 --- a/src/travis2docker/templates/entrypoint.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -{% if image == 'quay.io/travisci/travis-python' -%} -source /home/travis/virtualenv/python{{ python_version }}/bin/activate -{% elif image == 'vauxoo/odoo-80-image-shippable-auto' -%} -source ${REPO_REQUIREMENTS}/virtualenv/python{{ python_version }}/bin/activate && source ${REPO_REQUIREMENTS}/virtualenv/nodejs/bin/activate -{%- endif %} -source /rvm_env.sh -{% for entrypoint in entrypoints %} -{{ entrypoint }} -{% endfor %} diff --git a/src/travis2docker/templates/rvm_env.sh b/src/travis2docker/templates/rvm_env.sh deleted file mode 100644 index 2ba94d8..0000000 --- a/src/travis2docker/templates/rvm_env.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -# Load RVM into a shell session *as a function* -if [[ -s "$HOME/.rvm/scripts/rvm" ]] ; then - - # First try to load from a user install - source "$HOME/.rvm/scripts/rvm" - -elif [[ -s "/usr/local/rvm/scripts/rvm" ]] ; then - - # Then try to load from a root install - source "/usr/local/rvm/scripts/rvm" - -else - - printf "An RVM installation was not found.\n" - -fi diff --git a/src/travis2docker/travis-ci-apt-source-whitelist b/src/travis2docker/travis-ci-apt-source-whitelist deleted file mode 160000 index 92dbd15..0000000 --- a/src/travis2docker/travis-ci-apt-source-whitelist +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 92dbd15f6a12e5f24c2f31f314060a349c37dc68 diff --git a/src/travis2docker/travis2docker.py b/src/travis2docker/travis2docker.py index 94b455b..14b20f8 100644 --- a/src/travis2docker/travis2docker.py +++ b/src/travis2docker/travis2docker.py @@ -1,8 +1,5 @@ -# pylint: disable=useless-object-inheritance,consider-using-with,too-complex,print-used -import collections -import errno -import json -import os +# pylint: disable=useless-object-inheritance,consider-using-with,print-used +import pathlib import re import shutil import stat @@ -10,24 +7,16 @@ import jinja2 -try: - from yaml import full_load as yaml_load -except ImportError: - from yaml import load as yaml_load - RE_ENV_STR = r"(?P[\w]*)[ ]*[\=][ ]*[\"\']{0,1}" + r"(?P[\w\.\-\_/\$\{\}\:,\(\)\#\* ]*)[\"\']{0,1}" RE_EXPORT_STR = r"^(?Pexport|EXPORT)( )+" + RE_ENV_STR class Travis2Docker: - re_export = re.compile(RE_EXPORT_STR, re.M) + re_export = re.compile(RE_EXPORT_STR, re.MULTILINE) @property def dockerfile_template(self): - dockerfile = "Dockerfile" - if self.deployv: - dockerfile += "_deployv" - return self.jinja_env.get_template(dockerfile) + return self.jinja_env.get_template("Dockerfile_deployv") @property def new_image(self): @@ -38,10 +27,6 @@ def new_image(self): revision = revision.replace(invalid_char, "_") return ("%s:%s" % (image_name, revision)).lower() - @property - def entrypoint_template(self): - return self.jinja_env.get_template("entrypoint.sh") - @property def build_template(self): return self.jinja_env.get_template("10-build.sh") @@ -52,352 +37,118 @@ def run_template(self): @staticmethod def chmod_execution(file_path): - os.chmod(file_path, os.stat(file_path).st_mode | stat.S_IEXEC) - - @staticmethod - def mkdir_p(path): - try: - os.makedirs(path) - except OSError as os_error: - if os_error.errno == errno.EEXIST and os.path.isdir(path): - pass - else: - raise + file_path.chmod(file_path.stat().st_mode | stat.S_IEXEC) def __init__( self, - yml_buffer, image=None, work_path=None, dockerfile=None, templates_path=None, os_kwargs=None, copy_paths=None, - runs_at_the_end_script=None, build_env_args=None, - deployv=None, build_extra_steps=None, ): - self._python_versions = [] self.curr_work_path = None - self.curr_exports = [] self.build_extra_params = {} self.run_extra_params = {} self.build_env_args = build_env_args - self.deployv = deployv - self.runs_at_the_end_script = ["sleep 2"] if runs_at_the_end_script is None else runs_at_the_end_script - self.variables_sh_data = {} self.build_extra_steps = build_extra_steps - if deployv: - self.variables_sh_data = { - var.lower(): value for _, _, var, value in self.re_export.findall(os_kwargs["variables_sh"]) - } - self.variables_sh_data.update({"sha_short": os_kwargs["sha"][:7]}) - image = ( - "%(docker_image_repo)s:%(main_app)s-%(version)s-%(sha_short)s" % self.variables_sh_data - if not image - else image - ) - build_sh = os.path.join(os.path.dirname(os.path.realpath(__file__)), "templates", "build.sh") - entrypoint_sh = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "templates", - "entrypoint_deployv.sh", - ) - docker_helper = os.path.join(os.path.dirname(os.path.realpath(__file__)), "docker_helper") - vscode_conf = os.path.join(os.path.dirname(os.path.realpath(__file__)), "templates", ".vscode") - coveragerc = os.path.join(os.path.dirname(os.path.realpath(__file__)), "templates", ".coveragerc") - copy_paths.append([build_sh, "/home/odoo/build.sh"]) - copy_paths.append([entrypoint_sh, "/entrypoint.sh"]) - copy_paths.append([docker_helper, "/home/odoo/build"]) - copy_paths.append([vscode_conf, "/home/odoo/.vscode"]) - copy_paths.append([coveragerc, "/home/odoo/.coveragerc"]) - if image is None: - image = "vauxoo/odoo-80-image-shippable-auto" if os_kwargs is None: os_kwargs = {} - default_user = "root" - if image == "vauxoo/odoo-80-image-shippable-auto" or deployv: - default_user = "odoo" - elif image == "quay.io/travisci/travis-python": - default_user = "travis" - os_kwargs.setdefault("user", default_user) + if copy_paths is None: + copy_paths = [] + self.variables_sh_data = { + var.lower(): value for _, _, var, value in self.re_export.findall(os_kwargs["variables_sh"]) + } + self.variables_sh_data.update({"sha_short": os_kwargs["sha"][:7]}) + if not image: + image = "%(docker_image_repo)s:%(main_app)s-%(version)s-%(sha_short)s" % self.variables_sh_data + module_dir = pathlib.Path(__file__).resolve().parent + templates_dir = module_dir / "templates" + copy_paths.append([templates_dir / "build.sh", "/home/odoo/build.sh"]) + copy_paths.append([templates_dir / "entrypoint_deployv.sh", "/entrypoint.sh"]) + copy_paths.append([module_dir / "docker_helper", "/home/odoo/build"]) + copy_paths.append([templates_dir / ".vscode", "/home/odoo/.vscode"]) + copy_paths.append([templates_dir / ".coveragerc", "/home/odoo/.coveragerc"]) + os_kwargs.setdefault("user", "odoo") if dockerfile is None: dockerfile = "Dockerfile" if templates_path is None: - templates_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "templates") + templates_path = templates_dir self.copy_paths = copy_paths self.os_kwargs = os_kwargs self.jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(templates_path)) self.image = image - self._sections = collections.OrderedDict() - self._sections["env"] = "env" - self._sections["addons"] = "addons" - self._sections["before_install"] = "run" - self._sections["install"] = "run" - self._sections["script"] = "entrypoint" - self._sections["after_success"] = "entrypoint" - self.yml = yaml_load(yml_buffer) if work_path is None: - base_name = os.path.splitext(os.path.basename(__file__))[0] - self.work_path = os.path.join(gettempdir(), base_name) + self.work_path = pathlib.Path(gettempdir()) / pathlib.Path(__file__).stem else: - self.work_path = os.path.expandvars(os.path.expanduser(work_path)) + self.work_path = pathlib.Path(work_path).expanduser() self.dockerfile = dockerfile - travis_ci_apt_src = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "travis-ci-apt-source-whitelist", - ) - self.ubuntu_json = json.load( - open(os.path.join(travis_ci_apt_src, "ubuntu.json")) - ) # pylint: disable=consider-using-with - - def _compute(self, section, yml=None): - if yml is None: - yml = self.yml - section_type = self._sections.get(section) - if not section_type: - return None - section_data = yml.get(section, "") - if section != "env" and not section_data: - return None - if not isinstance(section_data, (list, dict, tuple)): - section_data = [section_data] - job_method = getattr(self, "_compute_" + section_type) - return job_method(section_data, section, yml) - - def _compute_env(self, data, _, yml): - if isinstance(data, list): - # old version without matrix - data = {"matrix": data} - env_globals = "" - for env_global in data.get("global", []): - if isinstance(env_global, dict): - # we can't use the secure encrypted variables - continue - env_globals += " " + env_global - env_globals = env_globals.strip() - psql_version = (yml.get("addons") or {}).get("postgresql") - if psql_version: - env_globals += ' PSQL_VERSION="%s"' % psql_version - - for env_matrix in data.get("matrix", [""]): - yield (env_globals + " " + env_matrix).strip() - - def _compute_run(self, data, section, _): - args = self._make_script(data, section, add_run=True, prefix="files") - return args - - def _compute_entrypoint(self, data, section, _): - args = self._make_script(data, section, add_entrypoint=True, prefix="files") - return args - - def _compute_addons(self, data, section, _): - if "apt" not in data: - return - sources = [] - for alias in data["apt"].get("sources") or []: - for ubuntu_source in self.ubuntu_json: - if alias == ubuntu_source["alias"]: - if ubuntu_source["key_url"]: - sources.append('curl -sSL "' + ubuntu_source["key_url"] + '" | apt-key add -') - if ubuntu_source["sourceline"].startswith("ppa:"): - sources.append('apt-add-repository -y "' + ubuntu_source["sourceline"] + '"') - else: - sources.append( - 'echo "' + ubuntu_source["sourceline"] + '" | tee -a /etc/apt/sources.list > /dev/null' - ) - new_data = data["apt"].copy() - new_data["sources"] = sources - return new_data - - def _make_script(self, data, section, add_entrypoint=False, add_run=False, prefix=""): - file_path = os.path.join(self.curr_work_path, prefix, section) - self.mkdir_p(os.path.dirname(file_path)) - with open(file_path, "w") as f_section: - f_section.write("#!/bin/bash\n") - for var, value in self.curr_exports: - f_section.write("\nexport %s=%s" % (var, value)) - for line in data: - self.curr_exports.extend([(var, value) for _, _, var, value in self.re_export.findall(line)]) - f_section.write("\n" + line) - if section == "script": - for run_at_the_end_script in self.runs_at_the_end_script: - f_section.write("\n%s" % run_at_the_end_script) - src = "./" + os.path.relpath(file_path, self.curr_work_path) - dest = "/" + section - args = { - "copies": [(src, dest)], - "entrypoints": [dest] if add_entrypoint else [], - "runs": [dest] if add_run else [], - } - self.chmod_execution(file_path) - return args - - def reset(self): - self.curr_work_path = None - self.curr_exports = [] - - def compute_build_scripts(self, prefix_build, version): - build_path = os.path.join(self.curr_work_path, "10-build.sh") - run_path = os.path.join(self.curr_work_path, "20-run.sh") - new_image = self.new_image + "_" + version.replace(".", "_") + "_" + str(prefix_build) - with open(build_path, "w") as f_build, open(run_path, "w") as f_run: + def compute_build_scripts(self): + build_path = self.curr_work_path / "10-build.sh" + run_path = self.curr_work_path / "20-run.sh" + with build_path.open("w") as f_build, run_path.open("w") as f_run: build_content = self.build_template.render( - image=new_image, dirname_dockerfile=self.curr_work_path, **self.build_extra_params + image=self.new_image, dirname_dockerfile=self.curr_work_path, **self.build_extra_params ).strip("\n ") - try: - f_build.write(build_content.encode("utf-8")) - except TypeError: - f_build.write(build_content) - - run_content = self.run_template.render(image=new_image, **self.run_extra_params).strip("\n ") - try: - f_run.write(run_content.encode("utf-8")) - except TypeError: - f_run.write(run_content) + f_build.write(build_content) + run_content = self.run_template.render(image=self.new_image, **self.run_extra_params).strip("\n ") + f_run.write(run_content) self.chmod_execution(build_path) self.chmod_execution(run_path) - def _python_version_env(self): - versions = self.yml.pop("python", {}) - if not versions: - self._python_versions = ["3.5"] # 3.5 by default - return - if not isinstance(versions, list): - versions = [versions] - # TODO: Use full version if in the default base image are installed - new_versions = {".".join(version.split(".")[:2]) for version in versions} - self._python_versions = list(set(self._python_versions) | new_versions) - - def _transform_yml_matrix2env(self): - matrix = self.yml.pop("matrix", {}) - envs = [include["env"] for include in matrix.get("include", []) if include.get("env")] - if envs: - self.yml["env"] = envs - - def compute_dockerfile(self, skip_after_success=False): - work_paths = [] - self._transform_yml_matrix2env() - self._python_version_env() - jobs_stages = self.yml.pop("jobs", {}).get("include", {}) - for global_version in self._python_versions: - for count, global_env in enumerate(self._compute("env"), 1): - for job_count, job_stage in enumerate(jobs_stages or [{}], 1): - job_env = self._compute("env", job_stage) or "" - if job_env is not None: - job_env = next(job_env) - env = "%s %s" % (global_env, job_env) - env = env.strip() - version = global_version - try: - version = job_stage["python"] - except KeyError: # pylint: disable=except-pass - pass - version = "%s" % version - - self.reset() - self.curr_work_path = os.path.join( - self.work_path, - version.replace(".", "_"), - "env_%d_job_%d" % (count, job_count), - ) - curr_dockerfile = os.path.join(self.curr_work_path, self.dockerfile) - entryp_path = os.path.join(self.curr_work_path, "files", "entrypoint.sh") - self.mkdir_p(os.path.dirname(entryp_path)) - entryp_relpath = os.path.relpath(entryp_path, self.curr_work_path) - rvm_env_path = os.path.join(self.curr_work_path, "files", "rvm_env.sh") - rvm_env_relpath = os.path.relpath(rvm_env_path, self.curr_work_path) - copies = [] - for copy_path, dest in self.copy_paths: - copies.append((self.copy_path(copy_path), dest)) - self.set_authorized_key() - kwargs = { - "runs": [], - "copies": copies, - "entrypoints": [], - "entrypoint_path": entryp_relpath, - "python_version": version, - "image": self.image, - "env": env, - "packages": [], - "sources": [], - "rvm_env_path": rvm_env_relpath, - "build_env_args": self.build_env_args, - "build_extra_steps": self.build_extra_steps, - } - with open(curr_dockerfile, "w") as f_dockerfile, open(entryp_path, "w") as f_entrypoint, open( - rvm_env_path, "w" - ) as f_rvm: - for section, _ in self._sections.items(): - if section == "env": - continue - if skip_after_success and section == "after_success": - continue - # job section replace global one - result = self._compute(section, job_stage) - if not result: - result = self._compute(section) - if not result: - continue - keys_to_extend = ( - ["copies", "runs", "entrypoints", "packages", "sources"] - if isinstance(result, dict) - else [] - ) - for key_to_extend in keys_to_extend: - if key_to_extend in result: - kwargs[key_to_extend].extend(result[key_to_extend]) - kwargs.update(self.os_kwargs) - dockerfile_content = self.dockerfile_template.render(kwargs).strip("\n ") - try: - f_dockerfile.write(dockerfile_content.encode("utf-8")) - except TypeError: - f_dockerfile.write(dockerfile_content) - entrypoint_content = self.entrypoint_template.render(kwargs).strip("\n ") - try: - f_entrypoint.write(entrypoint_content.encode("utf-8")) - except TypeError: - f_entrypoint.write(entrypoint_content) - rvm_env_content = self.jinja_env.get_template("rvm_env.sh").render(kwargs).strip("\n ") - try: - f_rvm.write(rvm_env_content.encode("UTF-8")) - except TypeError: - f_rvm.write(rvm_env_content) - self.compute_build_scripts(count, version) - self.chmod_execution(entryp_path) - work_paths.append(self.curr_work_path) - self.reset() + def compute_dockerfile(self): + self.curr_work_path = self.work_path + curr_dockerfile = self.curr_work_path / self.dockerfile + self.curr_work_path.mkdir(exist_ok=True, parents=True) + copies = [] + for copy_path, dest in self.copy_paths: + copies.append((self.copy_path(copy_path), dest)) + self.set_authorized_key() + kwargs = { + "copies": copies, + "image": self.image, + "build_env_args": self.build_env_args, + "build_extra_steps": self.build_extra_steps, + } + kwargs.update(self.os_kwargs) + with curr_dockerfile.open("w") as f_dockerfile: + dockerfile_content = self.dockerfile_template.render(kwargs).strip("\n ") + f_dockerfile.write(dockerfile_content) + self.compute_build_scripts() + work_paths = [str(self.curr_work_path)] + self.curr_work_path = None return work_paths def copy_path(self, path): """:param paths list: List of paths to copy""" - src = os.path.expandvars(os.path.expanduser(path)) - basename = os.path.basename(src) - dest_path = os.path.expandvars(os.path.expanduser(os.path.join(self.curr_work_path, basename))) - if os.path.isdir(dest_path): + src = pathlib.Path(path).expanduser() + dest_path = self.curr_work_path / src.name + if dest_path.is_dir(): shutil.rmtree(dest_path) - if os.path.isdir(src): + if src.is_dir(): try: shutil.copytree(src, dest_path) except shutil.Error: # pylint: disable=except-pass pass # There are permissions errors to copy - elif os.path.isfile(src): + elif src.is_file(): shutil.copy(src, dest_path) else: raise UserWarning("Just directory or file is supported to copy [%s]" % src) - return os.path.relpath(dest_path, self.curr_work_path) + return str(dest_path.relative_to(self.curr_work_path)) def set_authorized_key(self): - ssh_dir = os.path.expanduser("~/.ssh") - ed_key = os.path.join(ssh_dir, "id_ed25519.pub") - rsa_key = os.path.join(ssh_dir, "id_rsa.pub") + ssh_dir = pathlib.Path("~/.ssh").expanduser() + ed_key = ssh_dir / "id_ed25519.pub" + rsa_key = ssh_dir / "id_rsa.pub" - to_copy = False - if os.path.isfile(ed_key): + to_copy = None + if ed_key.is_file(): to_copy = ed_key - elif os.path.isfile(rsa_key): + elif rsa_key.is_file(): print("RSA keys are deprecated, consider changing to ed25519") to_copy = rsa_key @@ -405,12 +156,7 @@ def set_authorized_key(self): print("No public key found. No key added to ~/.ssh/authorized_keys. SSH login won't work.") return - with open(to_copy, encoding="utf-8") as key_fd: - pub_key = key_fd.read() + pub_key = to_copy.read_text(encoding="utf-8") - with open( - os.path.join(self.curr_work_path, ".ssh", "authorized_keys"), - "a", - encoding="utf-8", - ) as auth_fd: + with (self.curr_work_path / ".ssh" / "authorized_keys").open("a", encoding="utf-8") as auth_fd: auth_fd.write(pub_key) diff --git a/tests/test_travis2docker.py b/tests/test_travis2docker.py index da70622..630dfd8 100644 --- a/tests/test_travis2docker.py +++ b/tests/test_travis2docker.py @@ -2,22 +2,24 @@ # pylint: disable=print-used,consider-using-with import os +import pathlib import subprocess import sys +from shutil import which -from travis2docker.cli import main as cli_main - -try: - from shutil import which # python3.x -except ImportError: - from whichcraft import which +import pytest +from travis2docker.cli import main as cli_main +from travis2docker.exceptions import InvalidRepoBranchError -def main(): - return cli_main(return_result=True) +VARIABLES_SH = """export DOCKER_IMAGE_REPO=quay.io/vauxoo/myproject +export MAIN_APP=myproject +export VERSION=16.0 +export CUSTOM_VAR="custom value" +""" -def check_failed_dockerfile(scripts, lines_required=None): +def check_dockerfile_lint(scripts): npm_bin = which("npm") npm_bin_path = subprocess.check_output([npm_bin, "list"]).decode("UTF-8").strip("\n") if npm_bin else "" npm_bin_path_g = subprocess.check_output([npm_bin, "list", "-g"]).decode("UTF-8").strip("\n") if npm_bin else "" @@ -25,7 +27,7 @@ def check_failed_dockerfile(scripts, lines_required=None): lint_bin = which(lint_bin_name) or which(lint_bin_name, path=npm_bin_path + os.pathsep + npm_bin_path_g) assert lint_bin, "'%s' not found." % lint_bin_name for script in scripts: - fname_dkr = os.path.join(script, "Dockerfile") + fname_dkr = str(pathlib.Path(script) / "Dockerfile") pipe = subprocess.Popen( [lint_bin, "-f", fname_dkr], stderr=subprocess.STDOUT, @@ -33,124 +35,136 @@ def check_failed_dockerfile(scripts, lines_required=None): ) output = pipe.stdout.read().decode("utf-8") assert "Check passed" in output, fname_dkr - print("Check dockerfile output", output) - if not lines_required: - continue - with open(fname_dkr) as fdkr: - fdkr_lines = fdkr.readlines() - fdkr_lines[-1] = fdkr_lines[-1].strip("\n") + "\n" - for line_required in lines_required: - assert line_required + "\n" in fdkr_lines - print(fdkr_lines) - - -def test_main(): - # TODO: fix duplicated code - dirname_example = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "examples") - argv = ["travis2docker", "foo", "bar", "--no-clone"] - sources_py = "source ${REPO_REQUIREMENTS}/virtualenv/" + "python2.7/bin/activate" - sources_js = "source ${REPO_REQUIREMENTS}/virtualenv/nodejs/bin/activate" - lines_required = [ - 'RUN /bin/bash -c "{source_py} && {source_js} ' - '&& source /rvm_env.sh && /install"'.format(source_py=sources_py, source_js=sources_js), - "ENTRYPOINT /entrypoint.sh", - ] - example = os.path.join(dirname_example, "example_1.yml") - sys.argv = argv + [ - "--travis-yml-path", - example, - "--add-rcfile=%s,%s" % (example, dirname_example), - ] - scripts = main() - assert len(scripts) == 1, "Scripts returned should be 1 for %s" % example - check_failed_dockerfile(scripts, lines_required) - assert os.path.isdir(os.path.join(scripts[0], os.path.basename(dirname_example))) - assert os.path.isfile(os.path.join(scripts[0], os.path.basename(example))) - sys.argv = argv + [ +def create_repo(base_path, files): + repo_path = pathlib.Path(base_path) / "myrepo" + repo_path.mkdir(parents=True) + subprocess.check_call(["git", "init", "-b", "main", str(repo_path)]) + for fname, content in files.items(): + (repo_path / fname).write_text(content) + subprocess.check_call(["git", "-C", repo_path, "add", "-A"]) + subprocess.check_call([ + "git", + "-C", + repo_path, + "-c", + "user.email=test@test.com", + "-c", + "user.name=test", + "commit", + "-m", + "initial", + ]) + # Return the path relative to base_path: an absolute path would be flattened + # into the clone dirname, exceeding the Windows MAX_PATH limit + return repo_path.name + + +def test_main_deployv(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + repo = create_repo(tmp_path, {"variables.sh": VARIABLES_SH}) + sys.argv = [ + "travis2docker", + repo, + "main", + "--root-path", + str(tmp_path / "t2d"), + "--build-env-args", + "BUILD_ENV1", + "--build-env-args", + "BUILD_ENV2", + "--build-extra-steps", + "touch /home/odoo/extra_step_done", + # Deprecated parameters must still be accepted (and ignored) + "--deployv", + "--exclude-after-success", "--travis-yml-path", - example, + "foo.yml", + "--runs-at-the-end-script", + "echo done", + ] + scripts = cli_main(return_result=True) + assert len(scripts) == 1, "Scripts returned should be 1" + dkr_content = (pathlib.Path(scripts[0]) / "Dockerfile").read_text() + sha_short = subprocess.check_output(["git", "-C", repo, "rev-parse", "HEAD"]).decode("UTF-8")[:7] + assert "FROM quay.io/vauxoo/myproject:myproject-16.0-%s" % sha_short in dkr_content + assert "ENV BUILD_ENV1=TRUE" in dkr_content + assert "ENV BUILD_ENV2=TRUE" in dkr_content + assert "RUN touch /home/odoo/extra_step_done" in dkr_content + assert "ENTRYPOINT /entrypoint.sh" in dkr_content + assert "COPY build.sh /home/odoo/build.sh" in dkr_content + assert "COPY entrypoint_deployv.sh /entrypoint.sh" in dkr_content + assert "COPY docker_helper /home/odoo/build" in dkr_content + for script in ("10-build.sh", "20-run.sh"): + script_path = pathlib.Path(scripts[0]) / script + assert script_path.is_file() + assert os.access(script_path, os.X_OK), "%s should be executable" % script + check_dockerfile_lint(scripts) + + +def test_main_docker_image_parameter(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + repo = create_repo(tmp_path, {"variables.sh": VARIABLES_SH}) + sys.argv = [ + "travis2docker", + repo, + "main", + "--root-path", + str(tmp_path / "t2d"), "--docker-image", - "quay.io/travisci/travis-python", + "quay.io/vauxoo/myproject:custom-tag", + ] + scripts = cli_main(return_result=True) + assert len(scripts) == 1, "Scripts returned should be 1" + dkr_content = (pathlib.Path(scripts[0]) / "Dockerfile").read_text() + assert "FROM quay.io/vauxoo/myproject:custom-tag" in dkr_content + + +def test_main_no_clone(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + (tmp_path / "variables.sh").write_text(VARIABLES_SH) + sys.argv = [ + "travis2docker", + "foo", + "bar", + "--no-clone", + "--variables-sh-path", + str(tmp_path), + "--root-path", + str(tmp_path / "t2d"), + ] + scripts = cli_main(return_result=True) + assert len(scripts) == 1, "Scripts returned should be 1" + dkr_content = (pathlib.Path(scripts[0]) / "Dockerfile").read_text() + assert "FROM quay.io/vauxoo/myproject:myproject-16.0-local_f" in dkr_content + + +def test_main_no_clone_missing_variables_sh(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + sys.argv = [ + "travis2docker", + "foo", + "bar", + "--no-clone", + "--variables-sh-path", + str(tmp_path / "missing.sh"), + "--root-path", + str(tmp_path / "t2d"), + ] + with pytest.raises(InvalidRepoBranchError): + cli_main(return_result=True) + + +def test_main_without_variables_sh(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + repo = create_repo(tmp_path, {"README.md": "no variables.sh here"}) + sys.argv = [ + "travis2docker", + repo, + "main", + "--root-path", + str(tmp_path / "t2d"), ] - scripts = main() - assert len(scripts) == 1, "Scripts returned should be 1 for %s" % example - check_failed_dockerfile(scripts, ["FROM quay.io/travisci/travis-python"]) - - example = os.path.join(dirname_example, "example_2.yml") - sys.argv = argv + ["--travis-yml-path", example] - scripts = main() - assert len(scripts) == 1, "Scripts returned should be 1 for %s" % example - check_failed_dockerfile(scripts, lines_required + ['ENV VARIABLE="value"']) - - example = os.path.join(dirname_example, "example_3.yml") - sys.argv = argv + ["--travis-yml-path", example] - scripts = main() - assert len(scripts) == 2, "Scripts returned should be 2 for %s" % example - check_failed_dockerfile(scripts, lines_required) - with open(os.path.join(scripts[0], "Dockerfile")) as f_dkr: - dkr_content = f_dkr.read() - assert 'VARIABLE_MATRIX_1="value matrix 1"' in dkr_content - assert 'ENV VARIABLE_GLOBAL="value global"' in dkr_content - assert "RUN apt-add-repository" in dkr_content - with open(os.path.join(scripts[1], "Dockerfile")) as f_dkr: - dkr_content = f_dkr.read() - assert 'VARIABLE_MATRIX_2="value matrix 2"' in dkr_content - assert 'ENV VARIABLE_GLOBAL="value global"' in dkr_content - assert "RUN apt-add-repository" in dkr_content - - example = os.path.join(dirname_example, "example_4.yml") - sys.argv = argv + ["--travis-yml-path", example] - scripts = main() - assert len(scripts) == 2, "Scripts returned should be 2 for %s" % example - check_failed_dockerfile(scripts) - with open(os.path.join(scripts[0], "Dockerfile")) as f_dkr: - dkr_content = f_dkr.read() - assert 'VARIABLE_INCLUDE_1="value include 1"' in dkr_content - with open(os.path.join(scripts[1], "Dockerfile")) as f_dkr: - dkr_content = f_dkr.read() - assert 'VARIABLE_INCLUDE_2="value include 2"' in dkr_content - - # Tests that, when specified, the postgresql key sets - # automatically the environment variable $PSQL_VERSION - example = os.path.join(dirname_example, "example_5.yml") - sys.argv = argv + ["--travis-yml-path", example] - scripts = main() - assert len(scripts) == 2, "Scripts returned should be 2 for %s" % example - check_failed_dockerfile(scripts) - with open(os.path.join(scripts[0], "Dockerfile")) as f_dkr: - dkr_content = f_dkr.read() - assert ' PSQL_VERSION="9.5" ' in dkr_content - with open(os.path.join(scripts[1], "Dockerfile")) as f_dkr: - dkr_content = f_dkr.read() - assert ' PSQL_VERSION="9.5" ' in dkr_content - - url = "https://github.com/Vauxoo/travis2docker.git" - sys.argv = ["travis2docker", url, "main"] - scripts = main() - sources_py = "source ${REPO_REQUIREMENTS}/virtualenv/" + "python3.5/bin/activate" - lines_required.pop(0) - lines_required.append( - 'RUN /bin/bash -c "{source_py} && {source_js} && ' - "source /rvm_env.sh && " - '/before_install && /install"'.format(source_py=sources_py, source_js=sources_js), - ) - check_failed_dockerfile(scripts, lines_required + ["ENV TRAVIS_REPO_SLUG=Vauxoo/travis2docker"]) - - sys.argv = ["travis2docker", url, "pull/54"] - scripts = main() - check_failed_dockerfile(scripts, lines_required + ["ENV TRAVIS_REPO_SLUG=Vauxoo/travis2docker"]) - - sys.argv += ["--build-env-args", "BUILD_ENV1", "--build-env-args", "BUILD_ENV2"] - scripts = main() - check_failed_dockerfile( - scripts, - lines_required - + [ - "ARG BUILD_ENV1", - "ENV BUILD_ENV1=$BUILD_ENV1", - "ARG BUILD_ENV2", - "ENV BUILD_ENV2=$BUILD_ENV2", - ], - ) + with pytest.raises(InvalidRepoBranchError): + cli_main(return_result=True) diff --git a/tox.ini b/tox.ini index 8cc5b92..a7a34e6 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ envlist = build, clean, - check, + lint, report, docs @@ -20,7 +20,6 @@ deps = -r{toxinidir}/requirements.txt # test deps pytest pytest-cov - whichcraft nodeenv commands = nodeenv --python-virtualenv @@ -56,7 +55,7 @@ commands = sphinx-build {posargs:-E} -b html docs dist/docs sphinx-build -b linkcheck docs dist/docs -[testenv:check] +[testenv:lint] deps = docutils check-manifest @@ -70,7 +69,7 @@ commands = python setup.py check --strict --metadata --restructuredtext ; We are using git submodule in the package ; check-manifest {toxinidir} - pre-commit-vauxoo --compatibility-version=20.20.20.20.20.20.20.20.20.20 -t all -p src/ -p tests/ -p setup.py + pre-commit-vauxoo --compatibility-version=900.900.900.900.900.900.900.900.900.900 -t all -p src/ -p tests/ -p setup.py [testenv:build] skip_install = true diff --git a/variables.sh b/variables.sh new file mode 100644 index 0000000..36b8d8d --- /dev/null +++ b/variables.sh @@ -0,0 +1 @@ +export RUFF_DISABLE_CHECKS="print"