diff --git a/.gitignore b/.gitignore index 5d4f3c5..95006b3 100644 --- a/.gitignore +++ b/.gitignore @@ -120,3 +120,4 @@ node_modules/ bandit*.yaml doc8.ini pyproject.toml +.claude/ diff --git a/README.rst b/README.rst index db44839..45d63ed 100644 --- a/README.rst +++ b/README.rst @@ -68,15 +68,16 @@ Usage ===== `travisfile2dockerfile REPO_URL BRANCH` - + Or with pull request `travisfile2dockerfile REPO_URL pull/##` - -In REPO_URL use the ssh url of github. + +In REPO_URL use the ssh or https url of the git repository. +For private repositories use the ssh url. For more information execute: `travisfile2dockerfile --help` - + Example: `travisfile2dockerfile --root-path=$HOME/t2d git@github.com:Vauxoo/forecast.git 8.0` @@ -145,6 +146,17 @@ SSH key without password Dockerfile doesn't support a prompt to enter your password, so you need to remove it from your ssh keys. +Recommended: use Ed25519 keys. The tool copies ``~/.ssh/id_ed25519.pub`` to the +container's ``authorized_keys`` and warns if only RSA keys are found. + +:: + + export fname=~/.ssh/id_ed25519 + cp ${fname} ${fname}_with_pwd + ssh-keygen -p -N "" -f ${fname} + +For legacy RSA keys: + :: export fname=~/.ssh/id_rsa diff --git a/src/travis2docker/cli.py b/src/travis2docker/cli.py index f077b9f..08e73da 100644 --- a/src/travis2docker/cli.py +++ b/src/travis2docker/cli.py @@ -14,15 +14,18 @@ """ import argparse +import logging import os import pathlib -from sys import stdout from . import __version__ from .exceptions import InvalidRepoBranchError from .git_run import GitRun +from .logging_colored import FORMAT_STR, ColoredFormatter from .travis2docker import Travis2Docker +_logger = logging.getLogger(__name__) + def variables_sh_read(variables_sh_path): variables_sh_path = pathlib.Path(variables_sh_path).expanduser() @@ -50,50 +53,76 @@ def get_git_data(project, path, revision): def main(return_result=False): - parser = argparse.ArgumentParser() + handler = logging.StreamHandler() + handler.setFormatter(ColoredFormatter(FORMAT_STR)) + logging.basicConfig(level=logging.DEBUG, handlers=[handler]) + default_root_path = os.environ.get("TRAVIS2DOCKER_ROOT_PATH") + if not default_root_path: + default_root_path = pathlib.Path("~").expanduser() + default_root_path = str(pathlib.Path(default_root_path) / ".t2d") + parser = argparse.ArgumentParser( + prog="travisfile2dockerfile", + formatter_class=argparse.RawDescriptionHelpFormatter, + description=( + "travis2docker (t2d) - Generate a development Dockerfile from the\n" + "deployv image of a repository.\n" + "\n" + "Clones the git repository, reads the variables.sh file of the given\n" + "revision and generates a Dockerfile plus 10-build.sh and 20-run.sh\n" + "helper scripts to build the image and run a development container." + ), + epilog=( + "examples:\n" + " %(prog)s git@github.com:Vauxoo/forecast.git 16.0\n" + " %(prog)s git@github.com:Vauxoo/forecast.git pull/42\n" + " %(prog)s --docker-image quay.io/vauxoo/proj:tag git@github.com:org/proj.git 16.0\n" + " %(prog)s --no-clone --variables-sh-path ./variables.sh foo bar\n" + "\n" + "environment variables:\n" + " TRAVIS2DOCKER_ROOT_PATH Override the default root path (~) where\n" + " the .t2d working directory is created.\n" + ), + ) parser.add_argument( "git_repo_url", - help="Specify repository git of work." - "\nThis is used to clone it " - "and get the variables.sh file of the deployv image" - "\nIf your repository is private, " - "don't use https url, " - "use ssh url", + help="Git URL of the repository to process. " + "It is cloned locally to extract the variables.sh file of the deployv image. " + "For private repositories use the SSH URL (git@...) instead of HTTPS.", ) parser.add_argument( "git_revision", - help="Revision git of work." - "\nYou can use " - "branch name e.g. master or 8.0 " - "or pull number with 'pull/#' e.g. pull/1 " - "NOTE: A sha e.g. b48228 NOT IMPLEMENTED YET", + help="Git revision to process. Accepts a branch name e.g. 'main' or '16.0', " + "or a pull request with 'pull/#' e.g. 'pull/1'. " + "NOTE: A sha e.g. b48228 is not supported yet.", ) parser.add_argument( "--docker-user", dest="docker_user", - help="User of work into Dockerfile.\nBased on your docker image.\nDefault: odoo", + help="Unix user that runs the commands inside the container. " + "It must exist in the base docker image. " + "Default: odoo", ) parser.add_argument( "--docker-image", dest="default_docker_image", - help="Docker image to use by default in Dockerfile." - "\nDefault: built from variables.sh as " - "'DOCKER_IMAGE_REPO:MAIN_APP-VERSION-SHA_SHORT'", + help="Base docker image for the generated Dockerfile, e.g. the one pushed " + "by the 'build_docker' pipeline as 'quay.io/vauxoo/PROJECT:TAG'. " + "Default: built from variables.sh values as 'DOCKER_IMAGE_REPO:MAIN_APP-VERSION-SHA_SHORT'", ) - default_root_path = os.environ.get("TRAVIS2DOCKER_ROOT_PATH") - if not default_root_path: - default_root_path = pathlib.Path("~").expanduser() - default_root_path = str(pathlib.Path(default_root_path) / ".t2d") parser.add_argument( "--root-path", dest="root_path", - help=f"Root path to save scripts generated.\nDefault: {default_root_path}", default=default_root_path, + help="Root directory to store the generated scripts and the cloned repositories. " + "The 'repo/' and 'script/' sub-directories are created inside it. " + f"Default: {default_root_path}", ) parser.add_argument( "--add-remote", dest="remotes", - help="Add git remote to git of build path, separated by a comma.\nUse remote name. E.g. 'Vauxoo,moylop260'", + help="Comma-separated list of GitHub user/organization names to add as git " + "remotes in the instance repositories. E.g. 'Vauxoo,moylop260'. " + "Default: none", ) parser.add_argument( "--exclude-after-success", @@ -105,22 +134,25 @@ def main(return_result=False): parser.add_argument( "--run-extra-args", dest="run_extra_args", - help="Extra arguments to `docker run RUN_EXTRA_ARGS` command", - default="-itP -e LANG=C.UTF-8", + help="Extra arguments appended to the `docker run` command of 20-run.sh. " + "Note: '-ditP' is always used, no need to add it here. " + "Default: '-e LANG=C.UTF-8'", + default="-e LANG=C.UTF-8", ) parser.add_argument( "--run-extra-cmds", dest="run_extra_cmds", nargs="*", default="", - help='Extra commands to run after "run" script. ' - "Note: You can use \\$IMAGE escaped environment variable." - 'E.g. "docker rmi -f \\$IMAGE"', + help="Extra commands to run at the end of the 20-run.sh script. " + "The built image can be referenced with the escaped variable \\$IMAGE. " + 'E.g. "docker rmi -f \\$IMAGE". ' + "Default: none", ) parser.add_argument( "--build-extra-args", dest="build_extra_args", - help="Extra arguments to `docker build BUILD_EXTRA_ARGS` command", + help="Extra arguments appended to the `docker build` command of 10-build.sh. Default: '--rm'", default="--rm", ) parser.add_argument( @@ -128,7 +160,9 @@ def main(return_result=False): dest="build_extra_cmds", nargs="*", default="", - help='Extra commands to run after "build" script. Note: You can use \\$IMAGE escaped environment variable.', + help="Extra commands to run at the end of the 10-build.sh script. " + "The built image can be referenced with the escaped variable \\$IMAGE. " + "Default: none", ) parser.add_argument( "--travis-yml-path", @@ -140,22 +174,26 @@ def main(return_result=False): "--variables-sh-path", dest="variables_sh_path", default=None, - help="Optional path of the variables.sh file (or the directory containing it) to use.\n" - "Default: Extracted from git repo and git revision.", + help="Use a local variables.sh file (or the directory containing it) " + "instead of extracting it from the cloned repository. " + "Default: extracted from git_repo_url at git_revision", ) parser.add_argument( "--no-clone", dest="no_clone", action="store_true", default=False, - help="Avoid cloning the repository. It requires --variables-sh-path", + help="Skip cloning the repository. It requires --variables-sh-path pointing " + "to a local variables.sh file. " + "Default: False", ) parser.add_argument( "--add-rcfile", dest="add_rcfile", default="", - help="Optional paths of configuration files to " - "copy for user's HOME path into container, separated by a comma.", + help="Comma-separated list of configuration file paths (e.g. '~/.gitconfig,~/.vimrc') " + "to copy into the container user's $HOME directory. " + "Default: none", ) parser.add_argument("-v", "--version", action="version", version="%(prog)s " + __version__) parser.add_argument( @@ -171,11 +209,10 @@ def main(return_result=False): nargs="*", action="append", default=[], - help="Args used as environment variables " - "More info about: https://vsupalov.com/docker-build-time-env-values\n" - "E.g. --build-env-args ENVAR1\n" - "It generates the following line for Dockerfile:\n" - "ARG ENVVAR1\nENV ENVVAR1=$ENVVAR1", + help="Environment variable names to enable in the generated Dockerfile. " + "Each NAME generates an 'ENV NAME=TRUE' line, used to activate optional " + "installation steps of the image. E.g. '--build-env-args VIM_INSTALL ZSH_INSTALL'. " + "Default: none", ) parser.add_argument( "--deployv", @@ -189,7 +226,9 @@ def main(return_result=False): nargs="*", default="", dest="build_extra_steps", - help="Append these extra steps at the end of the Dockerfile", + help="Extra Dockerfile instructions appended at the end of the generated " + "Dockerfile, each value as a separate line. " + "Default: none", ) args = parser.parse_args() @@ -200,7 +239,7 @@ def main(return_result=False): } for deprecated_arg, value in deprecated_args.items(): if value: - stdout.write("WARNING: %s is deprecated and its value will be ignored\n" % deprecated_arg) + _logger.warning("%s is deprecated and its value will be ignored", deprecated_arg) revision = args.git_revision git_repo = args.git_repo_url git_base = GitRun.get_data_url(git_repo, False)[0] @@ -213,7 +252,7 @@ def main(return_result=False): build_extra_cmds = "\n".join(args.build_extra_cmds) run_extra_cmds = "\n".join(args.run_extra_cmds) rcfiles_args = args.add_rcfile and args.add_rcfile.split(",") - build_env_args = [build_env_args[0] for build_env_args in args.build_env_args] + build_env_args = [build_env_arg for build_env_args in args.build_env_args for build_env_arg in build_env_args] rcfiles = [ (pathlib.Path(rc_file).expanduser(), "$HOME/%s" % pathlib.Path(rc_file).name) for rc_file in rcfiles_args ] @@ -264,18 +303,16 @@ def main(return_result=False): fname_scripts = t2d.compute_dockerfile() if fname_scripts: fname_list = "- " + "\n- ".join(fname_scripts) - stdout.write("\nGenerated scripts:\n%s\n" % fname_list) + _logger.info("Generated scripts:\n%s", fname_list) if not default_docker_image: - stdout.write("=" * 80) # TODO: Add the URL to open the pipelines - stdout.write( - '\nTIP: Use the parameter "--docker-image=quay.io/vauxoo/PROJECT:TAG" ' + _logger.info( + 'TIP: Use the parameter "--docker-image=quay.io/vauxoo/PROJECT:TAG" ' 'get the PROJECT:TAG info in your "build_docker" pipeline similar to ' '\n"... INFO - deployv.deployv_addon_gitlab_tools.common.common.push_image - ' - 'Pushing image ... to quay.io/vauxoo/PROJECT:TAG"\n' + 'Pushing image ... to quay.io/vauxoo/PROJECT:TAG"' ) - stdout.write("=" * 80) else: - stdout.write("\nNo scripts were generated.") + _logger.info("No scripts were generated.") if return_result: return fname_scripts diff --git a/src/travis2docker/docker_helper/build.py b/src/travis2docker/docker_helper/build.py index e68c993..64777be 100644 --- a/src/travis2docker/docker_helper/build.py +++ b/src/travis2docker/docker_helper/build.py @@ -1,12 +1,15 @@ -# No plan to use logging here -# pylint: disable=print-used - import glob +import logging import pathlib import re import subprocess import sys +# Standalone script copied into the docker image and run with +# `python3 -c "import build;..."` so it configures its own logging +logging.basicConfig(level=logging.DEBUG, format="%(asctime)s %(levelname)s %(name)s: %(message)s") +_logger = logging.getLogger("docker_helper.build") + def ssh_keyscan2known_hosts(url, known_hosts_path=None): # python3 -c "import build;build.ssh_keyscan2known_hosts('url')" @@ -15,12 +18,12 @@ def ssh_keyscan2known_hosts(url, known_hosts_path=None): # Clear current known hosts cmd = ["ssh-keygen", "-R", url] - print(" ".join(cmd)) + _logger.info(" ".join(cmd)) subprocess.call(cmd) # Scan new key of host and store cmd = ["ssh-keyscan", "-p", "22", url] - print(" ".join(cmd)) + _logger.info(" ".join(cmd)) keys_scanned = subprocess.check_output(cmd).decode(sys.stdout.encoding).strip() with pathlib.Path(known_hosts_path).open("r+") as known_hosts_f: known_hosts_f.write("\n" + keys_scanned) @@ -54,7 +57,7 @@ def git_set_remote(path=None): subprocess.call(cmd) if not git_re_match: - print("Remote not matched %s" % remote) + _logger.warning("Remote not matched %s", remote) continue git_re_groups = git_re_match.groups() @@ -65,7 +68,7 @@ def git_set_remote(path=None): # Transform https url to ssh format ssh_url_stb = "git@%s:%s/%s" % (host, org, repo) cmd = git_cmd + ["remote", "set-url", "origin", ssh_url_stb] - print(" ".join(cmd)) + _logger.info(" ".join(cmd)) subprocess.call(cmd) # Unshallow repository @@ -76,7 +79,7 @@ def git_set_remote(path=None): ssh_keyscan2known_hosts(host) hosts_scanned.add(host) cmd = git_cmd + ["fetch", "--unshallow"] - print(" ".join(cmd)) + _logger.info(" ".join(cmd)) subprocess.call(cmd) # Add extra remote if "stb" so add "dev" if "dev" so add "stb" @@ -84,5 +87,5 @@ def git_set_remote(path=None): new_remote = "dev" if "dev" in new_org else "stb" ssh_url_dev = "git@%s:%s/%s" % (host, new_org, repo) cmd = git_cmd + ["remote", "add", new_remote, ssh_url_dev] - print(" ".join(cmd)) + _logger.info(" ".join(cmd)) subprocess.call(cmd) diff --git a/src/travis2docker/git_run.py b/src/travis2docker/git_run.py index 858565a..9110882 100644 --- a/src/travis2docker/git_run.py +++ b/src/travis2docker/git_run.py @@ -1,10 +1,13 @@ -# pylint: disable=useless-object-inheritance,print-used,except-pass +# pylint: disable=useless-object-inheritance,except-pass import contextlib +import logging import pathlib import re import subprocess +_logger = logging.getLogger(__name__) + def decode_utf(field): try: @@ -60,8 +63,8 @@ def get_config_data(self, field=None): def run(self, cmd): """Execute git command in bash""" cmd = ["git", "--git-dir=%s" % self.path] + cmd - print("cmd list", cmd) - print("cmd", " ".join(cmd)) + _logger.debug("cmd list %s", cmd) + _logger.debug("cmd %s", " ".join(cmd)) res = None with contextlib.suppress(BaseException): res = subprocess.check_output(cmd) diff --git a/src/travis2docker/logging_colored.py b/src/travis2docker/logging_colored.py new file mode 100644 index 0000000..6f017bc --- /dev/null +++ b/src/travis2docker/logging_colored.py @@ -0,0 +1,30 @@ +import logging + +BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE, _NOTHING, DEFAULT = range(10) +# The background is set with 40 plus the number of the color, and the foreground with 30 +# These are the sequences needed to get colored output +RESET_SEQ = "\033[0m" +COLOR_SEQ = "\033[1;%dm" +BOLD_SEQ = "\033[1m" +COLOR_PATTERN = "%s%s%%s%s" % (COLOR_SEQ, COLOR_SEQ, RESET_SEQ) +LEVEL_COLOR_MAPPING = { + logging.DEBUG: (BLUE, DEFAULT), + logging.INFO: (GREEN, DEFAULT), + logging.WARNING: (YELLOW, DEFAULT), + logging.ERROR: (RED, DEFAULT), + logging.CRITICAL: (WHITE, RED), +} +FORMAT_STR = "%(asctime)s %(levelname)s %(name)s: %(message)s" + + +def colorized_msg(msg, level): + fg_color, bg_color = LEVEL_COLOR_MAPPING.get(level, (GREEN, DEFAULT)) + colorized_msg_str = COLOR_PATTERN % (30 + fg_color, 40 + bg_color, msg) + return colorized_msg_str + + +class ColoredFormatter(logging.Formatter): + def format(self, record): + level_colorized = colorized_msg(record.levelname, record.levelno) + record.levelname = level_colorized + return logging.Formatter.format(self, record) diff --git a/src/travis2docker/travis2docker.py b/src/travis2docker/travis2docker.py index 14b20f8..f42102b 100644 --- a/src/travis2docker/travis2docker.py +++ b/src/travis2docker/travis2docker.py @@ -1,4 +1,5 @@ -# pylint: disable=useless-object-inheritance,consider-using-with,print-used +# pylint: disable=useless-object-inheritance,consider-using-with +import logging import pathlib import re import shutil @@ -7,6 +8,8 @@ import jinja2 +_logger = logging.getLogger(__name__) + RE_ENV_STR = r"(?P[\w]*)[ ]*[\=][ ]*[\"\']{0,1}" + r"(?P[\w\.\-\_/\$\{\}\:,\(\)\#\* ]*)[\"\']{0,1}" RE_EXPORT_STR = r"^(?Pexport|EXPORT)( )+" + RE_ENV_STR @@ -149,11 +152,11 @@ def set_authorized_key(self): if ed_key.is_file(): to_copy = ed_key elif rsa_key.is_file(): - print("RSA keys are deprecated, consider changing to ed25519") + _logger.warning("RSA keys are deprecated, consider changing to ed25519") to_copy = rsa_key if not to_copy: - print("No public key found. No key added to ~/.ssh/authorized_keys. SSH login won't work.") + _logger.warning("No public key found. No key added to ~/.ssh/authorized_keys. SSH login won't work.") return pub_key = to_copy.read_text(encoding="utf-8") diff --git a/tests/test_travis2docker.py b/tests/test_travis2docker.py index 630dfd8..c0e4412 100644 --- a/tests/test_travis2docker.py +++ b/tests/test_travis2docker.py @@ -1,5 +1,4 @@ -# No logger planned to use here -# pylint: disable=print-used,consider-using-with +# pylint: disable=consider-using-with import os import pathlib @@ -74,6 +73,7 @@ def test_main_deployv(tmp_path, monkeypatch): "BUILD_ENV1", "--build-env-args", "BUILD_ENV2", + "BUILD_ENV3", "--build-extra-steps", "touch /home/odoo/extra_step_done", # Deprecated parameters must still be accepted (and ignored) @@ -91,6 +91,7 @@ def test_main_deployv(tmp_path, monkeypatch): assert "FROM quay.io/vauxoo/myproject:myproject-16.0-%s" % sha_short in dkr_content assert "ENV BUILD_ENV1=TRUE" in dkr_content assert "ENV BUILD_ENV2=TRUE" in dkr_content + assert "ENV BUILD_ENV3=TRUE" in dkr_content assert "RUN touch /home/odoo/extra_step_done" in dkr_content assert "ENTRYPOINT /entrypoint.sh" in dkr_content assert "COPY build.sh /home/odoo/build.sh" in dkr_content diff --git a/variables.sh b/variables.sh deleted file mode 100644 index 36b8d8d..0000000 --- a/variables.sh +++ /dev/null @@ -1 +0,0 @@ -export RUFF_DISABLE_CHECKS="print"