From 6a65495b505bed777d02649806992af119f9ec0c Mon Sep 17 00:00:00 2001 From: jo Date: Sun, 12 Feb 2023 15:41:10 +0100 Subject: [PATCH] chore: add build_metadata script --- .gitlab-ci.yml | 21 ++ .../add-build-metadata-script.misc | 1 + scripts/Makefile | 18 ++ scripts/build_metadata.py | 254 ++++++++++++++++++ scripts/build_metadata_test.py | 236 ++++++++++++++++ scripts/poetry.lock | 108 ++++++++ scripts/pyproject.toml | 20 ++ 7 files changed, 658 insertions(+) create mode 100644 changes/changelog.d/add-build-metadata-script.misc create mode 100644 scripts/Makefile create mode 100755 scripts/build_metadata.py create mode 100644 scripts/build_metadata_test.py create mode 100644 scripts/poetry.lock create mode 100644 scripts/pyproject.toml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2e4765fb1..d3db3d4bf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -202,6 +202,27 @@ lint_front: - yarn lint --max-warnings 0 - yarn lint:tsc +test_scripts: + stage: test + needs: [] + rules: + - if: $CI_COMMIT_BRANCH =~ /(stable|develop)/ + - changes: [scripts/**/*] + + image: $CI_REGISTRY/funkwhale/ci/python:3.11 + cache: + - key: scripts-pip + paths: [$PIP_CACHE_DIR] + - key: + prefix: scripts-venv + files: [scripts/poetry.lock] + paths: [scripts/.venv] + before_script: + - cd scripts + - make install + script: + - make test + test_api: retry: 1 stage: test diff --git a/changes/changelog.d/add-build-metadata-script.misc b/changes/changelog.d/add-build-metadata-script.misc new file mode 100644 index 000000000..863865ef3 --- /dev/null +++ b/changes/changelog.d/add-build-metadata-script.misc @@ -0,0 +1 @@ +Add build metadata script diff --git a/scripts/Makefile b/scripts/Makefile new file mode 100644 index 000000000..6717f5af1 --- /dev/null +++ b/scripts/Makefile @@ -0,0 +1,18 @@ +SHELL = bash +CPU_CORES = $(shell N=$$(nproc); echo $$(( $$N > 4 ? 4 : $$N ))) + +.PHONY: install test clean + +# Install +VENV = .venv +export POETRY_VIRTUALENVS_IN_PROJECT=true + +install: $(VENV) +$(VENV): + poetry install + +test: $(VENV) + poetry run pytest -s -vv + +clean: + rm -Rf $(VENV) diff --git a/scripts/build_metadata.py b/scripts/build_metadata.py new file mode 100755 index 000000000..551fda2ad --- /dev/null +++ b/scripts/build_metadata.py @@ -0,0 +1,254 @@ +#!/usr/bin/env python3 + +""" +Build metadata is a script that will extract build information from the environment, +either from CI variables or from git commands. + +The build information are then mapped to the output format you need. The default output +format will print the build information as json. + +- Release tags are stable version tags (e.g. 1.2.9), +- Prerelease tags are unstable version tags (e.g. 1.3.0-rc3), +- Development branches are both the 'stable' and 'develop' branches, +- Feature branches are any branch that will be merged in the development branches. +""" + +import json +import logging +import os +import shlex +from argparse import ArgumentParser +from subprocess import check_output +from typing import List, Optional, TypedDict + +from packaging.version import Version + +logger = logging.getLogger(__name__) + +PROJECT_NAME = "Funkwhale" +PROJECT_DESCRIPTION = "Funkwhale platform" +AUTHORS = "Funkwhale Collective" +WEBSITE_URL = "https://funkwhale.audio/" +SOURCE_URL = "https://dev.funkwhale.audio/funkwhale/funkwhale" +DOCUMENTATION_URL = "https://docs.funkwhale.audio" +LICENSE = "AGPL-3.0" + + +class Metadata(TypedDict): + commit_tag: str + commit_branch: str + commit_sha: str + commit_timestamp: str + commit_ref_name: str + + version: str + """ + Version is: + - on release tags, the current tag name, + - on prerelease tags, the current tag name, + - on development branches, the latest tag name in the branch and the commit sha suffix, + - on feature branches, an empty string. + """ + tags: List[str] + """ + Tags are: + - on release tags, the current tag name and aliases in the form 'X.Y.Z', 'X.Y', 'X' and 'latest', + - on prerelease tags, the current tag name, + - on development branches, the current commit branch name, + - on feature branches, an empty list. + """ + latest: bool + """ + Latest is true when the current tag name is not a prerelease: + - on release tags: true, + - on prerelease tags: false, + - on development branches: false, + - on feature branches: false. + """ + + +def sh(cmd: str): + logger.debug("running command: %s", cmd) + return check_output(shlex.split(cmd), text=True).strip() + + +def latest_tag_on_branch() -> str: + """ + Return the latest tag on the current branch. + """ + return sh("git describe --tags --abbrev=0") + + +def env_or_cmd(key: str, cmd: str) -> str: + if "CI" in os.environ: + return os.environ.get(key, "") + + return sh(cmd) + + +def extract_metadata() -> Metadata: + commit_tag = env_or_cmd( + "CI_COMMIT_TAG", + "git tag --points-at HEAD", + ) + commit_branch = env_or_cmd( + "CI_COMMIT_BRANCH", + "git rev-parse --abbrev-ref HEAD", + ) + commit_sha = env_or_cmd( + "CI_COMMIT_SHA", + "git rev-parse HEAD", + ) + commit_timestamp = env_or_cmd( + "CI_COMMIT_TIMESTAMP", + "git show -s --format=%cI HEAD", + ) + commit_ref_name = os.environ.get( + "CI_COMMIT_REF_NAME", + default=commit_tag or commit_branch, + ) + + logger.info("found commit_tag: %s", commit_tag) + logger.info("found commit_branch: %s", commit_branch) + logger.info("found commit_sha: %s", commit_sha) + logger.info("found commit_timestamp: %s", commit_timestamp) + logger.info("found commit_ref_name: %s", commit_ref_name) + + version = "" + tags = [] + latest = False + if commit_tag: # Tagged version + version = Version(commit_tag) + if version.is_prerelease: + logger.info("build is for a prerelease tag") + tags.append(commit_tag) + + else: + logger.info("build is for a release tag") + tags.append(f"{version.major}.{version.minor}.{version.micro}") + tags.append(f"{version.major}.{version.minor}") + tags.append(f"{version.major}") + tags.append("latest") + latest = True + + version = tags[0] + + else: # Branch version + if commit_branch in ("stable", "develop"): + logger.info("build is for a development branch") + tags.append(commit_branch) + + previous_tag = latest_tag_on_branch() + previous_version = Version(previous_tag) + version = f"{previous_version.base_version}-dev+{commit_sha[:7]}" + + else: + logger.info("build is for a feature branch") + + return { + "commit_tag": commit_tag, + "commit_branch": commit_branch, + "commit_sha": commit_sha, + "commit_timestamp": commit_timestamp, + "commit_ref_name": commit_ref_name, + "version": version, + "tags": tags, + "latest": latest, + } + + +def bake_output( + metadata: Metadata, + target: Optional[str], + images: Optional[List[str]], +) -> dict: + if target is None: + logger.error("no bake target provided, exiting...") + raise SystemExit(1) + if images is None: + logger.error("no bake images provided, exiting...") + raise SystemExit(1) + + docker_tags = [f"{img}:{tag}" for img in images for tag in metadata["tags"]] + + docker_labels = { + "org.opencontainers.image.title": PROJECT_NAME, + "org.opencontainers.image.description": PROJECT_DESCRIPTION, + "org.opencontainers.image.url": WEBSITE_URL, + "org.opencontainers.image.source": SOURCE_URL, + "org.opencontainers.image.documentation": DOCUMENTATION_URL, + "org.opencontainers.image.licenses": LICENSE, + "org.opencontainers.image.vendor": AUTHORS, + "org.opencontainers.image.version": metadata["commit_ref_name"], + "org.opencontainers.image.created": metadata["commit_timestamp"], + "org.opencontainers.image.revision": metadata["commit_sha"], + } + + return { + "target": { + target: { + "tags": docker_tags, + "labels": docker_labels, + } + } + } + + +def env_output(metadata: Metadata) -> list[str]: + env_dict = { + "BUILD_COMMIT_TAG": str(metadata["commit_tag"]), + "BUILD_COMMIT_BRANCH": str(metadata["commit_branch"]), + "BUILD_COMMIT_SHA": str(metadata["commit_sha"]), + "BUILD_COMMIT_TIMESTAMP": str(metadata["commit_timestamp"]), + "BUILD_COMMIT_REF_NAME": str(metadata["commit_ref_name"]), + "BUILD_VERSION": str(metadata["version"]), + "BUILD_TAGS": ",".join(metadata["tags"]), + "BUILD_LATEST": str(metadata["latest"]).lower(), + } + return [f"{key}={value}" for key, value in env_dict.items()] + + +def main( + format_: str, + bake_target: Optional[str], + bake_images: Optional[List[str]], +) -> int: + metadata = extract_metadata() + + if format_ == "bake": + result = json.dumps( + bake_output(metadata=metadata, target=bake_target, images=bake_images), + indent=2, + ) + elif format_ == "env": + result = "\n".join(env_output(metadata=metadata)) + else: + result = json.dumps(metadata, indent=2) + + print(result) + return 0 + + +if __name__ == "__main__": + parser = ArgumentParser() + parser.add_argument( + "-f", + "--format", + choices=["bake", "env"], + default=None, + help="Print format for the metadata", + ) + parser.add_argument( + "--bake-target", + help="Target for the bake metadata", + ) + parser.add_argument( + "--bake-image", + action="append", + dest="bake_images", + help="Image names for the bake metadata", + ) + args = parser.parse_args() + logging.basicConfig(level=logging.DEBUG, format="%(levelname)s: %(message)s") + + raise SystemExit(main(args.format, args.bake_target, args.bake_images)) diff --git a/scripts/build_metadata_test.py b/scripts/build_metadata_test.py new file mode 100644 index 000000000..66b4b521f --- /dev/null +++ b/scripts/build_metadata_test.py @@ -0,0 +1,236 @@ +from unittest import mock + +import pytest +from build_metadata import ( + AUTHORS, + DOCUMENTATION_URL, + LICENSE, + PROJECT_DESCRIPTION, + PROJECT_NAME, + SOURCE_URL, + WEBSITE_URL, + bake_output, + env_output, + extract_metadata, +) + +common_docker_labels = { + "org.opencontainers.image.title": PROJECT_NAME, + "org.opencontainers.image.description": PROJECT_DESCRIPTION, + "org.opencontainers.image.url": WEBSITE_URL, + "org.opencontainers.image.source": SOURCE_URL, + "org.opencontainers.image.documentation": DOCUMENTATION_URL, + "org.opencontainers.image.licenses": LICENSE, + "org.opencontainers.image.vendor": AUTHORS, +} + +test_cases = [ + { # On a random feature branch + "environ": { + "CI": "true", + "CI_COMMIT_TAG": "", + "CI_COMMIT_BRANCH": "ci_build_metadata", + "CI_COMMIT_SHA": "de206ac559a171b68fb894b2d61db298fc386705", + "CI_COMMIT_TIMESTAMP": "2023-01-31T13:31:13+01:00", + "CI_COMMIT_REF_NAME": "ci_build_metadata", + }, + "metadata": { + "commit_tag": "", + "commit_branch": "ci_build_metadata", + "commit_sha": "de206ac559a171b68fb894b2d61db298fc386705", + "commit_timestamp": "2023-01-31T13:31:13+01:00", + "commit_ref_name": "ci_build_metadata", + "version": "", + "tags": [], + "latest": False, + }, + "bake_output": { + "target": { + "api": { + "tags": [], + "labels": { + **common_docker_labels, + "org.opencontainers.image.version": "ci_build_metadata", + "org.opencontainers.image.created": "2023-01-31T13:31:13+01:00", + "org.opencontainers.image.revision": "de206ac559a171b68fb894b2d61db298fc386705", + }, + } + } + }, + "env_output": [ + "BUILD_COMMIT_TAG=", + "BUILD_COMMIT_BRANCH=ci_build_metadata", + "BUILD_COMMIT_SHA=de206ac559a171b68fb894b2d61db298fc386705", + "BUILD_COMMIT_TIMESTAMP=2023-01-31T13:31:13+01:00", + "BUILD_COMMIT_REF_NAME=ci_build_metadata", + "BUILD_VERSION=", + "BUILD_TAGS=", + "BUILD_LATEST=false", + ], + }, + { # On the develop (or stable) branch + "environ": { + "CI": "true", + "CI_COMMIT_TAG": "", + "CI_COMMIT_BRANCH": "develop", + "CI_COMMIT_SHA": "de206ac559a171b68fb894b2d61db298fc386705", + "CI_COMMIT_TIMESTAMP": "2023-01-31T13:31:13+01:00", + "CI_COMMIT_REF_NAME": "develop", + }, + "metadata": { + "commit_tag": "", + "commit_branch": "develop", + "commit_sha": "de206ac559a171b68fb894b2d61db298fc386705", + "commit_timestamp": "2023-01-31T13:31:13+01:00", + "commit_ref_name": "develop", + "version": "1.7.2-dev+de206ac", + "tags": ["develop"], + "latest": False, + }, + "bake_output": { + "target": { + "api": { + "tags": ["funkwhale/api:develop"], + "labels": { + **common_docker_labels, + "org.opencontainers.image.version": "develop", + "org.opencontainers.image.created": "2023-01-31T13:31:13+01:00", + "org.opencontainers.image.revision": "de206ac559a171b68fb894b2d61db298fc386705", + }, + } + } + }, + "env_output": [ + "BUILD_COMMIT_TAG=", + "BUILD_COMMIT_BRANCH=develop", + "BUILD_COMMIT_SHA=de206ac559a171b68fb894b2d61db298fc386705", + "BUILD_COMMIT_TIMESTAMP=2023-01-31T13:31:13+01:00", + "BUILD_COMMIT_REF_NAME=develop", + "BUILD_VERSION=1.7.2-dev+de206ac", + "BUILD_TAGS=develop", + "BUILD_LATEST=false", + ], + }, + { # A release tag + "environ": { + "CI": "true", + "CI_COMMIT_TAG": "1.2.9", + "CI_COMMIT_BRANCH": "", + "CI_COMMIT_SHA": "817c8fbcaa0706ccc9b724da8546f44ba7d2d841", + "CI_COMMIT_TIMESTAMP": "2022-11-25T17:59:23+01:00", + "CI_COMMIT_REF_NAME": "1.2.9", + }, + "metadata": { + "commit_tag": "1.2.9", + "commit_branch": "", + "commit_sha": "817c8fbcaa0706ccc9b724da8546f44ba7d2d841", + "commit_timestamp": "2022-11-25T17:59:23+01:00", + "commit_ref_name": "1.2.9", + "version": "1.2.9", + "tags": ["1.2.9", "1.2", "1", "latest"], + "latest": True, + }, + "bake_output": { + "target": { + "api": { + "tags": [ + "funkwhale/api:1.2.9", + "funkwhale/api:1.2", + "funkwhale/api:1", + "funkwhale/api:latest", + ], + "labels": { + **common_docker_labels, + "org.opencontainers.image.version": "1.2.9", + "org.opencontainers.image.created": "2022-11-25T17:59:23+01:00", + "org.opencontainers.image.revision": "817c8fbcaa0706ccc9b724da8546f44ba7d2d841", + }, + } + } + }, + "env_output": [ + "BUILD_COMMIT_TAG=1.2.9", + "BUILD_COMMIT_BRANCH=", + "BUILD_COMMIT_SHA=817c8fbcaa0706ccc9b724da8546f44ba7d2d841", + "BUILD_COMMIT_TIMESTAMP=2022-11-25T17:59:23+01:00", + "BUILD_COMMIT_REF_NAME=1.2.9", + "BUILD_VERSION=1.2.9", + "BUILD_TAGS=1.2.9,1.2,1,latest", + "BUILD_LATEST=true", + ], + }, + { # A prerelease tag + "environ": { + "CI": "true", + "CI_COMMIT_TAG": "1.3.0-rc3", + "CI_COMMIT_BRANCH": "", + "CI_COMMIT_SHA": "e04a1b188d3f463e7b3e2484578d63d754b09b9d", + "CI_COMMIT_TIMESTAMP": "2023-01-23T14:24:46+01:00", + "CI_COMMIT_REF_NAME": "1.3.0-rc3", + }, + "metadata": { + "commit_tag": "1.3.0-rc3", + "commit_branch": "", + "commit_sha": "e04a1b188d3f463e7b3e2484578d63d754b09b9d", + "commit_timestamp": "2023-01-23T14:24:46+01:00", + "commit_ref_name": "1.3.0-rc3", + "version": "1.3.0-rc3", + "tags": ["1.3.0-rc3"], + "latest": False, + }, + "bake_output": { + "target": { + "api": { + "tags": ["funkwhale/api:1.3.0-rc3"], + "labels": { + **common_docker_labels, + "org.opencontainers.image.version": "1.3.0-rc3", + "org.opencontainers.image.created": "2023-01-23T14:24:46+01:00", + "org.opencontainers.image.revision": "e04a1b188d3f463e7b3e2484578d63d754b09b9d", + }, + } + } + }, + "env_output": [ + "BUILD_COMMIT_TAG=1.3.0-rc3", + "BUILD_COMMIT_BRANCH=", + "BUILD_COMMIT_SHA=e04a1b188d3f463e7b3e2484578d63d754b09b9d", + "BUILD_COMMIT_TIMESTAMP=2023-01-23T14:24:46+01:00", + "BUILD_COMMIT_REF_NAME=1.3.0-rc3", + "BUILD_VERSION=1.3.0-rc3", + "BUILD_TAGS=1.3.0-rc3", + "BUILD_LATEST=false", + ], + }, +] + + +@pytest.mark.parametrize( + "environ, expected_metadata, expected_bake_output, expected_env_output", + map( + lambda i: (i["environ"], i["metadata"], i["bake_output"], i["env_output"]), + test_cases, + ), +) +def test_extract_metadata( + environ, + expected_metadata, + expected_bake_output, + expected_env_output, +): + with mock.patch("build_metadata.latest_tag_on_branch") as latest_tag_on_branch_mock: + latest_tag_on_branch_mock.return_value = "1.7.2-rc5" + with mock.patch.dict("os.environ", environ, clear=True): + found_metadata = extract_metadata() + + assert found_metadata == expected_metadata + + found_bake_output = bake_output( + metadata=found_metadata, + target="api", + images=["funkwhale/api"], + ) + assert found_bake_output == expected_bake_output + + found_env_output = env_output(metadata=found_metadata) + assert found_env_output == expected_env_output diff --git a/scripts/poetry.lock b/scripts/poetry.lock new file mode 100644 index 000000000..b40c2ea74 --- /dev/null +++ b/scripts/poetry.lock @@ -0,0 +1,108 @@ +# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +category = "dev" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.1.1" +description = "Backport of PEP 654 (exception groups)" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.1.1-py3-none-any.whl", hash = "sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e"}, + {file = "exceptiongroup-1.1.1.tar.gz", hash = "sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "packaging" +version = "23.1" +description = "Core utilities for Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, + {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, +] + +[[package]] +name = "pluggy" +version = "1.0.0" +description = "plugin and hook calling mechanisms for python" +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, + {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "pytest" +version = "7.3.1" +description = "pytest: simple powerful testing with Python" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-7.3.1-py3-none-any.whl", hash = "sha256:3799fa815351fea3a5e96ac7e503a96fa51cc9942c3753cda7651b93c1cfa362"}, + {file = "pytest-7.3.1.tar.gz", hash = "sha256:434afafd78b1d78ed0addf160ad2b77a30d35d4bdf8af234fe621919d9ed15e3"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.8" +content-hash = "ca70aa89c1605b6872de58133c5bc514c167f79b42ca9d41a155a96f7bd5d9e9" diff --git a/scripts/pyproject.toml b/scripts/pyproject.toml new file mode 100644 index 000000000..ec5e7df60 --- /dev/null +++ b/scripts/pyproject.toml @@ -0,0 +1,20 @@ +[tool.poetry] +name = "funkwhale-scripts" +version = "0.0.0" +description = "Funkwhale Scripts" +authors = ["Funkwhale Collective"] +license = "AGPL-3.0-only" + +[tool.poetry.dependencies] +python = "^3.8" + +[tool.poetry.group.dev.dependencies] +pytest = "^7.2.1" + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" + +[tool.pytest.ini_options] +log_cli = "true" +log_level = "DEBUG"