chore: add build_metadata script

pipelines/30920
jo 2023-02-12 15:41:10 +01:00
rodzic 1045d8b11c
commit 6a65495b50
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: B2FEC9B22722B984
7 zmienionych plików z 658 dodań i 0 usunięć

Wyświetl plik

@ -202,6 +202,27 @@ lint_front:
- yarn lint --max-warnings 0
- yarn lint:tsc
test_scripts:
stage: test
needs: []
rules:
- if: $CI_COMMIT_BRANCH =~ /(stable|develop)/
- changes: [scripts/**/*]
image: $CI_REGISTRY/funkwhale/ci/python:3.11
cache:
- key: scripts-pip
paths: [$PIP_CACHE_DIR]
- key:
prefix: scripts-venv
files: [scripts/poetry.lock]
paths: [scripts/.venv]
before_script:
- cd scripts
- make install
script:
- make test
test_api:
retry: 1
stage: test

Wyświetl plik

@ -0,0 +1 @@
Add build metadata script

18
scripts/Makefile 100644
Wyświetl plik

@ -0,0 +1,18 @@
SHELL = bash
CPU_CORES = $(shell N=$$(nproc); echo $$(( $$N > 4 ? 4 : $$N )))
.PHONY: install test clean
# Install
VENV = .venv
export POETRY_VIRTUALENVS_IN_PROJECT=true
install: $(VENV)
$(VENV):
poetry install
test: $(VENV)
poetry run pytest -s -vv
clean:
rm -Rf $(VENV)

Wyświetl plik

@ -0,0 +1,254 @@
#!/usr/bin/env python3
"""
Build metadata is a script that will extract build information from the environment,
either from CI variables or from git commands.
The build information are then mapped to the output format you need. The default output
format will print the build information as json.
- Release tags are stable version tags (e.g. 1.2.9),
- Prerelease tags are unstable version tags (e.g. 1.3.0-rc3),
- Development branches are both the 'stable' and 'develop' branches,
- Feature branches are any branch that will be merged in the development branches.
"""
import json
import logging
import os
import shlex
from argparse import ArgumentParser
from subprocess import check_output
from typing import List, Optional, TypedDict
from packaging.version import Version
logger = logging.getLogger(__name__)
PROJECT_NAME = "Funkwhale"
PROJECT_DESCRIPTION = "Funkwhale platform"
AUTHORS = "Funkwhale Collective"
WEBSITE_URL = "https://funkwhale.audio/"
SOURCE_URL = "https://dev.funkwhale.audio/funkwhale/funkwhale"
DOCUMENTATION_URL = "https://docs.funkwhale.audio"
LICENSE = "AGPL-3.0"
class Metadata(TypedDict):
commit_tag: str
commit_branch: str
commit_sha: str
commit_timestamp: str
commit_ref_name: str
version: str
"""
Version is:
- on release tags, the current tag name,
- on prerelease tags, the current tag name,
- on development branches, the latest tag name in the branch and the commit sha suffix,
- on feature branches, an empty string.
"""
tags: List[str]
"""
Tags are:
- on release tags, the current tag name and aliases in the form 'X.Y.Z', 'X.Y', 'X' and 'latest',
- on prerelease tags, the current tag name,
- on development branches, the current commit branch name,
- on feature branches, an empty list.
"""
latest: bool
"""
Latest is true when the current tag name is not a prerelease:
- on release tags: true,
- on prerelease tags: false,
- on development branches: false,
- on feature branches: false.
"""
def sh(cmd: str):
logger.debug("running command: %s", cmd)
return check_output(shlex.split(cmd), text=True).strip()
def latest_tag_on_branch() -> str:
"""
Return the latest tag on the current branch.
"""
return sh("git describe --tags --abbrev=0")
def env_or_cmd(key: str, cmd: str) -> str:
if "CI" in os.environ:
return os.environ.get(key, "")
return sh(cmd)
def extract_metadata() -> Metadata:
commit_tag = env_or_cmd(
"CI_COMMIT_TAG",
"git tag --points-at HEAD",
)
commit_branch = env_or_cmd(
"CI_COMMIT_BRANCH",
"git rev-parse --abbrev-ref HEAD",
)
commit_sha = env_or_cmd(
"CI_COMMIT_SHA",
"git rev-parse HEAD",
)
commit_timestamp = env_or_cmd(
"CI_COMMIT_TIMESTAMP",
"git show -s --format=%cI HEAD",
)
commit_ref_name = os.environ.get(
"CI_COMMIT_REF_NAME",
default=commit_tag or commit_branch,
)
logger.info("found commit_tag: %s", commit_tag)
logger.info("found commit_branch: %s", commit_branch)
logger.info("found commit_sha: %s", commit_sha)
logger.info("found commit_timestamp: %s", commit_timestamp)
logger.info("found commit_ref_name: %s", commit_ref_name)
version = ""
tags = []
latest = False
if commit_tag: # Tagged version
version = Version(commit_tag)
if version.is_prerelease:
logger.info("build is for a prerelease tag")
tags.append(commit_tag)
else:
logger.info("build is for a release tag")
tags.append(f"{version.major}.{version.minor}.{version.micro}")
tags.append(f"{version.major}.{version.minor}")
tags.append(f"{version.major}")
tags.append("latest")
latest = True
version = tags[0]
else: # Branch version
if commit_branch in ("stable", "develop"):
logger.info("build is for a development branch")
tags.append(commit_branch)
previous_tag = latest_tag_on_branch()
previous_version = Version(previous_tag)
version = f"{previous_version.base_version}-dev+{commit_sha[:7]}"
else:
logger.info("build is for a feature branch")
return {
"commit_tag": commit_tag,
"commit_branch": commit_branch,
"commit_sha": commit_sha,
"commit_timestamp": commit_timestamp,
"commit_ref_name": commit_ref_name,
"version": version,
"tags": tags,
"latest": latest,
}
def bake_output(
metadata: Metadata,
target: Optional[str],
images: Optional[List[str]],
) -> dict:
if target is None:
logger.error("no bake target provided, exiting...")
raise SystemExit(1)
if images is None:
logger.error("no bake images provided, exiting...")
raise SystemExit(1)
docker_tags = [f"{img}:{tag}" for img in images for tag in metadata["tags"]]
docker_labels = {
"org.opencontainers.image.title": PROJECT_NAME,
"org.opencontainers.image.description": PROJECT_DESCRIPTION,
"org.opencontainers.image.url": WEBSITE_URL,
"org.opencontainers.image.source": SOURCE_URL,
"org.opencontainers.image.documentation": DOCUMENTATION_URL,
"org.opencontainers.image.licenses": LICENSE,
"org.opencontainers.image.vendor": AUTHORS,
"org.opencontainers.image.version": metadata["commit_ref_name"],
"org.opencontainers.image.created": metadata["commit_timestamp"],
"org.opencontainers.image.revision": metadata["commit_sha"],
}
return {
"target": {
target: {
"tags": docker_tags,
"labels": docker_labels,
}
}
}
def env_output(metadata: Metadata) -> list[str]:
env_dict = {
"BUILD_COMMIT_TAG": str(metadata["commit_tag"]),
"BUILD_COMMIT_BRANCH": str(metadata["commit_branch"]),
"BUILD_COMMIT_SHA": str(metadata["commit_sha"]),
"BUILD_COMMIT_TIMESTAMP": str(metadata["commit_timestamp"]),
"BUILD_COMMIT_REF_NAME": str(metadata["commit_ref_name"]),
"BUILD_VERSION": str(metadata["version"]),
"BUILD_TAGS": ",".join(metadata["tags"]),
"BUILD_LATEST": str(metadata["latest"]).lower(),
}
return [f"{key}={value}" for key, value in env_dict.items()]
def main(
format_: str,
bake_target: Optional[str],
bake_images: Optional[List[str]],
) -> int:
metadata = extract_metadata()
if format_ == "bake":
result = json.dumps(
bake_output(metadata=metadata, target=bake_target, images=bake_images),
indent=2,
)
elif format_ == "env":
result = "\n".join(env_output(metadata=metadata))
else:
result = json.dumps(metadata, indent=2)
print(result)
return 0
if __name__ == "__main__":
parser = ArgumentParser()
parser.add_argument(
"-f",
"--format",
choices=["bake", "env"],
default=None,
help="Print format for the metadata",
)
parser.add_argument(
"--bake-target",
help="Target for the bake metadata",
)
parser.add_argument(
"--bake-image",
action="append",
dest="bake_images",
help="Image names for the bake metadata",
)
args = parser.parse_args()
logging.basicConfig(level=logging.DEBUG, format="%(levelname)s: %(message)s")
raise SystemExit(main(args.format, args.bake_target, args.bake_images))

Wyświetl plik

@ -0,0 +1,236 @@
from unittest import mock
import pytest
from build_metadata import (
AUTHORS,
DOCUMENTATION_URL,
LICENSE,
PROJECT_DESCRIPTION,
PROJECT_NAME,
SOURCE_URL,
WEBSITE_URL,
bake_output,
env_output,
extract_metadata,
)
common_docker_labels = {
"org.opencontainers.image.title": PROJECT_NAME,
"org.opencontainers.image.description": PROJECT_DESCRIPTION,
"org.opencontainers.image.url": WEBSITE_URL,
"org.opencontainers.image.source": SOURCE_URL,
"org.opencontainers.image.documentation": DOCUMENTATION_URL,
"org.opencontainers.image.licenses": LICENSE,
"org.opencontainers.image.vendor": AUTHORS,
}
test_cases = [
{ # On a random feature branch
"environ": {
"CI": "true",
"CI_COMMIT_TAG": "",
"CI_COMMIT_BRANCH": "ci_build_metadata",
"CI_COMMIT_SHA": "de206ac559a171b68fb894b2d61db298fc386705",
"CI_COMMIT_TIMESTAMP": "2023-01-31T13:31:13+01:00",
"CI_COMMIT_REF_NAME": "ci_build_metadata",
},
"metadata": {
"commit_tag": "",
"commit_branch": "ci_build_metadata",
"commit_sha": "de206ac559a171b68fb894b2d61db298fc386705",
"commit_timestamp": "2023-01-31T13:31:13+01:00",
"commit_ref_name": "ci_build_metadata",
"version": "",
"tags": [],
"latest": False,
},
"bake_output": {
"target": {
"api": {
"tags": [],
"labels": {
**common_docker_labels,
"org.opencontainers.image.version": "ci_build_metadata",
"org.opencontainers.image.created": "2023-01-31T13:31:13+01:00",
"org.opencontainers.image.revision": "de206ac559a171b68fb894b2d61db298fc386705",
},
}
}
},
"env_output": [
"BUILD_COMMIT_TAG=",
"BUILD_COMMIT_BRANCH=ci_build_metadata",
"BUILD_COMMIT_SHA=de206ac559a171b68fb894b2d61db298fc386705",
"BUILD_COMMIT_TIMESTAMP=2023-01-31T13:31:13+01:00",
"BUILD_COMMIT_REF_NAME=ci_build_metadata",
"BUILD_VERSION=",
"BUILD_TAGS=",
"BUILD_LATEST=false",
],
},
{ # On the develop (or stable) branch
"environ": {
"CI": "true",
"CI_COMMIT_TAG": "",
"CI_COMMIT_BRANCH": "develop",
"CI_COMMIT_SHA": "de206ac559a171b68fb894b2d61db298fc386705",
"CI_COMMIT_TIMESTAMP": "2023-01-31T13:31:13+01:00",
"CI_COMMIT_REF_NAME": "develop",
},
"metadata": {
"commit_tag": "",
"commit_branch": "develop",
"commit_sha": "de206ac559a171b68fb894b2d61db298fc386705",
"commit_timestamp": "2023-01-31T13:31:13+01:00",
"commit_ref_name": "develop",
"version": "1.7.2-dev+de206ac",
"tags": ["develop"],
"latest": False,
},
"bake_output": {
"target": {
"api": {
"tags": ["funkwhale/api:develop"],
"labels": {
**common_docker_labels,
"org.opencontainers.image.version": "develop",
"org.opencontainers.image.created": "2023-01-31T13:31:13+01:00",
"org.opencontainers.image.revision": "de206ac559a171b68fb894b2d61db298fc386705",
},
}
}
},
"env_output": [
"BUILD_COMMIT_TAG=",
"BUILD_COMMIT_BRANCH=develop",
"BUILD_COMMIT_SHA=de206ac559a171b68fb894b2d61db298fc386705",
"BUILD_COMMIT_TIMESTAMP=2023-01-31T13:31:13+01:00",
"BUILD_COMMIT_REF_NAME=develop",
"BUILD_VERSION=1.7.2-dev+de206ac",
"BUILD_TAGS=develop",
"BUILD_LATEST=false",
],
},
{ # A release tag
"environ": {
"CI": "true",
"CI_COMMIT_TAG": "1.2.9",
"CI_COMMIT_BRANCH": "",
"CI_COMMIT_SHA": "817c8fbcaa0706ccc9b724da8546f44ba7d2d841",
"CI_COMMIT_TIMESTAMP": "2022-11-25T17:59:23+01:00",
"CI_COMMIT_REF_NAME": "1.2.9",
},
"metadata": {
"commit_tag": "1.2.9",
"commit_branch": "",
"commit_sha": "817c8fbcaa0706ccc9b724da8546f44ba7d2d841",
"commit_timestamp": "2022-11-25T17:59:23+01:00",
"commit_ref_name": "1.2.9",
"version": "1.2.9",
"tags": ["1.2.9", "1.2", "1", "latest"],
"latest": True,
},
"bake_output": {
"target": {
"api": {
"tags": [
"funkwhale/api:1.2.9",
"funkwhale/api:1.2",
"funkwhale/api:1",
"funkwhale/api:latest",
],
"labels": {
**common_docker_labels,
"org.opencontainers.image.version": "1.2.9",
"org.opencontainers.image.created": "2022-11-25T17:59:23+01:00",
"org.opencontainers.image.revision": "817c8fbcaa0706ccc9b724da8546f44ba7d2d841",
},
}
}
},
"env_output": [
"BUILD_COMMIT_TAG=1.2.9",
"BUILD_COMMIT_BRANCH=",
"BUILD_COMMIT_SHA=817c8fbcaa0706ccc9b724da8546f44ba7d2d841",
"BUILD_COMMIT_TIMESTAMP=2022-11-25T17:59:23+01:00",
"BUILD_COMMIT_REF_NAME=1.2.9",
"BUILD_VERSION=1.2.9",
"BUILD_TAGS=1.2.9,1.2,1,latest",
"BUILD_LATEST=true",
],
},
{ # A prerelease tag
"environ": {
"CI": "true",
"CI_COMMIT_TAG": "1.3.0-rc3",
"CI_COMMIT_BRANCH": "",
"CI_COMMIT_SHA": "e04a1b188d3f463e7b3e2484578d63d754b09b9d",
"CI_COMMIT_TIMESTAMP": "2023-01-23T14:24:46+01:00",
"CI_COMMIT_REF_NAME": "1.3.0-rc3",
},
"metadata": {
"commit_tag": "1.3.0-rc3",
"commit_branch": "",
"commit_sha": "e04a1b188d3f463e7b3e2484578d63d754b09b9d",
"commit_timestamp": "2023-01-23T14:24:46+01:00",
"commit_ref_name": "1.3.0-rc3",
"version": "1.3.0-rc3",
"tags": ["1.3.0-rc3"],
"latest": False,
},
"bake_output": {
"target": {
"api": {
"tags": ["funkwhale/api:1.3.0-rc3"],
"labels": {
**common_docker_labels,
"org.opencontainers.image.version": "1.3.0-rc3",
"org.opencontainers.image.created": "2023-01-23T14:24:46+01:00",
"org.opencontainers.image.revision": "e04a1b188d3f463e7b3e2484578d63d754b09b9d",
},
}
}
},
"env_output": [
"BUILD_COMMIT_TAG=1.3.0-rc3",
"BUILD_COMMIT_BRANCH=",
"BUILD_COMMIT_SHA=e04a1b188d3f463e7b3e2484578d63d754b09b9d",
"BUILD_COMMIT_TIMESTAMP=2023-01-23T14:24:46+01:00",
"BUILD_COMMIT_REF_NAME=1.3.0-rc3",
"BUILD_VERSION=1.3.0-rc3",
"BUILD_TAGS=1.3.0-rc3",
"BUILD_LATEST=false",
],
},
]
@pytest.mark.parametrize(
"environ, expected_metadata, expected_bake_output, expected_env_output",
map(
lambda i: (i["environ"], i["metadata"], i["bake_output"], i["env_output"]),
test_cases,
),
)
def test_extract_metadata(
environ,
expected_metadata,
expected_bake_output,
expected_env_output,
):
with mock.patch("build_metadata.latest_tag_on_branch") as latest_tag_on_branch_mock:
latest_tag_on_branch_mock.return_value = "1.7.2-rc5"
with mock.patch.dict("os.environ", environ, clear=True):
found_metadata = extract_metadata()
assert found_metadata == expected_metadata
found_bake_output = bake_output(
metadata=found_metadata,
target="api",
images=["funkwhale/api"],
)
assert found_bake_output == expected_bake_output
found_env_output = env_output(metadata=found_metadata)
assert found_env_output == expected_env_output

108
scripts/poetry.lock wygenerowano 100644
Wyświetl plik

@ -0,0 +1,108 @@
# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand.
[[package]]
name = "colorama"
version = "0.4.6"
description = "Cross-platform colored terminal text."
category = "dev"
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
files = [
{file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"},
{file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"},
]
[[package]]
name = "exceptiongroup"
version = "1.1.1"
description = "Backport of PEP 654 (exception groups)"
category = "dev"
optional = false
python-versions = ">=3.7"
files = [
{file = "exceptiongroup-1.1.1-py3-none-any.whl", hash = "sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e"},
{file = "exceptiongroup-1.1.1.tar.gz", hash = "sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785"},
]
[package.extras]
test = ["pytest (>=6)"]
[[package]]
name = "iniconfig"
version = "2.0.0"
description = "brain-dead simple config-ini parsing"
category = "dev"
optional = false
python-versions = ">=3.7"
files = [
{file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"},
{file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"},
]
[[package]]
name = "packaging"
version = "23.1"
description = "Core utilities for Python packages"
category = "dev"
optional = false
python-versions = ">=3.7"
files = [
{file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"},
{file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"},
]
[[package]]
name = "pluggy"
version = "1.0.0"
description = "plugin and hook calling mechanisms for python"
category = "dev"
optional = false
python-versions = ">=3.6"
files = [
{file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"},
{file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"},
]
[package.extras]
dev = ["pre-commit", "tox"]
testing = ["pytest", "pytest-benchmark"]
[[package]]
name = "pytest"
version = "7.3.1"
description = "pytest: simple powerful testing with Python"
category = "dev"
optional = false
python-versions = ">=3.7"
files = [
{file = "pytest-7.3.1-py3-none-any.whl", hash = "sha256:3799fa815351fea3a5e96ac7e503a96fa51cc9942c3753cda7651b93c1cfa362"},
{file = "pytest-7.3.1.tar.gz", hash = "sha256:434afafd78b1d78ed0addf160ad2b77a30d35d4bdf8af234fe621919d9ed15e3"},
]
[package.dependencies]
colorama = {version = "*", markers = "sys_platform == \"win32\""}
exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""}
iniconfig = "*"
packaging = "*"
pluggy = ">=0.12,<2.0"
tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""}
[package.extras]
testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"]
[[package]]
name = "tomli"
version = "2.0.1"
description = "A lil' TOML parser"
category = "dev"
optional = false
python-versions = ">=3.7"
files = [
{file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"},
{file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"},
]
[metadata]
lock-version = "2.0"
python-versions = "^3.8"
content-hash = "ca70aa89c1605b6872de58133c5bc514c167f79b42ca9d41a155a96f7bd5d9e9"

Wyświetl plik

@ -0,0 +1,20 @@
[tool.poetry]
name = "funkwhale-scripts"
version = "0.0.0"
description = "Funkwhale Scripts"
authors = ["Funkwhale Collective"]
license = "AGPL-3.0-only"
[tool.poetry.dependencies]
python = "^3.8"
[tool.poetry.group.dev.dependencies]
pytest = "^7.2.1"
[build-system]
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"
[tool.pytest.ini_options]
log_cli = "true"
log_level = "DEBUG"